summaryrefslogtreecommitdiff
path: root/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c')
-rw-r--r--drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c2474
1 files changed, 1828 insertions, 646 deletions
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 73fc2479b4f4..043733c5c812 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -51,7 +51,6 @@
#include <linux/mutex.h>
#include <linux/netdevice.h>
#include <linux/pci.h>
-#include <linux/aer.h>
#include <linux/rtnetlink.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
@@ -65,6 +64,10 @@
#include <linux/uaccess.h>
#include <linux/crash_dump.h>
#include <net/udp_tunnel.h>
+#include <net/xfrm.h>
+#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE)
+#include <net/tls.h>
+#endif
#include "cxgb4.h"
#include "cxgb4_filter.h"
@@ -82,16 +85,13 @@
#include "sched.h"
#include "cxgb4_tc_u32.h"
#include "cxgb4_tc_flower.h"
+#include "cxgb4_tc_mqprio.h"
+#include "cxgb4_tc_matchall.h"
#include "cxgb4_ptp.h"
#include "cxgb4_cudbg.h"
char cxgb4_driver_name[] = KBUILD_MODNAME;
-#ifdef DRV_VERSION
-#undef DRV_VERSION
-#endif
-#define DRV_VERSION "2.0.0-ko"
-const char cxgb4_driver_version[] = DRV_VERSION;
#define DRV_DESC "Chelsio T4/T5/T6 Network Driver"
#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
@@ -134,7 +134,6 @@ const char cxgb4_driver_version[] = DRV_VERSION;
MODULE_DESCRIPTION(DRV_DESC);
MODULE_AUTHOR("Chelsio Communications");
MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
MODULE_FIRMWARE(FW4_FNAME);
MODULE_FIRMWARE(FW5_FNAME);
@@ -183,6 +182,9 @@ static struct dentry *cxgb4_debugfs_root;
LIST_HEAD(adapter_list);
DEFINE_MUTEX(uld_mutex);
+LIST_HEAD(uld_list);
+
+static int cfg_queues(struct adapter *adap);
static void link_report(struct net_device *dev)
{
@@ -366,13 +368,19 @@ static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr)
int ret;
u64 mhash = 0;
u64 uhash = 0;
+ /* idx stores the index of allocated filters,
+ * its size should be modified based on the number of
+ * MAC addresses that we allocate filters for
+ */
+
+ u16 idx[1] = {};
bool free = false;
bool ucast = is_unicast_ether_addr(mac_addr);
const u8 *maclist[1] = {mac_addr};
struct hash_mac_addr *new_entry;
- ret = t4_alloc_mac_filt(adap, adap->mbox, pi->viid, free, 1, maclist,
- NULL, ucast ? &uhash : &mhash, false);
+ ret = cxgb4_alloc_mac_filt(adap, pi->viid, free, 1, maclist,
+ idx, ucast ? &uhash : &mhash, false);
if (ret < 0)
goto out;
/* if hash != 0, then add the addr to hash addr list
@@ -410,7 +418,7 @@ static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr)
}
}
- ret = t4_free_mac_filt(adap, adap->mbox, pi->viid, 1, maclist, false);
+ ret = cxgb4_free_mac_filt(adap, pi->viid, 1, maclist, false);
return ret < 0 ? -EINVAL : 0;
}
@@ -426,13 +434,67 @@ static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
__dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
__dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync);
- return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu,
- (dev->flags & IFF_PROMISC) ? 1 : 0,
+ return t4_set_rxmode(adapter, adapter->mbox, pi->viid, pi->viid_mirror,
+ mtu, (dev->flags & IFF_PROMISC) ? 1 : 0,
(dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
sleep_ok);
}
/**
+ * cxgb4_change_mac - Update match filter for a MAC address.
+ * @pi: the port_info
+ * @viid: the VI id
+ * @tcam_idx: TCAM index of existing filter for old value of MAC address,
+ * or -1
+ * @addr: the new MAC address value
+ * @persist: whether a new MAC allocation should be persistent
+ * @smt_idx: the destination to store the new SMT index.
+ *
+ * Modifies an MPS filter and sets it to the new MAC address if
+ * @tcam_idx >= 0, or adds the MAC address to a new filter if
+ * @tcam_idx < 0. In the latter case the address is added persistently
+ * if @persist is %true.
+ * Addresses are programmed to hash region, if tcam runs out of entries.
+ *
+ */
+int cxgb4_change_mac(struct port_info *pi, unsigned int viid,
+ int *tcam_idx, const u8 *addr, bool persist,
+ u8 *smt_idx)
+{
+ struct adapter *adapter = pi->adapter;
+ struct hash_mac_addr *entry, *new_entry;
+ int ret;
+
+ ret = t4_change_mac(adapter, adapter->mbox, viid,
+ *tcam_idx, addr, persist, smt_idx);
+ /* We ran out of TCAM entries. try programming hash region. */
+ if (ret == -ENOMEM) {
+ /* If the MAC address to be updated is in the hash addr
+ * list, update it from the list
+ */
+ list_for_each_entry(entry, &adapter->mac_hlist, list) {
+ if (entry->iface_mac) {
+ ether_addr_copy(entry->addr, addr);
+ goto set_hash;
+ }
+ }
+ new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL);
+ if (!new_entry)
+ return -ENOMEM;
+ ether_addr_copy(new_entry->addr, addr);
+ new_entry->iface_mac = true;
+ list_add_tail(&new_entry->list, &adapter->mac_hlist);
+set_hash:
+ ret = cxgb4_set_addr_hash(pi);
+ } else if (ret >= 0) {
+ *tcam_idx = ret;
+ ret = 0;
+ }
+
+ return ret;
+}
+
+/*
* link_start - enable a port
* @dev: the port to enable
*
@@ -440,25 +502,20 @@ static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
*/
static int link_start(struct net_device *dev)
{
- int ret;
struct port_info *pi = netdev_priv(dev);
- unsigned int mb = pi->adapter->pf;
+ unsigned int mb = pi->adapter->mbox;
+ int ret;
/*
* We do not set address filters and promiscuity here, the stack does
* that step explicitly.
*/
- ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
+ ret = t4_set_rxmode(pi->adapter, mb, pi->viid, pi->viid_mirror,
+ dev->mtu, -1, -1, -1,
!!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
- if (ret == 0) {
- ret = t4_change_mac(pi->adapter, mb, pi->viid,
- pi->xact_addr_filt, dev->dev_addr, true,
- &pi->smt_idx);
- if (ret >= 0) {
- pi->xact_addr_filt = ret;
- ret = 0;
- }
- }
+ if (ret == 0)
+ ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt,
+ dev->dev_addr, true, &pi->smt_idx);
if (ret == 0)
ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan,
&pi->link_cfg);
@@ -527,7 +584,7 @@ static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
struct sge_eth_txq *eq;
eq = container_of(txq, struct sge_eth_txq, q);
- netif_tx_wake_queue(eq->txq);
+ t4_sge_eth_txq_egress_update(q->adap, eq, -1);
} else {
struct sge_uld_txq *oq;
@@ -603,12 +660,12 @@ out:
static void disable_msi(struct adapter *adapter)
{
- if (adapter->flags & USING_MSIX) {
+ if (adapter->flags & CXGB4_USING_MSIX) {
pci_disable_msix(adapter->pdev);
- adapter->flags &= ~USING_MSIX;
- } else if (adapter->flags & USING_MSI) {
+ adapter->flags &= ~CXGB4_USING_MSIX;
+ } else if (adapter->flags & CXGB4_USING_MSI) {
pci_disable_msi(adapter->pdev);
- adapter->flags &= ~USING_MSI;
+ adapter->flags &= ~CXGB4_USING_MSI;
}
}
@@ -624,74 +681,170 @@ static irqreturn_t t4_nondata_intr(int irq, void *cookie)
adap->swintr = 1;
t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
}
- if (adap->flags & MASTER_PF)
+ if (adap->flags & CXGB4_MASTER_PF)
t4_slow_intr_handler(adap);
return IRQ_HANDLED;
}
-/*
- * Name the MSI-X interrupts.
- */
-static void name_msix_vecs(struct adapter *adap)
+int cxgb4_set_msix_aff(struct adapter *adap, unsigned short vec,
+ cpumask_var_t *aff_mask, int idx)
{
- int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
+ int rv;
+
+ if (!zalloc_cpumask_var(aff_mask, GFP_KERNEL)) {
+ dev_err(adap->pdev_dev, "alloc_cpumask_var failed\n");
+ return -ENOMEM;
+ }
- /* non-data interrupts */
- snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
+ cpumask_set_cpu(cpumask_local_spread(idx, dev_to_node(adap->pdev_dev)),
+ *aff_mask);
- /* FW events */
- snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
- adap->port[0]->name);
+ rv = irq_set_affinity_hint(vec, *aff_mask);
+ if (rv)
+ dev_warn(adap->pdev_dev,
+ "irq_set_affinity_hint %u failed %d\n",
+ vec, rv);
- /* Ethernet queues */
- for_each_port(adap, j) {
- struct net_device *d = adap->port[j];
- const struct port_info *pi = netdev_priv(d);
+ return 0;
+}
- for (i = 0; i < pi->nqsets; i++, msi_idx++)
- snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
- d->name, i);
- }
+void cxgb4_clear_msix_aff(unsigned short vec, cpumask_var_t aff_mask)
+{
+ irq_set_affinity_hint(vec, NULL);
+ free_cpumask_var(aff_mask);
}
static int request_msix_queue_irqs(struct adapter *adap)
{
struct sge *s = &adap->sge;
+ struct msix_info *minfo;
int err, ethqidx;
- int msi_index = 2;
- err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
- adap->msix_info[1].desc, &s->fw_evtq);
+ if (s->fwevtq_msix_idx < 0)
+ return -ENOMEM;
+
+ err = request_irq(adap->msix_info[s->fwevtq_msix_idx].vec,
+ t4_sge_intr_msix, 0,
+ adap->msix_info[s->fwevtq_msix_idx].desc,
+ &s->fw_evtq);
if (err)
return err;
for_each_ethrxq(s, ethqidx) {
- err = request_irq(adap->msix_info[msi_index].vec,
+ minfo = s->ethrxq[ethqidx].msix;
+ err = request_irq(minfo->vec,
t4_sge_intr_msix, 0,
- adap->msix_info[msi_index].desc,
+ minfo->desc,
&s->ethrxq[ethqidx].rspq);
if (err)
goto unwind;
- msi_index++;
+
+ cxgb4_set_msix_aff(adap, minfo->vec,
+ &minfo->aff_mask, ethqidx);
}
return 0;
unwind:
- while (--ethqidx >= 0)
- free_irq(adap->msix_info[--msi_index].vec,
- &s->ethrxq[ethqidx].rspq);
- free_irq(adap->msix_info[1].vec, &s->fw_evtq);
+ while (--ethqidx >= 0) {
+ minfo = s->ethrxq[ethqidx].msix;
+ cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask);
+ free_irq(minfo->vec, &s->ethrxq[ethqidx].rspq);
+ }
+ free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq);
return err;
}
static void free_msix_queue_irqs(struct adapter *adap)
{
- int i, msi_index = 2;
struct sge *s = &adap->sge;
+ struct msix_info *minfo;
+ int i;
+
+ free_irq(adap->msix_info[s->fwevtq_msix_idx].vec, &s->fw_evtq);
+ for_each_ethrxq(s, i) {
+ minfo = s->ethrxq[i].msix;
+ cxgb4_clear_msix_aff(minfo->vec, minfo->aff_mask);
+ free_irq(minfo->vec, &s->ethrxq[i].rspq);
+ }
+}
+
+static int setup_ppod_edram(struct adapter *adap)
+{
+ unsigned int param, val;
+ int ret;
+
+ /* Driver sends FW_PARAMS_PARAM_DEV_PPOD_EDRAM read command to check
+ * if firmware supports ppod edram feature or not. If firmware
+ * returns 1, then driver can enable this feature by sending
+ * FW_PARAMS_PARAM_DEV_PPOD_EDRAM write command with value 1 to
+ * enable ppod edram feature.
+ */
+ param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
+ FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PPOD_EDRAM));
+
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &param, &val);
+ if (ret < 0) {
+ dev_warn(adap->pdev_dev,
+ "querying PPOD_EDRAM support failed: %d\n",
+ ret);
+ return -1;
+ }
- free_irq(adap->msix_info[1].vec, &s->fw_evtq);
- for_each_ethrxq(s, i)
- free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
+ if (val != 1)
+ return -1;
+
+ ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, &param, &val);
+ if (ret < 0) {
+ dev_err(adap->pdev_dev,
+ "setting PPOD_EDRAM failed: %d\n", ret);
+ return -1;
+ }
+ return 0;
+}
+
+static void adap_config_hpfilter(struct adapter *adapter)
+{
+ u32 param, val = 0;
+ int ret;
+
+ /* Enable HP filter region. Older fw will fail this request and
+ * it is fine.
+ */
+ param = FW_PARAM_DEV(HPFILTER_REGION_SUPPORT);
+ ret = t4_set_params(adapter, adapter->mbox, adapter->pf, 0,
+ 1, &param, &val);
+
+ /* An error means FW doesn't know about HP filter support,
+ * it's not a problem, don't return an error.
+ */
+ if (ret < 0)
+ dev_err(adapter->pdev_dev,
+ "HP filter region isn't supported by FW\n");
+}
+
+static int cxgb4_config_rss(const struct port_info *pi, u16 *rss,
+ u16 rss_size, u16 viid)
+{
+ struct adapter *adap = pi->adapter;
+ int ret;
+
+ ret = t4_config_rss_range(adap, adap->mbox, viid, 0, rss_size, rss,
+ rss_size);
+ if (ret)
+ return ret;
+
+ /* If Tunnel All Lookup isn't specified in the global RSS
+ * Configuration, then we need to specify a default Ingress
+ * Queue for any ingress packets which aren't hashed. We'll
+ * use our first ingress queue ...
+ */
+ return t4_config_vi_rss(adap, adap->mbox, viid,
+ FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F |
+ FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F |
+ FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F |
+ FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F |
+ FW_RSS_VI_CONFIG_CMD_UDPEN_F,
+ rss[0]);
}
/**
@@ -705,10 +858,10 @@ static void free_msix_queue_irqs(struct adapter *adap)
*/
int cxgb4_write_rss(const struct port_info *pi, const u16 *queues)
{
- u16 *rss;
- int i, err;
struct adapter *adapter = pi->adapter;
const struct sge_eth_rxq *rxq;
+ int i, err;
+ u16 *rss;
rxq = &adapter->sge.ethrxq[pi->first_qset];
rss = kmalloc_array(pi->rss_size, sizeof(u16), GFP_KERNEL);
@@ -719,21 +872,7 @@ int cxgb4_write_rss(const struct port_info *pi, const u16 *queues)
for (i = 0; i < pi->rss_size; i++, queues++)
rss[i] = rxq[*queues].rspq.abs_id;
- err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0,
- pi->rss_size, rss, pi->rss_size);
- /* If Tunnel All Lookup isn't specified in the global RSS
- * Configuration, then we need to specify a default Ingress
- * Queue for any ingress packets which aren't hashed. We'll
- * use our first ingress queue ...
- */
- if (!err)
- err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid,
- FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F |
- FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F |
- FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F |
- FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F |
- FW_RSS_VI_CONFIG_CMD_UDPEN_F,
- rss[0]);
+ err = cxgb4_config_rss(pi, rss, pi->rss_size, pi->viid);
kfree(rss);
return err;
}
@@ -771,6 +910,12 @@ static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
}
+void cxgb4_quiesce_rx(struct sge_rspq *q)
+{
+ if (q->handler)
+ napi_disable(&q->napi);
+}
+
/*
* Wait until all NAPI handlers are descheduled.
*/
@@ -781,19 +926,24 @@ static void quiesce_rx(struct adapter *adap)
for (i = 0; i < adap->sge.ingr_sz; i++) {
struct sge_rspq *q = adap->sge.ingr_map[i];
- if (q && q->handler)
- napi_disable(&q->napi);
+ if (!q)
+ continue;
+
+ cxgb4_quiesce_rx(q);
}
}
/* Disable interrupt and napi handler */
static void disable_interrupts(struct adapter *adap)
{
- if (adap->flags & FULL_INIT_DONE) {
+ struct sge *s = &adap->sge;
+
+ if (adap->flags & CXGB4_FULL_INIT_DONE) {
t4_intr_disable(adap);
- if (adap->flags & USING_MSIX) {
+ if (adap->flags & CXGB4_USING_MSIX) {
free_msix_queue_irqs(adap);
- free_irq(adap->msix_info[0].vec, adap);
+ free_irq(adap->msix_info[s->nd_msix_idx].vec,
+ adap);
} else {
free_irq(adap->pdev->irq, adap);
}
@@ -801,6 +951,17 @@ static void disable_interrupts(struct adapter *adap)
}
}
+void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q)
+{
+ if (q->handler)
+ napi_enable(&q->napi);
+
+ /* 0-increment GTS to start the timer and enable interrupts */
+ t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
+ SEINTARM_V(q->intr_params) |
+ INGRESSQID_V(q->cntxt_id));
+}
+
/*
* Enable NAPI scheduling and interrupt generation for all Rx queues.
*/
@@ -813,37 +974,63 @@ static void enable_rx(struct adapter *adap)
if (!q)
continue;
- if (q->handler)
- napi_enable(&q->napi);
- /* 0-increment GTS to start the timer and enable interrupts */
- t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
- SEINTARM_V(q->intr_params) |
- INGRESSQID_V(q->cntxt_id));
+ cxgb4_enable_rx(adap, q);
}
}
+static int setup_non_data_intr(struct adapter *adap)
+{
+ int msix;
+
+ adap->sge.nd_msix_idx = -1;
+ if (!(adap->flags & CXGB4_USING_MSIX))
+ return 0;
+
+ /* Request MSI-X vector for non-data interrupt */
+ msix = cxgb4_get_msix_idx_from_bmap(adap);
+ if (msix < 0)
+ return -ENOMEM;
+
+ snprintf(adap->msix_info[msix].desc,
+ sizeof(adap->msix_info[msix].desc),
+ "%s", adap->port[0]->name);
+
+ adap->sge.nd_msix_idx = msix;
+ return 0;
+}
static int setup_fw_sge_queues(struct adapter *adap)
{
struct sge *s = &adap->sge;
- int err = 0;
+ int msix, err = 0;
bitmap_zero(s->starving_fl, s->egr_sz);
bitmap_zero(s->txq_maperr, s->egr_sz);
- if (adap->flags & USING_MSIX)
- adap->msi_idx = 1; /* vector 0 is for non-queue interrupts */
- else {
+ if (adap->flags & CXGB4_USING_MSIX) {
+ s->fwevtq_msix_idx = -1;
+ msix = cxgb4_get_msix_idx_from_bmap(adap);
+ if (msix < 0)
+ return -ENOMEM;
+
+ snprintf(adap->msix_info[msix].desc,
+ sizeof(adap->msix_info[msix].desc),
+ "%s-FWeventq", adap->port[0]->name);
+ } else {
err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
NULL, NULL, NULL, -1);
if (err)
return err;
- adap->msi_idx = -((int)s->intrq.abs_id + 1);
+ msix = -((int)s->intrq.abs_id + 1);
}
err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
- adap->msi_idx, NULL, fwevtq_handler, NULL, -1);
+ msix, NULL, fwevtq_handler, NULL, -1);
+ if (err && msix >= 0)
+ cxgb4_free_msix_idx_in_bmap(adap, msix);
+
+ s->fwevtq_msix_idx = msix;
return err;
}
@@ -857,14 +1044,17 @@ static int setup_fw_sge_queues(struct adapter *adap)
*/
static int setup_sge_queues(struct adapter *adap)
{
- int err, i, j;
- struct sge *s = &adap->sge;
struct sge_uld_rxq_info *rxq_info = NULL;
+ struct sge *s = &adap->sge;
unsigned int cmplqid = 0;
+ int err, i, j, msix = 0;
if (is_uld(adap))
rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA];
+ if (!(adap->flags & CXGB4_USING_MSIX))
+ msix = -((int)s->intrq.abs_id + 1);
+
for_each_port(adap, i) {
struct net_device *dev = adap->port[i];
struct port_info *pi = netdev_priv(dev);
@@ -872,10 +1062,21 @@ static int setup_sge_queues(struct adapter *adap)
struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
for (j = 0; j < pi->nqsets; j++, q++) {
- if (adap->msi_idx > 0)
- adap->msi_idx++;
+ if (msix >= 0) {
+ msix = cxgb4_get_msix_idx_from_bmap(adap);
+ if (msix < 0) {
+ err = msix;
+ goto freeout;
+ }
+
+ snprintf(adap->msix_info[msix].desc,
+ sizeof(adap->msix_info[msix].desc),
+ "%s-Rx%d", dev->name, j);
+ q->msix = &adap->msix_info[msix];
+ }
+
err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
- adap->msi_idx, &q->fl,
+ msix, &q->fl,
t4_ethrx_handler,
NULL,
t4_get_tp_ch_map(adap,
@@ -885,10 +1086,13 @@ static int setup_sge_queues(struct adapter *adap)
q->rspq.idx = j;
memset(&q->stats, 0, sizeof(q->stats));
}
- for (j = 0; j < pi->nqsets; j++, t++) {
+
+ q = &s->ethrxq[pi->first_qset];
+ for (j = 0; j < pi->nqsets; j++, t++, q++) {
err = t4_sge_alloc_eth_txq(adap, t, dev,
netdev_get_tx_queue(dev, j),
- s->fw_evtq.cntxt_id);
+ q->rspq.cntxt_id,
+ !!(adap->flags & CXGB4_SGE_DBQ_TIMER));
if (err)
goto freeout;
}
@@ -910,7 +1114,7 @@ static int setup_sge_queues(struct adapter *adap)
if (!is_t4(adap->params.chip)) {
err = t4_sge_alloc_eth_txq(adap, &s->ptptxq, adap->port[0],
netdev_get_tx_queue(adap->port[0], 0)
- , s->fw_evtq.cntxt_id);
+ , s->fw_evtq.cntxt_id, false);
if (err)
goto freeout;
}
@@ -928,8 +1132,7 @@ freeout:
}
static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
- struct net_device *sb_dev,
- select_queue_fallback_t fallback)
+ struct net_device *sb_dev)
{
int txq;
@@ -960,6 +1163,25 @@ static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
}
#endif /* CONFIG_CHELSIO_T4_DCB */
+ if (dev->num_tc) {
+ struct port_info *pi = netdev2pinfo(dev);
+ u8 ver, proto;
+
+ ver = ip_hdr(skb)->version;
+ proto = (ver == 6) ? ipv6_hdr(skb)->nexthdr :
+ ip_hdr(skb)->protocol;
+
+ /* Send unsupported traffic pattern to normal NIC queues. */
+ txq = netdev_pick_tx(dev, skb, sb_dev);
+ if (xfrm_offload(skb) || is_ptp_enabled(skb, dev) ||
+ skb->encapsulation ||
+ tls_is_skb_tx_device_offloaded(skb) ||
+ (proto != IPPROTO_TCP && proto != IPPROTO_UDP))
+ txq = txq % pi->nqsets;
+
+ return txq;
+ }
+
if (select_queue) {
txq = (skb_rx_queue_recorded(skb)
? skb_get_rx_queue(skb)
@@ -971,7 +1193,7 @@ static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
return txq;
}
- return fallback(dev, skb, NULL) % dev->real_num_tx_queues;
+ return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
}
static int closest_timer(const struct sge *s, int time)
@@ -1049,15 +1271,15 @@ int cxgb4_set_rspq_intr_params(struct sge_rspq *q,
static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
{
- const struct port_info *pi = netdev_priv(dev);
netdev_features_t changed = dev->features ^ features;
+ const struct port_info *pi = netdev_priv(dev);
int err;
if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
return 0;
- err = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, -1,
- -1, -1, -1,
+ err = t4_set_rxmode(pi->adapter, pi->adapter->mbox, pi->viid,
+ pi->viid_mirror, -1, -1, -1, -1,
!!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
if (unlikely(err))
dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
@@ -1075,6 +1297,292 @@ static int setup_debugfs(struct adapter *adap)
return 0;
}
+static void cxgb4_port_mirror_free_rxq(struct adapter *adap,
+ struct sge_eth_rxq *mirror_rxq)
+{
+ if ((adap->flags & CXGB4_FULL_INIT_DONE) &&
+ !(adap->flags & CXGB4_SHUTTING_DOWN))
+ cxgb4_quiesce_rx(&mirror_rxq->rspq);
+
+ if (adap->flags & CXGB4_USING_MSIX) {
+ cxgb4_clear_msix_aff(mirror_rxq->msix->vec,
+ mirror_rxq->msix->aff_mask);
+ free_irq(mirror_rxq->msix->vec, &mirror_rxq->rspq);
+ cxgb4_free_msix_idx_in_bmap(adap, mirror_rxq->msix->idx);
+ }
+
+ free_rspq_fl(adap, &mirror_rxq->rspq, &mirror_rxq->fl);
+}
+
+static int cxgb4_port_mirror_alloc_queues(struct net_device *dev)
+{
+ struct port_info *pi = netdev2pinfo(dev);
+ struct adapter *adap = netdev2adap(dev);
+ struct sge_eth_rxq *mirror_rxq;
+ struct sge *s = &adap->sge;
+ int ret = 0, msix = 0;
+ u16 i, rxqid;
+ u16 *rss;
+
+ if (!pi->vi_mirror_count)
+ return 0;
+
+ if (s->mirror_rxq[pi->port_id])
+ return 0;
+
+ mirror_rxq = kcalloc(pi->nmirrorqsets, sizeof(*mirror_rxq), GFP_KERNEL);
+ if (!mirror_rxq)
+ return -ENOMEM;
+
+ s->mirror_rxq[pi->port_id] = mirror_rxq;
+
+ if (!(adap->flags & CXGB4_USING_MSIX))
+ msix = -((int)adap->sge.intrq.abs_id + 1);
+
+ for (i = 0, rxqid = 0; i < pi->nmirrorqsets; i++, rxqid++) {
+ mirror_rxq = &s->mirror_rxq[pi->port_id][i];
+
+ /* Allocate Mirror Rxqs */
+ if (msix >= 0) {
+ msix = cxgb4_get_msix_idx_from_bmap(adap);
+ if (msix < 0) {
+ ret = msix;
+ goto out_free_queues;
+ }
+
+ mirror_rxq->msix = &adap->msix_info[msix];
+ snprintf(mirror_rxq->msix->desc,
+ sizeof(mirror_rxq->msix->desc),
+ "%s-mirrorrxq%d", dev->name, i);
+ }
+
+ init_rspq(adap, &mirror_rxq->rspq,
+ CXGB4_MIRROR_RXQ_DEFAULT_INTR_USEC,
+ CXGB4_MIRROR_RXQ_DEFAULT_PKT_CNT,
+ CXGB4_MIRROR_RXQ_DEFAULT_DESC_NUM,
+ CXGB4_MIRROR_RXQ_DEFAULT_DESC_SIZE);
+
+ mirror_rxq->fl.size = CXGB4_MIRROR_FLQ_DEFAULT_DESC_NUM;
+
+ ret = t4_sge_alloc_rxq(adap, &mirror_rxq->rspq, false,
+ dev, msix, &mirror_rxq->fl,
+ t4_ethrx_handler, NULL, 0);
+ if (ret)
+ goto out_free_msix_idx;
+
+ /* Setup MSI-X vectors for Mirror Rxqs */
+ if (adap->flags & CXGB4_USING_MSIX) {
+ ret = request_irq(mirror_rxq->msix->vec,
+ t4_sge_intr_msix, 0,
+ mirror_rxq->msix->desc,
+ &mirror_rxq->rspq);
+ if (ret)
+ goto out_free_rxq;
+
+ cxgb4_set_msix_aff(adap, mirror_rxq->msix->vec,
+ &mirror_rxq->msix->aff_mask, i);
+ }
+
+ /* Start NAPI for Mirror Rxqs */
+ cxgb4_enable_rx(adap, &mirror_rxq->rspq);
+ }
+
+ /* Setup RSS for Mirror Rxqs */
+ rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
+ if (!rss) {
+ ret = -ENOMEM;
+ goto out_free_queues;
+ }
+
+ mirror_rxq = &s->mirror_rxq[pi->port_id][0];
+ for (i = 0; i < pi->rss_size; i++)
+ rss[i] = mirror_rxq[i % pi->nmirrorqsets].rspq.abs_id;
+
+ ret = cxgb4_config_rss(pi, rss, pi->rss_size, pi->viid_mirror);
+ kfree(rss);
+ if (ret)
+ goto out_free_queues;
+
+ return 0;
+
+out_free_rxq:
+ free_rspq_fl(adap, &mirror_rxq->rspq, &mirror_rxq->fl);
+
+out_free_msix_idx:
+ cxgb4_free_msix_idx_in_bmap(adap, mirror_rxq->msix->idx);
+
+out_free_queues:
+ while (rxqid-- > 0)
+ cxgb4_port_mirror_free_rxq(adap,
+ &s->mirror_rxq[pi->port_id][rxqid]);
+
+ kfree(s->mirror_rxq[pi->port_id]);
+ s->mirror_rxq[pi->port_id] = NULL;
+ return ret;
+}
+
+static void cxgb4_port_mirror_free_queues(struct net_device *dev)
+{
+ struct port_info *pi = netdev2pinfo(dev);
+ struct adapter *adap = netdev2adap(dev);
+ struct sge *s = &adap->sge;
+ u16 i;
+
+ if (!pi->vi_mirror_count)
+ return;
+
+ if (!s->mirror_rxq[pi->port_id])
+ return;
+
+ for (i = 0; i < pi->nmirrorqsets; i++)
+ cxgb4_port_mirror_free_rxq(adap,
+ &s->mirror_rxq[pi->port_id][i]);
+
+ kfree(s->mirror_rxq[pi->port_id]);
+ s->mirror_rxq[pi->port_id] = NULL;
+}
+
+static int cxgb4_port_mirror_start(struct net_device *dev)
+{
+ struct port_info *pi = netdev2pinfo(dev);
+ struct adapter *adap = netdev2adap(dev);
+ int ret, idx = -1;
+
+ if (!pi->vi_mirror_count)
+ return 0;
+
+ /* Mirror VIs can be created dynamically after stack had
+ * already setup Rx modes like MTU, promisc, allmulti, etc.
+ * on main VI. So, parse what the stack had setup on the
+ * main VI and update the same on the mirror VI.
+ */
+ ret = t4_set_rxmode(adap, adap->mbox, pi->viid, pi->viid_mirror,
+ dev->mtu, (dev->flags & IFF_PROMISC) ? 1 : 0,
+ (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1,
+ !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
+ if (ret) {
+ dev_err(adap->pdev_dev,
+ "Failed start up Rx mode for Mirror VI 0x%x, ret: %d\n",
+ pi->viid_mirror, ret);
+ return ret;
+ }
+
+ /* Enable replication bit for the device's MAC address
+ * in MPS TCAM, so that the packets for the main VI are
+ * replicated to mirror VI.
+ */
+ ret = cxgb4_update_mac_filt(pi, pi->viid_mirror, &idx,
+ dev->dev_addr, true, NULL);
+ if (ret) {
+ dev_err(adap->pdev_dev,
+ "Failed updating MAC filter for Mirror VI 0x%x, ret: %d\n",
+ pi->viid_mirror, ret);
+ return ret;
+ }
+
+ /* Enabling a Virtual Interface can result in an interrupt
+ * during the processing of the VI Enable command and, in some
+ * paths, result in an attempt to issue another command in the
+ * interrupt context. Thus, we disable interrupts during the
+ * course of the VI Enable command ...
+ */
+ local_bh_disable();
+ ret = t4_enable_vi_params(adap, adap->mbox, pi->viid_mirror, true, true,
+ false);
+ local_bh_enable();
+ if (ret)
+ dev_err(adap->pdev_dev,
+ "Failed starting Mirror VI 0x%x, ret: %d\n",
+ pi->viid_mirror, ret);
+
+ return ret;
+}
+
+static void cxgb4_port_mirror_stop(struct net_device *dev)
+{
+ struct port_info *pi = netdev2pinfo(dev);
+ struct adapter *adap = netdev2adap(dev);
+
+ if (!pi->vi_mirror_count)
+ return;
+
+ t4_enable_vi_params(adap, adap->mbox, pi->viid_mirror, false, false,
+ false);
+}
+
+int cxgb4_port_mirror_alloc(struct net_device *dev)
+{
+ struct port_info *pi = netdev2pinfo(dev);
+ struct adapter *adap = netdev2adap(dev);
+ int ret = 0;
+
+ if (!pi->nmirrorqsets)
+ return -EOPNOTSUPP;
+
+ mutex_lock(&pi->vi_mirror_mutex);
+ if (pi->viid_mirror) {
+ pi->vi_mirror_count++;
+ goto out_unlock;
+ }
+
+ ret = t4_init_port_mirror(pi, adap->mbox, pi->port_id, adap->pf, 0,
+ &pi->viid_mirror);
+ if (ret)
+ goto out_unlock;
+
+ pi->vi_mirror_count = 1;
+
+ if (adap->flags & CXGB4_FULL_INIT_DONE) {
+ ret = cxgb4_port_mirror_alloc_queues(dev);
+ if (ret)
+ goto out_free_vi;
+
+ ret = cxgb4_port_mirror_start(dev);
+ if (ret)
+ goto out_free_queues;
+ }
+
+ mutex_unlock(&pi->vi_mirror_mutex);
+ return 0;
+
+out_free_queues:
+ cxgb4_port_mirror_free_queues(dev);
+
+out_free_vi:
+ pi->vi_mirror_count = 0;
+ t4_free_vi(adap, adap->mbox, adap->pf, 0, pi->viid_mirror);
+ pi->viid_mirror = 0;
+
+out_unlock:
+ mutex_unlock(&pi->vi_mirror_mutex);
+ return ret;
+}
+
+void cxgb4_port_mirror_free(struct net_device *dev)
+{
+ struct port_info *pi = netdev2pinfo(dev);
+ struct adapter *adap = netdev2adap(dev);
+
+ mutex_lock(&pi->vi_mirror_mutex);
+ if (!pi->viid_mirror)
+ goto out_unlock;
+
+ if (pi->vi_mirror_count > 1) {
+ pi->vi_mirror_count--;
+ goto out_unlock;
+ }
+
+ cxgb4_port_mirror_stop(dev);
+ cxgb4_port_mirror_free_queues(dev);
+
+ pi->vi_mirror_count = 0;
+ t4_free_vi(adap, adap->mbox, adap->pf, 0, pi->viid_mirror);
+ pi->viid_mirror = 0;
+
+out_unlock:
+ mutex_unlock(&pi->vi_mirror_mutex);
+}
+
/*
* upper-layer driver support
*/
@@ -1235,8 +1743,8 @@ static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
unsigned int tid)
{
- void **p = &t->tid_tab[tid];
struct adapter *adap = container_of(t, struct adapter, tids);
+ void **p = &t->tid_tab[tid - t->tid_base];
spin_lock_bh(&adap->tid_release_lock);
*p = adap->tid_release_head;
@@ -1288,13 +1796,16 @@ static void process_tid_release_list(struct work_struct *work)
void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid,
unsigned short family)
{
- struct sk_buff *skb;
struct adapter *adap = container_of(t, struct adapter, tids);
+ struct sk_buff *skb;
- WARN_ON(tid >= t->ntids);
+ if (tid_out_of_range(&adap->tids, tid)) {
+ dev_err(adap->pdev_dev, "tid %d out of range\n", tid);
+ return;
+ }
- if (t->tid_tab[tid]) {
- t->tid_tab[tid] = NULL;
+ if (t->tid_tab[tid - adap->tids.tid_base]) {
+ t->tid_tab[tid - adap->tids.tid_base] = NULL;
atomic_dec(&t->conns_in_use);
if (t->hash_base && (tid >= t->hash_base)) {
if (family == AF_INET6)
@@ -1326,19 +1837,27 @@ static int tid_init(struct tid_info *t)
struct adapter *adap = container_of(t, struct adapter, tids);
unsigned int max_ftids = t->nftids + t->nsftids;
unsigned int natids = t->natids;
+ unsigned int hpftid_bmap_size;
+ unsigned int eotid_bmap_size;
unsigned int stid_bmap_size;
unsigned int ftid_bmap_size;
size_t size;
stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
ftid_bmap_size = BITS_TO_LONGS(t->nftids);
+ hpftid_bmap_size = BITS_TO_LONGS(t->nhpftids);
+ eotid_bmap_size = BITS_TO_LONGS(t->neotids);
size = t->ntids * sizeof(*t->tid_tab) +
natids * sizeof(*t->atid_tab) +
t->nstids * sizeof(*t->stid_tab) +
t->nsftids * sizeof(*t->stid_tab) +
stid_bmap_size * sizeof(long) +
+ t->nhpftids * sizeof(*t->hpftid_tab) +
+ hpftid_bmap_size * sizeof(long) +
max_ftids * sizeof(*t->ftid_tab) +
- ftid_bmap_size * sizeof(long);
+ ftid_bmap_size * sizeof(long) +
+ t->neotids * sizeof(*t->eotid_tab) +
+ eotid_bmap_size * sizeof(long);
t->tid_tab = kvzalloc(size, GFP_KERNEL);
if (!t->tid_tab)
@@ -1347,8 +1866,12 @@ static int tid_init(struct tid_info *t)
t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
- t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
+ t->hpftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
+ t->hpftid_bmap = (unsigned long *)&t->hpftid_tab[t->nhpftids];
+ t->ftid_tab = (struct filter_entry *)&t->hpftid_bmap[hpftid_bmap_size];
t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids];
+ t->eotid_tab = (struct eotid_entry *)&t->ftid_bmap[ftid_bmap_size];
+ t->eotid_bmap = (unsigned long *)&t->eotid_tab[t->neotids];
spin_lock_init(&t->stid_lock);
spin_lock_init(&t->atid_lock);
spin_lock_init(&t->ftid_lock);
@@ -1361,6 +1884,7 @@ static int tid_init(struct tid_info *t)
atomic_set(&t->tids_in_use, 0);
atomic_set(&t->conns_in_use, 0);
atomic_set(&t->hash_tids_in_use, 0);
+ atomic_set(&t->eotids_in_use, 0);
/* Setup the free list for atid_tab and clear the stid bitmap. */
if (natids) {
@@ -1375,8 +1899,13 @@ static int tid_init(struct tid_info *t)
if (!t->stid_base &&
CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
__set_bit(0, t->stid_bmap);
+
+ if (t->neotids)
+ bitmap_zero(t->eotid_bmap, t->neotids);
}
+ if (t->nhpftids)
+ bitmap_zero(t->hpftid_bmap, t->nhpftids);
bitmap_zero(t->ftid_bmap, t->nftids);
return 0;
}
@@ -1387,6 +1916,7 @@ static int tid_init(struct tid_info *t)
* @stid: the server TID
* @sip: local IP address to bind server to
* @sport: the server's TCP port
+ * @vlan: the VLAN header information
* @queue: queue to direct messages from this server to
*
* Create an IP server for the given port and address.
@@ -1596,6 +2126,18 @@ unsigned int cxgb4_port_chan(const struct net_device *dev)
}
EXPORT_SYMBOL(cxgb4_port_chan);
+/**
+ * cxgb4_port_e2cchan - get the HW c-channel of a port
+ * @dev: the net device for the port
+ *
+ * Return the HW RX c-channel of the given port.
+ */
+unsigned int cxgb4_port_e2cchan(const struct net_device *dev)
+{
+ return netdev2pinfo(dev)->rx_cchan;
+}
+EXPORT_SYMBOL(cxgb4_port_e2cchan);
+
unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
{
struct adapter *adap = netdev2adap(dev);
@@ -1649,18 +2191,6 @@ void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
}
EXPORT_SYMBOL(cxgb4_get_tcp_stats);
-void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
- const unsigned int *pgsz_order)
-{
- struct adapter *adap = netdev2adap(dev);
-
- t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
- t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
- HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
- HPZ3_V(pgsz_order[3]));
-}
-EXPORT_SYMBOL(cxgb4_iscsi_init);
-
int cxgb4_flush_eq_cache(struct net_device *dev)
{
struct adapter *adap = netdev2adap(dev);
@@ -2103,6 +2633,9 @@ static void detach_ulds(struct adapter *adap)
{
unsigned int i;
+ if (!is_uld(adap))
+ return;
+
mutex_lock(&uld_mutex);
list_del(&adap->list_node);
@@ -2219,6 +2752,7 @@ static void update_clip(const struct adapter *adap)
*/
static int cxgb_up(struct adapter *adap)
{
+ struct sge *s = &adap->sge;
int err;
mutex_lock(&uld_mutex);
@@ -2229,20 +2763,25 @@ static int cxgb_up(struct adapter *adap)
if (err)
goto freeq;
- if (adap->flags & USING_MSIX) {
- name_msix_vecs(adap);
- err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
- adap->msix_info[0].desc, adap);
+ if (adap->flags & CXGB4_USING_MSIX) {
+ if (s->nd_msix_idx < 0) {
+ err = -ENOMEM;
+ goto irq_err;
+ }
+
+ err = request_irq(adap->msix_info[s->nd_msix_idx].vec,
+ t4_nondata_intr, 0,
+ adap->msix_info[s->nd_msix_idx].desc, adap);
if (err)
goto irq_err;
+
err = request_msix_queue_irqs(adap);
- if (err) {
- free_irq(adap->msix_info[0].vec, adap);
- goto irq_err;
- }
+ if (err)
+ goto irq_err_free_nd_msix;
} else {
err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
- (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
+ (adap->flags & CXGB4_USING_MSI) ? 0
+ : IRQF_SHARED,
adap->port[0]->name, adap);
if (err)
goto irq_err;
@@ -2251,7 +2790,7 @@ static int cxgb_up(struct adapter *adap)
enable_rx(adap);
t4_sge_start(adap);
t4_intr_enable(adap);
- adap->flags |= FULL_INIT_DONE;
+ adap->flags |= CXGB4_FULL_INIT_DONE;
mutex_unlock(&uld_mutex);
notify_ulds(adap, CXGB4_STATE_UP);
@@ -2260,11 +2799,13 @@ static int cxgb_up(struct adapter *adap)
#endif
return err;
- irq_err:
+irq_err_free_nd_msix:
+ free_irq(adap->msix_info[s->nd_msix_idx].vec, adap);
+irq_err:
dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
- freeq:
+freeq:
t4_free_sge_resources(adap);
- rel_lock:
+rel_lock:
mutex_unlock(&uld_mutex);
return err;
}
@@ -2280,7 +2821,7 @@ static void cxgb_down(struct adapter *adapter)
t4_sge_stop(adapter);
t4_free_sge_resources(adapter);
- adapter->flags &= ~FULL_INIT_DONE;
+ adapter->flags &= ~CXGB4_FULL_INIT_DONE;
}
/*
@@ -2288,13 +2829,13 @@ static void cxgb_down(struct adapter *adapter)
*/
static int cxgb_open(struct net_device *dev)
{
- int err;
struct port_info *pi = netdev_priv(dev);
struct adapter *adapter = pi->adapter;
+ int err;
netif_carrier_off(dev);
- if (!(adapter->flags & FULL_INIT_DONE)) {
+ if (!(adapter->flags & CXGB4_FULL_INIT_DONE)) {
err = cxgb_up(adapter);
if (err < 0)
return err;
@@ -2308,8 +2849,29 @@ static int cxgb_open(struct net_device *dev)
return err;
err = link_start(dev);
- if (!err)
- netif_tx_start_all_queues(dev);
+ if (err)
+ return err;
+
+ if (pi->nmirrorqsets) {
+ mutex_lock(&pi->vi_mirror_mutex);
+ err = cxgb4_port_mirror_alloc_queues(dev);
+ if (err)
+ goto out_unlock;
+
+ err = cxgb4_port_mirror_start(dev);
+ if (err)
+ goto out_free_queues;
+ mutex_unlock(&pi->vi_mirror_mutex);
+ }
+
+ netif_tx_start_all_queues(dev);
+ return 0;
+
+out_free_queues:
+ cxgb4_port_mirror_free_queues(dev);
+
+out_unlock:
+ mutex_unlock(&pi->vi_mirror_mutex);
return err;
}
@@ -2327,7 +2889,17 @@ static int cxgb_close(struct net_device *dev)
cxgb4_dcb_reset(dev);
dcb_tx_queue_prio_enable(dev, false);
#endif
- return ret;
+ if (ret)
+ return ret;
+
+ if (pi->nmirrorqsets) {
+ mutex_lock(&pi->vi_mirror_mutex);
+ cxgb4_port_mirror_stop(dev);
+ cxgb4_port_mirror_free_queues(dev);
+ mutex_unlock(&pi->vi_mirror_mutex);
+ }
+
+ return 0;
}
int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
@@ -2361,7 +2933,7 @@ int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
/* Clear out filter specifications */
memset(&f->fs, 0, sizeof(struct ch_filter_specification));
- f->fs.val.lport = cpu_to_be16(sport);
+ f->fs.val.lport = be16_to_cpu(sport);
f->fs.mask.lport = ~0;
val = (u8 *)&sip;
if ((val[0] | val[1] | val[2] | val[3]) != 0) {
@@ -2470,12 +3042,87 @@ static void cxgb_get_stats(struct net_device *dev,
ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
}
+static int cxgb_hwtstamp_get(struct net_device *dev,
+ struct kernel_hwtstamp_config *config)
+{
+ struct port_info *pi = netdev_priv(dev);
+
+ *config = pi->tstamp_config;
+ return 0;
+}
+
+static int cxgb_hwtstamp_set(struct net_device *dev,
+ struct kernel_hwtstamp_config *config,
+ struct netlink_ext_ack *extack)
+{
+ struct port_info *pi = netdev_priv(dev);
+ struct adapter *adapter = pi->adapter;
+
+ if (is_t4(adapter->params.chip)) {
+ /* For T4 Adapters */
+ switch (config->rx_filter) {
+ case HWTSTAMP_FILTER_NONE:
+ pi->rxtstamp = false;
+ break;
+ case HWTSTAMP_FILTER_ALL:
+ pi->rxtstamp = true;
+ break;
+ default:
+ return -ERANGE;
+ }
+ pi->tstamp_config = *config;
+ return 0;
+ }
+
+ switch (config->tx_type) {
+ case HWTSTAMP_TX_OFF:
+ case HWTSTAMP_TX_ON:
+ break;
+ default:
+ return -ERANGE;
+ }
+
+ switch (config->rx_filter) {
+ case HWTSTAMP_FILTER_NONE:
+ pi->rxtstamp = false;
+ break;
+ case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
+ case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+ cxgb4_ptprx_timestamping(pi, pi->port_id, PTP_TS_L4);
+ break;
+ case HWTSTAMP_FILTER_PTP_V2_EVENT:
+ cxgb4_ptprx_timestamping(pi, pi->port_id, PTP_TS_L2_L4);
+ break;
+ case HWTSTAMP_FILTER_ALL:
+ case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
+ case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
+ case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+ case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+ pi->rxtstamp = true;
+ break;
+ default:
+ return -ERANGE;
+ }
+
+ if (config->tx_type == HWTSTAMP_TX_OFF &&
+ config->rx_filter == HWTSTAMP_FILTER_NONE) {
+ if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0)
+ pi->ptp_enable = false;
+ }
+
+ if (config->rx_filter != HWTSTAMP_FILTER_NONE) {
+ if (cxgb4_ptp_redirect_rx_packet(adapter, pi) >= 0)
+ pi->ptp_enable = true;
+ }
+ pi->tstamp_config = *config;
+ return 0;
+}
+
static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
{
unsigned int mbox;
int ret = 0, prtad, devad;
struct port_info *pi = netdev_priv(dev);
- struct adapter *adapter = pi->adapter;
struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
switch (cmd) {
@@ -2504,81 +3151,6 @@ static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
data->reg_num, data->val_in);
break;
- case SIOCGHWTSTAMP:
- return copy_to_user(req->ifr_data, &pi->tstamp_config,
- sizeof(pi->tstamp_config)) ?
- -EFAULT : 0;
- case SIOCSHWTSTAMP:
- if (copy_from_user(&pi->tstamp_config, req->ifr_data,
- sizeof(pi->tstamp_config)))
- return -EFAULT;
-
- if (!is_t4(adapter->params.chip)) {
- switch (pi->tstamp_config.tx_type) {
- case HWTSTAMP_TX_OFF:
- case HWTSTAMP_TX_ON:
- break;
- default:
- return -ERANGE;
- }
-
- switch (pi->tstamp_config.rx_filter) {
- case HWTSTAMP_FILTER_NONE:
- pi->rxtstamp = false;
- break;
- case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
- case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
- cxgb4_ptprx_timestamping(pi, pi->port_id,
- PTP_TS_L4);
- break;
- case HWTSTAMP_FILTER_PTP_V2_EVENT:
- cxgb4_ptprx_timestamping(pi, pi->port_id,
- PTP_TS_L2_L4);
- break;
- case HWTSTAMP_FILTER_ALL:
- case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
- case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
- case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
- case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
- pi->rxtstamp = true;
- break;
- default:
- pi->tstamp_config.rx_filter =
- HWTSTAMP_FILTER_NONE;
- return -ERANGE;
- }
-
- if ((pi->tstamp_config.tx_type == HWTSTAMP_TX_OFF) &&
- (pi->tstamp_config.rx_filter ==
- HWTSTAMP_FILTER_NONE)) {
- if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0)
- pi->ptp_enable = false;
- }
-
- if (pi->tstamp_config.rx_filter !=
- HWTSTAMP_FILTER_NONE) {
- if (cxgb4_ptp_redirect_rx_packet(adapter,
- pi) >= 0)
- pi->ptp_enable = true;
- }
- } else {
- /* For T4 Adapters */
- switch (pi->tstamp_config.rx_filter) {
- case HWTSTAMP_FILTER_NONE:
- pi->rxtstamp = false;
- break;
- case HWTSTAMP_FILTER_ALL:
- pi->rxtstamp = true;
- break;
- default:
- pi->tstamp_config.rx_filter =
- HWTSTAMP_FILTER_NONE;
- return -ERANGE;
- }
- }
- return copy_to_user(req->ifr_data, &pi->tstamp_config,
- sizeof(pi->tstamp_config)) ?
- -EFAULT : 0;
default:
return -EOPNOTSUPP;
}
@@ -2593,13 +3165,13 @@ static void cxgb_set_rxmode(struct net_device *dev)
static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
{
- int ret;
struct port_info *pi = netdev_priv(dev);
+ int ret;
- ret = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, new_mtu, -1,
- -1, -1, -1, true);
+ ret = t4_set_rxmode(pi->adapter, pi->adapter->mbox, pi->viid,
+ pi->viid_mirror, new_mtu, -1, -1, -1, -1, true);
if (!ret)
- dev->mtu = new_mtu;
+ WRITE_ONCE(dev->mtu, new_mtu);
return ret;
}
@@ -2622,8 +3194,6 @@ static void cxgb4_mgmt_fill_vf_station_mac_addr(struct adapter *adap)
int err;
u8 *na;
- adap->params.pci.vpd_cap_addr = pci_find_capability(adap->pdev,
- PCI_CAP_ID_VPD);
err = t4_get_raw_vpd_params(adap, &adap->params.vpd);
if (err)
return;
@@ -2667,7 +3237,7 @@ static int cxgb4_mgmt_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
dev_info(pi->adapter->pdev_dev,
"Setting MAC %pM on VF %d\n", mac, vf);
- ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac);
+ ret = t4_set_vf_mac_acl(adap, vf + 1, pi->lport, 1, mac);
if (!ret)
ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac);
return ret;
@@ -2689,6 +3259,7 @@ static int cxgb4_mgmt_get_vf_config(struct net_device *dev,
ivi->min_tx_rate = 0;
ether_addr_copy(ivi->mac, vfinfo->vf_mac_addr);
ivi->vlan = vfinfo->vlan;
+ ivi->linkstate = vfinfo->link_state;
return 0;
}
@@ -2726,7 +3297,7 @@ static int cxgb4_mgmt_set_vf_rate(struct net_device *dev, int vf,
}
if (max_tx_rate == 0) {
- /* unbind VF to to any Traffic Class */
+ /* unbind VF to any Traffic Class */
fw_pfvf =
(FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH));
@@ -2777,7 +3348,7 @@ static int cxgb4_mgmt_set_vf_rate(struct net_device *dev, int vf,
SCHED_CLASS_RATEUNIT_BITS,
SCHED_CLASS_RATEMODE_ABS,
pi->tx_chan, class_id, 0,
- max_tx_rate * 1000, 0, pktsize);
+ max_tx_rate * 1000, 0, pktsize, 0);
if (ret) {
dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n",
ret);
@@ -2828,6 +3399,49 @@ static int cxgb4_mgmt_set_vf_vlan(struct net_device *dev, int vf,
ret, (vlan ? "setting" : "clearing"), adap->pf, vf);
return ret;
}
+
+static int cxgb4_mgmt_set_vf_link_state(struct net_device *dev, int vf,
+ int link)
+{
+ struct port_info *pi = netdev_priv(dev);
+ struct adapter *adap = pi->adapter;
+ u32 param, val;
+ int ret = 0;
+
+ if (vf >= adap->num_vfs)
+ return -EINVAL;
+
+ switch (link) {
+ case IFLA_VF_LINK_STATE_AUTO:
+ val = FW_VF_LINK_STATE_AUTO;
+ break;
+
+ case IFLA_VF_LINK_STATE_ENABLE:
+ val = FW_VF_LINK_STATE_ENABLE;
+ break;
+
+ case IFLA_VF_LINK_STATE_DISABLE:
+ val = FW_VF_LINK_STATE_DISABLE;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
+ FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_LINK_STATE));
+ ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1,
+ &param, &val);
+ if (ret) {
+ dev_err(adap->pdev_dev,
+ "Error %d in setting PF %d VF %d link state\n",
+ ret, adap->pf, vf);
+ return -EINVAL;
+ }
+
+ adap->vfinfo[vf].link_state = link;
+ return ret;
+}
#endif /* CONFIG_PCI_IOV */
static int cxgb_set_mac_addr(struct net_device *dev, void *p)
@@ -2839,14 +3453,12 @@ static int cxgb_set_mac_addr(struct net_device *dev, void *p)
if (!is_valid_ether_addr(addr->sa_data))
return -EADDRNOTAVAIL;
- ret = t4_change_mac(pi->adapter, pi->adapter->pf, pi->viid,
- pi->xact_addr_filt, addr->sa_data, true,
- &pi->smt_idx);
+ ret = cxgb4_update_mac_filt(pi, pi->viid, &pi->xact_addr_filt,
+ addr->sa_data, true, &pi->smt_idx);
if (ret < 0)
return ret;
- memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
- pi->xact_addr_filt = ret;
+ eth_hw_addr_set(dev, addr->sa_data);
return 0;
}
@@ -2856,7 +3468,7 @@ static void cxgb_netpoll(struct net_device *dev)
struct port_info *pi = netdev_priv(dev);
struct adapter *adap = pi->adapter;
- if (adap->flags & USING_MSIX) {
+ if (adap->flags & CXGB4_USING_MSIX) {
int i;
struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
@@ -2871,9 +3483,9 @@ static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
{
struct port_info *pi = netdev_priv(dev);
struct adapter *adap = pi->adapter;
- struct sched_class *e;
- struct ch_sched_params p;
- struct ch_sched_queue qe;
+ struct ch_sched_queue qe = { 0 };
+ struct ch_sched_params p = { 0 };
+ struct ch_sched_class *e;
u32 req_rate;
int err = 0;
@@ -2883,13 +3495,22 @@ static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
if (index < 0 || index > pi->nqsets - 1)
return -EINVAL;
- if (!(adap->flags & FULL_INIT_DONE)) {
+ if (!(adap->flags & CXGB4_FULL_INIT_DONE)) {
dev_err(adap->pdev_dev,
"Failed to rate limit on queue %d. Link Down?\n",
index);
return -EINVAL;
}
+ qe.queue = index;
+ e = cxgb4_sched_queue_lookup(dev, &qe);
+ if (e && e->info.u.params.level != SCHED_CLASS_LEVEL_CL_RL) {
+ dev_err(adap->pdev_dev,
+ "Queue %u already bound to class %u of type: %u\n",
+ index, e->idx, e->info.u.params.level);
+ return -EBUSY;
+ }
+
/* Convert from Mbps to Kbps */
req_rate = rate * 1000;
@@ -2919,7 +3540,6 @@ static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
return 0;
/* Fetch any available unused or matching scheduling class */
- memset(&p, 0, sizeof(p));
p.type = SCHED_CLASS_TYPE_PACKET;
p.u.params.level = SCHED_CLASS_LEVEL_CL_RL;
p.u.params.mode = SCHED_CLASS_MODE_CLASS;
@@ -2949,14 +3569,14 @@ static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
}
static int cxgb_setup_tc_flower(struct net_device *dev,
- struct tc_cls_flower_offload *cls_flower)
+ struct flow_cls_offload *cls_flower)
{
switch (cls_flower->command) {
- case TC_CLSFLOWER_REPLACE:
+ case FLOW_CLS_REPLACE:
return cxgb4_tc_flower_replace(dev, cls_flower);
- case TC_CLSFLOWER_DESTROY:
+ case FLOW_CLS_DESTROY:
return cxgb4_tc_flower_destroy(dev, cls_flower);
- case TC_CLSFLOWER_STATS:
+ case FLOW_CLS_STATS:
return cxgb4_tc_flower_stats(dev, cls_flower);
default:
return -EOPNOTSUPP;
@@ -2977,14 +3597,39 @@ static int cxgb_setup_tc_cls_u32(struct net_device *dev,
}
}
-static int cxgb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
- void *cb_priv)
+static int cxgb_setup_tc_matchall(struct net_device *dev,
+ struct tc_cls_matchall_offload *cls_matchall,
+ bool ingress)
+{
+ struct adapter *adap = netdev2adap(dev);
+
+ if (!adap->tc_matchall)
+ return -ENOMEM;
+
+ switch (cls_matchall->command) {
+ case TC_CLSMATCHALL_REPLACE:
+ return cxgb4_tc_matchall_replace(dev, cls_matchall, ingress);
+ case TC_CLSMATCHALL_DESTROY:
+ return cxgb4_tc_matchall_destroy(dev, cls_matchall, ingress);
+ case TC_CLSMATCHALL_STATS:
+ if (ingress)
+ return cxgb4_tc_matchall_stats(dev, cls_matchall);
+ break;
+ default:
+ break;
+ }
+
+ return -EOPNOTSUPP;
+}
+
+static int cxgb_setup_tc_block_ingress_cb(enum tc_setup_type type,
+ void *type_data, void *cb_priv)
{
struct net_device *dev = cb_priv;
struct port_info *pi = netdev2pinfo(dev);
struct adapter *adap = netdev2adap(dev);
- if (!(adap->flags & FULL_INIT_DONE)) {
+ if (!(adap->flags & CXGB4_FULL_INIT_DONE)) {
dev_err(adap->pdev_dev,
"Failed to setup tc on port %d. Link Down?\n",
pi->port_id);
@@ -2999,35 +3644,79 @@ static int cxgb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
return cxgb_setup_tc_cls_u32(dev, type_data);
case TC_SETUP_CLSFLOWER:
return cxgb_setup_tc_flower(dev, type_data);
+ case TC_SETUP_CLSMATCHALL:
+ return cxgb_setup_tc_matchall(dev, type_data, true);
default:
return -EOPNOTSUPP;
}
}
-static int cxgb_setup_tc_block(struct net_device *dev,
- struct tc_block_offload *f)
+static int cxgb_setup_tc_block_egress_cb(enum tc_setup_type type,
+ void *type_data, void *cb_priv)
{
+ struct net_device *dev = cb_priv;
struct port_info *pi = netdev2pinfo(dev);
+ struct adapter *adap = netdev2adap(dev);
+
+ if (!(adap->flags & CXGB4_FULL_INIT_DONE)) {
+ dev_err(adap->pdev_dev,
+ "Failed to setup tc on port %d. Link Down?\n",
+ pi->port_id);
+ return -EINVAL;
+ }
- if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+ if (!tc_cls_can_offload_and_chain0(dev, type_data))
return -EOPNOTSUPP;
- switch (f->command) {
- case TC_BLOCK_BIND:
- return tcf_block_cb_register(f->block, cxgb_setup_tc_block_cb,
- pi, dev, f->extack);
- case TC_BLOCK_UNBIND:
- tcf_block_cb_unregister(f->block, cxgb_setup_tc_block_cb, pi);
- return 0;
+ switch (type) {
+ case TC_SETUP_CLSMATCHALL:
+ return cxgb_setup_tc_matchall(dev, type_data, false);
default:
- return -EOPNOTSUPP;
+ break;
+ }
+
+ return -EOPNOTSUPP;
+}
+
+static int cxgb_setup_tc_mqprio(struct net_device *dev,
+ struct tc_mqprio_qopt_offload *mqprio)
+{
+ struct adapter *adap = netdev2adap(dev);
+
+ if (!is_ethofld(adap) || !adap->tc_mqprio)
+ return -ENOMEM;
+
+ return cxgb4_setup_tc_mqprio(dev, mqprio);
+}
+
+static LIST_HEAD(cxgb_block_cb_list);
+
+static int cxgb_setup_tc_block(struct net_device *dev,
+ struct flow_block_offload *f)
+{
+ struct port_info *pi = netdev_priv(dev);
+ flow_setup_cb_t *cb;
+ bool ingress_only;
+
+ pi->tc_block_shared = f->block_shared;
+ if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
+ cb = cxgb_setup_tc_block_egress_cb;
+ ingress_only = false;
+ } else {
+ cb = cxgb_setup_tc_block_ingress_cb;
+ ingress_only = true;
}
+
+ return flow_block_cb_setup_simple(f, &cxgb_block_cb_list,
+ cb, pi, dev, ingress_only);
}
static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type,
void *type_data)
{
switch (type) {
+ case TC_SETUP_QDISC_MQPRIO:
+ return cxgb_setup_tc_mqprio(dev, type_data);
case TC_SETUP_BLOCK:
return cxgb_setup_tc_block(dev, type_data);
default:
@@ -3035,131 +3724,71 @@ static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type,
}
}
-static void cxgb_del_udp_tunnel(struct net_device *netdev,
- struct udp_tunnel_info *ti)
+static int cxgb_udp_tunnel_unset_port(struct net_device *netdev,
+ unsigned int table, unsigned int entry,
+ struct udp_tunnel_info *ti)
{
struct port_info *pi = netdev_priv(netdev);
struct adapter *adapter = pi->adapter;
- unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
int ret = 0, i;
- if (chip_ver < CHELSIO_T6)
- return;
-
switch (ti->type) {
case UDP_TUNNEL_TYPE_VXLAN:
- if (!adapter->vxlan_port_cnt ||
- adapter->vxlan_port != ti->port)
- return; /* Invalid VxLAN destination port */
-
- adapter->vxlan_port_cnt--;
- if (adapter->vxlan_port_cnt)
- return;
-
adapter->vxlan_port = 0;
t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 0);
break;
case UDP_TUNNEL_TYPE_GENEVE:
- if (!adapter->geneve_port_cnt ||
- adapter->geneve_port != ti->port)
- return; /* Invalid GENEVE destination port */
-
- adapter->geneve_port_cnt--;
- if (adapter->geneve_port_cnt)
- return;
-
adapter->geneve_port = 0;
t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 0);
break;
default:
- return;
+ return -EINVAL;
}
/* Matchall mac entries can be deleted only after all tunnel ports
* are brought down or removed.
*/
if (!adapter->rawf_cnt)
- return;
+ return 0;
for_each_port(adapter, i) {
pi = adap2pinfo(adapter, i);
ret = t4_free_raw_mac_filt(adapter, pi->viid,
match_all_mac, match_all_mac,
- adapter->rawf_start +
- pi->port_id,
+ adapter->rawf_start + pi->port_id,
1, pi->port_id, false);
if (ret < 0) {
netdev_info(netdev, "Failed to free mac filter entry, for port %d\n",
i);
- return;
+ return ret;
}
- atomic_dec(&adapter->mps_encap[adapter->rawf_start +
- pi->port_id].refcnt);
}
+
+ return 0;
}
-static void cxgb_add_udp_tunnel(struct net_device *netdev,
- struct udp_tunnel_info *ti)
+static int cxgb_udp_tunnel_set_port(struct net_device *netdev,
+ unsigned int table, unsigned int entry,
+ struct udp_tunnel_info *ti)
{
struct port_info *pi = netdev_priv(netdev);
struct adapter *adapter = pi->adapter;
- unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip);
u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 };
int i, ret;
- if (chip_ver < CHELSIO_T6 || !adapter->rawf_cnt)
- return;
-
switch (ti->type) {
case UDP_TUNNEL_TYPE_VXLAN:
- /* Callback for adding vxlan port can be called with the same
- * port for both IPv4 and IPv6. We should not disable the
- * offloading when the same port for both protocols is added
- * and later one of them is removed.
- */
- if (adapter->vxlan_port_cnt &&
- adapter->vxlan_port == ti->port) {
- adapter->vxlan_port_cnt++;
- return;
- }
-
- /* We will support only one VxLAN port */
- if (adapter->vxlan_port_cnt) {
- netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n",
- be16_to_cpu(adapter->vxlan_port),
- be16_to_cpu(ti->port));
- return;
- }
-
adapter->vxlan_port = ti->port;
- adapter->vxlan_port_cnt = 1;
-
t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A,
VXLAN_V(be16_to_cpu(ti->port)) | VXLAN_EN_F);
break;
case UDP_TUNNEL_TYPE_GENEVE:
- if (adapter->geneve_port_cnt &&
- adapter->geneve_port == ti->port) {
- adapter->geneve_port_cnt++;
- return;
- }
-
- /* We will support only one GENEVE port */
- if (adapter->geneve_port_cnt) {
- netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n",
- be16_to_cpu(adapter->geneve_port),
- be16_to_cpu(ti->port));
- return;
- }
-
adapter->geneve_port = ti->port;
- adapter->geneve_port_cnt = 1;
-
t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A,
GENEVE_V(be16_to_cpu(ti->port)) | GENEVE_EN_F);
break;
default:
- return;
+ return -EINVAL;
}
/* Create a 'match all' mac filter entry for inner mac,
@@ -3174,19 +3803,27 @@ static void cxgb_add_udp_tunnel(struct net_device *netdev,
ret = t4_alloc_raw_mac_filt(adapter, pi->viid,
match_all_mac,
match_all_mac,
- adapter->rawf_start +
- pi->port_id,
+ adapter->rawf_start + pi->port_id,
1, pi->port_id, false);
if (ret < 0) {
netdev_info(netdev, "Failed to allocate a mac filter entry, not adding port %d\n",
be16_to_cpu(ti->port));
- cxgb_del_udp_tunnel(netdev, ti);
- return;
+ return ret;
}
- atomic_inc(&adapter->mps_encap[ret].refcnt);
}
+
+ return 0;
}
+static const struct udp_tunnel_nic_info cxgb_udp_tunnels = {
+ .set_port = cxgb_udp_tunnel_set_port,
+ .unset_port = cxgb_udp_tunnel_unset_port,
+ .tables = {
+ { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },
+ { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
+ },
+};
+
static netdev_features_t cxgb_features_check(struct sk_buff *skb,
struct net_device *dev,
netdev_features_t features)
@@ -3225,7 +3862,7 @@ static const struct net_device_ops cxgb4_netdev_ops = {
.ndo_set_mac_address = cxgb_set_mac_addr,
.ndo_set_features = cxgb_set_features,
.ndo_validate_addr = eth_validate_addr,
- .ndo_do_ioctl = cxgb_ioctl,
+ .ndo_eth_ioctl = cxgb_ioctl,
.ndo_change_mtu = cxgb_change_mtu,
#ifdef CONFIG_NET_POLL_CONTROLLER
.ndo_poll_controller = cxgb_netpoll,
@@ -3236,38 +3873,37 @@ static const struct net_device_ops cxgb4_netdev_ops = {
#endif /* CONFIG_CHELSIO_T4_FCOE */
.ndo_set_tx_maxrate = cxgb_set_tx_maxrate,
.ndo_setup_tc = cxgb_setup_tc,
- .ndo_udp_tunnel_add = cxgb_add_udp_tunnel,
- .ndo_udp_tunnel_del = cxgb_del_udp_tunnel,
.ndo_features_check = cxgb_features_check,
.ndo_fix_features = cxgb_fix_features,
+ .ndo_hwtstamp_get = cxgb_hwtstamp_get,
+ .ndo_hwtstamp_set = cxgb_hwtstamp_set,
};
#ifdef CONFIG_PCI_IOV
static const struct net_device_ops cxgb4_mgmt_netdev_ops = {
- .ndo_open = cxgb4_mgmt_open,
- .ndo_set_vf_mac = cxgb4_mgmt_set_vf_mac,
- .ndo_get_vf_config = cxgb4_mgmt_get_vf_config,
- .ndo_set_vf_rate = cxgb4_mgmt_set_vf_rate,
- .ndo_get_phys_port_id = cxgb4_mgmt_get_phys_port_id,
- .ndo_set_vf_vlan = cxgb4_mgmt_set_vf_vlan,
+ .ndo_open = cxgb4_mgmt_open,
+ .ndo_set_vf_mac = cxgb4_mgmt_set_vf_mac,
+ .ndo_get_vf_config = cxgb4_mgmt_get_vf_config,
+ .ndo_set_vf_rate = cxgb4_mgmt_set_vf_rate,
+ .ndo_get_phys_port_id = cxgb4_mgmt_get_phys_port_id,
+ .ndo_set_vf_vlan = cxgb4_mgmt_set_vf_vlan,
+ .ndo_set_vf_link_state = cxgb4_mgmt_set_vf_link_state,
};
-#endif
static void cxgb4_mgmt_get_drvinfo(struct net_device *dev,
struct ethtool_drvinfo *info)
{
struct adapter *adapter = netdev2adap(dev);
- strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
- strlcpy(info->version, cxgb4_driver_version,
- sizeof(info->version));
- strlcpy(info->bus_info, pci_name(adapter->pdev),
+ strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
+ strscpy(info->bus_info, pci_name(adapter->pdev),
sizeof(info->bus_info));
}
static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = {
.get_drvinfo = cxgb4_mgmt_get_drvinfo,
};
+#endif
static void notify_fatal_err(struct work_struct *work)
{
@@ -3364,7 +4000,7 @@ static void adap_free_hma_mem(struct adapter *adapter)
if (adapter->hma.flags & HMA_DMA_MAPPED_FLAG) {
dma_unmap_sg(adapter->pdev_dev, adapter->hma.sgt->sgl,
- adapter->hma.sgt->nents, PCI_DMA_BIDIRECTIONAL);
+ adapter->hma.sgt->nents, DMA_BIDIRECTIONAL);
adapter->hma.flags &= ~HMA_DMA_MAPPED_FLAG;
}
@@ -3783,8 +4419,7 @@ static int adap_init0_phy(struct adapter *adap)
/* Load PHY Firmware onto adapter.
*/
- ret = t4_load_phy_fw(adap, MEMWIN_NIC, &adap->win0_lock,
- phy_info->phy_fw_version,
+ ret = t4_load_phy_fw(adap, MEMWIN_NIC, phy_info->phy_fw_version,
(u8 *)phyf->data, phyf->size);
if (ret < 0)
dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n",
@@ -3810,14 +4445,14 @@ static int adap_init0_phy(struct adapter *adap)
*/
static int adap_init0_config(struct adapter *adapter, int reset)
{
+ char *fw_config_file, fw_config_file_path[256];
+ u32 finiver, finicsum, cfcsum, param, val;
struct fw_caps_config_cmd caps_cmd;
- const struct firmware *cf;
unsigned long mtype = 0, maddr = 0;
- u32 finiver, finicsum, cfcsum;
- int ret;
- int config_issued = 0;
- char *fw_config_file, fw_config_file_path[256];
+ const struct firmware *cf;
char *config_name = NULL;
+ int config_issued = 0;
+ int ret;
/*
* Reset device if necessary.
@@ -3925,6 +4560,24 @@ static int adap_init0_config(struct adapter *adapter, int reset)
goto bye;
}
+ val = 0;
+
+ /* Ofld + Hash filter is supported. Older fw will fail this request and
+ * it is fine.
+ */
+ param = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
+ FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_HASHFILTER_WITH_OFLD));
+ ret = t4_set_params(adapter, adapter->mbox, adapter->pf, 0,
+ 1, &param, &val);
+
+ /* FW doesn't know about Hash filter + ofld support,
+ * it's not a problem, don't return an error.
+ */
+ if (ret < 0) {
+ dev_warn(adapter->pdev_dev,
+ "Hash filter with ofld is not supported by FW\n");
+ }
+
/*
* Issue a Capability Configuration command to the firmware to get it
* to parse the Configuration File. We don't use t4_fw_config_file()
@@ -4001,6 +4654,14 @@ static int adap_init0_config(struct adapter *adapter, int reset)
dev_err(adapter->pdev_dev,
"HMA configuration failed with error %d\n", ret);
+ if (is_t6(adapter->params.chip)) {
+ adap_config_hpfilter(adapter);
+ ret = setup_ppod_edram(adapter);
+ if (!ret)
+ dev_info(adapter->pdev_dev, "Successfully enabled "
+ "ppod edram feature\n");
+ }
+
/*
* And finally tell the firmware to initialize itself using the
* parameters from the Configuration File.
@@ -4090,14 +4751,14 @@ static struct fw_info *find_fw_info(int chip)
/*
* Phase 0 of initialization: contact FW, obtain config, perform basic init.
*/
-static int adap_init0(struct adapter *adap)
+static int adap_init0(struct adapter *adap, int vpd_skip)
{
- int ret;
- u32 v, port_vec;
- enum dev_state state;
- u32 params[7], val[7];
struct fw_caps_config_cmd caps_cmd;
+ u32 params[7], val[7];
+ enum dev_state state;
+ u32 v, port_vec;
int reset = 1;
+ int ret;
/* Grab Firmware Device Log parameters as early as possible so we have
* access to it for debugging, etc.
@@ -4115,7 +4776,7 @@ static int adap_init0(struct adapter *adap)
return ret;
}
if (ret == adap->mbox)
- adap->flags |= MASTER_PF;
+ adap->flags |= CXGB4_MASTER_PF;
/*
* If we're the Master PF Driver and the device is uninitialized,
@@ -4130,7 +4791,7 @@ static int adap_init0(struct adapter *adap)
/* If firmware is too old (not supported by driver) force an update. */
if (ret)
state = DEV_STATE_UNINIT;
- if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
+ if ((adap->flags & CXGB4_MASTER_PF) && state != DEV_STATE_INIT) {
struct fw_info *fw_info;
struct fw_hdr *card_fw;
const struct firmware *fw;
@@ -4157,7 +4818,7 @@ static int adap_init0(struct adapter *adap)
goto bye;
}
- /* Get FW from from /lib/firmware/ */
+ /* Get FW from /lib/firmware/ */
ret = request_firmware(&fw, fw_info->fw_mod_name,
adap->pdev_dev);
if (ret < 0) {
@@ -4192,7 +4853,7 @@ static int adap_init0(struct adapter *adap)
ret);
dev_info(adap->pdev_dev, "Coming up as %s: "\
"Adapter already initialized\n",
- adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
+ adap->flags & CXGB4_MASTER_PF ? "MASTER" : "SLAVE");
} else {
dev_info(adap->pdev_dev, "Coming up as MASTER: "\
"Initializing adapter\n");
@@ -4252,9 +4913,11 @@ static int adap_init0(struct adapter *adap)
* could have FLASHed a new VPD which won't be read by the firmware
* until we do the RESET ...
*/
- ret = t4_get_vpd_params(adap, &adap->params.vpd);
- if (ret < 0)
- goto bye;
+ if (!vpd_skip) {
+ ret = t4_get_vpd_params(adap, &adap->params.vpd);
+ if (ret < 0)
+ goto bye;
+ }
/* Find out what ports are available to us. Note that we need to do
* this before calling adap_init0_no_config() since it needs nports
@@ -4278,22 +4941,30 @@ static int adap_init0(struct adapter *adap)
if (ret < 0)
goto bye;
+ /* Grab the SGE Doorbell Queue Timer values. If successful, that
+ * indicates that the Firmware and Hardware support this.
+ */
+ params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
+ FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_DBQ_TIMERTICK));
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
+ 1, params, val);
+
+ if (!ret) {
+ adap->sge.dbqtimer_tick = val[0];
+ ret = t4_read_sge_dbqtimers(adap,
+ ARRAY_SIZE(adap->sge.dbqtimer_val),
+ adap->sge.dbqtimer_val);
+ }
+
+ if (!ret)
+ adap->flags |= CXGB4_SGE_DBQ_TIMER;
+
if (is_bypass_device(adap->pdev->device))
adap->params.bypass = 1;
/*
* Grab some of our basic fundamental operating parameters.
*/
-#define FW_PARAM_DEV(param) \
- (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
- FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
-
-#define FW_PARAM_PFVF(param) \
- FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
- FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \
- FW_PARAMS_PARAM_Y_V(0) | \
- FW_PARAMS_PARAM_Z_V(0)
-
params[0] = FW_PARAM_PFVF(EQ_START);
params[1] = FW_PARAM_PFVF(L2T_START);
params[2] = FW_PARAM_PFVF(L2T_END);
@@ -4311,6 +4982,16 @@ static int adap_init0(struct adapter *adap)
adap->sge.ingr_start = val[5];
if (CHELSIO_CHIP_VERSION(adap->params.chip) > CHELSIO_T5) {
+ params[0] = FW_PARAM_PFVF(HPFILTER_START);
+ params[1] = FW_PARAM_PFVF(HPFILTER_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
+ params, val);
+ if (ret < 0)
+ goto bye;
+
+ adap->tids.hpftid_base = val[0];
+ adap->tids.nhpftids = val[1] - val[0] + 1;
+
/* Read the raw mps entries. In T6, the last 2 tcam entries
* are reserved for raw mac addresses (rawf = 2, one per port).
*/
@@ -4322,6 +5003,9 @@ static int adap_init0(struct adapter *adap)
adap->rawf_start = val[0];
adap->rawf_cnt = val[1] - val[0] + 1;
}
+
+ adap->tids.tid_base =
+ t4_read_reg(adap, LE_DB_ACTIVE_TABLE_START_INDEX_A);
}
/* qids (ingress/egress) returned from firmware can be anywhere
@@ -4355,23 +5039,20 @@ static int adap_init0(struct adapter *adap)
/* Allocate the memory for the vaious egress queue bitmaps
* ie starving_fl, txq_maperr and blocked_fl.
*/
- adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
- sizeof(long), GFP_KERNEL);
+ adap->sge.starving_fl = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL);
if (!adap->sge.starving_fl) {
ret = -ENOMEM;
goto bye;
}
- adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
- sizeof(long), GFP_KERNEL);
+ adap->sge.txq_maperr = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL);
if (!adap->sge.txq_maperr) {
ret = -ENOMEM;
goto bye;
}
#ifdef CONFIG_DEBUG_FS
- adap->sge.blocked_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz),
- sizeof(long), GFP_KERNEL);
+ adap->sge.blocked_fl = bitmap_zalloc(adap->sge.egr_sz, GFP_KERNEL);
if (!adap->sge.blocked_fl) {
ret = -ENOMEM;
goto bye;
@@ -4386,11 +5067,18 @@ static int adap_init0(struct adapter *adap)
adap->clipt_start = val[0];
adap->clipt_end = val[1];
- /* We don't yet have a PARAMs calls to retrieve the number of Traffic
- * Classes supported by the hardware/firmware so we hard code it here
- * for now.
- */
- adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16;
+ /* Get the supported number of traffic classes */
+ params[0] = FW_PARAM_DEV(NUM_TM_CLASS);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, params, val);
+ if (ret < 0) {
+ /* We couldn't retrieve the number of Traffic Classes
+ * supported by the hardware/firmware. So we hard
+ * code it here.
+ */
+ adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16;
+ } else {
+ adap->params.nsched_cls = val[0];
+ }
/* query params related to active filter region */
params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
@@ -4400,7 +5088,7 @@ static int adap_init0(struct adapter *adap)
* offload connection through firmware work request
*/
if ((val[0] != val[1]) && (ret >= 0)) {
- adap->flags |= FW_OFLD_CONN;
+ adap->flags |= CXGB4_FW_OFLD_CONN;
adap->tids.aftid_base = val[0];
adap->tids.aftid_end = val[1];
}
@@ -4437,7 +5125,7 @@ static int adap_init0(struct adapter *adap)
/* See if FW supports FW_FILTER2 work request */
if (is_t4(adap->params.chip)) {
- adap->params.filter2_wr_support = 0;
+ adap->params.filter2_wr_support = false;
} else {
params[0] = FW_PARAM_DEV(FILTER2_WR);
ret = t4_query_params(adap, adap->mbox, adap->pf, 0,
@@ -4467,8 +5155,16 @@ static int adap_init0(struct adapter *adap)
if (ret < 0)
goto bye;
+ /* hash filter has some mandatory register settings to be tested and for
+ * that it needs to test whether offload is enabled or not, hence
+ * checking and setting it here.
+ */
+ if (caps_cmd.ofldcaps)
+ adap->params.offload = 1;
+
if (caps_cmd.ofldcaps ||
- (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER))) {
+ (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) ||
+ (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD))) {
/* query offload-related parameters */
params[0] = FW_PARAM_DEV(NTID);
params[1] = FW_PARAM_PFVF(SERVER_START);
@@ -4493,7 +5189,7 @@ static int adap_init0(struct adapter *adap)
* 2. Server filter: This are special filters which are used
* to redirect SYN packets to offload queue.
*/
- if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
+ if (adap->flags & CXGB4_FW_OFLD_CONN && !is_bypass(adap)) {
adap->tids.sftid_base = adap->tids.ftid_base +
DIV_ROUND_UP(adap->tids.nftids, 3);
adap->tids.nsftids = adap->tids.nftids -
@@ -4506,13 +5202,23 @@ static int adap_init0(struct adapter *adap)
adap->params.ofldq_wr_cred = val[5];
if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) {
- ret = init_hash_filter(adap);
- if (ret < 0)
- goto bye;
+ init_hash_filter(adap);
} else {
- adap->params.offload = 1;
adap->num_ofld_uld += 1;
}
+
+ if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_ETHOFLD)) {
+ params[0] = FW_PARAM_PFVF(ETHOFLD_START);
+ params[1] = FW_PARAM_PFVF(ETHOFLD_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
+ params, val);
+ if (!ret) {
+ adap->tids.eotid_base = val[0];
+ adap->tids.neotids = min_t(u32, MAX_ATIDS,
+ val[1] - val[0] + 1);
+ adap->params.ethofld = 1;
+ }
+ }
}
if (caps_cmd.rdmacaps) {
params[0] = FW_PARAM_PFVF(STAG_START);
@@ -4602,6 +5308,22 @@ static int adap_init0(struct adapter *adap)
goto bye;
adap->vres.iscsi.start = val[0];
adap->vres.iscsi.size = val[1] - val[0] + 1;
+ if (is_t6(adap->params.chip)) {
+ params[0] = FW_PARAM_PFVF(PPOD_EDRAM_START);
+ params[1] = FW_PARAM_PFVF(PPOD_EDRAM_END);
+ ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2,
+ params, val);
+ if (!ret) {
+ adap->vres.ppod_edram.start = val[0];
+ adap->vres.ppod_edram.size =
+ val[1] - val[0] + 1;
+
+ dev_info(adap->pdev_dev,
+ "ppod edram start 0x%x end 0x%x size 0x%x\n",
+ val[0], val[1],
+ adap->vres.ppod_edram.size);
+ }
+ }
/* LIO target and cxgb4i initiaitor */
adap->num_ofld_uld += 2;
}
@@ -4633,8 +5355,6 @@ static int adap_init0(struct adapter *adap)
}
adap->params.crypto = ntohs(caps_cmd.cryptocaps);
}
-#undef FW_PARAM_PFVF
-#undef FW_PARAM_DEV
/* The MTU/MSS Table is initialized by now, so load their values. If
* we're initializing the adapter, then we'll make any modifications
@@ -4672,7 +5392,7 @@ static int adap_init0(struct adapter *adap)
adap->params.b_wnd);
}
t4_init_sge_params(adap);
- adap->flags |= FW_OK;
+ adap->flags |= CXGB4_FW_OK;
t4_init_tp_params(adap, true);
return 0;
@@ -4685,10 +5405,10 @@ bye:
adap_free_hma_mem(adap);
kfree(adap->sge.egr_map);
kfree(adap->sge.ingr_map);
- kfree(adap->sge.starving_fl);
- kfree(adap->sge.txq_maperr);
+ bitmap_free(adap->sge.starving_fl);
+ bitmap_free(adap->sge.txq_maperr);
#ifdef CONFIG_DEBUG_FS
- kfree(adap->sge.blocked_fl);
+ bitmap_free(adap->sge.blocked_fl);
#endif
if (ret != -ETIMEDOUT && ret != -EIO)
t4_fw_bye(adap, adap->mbox);
@@ -4707,7 +5427,7 @@ static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
goto out;
rtnl_lock();
- adap->flags &= ~FW_OK;
+ adap->flags &= ~CXGB4_FW_OK;
notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
spin_lock(&adap->stats_lock);
for_each_port(adap, i) {
@@ -4719,12 +5439,12 @@ static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
}
spin_unlock(&adap->stats_lock);
disable_interrupts(adap);
- if (adap->flags & FULL_INIT_DONE)
+ if (adap->flags & CXGB4_FULL_INIT_DONE)
cxgb_down(adap);
rtnl_unlock();
- if ((adap->flags & DEV_ENABLED)) {
+ if ((adap->flags & CXGB4_DEV_ENABLED)) {
pci_disable_device(pdev);
- adap->flags &= ~DEV_ENABLED;
+ adap->flags &= ~CXGB4_DEV_ENABLED;
}
out: return state == pci_channel_io_perm_failure ?
PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
@@ -4738,28 +5458,26 @@ static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
if (!adap) {
pci_restore_state(pdev);
- pci_save_state(pdev);
return PCI_ERS_RESULT_RECOVERED;
}
- if (!(adap->flags & DEV_ENABLED)) {
+ if (!(adap->flags & CXGB4_DEV_ENABLED)) {
if (pci_enable_device(pdev)) {
dev_err(&pdev->dev, "Cannot reenable PCI "
"device after reset\n");
return PCI_ERS_RESULT_DISCONNECT;
}
- adap->flags |= DEV_ENABLED;
+ adap->flags |= CXGB4_DEV_ENABLED;
}
pci_set_master(pdev);
pci_restore_state(pdev);
- pci_save_state(pdev);
if (t4_wait_dev_ready(adap->regs) < 0)
return PCI_ERS_RESULT_DISCONNECT;
if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0)
return PCI_ERS_RESULT_DISCONNECT;
- adap->flags |= FW_OK;
+ adap->flags |= CXGB4_FW_OK;
if (adap_init1(adap, &c))
return PCI_ERS_RESULT_DISCONNECT;
@@ -4816,10 +5534,93 @@ static void eeh_resume(struct pci_dev *pdev)
rtnl_unlock();
}
+static void eeh_reset_prepare(struct pci_dev *pdev)
+{
+ struct adapter *adapter = pci_get_drvdata(pdev);
+ int i;
+
+ if (adapter->pf != 4)
+ return;
+
+ adapter->flags &= ~CXGB4_FW_OK;
+
+ notify_ulds(adapter, CXGB4_STATE_DOWN);
+
+ for_each_port(adapter, i)
+ if (adapter->port[i]->reg_state == NETREG_REGISTERED)
+ cxgb_close(adapter->port[i]);
+
+ disable_interrupts(adapter);
+ cxgb4_free_mps_ref_entries(adapter);
+
+ adap_free_hma_mem(adapter);
+
+ if (adapter->flags & CXGB4_FULL_INIT_DONE)
+ cxgb_down(adapter);
+}
+
+static void eeh_reset_done(struct pci_dev *pdev)
+{
+ struct adapter *adapter = pci_get_drvdata(pdev);
+ int err, i;
+
+ if (adapter->pf != 4)
+ return;
+
+ err = t4_wait_dev_ready(adapter->regs);
+ if (err < 0) {
+ dev_err(adapter->pdev_dev,
+ "Device not ready, err %d", err);
+ return;
+ }
+
+ setup_memwin(adapter);
+
+ err = adap_init0(adapter, 1);
+ if (err) {
+ dev_err(adapter->pdev_dev,
+ "Adapter init failed, err %d", err);
+ return;
+ }
+
+ setup_memwin_rdma(adapter);
+
+ if (adapter->flags & CXGB4_FW_OK) {
+ err = t4_port_init(adapter, adapter->pf, adapter->pf, 0);
+ if (err) {
+ dev_err(adapter->pdev_dev,
+ "Port init failed, err %d", err);
+ return;
+ }
+ }
+
+ err = cfg_queues(adapter);
+ if (err) {
+ dev_err(adapter->pdev_dev,
+ "Config queues failed, err %d", err);
+ return;
+ }
+
+ cxgb4_init_mps_ref_entries(adapter);
+
+ err = setup_fw_sge_queues(adapter);
+ if (err) {
+ dev_err(adapter->pdev_dev,
+ "FW sge queue allocation failed, err %d", err);
+ return;
+ }
+
+ for_each_port(adapter, i)
+ if (adapter->port[i]->reg_state == NETREG_REGISTERED)
+ cxgb_open(adapter->port[i]);
+}
+
static const struct pci_error_handlers cxgb4_eeh = {
.error_detected = eeh_err_detected,
.slot_reset = eeh_slot_reset,
.resume = eeh_resume,
+ .reset_prepare = eeh_reset_prepare,
+ .reset_done = eeh_reset_done,
};
/* Return true if the Link Configuration supports "High Speeds" (those greater
@@ -4836,26 +5637,24 @@ static inline bool is_x_10g_port(const struct link_config *lc)
return high_speeds != 0;
}
-/*
- * Perform default configuration of DMA queues depending on the number and type
+/* Perform default configuration of DMA queues depending on the number and type
* of ports we found and the number of available CPUs. Most settings can be
* modified by the admin prior to actual use.
*/
static int cfg_queues(struct adapter *adap)
{
+ u32 avail_qsets, avail_eth_qsets, avail_uld_qsets;
+ u32 ncpus = num_online_cpus();
+ u32 niqflint, neq, num_ulds;
struct sge *s = &adap->sge;
- int i, n10g = 0, qidx = 0;
- int niqflint, neq, avail_eth_qsets;
- int max_eth_qsets = 32;
-#ifndef CONFIG_CHELSIO_T4_DCB
- int q10g = 0;
-#endif
+ u32 i, n10g = 0, qidx = 0;
+ u32 q10g = 0, q1g;
- /* Reduce memory usage in kdump environment, disable all offload.
- */
+ /* Reduce memory usage in kdump environment, disable all offload. */
if (is_kdump_kernel() || (is_uld(adap) && t4_uld_mem_alloc(adap))) {
adap->params.offload = 0;
adap->params.crypto = 0;
+ adap->params.ethofld = 0;
}
/* Calculate the number of Ethernet Queue Sets available based on
@@ -4871,17 +5670,14 @@ static int cfg_queues(struct adapter *adap)
* at all is problematic ...
*/
niqflint = adap->params.pfres.niqflint - 1;
- if (!(adap->flags & USING_MSIX))
+ if (!(adap->flags & CXGB4_USING_MSIX))
niqflint--;
neq = adap->params.pfres.neq / 2;
- avail_eth_qsets = min(niqflint, neq);
-
- if (avail_eth_qsets > max_eth_qsets)
- avail_eth_qsets = max_eth_qsets;
+ avail_qsets = min(niqflint, neq);
- if (avail_eth_qsets < adap->params.nports) {
+ if (avail_qsets < adap->params.nports) {
dev_err(adap->pdev_dev, "avail_eth_qsets=%d < nports=%d\n",
- avail_eth_qsets, adap->params.nports);
+ avail_qsets, adap->params.nports);
return -ENOMEM;
}
@@ -4889,63 +5685,103 @@ static int cfg_queues(struct adapter *adap)
for_each_port(adap, i)
n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
+ avail_eth_qsets = min_t(u32, avail_qsets, MAX_ETH_QSETS);
+
+ /* We default to 1 queue per non-10G port and up to # of cores queues
+ * per 10G port.
+ */
+ if (n10g)
+ q10g = (avail_eth_qsets - (adap->params.nports - n10g)) / n10g;
+
#ifdef CONFIG_CHELSIO_T4_DCB
/* For Data Center Bridging support we need to be able to support up
* to 8 Traffic Priorities; each of which will be assigned to its
* own TX Queue in order to prevent Head-Of-Line Blocking.
*/
+ q1g = 8;
if (adap->params.nports * 8 > avail_eth_qsets) {
dev_err(adap->pdev_dev, "DCB avail_eth_qsets=%d < %d!\n",
avail_eth_qsets, adap->params.nports * 8);
return -ENOMEM;
}
- for_each_port(adap, i) {
- struct port_info *pi = adap2pinfo(adap, i);
+ if (adap->params.nports * ncpus < avail_eth_qsets)
+ q10g = max(8U, ncpus);
+ else
+ q10g = max(8U, q10g);
- pi->first_qset = qidx;
- pi->nqsets = is_kdump_kernel() ? 1 : 8;
- qidx += pi->nqsets;
- }
-#else /* !CONFIG_CHELSIO_T4_DCB */
- /*
- * We default to 1 queue per non-10G port and up to # of cores queues
- * per 10G port.
- */
- if (n10g)
- q10g = (avail_eth_qsets - (adap->params.nports - n10g)) / n10g;
- if (q10g > netif_get_num_default_rss_queues())
- q10g = netif_get_num_default_rss_queues();
+ while ((q10g * n10g) >
+ (avail_eth_qsets - (adap->params.nports - n10g) * q1g))
+ q10g--;
- if (is_kdump_kernel())
+#else /* !CONFIG_CHELSIO_T4_DCB */
+ q1g = 1;
+ q10g = min(q10g, ncpus);
+#endif /* !CONFIG_CHELSIO_T4_DCB */
+ if (is_kdump_kernel()) {
q10g = 1;
+ q1g = 1;
+ }
for_each_port(adap, i) {
struct port_info *pi = adap2pinfo(adap, i);
pi->first_qset = qidx;
- pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
+ pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : q1g;
qidx += pi->nqsets;
}
-#endif /* !CONFIG_CHELSIO_T4_DCB */
s->ethqsets = qidx;
s->max_ethqsets = qidx; /* MSI-X may lower it later */
+ avail_qsets -= qidx;
if (is_uld(adap)) {
- /*
- * For offload we use 1 queue/channel if all ports are up to 1G,
+ /* For offload we use 1 queue/channel if all ports are up to 1G,
* otherwise we divide all available queues amongst the channels
* capped by the number of available cores.
*/
- if (n10g) {
- i = min_t(int, MAX_OFLD_QSETS, num_online_cpus());
- s->ofldqsets = roundup(i, adap->params.nports);
- } else {
+ num_ulds = adap->num_uld + adap->num_ofld_uld;
+ i = min_t(u32, MAX_OFLD_QSETS, ncpus);
+ avail_uld_qsets = roundup(i, adap->params.nports);
+ if (avail_qsets < num_ulds * adap->params.nports) {
+ adap->params.offload = 0;
+ adap->params.crypto = 0;
+ s->ofldqsets = 0;
+ } else if (avail_qsets < num_ulds * avail_uld_qsets || !n10g) {
s->ofldqsets = adap->params.nports;
+ } else {
+ s->ofldqsets = avail_uld_qsets;
}
+
+ avail_qsets -= num_ulds * s->ofldqsets;
}
+ /* ETHOFLD Queues used for QoS offload should follow same
+ * allocation scheme as normal Ethernet Queues.
+ */
+ if (is_ethofld(adap)) {
+ if (avail_qsets < s->max_ethqsets) {
+ adap->params.ethofld = 0;
+ s->eoqsets = 0;
+ } else {
+ s->eoqsets = s->max_ethqsets;
+ }
+ avail_qsets -= s->eoqsets;
+ }
+
+ /* Mirror queues must follow same scheme as normal Ethernet
+ * Queues, when there are enough queues available. Otherwise,
+ * allocate at least 1 queue per port. If even 1 queue is not
+ * available, then disable mirror queues support.
+ */
+ if (avail_qsets >= s->max_ethqsets)
+ s->mirrorqsets = s->max_ethqsets;
+ else if (avail_qsets >= adap->params.nports)
+ s->mirrorqsets = adap->params.nports;
+ else
+ s->mirrorqsets = 0;
+ avail_qsets -= s->mirrorqsets;
+
for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
struct sge_eth_rxq *r = &s->ethrxq[i];
@@ -4996,42 +5832,61 @@ static void reduce_ethqs(struct adapter *adap, int n)
}
}
-static int get_msix_info(struct adapter *adap)
+static int alloc_msix_info(struct adapter *adap, u32 num_vec)
{
- struct uld_msix_info *msix_info;
- unsigned int max_ingq = 0;
-
- if (is_offload(adap))
- max_ingq += MAX_OFLD_QSETS * adap->num_ofld_uld;
- if (is_pci_uld(adap))
- max_ingq += MAX_OFLD_QSETS * adap->num_uld;
-
- if (!max_ingq)
- goto out;
+ struct msix_info *msix_info;
- msix_info = kcalloc(max_ingq, sizeof(*msix_info), GFP_KERNEL);
+ msix_info = kcalloc(num_vec, sizeof(*msix_info), GFP_KERNEL);
if (!msix_info)
return -ENOMEM;
- adap->msix_bmap_ulds.msix_bmap = kcalloc(BITS_TO_LONGS(max_ingq),
- sizeof(long), GFP_KERNEL);
- if (!adap->msix_bmap_ulds.msix_bmap) {
+ adap->msix_bmap.msix_bmap = bitmap_zalloc(num_vec, GFP_KERNEL);
+ if (!adap->msix_bmap.msix_bmap) {
kfree(msix_info);
return -ENOMEM;
}
- spin_lock_init(&adap->msix_bmap_ulds.lock);
- adap->msix_info_ulds = msix_info;
-out:
+
+ spin_lock_init(&adap->msix_bmap.lock);
+ adap->msix_bmap.mapsize = num_vec;
+
+ adap->msix_info = msix_info;
return 0;
}
static void free_msix_info(struct adapter *adap)
{
- if (!(adap->num_uld && adap->num_ofld_uld))
- return;
+ bitmap_free(adap->msix_bmap.msix_bmap);
+ kfree(adap->msix_info);
+}
+
+int cxgb4_get_msix_idx_from_bmap(struct adapter *adap)
+{
+ struct msix_bmap *bmap = &adap->msix_bmap;
+ unsigned int msix_idx;
+ unsigned long flags;
+
+ spin_lock_irqsave(&bmap->lock, flags);
+ msix_idx = find_first_zero_bit(bmap->msix_bmap, bmap->mapsize);
+ if (msix_idx < bmap->mapsize) {
+ __set_bit(msix_idx, bmap->msix_bmap);
+ } else {
+ spin_unlock_irqrestore(&bmap->lock, flags);
+ return -ENOSPC;
+ }
+
+ spin_unlock_irqrestore(&bmap->lock, flags);
+ return msix_idx;
+}
- kfree(adap->msix_info_ulds);
- kfree(adap->msix_bmap_ulds.msix_bmap);
+void cxgb4_free_msix_idx_in_bmap(struct adapter *adap,
+ unsigned int msix_idx)
+{
+ struct msix_bmap *bmap = &adap->msix_bmap;
+ unsigned long flags;
+
+ spin_lock_irqsave(&bmap->lock, flags);
+ __clear_bit(msix_idx, bmap->msix_bmap);
+ spin_unlock_irqrestore(&bmap->lock, flags);
}
/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
@@ -5039,88 +5894,194 @@ static void free_msix_info(struct adapter *adap)
static int enable_msix(struct adapter *adap)
{
- int ofld_need = 0, uld_need = 0;
- int i, j, want, need, allocated;
+ u32 eth_need, uld_need = 0, ethofld_need = 0, mirror_need = 0;
+ u32 ethqsets = 0, ofldqsets = 0, eoqsets = 0, mirrorqsets = 0;
+ u8 num_uld = 0, nchan = adap->params.nports;
+ u32 i, want, need, num_vec;
struct sge *s = &adap->sge;
- unsigned int nchan = adap->params.nports;
struct msix_entry *entries;
- int max_ingq = MAX_INGQ;
-
- if (is_pci_uld(adap))
- max_ingq += (MAX_OFLD_QSETS * adap->num_uld);
- if (is_offload(adap))
- max_ingq += (MAX_OFLD_QSETS * adap->num_ofld_uld);
- entries = kmalloc_array(max_ingq + 1, sizeof(*entries),
- GFP_KERNEL);
- if (!entries)
- return -ENOMEM;
-
- /* map for msix */
- if (get_msix_info(adap)) {
- adap->params.offload = 0;
- adap->params.crypto = 0;
- }
-
- for (i = 0; i < max_ingq + 1; ++i)
- entries[i].entry = i;
+ struct port_info *pi;
+ int allocated, ret;
- want = s->max_ethqsets + EXTRA_VECS;
- if (is_offload(adap)) {
- want += adap->num_ofld_uld * s->ofldqsets;
- ofld_need = adap->num_ofld_uld * nchan;
- }
- if (is_pci_uld(adap)) {
- want += adap->num_uld * s->ofldqsets;
- uld_need = adap->num_uld * nchan;
- }
+ want = s->max_ethqsets;
#ifdef CONFIG_CHELSIO_T4_DCB
/* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
* each port.
*/
- need = 8 * adap->params.nports + EXTRA_VECS + ofld_need + uld_need;
+ need = 8 * nchan;
#else
- need = adap->params.nports + EXTRA_VECS + ofld_need + uld_need;
+ need = nchan;
#endif
+ eth_need = need;
+ if (is_uld(adap)) {
+ num_uld = adap->num_ofld_uld + adap->num_uld;
+ want += num_uld * s->ofldqsets;
+ uld_need = num_uld * nchan;
+ need += uld_need;
+ }
+
+ if (is_ethofld(adap)) {
+ want += s->eoqsets;
+ ethofld_need = eth_need;
+ need += ethofld_need;
+ }
+
+ if (s->mirrorqsets) {
+ want += s->mirrorqsets;
+ mirror_need = nchan;
+ need += mirror_need;
+ }
+
+ want += EXTRA_VECS;
+ need += EXTRA_VECS;
+
+ entries = kmalloc_array(want, sizeof(*entries), GFP_KERNEL);
+ if (!entries)
+ return -ENOMEM;
+
+ for (i = 0; i < want; i++)
+ entries[i].entry = i;
+
allocated = pci_enable_msix_range(adap->pdev, entries, need, want);
if (allocated < 0) {
- dev_info(adap->pdev_dev, "not enough MSI-X vectors left,"
- " not using MSI-X\n");
- kfree(entries);
- return allocated;
+ /* Disable offload and attempt to get vectors for NIC
+ * only mode.
+ */
+ want = s->max_ethqsets + EXTRA_VECS;
+ need = eth_need + EXTRA_VECS;
+ allocated = pci_enable_msix_range(adap->pdev, entries,
+ need, want);
+ if (allocated < 0) {
+ dev_info(adap->pdev_dev,
+ "Disabling MSI-X due to insufficient MSI-X vectors\n");
+ ret = allocated;
+ goto out_free;
+ }
+
+ dev_info(adap->pdev_dev,
+ "Disabling offload due to insufficient MSI-X vectors\n");
+ adap->params.offload = 0;
+ adap->params.crypto = 0;
+ adap->params.ethofld = 0;
+ s->ofldqsets = 0;
+ s->eoqsets = 0;
+ s->mirrorqsets = 0;
+ uld_need = 0;
+ ethofld_need = 0;
+ mirror_need = 0;
+ }
+
+ num_vec = allocated;
+ if (num_vec < want) {
+ /* Distribute available vectors to the various queue groups.
+ * Every group gets its minimum requirement and NIC gets top
+ * priority for leftovers.
+ */
+ ethqsets = eth_need;
+ if (is_uld(adap))
+ ofldqsets = nchan;
+ if (is_ethofld(adap))
+ eoqsets = ethofld_need;
+ if (s->mirrorqsets)
+ mirrorqsets = mirror_need;
+
+ num_vec -= need;
+ while (num_vec) {
+ if (num_vec < eth_need + ethofld_need ||
+ ethqsets > s->max_ethqsets)
+ break;
+
+ for_each_port(adap, i) {
+ pi = adap2pinfo(adap, i);
+ if (pi->nqsets < 2)
+ continue;
+
+ ethqsets++;
+ num_vec--;
+ if (ethofld_need) {
+ eoqsets++;
+ num_vec--;
+ }
+ }
+ }
+
+ if (is_uld(adap)) {
+ while (num_vec) {
+ if (num_vec < uld_need ||
+ ofldqsets > s->ofldqsets)
+ break;
+
+ ofldqsets++;
+ num_vec -= uld_need;
+ }
+ }
+
+ if (s->mirrorqsets) {
+ while (num_vec) {
+ if (num_vec < mirror_need ||
+ mirrorqsets > s->mirrorqsets)
+ break;
+
+ mirrorqsets++;
+ num_vec -= mirror_need;
+ }
+ }
+ } else {
+ ethqsets = s->max_ethqsets;
+ if (is_uld(adap))
+ ofldqsets = s->ofldqsets;
+ if (is_ethofld(adap))
+ eoqsets = s->eoqsets;
+ if (s->mirrorqsets)
+ mirrorqsets = s->mirrorqsets;
}
- /* Distribute available vectors to the various queue groups.
- * Every group gets its minimum requirement and NIC gets top
- * priority for leftovers.
- */
- i = allocated - EXTRA_VECS - ofld_need - uld_need;
- if (i < s->max_ethqsets) {
- s->max_ethqsets = i;
- if (i < s->ethqsets)
- reduce_ethqs(adap, i);
+ if (ethqsets < s->max_ethqsets) {
+ s->max_ethqsets = ethqsets;
+ reduce_ethqs(adap, ethqsets);
}
+
if (is_uld(adap)) {
- if (allocated < want)
- s->nqs_per_uld = nchan;
- else
- s->nqs_per_uld = s->ofldqsets;
+ s->ofldqsets = ofldqsets;
+ s->nqs_per_uld = s->ofldqsets;
}
- for (i = 0; i < (s->max_ethqsets + EXTRA_VECS); ++i)
- adap->msix_info[i].vec = entries[i].vector;
- if (is_uld(adap)) {
- for (j = 0 ; i < allocated; ++i, j++) {
- adap->msix_info_ulds[j].vec = entries[i].vector;
- adap->msix_info_ulds[j].idx = i;
+ if (is_ethofld(adap))
+ s->eoqsets = eoqsets;
+
+ if (s->mirrorqsets) {
+ s->mirrorqsets = mirrorqsets;
+ for_each_port(adap, i) {
+ pi = adap2pinfo(adap, i);
+ pi->nmirrorqsets = s->mirrorqsets / nchan;
+ mutex_init(&pi->vi_mirror_mutex);
}
- adap->msix_bmap_ulds.mapsize = j;
}
- dev_info(adap->pdev_dev, "%d MSI-X vectors allocated, "
- "nic %d per uld %d\n",
- allocated, s->max_ethqsets, s->nqs_per_uld);
+
+ /* map for msix */
+ ret = alloc_msix_info(adap, allocated);
+ if (ret)
+ goto out_disable_msix;
+
+ for (i = 0; i < allocated; i++) {
+ adap->msix_info[i].vec = entries[i].vector;
+ adap->msix_info[i].idx = i;
+ }
+
+ dev_info(adap->pdev_dev,
+ "%d MSI-X vectors allocated, nic %d eoqsets %d per uld %d mirrorqsets %d\n",
+ allocated, s->max_ethqsets, s->eoqsets, s->nqs_per_uld,
+ s->mirrorqsets);
kfree(entries);
return 0;
+
+out_disable_msix:
+ pci_disable_msix(adap->pdev);
+
+out_free:
+ kfree(entries);
+ return ret;
}
#undef EXTRA_VECS
@@ -5153,8 +6114,8 @@ static void print_adapter_info(struct adapter *adapter)
/* Software/Hardware configuration */
dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n",
is_offload(adapter) ? "R" : "",
- ((adapter->flags & USING_MSIX) ? "MSI-X" :
- (adapter->flags & USING_MSI) ? "MSI" : ""),
+ ((adapter->flags & CXGB4_USING_MSIX) ? "MSI-X" :
+ (adapter->flags & CXGB4_USING_MSI) ? "MSI" : ""),
is_offload(adapter) ? "Offload" : "non-Offload");
}
@@ -5187,8 +6148,7 @@ static void print_port_info(const struct net_device *dev)
--bufp;
sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
- netdev_info(dev, "%s: Chelsio %s (%s) %s\n",
- dev->name, adap->params.vpd.id, adap->name, buf);
+ netdev_info(dev, "Chelsio %s %s\n", adap->params.vpd.id, buf);
}
/*
@@ -5202,20 +6162,22 @@ static void free_some_resources(struct adapter *adapter)
{
unsigned int i;
- kvfree(adapter->mps_encap);
kvfree(adapter->smt);
kvfree(adapter->l2t);
kvfree(adapter->srq);
t4_cleanup_sched(adapter);
kvfree(adapter->tids.tid_tab);
+ cxgb4_cleanup_tc_matchall(adapter);
+ cxgb4_cleanup_tc_mqprio(adapter);
cxgb4_cleanup_tc_flower(adapter);
cxgb4_cleanup_tc_u32(adapter);
+ cxgb4_cleanup_ethtool_filters(adapter);
kfree(adapter->sge.egr_map);
kfree(adapter->sge.ingr_map);
- kfree(adapter->sge.starving_fl);
- kfree(adapter->sge.txq_maperr);
+ bitmap_free(adapter->sge.starving_fl);
+ bitmap_free(adapter->sge.txq_maperr);
#ifdef CONFIG_DEBUG_FS
- kfree(adapter->sge.blocked_fl);
+ bitmap_free(adapter->sge.blocked_fl);
#endif
disable_msi(adapter);
@@ -5229,13 +6191,14 @@ static void free_some_resources(struct adapter *adapter)
kfree(adap2pinfo(adapter, i)->rss);
free_netdev(adapter->port[i]);
}
- if (adapter->flags & FW_OK)
+ if (adapter->flags & CXGB4_FW_OK)
t4_fw_bye(adapter, adapter->pf);
}
-#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
+#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN | \
+ NETIF_F_GSO_UDP_L4)
#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
- NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
+ NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
#define SEGMENT_SIZE 128
static int t4_get_chip_type(struct adapter *adap, int ver)
@@ -5328,7 +6291,6 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
char name[IFNAMSIZ];
u32 devcap2;
u16 flags;
- int pos;
/* If we want to instantiate Virtual Functions, then our
* parent bridge's PCI-E needs to support Alternative Routing
@@ -5336,9 +6298,8 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
* and above.
*/
pbridge = pdev->bus->self;
- pos = pci_find_capability(pbridge, PCI_CAP_ID_EXP);
- pci_read_config_word(pbridge, pos + PCI_EXP_FLAGS, &flags);
- pci_read_config_dword(pbridge, pos + PCI_EXP_DEVCAP2, &devcap2);
+ pcie_capability_read_word(pbridge, PCI_EXP_FLAGS, &flags);
+ pcie_capability_read_dword(pbridge, PCI_EXP_DEVCAP2, &devcap2);
if ((flags & PCI_EXP_FLAGS_VERS) < 2 ||
!(devcap2 & PCI_EXP_DEVCAP2_ARI)) {
@@ -5418,6 +6379,197 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
}
#endif /* CONFIG_PCI_IOV */
+#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) || IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE)
+
+static int chcr_offload_state(struct adapter *adap,
+ enum cxgb4_netdev_tls_ops op_val)
+{
+ switch (op_val) {
+#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE)
+ case CXGB4_TLSDEV_OPS:
+ if (!adap->uld[CXGB4_ULD_KTLS].handle) {
+ dev_dbg(adap->pdev_dev, "ch_ktls driver is not loaded\n");
+ return -EOPNOTSUPP;
+ }
+ if (!adap->uld[CXGB4_ULD_KTLS].tlsdev_ops) {
+ dev_dbg(adap->pdev_dev,
+ "ch_ktls driver has no registered tlsdev_ops\n");
+ return -EOPNOTSUPP;
+ }
+ break;
+#endif /* CONFIG_CHELSIO_TLS_DEVICE */
+#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE)
+ case CXGB4_XFRMDEV_OPS:
+ if (!adap->uld[CXGB4_ULD_IPSEC].handle) {
+ dev_dbg(adap->pdev_dev, "chipsec driver is not loaded\n");
+ return -EOPNOTSUPP;
+ }
+ if (!adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops) {
+ dev_dbg(adap->pdev_dev,
+ "chipsec driver has no registered xfrmdev_ops\n");
+ return -EOPNOTSUPP;
+ }
+ break;
+#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
+ default:
+ dev_dbg(adap->pdev_dev,
+ "driver has no support for offload %d\n", op_val);
+ return -EOPNOTSUPP;
+ }
+
+ return 0;
+}
+
+#endif /* CONFIG_CHELSIO_TLS_DEVICE || CONFIG_CHELSIO_IPSEC_INLINE */
+
+#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE)
+
+static int cxgb4_ktls_dev_add(struct net_device *netdev, struct sock *sk,
+ enum tls_offload_ctx_dir direction,
+ struct tls_crypto_info *crypto_info,
+ u32 tcp_sn)
+{
+ struct adapter *adap = netdev2adap(netdev);
+ int ret;
+
+ mutex_lock(&uld_mutex);
+ ret = chcr_offload_state(adap, CXGB4_TLSDEV_OPS);
+ if (ret)
+ goto out_unlock;
+
+ ret = cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_ENABLE);
+ if (ret)
+ goto out_unlock;
+
+ ret = adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_add(netdev, sk,
+ direction,
+ crypto_info,
+ tcp_sn);
+ /* if there is a failure, clear the refcount */
+ if (ret)
+ cxgb4_set_ktls_feature(adap,
+ FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE);
+out_unlock:
+ mutex_unlock(&uld_mutex);
+ return ret;
+}
+
+static void cxgb4_ktls_dev_del(struct net_device *netdev,
+ struct tls_context *tls_ctx,
+ enum tls_offload_ctx_dir direction)
+{
+ struct adapter *adap = netdev2adap(netdev);
+
+ mutex_lock(&uld_mutex);
+ if (chcr_offload_state(adap, CXGB4_TLSDEV_OPS))
+ goto out_unlock;
+
+ adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_del(netdev, tls_ctx,
+ direction);
+
+out_unlock:
+ cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE);
+ mutex_unlock(&uld_mutex);
+}
+
+static const struct tlsdev_ops cxgb4_ktls_ops = {
+ .tls_dev_add = cxgb4_ktls_dev_add,
+ .tls_dev_del = cxgb4_ktls_dev_del,
+};
+#endif /* CONFIG_CHELSIO_TLS_DEVICE */
+
+#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE)
+
+static int cxgb4_xfrm_add_state(struct net_device *dev,
+ struct xfrm_state *x,
+ struct netlink_ext_ack *extack)
+{
+ struct adapter *adap = netdev2adap(dev);
+ int ret;
+
+ if (!mutex_trylock(&uld_mutex)) {
+ NL_SET_ERR_MSG_MOD(extack, "crypto uld critical resource is under use");
+ return -EBUSY;
+ }
+ ret = chcr_offload_state(adap, CXGB4_XFRMDEV_OPS);
+ if (ret)
+ goto out_unlock;
+
+ ret = adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_add(dev, x,
+ extack);
+
+out_unlock:
+ mutex_unlock(&uld_mutex);
+
+ return ret;
+}
+
+static void cxgb4_xfrm_del_state(struct net_device *dev, struct xfrm_state *x)
+{
+ struct adapter *adap = netdev2adap(dev);
+
+ if (!mutex_trylock(&uld_mutex)) {
+ dev_dbg(adap->pdev_dev,
+ "crypto uld critical resource is under use\n");
+ return;
+ }
+ if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS))
+ goto out_unlock;
+
+ adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_delete(dev, x);
+
+out_unlock:
+ mutex_unlock(&uld_mutex);
+}
+
+static void cxgb4_xfrm_free_state(struct net_device *dev, struct xfrm_state *x)
+{
+ struct adapter *adap = netdev2adap(dev);
+
+ if (!mutex_trylock(&uld_mutex)) {
+ dev_dbg(adap->pdev_dev,
+ "crypto uld critical resource is under use\n");
+ return;
+ }
+ if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS))
+ goto out_unlock;
+
+ adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_free(dev, x);
+
+out_unlock:
+ mutex_unlock(&uld_mutex);
+}
+
+static void cxgb4_advance_esn_state(struct xfrm_state *x)
+{
+ struct adapter *adap = netdev2adap(x->xso.dev);
+
+ if (x->xso.dir != XFRM_DEV_OFFLOAD_IN)
+ return;
+
+ if (!mutex_trylock(&uld_mutex)) {
+ dev_dbg(adap->pdev_dev,
+ "crypto uld critical resource is under use\n");
+ return;
+ }
+ if (chcr_offload_state(adap, CXGB4_XFRMDEV_OPS))
+ goto out_unlock;
+
+ adap->uld[CXGB4_ULD_IPSEC].xfrmdev_ops->xdo_dev_state_advance_esn(x);
+
+out_unlock:
+ mutex_unlock(&uld_mutex);
+}
+
+static const struct xfrmdev_ops cxgb4_xfrmdev_ops = {
+ .xdo_dev_state_add = cxgb4_xfrm_add_state,
+ .xdo_dev_state_delete = cxgb4_xfrm_del_state,
+ .xdo_dev_state_free = cxgb4_xfrm_free_state,
+ .xdo_dev_state_advance_esn = cxgb4_advance_esn_state,
+};
+
+#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
+
static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
{
struct net_device *netdev;
@@ -5425,7 +6577,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
static int adap_idx = 1;
int s_qpp, qpp, num_seg;
struct port_info *pi;
- bool highdma = false;
enum chip_type chip;
void __iomem *regs;
int func, chip_ver;
@@ -5433,8 +6584,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
int i, err;
u32 whoami;
- printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
-
err = pci_request_regions(pdev, KBUILD_MODNAME);
if (err) {
/* Just info, some other driver may have claimed the device. */
@@ -5470,7 +6619,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
whoami = t4_read_reg(adapter, PL_WHOAMI_A);
pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id);
chip = t4_get_chip_type(adapter, CHELSIO_PCI_ID_VER(device_id));
- if (chip < 0) {
+ if ((int)chip < 0) {
dev_err(&pdev->dev, "Device %d is not supported\n", device_id);
err = chip;
goto out_free_adapter;
@@ -5506,23 +6655,12 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
return 0;
}
- if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
- highdma = true;
- err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
- if (err) {
- dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
- "coherent allocations\n");
- goto out_free_adapter;
- }
- } else {
- err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
- if (err) {
- dev_err(&pdev->dev, "no usable DMA configuration\n");
- goto out_free_adapter;
- }
+ err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+ if (err) {
+ dev_err(&pdev->dev, "no usable DMA configuration\n");
+ goto out_free_adapter;
}
- pci_enable_pcie_error_reporting(pdev);
pci_set_master(pdev);
pci_save_state(pdev);
adap_idx++;
@@ -5533,7 +6671,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
}
/* PCI device has been enabled */
- adapter->flags |= DEV_ENABLED;
+ adapter->flags |= CXGB4_DEV_ENABLED;
memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
/* If possible, we use PCIe Relaxed Ordering Attribute to deliver
@@ -5551,7 +6689,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
* using Relaxed Ordering.
*/
if (!pcie_relaxed_ordering_enabled(pdev))
- adapter->flags |= ROOT_NO_RELAXED_ORDERING;
+ adapter->flags |= CXGB4_ROOT_NO_RELAXED_ORDERING;
spin_lock_init(&adapter->stats_lock);
spin_lock_init(&adapter->tid_release_lock);
@@ -5606,14 +6744,12 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
}
setup_memwin(adapter);
- err = adap_init0(adapter);
-#ifdef CONFIG_DEBUG_FS
- bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz);
-#endif
- setup_memwin_rdma(adapter);
+ err = adap_init0(adapter, 0);
if (err)
goto out_unmap_bar;
+ setup_memwin_rdma(adapter);
+
/* configure SGE_STAT_CFG_A to read WC stats */
if (!is_t4(adapter->params.chip))
t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) |
@@ -5624,8 +6760,14 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
INIT_LIST_HEAD(&adapter->mac_hlist);
for_each_port(adapter, i) {
+ /* For supporting MQPRIO Offload, need some extra
+ * queues for each ETHOFLD TIDs. Keep it equal to
+ * MAX_ATIDs for now. Once we connect to firmware
+ * later and query the EOTID params, we'll come to
+ * know the actual # of EOTIDs supported.
+ */
netdev = alloc_etherdev_mq(sizeof(struct port_info),
- MAX_ETH_QSETS);
+ MAX_ETH_QSETS + MAX_ATIDS);
if (!netdev) {
err = -ENOMEM;
goto out_free_dev;
@@ -5642,25 +6784,43 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
- NETIF_F_RXCSUM | NETIF_F_RXHASH |
+ NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_GRO |
NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
- NETIF_F_HW_TC;
+ NETIF_F_HW_TC | NETIF_F_NTUPLE | NETIF_F_HIGHDMA;
if (chip_ver > CHELSIO_T5) {
netdev->hw_enc_features |= NETIF_F_IP_CSUM |
NETIF_F_IPV6_CSUM |
NETIF_F_RXCSUM |
NETIF_F_GSO_UDP_TUNNEL |
+ NETIF_F_GSO_UDP_TUNNEL_CSUM |
NETIF_F_TSO | NETIF_F_TSO6;
netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
+ NETIF_F_GSO_UDP_TUNNEL_CSUM |
NETIF_F_HW_TLS_RECORD;
+
+ if (adapter->rawf_cnt)
+ netdev->udp_tunnel_nic_info = &cxgb_udp_tunnels;
}
- if (highdma)
- netdev->hw_features |= NETIF_F_HIGHDMA;
netdev->features |= netdev->hw_features;
netdev->vlan_features = netdev->features & VLAN_FEAT;
+#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE)
+ if (pi->adapter->params.crypto & FW_CAPS_CONFIG_TLS_HW) {
+ netdev->hw_features |= NETIF_F_HW_TLS_TX;
+ netdev->tlsdev_ops = &cxgb4_ktls_ops;
+ /* initialize the refcount */
+ refcount_set(&pi->adapter->chcr_ktls.ktls_refcount, 0);
+ }
+#endif /* CONFIG_CHELSIO_TLS_DEVICE */
+#if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE)
+ if (pi->adapter->params.crypto & FW_CAPS_CONFIG_IPSEC_INLINE) {
+ netdev->hw_enc_features |= NETIF_F_HW_ESP;
+ netdev->features |= NETIF_F_HW_ESP;
+ netdev->xfrmdev_ops = &cxgb4_xfrmdev_ops;
+ }
+#endif /* CONFIG_CHELSIO_IPSEC_INLINE */
netdev->priv_flags |= IFF_UNICAST_FLT;
@@ -5681,7 +6841,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
pci_set_drvdata(pdev, adapter);
- if (adapter->flags & FW_OK) {
+ if (adapter->flags & CXGB4_FW_OK) {
err = t4_port_init(adapter, func, func, 0);
if (err)
goto out_free_dev;
@@ -5703,7 +6863,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
}
}
- if (!(adapter->flags & FW_OK))
+ if (!(adapter->flags & CXGB4_FW_OK))
goto fw_attach_fail;
/* Configure queues and allocate tables now, they can be needed as
@@ -5726,12 +6886,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
adapter->params.offload = 0;
}
- adapter->mps_encap = kvcalloc(adapter->params.arch.mps_tcam_size,
- sizeof(struct mps_encap_entry),
- GFP_KERNEL);
- if (!adapter->mps_encap)
- dev_warn(&pdev->dev, "could not allocate MPS Encap entries, continuing\n");
-
#if IS_ENABLED(CONFIG_IPV6)
if (chip_ver <= CHELSIO_T5 &&
(!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) {
@@ -5764,6 +6918,24 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
i);
}
+ if (is_offload(adapter) || is_hashfilter(adapter)) {
+ if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) {
+ u32 v;
+
+ v = t4_read_reg(adapter, LE_DB_HASH_CONFIG_A);
+ if (chip_ver <= CHELSIO_T5) {
+ adapter->tids.nhash = 1 << HASHTIDSIZE_G(v);
+ v = t4_read_reg(adapter, LE_DB_TID_HASHBASE_A);
+ adapter->tids.hash_base = v / 4;
+ } else {
+ adapter->tids.nhash = HASHTBLSIZE_G(v) << 3;
+ v = t4_read_reg(adapter,
+ T6_LE_DB_HASH_TID_BASE_A);
+ adapter->tids.hash_base = v;
+ }
+ }
+ }
+
if (tid_init(&adapter->tids) < 0) {
dev_warn(&pdev->dev, "could not allocate TID table, "
"continuing\n");
@@ -5777,29 +6949,24 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
if (cxgb4_init_tc_flower(adapter))
dev_warn(&pdev->dev,
"could not offload tc flower, continuing\n");
- }
- if (is_offload(adapter) || is_hashfilter(adapter)) {
- if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) {
- u32 hash_base, hash_reg;
+ if (cxgb4_init_tc_mqprio(adapter))
+ dev_warn(&pdev->dev,
+ "could not offload tc mqprio, continuing\n");
- if (chip_ver <= CHELSIO_T5) {
- hash_reg = LE_DB_TID_HASHBASE_A;
- hash_base = t4_read_reg(adapter, hash_reg);
- adapter->tids.hash_base = hash_base / 4;
- } else {
- hash_reg = T6_LE_DB_HASH_TID_BASE_A;
- hash_base = t4_read_reg(adapter, hash_reg);
- adapter->tids.hash_base = hash_base;
- }
- }
+ if (cxgb4_init_tc_matchall(adapter))
+ dev_warn(&pdev->dev,
+ "could not offload tc matchall, continuing\n");
+ if (cxgb4_init_ethtool_filters(adapter))
+ dev_warn(&pdev->dev,
+ "could not initialize ethtool filters, continuing\n");
}
/* See what interrupts we'll be using */
if (msi > 1 && enable_msix(adapter) == 0)
- adapter->flags |= USING_MSIX;
+ adapter->flags |= CXGB4_USING_MSIX;
else if (msi > 0 && pci_enable_msi(pdev) == 0) {
- adapter->flags |= USING_MSI;
+ adapter->flags |= CXGB4_USING_MSI;
if (msi > 1)
free_msix_info(adapter);
}
@@ -5807,10 +6974,19 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
/* check for PCI Express bandwidth capabiltites */
pcie_print_link_status(pdev);
+ cxgb4_init_mps_ref_entries(adapter);
+
err = init_rss(adapter);
if (err)
goto out_free_dev;
+ err = setup_non_data_intr(adapter);
+ if (err) {
+ dev_err(adapter->pdev_dev,
+ "Non Data interrupt allocation failed, err: %d\n", err);
+ goto out_free_dev;
+ }
+
err = setup_fw_sge_queues(adapter);
if (err) {
dev_err(adapter->pdev_dev,
@@ -5857,17 +7033,14 @@ fw_attach_fail:
/* PCIe EEH recovery on powerpc platforms needs fundamental reset */
pdev->needs_freset = 1;
- if (is_uld(adapter)) {
- mutex_lock(&uld_mutex);
- list_add_tail(&adapter->list_node, &adapter_list);
- mutex_unlock(&uld_mutex);
- }
+ if (is_uld(adapter))
+ cxgb4_uld_enable(adapter);
if (!is_t4(adapter->params.chip))
cxgb4_ptp_init(adapter);
if (IS_REACHABLE(CONFIG_THERMAL) &&
- !is_t4(adapter->params.chip) && (adapter->flags & FW_OK))
+ !is_t4(adapter->params.chip) && (adapter->flags & CXGB4_FW_OK))
cxgb4_thermal_init(adapter);
print_adapter_info(adapter);
@@ -5876,7 +7049,7 @@ fw_attach_fail:
out_free_dev:
t4_free_sge_resources(adapter);
free_some_resources(adapter);
- if (adapter->flags & USING_MSIX)
+ if (adapter->flags & CXGB4_USING_MSIX)
free_msix_info(adapter);
if (adapter->num_uld || adapter->num_ofld_uld)
t4_uld_mem_free(adapter);
@@ -5892,7 +7065,6 @@ fw_attach_fail:
out_unmap_bar0:
iounmap(regs);
out_disable_device:
- pci_disable_pcie_error_reporting(pdev);
pci_disable_device(pdev);
out_release_regions:
pci_release_regions(pdev);
@@ -5909,7 +7081,12 @@ static void remove_one(struct pci_dev *pdev)
return;
}
- adapter->flags |= SHUTTING_DOWN;
+ /* If we allocated filters, free up state associated with any
+ * valid filters ...
+ */
+ clear_all_filters(adapter);
+
+ adapter->flags |= CXGB4_SHUTTING_DOWN;
if (adapter->pf == 4) {
int i;
@@ -5919,18 +7096,19 @@ static void remove_one(struct pci_dev *pdev)
*/
destroy_workqueue(adapter->workq);
- if (is_uld(adapter)) {
- detach_ulds(adapter);
- t4_uld_clean_up(adapter);
- }
+ detach_ulds(adapter);
+
+ for_each_port(adapter, i)
+ if (adapter->port[i]->reg_state == NETREG_REGISTERED)
+ unregister_netdev(adapter->port[i]);
+
+ t4_uld_clean_up(adapter);
adap_free_hma_mem(adapter);
disable_interrupts(adapter);
- for_each_port(adapter, i)
- if (adapter->port[i]->reg_state == NETREG_REGISTERED)
- unregister_netdev(adapter->port[i]);
+ cxgb4_free_mps_ref_entries(adapter);
debugfs_remove_recursive(adapter->debugfs_root);
@@ -5939,15 +7117,10 @@ static void remove_one(struct pci_dev *pdev)
if (IS_REACHABLE(CONFIG_THERMAL))
cxgb4_thermal_remove(adapter);
- /* If we allocated filters, free up state associated with any
- * valid filters ...
- */
- clear_all_filters(adapter);
-
- if (adapter->flags & FULL_INIT_DONE)
+ if (adapter->flags & CXGB4_FULL_INIT_DONE)
cxgb_down(adapter);
- if (adapter->flags & USING_MSIX)
+ if (adapter->flags & CXGB4_USING_MSIX)
free_msix_info(adapter);
if (adapter->num_uld || adapter->num_ofld_uld)
t4_uld_mem_free(adapter);
@@ -5970,10 +7143,9 @@ static void remove_one(struct pci_dev *pdev)
}
#endif
iounmap(adapter->regs);
- pci_disable_pcie_error_reporting(pdev);
- if ((adapter->flags & DEV_ENABLED)) {
+ if ((adapter->flags & CXGB4_DEV_ENABLED)) {
pci_disable_device(pdev);
- adapter->flags &= ~DEV_ENABLED;
+ adapter->flags &= ~CXGB4_DEV_ENABLED;
}
pci_release_regions(pdev);
kfree(adapter->mbox_log);
@@ -5999,7 +7171,7 @@ static void shutdown_one(struct pci_dev *pdev)
return;
}
- adapter->flags |= SHUTTING_DOWN;
+ adapter->flags |= CXGB4_SHUTTING_DOWN;
if (adapter->pf == 4) {
int i;
@@ -6008,6 +7180,10 @@ static void shutdown_one(struct pci_dev *pdev)
if (adapter->port[i]->reg_state == NETREG_REGISTERED)
cxgb_close(adapter->port[i]);
+ rtnl_lock();
+ cxgb4_mqprio_stop_offload(adapter);
+ rtnl_unlock();
+
if (is_uld(adapter)) {
detach_ulds(adapter);
t4_uld_clean_up(adapter);
@@ -6017,7 +7193,7 @@ static void shutdown_one(struct pci_dev *pdev)
disable_msi(adapter);
t4_sge_stop(adapter);
- if (adapter->flags & FW_OK)
+ if (adapter->flags & CXGB4_FW_OK)
t4_fw_bye(adapter, adapter->mbox);
}
}
@@ -6038,22 +7214,28 @@ static int __init cxgb4_init_module(void)
{
int ret;
- /* Debugfs support is optional, just warn if this fails */
cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
- if (!cxgb4_debugfs_root)
- pr_warn("could not create debugfs entry, continuing\n");
ret = pci_register_driver(&cxgb4_driver);
if (ret < 0)
- debugfs_remove(cxgb4_debugfs_root);
+ goto err_pci;
#if IS_ENABLED(CONFIG_IPV6)
if (!inet6addr_registered) {
- register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
- inet6addr_registered = true;
+ ret = register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
+ if (ret)
+ pci_unregister_driver(&cxgb4_driver);
+ else
+ inet6addr_registered = true;
}
#endif
+ if (ret == 0)
+ return ret;
+
+err_pci:
+ debugfs_remove(cxgb4_debugfs_root);
+
return ret;
}
generated by cgit (git 2.34.1) at 2025-12-26 04:50:42 +0000