summaryrefslogtreecommitdiff
path: root/drivers/net/ethernet/pensando/ionic
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/ethernet/pensando/ionic')
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic.h6
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c11
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_debugfs.c4
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_dev.c132
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_dev.h37
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_ethtool.c152
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_if.h276
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_lif.c335
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_lif.h14
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_main.c5
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c4
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_txrx.c516
-rw-r--r--drivers/net/ethernet/pensando/ionic/ionic_txrx.h4
13 files changed, 970 insertions, 526 deletions
diff --git a/drivers/net/ethernet/pensando/ionic/ionic.h b/drivers/net/ethernet/pensando/ionic/ionic.h
index 2ccc2c2a06e3..04f00ea94230 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic.h
@@ -47,6 +47,7 @@ struct ionic {
struct ionic_dev_bar bars[IONIC_BARS_MAX];
unsigned int num_bars;
struct ionic_identity ident;
+ struct workqueue_struct *wq;
struct ionic_lif *lif;
unsigned int nnqs_per_lif;
unsigned int neqs_per_lif;
@@ -54,7 +55,8 @@ struct ionic {
unsigned int nrxqs_per_lif;
unsigned int nintrs;
DECLARE_BITMAP(intrs, IONIC_INTR_CTRL_REGS_MAX);
- struct work_struct nb_work;
+ cpumask_var_t *affinity_masks;
+ struct delayed_work doorbell_check_dwork;
struct notifier_block nb;
struct rw_semaphore vf_op_lock; /* lock for VF operations */
struct ionic_vf *vfs;
@@ -93,4 +95,6 @@ int ionic_port_identify(struct ionic *ionic);
int ionic_port_init(struct ionic *ionic);
int ionic_port_reset(struct ionic *ionic);
+bool ionic_doorbell_wa(struct ionic *ionic);
+
#endif /* _IONIC_H_ */
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
index 6ba8d4aca0a0..4c377bdc62c8 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
@@ -326,6 +326,11 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto err_out;
}
+#ifdef CONFIG_PPC64
+ /* Ensure MSI/MSI-X interrupts lie within addressable physical memory */
+ pdev->no_64bit_msi = 1;
+#endif
+
err = ionic_setup_one(ionic);
if (err)
goto err_out;
@@ -372,6 +377,7 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
mod_timer(&ionic->watchdog_timer,
round_jiffies(jiffies + ionic->watchdog_period));
+ ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
return 0;
@@ -388,6 +394,7 @@ err_out_free_irqs:
err_out_pci:
ionic_dev_teardown(ionic);
ionic_clear_pci(ionic);
+ ionic_debugfs_del_dev(ionic);
err_out:
mutex_destroy(&ionic->dev_cmd_lock);
ionic_devlink_free(ionic);
@@ -406,6 +413,8 @@ static void ionic_remove(struct pci_dev *pdev)
if (test_and_clear_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state))
set_bit(IONIC_LIF_F_FW_STOPPING, ionic->lif->state);
+ if (ionic->lif->doorbell_wa)
+ cancel_delayed_work_sync(&ionic->doorbell_check_dwork);
ionic_lif_unregister(ionic->lif);
ionic_devlink_unregister(ionic);
ionic_lif_deinit(ionic->lif);
@@ -432,7 +441,7 @@ static void ionic_reset_prepare(struct pci_dev *pdev)
set_bit(IONIC_LIF_F_FW_RESET, lif->state);
- del_timer_sync(&ionic->watchdog_timer);
+ timer_delete_sync(&ionic->watchdog_timer);
cancel_work_sync(&lif->deferred.work);
mutex_lock(&lif->queue_lock);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c
index c3ae11a48024..c98b4e75e288 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c
@@ -123,7 +123,7 @@ void ionic_debugfs_add_qcq(struct ionic_lif *lif, struct ionic_qcq *qcq)
struct ionic_cq *cq = &qcq->cq;
qcq_dentry = debugfs_create_dir(q->name, lif->dentry);
- if (IS_ERR_OR_NULL(qcq_dentry))
+ if (IS_ERR(qcq_dentry))
return;
qcq->dentry = qcq_dentry;
@@ -220,7 +220,7 @@ static int netdev_show(struct seq_file *seq, void *v)
{
struct net_device *netdev = seq->private;
- seq_printf(seq, "%s\n", netdev->name);
+ seq_printf(seq, "%s\n", netdev_name(netdev));
return 0;
}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.c b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
index 874499337132..57edcde9e6f8 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
@@ -43,11 +43,99 @@ static void ionic_watchdog_cb(struct timer_list *t)
work->type = IONIC_DW_TYPE_RX_MODE;
netdev_dbg(lif->netdev, "deferred: rx_mode\n");
- ionic_lif_deferred_enqueue(&lif->deferred, work);
+ ionic_lif_deferred_enqueue(lif, work);
}
}
-static void ionic_watchdog_init(struct ionic *ionic)
+static void ionic_napi_schedule_do_softirq(struct napi_struct *napi)
+{
+ local_bh_disable();
+ napi_schedule(napi);
+ local_bh_enable();
+}
+
+void ionic_doorbell_napi_work(struct work_struct *work)
+{
+ struct ionic_qcq *qcq = container_of(work, struct ionic_qcq,
+ doorbell_napi_work);
+ unsigned long now, then, dif;
+
+ now = READ_ONCE(jiffies);
+ then = qcq->q.dbell_jiffies;
+ dif = now - then;
+
+ if (dif > qcq->q.dbell_deadline)
+ ionic_napi_schedule_do_softirq(&qcq->napi);
+}
+
+static int ionic_get_preferred_cpu(struct ionic *ionic,
+ struct ionic_intr_info *intr)
+{
+ int cpu;
+
+ cpu = cpumask_first_and(*intr->affinity_mask, cpu_online_mask);
+ if (cpu >= nr_cpu_ids)
+ cpu = cpumask_local_spread(0, dev_to_node(ionic->dev));
+
+ return cpu;
+}
+
+static void ionic_queue_dbell_napi_work(struct ionic *ionic,
+ struct ionic_qcq *qcq)
+{
+ int cpu;
+
+ if (!(qcq->flags & IONIC_QCQ_F_INTR))
+ return;
+
+ cpu = ionic_get_preferred_cpu(ionic, &qcq->intr);
+ queue_work_on(cpu, ionic->wq, &qcq->doorbell_napi_work);
+}
+
+static void ionic_doorbell_check_dwork(struct work_struct *work)
+{
+ struct ionic *ionic = container_of(work, struct ionic,
+ doorbell_check_dwork.work);
+ struct ionic_lif *lif = ionic->lif;
+
+ mutex_lock(&lif->queue_lock);
+
+ if (test_bit(IONIC_LIF_F_FW_STOPPING, lif->state) ||
+ test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
+ mutex_unlock(&lif->queue_lock);
+ return;
+ }
+
+ ionic_napi_schedule_do_softirq(&lif->adminqcq->napi);
+
+ if (test_bit(IONIC_LIF_F_UP, lif->state)) {
+ int i;
+
+ for (i = 0; i < lif->nxqs; i++) {
+ ionic_queue_dbell_napi_work(ionic, lif->txqcqs[i]);
+ ionic_queue_dbell_napi_work(ionic, lif->rxqcqs[i]);
+ }
+
+ if (lif->hwstamp_txq &&
+ lif->hwstamp_txq->flags & IONIC_QCQ_F_INTR)
+ ionic_napi_schedule_do_softirq(&lif->hwstamp_txq->napi);
+ if (lif->hwstamp_rxq &&
+ lif->hwstamp_rxq->flags & IONIC_QCQ_F_INTR)
+ ionic_napi_schedule_do_softirq(&lif->hwstamp_rxq->napi);
+ }
+ mutex_unlock(&lif->queue_lock);
+
+ ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
+}
+
+bool ionic_doorbell_wa(struct ionic *ionic)
+{
+ u8 asic_type = ionic->idev.dev_info.asic_type;
+
+ return !asic_type || asic_type == IONIC_ASIC_TYPE_ELBA;
+}
+
+static int ionic_watchdog_init(struct ionic *ionic)
{
struct ionic_dev *idev = &ionic->idev;
@@ -63,6 +151,31 @@ static void ionic_watchdog_init(struct ionic *ionic)
idev->fw_status_ready = true;
idev->fw_generation = IONIC_FW_STS_F_GENERATION &
ioread8(&idev->dev_info_regs->fw_status);
+
+ ionic->wq = alloc_workqueue("%s-wq", WQ_UNBOUND, 0,
+ dev_name(ionic->dev));
+ if (!ionic->wq) {
+ dev_err(ionic->dev, "alloc_workqueue failed");
+ return -ENOMEM;
+ }
+
+ if (ionic_doorbell_wa(ionic))
+ INIT_DELAYED_WORK(&ionic->doorbell_check_dwork,
+ ionic_doorbell_check_dwork);
+
+ return 0;
+}
+
+void ionic_queue_doorbell_check(struct ionic *ionic, int delay)
+{
+ int cpu;
+
+ if (!ionic->lif->doorbell_wa)
+ return;
+
+ cpu = ionic_get_preferred_cpu(ionic, &ionic->lif->adminqcq->intr);
+ queue_delayed_work_on(cpu, ionic->wq, &ionic->doorbell_check_dwork,
+ delay);
}
void ionic_init_devinfo(struct ionic *ionic)
@@ -94,6 +207,7 @@ int ionic_dev_setup(struct ionic *ionic)
struct device *dev = ionic->dev;
int size;
u32 sig;
+ int err;
/* BAR0: dev_cmd and interrupts */
if (num_bars < 1) {
@@ -129,7 +243,9 @@ int ionic_dev_setup(struct ionic *ionic)
return -EFAULT;
}
- ionic_watchdog_init(ionic);
+ err = ionic_watchdog_init(ionic);
+ if (err)
+ return err;
idev->db_pages = bar->vaddr;
idev->phy_db_pages = bar->bus_addr;
@@ -161,6 +277,10 @@ void ionic_dev_teardown(struct ionic *ionic)
idev->phy_cmb_pages = 0;
idev->cmb_npages = 0;
+ if (ionic->wq) {
+ destroy_workqueue(ionic->wq);
+ ionic->wq = NULL;
+ }
mutex_destroy(&idev->cmb_inuse_lock);
}
@@ -273,7 +393,7 @@ do_check_time:
if (work) {
work->type = IONIC_DW_TYPE_LIF_RESET;
work->fw_status = fw_status_ready;
- ionic_lif_deferred_enqueue(&lif->deferred, work);
+ ionic_lif_deferred_enqueue(lif, work);
}
}
}
@@ -703,10 +823,6 @@ void ionic_q_post(struct ionic_queue *q, bool ring_doorbell)
q->dbval | q->head_idx);
q->dbell_jiffies = jiffies;
-
- if (q_to_qcq(q)->napi_qcq)
- mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
- jiffies + IONIC_NAPI_DEADLINE);
}
}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
index f30eee4a5a80..c8c710cfe70c 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
@@ -28,11 +28,11 @@
#define IONIC_DEV_INFO_REG_COUNT 32
#define IONIC_DEV_CMD_REG_COUNT 32
-#define IONIC_NAPI_DEADLINE (HZ / 200) /* 5ms */
+#define IONIC_NAPI_DEADLINE (HZ) /* 1 sec */
#define IONIC_ADMIN_DOORBELL_DEADLINE (HZ / 2) /* 500ms */
#define IONIC_TX_DOORBELL_DEADLINE (HZ / 100) /* 10ms */
#define IONIC_RX_MIN_DOORBELL_DEADLINE (HZ / 100) /* 10ms */
-#define IONIC_RX_MAX_DOORBELL_DEADLINE (HZ * 5) /* 5s */
+#define IONIC_RX_MAX_DOORBELL_DEADLINE (HZ * 4) /* 4s */
struct ionic_dev_bar {
void __iomem *vaddr;
@@ -181,10 +181,7 @@ struct ionic_queue;
struct ionic_qcq;
#define IONIC_MAX_BUF_LEN ((u16)-1)
-#define IONIC_PAGE_SIZE PAGE_SIZE
-#define IONIC_PAGE_SPLIT_SZ (PAGE_SIZE / 2)
-#define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\
- __GFP_COMP | __GFP_MEMALLOC)
+#define IONIC_PAGE_SIZE MIN(PAGE_SIZE, IONIC_MAX_BUF_LEN)
#define IONIC_XDP_MAX_LINEAR_MTU (IONIC_PAGE_SIZE - \
(VLAN_ETH_HLEN + \
@@ -238,9 +235,8 @@ struct ionic_queue {
unsigned int index;
unsigned int num_descs;
unsigned int max_sg_elems;
+
u64 features;
- unsigned int type;
- unsigned int hw_index;
unsigned int hw_type;
bool xdp_flush;
union {
@@ -250,18 +246,23 @@ struct ionic_queue {
struct ionic_admin_cmd *adminq;
};
union {
- void __iomem *cmb_base;
- struct ionic_txq_desc __iomem *cmb_txq;
- struct ionic_rxq_desc __iomem *cmb_rxq;
- };
- union {
void *sg_base;
struct ionic_txq_sg_desc *txq_sgl;
struct ionic_txq_sg_desc_v1 *txq_sgl_v1;
struct ionic_rxq_sg_desc *rxq_sgl;
};
struct xdp_rxq_info *xdp_rxq_info;
+ struct bpf_prog *xdp_prog;
+ struct page_pool *page_pool;
struct ionic_queue *partner;
+
+ union {
+ void __iomem *cmb_base;
+ struct ionic_txq_desc __iomem *cmb_txq;
+ struct ionic_rxq_desc __iomem *cmb_rxq;
+ };
+ unsigned int type;
+ unsigned int hw_index;
dma_addr_t base_pa;
dma_addr_t cmb_base_pa;
dma_addr_t sg_base_pa;
@@ -280,9 +281,9 @@ struct ionic_intr_info {
u64 rearm_count;
unsigned int index;
unsigned int vector;
- unsigned int cpu;
u32 dim_coal_hw;
- cpumask_t affinity_mask;
+ cpumask_var_t *affinity_mask;
+ struct irq_affinity_notify aff_notify;
};
struct ionic_cq {
@@ -375,7 +376,9 @@ typedef void (*ionic_cq_done_cb)(void *done_arg);
unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do,
ionic_cq_cb cb, ionic_cq_done_cb done_cb,
void *done_arg);
-unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do);
+unsigned int ionic_tx_cq_service(struct ionic_cq *cq,
+ unsigned int work_to_do,
+ bool in_napi);
int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev,
struct ionic_queue *q, unsigned int index, const char *name,
@@ -386,6 +389,8 @@ bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos);
int ionic_heartbeat_check(struct ionic *ionic);
bool ionic_is_fw_running(struct ionic_dev *idev);
+void ionic_doorbell_napi_work(struct work_struct *work);
+void ionic_queue_doorbell_check(struct ionic *ionic, int delay);
bool ionic_adminq_poke_doorbell(struct ionic_queue *q);
bool ionic_txq_poke_doorbell(struct ionic_queue *q);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
index 91183965a6b7..92f30ff2d631 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
@@ -11,6 +11,8 @@
#include "ionic_ethtool.h"
#include "ionic_stats.h"
+#define IONIC_MAX_RX_COPYBREAK min(U16_MAX, IONIC_MAX_BUF_LEN)
+
static void ionic_get_stats_strings(struct ionic_lif *lif, u8 *buf)
{
u32 i;
@@ -156,6 +158,20 @@ static int ionic_get_link_ksettings(struct net_device *netdev,
25000baseCR_Full);
copper_seen++;
break;
+ case IONIC_XCVR_PID_QSFP_50G_CR2_FC:
+ case IONIC_XCVR_PID_QSFP_50G_CR2:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 50000baseCR2_Full);
+ copper_seen++;
+ break;
+ case IONIC_XCVR_PID_QSFP_200G_CR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported, 200000baseCR4_Full);
+ copper_seen++;
+ break;
+ case IONIC_XCVR_PID_QSFP_400G_CR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported, 400000baseCR4_Full);
+ copper_seen++;
+ break;
case IONIC_XCVR_PID_SFP_10GBASE_AOC:
case IONIC_XCVR_PID_SFP_10GBASE_CU:
ethtool_link_ksettings_add_link_mode(ks, supported,
@@ -194,6 +210,31 @@ static int ionic_get_link_ksettings(struct net_device *netdev,
ethtool_link_ksettings_add_link_mode(ks, supported,
25000baseSR_Full);
break;
+ case IONIC_XCVR_PID_QSFP_200G_AOC:
+ case IONIC_XCVR_PID_QSFP_200G_SR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 200000baseSR4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_200G_FR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 200000baseLR4_ER4_FR4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_200G_DR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 200000baseDR4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_400G_FR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 400000baseLR4_ER4_FR4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_400G_DR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 400000baseDR4_Full);
+ break;
+ case IONIC_XCVR_PID_QSFP_400G_SR4:
+ ethtool_link_ksettings_add_link_mode(ks, supported,
+ 400000baseSR4_Full);
+ break;
case IONIC_XCVR_PID_SFP_10GBASE_SR:
ethtool_link_ksettings_add_link_mode(ks, supported,
10000baseSR_Full);
@@ -872,10 +913,17 @@ static int ionic_set_tunable(struct net_device *dev,
const void *data)
{
struct ionic_lif *lif = netdev_priv(dev);
+ u32 rx_copybreak;
switch (tuna->id) {
case ETHTOOL_RX_COPYBREAK:
- lif->rx_copybreak = *(u32 *)data;
+ rx_copybreak = *(u32 *)data;
+ if (rx_copybreak > IONIC_MAX_RX_COPYBREAK) {
+ netdev_err(dev, "Max supported rx_copybreak size: %u\n",
+ IONIC_MAX_RX_COPYBREAK);
+ return -EINVAL;
+ }
+ lif->rx_copybreak = (u16)rx_copybreak;
break;
default:
return -EOPNOTSUPP;
@@ -900,63 +948,20 @@ static int ionic_get_tunable(struct net_device *netdev,
return 0;
}
-static int ionic_get_module_info(struct net_device *netdev,
- struct ethtool_modinfo *modinfo)
-
-{
- struct ionic_lif *lif = netdev_priv(netdev);
- struct ionic_dev *idev = &lif->ionic->idev;
- struct ionic_xcvr_status *xcvr;
- struct sfp_eeprom_base *sfp;
-
- xcvr = &idev->port_info->status.xcvr;
- sfp = (struct sfp_eeprom_base *) xcvr->sprom;
-
- /* report the module data type and length */
- switch (sfp->phys_id) {
- case SFF8024_ID_SFP:
- modinfo->type = ETH_MODULE_SFF_8079;
- modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
- break;
- case SFF8024_ID_QSFP_8436_8636:
- case SFF8024_ID_QSFP28_8636:
- modinfo->type = ETH_MODULE_SFF_8436;
- modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN;
- break;
- default:
- netdev_info(netdev, "unknown xcvr type 0x%02x\n",
- xcvr->sprom[0]);
- modinfo->type = 0;
- modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN;
- break;
- }
-
- return 0;
-}
-
-static int ionic_get_module_eeprom(struct net_device *netdev,
- struct ethtool_eeprom *ee,
- u8 *data)
+static int ionic_do_module_copy(u8 *dst, u8 *src, u32 len)
{
- struct ionic_lif *lif = netdev_priv(netdev);
- struct ionic_dev *idev = &lif->ionic->idev;
- struct ionic_xcvr_status *xcvr;
- char tbuf[sizeof(xcvr->sprom)];
+ char tbuf[sizeof_field(struct ionic_xcvr_status, sprom)];
int count = 10;
- u32 len;
/* The NIC keeps the module prom up-to-date in the DMA space
* so we can simply copy the module bytes into the data buffer.
*/
- xcvr = &idev->port_info->status.xcvr;
- len = min_t(u32, sizeof(xcvr->sprom), ee->len);
-
do {
- memcpy(data, xcvr->sprom, len);
- memcpy(tbuf, xcvr->sprom, len);
+ memcpy(dst, src, len);
+ memcpy(tbuf, src, len);
/* Let's make sure we got a consistent copy */
- if (!memcmp(data, tbuf, len))
+ if (!memcmp(dst, tbuf, len))
break;
} while (--count);
@@ -967,8 +972,50 @@ static int ionic_get_module_eeprom(struct net_device *netdev,
return 0;
}
+static int ionic_get_module_eeprom_by_page(struct net_device *netdev,
+ const struct ethtool_module_eeprom *page_data,
+ struct netlink_ext_ack *extack)
+{
+ struct ionic_lif *lif = netdev_priv(netdev);
+ struct ionic_dev *idev = &lif->ionic->idev;
+ u32 err = -EINVAL;
+ u8 *src;
+
+ if (!page_data->length)
+ return -EINVAL;
+
+ if (page_data->bank != 0) {
+ NL_SET_ERR_MSG_MOD(extack, "Only bank 0 is supported");
+ return -EINVAL;
+ }
+
+ switch (page_data->page) {
+ case 0:
+ src = &idev->port_info->status.xcvr.sprom[page_data->offset];
+ break;
+ case 1:
+ src = &idev->port_info->sprom_page1[page_data->offset - 128];
+ break;
+ case 2:
+ src = &idev->port_info->sprom_page2[page_data->offset - 128];
+ break;
+ case 17:
+ src = &idev->port_info->sprom_page17[page_data->offset - 128];
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ memset(page_data->data, 0, page_data->length);
+ err = ionic_do_module_copy(page_data->data, src, page_data->length);
+ if (err)
+ return err;
+
+ return page_data->length;
+}
+
static int ionic_get_ts_info(struct net_device *netdev,
- struct ethtool_ts_info *info)
+ struct kernel_ethtool_ts_info *info)
{
struct ionic_lif *lif = netdev_priv(netdev);
struct ionic *ionic = lif->ionic;
@@ -980,8 +1027,6 @@ static int ionic_get_ts_info(struct net_device *netdev,
info->phc_index = ptp_clock_index(lif->phc->ptp);
info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
- SOF_TIMESTAMPING_RX_SOFTWARE |
- SOF_TIMESTAMPING_SOFTWARE |
SOF_TIMESTAMPING_TX_HARDWARE |
SOF_TIMESTAMPING_RX_HARDWARE |
SOF_TIMESTAMPING_RAW_HARDWARE;
@@ -1114,8 +1159,7 @@ static const struct ethtool_ops ionic_ethtool_ops = {
.set_rxfh = ionic_set_rxfh,
.get_tunable = ionic_get_tunable,
.set_tunable = ionic_set_tunable,
- .get_module_info = ionic_get_module_info,
- .get_module_eeprom = ionic_get_module_eeprom,
+ .get_module_eeprom_by_page = ionic_get_module_eeprom_by_page,
.get_pauseparam = ionic_get_pauseparam,
.set_pauseparam = ionic_set_pauseparam,
.get_fecparam = ionic_get_fecparam,
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h
index 9a1825edf0d0..f1ddbe9994a3 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_if.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h
@@ -71,7 +71,7 @@ enum ionic_cmd_opcode {
IONIC_CMD_FW_CONTROL_V1 = 255,
};
-/**
+/*
* enum ionic_status_code - Device command return codes
*/
enum ionic_status_code {
@@ -112,6 +112,7 @@ enum ionic_notifyq_opcode {
/**
* struct ionic_admin_cmd - General admin command format
* @opcode: Opcode for the command
+ * @rsvd: reserved byte(s)
* @lif_index: LIF index
* @cmd_data: Opcode-specific command bytes
*/
@@ -125,6 +126,7 @@ struct ionic_admin_cmd {
/**
* struct ionic_admin_comp - General admin command completion format
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
* @cmd_data: Command-specific bytes
* @color: Color bit (Always 0 for commands issued to the
@@ -147,6 +149,7 @@ static inline u8 color_match(u8 color, u8 done_color)
/**
* struct ionic_nop_cmd - NOP command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
*/
struct ionic_nop_cmd {
u8 opcode;
@@ -156,6 +159,7 @@ struct ionic_nop_cmd {
/**
* struct ionic_nop_comp - NOP command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
*/
struct ionic_nop_comp {
u8 status;
@@ -166,6 +170,7 @@ struct ionic_nop_comp {
* struct ionic_dev_init_cmd - Device init command
* @opcode: opcode
* @type: Device type
+ * @rsvd: reserved byte(s)
*/
struct ionic_dev_init_cmd {
u8 opcode;
@@ -176,6 +181,7 @@ struct ionic_dev_init_cmd {
/**
* struct ionic_dev_init_comp - Device init command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
*/
struct ionic_dev_init_comp {
u8 status;
@@ -185,6 +191,7 @@ struct ionic_dev_init_comp {
/**
* struct ionic_dev_reset_cmd - Device reset command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
*/
struct ionic_dev_reset_cmd {
u8 opcode;
@@ -194,6 +201,7 @@ struct ionic_dev_reset_cmd {
/**
* struct ionic_dev_reset_comp - Reset command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
*/
struct ionic_dev_reset_comp {
u8 status;
@@ -207,6 +215,7 @@ struct ionic_dev_reset_comp {
* struct ionic_dev_identify_cmd - Driver/device identify command
* @opcode: opcode
* @ver: Highest version of identify supported by driver
+ * @rsvd: reserved byte(s)
*/
struct ionic_dev_identify_cmd {
u8 opcode;
@@ -218,6 +227,7 @@ struct ionic_dev_identify_cmd {
* struct ionic_dev_identify_comp - Driver/device identify command completion
* @status: Status of the command (enum ionic_status_code)
* @ver: Version of identify returned by device
+ * @rsvd: reserved byte(s)
*/
struct ionic_dev_identify_comp {
u8 status;
@@ -242,6 +252,7 @@ enum ionic_os_type {
* @kernel_ver: Kernel version, numeric format
* @kernel_ver_str: Kernel version, string format
* @driver_ver_str: Driver version, string format
+ * @words: word access to struct contents
*/
union ionic_drv_identity {
struct {
@@ -267,7 +278,9 @@ enum ionic_dev_capability {
* union ionic_dev_identity - device identity information
* @version: Version of device identify
* @type: Identify type (0 for now)
+ * @rsvd: reserved byte(s)
* @nports: Number of ports provisioned
+ * @rsvd2: reserved byte(s)
* @nlifs: Number of LIFs provisioned
* @nintrs: Number of interrupts provisioned
* @ndbpgs_per_lif: Number of doorbell pages per LIF
@@ -284,6 +297,7 @@ enum ionic_dev_capability {
* @hwstamp_mult: Hardware tick to nanosecond multiplier.
* @hwstamp_shift: Hardware tick to nanosecond divisor (power of two).
* @capabilities: Device capabilities
+ * @words: word access to struct contents
*/
union ionic_dev_identity {
struct {
@@ -317,6 +331,7 @@ enum ionic_lif_type {
* @opcode: opcode
* @type: LIF type (enum ionic_lif_type)
* @ver: Version of identify returned by device
+ * @rsvd: reserved byte(s)
*/
struct ionic_lif_identify_cmd {
u8 opcode;
@@ -329,6 +344,7 @@ struct ionic_lif_identify_cmd {
* struct ionic_lif_identify_comp - LIF identify command completion
* @status: Status of the command (enum ionic_status_code)
* @ver: Version of identify returned by device
+ * @rsvd2: reserved byte(s)
*/
struct ionic_lif_identify_comp {
u8 status;
@@ -416,7 +432,7 @@ enum ionic_txq_feature {
};
/**
- * struct ionic_hwstamp_bits - Hardware timestamp decoding bits
+ * enum ionic_hwstamp_bits - Hardware timestamp decoding bits
* @IONIC_HWSTAMP_INVALID: Invalid hardware timestamp value
* @IONIC_HWSTAMP_CQ_NEGOFFSET: Timestamp field negative offset
* from the base cq descriptor.
@@ -429,6 +445,7 @@ enum ionic_hwstamp_bits {
/**
* struct ionic_lif_logical_qtype - Descriptor of logical to HW queue type
* @qtype: Hardware Queue Type
+ * @rsvd: reserved byte(s)
* @qid_count: Number of Queue IDs of the logical type
* @qid_base: Minimum Queue ID of the logical type
*/
@@ -454,12 +471,14 @@ enum ionic_lif_state {
/**
* union ionic_lif_config - LIF configuration
* @state: LIF state (enum ionic_lif_state)
+ * @rsvd: reserved byte(s)
* @name: LIF name
* @mtu: MTU
* @mac: Station MAC address
* @vlan: Default Vlan ID
* @features: Features (enum ionic_eth_hw_features)
* @queue_count: Queue counts per queue-type
+ * @words: word access to struct contents
*/
union ionic_lif_config {
struct {
@@ -481,33 +500,39 @@ union ionic_lif_config {
* @capabilities: LIF capabilities
*
* @eth: Ethernet identify structure
- * @version: Ethernet identify structure version
- * @max_ucast_filters: Number of perfect unicast addresses supported
- * @max_mcast_filters: Number of perfect multicast addresses supported
- * @min_frame_size: Minimum size of frames to be sent
- * @max_frame_size: Maximum size of frames to be sent
- * @hwstamp_tx_modes: Bitmask of BIT_ULL(enum ionic_txstamp_mode)
- * @hwstamp_rx_filters: Bitmask of enum ionic_pkt_class
- * @config: LIF config struct with features, mtu, mac, q counts
+ * @eth.version: Ethernet identify structure version
+ * @eth.rsvd: reserved byte(s)
+ * @eth.max_ucast_filters: Number of perfect unicast addresses supported
+ * @eth.max_mcast_filters: Number of perfect multicast addresses supported
+ * @eth.min_frame_size: Minimum size of frames to be sent
+ * @eth.max_frame_size: Maximum size of frames to be sent
+ * @eth.rsvd2: reserved byte(s)
+ * @eth.hwstamp_tx_modes: Bitmask of BIT_ULL(enum ionic_txstamp_mode)
+ * @eth.hwstamp_rx_filters: Bitmask of enum ionic_pkt_class
+ * @eth.rsvd3: reserved byte(s)
+ * @eth.config: LIF config struct with features, mtu, mac, q counts
*
* @rdma: RDMA identify structure
- * @version: RDMA version of opcodes and queue descriptors
- * @qp_opcodes: Number of RDMA queue pair opcodes supported
- * @admin_opcodes: Number of RDMA admin opcodes supported
- * @npts_per_lif: Page table size per LIF
- * @nmrs_per_lif: Number of memory regions per LIF
- * @nahs_per_lif: Number of address handles per LIF
- * @max_stride: Max work request stride
- * @cl_stride: Cache line stride
- * @pte_stride: Page table entry stride
- * @rrq_stride: Remote RQ work request stride
- * @rsq_stride: Remote SQ work request stride
- * @dcqcn_profiles: Number of DCQCN profiles
- * @aq_qtype: RDMA Admin Qtype
- * @sq_qtype: RDMA Send Qtype
- * @rq_qtype: RDMA Receive Qtype
- * @cq_qtype: RDMA Completion Qtype
- * @eq_qtype: RDMA Event Qtype
+ * @rdma.version: RDMA version of opcodes and queue descriptors
+ * @rdma.qp_opcodes: Number of RDMA queue pair opcodes supported
+ * @rdma.admin_opcodes: Number of RDMA admin opcodes supported
+ * @rdma.rsvd: reserved byte(s)
+ * @rdma.npts_per_lif: Page table size per LIF
+ * @rdma.nmrs_per_lif: Number of memory regions per LIF
+ * @rdma.nahs_per_lif: Number of address handles per LIF
+ * @rdma.max_stride: Max work request stride
+ * @rdma.cl_stride: Cache line stride
+ * @rdma.pte_stride: Page table entry stride
+ * @rdma.rrq_stride: Remote RQ work request stride
+ * @rdma.rsq_stride: Remote SQ work request stride
+ * @rdma.dcqcn_profiles: Number of DCQCN profiles
+ * @rdma.rsvd_dimensions: reserved byte(s)
+ * @rdma.aq_qtype: RDMA Admin Qtype
+ * @rdma.sq_qtype: RDMA Send Qtype
+ * @rdma.rq_qtype: RDMA Receive Qtype
+ * @rdma.cq_qtype: RDMA Completion Qtype
+ * @rdma.eq_qtype: RDMA Event Qtype
+ * @words: word access to struct contents
*/
union ionic_lif_identity {
struct {
@@ -558,7 +583,9 @@ union ionic_lif_identity {
* @opcode: Opcode
* @type: LIF type (enum ionic_lif_type)
* @index: LIF index
+ * @rsvd: reserved byte(s)
* @info_pa: Destination address for LIF info (struct ionic_lif_info)
+ * @rsvd2: reserved byte(s)
*/
struct ionic_lif_init_cmd {
u8 opcode;
@@ -572,7 +599,9 @@ struct ionic_lif_init_cmd {
/**
* struct ionic_lif_init_comp - LIF init command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @hw_index: Hardware index of the initialized LIF
+ * @rsvd2: reserved byte(s)
*/
struct ionic_lif_init_comp {
u8 status;
@@ -584,9 +613,11 @@ struct ionic_lif_init_comp {
/**
* struct ionic_q_identify_cmd - queue identify command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @lif_type: LIF type (enum ionic_lif_type)
* @type: Logical queue type (enum ionic_logical_qtype)
* @ver: Highest queue type version that the driver supports
+ * @rsvd2: reserved byte(s)
*/
struct ionic_q_identify_cmd {
u8 opcode;
@@ -600,8 +631,10 @@ struct ionic_q_identify_cmd {
/**
* struct ionic_q_identify_comp - queue identify command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
* @ver: Queue type version that can be used with FW
+ * @rsvd2: reserved byte(s)
*/
struct ionic_q_identify_comp {
u8 status;
@@ -615,12 +648,14 @@ struct ionic_q_identify_comp {
* union ionic_q_identity - queue identity information
* @version: Queue type version that can be used with FW
* @supported: Bitfield of queue versions, first bit = ver 0
+ * @rsvd: reserved byte(s)
* @features: Queue features (enum ionic_q_feature, etc)
* @desc_sz: Descriptor size
* @comp_sz: Completion descriptor size
* @sg_desc_sz: Scatter/Gather descriptor size
* @max_sg_elems: Maximum number of Scatter/Gather elements
* @sg_desc_stride: Number of Scatter/Gather elements per descriptor
+ * @words: word access to struct contents
*/
union ionic_q_identity {
struct {
@@ -640,8 +675,10 @@ union ionic_q_identity {
/**
* struct ionic_q_init_cmd - Queue init command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @type: Logical queue type
* @ver: Queue type version
+ * @rsvd1: reserved byte(s)
* @lif_index: LIF index
* @index: (LIF, qtype) relative admin queue index
* @intr_index: Interrupt control register index, or Event queue index
@@ -667,6 +704,7 @@ union ionic_q_identity {
* @ring_base: Queue ring base address
* @cq_ring_base: Completion queue ring base address
* @sg_ring_base: Scatter/Gather ring base address
+ * @rsvd2: reserved byte(s)
* @features: Mask of queue features to enable, if not in the flags above.
*/
struct ionic_q_init_cmd {
@@ -698,9 +736,11 @@ struct ionic_q_init_cmd {
/**
* struct ionic_q_init_comp - Queue init command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
* @hw_index: Hardware Queue ID
* @hw_type: Hardware Queue type
+ * @rsvd2: reserved byte(s)
* @color: Color
*/
struct ionic_q_init_comp {
@@ -800,7 +840,7 @@ enum ionic_txq_desc_opcode {
* will set CWR flag in the first segment if
* CWR is set in the template header, and
* clear CWR in remaining segments.
- * @flags:
+ * flags:
* vlan:
* Insert an L2 VLAN header using @vlan_tci
* encap:
@@ -813,13 +853,14 @@ enum ionic_txq_desc_opcode {
* TSO start
* tso_eot:
* TSO end
- * @num_sg_elems: Number of scatter-gather elements in SG
+ * num_sg_elems: Number of scatter-gather elements in SG
* descriptor
- * @addr: First data buffer's DMA address
+ * addr: First data buffer's DMA address
* (Subsequent data buffers are on txq_sg_desc)
* @len: First data buffer's length, in bytes
* @vlan_tci: VLAN tag to insert in the packet (if requested
* by @V-bit). Includes .1p and .1q tags
+ * @hword0: half word padding
* @hdr_len: Length of packet headers, including
* encapsulating outer header, if applicable
* Valid for opcodes IONIC_TXQ_DESC_OPCODE_CALC_CSUM and
@@ -830,10 +871,12 @@ enum ionic_txq_desc_opcode {
* IONIC_TXQ_DESC_OPCODE_TSO, @hdr_len is up to
* inner-most L4 payload, so inclusive of
* inner-most L4 header.
+ * @hword1: half word padding
* @mss: Desired MSS value for TSO; only applicable for
* IONIC_TXQ_DESC_OPCODE_TSO
* @csum_start: Offset from packet to first byte checked in L4 checksum
* @csum_offset: Offset from csum_start to L4 checksum field
+ * @hword2: half word padding
*/
struct ionic_txq_desc {
__le64 cmd;
@@ -901,6 +944,7 @@ static inline void decode_txq_desc_cmd(u64 cmd, u8 *opcode, u8 *flags,
* struct ionic_txq_sg_elem - Transmit scatter-gather (SG) descriptor element
* @addr: DMA address of SG element data buffer
* @len: Length of SG element data buffer, in bytes
+ * @rsvd: reserved byte(s)
*/
struct ionic_txq_sg_elem {
__le64 addr;
@@ -927,7 +971,9 @@ struct ionic_txq_sg_desc_v1 {
/**
* struct ionic_txq_comp - Ethernet transmit queue completion descriptor
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
+ * @rsvd2: reserved byte(s)
* @color: Color bit
*/
struct ionic_txq_comp {
@@ -953,6 +999,7 @@ enum ionic_rxq_desc_opcode {
* receive, including actual bytes received,
* are recorded in Rx completion descriptor.
*
+ * @rsvd: reserved byte(s)
* @len: Data buffer's length, in bytes
* @addr: Data buffer's DMA address
*/
@@ -967,6 +1014,7 @@ struct ionic_rxq_desc {
* struct ionic_rxq_sg_elem - Receive scatter-gather (SG) descriptor element
* @addr: DMA address of SG element data buffer
* @len: Length of SG element data buffer, in bytes
+ * @rsvd: reserved byte(s)
*/
struct ionic_rxq_sg_elem {
__le64 addr;
@@ -1170,6 +1218,7 @@ enum ionic_pkt_class {
* @lif_index: LIF index
* @index: Queue index
* @oper: Operation (enum ionic_q_control_oper)
+ * @rsvd: reserved byte(s)
*/
struct ionic_q_control_cmd {
u8 opcode;
@@ -1182,7 +1231,7 @@ struct ionic_q_control_cmd {
typedef struct ionic_admin_comp ionic_q_control_comp;
-enum q_control_oper {
+enum ionic_q_control_oper {
IONIC_Q_DISABLE = 0,
IONIC_Q_ENABLE = 1,
IONIC_Q_HANG_RESET = 2,
@@ -1216,7 +1265,7 @@ enum ionic_xcvr_state {
IONIC_XCVR_STATE_SPROM_READ_ERR = 4,
};
-/**
+/*
* enum ionic_xcvr_pid - Supported link modes
*/
enum ionic_xcvr_pid {
@@ -1228,7 +1277,10 @@ enum ionic_xcvr_pid {
IONIC_XCVR_PID_SFP_25GBASE_CR_S = 3,
IONIC_XCVR_PID_SFP_25GBASE_CR_L = 4,
IONIC_XCVR_PID_SFP_25GBASE_CR_N = 5,
-
+ IONIC_XCVR_PID_QSFP_50G_CR2_FC = 6,
+ IONIC_XCVR_PID_QSFP_50G_CR2 = 7,
+ IONIC_XCVR_PID_QSFP_200G_CR4 = 8,
+ IONIC_XCVR_PID_QSFP_400G_CR4 = 9,
/* Fiber */
IONIC_XCVR_PID_QSFP_100G_AOC = 50,
IONIC_XCVR_PID_QSFP_100G_ACC = 51,
@@ -1254,6 +1306,15 @@ enum ionic_xcvr_pid {
IONIC_XCVR_PID_SFP_25GBASE_ACC = 71,
IONIC_XCVR_PID_SFP_10GBASE_T = 72,
IONIC_XCVR_PID_SFP_1000BASE_T = 73,
+ IONIC_XCVR_PID_QSFP_200G_AOC = 74,
+ IONIC_XCVR_PID_QSFP_200G_FR4 = 75,
+ IONIC_XCVR_PID_QSFP_200G_DR4 = 76,
+ IONIC_XCVR_PID_QSFP_200G_SR4 = 77,
+ IONIC_XCVR_PID_QSFP_200G_ACC = 78,
+ IONIC_XCVR_PID_QSFP_400G_FR4 = 79,
+ IONIC_XCVR_PID_QSFP_400G_DR4 = 80,
+ IONIC_XCVR_PID_QSFP_400G_SR4 = 81,
+ IONIC_XCVR_PID_QSFP_400G_VR4 = 82,
};
/**
@@ -1351,9 +1412,12 @@ struct ionic_xcvr_status {
* @fec_type: fec type (enum ionic_port_fec_type)
* @pause_type: pause type (enum ionic_port_pause_type)
* @loopback_mode: loopback mode (enum ionic_port_loopback_mode)
+ * @words: word access to struct contents
*/
union ionic_port_config {
struct {
+#define IONIC_SPEED_400G 400000 /* 400G in Mbps */
+#define IONIC_SPEED_200G 200000 /* 200G in Mbps */
#define IONIC_SPEED_100G 100000 /* 100G in Mbps */
#define IONIC_SPEED_50G 50000 /* 50G in Mbps */
#define IONIC_SPEED_40G 40000 /* 40G in Mbps */
@@ -1382,6 +1446,7 @@ union ionic_port_config {
* @speed: link speed (in Mbps)
* @link_down_count: number of times link went from up to down
* @fec_type: fec type (enum ionic_port_fec_type)
+ * @rsvd: reserved byte(s)
* @xcvr: transceiver status
*/
struct ionic_port_status {
@@ -1399,6 +1464,7 @@ struct ionic_port_status {
* @opcode: opcode
* @index: port index
* @ver: Highest version of identify supported by driver
+ * @rsvd: reserved byte(s)
*/
struct ionic_port_identify_cmd {
u8 opcode;
@@ -1411,6 +1477,7 @@ struct ionic_port_identify_cmd {
* struct ionic_port_identify_comp - Port identify command completion
* @status: Status of the command (enum ionic_status_code)
* @ver: Version of identify returned by device
+ * @rsvd: reserved byte(s)
*/
struct ionic_port_identify_comp {
u8 status;
@@ -1422,7 +1489,9 @@ struct ionic_port_identify_comp {
* struct ionic_port_init_cmd - Port initialization command
* @opcode: opcode
* @index: port index
+ * @rsvd: reserved byte(s)
* @info_pa: destination address for port info (struct ionic_port_info)
+ * @rsvd2: reserved byte(s)
*/
struct ionic_port_init_cmd {
u8 opcode;
@@ -1435,6 +1504,7 @@ struct ionic_port_init_cmd {
/**
* struct ionic_port_init_comp - Port initialization command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
*/
struct ionic_port_init_comp {
u8 status;
@@ -1445,6 +1515,7 @@ struct ionic_port_init_comp {
* struct ionic_port_reset_cmd - Port reset command
* @opcode: opcode
* @index: port index
+ * @rsvd: reserved byte(s)
*/
struct ionic_port_reset_cmd {
u8 opcode;
@@ -1455,6 +1526,7 @@ struct ionic_port_reset_cmd {
/**
* struct ionic_port_reset_comp - Port reset command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
*/
struct ionic_port_reset_comp {
u8 status;
@@ -1510,6 +1582,7 @@ enum ionic_port_attr {
* @opcode: Opcode
* @index: Port index
* @attr: Attribute type (enum ionic_port_attr)
+ * @rsvd: reserved byte(s)
* @state: Port state
* @speed: Port speed
* @mtu: Port MTU
@@ -1518,6 +1591,7 @@ enum ionic_port_attr {
* @pause_type: Port pause type setting
* @loopback_mode: Port loopback mode
* @stats_ctl: Port stats setting
+ * @rsvd2: reserved byte(s)
*/
struct ionic_port_setattr_cmd {
u8 opcode;
@@ -1540,6 +1614,7 @@ struct ionic_port_setattr_cmd {
/**
* struct ionic_port_setattr_comp - Port set attr command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @color: Color bit
*/
struct ionic_port_setattr_comp {
@@ -1553,6 +1628,7 @@ struct ionic_port_setattr_comp {
* @opcode: Opcode
* @index: port index
* @attr: Attribute type (enum ionic_port_attr)
+ * @rsvd: reserved byte(s)
*/
struct ionic_port_getattr_cmd {
u8 opcode;
@@ -1564,6 +1640,7 @@ struct ionic_port_getattr_cmd {
/**
* struct ionic_port_getattr_comp - Port get attr command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @state: Port state
* @speed: Port speed
* @mtu: Port MTU
@@ -1571,6 +1648,7 @@ struct ionic_port_getattr_cmd {
* @fec_type: Port FEC type setting
* @pause_type: Port pause type setting
* @loopback_mode: Port loopback mode
+ * @rsvd2: reserved byte(s)
* @color: Color bit
*/
struct ionic_port_getattr_comp {
@@ -1593,9 +1671,11 @@ struct ionic_port_getattr_comp {
* struct ionic_lif_status - LIF status register
* @eid: most recent NotifyQ event id
* @port_num: port the LIF is connected to
+ * @rsvd: reserved byte(s)
* @link_status: port status (enum ionic_port_oper_status)
* @link_speed: speed of link in Mbps
* @link_down_count: number of times link went from up to down
+ * @rsvd2: reserved byte(s)
*/
struct ionic_lif_status {
__le64 eid;
@@ -1610,7 +1690,9 @@ struct ionic_lif_status {
/**
* struct ionic_lif_reset_cmd - LIF reset command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @index: LIF index
+ * @rsvd2: reserved byte(s)
*/
struct ionic_lif_reset_cmd {
u8 opcode;
@@ -1643,9 +1725,11 @@ enum ionic_dev_attr {
* struct ionic_dev_setattr_cmd - Set Device attributes on the NIC
* @opcode: Opcode
* @attr: Attribute type (enum ionic_dev_attr)
+ * @rsvd: reserved byte(s)
* @state: Device state (enum ionic_dev_state)
* @name: The bus info, e.g. PCI slot-device-function, 0 terminated
* @features: Device features
+ * @rsvd2: reserved byte(s)
*/
struct ionic_dev_setattr_cmd {
u8 opcode;
@@ -1662,7 +1746,9 @@ struct ionic_dev_setattr_cmd {
/**
* struct ionic_dev_setattr_comp - Device set attr command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @features: Device features
+ * @rsvd2: reserved byte(s)
* @color: Color bit
*/
struct ionic_dev_setattr_comp {
@@ -1679,6 +1765,7 @@ struct ionic_dev_setattr_comp {
* struct ionic_dev_getattr_cmd - Get Device attributes from the NIC
* @opcode: opcode
* @attr: Attribute type (enum ionic_dev_attr)
+ * @rsvd: reserved byte(s)
*/
struct ionic_dev_getattr_cmd {
u8 opcode;
@@ -1687,9 +1774,11 @@ struct ionic_dev_getattr_cmd {
};
/**
- * struct ionic_dev_setattr_comp - Device set attr command completion
+ * struct ionic_dev_getattr_comp - Device set attr command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @features: Device features
+ * @rsvd2: reserved byte(s)
* @color: Color bit
*/
struct ionic_dev_getattr_comp {
@@ -1702,7 +1791,7 @@ struct ionic_dev_getattr_comp {
u8 color;
};
-/**
+/*
* RSS parameters
*/
#define IONIC_RSS_HASH_KEY_SIZE 40
@@ -1726,6 +1815,7 @@ enum ionic_rss_hash_types {
* @IONIC_LIF_ATTR_RSS: LIF RSS attribute
* @IONIC_LIF_ATTR_STATS_CTRL: LIF statistics control attribute
* @IONIC_LIF_ATTR_TXSTAMP: LIF TX timestamping mode
+ * @IONIC_LIF_ATTR_MAX: maximum attribute value
*/
enum ionic_lif_attr {
IONIC_LIF_ATTR_STATE = 0,
@@ -1736,6 +1826,7 @@ enum ionic_lif_attr {
IONIC_LIF_ATTR_RSS = 5,
IONIC_LIF_ATTR_STATS_CTRL = 6,
IONIC_LIF_ATTR_TXSTAMP = 7,
+ IONIC_LIF_ATTR_MAX = 255,
};
/**
@@ -1749,11 +1840,13 @@ enum ionic_lif_attr {
* @mac: Station mac
* @features: Features (enum ionic_eth_hw_features)
* @rss: RSS properties
- * @types: The hash types to enable (see rss_hash_types)
- * @key: The hash secret key
- * @addr: Address for the indirection table shared memory
+ * @rss.types: The hash types to enable (see rss_hash_types)
+ * @rss.key: The hash secret key
+ * @rss.rsvd: reserved byte(s)
+ * @rss.addr: Address for the indirection table shared memory
* @stats_ctl: stats control commands (enum ionic_stats_ctl_cmd)
- * @txstamp: TX Timestamping Mode (enum ionic_txstamp_mode)
+ * @txstamp_mode: TX Timestamping Mode (enum ionic_txstamp_mode)
+ * @rsvd: reserved byte(s)
*/
struct ionic_lif_setattr_cmd {
u8 opcode;
@@ -1772,7 +1865,7 @@ struct ionic_lif_setattr_cmd {
__le64 addr;
} rss;
u8 stats_ctl;
- __le16 txstamp_mode;
+ __le16 txstamp_mode;
u8 rsvd[60];
} __packed;
};
@@ -1780,8 +1873,10 @@ struct ionic_lif_setattr_cmd {
/**
* struct ionic_lif_setattr_comp - LIF set attr command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
* @features: features (enum ionic_eth_hw_features)
+ * @rsvd2: reserved byte(s)
* @color: Color bit
*/
struct ionic_lif_setattr_comp {
@@ -1800,6 +1895,7 @@ struct ionic_lif_setattr_comp {
* @opcode: Opcode
* @attr: Attribute type (enum ionic_lif_attr)
* @index: LIF index
+ * @rsvd: reserved byte(s)
*/
struct ionic_lif_getattr_cmd {
u8 opcode;
@@ -1811,13 +1907,14 @@ struct ionic_lif_getattr_cmd {
/**
* struct ionic_lif_getattr_comp - LIF get attr command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
* @state: LIF state (enum ionic_lif_state)
- * @name: The netdev name string, 0 terminated
* @mtu: Mtu
* @mac: Station mac
* @features: Features (enum ionic_eth_hw_features)
- * @txstamp: TX Timestamping Mode (enum ionic_txstamp_mode)
+ * @txstamp_mode: TX Timestamping Mode (enum ionic_txstamp_mode)
+ * @rsvd2: reserved byte(s)
* @color: Color bit
*/
struct ionic_lif_getattr_comp {
@@ -1838,12 +1935,15 @@ struct ionic_lif_getattr_comp {
/**
* struct ionic_lif_setphc_cmd - Set LIF PTP Hardware Clock
* @opcode: Opcode
+ * @rsvd1: reserved byte(s)
* @lif_index: LIF index
+ * @rsvd2: reserved byte(s)
* @tick: Hardware stamp tick of an instant in time.
* @nsec: Nanosecond stamp of the same instant.
* @frac: Fractional nanoseconds at the same instant.
* @mult: Cycle to nanosecond multiplier.
* @shift: Cycle to nanosecond divisor (power of two).
+ * @rsvd3: reserved byte(s)
*/
struct ionic_lif_setphc_cmd {
u8 opcode;
@@ -1870,6 +1970,7 @@ enum ionic_rx_mode {
/**
* struct ionic_rx_mode_set_cmd - Set LIF's Rx mode command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @lif_index: LIF index
* @rx_mode: Rx mode flags:
* IONIC_RX_MODE_F_UNICAST: Accept known unicast packets
@@ -1878,6 +1979,7 @@ enum ionic_rx_mode {
* IONIC_RX_MODE_F_PROMISC: Accept any packets
* IONIC_RX_MODE_F_ALLMULTI: Accept any multicast packets
* IONIC_RX_MODE_F_RDMA_SNIFFER: Sniff RDMA packets
+ * @rsvd2: reserved byte(s)
*/
struct ionic_rx_mode_set_cmd {
u8 opcode;
@@ -1904,13 +2006,14 @@ enum ionic_rx_filter_match_type {
* @qid: Queue ID
* @match: Rx filter match type (see IONIC_RX_FILTER_MATCH_xxx)
* @vlan: VLAN filter
- * @vlan: VLAN ID
+ * @vlan.vlan: VLAN ID
* @mac: MAC filter
- * @addr: MAC address (network-byte order)
+ * @mac.addr: MAC address (network-byte order)
* @mac_vlan: MACVLAN filter
- * @vlan: VLAN ID
- * @addr: MAC address (network-byte order)
+ * @mac_vlan.vlan: VLAN ID
+ * @mac_vlan.addr: MAC address (network-byte order)
* @pkt_class: Packet classification filter
+ * @rsvd: reserved byte(s)
*/
struct ionic_rx_filter_add_cmd {
u8 opcode;
@@ -1937,8 +2040,10 @@ struct ionic_rx_filter_add_cmd {
/**
* struct ionic_rx_filter_add_comp - Add LIF Rx filter command completion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
* @filter_id: Filter ID
+ * @rsvd2: reserved byte(s)
* @color: Color bit
*/
struct ionic_rx_filter_add_comp {
@@ -1953,8 +2058,10 @@ struct ionic_rx_filter_add_comp {
/**
* struct ionic_rx_filter_del_cmd - Delete LIF Rx filter command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @lif_index: LIF index
* @filter_id: Filter ID
+ * @rsvd2: reserved byte(s)
*/
struct ionic_rx_filter_del_cmd {
u8 opcode;
@@ -2000,6 +2107,7 @@ enum ionic_vf_link_status {
* @trust: enable VF trust
* @linkstate: set link up or down
* @stats_pa: set DMA address for VF stats
+ * @pad: reserved byte(s)
*/
struct ionic_vf_setattr_cmd {
u8 opcode;
@@ -2031,6 +2139,7 @@ struct ionic_vf_setattr_comp {
* @opcode: Opcode
* @attr: Attribute type (enum ionic_vf_attr)
* @vf_index: VF index
+ * @rsvd: reserved byte(s)
*/
struct ionic_vf_getattr_cmd {
u8 opcode;
@@ -2064,8 +2173,8 @@ enum ionic_vf_ctrl_opcode {
/**
* struct ionic_vf_ctrl_cmd - VF control command
* @opcode: Opcode for the command
- * @vf_index: VF Index. It is unused if op START_ALL is used.
* @ctrl_opcode: VF control operation type
+ * @vf_index: VF Index. It is unused if op START_ALL is used.
*/
struct ionic_vf_ctrl_cmd {
u8 opcode;
@@ -2089,7 +2198,7 @@ struct ionic_vf_ctrl_comp {
* struct ionic_qos_identify_cmd - QoS identify command
* @opcode: opcode
* @ver: Highest version of identify supported by driver
- *
+ * @rsvd: reserved byte(s)
*/
struct ionic_qos_identify_cmd {
u8 opcode;
@@ -2101,6 +2210,7 @@ struct ionic_qos_identify_cmd {
* struct ionic_qos_identify_comp - QoS identify command completion
* @status: Status of the command (enum ionic_status_code)
* @ver: Version of identify returned by device
+ * @rsvd: reserved byte(s)
*/
struct ionic_qos_identify_comp {
u8 status;
@@ -2118,7 +2228,7 @@ struct ionic_qos_identify_comp {
#define IONIC_QOS_ALL_PCP 0xFF
#define IONIC_DSCP_BLOCK_SIZE 8
-/**
+/*
* enum ionic_qos_class
*/
enum ionic_qos_class {
@@ -2174,6 +2284,7 @@ enum ionic_qos_sched_type {
* @dot1q_pcp: Dot1q pcp value
* @ndscp: Number of valid dscp values in the ip_dscp field
* @ip_dscp: IP dscp values
+ * @words: word access to struct contents
*/
union ionic_qos_config {
struct {
@@ -2219,8 +2330,9 @@ union ionic_qos_config {
* union ionic_qos_identity - QoS identity structure
* @version: Version of the identify structure
* @type: QoS system type
- * @nclasses: Number of usable QoS classes
+ * @rsvd: reserved byte(s)
* @config: Current configuration of classes
+ * @words: word access to struct contents
*/
union ionic_qos_identity {
struct {
@@ -2236,7 +2348,9 @@ union ionic_qos_identity {
* struct ionic_qos_init_cmd - QoS config init command
* @opcode: Opcode
* @group: QoS class id
+ * @rsvd: reserved byte(s)
* @info_pa: destination address for qos info
+ * @rsvd1: reserved byte(s)
*/
struct ionic_qos_init_cmd {
u8 opcode;
@@ -2252,6 +2366,7 @@ typedef struct ionic_admin_comp ionic_qos_init_comp;
* struct ionic_qos_reset_cmd - QoS config reset command
* @opcode: Opcode
* @group: QoS class id
+ * @rsvd: reserved byte(s)
*/
struct ionic_qos_reset_cmd {
u8 opcode;
@@ -2260,8 +2375,10 @@ struct ionic_qos_reset_cmd {
};
/**
- * struct ionic_qos_clear_port_stats_cmd - Qos config reset command
+ * struct ionic_qos_clear_stats_cmd - Qos config reset command
* @opcode: Opcode
+ * @group_bitmap: bitmap of groups to be cleared
+ * @rsvd: reserved byte(s)
*/
struct ionic_qos_clear_stats_cmd {
u8 opcode;
@@ -2274,6 +2391,7 @@ typedef struct ionic_admin_comp ionic_qos_reset_comp;
/**
* struct ionic_fw_download_cmd - Firmware download command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @addr: dma address of the firmware buffer
* @offset: offset of the firmware buffer within the full image
* @length: number of valid bytes in the firmware buffer
@@ -2297,6 +2415,7 @@ typedef struct ionic_admin_comp ionic_fw_download_comp;
* @IONIC_FW_INSTALL_STATUS: Firmware installation status
* @IONIC_FW_ACTIVATE_ASYNC: Activate firmware asynchronously
* @IONIC_FW_ACTIVATE_STATUS: Firmware activate status
+ * @IONIC_FW_UPDATE_CLEANUP: Clean up after an interrupted fw update
*/
enum ionic_fw_control_oper {
IONIC_FW_RESET = 0,
@@ -2312,8 +2431,10 @@ enum ionic_fw_control_oper {
/**
* struct ionic_fw_control_cmd - Firmware control command
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @oper: firmware control operation (enum ionic_fw_control_oper)
* @slot: slot to activate
+ * @rsvd1: reserved byte(s)
*/
struct ionic_fw_control_cmd {
u8 opcode;
@@ -2326,8 +2447,10 @@ struct ionic_fw_control_cmd {
/**
* struct ionic_fw_control_comp - Firmware control copletion
* @status: Status of the command (enum ionic_status_code)
+ * @rsvd: reserved byte(s)
* @comp_index: Index in the descriptor ring for which this is the completion
* @slot: Slot where the firmware was installed
+ * @rsvd1: reserved byte(s)
* @color: Color bit
*/
struct ionic_fw_control_comp {
@@ -2346,7 +2469,9 @@ struct ionic_fw_control_comp {
/**
* struct ionic_rdma_reset_cmd - Reset RDMA LIF cmd
* @opcode: opcode
+ * @rsvd: reserved byte(s)
* @lif_index: LIF index
+ * @rsvd2: reserved byte(s)
*
* There is no RDMA specific dev command completion struct. Completion uses
* the common struct ionic_admin_comp. Only the status is indicated.
@@ -2362,6 +2487,7 @@ struct ionic_rdma_reset_cmd {
/**
* struct ionic_rdma_queue_cmd - Create RDMA Queue command
* @opcode: opcode, 52, 53
+ * @rsvd: reserved byte(s)
* @lif_index: LIF index
* @qid_ver: (qid | (RDMA version << 24))
* @cid: intr, eq_id, or cq_id
@@ -2369,6 +2495,7 @@ struct ionic_rdma_reset_cmd {
* @depth_log2: log base two of queue depth
* @stride_log2: log base two of queue stride
* @dma_addr: address of the queue memory
+ * @rsvd2: reserved byte(s)
*
* The same command struct is used to create an RDMA event queue, completion
* queue, or RDMA admin queue. The cid is an interrupt number for an event
@@ -2425,6 +2552,7 @@ struct ionic_notifyq_event {
* @ecode: event code = IONIC_EVENT_LINK_CHANGE
* @link_status: link up/down, with error bits (enum ionic_port_status)
* @link_speed: speed of the network link
+ * @rsvd: reserved byte(s)
*
* Sent when the network link state changes between UP and DOWN
*/
@@ -2442,6 +2570,7 @@ struct ionic_link_change_event {
* @ecode: event code = IONIC_EVENT_RESET
* @reset_code: reset type
* @state: 0=pending, 1=complete, 2=error
+ * @rsvd: reserved byte(s)
*
* Sent when the NIC or some subsystem is going to be or
* has been reset.
@@ -2458,6 +2587,7 @@ struct ionic_reset_event {
* struct ionic_heartbeat_event - Sent periodically by NIC to indicate health
* @eid: event number
* @ecode: event code = IONIC_EVENT_HEARTBEAT
+ * @rsvd: reserved byte(s)
*/
struct ionic_heartbeat_event {
__le64 eid;
@@ -2481,6 +2611,7 @@ struct ionic_log_event {
* struct ionic_xcvr_event - Transceiver change event
* @eid: event number
* @ecode: event code = IONIC_EVENT_XCVR
+ * @rsvd: reserved byte(s)
*/
struct ionic_xcvr_event {
__le64 eid;
@@ -2488,7 +2619,7 @@ struct ionic_xcvr_event {
u8 rsvd[54];
};
-/**
+/*
* struct ionic_port_stats - Port statistics structure
*/
struct ionic_port_stats {
@@ -2646,8 +2777,7 @@ enum ionic_oflow_drop_stats {
IONIC_OFLOW_DROP_MAX,
};
-/**
- * struct port_pb_stats - packet buffers system stats
+/* struct ionic_port_pb_stats - packet buffers system stats
* uses ionic_pb_buffer_drop_stats for drop_counts[]
*/
struct ionic_port_pb_stats {
@@ -2681,7 +2811,9 @@ struct ionic_port_pb_stats {
* @pause_type: supported pause types
* @loopback_mode: supported loopback mode
* @speeds: supported speeds
+ * @rsvd2: reserved byte(s)
* @config: current port configuration
+ * @words: word access to struct contents
*/
union ionic_port_identity {
struct {
@@ -2707,7 +2839,12 @@ union ionic_port_identity {
* @status: Port status data
* @stats: Port statistics data
* @mgmt_stats: Port management statistics data
- * @port_pb_drop_stats: uplink pb drop stats
+ * @sprom_epage: Extended Transceiver sprom
+ * @sprom_page1: Extended Transceiver sprom, page 1
+ * @sprom_page2: Extended Transceiver sprom, page 2
+ * @sprom_page17: Extended Transceiver sprom, page 17
+ * @rsvd: reserved byte(s)
+ * @pb_stats: uplink pb drop stats
*/
struct ionic_port_info {
union ionic_port_config config;
@@ -2716,12 +2853,21 @@ struct ionic_port_info {
struct ionic_port_stats stats;
struct ionic_mgmt_port_stats mgmt_stats;
};
- /* room for pb_stats to start at 2k offset */
- u8 rsvd[760];
+ union {
+ u8 sprom_epage[384];
+ struct {
+ u8 sprom_page1[128];
+ u8 sprom_page2[128];
+ u8 sprom_page17[128];
+ };
+ };
+ u8 rsvd[376];
+
+ /* pb_stats must start at 2k offset */
struct ionic_port_pb_stats pb_stats;
};
-/**
+/*
* struct ionic_lif_stats - LIF statistics structure
*/
struct ionic_lif_stats {
@@ -2983,8 +3129,10 @@ struct ionic_hwstamp_regs {
* bit 4-7 - 4 bit generation number, changes on fw restart
* @fw_heartbeat: Firmware heartbeat counter
* @serial_num: Serial number
+ * @rsvd_pad1024: reserved byte(s)
* @fw_version: Firmware version
- * @hwstamp_regs: Hardware current timestamp registers
+ * @hwstamp: Hardware current timestamp registers
+ * @words: word access to struct contents
*/
union ionic_dev_info_regs {
#define IONIC_DEVINFO_FWVERS_BUFLEN 32
@@ -3014,7 +3162,9 @@ union ionic_dev_info_regs {
* @done: Done indicator, bit 0 == 1 when command is complete
* @cmd: Opcode-specific command bytes
* @comp: Opcode-specific response bytes
+ * @rsvd: reserved byte(s)
* @data: Opcode-specific side-data
+ * @words: word access to struct contents
*/
union ionic_dev_cmd_regs {
struct {
@@ -3032,6 +3182,7 @@ union ionic_dev_cmd_regs {
* union ionic_dev_regs - Device register format for bar 0 page 0
* @info: Device info registers
* @devcmd: Device command registers
+ * @words: word access to struct contents
*/
union ionic_dev_regs {
struct {
@@ -3085,7 +3236,11 @@ union ionic_adminq_comp {
#define IONIC_BAR0_INTR_CTRL_OFFSET 0x2000
#define IONIC_DEV_CMD_DONE 0x00000001
-#define IONIC_ASIC_TYPE_CAPRI 0
+#define IONIC_ASIC_TYPE_NONE 0
+#define IONIC_ASIC_TYPE_CAPRI 1
+#define IONIC_ASIC_TYPE_ELBA 2
+#define IONIC_ASIC_TYPE_GIGLIO 3
+#define IONIC_ASIC_TYPE_SALINA 4
/**
* struct ionic_doorbell - Doorbell register layout
@@ -3098,6 +3253,7 @@ union ionic_adminq_comp {
* interrupts when armed.
* @qid_lo: Queue destination for the producer index and flags (low bits)
* @qid_hi: Queue destination for the producer index and flags (high bits)
+ * @rsvd2: reserved byte(s)
*/
struct ionic_doorbell {
__le16 p_index;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index 7f0c6cdc375e..7707a9e53c43 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -13,6 +13,7 @@
#include <linux/cpumask.h>
#include <linux/crash_dump.h>
#include <linux/vmalloc.h>
+#include <net/page_pool/helpers.h>
#include "ionic.h"
#include "ionic_bus.h"
@@ -46,8 +47,9 @@ static int ionic_start_queues(struct ionic_lif *lif);
static void ionic_stop_queues(struct ionic_lif *lif);
static void ionic_lif_queue_identify(struct ionic_lif *lif);
-static int ionic_xdp_queues_config(struct ionic_lif *lif);
-static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q);
+static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif);
+static void ionic_unregister_rxq_info(struct ionic_queue *q);
+static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id);
static void ionic_dim_work(struct work_struct *work)
{
@@ -126,13 +128,13 @@ static void ionic_lif_deferred_work(struct work_struct *work)
} while (true);
}
-void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
+void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
struct ionic_deferred_work *work)
{
- spin_lock_bh(&def->lock);
- list_add_tail(&work->list, &def->list);
- spin_unlock_bh(&def->lock);
- schedule_work(&def->work);
+ spin_lock_bh(&lif->deferred.lock);
+ list_add_tail(&work->list, &lif->deferred.list);
+ spin_unlock_bh(&lif->deferred.lock);
+ queue_work(lif->ionic->wq, &lif->deferred.work);
}
static void ionic_link_status_check(struct ionic_lif *lif)
@@ -207,19 +209,12 @@ void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
}
work->type = IONIC_DW_TYPE_LINK_STATUS;
- ionic_lif_deferred_enqueue(&lif->deferred, work);
+ ionic_lif_deferred_enqueue(lif, work);
} else {
ionic_link_status_check(lif);
}
}
-static void ionic_napi_deadline(struct timer_list *timer)
-{
- struct ionic_qcq *qcq = container_of(timer, struct ionic_qcq, napi_deadline);
-
- napi_schedule(&qcq->napi);
-}
-
static irqreturn_t ionic_isr(int irq, void *data)
{
struct napi_struct *napi = data;
@@ -237,12 +232,12 @@ static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
const char *name;
if (lif->registered)
- name = lif->netdev->name;
+ name = netdev_name(lif->netdev);
else
name = dev_name(dev);
snprintf(intr->name, sizeof(intr->name),
- "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
+ "%.5s-%.16s-%.8s", IONIC_DRV_NAME, name, q->name);
return devm_request_irq(dev, intr->vector, ionic_isr,
0, intr->name, &qcq->napi);
@@ -272,6 +267,18 @@ static void ionic_intr_free(struct ionic *ionic, int index)
clear_bit(index, ionic->intrs);
}
+static void ionic_irq_aff_notify(struct irq_affinity_notify *notify,
+ const cpumask_t *mask)
+{
+ struct ionic_intr_info *intr = container_of(notify, struct ionic_intr_info, aff_notify);
+
+ cpumask_copy(*intr->affinity_mask, mask);
+}
+
+static void ionic_irq_aff_release(struct kref __always_unused *ref)
+{
+}
+
static int ionic_qcq_enable(struct ionic_qcq *qcq)
{
struct ionic_queue *q = &qcq->q;
@@ -304,12 +311,12 @@ static int ionic_qcq_enable(struct ionic_qcq *qcq)
if (ret)
return ret;
- if (qcq->napi.poll)
- napi_enable(&qcq->napi);
-
if (qcq->flags & IONIC_QCQ_F_INTR) {
+ napi_enable(&qcq->napi);
+ irq_set_affinity_notifier(qcq->intr.vector,
+ &qcq->intr.aff_notify);
irq_set_affinity_hint(qcq->intr.vector,
- &qcq->intr.affinity_mask);
+ *qcq->intr.affinity_mask);
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
IONIC_INTR_MASK_CLEAR);
}
@@ -339,13 +346,15 @@ static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int f
if (qcq->flags & IONIC_QCQ_F_INTR) {
struct ionic_dev *idev = &lif->ionic->idev;
+ if (lif->doorbell_wa)
+ cancel_work_sync(&qcq->doorbell_napi_work);
cancel_work_sync(&qcq->dim.work);
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
IONIC_INTR_MASK_SET);
synchronize_irq(qcq->intr.vector);
+ irq_set_affinity_notifier(qcq->intr.vector, NULL);
irq_set_affinity_hint(qcq->intr.vector, NULL);
napi_disable(&qcq->napi);
- del_timer_sync(&qcq->napi_deadline);
}
/* If there was a previous fw communcation error, don't bother with
@@ -373,6 +382,7 @@ static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
if (!(qcq->flags & IONIC_QCQ_F_INITED))
return;
+ ionic_unregister_rxq_info(&qcq->q);
if (qcq->flags & IONIC_QCQ_F_INTR) {
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
IONIC_INTR_MASK_SET);
@@ -430,9 +440,10 @@ static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
qcq->sg_base_pa = 0;
}
- ionic_xdp_unregister_rxq_info(&qcq->q);
- ionic_qcq_intr_free(lif, qcq);
+ page_pool_destroy(qcq->q.page_pool);
+ qcq->q.page_pool = NULL;
+ ionic_qcq_intr_free(lif, qcq);
vfree(qcq->q.info);
qcq->q.info = NULL;
}
@@ -480,11 +491,11 @@ static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
{
n_qcq->intr.vector = src_qcq->intr.vector;
n_qcq->intr.index = src_qcq->intr.index;
- n_qcq->napi_qcq = src_qcq->napi_qcq;
}
static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
{
+ cpumask_var_t *affinity_mask;
int err;
if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
@@ -516,10 +527,19 @@ static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qc
}
/* try to get the irq on the local numa node first */
- qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
- dev_to_node(lif->ionic->dev));
- if (qcq->intr.cpu != -1)
- cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
+ affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];
+ if (cpumask_empty(*affinity_mask)) {
+ unsigned int cpu;
+
+ cpu = cpumask_local_spread(qcq->intr.index,
+ dev_to_node(lif->ionic->dev));
+ if (cpu != -1)
+ cpumask_set_cpu(cpu, *affinity_mask);
+ }
+
+ qcq->intr.affinity_mask = affinity_mask;
+ qcq->intr.aff_notify.notify = ionic_irq_aff_notify;
+ qcq->intr.aff_notify.release = ionic_irq_aff_release;
netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
return 0;
@@ -537,7 +557,8 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
unsigned int cq_desc_size,
unsigned int sg_desc_size,
unsigned int desc_info_size,
- unsigned int pid, struct ionic_qcq **qcq)
+ unsigned int pid, struct bpf_prog *xdp_prog,
+ struct ionic_qcq **qcq)
{
struct ionic_dev *idev = &lif->ionic->idev;
struct device *dev = lif->ionic->dev;
@@ -563,6 +584,31 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
goto err_out_free_qcq;
}
+ if (type == IONIC_QTYPE_RXQ) {
+ struct page_pool_params pp_params = {
+ .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+ .order = 0,
+ .pool_size = num_descs,
+ .nid = NUMA_NO_NODE,
+ .dev = lif->ionic->dev,
+ .napi = &new->napi,
+ .dma_dir = DMA_FROM_DEVICE,
+ .max_len = PAGE_SIZE,
+ .netdev = lif->netdev,
+ };
+
+ if (xdp_prog)
+ pp_params.dma_dir = DMA_BIDIRECTIONAL;
+
+ new->q.page_pool = page_pool_create(&pp_params);
+ if (IS_ERR(new->q.page_pool)) {
+ netdev_err(lif->netdev, "Cannot create page_pool\n");
+ err = PTR_ERR(new->q.page_pool);
+ new->q.page_pool = NULL;
+ goto err_out_free_q_info;
+ }
+ }
+
new->q.type = type;
new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
@@ -570,12 +616,12 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
desc_size, sg_desc_size, pid);
if (err) {
netdev_err(lif->netdev, "Cannot initialize queue\n");
- goto err_out_free_q_info;
+ goto err_out_free_page_pool;
}
err = ionic_alloc_qcq_interrupt(lif, new);
if (err)
- goto err_out_free_q_info;
+ goto err_out_free_page_pool;
err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
if (err) {
@@ -676,6 +722,8 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
INIT_WORK(&new->dim.work, ionic_dim_work);
new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
+ if (lif->doorbell_wa)
+ INIT_WORK(&new->doorbell_napi_work, ionic_doorbell_napi_work);
*qcq = new;
@@ -694,6 +742,8 @@ err_out_free_irq:
devm_free_irq(dev, new->intr.vector, &new->napi);
ionic_intr_free(lif->ionic, new->intr.index);
}
+err_out_free_page_pool:
+ page_pool_destroy(new->q.page_pool);
err_out_free_q_info:
vfree(new->q.info);
err_out_free_qcq:
@@ -716,7 +766,7 @@ static int ionic_qcqs_alloc(struct ionic_lif *lif)
sizeof(struct ionic_admin_comp),
0,
sizeof(struct ionic_admin_desc_info),
- lif->kern_pid, &lif->adminqcq);
+ lif->kern_pid, NULL, &lif->adminqcq);
if (err)
return err;
ionic_debugfs_add_qcq(lif, lif->adminqcq);
@@ -729,7 +779,7 @@ static int ionic_qcqs_alloc(struct ionic_lif *lif)
sizeof(union ionic_notifyq_comp),
0,
sizeof(struct ionic_admin_desc_info),
- lif->kern_pid, &lif->notifyqcq);
+ lif->kern_pid, NULL, &lif->notifyqcq);
if (err)
goto err_out;
ionic_debugfs_add_qcq(lif, lif->notifyqcq);
@@ -834,11 +884,8 @@ static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
q->dbell_jiffies = jiffies;
- if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
+ if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
- qcq->napi_qcq = qcq;
- timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
- }
qcq->flags |= IONIC_QCQ_F_INITED;
@@ -910,9 +957,11 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
else
netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
-
- qcq->napi_qcq = qcq;
- timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
+ err = ionic_register_rxq_info(q, qcq->napi.napi_id);
+ if (err) {
+ netif_napi_del(&qcq->napi);
+ return err;
+ }
qcq->flags |= IONIC_QCQ_F_INITED;
@@ -948,7 +997,7 @@ int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
num_desc, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_tx_desc_info),
- lif->kern_pid, &txq);
+ lif->kern_pid, NULL, &txq);
if (err)
goto err_qcq_alloc;
@@ -1008,7 +1057,7 @@ int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
num_desc, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_rx_desc_info),
- lif->kern_pid, &rxq);
+ lif->kern_pid, NULL, &rxq);
if (err)
goto err_qcq_alloc;
@@ -1025,7 +1074,7 @@ int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
goto err_qcq_init;
if (test_bit(IONIC_LIF_F_UP, lif->state)) {
- ionic_rx_fill(&rxq->q);
+ ionic_rx_fill(&rxq->q, NULL);
err = ionic_qcq_enable(rxq);
if (err)
goto err_qcq_enable;
@@ -1168,7 +1217,6 @@ static int ionic_adminq_napi(struct napi_struct *napi, int budget)
struct ionic_dev *idev = &lif->ionic->idev;
unsigned long irqflags;
unsigned int flags = 0;
- bool resched = false;
int rx_work = 0;
int tx_work = 0;
int n_work = 0;
@@ -1184,6 +1232,7 @@ static int ionic_adminq_napi(struct napi_struct *napi, int budget)
if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
ionic_adminq_service, NULL, NULL);
+
spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
if (lif->hwstamp_rxq)
@@ -1191,7 +1240,7 @@ static int ionic_adminq_napi(struct napi_struct *napi, int budget)
ionic_rx_service, NULL, NULL);
if (lif->hwstamp_txq)
- tx_work = ionic_tx_cq_service(&lif->hwstamp_txq->cq, budget);
+ tx_work = ionic_tx_cq_service(&lif->hwstamp_txq->cq, budget, !!budget);
work_done = max(max(n_work, a_work), max(rx_work, tx_work));
if (work_done < budget && napi_complete_done(napi, work_done)) {
@@ -1205,15 +1254,14 @@ static int ionic_adminq_napi(struct napi_struct *napi, int budget)
ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
}
- if (!a_work && ionic_adminq_poke_doorbell(&lif->adminqcq->q))
- resched = true;
- if (lif->hwstamp_rxq && !rx_work && ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q))
- resched = true;
- if (lif->hwstamp_txq && !tx_work && ionic_txq_poke_doorbell(&lif->hwstamp_txq->q))
- resched = true;
- if (resched)
- mod_timer(&lif->adminqcq->napi_deadline,
- jiffies + IONIC_NAPI_DEADLINE);
+ if (lif->doorbell_wa) {
+ if (!a_work)
+ ionic_adminq_poke_doorbell(&lif->adminqcq->q);
+ if (lif->hwstamp_rxq && !rx_work)
+ ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q);
+ if (lif->hwstamp_txq && !tx_work)
+ ionic_txq_poke_doorbell(&lif->hwstamp_txq->q);
+ }
return work_done;
}
@@ -1385,7 +1433,7 @@ static void ionic_ndo_set_rx_mode(struct net_device *netdev)
}
work->type = IONIC_DW_TYPE_RX_MODE;
netdev_dbg(lif->netdev, "deferred: rx_mode\n");
- ionic_lif_deferred_enqueue(&lif->deferred, work);
+ ionic_lif_deferred_enqueue(lif, work);
}
static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
@@ -1761,13 +1809,13 @@ static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
/* if we're not running, nothing more to do */
if (!netif_running(netdev)) {
- netdev->mtu = new_mtu;
+ WRITE_ONCE(netdev->mtu, new_mtu);
return 0;
}
mutex_lock(&lif->queue_lock);
ionic_stop_queues_reconfig(lif);
- netdev->mtu = new_mtu;
+ WRITE_ONCE(netdev->mtu, new_mtu);
err = ionic_start_queues_reconfig(lif);
mutex_unlock(&lif->queue_lock);
@@ -2035,7 +2083,7 @@ static int ionic_txrx_alloc(struct ionic_lif *lif)
err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
num_desc, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_tx_desc_info),
- lif->kern_pid, &lif->txqcqs[i]);
+ lif->kern_pid, NULL, &lif->txqcqs[i]);
if (err)
goto err_out;
@@ -2067,7 +2115,8 @@ static int ionic_txrx_alloc(struct ionic_lif *lif)
err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
num_desc, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_rx_desc_info),
- lif->kern_pid, &lif->rxqcqs[i]);
+ lif->kern_pid, lif->xdp_prog,
+ &lif->rxqcqs[i]);
if (err)
goto err_out;
@@ -2132,9 +2181,7 @@ static int ionic_txrx_enable(struct ionic_lif *lif)
int derr = 0;
int i, err;
- err = ionic_xdp_queues_config(lif);
- if (err)
- return err;
+ ionic_xdp_rxqs_prog_update(lif);
for (i = 0; i < lif->nxqs; i++) {
if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
@@ -2143,7 +2190,8 @@ static int ionic_txrx_enable(struct ionic_lif *lif)
goto err_out;
}
- ionic_rx_fill(&lif->rxqcqs[i]->q);
+ ionic_rx_fill(&lif->rxqcqs[i]->q,
+ READ_ONCE(lif->rxqcqs[i]->q.xdp_prog));
err = ionic_qcq_enable(lif->rxqcqs[i]);
if (err)
goto err_out;
@@ -2156,7 +2204,7 @@ static int ionic_txrx_enable(struct ionic_lif *lif)
}
if (lif->hwstamp_rxq) {
- ionic_rx_fill(&lif->hwstamp_rxq->q);
+ ionic_rx_fill(&lif->hwstamp_rxq->q, NULL);
err = ionic_qcq_enable(lif->hwstamp_rxq);
if (err)
goto err_out_hwstamp_rx;
@@ -2181,7 +2229,7 @@ err_out:
derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
}
- ionic_xdp_queues_config(lif);
+ ionic_xdp_rxqs_prog_update(lif);
return err;
}
@@ -2640,7 +2688,7 @@ static void ionic_vf_attr_replay(struct ionic_lif *lif)
ionic_vf_start(ionic);
}
-static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q)
+static void ionic_unregister_rxq_info(struct ionic_queue *q)
{
struct xdp_rxq_info *xi;
@@ -2654,7 +2702,7 @@ static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q)
kfree(xi);
}
-static int ionic_xdp_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)
+static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)
{
struct xdp_rxq_info *rxq_info;
int err;
@@ -2665,15 +2713,15 @@ static int ionic_xdp_register_rxq_info(struct ionic_queue *q, unsigned int napi_
err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id);
if (err) {
- dev_err(q->dev, "Queue %d xdp_rxq_info_reg failed, err %d\n",
- q->index, err);
+ netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n",
+ q->index, err);
goto err_out;
}
- err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_ORDER0, NULL);
+ err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool);
if (err) {
- dev_err(q->dev, "Queue %d xdp_rxq_info_reg_mem_model failed, err %d\n",
- q->index, err);
+ netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n",
+ q->index, err);
xdp_rxq_info_unreg(rxq_info);
goto err_out;
}
@@ -2687,44 +2735,20 @@ err_out:
return err;
}
-static int ionic_xdp_queues_config(struct ionic_lif *lif)
+static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif)
{
+ struct bpf_prog *xdp_prog;
unsigned int i;
- int err;
if (!lif->rxqcqs)
- return 0;
-
- /* There's no need to rework memory if not going to/from NULL program.
- * If there is no lif->xdp_prog, there should also be no q.xdp_rxq_info
- * This way we don't need to keep an *xdp_prog in every queue struct.
- */
- if (!lif->xdp_prog == !lif->rxqcqs[0]->q.xdp_rxq_info)
- return 0;
+ return;
+ xdp_prog = READ_ONCE(lif->xdp_prog);
for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
struct ionic_queue *q = &lif->rxqcqs[i]->q;
- if (q->xdp_rxq_info) {
- ionic_xdp_unregister_rxq_info(q);
- continue;
- }
-
- err = ionic_xdp_register_rxq_info(q, lif->rxqcqs[i]->napi.napi_id);
- if (err) {
- dev_err(lif->ionic->dev, "failed to register RX queue %d info for XDP, err %d\n",
- i, err);
- goto err_out;
- }
+ WRITE_ONCE(q->xdp_prog, xdp_prog);
}
-
- return 0;
-
-err_out:
- for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++)
- ionic_xdp_unregister_rxq_info(&lif->rxqcqs[i]->q);
-
- return err;
}
static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
@@ -2754,11 +2778,17 @@ static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
if (!netif_running(netdev)) {
old_prog = xchg(&lif->xdp_prog, bpf->prog);
+ } else if (lif->xdp_prog && bpf->prog) {
+ old_prog = xchg(&lif->xdp_prog, bpf->prog);
+ ionic_xdp_rxqs_prog_update(lif);
} else {
+ struct ionic_queue_params qparams;
+
+ ionic_init_queue_params(lif, &qparams);
+ qparams.xdp_prog = bpf->prog;
mutex_lock(&lif->queue_lock);
- ionic_stop_queues_reconfig(lif);
+ ionic_reconfigure_queues(lif, &qparams);
old_prog = xchg(&lif->xdp_prog, bpf->prog);
- ionic_start_queues_reconfig(lif);
mutex_unlock(&lif->queue_lock);
}
@@ -2860,13 +2890,23 @@ err_out:
static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
{
- /* only swapping the queues, not the napi, flags, or other stuff */
+ /* only swapping the queues and napi, not flags or other stuff */
+ swap(a->napi, b->napi);
+
+ if (a->q.type == IONIC_QTYPE_RXQ) {
+ swap(a->q.page_pool, b->q.page_pool);
+ a->q.page_pool->p.napi = &a->napi;
+ if (b->q.page_pool) /* is NULL when increasing queue count */
+ b->q.page_pool->p.napi = &b->napi;
+ }
+
swap(a->q.features, b->q.features);
swap(a->q.num_descs, b->q.num_descs);
swap(a->q.desc_size, b->q.desc_size);
swap(a->q.base, b->q.base);
swap(a->q.base_pa, b->q.base_pa);
swap(a->q.info, b->q.info);
+ swap(a->q.xdp_prog, b->q.xdp_prog);
swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
swap(a->q.partner, b->q.partner);
swap(a->q_base, b->q_base);
@@ -2917,7 +2957,8 @@ int ionic_reconfigure_queues(struct ionic_lif *lif,
}
if (qparam->nxqs != lif->nxqs ||
qparam->nrxq_descs != lif->nrxq_descs ||
- qparam->rxq_features != lif->rxq_features) {
+ qparam->rxq_features != lif->rxq_features ||
+ qparam->xdp_prog != lif->xdp_prog) {
rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
sizeof(struct ionic_qcq *), GFP_KERNEL);
if (!rx_qcqs) {
@@ -2948,7 +2989,7 @@ int ionic_reconfigure_queues(struct ionic_lif *lif,
err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
4, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_tx_desc_info),
- lif->kern_pid, &lif->txqcqs[i]);
+ lif->kern_pid, NULL, &lif->txqcqs[i]);
if (err)
goto err_out;
}
@@ -2957,7 +2998,7 @@ int ionic_reconfigure_queues(struct ionic_lif *lif,
err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
num_desc, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_tx_desc_info),
- lif->kern_pid, &tx_qcqs[i]);
+ lif->kern_pid, NULL, &tx_qcqs[i]);
if (err)
goto err_out;
}
@@ -2979,7 +3020,7 @@ int ionic_reconfigure_queues(struct ionic_lif *lif,
err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
4, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_rx_desc_info),
- lif->kern_pid, &lif->rxqcqs[i]);
+ lif->kern_pid, NULL, &lif->rxqcqs[i]);
if (err)
goto err_out;
}
@@ -2988,11 +3029,12 @@ int ionic_reconfigure_queues(struct ionic_lif *lif,
err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
num_desc, desc_sz, comp_sz, sg_desc_sz,
sizeof(struct ionic_rx_desc_info),
- lif->kern_pid, &rx_qcqs[i]);
+ lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]);
if (err)
goto err_out;
rx_qcqs[i]->q.features = qparam->rxq_features;
+ rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog;
}
}
@@ -3141,6 +3183,44 @@ err_out:
return err;
}
+static int ionic_affinity_masks_alloc(struct ionic *ionic)
+{
+ cpumask_var_t *affinity_masks;
+ int nintrs = ionic->nintrs;
+ int i;
+
+ affinity_masks = kcalloc(nintrs, sizeof(cpumask_var_t), GFP_KERNEL);
+ if (!affinity_masks)
+ return -ENOMEM;
+
+ for (i = 0; i < nintrs; i++) {
+ if (!zalloc_cpumask_var_node(&affinity_masks[i], GFP_KERNEL,
+ dev_to_node(ionic->dev)))
+ goto err_out;
+ }
+
+ ionic->affinity_masks = affinity_masks;
+
+ return 0;
+
+err_out:
+ for (--i; i >= 0; i--)
+ free_cpumask_var(affinity_masks[i]);
+ kfree(affinity_masks);
+
+ return -ENOMEM;
+}
+
+static void ionic_affinity_masks_free(struct ionic *ionic)
+{
+ int i;
+
+ for (i = 0; i < ionic->nintrs; i++)
+ free_cpumask_var(ionic->affinity_masks[i]);
+ kfree(ionic->affinity_masks);
+ ionic->affinity_masks = NULL;
+}
+
int ionic_lif_alloc(struct ionic *ionic)
{
struct device *dev = ionic->dev;
@@ -3171,7 +3251,7 @@ int ionic_lif_alloc(struct ionic *ionic)
netdev->netdev_ops = &ionic_netdev_ops;
ionic_ethtool_set_ops(netdev);
- netdev->watchdog_timeo = 2 * HZ;
+ netdev->watchdog_timeo = 5 * HZ;
netif_carrier_off(netdev);
lif->identity = lid;
@@ -3185,7 +3265,7 @@ int ionic_lif_alloc(struct ionic *ionic)
lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
le32_to_cpu(lif->identity->eth.min_frame_size));
lif->netdev->max_mtu =
- le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
+ le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
lif->neqs = ionic->neqs_per_lif;
lif->nxqs = ionic->ntxqs_per_lif;
@@ -3232,11 +3312,15 @@ int ionic_lif_alloc(struct ionic *ionic)
ionic_debugfs_add_lif(lif);
+ err = ionic_affinity_masks_alloc(ionic);
+ if (err)
+ goto err_out_free_lif_info;
+
/* allocate control queues and txrx queue arrays */
ionic_lif_queue_identify(lif);
err = ionic_qcqs_alloc(lif);
if (err)
- goto err_out_free_lif_info;
+ goto err_out_free_affinity_masks;
/* allocate rss indirection table */
tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
@@ -3258,6 +3342,8 @@ int ionic_lif_alloc(struct ionic *ionic)
err_out_free_qcqs:
ionic_qcqs_free(lif);
+err_out_free_affinity_masks:
+ ionic_affinity_masks_free(lif->ionic);
err_out_free_lif_info:
dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
lif->info = NULL;
@@ -3358,6 +3444,7 @@ int ionic_restart_lif(struct ionic_lif *lif)
clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
ionic_link_status_check_request(lif, CAN_SLEEP);
netif_device_attach(lif->netdev);
+ ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
return 0;
@@ -3388,6 +3475,7 @@ static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
* just need to reanimate it.
*/
ionic_init_devinfo(ionic);
+ ionic_reset(ionic);
err = ionic_identify(ionic);
if (err)
goto err_out;
@@ -3430,6 +3518,8 @@ void ionic_lif_free(struct ionic_lif *lif)
if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
ionic_lif_reset(lif);
+ ionic_affinity_masks_free(lif->ionic);
+
/* free lif info */
kfree(lif->identity);
dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
@@ -3503,14 +3593,11 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif)
netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
- qcq->napi_qcq = qcq;
- timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
-
napi_enable(&qcq->napi);
if (qcq->flags & IONIC_QCQ_F_INTR) {
irq_set_affinity_hint(qcq->intr.vector,
- &qcq->intr.affinity_mask);
+ *qcq->intr.affinity_mask);
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
IONIC_INTR_MASK_CLEAR);
}
@@ -3697,6 +3784,7 @@ int ionic_lif_init(struct ionic_lif *lif)
goto err_out_notifyq_deinit;
lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
+ lif->doorbell_wa = ionic_doorbell_wa(lif->ionic);
set_bit(IONIC_LIF_F_INITED, lif->state);
@@ -3716,10 +3804,6 @@ err_out_adminq_deinit:
return err;
}
-static void ionic_lif_notify_work(struct work_struct *ws)
-{
-}
-
static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
{
struct ionic_admin_ctx ctx = {
@@ -3731,7 +3815,7 @@ static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
},
};
- strscpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
+ strscpy(ctx.cmd.lif_setattr.name, netdev_name(lif->netdev),
sizeof(ctx.cmd.lif_setattr.name));
ionic_adminq_post_wait(lif, &ctx);
@@ -3770,8 +3854,6 @@ int ionic_lif_register(struct ionic_lif *lif)
ionic_lif_register_phc(lif);
- INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
-
lif->ionic->nb.notifier_call = ionic_lif_notify;
err = register_netdevice_notifier(&lif->ionic->nb);
@@ -3781,8 +3863,8 @@ int ionic_lif_register(struct ionic_lif *lif)
/* only register LIF0 for now */
err = register_netdev(lif->netdev);
if (err) {
- dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
- ionic_lif_unregister_phc(lif);
+ dev_err(lif->ionic->dev, "Cannot register net device: %d, aborting\n", err);
+ ionic_lif_unregister(lif);
return err;
}
@@ -3797,7 +3879,6 @@ void ionic_lif_unregister(struct ionic_lif *lif)
{
if (lif->ionic->nb.notifier_call) {
unregister_netdevice_notifier(&lif->ionic->nb);
- cancel_work_sync(&lif->ionic->nb_work);
lif->ionic->nb.notifier_call = NULL;
}
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
index 08f4266fe2aa..e01756fb7fdd 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h
@@ -84,12 +84,11 @@ struct ionic_qcq {
u32 cmb_pgid;
u32 cmb_order;
struct dim dim;
- struct timer_list napi_deadline;
struct ionic_queue q;
struct ionic_cq cq;
struct napi_struct napi;
- struct ionic_qcq *napi_qcq;
struct ionic_intr_info intr;
+ struct work_struct doorbell_napi_work;
struct dentry *dentry;
};
@@ -207,11 +206,12 @@ struct ionic_lif {
unsigned int nxqs;
unsigned int ntxq_descs;
unsigned int nrxq_descs;
- u32 rx_copybreak;
u64 rxq_features;
- u16 rx_mode;
u64 hw_features;
+ u16 rx_copybreak;
+ u16 rx_mode;
bool registered;
+ bool doorbell_wa;
u16 lif_type;
unsigned int link_down_count;
unsigned int nmcast;
@@ -226,11 +226,11 @@ struct ionic_lif {
u32 info_sz;
struct ionic_qtype_info qtype_info[IONIC_QTYPE_MAX];
- u16 rss_types;
u8 rss_hash_key[IONIC_RSS_HASH_KEY_SIZE];
u8 *rss_ind_tbl;
dma_addr_t rss_ind_tbl_pa;
u32 rss_ind_tbl_sz;
+ u16 rss_types;
struct ionic_rx_filters rx_filters;
u32 rx_coalesce_usecs; /* what the user asked for */
@@ -268,6 +268,7 @@ struct ionic_queue_params {
unsigned int ntxq_descs;
unsigned int nrxq_descs;
u64 rxq_features;
+ struct bpf_prog *xdp_prog;
bool intr_split;
bool cmb_tx;
bool cmb_rx;
@@ -280,6 +281,7 @@ static inline void ionic_init_queue_params(struct ionic_lif *lif,
qparam->ntxq_descs = lif->ntxq_descs;
qparam->nrxq_descs = lif->nrxq_descs;
qparam->rxq_features = lif->rxq_features;
+ qparam->xdp_prog = lif->xdp_prog;
qparam->intr_split = test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
qparam->cmb_tx = test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state);
qparam->cmb_rx = test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state);
@@ -333,7 +335,7 @@ static inline bool ionic_txq_hwstamp_enabled(struct ionic_queue *q)
void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep);
void ionic_get_stats64(struct net_device *netdev,
struct rtnl_link_stats64 *ns);
-void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
+void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
struct ionic_deferred_work *work);
int ionic_lif_alloc(struct ionic *ionic);
int ionic_lif_init(struct ionic_lif *lif);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c
index c1259324b0be..daf1e82cb76b 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_main.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c
@@ -81,8 +81,9 @@ static int ionic_error_to_errno(enum ionic_status_code code)
case IONIC_RC_EQTYPE:
case IONIC_RC_EQID:
case IONIC_RC_EINVAL:
- case IONIC_RC_ENOSUPP:
return -EINVAL;
+ case IONIC_RC_ENOSUPP:
+ return -EOPNOTSUPP;
case IONIC_RC_EPERM:
return -EPERM;
case IONIC_RC_ENOENT:
@@ -287,7 +288,7 @@ bool ionic_notifyq_service(struct ionic_cq *cq)
clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
} else {
work->type = IONIC_DW_TYPE_LIF_RESET;
- ionic_lif_deferred_enqueue(&lif->deferred, work);
+ ionic_lif_deferred_enqueue(lif, work);
}
}
break;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
index 1ee2f285cb42..528114877677 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c
@@ -312,8 +312,8 @@ static int ionic_lif_filter_add(struct ionic_lif *lif,
int err = 0;
ctx.cmd.rx_filter_add = *ac;
- ctx.cmd.rx_filter_add.opcode = IONIC_CMD_RX_FILTER_ADD,
- ctx.cmd.rx_filter_add.lif_index = cpu_to_le16(lif->index),
+ ctx.cmd.rx_filter_add.opcode = IONIC_CMD_RX_FILTER_ADD;
+ ctx.cmd.rx_filter_add.lif_index = cpu_to_le16(lif->index);
spin_lock_bh(&lif->rx_filters.lock);
f = ionic_rx_filter_find(lif, &ctx.cmd.rx_filter_add);
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index 5dba6d2d633c..2ac59564ded1 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -6,6 +6,7 @@
#include <linux/if_vlan.h>
#include <net/ip6_checksum.h>
#include <net/netdev_queues.h>
+#include <net/page_pool/helpers.h>
#include "ionic.h"
#include "ionic_lif.h"
@@ -23,7 +24,8 @@ static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
static void ionic_tx_clean(struct ionic_queue *q,
struct ionic_tx_desc_info *desc_info,
- struct ionic_txq_comp *comp);
+ struct ionic_txq_comp *comp,
+ bool in_napi);
static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell)
{
@@ -117,108 +119,57 @@ static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info)
static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info)
{
- return buf_info->dma_addr + buf_info->page_offset;
+ return page_pool_get_dma_addr(buf_info->page) + buf_info->page_offset;
}
-static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info)
+static void __ionic_rx_put_buf(struct ionic_queue *q,
+ struct ionic_buf_info *buf_info,
+ bool recycle_direct)
{
- return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset);
-}
-
-static int ionic_rx_page_alloc(struct ionic_queue *q,
- struct ionic_buf_info *buf_info)
-{
- struct device *dev = q->dev;
- dma_addr_t dma_addr;
- struct page *page;
-
- page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
- if (unlikely(!page)) {
- net_err_ratelimited("%s: %s page alloc failed\n",
- dev_name(dev), q->name);
- q_to_rx_stats(q)->alloc_err++;
- return -ENOMEM;
- }
-
- dma_addr = dma_map_page(dev, page, 0,
- IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
- if (unlikely(dma_mapping_error(dev, dma_addr))) {
- __free_pages(page, 0);
- net_err_ratelimited("%s: %s dma map failed\n",
- dev_name(dev), q->name);
- q_to_rx_stats(q)->dma_map_err++;
- return -EIO;
- }
-
- buf_info->dma_addr = dma_addr;
- buf_info->page = page;
- buf_info->page_offset = 0;
-
- return 0;
-}
-
-static void ionic_rx_page_free(struct ionic_queue *q,
- struct ionic_buf_info *buf_info)
-{
- struct device *dev = q->dev;
-
- if (unlikely(!buf_info)) {
- net_err_ratelimited("%s: %s invalid buf_info in free\n",
- dev_name(dev), q->name);
- return;
- }
-
if (!buf_info->page)
return;
- dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
- __free_pages(buf_info->page, 0);
+ page_pool_put_full_page(q->page_pool, buf_info->page, recycle_direct);
buf_info->page = NULL;
+ buf_info->len = 0;
+ buf_info->page_offset = 0;
}
-static bool ionic_rx_buf_recycle(struct ionic_queue *q,
- struct ionic_buf_info *buf_info, u32 len)
-{
- u32 size;
-
- /* don't re-use pages allocated in low-mem condition */
- if (page_is_pfmemalloc(buf_info->page))
- return false;
- /* don't re-use buffers from non-local numa nodes */
- if (page_to_nid(buf_info->page) != numa_mem_id())
- return false;
-
- size = ALIGN(len, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ);
- buf_info->page_offset += size;
- if (buf_info->page_offset >= IONIC_PAGE_SIZE)
- return false;
-
- get_page(buf_info->page);
+static void ionic_rx_put_buf(struct ionic_queue *q,
+ struct ionic_buf_info *buf_info)
+{
+ __ionic_rx_put_buf(q, buf_info, false);
+}
- return true;
+static void ionic_rx_put_buf_direct(struct ionic_queue *q,
+ struct ionic_buf_info *buf_info)
+{
+ __ionic_rx_put_buf(q, buf_info, true);
}
static void ionic_rx_add_skb_frag(struct ionic_queue *q,
struct sk_buff *skb,
struct ionic_buf_info *buf_info,
- u32 off, u32 len,
+ u32 headroom, u32 len,
bool synced)
{
if (!synced)
- dma_sync_single_range_for_cpu(q->dev, ionic_rx_buf_pa(buf_info),
- off, len, DMA_FROM_DEVICE);
+ page_pool_dma_sync_for_cpu(q->page_pool,
+ buf_info->page,
+ buf_info->page_offset + headroom,
+ len);
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
- buf_info->page, buf_info->page_offset + off,
- len,
- IONIC_PAGE_SIZE);
+ buf_info->page, buf_info->page_offset + headroom,
+ len, buf_info->len);
- if (!ionic_rx_buf_recycle(q, buf_info, len)) {
- dma_unmap_page(q->dev, buf_info->dma_addr,
- IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
- buf_info->page = NULL;
- }
+ /* napi_gro_frags() will release/recycle the
+ * page_pool buffers from the frags list
+ */
+ buf_info->page = NULL;
+ buf_info->len = 0;
+ buf_info->page_offset = 0;
}
static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q,
@@ -243,12 +194,13 @@ static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q,
q_to_rx_stats(q)->alloc_err++;
return NULL;
}
+ skb_mark_for_recycle(skb);
if (headroom)
frag_len = min_t(u16, len,
IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
else
- frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
+ frag_len = min_t(u16, len, IONIC_PAGE_SIZE);
if (unlikely(!buf_info->page))
goto err_bad_buf_page;
@@ -259,7 +211,7 @@ static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q,
for (i = 0; i < num_sg_elems; i++, buf_info++) {
if (unlikely(!buf_info->page))
goto err_bad_buf_page;
- frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info));
+ frag_len = min_t(u16, len, buf_info->len);
ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced);
len -= frag_len;
}
@@ -276,11 +228,13 @@ static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev,
struct ionic_rx_desc_info *desc_info,
unsigned int headroom,
unsigned int len,
+ unsigned int num_sg_elems,
bool synced)
{
struct ionic_buf_info *buf_info;
struct device *dev = q->dev;
struct sk_buff *skb;
+ int i;
buf_info = &desc_info->bufs[0];
@@ -291,54 +245,52 @@ static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev,
q_to_rx_stats(q)->alloc_err++;
return NULL;
}
-
- if (unlikely(!buf_info->page)) {
- dev_kfree_skb(skb);
- return NULL;
- }
+ skb_mark_for_recycle(skb);
if (!synced)
- dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info),
- headroom, len, DMA_FROM_DEVICE);
+ page_pool_dma_sync_for_cpu(q->page_pool,
+ buf_info->page,
+ buf_info->page_offset + headroom,
+ len);
+
skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len);
- dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info),
- headroom, len, DMA_FROM_DEVICE);
skb_put(skb, len);
skb->protocol = eth_type_trans(skb, netdev);
+ /* recycle the Rx buffer now that we're done with it */
+ ionic_rx_put_buf_direct(q, buf_info);
+ buf_info++;
+ for (i = 0; i < num_sg_elems; i++, buf_info++)
+ ionic_rx_put_buf_direct(q, buf_info);
+
return skb;
}
static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
- struct ionic_tx_desc_info *desc_info)
+ struct ionic_tx_desc_info *desc_info,
+ bool in_napi)
{
- unsigned int nbufs = desc_info->nbufs;
- struct ionic_buf_info *buf_info;
- struct device *dev = q->dev;
- int i;
+ struct xdp_frame_bulk bq;
- if (!nbufs)
+ if (!desc_info->nbufs)
return;
- buf_info = desc_info->bufs;
- dma_unmap_single(dev, buf_info->dma_addr,
- buf_info->len, DMA_TO_DEVICE);
- if (desc_info->act == XDP_TX)
- __free_pages(buf_info->page, 0);
- buf_info->page = NULL;
+ xdp_frame_bulk_init(&bq);
+ rcu_read_lock(); /* need for xdp_return_frame_bulk */
- buf_info++;
- for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) {
- dma_unmap_page(dev, buf_info->dma_addr,
- buf_info->len, DMA_TO_DEVICE);
- if (desc_info->act == XDP_TX)
- __free_pages(buf_info->page, 0);
- buf_info->page = NULL;
+ if (desc_info->act == XDP_TX) {
+ if (likely(in_napi))
+ xdp_return_frame_rx_napi(desc_info->xdpf);
+ else
+ xdp_return_frame(desc_info->xdpf);
+ } else if (desc_info->act == XDP_REDIRECT) {
+ ionic_tx_desc_unmap_bufs(q, desc_info);
+ xdp_return_frame_bulk(desc_info->xdpf, &bq);
}
- if (desc_info->act == XDP_REDIRECT)
- xdp_return_frame(desc_info->xdpf);
+ xdp_flush_frame_bulk(&bq);
+ rcu_read_unlock();
desc_info->nbufs = 0;
desc_info->xdpf = NULL;
@@ -362,9 +314,17 @@ static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame,
buf_info = desc_info->bufs;
stats = q_to_tx_stats(q);
- dma_addr = ionic_tx_map_single(q, frame->data, len);
- if (!dma_addr)
- return -EIO;
+ if (act == XDP_TX) {
+ dma_addr = page_pool_get_dma_addr(page) +
+ off + XDP_PACKET_HEADROOM;
+ dma_sync_single_for_device(q->dev, dma_addr,
+ len, DMA_TO_DEVICE);
+ } else /* XDP_REDIRECT */ {
+ dma_addr = ionic_tx_map_single(q, frame->data, len);
+ if (!dma_addr)
+ return -EIO;
+ }
+
buf_info->dma_addr = dma_addr;
buf_info->len = len;
buf_info->page = page;
@@ -386,10 +346,21 @@ static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame,
frag = sinfo->frags;
elem = ionic_tx_sg_elems(q);
for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) {
- dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
- if (!dma_addr) {
- ionic_tx_desc_unmap_bufs(q, desc_info);
- return -EIO;
+ if (act == XDP_TX) {
+ struct page *pg = skb_frag_page(frag);
+
+ dma_addr = page_pool_get_dma_addr(pg) +
+ skb_frag_off(frag);
+ dma_sync_single_for_device(q->dev, dma_addr,
+ skb_frag_size(frag),
+ DMA_TO_DEVICE);
+ } else {
+ dma_addr = ionic_tx_map_frag(q, frag, 0,
+ skb_frag_size(frag));
+ if (dma_mapping_error(q->dev, dma_addr)) {
+ ionic_tx_desc_unmap_bufs(q, desc_info);
+ return -EIO;
+ }
}
bi->dma_addr = dma_addr;
bi->len = skb_frag_size(frag);
@@ -480,6 +451,18 @@ int ionic_xdp_xmit(struct net_device *netdev, int n,
return nxmit;
}
+static void ionic_xdp_rx_unlink_bufs(struct ionic_queue *q,
+ struct ionic_buf_info *buf_info,
+ int nbufs)
+{
+ int i;
+
+ for (i = 0; i < nbufs; i++) {
+ buf_info->page = NULL;
+ buf_info++;
+ }
+}
+
static bool ionic_run_xdp(struct ionic_rx_stats *stats,
struct net_device *netdev,
struct bpf_prog *xdp_prog,
@@ -493,6 +476,7 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
struct netdev_queue *nq;
struct xdp_frame *xdpf;
int remain_len;
+ int nbufs = 1;
int frag_len;
int err = 0;
@@ -500,11 +484,9 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
XDP_PACKET_HEADROOM, frag_len, false);
-
- dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info),
- XDP_PACKET_HEADROOM, len,
- DMA_FROM_DEVICE);
-
+ page_pool_dma_sync_for_cpu(rxq->page_pool, buf_info->page,
+ buf_info->page_offset + XDP_PACKET_HEADROOM,
+ frag_len);
prefetchw(&xdp_buf.data_hard_start);
/* We limit MTU size to one buffer if !xdp_has_frags, so
@@ -526,15 +508,16 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
do {
if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
err = -ENOSPC;
- goto out_xdp_abort;
+ break;
}
frag = &sinfo->frags[sinfo->nr_frags];
sinfo->nr_frags++;
bi++;
- frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi));
- dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi),
- 0, frag_len, DMA_FROM_DEVICE);
+ frag_len = min_t(u16, remain_len, bi->len);
+ page_pool_dma_sync_for_cpu(rxq->page_pool, bi->page,
+ buf_info->page_offset,
+ frag_len);
skb_frag_fill_page_desc(frag, bi->page, 0, frag_len);
sinfo->xdp_frags_size += frag_len;
remain_len -= frag_len;
@@ -542,6 +525,7 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
if (page_is_pfmemalloc(bi->page))
xdp_buff_set_frag_pfmemalloc(&xdp_buf);
} while (remain_len > 0);
+ nbufs += sinfo->nr_frags;
}
xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf);
@@ -552,14 +536,16 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
return false; /* false = we didn't consume the packet */
case XDP_DROP:
- ionic_rx_page_free(rxq, buf_info);
+ ionic_rx_put_buf_direct(rxq, buf_info);
stats->xdp_drop++;
break;
case XDP_TX:
xdpf = xdp_convert_buff_to_frame(&xdp_buf);
- if (!xdpf)
- goto out_xdp_abort;
+ if (!xdpf) {
+ err = -ENOSPC;
+ break;
+ }
txq = rxq->partner;
nq = netdev_get_tx_queue(netdev, txq->index);
@@ -571,65 +557,58 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
ionic_q_space_avail(txq),
1, 1)) {
__netif_tx_unlock(nq);
- goto out_xdp_abort;
+ err = -EIO;
+ break;
}
- dma_unmap_page(rxq->dev, buf_info->dma_addr,
- IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
-
err = ionic_xdp_post_frame(txq, xdpf, XDP_TX,
buf_info->page,
buf_info->page_offset,
true);
__netif_tx_unlock(nq);
- if (err) {
+ if (unlikely(err)) {
netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
- goto out_xdp_abort;
+ break;
}
+ ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs);
stats->xdp_tx++;
-
- /* the Tx completion will free the buffers */
break;
case XDP_REDIRECT:
- /* unmap the pages before handing them to a different device */
- dma_unmap_page(rxq->dev, buf_info->dma_addr,
- IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
-
err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog);
- if (err) {
+ if (unlikely(err)) {
netdev_dbg(netdev, "xdp_do_redirect err %d\n", err);
- goto out_xdp_abort;
+ break;
}
- buf_info->page = NULL;
+ ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs);
rxq->xdp_flush = true;
stats->xdp_redirect++;
break;
case XDP_ABORTED:
default:
- goto out_xdp_abort;
+ err = -EIO;
+ break;
}
- return true;
-
-out_xdp_abort:
- trace_xdp_exception(netdev, xdp_prog, xdp_action);
- ionic_rx_page_free(rxq, buf_info);
- stats->xdp_aborted++;
+ if (err) {
+ ionic_rx_put_buf_direct(rxq, buf_info);
+ trace_xdp_exception(netdev, xdp_prog, xdp_action);
+ stats->xdp_aborted++;
+ }
return true;
}
static void ionic_rx_clean(struct ionic_queue *q,
struct ionic_rx_desc_info *desc_info,
- struct ionic_rxq_comp *comp)
+ struct ionic_rxq_comp *comp,
+ struct bpf_prog *xdp_prog)
{
struct net_device *netdev = q->lif->netdev;
struct ionic_qcq *qcq = q_to_qcq(q);
struct ionic_rx_stats *stats;
- struct bpf_prog *xdp_prog;
- unsigned int headroom;
+ unsigned int headroom = 0;
struct sk_buff *skb;
bool synced = false;
bool use_copybreak;
@@ -637,7 +616,14 @@ static void ionic_rx_clean(struct ionic_queue *q,
stats = q_to_rx_stats(q);
- if (comp->status) {
+ if (unlikely(comp->status)) {
+ /* Most likely status==2 and the pkt received was bigger
+ * than the buffer available: comp->len will show the
+ * pkt size received that didn't fit the advertised desc.len
+ */
+ dev_dbg(q->dev, "q%d drop comp->status %d comp->len %d desc->len %d\n",
+ q->index, comp->status, comp->len, q->rxq[q->head_idx].len);
+
stats->dropped++;
return;
}
@@ -646,18 +632,18 @@ static void ionic_rx_clean(struct ionic_queue *q,
stats->pkts++;
stats->bytes += len;
- xdp_prog = READ_ONCE(q->lif->xdp_prog);
if (xdp_prog) {
if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len))
return;
synced = true;
+ headroom = XDP_PACKET_HEADROOM;
}
- headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
use_copybreak = len <= q->lif->rx_copybreak;
if (use_copybreak)
skb = ionic_rx_copybreak(netdev, q, desc_info,
- headroom, len, synced);
+ headroom, len,
+ comp->num_sg_elems, synced);
else
skb = ionic_rx_build_skb(q, desc_info, headroom, len,
comp->num_sg_elems, synced);
@@ -733,7 +719,7 @@ static void ionic_rx_clean(struct ionic_queue *q,
napi_gro_frags(&qcq->napi);
}
-bool ionic_rx_service(struct ionic_cq *cq)
+static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog)
{
struct ionic_rx_desc_info *desc_info;
struct ionic_queue *q = cq->bound_q;
@@ -755,11 +741,16 @@ bool ionic_rx_service(struct ionic_cq *cq)
q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
/* clean the related q entry, only one per qc completion */
- ionic_rx_clean(q, desc_info, comp);
+ ionic_rx_clean(q, desc_info, comp, xdp_prog);
return true;
}
+bool ionic_rx_service(struct ionic_cq *cq)
+{
+ return __ionic_rx_service(cq, NULL);
+}
+
static inline void ionic_write_cmb_desc(struct ionic_queue *q,
void *desc)
{
@@ -770,7 +761,7 @@ static inline void ionic_write_cmb_desc(struct ionic_queue *q,
memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0]));
}
-void ionic_rx_fill(struct ionic_queue *q)
+void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog)
{
struct net_device *netdev = q->lif->netdev;
struct ionic_rx_desc_info *desc_info;
@@ -778,6 +769,9 @@ void ionic_rx_fill(struct ionic_queue *q)
struct ionic_buf_info *buf_info;
unsigned int fill_threshold;
struct ionic_rxq_desc *desc;
+ unsigned int first_frag_len;
+ unsigned int first_buf_len;
+ unsigned int headroom = 0;
unsigned int remain_len;
unsigned int frag_len;
unsigned int nfrags;
@@ -795,35 +789,43 @@ void ionic_rx_fill(struct ionic_queue *q)
len = netdev->mtu + VLAN_ETH_HLEN;
- for (i = n_fill; i; i--) {
- unsigned int headroom;
- unsigned int buf_len;
+ if (xdp_prog) {
+ /* Always alloc the full size buffer, but only need
+ * the actual frag_len in the descriptor
+ * XDP uses space in the first buffer, so account for
+ * head room, tail room, and ip header in the first frag size.
+ */
+ headroom = XDP_PACKET_HEADROOM;
+ first_buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN + headroom;
+ first_frag_len = min_t(u16, len + headroom, first_buf_len);
+ } else {
+ /* Use MTU size if smaller than max buffer size */
+ first_frag_len = min_t(u16, len, IONIC_PAGE_SIZE);
+ first_buf_len = first_frag_len;
+ }
+ for (i = n_fill; i; i--) {
+ /* fill main descriptor - buf[0] */
nfrags = 0;
remain_len = len;
desc = &q->rxq[q->head_idx];
desc_info = &q->rx_info[q->head_idx];
buf_info = &desc_info->bufs[0];
- if (!buf_info->page) { /* alloc a new buffer? */
- if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
- desc->addr = 0;
- desc->len = 0;
- return;
- }
+ buf_info->len = first_buf_len;
+ frag_len = first_frag_len - headroom;
+
+ /* get a new buffer if we can't reuse one */
+ if (!buf_info->page)
+ buf_info->page = page_pool_alloc(q->page_pool,
+ &buf_info->page_offset,
+ &buf_info->len,
+ GFP_ATOMIC);
+ if (unlikely(!buf_info->page)) {
+ buf_info->len = 0;
+ return;
}
- /* fill main descriptor - buf[0]
- * XDP uses space in the first buffer, so account for
- * head room, tail room, and ip header in the first frag size.
- */
- headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0;
- if (q->xdp_rxq_info)
- buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN;
- else
- buf_len = ionic_rx_buf_size(buf_info);
- frag_len = min_t(u16, len, buf_len);
-
desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom);
desc->len = cpu_to_le16(frag_len);
remain_len -= frag_len;
@@ -833,16 +835,26 @@ void ionic_rx_fill(struct ionic_queue *q)
/* fill sg descriptors - buf[1..n] */
sg_elem = q->rxq_sgl[q->head_idx].elems;
for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) {
- if (!buf_info->page) { /* alloc a new sg buffer? */
- if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
- sg_elem->addr = 0;
- sg_elem->len = 0;
+ frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE);
+
+ /* Recycle any leftover buffers that are too small to reuse */
+ if (unlikely(buf_info->page && buf_info->len < frag_len))
+ ionic_rx_put_buf_direct(q, buf_info);
+
+ /* Get new buffer if needed */
+ if (!buf_info->page) {
+ buf_info->len = frag_len;
+ buf_info->page = page_pool_alloc(q->page_pool,
+ &buf_info->page_offset,
+ &buf_info->len,
+ GFP_ATOMIC);
+ if (unlikely(!buf_info->page)) {
+ buf_info->len = 0;
return;
}
}
sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
- frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info));
sg_elem->len = cpu_to_le16(frag_len);
remain_len -= frag_len;
buf_info++;
@@ -867,25 +879,17 @@ void ionic_rx_fill(struct ionic_queue *q)
q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
q->dbell_jiffies = jiffies;
-
- mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
- jiffies + IONIC_NAPI_DEADLINE);
}
void ionic_rx_empty(struct ionic_queue *q)
{
struct ionic_rx_desc_info *desc_info;
- struct ionic_buf_info *buf_info;
unsigned int i, j;
for (i = 0; i < q->num_descs; i++) {
desc_info = &q->rx_info[i];
- for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) {
- buf_info = &desc_info->bufs[j];
- if (buf_info->page)
- ionic_rx_page_free(q, buf_info);
- }
-
+ for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++)
+ ionic_rx_put_buf(q, &desc_info->bufs[j]);
desc_info->nbufs = 0;
}
@@ -924,7 +928,7 @@ static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
dim_update_sample(qcq->cq.bound_intr->rearm_count,
pkts, bytes, &dim_sample);
- net_dim(&qcq->dim, dim_sample);
+ net_dim(&qcq->dim, &dim_sample);
}
int ionic_tx_napi(struct napi_struct *napi, int budget)
@@ -934,7 +938,7 @@ int ionic_tx_napi(struct napi_struct *napi, int budget)
u32 work_done = 0;
u32 flags = 0;
- work_done = ionic_tx_cq_service(cq, budget);
+ work_done = ionic_tx_cq_service(cq, budget, !!budget);
if (unlikely(!budget))
return budget;
@@ -952,8 +956,8 @@ int ionic_tx_napi(struct napi_struct *napi, int budget)
work_done, flags);
}
- if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
- mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+ if (!work_done && cq->bound_q->lif->doorbell_wa)
+ ionic_txq_poke_doorbell(&qcq->q);
return work_done;
}
@@ -966,6 +970,32 @@ static void ionic_xdp_do_flush(struct ionic_cq *cq)
}
}
+static unsigned int ionic_rx_cq_service(struct ionic_cq *cq,
+ unsigned int work_to_do)
+{
+ struct ionic_queue *q = cq->bound_q;
+ unsigned int work_done = 0;
+ struct bpf_prog *xdp_prog;
+
+ if (work_to_do == 0)
+ return 0;
+
+ xdp_prog = READ_ONCE(q->xdp_prog);
+ while (__ionic_rx_service(cq, xdp_prog)) {
+ if (cq->tail_idx == cq->num_descs - 1)
+ cq->done_color = !cq->done_color;
+
+ cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
+
+ if (++work_done >= work_to_do)
+ break;
+ }
+ ionic_rx_fill(q, xdp_prog);
+ ionic_xdp_do_flush(cq);
+
+ return work_done;
+}
+
int ionic_rx_napi(struct napi_struct *napi, int budget)
{
struct ionic_qcq *qcq = napi_to_qcq(napi);
@@ -976,12 +1006,8 @@ int ionic_rx_napi(struct napi_struct *napi, int budget)
if (unlikely(!budget))
return budget;
- work_done = ionic_cq_service(cq, budget,
- ionic_rx_service, NULL, NULL);
-
- ionic_rx_fill(cq->bound_q);
+ work_done = ionic_rx_cq_service(cq, budget);
- ionic_xdp_do_flush(cq);
if (work_done < budget && napi_complete_done(napi, work_done)) {
ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
flags |= IONIC_INTR_CRED_UNMASK;
@@ -995,8 +1021,8 @@ int ionic_rx_napi(struct napi_struct *napi, int budget)
work_done, flags);
}
- if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
- mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+ if (!work_done && cq->bound_q->lif->doorbell_wa)
+ ionic_rxq_poke_doorbell(&qcq->q);
return work_done;
}
@@ -1009,7 +1035,6 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)
struct ionic_qcq *txqcq;
struct ionic_lif *lif;
struct ionic_cq *txcq;
- bool resched = false;
u32 rx_work_done = 0;
u32 tx_work_done = 0;
u32 flags = 0;
@@ -1018,17 +1043,13 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)
txqcq = lif->txqcqs[qi];
txcq = &lif->txqcqs[qi]->cq;
- tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT);
+ tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, !!budget);
if (unlikely(!budget))
return budget;
- rx_work_done = ionic_cq_service(rxcq, budget,
- ionic_rx_service, NULL, NULL);
+ rx_work_done = ionic_rx_cq_service(rxcq, budget);
- ionic_rx_fill(rxcq->bound_q);
-
- ionic_xdp_do_flush(rxcq);
if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
ionic_dim_update(rxqcq, 0);
flags |= IONIC_INTR_CRED_UNMASK;
@@ -1041,12 +1062,12 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)
tx_work_done + rx_work_done, flags);
}
- if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
- resched = true;
- if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
- resched = true;
- if (resched)
- mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+ if (lif->doorbell_wa) {
+ if (!rx_work_done)
+ ionic_rxq_poke_doorbell(&rxqcq->q);
+ if (!tx_work_done)
+ ionic_txq_poke_doorbell(&txqcq->q);
+ }
return rx_work_done;
}
@@ -1058,7 +1079,7 @@ static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
dma_addr_t dma_addr;
dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
- if (dma_mapping_error(dev, dma_addr)) {
+ if (unlikely(dma_mapping_error(dev, dma_addr))) {
net_warn_ratelimited("%s: DMA single map failed on %s!\n",
dev_name(dev), q->name);
q_to_tx_stats(q)->dma_map_err++;
@@ -1075,7 +1096,7 @@ static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
dma_addr_t dma_addr;
dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
- if (dma_mapping_error(dev, dma_addr)) {
+ if (unlikely(dma_mapping_error(dev, dma_addr))) {
net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
dev_name(dev), q->name);
q_to_tx_stats(q)->dma_map_err++;
@@ -1151,14 +1172,15 @@ static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
static void ionic_tx_clean(struct ionic_queue *q,
struct ionic_tx_desc_info *desc_info,
- struct ionic_txq_comp *comp)
+ struct ionic_txq_comp *comp,
+ bool in_napi)
{
struct ionic_tx_stats *stats = q_to_tx_stats(q);
struct ionic_qcq *qcq = q_to_qcq(q);
struct sk_buff *skb;
if (desc_info->xdpf) {
- ionic_xdp_tx_desc_clean(q->partner, desc_info);
+ ionic_xdp_tx_desc_clean(q->partner, desc_info, in_napi);
stats->clean++;
if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
@@ -1203,11 +1225,13 @@ static void ionic_tx_clean(struct ionic_queue *q,
desc_info->bytes = skb->len;
stats->clean++;
- napi_consume_skb(skb, 1);
+ napi_consume_skb(skb, likely(in_napi) ? 1 : 0);
}
static bool ionic_tx_service(struct ionic_cq *cq,
- unsigned int *total_pkts, unsigned int *total_bytes)
+ unsigned int *total_pkts,
+ unsigned int *total_bytes,
+ bool in_napi)
{
struct ionic_tx_desc_info *desc_info;
struct ionic_queue *q = cq->bound_q;
@@ -1229,7 +1253,7 @@ static bool ionic_tx_service(struct ionic_cq *cq,
desc_info->bytes = 0;
index = q->tail_idx;
q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
- ionic_tx_clean(q, desc_info, comp);
+ ionic_tx_clean(q, desc_info, comp, in_napi);
if (desc_info->skb) {
pkts++;
bytes += desc_info->bytes;
@@ -1243,7 +1267,9 @@ static bool ionic_tx_service(struct ionic_cq *cq,
return true;
}
-unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do)
+unsigned int ionic_tx_cq_service(struct ionic_cq *cq,
+ unsigned int work_to_do,
+ bool in_napi)
{
unsigned int work_done = 0;
unsigned int bytes = 0;
@@ -1252,7 +1278,7 @@ unsigned int ionic_tx_cq_service(struct ionic_cq *cq, unsigned int work_to_do)
if (work_to_do == 0)
return 0;
- while (ionic_tx_service(cq, &pkts, &bytes)) {
+ while (ionic_tx_service(cq, &pkts, &bytes, in_napi)) {
if (cq->tail_idx == cq->num_descs - 1)
cq->done_color = !cq->done_color;
cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
@@ -1278,7 +1304,7 @@ void ionic_tx_flush(struct ionic_cq *cq)
{
u32 work_done;
- work_done = ionic_tx_cq_service(cq, cq->num_descs);
+ work_done = ionic_tx_cq_service(cq, cq->num_descs, false);
if (work_done)
ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index,
work_done, IONIC_INTR_CRED_RESET_COALESCE);
@@ -1295,7 +1321,7 @@ void ionic_tx_empty(struct ionic_queue *q)
desc_info = &q->tx_info[q->tail_idx];
desc_info->bytes = 0;
q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
- ionic_tx_clean(q, desc_info, NULL);
+ ionic_tx_clean(q, desc_info, NULL, false);
if (desc_info->skb) {
pkts++;
bytes += desc_info->bytes;
@@ -1316,7 +1342,7 @@ static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
int err;
err = skb_cow_head(skb, 0);
- if (err)
+ if (unlikely(err))
return err;
if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
@@ -1340,7 +1366,7 @@ static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
int err;
err = skb_cow_head(skb, 0);
- if (err)
+ if (unlikely(err))
return err;
if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
@@ -1357,7 +1383,7 @@ static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
}
static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q,
- struct ionic_tx_desc_info *desc_info,
+ struct ionic_txq_desc *desc,
struct sk_buff *skb,
dma_addr_t addr, u8 nsge, u16 len,
unsigned int hdrlen, unsigned int mss,
@@ -1365,7 +1391,6 @@ static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q,
u16 vlan_tci, bool has_vlan,
bool start, bool done)
{
- struct ionic_txq_desc *desc = &q->txq[q->head_idx];
u8 flags = 0;
u64 cmd;
@@ -1445,7 +1470,7 @@ static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q,
err = ionic_tx_tcp_inner_pseudo_csum(skb);
else
err = ionic_tx_tcp_pseudo_csum(skb);
- if (err) {
+ if (unlikely(err)) {
/* clean up mapping from ionic_tx_map_skb */
ionic_tx_desc_unmap_bufs(q, desc_info);
return err;
@@ -1503,10 +1528,9 @@ static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q,
seg_rem = min(tso_rem, mss);
done = (tso_rem == 0);
/* post descriptor */
- ionic_tx_tso_post(netdev, q, desc_info, skb,
- desc_addr, desc_nsge, desc_len,
- hdrlen, mss, outer_csum, vlan_tci, has_vlan,
- start, done);
+ ionic_tx_tso_post(netdev, q, desc, skb, desc_addr, desc_nsge,
+ desc_len, hdrlen, mss, outer_csum, vlan_tci,
+ has_vlan, start, done);
start = false;
/* Buffer information is stored with the first tso descriptor */
desc_info = &q->tx_info[q->head_idx];
@@ -1731,7 +1755,7 @@ static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
linearize:
if (too_many_frags) {
err = skb_linearize(skb);
- if (err)
+ if (unlikely(err))
return err;
q_to_tx_stats(q)->linearize++;
}
@@ -1765,7 +1789,7 @@ static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
else
err = ionic_tx(netdev, q, skb);
- if (err)
+ if (unlikely(err))
goto err_out_drop;
return NETDEV_TX_OK;
@@ -1811,7 +1835,7 @@ netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
else
err = ionic_tx(netdev, q, skb);
- if (err)
+ if (unlikely(err))
goto err_out_drop;
return NETDEV_TX_OK;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.h b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
index 9e73e324e7a1..b2b9a2dc9eb8 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h
@@ -4,9 +4,11 @@
#ifndef _IONIC_TXRX_H_
#define _IONIC_TXRX_H_
+struct bpf_prog;
+
void ionic_tx_flush(struct ionic_cq *cq);
-void ionic_rx_fill(struct ionic_queue *q);
+void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog);
void ionic_rx_empty(struct ionic_queue *q);
void ionic_tx_empty(struct ionic_queue *q);
int ionic_rx_napi(struct napi_struct *napi, int budget);
generated by cgit (git 2.34.1) at 2025-07-24 18:18:41 +0000