1 files changed, 788 insertions, 381 deletions

diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index 6f4776759863..2ac59564ded1 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -5,27 +5,42 @@
 #include <linux/ipv6.h>
 #include <linux/if_vlan.h>
 #include <net/ip6_checksum.h>
+#include <net/netdev_queues.h>
+#include <net/page_pool/helpers.h>
 
 #include "ionic.h"
 #include "ionic_lif.h"
 #include "ionic_txrx.h"
 
-static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
-				  ionic_desc_cb cb_func, void *cb_arg)
+static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
+				      void *data, size_t len);
+
+static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
+				    const skb_frag_t *frag,
+				    size_t offset, size_t len);
+
+static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
+				     struct ionic_tx_desc_info *desc_info);
+
+static void ionic_tx_clean(struct ionic_queue *q,
+			   struct ionic_tx_desc_info *desc_info,
+			   struct ionic_txq_comp *comp,
+			   bool in_napi);
+
+static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell)
 {
-	ionic_q_post(q, ring_dbell, cb_func, cb_arg);
+	ionic_q_post(q, ring_dbell);
 }
 
-static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
-				  ionic_desc_cb cb_func, void *cb_arg)
+static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell)
 {
-	ionic_q_post(q, ring_dbell, cb_func, cb_arg);
+	ionic_q_post(q, ring_dbell);
 }
 
 bool ionic_txq_poke_doorbell(struct ionic_queue *q)
 {
-	unsigned long now, then, dif;
 	struct netdev_queue *netdev_txq;
+	unsigned long now, then, dif;
 	struct net_device *netdev;
 
 	netdev = q->lif->netdev;
@@ -83,224 +98,555 @@ bool ionic_rxq_poke_doorbell(struct ionic_queue *q)
 	return true;
 }
 
-static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
+static inline struct ionic_txq_sg_elem *ionic_tx_sg_elems(struct ionic_queue *q)
 {
-	return netdev_get_tx_queue(q->lif->netdev, q->index);
+	if (likely(q->sg_desc_size == sizeof(struct ionic_txq_sg_desc_v1)))
+		return q->txq_sgl_v1[q->head_idx].elems;
+	else
+		return q->txq_sgl[q->head_idx].elems;
 }
 
-static int ionic_rx_page_alloc(struct ionic_queue *q,
-			       struct ionic_buf_info *buf_info)
+static inline struct netdev_queue *q_to_ndq(struct net_device *netdev,
+					    struct ionic_queue *q)
 {
-	struct net_device *netdev = q->lif->netdev;
-	struct ionic_rx_stats *stats;
-	struct device *dev;
-	struct page *page;
-
-	dev = q->dev;
-	stats = q_to_rx_stats(q);
-
-	if (unlikely(!buf_info)) {
-		net_err_ratelimited("%s: %s invalid buf_info in alloc\n",
-				    netdev->name, q->name);
-		return -EINVAL;
-	}
-
-	page = alloc_pages(IONIC_PAGE_GFP_MASK, 0);
-	if (unlikely(!page)) {
-		net_err_ratelimited("%s: %s page alloc failed\n",
-				    netdev->name, q->name);
-		stats->alloc_err++;
-		return -ENOMEM;
-	}
-
-	buf_info->dma_addr = dma_map_page(dev, page, 0,
-					  IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
-	if (unlikely(dma_mapping_error(dev, buf_info->dma_addr))) {
-		__free_pages(page, 0);
-		net_err_ratelimited("%s: %s dma map failed\n",
-				    netdev->name, q->name);
-		stats->dma_map_err++;
-		return -EIO;
-	}
-
-	buf_info->page = page;
-	buf_info->page_offset = 0;
-
-	return 0;
+	return netdev_get_tx_queue(netdev, q->index);
 }
 
-static void ionic_rx_page_free(struct ionic_queue *q,
-			       struct ionic_buf_info *buf_info)
+static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info)
 {
-	struct net_device *netdev = q->lif->netdev;
-	struct device *dev = q->dev;
+	return page_address(buf_info->page) + buf_info->page_offset;
+}
 
-	if (unlikely(!buf_info)) {
-		net_err_ratelimited("%s: %s invalid buf_info in free\n",
-				    netdev->name, q->name);
-		return;
-	}
+static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info)
+{
+	return page_pool_get_dma_addr(buf_info->page) + buf_info->page_offset;
+}
 
+static void __ionic_rx_put_buf(struct ionic_queue *q,
+			       struct ionic_buf_info *buf_info,
+			       bool recycle_direct)
+{
 	if (!buf_info->page)
 		return;
 
-	dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
-	__free_pages(buf_info->page, 0);
+	page_pool_put_full_page(q->page_pool, buf_info->page, recycle_direct);
 	buf_info->page = NULL;
+	buf_info->len = 0;
+	buf_info->page_offset = 0;
 }
 
-static bool ionic_rx_buf_recycle(struct ionic_queue *q,
-				 struct ionic_buf_info *buf_info, u32 used)
-{
-	u32 size;
-
-	/* don't re-use pages allocated in low-mem condition */
-	if (page_is_pfmemalloc(buf_info->page))
-		return false;
-
-	/* don't re-use buffers from non-local numa nodes */
-	if (page_to_nid(buf_info->page) != numa_mem_id())
-		return false;
 
-	size = ALIGN(used, IONIC_PAGE_SPLIT_SZ);
-	buf_info->page_offset += size;
-	if (buf_info->page_offset >= IONIC_PAGE_SIZE)
-		return false;
+static void ionic_rx_put_buf(struct ionic_queue *q,
+			     struct ionic_buf_info *buf_info)
+{
+	__ionic_rx_put_buf(q, buf_info, false);
+}
 
-	get_page(buf_info->page);
+static void ionic_rx_put_buf_direct(struct ionic_queue *q,
+				    struct ionic_buf_info *buf_info)
+{
+	__ionic_rx_put_buf(q, buf_info, true);
+}
 
-	return true;
+static void ionic_rx_add_skb_frag(struct ionic_queue *q,
+				  struct sk_buff *skb,
+				  struct ionic_buf_info *buf_info,
+				  u32 headroom, u32 len,
+				  bool synced)
+{
+	if (!synced)
+		page_pool_dma_sync_for_cpu(q->page_pool,
+					   buf_info->page,
+					   buf_info->page_offset + headroom,
+					   len);
+
+	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
+			buf_info->page, buf_info->page_offset + headroom,
+			len, buf_info->len);
+
+	/* napi_gro_frags() will release/recycle the
+	 * page_pool buffers from the frags list
+	 */
+	buf_info->page = NULL;
+	buf_info->len = 0;
+	buf_info->page_offset = 0;
 }
 
-static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
-				      struct ionic_desc_info *desc_info,
-				      struct ionic_rxq_comp *comp)
+static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q,
+					  struct ionic_rx_desc_info *desc_info,
+					  unsigned int headroom,
+					  unsigned int len,
+					  unsigned int num_sg_elems,
+					  bool synced)
 {
-	struct net_device *netdev = q->lif->netdev;
 	struct ionic_buf_info *buf_info;
-	struct ionic_rx_stats *stats;
-	struct device *dev = q->dev;
 	struct sk_buff *skb;
 	unsigned int i;
 	u16 frag_len;
-	u16 len;
-
-	stats = q_to_rx_stats(q);
 
 	buf_info = &desc_info->bufs[0];
-	len = le16_to_cpu(comp->len);
-
 	prefetchw(buf_info->page);
 
 	skb = napi_get_frags(&q_to_qcq(q)->napi);
 	if (unlikely(!skb)) {
 		net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
-				     netdev->name, q->name);
-		stats->alloc_err++;
+				     dev_name(q->dev), q->name);
+		q_to_rx_stats(q)->alloc_err++;
 		return NULL;
 	}
+	skb_mark_for_recycle(skb);
 
-	i = comp->num_sg_elems + 1;
-	do {
-		if (unlikely(!buf_info->page)) {
-			dev_kfree_skb(skb);
-			return NULL;
-		}
-
-		frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
-						 IONIC_PAGE_SIZE - buf_info->page_offset));
-		len -= frag_len;
-
-		dma_sync_single_for_cpu(dev,
-					buf_info->dma_addr + buf_info->page_offset,
-					frag_len, DMA_FROM_DEVICE);
-
-		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
-				buf_info->page, buf_info->page_offset, frag_len,
-				IONIC_PAGE_SIZE);
-
-		if (!ionic_rx_buf_recycle(q, buf_info, frag_len)) {
-			dma_unmap_page(dev, buf_info->dma_addr,
-				       IONIC_PAGE_SIZE, DMA_FROM_DEVICE);
-			buf_info->page = NULL;
-		}
+	if (headroom)
+		frag_len = min_t(u16, len,
+				 IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
+	else
+		frag_len = min_t(u16, len, IONIC_PAGE_SIZE);
 
-		buf_info++;
+	if (unlikely(!buf_info->page))
+		goto err_bad_buf_page;
+	ionic_rx_add_skb_frag(q, skb, buf_info, headroom, frag_len, synced);
+	len -= frag_len;
+	buf_info++;
 
-		i--;
-	} while (i > 0);
+	for (i = 0; i < num_sg_elems; i++, buf_info++) {
+		if (unlikely(!buf_info->page))
+			goto err_bad_buf_page;
+		frag_len = min_t(u16, len, buf_info->len);
+		ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced);
+		len -= frag_len;
+	}
 
 	return skb;
+
+err_bad_buf_page:
+	dev_kfree_skb(skb);
+	return NULL;
 }
 
-static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
-					  struct ionic_desc_info *desc_info,
-					  struct ionic_rxq_comp *comp)
+static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev,
+					  struct ionic_queue *q,
+					  struct ionic_rx_desc_info *desc_info,
+					  unsigned int headroom,
+					  unsigned int len,
+					  unsigned int num_sg_elems,
+					  bool synced)
 {
-	struct net_device *netdev = q->lif->netdev;
 	struct ionic_buf_info *buf_info;
-	struct ionic_rx_stats *stats;
 	struct device *dev = q->dev;
 	struct sk_buff *skb;
-	u16 len;
-
-	stats = q_to_rx_stats(q);
+	int i;
 
 	buf_info = &desc_info->bufs[0];
-	len = le16_to_cpu(comp->len);
 
 	skb = napi_alloc_skb(&q_to_qcq(q)->napi, len);
 	if (unlikely(!skb)) {
 		net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
-				     netdev->name, q->name);
-		stats->alloc_err++;
+				     dev_name(dev), q->name);
+		q_to_rx_stats(q)->alloc_err++;
 		return NULL;
 	}
+	skb_mark_for_recycle(skb);
 
-	if (unlikely(!buf_info->page)) {
-		dev_kfree_skb(skb);
-		return NULL;
-	}
+	if (!synced)
+		page_pool_dma_sync_for_cpu(q->page_pool,
+					   buf_info->page,
+					   buf_info->page_offset + headroom,
+					   len);
 
-	dma_sync_single_for_cpu(dev, buf_info->dma_addr + buf_info->page_offset,
-				len, DMA_FROM_DEVICE);
-	skb_copy_to_linear_data(skb, page_address(buf_info->page) + buf_info->page_offset, len);
-	dma_sync_single_for_device(dev, buf_info->dma_addr + buf_info->page_offset,
-				   len, DMA_FROM_DEVICE);
+	skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len);
 
 	skb_put(skb, len);
-	skb->protocol = eth_type_trans(skb, q->lif->netdev);
+	skb->protocol = eth_type_trans(skb, netdev);
+
+	/* recycle the Rx buffer now that we're done with it */
+	ionic_rx_put_buf_direct(q, buf_info);
+	buf_info++;
+	for (i = 0; i < num_sg_elems; i++, buf_info++)
+		ionic_rx_put_buf_direct(q, buf_info);
 
 	return skb;
 }
 
+static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
+				    struct ionic_tx_desc_info *desc_info,
+				    bool in_napi)
+{
+	struct xdp_frame_bulk bq;
+
+	if (!desc_info->nbufs)
+		return;
+
+	xdp_frame_bulk_init(&bq);
+	rcu_read_lock(); /* need for xdp_return_frame_bulk */
+
+	if (desc_info->act == XDP_TX) {
+		if (likely(in_napi))
+			xdp_return_frame_rx_napi(desc_info->xdpf);
+		else
+			xdp_return_frame(desc_info->xdpf);
+	} else if (desc_info->act == XDP_REDIRECT) {
+		ionic_tx_desc_unmap_bufs(q, desc_info);
+		xdp_return_frame_bulk(desc_info->xdpf, &bq);
+	}
+
+	xdp_flush_frame_bulk(&bq);
+	rcu_read_unlock();
+
+	desc_info->nbufs = 0;
+	desc_info->xdpf = NULL;
+	desc_info->act = 0;
+}
+
+static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame,
+				enum xdp_action act, struct page *page, int off,
+				bool ring_doorbell)
+{
+	struct ionic_tx_desc_info *desc_info;
+	struct ionic_buf_info *buf_info;
+	struct ionic_tx_stats *stats;
+	struct ionic_txq_desc *desc;
+	size_t len = frame->len;
+	dma_addr_t dma_addr;
+	u64 cmd;
+
+	desc_info = &q->tx_info[q->head_idx];
+	desc = &q->txq[q->head_idx];
+	buf_info = desc_info->bufs;
+	stats = q_to_tx_stats(q);
+
+	if (act == XDP_TX) {
+		dma_addr = page_pool_get_dma_addr(page) +
+			   off + XDP_PACKET_HEADROOM;
+		dma_sync_single_for_device(q->dev, dma_addr,
+					   len, DMA_TO_DEVICE);
+	} else /* XDP_REDIRECT */ {
+		dma_addr = ionic_tx_map_single(q, frame->data, len);
+		if (!dma_addr)
+			return -EIO;
+	}
+
+	buf_info->dma_addr = dma_addr;
+	buf_info->len = len;
+	buf_info->page = page;
+	buf_info->page_offset = off;
+
+	desc_info->nbufs = 1;
+	desc_info->xdpf = frame;
+	desc_info->act = act;
+
+	if (xdp_frame_has_frags(frame)) {
+		struct ionic_txq_sg_elem *elem;
+		struct skb_shared_info *sinfo;
+		struct ionic_buf_info *bi;
+		skb_frag_t *frag;
+		int i;
+
+		bi = &buf_info[1];
+		sinfo = xdp_get_shared_info_from_frame(frame);
+		frag = sinfo->frags;
+		elem = ionic_tx_sg_elems(q);
+		for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) {
+			if (act == XDP_TX) {
+				struct page *pg = skb_frag_page(frag);
+
+				dma_addr = page_pool_get_dma_addr(pg) +
+					   skb_frag_off(frag);
+				dma_sync_single_for_device(q->dev, dma_addr,
+							   skb_frag_size(frag),
+							   DMA_TO_DEVICE);
+			} else {
+				dma_addr = ionic_tx_map_frag(q, frag, 0,
+							     skb_frag_size(frag));
+				if (dma_mapping_error(q->dev, dma_addr)) {
+					ionic_tx_desc_unmap_bufs(q, desc_info);
+					return -EIO;
+				}
+			}
+			bi->dma_addr = dma_addr;
+			bi->len = skb_frag_size(frag);
+			bi->page = skb_frag_page(frag);
+
+			elem->addr = cpu_to_le64(bi->dma_addr);
+			elem->len = cpu_to_le16(bi->len);
+			elem++;
+
+			desc_info->nbufs++;
+		}
+	}
+
+	cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
+				  0, (desc_info->nbufs - 1), buf_info->dma_addr);
+	desc->cmd = cpu_to_le64(cmd);
+	desc->len = cpu_to_le16(len);
+	desc->csum_start = 0;
+	desc->csum_offset = 0;
+
+	stats->xdp_frames++;
+	stats->pkts++;
+	stats->bytes += len;
+
+	ionic_txq_post(q, ring_doorbell);
+
+	return 0;
+}
+
+int ionic_xdp_xmit(struct net_device *netdev, int n,
+		   struct xdp_frame **xdp_frames, u32 flags)
+{
+	struct ionic_lif *lif = netdev_priv(netdev);
+	struct ionic_queue *txq;
+	struct netdev_queue *nq;
+	int nxmit;
+	int space;
+	int cpu;
+	int qi;
+
+	if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state)))
+		return -ENETDOWN;
+
+	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+		return -EINVAL;
+
+	/* AdminQ is assumed on cpu 0, while we attempt to affinitize the
+	 * TxRx queue pairs 0..n-1 on cpus 1..n.  We try to keep with that
+	 * affinitization here, but of course irqbalance and friends might
+	 * have juggled things anyway, so we have to check for the 0 case.
+	 */
+	cpu = smp_processor_id();
+	qi = cpu ? (cpu - 1) % lif->nxqs : cpu;
+
+	txq = &lif->txqcqs[qi]->q;
+	nq = netdev_get_tx_queue(netdev, txq->index);
+	__netif_tx_lock(nq, cpu);
+	txq_trans_cond_update(nq);
+
+	if (netif_tx_queue_stopped(nq) ||
+	    !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
+				  ionic_q_space_avail(txq),
+				  1, 1)) {
+		__netif_tx_unlock(nq);
+		return -EIO;
+	}
+
+	space = min_t(int, n, ionic_q_space_avail(txq));
+	for (nxmit = 0; nxmit < space ; nxmit++) {
+		if (ionic_xdp_post_frame(txq, xdp_frames[nxmit],
+					 XDP_REDIRECT,
+					 virt_to_page(xdp_frames[nxmit]->data),
+					 0, false)) {
+			nxmit--;
+			break;
+		}
+	}
+
+	if (flags & XDP_XMIT_FLUSH)
+		ionic_dbell_ring(lif->kern_dbpage, txq->hw_type,
+				 txq->dbval | txq->head_idx);
+
+	netif_txq_maybe_stop(q_to_ndq(netdev, txq),
+			     ionic_q_space_avail(txq),
+			     4, 4);
+	__netif_tx_unlock(nq);
+
+	return nxmit;
+}
+
+static void ionic_xdp_rx_unlink_bufs(struct ionic_queue *q,
+				     struct ionic_buf_info *buf_info,
+				     int nbufs)
+{
+	int i;
+
+	for (i = 0; i < nbufs; i++) {
+		buf_info->page = NULL;
+		buf_info++;
+	}
+}
+
+static bool ionic_run_xdp(struct ionic_rx_stats *stats,
+			  struct net_device *netdev,
+			  struct bpf_prog *xdp_prog,
+			  struct ionic_queue *rxq,
+			  struct ionic_buf_info *buf_info,
+			  int len)
+{
+	u32 xdp_action = XDP_ABORTED;
+	struct xdp_buff xdp_buf;
+	struct ionic_queue *txq;
+	struct netdev_queue *nq;
+	struct xdp_frame *xdpf;
+	int remain_len;
+	int nbufs = 1;
+	int frag_len;
+	int err = 0;
+
+	xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
+	frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN);
+	xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
+			 XDP_PACKET_HEADROOM, frag_len, false);
+	page_pool_dma_sync_for_cpu(rxq->page_pool, buf_info->page,
+				   buf_info->page_offset + XDP_PACKET_HEADROOM,
+				   frag_len);
+	prefetchw(&xdp_buf.data_hard_start);
+
+	/*  We limit MTU size to one buffer if !xdp_has_frags, so
+	 *  if the recv len is bigger than one buffer
+	 *     then we know we have frag info to gather
+	 */
+	remain_len = len - frag_len;
+	if (remain_len) {
+		struct skb_shared_info *sinfo;
+		struct ionic_buf_info *bi;
+		skb_frag_t *frag;
+
+		bi = buf_info;
+		sinfo = xdp_get_shared_info_from_buff(&xdp_buf);
+		sinfo->nr_frags = 0;
+		sinfo->xdp_frags_size = 0;
+		xdp_buff_set_frags_flag(&xdp_buf);
+
+		do {
+			if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) {
+				err = -ENOSPC;
+				break;
+			}
+
+			frag = &sinfo->frags[sinfo->nr_frags];
+			sinfo->nr_frags++;
+			bi++;
+			frag_len = min_t(u16, remain_len, bi->len);
+			page_pool_dma_sync_for_cpu(rxq->page_pool, bi->page,
+						   buf_info->page_offset,
+						   frag_len);
+			skb_frag_fill_page_desc(frag, bi->page, 0, frag_len);
+			sinfo->xdp_frags_size += frag_len;
+			remain_len -= frag_len;
+
+			if (page_is_pfmemalloc(bi->page))
+				xdp_buff_set_frag_pfmemalloc(&xdp_buf);
+		} while (remain_len > 0);
+		nbufs += sinfo->nr_frags;
+	}
+
+	xdp_action = bpf_prog_run_xdp(xdp_prog, &xdp_buf);
+
+	switch (xdp_action) {
+	case XDP_PASS:
+		stats->xdp_pass++;
+		return false;  /* false = we didn't consume the packet */
+
+	case XDP_DROP:
+		ionic_rx_put_buf_direct(rxq, buf_info);
+		stats->xdp_drop++;
+		break;
+
+	case XDP_TX:
+		xdpf = xdp_convert_buff_to_frame(&xdp_buf);
+		if (!xdpf) {
+			err = -ENOSPC;
+			break;
+		}
+
+		txq = rxq->partner;
+		nq = netdev_get_tx_queue(netdev, txq->index);
+		__netif_tx_lock(nq, smp_processor_id());
+		txq_trans_cond_update(nq);
+
+		if (netif_tx_queue_stopped(nq) ||
+		    !netif_txq_maybe_stop(q_to_ndq(netdev, txq),
+					  ionic_q_space_avail(txq),
+					  1, 1)) {
+			__netif_tx_unlock(nq);
+			err = -EIO;
+			break;
+		}
+
+		err = ionic_xdp_post_frame(txq, xdpf, XDP_TX,
+					   buf_info->page,
+					   buf_info->page_offset,
+					   true);
+		__netif_tx_unlock(nq);
+		if (unlikely(err)) {
+			netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
+			break;
+		}
+		ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs);
+		stats->xdp_tx++;
+		break;
+
+	case XDP_REDIRECT:
+		err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog);
+		if (unlikely(err)) {
+			netdev_dbg(netdev, "xdp_do_redirect err %d\n", err);
+			break;
+		}
+		ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs);
+		rxq->xdp_flush = true;
+		stats->xdp_redirect++;
+		break;
+
+	case XDP_ABORTED:
+	default:
+		err = -EIO;
+		break;
+	}
+
+	if (err) {
+		ionic_rx_put_buf_direct(rxq, buf_info);
+		trace_xdp_exception(netdev, xdp_prog, xdp_action);
+		stats->xdp_aborted++;
+	}
+
+	return true;
+}
+
 static void ionic_rx_clean(struct ionic_queue *q,
-			   struct ionic_desc_info *desc_info,
-			   struct ionic_cq_info *cq_info,
-			   void *cb_arg)
+			   struct ionic_rx_desc_info *desc_info,
+			   struct ionic_rxq_comp *comp,
+			   struct bpf_prog *xdp_prog)
 {
 	struct net_device *netdev = q->lif->netdev;
 	struct ionic_qcq *qcq = q_to_qcq(q);
 	struct ionic_rx_stats *stats;
-	struct ionic_rxq_comp *comp;
+	unsigned int headroom = 0;
 	struct sk_buff *skb;
-
-	comp = cq_info->cq_desc + qcq->cq.desc_size - sizeof(*comp);
+	bool synced = false;
+	bool use_copybreak;
+	u16 len;
 
 	stats = q_to_rx_stats(q);
 
-	if (comp->status) {
+	if (unlikely(comp->status)) {
+		/* Most likely status==2 and the pkt received was bigger
+		 * than the buffer available: comp->len will show the
+		 * pkt size received that didn't fit the advertised desc.len
+		 */
+		dev_dbg(q->dev, "q%d drop comp->status %d comp->len %d desc->len %d\n",
+			q->index, comp->status, comp->len, q->rxq[q->head_idx].len);
+
 		stats->dropped++;
 		return;
 	}
 
+	len = le16_to_cpu(comp->len);
 	stats->pkts++;
-	stats->bytes += le16_to_cpu(comp->len);
+	stats->bytes += len;
 
-	if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
-		skb = ionic_rx_copybreak(q, desc_info, comp);
+	if (xdp_prog) {
+		if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len))
+			return;
+		synced = true;
+		headroom = XDP_PACKET_HEADROOM;
+	}
+
+	use_copybreak = len <= q->lif->rx_copybreak;
+	if (use_copybreak)
+		skb = ionic_rx_copybreak(netdev, q, desc_info,
+					 headroom, len,
+					 comp->num_sg_elems, synced);
 	else
-		skb = ionic_rx_frags(q, desc_info, comp);
+		skb = ionic_rx_build_skb(q, desc_info, headroom, len,
+					 comp->num_sg_elems, synced);
 
 	if (unlikely(!skb)) {
 		stats->dropped++;
@@ -352,7 +698,7 @@ static void ionic_rx_clean(struct ionic_queue *q,
 		u64 hwstamp;
 
 		cq_desc_hwstamp =
-			cq_info->cq_desc +
+			(void *)comp +
 			qcq->cq.desc_size -
 			sizeof(struct ionic_rxq_comp) -
 			IONIC_HWSTAMP_CQ_NEGOFFSET;
@@ -367,19 +713,19 @@ static void ionic_rx_clean(struct ionic_queue *q,
 		}
 	}
 
-	if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
+	if (use_copybreak)
 		napi_gro_receive(&qcq->napi, skb);
 	else
 		napi_gro_frags(&qcq->napi);
 }
 
-bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
+static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog)
 {
+	struct ionic_rx_desc_info *desc_info;
 	struct ionic_queue *q = cq->bound_q;
-	struct ionic_desc_info *desc_info;
 	struct ionic_rxq_comp *comp;
 
-	comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
+	comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx];
 
 	if (!color_match(comp->pkt_type_color, cq->done_color))
 		return false;
@@ -391,41 +737,48 @@ bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
 	if (q->tail_idx != le16_to_cpu(comp->comp_index))
 		return false;
 
-	desc_info = &q->info[q->tail_idx];
+	desc_info = &q->rx_info[q->tail_idx];
 	q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 
 	/* clean the related q entry, only one per qc completion */
-	ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
-
-	desc_info->cb = NULL;
-	desc_info->cb_arg = NULL;
+	ionic_rx_clean(q, desc_info, comp, xdp_prog);
 
 	return true;
 }
 
+bool ionic_rx_service(struct ionic_cq *cq)
+{
+	return __ionic_rx_service(cq, NULL);
+}
+
 static inline void ionic_write_cmb_desc(struct ionic_queue *q,
-					void __iomem *cmb_desc,
 					void *desc)
 {
-	if (q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS)
-		memcpy_toio(cmb_desc, desc, q->desc_size);
+	/* Since Rx and Tx descriptors are the same size, we can
+	 * save an instruction or two and skip the qtype check.
+	 */
+	if (unlikely(q_to_qcq(q)->flags & IONIC_QCQ_F_CMB_RINGS))
+		memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0]));
 }
 
-void ionic_rx_fill(struct ionic_queue *q)
+void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog)
 {
 	struct net_device *netdev = q->lif->netdev;
-	struct ionic_desc_info *desc_info;
-	struct ionic_rxq_sg_desc *sg_desc;
+	struct ionic_rx_desc_info *desc_info;
 	struct ionic_rxq_sg_elem *sg_elem;
 	struct ionic_buf_info *buf_info;
 	unsigned int fill_threshold;
 	struct ionic_rxq_desc *desc;
+	unsigned int first_frag_len;
+	unsigned int first_buf_len;
+	unsigned int headroom = 0;
 	unsigned int remain_len;
 	unsigned int frag_len;
 	unsigned int nfrags;
 	unsigned int n_fill;
-	unsigned int i, j;
 	unsigned int len;
+	unsigned int i;
+	unsigned int j;
 
 	n_fill = ionic_q_space_avail(q);
 
@@ -434,48 +787,74 @@ void ionic_rx_fill(struct ionic_queue *q)
 	if (n_fill < fill_threshold)
 		return;
 
-	len = netdev->mtu + ETH_HLEN + VLAN_HLEN;
+	len = netdev->mtu + VLAN_ETH_HLEN;
+
+	if (xdp_prog) {
+		/* Always alloc the full size buffer, but only need
+		 * the actual frag_len in the descriptor
+		 * XDP uses space in the first buffer, so account for
+		 * head room, tail room, and ip header in the first frag size.
+		 */
+		headroom = XDP_PACKET_HEADROOM;
+		first_buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN + headroom;
+		first_frag_len = min_t(u16, len + headroom, first_buf_len);
+	} else {
+		/* Use MTU size if smaller than max buffer size */
+		first_frag_len = min_t(u16, len, IONIC_PAGE_SIZE);
+		first_buf_len = first_frag_len;
+	}
 
 	for (i = n_fill; i; i--) {
+		/* fill main descriptor - buf[0] */
 		nfrags = 0;
 		remain_len = len;
-		desc_info = &q->info[q->head_idx];
-		desc = desc_info->desc;
+		desc = &q->rxq[q->head_idx];
+		desc_info = &q->rx_info[q->head_idx];
 		buf_info = &desc_info->bufs[0];
 
-		if (!buf_info->page) { /* alloc a new buffer? */
-			if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
-				desc->addr = 0;
-				desc->len = 0;
-				return;
-			}
+		buf_info->len = first_buf_len;
+		frag_len = first_frag_len - headroom;
+
+		/* get a new buffer if we can't reuse one */
+		if (!buf_info->page)
+			buf_info->page = page_pool_alloc(q->page_pool,
+							 &buf_info->page_offset,
+							 &buf_info->len,
+							 GFP_ATOMIC);
+		if (unlikely(!buf_info->page)) {
+			buf_info->len = 0;
+			return;
 		}
 
-		/* fill main descriptor - buf[0] */
-		desc->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
-		frag_len = min_t(u16, len, min_t(u32, IONIC_MAX_BUF_LEN,
-						 IONIC_PAGE_SIZE - buf_info->page_offset));
+		desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom);
 		desc->len = cpu_to_le16(frag_len);
 		remain_len -= frag_len;
 		buf_info++;
 		nfrags++;
 
 		/* fill sg descriptors - buf[1..n] */
-		sg_desc = desc_info->sg_desc;
-		for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++) {
-			sg_elem = &sg_desc->elems[j];
-			if (!buf_info->page) { /* alloc a new sg buffer? */
-				if (unlikely(ionic_rx_page_alloc(q, buf_info))) {
-					sg_elem->addr = 0;
-					sg_elem->len = 0;
+		sg_elem = q->rxq_sgl[q->head_idx].elems;
+		for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) {
+			frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE);
+
+			/* Recycle any leftover buffers that are too small to reuse */
+			if (unlikely(buf_info->page && buf_info->len < frag_len))
+				ionic_rx_put_buf_direct(q, buf_info);
+
+			/* Get new buffer if needed */
+			if (!buf_info->page) {
+				buf_info->len = frag_len;
+				buf_info->page = page_pool_alloc(q->page_pool,
+								 &buf_info->page_offset,
+								 &buf_info->len,
+								 GFP_ATOMIC);
+				if (unlikely(!buf_info->page)) {
+					buf_info->len = 0;
 					return;
 				}
 			}
 
-			sg_elem->addr = cpu_to_le64(buf_info->dma_addr + buf_info->page_offset);
-			frag_len = min_t(u16, remain_len, min_t(u32, IONIC_MAX_BUF_LEN,
-								IONIC_PAGE_SIZE -
-								buf_info->page_offset));
+			sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info));
 			sg_elem->len = cpu_to_le16(frag_len);
 			remain_len -= frag_len;
 			buf_info++;
@@ -483,18 +862,16 @@ void ionic_rx_fill(struct ionic_queue *q)
 		}
 
 		/* clear end sg element as a sentinel */
-		if (j < q->max_sg_elems) {
-			sg_elem = &sg_desc->elems[j];
+		if (j < q->max_sg_elems)
 			memset(sg_elem, 0, sizeof(*sg_elem));
-		}
 
 		desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 					      IONIC_RXQ_DESC_OPCODE_SIMPLE;
 		desc_info->nbufs = nfrags;
 
-		ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
+		ionic_write_cmb_desc(q, desc);
 
-		ionic_rxq_post(q, false, ionic_rx_clean, NULL);
+		ionic_rxq_post(q, false);
 	}
 
 	ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
@@ -502,28 +879,18 @@ void ionic_rx_fill(struct ionic_queue *q)
 
 	q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
 	q->dbell_jiffies = jiffies;
-
-	mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
-		  jiffies + IONIC_NAPI_DEADLINE);
 }
 
 void ionic_rx_empty(struct ionic_queue *q)
 {
-	struct ionic_desc_info *desc_info;
-	struct ionic_buf_info *buf_info;
+	struct ionic_rx_desc_info *desc_info;
 	unsigned int i, j;
 
 	for (i = 0; i < q->num_descs; i++) {
-		desc_info = &q->info[i];
-		for (j = 0; j < IONIC_RX_MAX_SG_ELEMS + 1; j++) {
-			buf_info = &desc_info->bufs[j];
-			if (buf_info->page)
-				ionic_rx_page_free(q, buf_info);
-		}
-
+		desc_info = &q->rx_info[i];
+		for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++)
+			ionic_rx_put_buf(q, &desc_info->bufs[j]);
 		desc_info->nbufs = 0;
-		desc_info->cb = NULL;
-		desc_info->cb_arg = NULL;
 	}
 
 	q->head_idx = 0;
@@ -561,23 +928,17 @@ static void ionic_dim_update(struct ionic_qcq *qcq, int napi_mode)
 	dim_update_sample(qcq->cq.bound_intr->rearm_count,
 			  pkts, bytes, &dim_sample);
 
-	net_dim(&qcq->dim, dim_sample);
+	net_dim(&qcq->dim, &dim_sample);
 }
 
 int ionic_tx_napi(struct napi_struct *napi, int budget)
 {
 	struct ionic_qcq *qcq = napi_to_qcq(napi);
 	struct ionic_cq *cq = napi_to_cq(napi);
-	struct ionic_dev *idev;
-	struct ionic_lif *lif;
 	u32 work_done = 0;
 	u32 flags = 0;
 
-	lif = cq->bound_q->lif;
-	idev = &lif->ionic->idev;
-
-	work_done = ionic_cq_service(cq, budget,
-				     ionic_tx_service, NULL, NULL);
+	work_done = ionic_tx_cq_service(cq, budget, !!budget);
 
 	if (unlikely(!budget))
 		return budget;
@@ -590,13 +951,47 @@ int ionic_tx_napi(struct napi_struct *napi, int budget)
 
 	if (work_done || flags) {
 		flags |= IONIC_INTR_CRED_RESET_COALESCE;
-		ionic_intr_credits(idev->intr_ctrl,
+		ionic_intr_credits(cq->idev->intr_ctrl,
 				   cq->bound_intr->index,
 				   work_done, flags);
 	}
 
-	if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
-		mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+	if (!work_done && cq->bound_q->lif->doorbell_wa)
+		ionic_txq_poke_doorbell(&qcq->q);
+
+	return work_done;
+}
+
+static void ionic_xdp_do_flush(struct ionic_cq *cq)
+{
+	if (cq->bound_q->xdp_flush) {
+		xdp_do_flush();
+		cq->bound_q->xdp_flush = false;
+	}
+}
+
+static unsigned int ionic_rx_cq_service(struct ionic_cq *cq,
+					unsigned int work_to_do)
+{
+	struct ionic_queue *q = cq->bound_q;
+	unsigned int work_done = 0;
+	struct bpf_prog *xdp_prog;
+
+	if (work_to_do == 0)
+		return 0;
+
+	xdp_prog = READ_ONCE(q->xdp_prog);
+	while (__ionic_rx_service(cq, xdp_prog)) {
+		if (cq->tail_idx == cq->num_descs - 1)
+			cq->done_color = !cq->done_color;
+
+		cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
+
+		if (++work_done >= work_to_do)
+			break;
+	}
+	ionic_rx_fill(q, xdp_prog);
+	ionic_xdp_do_flush(cq);
 
 	return work_done;
 }
@@ -605,21 +1000,13 @@ int ionic_rx_napi(struct napi_struct *napi, int budget)
 {
 	struct ionic_qcq *qcq = napi_to_qcq(napi);
 	struct ionic_cq *cq = napi_to_cq(napi);
-	struct ionic_dev *idev;
-	struct ionic_lif *lif;
 	u32 work_done = 0;
 	u32 flags = 0;
 
 	if (unlikely(!budget))
 		return budget;
 
-	lif = cq->bound_q->lif;
-	idev = &lif->ionic->idev;
-
-	work_done = ionic_cq_service(cq, budget,
-				     ionic_rx_service, NULL, NULL);
-
-	ionic_rx_fill(cq->bound_q);
+	work_done = ionic_rx_cq_service(cq, budget);
 
 	if (work_done < budget && napi_complete_done(napi, work_done)) {
 		ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR);
@@ -629,13 +1016,13 @@ int ionic_rx_napi(struct napi_struct *napi, int budget)
 
 	if (work_done || flags) {
 		flags |= IONIC_INTR_CRED_RESET_COALESCE;
-		ionic_intr_credits(idev->intr_ctrl,
+		ionic_intr_credits(cq->idev->intr_ctrl,
 				   cq->bound_intr->index,
 				   work_done, flags);
 	}
 
-	if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
-		mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+	if (!work_done && cq->bound_q->lif->doorbell_wa)
+		ionic_rxq_poke_doorbell(&qcq->q);
 
 	return work_done;
 }
@@ -646,29 +1033,22 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)
 	struct ionic_cq *rxcq = napi_to_cq(napi);
 	unsigned int qi = rxcq->bound_q->index;
 	struct ionic_qcq *txqcq;
-	struct ionic_dev *idev;
 	struct ionic_lif *lif;
 	struct ionic_cq *txcq;
-	bool resched = false;
 	u32 rx_work_done = 0;
 	u32 tx_work_done = 0;
 	u32 flags = 0;
 
 	lif = rxcq->bound_q->lif;
-	idev = &lif->ionic->idev;
 	txqcq = lif->txqcqs[qi];
 	txcq = &lif->txqcqs[qi]->cq;
 
-	tx_work_done = ionic_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT,
-					ionic_tx_service, NULL, NULL);
+	tx_work_done = ionic_tx_cq_service(txcq, IONIC_TX_BUDGET_DEFAULT, !!budget);
 
 	if (unlikely(!budget))
 		return budget;
 
-	rx_work_done = ionic_cq_service(rxcq, budget,
-					ionic_rx_service, NULL, NULL);
-
-	ionic_rx_fill(rxcq->bound_q);
+	rx_work_done = ionic_rx_cq_service(rxcq, budget);
 
 	if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
 		ionic_dim_update(rxqcq, 0);
@@ -678,16 +1058,16 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)
 
 	if (rx_work_done || flags) {
 		flags |= IONIC_INTR_CRED_RESET_COALESCE;
-		ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
+		ionic_intr_credits(rxcq->idev->intr_ctrl, rxcq->bound_intr->index,
 				   tx_work_done + rx_work_done, flags);
 	}
 
-	if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
-		resched = true;
-	if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
-		resched = true;
-	if (resched)
-		mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
+	if (lif->doorbell_wa) {
+		if (!rx_work_done)
+			ionic_rxq_poke_doorbell(&rxqcq->q);
+		if (!tx_work_done)
+			ionic_txq_poke_doorbell(&txqcq->q);
+	}
 
 	return rx_work_done;
 }
@@ -695,15 +1075,14 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget)
 static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
 				      void *data, size_t len)
 {
-	struct ionic_tx_stats *stats = q_to_tx_stats(q);
 	struct device *dev = q->dev;
 	dma_addr_t dma_addr;
 
 	dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
-	if (dma_mapping_error(dev, dma_addr)) {
+	if (unlikely(dma_mapping_error(dev, dma_addr))) {
 		net_warn_ratelimited("%s: DMA single map failed on %s!\n",
-				     q->lif->netdev->name, q->name);
-		stats->dma_map_err++;
+				     dev_name(dev), q->name);
+		q_to_tx_stats(q)->dma_map_err++;
 		return 0;
 	}
 	return dma_addr;
@@ -713,24 +1092,23 @@ static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
 				    const skb_frag_t *frag,
 				    size_t offset, size_t len)
 {
-	struct ionic_tx_stats *stats = q_to_tx_stats(q);
 	struct device *dev = q->dev;
 	dma_addr_t dma_addr;
 
 	dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
-	if (dma_mapping_error(dev, dma_addr)) {
+	if (unlikely(dma_mapping_error(dev, dma_addr))) {
 		net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
-				     q->lif->netdev->name, q->name);
-		stats->dma_map_err++;
+				     dev_name(dev), q->name);
+		q_to_tx_stats(q)->dma_map_err++;
+		return 0;
 	}
 	return dma_addr;
 }
 
 static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
-			    struct ionic_desc_info *desc_info)
+			    struct ionic_tx_desc_info *desc_info)
 {
 	struct ionic_buf_info *buf_info = desc_info->bufs;
-	struct ionic_tx_stats *stats = q_to_tx_stats(q);
 	struct device *dev = q->dev;
 	dma_addr_t dma_addr;
 	unsigned int nfrags;
@@ -738,10 +1116,8 @@ static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
 	int frag_idx;
 
 	dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
-	if (dma_mapping_error(dev, dma_addr)) {
-		stats->dma_map_err++;
+	if (!dma_addr)
 		return -EIO;
-	}
 	buf_info->dma_addr = dma_addr;
 	buf_info->len = skb_headlen(skb);
 	buf_info++;
@@ -750,10 +1126,8 @@ static int ionic_tx_map_skb(struct ionic_queue *q, struct sk_buff *skb,
 	nfrags = skb_shinfo(skb)->nr_frags;
 	for (frag_idx = 0; frag_idx < nfrags; frag_idx++, frag++) {
 		dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag));
-		if (dma_mapping_error(dev, dma_addr)) {
-			stats->dma_map_err++;
+		if (!dma_addr)
 			goto dma_fail;
-		}
 		buf_info->dma_addr = dma_addr;
 		buf_info->len = skb_frag_size(frag);
 		buf_info++;
@@ -771,12 +1145,13 @@ dma_fail:
 		dma_unmap_page(dev, buf_info->dma_addr,
 			       buf_info->len, DMA_TO_DEVICE);
 	}
-	dma_unmap_single(dev, buf_info->dma_addr, buf_info->len, DMA_TO_DEVICE);
+	dma_unmap_single(dev, desc_info->bufs[0].dma_addr,
+			 desc_info->bufs[0].len, DMA_TO_DEVICE);
 	return -EIO;
 }
 
 static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
-				     struct ionic_desc_info *desc_info)
+				     struct ionic_tx_desc_info *desc_info)
 {
 	struct ionic_buf_info *buf_info = desc_info->bufs;
 	struct device *dev = q->dev;
@@ -785,41 +1160,49 @@ static void ionic_tx_desc_unmap_bufs(struct ionic_queue *q,
 	if (!desc_info->nbufs)
 		return;
 
-	dma_unmap_single(dev, (dma_addr_t)buf_info->dma_addr,
+	dma_unmap_single(dev, buf_info->dma_addr,
 			 buf_info->len, DMA_TO_DEVICE);
 	buf_info++;
 	for (i = 1; i < desc_info->nbufs; i++, buf_info++)
-		dma_unmap_page(dev, (dma_addr_t)buf_info->dma_addr,
+		dma_unmap_page(dev, buf_info->dma_addr,
 			       buf_info->len, DMA_TO_DEVICE);
 
 	desc_info->nbufs = 0;
 }
 
 static void ionic_tx_clean(struct ionic_queue *q,
-			   struct ionic_desc_info *desc_info,
-			   struct ionic_cq_info *cq_info,
-			   void *cb_arg)
+			   struct ionic_tx_desc_info *desc_info,
+			   struct ionic_txq_comp *comp,
+			   bool in_napi)
 {
 	struct ionic_tx_stats *stats = q_to_tx_stats(q);
 	struct ionic_qcq *qcq = q_to_qcq(q);
-	struct sk_buff *skb = cb_arg;
-	u16 qi;
+	struct sk_buff *skb;
+
+	if (desc_info->xdpf) {
+		ionic_xdp_tx_desc_clean(q->partner, desc_info, in_napi);
+		stats->clean++;
+
+		if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
+			netif_wake_subqueue(q->lif->netdev, q->index);
+
+		return;
+	}
 
 	ionic_tx_desc_unmap_bufs(q, desc_info);
 
+	skb = desc_info->skb;
 	if (!skb)
 		return;
 
-	qi = skb_get_queue_mapping(skb);
-
-	if (ionic_txq_hwstamp_enabled(q)) {
-		if (cq_info) {
+	if (unlikely(ionic_txq_hwstamp_enabled(q))) {
+		if (comp) {
 			struct skb_shared_hwtstamps hwts = {};
 			__le64 *cq_desc_hwstamp;
 			u64 hwstamp;
 
 			cq_desc_hwstamp =
-				cq_info->cq_desc +
+				(void *)comp +
 				qcq->cq.desc_size -
 				sizeof(struct ionic_txq_comp) -
 				IONIC_HWSTAMP_CQ_NEGOFFSET;
@@ -837,27 +1220,27 @@ static void ionic_tx_clean(struct ionic_queue *q,
 				stats->hwstamp_invalid++;
 			}
 		}
-
-	} else if (unlikely(__netif_subqueue_stopped(q->lif->netdev, qi))) {
-		netif_wake_subqueue(q->lif->netdev, qi);
 	}
 
 	desc_info->bytes = skb->len;
 	stats->clean++;
 
-	dev_consume_skb_any(skb);
+	napi_consume_skb(skb, likely(in_napi) ? 1 : 0);
 }
 
-bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
+static bool ionic_tx_service(struct ionic_cq *cq,
+			     unsigned int *total_pkts,
+			     unsigned int *total_bytes,
+			     bool in_napi)
 {
+	struct ionic_tx_desc_info *desc_info;
 	struct ionic_queue *q = cq->bound_q;
-	struct ionic_desc_info *desc_info;
 	struct ionic_txq_comp *comp;
-	int bytes = 0;
-	int pkts = 0;
+	unsigned int bytes = 0;
+	unsigned int pkts = 0;
 	u16 index;
 
-	comp = cq_info->cq_desc + cq->desc_size - sizeof(*comp);
+	comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx];
 
 	if (!color_match(comp->color, cq->done_color))
 		return false;
@@ -866,59 +1249,92 @@ bool ionic_tx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
 	 * several q entries completed for each cq completion
 	 */
 	do {
-		desc_info = &q->info[q->tail_idx];
+		desc_info = &q->tx_info[q->tail_idx];
 		desc_info->bytes = 0;
 		index = q->tail_idx;
 		q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
-		ionic_tx_clean(q, desc_info, cq_info, desc_info->cb_arg);
-		if (desc_info->cb_arg) {
+		ionic_tx_clean(q, desc_info, comp, in_napi);
+		if (desc_info->skb) {
 			pkts++;
 			bytes += desc_info->bytes;
+			desc_info->skb = NULL;
 		}
-		desc_info->cb = NULL;
-		desc_info->cb_arg = NULL;
 	} while (index != le16_to_cpu(comp->comp_index));
 
-	if (pkts && bytes && !ionic_txq_hwstamp_enabled(q))
-		netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
+	(*total_pkts) += pkts;
+	(*total_bytes) += bytes;
 
 	return true;
 }
 
+unsigned int ionic_tx_cq_service(struct ionic_cq *cq,
+				 unsigned int work_to_do,
+				 bool in_napi)
+{
+	unsigned int work_done = 0;
+	unsigned int bytes = 0;
+	unsigned int pkts = 0;
+
+	if (work_to_do == 0)
+		return 0;
+
+	while (ionic_tx_service(cq, &pkts, &bytes, in_napi)) {
+		if (cq->tail_idx == cq->num_descs - 1)
+			cq->done_color = !cq->done_color;
+		cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
+
+		if (++work_done >= work_to_do)
+			break;
+	}
+
+	if (work_done) {
+		struct ionic_queue *q = cq->bound_q;
+
+		if (likely(!ionic_txq_hwstamp_enabled(q)))
+			netif_txq_completed_wake(q_to_ndq(q->lif->netdev, q),
+						 pkts, bytes,
+						 ionic_q_space_avail(q),
+						 IONIC_TSO_DESCS_NEEDED);
+	}
+
+	return work_done;
+}
+
 void ionic_tx_flush(struct ionic_cq *cq)
 {
-	struct ionic_dev *idev = &cq->lif->ionic->idev;
 	u32 work_done;
 
-	work_done = ionic_cq_service(cq, cq->num_descs,
-				     ionic_tx_service, NULL, NULL);
+	work_done = ionic_tx_cq_service(cq, cq->num_descs, false);
 	if (work_done)
-		ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
+		ionic_intr_credits(cq->idev->intr_ctrl, cq->bound_intr->index,
 				   work_done, IONIC_INTR_CRED_RESET_COALESCE);
 }
 
 void ionic_tx_empty(struct ionic_queue *q)
 {
-	struct ionic_desc_info *desc_info;
+	struct ionic_tx_desc_info *desc_info;
 	int bytes = 0;
 	int pkts = 0;
 
 	/* walk the not completed tx entries, if any */
 	while (q->head_idx != q->tail_idx) {
-		desc_info = &q->info[q->tail_idx];
+		desc_info = &q->tx_info[q->tail_idx];
 		desc_info->bytes = 0;
 		q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
-		ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
-		if (desc_info->cb_arg) {
+		ionic_tx_clean(q, desc_info, NULL, false);
+		if (desc_info->skb) {
 			pkts++;
 			bytes += desc_info->bytes;
+			desc_info->skb = NULL;
 		}
-		desc_info->cb = NULL;
-		desc_info->cb_arg = NULL;
 	}
 
-	if (pkts && bytes && !ionic_txq_hwstamp_enabled(q))
-		netdev_tx_completed_queue(q_to_ndq(q), pkts, bytes);
+	if (likely(!ionic_txq_hwstamp_enabled(q))) {
+		struct netdev_queue *ndq = q_to_ndq(q->lif->netdev, q);
+
+		netdev_tx_completed_queue(ndq, pkts, bytes);
+		netdev_tx_reset_queue(ndq);
+	}
 }
 
 static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
@@ -926,7 +1342,7 @@ static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
 	int err;
 
 	err = skb_cow_head(skb, 0);
-	if (err)
+	if (unlikely(err))
 		return err;
 
 	if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
@@ -950,7 +1366,7 @@ static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
 	int err;
 
 	err = skb_cow_head(skb, 0);
-	if (err)
+	if (unlikely(err))
 		return err;
 
 	if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
@@ -966,8 +1382,8 @@ static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
 	return 0;
 }
 
-static void ionic_tx_tso_post(struct ionic_queue *q,
-			      struct ionic_desc_info *desc_info,
+static void ionic_tx_tso_post(struct net_device *netdev, struct ionic_queue *q,
+			      struct ionic_txq_desc *desc,
 			      struct sk_buff *skb,
 			      dma_addr_t addr, u8 nsge, u16 len,
 			      unsigned int hdrlen, unsigned int mss,
@@ -975,7 +1391,6 @@ static void ionic_tx_tso_post(struct ionic_queue *q,
 			      u16 vlan_tci, bool has_vlan,
 			      bool start, bool done)
 {
-	struct ionic_txq_desc *desc = desc_info->desc;
 	u8 flags = 0;
 	u64 cmd;
 
@@ -991,22 +1406,23 @@ static void ionic_tx_tso_post(struct ionic_queue *q,
 	desc->hdr_len = cpu_to_le16(hdrlen);
 	desc->mss = cpu_to_le16(mss);
 
-	ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
+	ionic_write_cmb_desc(q, desc);
 
 	if (start) {
 		skb_tx_timestamp(skb);
-		if (!ionic_txq_hwstamp_enabled(q))
-			netdev_tx_sent_queue(q_to_ndq(q), skb->len);
-		ionic_txq_post(q, false, ionic_tx_clean, skb);
+		if (likely(!ionic_txq_hwstamp_enabled(q)))
+			netdev_tx_sent_queue(q_to_ndq(netdev, q), skb->len);
+		ionic_txq_post(q, false);
 	} else {
-		ionic_txq_post(q, done, NULL, NULL);
+		ionic_txq_post(q, done);
 	}
 }
 
-static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
+static int ionic_tx_tso(struct net_device *netdev, struct ionic_queue *q,
+			struct sk_buff *skb)
 {
 	struct ionic_tx_stats *stats = q_to_tx_stats(q);
-	struct ionic_desc_info *desc_info;
+	struct ionic_tx_desc_info *desc_info;
 	struct ionic_buf_info *buf_info;
 	struct ionic_txq_sg_elem *elem;
 	struct ionic_txq_desc *desc;
@@ -1028,8 +1444,7 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
 	bool encap;
 	int err;
 
-	desc_info = &q->info[q->head_idx];
-	buf_info = desc_info->bufs;
+	desc_info = &q->tx_info[q->head_idx];
 
 	if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
 		return -EIO;
@@ -1055,7 +1470,7 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
 		err = ionic_tx_tcp_inner_pseudo_csum(skb);
 	else
 		err = ionic_tx_tcp_pseudo_csum(skb);
-	if (err) {
+	if (unlikely(err)) {
 		/* clean up mapping from ionic_tx_map_skb */
 		ionic_tx_desc_unmap_bufs(q, desc_info);
 		return err;
@@ -1066,6 +1481,8 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
 	else
 		hdrlen = skb_tcp_all_headers(skb);
 
+	desc_info->skb = skb;
+	buf_info = desc_info->bufs;
 	tso_rem = len;
 	seg_rem = min(tso_rem, hdrlen + mss);
 
@@ -1092,8 +1509,8 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
 			chunk_len = min(frag_rem, seg_rem);
 			if (!desc) {
 				/* fill main descriptor */
-				desc = desc_info->txq_desc;
-				elem = desc_info->txq_sg_desc->elems;
+				desc = &q->txq[q->head_idx];
+				elem = ionic_tx_sg_elems(q);
 				desc_addr = frag_addr;
 				desc_len = chunk_len;
 			} else {
@@ -1111,13 +1528,12 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
 		seg_rem = min(tso_rem, mss);
 		done = (tso_rem == 0);
 		/* post descriptor */
-		ionic_tx_tso_post(q, desc_info, skb,
-				  desc_addr, desc_nsge, desc_len,
-				  hdrlen, mss, outer_csum, vlan_tci, has_vlan,
-				  start, done);
+		ionic_tx_tso_post(netdev, q, desc, skb, desc_addr, desc_nsge,
+				  desc_len, hdrlen, mss, outer_csum, vlan_tci,
+				  has_vlan, start, done);
 		start = false;
 		/* Buffer information is stored with the first tso descriptor */
-		desc_info = &q->info[q->head_idx];
+		desc_info = &q->tx_info[q->head_idx];
 		desc_info->nbufs = 0;
 	}
 
@@ -1130,9 +1546,9 @@ static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
 }
 
 static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
-			       struct ionic_desc_info *desc_info)
+			       struct ionic_tx_desc_info *desc_info)
 {
-	struct ionic_txq_desc *desc = desc_info->txq_desc;
+	struct ionic_txq_desc *desc = &q->txq[q->head_idx];
 	struct ionic_buf_info *buf_info = desc_info->bufs;
 	struct ionic_tx_stats *stats = q_to_tx_stats(q);
 	bool has_vlan;
@@ -1160,7 +1576,7 @@ static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
 	desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
 	desc->csum_offset = cpu_to_le16(skb->csum_offset);
 
-	ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
+	ionic_write_cmb_desc(q, desc);
 
 	if (skb_csum_is_sctp(skb))
 		stats->crc32_csum++;
@@ -1169,9 +1585,9 @@ static void ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb,
 }
 
 static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
-				  struct ionic_desc_info *desc_info)
+				  struct ionic_tx_desc_info *desc_info)
 {
-	struct ionic_txq_desc *desc = desc_info->txq_desc;
+	struct ionic_txq_desc *desc = &q->txq[q->head_idx];
 	struct ionic_buf_info *buf_info = desc_info->bufs;
 	struct ionic_tx_stats *stats = q_to_tx_stats(q);
 	bool has_vlan;
@@ -1199,20 +1615,20 @@ static void ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb,
 	desc->csum_start = 0;
 	desc->csum_offset = 0;
 
-	ionic_write_cmb_desc(q, desc_info->cmb_desc, desc);
+	ionic_write_cmb_desc(q, desc);
 
 	stats->csum_none++;
 }
 
 static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
-			       struct ionic_desc_info *desc_info)
+			       struct ionic_tx_desc_info *desc_info)
 {
-	struct ionic_txq_sg_desc *sg_desc = desc_info->txq_sg_desc;
 	struct ionic_buf_info *buf_info = &desc_info->bufs[1];
-	struct ionic_txq_sg_elem *elem = sg_desc->elems;
 	struct ionic_tx_stats *stats = q_to_tx_stats(q);
+	struct ionic_txq_sg_elem *elem;
 	unsigned int i;
 
+	elem = ionic_tx_sg_elems(q);
 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, buf_info++, elem++) {
 		elem->addr = cpu_to_le64(buf_info->dma_addr);
 		elem->len = cpu_to_le16(buf_info->len);
@@ -1221,14 +1637,18 @@ static void ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb,
 	stats->frags += skb_shinfo(skb)->nr_frags;
 }
 
-static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
+static int ionic_tx(struct net_device *netdev, struct ionic_queue *q,
+		    struct sk_buff *skb)
 {
-	struct ionic_desc_info *desc_info = &q->info[q->head_idx];
+	struct ionic_tx_desc_info *desc_info = &q->tx_info[q->head_idx];
 	struct ionic_tx_stats *stats = q_to_tx_stats(q);
+	bool ring_dbell = true;
 
 	if (unlikely(ionic_tx_map_skb(q, skb, desc_info)))
 		return -EIO;
 
+	desc_info->skb = skb;
+
 	/* set up the initial descriptor */
 	if (skb->ip_summed == CHECKSUM_PARTIAL)
 		ionic_tx_calc_csum(q, skb, desc_info);
@@ -1242,16 +1662,22 @@ static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
 	stats->pkts++;
 	stats->bytes += skb->len;
 
-	if (!ionic_txq_hwstamp_enabled(q))
-		netdev_tx_sent_queue(q_to_ndq(q), skb->len);
-	ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
+	if (likely(!ionic_txq_hwstamp_enabled(q))) {
+		struct netdev_queue *ndq = q_to_ndq(netdev, q);
+
+		if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1)))
+			netif_tx_stop_queue(ndq);
+		ring_dbell = __netdev_tx_sent_queue(ndq, skb->len,
+						    netdev_xmit_more());
+	}
+	ionic_txq_post(q, ring_dbell);
 
 	return 0;
 }
 
 static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
 {
-	struct ionic_tx_stats *stats = q_to_tx_stats(q);
+	int nr_frags = skb_shinfo(skb)->nr_frags;
 	bool too_many_frags = false;
 	skb_frag_t *frag;
 	int desc_bufs;
@@ -1267,17 +1693,20 @@ static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
 	/* Each desc is mss long max, so a descriptor for each gso_seg */
 	if (skb_is_gso(skb)) {
 		ndescs = skb_shinfo(skb)->gso_segs;
+		if (!nr_frags)
+			return ndescs;
 	} else {
 		ndescs = 1;
-		if (skb_shinfo(skb)->nr_frags > q->max_sg_elems) {
+		if (!nr_frags)
+			return ndescs;
+
+		if (unlikely(nr_frags > q->max_sg_elems)) {
 			too_many_frags = true;
 			goto linearize;
 		}
-	}
 
-	/* If non-TSO, or no frags to check, we're done */
-	if (!skb_is_gso(skb) || !skb_shinfo(skb)->nr_frags)
 		return ndescs;
+	}
 
 	/* We need to scan the skb to be sure that none of the MTU sized
 	 * packets in the TSO will require more sgs per descriptor than we
@@ -1326,38 +1755,19 @@ static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
 linearize:
 	if (too_many_frags) {
 		err = skb_linearize(skb);
-		if (err)
+		if (unlikely(err))
 			return err;
-		stats->linearize++;
+		q_to_tx_stats(q)->linearize++;
 	}
 
 	return ndescs;
 }
 
-static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
-{
-	int stopped = 0;
-
-	if (unlikely(!ionic_q_has_space(q, ndescs))) {
-		netif_stop_subqueue(q->lif->netdev, q->index);
-		stopped = 1;
-
-		/* Might race with ionic_tx_clean, check again */
-		smp_rmb();
-		if (ionic_q_has_space(q, ndescs)) {
-			netif_wake_subqueue(q->lif->netdev, q->index);
-			stopped = 0;
-		}
-	}
-
-	return stopped;
-}
-
 static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
 					    struct net_device *netdev)
 {
 	struct ionic_lif *lif = netdev_priv(netdev);
-	struct ionic_queue *q = &lif->hwstamp_txq->q;
+	struct ionic_queue *q;
 	int err, ndescs;
 
 	/* Does not stop/start txq, because we post to a separate tx queue
@@ -1365,6 +1775,7 @@ static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
 	 * the timestamping queue, it is dropped.
 	 */
 
+	q = &lif->hwstamp_txq->q;
 	ndescs = ionic_tx_descs_needed(q, skb);
 	if (unlikely(ndescs < 0))
 		goto err_out_drop;
@@ -1374,11 +1785,11 @@ static netdev_tx_t ionic_start_hwstamp_xmit(struct sk_buff *skb,
 
 	skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP;
 	if (skb_is_gso(skb))
-		err = ionic_tx_tso(q, skb);
+		err = ionic_tx_tso(netdev, q, skb);
 	else
-		err = ionic_tx(q, skb);
+		err = ionic_tx(netdev, q, skb);
 
-	if (err)
+	if (unlikely(err))
 		goto err_out_drop;
 
 	return NETDEV_TX_OK;
@@ -1414,23 +1825,19 @@ netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 	if (ndescs < 0)
 		goto err_out_drop;
 
-	if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
+	if (!netif_txq_maybe_stop(q_to_ndq(netdev, q),
+				  ionic_q_space_avail(q),
+				  ndescs, ndescs))
 		return NETDEV_TX_BUSY;
 
 	if (skb_is_gso(skb))
-		err = ionic_tx_tso(q, skb);
+		err = ionic_tx_tso(netdev, q, skb);
 	else
-		err = ionic_tx(q, skb);
+		err = ionic_tx(netdev, q, skb);
 
-	if (err)
+	if (unlikely(err))
 		goto err_out_drop;
 
-	/* Stop the queue if there aren't descriptors for the next packet.
-	 * Since our SG lists per descriptor take care of most of the possible
-	 * fragmentation, we don't need to have many descriptors available.
-	 */
-	ionic_maybe_stop_tx(q, 4);
-
 	return NETDEV_TX_OK;
 
 err_out_drop: