diff options
Diffstat (limited to 'drivers/net/ethernet/pensando/ionic/ionic_txrx.c')
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 136 |
1 files changed, 133 insertions, 3 deletions
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index a4f6dd5030c0..3d4cadcf1c0c 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -10,6 +10,16 @@ #include "ionic_lif.h" #include "ionic_txrx.h" +static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs); + +static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, + void *data, size_t len); + +static void ionic_tx_clean(struct ionic_queue *q, + struct ionic_desc_info *desc_info, + struct ionic_cq_info *cq_info, + void *cb_arg); + static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell, ionic_desc_cb cb_func, void *cb_arg) { @@ -297,6 +307,75 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q, return skb; } +static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, + struct ionic_desc_info *desc_info) +{ + unsigned int nbufs = desc_info->nbufs; + struct ionic_buf_info *buf_info; + struct device *dev = q->dev; + + if (!nbufs) + return; + + buf_info = desc_info->bufs; + dma_unmap_single(dev, buf_info->dma_addr, + buf_info->len, DMA_TO_DEVICE); + __free_pages(buf_info->page, 0); + buf_info->page = NULL; + + desc_info->nbufs = 0; + desc_info->xdpf = NULL; + desc_info->act = 0; +} + +static int ionic_xdp_post_frame(struct net_device *netdev, + struct ionic_queue *q, struct xdp_frame *frame, + enum xdp_action act, struct page *page, int off, + bool ring_doorbell) +{ + struct ionic_desc_info *desc_info; + struct ionic_buf_info *buf_info; + struct ionic_tx_stats *stats; + struct ionic_txq_desc *desc; + size_t len = frame->len; + dma_addr_t dma_addr; + u64 cmd; + + desc_info = &q->info[q->head_idx]; + desc = desc_info->txq_desc; + buf_info = desc_info->bufs; + stats = q_to_tx_stats(q); + + dma_addr = ionic_tx_map_single(q, frame->data, len); + if (dma_mapping_error(q->dev, dma_addr)) { + stats->dma_map_err++; + return -EIO; + } + buf_info->dma_addr = dma_addr; + buf_info->len = len; + buf_info->page = page; + buf_info->page_offset = off; + + desc_info->nbufs = 1; + desc_info->xdpf = frame; + desc_info->act = act; + + cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE, + 0, 0, buf_info->dma_addr); + desc->cmd = cpu_to_le64(cmd); + desc->len = cpu_to_le16(len); + desc->csum_start = 0; + desc->csum_offset = 0; + + stats->xdp_frames++; + stats->pkts++; + stats->bytes += len; + + ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL); + + return 0; +} + static bool ionic_run_xdp(struct ionic_rx_stats *stats, struct net_device *netdev, struct bpf_prog *xdp_prog, @@ -306,6 +385,10 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, { u32 xdp_action = XDP_ABORTED; struct xdp_buff xdp_buf; + struct ionic_queue *txq; + struct netdev_queue *nq; + struct xdp_frame *xdpf; + int err = 0; xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info); xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), @@ -330,15 +413,52 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, break; case XDP_TX: + xdpf = xdp_convert_buff_to_frame(&xdp_buf); + if (!xdpf) + goto out_xdp_abort; + + txq = rxq->partner; + nq = netdev_get_tx_queue(netdev, txq->index); + __netif_tx_lock(nq, smp_processor_id()); + txq_trans_cond_update(nq); + + if (netif_tx_queue_stopped(nq) || + unlikely(ionic_maybe_stop_tx(txq, 1))) { + __netif_tx_unlock(nq); + goto out_xdp_abort; + } + + dma_unmap_page(rxq->dev, buf_info->dma_addr, + IONIC_PAGE_SIZE, DMA_FROM_DEVICE); + + err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX, + buf_info->page, + buf_info->page_offset, + true); + __netif_tx_unlock(nq); + if (err) { + netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); + goto out_xdp_abort; + } + stats->xdp_tx++; + + /* the Tx completion will free the buffers */ + break; + case XDP_REDIRECT: case XDP_ABORTED: default: - trace_xdp_exception(netdev, xdp_prog, xdp_action); - ionic_rx_page_free(rxq, buf_info); - stats->xdp_aborted++; + goto out_xdp_abort; } return true; + +out_xdp_abort: + trace_xdp_exception(netdev, xdp_prog, xdp_action); + ionic_rx_page_free(rxq, buf_info); + stats->xdp_aborted++; + + return true; } static void ionic_rx_clean(struct ionic_queue *q, @@ -893,6 +1013,16 @@ static void ionic_tx_clean(struct ionic_queue *q, struct sk_buff *skb = cb_arg; u16 qi; + if (desc_info->xdpf) { + ionic_xdp_tx_desc_clean(q->partner, desc_info); + stats->clean++; + + if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) + netif_wake_subqueue(q->lif->netdev, q->index); + + return; + } + ionic_tx_desc_unmap_bufs(q, desc_info); if (!skb) |