net:fec: implement almost zero-copy receive path

iMX6 SoCs have an IP accelerator which can pad and strip two bytes in
the FIFO at the beginning of each packet.  Since the FEC has alignment
restrictions on the buffer addresses, this feature is particularly
interesting for receive as it allows us to receive an appropriately
aligned packet for IP.

This allows us to reduce the CPU overhead by avoiding copying large
packets - and in the spirit of all network drivers which use this trick,
we provide a copybreak tunable which allows the packet size to be
copied to be adjusted.

Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

1 files changed, 149 insertions, 46 deletions

diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index 496e60228521..45ceac59ae61 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -109,6 +109,8 @@ static void set_multicast_list(struct net_device *ndev);
 #define FEC_QUIRK_HAS_CSUM		(1 << 5)
 /* Controller has hardware vlan support */
 #define FEC_QUIRK_HAS_VLAN		(1 << 6)
+/* Controller has ability to offset rx packets */
+#define FEC_QUIRK_RX_SHIFT16            (1 << 8)
 
 static struct platform_device_id fec_devtype[] = {
 	{
@@ -128,7 +130,7 @@ static struct platform_device_id fec_devtype[] = {
 		.name = "imx6q-fec",
 		.driver_data = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT |
 				FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM |
-				FEC_QUIRK_HAS_VLAN,
+				FEC_QUIRK_HAS_VLAN | FEC_QUIRK_RX_SHIFT16,
 	}, {
 		.name = "mvf600-fec",
 		.driver_data = FEC_QUIRK_ENET_MAC,
@@ -207,6 +209,7 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address");
 /* FEC receive acceleration */
 #define FEC_RACC_IPDIS		(1 << 1)
 #define FEC_RACC_PRODIS		(1 << 2)
+#define FEC_RACC_SHIFT16	BIT(7)
 #define FEC_RACC_OPTIONS	(FEC_RACC_IPDIS | FEC_RACC_PRODIS)
 
 /*
@@ -247,6 +250,20 @@ MODULE_PARM_DESC(macaddr, "FEC Ethernet MAC address");
 
 static int mii_cnt;
 
+static unsigned copybreak = 200;
+module_param(copybreak, uint, 0644);
+MODULE_PARM_DESC(copybreak,
+		 "Maximum size of packet that is copied to a new buffer on receive");
+
+static bool fec_enet_rx_zerocopy(struct fec_enet_private *fep, unsigned pktlen)
+{
+#ifndef CONFIG_M5272
+	if (fep->quirks & FEC_QUIRK_RX_SHIFT16 && pktlen >= copybreak)
+		return true;
+#endif
+	return false;
+}
+
 static union bufdesc_u *
 fec_enet_tx_get(unsigned int index, struct fec_enet_private *fep)
 {
@@ -669,6 +686,8 @@ fec_restart(struct net_device *ndev)
 #if !defined(CONFIG_M5272)
 	/* set RX checksum */
 	val = readl(fep->hwp + FEC_RACC);
+	if (fep->quirks & FEC_QUIRK_RX_SHIFT16)
+		val |= FEC_RACC_SHIFT16;
 	if (fep->flags & FEC_FLAG_RX_CSUM)
 		val |= FEC_RACC_OPTIONS;
 	else
@@ -987,6 +1006,68 @@ fec_enet_receive(struct sk_buff *skb, union bufdesc_u *bdp, struct net_device *n
 	napi_gro_receive(&fep->napi, skb);
 }
 
+static void
+fec_enet_receive_nocopy(unsigned pkt_len, unsigned index, union bufdesc_u *bdp,
+	struct net_device *ndev)
+{
+	struct fec_enet_private *fep = netdev_priv(ndev);
+	struct sk_buff *skb, *skb_new;
+	unsigned char *data;
+	dma_addr_t addr;
+
+	skb_new = netdev_alloc_skb(ndev, FEC_ENET_RX_FRSIZE);
+	if (!skb_new) {
+		ndev->stats.rx_dropped++;
+		return;
+	}
+
+	addr = dma_map_single(&fep->pdev->dev, skb_new->data,
+			      FEC_ENET_RX_FRSIZE, DMA_FROM_DEVICE);
+	if (dma_mapping_error(&fep->pdev->dev, addr)) {
+		dev_kfree_skb(skb_new);
+		ndev->stats.rx_dropped++;
+		return;
+	}
+
+	/* We have the new skb, so proceed to deal with the received data. */
+	dma_unmap_single(&fep->pdev->dev, bdp->bd.cbd_bufaddr,
+			 FEC_ENET_RX_FRSIZE, DMA_FROM_DEVICE);
+
+	skb = fep->rx_skbuff[index];
+
+	/* Now subsitute in the new skb */
+	fep->rx_skbuff[index] = skb_new;
+	bdp->bd.cbd_bufaddr = addr;
+
+	/*
+	 * Update the skb length according to the raw packet length.
+	 * Then remove the two bytes of additional padding.
+	 */
+	skb_put(skb, pkt_len);
+	data = skb_pull_inline(skb, 2);
+
+	if (fep->quirks & FEC_QUIRK_SWAP_FRAME)
+		swap_buffer(data, skb->len);
+
+	/*
+	 * Now juggle things for the VLAN tag - if the hardware
+	 * flags this as present, we need to read the tag, and
+	 * then shuffle the ethernet addresses up.
+	 */
+	if (ndev->features & NETIF_F_HW_VLAN_CTAG_RX &&
+	    bdp->ebd.cbd_esc & BD_ENET_RX_VLAN) {
+		struct vlan_hdr *vlan = (struct vlan_hdr *)(data + ETH_HLEN);
+
+		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
+				       ntohs(vlan->h_vlan_TCI));
+
+		memmove(data + VLAN_HLEN, data, 2 * ETH_ALEN);
+		skb_pull_inline(skb, VLAN_HLEN);
+	}
+
+	fec_enet_receive(skb, bdp, ndev);
+}
+
 /* During a receive, the rx_next points to the current incoming buffer.
  * When we update through the ring, if the next incoming buffer has
  * not been given to the system, we just set the empty indicator,
@@ -996,7 +1077,6 @@ static int
 fec_enet_rx(struct net_device *ndev, int budget)
 {
 	struct fec_enet_private *fep = netdev_priv(ndev);
-	struct	sk_buff	*skb;
 	ushort	pkt_len;
 	__u8 *data;
 	int	pkt_received = 0;
@@ -1012,6 +1092,7 @@ fec_enet_rx(struct net_device *ndev, int budget)
 	do {
 		union bufdesc_u *bdp = fec_enet_rx_get(index, fep);
 		unsigned status, cbd_esc;
+		struct sk_buff *skb;
 
 		status = bdp->bd.cbd_sc;
 		if (status & BD_ENET_RX_EMPTY)
@@ -1074,62 +1155,84 @@ fec_enet_rx(struct net_device *ndev, int budget)
 		pkt_len = bdp->bd.cbd_datlen - 4;
 		ndev->stats.rx_bytes += pkt_len;
 
-		if (fep->flags & FEC_FLAG_BUFDESC_EX) {
-			cbd_esc = bdp->ebd.cbd_esc;
-			if (!(fep->flags & FEC_FLAG_RX_VLAN))
-				cbd_esc &= ~BD_ENET_RX_VLAN;
+		if (fec_enet_rx_zerocopy(fep, pkt_len)) {
+			fec_enet_receive_nocopy(pkt_len, index, bdp, ndev);
 		} else {
-			cbd_esc = 0;
-		}
+			if (fep->flags & FEC_FLAG_BUFDESC_EX) {
+				cbd_esc = bdp->ebd.cbd_esc;
+				if (!(fep->flags & FEC_FLAG_RX_VLAN))
+					cbd_esc &= ~BD_ENET_RX_VLAN;
+			} else {
+				cbd_esc = 0;
+			}
 
-		/*
-		 * Detect the presence of the VLANG tag, and just
-		 * the packet length appropriately.
-		 */
-		if (cbd_esc & BD_ENET_RX_VLAN)
-			pkt_len -= VLAN_HLEN;
+			/*
+			 * Detect the presence of the VLANG tag, and just
+			 * the packet length appropriately.
+			 */
+			if (cbd_esc & BD_ENET_RX_VLAN)
+				pkt_len -= VLAN_HLEN;
 
-		/* This does 16 byte alignment, exactly what we need. */
-		skb = netdev_alloc_skb(ndev, pkt_len + NET_IP_ALIGN);
-		if (unlikely(!skb)) {
-			ndev->stats.rx_dropped++;
-			goto rx_processing_done;
-		}
+			/*
+			 * This does 16 byte alignment, exactly what we need.
+			 */
+			skb = netdev_alloc_skb(ndev, pkt_len + NET_IP_ALIGN);
+			if (unlikely(!skb)) {
+				ndev->stats.rx_dropped++;
+				goto rx_processing_done;
+			}
 
-		data = fep->rx_skbuff[index]->data;
-		dma_sync_single_for_cpu(&fep->pdev->dev, bdp->bd.cbd_bufaddr,
-					FEC_ENET_RX_FRSIZE, DMA_FROM_DEVICE);
+			dma_sync_single_for_cpu(&fep->pdev->dev,
+					        bdp->bd.cbd_bufaddr,
+						FEC_ENET_RX_FRSIZE,
+						DMA_FROM_DEVICE);
 
-		if (fep->quirks & FEC_QUIRK_SWAP_FRAME)
-			swap_buffer(data, pkt_len);
+			data = fep->rx_skbuff[index]->data;
 
-		skb_reserve(skb, NET_IP_ALIGN);
-		skb_put(skb, pkt_len);	/* Make room */
+#ifndef CONFIG_M5272
+			/*
+			 * If we have enabled this feature, we need to discard
+			 * the two bytes at the beginning of the packet before
+			 * copying it.
+			 */
+			if (fep->quirks & FEC_QUIRK_RX_SHIFT16) {
+				pkt_len -= 2;
+				data += 2;
+			}
+#endif
 
-		/* If this is a VLAN packet remove the VLAN Tag */
-		if (cbd_esc & BD_ENET_RX_VLAN) {
-			/* Push and remove the vlan tag */
-			struct vlan_hdr *vlan =
+			if (fep->quirks & FEC_QUIRK_SWAP_FRAME)
+				swap_buffer(data, pkt_len);
+
+			skb_reserve(skb, NET_IP_ALIGN);
+			skb_put(skb, pkt_len);	/* Make room */
+
+			/* If this is a VLAN packet remove the VLAN Tag */
+			if (cbd_esc & BD_ENET_RX_VLAN) {
+				/* Push and remove the vlan tag */
+				struct vlan_hdr *vlan =
 					(struct vlan_hdr *) (data + ETH_HLEN);
 
-			__vlan_hwaccel_put_tag(skb,
-					       htons(ETH_P_8021Q),
-					       ntohs(vlan->h_vlan_TCI));
+				__vlan_hwaccel_put_tag(skb,
+						       htons(ETH_P_8021Q),
+						       ntohs(vlan->h_vlan_TCI));
 
-			/* Extract the frame data without the VLAN header. */
-			skb_copy_to_linear_data(skb, data, 2 * ETH_ALEN);
-			skb_copy_to_linear_data_offset(skb, 2 * ETH_ALEN,
-						       data + 2 * ETH_ALEN +
-						        VLAN_HLEN,
-						       pkt_len - 2 * ETH_ALEN);
-		} else {
-			skb_copy_to_linear_data(skb, data, pkt_len);
-		}
+				/* Extract the frame data without the VLAN header. */
+				skb_copy_to_linear_data(skb, data, 2 * ETH_ALEN);
+				skb_copy_to_linear_data_offset(skb, 2 * ETH_ALEN,
+							       data + 2 * ETH_ALEN + VLAN_HLEN,
+							       pkt_len - 2 * ETH_ALEN);
+			} else {
+				skb_copy_to_linear_data(skb, data, pkt_len);
+			}
 
-		dma_sync_single_for_device(&fep->pdev->dev, bdp->bd.cbd_bufaddr,
-					   FEC_ENET_RX_FRSIZE, DMA_FROM_DEVICE);
+			dma_sync_single_for_device(&fep->pdev->dev,
+						   bdp->bd.cbd_bufaddr,
+						   FEC_ENET_RX_FRSIZE,
+						   DMA_FROM_DEVICE);
 
-		fec_enet_receive(skb, bdp, ndev);
+			fec_enet_receive(skb, bdp, ndev);
+		}
 
 rx_processing_done:
 		if (fep->flags & FEC_FLAG_BUFDESC_EX) {