summaryrefslogtreecommitdiff
path: root/drivers/net/ethernet/fungible/funeth/funeth_txrx.h
blob: 5eec552a1f24d9a0b34186730cf70b2c4316920d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */

#ifndef _FUNETH_TXRX_H
#define _FUNETH_TXRX_H

#include <linux/netdevice.h>
#include <linux/u64_stats_sync.h>
#include <net/xdp.h>

/* Tx descriptor size */
#define FUNETH_SQE_SIZE 64U

/* Size of device headers per Tx packet */
#define FUNETH_FUNOS_HDR_SZ (sizeof(struct fun_eth_tx_req))

/* Number of gather list entries per Tx descriptor */
#define FUNETH_GLE_PER_DESC (FUNETH_SQE_SIZE / sizeof(struct fun_dataop_gl))

/* Max gather list size in bytes for an sk_buff. */
#define FUNETH_MAX_GL_SZ ((MAX_SKB_FRAGS + 1) * sizeof(struct fun_dataop_gl))

#if IS_ENABLED(CONFIG_TLS_DEVICE)
# define FUNETH_TLS_SZ sizeof(struct fun_eth_tls)
#else
# define FUNETH_TLS_SZ 0
#endif

/* Max number of Tx descriptors for an sk_buff using a gather list. */
#define FUNETH_MAX_GL_DESC \
	DIV_ROUND_UP((FUNETH_FUNOS_HDR_SZ + FUNETH_MAX_GL_SZ + FUNETH_TLS_SZ), \
		     FUNETH_SQE_SIZE)

/* Max number of Tx descriptors for any packet. */
#define FUNETH_MAX_PKT_DESC FUNETH_MAX_GL_DESC

/* Rx CQ descriptor size. */
#define FUNETH_CQE_SIZE 64U

/* Offset of cqe_info within a CQE. */
#define FUNETH_CQE_INFO_OFFSET (FUNETH_CQE_SIZE - sizeof(struct fun_cqe_info))

/* Construct the IRQ portion of a CQ doorbell. The resulting value arms the
 * interrupt with the supplied time delay and packet count moderation settings.
 */
#define FUN_IRQ_CQ_DB(usec, pkts) \
	(FUN_DB_IRQ_ARM_F | ((usec) << FUN_DB_INTCOAL_USEC_S) | \
	 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))

/* As above for SQ doorbells. */
#define FUN_IRQ_SQ_DB(usec, pkts) \
	(FUN_DB_IRQ_ARM_F | \
	 ((usec) << FUN_DB_INTCOAL_USEC_S) | \
	 ((pkts) << FUN_DB_INTCOAL_ENTRIES_S))

/* Per packet tailroom. Present only for 1-frag packets. */
#define FUN_RX_TAILROOM SKB_DATA_ALIGN(sizeof(struct skb_shared_info))

/* Per packet headroom for XDP. Preferred over XDP_PACKET_HEADROOM to
 * accommodate two packets per buffer for 4K pages and 1500B MTUs.
 */
#define FUN_XDP_HEADROOM 192

/* Initialization state of a queue. */
enum {
	FUN_QSTATE_DESTROYED, /* what queue? */
	FUN_QSTATE_INIT_SW,   /* exists in SW, not on the device */
	FUN_QSTATE_INIT_FULL, /* exists both in SW and on device */
};

/* Initialization state of an interrupt. */
enum {
	FUN_IRQ_INIT,      /* initialized and in the XArray but inactive */
	FUN_IRQ_REQUESTED, /* request_irq() done */
	FUN_IRQ_ENABLED,   /* processing enabled */
	FUN_IRQ_DISABLED,  /* processing disabled */
};

struct bpf_prog;

struct funeth_txq_stats {  /* per Tx queue SW counters */
	u64 tx_pkts;       /* # of Tx packets */
	u64 tx_bytes;      /* total bytes of Tx packets */
	u64 tx_cso;        /* # of packets with checksum offload */
	u64 tx_tso;        /* # of non-encapsulated TSO super-packets */
	u64 tx_encap_tso;  /* # of encapsulated TSO super-packets */
	u64 tx_uso;        /* # of non-encapsulated UDP LSO super-packets */
	u64 tx_more;       /* # of DBs elided due to xmit_more */
	u64 tx_nstops;     /* # of times the queue has stopped */
	u64 tx_nrestarts;  /* # of times the queue has restarted */
	u64 tx_map_err;    /* # of packets dropped due to DMA mapping errors */
	u64 tx_xdp_full;   /* # of XDP packets that could not be enqueued */
	u64 tx_tls_pkts;   /* # of Tx TLS packets offloaded to HW */
	u64 tx_tls_bytes;  /* Tx bytes of HW-handled TLS payload */
	u64 tx_tls_fallback; /* attempted Tx TLS offloads punted to SW */
	u64 tx_tls_drops;  /* attempted Tx TLS offloads dropped */
};

struct funeth_tx_info {      /* per Tx descriptor state */
	union {
		struct sk_buff *skb;    /* associated packet (sk_buff path) */
		struct xdp_frame *xdpf; /* associated XDP frame (XDP path) */
	};
};

struct funeth_txq {
	/* RO cacheline of frequently accessed data */
	u32 mask;               /* queue depth - 1 */
	u32 hw_qid;             /* device ID of the queue */
	void *desc;             /* base address of descriptor ring */
	struct funeth_tx_info *info;
	struct device *dma_dev; /* device for DMA mappings */
	volatile __be64 *hw_wb; /* HW write-back location */
	u32 __iomem *db;        /* SQ doorbell register address */
	struct netdev_queue *ndq;
	dma_addr_t dma_addr;    /* DMA address of descriptor ring */
	/* producer R/W cacheline */
	u16 qidx;               /* queue index within net_device */
	u16 ethid;
	u32 prod_cnt;           /* producer counter */
	struct funeth_txq_stats stats;
	/* shared R/W cacheline, primarily accessed by consumer */
	u32 irq_db_val;         /* value written to IRQ doorbell */
	u32 cons_cnt;           /* consumer (cleanup) counter */
	struct net_device *netdev;
	struct fun_irq *irq;
	int numa_node;
	u8 init_state;          /* queue initialization state */
	struct u64_stats_sync syncp;
};

struct funeth_rxq_stats {  /* per Rx queue SW counters */
	u64 rx_pkts;       /* # of received packets, including SW drops */
	u64 rx_bytes;      /* total size of received packets */
	u64 rx_cso;        /* # of packets with checksum offload */
	u64 rx_bufs;       /* total # of Rx buffers provided to device */
	u64 gro_pkts;      /* # of GRO superpackets */
	u64 gro_merged;    /* # of pkts merged into existing GRO superpackets */
	u64 rx_page_alloc; /* # of page allocations for Rx buffers */
	u64 rx_budget;     /* NAPI iterations that exhausted their budget */
	u64 rx_mem_drops;  /* # of packets dropped due to memory shortage */
	u64 rx_map_err;    /* # of page DMA mapping errors */
	u64 xdp_drops;     /* XDP_DROPped packets */
	u64 xdp_tx;        /* successful XDP transmits */
	u64 xdp_redir;     /* successful XDP redirects */
	u64 xdp_err;       /* packets dropped due to XDP errors */
};

struct funeth_rxbuf {          /* per Rx buffer state */
	struct page *page;     /* associated page */
	dma_addr_t dma_addr;   /* DMA address of page start */
	int pg_refs;           /* page refs held by driver */
	int node;              /* page node, or -1 if it is PF_MEMALLOC */
};

struct funeth_rx_cache {       /* cache of DMA-mapped previously used buffers */
	struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */
	unsigned int prod_cnt;     /* producer counter */
	unsigned int cons_cnt;     /* consumer counter */
	unsigned int mask;         /* depth - 1 */
};

/* An Rx queue consists of a CQ and an SQ used to provide Rx buffers. */
struct funeth_rxq {
	struct net_device *netdev;
	struct napi_struct *napi;
	struct device *dma_dev;    /* device for DMA mappings */
	void *cqes;                /* base of CQ descriptor ring */
	const void *next_cqe_info; /* fun_cqe_info of next CQE */
	u32 __iomem *cq_db;        /* CQ doorbell register address */
	unsigned int cq_head;      /* CQ head index */
	unsigned int cq_mask;      /* CQ depth - 1 */
	u16 phase;                 /* CQ phase tag */
	u16 qidx;                  /* queue index within net_device */
	unsigned int irq_db_val;   /* IRQ info for CQ doorbell */
	struct fun_eprq_rqbuf *rqes; /* base of RQ descriptor ring */
	struct funeth_rxbuf *bufs; /* base of Rx buffer state ring */
	struct funeth_rxbuf *cur_buf; /* currently active buffer */
	u32 __iomem *rq_db;        /* RQ doorbell register address */
	unsigned int rq_cons;      /* RQ consumer counter */
	unsigned int rq_mask;      /* RQ depth - 1 */
	unsigned int buf_offset;   /* offset of next pkt in head buffer */
	u8 xdp_flush;              /* XDP flush types needed at NAPI end */
	u8 init_state;             /* queue initialization state */
	u16 headroom;              /* per packet headroom */
	unsigned int rq_cons_db;   /* value of rq_cons at last RQ db */
	unsigned int rq_db_thres;  /* # of new buffers needed to write RQ db */
	struct funeth_rxbuf spare_buf; /* spare for next buffer replacement */
	struct funeth_rx_cache cache; /* used buffer cache */
	struct bpf_prog *xdp_prog; /* optional XDP BPF program */
	struct funeth_rxq_stats stats;
	dma_addr_t cq_dma_addr;    /* DMA address of CQE ring */
	dma_addr_t rq_dma_addr;    /* DMA address of RQE ring */
	u16 irq_cnt;
	u32 hw_cqid;               /* device ID of the queue's CQ */
	u32 hw_sqid;               /* device ID of the queue's SQ */
	int numa_node;
	struct u64_stats_sync syncp;
	struct xdp_rxq_info xdp_rxq;
};

#define FUN_QSTAT_INC(q, counter) \
	do { \
		u64_stats_update_begin(&(q)->syncp); \
		(q)->stats.counter++; \
		u64_stats_update_end(&(q)->syncp); \
	} while (0)

#define FUN_QSTAT_READ(q, seq, stats_copy) \
	do { \
		seq = u64_stats_fetch_begin(&(q)->syncp); \
		stats_copy = (q)->stats; \
	} while (u64_stats_fetch_retry(&(q)->syncp, (seq)))

#define FUN_INT_NAME_LEN (IFNAMSIZ + 16)

struct fun_irq {
	struct napi_struct napi;
	struct funeth_txq *txq;
	struct funeth_rxq *rxq;
	u8 state;
	u16 irq_idx;              /* index of MSI-X interrupt */
	int irq;                  /* Linux IRQ vector */
	cpumask_t affinity_mask;  /* IRQ affinity */
	struct irq_affinity_notify aff_notify;
	char name[FUN_INT_NAME_LEN];
} ____cacheline_internodealigned_in_smp;

/* Return the start address of the idx-th Tx descriptor. */
static inline void *fun_tx_desc_addr(const struct funeth_txq *q,
				     unsigned int idx)
{
	return q->desc + idx * FUNETH_SQE_SIZE;
}

static inline void fun_txq_wr_db(const struct funeth_txq *q)
{
	unsigned int tail = q->prod_cnt & q->mask;

	writel(tail, q->db);
}

static inline int fun_irq_node(const struct fun_irq *p)
{
	return cpu_to_mem(cpumask_first(&p->affinity_mask));
}

int fun_rxq_napi_poll(struct napi_struct *napi, int budget);
int fun_txq_napi_poll(struct napi_struct *napi, int budget);
netdev_tx_t fun_start_xmit(struct sk_buff *skb, struct net_device *netdev);
bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf);
int fun_xdp_xmit_frames(struct net_device *dev, int n,
			struct xdp_frame **frames, u32 flags);

int funeth_txq_create(struct net_device *dev, unsigned int qidx,
		      unsigned int ndesc, struct fun_irq *irq, int state,
		      struct funeth_txq **qp);
int fun_txq_create_dev(struct funeth_txq *q, struct fun_irq *irq);
struct funeth_txq *funeth_txq_free(struct funeth_txq *q, int state);
int funeth_rxq_create(struct net_device *dev, unsigned int qidx,
		      unsigned int ncqe, unsigned int nrqe, struct fun_irq *irq,
		      int state, struct funeth_rxq **qp);
int fun_rxq_create_dev(struct funeth_rxq *q, struct fun_irq *irq);
struct funeth_rxq *funeth_rxq_free(struct funeth_rxq *q, int state);
int fun_rxq_set_bpf(struct funeth_rxq *q, struct bpf_prog *prog);

#endif /* _FUNETH_TXRX_H */