diff options
Diffstat (limited to 'drivers/net/ethernet/meta')
21 files changed, 1603 insertions, 12 deletions
diff --git a/drivers/net/ethernet/meta/Kconfig b/drivers/net/ethernet/meta/Kconfig index 85519690b837..831921b9d4d5 100644 --- a/drivers/net/ethernet/meta/Kconfig +++ b/drivers/net/ethernet/meta/Kconfig @@ -23,6 +23,7 @@ config FBNIC depends on !S390 depends on MAX_SKB_FRAGS < 22 depends on PCI_MSI + depends on PTP_1588_CLOCK_OPTIONAL select NET_DEVLINK select PAGE_POOL select PHYLINK diff --git a/drivers/net/ethernet/meta/fbnic/Makefile b/drivers/net/ethernet/meta/fbnic/Makefile index ed4533a73c57..239b2258ec65 100644 --- a/drivers/net/ethernet/meta/fbnic/Makefile +++ b/drivers/net/ethernet/meta/fbnic/Makefile @@ -7,10 +7,13 @@ obj-$(CONFIG_FBNIC) += fbnic.o -fbnic-y := fbnic_devlink.o \ +fbnic-y := fbnic_csr.o \ + fbnic_debugfs.o \ + fbnic_devlink.o \ fbnic_ethtool.o \ fbnic_fw.o \ fbnic_hw_stats.o \ + fbnic_hwmon.o \ fbnic_irq.o \ fbnic_mac.o \ fbnic_netdev.o \ @@ -18,4 +21,5 @@ fbnic-y := fbnic_devlink.o \ fbnic_phylink.o \ fbnic_rpc.o \ fbnic_tlv.o \ - fbnic_txrx.o + fbnic_txrx.o \ + fbnic_time.o diff --git a/drivers/net/ethernet/meta/fbnic/fbnic.h b/drivers/net/ethernet/meta/fbnic/fbnic.h index 0f9e8d79461c..706ae6104c8e 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic.h @@ -6,6 +6,7 @@ #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/ptp_clock_kernel.h> #include <linux/types.h> #include <linux/workqueue.h> @@ -18,6 +19,8 @@ struct fbnic_dev { struct device *dev; struct net_device *netdev; + struct dentry *dbg_fbd; + struct device *hwmon; u32 __iomem *uc_addr0; u32 __iomem *uc_addr4; @@ -30,6 +33,7 @@ struct fbnic_dev { struct fbnic_fw_mbx mbx[FBNIC_IPC_MBX_INDICES]; struct fbnic_fw_cap fw_cap; + struct fbnic_fw_completion *cmpl_data; /* Lock protecting Tx Mailbox queue to prevent possible races */ spinlock_t fw_tx_lock; @@ -45,10 +49,21 @@ struct fbnic_dev { struct fbnic_act_tcam act_tcam[FBNIC_RPC_TCAM_ACT_NUM_ENTRIES]; struct fbnic_mac_addr mac_addr[FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES]; u8 mac_addr_boundary; + u8 tce_tcam_last; /* Number of TCQs/RCQs available on hardware */ u16 max_num_queues; + /* Lock protecting writes to @time_high, @time_offset of fbnic_netdev, + * and the HW time CSR machinery. + */ + spinlock_t time_lock; + /* Externally accessible PTP clock, may be NULL */ + struct ptp_clock *ptp; + struct ptp_clock_info ptp_info; + /* Last @time_high refresh time in jiffies (to catch stalls) */ + unsigned long last_read; + /* Local copy of hardware statistics */ struct fbnic_hw_stats hw_stats; }; @@ -127,6 +142,9 @@ void fbnic_devlink_unregister(struct fbnic_dev *fbd); int fbnic_fw_enable_mbx(struct fbnic_dev *fbd); void fbnic_fw_disable_mbx(struct fbnic_dev *fbd); +void fbnic_hwmon_register(struct fbnic_dev *fbd); +void fbnic_hwmon_unregister(struct fbnic_dev *fbd); + int fbnic_pcs_irq_enable(struct fbnic_dev *fbd); void fbnic_pcs_irq_disable(struct fbnic_dev *fbd); @@ -139,6 +157,14 @@ int fbnic_alloc_irqs(struct fbnic_dev *fbd); void fbnic_get_fw_ver_commit_str(struct fbnic_dev *fbd, char *fw_version, const size_t str_sz); +void fbnic_dbg_fbd_init(struct fbnic_dev *fbd); +void fbnic_dbg_fbd_exit(struct fbnic_dev *fbd); +void fbnic_dbg_init(void); +void fbnic_dbg_exit(void); + +void fbnic_csr_get_regs(struct fbnic_dev *fbd, u32 *data, u32 *regs_version); +int fbnic_csr_regs_len(struct fbnic_dev *fbd); + enum fbnic_boards { fbnic_board_asic }; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.c b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c new file mode 100644 index 000000000000..2118901b25e9 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include "fbnic.h" + +#define FBNIC_BOUNDS(section) { \ + .start = FBNIC_CSR_START_##section, \ + .end = FBNIC_CSR_END_##section + 1, \ +} + +struct fbnic_csr_bounds { + u32 start; + u32 end; +}; + +static const struct fbnic_csr_bounds fbnic_csr_sects[] = { + FBNIC_BOUNDS(INTR), + FBNIC_BOUNDS(INTR_CQ), + FBNIC_BOUNDS(QM_TX), + FBNIC_BOUNDS(QM_RX), + FBNIC_BOUNDS(TCE), + FBNIC_BOUNDS(TCE_RAM), + FBNIC_BOUNDS(TMI), + FBNIC_BOUNDS(PTP), + FBNIC_BOUNDS(RXB), + FBNIC_BOUNDS(RPC), + FBNIC_BOUNDS(FAB), + FBNIC_BOUNDS(MASTER), + FBNIC_BOUNDS(PCS), + FBNIC_BOUNDS(RSFEC), + FBNIC_BOUNDS(MAC_MAC), + FBNIC_BOUNDS(SIG), + FBNIC_BOUNDS(PUL_USER), + FBNIC_BOUNDS(QUEUE), + FBNIC_BOUNDS(RPC_RAM), +}; + +#define FBNIC_RPC_TCAM_ACT_DW_PER_ENTRY 14 +#define FBNIC_RPC_TCAM_ACT_NUM_ENTRIES 64 + +#define FBNIC_RPC_TCAM_MACDA_DW_PER_ENTRY 4 +#define FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES 32 + +#define FBNIC_RPC_TCAM_OUTER_IPSRC_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_OUTER_IPSRC_NUM_ENTRIES 8 + +#define FBNIC_RPC_TCAM_OUTER_IPDST_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_OUTER_IPDST_NUM_ENTRIES 8 + +#define FBNIC_RPC_TCAM_IPSRC_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_IPSRC_NUM_ENTRIES 8 + +#define FBNIC_RPC_TCAM_IPDST_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_IPDST_NUM_ENTRIES 8 + +#define FBNIC_RPC_RSS_TBL_DW_PER_ENTRY 2 +#define FBNIC_RPC_RSS_TBL_NUM_ENTRIES 256 + +static void fbnic_csr_get_regs_rpc_ram(struct fbnic_dev *fbd, u32 **data_p) +{ + u32 start = FBNIC_CSR_START_RPC_RAM; + u32 end = FBNIC_CSR_END_RPC_RAM; + u32 *data = *data_p; + u32 i, j; + + *(data++) = start; + *(data++) = end - 1; + + /* FBNIC_RPC_TCAM_ACT */ + for (i = 0; i < FBNIC_RPC_TCAM_ACT_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_ACT_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_ACT(i, j)); + } + + /* FBNIC_RPC_TCAM_MACDA */ + for (i = 0; i < FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_MACDA_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_MACDA(i, j)); + } + + /* FBNIC_RPC_TCAM_OUTER_IPSRC */ + for (i = 0; i < FBNIC_RPC_TCAM_OUTER_IPSRC_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_OUTER_IPSRC_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_OUTER_IPSRC(i, j)); + } + + /* FBNIC_RPC_TCAM_OUTER_IPDST */ + for (i = 0; i < FBNIC_RPC_TCAM_OUTER_IPDST_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_OUTER_IPDST_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_OUTER_IPDST(i, j)); + } + + /* FBNIC_RPC_TCAM_IPSRC */ + for (i = 0; i < FBNIC_RPC_TCAM_IPSRC_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_IPSRC_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_IPSRC(i, j)); + } + + /* FBNIC_RPC_TCAM_IPDST */ + for (i = 0; i < FBNIC_RPC_TCAM_IPDST_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_IPDST_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_IPDST(i, j)); + } + + /* FBNIC_RPC_RSS_TBL */ + for (i = 0; i < FBNIC_RPC_RSS_TBL_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_RSS_TBL_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_RSS_TBL(i, j)); + } + + *data_p = data; +} + +void fbnic_csr_get_regs(struct fbnic_dev *fbd, u32 *data, u32 *regs_version) +{ + const struct fbnic_csr_bounds *bound; + u32 *start = data; + int i, j; + + *regs_version = 1u; + + /* Skip RPC_RAM section which cannot be dumped linearly */ + for (i = 0, bound = fbnic_csr_sects; + i < ARRAY_SIZE(fbnic_csr_sects) - 1; i++, ++bound) { + *(data++) = bound->start; + *(data++) = bound->end - 1; + for (j = bound->start; j < bound->end; j++) + *(data++) = rd32(fbd, j); + } + + /* Dump the RPC_RAM as special case registers */ + fbnic_csr_get_regs_rpc_ram(fbd, &data); + + WARN_ON(data - start != fbnic_csr_regs_len(fbd)); +} + +int fbnic_csr_regs_len(struct fbnic_dev *fbd) +{ + int i, len = 0; + + /* Dump includes start and end information of each section + * which results in an offset of 2 + */ + for (i = 0; i < ARRAY_SIZE(fbnic_csr_sects); i++) + len += fbnic_csr_sects[i].end - fbnic_csr_sects[i].start + 2; + + return len; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 21db509acbc1..02bb81b3c506 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -397,6 +397,14 @@ enum { #define FBNIC_TCE_DROP_CTRL_TTI_FRM_DROP_EN CSR_BIT(1) #define FBNIC_TCE_DROP_CTRL_TTI_TBI_DROP_EN CSR_BIT(2) +#define FBNIC_TCE_TCAM_IDX2DEST_MAP 0x0404A /* 0x10128 */ +#define FBNIC_TCE_TCAM_IDX2DEST_MAP_DEST_ID_0 CSR_GENMASK(3, 0) +enum { + FBNIC_TCE_TCAM_DEST_MAC = 1, + FBNIC_TCE_TCAM_DEST_BMC = 2, + FBNIC_TCE_TCAM_DEST_FW = 4, +}; + #define FBNIC_TCE_TXB_TX_BMC_Q_CTRL 0x0404B /* 0x1012c */ #define FBNIC_TCE_TXB_BMC_DWRR_CTRL 0x0404C /* 0x10130 */ #define FBNIC_TCE_TXB_BMC_DWRR_CTRL_QUANTUM0 CSR_GENMASK(7, 0) @@ -407,12 +415,62 @@ enum { #define FBNIC_TCE_TXB_BMC_DWRR_CTRL_EXT 0x0404F /* 0x1013c */ #define FBNIC_CSR_END_TCE 0x04050 /* CSR section delimiter */ +/* TCE RAM registers */ +#define FBNIC_CSR_START_TCE_RAM 0x04200 /* CSR section delimiter */ +#define FBNIC_TCE_RAM_TCAM(m, n) \ + (0x04200 + 0x8 * (n) + (m)) /* 0x10800 + 32*n + 4*m */ +#define FBNIC_TCE_RAM_TCAM_MASK CSR_GENMASK(15, 0) +#define FBNIC_TCE_RAM_TCAM_VALUE CSR_GENMASK(31, 16) +#define FBNIC_TCE_RAM_TCAM3(n) (0x04218 + (n)) /* 0x010860 + 4*n */ +#define FBNIC_TCE_RAM_TCAM3_DEST_MASK CSR_GENMASK(5, 3) +#define FBNIC_TCE_RAM_TCAM3_MCQ_MASK CSR_BIT(7) +#define FBNIC_TCE_RAM_TCAM3_VALIDATE CSR_BIT(31) +#define FBNIC_CSR_END_TCE_RAM 0x0421F /* CSR section delimiter */ + /* TMI registers */ #define FBNIC_CSR_START_TMI 0x04400 /* CSR section delimiter */ #define FBNIC_TMI_SOP_PROT_CTRL 0x04400 /* 0x11000 */ #define FBNIC_TMI_DROP_CTRL 0x04401 /* 0x11004 */ #define FBNIC_TMI_DROP_CTRL_EN CSR_BIT(0) #define FBNIC_CSR_END_TMI 0x0443f /* CSR section delimiter */ + +/* Precision Time Protocol Registers */ +#define FBNIC_CSR_START_PTP 0x04800 /* CSR section delimiter */ +#define FBNIC_PTP_REG_BASE 0x04800 /* 0x12000 */ + +#define FBNIC_PTP_CTRL 0x04800 /* 0x12000 */ +#define FBNIC_PTP_CTRL_EN CSR_BIT(0) +#define FBNIC_PTP_CTRL_MONO_EN CSR_BIT(4) +#define FBNIC_PTP_CTRL_TQS_OUT_EN CSR_BIT(8) +#define FBNIC_PTP_CTRL_MAC_OUT_IVAL CSR_GENMASK(16, 12) +#define FBNIC_PTP_CTRL_TICK_IVAL CSR_GENMASK(23, 20) + +#define FBNIC_PTP_ADJUST 0x04801 /* 0x12004 */ +#define FBNIC_PTP_ADJUST_INIT CSR_BIT(0) +#define FBNIC_PTP_ADJUST_SUB_NUDGE CSR_BIT(8) +#define FBNIC_PTP_ADJUST_ADD_NUDGE CSR_BIT(16) +#define FBNIC_PTP_ADJUST_ADDEND_SET CSR_BIT(24) + +#define FBNIC_PTP_INIT_HI 0x04802 /* 0x12008 */ +#define FBNIC_PTP_INIT_LO 0x04803 /* 0x1200c */ + +#define FBNIC_PTP_NUDGE_NS 0x04804 /* 0x12010 */ +#define FBNIC_PTP_NUDGE_SUBNS 0x04805 /* 0x12014 */ + +#define FBNIC_PTP_ADD_VAL_NS 0x04806 /* 0x12018 */ +#define FBNIC_PTP_ADD_VAL_NS_MASK CSR_GENMASK(15, 0) +#define FBNIC_PTP_ADD_VAL_SUBNS 0x04807 /* 0x1201c */ + +#define FBNIC_PTP_CTR_VAL_HI 0x04808 /* 0x12020 */ +#define FBNIC_PTP_CTR_VAL_LO 0x04809 /* 0x12024 */ + +#define FBNIC_PTP_MONO_PTP_CTR_HI 0x0480a /* 0x12028 */ +#define FBNIC_PTP_MONO_PTP_CTR_LO 0x0480b /* 0x1202c */ + +#define FBNIC_PTP_CDC_FIFO_STATUS 0x0480c /* 0x12030 */ +#define FBNIC_PTP_SPARE 0x0480d /* 0x12034 */ +#define FBNIC_CSR_END_PTP 0x0480d /* CSR section delimiter */ + /* Rx Buffer Registers */ #define FBNIC_CSR_START_RXB 0x08000 /* CSR section delimiter */ enum { @@ -548,6 +606,7 @@ enum { }; #define FBNIC_RPC_ACT_TBL0_DMA_HINT CSR_GENMASK(24, 16) +#define FBNIC_RPC_ACT_TBL0_TS_ENA CSR_BIT(28) #define FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID CSR_BIT(30) #define FBNIC_RPC_ACT_TBL1_DEFAULT 0x0840b /* 0x2102c */ @@ -579,6 +638,16 @@ enum { FBNIC_RPC_RSS_KEY_DWORD_LEN * 32 - \ FBNIC_RPC_RSS_KEY_BIT_LEN) +#define FBNIC_RPC_CNTR_TCP_OPT_ERR 0x0849e /* 0x21278 */ +#define FBNIC_RPC_CNTR_UNKN_ETYPE 0x0849f /* 0x2127c */ +#define FBNIC_RPC_CNTR_IPV4_FRAG 0x084a0 /* 0x21280 */ +#define FBNIC_RPC_CNTR_IPV6_FRAG 0x084a1 /* 0x21284 */ +#define FBNIC_RPC_CNTR_IPV4_ESP 0x084a2 /* 0x21288 */ +#define FBNIC_RPC_CNTR_IPV6_ESP 0x084a3 /* 0x2128c */ +#define FBNIC_RPC_CNTR_UNKN_EXT_HDR 0x084a4 /* 0x21290 */ +#define FBNIC_RPC_CNTR_OUT_OF_HDR_ERR 0x084a5 /* 0x21294 */ +#define FBNIC_RPC_CNTR_OVR_SIZE_ERR 0x084a6 /* 0x21298 */ + #define FBNIC_RPC_TCAM_MACDA_VALIDATE 0x0852d /* 0x214b4 */ #define FBNIC_CSR_END_RPC 0x0856b /* CSR section delimiter */ @@ -606,6 +675,15 @@ enum { #define FBNIC_RPC_TCAM_MACDA_VALUE CSR_GENMASK(15, 0) #define FBNIC_RPC_TCAM_MACDA_MASK CSR_GENMASK(31, 16) +#define FBNIC_RPC_TCAM_OUTER_IPSRC(m, n)\ + (0x08c00 + 0x08 * (n) + (m)) /* 0x023000 + 32*n + 4*m */ +#define FBNIC_RPC_TCAM_OUTER_IPDST(m, n)\ + (0x08c48 + 0x08 * (n) + (m)) /* 0x023120 + 32*n + 4*m */ +#define FBNIC_RPC_TCAM_IPSRC(m, n)\ + (0x08c90 + 0x08 * (n) + (m)) /* 0x023240 + 32*n + 4*m */ +#define FBNIC_RPC_TCAM_IPDST(m, n)\ + (0x08cd8 + 0x08 * (n) + (m)) /* 0x023360 + 32*n + 4*m */ + #define FBNIC_RPC_RSS_TBL(n, m) \ (0x08d20 + 0x100 * (n) + (m)) /* 0x023480 + 1024*n + 4*m */ #define FBNIC_RPC_RSS_TBL_COUNT 2 @@ -624,6 +702,13 @@ enum { #define FBNIC_MASTER_SPARE_0 0x0C41B /* 0x3106c */ #define FBNIC_CSR_END_MASTER 0x0C452 /* CSR section delimiter */ +/* MAC PCS registers */ +#define FBNIC_CSR_START_PCS 0x10000 /* CSR section delimiter */ +#define FBNIC_CSR_END_PCS 0x10668 /* CSR section delimiter */ + +#define FBNIC_CSR_START_RSFEC 0x10800 /* CSR section delimiter */ +#define FBNIC_CSR_END_RSFEC 0x108c8 /* CSR section delimiter */ + /* MAC MAC registers (ASIC only) */ #define FBNIC_CSR_START_MAC_MAC 0x11000 /* CSR section delimiter */ #define FBNIC_MAC_COMMAND_CONFIG 0x11002 /* 0x44008 */ @@ -843,6 +928,43 @@ enum { #define FBNIC_MAX_QUEUES 128 #define FBNIC_CSR_END_QUEUE (0x40000 + 0x400 * FBNIC_MAX_QUEUES - 1) +/* PUL User Registers*/ +#define FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0 \ + 0x3106e /* 0xc41b8 */ +#define FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0 \ + 0x31070 /* 0xc41c0 */ +#define FBNIC_PUL_USER_OB_RD_DWORD_CNT_63_32 \ + 0x31071 /* 0xc41c4 */ +#define FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0 \ + 0x31072 /* 0xc41c8 */ +#define FBNIC_PUL_USER_OB_WR_TLP_CNT_63_32 \ + 0x31073 /* 0xc41cc */ +#define FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0 \ + 0x31074 /* 0xc41d0 */ +#define FBNIC_PUL_USER_OB_WR_DWORD_CNT_63_32 \ + 0x31075 /* 0xc41d4 */ +#define FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0 \ + 0x31076 /* 0xc41d8 */ +#define FBNIC_PUL_USER_OB_CPL_TLP_CNT_63_32 \ + 0x31077 /* 0xc41dc */ +#define FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0 \ + 0x31078 /* 0xc41e0 */ +#define FBNIC_PUL_USER_OB_CPL_DWORD_CNT_63_32 \ + 0x31079 /* 0xc41e4 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0 \ + 0x3107a /* 0xc41e8 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_63_32 \ + 0x3107b /* 0xc41ec */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0 \ + 0x3107c /* 0xc41f0 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_63_32 \ + 0x3107d /* 0xc41f4 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0 \ + 0x3107e /* 0xc41f8 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_63_32 \ + 0x3107f /* 0xc41fc */ +#define FBNIC_CSR_END_PUL_USER 0x31080 /* CSR section delimiter */ + /* BAR 4 CSRs */ /* The IPC mailbox consists of 32 mailboxes, with each mailbox consisting diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c new file mode 100644 index 000000000000..59951b5abdb7 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/debugfs.h> +#include <linux/pci.h> +#include <linux/rtnetlink.h> +#include <linux/seq_file.h> + +#include "fbnic.h" + +static struct dentry *fbnic_dbg_root; + +static int fbnic_dbg_pcie_stats_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + + rtnl_lock(); + fbnic_get_hw_stats(fbd); + + seq_printf(s, "ob_rd_tlp: %llu\n", fbd->hw_stats.pcie.ob_rd_tlp.value); + seq_printf(s, "ob_rd_dword: %llu\n", + fbd->hw_stats.pcie.ob_rd_dword.value); + seq_printf(s, "ob_wr_tlp: %llu\n", fbd->hw_stats.pcie.ob_wr_tlp.value); + seq_printf(s, "ob_wr_dword: %llu\n", + fbd->hw_stats.pcie.ob_wr_dword.value); + seq_printf(s, "ob_cpl_tlp: %llu\n", + fbd->hw_stats.pcie.ob_cpl_tlp.value); + seq_printf(s, "ob_cpl_dword: %llu\n", + fbd->hw_stats.pcie.ob_cpl_dword.value); + seq_printf(s, "ob_rd_no_tag: %llu\n", + fbd->hw_stats.pcie.ob_rd_no_tag.value); + seq_printf(s, "ob_rd_no_cpl_cred: %llu\n", + fbd->hw_stats.pcie.ob_rd_no_cpl_cred.value); + seq_printf(s, "ob_rd_no_np_cred: %llu\n", + fbd->hw_stats.pcie.ob_rd_no_np_cred.value); + rtnl_unlock(); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_pcie_stats); + +void fbnic_dbg_fbd_init(struct fbnic_dev *fbd) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + const char *name = pci_name(pdev); + + fbd->dbg_fbd = debugfs_create_dir(name, fbnic_dbg_root); + debugfs_create_file("pcie_stats", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_pcie_stats_fops); +} + +void fbnic_dbg_fbd_exit(struct fbnic_dev *fbd) +{ + debugfs_remove_recursive(fbd->dbg_fbd); + fbd->dbg_fbd = NULL; +} + +void fbnic_dbg_init(void) +{ + fbnic_dbg_root = debugfs_create_dir(fbnic_driver_name, NULL); +} + +void fbnic_dbg_exit(void) +{ + debugfs_remove_recursive(fbnic_dbg_root); + fbnic_dbg_root = NULL; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 5d980e178941..cc8ca94529ca 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -1,3 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + #include <linux/ethtool.h> #include <linux/netdevice.h> #include <linux/pci.h> @@ -6,6 +9,66 @@ #include "fbnic_netdev.h" #include "fbnic_tlv.h" +struct fbnic_stat { + u8 string[ETH_GSTRING_LEN]; + unsigned int size; + unsigned int offset; +}; + +#define FBNIC_STAT_FIELDS(type, name, stat) { \ + .string = name, \ + .size = sizeof_field(struct type, stat), \ + .offset = offsetof(struct type, stat), \ +} + +/* Hardware statistics not captured in rtnl_link_stats */ +#define FBNIC_HW_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_hw_stats, name, stat) + +static const struct fbnic_stat fbnic_gstrings_hw_stats[] = { + /* RPC */ + FBNIC_HW_STAT("rpc_unkn_etype", rpc.unkn_etype), + FBNIC_HW_STAT("rpc_unkn_ext_hdr", rpc.unkn_ext_hdr), + FBNIC_HW_STAT("rpc_ipv4_frag", rpc.ipv4_frag), + FBNIC_HW_STAT("rpc_ipv6_frag", rpc.ipv6_frag), + FBNIC_HW_STAT("rpc_ipv4_esp", rpc.ipv4_esp), + FBNIC_HW_STAT("rpc_ipv6_esp", rpc.ipv6_esp), + FBNIC_HW_STAT("rpc_tcp_opt_err", rpc.tcp_opt_err), + FBNIC_HW_STAT("rpc_out_of_hdr_err", rpc.out_of_hdr_err), +}; + +#define FBNIC_HW_FIXED_STATS_LEN ARRAY_SIZE(fbnic_gstrings_hw_stats) +#define FBNIC_HW_STATS_LEN FBNIC_HW_FIXED_STATS_LEN + +static int +fbnic_get_ts_info(struct net_device *netdev, + struct kernel_ethtool_ts_info *tsinfo) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + tsinfo->phc_index = ptp_clock_index(fbn->fbd->ptp); + + tsinfo->so_timestamping = + SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + tsinfo->tx_types = + BIT(HWTSTAMP_TX_OFF) | + BIT(HWTSTAMP_TX_ON); + + tsinfo->rx_filters = + BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_PTP_V1_L4_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | + BIT(HWTSTAMP_FILTER_ALL); + + return 0; +} + static void fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { @@ -22,6 +85,43 @@ static void fbnic_set_counter(u64 *stat, struct fbnic_stat_counter *counter) *stat = counter->value; } +static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data) +{ + int i; + + switch (sset) { + case ETH_SS_STATS: + for (i = 0; i < FBNIC_HW_STATS_LEN; i++) + ethtool_puts(&data, fbnic_gstrings_hw_stats[i].string); + break; + } +} + +static int fbnic_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return FBNIC_HW_STATS_LEN; + default: + return -EOPNOTSUPP; + } +} + +static void fbnic_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(dev); + const struct fbnic_stat *stat; + int i; + + fbnic_get_hw_stats(fbn->fbd); + + for (i = 0; i < FBNIC_HW_STATS_LEN; i++) { + stat = &fbnic_gstrings_hw_stats[i]; + data[i] = *(u64 *)((u8 *)&fbn->fbd->hw_stats + stat->offset); + } +} + static void fbnic_get_eth_mac_stats(struct net_device *netdev, struct ethtool_eth_mac_stats *eth_mac_stats) @@ -64,8 +164,53 @@ fbnic_get_eth_mac_stats(struct net_device *netdev, &mac_stats->eth_mac.FrameTooLongErrors); } +static void fbnic_get_ts_stats(struct net_device *netdev, + struct ethtool_ts_stats *ts_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + u64 ts_packets, ts_lost; + struct fbnic_ring *ring; + unsigned int start; + int i; + + ts_stats->pkts = fbn->tx_stats.ts_packets; + ts_stats->lost = fbn->tx_stats.ts_lost; + for (i = 0; i < fbn->num_tx_queues; i++) { + ring = fbn->tx[i]; + do { + start = u64_stats_fetch_begin(&ring->stats.syncp); + ts_packets = ring->stats.ts_packets; + ts_lost = ring->stats.ts_lost; + } while (u64_stats_fetch_retry(&ring->stats.syncp, start)); + ts_stats->pkts += ts_packets; + ts_stats->lost += ts_lost; + } +} + +static void fbnic_get_regs(struct net_device *netdev, + struct ethtool_regs *regs, void *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + fbnic_csr_get_regs(fbn->fbd, data, ®s->version); +} + +static int fbnic_get_regs_len(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + return fbnic_csr_regs_len(fbn->fbd) * sizeof(u32); +} + static const struct ethtool_ops fbnic_ethtool_ops = { .get_drvinfo = fbnic_get_drvinfo, + .get_regs_len = fbnic_get_regs_len, + .get_regs = fbnic_get_regs, + .get_strings = fbnic_get_strings, + .get_ethtool_stats = fbnic_get_ethtool_stats, + .get_sset_count = fbnic_get_sset_count, + .get_ts_info = fbnic_get_ts_info, + .get_ts_stats = fbnic_get_ts_stats, .get_eth_mac_stats = fbnic_get_eth_mac_stats, }; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h index 221faf8c6756..7cd8841920e4 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -44,6 +44,13 @@ struct fbnic_fw_cap { u8 link_fec; }; +struct fbnic_fw_completion { + struct { + s32 millivolts; + s32 millidegrees; + } tsene; +}; + void fbnic_mbx_init(struct fbnic_dev *fbd); void fbnic_mbx_clean(struct fbnic_dev *fbd); void fbnic_mbx_poll(struct fbnic_dev *fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c index a0acc7606aa1..89ac6bc8c7fc 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c @@ -1,5 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + #include "fbnic.h" +static void fbnic_hw_stat_rst32(struct fbnic_dev *fbd, u32 reg, + struct fbnic_stat_counter *stat) +{ + /* We do not touch the "value" field here. + * It gets zeroed out on fbd structure allocation. + * After that we want it to grow continuously + * through device resets and power state changes. + */ + stat->u.old_reg_value_32 = rd32(fbd, reg); +} + +static void fbnic_hw_stat_rd32(struct fbnic_dev *fbd, u32 reg, + struct fbnic_stat_counter *stat) +{ + u32 new_reg_value; + + new_reg_value = rd32(fbd, reg); + stat->value += new_reg_value - stat->u.old_reg_value_32; + stat->u.old_reg_value_32 = new_reg_value; +} + u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset) { u32 prev_upper, upper, lower, diff; @@ -25,3 +49,172 @@ u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset) */ return ((u64)upper << 32); } + +static void fbnic_hw_stat_rst64(struct fbnic_dev *fbd, u32 reg, s32 offset, + struct fbnic_stat_counter *stat) +{ + /* Record initial counter values and compute deltas from there to ensure + * stats start at 0 after reboot/reset. This avoids exposing absolute + * hardware counter values to userspace. + */ + stat->u.old_reg_value_64 = fbnic_stat_rd64(fbd, reg, offset); +} + +static void fbnic_hw_stat_rd64(struct fbnic_dev *fbd, u32 reg, s32 offset, + struct fbnic_stat_counter *stat) +{ + u64 new_reg_value; + + new_reg_value = fbnic_stat_rd64(fbd, reg, offset); + stat->value += new_reg_value - stat->u.old_reg_value_64; + stat->u.old_reg_value_64 = new_reg_value; +} + +static void fbnic_reset_rpc_stats(struct fbnic_dev *fbd, + struct fbnic_rpc_stats *rpc) +{ + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_UNKN_ETYPE, + &rpc->unkn_etype); + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_UNKN_EXT_HDR, + &rpc->unkn_ext_hdr); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV4_FRAG, &rpc->ipv4_frag); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV6_FRAG, &rpc->ipv6_frag); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV4_ESP, &rpc->ipv4_esp); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV6_ESP, &rpc->ipv6_esp); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_TCP_OPT_ERR, &rpc->tcp_opt_err); + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_OUT_OF_HDR_ERR, + &rpc->out_of_hdr_err); + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_OVR_SIZE_ERR, + &rpc->ovr_size_err); +} + +static void fbnic_get_rpc_stats32(struct fbnic_dev *fbd, + struct fbnic_rpc_stats *rpc) +{ + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_UNKN_ETYPE, + &rpc->unkn_etype); + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_UNKN_EXT_HDR, + &rpc->unkn_ext_hdr); + + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV4_FRAG, &rpc->ipv4_frag); + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV6_FRAG, &rpc->ipv6_frag); + + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV4_ESP, &rpc->ipv4_esp); + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV6_ESP, &rpc->ipv6_esp); + + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_TCP_OPT_ERR, &rpc->tcp_opt_err); + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_OUT_OF_HDR_ERR, + &rpc->out_of_hdr_err); + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_OVR_SIZE_ERR, + &rpc->ovr_size_err); +} + +static void fbnic_reset_pcie_stats_asic(struct fbnic_dev *fbd, + struct fbnic_pcie_stats *pcie) +{ + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0, + 1, + &pcie->ob_rd_tlp); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0, + 1, + &pcie->ob_rd_dword); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0, + 1, + &pcie->ob_cpl_tlp); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0, + 1, + &pcie->ob_cpl_dword); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0, + 1, + &pcie->ob_wr_tlp); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0, + 1, + &pcie->ob_wr_dword); + + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0, + 1, + &pcie->ob_rd_no_tag); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0, + 1, + &pcie->ob_rd_no_cpl_cred); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0, + 1, + &pcie->ob_rd_no_np_cred); +} + +static void fbnic_get_pcie_stats_asic64(struct fbnic_dev *fbd, + struct fbnic_pcie_stats *pcie) +{ + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0, + 1, + &pcie->ob_rd_tlp); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0, + 1, + &pcie->ob_rd_dword); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0, + 1, + &pcie->ob_wr_tlp); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0, + 1, + &pcie->ob_wr_dword); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0, + 1, + &pcie->ob_cpl_tlp); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0, + 1, + &pcie->ob_cpl_dword); + + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0, + 1, + &pcie->ob_rd_no_tag); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0, + 1, + &pcie->ob_rd_no_cpl_cred); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0, + 1, + &pcie->ob_rd_no_np_cred); +} + +void fbnic_reset_hw_stats(struct fbnic_dev *fbd) +{ + fbnic_reset_rpc_stats(fbd, &fbd->hw_stats.rpc); + fbnic_reset_pcie_stats_asic(fbd, &fbd->hw_stats.pcie); +} + +void fbnic_get_hw_stats32(struct fbnic_dev *fbd) +{ + fbnic_get_rpc_stats32(fbd, &fbd->hw_stats.rpc); +} + +void fbnic_get_hw_stats(struct fbnic_dev *fbd) +{ + fbnic_get_hw_stats32(fbd); + + fbnic_get_pcie_stats_asic64(fbd, &fbd->hw_stats.pcie); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h index 30348904b510..78df56b87745 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h @@ -1,3 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_HW_STATS_H_ +#define _FBNIC_HW_STATS_H_ + #include <linux/ethtool.h> #include "fbnic_csr.h" @@ -31,10 +37,32 @@ struct fbnic_mac_stats { struct fbnic_eth_mac_stats eth_mac; }; +struct fbnic_rpc_stats { + struct fbnic_stat_counter unkn_etype, unkn_ext_hdr; + struct fbnic_stat_counter ipv4_frag, ipv6_frag, ipv4_esp, ipv6_esp; + struct fbnic_stat_counter tcp_opt_err, out_of_hdr_err, ovr_size_err; +}; + +struct fbnic_pcie_stats { + struct fbnic_stat_counter ob_rd_tlp, ob_rd_dword; + struct fbnic_stat_counter ob_wr_tlp, ob_wr_dword; + struct fbnic_stat_counter ob_cpl_tlp, ob_cpl_dword; + + struct fbnic_stat_counter ob_rd_no_tag; + struct fbnic_stat_counter ob_rd_no_cpl_cred; + struct fbnic_stat_counter ob_rd_no_np_cred; +}; + struct fbnic_hw_stats { struct fbnic_mac_stats mac; + struct fbnic_rpc_stats rpc; + struct fbnic_pcie_stats pcie; }; u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset); +void fbnic_reset_hw_stats(struct fbnic_dev *fbd); +void fbnic_get_hw_stats32(struct fbnic_dev *fbd); void fbnic_get_hw_stats(struct fbnic_dev *fbd); + +#endif /* _FBNIC_HW_STATS_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hwmon.c b/drivers/net/ethernet/meta/fbnic/fbnic_hwmon.c new file mode 100644 index 000000000000..bcd1086e3768 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hwmon.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/hwmon.h> + +#include "fbnic.h" +#include "fbnic_mac.h" + +static int fbnic_hwmon_sensor_id(enum hwmon_sensor_types type) +{ + if (type == hwmon_temp) + return FBNIC_SENSOR_TEMP; + if (type == hwmon_in) + return FBNIC_SENSOR_VOLTAGE; + + return -EOPNOTSUPP; +} + +static umode_t fbnic_hwmon_is_visible(const void *drvdata, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + if (type == hwmon_temp && attr == hwmon_temp_input) + return 0444; + if (type == hwmon_in && attr == hwmon_in_input) + return 0444; + + return 0; +} + +static int fbnic_hwmon_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct fbnic_dev *fbd = dev_get_drvdata(dev); + const struct fbnic_mac *mac = fbd->mac; + int id; + + id = fbnic_hwmon_sensor_id(type); + return id < 0 ? id : mac->get_sensor(fbd, id, val); +} + +static const struct hwmon_ops fbnic_hwmon_ops = { + .is_visible = fbnic_hwmon_is_visible, + .read = fbnic_hwmon_read, +}; + +static const struct hwmon_channel_info *fbnic_hwmon_info[] = { + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + HWMON_CHANNEL_INFO(in, HWMON_I_INPUT), + NULL +}; + +static const struct hwmon_chip_info fbnic_chip_info = { + .ops = &fbnic_hwmon_ops, + .info = fbnic_hwmon_info, +}; + +void fbnic_hwmon_register(struct fbnic_dev *fbd) +{ + if (!IS_REACHABLE(CONFIG_HWMON)) + return; + + fbd->hwmon = hwmon_device_register_with_info(fbd->dev, "fbnic", + fbd, &fbnic_chip_info, + NULL); + if (IS_ERR(fbd->hwmon)) { + dev_notice(fbd->dev, + "Failed to register hwmon device %pe\n", + fbd->hwmon); + fbd->hwmon = NULL; + } +} + +void fbnic_hwmon_unregister(struct fbnic_dev *fbd) +{ + if (!IS_REACHABLE(CONFIG_HWMON) || !fbd->hwmon) + return; + + hwmon_device_unregister(fbd->hwmon); + fbd->hwmon = NULL; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c index 7b654d0a6dac..80b82ff12c4d 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c @@ -686,6 +686,27 @@ fbnic_mac_get_eth_mac_stats(struct fbnic_dev *fbd, bool reset, MAC_STAT_TX_BROADCAST); } +static int fbnic_mac_get_sensor_asic(struct fbnic_dev *fbd, int id, long *val) +{ + struct fbnic_fw_completion fw_cmpl; + s32 *sensor; + + switch (id) { + case FBNIC_SENSOR_TEMP: + sensor = &fw_cmpl.tsene.millidegrees; + break; + case FBNIC_SENSOR_VOLTAGE: + sensor = &fw_cmpl.tsene.millivolts; + break; + default: + return -EINVAL; + } + + *val = *sensor; + + return 0; +} + static const struct fbnic_mac fbnic_mac_asic = { .init_regs = fbnic_mac_init_regs, .pcs_enable = fbnic_pcs_enable_asic, @@ -695,6 +716,7 @@ static const struct fbnic_mac fbnic_mac_asic = { .get_eth_mac_stats = fbnic_mac_get_eth_mac_stats, .link_down = fbnic_mac_link_down_asic, .link_up = fbnic_mac_link_up_asic, + .get_sensor = fbnic_mac_get_sensor_asic, }; /** diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h index 476239a9d381..05a591653e09 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h @@ -47,6 +47,11 @@ enum { #define FBNIC_LINK_MODE_PAM4 (FBNIC_LINK_50R1) #define FBNIC_LINK_MODE_MASK (FBNIC_LINK_AUTO - 1) +enum fbnic_sensor_id { + FBNIC_SENSOR_TEMP, /* Temp in millidegrees Centigrade */ + FBNIC_SENSOR_VOLTAGE, /* Voltage in millivolts */ +}; + /* This structure defines the interface hooks for the MAC. The MAC hooks * will be configured as a const struct provided with a set of function * pointers. @@ -83,6 +88,8 @@ struct fbnic_mac { void (*link_down)(struct fbnic_dev *fbd); void (*link_up)(struct fbnic_dev *fbd, bool tx_pause, bool rx_pause); + + int (*get_sensor)(struct fbnic_dev *fbd, int id, long *val); }; int fbnic_mac_init(struct fbnic_dev *fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index a400616a24d4..fc7d80db5fa6 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -42,18 +42,24 @@ int __fbnic_open(struct fbnic_net *fbn) goto free_resources; } - err = fbnic_fw_init_heartbeat(fbd, false); + err = fbnic_time_start(fbn); if (err) goto release_ownership; + err = fbnic_fw_init_heartbeat(fbd, false); + if (err) + goto time_stop; + err = fbnic_pcs_irq_enable(fbd); if (err) - goto release_ownership; + goto time_stop; /* Pull the BMC config and initialize the RPC */ fbnic_bmc_rpc_init(fbd); fbnic_rss_reinit(fbd, fbn); return 0; +time_stop: + fbnic_time_stop(fbn); release_ownership: fbnic_fw_xmit_ownership_msg(fbn->fbd, false); free_resources: @@ -82,6 +88,7 @@ static int fbnic_stop(struct net_device *netdev) fbnic_down(fbn); fbnic_pcs_irq_disable(fbn->fbd); + fbnic_time_stop(fbn); fbnic_fw_xmit_ownership_msg(fbn->fbd, false); fbnic_free_resources(fbn); @@ -266,6 +273,7 @@ void __fbnic_set_rx_mode(struct net_device *netdev) /* Write updates to hardware */ fbnic_write_rules(fbd); fbnic_write_macda(fbd); + fbnic_write_tce_tcam(fbd); } static void fbnic_set_rx_mode(struct net_device *netdev) @@ -317,6 +325,84 @@ void fbnic_clear_rx_mode(struct net_device *netdev) __dev_mc_unsync(netdev, NULL); } +static int fbnic_hwtstamp_get(struct net_device *netdev, + struct kernel_hwtstamp_config *config) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + *config = fbn->hwtstamp_config; + + return 0; +} + +static int fbnic_hwtstamp_set(struct net_device *netdev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int old_rx_filter; + + if (config->source != HWTSTAMP_SOURCE_NETDEV) + return -EOPNOTSUPP; + + if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config)) + return 0; + + /* Upscale the filters */ + switch (config->rx_filter) { + case HWTSTAMP_FILTER_NONE: + case HWTSTAMP_FILTER_ALL: + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_EVENT: + break; + case HWTSTAMP_FILTER_NTP_ALL: + config->rx_filter = HWTSTAMP_FILTER_ALL; + break; + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + break; + default: + return -ERANGE; + } + + /* Configure */ + old_rx_filter = fbn->hwtstamp_config.rx_filter; + memcpy(&fbn->hwtstamp_config, config, sizeof(*config)); + + if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) { + fbnic_rss_reinit(fbn->fbd, fbn); + fbnic_write_rules(fbn->fbd); + } + + /* Save / report back filter configuration + * Note that our filter configuration is inexact. Instead of + * filtering for a specific UDP port or L2 Ethertype we are + * filtering in all UDP or all non-IP packets for timestamping. So + * if anything other than FILTER_ALL is requested we report + * FILTER_SOME indicating that we will be timestamping a few + * additional packets. + */ + if (config->rx_filter > HWTSTAMP_FILTER_ALL) + config->rx_filter = HWTSTAMP_FILTER_SOME; + + return 0; +} + static void fbnic_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64) { @@ -394,6 +480,8 @@ static const struct net_device_ops fbnic_netdev_ops = { .ndo_set_mac_address = fbnic_set_mac, .ndo_set_rx_mode = fbnic_set_rx_mode, .ndo_get_stats64 = fbnic_get_stats64, + .ndo_hwtstamp_get = fbnic_hwtstamp_get, + .ndo_hwtstamp_set = fbnic_hwtstamp_set, }; static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h index 6c27da09a612..b8417b300778 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h @@ -33,6 +33,15 @@ struct fbnic_net { u8 fec; u8 link_mode; + /* Cached top bits of the HW time counter for 40b -> 64b conversion */ + u32 time_high; + /* Protect readers of @time_offset, writers take @time_lock. */ + struct u64_stats_sync time_seq; + /* Offset in ns between free running NIC PHC and time set via PTP + * clock callbacks + */ + s64 time_offset; + u16 num_tx_queues; u16 num_rx_queues; @@ -45,6 +54,9 @@ struct fbnic_net { struct fbnic_queue_stats rx_stats; u64 link_down_events; + /* Time stampinn filter config */ + struct kernel_hwtstamp_config hwtstamp_config; + struct list_head napis; }; @@ -60,6 +72,12 @@ void fbnic_reset_queues(struct fbnic_net *fbn, unsigned int tx, unsigned int rx); void fbnic_set_ethtool_ops(struct net_device *dev); +int fbnic_ptp_setup(struct fbnic_dev *fbd); +void fbnic_ptp_destroy(struct fbnic_dev *fbd); +void fbnic_time_init(struct fbnic_net *fbn); +int fbnic_time_start(struct fbnic_net *fbn); +void fbnic_time_stop(struct fbnic_net *fbn); + void __fbnic_set_rx_mode(struct net_device *netdev); void fbnic_clear_rx_mode(struct net_device *netdev); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index a4809fe0fc24..32702dc4a066 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -9,6 +9,7 @@ #include "fbnic.h" #include "fbnic_drvinfo.h" +#include "fbnic_hw_stats.h" #include "fbnic_netdev.h" char fbnic_driver_name[] = DRV_NAME; @@ -198,6 +199,8 @@ static void fbnic_service_task(struct work_struct *work) rtnl_lock(); + fbnic_get_hw_stats32(fbd); + fbnic_fw_check_heartbeat(fbd); fbnic_health_check(fbd); @@ -288,6 +291,12 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } fbnic_devlink_register(fbd); + fbnic_dbg_fbd_init(fbd); + + /* Capture snapshot of hardware stats so netdev can calculate delta */ + fbnic_reset_hw_stats(fbd); + + fbnic_hwmon_register(fbd); if (!fbd->dsn) { dev_warn(&pdev->dev, "Reading serial number failed\n"); @@ -300,14 +309,20 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto init_failure_mode; } + err = fbnic_ptp_setup(fbd); + if (err) + goto ifm_free_netdev; + err = fbnic_netdev_register(netdev); if (err) { dev_err(&pdev->dev, "Netdev registration failed: %d\n", err); - goto ifm_free_netdev; + goto ifm_destroy_ptp; } return 0; +ifm_destroy_ptp: + fbnic_ptp_destroy(fbd); ifm_free_netdev: fbnic_netdev_free(fbd); init_failure_mode: @@ -319,7 +334,6 @@ init_failure_mode: free_irqs: fbnic_free_irqs(fbd); free_fbd: - pci_disable_device(pdev); fbnic_devlink_free(fbd); return err; @@ -342,14 +356,16 @@ static void fbnic_remove(struct pci_dev *pdev) fbnic_netdev_unregister(netdev); cancel_delayed_work_sync(&fbd->service_task); + fbnic_ptp_destroy(fbd); fbnic_netdev_free(fbd); } + fbnic_hwmon_unregister(fbd); + fbnic_dbg_fbd_exit(fbd); fbnic_devlink_unregister(fbd); fbnic_fw_disable_mbx(fbd); fbnic_free_irqs(fbd); - pci_disable_device(pdev); fbnic_devlink_free(fbd); } @@ -542,9 +558,13 @@ static int __init fbnic_init_module(void) { int err; + fbnic_dbg_init(); + err = pci_register_driver(&fbnic_driver); - if (err) + if (err) { + fbnic_dbg_exit(); goto out; + } pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name); out: @@ -560,5 +580,7 @@ module_init(fbnic_init_module); static void __exit fbnic_exit_module(void) { pci_unregister_driver(&fbnic_driver); + + fbnic_dbg_exit(); } module_exit(fbnic_exit_module); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c index c8aa29fc052b..908c098cd59e 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -244,6 +244,12 @@ void fbnic_bmc_rpc_init(struct fbnic_dev *fbd) ((_ip) ? FBNIC_RPC_TCAM_ACT1_IP_VALID : 0) | \ ((_v6) ? FBNIC_RPC_TCAM_ACT1_IP_IS_V6 : 0)) +#define FBNIC_TSTAMP_MASK(_all, _udp, _ether) \ + (((_all) ? ((1u << FBNIC_NUM_HASH_OPT) - 1) : 0) | \ + ((_udp) ? (1u << FBNIC_UDP6_HASH_OPT) | \ + (1u << FBNIC_UDP4_HASH_OPT) : 0) | \ + ((_ether) ? (1u << FBNIC_ETHER_HASH_OPT) : 0)) + void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) { static const u32 act1_value[FBNIC_NUM_HASH_OPT] = { @@ -255,6 +261,7 @@ void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) FBNIC_ACT1_INIT(0, 0, 1, 0), /* IP4 */ 0 /* Ether */ }; + u32 tstamp_mask = 0; unsigned int i; /* To support scenarios where a BMC is present we must write the @@ -264,6 +271,28 @@ void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) BUILD_BUG_ON(FBNIC_RSS_EN_NUM_UNICAST * 2 != FBNIC_RSS_EN_NUM_ENTRIES); BUILD_BUG_ON(ARRAY_SIZE(act1_value) != FBNIC_NUM_HASH_OPT); + /* Set timestamp mask with 1b per flow type */ + if (fbn->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) { + switch (fbn->hwtstamp_config.rx_filter) { + case HWTSTAMP_FILTER_ALL: + tstamp_mask = FBNIC_TSTAMP_MASK(1, 1, 1); + break; + case HWTSTAMP_FILTER_PTP_V2_EVENT: + tstamp_mask = FBNIC_TSTAMP_MASK(0, 1, 1); + break; + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + tstamp_mask = FBNIC_TSTAMP_MASK(0, 1, 0); + break; + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + tstamp_mask = FBNIC_TSTAMP_MASK(0, 0, 1); + break; + default: + netdev_warn(fbn->netdev, "Unsupported hwtstamp_rx_filter\n"); + break; + } + } + /* Program RSS hash enable mask for host in action TCAM/table. */ for (i = fbnic_bmc_present(fbd) ? 0 : FBNIC_RSS_EN_NUM_UNICAST; i < FBNIC_RSS_EN_NUM_ENTRIES; i++) { @@ -287,6 +316,8 @@ void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) if (!dest) dest = FBNIC_RPC_ACT_TBL0_DROP; + else if (tstamp_mask & (1u << flow_type)) + dest |= FBNIC_RPC_ACT_TBL0_TS_ENA; if (act1_value[flow_type] & FBNIC_RPC_TCAM_ACT1_L4_VALID) dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, @@ -556,6 +587,116 @@ static void fbnic_clear_act_tcam(struct fbnic_dev *fbd, unsigned int idx) wr32(fbd, FBNIC_RPC_TCAM_ACT(idx, i), 0); } +static void fbnic_clear_tce_tcam_entry(struct fbnic_dev *fbd, unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_TCE_TCAM_WORD_LEN; i++) + wr32(fbd, FBNIC_TCE_RAM_TCAM(idx, i), 0); +} + +static void fbnic_write_tce_tcam_dest(struct fbnic_dev *fbd, unsigned int idx, + struct fbnic_mac_addr *mac_addr) +{ + u32 dest = FBNIC_TCE_TCAM_DEST_BMC; + u32 idx2dest_map; + + if (is_multicast_ether_addr(mac_addr->value.addr8)) + dest |= FBNIC_TCE_TCAM_DEST_MAC; + + idx2dest_map = rd32(fbd, FBNIC_TCE_TCAM_IDX2DEST_MAP); + idx2dest_map &= ~(FBNIC_TCE_TCAM_IDX2DEST_MAP_DEST_ID_0 << (4 * idx)); + idx2dest_map |= dest << (4 * idx); + + wr32(fbd, FBNIC_TCE_TCAM_IDX2DEST_MAP, idx2dest_map); +} + +static void fbnic_write_tce_tcam_entry(struct fbnic_dev *fbd, unsigned int idx, + struct fbnic_mac_addr *mac_addr) +{ + __be16 *mask, *value; + int i; + + mask = &mac_addr->mask.addr16[FBNIC_TCE_TCAM_WORD_LEN - 1]; + value = &mac_addr->value.addr16[FBNIC_TCE_TCAM_WORD_LEN - 1]; + + for (i = 0; i < FBNIC_TCE_TCAM_WORD_LEN; i++) + wr32(fbd, FBNIC_TCE_RAM_TCAM(idx, i), + FIELD_PREP(FBNIC_TCE_RAM_TCAM_MASK, ntohs(*mask--)) | + FIELD_PREP(FBNIC_TCE_RAM_TCAM_VALUE, ntohs(*value--))); + + wrfl(fbd); + + wr32(fbd, FBNIC_TCE_RAM_TCAM3(idx), FBNIC_TCE_RAM_TCAM3_MCQ_MASK | + FBNIC_TCE_RAM_TCAM3_DEST_MASK | + FBNIC_TCE_RAM_TCAM3_VALIDATE); +} + +static void __fbnic_write_tce_tcam_rev(struct fbnic_dev *fbd) +{ + int tcam_idx = FBNIC_TCE_TCAM_NUM_ENTRIES; + int mac_idx; + + for (mac_idx = ARRAY_SIZE(fbd->mac_addr); mac_idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[mac_idx]; + + /* Verify BMC bit is set */ + if (!test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) + continue; + + if (!tcam_idx) { + dev_err(fbd->dev, "TCE TCAM overflow\n"); + return; + } + + tcam_idx--; + fbnic_write_tce_tcam_dest(fbd, tcam_idx, mac_addr); + fbnic_write_tce_tcam_entry(fbd, tcam_idx, mac_addr); + } + + while (tcam_idx) + fbnic_clear_tce_tcam_entry(fbd, --tcam_idx); + + fbd->tce_tcam_last = tcam_idx; +} + +static void __fbnic_write_tce_tcam(struct fbnic_dev *fbd) +{ + int tcam_idx = 0; + int mac_idx; + + for (mac_idx = 0; mac_idx < ARRAY_SIZE(fbd->mac_addr); mac_idx++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[mac_idx]; + + /* Verify BMC bit is set */ + if (!test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) + continue; + + if (tcam_idx == FBNIC_TCE_TCAM_NUM_ENTRIES) { + dev_err(fbd->dev, "TCE TCAM overflow\n"); + return; + } + + fbnic_write_tce_tcam_dest(fbd, tcam_idx, mac_addr); + fbnic_write_tce_tcam_entry(fbd, tcam_idx, mac_addr); + tcam_idx++; + } + + while (tcam_idx < FBNIC_TCE_TCAM_NUM_ENTRIES) + fbnic_clear_tce_tcam_entry(fbd, tcam_idx++); + + fbd->tce_tcam_last = tcam_idx; +} + +void fbnic_write_tce_tcam(struct fbnic_dev *fbd) +{ + if (fbd->tce_tcam_last) + __fbnic_write_tce_tcam_rev(fbd); + else + __fbnic_write_tce_tcam(fbd); +} + void fbnic_clear_rules(struct fbnic_dev *fbd) { u32 dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h index d62935f722a2..0d8285fa5b45 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h @@ -35,6 +35,9 @@ enum { #define FBNIC_RPC_TCAM_ACT_WORD_LEN 11 #define FBNIC_RPC_TCAM_ACT_NUM_ENTRIES 64 +#define FBNIC_TCE_TCAM_WORD_LEN 3 +#define FBNIC_TCE_TCAM_NUM_ENTRIES 8 + struct fbnic_mac_addr { union { unsigned char addr8[ETH_ALEN]; @@ -186,4 +189,5 @@ static inline int __fbnic_mc_unsync(struct fbnic_mac_addr *mac_addr) void fbnic_clear_rules(struct fbnic_dev *fbd); void fbnic_write_rules(struct fbnic_dev *fbd); +void fbnic_write_tce_tcam(struct fbnic_dev *fbd); #endif /* _FBNIC_RPC_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_time.c b/drivers/net/ethernet/meta/fbnic/fbnic_time.c new file mode 100644 index 000000000000..39d99677b71e --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_time.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <linux/jiffies.h> +#include <linux/limits.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/timer.h> + +#include "fbnic.h" +#include "fbnic_csr.h" +#include "fbnic_netdev.h" + +/* FBNIC timing & PTP implementation + * Datapath uses truncated 40b timestamps for scheduling and event reporting. + * We need to promote those to full 64b, hence we periodically cache the top + * 32bit of the HW time counter. Since this makes our time reporting non-atomic + * we leave the HW clock free running and adjust time offsets in SW as needed. + * Time offset is 64bit - we need a seq counter for 32bit machines. + * Time offset and the cache of top bits are independent so we don't need + * a coherent snapshot of both - READ_ONCE()/WRITE_ONCE() + writer side lock + * are enough. + */ + +/* Period of refresh of top bits of timestamp, give ourselves a 8x margin. + * This should translate to once a minute. + * The use of nsecs_to_jiffies() should be safe for a <=40b nsec value. + */ +#define FBNIC_TS_HIGH_REFRESH_JIF nsecs_to_jiffies((1ULL << 40) / 16) + +static struct fbnic_dev *fbnic_from_ptp_info(struct ptp_clock_info *ptp) +{ + return container_of(ptp, struct fbnic_dev, ptp_info); +} + +/* This function is "slow" because we could try guessing which high part + * is correct based on low instead of re-reading, and skip reading @hi + * twice altogether if @lo is far enough from 0. + */ +static u64 __fbnic_time_get_slow(struct fbnic_dev *fbd) +{ + u32 hi, lo; + + lockdep_assert_held(&fbd->time_lock); + + do { + hi = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI); + lo = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_LO); + } while (hi != fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI)); + + return (u64)hi << 32 | lo; +} + +static void __fbnic_time_set_addend(struct fbnic_dev *fbd, u64 addend) +{ + lockdep_assert_held(&fbd->time_lock); + + fbnic_wr32(fbd, FBNIC_PTP_ADD_VAL_NS, + FIELD_PREP(FBNIC_PTP_ADD_VAL_NS_MASK, addend >> 32)); + fbnic_wr32(fbd, FBNIC_PTP_ADD_VAL_SUBNS, (u32)addend); +} + +static void fbnic_ptp_fresh_check(struct fbnic_dev *fbd) +{ + if (time_is_after_jiffies(fbd->last_read + + FBNIC_TS_HIGH_REFRESH_JIF * 3 / 2)) + return; + + dev_warn(fbd->dev, "NIC timestamp refresh stall, delayed by %lu sec\n", + (jiffies - fbd->last_read - FBNIC_TS_HIGH_REFRESH_JIF) / HZ); +} + +static void fbnic_ptp_refresh_time(struct fbnic_dev *fbd, struct fbnic_net *fbn) +{ + unsigned long flags; + u32 hi; + + spin_lock_irqsave(&fbd->time_lock, flags); + hi = fbnic_rd32(fbn->fbd, FBNIC_PTP_CTR_VAL_HI); + if (!fbnic_present(fbd)) + goto out; /* Don't bother handling, reset is pending */ + /* Let's keep high cached value a bit lower to avoid race with + * incoming timestamps. The logic in fbnic_ts40_to_ns() will + * take care of overflow in this case. It will make cached time + * ~1 minute lower and incoming timestamp will always be later + * then cached time. + */ + WRITE_ONCE(fbn->time_high, hi - 16); + fbd->last_read = jiffies; + out: + spin_unlock_irqrestore(&fbd->time_lock, flags); +} + +static long fbnic_ptp_do_aux_work(struct ptp_clock_info *ptp) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + + fbn = netdev_priv(fbd->netdev); + + fbnic_ptp_fresh_check(fbd); + fbnic_ptp_refresh_time(fbd, fbn); + + return FBNIC_TS_HIGH_REFRESH_JIF; +} + +static int fbnic_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + u64 addend, dclk_period; + unsigned long flags; + + /* d_clock is 600 MHz; which in Q16.32 fixed point ns is: */ + dclk_period = (((u64)1000000000) << 32) / FBNIC_CLOCK_FREQ; + addend = adjust_by_scaled_ppm(dclk_period, scaled_ppm); + + spin_lock_irqsave(&fbd->time_lock, flags); + __fbnic_time_set_addend(fbd, addend); + fbnic_wr32(fbd, FBNIC_PTP_ADJUST, FBNIC_PTP_ADJUST_ADDEND_SET); + + /* Flush, make sure FBNIC_PTP_ADD_VAL_* is stable for at least 4 clks */ + fbnic_rd32(fbd, FBNIC_PTP_SPARE); + spin_unlock_irqrestore(&fbd->time_lock, flags); + + return fbnic_present(fbd) ? 0 : -EIO; +} + +static int fbnic_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + unsigned long flags; + + fbn = netdev_priv(fbd->netdev); + + spin_lock_irqsave(&fbd->time_lock, flags); + u64_stats_update_begin(&fbn->time_seq); + WRITE_ONCE(fbn->time_offset, READ_ONCE(fbn->time_offset) + delta); + u64_stats_update_end(&fbn->time_seq); + spin_unlock_irqrestore(&fbd->time_lock, flags); + + return 0; +} + +static int +fbnic_ptp_gettimex64(struct ptp_clock_info *ptp, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + unsigned long flags; + u64 time_ns; + u32 hi, lo; + + fbn = netdev_priv(fbd->netdev); + + spin_lock_irqsave(&fbd->time_lock, flags); + + do { + hi = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI); + ptp_read_system_prets(sts); + lo = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_LO); + ptp_read_system_postts(sts); + /* Similarly to comment above __fbnic_time_get_slow() + * - this can be optimized if needed. + */ + } while (hi != fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI)); + + time_ns = ((u64)hi << 32 | lo) + fbn->time_offset; + spin_unlock_irqrestore(&fbd->time_lock, flags); + + if (!fbnic_present(fbd)) + return -EIO; + + *ts = ns_to_timespec64(time_ns); + + return 0; +} + +static int +fbnic_ptp_settime64(struct ptp_clock_info *ptp, const struct timespec64 *ts) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + unsigned long flags; + u64 dev_ns, host_ns; + int ret; + + fbn = netdev_priv(fbd->netdev); + + host_ns = timespec64_to_ns(ts); + + spin_lock_irqsave(&fbd->time_lock, flags); + + dev_ns = __fbnic_time_get_slow(fbd); + + if (fbnic_present(fbd)) { + u64_stats_update_begin(&fbn->time_seq); + WRITE_ONCE(fbn->time_offset, host_ns - dev_ns); + u64_stats_update_end(&fbn->time_seq); + ret = 0; + } else { + ret = -EIO; + } + spin_unlock_irqrestore(&fbd->time_lock, flags); + + return ret; +} + +static const struct ptp_clock_info fbnic_ptp_info = { + .owner = THIS_MODULE, + /* 1,000,000,000 - 1 PPB to ensure increment is positive + * after max negative adjustment. + */ + .max_adj = 999999999, + .do_aux_work = fbnic_ptp_do_aux_work, + .adjfine = fbnic_ptp_adjfine, + .adjtime = fbnic_ptp_adjtime, + .gettimex64 = fbnic_ptp_gettimex64, + .settime64 = fbnic_ptp_settime64, +}; + +static void fbnic_ptp_reset(struct fbnic_dev *fbd) +{ + struct fbnic_net *fbn = netdev_priv(fbd->netdev); + u64 dclk_period; + + fbnic_wr32(fbd, FBNIC_PTP_CTRL, + FBNIC_PTP_CTRL_EN | + FIELD_PREP(FBNIC_PTP_CTRL_TICK_IVAL, 1)); + + /* d_clock is 600 MHz; which in Q16.32 fixed point ns is: */ + dclk_period = (((u64)1000000000) << 32) / FBNIC_CLOCK_FREQ; + + __fbnic_time_set_addend(fbd, dclk_period); + + fbnic_wr32(fbd, FBNIC_PTP_INIT_HI, 0); + fbnic_wr32(fbd, FBNIC_PTP_INIT_LO, 0); + + fbnic_wr32(fbd, FBNIC_PTP_ADJUST, FBNIC_PTP_ADJUST_INIT); + + fbnic_wr32(fbd, FBNIC_PTP_CTRL, + FBNIC_PTP_CTRL_EN | + FBNIC_PTP_CTRL_TQS_OUT_EN | + FIELD_PREP(FBNIC_PTP_CTRL_MAC_OUT_IVAL, 3) | + FIELD_PREP(FBNIC_PTP_CTRL_TICK_IVAL, 1)); + + fbnic_rd32(fbd, FBNIC_PTP_SPARE); + + fbn->time_offset = 0; + fbn->time_high = 0; +} + +void fbnic_time_init(struct fbnic_net *fbn) +{ + /* This is not really a statistic, but the lockng primitive fits + * our usecase perfectly, we need an atomic 8 bytes READ_ONCE() / + * WRITE_ONCE() behavior. + */ + u64_stats_init(&fbn->time_seq); +} + +int fbnic_time_start(struct fbnic_net *fbn) +{ + fbnic_ptp_refresh_time(fbn->fbd, fbn); + /* Assume that fbnic_ptp_do_aux_work() will never be called if not + * scheduled here + */ + return ptp_schedule_worker(fbn->fbd->ptp, FBNIC_TS_HIGH_REFRESH_JIF); +} + +void fbnic_time_stop(struct fbnic_net *fbn) +{ + ptp_cancel_worker_sync(fbn->fbd->ptp); + fbnic_ptp_fresh_check(fbn->fbd); +} + +int fbnic_ptp_setup(struct fbnic_dev *fbd) +{ + struct device *dev = fbd->dev; + unsigned long flags; + + spin_lock_init(&fbd->time_lock); + + spin_lock_irqsave(&fbd->time_lock, flags); /* Appease lockdep */ + fbnic_ptp_reset(fbd); + spin_unlock_irqrestore(&fbd->time_lock, flags); + + memcpy(&fbd->ptp_info, &fbnic_ptp_info, sizeof(fbnic_ptp_info)); + + fbd->ptp = ptp_clock_register(&fbd->ptp_info, dev); + if (IS_ERR(fbd->ptp)) + dev_err(dev, "Failed to register PTP: %pe\n", fbd->ptp); + + return PTR_ERR_OR_ZERO(fbd->ptp); +} + +void fbnic_ptp_destroy(struct fbnic_dev *fbd) +{ + if (!fbd->ptp) + return; + ptp_clock_unregister(fbd->ptp); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 6a6d7e22f1a7..b5050fabe8fe 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -12,9 +12,14 @@ #include "fbnic_netdev.h" #include "fbnic_txrx.h" +enum { + FBNIC_XMIT_CB_TS = 0x01, +}; + struct fbnic_xmit_cb { u32 bytecount; u8 desc_count; + u8 flags; int hw_head; }; @@ -43,6 +48,46 @@ static void fbnic_ring_wr32(struct fbnic_ring *ring, unsigned int csr, u32 val) writel(val, csr_base + csr); } +/** + * fbnic_ts40_to_ns() - convert descriptor timestamp to PHC time + * @fbn: netdev priv of the FB NIC + * @ts40: timestamp read from a descriptor + * + * Return: u64 value of PHC time in nanoseconds + * + * Convert truncated 40 bit device timestamp as read from a descriptor + * to the full PHC time in nanoseconds. + */ +static __maybe_unused u64 fbnic_ts40_to_ns(struct fbnic_net *fbn, u64 ts40) +{ + unsigned int s; + u64 time_ns; + s64 offset; + u8 ts_top; + u32 high; + + do { + s = u64_stats_fetch_begin(&fbn->time_seq); + offset = READ_ONCE(fbn->time_offset); + } while (u64_stats_fetch_retry(&fbn->time_seq, s)); + + high = READ_ONCE(fbn->time_high); + + /* Bits 63..40 from periodic clock reads, 39..0 from ts40 */ + time_ns = (u64)(high >> 8) << 40 | ts40; + + /* Compare bits 32-39 between periodic reads and ts40, + * see if HW clock may have wrapped since last read. We are sure + * that periodic reads are always at least ~1 minute behind, so + * this logic works perfectly fine. + */ + ts_top = ts40 >> 32; + if (ts_top < (u8)high && (u8)high - ts_top > U8_MAX / 2) + time_ns += 1ULL << 40; + + return time_ns + offset; +} + static unsigned int fbnic_desc_unused(struct fbnic_ring *ring) { return (ring->head - ring->tail - 1) & ring->size_mask; @@ -110,11 +155,32 @@ static void fbnic_unmap_page_twd(struct device *dev, __le64 *twd) #define FBNIC_TWD_TYPE(_type) \ cpu_to_le64(FIELD_PREP(FBNIC_TWD_TYPE_MASK, FBNIC_TWD_TYPE_##_type)) +static bool fbnic_tx_tstamp(struct sk_buff *skb) +{ + struct fbnic_net *fbn; + + if (!unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + return false; + + fbn = netdev_priv(skb->dev); + if (fbn->hwtstamp_config.tx_type == HWTSTAMP_TX_OFF) + return false; + + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + FBNIC_XMIT_CB(skb)->flags |= FBNIC_XMIT_CB_TS; + FBNIC_XMIT_CB(skb)->hw_head = -1; + + return true; +} + static bool fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) { unsigned int l2len, i3len; + if (fbnic_tx_tstamp(skb)) + *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_TS); + if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) return false; @@ -205,6 +271,9 @@ fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) ring->tail = tail; + /* Record SW timestamp */ + skb_tx_timestamp(skb); + /* Verify there is room for another packet */ fbnic_maybe_stop_tx(skb->dev, ring, FBNIC_MAX_SKB_DESC); @@ -316,7 +385,7 @@ static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, struct fbnic_ring *ring, bool discard, unsigned int hw_head) { - u64 total_bytes = 0, total_packets = 0; + u64 total_bytes = 0, total_packets = 0, ts_lost = 0; unsigned int head = ring->head; struct netdev_queue *txq; unsigned int clean_desc; @@ -331,6 +400,13 @@ static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, if (desc_cnt > clean_desc) break; + if (unlikely(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)) { + FBNIC_XMIT_CB(skb)->hw_head = hw_head; + if (likely(!discard)) + break; + ts_lost++; + } + ring->tx_buf[head] = NULL; clean_desc -= desc_cnt; @@ -368,6 +444,7 @@ static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, if (unlikely(discard)) { u64_stats_update_begin(&ring->stats.syncp); ring->stats.dropped += total_packets; + ring->stats.ts_lost += ts_lost; u64_stats_update_end(&ring->stats.syncp); netdev_tx_completed_queue(txq, total_packets, total_bytes); @@ -384,6 +461,56 @@ static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, FBNIC_TX_DESC_WAKEUP); } +static void fbnic_clean_tsq(struct fbnic_napi_vector *nv, + struct fbnic_ring *ring, + u64 tcd, int *ts_head, int *head0) +{ + struct skb_shared_hwtstamps hwtstamp; + struct fbnic_net *fbn; + struct sk_buff *skb; + int head; + u64 ns; + + head = (*ts_head < 0) ? ring->head : *ts_head; + + do { + unsigned int desc_cnt; + + if (head == ring->tail) { + if (unlikely(net_ratelimit())) + netdev_err(nv->napi.dev, + "Tx timestamp without matching packet\n"); + return; + } + + skb = ring->tx_buf[head]; + desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; + + head += desc_cnt; + head &= ring->size_mask; + } while (!(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)); + + fbn = netdev_priv(nv->napi.dev); + ns = fbnic_ts40_to_ns(fbn, FIELD_GET(FBNIC_TCD_TYPE1_TS_MASK, tcd)); + + memset(&hwtstamp, 0, sizeof(hwtstamp)); + hwtstamp.hwtstamp = ns_to_ktime(ns); + + *ts_head = head; + + FBNIC_XMIT_CB(skb)->flags &= ~FBNIC_XMIT_CB_TS; + if (*head0 < 0) { + head = FBNIC_XMIT_CB(skb)->hw_head; + if (head >= 0) + *head0 = head; + } + + skb_tstamp_tx(skb, &hwtstamp); + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.ts_packets++; + u64_stats_update_end(&ring->stats.syncp); +} + static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, struct page *page) { @@ -417,10 +544,12 @@ static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx, } static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, - struct fbnic_q_triad *qt, s32 head0) + struct fbnic_q_triad *qt, s32 ts_head, s32 head0) { if (head0 >= 0) fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); + else if (ts_head >= 0) + fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head); } static void @@ -428,9 +557,9 @@ fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, int napi_budget) { struct fbnic_ring *cmpl = &qt->cmpl; + s32 head0 = -1, ts_head = -1; __le64 *raw_tcd, done; u32 head = cmpl->head; - s32 head0 = -1; done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(FBNIC_TCD_DONE); raw_tcd = &cmpl->desc[head & cmpl->size_mask]; @@ -453,6 +582,12 @@ fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, * they are skipped for now. */ break; + case FBNIC_TCD_TYPE_1: + if (WARN_ON_ONCE(tcd & FBNIC_TCD_TWQ1)) + break; + + fbnic_clean_tsq(nv, &qt->sub0, tcd, &ts_head, &head0); + break; default: break; } @@ -472,7 +607,7 @@ fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, } /* Unmap and free processed buffers */ - fbnic_clean_twq(nv, napi_budget, qt, head0); + fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0); } static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget, @@ -707,6 +842,10 @@ static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, /* Set MAC header specific fields */ skb->protocol = eth_type_trans(skb, nv->napi.dev); + /* Add timestamp if present */ + if (pkt->hwtstamp) + skb_hwtstamps(skb)->hwtstamp = pkt->hwtstamp; + return skb; } @@ -717,6 +856,23 @@ static enum pkt_hash_types fbnic_skb_hash_type(u64 rcd) PKT_HASH_TYPE_L2; } +static void fbnic_rx_tstamp(struct fbnic_napi_vector *nv, u64 rcd, + struct fbnic_pkt_buff *pkt) +{ + struct fbnic_net *fbn; + u64 ns, ts; + + if (!FIELD_GET(FBNIC_RCD_OPT_META_TS, rcd)) + return; + + fbn = netdev_priv(nv->napi.dev); + ts = FIELD_GET(FBNIC_RCD_OPT_META_TS_MASK, rcd); + ns = fbnic_ts40_to_ns(fbn, ts); + + /* Add timestamp to shared info */ + pkt->hwtstamp = ns_to_ktime(ns); +} + static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv, u64 rcd, struct sk_buff *skb, struct fbnic_q_triad *qt) @@ -781,6 +937,8 @@ static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, if (FIELD_GET(FBNIC_RCD_OPT_META_TYPE_MASK, rcd)) break; + fbnic_rx_tstamp(nv, rcd, pkt); + /* We currently ignore the action table index */ break; case FBNIC_RCD_TYPE_META: @@ -907,6 +1065,8 @@ static void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, fbn->tx_stats.bytes += stats->bytes; fbn->tx_stats.packets += stats->packets; fbn->tx_stats.dropped += stats->dropped; + fbn->tx_stats.ts_lost += stats->ts_lost; + fbn->tx_stats.ts_packets += stats->ts_packets; } static void fbnic_remove_tx_ring(struct fbnic_net *fbn, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index 2f91f68d11d5..8d626287c3f4 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -47,6 +47,7 @@ struct fbnic_net; struct fbnic_pkt_buff { struct xdp_buff buff; + ktime_t hwtstamp; u32 data_truesize; u16 data_len; u16 nr_frags; @@ -56,6 +57,8 @@ struct fbnic_queue_stats { u64 packets; u64 bytes; u64 dropped; + u64 ts_packets; + u64 ts_lost; struct u64_stats_sync syncp; }; |