diff options
Diffstat (limited to 'drivers/pci/controller/dwc/pcie-tegra194.c')
| -rw-r--r-- | drivers/pci/controller/dwc/pcie-tegra194.c | 1080 |
1 files changed, 588 insertions, 492 deletions
diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c index 92b77f7d8354..0ddeef70726d 100644 --- a/drivers/pci/controller/dwc/pcie-tegra194.c +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -1,25 +1,25 @@ // SPDX-License-Identifier: GPL-2.0+ /* - * PCIe host controller driver for Tegra194 SoC + * PCIe host controller driver for the following SoCs + * Tegra194 + * Tegra234 * - * Copyright (C) 2019 NVIDIA Corporation. + * Copyright (C) 2019-2022 NVIDIA Corporation. * * Author: Vidya Sagar <vidyas@nvidia.com> */ +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/debugfs.h> #include <linux/delay.h> -#include <linux/gpio.h> #include <linux/gpio/consumer.h> +#include <linux/interconnect.h> #include <linux/interrupt.h> #include <linux/iopoll.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_gpio.h> -#include <linux/of_irq.h> #include <linux/of_pci.h> #include <linux/pci.h> #include <linux/phy/phy.h> @@ -35,6 +35,9 @@ #include <soc/tegra/bpmp-abi.h> #include "../../pci.h" +#define TEGRA194_DWC_IP_VER 0x490A +#define TEGRA234_DWC_IP_VER 0x562A + #define APPL_PINMUX 0x0 #define APPL_PINMUX_PEX_RST BIT(0) #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) @@ -49,6 +52,7 @@ #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 +#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2 #define APPL_INTR_EN_L0_0 0x8 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) @@ -120,7 +124,7 @@ #define APPL_LTR_MSG_1 0xC4 #define LTR_MSG_REQ BIT(15) -#define LTR_MST_NO_SNOOP_SHIFT 16 +#define LTR_NOSNOOP_MSG_REQ BIT(31) #define APPL_LTR_MSG_2 0xC8 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) @@ -170,53 +174,15 @@ #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) -#define EVENT_COUNTER_ALL_CLEAR 0x3 -#define EVENT_COUNTER_ENABLE_ALL 0x7 -#define EVENT_COUNTER_ENABLE_SHIFT 2 -#define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) -#define EVENT_COUNTER_EVENT_SEL_SHIFT 16 -#define EVENT_COUNTER_EVENT_Tx_L0S 0x2 -#define EVENT_COUNTER_EVENT_Rx_L0S 0x3 -#define EVENT_COUNTER_EVENT_L1 0x5 -#define EVENT_COUNTER_EVENT_L1_1 0x7 -#define EVENT_COUNTER_EVENT_L1_2 0x8 -#define EVENT_COUNTER_GROUP_SEL_SHIFT 24 -#define EVENT_COUNTER_GROUP_5 0x5 - -#define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C -#define ENTER_ASPM BIT(30) -#define L0S_ENTRANCE_LAT_SHIFT 24 -#define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) -#define L1_ENTRANCE_LAT_SHIFT 27 -#define L1_ENTRANCE_LAT_MASK GENMASK(29, 27) -#define N_FTS_SHIFT 8 -#define N_FTS_MASK GENMASK(7, 0) #define N_FTS_VAL 52 - -#define PORT_LOGIC_GEN2_CTRL 0x80C -#define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17) -#define FTS_MASK GENMASK(7, 0) #define FTS_VAL 52 -#define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828 - -#define GEN3_EQ_CONTROL_OFF 0x8a8 -#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8 -#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8) -#define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0) - -#define GEN3_RELATED_OFF 0x890 -#define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) -#define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) -#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 -#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) - #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 -#define AMBA_ERROR_RESPONSE_CRS_SHIFT 3 -#define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0) -#define AMBA_ERROR_RESPONSE_CRS_OKAY 0 -#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1 -#define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2 +#define AMBA_ERROR_RESPONSE_RRS_SHIFT 3 +#define AMBA_ERROR_RESPONSE_RRS_MASK GENMASK(1, 0) +#define AMBA_ERROR_RESPONSE_RRS_OKAY 0 +#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFFFFFF 1 +#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 2 #define MSIX_ADDR_MATCH_LOW_OFF 0x940 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) @@ -251,28 +217,23 @@ #define EP_STATE_ENABLED 1 static const unsigned int pcie_gen_freq[] = { + GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */ GEN1_CORE_CLK_FREQ, GEN2_CORE_CLK_FREQ, GEN3_CORE_CLK_FREQ, GEN4_CORE_CLK_FREQ }; -static const u32 event_cntr_ctrl_offset[] = { - 0x1d8, - 0x1a8, - 0x1a8, - 0x1a8, - 0x1c4, - 0x1d8 -}; - -static const u32 event_cntr_data_offset[] = { - 0x1dc, - 0x1ac, - 0x1ac, - 0x1ac, - 0x1c8, - 0x1dc +struct tegra_pcie_dw_of_data { + u32 version; + enum dw_pcie_device_mode mode; + bool has_msix_doorbell_access_fix; + bool has_sbr_reset_fix; + bool has_l1ss_exit_fix; + bool has_ltr_req_fix; + u32 cdm_chk_int_en_bit; + u32 gen4_preset_vec; + u8 n_fts[2]; }; struct tegra_pcie_dw { @@ -287,18 +248,19 @@ struct tegra_pcie_dw { struct dw_pcie pci; struct tegra_bpmp *bpmp; - enum dw_pcie_device_mode mode; + struct tegra_pcie_dw_of_data *of_data; bool supports_clkreq; bool enable_cdm_check; + bool enable_srns; bool link_state; bool update_fc_fixup; + bool enable_ext_refclk; u8 init_link_width; u32 msi_ctrl_int; u32 num_lanes; - u32 max_speed; u32 cid; - u32 cfg_link_cap_l1sub; + u32 ras_des_cap; u32 pcie_cap_base; u32 aspm_cmrt; u32 aspm_pwr_on_t; @@ -318,10 +280,8 @@ struct tegra_pcie_dw { struct gpio_desc *pex_refclk_sel_gpiod; unsigned int pex_rst_irq; int ep_state; -}; - -struct tegra_pcie_dw_of_data { - enum dw_pcie_device_mode mode; + long link_status; + struct icc_path *icc_path; }; static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) @@ -340,11 +300,28 @@ static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) return readl_relaxed(pcie->appl_base + reg); } -struct tegra_pcie_soc { - enum dw_pcie_device_mode mode; -}; +static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 val, speed, width; -static void apply_bad_link_workaround(struct pcie_port *pp) + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); + + speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val); + + val = width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed]); + + if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0)) + dev_err(pcie->dev, "can't set bw[%u]\n", val); + + if (speed >= ARRAY_SIZE(pcie_gen_freq)) + speed = 0; + + clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]); +} + +static void apply_bad_link_workaround(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); @@ -358,8 +335,7 @@ static void apply_bad_link_workaround(struct pcie_port *pp) */ val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); if (val & PCI_EXP_LNKSTA_LBMS) { - current_link_width = (val & PCI_EXP_LNKSTA_NLW) >> - PCI_EXP_LNKSTA_NLW_SHIFT; + current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val); if (pcie->init_link_width > current_link_width) { dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + @@ -382,16 +358,16 @@ static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) { struct tegra_pcie_dw *pcie = arg; struct dw_pcie *pci = &pcie->pci; - struct pcie_port *pp = &pci->pp; - u32 val, tmp; + struct dw_pcie_rp *pp = &pci->pp; + u32 val, status_l0, status_l1; u16 val_w; - val = appl_readl(pcie, APPL_INTR_STATUS_L0); - if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { - val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); - if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { - appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); - + status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0); + if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0); + if (!pcie->of_data->has_sbr_reset_fix && + status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { /* SBR & Surprise Link Down WAR */ val = appl_readl(pcie, APPL_CAR_RESET_OVRD); val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; @@ -401,21 +377,27 @@ static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; appl_writel(pcie, val, APPL_CAR_RESET_OVRD); - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); - val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE; - dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val |= PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); } } - if (val & APPL_INTR_STATUS_L0_INT_INT) { - val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); - if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { + if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); + if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { appl_writel(pcie, APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, APPL_INTR_STATUS_L1_8_0); apply_bad_link_workaround(pp); } - if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { + if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { + val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + val_w |= PCI_EXP_LNKSTA_LBMS; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA, val_w); + appl_writel(pcie, APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, APPL_INTR_STATUS_L1_8_0); @@ -427,25 +409,24 @@ static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) } } - val = appl_readl(pcie, APPL_INTR_STATUS_L0); - if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { - val = appl_readl(pcie, APPL_INTR_STATUS_L1_18); - tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); - if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { + if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18); + val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); + if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { dev_info(pci->dev, "CDM check complete\n"); - tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; + val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; } - if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { + if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { dev_err(pci->dev, "CDM comparison mismatch\n"); - tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; + val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; } - if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { + if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { dev_err(pci->dev, "CDM Logic error\n"); - tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; + val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; } - dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp); - tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); - dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp); + dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val); + val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); + dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val); } return IRQ_HANDLED; @@ -480,16 +461,20 @@ static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie) static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) { struct tegra_pcie_dw *pcie = arg; + struct dw_pcie_ep *ep = &pcie->pci.ep; struct dw_pcie *pci = &pcie->pci; - u32 val, speed; + u32 val; - speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & - PCI_EXP_LNKSTA_CLS; - clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); + if (test_and_clear_bit(0, &pcie->link_status)) + dw_pcie_ep_linkup(ep); + + tegra_pcie_icc_set(pcie); + + if (pcie->of_data->has_ltr_req_fix) + return IRQ_HANDLED; /* If EP doesn't advertise L1SS, just return */ - val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); - if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2))) + if (!pci->l1ss_support) return IRQ_HANDLED; /* Check if BME is set to '1' */ @@ -498,8 +483,12 @@ static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) ktime_t timeout; /* 110us for both snoop and no-snoop */ - val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ; - val |= (val << LTR_MST_NO_SNOOP_SHIFT); + val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) | + FIELD_PREP(PCI_LTR_SCALE_MASK, 2) | + LTR_MSG_REQ | + FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) | + FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) | + LTR_NOSNOOP_MSG_REQ; appl_writel(pcie, val, APPL_LTR_MSG_1); /* Send LTR upstream */ @@ -526,34 +515,34 @@ static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) { struct tegra_pcie_dw *pcie = arg; - struct dw_pcie_ep *ep = &pcie->pci.ep; int spurious = 1; - u32 val, tmp; + u32 status_l0, status_l1, link_status; - val = appl_readl(pcie, APPL_INTR_STATUS_L0); - if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) { - val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); - appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0); + status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0); + if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0); - if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) + if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) pex_ep_event_hot_rst_done(pcie); - if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { - tmp = appl_readl(pcie, APPL_LINK_STATUS); - if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) { + if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { + link_status = appl_readl(pcie, APPL_LINK_STATUS); + if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) { dev_dbg(pcie->dev, "Link is up with Host\n"); - dw_pcie_ep_linkup(ep); + set_bit(0, &pcie->link_status); + return IRQ_WAKE_THREAD; } } spurious = 0; } - if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { - val = appl_readl(pcie, APPL_INTR_STATUS_L1_15); - appl_writel(pcie, val, APPL_INTR_STATUS_L1_15); + if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15); - if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) + if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) return IRQ_WAKE_THREAD; spurious = 0; @@ -561,17 +550,19 @@ static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) if (spurious) { dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", - val); - appl_writel(pcie, val, APPL_INTR_STATUS_L0); + status_l0); + appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0); } return IRQ_HANDLED; } -static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size, - u32 *val) +static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 *val) { + struct dw_pcie_rp *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); /* * This is an endpoint mode specific register happen to appear even @@ -579,18 +570,21 @@ static int tegra_pcie_dw_rd_own_conf(struct pcie_port *pp, int where, int size, * when it is accessed with link being in ASPM-L1 state. * So skip accessing it altogether */ - if (where == PORT_LOGIC_MSIX_DOORBELL) { + if (!pcie->of_data->has_msix_doorbell_access_fix && + !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) { *val = 0x00000000; return PCIBIOS_SUCCESSFUL; } - return dw_pcie_read(pci->dbi_base + where, size, val); + return pci_generic_config_read(bus, devfn, where, size, val); } -static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size, - u32 val) +static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 val) { + struct dw_pcie_rp *pp = bus->sysdata; struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); /* * This is an endpoint mode specific register happen to appear even @@ -598,42 +592,34 @@ static int tegra_pcie_dw_wr_own_conf(struct pcie_port *pp, int where, int size, * when it is accessed with link being in ASPM-L1 state. * So skip accessing it altogether */ - if (where == PORT_LOGIC_MSIX_DOORBELL) + if (!pcie->of_data->has_msix_doorbell_access_fix && + !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) return PCIBIOS_SUCCESSFUL; - return dw_pcie_write(pci->dbi_base + where, size, val); -} - -#if defined(CONFIG_PCIEASPM) -static void disable_aspm_l11(struct tegra_pcie_dw *pcie) -{ - u32 val; - - val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); - val &= ~PCI_L1SS_CAP_ASPM_L1_1; - dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); + return pci_generic_config_write(bus, devfn, where, size, val); } -static void disable_aspm_l12(struct tegra_pcie_dw *pcie) -{ - u32 val; - - val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub); - val &= ~PCI_L1SS_CAP_ASPM_L1_2; - dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val); -} +static struct pci_ops tegra_pci_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = tegra_pcie_dw_rd_own_conf, + .write = tegra_pcie_dw_wr_own_conf, +}; +#if defined(CONFIG_PCIEASPM) static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) { u32 val; - val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]); + val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL); val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; - dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); - val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]); + dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); + val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_DATA); return val; } @@ -660,13 +646,15 @@ static int aspm_state_cnt(struct seq_file *s, void *data) event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); /* Clear all counters */ - dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], + dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, EVENT_COUNTER_ALL_CLEAR); /* Re-enable counting */ val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; - dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val); + dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); return 0; } @@ -674,51 +662,57 @@ static int aspm_state_cnt(struct seq_file *s, void *data) static void init_host_aspm(struct tegra_pcie_dw *pcie) { struct dw_pcie *pci = &pcie->pci; - u32 val; + u32 l1ss, val; - val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); - pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP; + l1ss = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); + + pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci, + PCI_EXT_CAP_ID_VNDR); /* Enable ASPM counters */ val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; - dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val); + dw_pcie_writel_dbi(pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); /* Program T_cmrt and T_pwr_on values */ - val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub); + val = dw_pcie_readl_dbi(pci, l1ss + PCI_L1SS_CAP); val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); val |= (pcie->aspm_cmrt << 8); val |= (pcie->aspm_pwr_on_t << 19); - dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val); + dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, val); + + if (pcie->supports_clkreq) + pci->l1ss_support = true; /* Program L0s and L1 entrance latencies */ - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); - val &= ~L0S_ENTRANCE_LAT_MASK; - val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT); - val |= ENTER_ASPM; - dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); + val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); + val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; + val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT); + val |= PORT_AFR_ENTER_ASPM; + dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); } -static int init_debugfs(struct tegra_pcie_dw *pcie) +static void init_debugfs(struct tegra_pcie_dw *pcie) { - struct dentry *d; + struct device *dev = pcie->dev; + char *name; - d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt", - pcie->debugfs, aspm_state_cnt); - if (IS_ERR_OR_NULL(d)) - dev_err(pcie->dev, - "Failed to create debugfs file \"aspm_state_cnt\"\n"); + name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); + if (!name) + return; - return 0; + pcie->debugfs = debugfs_create_dir(name, NULL); + + debugfs_create_devm_seqfile(dev, "aspm_state_cnt", pcie->debugfs, + aspm_state_cnt); } #else -static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; } -static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; } static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } -static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; } +static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; } #endif -static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) +static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); @@ -729,13 +723,15 @@ static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; appl_writel(pcie, val, APPL_INTR_EN_L0_0); - val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); - val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; - appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); + if (!pcie->of_data->has_sbr_reset_fix) { + val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); + val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); + } if (pcie->enable_cdm_check) { val = appl_readl(pcie, APPL_INTR_EN_L0_0); - val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN; + val |= pcie->of_data->cdm_chk_int_en_bit; appl_writel(pcie, val, APPL_INTR_EN_L0_0); val = appl_readl(pcie, APPL_INTR_EN_L1_18); @@ -746,8 +742,7 @@ static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); - pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >> - PCI_EXP_LNKSTA_NLW_SHIFT; + pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w); val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL); @@ -756,13 +751,13 @@ static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp) val_w); } -static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) +static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val; - /* Enable legacy interrupt generation */ + /* Enable INTX interrupt generation */ val = appl_readl(pcie, APPL_INTR_EN_L0_0); val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; val |= APPL_INTR_EN_L0_0_INT_INT_EN; @@ -777,14 +772,12 @@ static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp) appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); } -static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) +static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val; - dw_pcie_msi_init(pp); - /* Enable MSI interrupt generation */ val = appl_readl(pcie, APPL_INTR_EN_L0_0); val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; @@ -792,7 +785,7 @@ static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp) appl_writel(pcie, val, APPL_INTR_EN_L0_0); } -static void tegra_pcie_enable_interrupts(struct pcie_port *pp) +static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); @@ -815,7 +808,7 @@ static void tegra_pcie_enable_interrupts(struct pcie_port *pp) appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); tegra_pcie_enable_system_interrupts(pp); - tegra_pcie_enable_legacy_interrupts(pp); + tegra_pcie_enable_intx_interrupts(pp); if (IS_ENABLED(CONFIG_PCI_MSI)) tegra_pcie_enable_msi_interrupts(pp); } @@ -827,26 +820,24 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) /* Program init preset */ for (i = 0; i < pcie->num_lanes; i++) { - dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF - + (i * 2), 2, &val); + val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2)); val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; val |= GEN3_GEN4_EQ_PRESET_INIT; val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; val |= (GEN3_GEN4_EQ_PRESET_INIT << CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); - dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF - + (i * 2), 2, val); + dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val); offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_PL_16GT) + PCI_PL_16GT_LE_CTRL; - dw_pcie_read(pci->dbi_base + offset + i, 1, &val); + val = dw_pcie_readb_dbi(pci, offset + i); val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; val |= GEN3_GEN4_EQ_PRESET_INIT; val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; val |= (GEN3_GEN4_EQ_PRESET_INIT << PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); - dw_pcie_write(pci->dbi_base + offset + i, 1, val); + dw_pcie_writeb_dbi(pci, offset + i, val); } val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); @@ -854,9 +845,9 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); - val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; - val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); - val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; + val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC; + val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, 0x3ff); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE; dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); @@ -865,9 +856,10 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); - val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK; - val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT); - val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK; + val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC; + val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, + pcie->of_data->gen4_preset_vec); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE; dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); @@ -875,11 +867,18 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); } -static void tegra_pcie_prepare_host(struct pcie_port *pp) +static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp) { struct dw_pcie *pci = to_dw_pcie_from_pp(pp); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val; + u16 val_16; + + pp->bridge->ops = &tegra_pci_ops; + + if (!pcie->pcie_cap_base) + pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, + PCI_CAP_ID_EXP); val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); @@ -892,47 +891,31 @@ static void tegra_pcie_prepare_host(struct pcie_port *pp) dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); - /* Configure FTS */ - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); - val &= ~(N_FTS_MASK << N_FTS_SHIFT); - val |= N_FTS_VAL << N_FTS_SHIFT; - dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); - - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); - val &= ~FTS_MASK; - val |= FTS_VAL; - dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); - - /* Enable as 0xFFFF0001 response for CRS */ + /* Enable as 0xFFFF0001 response for RRS */ val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); - val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT); - val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 << - AMBA_ERROR_RESPONSE_CRS_SHIFT); + val &= ~(AMBA_ERROR_RESPONSE_RRS_MASK << AMBA_ERROR_RESPONSE_RRS_SHIFT); + val |= (AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 << + AMBA_ERROR_RESPONSE_RRS_SHIFT); dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); - /* Configure Max Speed from DT */ - if (pcie->max_speed && pcie->max_speed != -EINVAL) { - val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + - PCI_EXP_LNKCAP); - val &= ~PCI_EXP_LNKCAP_SLS; - val |= pcie->max_speed; - dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, - val); + /* Clear Slot Clock Configuration bit if SRNS configuration */ + if (pcie->enable_srns) { + val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + val_16 &= ~PCI_EXP_LNKSTA_SLC; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA, + val_16); } - /* Configure Max lane width from DT */ - val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP); - val &= ~PCI_EXP_LNKCAP_MLW; - val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT); - dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val); - config_gen3_gen4_eq_presets(pcie); init_host_aspm(pcie); - val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); - val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; - dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + if (!pcie->of_data->has_l1ss_exit_fix) { + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + } if (pcie->update_fc_fixup) { val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); @@ -940,10 +923,24 @@ static void tegra_pcie_prepare_host(struct pcie_port *pp) dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); } - dw_pcie_setup_rc(pp); - clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); + return 0; +} + +static int tegra_pcie_dw_start_link(struct dw_pcie *pci) +{ + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + struct dw_pcie_rp *pp = &pci->pp; + u32 val, offset, tmp; + bool retry = true; + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { + enable_irq(pcie->pex_rst_irq); + return 0; + } + +retry_link: /* Assert RST */ val = appl_readl(pcie, APPL_PINMUX); val &= ~APPL_PINMUX_PEX_RST; @@ -962,17 +959,10 @@ static void tegra_pcie_prepare_host(struct pcie_port *pp) appl_writel(pcie, val, APPL_PINMUX); msleep(100); -} - -static int tegra_pcie_dw_host_init(struct pcie_port *pp) -{ - struct dw_pcie *pci = to_dw_pcie_from_pp(pp); - struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); - u32 val, tmp, offset, speed; - - tegra_pcie_prepare_host(pp); if (dw_pcie_wait_for_link(pci)) { + if (!retry) + return 0; /* * There are some endpoints which can't get the link up if * root port has Data Link Feature (DLF) enabled. @@ -1004,43 +994,28 @@ static int tegra_pcie_dw_host_init(struct pcie_port *pp) offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); val &= ~PCI_DLF_EXCHANGE_ENABLE; - dw_pcie_writel_dbi(pci, offset, val); + dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val); - tegra_pcie_prepare_host(pp); + tegra_pcie_dw_host_init(pp); + dw_pcie_setup_rc(pp); - if (dw_pcie_wait_for_link(pci)) - return 0; + retry = false; + goto retry_link; } - speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) & - PCI_EXP_LNKSTA_CLS; - clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]); + tegra_pcie_icc_set(pcie); tegra_pcie_enable_interrupts(pp); return 0; } -static int tegra_pcie_dw_link_up(struct dw_pcie *pci) +static bool tegra_pcie_dw_link_up(struct dw_pcie *pci) { struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); - return !!(val & PCI_EXP_LNKSTA_DLLLA); -} - -static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp) -{ - pp->num_vectors = MAX_MSI_IRQS; -} - -static int tegra_pcie_dw_start_link(struct dw_pcie *pci) -{ - struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); - - enable_irq(pcie->pex_rst_irq); - - return 0; + return val & PCI_EXP_LNKSTA_DLLLA; } static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) @@ -1056,11 +1031,8 @@ static const struct dw_pcie_ops tegra_dw_pcie_ops = { .stop_link = tegra_pcie_dw_stop_link, }; -static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { - .rd_own_conf = tegra_pcie_dw_rd_own_conf, - .wr_own_conf = tegra_pcie_dw_wr_own_conf, - .host_init = tegra_pcie_dw_host_init, - .set_num_vectors = tegra_pcie_set_msi_vec_num, +static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { + .init = tegra_pcie_dw_host_init, }; static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) @@ -1102,9 +1074,16 @@ phy_exit: static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) { + struct platform_device *pdev = to_platform_device(pcie->dev); struct device_node *np = pcie->dev->of_node; int ret; + pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + if (!pcie->dbi_res) { + dev_err(pcie->dev, "Failed to find \"dbi\" region\n"); + return -ENODEV; + } + ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); if (ret < 0) { dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); @@ -1129,8 +1108,6 @@ static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) return ret; } - pcie->max_speed = of_pci_get_max_link_speed(np); - ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); if (ret) { dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); @@ -1148,13 +1125,27 @@ static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) if (of_property_read_bool(np, "nvidia,update-fc-fixup")) pcie->update_fc_fixup = true; + /* RP using an external REFCLK is supported only in Tegra234 */ + if (pcie->of_data->version == TEGRA194_DWC_IP_VER) { + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + pcie->enable_ext_refclk = true; + } else { + pcie->enable_ext_refclk = + of_property_read_bool(pcie->dev->of_node, + "nvidia,enable-ext-refclk"); + } + pcie->supports_clkreq = of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); pcie->enable_cdm_check = of_property_read_bool(np, "snps,enable-cdm-check"); - if (pcie->mode == DW_PCIE_RC_TYPE) + if (pcie->of_data->version == TEGRA234_DWC_IP_VER) + pcie->enable_srns = + of_property_read_bool(np, "nvidia,enable-srns"); + + if (pcie->of_data->mode == DW_PCIE_RC_TYPE) return 0; /* Endpoint mode specific DT entries */ @@ -1197,9 +1188,13 @@ static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, struct mrq_uphy_response resp; struct tegra_bpmp_message msg; struct mrq_uphy_request req; + int err; - /* Controller-5 doesn't need to have its state set by BPMP-FW */ - if (pcie->cid == 5) + /* + * Controller-5 doesn't need to have its state set by BPMP-FW in + * Tegra194 + */ + if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5) return 0; memset(&req, 0, sizeof(req)); @@ -1216,7 +1211,13 @@ static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, msg.rx.data = &resp; msg.rx.size = sizeof(resp); - return tegra_bpmp_transfer(pcie->bpmp, &msg); + err = tegra_bpmp_transfer(pcie->bpmp, &msg); + if (err) + return err; + if (msg.rx.ret) + return -EINVAL; + + return 0; } static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, @@ -1225,6 +1226,7 @@ static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, struct mrq_uphy_response resp; struct tegra_bpmp_message msg; struct mrq_uphy_request req; + int err; memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); @@ -1244,13 +1246,19 @@ static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, msg.rx.data = &resp; msg.rx.size = sizeof(resp); - return tegra_bpmp_transfer(pcie->bpmp, &msg); + err = tegra_bpmp_transfer(pcie->bpmp, &msg); + if (err) + return err; + if (msg.rx.ret) + return -EINVAL; + + return 0; } static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) { - struct pcie_port *pp = &pcie->pci.pp; - struct pci_bus *child, *root_bus = NULL; + struct dw_pcie_rp *pp = &pcie->pci.pp; + struct pci_bus *child, *root_port_bus = NULL; struct pci_dev *pdev; /* @@ -1262,20 +1270,20 @@ static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie) * 5.2 Link State Power Management (Page #428). */ - list_for_each_entry(child, &pp->root_bus->children, node) { - /* Bring downstream devices to D0 if they are not already in */ - if (child->parent == pp->root_bus) { - root_bus = child; + list_for_each_entry(child, &pp->bridge->bus->children, node) { + if (child->parent == pp->bridge->bus) { + root_port_bus = child; break; } } - if (!root_bus) { - dev_err(pcie->dev, "Failed to find downstream devices\n"); + if (!root_port_bus) { + dev_err(pcie->dev, "Failed to find downstream bus of Root Port\n"); return; } - list_for_each_entry(pdev, &root_bus->devices, bus_list) { + /* Bring downstream devices to D0 if they are not already in */ + list_for_each_entry(pdev, &root_port_bus->devices, bus_list) { if (PCI_SLOT(pdev->devfn) == 0) { if (pci_set_power_state(pdev, PCI_D0)) dev_err(pcie->dev, @@ -1365,6 +1373,14 @@ static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, return ret; } + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, true); + if (ret) { + dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret); + goto fail_pll_init; + } + } + ret = tegra_pcie_enable_slot_regulators(pcie); if (ret < 0) goto fail_slot_reg_en; @@ -1388,11 +1404,13 @@ static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, goto fail_core_apb_rst; } - if (en_hw_hot_rst) { + if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) { /* Enable HW_HOT_RST mode */ val = appl_readl(pcie, APPL_CTRL); val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); val |= APPL_CTRL_HW_HOT_RST_EN; appl_writel(pcie, val, APPL_CTRL); } @@ -1419,6 +1437,19 @@ static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); appl_writel(pcie, val, APPL_CFG_MISC); + if (pcie->enable_srns || pcie->enable_ext_refclk) { + /* + * When Tegra PCIe RP is using external clock, it cannot supply + * same clock to its downstream hierarchy. Hence, gate PCIe RP + * REFCLK out pads when RP & EP are using separate clocks or RP + * is using an external REFCLK. + */ + val = appl_readl(pcie, APPL_PINMUX); + val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; + val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; + appl_writel(pcie, val, APPL_PINMUX); + } + if (!pcie->supports_clkreq) { val = appl_readl(pcie, APPL_PINMUX); val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN; @@ -1433,15 +1464,6 @@ static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, reset_control_deassert(pcie->core_rst); - pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, - PCI_CAP_ID_EXP); - - /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */ - if (!pcie->supports_clkreq) { - disable_aspm_l11(pcie); - disable_aspm_l12(pcie); - } - return ret; fail_phy: @@ -1453,54 +1475,52 @@ fail_core_clk: fail_reg_en: tegra_pcie_disable_slot_regulators(pcie); fail_slot_reg_en: + if (pcie->enable_ext_refclk) + tegra_pcie_bpmp_set_pll_state(pcie, false); +fail_pll_init: tegra_pcie_bpmp_set_ctrl_state(pcie, false); return ret; } -static int __deinit_controller(struct tegra_pcie_dw *pcie) +static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie) { int ret; ret = reset_control_assert(pcie->core_rst); - if (ret) { - dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", - ret); - return ret; - } + if (ret) + dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret); tegra_pcie_disable_phy(pcie); ret = reset_control_assert(pcie->core_apb_rst); - if (ret) { + if (ret) dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); - return ret; - } clk_disable_unprepare(pcie->core_clk); ret = regulator_disable(pcie->pex_ctl_supply); - if (ret) { + if (ret) dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); - return ret; - } tegra_pcie_disable_slot_regulators(pcie); + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, false); + if (ret) + dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret); + } + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); - if (ret) { + if (ret) dev_err(pcie->dev, "Failed to disable controller %d: %d\n", pcie->cid, ret); - return ret; - } - - return ret; } static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) { struct dw_pcie *pci = &pcie->pci; - struct pcie_port *pp = &pci->pp; + struct dw_pcie_rp *pp = &pci->pp; int ret; ret = tegra_pcie_config_controller(pcie, false); @@ -1518,7 +1538,8 @@ static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) return 0; fail_host_init: - return __deinit_controller(pcie); + tegra_pcie_unconfig_controller(pcie); + return ret; } static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) @@ -1547,6 +1568,16 @@ static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) return; } + /* + * PCIe controller exits from L2 only if reset is applied, so + * controller doesn't handle interrupts. But in cases where + * L2 entry fails, PERST# is asserted which can trigger surprise + * link down AER. However this function call happens in + * suspend_noirq(), so AER interrupt will not be processed. + * Disable all interrupts to avoid such a scenario. + */ + appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0); + if (tegra_pcie_try_link_l2(pcie)) { dev_info(pcie->dev, "Link didn't transition to L2 state\n"); /* @@ -1559,6 +1590,14 @@ static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) data &= ~APPL_PINMUX_PEX_RST; appl_writel(pcie, data, APPL_PINMUX); + /* + * Some cards do not go to detect state even after de-asserting + * PERST#. So, de-assert LTSSM to bring link to detect state. + */ + data = readl(pcie->appl_base + APPL_CTRL); + data &= ~APPL_CTRL_LTSSM_EN; + writel(data, pcie->appl_base + APPL_CTRL); + err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, data, ((data & @@ -1566,14 +1605,8 @@ static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) APPL_DEBUG_LTSSM_STATE_SHIFT) == LTSSM_STATE_PRE_DETECT, 1, LTSSM_TIMEOUT); - if (err) { + if (err) dev_info(pcie->dev, "Link didn't go to detect state\n"); - } else { - /* Disable LTSSM after link is in detect state */ - data = appl_readl(pcie, APPL_CTRL); - data &= ~APPL_CTRL_LTSSM_EN; - appl_writel(pcie, data, APPL_CTRL); - } } /* * DBI registers may not be accessible after this as PLL-E would be @@ -1587,30 +1620,19 @@ static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) appl_writel(pcie, data, APPL_PINMUX); } -static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) +static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) { tegra_pcie_downstream_dev_to_D0(pcie); dw_pcie_host_deinit(&pcie->pci.pp); tegra_pcie_dw_pme_turnoff(pcie); - - return __deinit_controller(pcie); + tegra_pcie_unconfig_controller(pcie); } static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) { - struct pcie_port *pp = &pcie->pci.pp; struct device *dev = pcie->dev; - char *name; int ret; - if (IS_ENABLED(CONFIG_PCI_MSI)) { - pp->msi_irq = of_irq_get_byname(dev->of_node, "msi"); - if (!pp->msi_irq) { - dev_err(dev, "Failed to get MSI interrupt\n"); - return -ENODEV; - } - } - pm_runtime_enable(dev); ret = pm_runtime_get_sync(dev); @@ -1626,7 +1648,11 @@ static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) goto fail_pm_get_sync; } - tegra_pcie_init_controller(pcie); + ret = tegra_pcie_init_controller(pcie); + if (ret < 0) { + dev_err(dev, "Failed to initialize controller: %d\n", ret); + goto fail_pm_get_sync; + } pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); if (!pcie->link_state) { @@ -1634,17 +1660,7 @@ static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) goto fail_host_init; } - name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); - if (!name) { - ret = -ENOMEM; - goto fail_host_init; - } - - pcie->debugfs = debugfs_create_dir(name, NULL); - if (!pcie->debugfs) - dev_err(dev, "Failed to create debugfs\n"); - else - init_debugfs(pcie); + init_debugfs(pcie); return ret; @@ -1687,9 +1703,16 @@ static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) pm_runtime_put_sync(pcie->dev); - ret = tegra_pcie_bpmp_set_pll_state(pcie, false); + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, false); + if (ret) + dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", + ret); + } + + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); if (ret) - dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret); + dev_err(pcie->dev, "Failed to disable controller: %d\n", ret); pcie->ep_state = EP_STATE_DISABLED; dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); @@ -1702,21 +1725,32 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) struct device *dev = pcie->dev; u32 val; int ret; + u16 val_16; if (pcie->ep_state == EP_STATE_ENABLED) return; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", ret); return; } - ret = tegra_pcie_bpmp_set_pll_state(pcie, true); + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); if (ret) { - dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret); - goto fail_pll_init; + dev_err(pcie->dev, "Failed to enable controller %u: %d\n", + pcie->cid, ret); + goto fail_set_ctrl_state; + } + + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, true); + if (ret) { + dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", + ret); + goto fail_pll_init; + } } ret = clk_prepare_enable(pcie->core_clk); @@ -1737,6 +1771,10 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) goto fail_phy; } + /* Perform cleanup that requires refclk */ + pci_epc_deinit_notify(pcie->pci.ep.epc); + dw_pcie_ep_cleanup(&pcie->pci.ep); + /* Clear any stale interrupt statuses */ appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); @@ -1807,54 +1845,46 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) init_host_aspm(pcie); - /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */ - if (!pcie->supports_clkreq) { - disable_aspm_l11(pcie); - disable_aspm_l12(pcie); + if (!pcie->of_data->has_l1ss_exit_fix) { + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); } - val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); - val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; - dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); - - /* Configure N_FTS & FTS */ - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL); - val &= ~(N_FTS_MASK << N_FTS_SHIFT); - val |= N_FTS_VAL << N_FTS_SHIFT; - dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val); - - val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL); - val &= ~FTS_MASK; - val |= FTS_VAL; - dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val); + pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, + PCI_CAP_ID_EXP); - /* Configure Max Speed from DT */ - if (pcie->max_speed && pcie->max_speed != -EINVAL) { - val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + - PCI_EXP_LNKCAP); - val &= ~PCI_EXP_LNKCAP_SLS; - val |= pcie->max_speed; - dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, - val); + /* Clear Slot Clock Configuration bit if SRNS configuration */ + if (pcie->enable_srns) { + val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + val_16 &= ~PCI_EXP_LNKSTA_SLC; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA, + val_16); } - pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, - PCI_CAP_ID_EXP); clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); val |= MSIX_ADDR_MATCH_LOW_OFF_EN; dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); - val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); + val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); - ret = dw_pcie_ep_init_complete(ep); + ret = dw_pcie_ep_init_registers(ep); if (ret) { dev_err(dev, "Failed to complete initialization: %d\n", ret); goto fail_init_complete; } - dw_pcie_ep_init_notify(ep); + pci_epc_init_notify(ep->epc); + + /* Program the private control to allow sending LTR upstream */ + if (pcie->of_data->has_ltr_req_fix) { + val = appl_readl(pcie, APPL_LTR_MSG_2); + val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; + appl_writel(pcie, val, APPL_LTR_MSG_2); + } /* Enable LTSSM */ val = appl_readl(pcie, APPL_CTRL); @@ -1876,6 +1906,8 @@ fail_core_apb_rst: fail_core_clk_enable: tegra_pcie_bpmp_set_pll_state(pcie, false); fail_pll_init: + tegra_pcie_bpmp_set_ctrl_state(pcie, false); +fail_set_ctrl_state: pm_runtime_put_sync(dev); } @@ -1891,7 +1923,16 @@ static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) return IRQ_HANDLED; } -static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) +static void tegra_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar; + + for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) + dw_pcie_ep_reset_bar(pci, bar); +}; + +static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq) { /* Tegra194 supports only INTA */ if (irq > 1) @@ -1905,10 +1946,10 @@ static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq) static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) { - if (unlikely(irq > 31)) + if (unlikely(irq > 32)) return -EINVAL; - appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1); + appl_writel(pcie, BIT(irq - 1), APPL_MSI_CTRL_1); return 0; } @@ -1923,20 +1964,19 @@ static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq) } static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, - enum pci_epc_irq_type type, - u16 interrupt_num) + unsigned int type, u16 interrupt_num) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); switch (type) { - case PCI_EPC_IRQ_LEGACY: - return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num); + case PCI_IRQ_INTX: + return tegra_pcie_ep_raise_intx_irq(pcie, interrupt_num); - case PCI_EPC_IRQ_MSI: + case PCI_IRQ_MSI: return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num); - case PCI_EPC_IRQ_MSIX: + case PCI_IRQ_MSIX: return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num); default: @@ -1949,12 +1989,15 @@ static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, static const struct pci_epc_features tegra_pcie_epc_features = { .linkup_notifier = true, - .core_init_notifier = true, - .msi_capable = false, - .msix_capable = false, - .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5, - .bar_fixed_64bit = 1 << BAR_0, - .bar_fixed_size[0] = SZ_1M, + .msi_capable = true, + .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = SZ_1M, + .only_64bit = true, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .type = BAR_RESERVED, }, + .bar[BAR_3] = { .type = BAR_RESERVED, }, + .bar[BAR_4] = { .type = BAR_RESERVED, }, + .bar[BAR_5] = { .type = BAR_RESERVED, }, + .align = SZ_64K, }; static const struct pci_epc_features* @@ -1963,7 +2006,8 @@ tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) return &tegra_pcie_epc_features; } -static struct dw_pcie_ep_ops pcie_ep_ops = { +static const struct dw_pcie_ep_ops pcie_ep_ops = { + .init = tegra_pcie_ep_init, .raise_irq = tegra_pcie_ep_raise_irq, .get_features = tegra_pcie_ep_get_features, }; @@ -1974,19 +2018,12 @@ static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, struct dw_pcie *pci = &pcie->pci; struct device *dev = pcie->dev; struct dw_pcie_ep *ep; - struct resource *res; char *name; int ret; ep = &pci->ep; ep->ops = &pcie_ep_ops; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); - if (!res) - return -EINVAL; - - ep->phys_base = res->start; - ep->addr_size = resource_size(res); ep->page_size = SZ_64K; ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME); @@ -2024,19 +2061,13 @@ static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, return ret; } - name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work", - pcie->cid); - if (!name) { - dev_err(dev, "Failed to create PCIe EP work thread string\n"); - return -ENOMEM; - } - pm_runtime_enable(dev); ret = dw_pcie_ep_init(ep); if (ret) { dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n", ret); + pm_runtime_disable(dev); return ret; } @@ -2049,8 +2080,7 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct resource *atu_dma_res; struct tegra_pcie_dw *pcie; - struct resource *dbi_res; - struct pcie_port *pp; + struct dw_pcie_rp *pp; struct dw_pcie *pci; struct phy **phys; char *name; @@ -2066,9 +2096,12 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) pci = &pcie->pci; pci->dev = &pdev->dev; pci->ops = &tegra_dw_pcie_ops; - pp = &pci->pp; pcie->dev = &pdev->dev; - pcie->mode = (enum dw_pcie_device_mode)data->mode; + pcie->of_data = (struct tegra_pcie_dw_of_data *)data; + pci->n_fts[0] = pcie->of_data->n_fts[0]; + pci->n_fts[1] = pcie->of_data->n_fts[1]; + pp = &pci->pp; + pp->num_vectors = MAX_MSI_IRQS; ret = tegra_pcie_dw_parse_dt(pcie); if (ret < 0) { @@ -2155,20 +2188,6 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) pcie->phys = phys; - dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); - if (!dbi_res) { - dev_err(dev, "Failed to find \"dbi\" region\n"); - return -ENODEV; - } - pcie->dbi_res = dbi_res; - - pci->dbi_base = devm_ioremap_resource(dev, dbi_res); - if (IS_ERR(pci->dbi_base)) - return PTR_ERR(pci->dbi_base); - - /* Tegra HW locates DBI2 at a fixed offset from DBI */ - pci->dbi_base2 = pci->dbi_base + 0x1000; - atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu_dma"); if (!atu_dma_res) { @@ -2177,6 +2196,7 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) } pcie->atu_dma_res = atu_dma_res; + pci->atu_size = resource_size(atu_dma_res); pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); if (IS_ERR(pci->atu_base)) return PTR_ERR(pci->atu_base); @@ -2189,10 +2209,8 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) } pp->irq = platform_get_irq_byname(pdev, "intr"); - if (pp->irq < 0) { - dev_err(dev, "Failed to get \"intr\" interrupt\n"); + if (pp->irq < 0) return pp->irq; - } pcie->bpmp = tegra_bpmp_get(dev); if (IS_ERR(pcie->bpmp)) @@ -2200,7 +2218,15 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) platform_set_drvdata(pdev, pcie); - switch (pcie->mode) { + pcie->icc_path = devm_of_icc_get(&pdev->dev, "write"); + ret = PTR_ERR_OR_ZERO(pcie->icc_path); + if (ret) { + tegra_bpmp_put(pcie->bpmp); + dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n"); + return ret; + } + + switch (pcie->of_data->mode) { case DW_PCIE_RC_TYPE: ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler, IRQF_SHARED, "tegra-pcie-intr", pcie); @@ -2232,10 +2258,14 @@ static int tegra_pcie_dw_probe(struct platform_device *pdev) ret = tegra_pcie_config_ep(pcie, pdev); if (ret < 0) goto fail; + else + return 0; break; default: - dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode); + dev_err(dev, "Invalid PCIe device type %d\n", + pcie->of_data->mode); + ret = -EINVAL; } fail: @@ -2243,22 +2273,26 @@ fail: return ret; } -static int tegra_pcie_dw_remove(struct platform_device *pdev) +static void tegra_pcie_dw_remove(struct platform_device *pdev) { struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); - if (!pcie->link_state) - return 0; + if (pcie->of_data->mode == DW_PCIE_RC_TYPE) { + if (!pcie->link_state) + return; + + debugfs_remove_recursive(pcie->debugfs); + tegra_pcie_deinit_controller(pcie); + pm_runtime_put_sync(pcie->dev); + } else { + disable_irq(pcie->pex_rst_irq); + pex_ep_event_pex_rst_assert(pcie); + } - debugfs_remove_recursive(pcie->debugfs); - tegra_pcie_deinit_controller(pcie); - pm_runtime_put_sync(pcie->dev); pm_runtime_disable(pcie->dev); tegra_bpmp_put(pcie->bpmp); if (pcie->pex_refclk_sel_gpiod) gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); - - return 0; } static int tegra_pcie_dw_suspend_late(struct device *dev) @@ -2266,15 +2300,22 @@ static int tegra_pcie_dw_suspend_late(struct device *dev) struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); u32 val; + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { + dev_err(dev, "Failed to Suspend as Tegra PCIe is in EP mode\n"); + return -EPERM; + } + if (!pcie->link_state) return 0; /* Enable HW_HOT_RST mode */ - val = appl_readl(pcie, APPL_CTRL); - val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << - APPL_CTRL_HW_HOT_RST_MODE_SHIFT); - val |= APPL_CTRL_HW_HOT_RST_EN; - appl_writel(pcie, val, APPL_CTRL); + if (!pcie->of_data->has_sbr_reset_fix) { + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + } return 0; } @@ -2286,13 +2327,11 @@ static int tegra_pcie_dw_suspend_noirq(struct device *dev) if (!pcie->link_state) return 0; - /* Save MSI interrupt vector */ - pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci, - PORT_LOGIC_MSI_CTRL_INT_0_EN); tegra_pcie_downstream_dev_to_D0(pcie); tegra_pcie_dw_pme_turnoff(pcie); + tegra_pcie_unconfig_controller(pcie); - return __deinit_controller(pcie); + return 0; } static int tegra_pcie_dw_resume_noirq(struct device *dev) @@ -2313,14 +2352,17 @@ static int tegra_pcie_dw_resume_noirq(struct device *dev) goto fail_host_init; } - /* Restore MSI interrupt vector */ - dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN, - pcie->msi_ctrl_int); + dw_pcie_setup_rc(&pcie->pci.pp); + + ret = tegra_pcie_dw_start_link(&pcie->pci); + if (ret < 0) + goto fail_host_init; return 0; fail_host_init: - return __deinit_controller(pcie); + tegra_pcie_unconfig_controller(pcie); + return ret; } static int tegra_pcie_dw_resume_early(struct device *dev) @@ -2328,17 +2370,24 @@ static int tegra_pcie_dw_resume_early(struct device *dev) struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); u32 val; + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { + dev_err(dev, "Suspend is not supported in EP mode"); + return -ENOTSUPP; + } + if (!pcie->link_state) return 0; /* Disable HW_HOT_RST mode */ - val = appl_readl(pcie, APPL_CTRL); - val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << - APPL_CTRL_HW_HOT_RST_MODE_SHIFT); - val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << - APPL_CTRL_HW_HOT_RST_MODE_SHIFT; - val &= ~APPL_CTRL_HW_HOT_RST_EN; - appl_writel(pcie, val, APPL_CTRL); + if (!pcie->of_data->has_sbr_reset_fix) { + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT; + val &= ~APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + } return 0; } @@ -2347,38 +2396,85 @@ static void tegra_pcie_dw_shutdown(struct platform_device *pdev) { struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); - if (!pcie->link_state) - return; + if (pcie->of_data->mode == DW_PCIE_RC_TYPE) { + if (!pcie->link_state) + return; - debugfs_remove_recursive(pcie->debugfs); - tegra_pcie_downstream_dev_to_D0(pcie); + debugfs_remove_recursive(pcie->debugfs); + tegra_pcie_downstream_dev_to_D0(pcie); - disable_irq(pcie->pci.pp.irq); - if (IS_ENABLED(CONFIG_PCI_MSI)) - disable_irq(pcie->pci.pp.msi_irq); + disable_irq(pcie->pci.pp.irq); + if (IS_ENABLED(CONFIG_PCI_MSI)) + disable_irq(pcie->pci.pp.msi_irq[0]); - tegra_pcie_dw_pme_turnoff(pcie); - __deinit_controller(pcie); + tegra_pcie_dw_pme_turnoff(pcie); + tegra_pcie_unconfig_controller(pcie); + pm_runtime_put_sync(pcie->dev); + } else { + disable_irq(pcie->pex_rst_irq); + pex_ep_event_pex_rst_assert(pcie); + } } -static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = { +static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = { + .version = TEGRA194_DWC_IP_VER, + .mode = DW_PCIE_RC_TYPE, + .cdm_chk_int_en_bit = BIT(19), + /* Gen4 - 5, 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x360, + .n_fts = { 52, 52 }, +}; + +static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = { + .version = TEGRA194_DWC_IP_VER, + .mode = DW_PCIE_EP_TYPE, + .cdm_chk_int_en_bit = BIT(19), + /* Gen4 - 5, 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x360, + .n_fts = { 52, 52 }, +}; + +static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = { + .version = TEGRA234_DWC_IP_VER, .mode = DW_PCIE_RC_TYPE, + .has_msix_doorbell_access_fix = true, + .has_sbr_reset_fix = true, + .has_l1ss_exit_fix = true, + .cdm_chk_int_en_bit = BIT(18), + /* Gen4 - 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x340, + .n_fts = { 52, 80 }, }; -static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = { +static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = { + .version = TEGRA234_DWC_IP_VER, .mode = DW_PCIE_EP_TYPE, + .has_l1ss_exit_fix = true, + .has_ltr_req_fix = true, + .cdm_chk_int_en_bit = BIT(18), + /* Gen4 - 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x340, + .n_fts = { 52, 80 }, }; static const struct of_device_id tegra_pcie_dw_of_match[] = { { .compatible = "nvidia,tegra194-pcie", - .data = &tegra_pcie_dw_rc_of_data, + .data = &tegra194_pcie_dw_rc_of_data, }, { .compatible = "nvidia,tegra194-pcie-ep", - .data = &tegra_pcie_dw_ep_of_data, + .data = &tegra194_pcie_dw_ep_of_data, + }, + { + .compatible = "nvidia,tegra234-pcie", + .data = &tegra234_pcie_dw_rc_of_data, + }, + { + .compatible = "nvidia,tegra234-pcie-ep", + .data = &tegra234_pcie_dw_ep_of_data, }, - {}, + {} }; static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { |