diff options
Diffstat (limited to 'drivers/memory/tegra/mc.c')
| -rw-r--r-- | drivers/memory/tegra/mc.c | 726 |
1 files changed, 505 insertions, 221 deletions
diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c index 24afc36833bf..6edb210287dc 100644 --- a/drivers/memory/tegra/mc.c +++ b/drivers/memory/tegra/mc.c @@ -1,57 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (C) 2014 NVIDIA CORPORATION. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. + * Copyright (C) 2014-2025 NVIDIA CORPORATION. All rights reserved. */ #include <linux/clk.h> #include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/export.h> #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/sort.h> +#include <linux/tegra-icc.h> #include <soc/tegra/fuse.h> #include "mc.h" -#define MC_INTSTATUS 0x000 - -#define MC_INTMASK 0x004 - -#define MC_ERR_STATUS 0x08 -#define MC_ERR_STATUS_TYPE_SHIFT 28 -#define MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE (6 << MC_ERR_STATUS_TYPE_SHIFT) -#define MC_ERR_STATUS_TYPE_MASK (0x7 << MC_ERR_STATUS_TYPE_SHIFT) -#define MC_ERR_STATUS_READABLE (1 << 27) -#define MC_ERR_STATUS_WRITABLE (1 << 26) -#define MC_ERR_STATUS_NONSECURE (1 << 25) -#define MC_ERR_STATUS_ADR_HI_SHIFT 20 -#define MC_ERR_STATUS_ADR_HI_MASK 0x3 -#define MC_ERR_STATUS_SECURITY (1 << 17) -#define MC_ERR_STATUS_RW (1 << 16) - -#define MC_ERR_ADR 0x0c - -#define MC_DECERR_EMEM_OTHERS_STATUS 0x58 -#define MC_SECURITY_VIOLATION_STATUS 0x74 - -#define MC_EMEM_ARB_CFG 0x90 -#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(x) (((x) & 0x1ff) << 0) -#define MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK 0x1ff -#define MC_EMEM_ARB_MISC0 0xd8 - -#define MC_EMEM_ADR_CFG 0x54 -#define MC_EMEM_ADR_CFG_EMEM_NUMDEV BIT(0) - static const struct of_device_id tegra_mc_of_match[] = { #ifdef CONFIG_ARCH_TEGRA_2x_SOC - { .compatible = "nvidia,tegra20-mc", .data = &tegra20_mc_soc }, + { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc }, #endif #ifdef CONFIG_ARCH_TEGRA_3x_SOC { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc }, @@ -68,11 +40,103 @@ static const struct of_device_id tegra_mc_of_match[] = { #ifdef CONFIG_ARCH_TEGRA_210_SOC { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc }, #endif - { } +#ifdef CONFIG_ARCH_TEGRA_186_SOC + { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc }, +#endif +#ifdef CONFIG_ARCH_TEGRA_194_SOC + { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc }, +#endif +#ifdef CONFIG_ARCH_TEGRA_234_SOC + { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc }, +#endif +#ifdef CONFIG_ARCH_TEGRA_264_SOC + { .compatible = "nvidia,tegra264-mc", .data = &tegra264_mc_soc }, +#endif + { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, tegra_mc_of_match); -static int terga_mc_block_dma_common(struct tegra_mc *mc, +static void tegra_mc_devm_action_put_device(void *data) +{ + struct tegra_mc *mc = data; + + put_device(mc->dev); +} + +/** + * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle + * @dev: device pointer for the consumer device + * + * This function will search for the Memory Controller node in a device-tree + * and retrieve the Memory Controller handle. + * + * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc. + */ +struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev) +{ + struct platform_device *pdev; + struct device_node *np; + struct tegra_mc *mc; + int err; + + np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0); + if (!np) + return ERR_PTR(-ENOENT); + + pdev = of_find_device_by_node(np); + of_node_put(np); + if (!pdev) + return ERR_PTR(-ENODEV); + + mc = platform_get_drvdata(pdev); + if (!mc) { + put_device(&pdev->dev); + return ERR_PTR(-EPROBE_DEFER); + } + + err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc); + if (err) + return ERR_PTR(err); + + return mc; +} +EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get); + +int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev) +{ + if (mc->soc->ops && mc->soc->ops->probe_device) + return mc->soc->ops->probe_device(mc, dev); + + return 0; +} +EXPORT_SYMBOL_GPL(tegra_mc_probe_device); + +int tegra_mc_get_carveout_info(struct tegra_mc *mc, unsigned int id, + phys_addr_t *base, u64 *size) +{ + u32 offset; + + if (id < 1 || id >= mc->soc->num_carveouts) + return -EINVAL; + + if (id < 6) + offset = 0xc0c + 0x50 * (id - 1); + else + offset = 0x2004 + 0x50 * (id - 6); + + *base = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x0); +#ifdef CONFIG_PHYS_ADDR_T_64BIT + *base |= (phys_addr_t)mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x4) << 32; +#endif + + if (size) + *size = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, offset + 0x8) << 17; + + return 0; +} +EXPORT_SYMBOL_GPL(tegra_mc_get_carveout_info); + +static int tegra_mc_block_dma_common(struct tegra_mc *mc, const struct tegra_mc_reset *rst) { unsigned long flags; @@ -88,13 +152,13 @@ static int terga_mc_block_dma_common(struct tegra_mc *mc, return 0; } -static bool terga_mc_dma_idling_common(struct tegra_mc *mc, +static bool tegra_mc_dma_idling_common(struct tegra_mc *mc, const struct tegra_mc_reset *rst) { return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0; } -static int terga_mc_unblock_dma_common(struct tegra_mc *mc, +static int tegra_mc_unblock_dma_common(struct tegra_mc *mc, const struct tegra_mc_reset *rst) { unsigned long flags; @@ -110,17 +174,17 @@ static int terga_mc_unblock_dma_common(struct tegra_mc *mc, return 0; } -static int terga_mc_reset_status_common(struct tegra_mc *mc, +static int tegra_mc_reset_status_common(struct tegra_mc *mc, const struct tegra_mc_reset *rst) { return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0; } -const struct tegra_mc_reset_ops terga_mc_reset_ops_common = { - .block_dma = terga_mc_block_dma_common, - .dma_idling = terga_mc_dma_idling_common, - .unblock_dma = terga_mc_unblock_dma_common, - .reset_status = terga_mc_reset_status_common, +const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = { + .block_dma = tegra_mc_block_dma_common, + .dma_idling = tegra_mc_dma_idling_common, + .unblock_dma = tegra_mc_unblock_dma_common, + .reset_status = tegra_mc_reset_status_common, }; static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev) @@ -157,11 +221,18 @@ static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, if (!rst_ops) return -ENODEV; + /* DMA flushing will fail if reset is already asserted */ + if (rst_ops->reset_status) { + /* check whether reset is asserted */ + if (rst_ops->reset_status(mc, rst)) + return 0; + } + if (rst_ops->block_dma) { /* block clients DMA requests */ err = rst_ops->block_dma(mc, rst); if (err) { - dev_err(mc->dev, "Failed to block %s DMA: %d\n", + dev_err(mc->dev, "failed to block %s DMA: %d\n", rst->name, err); return err; } @@ -171,7 +242,7 @@ static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, /* wait for completion of the outstanding DMA requests */ while (!rst_ops->dma_idling(mc, rst)) { if (!retries--) { - dev_err(mc->dev, "Failed to flush %s DMA\n", + dev_err(mc->dev, "failed to flush %s DMA\n", rst->name); return -EBUSY; } @@ -184,7 +255,7 @@ static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev, /* clear clients DMA requests sitting before arbitration */ err = rst_ops->hotreset_assert(mc, rst); if (err) { - dev_err(mc->dev, "Failed to hot reset %s: %d\n", + dev_err(mc->dev, "failed to hot reset %s: %d\n", rst->name, err); return err; } @@ -213,7 +284,7 @@ static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev, /* take out client from hot reset */ err = rst_ops->hotreset_deassert(mc, rst); if (err) { - dev_err(mc->dev, "Failed to deassert hot reset %s: %d\n", + dev_err(mc->dev, "failed to deassert hot reset %s: %d\n", rst->name, err); return err; } @@ -223,7 +294,7 @@ static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev, /* allow new DMA requests to proceed to arbitration */ err = rst_ops->unblock_dma(mc, rst); if (err) { - dev_err(mc->dev, "Failed to unblock %s DMA : %d\n", + dev_err(mc->dev, "failed to unblock %s DMA : %d\n", rst->name, err); return err; } @@ -273,36 +344,7 @@ static int tegra_mc_reset_setup(struct tegra_mc *mc) return 0; } -static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc) -{ - unsigned long long tick; - unsigned int i; - u32 value; - - /* compute the number of MC clock cycles per tick */ - tick = mc->tick * clk_get_rate(mc->clk); - do_div(tick, NSEC_PER_SEC); - - value = readl(mc->regs + MC_EMEM_ARB_CFG); - value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; - value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); - writel(value, mc->regs + MC_EMEM_ARB_CFG); - - /* write latency allowance defaults */ - for (i = 0; i < mc->soc->num_clients; i++) { - const struct tegra_mc_la *la = &mc->soc->clients[i].la; - u32 value; - - value = readl(mc->regs + la->reg); - value &= ~(la->mask << la->shift); - value |= (la->def & la->mask) << la->shift; - writel(value, mc->regs + la->reg); - } - - return 0; -} - -void tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) +int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) { unsigned int i; struct tegra_mc_timing *timing = NULL; @@ -317,12 +359,15 @@ void tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate) if (!timing) { dev_err(mc->dev, "no memory timing registered for rate %lu\n", rate); - return; + return -EINVAL; } for (i = 0; i < mc->soc->num_emem_regs; ++i) mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]); + + return 0; } +EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration); unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) { @@ -334,6 +379,44 @@ unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc) return dram_count; } +EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count); + +#if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \ + defined(CONFIG_ARCH_TEGRA_114_SOC) || \ + defined(CONFIG_ARCH_TEGRA_124_SOC) || \ + defined(CONFIG_ARCH_TEGRA_132_SOC) || \ + defined(CONFIG_ARCH_TEGRA_210_SOC) +static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc) +{ + unsigned long long tick; + unsigned int i; + u32 value; + + /* compute the number of MC clock cycles per tick */ + tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk); + do_div(tick, NSEC_PER_SEC); + + value = mc_readl(mc, MC_EMEM_ARB_CFG); + value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK; + value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick); + mc_writel(mc, value, MC_EMEM_ARB_CFG); + + /* write latency allowance defaults */ + for (i = 0; i < mc->soc->num_clients; i++) { + const struct tegra_mc_client *client = &mc->soc->clients[i]; + u32 value; + + value = mc_readl(mc, client->regs.la.reg); + value &= ~(client->regs.la.mask << client->regs.la.shift); + value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift; + mc_writel(mc, value, client->regs.la.reg); + } + + /* latch new values */ + mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL); + + return 0; +} static int load_one_timing(struct tegra_mc *mc, struct tegra_mc_timing *timing, @@ -370,7 +453,6 @@ static int load_one_timing(struct tegra_mc *mc, static int load_timings(struct tegra_mc *mc, struct device_node *node) { - struct device_node *child; struct tegra_mc_timing *timing; int child_count = of_get_child_count(node); int i = 0, err; @@ -382,14 +464,12 @@ static int load_timings(struct tegra_mc *mc, struct device_node *node) mc->num_timings = child_count; - for_each_child_of_node(node, child) { + for_each_child_of_node_scoped(node, child) { timing = &mc->timings[i++]; err = load_one_timing(mc, timing, child); - if (err) { - of_node_put(child); + if (err) return err; - } } return 0; @@ -397,7 +477,6 @@ static int load_timings(struct tegra_mc *mc, struct device_node *node) static int tegra_mc_setup_timings(struct tegra_mc *mc) { - struct device_node *node; u32 ram_code, node_ram_code; int err; @@ -405,14 +484,13 @@ static int tegra_mc_setup_timings(struct tegra_mc *mc) mc->num_timings = 0; - for_each_child_of_node(mc->dev->of_node, node) { + for_each_child_of_node_scoped(mc->dev->of_node, node) { err = of_property_read_u32(node, "nvidia,ram-code", &node_ram_code); if (err || (node_ram_code != ram_code)) continue; err = load_timings(mc, node); - of_node_put(node); if (err) return err; break; @@ -426,53 +504,153 @@ static int tegra_mc_setup_timings(struct tegra_mc *mc) return 0; } -static const char *const status_names[32] = { - [ 1] = "External interrupt", - [ 6] = "EMEM address decode error", - [ 7] = "GART page fault", - [ 8] = "Security violation", - [ 9] = "EMEM arbitration error", - [10] = "Page fault", - [11] = "Invalid APB ASID update", - [12] = "VPR violation", - [13] = "Secure carveout violation", - [16] = "MTS carveout violation", -}; +int tegra30_mc_probe(struct tegra_mc *mc) +{ + int err; -static const char *const error_names[8] = { - [2] = "EMEM decode error", - [3] = "TrustZone violation", - [4] = "Carveout violation", - [6] = "SMMU translation error", + mc->clk = devm_clk_get_optional(mc->dev, "mc"); + if (IS_ERR(mc->clk)) { + dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk)); + return PTR_ERR(mc->clk); + } + + /* ensure that debug features are disabled */ + mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG); + + err = tegra_mc_setup_latency_allowance(mc); + if (err < 0) { + dev_err(mc->dev, "failed to setup latency allowance: %d\n", err); + return err; + } + + err = tegra_mc_setup_timings(mc); + if (err < 0) { + dev_err(mc->dev, "failed to setup timings: %d\n", err); + return err; + } + + return 0; +} + +const struct tegra_mc_ops tegra30_mc_ops = { + .probe = tegra30_mc_probe, + .handle_irq = tegra30_mc_handle_irq, }; +#endif + +static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status, + unsigned int *mc_channel) +{ + if ((status & mc->soc->ch_intmask) == 0) + return -EINVAL; + + *mc_channel = __ffs((status & mc->soc->ch_intmask) >> + mc->soc->global_intstatus_channel_shift); + + return 0; +} -static irqreturn_t tegra_mc_irq(int irq, void *data) +static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc, + unsigned int channel) +{ + return BIT(channel) << mc->soc->global_intstatus_channel_shift; +} + +irqreturn_t tegra30_mc_handle_irq(int irq, void *data) { struct tegra_mc *mc = data; + unsigned int bit, channel; unsigned long status; - unsigned int bit; - /* mask all interrupts to avoid flooding */ - status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; + if (mc->soc->num_channels) { + u32 global_status; + int err; + + global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS); + err = mc_global_intstatus_to_channel(mc, global_status, &channel); + if (err < 0) { + dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n", + global_status); + return IRQ_NONE; + } + + /* mask all interrupts to avoid flooding */ + status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask; + } else { + status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; + } + if (!status) return IRQ_NONE; for_each_set_bit(bit, &status, 32) { - const char *error = status_names[bit] ?: "unknown"; + const char *error = tegra_mc_status_names[bit] ?: "unknown"; const char *client = "unknown", *desc; const char *direction, *secure; + u32 status_reg, addr_reg; + u32 intmask = BIT(bit); phys_addr_t addr = 0; +#ifdef CONFIG_PHYS_ADDR_T_64BIT + u32 addr_hi_reg = 0; +#endif unsigned int i; char perm[7]; u8 id, type; u32 value; - value = mc_readl(mc, MC_ERR_STATUS); + switch (intmask) { + case MC_INT_DECERR_VPR: + status_reg = MC_ERR_VPR_STATUS; + addr_reg = MC_ERR_VPR_ADR; + break; + + case MC_INT_SECERR_SEC: + status_reg = MC_ERR_SEC_STATUS; + addr_reg = MC_ERR_SEC_ADR; + break; + + case MC_INT_DECERR_MTS: + status_reg = MC_ERR_MTS_STATUS; + addr_reg = MC_ERR_MTS_ADR; + break; + + case MC_INT_DECERR_GENERALIZED_CARVEOUT: + status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS; + addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR; + break; + + case MC_INT_DECERR_ROUTE_SANITY: + status_reg = MC_ERR_ROUTE_SANITY_STATUS; + addr_reg = MC_ERR_ROUTE_SANITY_ADR; + break; + + default: + status_reg = MC_ERR_STATUS; + addr_reg = MC_ERR_ADR; + +#ifdef CONFIG_PHYS_ADDR_T_64BIT + if (mc->soc->has_addr_hi_reg) + addr_hi_reg = MC_ERR_ADR_HI; +#endif + break; + } + + if (mc->soc->num_channels) + value = mc_ch_readl(mc, channel, status_reg); + else + value = mc_readl(mc, status_reg); #ifdef CONFIG_PHYS_ADDR_T_64BIT if (mc->soc->num_address_bits > 32) { - addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & - MC_ERR_STATUS_ADR_HI_MASK); + if (addr_hi_reg) { + if (mc->soc->num_channels) + addr = mc_ch_readl(mc, channel, addr_hi_reg); + else + addr = mc_readl(mc, addr_hi_reg); + } else { + addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) & + MC_ERR_STATUS_ADR_HI_MASK); + } addr <<= 32; } #endif @@ -498,7 +676,7 @@ static irqreturn_t tegra_mc_irq(int irq, void *data) type = (value & MC_ERR_STATUS_TYPE_MASK) >> MC_ERR_STATUS_TYPE_SHIFT; - desc = error_names[type]; + desc = tegra_mc_error_names[type]; switch (value & MC_ERR_STATUS_TYPE_MASK) { case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE: @@ -529,7 +707,10 @@ static irqreturn_t tegra_mc_irq(int irq, void *data) break; } - value = mc_readl(mc, MC_ERR_ADR); + if (mc->soc->num_channels) + value = mc_ch_readl(mc, channel, addr_reg); + else + value = mc_readl(mc, addr_reg); addr |= value; dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n", @@ -538,178 +719,282 @@ static irqreturn_t tegra_mc_irq(int irq, void *data) } /* clear interrupts */ - mc_writel(mc, status, MC_INTSTATUS); + if (mc->soc->num_channels) { + mc_ch_writel(mc, channel, status, MC_INTSTATUS); + mc_ch_writel(mc, MC_BROADCAST_CHANNEL, + mc_channel_to_global_intstatus(mc, channel), + MC_GLOBAL_INTSTATUS); + } else { + mc_writel(mc, status, MC_INTSTATUS); + } return IRQ_HANDLED; } -static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data) +const char *const tegra_mc_status_names[32] = { + [ 1] = "External interrupt", + [ 6] = "EMEM address decode error", + [ 7] = "GART page fault", + [ 8] = "Security violation", + [ 9] = "EMEM arbitration error", + [10] = "Page fault", + [11] = "Invalid APB ASID update", + [12] = "VPR violation", + [13] = "Secure carveout violation", + [16] = "MTS carveout violation", + [17] = "Generalized carveout violation", + [20] = "Route Sanity error", +}; + +const char *const tegra_mc_error_names[8] = { + [2] = "EMEM decode error", + [3] = "TrustZone violation", + [4] = "Carveout violation", + [6] = "SMMU translation error", +}; + +struct icc_node *tegra_mc_icc_xlate(const struct of_phandle_args *spec, void *data) { - struct tegra_mc *mc = data; - unsigned long status; - unsigned int bit; + struct tegra_mc *mc = icc_provider_to_tegra_mc(data); + struct icc_node *node; - /* mask all interrupts to avoid flooding */ - status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask; - if (!status) - return IRQ_NONE; + list_for_each_entry(node, &mc->provider.nodes, node_list) { + if (node->id == spec->args[0]) + return node; + } - for_each_set_bit(bit, &status, 32) { - const char *direction = "read", *secure = ""; - const char *error = status_names[bit]; - const char *client, *desc; - phys_addr_t addr; - u32 value, reg; - u8 id, type; + /* + * If a client driver calls devm_of_icc_get() before the MC driver + * is probed, then return EPROBE_DEFER to the client driver. + */ + return ERR_PTR(-EPROBE_DEFER); +} - switch (BIT(bit)) { - case MC_INT_DECERR_EMEM: - reg = MC_DECERR_EMEM_OTHERS_STATUS; - value = mc_readl(mc, reg); +static int tegra_mc_icc_get(struct icc_node *node, u32 *average, u32 *peak) +{ + *average = 0; + *peak = 0; - id = value & mc->soc->client_id_mask; - desc = error_names[2]; + return 0; +} - if (value & BIT(31)) - direction = "write"; - break; +static int tegra_mc_icc_set(struct icc_node *src, struct icc_node *dst) +{ + return 0; +} - case MC_INT_INVALID_GART_PAGE: - dev_err_ratelimited(mc->dev, "%s\n", error); - continue; +const struct tegra_mc_icc_ops tegra_mc_icc_ops = { + .xlate = tegra_mc_icc_xlate, + .aggregate = icc_std_aggregate, + .get_bw = tegra_mc_icc_get, + .set = tegra_mc_icc_set, +}; - case MC_INT_SECURITY_VIOLATION: - reg = MC_SECURITY_VIOLATION_STATUS; - value = mc_readl(mc, reg); +/* + * Memory Controller (MC) has few Memory Clients that are issuing memory + * bandwidth allocation requests to the MC interconnect provider. The MC + * provider aggregates the requests and then sends the aggregated request + * up to the External Memory Controller (EMC) interconnect provider which + * re-configures hardware interface to External Memory (EMEM) in accordance + * to the required bandwidth. Each MC interconnect node represents an + * individual Memory Client. + * + * Memory interconnect topology: + * + * +----+ + * +--------+ | | + * | TEXSRD +--->+ | + * +--------+ | | + * | | +-----+ +------+ + * ... | MC +--->+ EMC +--->+ EMEM | + * | | +-----+ +------+ + * +--------+ | | + * | DISP.. +--->+ | + * +--------+ | | + * +----+ + */ +static int tegra_mc_interconnect_setup(struct tegra_mc *mc) +{ + struct icc_node *node; + unsigned int i; + int err; - id = value & mc->soc->client_id_mask; - type = (value & BIT(30)) ? 4 : 3; - desc = error_names[type]; - secure = "secure "; + /* older device-trees don't have interconnect properties */ + if (!device_property_present(mc->dev, "#interconnect-cells") || + !mc->soc->icc_ops) + return 0; - if (value & BIT(31)) - direction = "write"; - break; + mc->provider.dev = mc->dev; + mc->provider.data = &mc->provider; + mc->provider.set = mc->soc->icc_ops->set; + mc->provider.aggregate = mc->soc->icc_ops->aggregate; + mc->provider.get_bw = mc->soc->icc_ops->get_bw; + mc->provider.xlate = mc->soc->icc_ops->xlate; + mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended; - default: - continue; + icc_provider_init(&mc->provider); + + /* create Memory Controller node */ + node = icc_node_create(TEGRA_ICC_MC); + if (IS_ERR(node)) + return PTR_ERR(node); + + node->name = "Memory Controller"; + icc_node_add(node, &mc->provider); + + /* link Memory Controller to External Memory Controller */ + err = icc_link_create(node, TEGRA_ICC_EMC); + if (err) + goto remove_nodes; + + for (i = 0; i < mc->soc->num_clients; i++) { + /* create MC client node */ + node = icc_node_create(mc->soc->clients[i].id); + if (IS_ERR(node)) { + err = PTR_ERR(node); + goto remove_nodes; } - client = mc->soc->clients[id].name; - addr = mc_readl(mc, reg + sizeof(u32)); + node->name = mc->soc->clients[i].name; + icc_node_add(node, &mc->provider); - dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n", - client, secure, direction, &addr, error, - desc); + /* link Memory Client to Memory Controller */ + err = icc_link_create(node, TEGRA_ICC_MC); + if (err) + goto remove_nodes; + + node->data = (struct tegra_mc_client *)&(mc->soc->clients[i]); } - /* clear interrupts */ - mc_writel(mc, status, MC_INTSTATUS); + err = icc_provider_register(&mc->provider); + if (err) + goto remove_nodes; - return IRQ_HANDLED; + return 0; + +remove_nodes: + icc_nodes_remove(&mc->provider); + + return err; +} + +static void tegra_mc_num_channel_enabled(struct tegra_mc *mc) +{ + unsigned int i; + u32 value; + + value = mc_ch_readl(mc, 0, MC_EMEM_ADR_CFG_CHANNEL_ENABLE); + if (value <= 0) { + mc->num_channels = mc->soc->num_channels; + return; + } + + for (i = 0; i < 32; i++) { + if (value & BIT(i)) + mc->num_channels++; + } } static int tegra_mc_probe(struct platform_device *pdev) { - const struct of_device_id *match; - struct resource *res; struct tegra_mc *mc; - void *isr; + u64 mask; int err; - match = of_match_node(tegra_mc_of_match, pdev->dev.of_node); - if (!match) - return -ENODEV; - mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL); if (!mc) return -ENOMEM; platform_set_drvdata(pdev, mc); spin_lock_init(&mc->lock); - mc->soc = match->data; + mc->soc = of_device_get_match_data(&pdev->dev); mc->dev = &pdev->dev; + mask = DMA_BIT_MASK(mc->soc->num_address_bits); + + err = dma_coerce_mask_and_coherent(&pdev->dev, mask); + if (err < 0) { + dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err); + return err; + } + /* length of MC tick in nanoseconds */ mc->tick = 30; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mc->regs = devm_ioremap_resource(&pdev->dev, res); + mc->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mc->regs)) return PTR_ERR(mc->regs); -#ifdef CONFIG_ARCH_TEGRA_2x_SOC - if (mc->soc == &tegra20_mc_soc) { - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - mc->regs2 = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(mc->regs2)) - return PTR_ERR(mc->regs2); - - isr = tegra20_mc_irq; - } else -#endif - { - mc->clk = devm_clk_get(&pdev->dev, "mc"); - if (IS_ERR(mc->clk)) { - dev_err(&pdev->dev, "failed to get MC clock: %ld\n", - PTR_ERR(mc->clk)); - return PTR_ERR(mc->clk); - } + mc->debugfs.root = debugfs_create_dir("mc", NULL); - err = tegra_mc_setup_latency_allowance(mc); - if (err < 0) { - dev_err(&pdev->dev, "failed to setup latency allowance: %d\n", - err); + if (mc->soc->ops && mc->soc->ops->probe) { + err = mc->soc->ops->probe(mc); + if (err < 0) return err; - } - - isr = tegra_mc_irq; } - err = tegra_mc_setup_timings(mc); - if (err < 0) { - dev_err(&pdev->dev, "failed to setup timings: %d\n", err); - return err; - } + tegra_mc_num_channel_enabled(mc); - mc->irq = platform_get_irq(pdev, 0); - if (mc->irq < 0) { - dev_err(&pdev->dev, "interrupt not specified\n"); - return mc->irq; - } + if (mc->soc->ops && mc->soc->ops->handle_irq) { + mc->irq = platform_get_irq(pdev, 0); + if (mc->irq < 0) + return mc->irq; - WARN(!mc->soc->client_id_mask, "Missing client ID mask for this SoC\n"); + WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n"); - mc_writel(mc, mc->soc->intmask, MC_INTMASK); + if (mc->soc->num_channels) + mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask, + MC_INTMASK); + else + mc_writel(mc, mc->soc->intmask, MC_INTMASK); - err = devm_request_irq(&pdev->dev, mc->irq, isr, IRQF_SHARED, - dev_name(&pdev->dev), mc); - if (err < 0) { - dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq, - err); - return err; + err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0, + dev_name(&pdev->dev), mc); + if (err < 0) { + dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq, + err); + return err; + } + } + + if (mc->soc->reset_ops) { + err = tegra_mc_reset_setup(mc); + if (err < 0) + dev_err(&pdev->dev, "failed to register reset controller: %d\n", err); } - err = tegra_mc_reset_setup(mc); + err = tegra_mc_interconnect_setup(mc); if (err < 0) - dev_err(&pdev->dev, "failed to register reset controller: %d\n", + dev_err(&pdev->dev, "failed to initialize interconnect: %d\n", err); - if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU)) { + if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) { mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc); - if (IS_ERR(mc->smmu)) + if (IS_ERR(mc->smmu)) { dev_err(&pdev->dev, "failed to probe SMMU: %ld\n", PTR_ERR(mc->smmu)); + mc->smmu = NULL; + } } return 0; } +static void tegra_mc_sync_state(struct device *dev) +{ + struct tegra_mc *mc = dev_get_drvdata(dev); + + /* check whether ICC provider is registered */ + if (mc->provider.dev == dev) + icc_sync_state(dev); +} + static struct platform_driver tegra_mc_driver = { .driver = { .name = "tegra-mc", .of_match_table = tegra_mc_of_match, .suppress_bind_attrs = true, + .sync_state = tegra_mc_sync_state, }, .prevent_deferred_probe = true, .probe = tegra_mc_probe, @@ -723,4 +1008,3 @@ arch_initcall(tegra_mc_init); MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver"); -MODULE_LICENSE("GPL v2"); |