From 0fdd1c4ea99e188dfa8ab7bafbe4004cc72dca30 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sun, 5 Nov 2023 10:34:17 +0100 Subject: dmaengine: milbeaut-hdmac: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). There is an error path that has the above mentioned problem. This patch only adds a more drastic error message. To properly fix it, dmaengine_terminate_sync() must be known to have succeeded (or that it's safe to not call it as other drivers seem to assume). Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20231105093415.3704633-7-u.kleine-koenig@pengutronix.de Signed-off-by: Vinod Koul --- drivers/dma/milbeaut-hdmac.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/milbeaut-hdmac.c b/drivers/dma/milbeaut-hdmac.c index 1b0a95892627..7b41c670970a 100644 --- a/drivers/dma/milbeaut-hdmac.c +++ b/drivers/dma/milbeaut-hdmac.c @@ -531,7 +531,7 @@ disable_clk: return ret; } -static int milbeaut_hdmac_remove(struct platform_device *pdev) +static void milbeaut_hdmac_remove(struct platform_device *pdev) { struct milbeaut_hdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -546,16 +546,21 @@ static int milbeaut_hdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } milbeaut_hdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&mdev->ddev); clk_disable_unprepare(mdev->clk); - - return 0; } static const struct of_device_id milbeaut_hdmac_match[] = { @@ -566,7 +571,7 @@ MODULE_DEVICE_TABLE(of, milbeaut_hdmac_match); static struct platform_driver milbeaut_hdmac_driver = { .probe = milbeaut_hdmac_probe, - .remove = milbeaut_hdmac_remove, + .remove_new = milbeaut_hdmac_remove, .driver = { .name = "milbeaut-m10v-hdmac", .of_match_table = milbeaut_hdmac_match, -- cgit From 47ee210011ddb8b366c7dcf9c1a9c3818573df29 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sun, 5 Nov 2023 10:34:18 +0100 Subject: dmaengine: milbeaut-xdmac: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). There is an error path that has the above mentioned problem. This patch only adds a more drastic error message. To properly fix it, dmaengine_terminate_sync() must be known to have succeeded (or that it's safe to not call it as other drivers seem to assume). Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20231105093415.3704633-8-u.kleine-koenig@pengutronix.de Signed-off-by: Vinod Koul --- drivers/dma/milbeaut-xdmac.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/milbeaut-xdmac.c b/drivers/dma/milbeaut-xdmac.c index d29d01e730aa..2cce529b448e 100644 --- a/drivers/dma/milbeaut-xdmac.c +++ b/drivers/dma/milbeaut-xdmac.c @@ -368,7 +368,7 @@ disable_xdmac: return ret; } -static int milbeaut_xdmac_remove(struct platform_device *pdev) +static void milbeaut_xdmac_remove(struct platform_device *pdev) { struct milbeaut_xdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -383,8 +383,15 @@ static int milbeaut_xdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } milbeaut_xdmac_free_chan_resources(chan); } @@ -392,8 +399,6 @@ static int milbeaut_xdmac_remove(struct platform_device *pdev) dma_async_device_unregister(&mdev->ddev); disable_xdmac(mdev); - - return 0; } static const struct of_device_id milbeaut_xdmac_match[] = { @@ -404,7 +409,7 @@ MODULE_DEVICE_TABLE(of, milbeaut_xdmac_match); static struct platform_driver milbeaut_xdmac_driver = { .probe = milbeaut_xdmac_probe, - .remove = milbeaut_xdmac_remove, + .remove_new = milbeaut_xdmac_remove, .driver = { .name = "milbeaut-m10v-xdmac", .of_match_table = milbeaut_xdmac_match, -- cgit From 5d4304a8d5646c268d73383fbc179db53f85b921 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sun, 5 Nov 2023 10:34:19 +0100 Subject: dmaengine: uniphier-mdmac: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). There is an error path that has the above mentioned problem. This patch only adds a more drastic error message. To properly fix it, dmaengine_terminate_sync() must be known to have succeeded (or that it's safe to not call it as other drivers seem to assume). Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20231105093415.3704633-9-u.kleine-koenig@pengutronix.de Signed-off-by: Vinod Koul --- drivers/dma/uniphier-mdmac.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/uniphier-mdmac.c b/drivers/dma/uniphier-mdmac.c index 618839df0748..ad7125f6e2ca 100644 --- a/drivers/dma/uniphier-mdmac.c +++ b/drivers/dma/uniphier-mdmac.c @@ -453,7 +453,7 @@ disable_clk: return ret; } -static int uniphier_mdmac_remove(struct platform_device *pdev) +static void uniphier_mdmac_remove(struct platform_device *pdev) { struct uniphier_mdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -468,16 +468,21 @@ static int uniphier_mdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } uniphier_mdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&mdev->ddev); clk_disable_unprepare(mdev->clk); - - return 0; } static const struct of_device_id uniphier_mdmac_match[] = { @@ -488,7 +493,7 @@ MODULE_DEVICE_TABLE(of, uniphier_mdmac_match); static struct platform_driver uniphier_mdmac_driver = { .probe = uniphier_mdmac_probe, - .remove = uniphier_mdmac_remove, + .remove_new = uniphier_mdmac_remove, .driver = { .name = "uniphier-mio-dmac", .of_match_table = uniphier_mdmac_match, -- cgit From ead0e402e50d1101939e4af67891d5b2fa9678b3 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Sun, 5 Nov 2023 10:34:20 +0100 Subject: dmaengine: uniphier-xdmac: Convert to platform remove callback returning void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .remove() callback for a platform driver returns an int which makes many driver authors wrongly assume it's possible to do error handling by returning an error code. However the value returned is ignored (apart from emitting a warning) and this typically results in resource leaks. To improve here there is a quest to make the remove callback return void. In the first step of this quest all drivers are converted to .remove_new(), which already returns void. Eventually after all drivers are converted, .remove_new() will be renamed to .remove(). There is an error path that has the above mentioned problem. This patch only adds a more drastic error message. To properly fix it, dmaengine_terminate_sync() must be known to have succeeded (or that it's safe to not call it as other drivers seem to assume). Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20231105093415.3704633-10-u.kleine-koenig@pengutronix.de Signed-off-by: Vinod Koul --- drivers/dma/uniphier-xdmac.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/uniphier-xdmac.c b/drivers/dma/uniphier-xdmac.c index 3a8ee2b173b5..3ce2dc2ad9de 100644 --- a/drivers/dma/uniphier-xdmac.c +++ b/drivers/dma/uniphier-xdmac.c @@ -563,7 +563,7 @@ out_unregister_dmac: return ret; } -static int uniphier_xdmac_remove(struct platform_device *pdev) +static void uniphier_xdmac_remove(struct platform_device *pdev) { struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev); struct dma_device *ddev = &xdev->ddev; @@ -579,15 +579,20 @@ static int uniphier_xdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &ddev->channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *xdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } uniphier_xdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(ddev); - - return 0; } static const struct of_device_id uniphier_xdmac_match[] = { @@ -598,7 +603,7 @@ MODULE_DEVICE_TABLE(of, uniphier_xdmac_match); static struct platform_driver uniphier_xdmac_driver = { .probe = uniphier_xdmac_probe, - .remove = uniphier_xdmac_remove, + .remove_new = uniphier_xdmac_remove, .driver = { .name = "uniphier-xdmac", .of_match_table = uniphier_xdmac_match, -- cgit From 306f5df81fcc89b462fbeb9dbe26d9a8ad7c7582 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Sun, 29 Oct 2023 18:07:04 +0100 Subject: dmaengine: apple-admac: Keep upper bits of REG_BUS_WIDTH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For RX channels, REG_BUS_WIDTH seems to default to a value of 0xf00, and macOS preserves the upper bits when setting the configuration in the lower ones. If we reset the upper bits to 0, this causes framing errors on suspend/resume (the data stream "tears" and channels get swapped around). Keeping the upper bits untouched, like the macOS driver does, fixes this issue. Signed-off-by: Hector Martin Reviewed-by: Martin Povišer Signed-off-by: Martin Povišer Link: https://lore.kernel.org/r/20231029170704.82238-1-povik+lin@cutebit.org Signed-off-by: Vinod Koul --- drivers/dma/apple-admac.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index 5b63996640d9..9588773dd2eb 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -57,6 +57,8 @@ #define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) +#define BUS_WIDTH_WORD_SIZE GENMASK(3, 0) +#define BUS_WIDTH_FRAME_SIZE GENMASK(7, 4) #define BUS_WIDTH_8BIT 0x00 #define BUS_WIDTH_16BIT 0x01 #define BUS_WIDTH_32BIT 0x02 @@ -740,7 +742,8 @@ static int admac_device_config(struct dma_chan *chan, struct admac_data *ad = adchan->host; bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV; int wordsize = 0; - u32 bus_width = 0; + u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) & + ~(BUS_WIDTH_WORD_SIZE | BUS_WIDTH_FRAME_SIZE); switch (is_tx ? config->dst_addr_width : config->src_addr_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: -- cgit From 8e578b47e6d92d5e43982ddc54045973dd4a7de5 Mon Sep 17 00:00:00 2001 From: Shravan Chippa Date: Fri, 8 Dec 2023 16:08:53 +0530 Subject: dmaengine: sf-pdma: Support of_dma_controller_register() Update sf-pdma driver to adopt generic DMA device tree bindings. It calls of_dma_controller_register() with of_dma_xlate_by_chan_id to get the generic DMA device tree helper support and the DMA clients can look up the sf-pdma controller using standard APIs. Signed-off-by: Shravan Chippa Link: https://lore.kernel.org/r/20231208103856.3732998-2-shravan.chippa@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/sf-pdma/sf-pdma.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index 3125a2f162b4..6109e1c5a09e 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "sf-pdma.h" @@ -563,7 +564,20 @@ static int sf_pdma_probe(struct platform_device *pdev) return ret; } + ret = of_dma_controller_register(pdev->dev.of_node, + of_dma_xlate_by_chan_id, pdma); + if (ret < 0) { + dev_err(&pdev->dev, + "Can't register SiFive Platform OF_DMA. (%d)\n", ret); + goto err_unregister; + } + return 0; + +err_unregister: + dma_async_device_unregister(&pdma->dma_dev); + + return ret; } static void sf_pdma_remove(struct platform_device *pdev) @@ -583,6 +597,9 @@ static void sf_pdma_remove(struct platform_device *pdev) tasklet_kill(&ch->err_tasklet); } + if (pdev->dev.of_node) + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&pdma->dma_dev); } -- cgit From 58eea79a1cf285a62af886851b1a91ed5aceb401 Mon Sep 17 00:00:00 2001 From: Shravan Chippa Date: Fri, 8 Dec 2023 16:08:55 +0530 Subject: dmaengine: sf-pdma: add mpfs-pdma compatible name Sifive platform dma (sf-pdma) has both in-order and out-of-order configurations but sf-pdam driver configured to do in-order DMA transfers, with out-of-order configuration got better throughput in the PolarFire SoC platform. Add a PolarFire SoC specific compatible and code to support for out-of-order dma transfers Reviewed-by: Emil Renner Berthing Signed-off-by: Shravan Chippa Link: https://lore.kernel.org/r/20231208103856.3732998-4-shravan.chippa@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/sf-pdma/sf-pdma.c | 27 ++++++++++++++++++++++++--- drivers/dma/sf-pdma/sf-pdma.h | 8 +++++++- 2 files changed, 31 insertions(+), 4 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index 6109e1c5a09e..428473611115 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -25,6 +25,8 @@ #include "sf-pdma.h" +#define PDMA_QUIRK_NO_STRICT_ORDERING BIT(0) + #ifndef readq static inline unsigned long long readq(void __iomem *addr) { @@ -66,7 +68,7 @@ static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan) static void sf_pdma_fill_desc(struct sf_pdma_desc *desc, u64 dst, u64 src, u64 size) { - desc->xfer_type = PDMA_FULL_SPEED; + desc->xfer_type = desc->chan->pdma->transfer_type; desc->xfer_size = size; desc->dst_addr = dst; desc->src_addr = src; @@ -493,6 +495,7 @@ static void sf_pdma_setup_chans(struct sf_pdma *pdma) static int sf_pdma_probe(struct platform_device *pdev) { + const struct sf_pdma_driver_platdata *ddata; struct sf_pdma *pdma; int ret, n_chans; const enum dma_slave_buswidth widths = @@ -518,6 +521,14 @@ static int sf_pdma_probe(struct platform_device *pdev) pdma->n_chans = n_chans; + pdma->transfer_type = PDMA_FULL_SPEED | PDMA_STRICT_ORDERING; + + ddata = device_get_match_data(&pdev->dev); + if (ddata) { + if (ddata->quirks & PDMA_QUIRK_NO_STRICT_ORDERING) + pdma->transfer_type &= ~PDMA_STRICT_ORDERING; + } + pdma->membase = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pdma->membase)) return PTR_ERR(pdma->membase); @@ -603,9 +614,19 @@ static void sf_pdma_remove(struct platform_device *pdev) dma_async_device_unregister(&pdma->dma_dev); } +static const struct sf_pdma_driver_platdata mpfs_pdma = { + .quirks = PDMA_QUIRK_NO_STRICT_ORDERING, +}; + static const struct of_device_id sf_pdma_dt_ids[] = { - { .compatible = "sifive,fu540-c000-pdma" }, - { .compatible = "sifive,pdma0" }, + { + .compatible = "sifive,fu540-c000-pdma", + }, { + .compatible = "sifive,pdma0", + }, { + .compatible = "microchip,mpfs-pdma", + .data = &mpfs_pdma, + }, {}, }; MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids); diff --git a/drivers/dma/sf-pdma/sf-pdma.h b/drivers/dma/sf-pdma/sf-pdma.h index d05772b5d8d3..215e07183d7e 100644 --- a/drivers/dma/sf-pdma/sf-pdma.h +++ b/drivers/dma/sf-pdma/sf-pdma.h @@ -48,7 +48,8 @@ #define PDMA_ERR_STATUS_MASK GENMASK(31, 31) /* Transfer Type */ -#define PDMA_FULL_SPEED 0xFF000008 +#define PDMA_FULL_SPEED 0xFF000000 +#define PDMA_STRICT_ORDERING BIT(3) /* Error Recovery */ #define MAX_RETRY 1 @@ -112,8 +113,13 @@ struct sf_pdma { struct dma_device dma_dev; void __iomem *membase; void __iomem *mappedbase; + u32 transfer_type; u32 n_chans; struct sf_pdma_chan chans[] __counted_by(n_chans); }; +struct sf_pdma_driver_platdata { + u32 quirks; +}; + #endif /* _SF_PDMA_H */ -- cgit From 25b636225a0816eac20b02fcb37daf6c722d0bed Mon Sep 17 00:00:00 2001 From: Mohan Kumar Date: Tue, 28 Nov 2023 12:46:15 +0530 Subject: dmaengine: tegra210-adma: Support dma-channel-mask property To support the flexibility to reserve the specific dma channels add the support of dma-channel-mask property in the tegra210-adma driver Signed-off-by: Mohan Kumar Link: https://lore.kernel.org/r/20231128071615.31447-3-mkumard@nvidia.com Signed-off-by: Vinod Koul --- drivers/dma/tegra210-adma.c | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index 7a0586633bf3..24ad7077c53b 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -153,6 +153,7 @@ struct tegra_adma { void __iomem *base_addr; struct clk *ahub_clk; unsigned int nr_channels; + unsigned long *dma_chan_mask; unsigned long rx_requests_reserved; unsigned long tx_requests_reserved; @@ -741,6 +742,10 @@ static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev) for (i = 0; i < tdma->nr_channels; i++) { tdc = &tdma->channels[i]; + /* skip for reserved channels */ + if (!tdc->tdma) + continue; + ch_reg = &tdc->ch_regs; ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); /* skip if channel is not active */ @@ -779,6 +784,9 @@ static int __maybe_unused tegra_adma_runtime_resume(struct device *dev) for (i = 0; i < tdma->nr_channels; i++) { tdc = &tdma->channels[i]; + /* skip for reserved channels */ + if (!tdc->tdma) + continue; ch_reg = &tdc->ch_regs; /* skip if channel was not active earlier */ if (!ch_reg->cmd) @@ -867,10 +875,31 @@ static int tegra_adma_probe(struct platform_device *pdev) return PTR_ERR(tdma->ahub_clk); } + tdma->dma_chan_mask = devm_kzalloc(&pdev->dev, + BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long), + GFP_KERNEL); + if (!tdma->dma_chan_mask) + return -ENOMEM; + + /* Enable all channels by default */ + bitmap_fill(tdma->dma_chan_mask, tdma->nr_channels); + + ret = of_property_read_u32_array(pdev->dev.of_node, "dma-channel-mask", + (u32 *)tdma->dma_chan_mask, + BITS_TO_U32(tdma->nr_channels)); + if (ret < 0 && (ret != -EINVAL)) { + dev_err(&pdev->dev, "dma-channel-mask is not complete.\n"); + return ret; + } + INIT_LIST_HEAD(&tdma->dma_dev.channels); for (i = 0; i < tdma->nr_channels; i++) { struct tegra_adma_chan *tdc = &tdma->channels[i]; + /* skip for reserved channels */ + if (!test_bit(i, tdma->dma_chan_mask)) + continue; + tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset + (cdata->ch_reg_size * i); @@ -957,8 +986,10 @@ static void tegra_adma_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&tdma->dma_dev); - for (i = 0; i < tdma->nr_channels; ++i) - irq_dispose_mapping(tdma->channels[i].irq); + for (i = 0; i < tdma->nr_channels; ++i) { + if (tdma->channels[i].irq) + irq_dispose_mapping(tdma->channels[i].irq); + } pm_runtime_disable(&pdev->dev); } -- cgit From 70f008fb3ea9bd2e6727eebc858405acd49a212b Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Fri, 24 Nov 2023 17:02:35 +0100 Subject: dmaengine: dmatest: prevent using swiotlb buffer with nobounce parameter Source and destination data buffers are allocated with GPF_KERNEL flag. It means that, if the DDR is more than 2GB, buffers can be allocated above the 32-bit addressable space. In this case, and if the dma controller is only 32-bit compatible, swiotlb bounce buffer, located in the 32-bit addressable space, is used and introduces a memcpy. To prevent this extra memcpy, due to swiotlb bounce buffer use because source or destination data buffer is allocated above the 32-bit addressable space, force source and destination data buffers allocation with GPF_DMA instead, when nobounce parameter is true. Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20231124160235.2459326-1-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/dma/dmatest.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index ffe621695e47..a4f608837849 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -21,6 +21,10 @@ #include #include +static bool nobounce; +module_param(nobounce, bool, 0644); +MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)"); + static unsigned int test_buf_size = 16384; module_param(test_buf_size, uint, 0644); MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); @@ -90,6 +94,7 @@ MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts"); /** * struct dmatest_params - test parameters. + * @nobounce: prevent using swiotlb buffer * @buf_size: size of the memcpy test buffer * @channel: bus ID of the channel to test * @device: bus ID of the DMA Engine to test @@ -106,6 +111,7 @@ MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts"); * @polled: use polling for completion instead of interrupts */ struct dmatest_params { + bool nobounce; unsigned int buf_size; char channel[20]; char device[32]; @@ -215,6 +221,7 @@ struct dmatest_done { struct dmatest_data { u8 **raw; u8 **aligned; + gfp_t gfp_flags; unsigned int cnt; unsigned int off; }; @@ -533,7 +540,7 @@ static int dmatest_alloc_test_data(struct dmatest_data *d, goto err; for (i = 0; i < d->cnt; i++) { - d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); + d->raw[i] = kmalloc(buf_size + align, d->gfp_flags); if (!d->raw[i]) goto err; @@ -655,6 +662,13 @@ static int dmatest_func(void *data) goto err_free_coefs; } + src->gfp_flags = GFP_KERNEL; + dst->gfp_flags = GFP_KERNEL; + if (params->nobounce) { + src->gfp_flags = GFP_DMA; + dst->gfp_flags = GFP_DMA; + } + if (dmatest_alloc_test_data(src, buf_size, align) < 0) goto err_free_coefs; @@ -1093,6 +1107,7 @@ static void add_threaded_test(struct dmatest_info *info) struct dmatest_params *params = &info->params; /* Copy test parameters */ + params->nobounce = nobounce; params->buf_size = test_buf_size; strscpy(params->channel, strim(test_channel), sizeof(params->channel)); strscpy(params->device, strim(test_device), sizeof(params->device)); -- cgit From 1075ee66a8c19bfa375b19c236fd6a22a867f138 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Tue, 19 Dec 2023 20:33:50 +0100 Subject: dmaengine: idxd: Remove usage of the deprecated ida_simple_xx() API ida_alloc() and ida_free() should be preferred to the deprecated ida_simple_get() and ida_simple_remove(). This is less verbose. Note that the upper limit of ida_simple_get() is exclusive, but the one of ida_alloc_range() is inclusive. Sothis change allows one more device. MINORMASK is ((1U << MINORBITS) - 1), so allowing MINORMASK as a maximum value makes sense. It is also consistent with other "ida_.*MINORMASK" and "ida_*MINOR()" usages. Signed-off-by: Christophe JAILLET Reviewed-by: Fenghua Yu Acked-by: Lijun Pan Link: https://lore.kernel.org/r/ac991f5f42112fa782a881d391d447529cbc4a23.1702967302.git.christophe.jaillet@wanadoo.fr Signed-off-by: Vinod Koul --- drivers/dma/idxd/cdev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 0423655f5a88..b00926abc69a 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -165,7 +165,7 @@ static void idxd_cdev_dev_release(struct device *dev) struct idxd_wq *wq = idxd_cdev->wq; cdev_ctx = &ictx[wq->idxd->data->type]; - ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); kfree(idxd_cdev); } @@ -463,7 +463,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) cdev = &idxd_cdev->cdev; dev = cdev_dev(idxd_cdev); cdev_ctx = &ictx[wq->idxd->data->type]; - minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); + minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL); if (minor < 0) { kfree(idxd_cdev); return minor; -- cgit From 71a5197e2b872afeef8ade3099ffc4050466b542 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sun, 17 Dec 2023 22:08:34 -0800 Subject: dmaengine: std_dma40: fix kernel-doc warnings and spelling Correct kernel-doc warnings as reported by kernel test robot: ste_dma40.c:57: warning: Excess struct member 'dev_tx' description in 'stedma40_platform_data' ste_dma40.c:57: warning: Excess struct member 'dev_rx' description in 'stedma40_platform_data' Correct spellos as reported by codespell. Signed-off-by: Randy Dunlap Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202312171417.izbQThoU-lkp@intel.com/ Cc: Linus Walleij Cc: linux-arm-kernel@lists.infradead.org Cc: Vinod Koul Cc: dmaengine@vger.kernel.org Reviewed-by: Linus Walleij Link: https://lore.kernel.org/r/20231218060834.19222-1-rdunlap@infradead.org Signed-off-by: Vinod Koul --- drivers/dma/ste_dma40.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 002833fb1fa0..2c489299148e 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -31,13 +31,11 @@ /** * struct stedma40_platform_data - Configuration struct for the dma device. * - * @dev_tx: mapping between destination event line and io address - * @dev_rx: mapping between source event line and io address * @disabled_channels: A vector, ending with -1, that marks physical channels * that are for different reasons not available for the driver. * @soft_lli_chans: A vector, that marks physical channels will use LLI by SW * which avoids HW bug that exists in some versions of the controller. - * SoftLLI introduces relink overhead that could impact performace for + * SoftLLI introduces relink overhead that could impact performance for * certain use cases. * @num_of_soft_lli_chans: The number of channels that needs to be configured * to use SoftLLI. @@ -184,7 +182,7 @@ static __maybe_unused u32 d40_backup_regs[] = { /* * since 9540 and 8540 has the same HW revision - * use v4a for 9540 or ealier + * use v4a for 9540 or earlier * use v4b for 8540 or later * HW revision: * DB8500ed has revision 0 @@ -411,7 +409,7 @@ struct d40_desc { * * @base: The virtual address of LCLA. 18 bit aligned. * @dma_addr: DMA address, if mapped - * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used. + * @base_unaligned: The original kmalloc pointer, if kmalloc is used. * This pointer is only there for clean-up on error. * @pages: The number of pages needed for all physical channels. * Only used later for clean-up on error @@ -1655,7 +1653,7 @@ static void dma_tasklet(struct tasklet_struct *t) return; check_pending_tx: - /* Rescue manouver if receiving double interrupts */ + /* Rescue maneuver if receiving double interrupts */ if (d40c->pending_tx > 0) d40c->pending_tx--; spin_unlock_irqrestore(&d40c->lock, flags); @@ -3412,7 +3410,7 @@ static int __init d40_lcla_allocate(struct d40_base *base) base->lcla_pool.base = (void *)page_list[i]; } else { /* - * After many attempts and no succees with finding the correct + * After many attempts and no success with finding the correct * alignment, try with allocating a big buffer. */ dev_warn(base->dev, -- cgit From 71e7d3cb6e55ae2eadcdb178f9243dc18499d369 Mon Sep 17 00:00:00 2001 From: Binbin Zhou Date: Mon, 18 Dec 2023 09:56:39 +0800 Subject: dmaengine: ls2x-apb: New driver for the Loongson LS2X APB DMA controller The Loongson LS2X APB DMA controller is available on Loongson-2K chips. It is a single-channel, configurable DMA controller IP core based on the AXI bus, whose main function is to integrate DMA functionality on a chip dedicated to carrying data between memory and peripherals in APB bus (e.g. nand). Signed-off-by: Binbin Zhou Signed-off-by: Yingkun Meng Link: https://lore.kernel.org/r/8df2a0199434fba3535831082966c2442ecf1cae.1702365725.git.zhoubinbin@loongson.cn Signed-off-by: Vinod Koul --- drivers/dma/Kconfig | 14 + drivers/dma/Makefile | 1 + drivers/dma/ls2x-apb-dma.c | 705 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 720 insertions(+) create mode 100644 drivers/dma/ls2x-apb-dma.c (limited to 'drivers/dma') diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 70ba506dabab..e928f2ca0f1e 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -378,6 +378,20 @@ config LPC18XX_DMAMUX Enable support for DMA on NXP LPC18xx/43xx platforms with PL080 and multiplexed DMA request lines. +config LS2X_APB_DMA + tristate "Loongson LS2X APB DMA support" + depends on LOONGARCH || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Loongson LS2X APB DMA controller driver. The + DMA controller is having single DMA channel which can be + configured for different peripherals like audio, nand, sdio + etc which is in APB bus. + + This DMA controller transfers data from memory to peripheral fifo. + It does not support memory to memory data transfer. + config MCF_EDMA tristate "Freescale eDMA engine support, ColdFire mcf5441x SoCs" depends on M5441x || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 83553a97a010..dfd40d14e408 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -48,6 +48,7 @@ obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-y += idxd/ obj-$(CONFIG_K3_DMA) += k3dma.o obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o +obj-$(CONFIG_LS2X_APB_DMA) += ls2x-apb-dma.o obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o obj-$(CONFIG_MILBEAUT_XDMAC) += milbeaut-xdmac.o obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o diff --git a/drivers/dma/ls2x-apb-dma.c b/drivers/dma/ls2x-apb-dma.c new file mode 100644 index 000000000000..a49913f3ed3f --- /dev/null +++ b/drivers/dma/ls2x-apb-dma.c @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for the Loongson LS2X APB DMA Controller + * + * Copyright (C) 2017-2023 Loongson Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dmaengine.h" +#include "virt-dma.h" + +/* Global Configuration Register */ +#define LDMA_ORDER_ERG 0x0 + +/* Bitfield definitions */ + +/* Bitfields in Global Configuration Register */ +#define LDMA_64BIT_EN BIT(0) /* 1: 64 bit support */ +#define LDMA_UNCOHERENT_EN BIT(1) /* 0: cache, 1: uncache */ +#define LDMA_ASK_VALID BIT(2) +#define LDMA_START BIT(3) /* DMA start operation */ +#define LDMA_STOP BIT(4) /* DMA stop operation */ +#define LDMA_CONFIG_MASK GENMASK(4, 0) /* DMA controller config bits mask */ + +/* Bitfields in ndesc_addr field of HW decriptor */ +#define LDMA_DESC_EN BIT(0) /*1: The next descriptor is valid */ +#define LDMA_DESC_ADDR_LOW GENMASK(31, 1) + +/* Bitfields in cmd field of HW decriptor */ +#define LDMA_INT BIT(1) /* Enable DMA interrupts */ +#define LDMA_DATA_DIRECTION BIT(12) /* 1: write to device, 0: read from device */ + +#define LDMA_SLAVE_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +#define LDMA_MAX_TRANS_LEN U32_MAX + +/*-- descriptors -----------------------------------------------------*/ + +/* + * struct ls2x_dma_hw_desc - DMA HW descriptor + * @ndesc_addr: the next descriptor low address. + * @mem_addr: memory low address. + * @apb_addr: device buffer address. + * @len: length of a piece of carried content, in words. + * @step_len: length between two moved memory data blocks. + * @step_times: number of blocks to be carried in a single DMA operation. + * @cmd: descriptor command or state. + * @stats: DMA status. + * @high_ndesc_addr: the next descriptor high address. + * @high_mem_addr: memory high address. + * @reserved: reserved + */ +struct ls2x_dma_hw_desc { + u32 ndesc_addr; + u32 mem_addr; + u32 apb_addr; + u32 len; + u32 step_len; + u32 step_times; + u32 cmd; + u32 stats; + u32 high_ndesc_addr; + u32 high_mem_addr; + u32 reserved[2]; +} __packed; + +/* + * struct ls2x_dma_sg - ls2x dma scatter gather entry + * @hw: the pointer to DMA HW descriptor. + * @llp: physical address of the DMA HW descriptor. + * @phys: destination or source address(mem). + * @len: number of Bytes to read. + */ +struct ls2x_dma_sg { + struct ls2x_dma_hw_desc *hw; + dma_addr_t llp; + dma_addr_t phys; + u32 len; +}; + +/* + * struct ls2x_dma_desc - software descriptor + * @vdesc: pointer to the virtual dma descriptor. + * @cyclic: flag to dma cyclic + * @burst_size: burst size of transaction, in words. + * @desc_num: number of sg entries. + * @direction: transfer direction, to or from device. + * @status: dma controller status. + * @sg: array of sgs. + */ +struct ls2x_dma_desc { + struct virt_dma_desc vdesc; + bool cyclic; + size_t burst_size; + u32 desc_num; + enum dma_transfer_direction direction; + enum dma_status status; + struct ls2x_dma_sg sg[] __counted_by(desc_num); +}; + +/*-- Channels --------------------------------------------------------*/ + +/* + * struct ls2x_dma_chan - internal representation of an LS2X APB DMA channel + * @vchan: virtual dma channel entry. + * @desc: pointer to the ls2x sw dma descriptor. + * @pool: hw desc table + * @irq: irq line + * @sconfig: configuration for slave transfers, passed via .device_config + */ +struct ls2x_dma_chan { + struct virt_dma_chan vchan; + struct ls2x_dma_desc *desc; + void *pool; + int irq; + struct dma_slave_config sconfig; +}; + +/*-- Controller ------------------------------------------------------*/ + +/* + * struct ls2x_dma_priv - LS2X APB DMAC specific information + * @ddev: dmaengine dma_device object members + * @dma_clk: DMAC clock source + * @regs: memory mapped register base + * @lchan: channel to store ls2x_dma_chan structures + */ +struct ls2x_dma_priv { + struct dma_device ddev; + struct clk *dma_clk; + void __iomem *regs; + struct ls2x_dma_chan lchan; +}; + +/*-- Helper functions ------------------------------------------------*/ + +static inline struct ls2x_dma_desc *to_ldma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct ls2x_dma_desc, vdesc); +} + +static inline struct ls2x_dma_chan *to_ldma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct ls2x_dma_chan, vchan.chan); +} + +static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev) +{ + return container_of(ddev, struct ls2x_dma_priv, ddev); +} + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan); + struct ls2x_dma_desc *desc = to_ldma_desc(vdesc); + int i; + + for (i = 0; i < desc->desc_num; i++) { + if (desc->sg[i].hw) + dma_pool_free(lchan->pool, desc->sg[i].hw, + desc->sg[i].llp); + } + + kfree(desc); +} + +static void ls2x_dma_write_cmd(struct ls2x_dma_chan *lchan, bool cmd) +{ + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); + u64 val; + + val = lo_hi_readq(priv->regs + LDMA_ORDER_ERG) & ~LDMA_CONFIG_MASK; + val |= LDMA_64BIT_EN | cmd; + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); +} + +static void ls2x_dma_start_transfer(struct ls2x_dma_chan *lchan) +{ + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); + struct ls2x_dma_sg *ldma_sg; + struct virt_dma_desc *vdesc; + u64 val; + + /* Get the next descriptor */ + vdesc = vchan_next_desc(&lchan->vchan); + if (!vdesc) { + lchan->desc = NULL; + return; + } + + list_del(&vdesc->node); + lchan->desc = to_ldma_desc(vdesc); + ldma_sg = &lchan->desc->sg[0]; + + /* Start DMA */ + lo_hi_writeq(0, priv->regs + LDMA_ORDER_ERG); + val = (ldma_sg->llp & ~LDMA_CONFIG_MASK) | LDMA_64BIT_EN | LDMA_START; + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); +} + +static size_t ls2x_dmac_detect_burst(struct ls2x_dma_chan *lchan) +{ + u32 maxburst, buswidth; + + /* Reject definitely invalid configurations */ + if ((lchan->sconfig.src_addr_width & LDMA_SLAVE_BUSWIDTHS) && + (lchan->sconfig.dst_addr_width & LDMA_SLAVE_BUSWIDTHS)) + return 0; + + if (lchan->sconfig.direction == DMA_MEM_TO_DEV) { + maxburst = lchan->sconfig.dst_maxburst; + buswidth = lchan->sconfig.dst_addr_width; + } else { + maxburst = lchan->sconfig.src_maxburst; + buswidth = lchan->sconfig.src_addr_width; + } + + /* If maxburst is zero, fallback to LDMA_MAX_TRANS_LEN */ + return maxburst ? (maxburst * buswidth) >> 2 : LDMA_MAX_TRANS_LEN; +} + +static void ls2x_dma_fill_desc(struct ls2x_dma_chan *lchan, u32 sg_index, + struct ls2x_dma_desc *desc) +{ + struct ls2x_dma_sg *ldma_sg = &desc->sg[sg_index]; + u32 num_segments, segment_size; + + if (desc->direction == DMA_MEM_TO_DEV) { + ldma_sg->hw->cmd = LDMA_INT | LDMA_DATA_DIRECTION; + ldma_sg->hw->apb_addr = lchan->sconfig.dst_addr; + } else { + ldma_sg->hw->cmd = LDMA_INT; + ldma_sg->hw->apb_addr = lchan->sconfig.src_addr; + } + + ldma_sg->hw->mem_addr = lower_32_bits(ldma_sg->phys); + ldma_sg->hw->high_mem_addr = upper_32_bits(ldma_sg->phys); + + /* Split into multiple equally sized segments if necessary */ + num_segments = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, desc->burst_size); + segment_size = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, num_segments); + + /* Word count register takes input in words */ + ldma_sg->hw->len = segment_size; + ldma_sg->hw->step_times = num_segments; + ldma_sg->hw->step_len = 0; + + /* lets make a link list */ + if (sg_index) { + desc->sg[sg_index - 1].hw->ndesc_addr = ldma_sg->llp | LDMA_DESC_EN; + desc->sg[sg_index - 1].hw->high_ndesc_addr = upper_32_bits(ldma_sg->llp); + } +} + +/*-- DMA Engine API --------------------------------------------------*/ + +/* + * ls2x_dma_alloc_chan_resources - allocate resources for DMA channel + * @chan: allocate descriptor resources for this channel + * + * return - the number of allocated descriptors + */ +static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + /* Create a pool of consistent memory blocks for hardware descriptors */ + lchan->pool = dma_pool_create(dev_name(chan2dev(chan)), + chan->device->dev, PAGE_SIZE, + __alignof__(struct ls2x_dma_hw_desc), 0); + if (!lchan->pool) { + dev_err(chan2dev(chan), "No memory for descriptors\n"); + return -ENOMEM; + } + + return 1; +} + +/* + * ls2x_dma_free_chan_resources - free all channel resources + * @chan: DMA channel + */ +static void ls2x_dma_free_chan_resources(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + vchan_free_chan_resources(to_virt_chan(chan)); + dma_pool_destroy(lchan->pool); + lchan->pool = NULL; +} + +/* + * ls2x_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction + * @chan: DMA channel + * @sgl: scatterlist to transfer to/from + * @sg_len: number of entries in @scatterlist + * @direction: DMA direction + * @flags: tx descriptor status flags + * @context: transaction context (ignored) + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +ls2x_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + u32 sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + struct ls2x_dma_desc *desc; + struct scatterlist *sg; + size_t burst_size; + int i; + + if (unlikely(!sg_len || !is_slave_direction(direction))) + return NULL; + + burst_size = ls2x_dmac_detect_burst(lchan); + if (!burst_size) + return NULL; + + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->desc_num = sg_len; + desc->direction = direction; + desc->burst_size = burst_size; + + for_each_sg(sgl, sg, sg_len, i) { + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; + + /* Allocate DMA capable memory for hardware descriptor */ + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); + if (!ldma_sg->hw) { + desc->desc_num = i; + ls2x_dma_desc_free(&desc->vdesc); + return NULL; + } + + ldma_sg->phys = sg_dma_address(sg); + ldma_sg->len = sg_dma_len(sg); + + ls2x_dma_fill_desc(lchan, i, desc); + } + + /* Setting the last descriptor enable bit */ + desc->sg[sg_len - 1].hw->ndesc_addr &= ~LDMA_DESC_EN; + desc->status = DMA_IN_PROGRESS; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +/* + * ls2x_dma_prep_dma_cyclic - prepare the cyclic DMA transfer + * @chan: the DMA channel to prepare + * @buf_addr: physical DMA address where the buffer starts + * @buf_len: total number of bytes for the entire buffer + * @period_len: number of bytes for each period + * @direction: transfer direction, to or from device + * @flags: tx descriptor status flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +ls2x_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + struct ls2x_dma_desc *desc; + size_t burst_size; + u32 num_periods; + int i; + + if (unlikely(!buf_len || !period_len)) + return NULL; + + if (unlikely(!is_slave_direction(direction))) + return NULL; + + burst_size = ls2x_dmac_detect_burst(lchan); + if (!burst_size) + return NULL; + + num_periods = buf_len / period_len; + desc = kzalloc(struct_size(desc, sg, num_periods), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->desc_num = num_periods; + desc->direction = direction; + desc->burst_size = burst_size; + + /* Build cyclic linked list */ + for (i = 0; i < num_periods; i++) { + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; + + /* Allocate DMA capable memory for hardware descriptor */ + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); + if (!ldma_sg->hw) { + desc->desc_num = i; + ls2x_dma_desc_free(&desc->vdesc); + return NULL; + } + + ldma_sg->phys = buf_addr + period_len * i; + ldma_sg->len = period_len; + + ls2x_dma_fill_desc(lchan, i, desc); + } + + /* Lets make a cyclic list */ + desc->sg[num_periods - 1].hw->ndesc_addr = desc->sg[0].llp | LDMA_DESC_EN; + desc->sg[num_periods - 1].hw->high_ndesc_addr = upper_32_bits(desc->sg[0].llp); + desc->cyclic = true; + desc->status = DMA_IN_PROGRESS; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +/* + * ls2x_slave_config - set slave configuration for channel + * @chan: dma channel + * @cfg: slave configuration + * + * Sets slave configuration for channel + */ +static int ls2x_dma_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + memcpy(&lchan->sconfig, config, sizeof(*config)); + return 0; +} + +/* + * ls2x_dma_issue_pending - push pending transactions to the hardware + * @chan: channel + * + * When this function is called, all pending transactions are pushed to the + * hardware and executed. + */ +static void ls2x_dma_issue_pending(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) + ls2x_dma_start_transfer(lchan); + spin_unlock_irqrestore(&lchan->vchan.lock, flags); +} + +/* + * ls2x_dma_terminate_all - terminate all transactions + * @chan: channel + * + * Stops all DMA transactions. + */ +static int ls2x_dma_terminate_all(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&lchan->vchan.lock, flags); + /* Setting stop cmd */ + ls2x_dma_write_cmd(lchan, LDMA_STOP); + if (lchan->desc) { + vchan_terminate_vdesc(&lchan->desc->vdesc); + lchan->desc = NULL; + } + + vchan_get_all_descriptors(&lchan->vchan, &head); + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + vchan_dma_desc_free_list(&lchan->vchan, &head); + return 0; +} + +/* + * ls2x_dma_synchronize - Synchronizes the termination of transfers to the + * current context. + * @chan: channel + */ +static void ls2x_dma_synchronize(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + vchan_synchronize(&lchan->vchan); +} + +static int ls2x_dma_pause(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) { + ls2x_dma_write_cmd(lchan, LDMA_STOP); + lchan->desc->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + return 0; +} + +static int ls2x_dma_resume(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (lchan->desc && lchan->desc->status == DMA_PAUSED) { + lchan->desc->status = DMA_IN_PROGRESS; + ls2x_dma_write_cmd(lchan, LDMA_START); + } + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + return 0; +} + +/* + * ls2x_dma_isr - LS2X DMA Interrupt handler + * @irq: IRQ number + * @dev_id: Pointer to ls2x_dma_chan + * + * Return: IRQ_HANDLED/IRQ_NONE + */ +static irqreturn_t ls2x_dma_isr(int irq, void *dev_id) +{ + struct ls2x_dma_chan *lchan = dev_id; + struct ls2x_dma_desc *desc; + + spin_lock(&lchan->vchan.lock); + desc = lchan->desc; + if (desc) { + if (desc->cyclic) { + vchan_cyclic_callback(&desc->vdesc); + } else { + desc->status = DMA_COMPLETE; + vchan_cookie_complete(&desc->vdesc); + ls2x_dma_start_transfer(lchan); + } + + /* ls2x_dma_start_transfer() updates lchan->desc */ + if (!lchan->desc) + ls2x_dma_write_cmd(lchan, LDMA_STOP); + } + spin_unlock(&lchan->vchan.lock); + + return IRQ_HANDLED; +} + +static int ls2x_dma_chan_init(struct platform_device *pdev, + struct ls2x_dma_priv *priv) +{ + struct ls2x_dma_chan *lchan = &priv->lchan; + struct device *dev = &pdev->dev; + int ret; + + lchan->irq = platform_get_irq(pdev, 0); + if (lchan->irq < 0) + return lchan->irq; + + ret = devm_request_irq(dev, lchan->irq, ls2x_dma_isr, IRQF_TRIGGER_RISING, + dev_name(&pdev->dev), lchan); + if (ret) + return ret; + + /* Initialize channels related values */ + INIT_LIST_HEAD(&priv->ddev.channels); + lchan->vchan.desc_free = ls2x_dma_desc_free; + vchan_init(&lchan->vchan, &priv->ddev); + + return 0; +} + +/* + * ls2x_dma_probe - Driver probe function + * @pdev: Pointer to the platform_device structure + * + * Return: '0' on success and failure value on error + */ +static int ls2x_dma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ls2x_dma_priv *priv; + struct dma_device *ddev; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->regs)) + return dev_err_probe(dev, PTR_ERR(priv->regs), + "devm_platform_ioremap_resource failed.\n"); + + priv->dma_clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(priv->dma_clk)) + return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "devm_clk_get failed.\n"); + + ret = clk_prepare_enable(priv->dma_clk); + if (ret) + return dev_err_probe(dev, ret, "clk_prepare_enable failed.\n"); + + ret = ls2x_dma_chan_init(pdev, priv); + if (ret) + goto disable_clk; + + ddev = &priv->ddev; + ddev->dev = dev; + dma_cap_zero(ddev->cap_mask); + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); + + ddev->device_alloc_chan_resources = ls2x_dma_alloc_chan_resources; + ddev->device_free_chan_resources = ls2x_dma_free_chan_resources; + ddev->device_tx_status = dma_cookie_status; + ddev->device_issue_pending = ls2x_dma_issue_pending; + ddev->device_prep_slave_sg = ls2x_dma_prep_slave_sg; + ddev->device_prep_dma_cyclic = ls2x_dma_prep_dma_cyclic; + ddev->device_config = ls2x_dma_slave_config; + ddev->device_terminate_all = ls2x_dma_terminate_all; + ddev->device_synchronize = ls2x_dma_synchronize; + ddev->device_pause = ls2x_dma_pause; + ddev->device_resume = ls2x_dma_resume; + + ddev->src_addr_widths = LDMA_SLAVE_BUSWIDTHS; + ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS; + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + + ret = dma_async_device_register(&priv->ddev); + if (ret < 0) + goto disable_clk; + + ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv); + if (ret < 0) + goto unregister_dmac; + + platform_set_drvdata(pdev, priv); + + dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n"); + return 0; + +unregister_dmac: + dma_async_device_unregister(&priv->ddev); +disable_clk: + clk_disable_unprepare(priv->dma_clk); + + return ret; +} + +/* + * ls2x_dma_remove - Driver remove function + * @pdev: Pointer to the platform_device structure + */ +static void ls2x_dma_remove(struct platform_device *pdev) +{ + struct ls2x_dma_priv *priv = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&priv->ddev); + clk_disable_unprepare(priv->dma_clk); +} + +static const struct of_device_id ls2x_dma_of_match_table[] = { + { .compatible = "loongson,ls2k1000-apbdma" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ls2x_dma_of_match_table); + +static struct platform_driver ls2x_dmac_driver = { + .probe = ls2x_dma_probe, + .remove_new = ls2x_dma_remove, + .driver = { + .name = "ls2x-apbdma", + .of_match_table = ls2x_dma_of_match_table, + }, +}; +module_platform_driver(ls2x_dmac_driver); + +MODULE_DESCRIPTION("Loongson LS2X APB DMA Controller driver"); +MODULE_AUTHOR("Loongson Technology Corporation Limited"); +MODULE_LICENSE("GPL"); -- cgit From a2ab7045389feab1c26ebab105a8ad6bce74a4a7 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 15 Dec 2023 14:13:09 +0100 Subject: dmaengine: axi-dmac: Small code cleanup Use a for() loop instead of a while() loop in axi_dmac_fill_linear_sg(). This makes the code leaner and cleaner overall, and does not introduce any functional change. Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20231215131313.23840-2-paul@crapouillou.net Signed-off-by: Vinod Koul --- drivers/dma/dma-axi-dmac.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 2457a420c13d..760940b21eab 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -508,16 +508,13 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; for (i = 0; i < num_periods; i++) { - len = period_len; - - while (len > segment_size) { + for (len = period_len; len > segment_size; sg++) { if (direction == DMA_DEV_TO_MEM) sg->dest_addr = addr; else sg->src_addr = addr; sg->x_len = segment_size; sg->y_len = 1; - sg++; addr += segment_size; len -= segment_size; } -- cgit From 3f8fd25936ee5f52596f10d420f650c5b5e3285f Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 15 Dec 2023 14:13:10 +0100 Subject: dmaengine: axi-dmac: Allocate hardware descriptors Change where and how the DMA transfers meta-data is stored, to prepare for the upcoming introduction of scatter-gather support. Allocate hardware descriptors in the format that the HDL core will be expecting them when the scatter-gather feature is enabled, and use these fields to store the data that was previously stored in the axi_dmac_sg structure. Note that the 'x_len' and 'y_len' fields now contain the transfer length minus one, since that's what the hardware will expect in these fields. Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20231215131313.23840-3-paul@crapouillou.net Signed-off-by: Vinod Koul --- drivers/dma/dma-axi-dmac.c | 134 +++++++++++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 46 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 760940b21eab..185230a769b9 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -97,20 +97,31 @@ /* The maximum ID allocated by the hardware is 31 */ #define AXI_DMAC_SG_UNUSED 32U +struct axi_dmac_hw_desc { + u32 flags; + u32 id; + u64 dest_addr; + u64 src_addr; + u64 __unused; + u32 y_len; + u32 x_len; + u32 src_stride; + u32 dst_stride; + u64 __pad[2]; +}; + struct axi_dmac_sg { - dma_addr_t src_addr; - dma_addr_t dest_addr; - unsigned int x_len; - unsigned int y_len; - unsigned int dest_stride; - unsigned int src_stride; - unsigned int id; unsigned int partial_len; bool schedule_when_free; + + struct axi_dmac_hw_desc *hw; + dma_addr_t hw_phys; }; struct axi_dmac_desc { struct virt_dma_desc vdesc; + struct axi_dmac_chan *chan; + bool cyclic; bool have_partial_xfer; @@ -229,7 +240,7 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) sg = &desc->sg[desc->num_submitted]; /* Already queued in cyclic mode. Wait for it to finish */ - if (sg->id != AXI_DMAC_SG_UNUSED) { + if (sg->hw->id != AXI_DMAC_SG_UNUSED) { sg->schedule_when_free = true; return; } @@ -246,16 +257,16 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) chan->next_desc = desc; } - sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); + sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); if (axi_dmac_dest_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride); + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); } if (axi_dmac_src_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride); + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); } /* @@ -270,8 +281,8 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) if (chan->hw_partial_xfer) flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; - axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); - axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); } @@ -286,9 +297,9 @@ static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, struct axi_dmac_sg *sg) { if (chan->hw_2d) - return sg->x_len * sg->y_len; + return (sg->hw->x_len + 1) * (sg->hw->y_len + 1); else - return sg->x_len; + return (sg->hw->x_len + 1); } static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) @@ -307,9 +318,9 @@ static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) list_for_each_entry(desc, &chan->active_descs, vdesc.node) { for (i = 0; i < desc->num_sgs; i++) { sg = &desc->sg[i]; - if (sg->id == AXI_DMAC_SG_UNUSED) + if (sg->hw->id == AXI_DMAC_SG_UNUSED) continue; - if (sg->id == id) { + if (sg->hw->id == id) { desc->have_partial_xfer = true; sg->partial_len = len; found_sg = true; @@ -376,12 +387,12 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, do { sg = &active->sg[active->num_completed]; - if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ + if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ break; - if (!(BIT(sg->id) & completed_transfers)) + if (!(BIT(sg->hw->id) & completed_transfers)) break; active->num_completed++; - sg->id = AXI_DMAC_SG_UNUSED; + sg->hw->id = AXI_DMAC_SG_UNUSED; if (sg->schedule_when_free) { sg->schedule_when_free = false; start_next = true; @@ -476,22 +487,52 @@ static void axi_dmac_issue_pending(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); } -static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs) +static struct axi_dmac_desc * +axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) { + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + struct device *dev = dmac->dma_dev.dev; + struct axi_dmac_hw_desc *hws; struct axi_dmac_desc *desc; + dma_addr_t hw_phys; unsigned int i; desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT); if (!desc) return NULL; desc->num_sgs = num_sgs; + desc->chan = chan; - for (i = 0; i < num_sgs; i++) - desc->sg[i].id = AXI_DMAC_SG_UNUSED; + hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)), + &hw_phys, GFP_ATOMIC); + if (!hws) { + kfree(desc); + return NULL; + } + + for (i = 0; i < num_sgs; i++) { + desc->sg[i].hw = &hws[i]; + desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws); + + hws[i].id = AXI_DMAC_SG_UNUSED; + hws[i].flags = 0; + } return desc; } +static void axi_dmac_free_desc(struct axi_dmac_desc *desc) +{ + struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan); + struct device *dev = dmac->dma_dev.dev; + struct axi_dmac_hw_desc *hw = desc->sg[0].hw; + dma_addr_t hw_phys = desc->sg[0].hw_phys; + + dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)), + hw, hw_phys); + kfree(desc); +} + static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, enum dma_transfer_direction direction, dma_addr_t addr, unsigned int num_periods, unsigned int period_len, @@ -510,21 +551,22 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, for (i = 0; i < num_periods; i++) { for (len = period_len; len > segment_size; sg++) { if (direction == DMA_DEV_TO_MEM) - sg->dest_addr = addr; + sg->hw->dest_addr = addr; else - sg->src_addr = addr; - sg->x_len = segment_size; - sg->y_len = 1; + sg->hw->src_addr = addr; + sg->hw->x_len = segment_size - 1; + sg->hw->y_len = 0; + sg->hw->flags = 0; addr += segment_size; len -= segment_size; } if (direction == DMA_DEV_TO_MEM) - sg->dest_addr = addr; + sg->hw->dest_addr = addr; else - sg->src_addr = addr; - sg->x_len = len; - sg->y_len = 1; + sg->hw->src_addr = addr; + sg->hw->x_len = len - 1; + sg->hw->y_len = 0; sg++; addr += len; } @@ -551,7 +593,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length); - desc = axi_dmac_alloc_desc(num_sgs); + desc = axi_dmac_alloc_desc(chan, num_sgs); if (!desc) return NULL; @@ -560,7 +602,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) { if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || !axi_dmac_check_len(chan, sg_dma_len(sg))) { - kfree(desc); + axi_dmac_free_desc(desc); return NULL; } @@ -595,7 +637,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( num_periods = buf_len / period_len; num_segments = DIV_ROUND_UP(period_len, chan->max_length); - desc = axi_dmac_alloc_desc(num_periods * num_segments); + desc = axi_dmac_alloc_desc(chan, num_periods * num_segments); if (!desc) return NULL; @@ -650,26 +692,26 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( return NULL; } - desc = axi_dmac_alloc_desc(1); + desc = axi_dmac_alloc_desc(chan, 1); if (!desc) return NULL; if (axi_dmac_src_is_mem(chan)) { - desc->sg[0].src_addr = xt->src_start; - desc->sg[0].src_stride = xt->sgl[0].size + src_icg; + desc->sg[0].hw->src_addr = xt->src_start; + desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg; } if (axi_dmac_dest_is_mem(chan)) { - desc->sg[0].dest_addr = xt->dst_start; - desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg; + desc->sg[0].hw->dest_addr = xt->dst_start; + desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg; } if (chan->hw_2d) { - desc->sg[0].x_len = xt->sgl[0].size; - desc->sg[0].y_len = xt->numf; + desc->sg[0].hw->x_len = xt->sgl[0].size - 1; + desc->sg[0].hw->y_len = xt->numf - 1; } else { - desc->sg[0].x_len = xt->sgl[0].size * xt->numf; - desc->sg[0].y_len = 1; + desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1; + desc->sg[0].hw->y_len = 0; } if (flags & DMA_CYCLIC) @@ -685,7 +727,7 @@ static void axi_dmac_free_chan_resources(struct dma_chan *c) static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) { - kfree(container_of(vdesc, struct axi_dmac_desc, vdesc)); + axi_dmac_free_desc(to_axi_dmac_desc(vdesc)); } static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) -- cgit From e97dc7435972d28ac7d96d199d4aedb868d04fd8 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 15 Dec 2023 14:13:11 +0100 Subject: dmaengine: axi-dmac: Add support for scatter-gather transfers Implement support for scatter-gather transfers. Build a chain of hardware descriptors, each one corresponding to a segment of the transfer, and linked to the next one. The hardware will transfer the chain and only fire interrupts when the whole chain has been transferred. Support for scatter-gather is automatically enabled when the driver detects that the hardware supports it, by writing then reading the AXI_DMAC_REG_SG_ADDRESS register. If not available, the driver will fall back to standard DMA transfers. Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20231215131313.23840-4-paul@crapouillou.net Signed-off-by: Vinod Koul --- drivers/dma/dma-axi-dmac.c | 135 +++++++++++++++++++++++++++++++-------------- 1 file changed, 93 insertions(+), 42 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 185230a769b9..5109530b66de 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -81,9 +81,13 @@ #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 +#define AXI_DMAC_REG_CURRENT_SG_ID 0x454 +#define AXI_DMAC_REG_SG_ADDRESS 0x47c +#define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc #define AXI_DMAC_CTRL_ENABLE BIT(0) #define AXI_DMAC_CTRL_PAUSE BIT(1) +#define AXI_DMAC_CTRL_ENABLE_SG BIT(2) #define AXI_DMAC_IRQ_SOT BIT(0) #define AXI_DMAC_IRQ_EOT BIT(1) @@ -97,12 +101,16 @@ /* The maximum ID allocated by the hardware is 31 */ #define AXI_DMAC_SG_UNUSED 32U +/* Flags for axi_dmac_hw_desc.flags */ +#define AXI_DMAC_HW_FLAG_LAST BIT(0) +#define AXI_DMAC_HW_FLAG_IRQ BIT(1) + struct axi_dmac_hw_desc { u32 flags; u32 id; u64 dest_addr; u64 src_addr; - u64 __unused; + u64 next_sg_addr; u32 y_len; u32 x_len; u32 src_stride; @@ -150,6 +158,7 @@ struct axi_dmac_chan { bool hw_partial_xfer; bool hw_cyclic; bool hw_2d; + bool hw_sg; }; struct axi_dmac { @@ -224,9 +233,11 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) unsigned int flags = 0; unsigned int val; - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); - if (val) /* Queue is full, wait for the next SOT IRQ */ - return; + if (!chan->hw_sg) { + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; + } desc = chan->next_desc; @@ -245,9 +256,10 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) return; } - desc->num_submitted++; - if (desc->num_submitted == desc->num_sgs || - desc->have_partial_xfer) { + if (chan->hw_sg) { + chan->next_desc = NULL; + } else if (++desc->num_submitted == desc->num_sgs || + desc->have_partial_xfer) { if (desc->cyclic) desc->num_submitted = 0; /* Start again */ else @@ -259,14 +271,16 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); - if (axi_dmac_dest_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); - } + if (!chan->hw_sg) { + if (axi_dmac_dest_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); + } - if (axi_dmac_src_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); + if (axi_dmac_src_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); + } } /* @@ -281,8 +295,14 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) if (chan->hw_partial_xfer) flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; - axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); - axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); + if (chan->hw_sg) { + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys); + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH, + (u64)sg->hw_phys >> 32); + } else { + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); + } axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); } @@ -359,6 +379,9 @@ static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, rslt->result = DMA_TRANS_NOERROR; rslt->residue = 0; + if (chan->hw_sg) + return; + /* * We get here if the last completed segment is partial, which * means we can compute the residue from that segment onwards @@ -385,36 +408,46 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) axi_dmac_dequeue_partial_xfers(chan); - do { - sg = &active->sg[active->num_completed]; - if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ - break; - if (!(BIT(sg->hw->id) & completed_transfers)) - break; - active->num_completed++; - sg->hw->id = AXI_DMAC_SG_UNUSED; - if (sg->schedule_when_free) { - sg->schedule_when_free = false; - start_next = true; + if (chan->hw_sg) { + if (active->cyclic) { + vchan_cyclic_callback(&active->vdesc); + } else { + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + active = axi_dmac_active_desc(chan); } + } else { + do { + sg = &active->sg[active->num_completed]; + if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ + break; + if (!(BIT(sg->hw->id) & completed_transfers)) + break; + active->num_completed++; + sg->hw->id = AXI_DMAC_SG_UNUSED; + if (sg->schedule_when_free) { + sg->schedule_when_free = false; + start_next = true; + } - if (sg->partial_len) - axi_dmac_compute_residue(chan, active); + if (sg->partial_len) + axi_dmac_compute_residue(chan, active); - if (active->cyclic) - vchan_cyclic_callback(&active->vdesc); + if (active->cyclic) + vchan_cyclic_callback(&active->vdesc); - if (active->num_completed == active->num_sgs || - sg->partial_len) { - if (active->cyclic) { - active->num_completed = 0; /* wrap around */ - } else { - list_del(&active->vdesc.node); - vchan_cookie_complete(&active->vdesc); - active = axi_dmac_active_desc(chan); + if (active->num_completed == active->num_sgs || + sg->partial_len) { + if (active->cyclic) { + active->num_completed = 0; /* wrap around */ + } else { + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + active = axi_dmac_active_desc(chan); + } } - } - } while (active); + } while (active); + } return start_next; } @@ -478,8 +511,12 @@ static void axi_dmac_issue_pending(struct dma_chan *c) struct axi_dmac_chan *chan = to_axi_dmac_chan(c); struct axi_dmac *dmac = chan_to_axi_dmac(chan); unsigned long flags; + u32 ctrl = AXI_DMAC_CTRL_ENABLE; + + if (chan->hw_sg) + ctrl |= AXI_DMAC_CTRL_ENABLE_SG; - axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl); spin_lock_irqsave(&chan->vchan.lock, flags); if (vchan_issue_pending(&chan->vchan)) @@ -516,8 +553,14 @@ axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) hws[i].id = AXI_DMAC_SG_UNUSED; hws[i].flags = 0; + + /* Link hardware descriptors */ + hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws); } + /* The last hardware descriptor will trigger an interrupt */ + desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ; + return desc; } @@ -753,6 +796,9 @@ static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) case AXI_DMAC_REG_CURRENT_DEST_ADDR: case AXI_DMAC_REG_PARTIAL_XFER_LEN: case AXI_DMAC_REG_PARTIAL_XFER_ID: + case AXI_DMAC_REG_CURRENT_SG_ID: + case AXI_DMAC_REG_SG_ADDRESS: + case AXI_DMAC_REG_SG_ADDRESS_HIGH: return true; default: return false; @@ -905,6 +951,10 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) chan->hw_cyclic = true; + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff); + if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS)) + chan->hw_sg = true; + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) chan->hw_2d = true; @@ -1005,6 +1055,7 @@ static int axi_dmac_probe(struct platform_device *pdev) dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); dma_dev->directions = BIT(dmac->chan.direction); dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */ INIT_LIST_HEAD(&dma_dev->channels); dmac->chan.vchan.desc_free = axi_dmac_desc_free; -- cgit From 238f68a08e19a612b8912c8697901e9982f97811 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 15 Dec 2023 14:13:12 +0100 Subject: dmaengine: axi-dmac: Use only EOT interrupts when doing scatter-gather Instead of notifying userspace in the end-of-transfer (EOT) interrupt and program the hardware in the start-of-transfer (SOT) interrupt, we can do both things in the EOT, allowing us to mask the SOT, and halve the number of interrupts sent by the HDL core. Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20231215131313.23840-5-paul@crapouillou.net Signed-off-by: Vinod Koul --- drivers/dma/dma-axi-dmac.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 5109530b66de..f63acae511fb 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -411,10 +411,12 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, if (chan->hw_sg) { if (active->cyclic) { vchan_cyclic_callback(&active->vdesc); + start_next = true; } else { list_del(&active->vdesc.node); vchan_cookie_complete(&active->vdesc); active = axi_dmac_active_desc(chan); + start_next = !!active; } } else { do { @@ -1000,6 +1002,7 @@ static int axi_dmac_probe(struct platform_device *pdev) struct axi_dmac *dmac; struct regmap *regmap; unsigned int version; + u32 irq_mask = 0; int ret; dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); @@ -1067,7 +1070,10 @@ static int axi_dmac_probe(struct platform_device *pdev) dma_dev->copy_align = (dmac->chan.address_align_mask + 1); - axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00); + if (dmac->chan.hw_sg) + irq_mask |= AXI_DMAC_IRQ_SOT; + + axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask); if (of_dma_is_coherent(pdev->dev.of_node)) { ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC); -- cgit From f60dfe0c561a8f1b8e30d3770997cbaa636f57f9 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 15 Dec 2023 14:13:13 +0100 Subject: dmaengine: axi-dmac: Improve cyclic DMA transfers in SG mode For cyclic transfers, chain the last descriptor to the first one, and disable IRQ generation if there is no callback registered with the cyclic transfer. Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20231215131313.23840-6-paul@crapouillou.net Signed-off-by: Vinod Koul --- drivers/dma/dma-axi-dmac.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index f63acae511fb..4e339c04fc1e 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -285,12 +285,14 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) /* * If the hardware supports cyclic transfers and there is no callback to - * call and only a single segment, enable hw cyclic mode to avoid - * unnecessary interrupts. + * call, enable hw cyclic mode to avoid unnecessary interrupts. */ - if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback && - desc->num_sgs == 1) - flags |= AXI_DMAC_FLAG_CYCLIC; + if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { + if (chan->hw_sg) + desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; + else if (desc->num_sgs == 1) + flags |= AXI_DMAC_FLAG_CYCLIC; + } if (chan->hw_partial_xfer) flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; @@ -411,7 +413,6 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, if (chan->hw_sg) { if (active->cyclic) { vchan_cyclic_callback(&active->vdesc); - start_next = true; } else { list_del(&active->vdesc.node); vchan_cookie_complete(&active->vdesc); @@ -667,7 +668,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( { struct axi_dmac_chan *chan = to_axi_dmac_chan(c); struct axi_dmac_desc *desc; - unsigned int num_periods, num_segments; + unsigned int num_periods, num_segments, num_sgs; if (direction != chan->direction) return NULL; @@ -681,11 +682,16 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( num_periods = buf_len / period_len; num_segments = DIV_ROUND_UP(period_len, chan->max_length); + num_sgs = num_periods * num_segments; - desc = axi_dmac_alloc_desc(chan, num_periods * num_segments); + desc = axi_dmac_alloc_desc(chan, num_sgs); if (!desc) return NULL; + /* Chain the last descriptor to the first, and remove its "last" flag */ + desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; + desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; + axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods, period_len, desc->sg); -- cgit From dc51b4442dd94ab12c146c1897bbdb40e16d5636 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 14 Nov 2023 10:48:21 -0500 Subject: dmaengine: fsl-edma: fix eDMAv4 channel allocation issue The eDMAv4 channel mux has a limitation where certain requests must use even channels, while others must use odd numbers. Add two flags (ARGS_EVEN_CH and ARGS_ODD_CH) to reflect this limitation. The device tree source (dts) files need to be updated accordingly. This issue was identified by the following commit: commit a725990557e7 ("arm64: dts: imx93: Fix the dmas entries order") Reverting channel orders triggered this problem. Fixes: 72f5801a4e2b ("dmaengine: fsl-edma: integrate v3 support") Signed-off-by: Frank Li Link: https://lore.kernel.org/r/20231114154824.3617255-2-Frank.Li@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-main.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 4635e16d7705..3ee08f390f81 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -24,6 +24,8 @@ #define ARGS_RX BIT(0) #define ARGS_REMOTE BIT(1) #define ARGS_MULTI_FIFO BIT(2) +#define ARGS_EVEN_CH BIT(3) +#define ARGS_ODD_CH BIT(4) static void fsl_edma_synchronize(struct dma_chan *chan) { @@ -157,6 +159,12 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, fsl_chan->is_remote = dma_spec->args[2] & ARGS_REMOTE; fsl_chan->is_multi_fifo = dma_spec->args[2] & ARGS_MULTI_FIFO; + if ((dma_spec->args[2] & ARGS_EVEN_CH) && (i & 0x1)) + continue; + + if ((dma_spec->args[2] & ARGS_ODD_CH) && !(i & 0x1)) + continue; + if (!b_chmux && i == dma_spec->args[0]) { chan = dma_get_slave_channel(chan); chan->device->privatecnt++; -- cgit From d0e217b72f9f5c5ef35e3423d393ea8093ce98ec Mon Sep 17 00:00:00 2001 From: Frank Li Date: Tue, 14 Nov 2023 10:48:23 -0500 Subject: dmaengine: fsl-edma: utilize common dt-binding header file Refactor the code to use the common dt-binding header file, fsl-edma.h. Renaming ARGS* to FSL_EDMA*, ensuring no functional changes. Signed-off-by: Frank Li Link: https://lore.kernel.org/r/20231114154824.3617255-4-Frank.Li@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-main.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 3ee08f390f81..f53b0ec17bcb 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -9,6 +9,7 @@ * Vybrid and Layerscape SoCs. */ +#include #include #include #include @@ -21,12 +22,6 @@ #include "fsl-edma-common.h" -#define ARGS_RX BIT(0) -#define ARGS_REMOTE BIT(1) -#define ARGS_MULTI_FIFO BIT(2) -#define ARGS_EVEN_CH BIT(3) -#define ARGS_ODD_CH BIT(4) - static void fsl_edma_synchronize(struct dma_chan *chan) { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); @@ -155,14 +150,14 @@ static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, i = fsl_chan - fsl_edma->chans; fsl_chan->priority = dma_spec->args[1]; - fsl_chan->is_rxchan = dma_spec->args[2] & ARGS_RX; - fsl_chan->is_remote = dma_spec->args[2] & ARGS_REMOTE; - fsl_chan->is_multi_fifo = dma_spec->args[2] & ARGS_MULTI_FIFO; + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; - if ((dma_spec->args[2] & ARGS_EVEN_CH) && (i & 0x1)) + if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) continue; - if ((dma_spec->args[2] & ARGS_ODD_CH) && !(i & 0x1)) + if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) continue; if (!b_chmux && i == dma_spec->args[0]) { -- cgit From f5c24d94512f1b288262beda4d3dcb9629222fc7 Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Wed, 13 Dec 2023 17:04:52 +0100 Subject: dmaengine: fix NULL pointer in channel unregistration function __dma_async_device_channel_register() can fail. In case of failure, chan->local is freed (with free_percpu()), and chan->local is nullified. When dma_async_device_unregister() is called (because of managed API or intentionally by DMA controller driver), channels are unconditionally unregistered, leading to this NULL pointer: [ 1.318693] Unable to handle kernel NULL pointer dereference at virtual address 00000000000000d0 [...] [ 1.484499] Call trace: [ 1.486930] device_del+0x40/0x394 [ 1.490314] device_unregister+0x20/0x7c [ 1.494220] __dma_async_device_channel_unregister+0x68/0xc0 Look at dma_async_device_register() function error path, channel device unregistration is done only if chan->local is not NULL. Then add the same condition at the beginning of __dma_async_device_channel_unregister() function, to avoid NULL pointer issue whatever the API used to reach this function. Fixes: d2fb0a043838 ("dmaengine: break out channel registration") Signed-off-by: Amelie Delaunay Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/20231213160452.2598073-1-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/dma/dmaengine.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index b7388ae62d7f..491b22240221 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -1103,6 +1103,9 @@ EXPORT_SYMBOL_GPL(dma_async_device_channel_register); static void __dma_async_device_channel_unregister(struct dma_device *device, struct dma_chan *chan) { + if (chan->local == NULL) + return; + WARN_ONCE(!device->device_release && chan->client_count, "%s called while %d clients hold a reference\n", __func__, chan->client_count); -- cgit From 3b08b3775593442be52cbb99efbccbd7fe4fa3fe Mon Sep 17 00:00:00 2001 From: Vignesh Raghavendra Date: Wed, 13 Dec 2023 13:43:18 +0530 Subject: dmaengine: ti: k3-udma: Add PSIL threads for AM62P and J722S Add PSIL thread information and enable UDMA support for AM62P and J722S SoC. J722S SoC family is a superset of AM62P, thus common PSIL thread ID map is reused for both devices. For those interested, more details about the SoC can be found in the Technical Reference Manual here: AM62P - https://www.ti.com/lit/pdf/spruj83 J722S - https://www.ti.com/lit/zip/sprujb3 Signed-off-by: Vignesh Raghavendra Signed-off-by: Bryan Brattlof Signed-off-by: Vaishnav Achath Reviewed-by: Jai Luthra Link: https://lore.kernel.org/r/20231213081318.26203-1-vaishnav.a@ti.com Signed-off-by: Vinod Koul --- drivers/dma/ti/Makefile | 3 +- drivers/dma/ti/k3-psil-am62p.c | 325 +++++++++++++++++++++++++++++++++++++++++ drivers/dma/ti/k3-psil-priv.h | 1 + drivers/dma/ti/k3-psil.c | 2 + drivers/dma/ti/k3-udma.c | 2 + 5 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 drivers/dma/ti/k3-psil-am62p.c (limited to 'drivers/dma') diff --git a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile index acc950bf609c..d376c117cecf 100644 --- a/drivers/dma/ti/Makefile +++ b/drivers/dma/ti/Makefile @@ -12,6 +12,7 @@ k3-psil-lib-objs := k3-psil.o \ k3-psil-j721s2.o \ k3-psil-am62.o \ k3-psil-am62a.o \ - k3-psil-j784s4.o + k3-psil-j784s4.o \ + k3-psil-am62p.o obj-$(CONFIG_TI_K3_PSIL) += k3-psil-lib.o obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o diff --git a/drivers/dma/ti/k3-psil-am62p.c b/drivers/dma/ti/k3-psil-am62p.c new file mode 100644 index 000000000000..0f338e16d971 --- /dev/null +++ b/drivers/dma/ti/k3-psil-am62p.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Texas Instruments Incorporated - https://www.ti.com + */ + +#include + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_TR(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + }, \ + } + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = flow_base, \ + }, \ + } + +#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = default_flow, \ + .notdpkt = tx, \ + }, \ + } + +#define PSIL_PDMA_MCASP(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pdma_acc32 = 1, \ + .pdma_burst = 1, \ + }, \ + } + +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep am62p_src_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0x7504, 20, 35, 8, 35, 0), + PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), + PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), + PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), + PSIL_PDMA_XY_PKT(0x4302), + PSIL_PDMA_XY_PKT(0x4303), + PSIL_PDMA_XY_PKT(0x4304), + PSIL_PDMA_XY_PKT(0x4305), + PSIL_PDMA_XY_PKT(0x4306), + PSIL_PDMA_XY_PKT(0x4307), + PSIL_PDMA_XY_PKT(0x4308), + PSIL_PDMA_XY_PKT(0x4309), + PSIL_PDMA_XY_PKT(0x430a), + PSIL_PDMA_XY_PKT(0x430b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0x4400), + PSIL_PDMA_XY_PKT(0x4401), + PSIL_PDMA_XY_PKT(0x4402), + PSIL_PDMA_XY_PKT(0x4403), + PSIL_PDMA_XY_PKT(0x4404), + PSIL_PDMA_XY_PKT(0x4405), + PSIL_PDMA_XY_PKT(0x4406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0x4500), + PSIL_PDMA_MCASP(0x4501), + PSIL_PDMA_MCASP(0x4502), + /* CPSW3G */ + PSIL_ETHERNET(0x4600, 19, 19, 16), + /* CSI2RX */ + PSIL_CSI2RX(0x5000), + PSIL_CSI2RX(0x5001), + PSIL_CSI2RX(0x5002), + PSIL_CSI2RX(0x5003), + PSIL_CSI2RX(0x5004), + PSIL_CSI2RX(0x5005), + PSIL_CSI2RX(0x5006), + PSIL_CSI2RX(0x5007), + PSIL_CSI2RX(0x5008), + PSIL_CSI2RX(0x5009), + PSIL_CSI2RX(0x500a), + PSIL_CSI2RX(0x500b), + PSIL_CSI2RX(0x500c), + PSIL_CSI2RX(0x500d), + PSIL_CSI2RX(0x500e), + PSIL_CSI2RX(0x500f), + PSIL_CSI2RX(0x5010), + PSIL_CSI2RX(0x5011), + PSIL_CSI2RX(0x5012), + PSIL_CSI2RX(0x5013), + PSIL_CSI2RX(0x5014), + PSIL_CSI2RX(0x5015), + PSIL_CSI2RX(0x5016), + PSIL_CSI2RX(0x5017), + PSIL_CSI2RX(0x5018), + PSIL_CSI2RX(0x5019), + PSIL_CSI2RX(0x501a), + PSIL_CSI2RX(0x501b), + PSIL_CSI2RX(0x501c), + PSIL_CSI2RX(0x501d), + PSIL_CSI2RX(0x501e), + PSIL_CSI2RX(0x501f), + PSIL_CSI2RX(0x5000), + PSIL_CSI2RX(0x5001), + PSIL_CSI2RX(0x5002), + PSIL_CSI2RX(0x5003), + PSIL_CSI2RX(0x5004), + PSIL_CSI2RX(0x5005), + PSIL_CSI2RX(0x5006), + PSIL_CSI2RX(0x5007), + PSIL_CSI2RX(0x5008), + PSIL_CSI2RX(0x5009), + PSIL_CSI2RX(0x500a), + PSIL_CSI2RX(0x500b), + PSIL_CSI2RX(0x500c), + PSIL_CSI2RX(0x500d), + PSIL_CSI2RX(0x500e), + PSIL_CSI2RX(0x500f), + PSIL_CSI2RX(0x5010), + PSIL_CSI2RX(0x5011), + PSIL_CSI2RX(0x5012), + PSIL_CSI2RX(0x5013), + PSIL_CSI2RX(0x5014), + PSIL_CSI2RX(0x5015), + PSIL_CSI2RX(0x5016), + PSIL_CSI2RX(0x5017), + PSIL_CSI2RX(0x5018), + PSIL_CSI2RX(0x5019), + PSIL_CSI2RX(0x501a), + PSIL_CSI2RX(0x501b), + PSIL_CSI2RX(0x501c), + PSIL_CSI2RX(0x501d), + PSIL_CSI2RX(0x501e), + PSIL_CSI2RX(0x501f), + /* CSIRX 1-3 (only for J722S) */ + PSIL_CSI2RX(0x5100), + PSIL_CSI2RX(0x5101), + PSIL_CSI2RX(0x5102), + PSIL_CSI2RX(0x5103), + PSIL_CSI2RX(0x5104), + PSIL_CSI2RX(0x5105), + PSIL_CSI2RX(0x5106), + PSIL_CSI2RX(0x5107), + PSIL_CSI2RX(0x5108), + PSIL_CSI2RX(0x5109), + PSIL_CSI2RX(0x510a), + PSIL_CSI2RX(0x510b), + PSIL_CSI2RX(0x510c), + PSIL_CSI2RX(0x510d), + PSIL_CSI2RX(0x510e), + PSIL_CSI2RX(0x510f), + PSIL_CSI2RX(0x5110), + PSIL_CSI2RX(0x5111), + PSIL_CSI2RX(0x5112), + PSIL_CSI2RX(0x5113), + PSIL_CSI2RX(0x5114), + PSIL_CSI2RX(0x5115), + PSIL_CSI2RX(0x5116), + PSIL_CSI2RX(0x5117), + PSIL_CSI2RX(0x5118), + PSIL_CSI2RX(0x5119), + PSIL_CSI2RX(0x511a), + PSIL_CSI2RX(0x511b), + PSIL_CSI2RX(0x511c), + PSIL_CSI2RX(0x511d), + PSIL_CSI2RX(0x511e), + PSIL_CSI2RX(0x511f), + PSIL_CSI2RX(0x5200), + PSIL_CSI2RX(0x5201), + PSIL_CSI2RX(0x5202), + PSIL_CSI2RX(0x5203), + PSIL_CSI2RX(0x5204), + PSIL_CSI2RX(0x5205), + PSIL_CSI2RX(0x5206), + PSIL_CSI2RX(0x5207), + PSIL_CSI2RX(0x5208), + PSIL_CSI2RX(0x5209), + PSIL_CSI2RX(0x520a), + PSIL_CSI2RX(0x520b), + PSIL_CSI2RX(0x520c), + PSIL_CSI2RX(0x520d), + PSIL_CSI2RX(0x520e), + PSIL_CSI2RX(0x520f), + PSIL_CSI2RX(0x5210), + PSIL_CSI2RX(0x5211), + PSIL_CSI2RX(0x5212), + PSIL_CSI2RX(0x5213), + PSIL_CSI2RX(0x5214), + PSIL_CSI2RX(0x5215), + PSIL_CSI2RX(0x5216), + PSIL_CSI2RX(0x5217), + PSIL_CSI2RX(0x5218), + PSIL_CSI2RX(0x5219), + PSIL_CSI2RX(0x521a), + PSIL_CSI2RX(0x521b), + PSIL_CSI2RX(0x521c), + PSIL_CSI2RX(0x521d), + PSIL_CSI2RX(0x521e), + PSIL_CSI2RX(0x521f), + PSIL_CSI2RX(0x5300), + PSIL_CSI2RX(0x5301), + PSIL_CSI2RX(0x5302), + PSIL_CSI2RX(0x5303), + PSIL_CSI2RX(0x5304), + PSIL_CSI2RX(0x5305), + PSIL_CSI2RX(0x5306), + PSIL_CSI2RX(0x5307), + PSIL_CSI2RX(0x5308), + PSIL_CSI2RX(0x5309), + PSIL_CSI2RX(0x530a), + PSIL_CSI2RX(0x530b), + PSIL_CSI2RX(0x530c), + PSIL_CSI2RX(0x530d), + PSIL_CSI2RX(0x530e), + PSIL_CSI2RX(0x530f), + PSIL_CSI2RX(0x5310), + PSIL_CSI2RX(0x5311), + PSIL_CSI2RX(0x5312), + PSIL_CSI2RX(0x5313), + PSIL_CSI2RX(0x5314), + PSIL_CSI2RX(0x5315), + PSIL_CSI2RX(0x5316), + PSIL_CSI2RX(0x5317), + PSIL_CSI2RX(0x5318), + PSIL_CSI2RX(0x5319), + PSIL_CSI2RX(0x531a), + PSIL_CSI2RX(0x531b), + PSIL_CSI2RX(0x531c), + PSIL_CSI2RX(0x531d), + PSIL_CSI2RX(0x531e), + PSIL_CSI2RX(0x531f), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep am62p_dst_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), + PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0xc300), + PSIL_PDMA_XY_PKT(0xc301), + PSIL_PDMA_XY_PKT(0xc302), + PSIL_PDMA_XY_PKT(0xc303), + PSIL_PDMA_XY_PKT(0xc304), + PSIL_PDMA_XY_PKT(0xc305), + PSIL_PDMA_XY_PKT(0xc306), + PSIL_PDMA_XY_PKT(0xc307), + PSIL_PDMA_XY_PKT(0xc308), + PSIL_PDMA_XY_PKT(0xc309), + PSIL_PDMA_XY_PKT(0xc30a), + PSIL_PDMA_XY_PKT(0xc30b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0xc400), + PSIL_PDMA_XY_PKT(0xc401), + PSIL_PDMA_XY_PKT(0xc402), + PSIL_PDMA_XY_PKT(0xc403), + PSIL_PDMA_XY_PKT(0xc404), + PSIL_PDMA_XY_PKT(0xc405), + PSIL_PDMA_XY_PKT(0xc406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0xc500), + PSIL_PDMA_MCASP(0xc501), + PSIL_PDMA_MCASP(0xc502), + /* CPSW3G */ + PSIL_ETHERNET(0xc600, 19, 19, 8), + PSIL_ETHERNET(0xc601, 20, 27, 8), + PSIL_ETHERNET(0xc602, 21, 35, 8), + PSIL_ETHERNET(0xc603, 22, 43, 8), + PSIL_ETHERNET(0xc604, 23, 51, 8), + PSIL_ETHERNET(0xc605, 24, 59, 8), + PSIL_ETHERNET(0xc606, 25, 67, 8), + PSIL_ETHERNET(0xc607, 26, 75, 8), +}; + +struct psil_ep_map am62p_ep_map = { + .name = "am62p", + .src = am62p_src_ep_map, + .src_count = ARRAY_SIZE(am62p_src_ep_map), + .dst = am62p_dst_ep_map, + .dst_count = ARRAY_SIZE(am62p_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-priv.h b/drivers/dma/ti/k3-psil-priv.h index c383723d1c8f..a577be97e344 100644 --- a/drivers/dma/ti/k3-psil-priv.h +++ b/drivers/dma/ti/k3-psil-priv.h @@ -45,5 +45,6 @@ extern struct psil_ep_map j721s2_ep_map; extern struct psil_ep_map am62_ep_map; extern struct psil_ep_map am62a_ep_map; extern struct psil_ep_map j784s4_ep_map; +extern struct psil_ep_map am62p_ep_map; #endif /* K3_PSIL_PRIV_H_ */ diff --git a/drivers/dma/ti/k3-psil.c b/drivers/dma/ti/k3-psil.c index c11389d67a3f..25148d952472 100644 --- a/drivers/dma/ti/k3-psil.c +++ b/drivers/dma/ti/k3-psil.c @@ -26,6 +26,8 @@ static const struct soc_device_attribute k3_soc_devices[] = { { .family = "AM62X", .data = &am62_ep_map }, { .family = "AM62AX", .data = &am62a_ep_map }, { .family = "J784S4", .data = &j784s4_ep_map }, + { .family = "AM62PX", .data = &am62p_ep_map }, + { .family = "J722S", .data = &am62p_ep_map }, { /* sentinel */ } }; diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 30fd2f386f36..2841a539c264 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -4441,6 +4441,8 @@ static const struct soc_device_attribute k3_soc_devices[] = { { .family = "AM62X", .data = &am64_soc_data }, { .family = "AM62AX", .data = &am64_soc_data }, { .family = "J784S4", .data = &j721e_soc_data }, + { .family = "AM62PX", .data = &am64_soc_data }, + { .family = "J722S", .data = &am64_soc_data }, { /* sentinel */ } }; -- cgit From e271c0ba3f919c48e90c64b703538fbb7865cb63 Mon Sep 17 00:00:00 2001 From: Rex Zhang Date: Tue, 12 Dec 2023 10:21:58 +0800 Subject: dmaengine: idxd: Move dma_free_coherent() out of spinlocked context Task may be rescheduled within dma_free_coherent(). So dma_free_coherent() can't be called between spin_lock() and spin_unlock() to avoid Call Trace: Call Trace: dump_stack_lvl+0x37/0x50 __might_resched+0x16a/0x1c0 vunmap+0x2c/0x70 __iommu_dma_free+0x96/0x100 idxd_device_evl_free+0xd5/0x100 [idxd] device_release_driver_internal+0x197/0x200 unbind_store+0xa1/0xb0 kernfs_fop_write_iter+0x120/0x1c0 vfs_write+0x2d3/0x400 ksys_write+0x63/0xe0 do_syscall_64+0x44/0xa0 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 Move it out of the context. Fixes: 244da66cda35 ("dmaengine: idxd: setup event log configuration") Signed-off-by: Rex Zhang Reviewed-by: Dave Jiang Reviewed-by: Fenghua Yu Link: https://lore.kernel.org/r/20231212022158.358619-2-rex.zhang@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/device.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 8f754f922217..fa0f880beae6 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -802,6 +802,9 @@ err_bmap: static void idxd_device_evl_free(struct idxd_device *idxd) { + void *evl_log; + unsigned int evl_log_size; + dma_addr_t evl_dma; union gencfg_reg gencfg; union genctrl_reg genctrl; struct device *dev = &idxd->pdev->dev; @@ -822,11 +825,15 @@ static void idxd_device_evl_free(struct idxd_device *idxd) iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET); iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET + 8); - dma_free_coherent(dev, evl->log_size, evl->log, evl->dma); bitmap_free(evl->bmap); + evl_log = evl->log; + evl_log_size = evl->log_size; + evl_dma = evl->dma; evl->log = NULL; evl->size = IDXD_EVL_SIZE_MIN; spin_unlock(&evl->lock); + + dma_free_coherent(dev, evl_log_size, evl_log, evl_dma); } static void idxd_group_config_write(struct idxd_group *group) -- cgit From 26ee018ff6d1c326ac9b9be36513e35870ed09db Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Thu, 30 Nov 2023 12:13:12 +0100 Subject: dmaengine: xilinx: xdma: Fix the count of elapsed periods in cyclic mode Xilinx DMA engine is capable of keeping track of the number of elapsed periods and this is an increasing 32-bit counter which is only reset when turning off the engine. No need to add this value to our local counter. Fixes: cd8c732ce1a5 ("dmaengine: xilinx: xdma: Support cyclic transfers") Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/r/20231130111315.729430-2-miquel.raynal@bootlin.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 84a88029226f..2c9c72d4b5a2 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -754,9 +754,9 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) if (ret) goto out; - desc->completed_desc_num += complete_desc_num; - if (desc->cyclic) { + desc->completed_desc_num = complete_desc_num; + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS, &st); if (ret) @@ -768,6 +768,8 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) goto out; } + desc->completed_desc_num += complete_desc_num; + /* * if all data blocks are transferred, remove and complete the request */ -- cgit From 58b61fc75ba901b1fd63c911b31249f36d17e9c4 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Thu, 30 Nov 2023 12:13:13 +0100 Subject: dmaengine: xilinx: xdma: Clarify the logic between cyclic/sg modes We support both modes, but they perform totally different taks in the interrupt handler. Clarify what shall be done in each case. Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/r/20231130111315.729430-3-miquel.raynal@bootlin.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 2c9c72d4b5a2..4efef1b5f89c 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -765,26 +765,23 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_STATUS, st); vchan_cyclic_callback(vd); - goto out; - } - - desc->completed_desc_num += complete_desc_num; + } else { + desc->completed_desc_num += complete_desc_num; - /* - * if all data blocks are transferred, remove and complete the request - */ - if (desc->completed_desc_num == desc->desc_num) { - list_del(&vd->node); - vchan_cookie_complete(vd); - goto out; - } + /* if all data blocks are transferred, remove and complete the request */ + if (desc->completed_desc_num == desc->desc_num) { + list_del(&vd->node); + vchan_cookie_complete(vd); + goto out; + } - if (desc->completed_desc_num > desc->desc_num || - complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) - goto out; + if (desc->completed_desc_num > desc->desc_num || + complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) + goto out; - /* transfer the rest of data (SG only) */ - xdma_xfer_start(xchan); + /* transfer the rest of data */ + xdma_xfer_start(xchan); + } out: spin_unlock(&xchan->vchan.lock); -- cgit From b3072be7f955e56789a0508c18e9870f45cd9a11 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Thu, 30 Nov 2023 12:13:14 +0100 Subject: dmaengine: xilinx: xdma: Better handling of the busy variable The driver internal scatter-gather logic is: * set busy to true * start transfer * set busy to false * trigger next transfer if any * set busy to true Setting busy to false in cyclic transfers does not make any sense and is conceptually wrong. In order to ease the integration of additional callbacks let's move this change to the scatter-gather path. Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/r/20231130111315.729430-4-miquel.raynal@bootlin.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 4efef1b5f89c..e931ff42209c 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -745,7 +745,6 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) if (!vd) goto out; - xchan->busy = false; desc = to_xdma_desc(vd); xdev = xchan->xdev_hdl; @@ -766,6 +765,7 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) vchan_cyclic_callback(vd); } else { + xchan->busy = false; desc->completed_desc_num += complete_desc_num; /* if all data blocks are transferred, remove and complete the request */ -- cgit From f5c392d106e7cc58c7705799ef4c36c3b2f60b31 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Thu, 30 Nov 2023 12:13:15 +0100 Subject: dmaengine: xilinx: xdma: Add terminate_all/synchronize callbacks The driver is capable of starting scatter-gather transfers and needs to wait until their end. It is also capable of starting cyclic transfers and will only be "reset" next time the channel will be reused. In practice most of the time we hear no audio glitch because the sound card stops the flow on its side so the DMA transfers are just discarded. There are however some cases (when playing a bit with a number of frames and with a discontinuous sound file) when the sound card seems to be slightly too slow at stopping the flow, leading to a glitch that can be heard. In all cases, we need to earn better control of the DMA engine and adding proper ->device_terminate_all() and ->device_synchronize() callbacks feels totally relevant. With these two callbacks, no glitch can be heard anymore. Fixes: cd8c732ce1a5 ("dmaengine: xilinx: xdma: Support cyclic transfers") Signed-off-by: Miquel Raynal Tested-by: Lizhi Hou Link: https://lore.kernel.org/r/20231130111315.729430-5-miquel.raynal@bootlin.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index e931ff42209c..290bb5d2d1e2 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -371,6 +371,31 @@ static int xdma_xfer_start(struct xdma_chan *xchan) return ret; xchan->busy = true; + + return 0; +} + +/** + * xdma_xfer_stop - Stop DMA transfer + * @xchan: DMA channel pointer + */ +static int xdma_xfer_stop(struct xdma_chan *xchan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&xchan->vchan); + struct xdma_device *xdev = xchan->xdev_hdl; + int ret; + + if (!vd || !xchan->busy) + return -EINVAL; + + /* clear run stop bit to prevent any further auto-triggering */ + ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, + CHAN_CTRL_RUN_STOP); + if (ret) + return ret; + + xchan->busy = false; + return 0; } @@ -475,6 +500,47 @@ static void xdma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); } +/** + * xdma_terminate_all - Terminate all transactions + * @chan: DMA channel pointer + */ +static int xdma_terminate_all(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_desc *desc = NULL; + struct virt_dma_desc *vd; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); + xdma_xfer_stop(xdma_chan); + + vd = vchan_next_desc(&xdma_chan->vchan); + if (vd) + desc = to_xdma_desc(vd); + if (desc) { + dma_cookie_complete(&desc->vdesc.tx); + vchan_terminate_vdesc(&desc->vdesc); + } + + vchan_get_all_descriptors(&xdma_chan->vchan, &head); + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); + vchan_dma_desc_free_list(&xdma_chan->vchan, &head); + + return 0; +} + +/** + * xdma_synchronize - Synchronize terminated transactions + * @chan: DMA channel pointer + */ +static void xdma_synchronize(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + + vchan_synchronize(&xdma_chan->vchan); +} + /** * xdma_prep_device_sg - prepare a descriptor for a DMA transaction * @chan: DMA channel pointer @@ -1088,6 +1154,8 @@ static int xdma_probe(struct platform_device *pdev) xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg; xdev->dma_dev.device_config = xdma_device_config; xdev->dma_dev.device_issue_pending = xdma_issue_pending; + xdev->dma_dev.device_terminate_all = xdma_terminate_all; + xdev->dma_dev.device_synchronize = xdma_synchronize; xdev->dma_dev.filter.map = pdata->device_map; xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt; xdev->dma_dev.filter.fn = xdma_filter_fn; -- cgit From 6e2387183312cdfce6326b2626c0b801c2ffe686 Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:36 +0100 Subject: dmaengine: xilinx: xdma: Get rid of unused code Get rid of duplicated macro definitions, as these macros are defined earlier in the file. Also, get rid of unused member of 'struct xdma_desc'. Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-2-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma-regs.h | 12 ------------ drivers/dma/xilinx/xdma.c | 2 -- 2 files changed, 14 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h index e641a5083e14..0b17a931f583 100644 --- a/drivers/dma/xilinx/xdma-regs.h +++ b/drivers/dma/xilinx/xdma-regs.h @@ -134,18 +134,6 @@ struct xdma_hw_desc { #define XDMA_SGDMA_DESC_ADJ 0x4088 #define XDMA_SGDMA_DESC_CREDIT 0x408c -/* bits of the SG DMA control register */ -#define XDMA_CTRL_RUN_STOP BIT(0) -#define XDMA_CTRL_IE_DESC_STOPPED BIT(1) -#define XDMA_CTRL_IE_DESC_COMPLETED BIT(2) -#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH BIT(3) -#define XDMA_CTRL_IE_MAGIC_STOPPED BIT(4) -#define XDMA_CTRL_IE_IDLE_STOPPED BIT(6) -#define XDMA_CTRL_IE_READ_ERROR GENMASK(13, 9) -#define XDMA_CTRL_IE_DESC_ERROR GENMASK(23, 19) -#define XDMA_CTRL_NON_INCR_ADDR BIT(25) -#define XDMA_CTRL_POLL_MODE_WB BIT(26) - /* * interrupt registers */ diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 290bb5d2d1e2..ddb9e7d07461 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -78,7 +78,6 @@ struct xdma_chan { * @vdesc: Virtual DMA descriptor * @chan: DMA channel pointer * @dir: Transferring direction of the request - * @dev_addr: Physical address on DMA device side * @desc_blocks: Hardware descriptor blocks * @dblk_num: Number of hardware descriptor blocks * @desc_num: Number of hardware descriptors @@ -91,7 +90,6 @@ struct xdma_desc { struct virt_dma_desc vdesc; struct xdma_chan *chan; enum dma_transfer_direction dir; - u64 dev_addr; struct xdma_desc_block *desc_blocks; u32 dblk_num; u32 desc_num; -- cgit From 7a9c7f46bd0abea214d96f00f78622f24c798ad8 Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:37 +0100 Subject: dmaengine: xilinx: xdma: Add necessary macro definitions Complete lacking bits describing the status/control register values. Add macros describing the status/control registers. Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-3-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma-regs.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h index 0b17a931f583..98117e8a466f 100644 --- a/drivers/dma/xilinx/xdma-regs.h +++ b/drivers/dma/xilinx/xdma-regs.h @@ -76,6 +76,7 @@ struct xdma_hw_desc { #define XDMA_CHAN_CONTROL_W1S 0x8 #define XDMA_CHAN_CONTROL_W1C 0xc #define XDMA_CHAN_STATUS 0x40 +#define XDMA_CHAN_STATUS_RC 0x44 #define XDMA_CHAN_COMPLETED_DESC 0x48 #define XDMA_CHAN_ALIGNMENTS 0x4c #define XDMA_CHAN_INTR_ENABLE 0x90 @@ -101,6 +102,7 @@ struct xdma_hw_desc { #define CHAN_CTRL_IE_MAGIC_STOPPED BIT(4) #define CHAN_CTRL_IE_IDLE_STOPPED BIT(6) #define CHAN_CTRL_IE_READ_ERROR GENMASK(13, 9) +#define CHAN_CTRL_IE_WRITE_ERROR GENMASK(18, 14) #define CHAN_CTRL_IE_DESC_ERROR GENMASK(23, 19) #define CHAN_CTRL_NON_INCR_ADDR BIT(25) #define CHAN_CTRL_POLL_MODE_WB BIT(26) @@ -111,8 +113,17 @@ struct xdma_hw_desc { CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ CHAN_CTRL_IE_MAGIC_STOPPED | \ CHAN_CTRL_IE_READ_ERROR | \ + CHAN_CTRL_IE_WRITE_ERROR | \ CHAN_CTRL_IE_DESC_ERROR) +#define XDMA_CHAN_STATUS_MASK CHAN_CTRL_START + +#define XDMA_CHAN_ERROR_MASK (CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ + CHAN_CTRL_IE_MAGIC_STOPPED | \ + CHAN_CTRL_IE_READ_ERROR | \ + CHAN_CTRL_IE_WRITE_ERROR | \ + CHAN_CTRL_IE_DESC_ERROR) + /* bits of the channel interrupt enable mask */ #define CHAN_IM_DESC_ERROR BIT(19) #define CHAN_IM_READ_ERROR BIT(9) -- cgit From e5bc76b0e1c54906ca744ed1a7872f4f407d5d2e Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:38 +0100 Subject: dmaengine: xilinx: xdma: Ease dma_pool alignment requirements According to the XDMA datasheet (PG195), the address of any descriptor must be 32 byte aligned. The datasheet also states that a contiguous block of descriptors must not cross a 4k address boundary. Therefore, it is possible to ease the pressure put on the dma_pool allocator just by requiring sufficient alignment and boundary values. Add proper macro definition and change the values passed into the dma_pool_create(). Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-4-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma-regs.h | 7 ++++--- drivers/dma/xilinx/xdma.c | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h index 98117e8a466f..98f5f6fb9ff9 100644 --- a/drivers/dma/xilinx/xdma-regs.h +++ b/drivers/dma/xilinx/xdma-regs.h @@ -64,9 +64,10 @@ struct xdma_hw_desc { __le64 next_desc; }; -#define XDMA_DESC_SIZE sizeof(struct xdma_hw_desc) -#define XDMA_DESC_BLOCK_SIZE (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT) -#define XDMA_DESC_BLOCK_ALIGN 4096 +#define XDMA_DESC_SIZE sizeof(struct xdma_hw_desc) +#define XDMA_DESC_BLOCK_SIZE (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT) +#define XDMA_DESC_BLOCK_ALIGN 32 +#define XDMA_DESC_BLOCK_BOUNDARY 4096 /* * Channel registers diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index ddb9e7d07461..c22701e76b69 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -741,9 +741,8 @@ static int xdma_alloc_chan_resources(struct dma_chan *chan) return -EINVAL; } - xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), - dev, XDMA_DESC_BLOCK_SIZE, - XDMA_DESC_BLOCK_ALIGN, 0); + xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), dev, XDMA_DESC_BLOCK_SIZE, + XDMA_DESC_BLOCK_ALIGN, XDMA_DESC_BLOCK_BOUNDARY); if (!xdma_chan->desc_pool) { xdma_err(xdev, "unable to allocate descriptor pool"); return -ENOMEM; -- cgit From 855c2e1d1842f0101a051378094548ca581d7a7d Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:39 +0100 Subject: dmaengine: xilinx: xdma: Rework xdma_terminate_all() Simplify xdma_xfer_stop(). Stop the dma engine and clear its status register unconditionally - just do what its name states. This change also allows to call it without grabbing a lock, which minimizes the total time spent with a spinlock held. Delete the currently processed vd.node from the vc.desc_issued list prior to passing it to vchan_terminate_vdesc(). In case there's more than one descriptor pending on vc.desc_issued list, calling vchan_terminate_desc() results in losing the link between vc.desc_issued list head and the second descriptor on the list. Doing so results in resources leakege, as vchan_dma_desc_free_list() won't be able to properly free memory resources attached to descriptors, resulting in dma_pool_destroy() failure. Don't call vchan_dma_desc_free_list() from within xdma_terminate_all(). Move all terminated descriptors to the vc.desc_terminated list instead. This allows to postpone freeing memory resources associated with descriptors until the call to vchan_synchronize(), which is called from xdma_synchronize() callback. This is the right way to do it - xdma_terminate_all() should return as soon as possible, while freeing resources (that may be time consuming in case of large number of descriptors) can be done safely later. Fixes: f5c392d106e7 ("dmaengine: xilinx: xdma: Add terminate_all/synchronize callbacks") Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-5-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index c22701e76b69..0c7350863873 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -379,12 +379,9 @@ static int xdma_xfer_start(struct xdma_chan *xchan) */ static int xdma_xfer_stop(struct xdma_chan *xchan) { - struct virt_dma_desc *vd = vchan_next_desc(&xchan->vchan); - struct xdma_device *xdev = xchan->xdev_hdl; int ret; - - if (!vd || !xchan->busy) - return -EINVAL; + u32 val; + struct xdma_device *xdev = xchan->xdev_hdl; /* clear run stop bit to prevent any further auto-triggering */ ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, @@ -392,7 +389,10 @@ static int xdma_xfer_stop(struct xdma_chan *xchan) if (ret) return ret; - xchan->busy = false; + /* Clear the channel status register */ + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &val); + if (ret) + return ret; return 0; } @@ -505,25 +505,25 @@ static void xdma_issue_pending(struct dma_chan *chan) static int xdma_terminate_all(struct dma_chan *chan) { struct xdma_chan *xdma_chan = to_xdma_chan(chan); - struct xdma_desc *desc = NULL; struct virt_dma_desc *vd; unsigned long flags; LIST_HEAD(head); - spin_lock_irqsave(&xdma_chan->vchan.lock, flags); xdma_xfer_stop(xdma_chan); + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); + + xdma_chan->busy = false; vd = vchan_next_desc(&xdma_chan->vchan); - if (vd) - desc = to_xdma_desc(vd); - if (desc) { - dma_cookie_complete(&desc->vdesc.tx); - vchan_terminate_vdesc(&desc->vdesc); + if (vd) { + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); } - vchan_get_all_descriptors(&xdma_chan->vchan, &head); + list_splice_tail(&head, &xdma_chan->vchan.desc_terminated); + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); - vchan_dma_desc_free_list(&xdma_chan->vchan, &head); return 0; } -- cgit From d0f22a3f55044f91b98e5536aa2c4d51d41cf8e7 Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:40 +0100 Subject: dmaengine: xilinx: xdma: Add error checking in xdma_channel_isr() Check and clear the status register value before proceeding any further in xdma_channel_isr(). It is necessary to do it since the interrupt may occur on any error condition enabled at the start of a transfer. Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-6-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 0c7350863873..9a1d2939a333 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -811,6 +811,18 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) desc = to_xdma_desc(vd); xdev = xchan->xdev_hdl; + /* Clear-on-read the status register */ + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &st); + if (ret) + goto out; + + st &= XDMA_CHAN_STATUS_MASK; + if ((st & XDMA_CHAN_ERROR_MASK) || + !(st & (CHAN_CTRL_IE_DESC_COMPLETED | CHAN_CTRL_IE_DESC_STOPPED))) { + xdma_err(xdev, "channel error, status register value: 0x%x", st); + goto out; + } + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_COMPLETED_DESC, &complete_desc_num); if (ret) @@ -818,14 +830,6 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) if (desc->cyclic) { desc->completed_desc_num = complete_desc_num; - - ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS, - &st); - if (ret) - goto out; - - regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_STATUS, st); - vchan_cyclic_callback(vd); } else { xchan->busy = false; -- cgit From fd0e1d83a813d48285d39eae0053b38828cf7e1a Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:41 +0100 Subject: dmaengine: xilinx: xdma: Add transfer error reporting Extend the capability of transfer status reporting. Introduce error flag, which allows to report error in case of a interrupt-reported error condition. Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-7-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 9a1d2939a333..9f8597ed9be2 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -85,6 +85,7 @@ struct xdma_chan { * @cyclic: Cyclic transfer vs. scatter-gather * @periods: Number of periods in the cyclic transfer * @period_size: Size of a period in bytes in cyclic transfers + * @error: tx error flag */ struct xdma_desc { struct virt_dma_desc vdesc; @@ -97,6 +98,7 @@ struct xdma_desc { bool cyclic; u32 periods; u32 period_size; + bool error; }; #define XDMA_DEV_STATUS_REG_DMA BIT(0) @@ -274,6 +276,7 @@ xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num, bool cyclic) sw_desc->chan = chan; sw_desc->desc_num = desc_num; sw_desc->cyclic = cyclic; + sw_desc->error = false; dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT); sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks), GFP_NOWAIT); @@ -769,20 +772,20 @@ static enum dma_status xdma_tx_status(struct dma_chan *chan, dma_cookie_t cookie spin_lock_irqsave(&xdma_chan->vchan.lock, flags); vd = vchan_find_desc(&xdma_chan->vchan, cookie); - if (vd) - desc = to_xdma_desc(vd); - if (!desc || !desc->cyclic) { - spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); - return ret; - } - - period_idx = desc->completed_desc_num % desc->periods; - residue = (desc->periods - period_idx) * desc->period_size; + if (!vd) + goto out; + desc = to_xdma_desc(vd); + if (desc->error) { + ret = DMA_ERROR; + } else if (desc->cyclic) { + period_idx = desc->completed_desc_num % desc->periods; + residue = (desc->periods - period_idx) * desc->period_size; + dma_set_residue(state, residue); + } +out: spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); - dma_set_residue(state, residue); - return ret; } @@ -819,6 +822,7 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) st &= XDMA_CHAN_STATUS_MASK; if ((st & XDMA_CHAN_ERROR_MASK) || !(st & (CHAN_CTRL_IE_DESC_COMPLETED | CHAN_CTRL_IE_DESC_STOPPED))) { + desc->error = true; xdma_err(xdev, "channel error, status register value: 0x%x", st); goto out; } -- cgit From 3e184e64c2e5145e51dc57872b0a64e3a6736888 Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:42 +0100 Subject: dmaengine: xilinx: xdma: Prepare the introduction of interleaved DMA transfers Make generic code generic. As descriptor-filling logic stays the same regardless of a dmaengine's type of transfer, it is possible to write the descriptor-filling function in a generic way, so that it can be used for every single type of transfer preparation callback. Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-8-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 101 ++++++++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 44 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 9f8597ed9be2..618cc9af6eb9 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -542,6 +542,43 @@ static void xdma_synchronize(struct dma_chan *chan) vchan_synchronize(&xdma_chan->vchan); } +/** + * xdma_fill_descs - Fill hardware descriptors with contiguous memory block addresses + * @sw_desc - tx descriptor state container + * @src_addr - Value for a ->src_addr field of a first descriptor + * @dst_addr - Value for a ->dst_addr field of a first descriptor + * @size - Total size of a contiguous memory block + * @filled_descs_num - Number of filled hardware descriptors for corresponding sw_desc + */ +static inline u32 xdma_fill_descs(struct xdma_desc *sw_desc, u64 src_addr, + u64 dst_addr, u32 size, u32 filled_descs_num) +{ + u32 left = size, len, desc_num = filled_descs_num; + struct xdma_desc_block *dblk; + struct xdma_hw_desc *desc; + + dblk = sw_desc->desc_blocks + (desc_num / XDMA_DESC_ADJACENT); + desc = dblk->virt_addr; + desc += desc_num & XDMA_DESC_ADJACENT_MASK; + do { + len = min_t(u32, left, XDMA_DESC_BLEN_MAX); + /* set hardware descriptor */ + desc->bytes = cpu_to_le32(len); + desc->src_addr = cpu_to_le64(src_addr); + desc->dst_addr = cpu_to_le64(dst_addr); + if (!(++desc_num & XDMA_DESC_ADJACENT_MASK)) + desc = (++dblk)->virt_addr; + else + desc++; + + src_addr += len; + dst_addr += len; + left -= len; + } while (left); + + return desc_num - filled_descs_num; +} + /** * xdma_prep_device_sg - prepare a descriptor for a DMA transaction * @chan: DMA channel pointer @@ -558,13 +595,10 @@ xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, { struct xdma_chan *xdma_chan = to_xdma_chan(chan); struct dma_async_tx_descriptor *tx_desc; - u32 desc_num = 0, i, len, rest; - struct xdma_desc_block *dblk; - struct xdma_hw_desc *desc; struct xdma_desc *sw_desc; - u64 dev_addr, *src, *dst; + u32 desc_num = 0, i; + u64 addr, dev_addr, *src, *dst; struct scatterlist *sg; - u64 addr; for_each_sg(sgl, sg, sg_len, i) desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX); @@ -584,32 +618,11 @@ xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, dst = &addr; } - dblk = sw_desc->desc_blocks; - desc = dblk->virt_addr; - desc_num = 1; + desc_num = 0; for_each_sg(sgl, sg, sg_len, i) { addr = sg_dma_address(sg); - rest = sg_dma_len(sg); - - do { - len = min_t(u32, rest, XDMA_DESC_BLEN_MAX); - /* set hardware descriptor */ - desc->bytes = cpu_to_le32(len); - desc->src_addr = cpu_to_le64(*src); - desc->dst_addr = cpu_to_le64(*dst); - - if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) { - dblk++; - desc = dblk->virt_addr; - } else { - desc++; - } - - desc_num++; - dev_addr += len; - addr += len; - rest -= len; - } while (rest); + desc_num += xdma_fill_descs(sw_desc, *src, *dst, sg_dma_len(sg), desc_num); + dev_addr += sg_dma_len(sg); } tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); @@ -643,9 +656,9 @@ xdma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t address, struct xdma_device *xdev = xdma_chan->xdev_hdl; unsigned int periods = size / period_size; struct dma_async_tx_descriptor *tx_desc; - struct xdma_desc_block *dblk; - struct xdma_hw_desc *desc; struct xdma_desc *sw_desc; + u64 addr, dev_addr, *src, *dst; + u32 desc_num; unsigned int i; /* @@ -670,21 +683,21 @@ xdma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t address, sw_desc->period_size = period_size; sw_desc->dir = dir; - dblk = sw_desc->desc_blocks; - desc = dblk->virt_addr; + addr = address; + if (dir == DMA_MEM_TO_DEV) { + dev_addr = xdma_chan->cfg.dst_addr; + src = &addr; + dst = &dev_addr; + } else { + dev_addr = xdma_chan->cfg.src_addr; + src = &dev_addr; + dst = &addr; + } - /* fill hardware descriptor */ + desc_num = 0; for (i = 0; i < periods; i++) { - desc->bytes = cpu_to_le32(period_size); - if (dir == DMA_MEM_TO_DEV) { - desc->src_addr = cpu_to_le64(address + i * period_size); - desc->dst_addr = cpu_to_le64(xdma_chan->cfg.dst_addr); - } else { - desc->src_addr = cpu_to_le64(xdma_chan->cfg.src_addr); - desc->dst_addr = cpu_to_le64(address + i * period_size); - } - - desc++; + desc_num += xdma_fill_descs(sw_desc, *src, *dst, period_size, desc_num); + addr += i * period_size; } tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); -- cgit From 2f8f90cd2f8d237c51c2775a53ef0d8c8acaa707 Mon Sep 17 00:00:00 2001 From: Jan Kuliga Date: Mon, 18 Dec 2023 12:39:43 +0100 Subject: dmaengine: xilinx: xdma: Implement interleaved DMA transfers Interleaved DMA functionality allows dmaengine clients' to express DMA transfers in an arbitrary way. This is extremely useful in FPGA environments, where a greater transfer flexibility is needed. For instance, in one FPGA design there may be need to do DMA to/from a FIFO at a fixed address, and also to do DMA to/from a (non)contiguous RAM memory. Introduce separate tx preparation callback and add tx-flags handling logic. Their behavior is based on the description of interleaved DMA transfers in both source code and the DMAEngine's documentation. Since XDMA is a fully-fledged scatter-gather dma engine, the logic of xdma_prep_interleaved_dma() is fairly simple and similar to the other tx preparation callbacks. The whole tx-flags handling logic resides in xdma_channel_isr(). Transfer of a single frame from a interleaved DMA transfer template is pretty similar to the single sg transaction. Therefore, the transaction of the whole interleaved DMA transfer template is basically a cyclic dma transaction with finite cycles/periods (equal to the frame of count) of a single sg transfers. Signed-off-by: Jan Kuliga Link: https://lore.kernel.org/r/20231218113943.9099-9-jankul@alatek.krakow.pl Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 107 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 9 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 618cc9af6eb9..9360b85131ef 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -83,8 +83,10 @@ struct xdma_chan { * @desc_num: Number of hardware descriptors * @completed_desc_num: Completed hardware descriptors * @cyclic: Cyclic transfer vs. scatter-gather + * @interleaved_dma: Interleaved DMA transfer * @periods: Number of periods in the cyclic transfer * @period_size: Size of a period in bytes in cyclic transfers + * @frames_left: Number of frames left in interleaved DMA transfer * @error: tx error flag */ struct xdma_desc { @@ -96,8 +98,10 @@ struct xdma_desc { u32 desc_num; u32 completed_desc_num; bool cyclic; + bool interleaved_dma; u32 periods; u32 period_size; + u32 frames_left; bool error; }; @@ -607,6 +611,8 @@ xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!sw_desc) return NULL; sw_desc->dir = dir; + sw_desc->cyclic = false; + sw_desc->interleaved_dma = false; if (dir == DMA_MEM_TO_DEV) { dev_addr = xdma_chan->cfg.dst_addr; @@ -682,6 +688,7 @@ xdma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t address, sw_desc->periods = periods; sw_desc->period_size = period_size; sw_desc->dir = dir; + sw_desc->interleaved_dma = false; addr = address; if (dir == DMA_MEM_TO_DEV) { @@ -712,6 +719,57 @@ failed: return NULL; } +/** + * xdma_prep_interleaved_dma - Prepare virtual descriptor for interleaved DMA transfers + * @chan: DMA channel + * @xt: DMA transfer template + * @flags: tx flags + */ +struct dma_async_tx_descriptor * +xdma_prep_interleaved_dma(struct dma_chan *chan, + struct dma_interleaved_template *xt, + unsigned long flags) +{ + int i; + u32 desc_num = 0, period_size = 0; + struct dma_async_tx_descriptor *tx_desc; + struct xdma_chan *xchan = to_xdma_chan(chan); + struct xdma_desc *sw_desc; + u64 src_addr, dst_addr; + + for (i = 0; i < xt->frame_size; ++i) + desc_num += DIV_ROUND_UP(xt->sgl[i].size, XDMA_DESC_BLEN_MAX); + + sw_desc = xdma_alloc_desc(xchan, desc_num, false); + if (!sw_desc) + return NULL; + sw_desc->dir = xt->dir; + sw_desc->interleaved_dma = true; + sw_desc->cyclic = flags & DMA_PREP_REPEAT; + sw_desc->frames_left = xt->numf; + sw_desc->periods = xt->numf; + + desc_num = 0; + src_addr = xt->src_start; + dst_addr = xt->dst_start; + for (i = 0; i < xt->frame_size; ++i) { + desc_num += xdma_fill_descs(sw_desc, src_addr, dst_addr, xt->sgl[i].size, desc_num); + src_addr += dmaengine_get_src_icg(xt, &xt->sgl[i]) + xt->src_inc ? + xt->sgl[i].size : 0; + dst_addr += dmaengine_get_dst_icg(xt, &xt->sgl[i]) + xt->dst_inc ? + xt->sgl[i].size : 0; + period_size += xt->sgl[i].size; + } + sw_desc->period_size = period_size; + + tx_desc = vchan_tx_prep(&xchan->vchan, &sw_desc->vdesc, flags); + if (tx_desc) + return tx_desc; + + xdma_free_desc(&sw_desc->vdesc); + return NULL; +} + /** * xdma_device_config - Configure the DMA channel * @chan: DMA channel @@ -811,11 +869,12 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) { struct xdma_chan *xchan = dev_id; u32 complete_desc_num = 0; - struct xdma_device *xdev; - struct virt_dma_desc *vd; + struct xdma_device *xdev = xchan->xdev_hdl; + struct virt_dma_desc *vd, *next_vd; struct xdma_desc *desc; int ret; u32 st; + bool repeat_tx; spin_lock(&xchan->vchan.lock); @@ -824,9 +883,6 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) if (!vd) goto out; - desc = to_xdma_desc(vd); - xdev = xchan->xdev_hdl; - /* Clear-on-read the status register */ ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &st); if (ret) @@ -845,10 +901,36 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) if (ret) goto out; - if (desc->cyclic) { - desc->completed_desc_num = complete_desc_num; - vchan_cyclic_callback(vd); - } else { + desc = to_xdma_desc(vd); + if (desc->interleaved_dma) { + xchan->busy = false; + desc->completed_desc_num += complete_desc_num; + if (complete_desc_num == XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) { + xdma_xfer_start(xchan); + goto out; + } + + /* last desc of any frame */ + desc->frames_left--; + if (desc->frames_left) + goto out; + + /* last desc of the last frame */ + repeat_tx = vd->tx.flags & DMA_PREP_REPEAT; + next_vd = list_first_entry_or_null(&vd->node, struct virt_dma_desc, node); + if (next_vd) + repeat_tx = repeat_tx && !(next_vd->tx.flags & DMA_PREP_LOAD_EOT); + if (repeat_tx) { + desc->frames_left = desc->periods; + desc->completed_desc_num = 0; + vchan_cyclic_callback(vd); + } else { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + /* start (or continue) the tx of a first desc on the vc.desc_issued list, if any */ + xdma_xfer_start(xchan); + } else if (!desc->cyclic) { xchan->busy = false; desc->completed_desc_num += complete_desc_num; @@ -865,6 +947,9 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) /* transfer the rest of data */ xdma_xfer_start(xchan); + } else { + desc->completed_desc_num = complete_desc_num; + vchan_cyclic_callback(vd); } out: @@ -1163,6 +1248,9 @@ static int xdma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask); dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask); dma_cap_set(DMA_CYCLIC, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_INTERLEAVE, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_REPEAT, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_LOAD_EOT, xdev->dma_dev.cap_mask); xdev->dma_dev.dev = &pdev->dev; xdev->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; @@ -1178,6 +1266,7 @@ static int xdma_probe(struct platform_device *pdev) xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt; xdev->dma_dev.filter.fn = xdma_filter_fn; xdev->dma_dev.device_prep_dma_cyclic = xdma_prep_dma_cyclic; + xdev->dma_dev.device_prep_interleaved_dma = xdma_prep_interleaved_dma; ret = dma_async_device_register(&xdev->dma_dev); if (ret) { -- cgit From 22a9d9585812440211b0b34a6bc02ade62314be4 Mon Sep 17 00:00:00 2001 From: Bumyong Lee Date: Tue, 19 Dec 2023 14:50:26 +0900 Subject: dmaengine: pl330: issue_pending waits until WFP state According to DMA-330 errata notice[1] 71930, DMAKILL cannot clear internal signal, named pipeline_req_active. it makes that pl330 would wait forever in WFP state although dma already send dma request if pl330 gets dma request before entering WFP state. The errata suggests that polling until entering WFP state as workaround and then peripherals allows to issue dma request. [1]: https://developer.arm.com/documentation/genc008428/latest Signed-off-by: Bumyong Lee Link: https://lore.kernel.org/r/20231219055026.118695-1-bumyong.lee@samsung.com Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 3cf0b38387ae..c29744bfdf2c 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -1053,6 +1053,9 @@ static bool _trigger(struct pl330_thread *thrd) thrd->req_running = idx; + if (desc->rqtype == DMA_MEM_TO_DEV || desc->rqtype == DMA_DEV_TO_MEM) + UNTIL(thrd, PL330_STATE_WFP); + return true; } -- cgit From bbcd7b588b0bf967f90df60fccd16c9c49b768ea Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Fri, 22 Dec 2023 15:10:17 +0530 Subject: dmaengine: xilinx: xdma: Workaround truncation compilation error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Increase length to be copied to be large enough to overcome the following compilation error. The buf is large enough for this purpose. drivers/dma/xilinx/xilinx_dpdma.c: In function ‘xilinx_dpdma_debugfs_desc_done_irq_read’: drivers/dma/xilinx/xilinx_dpdma.c:313:39: error: ‘snprintf’ output may be truncated before the last format character [-Werror=format-truncation=] 313 | snprintf(buf, out_str_len, "%d", | ^ drivers/dma/xilinx/xilinx_dpdma.c:313:9: note: ‘snprintf’ output between 2 and 6 bytes into a destination of size 5 313 | snprintf(buf, out_str_len, "%d", | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 314 | dpdma_debugfs.xilinx_dpdma_irq_done_count); | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Signed-off-by: Vinod Koul Link: https://lore.kernel.org/r/20231222094017.731917-1-vkoul@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dpdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xilinx_dpdma.c b/drivers/dma/xilinx/xilinx_dpdma.c index 69587d85a7cd..b82815e64d24 100644 --- a/drivers/dma/xilinx/xilinx_dpdma.c +++ b/drivers/dma/xilinx/xilinx_dpdma.c @@ -309,7 +309,7 @@ static ssize_t xilinx_dpdma_debugfs_desc_done_irq_read(char *buf) out_str_len = strlen(XILINX_DPDMA_DEBUGFS_UINT16_MAX_STR); out_str_len = min_t(size_t, XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE, - out_str_len); + out_str_len + 1); snprintf(buf, out_str_len, "%d", dpdma_debugfs.xilinx_dpdma_irq_done_count); -- cgit From 3d0b2176e04261ab4ac095ff2a17db077fc1e46d Mon Sep 17 00:00:00 2001 From: Vinod Koul Date: Fri, 22 Dec 2023 15:10:01 +0530 Subject: dmaengine: xilinx: xdma: statify xdma_prep_interleaved_dma xdma_prep_interleaved_dma() was local to file but not declared static, leading to warning: drivers/dma/xilinx/xdma.c:729:1: warning: no previous prototype for 'xdma_prep_interleaved_dma' [-Wmissing-prototypes] 729 | xdma_prep_interleaved_dma(struct dma_chan *chan Reported-by: Stephen Rothwell Signed-off-by: Vinod Koul Link: https://lore.kernel.org/r/20231222094001.731889-1-vkoul@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index 9360b85131ef..4ebc90b41bdb 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -725,7 +725,7 @@ failed: * @xt: DMA transfer template * @flags: tx flags */ -struct dma_async_tx_descriptor * +static struct dma_async_tx_descriptor * xdma_prep_interleaved_dma(struct dma_chan *chan, struct dma_interleaved_template *xt, unsigned long flags) -- cgit