diff options
Diffstat (limited to 'drivers/dma')
185 files changed, 14215 insertions, 5653 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 08fdd0e2ed1b..8bb0a119ecd4 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -89,13 +89,20 @@ config APPLE_ADMAC tristate "Apple ADMAC support" depends on ARCH_APPLE || COMPILE_TEST select DMA_ENGINE - default ARCH_APPLE help Enable support for Audio DMA Controller found on Apple Silicon SoCs. +config ARM_DMA350 + tristate "Arm DMA-350 support" + depends on ARM || ARM64 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the Arm DMA-350 controller. + config AT_HDMAC tristate "Atmel AHB DMA support" - depends on ARCH_AT91 + depends on ARCH_AT91 || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -103,7 +110,7 @@ config AT_HDMAC config AT_XDMAC tristate "Atmel XDMA support" - depends on ARCH_AT91 + depends on ARCH_MICROCHIP select DMA_ENGINE help Support the Atmel XDMA controller. @@ -136,7 +143,7 @@ config BCM_SBA_RAID config DMA_BCM2835 tristate "BCM2835 DMA engine support" - depends on ARCH_BCM2835 + depends on ARCH_BCM2835 || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -162,8 +169,8 @@ config DMA_SA11X0 config DMA_SUN4I tristate "Allwinner A10 DMA SoCs support" - depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I - default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I) + depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I || MACH_SUNIV + default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I || MACH_SUNIV) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -362,13 +369,36 @@ config INTEL_IOATDMA config K3_DMA tristate "Hisilicon K3 DMA support" - depends on ARCH_HI3xxx || ARCH_HISI || COMPILE_TEST + depends on ARCH_HISI || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help Support the DMA engine for Hisilicon K3 platform devices. +config LOONGSON1_APB_DMA + tristate "Loongson1 APB DMA support" + depends on MACH_LOONGSON32 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + This selects support for the APB DMA controller in Loongson1 SoCs, + which is required by Loongson1 NAND and audio support. + +config LOONGSON2_APB_DMA + tristate "Loongson2 APB DMA support" + depends on LOONGARCH || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Loongson2 APB DMA controller driver. The + DMA controller is having single DMA channel which can be + configured for different peripherals like audio, nand, sdio + etc which is in APB bus. + + This DMA controller transfers data from memory to peripheral fifo. + It does not support memory to memory data transfer. + config LPC18XX_DMAMUX bool "NXP LPC18xx/43xx DMA MUX for PL080" depends on ARCH_LPC18XX || COMPILE_TEST @@ -378,9 +408,18 @@ config LPC18XX_DMAMUX Enable support for DMA on NXP LPC18xx/43xx platforms with PL080 and multiplexed DMA request lines. +config LPC32XX_DMAMUX + bool "NXP LPC32xx DMA MUX for PL080" + depends on ARCH_LPC32XX || COMPILE_TEST + depends on OF && AMBA_PL08X + select MFD_SYSCON + help + Support for PL080 multiplexed DMA request lines on + LPC32XX platrofm. + config MCF_EDMA tristate "Freescale eDMA engine support, ColdFire mcf5441x SoCs" - depends on M5441x || COMPILE_TEST + depends on M5441x || (COMPILE_TEST && FSL_EDMA=n) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -411,7 +450,7 @@ config MILBEAUT_XDMAC config MMP_PDMA tristate "MMP PDMA support" - depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST + depends on ARCH_MMP || ARCH_PXA || ARCH_SPACEMIT || COMPILE_TEST select DMA_ENGINE help Support the MMP PDMA engine for PXA and MMP platform. @@ -474,25 +513,6 @@ config MXS_DMA Support the MXS DMA engine. This engine including APBH-DMA and APBX-DMA is integrated into some Freescale chips. -config MX3_IPU - bool "MX3x Image Processing Unit support" - depends on ARCH_MXC - select DMA_ENGINE - default y - help - If you plan to use the Image Processing unit in the i.MX3x, say - Y here. If unsure, select Y. - -config MX3_IPU_IRQS - int "Number of dynamically mapped interrupts for IPU" - depends on MX3_IPU - range 2 137 - default 4 - help - Out of 137 interrupt sources on i.MX31 IPU only very few are used. - To avoid bloating the irq_desc[] array we allocate a sufficient - number of IRQ slots and map them dynamically to specific sources. - config NBPFAXI_DMA tristate "Renesas Type-AXI NBPF DMA support" select DMA_ENGINE @@ -533,7 +553,7 @@ config PL330_DMA config PXA_DMA bool "PXA DMA support" - depends on (ARCH_MMP || ARCH_PXA) + depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -551,6 +571,15 @@ config PLX_DMA These are exposed via extra functions on the switch's upstream port. Each function exposes one DMA channel. +config SOPHGO_CV1800B_DMAMUX + tristate "Sophgo CV1800/SG2000 series SoC DMA multiplexer support" + depends on MFD_SYSCON + depends on ARCH_SOPHGO || COMPILE_TEST + help + Support for the DMA multiplexer on Sophgo CV1800/SG2000 + series SoCs. + Say Y here if your board have this soc. + config STE_DMA40 bool "ST-Ericsson DMA40 support" depends on ARCH_U8500 @@ -573,38 +602,6 @@ config ST_FDMA Say Y here if you have such a chipset. If unsure, say N. -config STM32_DMA - bool "STMicroelectronics STM32 DMA support" - depends on ARCH_STM32 || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the on-chip DMA controller on STMicroelectronics - STM32 MCUs. - If you have a board based on such a MCU and wish to use DMA say Y - here. - -config STM32_DMAMUX - bool "STMicroelectronics STM32 dma multiplexer support" - depends on STM32_DMA || COMPILE_TEST - help - Enable support for the on-chip DMA multiplexer on STMicroelectronics - STM32 MCUs. - If you have a board based on such a MCU and wish to use DMAMUX say Y - here. - -config STM32_MDMA - bool "STMicroelectronics STM32 master dma support" - depends on ARCH_STM32 || COMPILE_TEST - depends on OF - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the on-chip MDMA controller on STMicroelectronics - STM32 platforms. - If you have a board based on STM32 SoC and wish to use the master DMA - say Y here. - config SPRD_DMA tristate "Spreadtrum DMA support" depends on ARCH_SPRD || COMPILE_TEST @@ -648,16 +645,16 @@ config TEGRA20_APB_DMA config TEGRA210_ADMA tristate "NVIDIA Tegra210 ADMA support" - depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) + depends on (ARCH_TEGRA || COMPILE_TEST) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help - Support for the NVIDIA Tegra210 ADMA controller driver. The - DMA controller has multiple DMA channels and is used to service - various audio clients in the Tegra210 audio processing engine - (APE). This DMA controller transfers data from memory to - peripheral and vice versa. It does not support memory to - memory data transfer. + Support for the NVIDIA Tegra210/Tegra186/Tegra194/Tegra234 ADMA + controller driver. The DMA controller has multiple DMA channels + and is used to service various audio clients in the Tegra210 + audio processing engine (APE). This DMA controller transfers + data from memory to peripheral and vice versa. It does not + support memory to memory data transfer. config TIMB_DMA tristate "Timberdale FPGA DMA support" @@ -699,7 +696,7 @@ config XGENE_DMA config XILINX_DMA tristate "Xilinx AXI DMAS Engine" - depends on (ARCH_ZYNQ || MICROBLAZE || ARM64) + depends on HAS_IOMEM select DMA_ENGINE help Enable support for Xilinx AXI VDMA Soft IP. @@ -753,12 +750,12 @@ config XILINX_ZYNQMP_DPDMA display driver. # driver files +source "drivers/dma/amd/Kconfig" + source "drivers/dma/bestcomm/Kconfig" source "drivers/dma/mediatek/Kconfig" -source "drivers/dma/ptdma/Kconfig" - source "drivers/dma/qcom/Kconfig" source "drivers/dma/dw/Kconfig" @@ -777,6 +774,8 @@ source "drivers/dma/fsl-dpaa2-qdma/Kconfig" source "drivers/dma/lgm/Kconfig" +source "drivers/dma/stm32/Kconfig" + # clients comment "DMA Clients" depends on DMA_ENGINE diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index a4fd1ce29510..a54d7688392b 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -16,8 +16,8 @@ obj-$(CONFIG_DMATEST) += dmatest.o obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ -obj-$(CONFIG_AMD_PTDMA) += ptdma/ obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o +obj-$(CONFIG_ARM_DMA350) += arm-dma350.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o @@ -31,9 +31,13 @@ obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ obj-$(CONFIG_DW_DMAC_CORE) += dw/ obj-$(CONFIG_DW_EDMA) += dw-edma/ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o +fsl-edma-trace-$(CONFIG_TRACING) := fsl-edma-trace.o +CFLAGS_fsl-edma-trace.o := -I$(src) obj-$(CONFIG_FSL_DMA) += fsldma.o -obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o -obj-$(CONFIG_MCF_EDMA) += mcf-edma.o fsl-edma-common.o +fsl-edma-objs := fsl-edma-main.o fsl-edma-common.o ${fsl-edma-trace-y} +obj-$(CONFIG_FSL_EDMA) += fsl-edma.o +mcf-edma-objs := mcf-edma-main.o fsl-edma-common.o ${fsl-edma-trace-y} +obj-$(CONFIG_MCF_EDMA) += mcf-edma.o obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o obj-$(CONFIG_FSL_RAID) += fsl_raid.o obj-$(CONFIG_HISI_DMA) += hisi_dma.o @@ -45,7 +49,10 @@ obj-$(CONFIG_INTEL_IDMA64) += idma64.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-y += idxd/ obj-$(CONFIG_K3_DMA) += k3dma.o +obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o +obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o +obj-$(CONFIG_LPC32XX_DMAMUX) += lpc32xx-dmamux.o obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o obj-$(CONFIG_MILBEAUT_XDMAC) += milbeaut-xdmac.o obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o @@ -55,7 +62,6 @@ obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o obj-$(CONFIG_MV_XOR) += mv_xor.o obj-$(CONFIG_MV_XOR_V2) += mv_xor_v2.o obj-$(CONFIG_MXS_DMA) += mxs-dma.o -obj-$(CONFIG_MX3_IPU) += ipu/ obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o obj-$(CONFIG_OWL_DMA) += owl-dma.o obj-$(CONFIG_PCH_DMA) += pch_dma.o @@ -65,10 +71,8 @@ obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ obj-$(CONFIG_PXA_DMA) += pxa_dma.o obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SF_PDMA) += sf-pdma/ +obj-$(CONFIG_SOPHGO_CV1800B_DMAMUX) += cv1800b-dmamux.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o -obj-$(CONFIG_STM32_DMA) += stm32-dma.o -obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o -obj-$(CONFIG_STM32_MDMA) += stm32-mdma.o obj-$(CONFIG_SPRD_DMA) += sprd-dma.o obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_TEGRA186_GPC_DMA) += tegra186-gpc-dma.o @@ -82,7 +86,9 @@ obj-$(CONFIG_ST_FDMA) += st_fdma.o obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ obj-$(CONFIG_INTEL_LDMA) += lgm/ +obj-y += amd/ obj-y += mediatek/ obj-y += qcom/ +obj-y += stm32/ obj-y += ti/ obj-y += xilinx/ diff --git a/drivers/dma/acpi-dma.c b/drivers/dma/acpi-dma.c index 5906eae26e2a..2abbe11e797e 100644 --- a/drivers/dma/acpi-dma.c +++ b/drivers/dma/acpi-dma.c @@ -9,18 +9,21 @@ * Mika Westerberg <mika.westerberg@linux.intel.com> */ +#include <linux/acpi.h> +#include <linux/acpi_dma.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/err.h> -#include <linux/module.h> +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/ioport.h> #include <linux/kernel.h> #include <linux/list.h> #include <linux/mutex.h> -#include <linux/slab.h> -#include <linux/ioport.h> -#include <linux/acpi.h> -#include <linux/acpi_dma.h> #include <linux/property.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/types.h> static LIST_HEAD(acpi_dma_list); static DEFINE_MUTEX(acpi_dma_lock); @@ -112,7 +115,7 @@ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp, } /** - * acpi_dma_parse_csrt - parse CSRT to exctract additional DMA resources + * acpi_dma_parse_csrt - parse CSRT to extract additional DMA resources * @adev: ACPI device to match with * @adma: struct acpi_dma of the given DMA controller * @@ -236,7 +239,7 @@ int acpi_dma_controller_free(struct device *dev) } EXPORT_SYMBOL_GPL(acpi_dma_controller_free); -static void devm_acpi_dma_release(struct device *dev, void *res) +static void devm_acpi_dma_free(void *dev) { acpi_dma_controller_free(dev); } @@ -259,37 +262,15 @@ int devm_acpi_dma_controller_register(struct device *dev, (struct acpi_dma_spec *, struct acpi_dma *), void *data) { - void *res; int ret; - res = devres_alloc(devm_acpi_dma_release, 0, GFP_KERNEL); - if (!res) - return -ENOMEM; - ret = acpi_dma_controller_register(dev, acpi_dma_xlate, data); - if (ret) { - devres_free(res); + if (ret) return ret; - } - devres_add(dev, res); - return 0; -} -EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_register); -/** - * devm_acpi_dma_controller_free - resource managed acpi_dma_controller_free() - * @dev: device that is unregistering as DMA controller - * - * Unregister a DMA controller registered with - * devm_acpi_dma_controller_register(). Normally this function will not need to - * be called and the resource management code will ensure that the resource is - * freed. - */ -void devm_acpi_dma_controller_free(struct device *dev) -{ - WARN_ON(devres_release(dev, devm_acpi_dma_release, NULL, NULL)); + return devm_add_action_or_reset(dev, devm_acpi_dma_free, dev); } -EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_free); +EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_register); /** * acpi_dma_update_dma_spec - prepare dma specifier to pass to translation function @@ -305,7 +286,7 @@ EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_free); * found. * * Return: - * 0, if no information is avaiable, -1 on mismatch, and 1 otherwise. + * 0, if no information is available, -1 on mismatch, and 1 otherwise. */ static int acpi_dma_update_dma_spec(struct acpi_dma *adma, struct acpi_dma_spec *dma_spec) diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c index 4153c2edb049..a203fdd84950 100644 --- a/drivers/dma/altera-msgdma.c +++ b/drivers/dma/altera-msgdma.c @@ -153,7 +153,7 @@ struct msgdma_extended_desc { /** * struct msgdma_sw_desc - implements a sw descriptor * @async_tx: support for the async_tx api - * @hw_desc: assosiated HW descriptor + * @hw_desc: associated HW descriptor * @node: node to move from the free list to the tx list * @tx_list: transmit list node */ @@ -233,7 +233,7 @@ static void msgdma_free_descriptor(struct msgdma_device *mdev, struct msgdma_sw_desc *child, *next; mdev->desc_free_cnt++; - list_add_tail(&desc->node, &mdev->free_list); + list_move_tail(&desc->node, &mdev->free_list); list_for_each_entry_safe(child, next, &desc->tx_list, node) { mdev->desc_free_cnt++; list_move_tail(&child->node, &mdev->free_list); @@ -511,7 +511,7 @@ static void msgdma_copy_one(struct msgdma_device *mdev, * of the DMA controller. The descriptor will get flushed to the * FIFO, once the last word (control word) is written. Since we * are not 100% sure that memcpy() writes all word in the "correct" - * oder (address from low to high) on all architectures, we make + * order (address from low to high) on all architectures, we make * sure this control word is written last by single coding it and * adding some write-barriers here. */ @@ -583,22 +583,25 @@ static void msgdma_issue_pending(struct dma_chan *chan) static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev) { struct msgdma_sw_desc *desc, *next; + unsigned long irqflags; + + spin_lock_irqsave(&mdev->lock, irqflags); list_for_each_entry_safe(desc, next, &mdev->done_list, node) { struct dmaengine_desc_callback cb; - list_del(&desc->node); - dmaengine_desc_get_callback(&desc->async_tx, &cb); if (dmaengine_desc_callback_valid(&cb)) { - spin_unlock(&mdev->lock); + spin_unlock_irqrestore(&mdev->lock, irqflags); dmaengine_desc_callback_invoke(&cb, NULL); - spin_lock(&mdev->lock); + spin_lock_irqsave(&mdev->lock, irqflags); } /* Run any dependencies, then free the descriptor */ msgdma_free_descriptor(mdev, desc); } + + spin_unlock_irqrestore(&mdev->lock, irqflags); } /** @@ -713,10 +716,11 @@ static void msgdma_tasklet(struct tasklet_struct *t) } msgdma_complete_descriptor(mdev); - msgdma_chan_desc_cleanup(mdev); } spin_unlock_irqrestore(&mdev->lock, flags); + + msgdma_chan_desc_cleanup(mdev); } /** @@ -923,7 +927,7 @@ fail: * * Return: Always '0' */ -static int msgdma_remove(struct platform_device *pdev) +static void msgdma_remove(struct platform_device *pdev) { struct msgdma_device *mdev = platform_get_drvdata(pdev); @@ -933,8 +937,6 @@ static int msgdma_remove(struct platform_device *pdev) msgdma_dev_remove(mdev); dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n"); - - return 0; } #ifdef CONFIG_OF diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index eea8bd33b4b7..38cdbca59485 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2006 ARM Ltd. * Copyright (c) 2010 ST-Ericsson SA - * Copyirght (c) 2017 Linaro Ltd. + * Copyright (c) 2017 Linaro Ltd. * * Author: Peter Pearse <peter.pearse@arm.com> * Author: Linus Walleij <linus.walleij@linaro.org> @@ -2239,7 +2239,7 @@ static int pl08x_resume(struct dma_chan *chan) bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) { struct pl08x_dma_chan *plchan; - char *name = chan_id; + const char *name = chan_id; /* Reject channels for devices not bound to this driver */ if (chan->device->dev->driver != &pl08x_amba_driver.drv) @@ -2855,8 +2855,8 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) } /* Initialize physical channels */ - pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), - GFP_KERNEL); + pl08x->phy_chans = kcalloc(vd->channels, sizeof(*pl08x->phy_chans), + GFP_KERNEL); if (!pl08x->phy_chans) { ret = -ENOMEM; goto out_no_phychans; diff --git a/drivers/dma/amd/Kconfig b/drivers/dma/amd/Kconfig new file mode 100644 index 000000000000..00d874872a8f --- /dev/null +++ b/drivers/dma/amd/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only +# + +config AMD_AE4DMA + tristate "AMD AE4DMA Engine" + depends on (X86_64 || COMPILE_TEST) && PCI + depends on AMD_PTDMA + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD AE4DMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. + +config AMD_PTDMA + tristate "AMD PassThru DMA Engine" + depends on X86_64 && PCI + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD PTDMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. + +config AMD_QDMA + tristate "AMD Queue-based DMA" + depends on HAS_IOMEM + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select REGMAP_MMIO + help + Enable support for the AMD Queue-based DMA subsystem. The primary + mechanism to transfer data using the QDMA is for the QDMA engine to + operate on instructions (descriptors) provided by the host operating + system. Using the descriptors, the QDMA can move data in either the + Host to Card (H2C) direction or the Card to Host (C2H) direction. diff --git a/drivers/dma/amd/Makefile b/drivers/dma/amd/Makefile new file mode 100644 index 000000000000..11278c06374d --- /dev/null +++ b/drivers/dma/amd/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_AMD_AE4DMA) += ae4dma/ +obj-$(CONFIG_AMD_PTDMA) += ptdma/ +obj-$(CONFIG_AMD_QDMA) += qdma/ diff --git a/drivers/dma/amd/ae4dma/Makefile b/drivers/dma/amd/ae4dma/Makefile new file mode 100644 index 000000000000..e918f85a80ec --- /dev/null +++ b/drivers/dma/amd/ae4dma/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# AMD AE4DMA driver +# + +obj-$(CONFIG_AMD_AE4DMA) += ae4dma.o + +ae4dma-objs := ae4dma-dev.o + +ae4dma-$(CONFIG_PCI) += ae4dma-pci.o diff --git a/drivers/dma/amd/ae4dma/ae4dma-dev.c b/drivers/dma/amd/ae4dma/ae4dma-dev.c new file mode 100644 index 000000000000..8de3bef41b58 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-dev.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com> + */ + +#include "ae4dma.h" + +static unsigned int max_hw_q = 1; +module_param(max_hw_q, uint, 0444); +MODULE_PARM_DESC(max_hw_q, "max hw queues supported by engine (any non-zero value, default: 1)"); + +static void ae4_pending_work(struct work_struct *work) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(work, struct ae4_cmd_queue, p_work.work); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct pt_cmd *cmd; + u32 cridx; + + for (;;) { + wait_event_interruptible(ae4cmd_q->q_w, + ((atomic64_read(&ae4cmd_q->done_cnt)) < + atomic64_read(&ae4cmd_q->intr_cnt))); + + atomic64_inc(&ae4cmd_q->done_cnt); + + mutex_lock(&ae4cmd_q->cmd_lock); + cridx = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); + while ((ae4cmd_q->dridx != cridx) && !list_empty(&ae4cmd_q->cmd)) { + cmd = list_first_entry(&ae4cmd_q->cmd, struct pt_cmd, entry); + list_del(&cmd->entry); + + ae4_check_status_error(ae4cmd_q, ae4cmd_q->dridx); + cmd->pt_cmd_callback(cmd->data, cmd->ret); + + ae4cmd_q->q_cmd_count--; + ae4cmd_q->dridx = (ae4cmd_q->dridx + 1) % CMD_Q_LEN; + + complete_all(&ae4cmd_q->cmp); + } + mutex_unlock(&ae4cmd_q->cmd_lock); + } +} + +static irqreturn_t ae4_core_irq_handler(int irq, void *data) +{ + struct ae4_cmd_queue *ae4cmd_q = data; + struct pt_cmd_queue *cmd_q; + struct pt_device *pt; + u32 status; + + cmd_q = &ae4cmd_q->cmd_q; + pt = cmd_q->pt; + + pt->total_interrupts++; + atomic64_inc(&ae4cmd_q->intr_cnt); + + status = readl(cmd_q->reg_control + AE4_INTR_STS_OFF); + if (status & BIT(0)) { + status &= GENMASK(31, 1); + writel(status, cmd_q->reg_control + AE4_INTR_STS_OFF); + } + + wake_up(&ae4cmd_q->q_w); + + return IRQ_HANDLED; +} + +void ae4_destroy_work(struct ae4_device *ae4) +{ + struct ae4_cmd_queue *ae4cmd_q; + int i; + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + if (!ae4cmd_q->pws) + break; + + cancel_delayed_work_sync(&ae4cmd_q->p_work); + destroy_workqueue(ae4cmd_q->pws); + } +} + +int ae4_core_init(struct ae4_device *ae4) +{ + struct pt_device *pt = &ae4->pt; + struct ae4_cmd_queue *ae4cmd_q; + struct device *dev = pt->dev; + struct pt_cmd_queue *cmd_q; + int i, ret = 0; + + writel(max_hw_q, pt->io_regs); + + for (i = 0; i < max_hw_q; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + ae4cmd_q->id = ae4->cmd_q_count; + ae4->cmd_q_count++; + + cmd_q = &ae4cmd_q->cmd_q; + cmd_q->pt = pt; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + ret = devm_request_irq(dev, ae4->ae4_irq[i], ae4_core_irq_handler, 0, + dev_name(pt->dev), ae4cmd_q); + if (ret) + return ret; + + cmd_q->qsize = Q_SIZE(sizeof(struct ae4dma_desc)); + + cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) + return -ENOMEM; + } + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + cmd_q = &ae4cmd_q->cmd_q; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + /* Update the device registers with queue information. */ + writel(CMD_Q_LEN, cmd_q->reg_control + AE4_MAX_IDX_OFF); + + cmd_q->qdma_tail = cmd_q->qbase_dma; + writel(lower_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_L_OFF); + writel(upper_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_H_OFF); + + INIT_LIST_HEAD(&ae4cmd_q->cmd); + init_waitqueue_head(&ae4cmd_q->q_w); + + ae4cmd_q->pws = alloc_ordered_workqueue("ae4dma_%d", WQ_MEM_RECLAIM, ae4cmd_q->id); + if (!ae4cmd_q->pws) { + ae4_destroy_work(ae4); + return -ENOMEM; + } + INIT_DELAYED_WORK(&ae4cmd_q->p_work, ae4_pending_work); + queue_delayed_work(ae4cmd_q->pws, &ae4cmd_q->p_work, usecs_to_jiffies(100)); + + init_completion(&ae4cmd_q->cmp); + } + + ret = pt_dmaengine_register(pt); + if (ret) + ae4_destroy_work(ae4); + else + ptdma_debugfs_setup(pt); + + return ret; +} diff --git a/drivers/dma/amd/ae4dma/ae4dma-pci.c b/drivers/dma/amd/ae4dma/ae4dma-pci.c new file mode 100644 index 000000000000..2c63907db228 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-pci.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com> + */ + +#include "ae4dma.h" + +static int ae4_get_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + int i, v, ret; + + pdev = to_pci_dev(dev); + + for (v = 0; v < ARRAY_SIZE(ae4_msix->msix_entry); v++) + ae4_msix->msix_entry[v].entry = v; + + ret = pci_alloc_irq_vectors(pdev, v, v, PCI_IRQ_MSIX); + if (ret != v) { + if (ret > 0) + pci_free_irq_vectors(pdev); + + dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + dev_err(dev, "could not enable MSI (%d)\n", ret); + return ret; + } + + ret = pci_irq_vector(pdev, 0); + if (ret < 0) { + pci_free_irq_vectors(pdev); + return ret; + } + + for (i = 0; i < MAX_AE4_HW_QUEUES; i++) + ae4->ae4_irq[i] = ret; + + } else { + ae4_msix->msix_count = ret; + for (i = 0; i < ae4_msix->msix_count; i++) + ae4->ae4_irq[i] = pci_irq_vector(pdev, i); + } + + return ret; +} + +static void ae4_free_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + + pdev = to_pci_dev(dev); + + if (ae4_msix && (ae4_msix->msix_count || ae4->ae4_irq[MAX_AE4_HW_QUEUES - 1])) + pci_free_irq_vectors(pdev); +} + +static void ae4_deinit(struct ae4_device *ae4) +{ + ae4_free_irqs(ae4); +} + +static int ae4_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct ae4_device *ae4; + struct pt_device *pt; + int bar_mask; + int ret = 0; + + ae4 = devm_kzalloc(dev, sizeof(*ae4), GFP_KERNEL); + if (!ae4) + return -ENOMEM; + + ae4->ae4_msix = devm_kzalloc(dev, sizeof(struct ae4_msix), GFP_KERNEL); + if (!ae4->ae4_msix) + return -ENOMEM; + + ret = pcim_enable_device(pdev); + if (ret) + goto ae4_error; + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ae4dma"); + if (ret) + goto ae4_error; + + pt = &ae4->pt; + pt->dev = dev; + pt->ver = AE4_DMA_VERSION; + + pt->io_regs = pcim_iomap_table(pdev)[0]; + if (!pt->io_regs) { + ret = -ENOMEM; + goto ae4_error; + } + + ret = ae4_get_irqs(ae4); + if (ret < 0) + goto ae4_error; + + pci_set_master(pdev); + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + + dev_set_drvdata(dev, ae4); + + ret = ae4_core_init(ae4); + if (ret) + goto ae4_error; + + return 0; + +ae4_error: + ae4_deinit(ae4); + + return ret; +} + +static void ae4_pci_remove(struct pci_dev *pdev) +{ + struct ae4_device *ae4 = dev_get_drvdata(&pdev->dev); + + ae4_destroy_work(ae4); + ae4_deinit(ae4); +} + +static const struct pci_device_id ae4_pci_table[] = { + { PCI_VDEVICE(AMD, 0x149B), }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, ae4_pci_table); + +static struct pci_driver ae4_pci_driver = { + .name = "ae4dma", + .id_table = ae4_pci_table, + .probe = ae4_pci_probe, + .remove = ae4_pci_remove, +}; + +module_pci_driver(ae4_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD AE4DMA driver"); diff --git a/drivers/dma/amd/ae4dma/ae4dma.h b/drivers/dma/amd/ae4dma/ae4dma.h new file mode 100644 index 000000000000..57f6048726bb --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com> + */ +#ifndef __AE4DMA_H__ +#define __AE4DMA_H__ + +#include <linux/device.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/list.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/wait.h> + +#include "../ptdma/ptdma.h" +#include "../../virt-dma.h" + +#define MAX_AE4_HW_QUEUES 16 + +#define AE4_DESC_COMPLETED 0x03 + +#define AE4_MAX_IDX_OFF 0x08 +#define AE4_RD_IDX_OFF 0x0c +#define AE4_WR_IDX_OFF 0x10 +#define AE4_INTR_STS_OFF 0x14 +#define AE4_Q_BASE_L_OFF 0x18 +#define AE4_Q_BASE_H_OFF 0x1c +#define AE4_Q_SZ 0x20 + +#define AE4_DMA_VERSION 4 +#define CMD_AE4_DESC_DW0_VAL 2 + +#define AE4_TIME_OUT 5000 + +struct ae4_msix { + int msix_count; + struct msix_entry msix_entry[MAX_AE4_HW_QUEUES]; +}; + +struct ae4_cmd_queue { + struct ae4_device *ae4; + struct pt_cmd_queue cmd_q; + struct list_head cmd; + /* protect command operations */ + struct mutex cmd_lock; + struct delayed_work p_work; + struct workqueue_struct *pws; + struct completion cmp; + wait_queue_head_t q_w; + atomic64_t intr_cnt; + atomic64_t done_cnt; + u64 q_cmd_count; + u32 dridx; + u32 tail_wi; + u32 id; +}; + +union dwou { + u32 dw0; + struct dword0 { + u8 byte0; + u8 byte1; + u16 timestamp; + } dws; +}; + +struct dword1 { + u8 status; + u8 err_code; + u16 desc_id; +}; + +struct ae4dma_desc { + union dwou dwouv; + struct dword1 dw1; + u32 length; + u32 rsvd; + u32 src_hi; + u32 src_lo; + u32 dst_hi; + u32 dst_lo; +}; + +struct ae4_device { + struct pt_device pt; + struct ae4_msix *ae4_msix; + struct ae4_cmd_queue ae4cmd_q[MAX_AE4_HW_QUEUES]; + unsigned int ae4_irq[MAX_AE4_HW_QUEUES]; + unsigned int cmd_q_count; +}; + +int ae4_core_init(struct ae4_device *ae4); +void ae4_destroy_work(struct ae4_device *ae4); +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx); +#endif diff --git a/drivers/dma/ptdma/Makefile b/drivers/dma/amd/ptdma/Makefile index ce5410268a9a..ce5410268a9a 100644 --- a/drivers/dma/ptdma/Makefile +++ b/drivers/dma/amd/ptdma/Makefile diff --git a/drivers/dma/ptdma/ptdma-debugfs.c b/drivers/dma/amd/ptdma/ptdma-debugfs.c index c8307d3044a3..c7c90bbf6fd8 100644 --- a/drivers/dma/ptdma/ptdma-debugfs.c +++ b/drivers/dma/amd/ptdma/ptdma-debugfs.c @@ -13,6 +13,7 @@ #include <linux/seq_file.h> #include "ptdma.h" +#include "../ae4dma/ae4dma.h" /* DebugFS helpers */ #define RI_VERSION_NUM 0x0000003F @@ -23,11 +24,19 @@ static int pt_debugfs_info_show(struct seq_file *s, void *p) { struct pt_device *pt = s->private; + struct ae4_device *ae4; unsigned int regval; seq_printf(s, "Device name: %s\n", dev_name(pt->dev)); - seq_printf(s, " # Queues: %d\n", 1); - seq_printf(s, " # Cmds: %d\n", pt->cmd_count); + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + seq_printf(s, " # Queues: %d\n", ae4->cmd_q_count); + seq_printf(s, " # Cmds per queue: %d\n", CMD_Q_LEN); + } else { + seq_printf(s, " # Queues: %d\n", 1); + seq_printf(s, " # Cmds: %d\n", pt->cmd_count); + } regval = ioread32(pt->io_regs + CMD_PT_VERSION); @@ -55,6 +64,7 @@ static int pt_debugfs_stats_show(struct seq_file *s, void *p) static int pt_debugfs_queue_show(struct seq_file *s, void *p) { struct pt_cmd_queue *cmd_q = s->private; + struct pt_device *pt; unsigned int regval; if (!cmd_q) @@ -62,18 +72,24 @@ static int pt_debugfs_queue_show(struct seq_file *s, void *p) seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops); - regval = ioread32(cmd_q->reg_control + 0x000C); - - seq_puts(s, " Enabled Interrupts:"); - if (regval & INT_EMPTY_QUEUE) - seq_puts(s, " EMPTY"); - if (regval & INT_QUEUE_STOPPED) - seq_puts(s, " STOPPED"); - if (regval & INT_ERROR) - seq_puts(s, " ERROR"); - if (regval & INT_COMPLETION) - seq_puts(s, " COMPLETION"); - seq_puts(s, "\n"); + pt = cmd_q->pt; + if (pt->ver == AE4_DMA_VERSION) { + regval = readl(cmd_q->reg_control + 0x4); + seq_printf(s, " Enabled Interrupts:: status 0x%x\n", regval); + } else { + regval = ioread32(cmd_q->reg_control + 0x000C); + + seq_puts(s, " Enabled Interrupts:"); + if (regval & INT_EMPTY_QUEUE) + seq_puts(s, " EMPTY"); + if (regval & INT_QUEUE_STOPPED) + seq_puts(s, " STOPPED"); + if (regval & INT_ERROR) + seq_puts(s, " ERROR"); + if (regval & INT_COMPLETION) + seq_puts(s, " COMPLETION"); + seq_puts(s, "\n"); + } return 0; } @@ -84,8 +100,12 @@ DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats); void ptdma_debugfs_setup(struct pt_device *pt) { - struct pt_cmd_queue *cmd_q; struct dentry *debugfs_q_instance; + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + char name[30]; + int i; if (!debugfs_initialized()) return; @@ -96,11 +116,28 @@ void ptdma_debugfs_setup(struct pt_device *pt) debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt, &pt_debugfs_stats_fops); - cmd_q = &pt->cmd_q; - - debugfs_q_instance = - debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); - debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, - &pt_debugfs_queue_fops); + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + cmd_q = &ae4cmd_q->cmd_q; + + memset(name, 0, sizeof(name)); + snprintf(name, 29, "q%d", ae4cmd_q->id); + + debugfs_q_instance = + debugfs_create_dir(name, pt->dma_dev.dbg_dev_root); + + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } + } else { + debugfs_q_instance = + debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); + cmd_q = &pt->cmd_q; + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } } +EXPORT_SYMBOL_GPL(ptdma_debugfs_setup); diff --git a/drivers/dma/ptdma/ptdma-dev.c b/drivers/dma/amd/ptdma/ptdma-dev.c index a2bf13ff18b6..a2bf13ff18b6 100644 --- a/drivers/dma/ptdma/ptdma-dev.c +++ b/drivers/dma/amd/ptdma/ptdma-dev.c diff --git a/drivers/dma/ptdma/ptdma-dmaengine.c b/drivers/dma/amd/ptdma/ptdma-dmaengine.c index 1aa65e5de0f3..628c49ce5de9 100644 --- a/drivers/dma/ptdma/ptdma-dmaengine.c +++ b/drivers/dma/amd/ptdma/ptdma-dmaengine.c @@ -9,9 +9,58 @@ * Author: Gary R Hook <gary.hook@amd.com> */ +#include <linux/bitfield.h> #include "ptdma.h" -#include "../dmaengine.h" -#include "../virt-dma.h" +#include "../ae4dma/ae4dma.h" +#include "../../dmaengine.h" + +static char *ae4_error_codes[] = { + "", + "ERR 01: INVALID HEADER DW0", + "ERR 02: INVALID STATUS", + "ERR 03: INVALID LENGTH - 4 BYTE ALIGNMENT", + "ERR 04: INVALID SRC ADDR - 4 BYTE ALIGNMENT", + "ERR 05: INVALID DST ADDR - 4 BYTE ALIGNMENT", + "ERR 06: INVALID ALIGNMENT", + "ERR 07: INVALID DESCRIPTOR", +}; + +static void ae4_log_error(struct pt_device *d, int e) +{ + /* ERR 01 - 07 represents Invalid AE4 errors */ + if (e <= 7) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", ae4_error_codes[e], e); + /* ERR 08 - 15 represents Invalid Descriptor errors */ + else if (e > 7 && e <= 15) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "INVALID DESCRIPTOR", e); + /* ERR 16 - 31 represents Firmware errors */ + else if (e > 15 && e <= 31) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FIRMWARE ERROR", e); + /* ERR 32 - 63 represents Fatal errors */ + else if (e > 31 && e <= 63) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FATAL ERROR", e); + /* ERR 64 - 255 represents PTE errors */ + else if (e > 63 && e <= 255) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "PTE ERROR", e); + else + dev_info(d->dev, "Unknown AE4DMA error"); +} + +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx) +{ + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct ae4dma_desc desc; + u8 status; + + memcpy(&desc, &cmd_q->qbase[idx], sizeof(struct ae4dma_desc)); + status = desc.dw1.status; + if (status && status != AE4_DESC_COMPLETED) { + cmd_q->cmd_error = desc.dw1.err_code; + if (cmd_q->cmd_error) + ae4_log_error(cmd_q->pt, cmd_q->cmd_error); + } +} +EXPORT_SYMBOL_GPL(ae4_check_status_error); static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) { @@ -45,7 +94,71 @@ static void pt_do_cleanup(struct virt_dma_desc *vd) kmem_cache_free(pt->dma_desc_cache, desc); } -static int pt_dma_start_desc(struct pt_dma_desc *desc) +static struct pt_cmd_queue *pt_get_cmd_queue(struct pt_device *pt, struct pt_dma_chan *chan) +{ + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + cmd_q = &ae4cmd_q->cmd_q; + } else { + cmd_q = &pt->cmd_q; + } + + return cmd_q; +} + +static int ae4_core_execute_cmd(struct ae4dma_desc *desc, struct ae4_cmd_queue *ae4cmd_q) +{ + bool soc = FIELD_GET(DWORD0_SOC, desc->dwouv.dw0); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + + if (soc) { + desc->dwouv.dw0 |= FIELD_PREP(DWORD0_IOC, desc->dwouv.dw0); + desc->dwouv.dw0 &= ~DWORD0_SOC; + } + + mutex_lock(&ae4cmd_q->cmd_lock); + memcpy(&cmd_q->qbase[ae4cmd_q->tail_wi], desc, sizeof(struct ae4dma_desc)); + ae4cmd_q->q_cmd_count++; + ae4cmd_q->tail_wi = (ae4cmd_q->tail_wi + 1) % CMD_Q_LEN; + writel(ae4cmd_q->tail_wi, cmd_q->reg_control + AE4_WR_IDX_OFF); + mutex_unlock(&ae4cmd_q->cmd_lock); + + wake_up(&ae4cmd_q->q_w); + + return 0; +} + +static int pt_core_perform_passthru_ae4(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + struct ae4dma_desc desc; + + cmd_q->cmd_error = 0; + cmd_q->total_pt_ops++; + memset(&desc, 0, sizeof(desc)); + desc.dwouv.dws.byte0 = CMD_AE4_DESC_DW0_VAL; + + desc.dw1.status = 0; + desc.dw1.err_code = 0; + desc.dw1.desc_id = 0; + + desc.length = pt_engine->src_len; + + desc.src_lo = upper_32_bits(pt_engine->src_dma); + desc.src_hi = lower_32_bits(pt_engine->src_dma); + desc.dst_lo = upper_32_bits(pt_engine->dst_dma); + desc.dst_hi = lower_32_bits(pt_engine->dst_dma); + + return ae4_core_execute_cmd(&desc, ae4cmd_q); +} + +static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan) { struct pt_passthru_engine *pt_engine; struct pt_device *pt; @@ -56,13 +169,18 @@ static int pt_dma_start_desc(struct pt_dma_desc *desc) pt_cmd = &desc->pt_cmd; pt = pt_cmd->pt; - cmd_q = &pt->cmd_q; + + cmd_q = pt_get_cmd_queue(pt, chan); + pt_engine = &pt_cmd->passthru; pt->tdata.cmd = pt_cmd; /* Execute the command */ - pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); + if (pt->ver == AE4_DMA_VERSION) + pt_cmd->ret = pt_core_perform_passthru_ae4(cmd_q, pt_engine); + else + pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); return 0; } @@ -80,8 +198,10 @@ static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, { struct dma_async_tx_descriptor *tx_desc; struct virt_dma_desc *vd; + struct pt_device *pt; unsigned long flags; + pt = chan->pt; /* Loop over descriptors until one is found with commands */ do { if (desc) { @@ -99,7 +219,7 @@ static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, spin_lock_irqsave(&chan->vc.lock, flags); - if (desc) { + if (pt->ver != AE4_DMA_VERSION && desc) { if (desc->status != DMA_COMPLETE) { if (desc->status != DMA_ERROR) desc->status = DMA_COMPLETE; @@ -117,7 +237,7 @@ static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, spin_unlock_irqrestore(&chan->vc.lock, flags); - if (tx_desc) { + if (pt->ver != AE4_DMA_VERSION && tx_desc) { dmaengine_desc_get_callback_invoke(tx_desc, NULL); dma_run_dependencies(tx_desc); vchan_vdesc_fini(vd); @@ -127,11 +247,25 @@ static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, return NULL; } +static inline bool ae4_core_queue_full(struct pt_cmd_queue *cmd_q) +{ + u32 front_wi = readl(cmd_q->reg_control + AE4_WR_IDX_OFF); + u32 rear_ri = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); + + if (((MAX_CMD_QLEN + front_wi - rear_ri) % MAX_CMD_QLEN) >= (MAX_CMD_QLEN - 1)) + return true; + + return false; +} + static void pt_cmd_callback(void *data, int err) { struct pt_dma_desc *desc = data; + struct ae4_cmd_queue *ae4cmd_q; struct dma_chan *dma_chan; struct pt_dma_chan *chan; + struct ae4_device *ae4; + struct pt_device *pt; int ret; if (err == -EINPROGRESS) @@ -139,11 +273,32 @@ static void pt_cmd_callback(void *data, int err) dma_chan = desc->vd.tx.chan; chan = to_pt_chan(dma_chan); + pt = chan->pt; if (err) desc->status = DMA_ERROR; while (true) { + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + + if (ae4cmd_q->q_cmd_count >= (CMD_Q_LEN - 1) || + ae4_core_queue_full(&ae4cmd_q->cmd_q)) { + wake_up(&ae4cmd_q->q_w); + + if (wait_for_completion_timeout(&ae4cmd_q->cmp, + msecs_to_jiffies(AE4_TIME_OUT)) + == 0) { + dev_err(pt->dev, "TIMEOUT %d:\n", ae4cmd_q->id); + break; + } + + reinit_completion(&ae4cmd_q->cmp); + continue; + } + } + /* Check for DMA descriptor completion */ desc = pt_handle_active_desc(chan, desc); @@ -151,7 +306,7 @@ static void pt_cmd_callback(void *data, int err) if (!desc) break; - ret = pt_dma_start_desc(desc); + ret = pt_dma_start_desc(desc, chan); if (!ret) break; @@ -178,6 +333,50 @@ static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, return desc; } +static void pt_cmd_callback_work(void *data, int err) +{ + struct dma_async_tx_descriptor *tx_desc; + struct pt_dma_desc *desc = data; + struct dma_chan *dma_chan; + struct virt_dma_desc *vd; + struct pt_dma_chan *chan; + unsigned long flags; + + if (!desc) + return; + + dma_chan = desc->vd.tx.chan; + chan = to_pt_chan(dma_chan); + + if (err == -EINPROGRESS) + return; + + tx_desc = &desc->vd.tx; + vd = &desc->vd; + + if (err) + desc->status = DMA_ERROR; + + spin_lock_irqsave(&chan->vc.lock, flags); + if (desc->status != DMA_COMPLETE) { + if (desc->status != DMA_ERROR) + desc->status = DMA_COMPLETE; + + dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); + } else { + tx_desc = NULL; + } + spin_unlock_irqrestore(&chan->vc.lock, flags); + + if (tx_desc) { + dmaengine_desc_get_callback_invoke(tx_desc, NULL); + dma_run_dependencies(tx_desc); + list_del(&desc->vd.node); + vchan_vdesc_fini(vd); + } +} + static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, @@ -186,7 +385,10 @@ static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, { struct pt_dma_chan *chan = to_pt_chan(dma_chan); struct pt_passthru_engine *pt_engine; + struct pt_device *pt = chan->pt; + struct ae4_cmd_queue *ae4cmd_q; struct pt_dma_desc *desc; + struct ae4_device *ae4; struct pt_cmd *pt_cmd; desc = pt_alloc_dma_desc(chan, flags); @@ -194,7 +396,7 @@ static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, return NULL; pt_cmd = &desc->pt_cmd; - pt_cmd->pt = chan->pt; + pt_cmd->pt = pt; pt_engine = &pt_cmd->passthru; pt_cmd->engine = PT_ENGINE_PASSTHRU; pt_engine->src_dma = src; @@ -205,6 +407,15 @@ static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, desc->len = len; + if (pt->ver == AE4_DMA_VERSION) { + pt_cmd->pt_cmd_callback = pt_cmd_callback_work; + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + mutex_lock(&ae4cmd_q->cmd_lock); + list_add_tail(&pt_cmd->entry, &ae4cmd_q->cmd); + mutex_unlock(&ae4cmd_q->cmd_lock); + } + return desc; } @@ -238,13 +449,16 @@ static void pt_issue_pending(struct dma_chan *dma_chan) { struct pt_dma_chan *chan = to_pt_chan(dma_chan); struct pt_dma_desc *desc; + struct pt_device *pt; unsigned long flags; bool engine_is_idle = true; + pt = chan->pt; + spin_lock_irqsave(&chan->vc.lock, flags); desc = pt_next_dma_desc(chan); - if (desc) + if (desc && pt->ver != AE4_DMA_VERSION) engine_is_idle = false; vchan_issue_pending(&chan->vc); @@ -258,24 +472,43 @@ static void pt_issue_pending(struct dma_chan *dma_chan) pt_cmd_callback(desc, 0); } +static void pt_check_status_trans_ae4(struct pt_device *pt, struct pt_cmd_queue *cmd_q) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + int i; + + for (i = 0; i < CMD_Q_LEN; i++) + ae4_check_status_error(ae4cmd_q, i); +} + static enum dma_status pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate) { - struct pt_device *pt = to_pt_chan(c)->pt; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; + struct pt_dma_chan *chan = to_pt_chan(c); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + + cmd_q = pt_get_cmd_queue(pt, chan); + + if (pt->ver == AE4_DMA_VERSION) + pt_check_status_trans_ae4(pt, cmd_q); + else + pt_check_status_trans(pt, cmd_q); - pt_check_status_trans(pt, cmd_q); return dma_cookie_status(c, cookie, txstate); } static int pt_pause(struct dma_chan *dma_chan) { struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); - pt_stop_queue(&chan->pt->cmd_q); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_stop_queue(cmd_q); spin_unlock_irqrestore(&chan->vc.lock, flags); return 0; @@ -285,10 +518,13 @@ static int pt_resume(struct dma_chan *dma_chan) { struct pt_dma_chan *chan = to_pt_chan(dma_chan); struct pt_dma_desc *desc = NULL; + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); - pt_start_queue(&chan->pt->cmd_q); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_start_queue(cmd_q); desc = pt_next_dma_desc(chan); spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -302,11 +538,17 @@ static int pt_resume(struct dma_chan *dma_chan) static int pt_terminate_all(struct dma_chan *dma_chan) { struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; unsigned long flags; - struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q; LIST_HEAD(head); - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + cmd_q = pt_get_cmd_queue(pt, chan); + if (pt->ver == AE4_DMA_VERSION) + pt_stop_queue(cmd_q); + else + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + spin_lock_irqsave(&chan->vc.lock, flags); vchan_get_all_descriptors(&chan->vc, &head); spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -319,38 +561,37 @@ static int pt_terminate_all(struct dma_chan *dma_chan) int pt_dmaengine_register(struct pt_device *pt) { - struct pt_dma_chan *chan; struct dma_device *dma_dev = &pt->dma_dev; - char *cmd_cache_name; + struct ae4_cmd_queue *ae4cmd_q = NULL; + struct ae4_device *ae4 = NULL; + struct pt_dma_chan *chan; char *desc_cache_name; - int ret; + int ret, i; - pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), - GFP_KERNEL); - if (!pt->pt_dma_chan) - return -ENOMEM; + if (pt->ver == AE4_DMA_VERSION) + ae4 = container_of(pt, struct ae4_device, pt); + + if (ae4) + pt->pt_dma_chan = devm_kcalloc(pt->dev, ae4->cmd_q_count, + sizeof(*pt->pt_dma_chan), GFP_KERNEL); + else + pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), + GFP_KERNEL); - cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, - "%s-dmaengine-cmd-cache", - dev_name(pt->dev)); - if (!cmd_cache_name) + if (!pt->pt_dma_chan) return -ENOMEM; desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, "%s-dmaengine-desc-cache", dev_name(pt->dev)); - if (!desc_cache_name) { - ret = -ENOMEM; - goto err_cache; - } + if (!desc_cache_name) + return -ENOMEM; pt->dma_desc_cache = kmem_cache_create(desc_cache_name, sizeof(struct pt_dma_desc), 0, SLAB_HWCACHE_ALIGN, NULL); - if (!pt->dma_desc_cache) { - ret = -ENOMEM; - goto err_cache; - } + if (!pt->dma_desc_cache) + return -ENOMEM; dma_dev->dev = pt->dev; dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; @@ -368,9 +609,6 @@ int pt_dmaengine_register(struct pt_device *pt) INIT_LIST_HEAD(&dma_dev->channels); - chan = pt->pt_dma_chan; - chan->pt = pt; - /* Set base and prep routines */ dma_dev->device_free_chan_resources = pt_free_chan_resources; dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; @@ -382,10 +620,21 @@ int pt_dmaengine_register(struct pt_device *pt) dma_dev->device_terminate_all = pt_terminate_all; dma_dev->device_synchronize = pt_synchronize; - chan->vc.desc_free = pt_do_cleanup; - vchan_init(&chan->vc, dma_dev); - - dma_set_mask_and_coherent(pt->dev, DMA_BIT_MASK(64)); + if (ae4) { + for (i = 0; i < ae4->cmd_q_count; i++) { + chan = pt->pt_dma_chan + i; + ae4cmd_q = &ae4->ae4cmd_q[i]; + chan->id = ae4cmd_q->id; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } + } else { + chan = pt->pt_dma_chan; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } ret = dma_async_device_register(dma_dev); if (ret) @@ -396,11 +645,9 @@ int pt_dmaengine_register(struct pt_device *pt) err_reg: kmem_cache_destroy(pt->dma_desc_cache); -err_cache: - kmem_cache_destroy(pt->dma_cmd_cache); - return ret; } +EXPORT_SYMBOL_GPL(pt_dmaengine_register); void pt_dmaengine_unregister(struct pt_device *pt) { @@ -409,5 +656,4 @@ void pt_dmaengine_unregister(struct pt_device *pt) dma_async_device_unregister(dma_dev); kmem_cache_destroy(pt->dma_desc_cache); - kmem_cache_destroy(pt->dma_cmd_cache); } diff --git a/drivers/dma/ptdma/ptdma-pci.c b/drivers/dma/amd/ptdma/ptdma-pci.c index 22739ff0c3c5..22739ff0c3c5 100644 --- a/drivers/dma/ptdma/ptdma-pci.c +++ b/drivers/dma/amd/ptdma/ptdma-pci.c diff --git a/drivers/dma/ptdma/ptdma.h b/drivers/dma/amd/ptdma/ptdma.h index 21b4bf895200..ef3f55632107 100644 --- a/drivers/dma/ptdma/ptdma.h +++ b/drivers/dma/amd/ptdma/ptdma.h @@ -22,7 +22,7 @@ #include <linux/wait.h> #include <linux/dmapool.h> -#include "../virt-dma.h" +#include "../../virt-dma.h" #define MAX_PT_NAME_LEN 16 #define MAX_DMAPOOL_NAME_LEN 32 @@ -184,6 +184,7 @@ struct pt_dma_desc { struct pt_dma_chan { struct virt_dma_chan vc; struct pt_device *pt; + u32 id; }; struct pt_cmd_queue { @@ -192,7 +193,7 @@ struct pt_cmd_queue { /* Queue dma pool */ struct dma_pool *dma_pool; - /* Queue base address (not neccessarily aligned)*/ + /* Queue base address (not necessarily aligned)*/ struct ptdma_desc *qbase; /* Aligned queue start address (per requirement) */ @@ -253,7 +254,6 @@ struct pt_device { /* Support for the DMA Engine capabilities */ struct dma_device dma_dev; struct pt_dma_chan *pt_dma_chan; - struct kmem_cache *dma_cmd_cache; struct kmem_cache *dma_desc_cache; wait_queue_head_t lsb_queue; @@ -262,6 +262,7 @@ struct pt_device { unsigned long total_interrupts; struct pt_tasklet_data tdata; + int ver; }; /* diff --git a/drivers/dma/amd/qdma/Makefile b/drivers/dma/amd/qdma/Makefile new file mode 100644 index 000000000000..011268fef377 --- /dev/null +++ b/drivers/dma/amd/qdma/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_AMD_QDMA) += amd-qdma.o + +amd-qdma-$(CONFIG_AMD_QDMA) := qdma.o qdma-comm-regs.o diff --git a/drivers/dma/amd/qdma/qdma-comm-regs.c b/drivers/dma/amd/qdma/qdma-comm-regs.c new file mode 100644 index 000000000000..9162f9d367cc --- /dev/null +++ b/drivers/dma/amd/qdma/qdma-comm-regs.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef __QDMA_REGS_DEF_H +#define __QDMA_REGS_DEF_H + +#include "qdma.h" + +const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX] = { + [QDMA_REGO_CTXT_DATA] = QDMA_REGO(0x804, 8), + [QDMA_REGO_CTXT_CMD] = QDMA_REGO(0x844, 1), + [QDMA_REGO_CTXT_MASK] = QDMA_REGO(0x824, 8), + [QDMA_REGO_MM_H2C_CTRL] = QDMA_REGO(0x1004, 1), + [QDMA_REGO_MM_C2H_CTRL] = QDMA_REGO(0x1204, 1), + [QDMA_REGO_QUEUE_COUNT] = QDMA_REGO(0x120, 1), + [QDMA_REGO_RING_SIZE] = QDMA_REGO(0x204, 1), + [QDMA_REGO_H2C_PIDX] = QDMA_REGO(0x18004, 1), + [QDMA_REGO_C2H_PIDX] = QDMA_REGO(0x18008, 1), + [QDMA_REGO_INTR_CIDX] = QDMA_REGO(0x18000, 1), + [QDMA_REGO_FUNC_ID] = QDMA_REGO(0x12c, 1), + [QDMA_REGO_ERR_INT] = QDMA_REGO(0xb04, 1), + [QDMA_REGO_ERR_STAT] = QDMA_REGO(0x248, 1), +}; + +const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX] = { + /* QDMA_REGO_CTXT_DATA fields */ + [QDMA_REGF_IRQ_ENABLE] = QDMA_REGF(53, 53), + [QDMA_REGF_WBK_ENABLE] = QDMA_REGF(52, 52), + [QDMA_REGF_WBI_CHECK] = QDMA_REGF(34, 34), + [QDMA_REGF_IRQ_ARM] = QDMA_REGF(16, 16), + [QDMA_REGF_IRQ_VEC] = QDMA_REGF(138, 128), + [QDMA_REGF_IRQ_AGG] = QDMA_REGF(139, 139), + [QDMA_REGF_WBI_INTVL_ENABLE] = QDMA_REGF(35, 35), + [QDMA_REGF_MRKR_DISABLE] = QDMA_REGF(62, 62), + [QDMA_REGF_QUEUE_ENABLE] = QDMA_REGF(32, 32), + [QDMA_REGF_QUEUE_MODE] = QDMA_REGF(63, 63), + [QDMA_REGF_DESC_BASE] = QDMA_REGF(127, 64), + [QDMA_REGF_DESC_SIZE] = QDMA_REGF(49, 48), + [QDMA_REGF_RING_ID] = QDMA_REGF(47, 44), + [QDMA_REGF_QUEUE_BASE] = QDMA_REGF(11, 0), + [QDMA_REGF_QUEUE_MAX] = QDMA_REGF(44, 32), + [QDMA_REGF_FUNCTION_ID] = QDMA_REGF(24, 17), + [QDMA_REGF_INTR_AGG_BASE] = QDMA_REGF(66, 15), + [QDMA_REGF_INTR_VECTOR] = QDMA_REGF(11, 1), + [QDMA_REGF_INTR_SIZE] = QDMA_REGF(69, 67), + [QDMA_REGF_INTR_VALID] = QDMA_REGF(0, 0), + [QDMA_REGF_INTR_COLOR] = QDMA_REGF(14, 14), + [QDMA_REGF_INTR_FUNCTION_ID] = QDMA_REGF(125, 114), + /* QDMA_REGO_CTXT_CMD fields */ + [QDMA_REGF_CMD_INDX] = QDMA_REGF(19, 7), + [QDMA_REGF_CMD_CMD] = QDMA_REGF(6, 5), + [QDMA_REGF_CMD_TYPE] = QDMA_REGF(4, 1), + [QDMA_REGF_CMD_BUSY] = QDMA_REGF(0, 0), + /* QDMA_REGO_QUEUE_COUNT fields */ + [QDMA_REGF_QUEUE_COUNT] = QDMA_REGF(11, 0), + /* QDMA_REGO_ERR_INT fields */ + [QDMA_REGF_ERR_INT_FUNC] = QDMA_REGF(11, 0), + [QDMA_REGF_ERR_INT_VEC] = QDMA_REGF(22, 12), + [QDMA_REGF_ERR_INT_ARM] = QDMA_REGF(24, 24), +}; + +#endif /* __QDMA_REGS_DEF_H */ diff --git a/drivers/dma/amd/qdma/qdma.c b/drivers/dma/amd/qdma/qdma.c new file mode 100644 index 000000000000..8fb2d5e1df20 --- /dev/null +++ b/drivers/dma/amd/qdma/qdma.c @@ -0,0 +1,1143 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DMA driver for AMD Queue-based DMA Subsystem + * + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <linux/platform_data/amd_qdma.h> +#include <linux/regmap.h> + +#include "qdma.h" + +#define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H") +#define QDMA_REG_OFF(d, r) ((d)->roffs[r].off) + +/* MMIO regmap config for all QDMA registers */ +static const struct regmap_config qdma_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan) +{ + return container_of(chan, struct qdma_queue, vchan.chan); +} + +static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct qdma_mm_vdesc, vdesc); +} + +static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev) +{ + u32 idx; + + idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx; + qdev->qintr_ring_idx %= qdev->qintr_ring_num; + + return idx; +} + +static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data, + enum qdma_reg_fields field) +{ + const struct qdma_reg_field *f = &qdev->rfields[field]; + u16 low_pos, hi_pos, low_bit, hi_bit; + u64 value = 0, mask; + + low_pos = f->lsb / BITS_PER_TYPE(*data); + hi_pos = f->msb / BITS_PER_TYPE(*data); + + if (low_pos == hi_pos) { + low_bit = f->lsb % BITS_PER_TYPE(*data); + hi_bit = f->msb % BITS_PER_TYPE(*data); + mask = GENMASK(hi_bit, low_bit); + value = (data[low_pos] & mask) >> low_bit; + } else if (hi_pos == low_pos + 1) { + low_bit = f->lsb % BITS_PER_TYPE(*data); + hi_bit = low_bit + (f->msb - f->lsb); + value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) | + data[low_pos]; + mask = GENMASK_ULL(hi_bit, low_bit); + value = (value & mask) >> low_bit; + } else { + hi_bit = f->msb % BITS_PER_TYPE(*data); + mask = GENMASK(hi_bit, 0); + value = data[hi_pos] & mask; + low_bit = f->msb - f->lsb - hi_bit; + value <<= low_bit; + low_bit -= 32; + value |= (u64)data[hi_pos - 1] << low_bit; + mask = GENMASK(31, 32 - low_bit); + value |= (data[hi_pos - 2] & mask) >> low_bit; + } + + return value; +} + +static void qdma_set_field(const struct qdma_device *qdev, u32 *data, + enum qdma_reg_fields field, u64 value) +{ + const struct qdma_reg_field *f = &qdev->rfields[field]; + u16 low_pos, hi_pos, low_bit; + + low_pos = f->lsb / BITS_PER_TYPE(*data); + hi_pos = f->msb / BITS_PER_TYPE(*data); + low_bit = f->lsb % BITS_PER_TYPE(*data); + + data[low_pos++] |= value << low_bit; + if (low_pos <= hi_pos) + data[low_pos++] |= (u32)(value >> (32 - low_bit)); + if (low_pos <= hi_pos) + data[low_pos] |= (u32)(value >> (64 - low_bit)); +} + +static inline int qdma_reg_write(const struct qdma_device *qdev, + const u32 *data, enum qdma_regs reg) +{ + const struct qdma_reg *r = &qdev->roffs[reg]; + int ret; + + if (r->count > 1) + ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count); + else + ret = regmap_write(qdev->regmap, r->off, *data); + + return ret; +} + +static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data, + enum qdma_regs reg) +{ + const struct qdma_reg *r = &qdev->roffs[reg]; + int ret; + + if (r->count > 1) + ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count); + else + ret = regmap_read(qdev->regmap, r->off, data); + + return ret; +} + +static int qdma_context_cmd_execute(const struct qdma_device *qdev, + enum qdma_ctxt_type type, + enum qdma_ctxt_cmd cmd, u16 index) +{ + u32 value = 0; + int ret; + + qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index); + qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd); + qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type); + + ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(qdev->regmap, + QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD), + value, + !qdma_get_field(qdev, &value, + QDMA_REGF_CMD_BUSY), + QDMA_POLL_INTRVL_US, + QDMA_POLL_TIMEOUT_US); + if (ret) { + qdma_err(qdev, "Context command execution timed out"); + return ret; + } + + return 0; +} + +static int qdma_context_write_data(const struct qdma_device *qdev, + const u32 *data) +{ + u32 mask[QDMA_CTXT_REGMAP_LEN]; + int ret; + + memset(mask, ~0, sizeof(mask)); + + ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK); + if (ret) + return ret; + + ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA); + if (ret) + return ret; + + return 0; +} + +static void qdma_prep_sw_desc_context(const struct qdma_device *qdev, + const struct qdma_ctxt_sw_desc *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec); + qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid); + + qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B); + qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1); +} + +static void qdma_prep_intr_context(const struct qdma_device *qdev, + const struct qdma_ctxt_intr *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base); + qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec); + qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size); + qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid); + qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color); + qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid); +} + +static void qdma_prep_fmap_context(const struct qdma_device *qdev, + const struct qdma_ctxt_fmap *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax); +} + +/* + * Program the indirect context register space + * + * Once the queue is enabled, context is dynamically updated by hardware. Any + * modification of the context through this API when the queue is enabled can + * result in unexpected behavior. Reading the context when the queue is enabled + * is not recommended as it can result in reduced performance. + */ +static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type, + enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt) +{ + int ret; + + mutex_lock(&qdev->ctxt_lock); + if (cmd == QDMA_CTXT_WRITE) { + ret = qdma_context_write_data(qdev, ctxt); + if (ret) + goto failed; + } + + ret = qdma_context_cmd_execute(qdev, type, cmd, index); + if (ret) + goto failed; + + if (cmd == QDMA_CTXT_READ) { + ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA); + if (ret) + goto failed; + } + +failed: + mutex_unlock(&qdev->ctxt_lock); + + return ret; +} + +static int qdma_check_queue_status(struct qdma_device *qdev, + enum dma_transfer_direction dir, u16 qid) +{ + u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0}; + enum qdma_ctxt_type type; + int ret; + + if (dir == DMA_MEM_TO_DEV) + type = QDMA_CTXT_DESC_SW_H2C; + else + type = QDMA_CTXT_DESC_SW_C2H; + ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data); + if (ret) + return ret; + + status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE); + if (status) { + qdma_err(qdev, "queue %d already in use", qid); + return -EBUSY; + } + + return 0; +} + +static int qdma_clear_queue_context(const struct qdma_queue *queue) +{ + static const enum qdma_ctxt_type h2c_types[] = { + QDMA_CTXT_DESC_SW_H2C, + QDMA_CTXT_DESC_HW_H2C, + QDMA_CTXT_DESC_CR_H2C, + QDMA_CTXT_PFTCH, + }; + static const enum qdma_ctxt_type c2h_types[] = { + QDMA_CTXT_DESC_SW_C2H, + QDMA_CTXT_DESC_HW_C2H, + QDMA_CTXT_DESC_CR_C2H, + QDMA_CTXT_PFTCH, + }; + struct qdma_device *qdev = queue->qdev; + const enum qdma_ctxt_type *type; + int ret, num, i; + + if (queue->dir == DMA_MEM_TO_DEV) { + type = h2c_types; + num = ARRAY_SIZE(h2c_types); + } else { + type = c2h_types; + num = ARRAY_SIZE(c2h_types); + } + for (i = 0; i < num; i++) { + ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR, + queue->qid, NULL); + if (ret) { + qdma_err(qdev, "Failed to clear ctxt %d", type[i]); + return ret; + } + } + + return 0; +} + +static int qdma_setup_fmap_context(struct qdma_device *qdev) +{ + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + struct qdma_ctxt_fmap fmap; + int ret; + + ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR, + qdev->fid, NULL); + if (ret) { + qdma_err(qdev, "Failed clearing context"); + return ret; + } + + fmap.qbase = 0; + fmap.qmax = qdev->chan_num * 2; + qdma_prep_fmap_context(qdev, &fmap, ctxt); + ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE, + qdev->fid, ctxt); + if (ret) + qdma_err(qdev, "Failed setup fmap, ret %d", ret); + + return ret; +} + +static int qdma_setup_queue_context(struct qdma_device *qdev, + const struct qdma_ctxt_sw_desc *sw_desc, + enum dma_transfer_direction dir, u16 qid) +{ + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + enum qdma_ctxt_type type; + int ret; + + if (dir == DMA_MEM_TO_DEV) + type = QDMA_CTXT_DESC_SW_H2C; + else + type = QDMA_CTXT_DESC_SW_C2H; + + qdma_prep_sw_desc_context(qdev, sw_desc, ctxt); + /* Setup SW descriptor context */ + ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt); + if (ret) + qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid); + + return ret; +} + +/* + * Enable or disable memory-mapped DMA engines + * 1: enable, 0: disable + */ +static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl) +{ + int ret; + + ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL); + ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL); + + return ret; +} + +static int qdma_get_hw_info(struct qdma_device *qdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); + u32 value = 0; + int ret; + + ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT); + if (ret) + return ret; + + value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1; + if (pdata->max_mm_channels * 2 > value) { + qdma_err(qdev, "not enough hw queues %d", value); + return -EINVAL; + } + qdev->chan_num = pdata->max_mm_channels; + + ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID); + if (ret) + return ret; + + qdma_info(qdev, "max channel %d, function id %d", + qdev->chan_num, qdev->fid); + + return 0; +} + +static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx) +{ + struct qdma_device *qdev = queue->qdev; + + return regmap_write(qdev->regmap, queue->pidx_reg, + pidx | QDMA_QUEUE_ARM_BIT); +} + +static inline int qdma_update_cidx(const struct qdma_queue *queue, + u16 ridx, u16 cidx) +{ + struct qdma_device *qdev = queue->qdev; + + return regmap_write(qdev->regmap, queue->cidx_reg, + ((u32)ridx << 16) | cidx); +} + +/** + * qdma_free_vdesc - Free descriptor + * @vdesc: Virtual DMA descriptor + */ +static void qdma_free_vdesc(struct virt_dma_desc *vdesc) +{ + struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc); + + kfree(vd); +} + +static int qdma_alloc_queues(struct qdma_device *qdev, + enum dma_transfer_direction dir) +{ + struct qdma_queue *q, **queues; + u32 i, pidx_base; + int ret; + + if (dir == DMA_MEM_TO_DEV) { + queues = &qdev->h2c_queues; + pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX); + } else { + queues = &qdev->c2h_queues; + pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX); + } + + *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q), + GFP_KERNEL); + if (!*queues) + return -ENOMEM; + + for (i = 0; i < qdev->chan_num; i++) { + ret = qdma_check_queue_status(qdev, dir, i); + if (ret) + return ret; + + q = &(*queues)[i]; + q->ring_size = QDMA_DEFAULT_RING_SIZE; + q->idx_mask = q->ring_size - 2; + q->qdev = qdev; + q->dir = dir; + q->qid = i; + q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE; + q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) + + i * QDMA_DMAP_REG_STRIDE; + q->vchan.desc_free = qdma_free_vdesc; + vchan_init(&q->vchan, &qdev->dma_dev); + } + + return 0; +} + +static int qdma_device_verify(struct qdma_device *qdev) +{ + u32 value; + int ret; + + ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value); + if (ret) + return ret; + + value = FIELD_GET(QDMA_IDENTIFIER_MASK, value); + if (value != QDMA_IDENTIFIER) { + qdma_err(qdev, "Invalid identifier"); + return -ENODEV; + } + qdev->rfields = qdma_regfs_default; + qdev->roffs = qdma_regos_default; + + return 0; +} + +static int qdma_device_setup(struct qdma_device *qdev) +{ + u32 ring_sz = QDMA_DEFAULT_RING_SIZE; + int ret = 0; + + ret = qdma_setup_fmap_context(qdev); + if (ret) { + qdma_err(qdev, "Failed setup fmap context"); + return ret; + } + + /* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */ + ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE); + if (ret) { + qdma_err(qdev, "Failed to setup ring %d of size %ld", + QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE); + return ret; + } + + /* Enable memory-mapped DMA engine in both directions */ + ret = qdma_sgdma_control(qdev, 1); + if (ret) { + qdma_err(qdev, "Failed to SGDMA with error %d", ret); + return ret; + } + + ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV); + if (ret) { + qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret); + return ret; + } + + ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM); + if (ret) { + qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret); + return ret; + } + + return 0; +} + +/** + * qdma_free_queue_resources() - Free queue resources + * @chan: DMA channel + */ +static void qdma_free_queue_resources(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_device *qdev = queue->qdev; + struct qdma_platdata *pdata; + + qdma_clear_queue_context(queue); + vchan_free_chan_resources(&queue->vchan); + pdata = dev_get_platdata(&qdev->pdev->dev); + dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE, + queue->desc_base, queue->dma_desc_base); +} + +/** + * qdma_alloc_queue_resources() - Allocate queue resources + * @chan: DMA channel + */ +static int qdma_alloc_queue_resources(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_device *qdev = queue->qdev; + struct qdma_ctxt_sw_desc desc; + struct qdma_platdata *pdata; + size_t size; + int ret; + + ret = qdma_clear_queue_context(queue); + if (ret) + return ret; + + pdata = dev_get_platdata(&qdev->pdev->dev); + size = queue->ring_size * QDMA_MM_DESC_SIZE; + queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size, + &queue->dma_desc_base, + GFP_KERNEL); + if (!queue->desc_base) { + qdma_err(qdev, "Failed to allocate descriptor ring"); + return -ENOMEM; + } + + /* Setup SW descriptor queue context for DMA memory map */ + desc.vec = qdma_get_intr_ring_idx(qdev); + desc.desc_base = queue->dma_desc_base; + ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid); + if (ret) { + qdma_err(qdev, "Failed to setup SW desc ctxt for %s", + chan->name); + dma_free_coherent(pdata->dma_dev, size, queue->desc_base, + queue->dma_desc_base); + return ret; + } + + queue->pidx = 0; + queue->cidx = 0; + + return 0; +} + +static bool qdma_filter_fn(struct dma_chan *chan, void *param) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_queue_info *info = param; + + return info->dir == queue->dir; +} + +static int qdma_xfer_start(struct qdma_queue *queue) +{ + struct qdma_device *qdev = queue->qdev; + int ret; + + if (!vchan_next_desc(&queue->vchan)) + return 0; + + qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d", + queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid); + + ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx); + if (ret) { + qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d", + queue->pidx, CHAN_STR(queue), queue->qid); + } + + return ret; +} + +static void qdma_issue_pending(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + unsigned long flags; + + spin_lock_irqsave(&queue->vchan.lock, flags); + if (vchan_issue_pending(&queue->vchan)) { + if (queue->submitted_vdesc) { + queue->issued_vdesc = queue->submitted_vdesc; + queue->submitted_vdesc = NULL; + } + qdma_xfer_start(queue); + } + + spin_unlock_irqrestore(&queue->vchan.lock, flags); +} + +static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q) +{ + struct qdma_mm_desc *desc; + + if (((q->pidx + 1) & q->idx_mask) == q->cidx) + return NULL; + + desc = q->desc_base + q->pidx; + q->pidx = (q->pidx + 1) & q->idx_mask; + + return desc; +} + +static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc) +{ + struct qdma_mm_desc *desc; + struct scatterlist *sg; + u64 addr, *src, *dst; + u32 rest, len; + int ret = 0; + u32 i; + + if (!vdesc->sg_len) + return 0; + + if (q->dir == DMA_MEM_TO_DEV) { + dst = &vdesc->dev_addr; + src = &addr; + } else { + dst = &addr; + src = &vdesc->dev_addr; + } + + for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) { + addr = sg_dma_address(sg) + vdesc->sg_off; + rest = sg_dma_len(sg) - vdesc->sg_off; + while (rest) { + len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN); + desc = qdma_get_desc(q); + if (!desc) { + ret = -EBUSY; + goto out; + } + + desc->src_addr = cpu_to_le64(*src); + desc->dst_addr = cpu_to_le64(*dst); + desc->len = cpu_to_le32(len); + + vdesc->dev_addr += len; + vdesc->sg_off += len; + vdesc->pending_descs++; + addr += len; + rest -= len; + } + vdesc->sg_off = 0; + } +out: + vdesc->sg_len -= i; + vdesc->pidx = q->pidx; + return ret; +} + +static void qdma_fill_pending_vdesc(struct qdma_queue *q) +{ + struct virt_dma_chan *vc = &q->vchan; + struct qdma_mm_vdesc *vdesc = NULL; + struct virt_dma_desc *vd; + int ret; + + if (!list_empty(&vc->desc_issued)) { + vd = &q->issued_vdesc->vdesc; + list_for_each_entry_from(vd, &vc->desc_issued, node) { + vdesc = to_qdma_vdesc(vd); + ret = qdma_hw_enqueue(q, vdesc); + if (ret) { + q->issued_vdesc = vdesc; + return; + } + } + q->issued_vdesc = vdesc; + } + + if (list_empty(&vc->desc_submitted)) + return; + + if (q->submitted_vdesc) + vd = &q->submitted_vdesc->vdesc; + else + vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node); + + list_for_each_entry_from(vd, &vc->desc_submitted, node) { + vdesc = to_qdma_vdesc(vd); + ret = qdma_hw_enqueue(q, vdesc); + if (ret) + break; + } + q->submitted_vdesc = vdesc; +} + +static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct virt_dma_chan *vc = to_virt_chan(tx->chan); + struct qdma_queue *q = to_qdma_queue(&vc->chan); + struct virt_dma_desc *vd; + unsigned long flags; + dma_cookie_t cookie; + + vd = container_of(tx, struct virt_dma_desc, tx); + spin_lock_irqsave(&vc->lock, flags); + cookie = dma_cookie_assign(tx); + + list_move_tail(&vd->node, &vc->desc_submitted); + qdma_fill_pending_vdesc(q); + spin_unlock_irqrestore(&vc->lock, flags); + + return cookie; +} + +static struct dma_async_tx_descriptor * +qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct qdma_queue *q = to_qdma_queue(chan); + struct dma_async_tx_descriptor *tx; + struct qdma_mm_vdesc *vdesc; + + vdesc = kzalloc(sizeof(*vdesc), GFP_NOWAIT); + if (!vdesc) + return NULL; + vdesc->sgl = sgl; + vdesc->sg_len = sg_len; + if (dir == DMA_MEM_TO_DEV) + vdesc->dev_addr = q->cfg.dst_addr; + else + vdesc->dev_addr = q->cfg.src_addr; + + tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags); + tx->tx_submit = qdma_tx_submit; + + return tx; +} + +static int qdma_device_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct qdma_queue *q = to_qdma_queue(chan); + + memcpy(&q->cfg, cfg, sizeof(*cfg)); + + return 0; +} + +static int qdma_arm_err_intr(const struct qdma_device *qdev) +{ + u32 value = 0; + + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid); + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx); + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1); + + return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT); +} + +static irqreturn_t qdma_error_isr(int irq, void *data) +{ + struct qdma_device *qdev = data; + u32 err_stat = 0; + int ret; + + ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT); + if (ret) { + qdma_err(qdev, "read error state failed, ret %d", ret); + goto out; + } + + qdma_err(qdev, "global error %d", err_stat); + ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT); + if (ret) + qdma_err(qdev, "clear error state failed, ret %d", ret); + +out: + qdma_arm_err_intr(qdev); + return IRQ_HANDLED; +} + +static irqreturn_t qdma_queue_isr(int irq, void *data) +{ + struct qdma_intr_ring *intr = data; + struct qdma_queue *q = NULL; + struct qdma_device *qdev; + u32 index, comp_desc; + u64 intr_ent; + u8 color; + int ret; + u16 qid; + + qdev = intr->qdev; + index = intr->cidx; + while (1) { + struct virt_dma_desc *vd; + struct qdma_mm_vdesc *vdesc; + unsigned long flags; + u32 cidx; + + intr_ent = le64_to_cpu(intr->base[index]); + color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent); + if (color != intr->color) + break; + + qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent); + if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent)) + q = qdev->c2h_queues; + else + q = qdev->h2c_queues; + q += qid; + + cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent); + + spin_lock_irqsave(&q->vchan.lock, flags); + comp_desc = (cidx - q->cidx) & q->idx_mask; + + vd = vchan_next_desc(&q->vchan); + if (!vd) + goto skip; + + vdesc = to_qdma_vdesc(vd); + while (comp_desc > vdesc->pending_descs) { + list_del(&vd->node); + vchan_cookie_complete(vd); + comp_desc -= vdesc->pending_descs; + vd = vchan_next_desc(&q->vchan); + vdesc = to_qdma_vdesc(vd); + } + vdesc->pending_descs -= comp_desc; + if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + q->cidx = cidx; + + qdma_fill_pending_vdesc(q); + qdma_xfer_start(q); + +skip: + spin_unlock_irqrestore(&q->vchan.lock, flags); + + /* + * Wrap the index value and flip the expected color value if + * interrupt aggregation PIDX has wrapped around. + */ + index++; + index &= QDMA_INTR_RING_IDX_MASK; + if (!index) + intr->color = !intr->color; + } + + /* + * Update the software interrupt aggregation ring CIDX if a valid entry + * was found. + */ + if (q) { + qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index); + + /* + * Record the last read index of status descriptor from the + * interrupt aggregation ring. + */ + intr->cidx = index; + + ret = qdma_update_cidx(q, intr->ridx, index); + if (ret) { + qdma_err(qdev, "Failed to update IRQ CIDX"); + return IRQ_NONE; + } + } + + return IRQ_HANDLED; +} + +static int qdma_init_error_irq(struct qdma_device *qdev) +{ + struct device *dev = &qdev->pdev->dev; + int ret; + u32 vec; + + vec = qdev->queue_irq_start - 1; + + ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr, + IRQF_ONESHOT, "amd-qdma-error", qdev); + if (ret) { + qdma_err(qdev, "Failed to request error IRQ vector: %d", vec); + return ret; + } + + ret = qdma_arm_err_intr(qdev); + if (ret) + qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret); + + return ret; +} + +static int qdmam_alloc_qintr_rings(struct qdma_device *qdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); + struct device *dev = &qdev->pdev->dev; + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + struct qdma_intr_ring *ring; + struct qdma_ctxt_intr intr_ctxt; + u32 vector; + int ret, i; + + qdev->qintr_ring_num = qdev->queue_irq_num; + qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num, + sizeof(*qdev->qintr_rings), + GFP_KERNEL); + if (!qdev->qintr_rings) + return -ENOMEM; + + vector = qdev->queue_irq_start; + for (i = 0; i < qdev->qintr_ring_num; i++, vector++) { + ring = &qdev->qintr_rings[i]; + ring->qdev = qdev; + ring->msix_id = qdev->err_irq_idx + i + 1; + ring->ridx = i; + ring->color = 1; + ring->base = dmam_alloc_coherent(pdata->dma_dev, + QDMA_INTR_RING_SIZE, + &ring->dev_base, GFP_KERNEL); + if (!ring->base) { + qdma_err(qdev, "Failed to alloc intr ring %d", i); + return -ENOMEM; + } + intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base); + intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096; + intr_ctxt.vec = ring->msix_id; + intr_ctxt.valid = true; + intr_ctxt.color = true; + ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, + QDMA_CTXT_CLEAR, ring->ridx, NULL); + if (ret) { + qdma_err(qdev, "Failed clear intr ctx, ret %d", ret); + return ret; + } + + qdma_prep_intr_context(qdev, &intr_ctxt, ctxt); + ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, + QDMA_CTXT_WRITE, ring->ridx, ctxt); + if (ret) { + qdma_err(qdev, "Failed setup intr ctx, ret %d", ret); + return ret; + } + + ret = devm_request_threaded_irq(dev, vector, NULL, + qdma_queue_isr, IRQF_ONESHOT, + "amd-qdma-queue", ring); + if (ret) { + qdma_err(qdev, "Failed to request irq %d", vector); + return ret; + } + } + + return 0; +} + +static int qdma_intr_init(struct qdma_device *qdev) +{ + int ret; + + ret = qdma_init_error_irq(qdev); + if (ret) { + qdma_err(qdev, "Failed to init error IRQs, ret %d", ret); + return ret; + } + + ret = qdmam_alloc_qintr_rings(qdev); + if (ret) { + qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret); + return ret; + } + + return 0; +} + +static void amd_qdma_remove(struct platform_device *pdev) +{ + struct qdma_device *qdev = platform_get_drvdata(pdev); + + qdma_sgdma_control(qdev, 0); + dma_async_device_unregister(&qdev->dma_dev); + + mutex_destroy(&qdev->ctxt_lock); +} + +static int amd_qdma_probe(struct platform_device *pdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev); + struct qdma_device *qdev; + struct resource *res; + void __iomem *regs; + int ret; + + qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL); + if (!qdev) + return -ENOMEM; + + platform_set_drvdata(pdev, qdev); + qdev->pdev = pdev; + mutex_init(&qdev->ctxt_lock); + + res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res) { + qdma_err(qdev, "Failed to get IRQ resource"); + ret = -ENODEV; + goto failed; + } + qdev->err_irq_idx = pdata->irq_index; + qdev->queue_irq_start = res->start + 1; + qdev->queue_irq_num = resource_size(res) - 1; + + regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(regs)) { + ret = PTR_ERR(regs); + qdma_err(qdev, "Failed to map IO resource, err %d", ret); + goto failed; + } + + qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs, + &qdma_regmap_config); + if (IS_ERR(qdev->regmap)) { + ret = PTR_ERR(qdev->regmap); + qdma_err(qdev, "Regmap init failed, err %d", ret); + goto failed; + } + + ret = qdma_device_verify(qdev); + if (ret) + goto failed; + + ret = qdma_get_hw_info(qdev); + if (ret) + goto failed; + + INIT_LIST_HEAD(&qdev->dma_dev.channels); + + ret = qdma_device_setup(qdev); + if (ret) + goto failed; + + ret = qdma_intr_init(qdev); + if (ret) { + qdma_err(qdev, "Failed to initialize IRQs %d", ret); + goto failed_disable_engine; + } + + dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask); + dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask); + + qdev->dma_dev.dev = &pdev->dev; + qdev->dma_dev.filter.map = pdata->device_map; + qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2; + qdev->dma_dev.filter.fn = qdma_filter_fn; + qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources; + qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources; + qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg; + qdev->dma_dev.device_config = qdma_device_config; + qdev->dma_dev.device_issue_pending = qdma_issue_pending; + qdev->dma_dev.device_tx_status = dma_cookie_status; + qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + + ret = dma_async_device_register(&qdev->dma_dev); + if (ret) { + qdma_err(qdev, "Failed to register AMD QDMA: %d", ret); + goto failed_disable_engine; + } + + return 0; + +failed_disable_engine: + qdma_sgdma_control(qdev, 0); +failed: + mutex_destroy(&qdev->ctxt_lock); + qdma_err(qdev, "Failed to probe AMD QDMA driver"); + return ret; +} + +static struct platform_driver amd_qdma_driver = { + .driver = { + .name = "amd-qdma", + }, + .probe = amd_qdma_probe, + .remove = amd_qdma_remove, +}; + +module_platform_driver(amd_qdma_driver); + +MODULE_DESCRIPTION("AMD QDMA driver"); +MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/amd/qdma/qdma.h b/drivers/dma/amd/qdma/qdma.h new file mode 100644 index 000000000000..94089f1f0c11 --- /dev/null +++ b/drivers/dma/amd/qdma/qdma.h @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * DMA header for AMD Queue-based DMA Subsystem + * + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef __QDMA_H +#define __QDMA_H + +#include <linux/bitfield.h> +#include <linux/dmaengine.h> +#include <linux/kernel.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include "../../virt-dma.h" + +#define DISABLE 0 +#define ENABLE 1 + +#define QDMA_MIN_IRQ 3 +#define QDMA_INTR_NAME_MAX_LEN 30 +#define QDMA_INTR_PREFIX "amd-qdma" + +#define QDMA_IDENTIFIER 0x1FD3 +#define QDMA_DEFAULT_RING_SIZE (BIT(10) + 1) +#define QDMA_DEFAULT_RING_ID 0 +#define QDMA_POLL_INTRVL_US 10 /* 10us */ +#define QDMA_POLL_TIMEOUT_US (500 * 1000) /* 500ms */ +#define QDMA_DMAP_REG_STRIDE 16 +#define QDMA_CTXT_REGMAP_LEN 8 /* 8 regs */ +#define QDMA_MM_DESC_SIZE 32 /* Bytes */ +#define QDMA_MM_DESC_LEN_BITS 28 +#define QDMA_MM_DESC_MAX_LEN (BIT(QDMA_MM_DESC_LEN_BITS) - 1) +#define QDMA_MIN_DMA_ALLOC_SIZE 4096 +#define QDMA_INTR_RING_SIZE BIT(13) +#define QDMA_INTR_RING_IDX_MASK GENMASK(9, 0) +#define QDMA_INTR_RING_BASE(_addr) ((_addr) >> 12) + +#define QDMA_IDENTIFIER_REGOFF 0x0 +#define QDMA_IDENTIFIER_MASK GENMASK(31, 16) +#define QDMA_QUEUE_ARM_BIT BIT(16) + +#define qdma_err(qdev, fmt, args...) \ + dev_err(&(qdev)->pdev->dev, fmt, ##args) + +#define qdma_dbg(qdev, fmt, args...) \ + dev_dbg(&(qdev)->pdev->dev, fmt, ##args) + +#define qdma_info(qdev, fmt, args...) \ + dev_info(&(qdev)->pdev->dev, fmt, ##args) + +enum qdma_reg_fields { + QDMA_REGF_IRQ_ENABLE, + QDMA_REGF_WBK_ENABLE, + QDMA_REGF_WBI_CHECK, + QDMA_REGF_IRQ_ARM, + QDMA_REGF_IRQ_VEC, + QDMA_REGF_IRQ_AGG, + QDMA_REGF_WBI_INTVL_ENABLE, + QDMA_REGF_MRKR_DISABLE, + QDMA_REGF_QUEUE_ENABLE, + QDMA_REGF_QUEUE_MODE, + QDMA_REGF_DESC_BASE, + QDMA_REGF_DESC_SIZE, + QDMA_REGF_RING_ID, + QDMA_REGF_CMD_INDX, + QDMA_REGF_CMD_CMD, + QDMA_REGF_CMD_TYPE, + QDMA_REGF_CMD_BUSY, + QDMA_REGF_QUEUE_COUNT, + QDMA_REGF_QUEUE_MAX, + QDMA_REGF_QUEUE_BASE, + QDMA_REGF_FUNCTION_ID, + QDMA_REGF_INTR_AGG_BASE, + QDMA_REGF_INTR_VECTOR, + QDMA_REGF_INTR_SIZE, + QDMA_REGF_INTR_VALID, + QDMA_REGF_INTR_COLOR, + QDMA_REGF_INTR_FUNCTION_ID, + QDMA_REGF_ERR_INT_FUNC, + QDMA_REGF_ERR_INT_VEC, + QDMA_REGF_ERR_INT_ARM, + QDMA_REGF_MAX +}; + +enum qdma_regs { + QDMA_REGO_CTXT_DATA, + QDMA_REGO_CTXT_CMD, + QDMA_REGO_CTXT_MASK, + QDMA_REGO_MM_H2C_CTRL, + QDMA_REGO_MM_C2H_CTRL, + QDMA_REGO_QUEUE_COUNT, + QDMA_REGO_RING_SIZE, + QDMA_REGO_H2C_PIDX, + QDMA_REGO_C2H_PIDX, + QDMA_REGO_INTR_CIDX, + QDMA_REGO_FUNC_ID, + QDMA_REGO_ERR_INT, + QDMA_REGO_ERR_STAT, + QDMA_REGO_MAX +}; + +struct qdma_reg_field { + u16 lsb; /* Least significant bit of field */ + u16 msb; /* Most significant bit of field */ +}; + +struct qdma_reg { + u32 off; + u32 count; +}; + +#define QDMA_REGF(_msb, _lsb) { \ + .lsb = (_lsb), \ + .msb = (_msb), \ +} + +#define QDMA_REGO(_off, _count) { \ + .off = (_off), \ + .count = (_count), \ +} + +enum qdma_desc_size { + QDMA_DESC_SIZE_8B, + QDMA_DESC_SIZE_16B, + QDMA_DESC_SIZE_32B, + QDMA_DESC_SIZE_64B, +}; + +enum qdma_queue_op_mode { + QDMA_QUEUE_OP_STREAM, + QDMA_QUEUE_OP_MM, +}; + +enum qdma_ctxt_type { + QDMA_CTXT_DESC_SW_C2H, + QDMA_CTXT_DESC_SW_H2C, + QDMA_CTXT_DESC_HW_C2H, + QDMA_CTXT_DESC_HW_H2C, + QDMA_CTXT_DESC_CR_C2H, + QDMA_CTXT_DESC_CR_H2C, + QDMA_CTXT_WRB, + QDMA_CTXT_PFTCH, + QDMA_CTXT_INTR_COAL, + QDMA_CTXT_RSVD, + QDMA_CTXT_HOST_PROFILE, + QDMA_CTXT_TIMER, + QDMA_CTXT_FMAP, + QDMA_CTXT_FNC_STS, +}; + +enum qdma_ctxt_cmd { + QDMA_CTXT_CLEAR, + QDMA_CTXT_WRITE, + QDMA_CTXT_READ, + QDMA_CTXT_INVALIDATE, + QDMA_CTXT_MAX +}; + +struct qdma_ctxt_sw_desc { + u64 desc_base; + u16 vec; +}; + +struct qdma_ctxt_intr { + u64 agg_base; + u16 vec; + u32 size; + bool valid; + bool color; +}; + +struct qdma_ctxt_fmap { + u16 qbase; + u16 qmax; +}; + +struct qdma_device; + +struct qdma_mm_desc { + __le64 src_addr; + __le32 len; + __le32 reserved1; + __le64 dst_addr; + __le64 reserved2; +} __packed; + +struct qdma_mm_vdesc { + struct virt_dma_desc vdesc; + struct qdma_queue *queue; + struct scatterlist *sgl; + u64 sg_off; + u32 sg_len; + u64 dev_addr; + u32 pidx; + u32 pending_descs; + struct dma_slave_config cfg; +}; + +#define QDMA_VDESC_QUEUED(vdesc) (!(vdesc)->sg_len) + +struct qdma_queue { + struct qdma_device *qdev; + struct virt_dma_chan vchan; + enum dma_transfer_direction dir; + struct dma_slave_config cfg; + struct qdma_mm_desc *desc_base; + struct qdma_mm_vdesc *submitted_vdesc; + struct qdma_mm_vdesc *issued_vdesc; + dma_addr_t dma_desc_base; + u32 pidx_reg; + u32 cidx_reg; + u32 ring_size; + u32 idx_mask; + u16 qid; + u32 pidx; + u32 cidx; +}; + +struct qdma_intr_ring { + struct qdma_device *qdev; + __le64 *base; + dma_addr_t dev_base; + char msix_name[QDMA_INTR_NAME_MAX_LEN]; + u32 msix_vector; + u16 msix_id; + u32 ring_size; + u16 ridx; + u16 cidx; + u8 color; +}; + +#define QDMA_INTR_MASK_PIDX GENMASK_ULL(15, 0) +#define QDMA_INTR_MASK_CIDX GENMASK_ULL(31, 16) +#define QDMA_INTR_MASK_DESC_COLOR GENMASK_ULL(32, 32) +#define QDMA_INTR_MASK_STATE GENMASK_ULL(34, 33) +#define QDMA_INTR_MASK_ERROR GENMASK_ULL(36, 35) +#define QDMA_INTR_MASK_TYPE GENMASK_ULL(38, 38) +#define QDMA_INTR_MASK_QID GENMASK_ULL(62, 39) +#define QDMA_INTR_MASK_COLOR GENMASK_ULL(63, 63) + +struct qdma_device { + struct platform_device *pdev; + struct dma_device dma_dev; + struct regmap *regmap; + struct mutex ctxt_lock; /* protect ctxt registers */ + const struct qdma_reg_field *rfields; + const struct qdma_reg *roffs; + struct qdma_queue *h2c_queues; + struct qdma_queue *c2h_queues; + struct qdma_intr_ring *qintr_rings; + u32 qintr_ring_num; + u32 qintr_ring_idx; + u32 chan_num; + u32 queue_irq_start; + u32 queue_irq_num; + u32 err_irq_idx; + u32 fid; +}; + +extern const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX]; +extern const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX]; + +#endif /* __QDMA_H */ diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index 4cf8da77bdd9..bd49f0374291 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -10,8 +10,9 @@ #include <linux/device.h> #include <linux/init.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/platform_device.h> #include <linux/reset.h> #include <linux/spinlock.h> #include <linux/interrupt.h> @@ -56,6 +57,8 @@ #define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) +#define BUS_WIDTH_WORD_SIZE GENMASK(3, 0) +#define BUS_WIDTH_FRAME_SIZE GENMASK(7, 4) #define BUS_WIDTH_8BIT 0x00 #define BUS_WIDTH_16BIT 0x01 #define BUS_WIDTH_32BIT 0x02 @@ -127,7 +130,7 @@ struct admac_data { int irq; int irq_index; int nchannels; - struct admac_chan channels[]; + struct admac_chan channels[] __counted_by(nchannels); }; struct admac_tx { @@ -150,6 +153,8 @@ static int admac_alloc_sram_carveout(struct admac_data *ad, { struct admac_sram *sram; int i, ret = 0, nblocks; + ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); + ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); if (dir == DMA_MEM_TO_DEV) sram = &ad->txcache; @@ -739,7 +744,8 @@ static int admac_device_config(struct dma_chan *chan, struct admac_data *ad = adchan->host; bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV; int wordsize = 0; - u32 bus_width = 0; + u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) & + ~(BUS_WIDTH_WORD_SIZE | BUS_WIDTH_FRAME_SIZE); switch (is_tx ? config->dst_addr_width : config->src_addr_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: @@ -908,12 +914,7 @@ static int admac_probe(struct platform_device *pdev) goto free_irq; } - ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); - ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); - dev_info(&pdev->dev, "Audio DMA Controller\n"); - dev_info(&pdev->dev, "imprint %x TX cache %u RX cache %u\n", - readl_relaxed(ad->base + REG_IMPRINT), ad->txcache.size, ad->rxcache.size); return 0; @@ -924,7 +925,7 @@ free_reset: return err; } -static int admac_remove(struct platform_device *pdev) +static void admac_remove(struct platform_device *pdev) { struct admac_data *ad = platform_get_drvdata(pdev); @@ -932,8 +933,6 @@ static int admac_remove(struct platform_device *pdev) dma_async_device_unregister(&ad->dma); free_irq(ad->irq, ad); reset_control_rearm(ad->rstc); - - return 0; } static const struct of_device_id admac_of_match[] = { diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c new file mode 100644 index 000000000000..9efe2ca7d5ec --- /dev/null +++ b/drivers/dma/arm-dma350.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2024-2025 Arm Limited +// Arm DMA-350 driver + +#include <linux/bitfield.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/of.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +#define DMAINFO 0x0f00 + +#define DMA_BUILDCFG0 0xb0 +#define DMA_CFG_DATA_WIDTH GENMASK(18, 16) +#define DMA_CFG_ADDR_WIDTH GENMASK(15, 10) +#define DMA_CFG_NUM_CHANNELS GENMASK(9, 4) + +#define DMA_BUILDCFG1 0xb4 +#define DMA_CFG_NUM_TRIGGER_IN GENMASK(8, 0) + +#define IIDR 0xc8 +#define IIDR_PRODUCTID GENMASK(31, 20) +#define IIDR_VARIANT GENMASK(19, 16) +#define IIDR_REVISION GENMASK(15, 12) +#define IIDR_IMPLEMENTER GENMASK(11, 0) + +#define PRODUCTID_DMA350 0x3a0 +#define IMPLEMENTER_ARM 0x43b + +#define DMACH(n) (0x1000 + 0x0100 * (n)) + +#define CH_CMD 0x00 +#define CH_CMD_RESUME BIT(5) +#define CH_CMD_PAUSE BIT(4) +#define CH_CMD_STOP BIT(3) +#define CH_CMD_DISABLE BIT(2) +#define CH_CMD_CLEAR BIT(1) +#define CH_CMD_ENABLE BIT(0) + +#define CH_STATUS 0x04 +#define CH_STAT_RESUMEWAIT BIT(21) +#define CH_STAT_PAUSED BIT(20) +#define CH_STAT_STOPPED BIT(19) +#define CH_STAT_DISABLED BIT(18) +#define CH_STAT_ERR BIT(17) +#define CH_STAT_DONE BIT(16) +#define CH_STAT_INTR_ERR BIT(1) +#define CH_STAT_INTR_DONE BIT(0) + +#define CH_INTREN 0x08 +#define CH_INTREN_ERR BIT(1) +#define CH_INTREN_DONE BIT(0) + +#define CH_CTRL 0x0c +#define CH_CTRL_USEDESTRIGIN BIT(26) +#define CH_CTRL_USESRCTRIGIN BIT(26) +#define CH_CTRL_DONETYPE GENMASK(23, 21) +#define CH_CTRL_REGRELOADTYPE GENMASK(20, 18) +#define CH_CTRL_XTYPE GENMASK(11, 9) +#define CH_CTRL_TRANSIZE GENMASK(2, 0) + +#define CH_SRCADDR 0x10 +#define CH_SRCADDRHI 0x14 +#define CH_DESADDR 0x18 +#define CH_DESADDRHI 0x1c +#define CH_XSIZE 0x20 +#define CH_XSIZEHI 0x24 +#define CH_SRCTRANSCFG 0x28 +#define CH_DESTRANSCFG 0x2c +#define CH_CFG_MAXBURSTLEN GENMASK(19, 16) +#define CH_CFG_PRIVATTR BIT(11) +#define CH_CFG_SHAREATTR GENMASK(9, 8) +#define CH_CFG_MEMATTR GENMASK(7, 0) + +#define TRANSCFG_DEVICE \ + FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ + FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ + FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE) +#define TRANSCFG_NC \ + FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ + FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ + FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_NC) +#define TRANSCFG_WB \ + FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ + FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_ISH) | \ + FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_WB) + +#define CH_XADDRINC 0x30 +#define CH_XY_DES GENMASK(31, 16) +#define CH_XY_SRC GENMASK(15, 0) + +#define CH_FILLVAL 0x38 +#define CH_SRCTRIGINCFG 0x4c +#define CH_DESTRIGINCFG 0x50 +#define CH_LINKATTR 0x70 +#define CH_LINK_SHAREATTR GENMASK(9, 8) +#define CH_LINK_MEMATTR GENMASK(7, 0) + +#define CH_AUTOCFG 0x74 +#define CH_LINKADDR 0x78 +#define CH_LINKADDR_EN BIT(0) + +#define CH_LINKADDRHI 0x7c +#define CH_ERRINFO 0x90 +#define CH_ERRINFO_AXIRDPOISERR BIT(18) +#define CH_ERRINFO_AXIWRRESPERR BIT(17) +#define CH_ERRINFO_AXIRDRESPERR BIT(16) + +#define CH_BUILDCFG0 0xf8 +#define CH_CFG_INC_WIDTH GENMASK(29, 26) +#define CH_CFG_DATA_WIDTH GENMASK(24, 22) +#define CH_CFG_DATA_BUF_SIZE GENMASK(7, 0) + +#define CH_BUILDCFG1 0xfc +#define CH_CFG_HAS_CMDLINK BIT(8) +#define CH_CFG_HAS_TRIGSEL BIT(7) +#define CH_CFG_HAS_TRIGIN BIT(5) +#define CH_CFG_HAS_WRAP BIT(1) + + +#define LINK_REGCLEAR BIT(0) +#define LINK_INTREN BIT(2) +#define LINK_CTRL BIT(3) +#define LINK_SRCADDR BIT(4) +#define LINK_SRCADDRHI BIT(5) +#define LINK_DESADDR BIT(6) +#define LINK_DESADDRHI BIT(7) +#define LINK_XSIZE BIT(8) +#define LINK_XSIZEHI BIT(9) +#define LINK_SRCTRANSCFG BIT(10) +#define LINK_DESTRANSCFG BIT(11) +#define LINK_XADDRINC BIT(12) +#define LINK_FILLVAL BIT(14) +#define LINK_SRCTRIGINCFG BIT(19) +#define LINK_DESTRIGINCFG BIT(20) +#define LINK_AUTOCFG BIT(29) +#define LINK_LINKADDR BIT(30) +#define LINK_LINKADDRHI BIT(31) + + +enum ch_ctrl_donetype { + CH_CTRL_DONETYPE_NONE = 0, + CH_CTRL_DONETYPE_CMD = 1, + CH_CTRL_DONETYPE_CYCLE = 3 +}; + +enum ch_ctrl_xtype { + CH_CTRL_XTYPE_DISABLE = 0, + CH_CTRL_XTYPE_CONTINUE = 1, + CH_CTRL_XTYPE_WRAP = 2, + CH_CTRL_XTYPE_FILL = 3 +}; + +enum ch_cfg_shareattr { + SHAREATTR_NSH = 0, + SHAREATTR_OSH = 2, + SHAREATTR_ISH = 3 +}; + +enum ch_cfg_memattr { + MEMATTR_DEVICE = 0x00, + MEMATTR_NC = 0x44, + MEMATTR_WB = 0xff +}; + +struct d350_desc { + struct virt_dma_desc vd; + u32 command[16]; + u16 xsize; + u16 xsizehi; + u8 tsz; +}; + +struct d350_chan { + struct virt_dma_chan vc; + struct d350_desc *desc; + void __iomem *base; + int irq; + enum dma_status status; + dma_cookie_t cookie; + u32 residue; + u8 tsz; + bool has_trig; + bool has_wrap; + bool coherent; +}; + +struct d350 { + struct dma_device dma; + int nchan; + int nreq; + struct d350_chan channels[] __counted_by(nchan); +}; + +static inline struct d350_chan *to_d350_chan(struct dma_chan *chan) +{ + return container_of(chan, struct d350_chan, vc.chan); +} + +static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct d350_desc, vd); +} + +static void d350_desc_free(struct virt_dma_desc *vd) +{ + kfree(to_d350_desc(vd)); +} + +static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan, + dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) +{ + struct d350_chan *dch = to_d350_chan(chan); + struct d350_desc *desc; + u32 *cmd; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->tsz = __ffs(len | dest | src | (1 << dch->tsz)); + desc->xsize = lower_16_bits(len >> desc->tsz); + desc->xsizehi = upper_16_bits(len >> desc->tsz); + + cmd = desc->command; + cmd[0] = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR | + LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG | + LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR; + + cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | + FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) | + FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); + + cmd[2] = lower_32_bits(src); + cmd[3] = upper_32_bits(src); + cmd[4] = lower_32_bits(dest); + cmd[5] = upper_32_bits(dest); + cmd[6] = FIELD_PREP(CH_XY_SRC, desc->xsize) | FIELD_PREP(CH_XY_DES, desc->xsize); + cmd[7] = FIELD_PREP(CH_XY_SRC, desc->xsizehi) | FIELD_PREP(CH_XY_DES, desc->xsizehi); + cmd[8] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; + cmd[9] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; + cmd[10] = FIELD_PREP(CH_XY_SRC, 1) | FIELD_PREP(CH_XY_DES, 1); + cmd[11] = 0; + + return vchan_tx_prep(&dch->vc, &desc->vd, flags); +} + +static struct dma_async_tx_descriptor *d350_prep_memset(struct dma_chan *chan, + dma_addr_t dest, int value, size_t len, unsigned long flags) +{ + struct d350_chan *dch = to_d350_chan(chan); + struct d350_desc *desc; + u32 *cmd; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->tsz = __ffs(len | dest | (1 << dch->tsz)); + desc->xsize = lower_16_bits(len >> desc->tsz); + desc->xsizehi = upper_16_bits(len >> desc->tsz); + + cmd = desc->command; + cmd[0] = LINK_CTRL | LINK_DESADDR | LINK_DESADDRHI | + LINK_XSIZE | LINK_XSIZEHI | LINK_DESTRANSCFG | + LINK_XADDRINC | LINK_FILLVAL | LINK_LINKADDR; + + cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | + FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_FILL) | + FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); + + cmd[2] = lower_32_bits(dest); + cmd[3] = upper_32_bits(dest); + cmd[4] = FIELD_PREP(CH_XY_DES, desc->xsize); + cmd[5] = FIELD_PREP(CH_XY_DES, desc->xsizehi); + cmd[6] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; + cmd[7] = FIELD_PREP(CH_XY_DES, 1); + cmd[8] = (u8)value * 0x01010101; + cmd[9] = 0; + + return vchan_tx_prep(&dch->vc, &desc->vd, flags); +} + +static int d350_pause(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dch->vc.lock, flags); + if (dch->status == DMA_IN_PROGRESS) { + writel_relaxed(CH_CMD_PAUSE, dch->base + CH_CMD); + dch->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&dch->vc.lock, flags); + + return 0; +} + +static int d350_resume(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dch->vc.lock, flags); + if (dch->status == DMA_PAUSED) { + writel_relaxed(CH_CMD_RESUME, dch->base + CH_CMD); + dch->status = DMA_IN_PROGRESS; + } + spin_unlock_irqrestore(&dch->vc.lock, flags); + + return 0; +} + +static u32 d350_get_residue(struct d350_chan *dch) +{ + u32 res, xsize, xsizehi, hi_new; + int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */ + + hi_new = readl_relaxed(dch->base + CH_XSIZEHI); + do { + xsizehi = hi_new; + xsize = readl_relaxed(dch->base + CH_XSIZE); + hi_new = readl_relaxed(dch->base + CH_XSIZEHI); + } while (xsizehi != hi_new && --retries); + + res = FIELD_GET(CH_XY_DES, xsize); + res |= FIELD_GET(CH_XY_DES, xsizehi) << 16; + + return res << dch->desc->tsz; +} + +static int d350_terminate_all(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + LIST_HEAD(list); + + spin_lock_irqsave(&dch->vc.lock, flags); + writel_relaxed(CH_CMD_STOP, dch->base + CH_CMD); + if (dch->desc) { + if (dch->status != DMA_ERROR) + vchan_terminate_vdesc(&dch->desc->vd); + dch->desc = NULL; + dch->status = DMA_COMPLETE; + } + vchan_get_all_descriptors(&dch->vc, &list); + list_splice_tail(&list, &dch->vc.desc_terminated); + spin_unlock_irqrestore(&dch->vc.lock, flags); + + return 0; +} + +static void d350_synchronize(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + + vchan_synchronize(&dch->vc); +} + +static u32 d350_desc_bytes(struct d350_desc *desc) +{ + return ((u32)desc->xsizehi << 16 | desc->xsize) << desc->tsz; +} + +static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct d350_chan *dch = to_d350_chan(chan); + struct virt_dma_desc *vd; + enum dma_status status; + unsigned long flags; + u32 residue = 0; + + status = dma_cookie_status(chan, cookie, state); + + spin_lock_irqsave(&dch->vc.lock, flags); + if (cookie == dch->cookie) { + status = dch->status; + if (status == DMA_IN_PROGRESS || status == DMA_PAUSED) + dch->residue = d350_get_residue(dch); + residue = dch->residue; + } else if ((vd = vchan_find_desc(&dch->vc, cookie))) { + residue = d350_desc_bytes(to_d350_desc(vd)); + } else if (status == DMA_IN_PROGRESS) { + /* Somebody else terminated it? */ + status = DMA_ERROR; + } + spin_unlock_irqrestore(&dch->vc.lock, flags); + + dma_set_residue(state, residue); + return status; +} + +static void d350_start_next(struct d350_chan *dch) +{ + u32 hdr, *reg; + + dch->desc = to_d350_desc(vchan_next_desc(&dch->vc)); + if (!dch->desc) + return; + + list_del(&dch->desc->vd.node); + dch->status = DMA_IN_PROGRESS; + dch->cookie = dch->desc->vd.tx.cookie; + dch->residue = d350_desc_bytes(dch->desc); + + hdr = dch->desc->command[0]; + reg = &dch->desc->command[1]; + + if (hdr & LINK_INTREN) + writel_relaxed(*reg++, dch->base + CH_INTREN); + if (hdr & LINK_CTRL) + writel_relaxed(*reg++, dch->base + CH_CTRL); + if (hdr & LINK_SRCADDR) + writel_relaxed(*reg++, dch->base + CH_SRCADDR); + if (hdr & LINK_SRCADDRHI) + writel_relaxed(*reg++, dch->base + CH_SRCADDRHI); + if (hdr & LINK_DESADDR) + writel_relaxed(*reg++, dch->base + CH_DESADDR); + if (hdr & LINK_DESADDRHI) + writel_relaxed(*reg++, dch->base + CH_DESADDRHI); + if (hdr & LINK_XSIZE) + writel_relaxed(*reg++, dch->base + CH_XSIZE); + if (hdr & LINK_XSIZEHI) + writel_relaxed(*reg++, dch->base + CH_XSIZEHI); + if (hdr & LINK_SRCTRANSCFG) + writel_relaxed(*reg++, dch->base + CH_SRCTRANSCFG); + if (hdr & LINK_DESTRANSCFG) + writel_relaxed(*reg++, dch->base + CH_DESTRANSCFG); + if (hdr & LINK_XADDRINC) + writel_relaxed(*reg++, dch->base + CH_XADDRINC); + if (hdr & LINK_FILLVAL) + writel_relaxed(*reg++, dch->base + CH_FILLVAL); + if (hdr & LINK_SRCTRIGINCFG) + writel_relaxed(*reg++, dch->base + CH_SRCTRIGINCFG); + if (hdr & LINK_DESTRIGINCFG) + writel_relaxed(*reg++, dch->base + CH_DESTRIGINCFG); + if (hdr & LINK_AUTOCFG) + writel_relaxed(*reg++, dch->base + CH_AUTOCFG); + if (hdr & LINK_LINKADDR) + writel_relaxed(*reg++, dch->base + CH_LINKADDR); + if (hdr & LINK_LINKADDRHI) + writel_relaxed(*reg++, dch->base + CH_LINKADDRHI); + + writel(CH_CMD_ENABLE, dch->base + CH_CMD); +} + +static void d350_issue_pending(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dch->vc.lock, flags); + if (vchan_issue_pending(&dch->vc) && !dch->desc) + d350_start_next(dch); + spin_unlock_irqrestore(&dch->vc.lock, flags); +} + +static irqreturn_t d350_irq(int irq, void *data) +{ + struct d350_chan *dch = data; + struct device *dev = dch->vc.chan.device->dev; + struct virt_dma_desc *vd = &dch->desc->vd; + u32 ch_status; + + ch_status = readl(dch->base + CH_STATUS); + if (!ch_status) + return IRQ_NONE; + + if (ch_status & CH_STAT_INTR_ERR) { + u32 errinfo = readl_relaxed(dch->base + CH_ERRINFO); + + if (errinfo & (CH_ERRINFO_AXIRDPOISERR | CH_ERRINFO_AXIRDRESPERR)) + vd->tx_result.result = DMA_TRANS_READ_FAILED; + else if (errinfo & CH_ERRINFO_AXIWRRESPERR) + vd->tx_result.result = DMA_TRANS_WRITE_FAILED; + else + vd->tx_result.result = DMA_TRANS_ABORTED; + + vd->tx_result.residue = d350_get_residue(dch); + } else if (!(ch_status & CH_STAT_INTR_DONE)) { + dev_warn(dev, "Unexpected IRQ source? 0x%08x\n", ch_status); + } + writel_relaxed(ch_status, dch->base + CH_STATUS); + + spin_lock(&dch->vc.lock); + vchan_cookie_complete(vd); + if (ch_status & CH_STAT_INTR_DONE) { + dch->status = DMA_COMPLETE; + dch->residue = 0; + d350_start_next(dch); + } else { + dch->status = DMA_ERROR; + dch->residue = vd->tx_result.residue; + } + spin_unlock(&dch->vc.lock); + + return IRQ_HANDLED; +} + +static int d350_alloc_chan_resources(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED, + dev_name(&dch->vc.chan.dev->device), dch); + if (!ret) + writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN); + + return ret; +} + +static void d350_free_chan_resources(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + + writel_relaxed(0, dch->base + CH_INTREN); + free_irq(dch->irq, dch); + vchan_free_chan_resources(&dch->vc); +} + +static int d350_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct d350 *dmac; + void __iomem *base; + u32 reg; + int ret, nchan, dw, aw, r, p; + bool coherent, memset; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + reg = readl_relaxed(base + DMAINFO + IIDR); + r = FIELD_GET(IIDR_VARIANT, reg); + p = FIELD_GET(IIDR_REVISION, reg); + if (FIELD_GET(IIDR_IMPLEMENTER, reg) != IMPLEMENTER_ARM || + FIELD_GET(IIDR_PRODUCTID, reg) != PRODUCTID_DMA350) + return dev_err_probe(dev, -ENODEV, "Not a DMA-350!"); + + reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG0); + nchan = FIELD_GET(DMA_CFG_NUM_CHANNELS, reg) + 1; + dw = 1 << FIELD_GET(DMA_CFG_DATA_WIDTH, reg); + aw = FIELD_GET(DMA_CFG_ADDR_WIDTH, reg) + 1; + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aw)); + coherent = device_get_dma_attr(dev) == DEV_DMA_COHERENT; + + dmac = devm_kzalloc(dev, struct_size(dmac, channels, nchan), GFP_KERNEL); + if (!dmac) + return -ENOMEM; + + dmac->nchan = nchan; + + reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1); + dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg); + + dev_dbg(dev, "DMA-350 r%dp%d with %d channels, %d requests\n", r, p, dmac->nchan, dmac->nreq); + + dmac->dma.dev = dev; + for (int i = min(dw, 16); i > 0; i /= 2) { + dmac->dma.src_addr_widths |= BIT(i); + dmac->dma.dst_addr_widths |= BIT(i); + } + dmac->dma.directions = BIT(DMA_MEM_TO_MEM); + dmac->dma.descriptor_reuse = true; + dmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dmac->dma.device_alloc_chan_resources = d350_alloc_chan_resources; + dmac->dma.device_free_chan_resources = d350_free_chan_resources; + dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask); + dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy; + dmac->dma.device_pause = d350_pause; + dmac->dma.device_resume = d350_resume; + dmac->dma.device_terminate_all = d350_terminate_all; + dmac->dma.device_synchronize = d350_synchronize; + dmac->dma.device_tx_status = d350_tx_status; + dmac->dma.device_issue_pending = d350_issue_pending; + INIT_LIST_HEAD(&dmac->dma.channels); + + /* Would be nice to have per-channel caps for this... */ + memset = true; + for (int i = 0; i < nchan; i++) { + struct d350_chan *dch = &dmac->channels[i]; + + dch->base = base + DMACH(i); + writel_relaxed(CH_CMD_CLEAR, dch->base + CH_CMD); + + reg = readl_relaxed(dch->base + CH_BUILDCFG1); + if (!(FIELD_GET(CH_CFG_HAS_CMDLINK, reg))) { + dev_warn(dev, "No command link support on channel %d\n", i); + continue; + } + dch->irq = platform_get_irq(pdev, i); + if (dch->irq < 0) + return dev_err_probe(dev, dch->irq, + "Failed to get IRQ for channel %d\n", i); + + dch->has_wrap = FIELD_GET(CH_CFG_HAS_WRAP, reg); + dch->has_trig = FIELD_GET(CH_CFG_HAS_TRIGIN, reg) & + FIELD_GET(CH_CFG_HAS_TRIGSEL, reg); + + /* Fill is a special case of Wrap */ + memset &= dch->has_wrap; + + reg = readl_relaxed(dch->base + CH_BUILDCFG0); + dch->tsz = FIELD_GET(CH_CFG_DATA_WIDTH, reg); + + reg = FIELD_PREP(CH_LINK_SHAREATTR, coherent ? SHAREATTR_ISH : SHAREATTR_OSH); + reg |= FIELD_PREP(CH_LINK_MEMATTR, coherent ? MEMATTR_WB : MEMATTR_NC); + writel_relaxed(reg, dch->base + CH_LINKATTR); + + dch->vc.desc_free = d350_desc_free; + vchan_init(&dch->vc, &dmac->dma); + } + + if (memset) { + dma_cap_set(DMA_MEMSET, dmac->dma.cap_mask); + dmac->dma.device_prep_dma_memset = d350_prep_memset; + } + + platform_set_drvdata(pdev, dmac); + + ret = dma_async_device_register(&dmac->dma); + if (ret) + return dev_err_probe(dev, ret, "Failed to register DMA device\n"); + + return 0; +} + +static void d350_remove(struct platform_device *pdev) +{ + struct d350 *dmac = platform_get_drvdata(pdev); + + dma_async_device_unregister(&dmac->dma); +} + +static const struct of_device_id d350_of_match[] __maybe_unused = { + { .compatible = "arm,dma-350" }, + {} +}; +MODULE_DEVICE_TABLE(of, d350_of_match); + +static struct platform_driver d350_driver = { + .driver = { + .name = "arm-dma350", + .of_match_table = of_match_ptr(d350_of_match), + }, + .probe = d350_probe, + .remove = d350_remove, +}; +module_platform_driver(d350_driver); + +MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); +MODULE_DESCRIPTION("Arm DMA-350 driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index ee3a219e3a89..7d226453961f 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -20,7 +20,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/overflow.h> -#include <linux/of_device.h> +#include <linux/of_platform.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -222,8 +222,14 @@ struct atdma_sg { * @vd: pointer to the virtual dma descriptor. * @atchan: pointer to the atmel dma channel. * @total_len: total transaction byte count - * @sg_len: number of sg entries. + * @sglen: number of sg entries. * @sg: array of sgs. + * @boundary: number of transfers to perform before the automatic address increment operation + * @dst_hole: value to add to the destination address when the boundary has been reached + * @src_hole: value to add to the source address when the boundary has been reached + * @memset_buffer: buffer used for the memset operation + * @memset_paddr: physical address of the buffer used for the memset operation + * @memset_vaddr: virtual address of the buffer used for the memset operation */ struct at_desc { struct virt_dma_desc vd; @@ -239,13 +245,16 @@ struct at_desc { bool memset_buffer; dma_addr_t memset_paddr; int *memset_vaddr; - struct atdma_sg sg[]; + struct atdma_sg sg[] __counted_by(sglen); }; /*-- Channels --------------------------------------------------------*/ /** - * atc_status - information bits stored in channel status flag + * enum atc_status - information bits stored in channel status flag + * + * @ATC_IS_PAUSED: If channel is pauses + * @ATC_IS_CYCLIC: If channel is cyclic * * Manipulated with atomic operations. */ @@ -282,7 +291,6 @@ struct at_dma_chan { u32 save_cfg; u32 save_dscr; struct dma_slave_config dma_sconfig; - bool cyclic; struct at_desc *desc; }; @@ -328,12 +336,12 @@ static inline u8 convert_buswidth(enum dma_slave_buswidth addr_width) /** * struct at_dma - internal representation of an Atmel HDMA Controller * @dma_device: dmaengine dma_device object members - * @atdma_devtype: identifier of DMA controller compatibility - * @ch_regs: memory mapped register base + * @regs: memory mapped register base * @clk: dma controller clock * @save_imr: interrupt mask register that is saved on suspend/resume cycle - * @all_chan_mask: all channels availlable in a mask + * @all_chan_mask: all channels available in a mask * @lli_pool: hw lli table + * @memset_pool: hw memset pool * @chan: channels table to store at_dma_chan structures */ struct at_dma { @@ -626,6 +634,9 @@ static inline u32 atc_calc_bytes_left(u32 current_len, u32 ctrla) /** * atc_get_llis_residue - Get residue for a hardware linked list transfer + * @atchan: pointer to an atmel hdmac channel. + * @desc: pointer to the descriptor for which the residue is calculated. + * @residue: residue to be set to dma_tx_state. * * Calculate the residue by removing the length of the Linked List Item (LLI) * already transferred from the total length. To get the current LLI we can use @@ -657,14 +668,12 @@ static inline u32 atc_calc_bytes_left(u32 current_len, u32 ctrla) * CTRLA is read in turn, next the DSCR is read a second time. If the two * consecutive read values of the DSCR are the same then we assume both refers * to the very same LLI as well as the CTRLA value read inbetween does. For - * cyclic tranfers, the assumption is that a full loop is "not so fast". If the + * cyclic transfers, the assumption is that a full loop is "not so fast". If the * two DSCR values are different, we read again the CTRLA then the DSCR till two * consecutive read values from DSCR are equal or till the maximum trials is * reach. This algorithm is very unlikely not to find a stable value for DSCR. - * @atchan: pointer to an atmel hdmac channel. - * @desc: pointer to the descriptor for which the residue is calculated. - * @residue: residue to be set to dma_tx_state. - * Returns 0 on success, -errno otherwise. + * + * Returns: %0 on success, -errno otherwise. */ static int atc_get_llis_residue(struct at_dma_chan *atchan, struct at_desc *desc, u32 *residue) @@ -691,7 +700,7 @@ static int atc_get_llis_residue(struct at_dma_chan *atchan, break; /* - * DSCR has changed inside the DMA controller, so the previouly + * DSCR has changed inside the DMA controller, so the previously * read value of CTRLA may refer to an already processed * descriptor hence could be outdated. We need to update ctrla * to match the current descriptor. @@ -731,7 +740,8 @@ static int atc_get_llis_residue(struct at_dma_chan *atchan, * @chan: DMA channel * @cookie: transaction identifier to check status of * @residue: residue to be updated. - * Return 0 on success, -errono otherwise. + * + * Return: %0 on success, -errno otherwise. */ static int atc_get_residue(struct dma_chan *chan, dma_cookie_t cookie, u32 *residue) @@ -877,7 +887,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan, first = xt->sgl; dev_info(chan2dev(chan), - "%s: src=%pad, dest=%pad, numf=%d, frame_size=%d, flags=0x%lx\n", + "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n", __func__, &xt->src_start, &xt->dst_start, xt->numf, xt->frame_size, flags); @@ -1164,7 +1174,7 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, int i; int ret; - dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%zx f0x%lx\n", __func__, + dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%x f0x%lx\n", __func__, value, sg_len, flags); if (unlikely(!sgl || !sg_len)) { @@ -1493,7 +1503,7 @@ atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, unsigned int periods = buf_len / period_len; unsigned int i; - dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%d/%d)\n", + dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%zu/%zu)\n", direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE", &buf_addr, periods, buf_len, period_len); @@ -1710,7 +1720,7 @@ static void atc_issue_pending(struct dma_chan *chan) * atc_alloc_chan_resources - allocate resources for DMA channel * @chan: allocate descriptor resources for this channel * - * return - the number of allocated descriptors + * Return: the number of allocated descriptors */ static int atc_alloc_chan_resources(struct dma_chan *chan) { @@ -2100,7 +2110,7 @@ err_irq: return err; } -static int at_dma_remove(struct platform_device *pdev) +static void at_dma_remove(struct platform_device *pdev) { struct at_dma *atdma = platform_get_drvdata(pdev); struct dma_chan *chan, *_chan; @@ -2122,8 +2132,6 @@ static int at_dma_remove(struct platform_device *pdev) } clk_disable_unprepare(atdma->clk); - - return 0; } static void at_dma_shutdown(struct platform_device *pdev) diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index c3b37168b21f..3fbc74710a13 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -1363,6 +1363,8 @@ at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, return NULL; desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value); + if (!desc) + return NULL; list_add_tail(&desc->desc_node, &desc->descs_list); desc->tx_dma_desc.cookie = -EBUSY; @@ -2031,10 +2033,8 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan) * at_xdmac_start_xfer() for this descriptor. Now it's time * to release it. */ - if (desc->active_xfer) { - pm_runtime_put_autosuspend(atxdmac->dev); - pm_runtime_mark_last_busy(atxdmac->dev); - } + if (desc->active_xfer) + pm_runtime_put_noidle(atxdmac->dev); } clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); @@ -2431,7 +2431,7 @@ err_free_irq: return ret; } -static int at_xdmac_remove(struct platform_device *pdev) +static void at_xdmac_remove(struct platform_device *pdev) { struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev); int i; @@ -2452,8 +2452,6 @@ static int at_xdmac_remove(struct platform_device *pdev) tasklet_kill(&atchan->tasklet); at_xdmac_free_chan_resources(&atchan->chan); } - - return 0; } static const struct dev_pm_ops __maybe_unused atmel_xdmac_dev_pm_ops = { diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index 064761289a73..7f0e76439ce5 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -15,7 +15,7 @@ * number of hardware rings over one or more SBA hardware devices. By * design, the internal buffer size of SBA hardware device is limited * but all offload operations supported by SBA can be broken down into - * multiple small size requests and executed parallely on multiple SBA + * multiple small size requests and executed parallelly on multiple SBA * hardware devices for achieving high through-put. * * The Broadcom SBA RAID driver does not require any register programming @@ -35,7 +35,9 @@ #include <linux/mailbox_client.h> #include <linux/mailbox/brcm-message.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/raid/pq.h> @@ -133,7 +135,7 @@ struct sba_device { u32 max_xor_srcs; u32 max_resp_pool_size; u32 max_cmds_pool_size; - /* Maibox client and Mailbox channels */ + /* Mailbox client and Mailbox channels */ struct mbox_client client; struct mbox_chan *mchan; struct device *mbox_dev; @@ -1732,7 +1734,7 @@ fail_free_mchan: return ret; } -static int sba_remove(struct platform_device *pdev) +static void sba_remove(struct platform_device *pdev) { struct sba_device *sba = platform_get_drvdata(pdev); @@ -1743,8 +1745,6 @@ static int sba_remove(struct platform_device *pdev) sba_freeup_channel_resources(sba); mbox_free_channel(sba->mchan); - - return 0; } static const struct of_device_id sba_of_match[] = { diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c index 0807fb9eb262..321748e2983e 100644 --- a/drivers/dma/bcm2835-dma.c +++ b/drivers/dma/bcm2835-dma.c @@ -369,7 +369,7 @@ static struct bcm2835_desc *bcm2835_dma_create_cb_chain( /* the last frame requires extra flags */ d->cb_list[d->frames - 1].cb->info |= finalextrainfo; - /* detect a size missmatch */ + /* detect a size mismatch */ if (buf_len && (d->size != buf_len)) goto error_cb; @@ -875,6 +875,27 @@ static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, return chan; } +static int bcm2835_dma_suspend_late(struct device *dev) +{ + struct bcm2835_dmadev *od = dev_get_drvdata(dev); + struct bcm2835_chan *c, *next; + + list_for_each_entry_safe(c, next, &od->ddev.channels, + vc.chan.device_node) { + void __iomem *chan_base = c->chan_base; + + /* Check if DMA channel is busy */ + if (readl(chan_base + BCM2835_DMA_ADDR)) + return -EBUSY; + } + + return 0; +} + +static const struct dev_pm_ops bcm2835_dma_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(bcm2835_dma_suspend_late, NULL) +}; + static int bcm2835_dma_probe(struct platform_device *pdev) { struct bcm2835_dmadev *od; @@ -1019,28 +1040,26 @@ err_no_dma: return rc; } -static int bcm2835_dma_remove(struct platform_device *pdev) +static void bcm2835_dma_remove(struct platform_device *pdev) { struct bcm2835_dmadev *od = platform_get_drvdata(pdev); dma_async_device_unregister(&od->ddev); bcm2835_dma_free(od); - - return 0; } static struct platform_driver bcm2835_dma_driver = { .probe = bcm2835_dma_probe, - .remove = bcm2835_dma_remove, + .remove = bcm2835_dma_remove, .driver = { .name = "bcm2835-dma", .of_match_table = of_match_ptr(bcm2835_dma_of_match), + .pm = pm_ptr(&bcm2835_dma_pm_ops), }, }; module_platform_driver(bcm2835_dma_driver); -MODULE_ALIAS("platform:bcm2835-dma"); MODULE_DESCRIPTION("BCM2835 DMA engine driver"); MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c index eabbcfcaa7cb..6c4d655ffe77 100644 --- a/drivers/dma/bestcomm/bestcomm.c +++ b/drivers/dma/bestcomm/bestcomm.c @@ -14,9 +14,8 @@ #include <linux/slab.h> #include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <asm/io.h> #include <asm/irq.h> #include <asm/mpc52xx.h> @@ -456,7 +455,7 @@ error_ofput: } -static int mpc52xx_bcom_remove(struct platform_device *op) +static void mpc52xx_bcom_remove(struct platform_device *op) { /* Clean up the engine */ bcom_engine_cleanup(); @@ -474,8 +473,6 @@ static int mpc52xx_bcom_remove(struct platform_device *op) /* Release memory */ kfree(bcom_eng); bcom_eng = NULL; - - return 0; } static const struct of_device_id mpc52xx_bcom_of_match[] = { diff --git a/drivers/dma/bestcomm/sram.c b/drivers/dma/bestcomm/sram.c index 0553956f7456..ad74d57cc3ab 100644 --- a/drivers/dma/bestcomm/sram.c +++ b/drivers/dma/bestcomm/sram.c @@ -90,13 +90,8 @@ int bcom_sram_init(struct device_node *sram_node, char *owner) bcom_sram->rh = rh_create(4); /* Attach the free zones */ -#if 0 - /* Currently disabled ... for future use only */ - reg_addr_p = of_get_property(sram_node, "available", &psize); -#else regaddr_p = NULL; psize = 0; -#endif if (!regaddr_p || !psize) { /* Attach the whole zone */ diff --git a/drivers/dma/cv1800b-dmamux.c b/drivers/dma/cv1800b-dmamux.c new file mode 100644 index 000000000000..e900d6595617 --- /dev/null +++ b/drivers/dma/cv1800b-dmamux.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025 Inochi Amaoto <inochiama@gmail.com> + */ + +#include <linux/bitops.h> +#include <linux/cleanup.h> +#include <linux/module.h> +#include <linux/of_dma.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/llist.h> +#include <linux/regmap.h> +#include <linux/spinlock.h> +#include <linux/mfd/syscon.h> + +#define REG_DMA_CHANNEL_REMAP0 0x154 +#define REG_DMA_CHANNEL_REMAP1 0x158 +#define REG_DMA_INT_MUX 0x298 + +#define DMAMUX_NCELLS 2 +#define MAX_DMA_MAPPING_ID 42 +#define MAX_DMA_CPU_ID 2 +#define MAX_DMA_CH_ID 7 + +#define DMAMUX_INTMUX_REGISTER_LEN 4 +#define DMAMUX_NR_CH_PER_REGISTER 4 +#define DMAMUX_BIT_PER_CH 8 +#define DMAMUX_CH_MASk GENMASK(5, 0) +#define DMAMUX_INT_BIT_PER_CPU 10 +#define DMAMUX_CH_UPDATE_BIT BIT(31) + +#define DMAMUX_CH_REGPOS(chid) \ + ((chid) / DMAMUX_NR_CH_PER_REGISTER) +#define DMAMUX_CH_REGOFF(chid) \ + ((chid) % DMAMUX_NR_CH_PER_REGISTER) +#define DMAMUX_CH_REG(chid) \ + ((DMAMUX_CH_REGPOS(chid) * sizeof(u32)) + \ + REG_DMA_CHANNEL_REMAP0) +#define DMAMUX_CH_SET(chid, val) \ + (((val) << (DMAMUX_CH_REGOFF(chid) * DMAMUX_BIT_PER_CH)) | \ + DMAMUX_CH_UPDATE_BIT) +#define DMAMUX_CH_MASK(chid) \ + DMAMUX_CH_SET(chid, DMAMUX_CH_MASk) + +#define DMAMUX_INT_BIT(chid, cpuid) \ + BIT((cpuid) * DMAMUX_INT_BIT_PER_CPU + (chid)) +#define DMAMUX_INTEN_BIT(cpuid) \ + DMAMUX_INT_BIT(8, cpuid) +#define DMAMUX_INT_CH_BIT(chid, cpuid) \ + (DMAMUX_INT_BIT(chid, cpuid) | DMAMUX_INTEN_BIT(cpuid)) +#define DMAMUX_INT_MASK(chid) \ + (DMAMUX_INT_BIT(chid, 0) | \ + DMAMUX_INT_BIT(chid, 1) | \ + DMAMUX_INT_BIT(chid, 2)) +#define DMAMUX_INT_CH_MASK(chid, cpuid) \ + (DMAMUX_INT_MASK(chid) | DMAMUX_INTEN_BIT(cpuid)) + +struct cv1800_dmamux_data { + struct dma_router dmarouter; + struct regmap *regmap; + spinlock_t lock; + struct llist_head free_maps; + struct llist_head reserve_maps; + DECLARE_BITMAP(mapped_peripherals, MAX_DMA_MAPPING_ID); +}; + +struct cv1800_dmamux_map { + struct llist_node node; + unsigned int channel; + unsigned int peripheral; + unsigned int cpu; +}; + +static void cv1800_dmamux_free(struct device *dev, void *route_data) +{ + struct cv1800_dmamux_data *dmamux = dev_get_drvdata(dev); + struct cv1800_dmamux_map *map = route_data; + + guard(spinlock_irqsave)(&dmamux->lock); + + regmap_update_bits(dmamux->regmap, + DMAMUX_CH_REG(map->channel), + DMAMUX_CH_MASK(map->channel), + DMAMUX_CH_UPDATE_BIT); + + regmap_update_bits(dmamux->regmap, REG_DMA_INT_MUX, + DMAMUX_INT_CH_MASK(map->channel, map->cpu), + DMAMUX_INTEN_BIT(map->cpu)); + + dev_dbg(dev, "free channel %u for req %u (cpu %u)\n", + map->channel, map->peripheral, map->cpu); +} + +static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct cv1800_dmamux_data *dmamux = platform_get_drvdata(pdev); + struct cv1800_dmamux_map *map; + struct llist_node *node; + unsigned long flags; + unsigned int chid, devid, cpuid; + int ret; + + if (dma_spec->args_count != DMAMUX_NCELLS) { + dev_err(&pdev->dev, "invalid number of dma mux args\n"); + return ERR_PTR(-EINVAL); + } + + devid = dma_spec->args[0]; + cpuid = dma_spec->args[1]; + dma_spec->args_count = 1; + + if (devid > MAX_DMA_MAPPING_ID) { + dev_err(&pdev->dev, "invalid device id: %u\n", devid); + return ERR_PTR(-EINVAL); + } + + if (cpuid > MAX_DMA_CPU_ID) { + dev_err(&pdev->dev, "invalid cpu id: %u\n", cpuid); + return ERR_PTR(-EINVAL); + } + + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); + if (!dma_spec->np) { + dev_err(&pdev->dev, "can't get dma master\n"); + return ERR_PTR(-EINVAL); + } + + spin_lock_irqsave(&dmamux->lock, flags); + + if (test_bit(devid, dmamux->mapped_peripherals)) { + llist_for_each_entry(map, dmamux->reserve_maps.first, node) { + if (map->peripheral == devid && map->cpu == cpuid) + goto found; + } + + ret = -EINVAL; + goto failed; + } else { + node = llist_del_first(&dmamux->free_maps); + if (!node) { + ret = -ENODEV; + goto failed; + } + + map = llist_entry(node, struct cv1800_dmamux_map, node); + llist_add(&map->node, &dmamux->reserve_maps); + set_bit(devid, dmamux->mapped_peripherals); + } + +found: + chid = map->channel; + map->peripheral = devid; + map->cpu = cpuid; + + regmap_set_bits(dmamux->regmap, + DMAMUX_CH_REG(chid), + DMAMUX_CH_SET(chid, devid)); + + regmap_update_bits(dmamux->regmap, REG_DMA_INT_MUX, + DMAMUX_INT_CH_MASK(chid, cpuid), + DMAMUX_INT_CH_BIT(chid, cpuid)); + + spin_unlock_irqrestore(&dmamux->lock, flags); + + dma_spec->args[0] = chid; + + dev_dbg(&pdev->dev, "register channel %u for req %u (cpu %u)\n", + chid, devid, cpuid); + + return map; + +failed: + spin_unlock_irqrestore(&dmamux->lock, flags); + of_node_put(dma_spec->np); + dev_err(&pdev->dev, "errno %d\n", ret); + return ERR_PTR(ret); +} + +static int cv1800_dmamux_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *mux_node = dev->of_node; + struct cv1800_dmamux_data *data; + struct cv1800_dmamux_map *tmp; + struct device *parent = dev->parent; + struct regmap *regmap = NULL; + unsigned int i; + + if (!parent) + return -ENODEV; + + regmap = device_node_to_regmap(parent->of_node); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + spin_lock_init(&data->lock); + init_llist_head(&data->free_maps); + init_llist_head(&data->reserve_maps); + + for (i = 0; i <= MAX_DMA_CH_ID; i++) { + tmp = devm_kmalloc(dev, sizeof(*tmp), GFP_KERNEL); + if (!tmp) { + /* It is OK for not allocating all channel */ + dev_warn(dev, "can not allocate channel %u\n", i); + continue; + } + + init_llist_node(&tmp->node); + tmp->channel = i; + llist_add(&tmp->node, &data->free_maps); + } + + /* if no channel is allocated, the probe must fail */ + if (llist_empty(&data->free_maps)) + return -ENOMEM; + + data->regmap = regmap; + data->dmarouter.dev = dev; + data->dmarouter.route_free = cv1800_dmamux_free; + + platform_set_drvdata(pdev, data); + + return of_dma_router_register(mux_node, + cv1800_dmamux_route_allocate, + &data->dmarouter); +} + +static void cv1800_dmamux_remove(struct platform_device *pdev) +{ + of_dma_controller_free(pdev->dev.of_node); +} + +static const struct of_device_id cv1800_dmamux_ids[] = { + { .compatible = "sophgo,cv1800b-dmamux", }, + { } +}; +MODULE_DEVICE_TABLE(of, cv1800_dmamux_ids); + +static struct platform_driver cv1800_dmamux_driver = { + .probe = cv1800_dmamux_probe, + .remove = cv1800_dmamux_remove, + .driver = { + .name = "cv1800-dmamux", + .of_match_table = cv1800_dmamux_ids, + }, +}; +module_platform_driver(cv1800_dmamux_driver); + +MODULE_AUTHOR("Inochi Amaoto <inochiama@gmail.com>"); +MODULE_DESCRIPTION("Sophgo CV1800/SG2000 Series SoC DMAMUX driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index fc7cdad37161..5b06b0dc67ee 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -6,6 +6,7 @@ * Author: Lars-Peter Clausen <lars@metafoo.de> */ +#include <linux/adi-axi-common.h> #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/device.h> @@ -22,7 +23,6 @@ #include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/slab.h> -#include <linux/fpga/adi-axi-common.h> #include <dt-bindings/dma/axi-dmac.h> @@ -81,9 +81,13 @@ #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 +#define AXI_DMAC_REG_CURRENT_SG_ID 0x454 +#define AXI_DMAC_REG_SG_ADDRESS 0x47c +#define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc #define AXI_DMAC_CTRL_ENABLE BIT(0) #define AXI_DMAC_CTRL_PAUSE BIT(1) +#define AXI_DMAC_CTRL_ENABLE_SG BIT(2) #define AXI_DMAC_IRQ_SOT BIT(0) #define AXI_DMAC_IRQ_EOT BIT(1) @@ -97,27 +101,42 @@ /* The maximum ID allocated by the hardware is 31 */ #define AXI_DMAC_SG_UNUSED 32U +/* Flags for axi_dmac_hw_desc.flags */ +#define AXI_DMAC_HW_FLAG_LAST BIT(0) +#define AXI_DMAC_HW_FLAG_IRQ BIT(1) + +struct axi_dmac_hw_desc { + u32 flags; + u32 id; + u64 dest_addr; + u64 src_addr; + u64 next_sg_addr; + u32 y_len; + u32 x_len; + u32 src_stride; + u32 dst_stride; + u64 __pad[2]; +}; + struct axi_dmac_sg { - dma_addr_t src_addr; - dma_addr_t dest_addr; - unsigned int x_len; - unsigned int y_len; - unsigned int dest_stride; - unsigned int src_stride; - unsigned int id; unsigned int partial_len; bool schedule_when_free; + + struct axi_dmac_hw_desc *hw; + dma_addr_t hw_phys; }; struct axi_dmac_desc { struct virt_dma_desc vdesc; + struct axi_dmac_chan *chan; + bool cyclic; bool have_partial_xfer; unsigned int num_submitted; unsigned int num_completed; unsigned int num_sgs; - struct axi_dmac_sg sg[]; + struct axi_dmac_sg sg[] __counted_by(num_sgs); }; struct axi_dmac_chan { @@ -139,6 +158,7 @@ struct axi_dmac_chan { bool hw_partial_xfer; bool hw_cyclic; bool hw_2d; + bool hw_sg; }; struct axi_dmac { @@ -213,9 +233,11 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) unsigned int flags = 0; unsigned int val; - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); - if (val) /* Queue is full, wait for the next SOT IRQ */ - return; + if (!chan->hw_sg) { + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; + } desc = chan->next_desc; @@ -229,14 +251,15 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) sg = &desc->sg[desc->num_submitted]; /* Already queued in cyclic mode. Wait for it to finish */ - if (sg->id != AXI_DMAC_SG_UNUSED) { + if (sg->hw->id != AXI_DMAC_SG_UNUSED) { sg->schedule_when_free = true; return; } - desc->num_submitted++; - if (desc->num_submitted == desc->num_sgs || - desc->have_partial_xfer) { + if (chan->hw_sg) { + chan->next_desc = NULL; + } else if (++desc->num_submitted == desc->num_sgs || + desc->have_partial_xfer) { if (desc->cyclic) desc->num_submitted = 0; /* Start again */ else @@ -246,32 +269,42 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) chan->next_desc = desc; } - sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); + sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); - if (axi_dmac_dest_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride); - } + if (!chan->hw_sg) { + if (axi_dmac_dest_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); + } - if (axi_dmac_src_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride); + if (axi_dmac_src_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); + } } /* * If the hardware supports cyclic transfers and there is no callback to - * call and only a single segment, enable hw cyclic mode to avoid - * unnecessary interrupts. + * call, enable hw cyclic mode to avoid unnecessary interrupts. */ - if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback && - desc->num_sgs == 1) - flags |= AXI_DMAC_FLAG_CYCLIC; + if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { + if (chan->hw_sg) + desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; + else if (desc->num_sgs == 1) + flags |= AXI_DMAC_FLAG_CYCLIC; + } if (chan->hw_partial_xfer) flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; - axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); - axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); + if (chan->hw_sg) { + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys); + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH, + (u64)sg->hw_phys >> 32); + } else { + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); + } axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); } @@ -286,9 +319,9 @@ static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, struct axi_dmac_sg *sg) { if (chan->hw_2d) - return sg->x_len * sg->y_len; + return (sg->hw->x_len + 1) * (sg->hw->y_len + 1); else - return sg->x_len; + return (sg->hw->x_len + 1); } static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) @@ -307,9 +340,9 @@ static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) list_for_each_entry(desc, &chan->active_descs, vdesc.node) { for (i = 0; i < desc->num_sgs; i++) { sg = &desc->sg[i]; - if (sg->id == AXI_DMAC_SG_UNUSED) + if (sg->hw->id == AXI_DMAC_SG_UNUSED) continue; - if (sg->id == id) { + if (sg->hw->id == id) { desc->have_partial_xfer = true; sg->partial_len = len; found_sg = true; @@ -348,6 +381,9 @@ static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, rslt->result = DMA_TRANS_NOERROR; rslt->residue = 0; + if (chan->hw_sg) + return; + /* * We get here if the last completed segment is partial, which * means we can compute the residue from that segment onwards @@ -374,36 +410,47 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) axi_dmac_dequeue_partial_xfers(chan); - do { - sg = &active->sg[active->num_completed]; - if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ - break; - if (!(BIT(sg->id) & completed_transfers)) - break; - active->num_completed++; - sg->id = AXI_DMAC_SG_UNUSED; - if (sg->schedule_when_free) { - sg->schedule_when_free = false; - start_next = true; + if (chan->hw_sg) { + if (active->cyclic) { + vchan_cyclic_callback(&active->vdesc); + } else { + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + active = axi_dmac_active_desc(chan); + start_next = !!active; } + } else { + do { + sg = &active->sg[active->num_completed]; + if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ + break; + if (!(BIT(sg->hw->id) & completed_transfers)) + break; + active->num_completed++; + sg->hw->id = AXI_DMAC_SG_UNUSED; + if (sg->schedule_when_free) { + sg->schedule_when_free = false; + start_next = true; + } - if (sg->partial_len) - axi_dmac_compute_residue(chan, active); + if (sg->partial_len) + axi_dmac_compute_residue(chan, active); - if (active->cyclic) - vchan_cyclic_callback(&active->vdesc); + if (active->cyclic) + vchan_cyclic_callback(&active->vdesc); - if (active->num_completed == active->num_sgs || - sg->partial_len) { - if (active->cyclic) { - active->num_completed = 0; /* wrap around */ - } else { - list_del(&active->vdesc.node); - vchan_cookie_complete(&active->vdesc); - active = axi_dmac_active_desc(chan); + if (active->num_completed == active->num_sgs || + sg->partial_len) { + if (active->cyclic) { + active->num_completed = 0; /* wrap around */ + } else { + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + active = axi_dmac_active_desc(chan); + } } - } - } while (active); + } while (active); + } return start_next; } @@ -467,8 +514,12 @@ static void axi_dmac_issue_pending(struct dma_chan *c) struct axi_dmac_chan *chan = to_axi_dmac_chan(c); struct axi_dmac *dmac = chan_to_axi_dmac(chan); unsigned long flags; + u32 ctrl = AXI_DMAC_CTRL_ENABLE; - axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); + if (chan->hw_sg) + ctrl |= AXI_DMAC_CTRL_ENABLE_SG; + + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl); spin_lock_irqsave(&chan->vchan.lock, flags); if (vchan_issue_pending(&chan->vchan)) @@ -476,23 +527,58 @@ static void axi_dmac_issue_pending(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); } -static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs) +static struct axi_dmac_desc * +axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) { + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + struct device *dev = dmac->dma_dev.dev; + struct axi_dmac_hw_desc *hws; struct axi_dmac_desc *desc; + dma_addr_t hw_phys; unsigned int i; desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = num_sgs; + desc->chan = chan; - for (i = 0; i < num_sgs; i++) - desc->sg[i].id = AXI_DMAC_SG_UNUSED; + hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)), + &hw_phys, GFP_ATOMIC); + if (!hws) { + kfree(desc); + return NULL; + } - desc->num_sgs = num_sgs; + for (i = 0; i < num_sgs; i++) { + desc->sg[i].hw = &hws[i]; + desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws); + + hws[i].id = AXI_DMAC_SG_UNUSED; + hws[i].flags = 0; + + /* Link hardware descriptors */ + hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws); + } + + /* The last hardware descriptor will trigger an interrupt */ + desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ; return desc; } +static void axi_dmac_free_desc(struct axi_dmac_desc *desc) +{ + struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan); + struct device *dev = dmac->dma_dev.dev; + struct axi_dmac_hw_desc *hw = desc->sg[0].hw; + dma_addr_t hw_phys = desc->sg[0].hw_phys; + + dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)), + hw, hw_phys); + kfree(desc); +} + static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, enum dma_transfer_direction direction, dma_addr_t addr, unsigned int num_periods, unsigned int period_len, @@ -509,26 +595,24 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; for (i = 0; i < num_periods; i++) { - len = period_len; - - while (len > segment_size) { + for (len = period_len; len > segment_size; sg++) { if (direction == DMA_DEV_TO_MEM) - sg->dest_addr = addr; + sg->hw->dest_addr = addr; else - sg->src_addr = addr; - sg->x_len = segment_size; - sg->y_len = 1; - sg++; + sg->hw->src_addr = addr; + sg->hw->x_len = segment_size - 1; + sg->hw->y_len = 0; + sg->hw->flags = 0; addr += segment_size; len -= segment_size; } if (direction == DMA_DEV_TO_MEM) - sg->dest_addr = addr; + sg->hw->dest_addr = addr; else - sg->src_addr = addr; - sg->x_len = len; - sg->y_len = 1; + sg->hw->src_addr = addr; + sg->hw->x_len = len - 1; + sg->hw->y_len = 0; sg++; addr += len; } @@ -536,6 +620,45 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, return sg; } +static struct dma_async_tx_descriptor * +axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, + size_t nb, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct axi_dmac_desc *desc; + unsigned int num_sgs = 0; + struct axi_dmac_sg *dsg; + size_t i; + + if (direction != chan->direction) + return NULL; + + for (i = 0; i < nb; i++) + num_sgs += DIV_ROUND_UP(vecs[i].len, chan->max_length); + + desc = axi_dmac_alloc_desc(chan, num_sgs); + if (!desc) + return NULL; + + dsg = desc->sg; + + for (i = 0; i < nb; i++) { + if (!axi_dmac_check_addr(chan, vecs[i].addr) || + !axi_dmac_check_len(chan, vecs[i].len)) { + kfree(desc); + return NULL; + } + + dsg = axi_dmac_fill_linear_sg(chan, direction, vecs[i].addr, 1, + vecs[i].len, dsg); + } + + desc->cyclic = false; + + return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); +} + static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( struct dma_chan *c, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, @@ -555,7 +678,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length); - desc = axi_dmac_alloc_desc(num_sgs); + desc = axi_dmac_alloc_desc(chan, num_sgs); if (!desc) return NULL; @@ -564,7 +687,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) { if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || !axi_dmac_check_len(chan, sg_dma_len(sg))) { - kfree(desc); + axi_dmac_free_desc(desc); return NULL; } @@ -584,7 +707,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( { struct axi_dmac_chan *chan = to_axi_dmac_chan(c); struct axi_dmac_desc *desc; - unsigned int num_periods, num_segments; + unsigned int num_periods, num_segments, num_sgs; if (direction != chan->direction) return NULL; @@ -598,11 +721,16 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( num_periods = buf_len / period_len; num_segments = DIV_ROUND_UP(period_len, chan->max_length); + num_sgs = num_periods * num_segments; - desc = axi_dmac_alloc_desc(num_periods * num_segments); + desc = axi_dmac_alloc_desc(chan, num_sgs); if (!desc) return NULL; + /* Chain the last descriptor to the first, and remove its "last" flag */ + desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; + desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; + axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods, period_len, desc->sg); @@ -654,26 +782,26 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( return NULL; } - desc = axi_dmac_alloc_desc(1); + desc = axi_dmac_alloc_desc(chan, 1); if (!desc) return NULL; if (axi_dmac_src_is_mem(chan)) { - desc->sg[0].src_addr = xt->src_start; - desc->sg[0].src_stride = xt->sgl[0].size + src_icg; + desc->sg[0].hw->src_addr = xt->src_start; + desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg; } if (axi_dmac_dest_is_mem(chan)) { - desc->sg[0].dest_addr = xt->dst_start; - desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg; + desc->sg[0].hw->dest_addr = xt->dst_start; + desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg; } if (chan->hw_2d) { - desc->sg[0].x_len = xt->sgl[0].size; - desc->sg[0].y_len = xt->numf; + desc->sg[0].hw->x_len = xt->sgl[0].size - 1; + desc->sg[0].hw->y_len = xt->numf - 1; } else { - desc->sg[0].x_len = xt->sgl[0].size * xt->numf; - desc->sg[0].y_len = 1; + desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1; + desc->sg[0].hw->y_len = 0; } if (flags & DMA_CYCLIC) @@ -689,7 +817,7 @@ static void axi_dmac_free_chan_resources(struct dma_chan *c) static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) { - kfree(container_of(vdesc, struct axi_dmac_desc, vdesc)); + axi_dmac_free_desc(to_axi_dmac_desc(vdesc)); } static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) @@ -715,6 +843,9 @@ static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) case AXI_DMAC_REG_CURRENT_DEST_ADDR: case AXI_DMAC_REG_PARTIAL_XFER_LEN: case AXI_DMAC_REG_PARTIAL_XFER_ID: + case AXI_DMAC_REG_CURRENT_SG_ID: + case AXI_DMAC_REG_SG_ADDRESS: + case AXI_DMAC_REG_SG_ADDRESS_HIGH: return true; default: return false; @@ -867,6 +998,10 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) chan->hw_cyclic = true; + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff); + if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS)) + chan->hw_sg = true; + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) chan->hw_2d = true; @@ -906,12 +1041,23 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) return 0; } +static void axi_dmac_tasklet_kill(void *task) +{ + tasklet_kill(task); +} + +static void axi_dmac_free_dma_controller(void *of_node) +{ + of_dma_controller_free(of_node); +} + static int axi_dmac_probe(struct platform_device *pdev) { struct dma_device *dma_dev; struct axi_dmac *dmac; struct regmap *regmap; unsigned int version; + u32 irq_mask = 0; int ret; dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); @@ -928,14 +1074,10 @@ static int axi_dmac_probe(struct platform_device *pdev) if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); - dmac->clk = devm_clk_get(&pdev->dev, NULL); + dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(dmac->clk)) return PTR_ERR(dmac->clk); - ret = clk_prepare_enable(dmac->clk); - if (ret < 0) - return ret; - version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); if (version >= ADI_AXI_PCORE_VER(4, 3, 'a')) @@ -944,7 +1086,7 @@ static int axi_dmac_probe(struct platform_device *pdev) ret = axi_dmac_parse_dt(&pdev->dev, dmac); if (ret < 0) - goto err_clk_disable; + return ret; INIT_LIST_HEAD(&dmac->chan.active_descs); @@ -958,6 +1100,7 @@ static int axi_dmac_probe(struct platform_device *pdev) dma_dev->device_tx_status = dma_cookie_status; dma_dev->device_issue_pending = axi_dmac_issue_pending; dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg; + dma_dev->device_prep_peripheral_dma_vec = axi_dmac_prep_peripheral_dma_vec; dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic; dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved; dma_dev->device_terminate_all = axi_dmac_terminate_all; @@ -967,6 +1110,7 @@ static int axi_dmac_probe(struct platform_device *pdev) dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); dma_dev->directions = BIT(dmac->chan.direction); dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */ INIT_LIST_HEAD(&dma_dev->channels); dmac->chan.vchan.desc_free = axi_dmac_desc_free; @@ -974,11 +1118,14 @@ static int axi_dmac_probe(struct platform_device *pdev) ret = axi_dmac_detect_caps(dmac, version); if (ret) - goto err_clk_disable; + return ret; dma_dev->copy_align = (dmac->chan.address_align_mask + 1); - axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00); + if (dmac->chan.hw_sg) + irq_mask |= AXI_DMAC_IRQ_SOT; + + axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask); if (of_dma_is_coherent(pdev->dev.of_node)) { ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC); @@ -987,59 +1134,42 @@ static int axi_dmac_probe(struct platform_device *pdev) !AXI_DMAC_DST_COHERENT_GET(ret)) { dev_err(dmac->dma_dev.dev, "Coherent DMA not supported in hardware"); - ret = -EINVAL; - goto err_clk_disable; + return -EINVAL; } } - ret = dma_async_device_register(dma_dev); + ret = dmaenginem_async_device_register(dma_dev); if (ret) - goto err_clk_disable; + return ret; + + /* + * Put the action in here so it get's done before unregistering the DMA + * device. + */ + ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill, + &dmac->chan.vchan.task); + if (ret) + return ret; ret = of_dma_controller_register(pdev->dev.of_node, of_dma_xlate_by_chan_id, dma_dev); if (ret) - goto err_unregister_device; + return ret; - ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED, - dev_name(&pdev->dev), dmac); + ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller, + pdev->dev.of_node); if (ret) - goto err_unregister_of; + return ret; - platform_set_drvdata(pdev, dmac); + ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler, + IRQF_SHARED, dev_name(&pdev->dev), dmac); + if (ret) + return ret; regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base, &axi_dmac_regmap_config); - if (IS_ERR(regmap)) { - ret = PTR_ERR(regmap); - goto err_free_irq; - } - - return 0; - -err_free_irq: - free_irq(dmac->irq, dmac); -err_unregister_of: - of_dma_controller_free(pdev->dev.of_node); -err_unregister_device: - dma_async_device_unregister(&dmac->dma_dev); -err_clk_disable: - clk_disable_unprepare(dmac->clk); - return ret; -} - -static int axi_dmac_remove(struct platform_device *pdev) -{ - struct axi_dmac *dmac = platform_get_drvdata(pdev); - - of_dma_controller_free(pdev->dev.of_node); - free_irq(dmac->irq, dmac); - tasklet_kill(&dmac->chan.vchan.task); - dma_async_device_unregister(&dmac->dma_dev); - clk_disable_unprepare(dmac->clk); - - return 0; + return PTR_ERR_OR_ZERO(regmap); } static const struct of_device_id axi_dmac_of_match_table[] = { @@ -1054,7 +1184,6 @@ static struct platform_driver axi_dmac_driver = { .of_match_table = axi_dmac_of_match_table, }, .probe = axi_dmac_probe, - .remove = axi_dmac_remove, }; module_platform_driver(axi_dmac_driver); diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index 9c1a6e9a9c03..100057603fd4 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -13,7 +13,6 @@ #include <linux/interrupt.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -1009,7 +1008,7 @@ err_disable_clk: return ret; } -static int jz4780_dma_remove(struct platform_device *pdev) +static void jz4780_dma_remove(struct platform_device *pdev) { struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); int i; @@ -1021,8 +1020,6 @@ static int jz4780_dma_remove(struct platform_device *pdev) for (i = 0; i < jzdma->soc_data->nb_channels; i++) tasklet_kill(&jzdma->chan[i].vchan.task); - - return 0; } static const struct jz4780_dma_soc_data jz4740_dma_soc_data = { diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 826b98284fa1..ca13cd39330b 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -40,6 +40,8 @@ #include <linux/dmaengine.h> #include <linux/hardirq.h> #include <linux/spinlock.h> +#include <linux/of.h> +#include <linux/property.h> #include <linux/percpu.h> #include <linux/rcupdate.h> #include <linux/mutex.h> @@ -812,15 +814,13 @@ static const struct dma_slave_map *dma_filter_match(struct dma_device *device, */ struct dma_chan *dma_request_chan(struct device *dev, const char *name) { + struct fwnode_handle *fwnode = dev_fwnode(dev); struct dma_device *d, *_d; struct dma_chan *chan = NULL; - /* If device-tree is present get slave info from here */ - if (dev->of_node) - chan = of_dma_request_slave_channel(dev->of_node, name); - - /* If device was enumerated by ACPI get slave info from here */ - if (has_acpi_companion(dev) && !chan) + if (is_of_node(fwnode)) + chan = of_dma_request_slave_channel(to_of_node(fwnode), name); + else if (is_acpi_device_node(fwnode)) chan = acpi_dma_request_slave_chan_by_name(dev, name); if (PTR_ERR(chan) == -EPROBE_DEFER) @@ -854,8 +854,8 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name) found: #ifdef CONFIG_DEBUG_FS - chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), - name); + chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), name); + /* No functional issue if it fails, users are supposed to test before use */ #endif chan->name = kasprintf(GFP_KERNEL, "dma:%s", name); @@ -926,6 +926,36 @@ void dma_release_channel(struct dma_chan *chan) } EXPORT_SYMBOL_GPL(dma_release_channel); +static void dmaenginem_release_channel(void *chan) +{ + dma_release_channel(chan); +} + +/** + * devm_dma_request_chan - try to allocate an exclusive slave channel + * @dev: pointer to client device structure + * @name: slave channel name + * + * Returns pointer to appropriate DMA channel on success or an error pointer. + * + * The operation is managed and will be undone on driver detach. + */ + +struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name) +{ + struct dma_chan *chan = dma_request_chan(dev, name); + int ret = 0; + + if (!IS_ERR(chan)) + ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan); + + if (ret) + return ERR_PTR(ret); + + return chan; +} +EXPORT_SYMBOL_GPL(devm_dma_request_chan); + /** * dmaengine_get - register interest in dma_channels */ @@ -1037,7 +1067,8 @@ static int get_dma_id(struct dma_device *device) } static int __dma_async_device_channel_register(struct dma_device *device, - struct dma_chan *chan) + struct dma_chan *chan, + const char *name) { int rc; @@ -1066,8 +1097,10 @@ static int __dma_async_device_channel_register(struct dma_device *device, chan->dev->device.parent = device->dev; chan->dev->chan = chan; chan->dev->dev_id = device->dev_id; - dev_set_name(&chan->dev->device, "dma%dchan%d", - device->dev_id, chan->chan_id); + if (!name) + dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id); + else + dev_set_name(&chan->dev->device, "%s", name); rc = device_register(&chan->dev->device); if (rc) goto err_out_ida; @@ -1087,11 +1120,12 @@ static int __dma_async_device_channel_register(struct dma_device *device, } int dma_async_device_channel_register(struct dma_device *device, - struct dma_chan *chan) + struct dma_chan *chan, + const char *name) { int rc; - rc = __dma_async_device_channel_register(device, chan); + rc = __dma_async_device_channel_register(device, chan, name); if (rc < 0) return rc; @@ -1103,6 +1137,9 @@ EXPORT_SYMBOL_GPL(dma_async_device_channel_register); static void __dma_async_device_channel_unregister(struct dma_device *device, struct dma_chan *chan) { + if (chan->local == NULL) + return; + WARN_ONCE(!device->device_release && chan->client_count, "%s called while %d clients hold a reference\n", __func__, chan->client_count); @@ -1147,69 +1184,27 @@ int dma_async_device_register(struct dma_device *device) device->owner = device->dev->driver->owner; - if (dma_has_cap(DMA_MEMCPY, device->cap_mask) && !device->device_prep_dma_memcpy) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_MEMCPY"); - return -EIO; - } - - if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_XOR"); - return -EIO; - } - - if (dma_has_cap(DMA_XOR_VAL, device->cap_mask) && !device->device_prep_dma_xor_val) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_XOR_VAL"); - return -EIO; - } - - if (dma_has_cap(DMA_PQ, device->cap_mask) && !device->device_prep_dma_pq) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_PQ"); - return -EIO; - } - - if (dma_has_cap(DMA_PQ_VAL, device->cap_mask) && !device->device_prep_dma_pq_val) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_PQ_VAL"); - return -EIO; - } - - if (dma_has_cap(DMA_MEMSET, device->cap_mask) && !device->device_prep_dma_memset) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_MEMSET"); - return -EIO; - } - - if (dma_has_cap(DMA_INTERRUPT, device->cap_mask) && !device->device_prep_dma_interrupt) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_INTERRUPT"); - return -EIO; - } - - if (dma_has_cap(DMA_CYCLIC, device->cap_mask) && !device->device_prep_dma_cyclic) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_CYCLIC"); - return -EIO; - } +#define CHECK_CAP(_name, _type) \ +{ \ + if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) { \ + dev_err(device->dev, \ + "Device claims capability %s, but op is not defined\n", \ + __stringify(_type)); \ + return -EIO; \ + } \ +} - if (dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && !device->device_prep_interleaved_dma) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_INTERLEAVE"); - return -EIO; - } + CHECK_CAP(dma_memcpy, DMA_MEMCPY); + CHECK_CAP(dma_xor, DMA_XOR); + CHECK_CAP(dma_xor_val, DMA_XOR_VAL); + CHECK_CAP(dma_pq, DMA_PQ); + CHECK_CAP(dma_pq_val, DMA_PQ_VAL); + CHECK_CAP(dma_memset, DMA_MEMSET); + CHECK_CAP(dma_interrupt, DMA_INTERRUPT); + CHECK_CAP(dma_cyclic, DMA_CYCLIC); + CHECK_CAP(interleaved_dma, DMA_INTERLEAVE); +#undef CHECK_CAP if (!device->device_tx_status) { dev_err(device->dev, "Device tx_status is not defined\n"); @@ -1242,7 +1237,7 @@ int dma_async_device_register(struct dma_device *device) /* represent channels in sysfs. Probably want devs too */ list_for_each_entry(chan, &device->channels, device_node) { - rc = __dma_async_device_channel_register(device, chan); + rc = __dma_async_device_channel_register(device, chan, NULL); if (rc < 0) goto err_out; } diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index ffe621695e47..91b2fbc0b864 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -21,6 +21,10 @@ #include <linux/slab.h> #include <linux/wait.h> +static bool nobounce; +module_param(nobounce, bool, 0644); +MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)"); + static unsigned int test_buf_size = 16384; module_param(test_buf_size, uint, 0644); MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); @@ -90,6 +94,7 @@ MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts"); /** * struct dmatest_params - test parameters. + * @nobounce: prevent using swiotlb buffer * @buf_size: size of the memcpy test buffer * @channel: bus ID of the channel to test * @device: bus ID of the DMA Engine to test @@ -106,6 +111,7 @@ MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts"); * @polled: use polling for completion instead of interrupts */ struct dmatest_params { + bool nobounce; unsigned int buf_size; char channel[20]; char device[32]; @@ -215,6 +221,7 @@ struct dmatest_done { struct dmatest_data { u8 **raw; u8 **aligned; + gfp_t gfp_flags; unsigned int cnt; unsigned int off; }; @@ -493,7 +500,7 @@ static unsigned long long dmatest_persec(s64 runtime, unsigned int val) per_sec *= val; per_sec = INT_TO_FIXPT(per_sec); - do_div(per_sec, runtime); + do_div(per_sec, (u32)runtime); return per_sec; } @@ -533,7 +540,7 @@ static int dmatest_alloc_test_data(struct dmatest_data *d, goto err; for (i = 0; i < d->cnt; i++) { - d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); + d->raw[i] = kmalloc(buf_size + align, d->gfp_flags); if (!d->raw[i]) goto err; @@ -655,6 +662,13 @@ static int dmatest_func(void *data) goto err_free_coefs; } + src->gfp_flags = GFP_KERNEL; + dst->gfp_flags = GFP_KERNEL; + if (params->nobounce) { + src->gfp_flags = GFP_DMA; + dst->gfp_flags = GFP_DMA; + } + if (dmatest_alloc_test_data(src, buf_size, align) < 0) goto err_free_coefs; @@ -1093,6 +1107,7 @@ static void add_threaded_test(struct dmatest_info *info) struct dmatest_params *params = &info->params; /* Copy test parameters */ + params->nobounce = nobounce; params->buf_size = test_buf_size; strscpy(params->channel, strim(test_channel), sizeof(params->channel)); strscpy(params->device, strim(test_device), sizeof(params->device)); @@ -1357,4 +1372,5 @@ static void __exit dmatest_exit(void) module_exit(dmatest_exit); MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); +MODULE_DESCRIPTION("DMA Engine test module"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 796b6caf0bab..b23536645ff7 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -21,7 +21,6 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -64,6 +63,17 @@ static inline u32 axi_dma_ioread32(struct axi_dma_chip *chip, u32 reg) } static inline void +axi_dma_iowrite64(struct axi_dma_chip *chip, u32 reg, u64 val) +{ + iowrite64(val, chip->regs + reg); +} + +static inline u64 axi_dma_ioread64(struct axi_dma_chip *chip, u32 reg) +{ + return ioread64(chip->regs + reg); +} + +static inline void axi_chan_iowrite32(struct axi_dma_chan *chan, u32 reg, u32 val) { iowrite32(val, chan->chan_regs + reg); @@ -183,38 +193,73 @@ static inline u32 axi_chan_irq_read(struct axi_dma_chan *chan) static inline void axi_chan_disable(struct axi_dma_chan *chan) { - u32 val; - - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT); - if (chan->chip->dw->hdata->reg_map_8_channels) - val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; - else - val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + u64 val; + + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHEN); + if (chan->id >= DMAC_CHAN_16) { + val &= ~((u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN_SHIFT + DMAC_CHAN_BLOCK_SHIFT)); + val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT); + } else { + val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT); + val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; + } + axi_dma_iowrite64(chan->chip, DMAC_CHEN, val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT); + if (chan->chip->dw->hdata->reg_map_8_channels) + val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; + else + val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } } static inline void axi_chan_enable(struct axi_dma_chan *chan) { - u32 val; - - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - if (chan->chip->dw->hdata->reg_map_8_channels) - val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | - BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; - else - val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | + u64 val; + + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHEN); + if (chan->id >= DMAC_CHAN_16) { + val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN_SHIFT + DMAC_CHAN_BLOCK_SHIFT) | + (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT); + } else { + val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + } + axi_dma_iowrite64(chan->chip, DMAC_CHEN, val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + if (chan->chip->dw->hdata->reg_map_8_channels) { + val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | + BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; + } else { + val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | + BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; + } + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } } static inline bool axi_chan_is_hw_enable(struct axi_dma_chan *chan) { - u32 val; + u64 val; - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) + val = axi_dma_ioread64(chan->chip, DMAC_CHEN); + else + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - return !!(val & (BIT(chan->id) << DMAC_CHAN_EN_SHIFT)); + if (chan->id >= DMAC_CHAN_16) + return !!(val & ((u64)(BIT(chan->id) >> DMAC_CHAN_16) << DMAC_CHAN_BLOCK_SHIFT)); + else + return !!(val & (BIT(chan->id) << DMAC_CHAN_EN_SHIFT)); } static void axi_dma_hw_init(struct axi_dma_chip *chip) @@ -257,6 +302,7 @@ static struct axi_dma_desc *axi_desc_alloc(u32 num) kfree(desc); return NULL; } + desc->nr_hw_descs = num; return desc; } @@ -283,7 +329,7 @@ static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan, static void axi_desc_put(struct axi_dma_desc *desc) { struct axi_dma_chan *chan = desc->chan; - int count = atomic_read(&chan->descs_allocated); + int count = desc->nr_hw_descs; struct axi_dma_hw_desc *hw_desc; int descs_put; @@ -1094,9 +1140,6 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* Remove the completed descriptor from issued list before completing */ list_del(&vd->node); vchan_cookie_complete(vd); - - /* Submit queued descriptors after processing the completed ones */ - axi_chan_start_first_queued(chan); } out: @@ -1176,20 +1219,34 @@ static int dma_chan_pause(struct dma_chan *dchan) struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); unsigned long flags; unsigned int timeout = 20; /* timeout iterations */ - u32 val; + u64 val; spin_lock_irqsave(&chan->vc.lock, flags); - if (chan->chip->dw->hdata->reg_map_8_channels) { - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT | - BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHSUSPREG); + if (chan->id >= DMAC_CHAN_16) { + val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_SHIFT + DMAC_CHAN_BLOCK_SHIFT) | + (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT); + } else { + val |= BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT | + BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT; + } + axi_dma_iowrite64(chan->chip, DMAC_CHSUSPREG, val); } else { - val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG); - val |= BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT | + if (chan->chip->dw->hdata->reg_map_8_channels) { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT | + BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT; + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG); + val |= BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT | BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, val); + axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, (u32)val); + } } do { @@ -1211,18 +1268,32 @@ static int dma_chan_pause(struct dma_chan *dchan) /* Called in chan locked context */ static inline void axi_chan_resume(struct axi_dma_chan *chan) { - u32 val; - - if (chan->chip->dw->hdata->reg_map_8_channels) { - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT); - val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT); - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + u64 val; + + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHSUSPREG); + if (chan->id >= DMAC_CHAN_16) { + val &= ~((u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_SHIFT + DMAC_CHAN_BLOCK_SHIFT)); + val |= ((u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT)); + } else { + val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT); + val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT); + } + axi_dma_iowrite64(chan->chip, DMAC_CHSUSPREG, val); } else { - val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG); - val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT); - val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT); - axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, val); + if (chan->chip->dw->hdata->reg_map_8_channels) { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT); + val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT); + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG); + val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT); + val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT); + axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, (u32)val); + } } chan->is_paused = false; @@ -1372,6 +1443,24 @@ static int parse_device_properties(struct axi_dma_chip *chip) return 0; } +static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip) +{ + int irq_count = platform_irq_count(pdev); + int ret; + + for (int i = 0; i < irq_count; i++) { + chip->irq[i] = platform_get_irq(pdev, i); + if (chip->irq[i] < 0) + return chip->irq[i]; + ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt, + IRQF_SHARED, KBUILD_MODNAME, chip); + if (ret < 0) + return ret; + } + + return 0; +} + static int dw_probe(struct platform_device *pdev) { struct axi_dma_chip *chip; @@ -1398,10 +1487,6 @@ static int dw_probe(struct platform_device *pdev) chip->dev = &pdev->dev; chip->dw->hdata = hdata; - chip->irq = platform_get_irq(pdev, 0); - if (chip->irq < 0) - return chip->irq; - chip->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); @@ -1442,8 +1527,7 @@ static int dw_probe(struct platform_device *pdev) if (!dw->chan) return -ENOMEM; - ret = devm_request_irq(chip->dev, chip->irq, dw_axi_dma_interrupt, - IRQF_SHARED, KBUILD_MODNAME, chip); + ret = axi_req_irqs(pdev, chip); if (ret) return ret; @@ -1536,7 +1620,7 @@ err_pm_disable: return ret; } -static int dw_remove(struct platform_device *pdev) +static void dw_remove(struct platform_device *pdev) { struct axi_dma_chip *chip = platform_get_drvdata(pdev); struct dw_axi_dma *dw = chip->dw; @@ -1556,7 +1640,9 @@ static int dw_remove(struct platform_device *pdev) pm_runtime_disable(chip->dev); axi_dma_suspend(chip); - devm_free_irq(chip->dev, chip->irq, chip); + for (i = 0; i < DMAC_MAX_CHANNELS; i++) + if (chip->irq[i] > 0) + devm_free_irq(chip->dev, chip->irq[i], chip); of_dma_controller_free(chip->dev->of_node); @@ -1565,8 +1651,6 @@ static int dw_remove(struct platform_device *pdev) list_del(&chan->vc.chan.device_node); tasklet_kill(&chan->vc.task); } - - return 0; } static const struct dev_pm_ops dw_axi_dma_pm_ops = { @@ -1582,6 +1666,9 @@ static const struct of_device_id dw_dma_of_id_table[] = { }, { .compatible = "starfive,jh7110-axi-dma", .data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2), + }, { + .compatible = "starfive,jh8100-axi-dma", + .data = (void *)AXI_DMA_FLAG_HAS_RESETS, }, {} }; diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index eb267cb24f67..b842e6a8d90d 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -18,7 +18,7 @@ #include "../virt-dma.h" -#define DMAC_MAX_CHANNELS 16 +#define DMAC_MAX_CHANNELS 32 #define DMAC_MAX_MASTERS 2 #define DMAC_MAX_BLK_SIZE 0x200000 @@ -65,7 +65,7 @@ struct dw_axi_dma { struct axi_dma_chip { struct device *dev; - int irq; + int irq[DMAC_MAX_CHANNELS]; void __iomem *regs; void __iomem *apb_regs; struct clk *core_clk; @@ -104,6 +104,7 @@ struct axi_dma_desc { u32 completed_blocks; u32 length; u32 period_len; + u32 nr_hw_descs; }; struct axi_dma_chan_config { @@ -222,6 +223,10 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) /* DMAC_CHEN2 */ #define DMAC_CHAN_EN2_WE_SHIFT 16 +/* DMAC CHAN BLOCKS */ +#define DMAC_CHAN_BLOCK_SHIFT 32 +#define DMAC_CHAN_16 16 + /* DMAC_CHSUSP */ #define DMAC_CHAN_SUSP2_SHIFT 0 #define DMAC_CHAN_SUSP2_WE_SHIFT 16 diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 68236247059d..8e5f7defa6b6 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -15,6 +15,7 @@ #include <linux/irq.h> #include <linux/dma/edma.h> #include <linux/dma-mapping.h> +#include <linux/string_choices.h> #include "dw-edma-core.h" #include "dw-edma-v0-core.h" @@ -23,18 +24,6 @@ #include "../virt-dma.h" static inline -struct device *dchan2dev(struct dma_chan *dchan) -{ - return &dchan->dev->device; -} - -static inline -struct device *chan2dev(struct dw_edma_chan *chan) -{ - return &chan->vc.chan.dev->device; -} - -static inline struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) { return container_of(vd, struct dw_edma_desc, vd); @@ -595,6 +584,25 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, return dw_edma_device_transfer(&xfer); } +static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, + enum dmaengine_tx_result result) +{ + u32 residue = 0; + struct dw_edma_desc *desc; + struct dmaengine_result *res; + + if (!vd->tx.callback_result) + return; + + desc = vd2dw_edma_desc(vd); + if (desc) + residue = desc->alloc_sz - desc->xfer_sz; + + res = &vd->tx_result; + res->result = result; + res->residue = residue; +} + static void dw_edma_done_interrupt(struct dw_edma_chan *chan) { struct dw_edma_desc *desc; @@ -608,6 +616,8 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) case EDMA_REQ_NONE: desc = vd2dw_edma_desc(vd); if (!desc->chunks_alloc) { + dw_hdma_set_callback_result(vd, + DMA_TRANS_NOERROR); list_del(&vd->node); vchan_cookie_complete(vd); } @@ -644,6 +654,7 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); if (vd) { + dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); } @@ -746,7 +757,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) chan->ll_max -= 1; dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", - chan->dir == EDMA_DIR_WRITE ? "write" : "read", + str_write_read(chan->dir == EDMA_DIR_WRITE), chan->id, chan->ll_max); if (dw->nr_irqs == 1) @@ -767,7 +778,8 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", - chan->dir == EDMA_DIR_WRITE ? "write" : "read", chan->id, + str_write_read(chan->dir == EDMA_DIR_WRITE), + chan->id, chan->msi.address_hi, chan->msi.address_lo, chan->msi.data); diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 1c6043751dc9..3371e0a76d3c 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -136,7 +136,8 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, map = FIELD_GET(DW_PCIE_VSEC_DMA_MAP, val); if (map != EDMA_MF_EDMA_LEGACY && map != EDMA_MF_EDMA_UNROLL && - map != EDMA_MF_HDMA_COMPAT) + map != EDMA_MF_HDMA_COMPAT && + map != EDMA_MF_HDMA_NATIVE) return; pdata->mf = map; @@ -160,12 +161,16 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { struct dw_edma_pcie_data *pdata = (void *)pid->driver_data; - struct dw_edma_pcie_data vsec_data; + struct dw_edma_pcie_data *vsec_data __free(kfree) = NULL; struct device *dev = &pdev->dev; struct dw_edma_chip *chip; int err, nr_irqs; int i, mask; + vsec_data = kmalloc(sizeof(*vsec_data), GFP_KERNEL); + if (!vsec_data) + return -ENOMEM; + /* Enable PCI device */ err = pcim_enable_device(pdev); if (err) { @@ -173,23 +178,23 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return err; } - memcpy(&vsec_data, pdata, sizeof(struct dw_edma_pcie_data)); + memcpy(vsec_data, pdata, sizeof(struct dw_edma_pcie_data)); /* * Tries to find if exists a PCIe Vendor-Specific Extended Capability * for the DMA, if one exists, then reconfigures it. */ - dw_edma_pcie_get_vsec_dma_data(pdev, &vsec_data); + dw_edma_pcie_get_vsec_dma_data(pdev, vsec_data); /* Mapping PCI BAR regions */ - mask = BIT(vsec_data.rg.bar); - for (i = 0; i < vsec_data.wr_ch_cnt; i++) { - mask |= BIT(vsec_data.ll_wr[i].bar); - mask |= BIT(vsec_data.dt_wr[i].bar); + mask = BIT(vsec_data->rg.bar); + for (i = 0; i < vsec_data->wr_ch_cnt; i++) { + mask |= BIT(vsec_data->ll_wr[i].bar); + mask |= BIT(vsec_data->dt_wr[i].bar); } - for (i = 0; i < vsec_data.rd_ch_cnt; i++) { - mask |= BIT(vsec_data.ll_rd[i].bar); - mask |= BIT(vsec_data.dt_rd[i].bar); + for (i = 0; i < vsec_data->rd_ch_cnt; i++) { + mask |= BIT(vsec_data->ll_rd[i].bar); + mask |= BIT(vsec_data->dt_rd[i].bar); } err = pcim_iomap_regions(pdev, mask, pci_name(pdev)); if (err) { @@ -212,7 +217,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; /* IRQs allocation */ - nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data.irqs, + nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data->irqs, PCI_IRQ_MSI | PCI_IRQ_MSIX); if (nr_irqs < 1) { pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n", @@ -223,22 +228,22 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, /* Data structure initialization */ chip->dev = dev; - chip->mf = vsec_data.mf; + chip->mf = vsec_data->mf; chip->nr_irqs = nr_irqs; chip->ops = &dw_edma_pcie_plat_ops; - chip->ll_wr_cnt = vsec_data.wr_ch_cnt; - chip->ll_rd_cnt = vsec_data.rd_ch_cnt; + chip->ll_wr_cnt = vsec_data->wr_ch_cnt; + chip->ll_rd_cnt = vsec_data->rd_ch_cnt; - chip->reg_base = pcim_iomap_table(pdev)[vsec_data.rg.bar]; + chip->reg_base = pcim_iomap_table(pdev)[vsec_data->rg.bar]; if (!chip->reg_base) return -ENOMEM; for (i = 0; i < chip->ll_wr_cnt; i++) { struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; - struct dw_edma_block *ll_block = &vsec_data.ll_wr[i]; - struct dw_edma_block *dt_block = &vsec_data.dt_wr[i]; + struct dw_edma_block *ll_block = &vsec_data->ll_wr[i]; + struct dw_edma_block *dt_block = &vsec_data->dt_wr[i]; ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; if (!ll_region->vaddr.io) @@ -262,8 +267,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, for (i = 0; i < chip->ll_rd_cnt; i++) { struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; - struct dw_edma_block *ll_block = &vsec_data.ll_rd[i]; - struct dw_edma_block *dt_block = &vsec_data.dt_rd[i]; + struct dw_edma_block *ll_block = &vsec_data->ll_rd[i]; + struct dw_edma_block *dt_block = &vsec_data->dt_rd[i]; ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; if (!ll_region->vaddr.io) @@ -291,35 +296,37 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, pci_dbg(pdev, "Version:\teDMA Unroll (0x%x)\n", chip->mf); else if (chip->mf == EDMA_MF_HDMA_COMPAT) pci_dbg(pdev, "Version:\tHDMA Compatible (0x%x)\n", chip->mf); + else if (chip->mf == EDMA_MF_HDMA_NATIVE) + pci_dbg(pdev, "Version:\tHDMA Native (0x%x)\n", chip->mf); else pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", chip->mf); pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n", - vsec_data.rg.bar, vsec_data.rg.off, vsec_data.rg.sz, + vsec_data->rg.bar, vsec_data->rg.off, vsec_data->rg.sz, chip->reg_base); for (i = 0; i < chip->ll_wr_cnt; i++) { pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.ll_wr[i].bar, - vsec_data.ll_wr[i].off, chip->ll_region_wr[i].sz, + i, vsec_data->ll_wr[i].bar, + vsec_data->ll_wr[i].off, chip->ll_region_wr[i].sz, chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr); pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.dt_wr[i].bar, - vsec_data.dt_wr[i].off, chip->dt_region_wr[i].sz, + i, vsec_data->dt_wr[i].bar, + vsec_data->dt_wr[i].off, chip->dt_region_wr[i].sz, chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr); } for (i = 0; i < chip->ll_rd_cnt; i++) { pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.ll_rd[i].bar, - vsec_data.ll_rd[i].off, chip->ll_region_rd[i].sz, + i, vsec_data->ll_rd[i].bar, + vsec_data->ll_rd[i].off, chip->ll_region_rd[i].sz, chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr); pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.dt_rd[i].bar, - vsec_data.dt_rd[i].off, chip->dt_region_rd[i].sz, + i, vsec_data->dt_rd[i].bar, + vsec_data->dt_rd[i].off, chip->dt_region_rd[i].sz, chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr); } diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index b38786f0ad79..b75fdaffad9a 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -346,6 +346,20 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); } +static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +{ + /* + * In case of remote eDMA engine setup, the DW PCIe RP/EP internal + * configuration registers and application memory are normally accessed + * over different buses. Ensure LL-data reaches the memory before the + * doorbell register is toggled by issuing the dummy-read from the remote + * LL memory in a hope that the MRd TLP will return only after the + * last MWr TLP is completed + */ + if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chunk->ll_region.vaddr.io); +} + static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; @@ -412,6 +426,9 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) SET_CH_32(dw, chan->dir, chan->id, llp.msb, upper_32_bits(chunk->ll_region.paddr)); } + + dw_edma_v0_sync_ll_data(chunk); + /* Doorbell */ SET_RW_32(dw, chan->dir, doorbell, FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c index 0745d9e7d259..406f169b09a7 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c @@ -176,7 +176,7 @@ dw_edma_debugfs_regs_wr(struct dw_edma *dw, struct dentry *dent) }; struct dentry *regs_dent, *ch_dent; int nr_entries, i; - char name[16]; + char name[32]; regs_dent = debugfs_create_dir(WRITE_STR, dent); @@ -239,7 +239,7 @@ static noinline_for_stack void dw_edma_debugfs_regs_rd(struct dw_edma *dw, }; struct dentry *regs_dent, *ch_dent; int nr_entries, i; - char name[16]; + char name[32]; regs_dent = debugfs_create_dir(READ_STR, dent); diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 00b735a0202a..e3f8db4fe909 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -17,8 +17,8 @@ enum dw_hdma_control { DW_HDMA_V0_CB = BIT(0), DW_HDMA_V0_TCB = BIT(1), DW_HDMA_V0_LLP = BIT(2), - DW_HDMA_V0_LIE = BIT(3), - DW_HDMA_V0_RIE = BIT(4), + DW_HDMA_V0_LWIE = BIT(3), + DW_HDMA_V0_RWIE = BIT(4), DW_HDMA_V0_CCS = BIT(8), DW_HDMA_V0_LLE = BIT(9), }; @@ -65,18 +65,12 @@ static void dw_hdma_v0_core_off(struct dw_edma *dw) static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { - u32 num_ch = 0; - int id; - - for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) { - if (GET_CH_32(dw, id, dir, ch_en) & BIT(0)) - num_ch++; - } - - if (num_ch > HDMA_V0_MAX_NR_CH) - num_ch = HDMA_V0_MAX_NR_CH; - - return (u16)num_ch; + /* + * The HDMA IP have no way to know the number of hardware channels + * available, we set it to maximum channels and let the platform + * set the right number of channels. + */ + return HDMA_V0_MAX_NR_CH; } static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan) @@ -201,25 +195,14 @@ static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) { struct dw_edma_burst *child; - struct dw_edma_chan *chan = chunk->chan; u32 control = 0, i = 0; - int j; if (chunk->cb) control = DW_HDMA_V0_CB; - j = chunk->bursts_alloc; - list_for_each_entry(child, &chunk->burst->list, list) { - j--; - if (!j) { - control |= DW_HDMA_V0_LIE; - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - control |= DW_HDMA_V0_RIE; - } - + list_for_each_entry(child, &chunk->burst->list, list) dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz, child->sar, child->dar); - } control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; if (!chunk->cb) @@ -228,6 +211,20 @@ static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); } +static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +{ + /* + * In case of remote HDMA engine setup, the DW PCIe RP/EP internal + * configuration registers and application memory are normally accessed + * over different buses. Ensure LL-data reaches the memory before the + * doorbell register is toggled by issuing the dummy-read from the remote + * LL memory in a hope that the MRd TLP will return only after the + * last MWr TLP is completed + */ + if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chunk->ll_region.vaddr.io); +} + static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; @@ -239,10 +236,13 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) if (first) { /* Enable engine */ SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); - /* Interrupt enable&unmask - done, abort */ - tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup) | - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK | - HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN; + /* Interrupt unmask - stop, abort */ + tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); + tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + /* Interrupt enable - stop, abort */ + tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; + if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN; SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); /* Channel control */ SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); @@ -256,6 +256,9 @@ static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) /* Set consumer cycle */ SET_CH_32(dw, chan->dir, chan->id, cycle_sync, HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); + + dw_hdma_v0_sync_ll_data(chunk); + /* Doorbell */ SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); } diff --git a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c index 520c81978b08..dcdc57fe976c 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c @@ -116,7 +116,7 @@ static void dw_hdma_debugfs_regs_ch(struct dw_edma *dw, enum dw_edma_dir dir, static void dw_hdma_debugfs_regs_wr(struct dw_edma *dw, struct dentry *dent) { struct dentry *regs_dent, *ch_dent; - char name[16]; + char name[32]; int i; regs_dent = debugfs_create_dir(WRITE_STR, dent); @@ -133,7 +133,7 @@ static void dw_hdma_debugfs_regs_wr(struct dw_edma *dw, struct dentry *dent) static void dw_hdma_debugfs_regs_rd(struct dw_edma *dw, struct dentry *dent) { struct dentry *regs_dent, *ch_dent; - char name[16]; + char name[32]; int i; regs_dent = debugfs_create_dir(READ_STR, dent); diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h index a974abdf8aaf..eab5fd7177e5 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h @@ -15,7 +15,7 @@ #define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6) #define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5) #define HDMA_V0_LOCAL_STOP_INT_EN BIT(4) -#define HDMA_V0_REMOTEL_STOP_INT_EN BIT(3) +#define HDMA_V0_REMOTE_STOP_INT_EN BIT(3) #define HDMA_V0_ABORT_INT_MASK BIT(2) #define HDMA_V0_STOP_INT_MASK BIT(0) #define HDMA_V0_LINKLIST_EN BIT(0) diff --git a/drivers/dma/dw/acpi.c b/drivers/dma/dw/acpi.c index c510c109d2c3..b6452fffa657 100644 --- a/drivers/dma/dw/acpi.c +++ b/drivers/dma/dw/acpi.c @@ -8,13 +8,15 @@ static bool dw_dma_acpi_filter(struct dma_chan *chan, void *param) { + struct dw_dma *dw = to_dw_dma(chan->device); + struct dw_dma_chip_pdata *data = dev_get_drvdata(dw->dma.dev); struct acpi_dma_spec *dma_spec = param; struct dw_dma_slave slave = { .dma_dev = dma_spec->dev, .src_id = dma_spec->slave_id, .dst_id = dma_spec->slave_id, - .m_master = 0, - .p_master = 1, + .m_master = data->m_master, + .p_master = data->p_master, }; return dw_dma_filter(chan, &slave); diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index 5f7d690e3dba..dd75f97a33b3 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -16,6 +16,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/log2.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/slab.h> @@ -621,12 +622,10 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct dw_desc *prev; struct dw_desc *first; u32 ctllo, ctlhi; - u8 m_master = dwc->dws.m_master; - u8 lms = DWC_LLP_LMS(m_master); + u8 lms = DWC_LLP_LMS(dwc->dws.m_master); dma_addr_t reg; unsigned int reg_width; unsigned int mem_width; - unsigned int data_width = dw->pdata->data_width[m_master]; unsigned int i; struct scatterlist *sg; size_t total_len = 0; @@ -660,7 +659,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, mem = sg_dma_address(sg); len = sg_dma_len(sg); - mem_width = __ffs(data_width | mem | len); + mem_width = __ffs(sconfig->src_addr_width | mem | len); slave_sg_todev_fill_desc: desc = dwc_desc_get(dwc); @@ -720,7 +719,7 @@ slave_sg_fromdev_fill_desc: lli_write(desc, sar, reg); lli_write(desc, dar, mem); lli_write(desc, ctlhi, ctlhi); - mem_width = __ffs(data_width | mem); + mem_width = __ffs(sconfig->dst_addr_width | mem); lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width)); desc->len = dlen; @@ -780,20 +779,108 @@ bool dw_dma_filter(struct dma_chan *chan, void *param) } EXPORT_SYMBOL_GPL(dw_dma_filter); -static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) +static int dwc_verify_maxburst(struct dma_chan *chan) { struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - struct dw_dma *dw = to_dw_dma(chan->device); - memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); + dwc->dma_sconfig.src_maxburst = + clamp(dwc->dma_sconfig.src_maxburst, 1U, dwc->max_burst); + dwc->dma_sconfig.dst_maxburst = + clamp(dwc->dma_sconfig.dst_maxburst, 1U, dwc->max_burst); dwc->dma_sconfig.src_maxburst = - clamp(dwc->dma_sconfig.src_maxburst, 0U, dwc->max_burst); + rounddown_pow_of_two(dwc->dma_sconfig.src_maxburst); dwc->dma_sconfig.dst_maxburst = - clamp(dwc->dma_sconfig.dst_maxburst, 0U, dwc->max_burst); + rounddown_pow_of_two(dwc->dma_sconfig.dst_maxburst); - dw->encode_maxburst(dwc, &dwc->dma_sconfig.src_maxburst); - dw->encode_maxburst(dwc, &dwc->dma_sconfig.dst_maxburst); + return 0; +} + +static int dwc_verify_p_buswidth(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + u32 reg_width, max_width; + + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) + reg_width = dwc->dma_sconfig.dst_addr_width; + else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) + reg_width = dwc->dma_sconfig.src_addr_width; + else /* DMA_MEM_TO_MEM */ + return 0; + + max_width = dw->pdata->data_width[dwc->dws.p_master]; + + /* Fall-back to 1-byte transfer width if undefined */ + if (reg_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + reg_width = DMA_SLAVE_BUSWIDTH_1_BYTE; + else if (!is_power_of_2(reg_width) || reg_width > max_width) + return -EINVAL; + else /* bus width is valid */ + return 0; + + /* Update undefined addr width value */ + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) + dwc->dma_sconfig.dst_addr_width = reg_width; + else /* DMA_DEV_TO_MEM */ + dwc->dma_sconfig.src_addr_width = reg_width; + + return 0; +} + +static int dwc_verify_m_buswidth(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + u32 reg_width, reg_burst, mem_width; + + mem_width = dw->pdata->data_width[dwc->dws.m_master]; + + /* + * It's possible to have a data portion locked in the DMA FIFO in case + * of the channel suspension. Subsequent channel disabling will cause + * that data silent loss. In order to prevent that maintain the src and + * dst transfer widths coherency by means of the relation: + * (CTLx.SRC_TR_WIDTH * CTLx.SRC_MSIZE >= CTLx.DST_TR_WIDTH) + * Look for the details in the commit message that brings this change. + * + * Note the DMA configs utilized in the calculations below must have + * been verified to have correct values by this method call. + */ + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) { + reg_width = dwc->dma_sconfig.dst_addr_width; + if (mem_width < reg_width) + return -EINVAL; + + dwc->dma_sconfig.src_addr_width = mem_width; + } else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) { + reg_width = dwc->dma_sconfig.src_addr_width; + reg_burst = dwc->dma_sconfig.src_maxburst; + + dwc->dma_sconfig.dst_addr_width = min(mem_width, reg_width * reg_burst); + } + + return 0; +} + +static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + int ret; + + memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); + + ret = dwc_verify_maxburst(chan); + if (ret) + return ret; + + ret = dwc_verify_p_buswidth(chan); + if (ret) + return ret; + + ret = dwc_verify_m_buswidth(chan); + if (ret) + return ret; return 0; } @@ -1068,7 +1155,7 @@ int do_dma_probe(struct dw_dma_chip *chip) bool autocfg = false; unsigned int dw_params; unsigned int i; - int err; + int ret; dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL); if (!dw->pdata) @@ -1084,7 +1171,7 @@ int do_dma_probe(struct dw_dma_chip *chip) autocfg = dw_params >> DW_PARAMS_EN & 1; if (!autocfg) { - err = -EINVAL; + ret = -EINVAL; goto err_pdata; } @@ -1104,7 +1191,7 @@ int do_dma_probe(struct dw_dma_chip *chip) pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; pdata->chan_priority = CHAN_PRIORITY_ASCENDING; } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { - err = -EINVAL; + ret = -EINVAL; goto err_pdata; } else { memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata)); @@ -1116,7 +1203,7 @@ int do_dma_probe(struct dw_dma_chip *chip) dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan), GFP_KERNEL); if (!dw->chan) { - err = -ENOMEM; + ret = -ENOMEM; goto err_pdata; } @@ -1134,15 +1221,15 @@ int do_dma_probe(struct dw_dma_chip *chip) sizeof(struct dw_desc), 4, 0); if (!dw->desc_pool) { dev_err(chip->dev, "No memory for descriptors dma pool\n"); - err = -ENOMEM; + ret = -ENOMEM; goto err_pdata; } tasklet_setup(&dw->tasklet, dw_dma_tasklet); - err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, + ret = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, dw->name, dw); - if (err) + if (ret) goto err_pdata; INIT_LIST_HEAD(&dw->dma.channels); @@ -1254,8 +1341,8 @@ int do_dma_probe(struct dw_dma_chip *chip) */ dma_set_max_seg_size(dw->dma.dev, dw->chan[0].block_size); - err = dma_async_device_register(&dw->dma); - if (err) + ret = dma_async_device_register(&dw->dma); + if (ret) goto err_dma_register; dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", @@ -1269,7 +1356,7 @@ err_dma_register: free_irq(chip->irq, dw); err_pdata: pm_runtime_put_sync_suspend(chip->dev); - return err; + return ret; } int do_dma_remove(struct dw_dma_chip *chip) diff --git a/drivers/dma/dw/dw.c b/drivers/dma/dw/dw.c index a4862263ff14..6766142884b6 100644 --- a/drivers/dma/dw/dw.c +++ b/drivers/dma/dw/dw.c @@ -64,30 +64,39 @@ static size_t dw_dma_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) return DWC_CTLH_BLOCK_TS(block) << width; } +static inline u8 dw_dma_encode_maxburst(u32 maxburst) +{ + /* + * Fix burst size according to dw_dmac. We need to convert them as: + * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. + */ + return maxburst > 1 ? fls(maxburst) - 2 : 0; +} + static u32 dw_dma_prepare_ctllo(struct dw_dma_chan *dwc) { struct dma_slave_config *sconfig = &dwc->dma_sconfig; - u8 smsize = (dwc->direction == DMA_DEV_TO_MEM) ? sconfig->src_maxburst : 0; - u8 dmsize = (dwc->direction == DMA_MEM_TO_DEV) ? sconfig->dst_maxburst : 0; - u8 p_master = dwc->dws.p_master; - u8 m_master = dwc->dws.m_master; - u8 dms = (dwc->direction == DMA_MEM_TO_DEV) ? p_master : m_master; - u8 sms = (dwc->direction == DMA_DEV_TO_MEM) ? p_master : m_master; + u8 smsize = 0, dmsize = 0; + u8 sms, dms; + + if (dwc->direction == DMA_MEM_TO_DEV) { + sms = dwc->dws.m_master; + dms = dwc->dws.p_master; + dmsize = dw_dma_encode_maxburst(sconfig->dst_maxburst); + } else if (dwc->direction == DMA_DEV_TO_MEM) { + sms = dwc->dws.p_master; + dms = dwc->dws.m_master; + smsize = dw_dma_encode_maxburst(sconfig->src_maxburst); + } else /* DMA_MEM_TO_MEM */ { + sms = dwc->dws.m_master; + dms = dwc->dws.m_master; + } return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize) | DWC_CTLL_DMS(dms) | DWC_CTLL_SMS(sms); } -static void dw_dma_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) -{ - /* - * Fix burst size according to dw_dmac. We need to convert them as: - * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. - */ - *maxburst = *maxburst > 1 ? fls(*maxburst) - 2 : 0; -} - static void dw_dma_set_device_name(struct dw_dma *dw, int id) { snprintf(dw->name, sizeof(dw->name), "dw:dmac%d", id); @@ -116,7 +125,6 @@ int dw_dma_probe(struct dw_dma_chip *chip) dw->suspend_chan = dw_dma_suspend_chan; dw->resume_chan = dw_dma_resume_chan; dw->prepare_ctllo = dw_dma_prepare_ctllo; - dw->encode_maxburst = dw_dma_encode_maxburst; dw->bytes2block = dw_dma_bytes2block; dw->block2bytes = dw_dma_block2bytes; diff --git a/drivers/dma/dw/idma32.c b/drivers/dma/dw/idma32.c index 58f4078d83fe..dac617c183e6 100644 --- a/drivers/dma/dw/idma32.c +++ b/drivers/dma/dw/idma32.c @@ -199,21 +199,25 @@ static size_t idma32_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) return IDMA32C_CTLH_BLOCK_TS(block); } +static inline u8 idma32_encode_maxburst(u32 maxburst) +{ + return maxburst > 1 ? fls(maxburst) - 1 : 0; +} + static u32 idma32_prepare_ctllo(struct dw_dma_chan *dwc) { struct dma_slave_config *sconfig = &dwc->dma_sconfig; - u8 smsize = (dwc->direction == DMA_DEV_TO_MEM) ? sconfig->src_maxburst : 0; - u8 dmsize = (dwc->direction == DMA_MEM_TO_DEV) ? sconfig->dst_maxburst : 0; + u8 smsize = 0, dmsize = 0; + + if (dwc->direction == DMA_MEM_TO_DEV) + dmsize = idma32_encode_maxburst(sconfig->dst_maxburst); + else if (dwc->direction == DMA_DEV_TO_MEM) + smsize = idma32_encode_maxburst(sconfig->src_maxburst); return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize); } -static void idma32_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) -{ - *maxburst = *maxburst > 1 ? fls(*maxburst) - 1 : 0; -} - static void idma32_set_device_name(struct dw_dma *dw, int id) { snprintf(dw->name, sizeof(dw->name), "idma32:dmac%d", id); @@ -270,7 +274,6 @@ int idma32_dma_probe(struct dw_dma_chip *chip) dw->suspend_chan = idma32_suspend_chan; dw->resume_chan = idma32_resume_chan; dw->prepare_ctllo = idma32_prepare_ctllo; - dw->encode_maxburst = idma32_encode_maxburst; dw->bytes2block = idma32_bytes2block; dw->block2bytes = idma32_block2bytes; diff --git a/drivers/dma/dw/internal.h b/drivers/dma/dw/internal.h index 563ce73488db..f1bd06a20cd6 100644 --- a/drivers/dma/dw/internal.h +++ b/drivers/dma/dw/internal.h @@ -51,11 +51,15 @@ struct dw_dma_chip_pdata { int (*probe)(struct dw_dma_chip *chip); int (*remove)(struct dw_dma_chip *chip); struct dw_dma_chip *chip; + u8 m_master; + u8 p_master; }; static __maybe_unused const struct dw_dma_chip_pdata dw_dma_chip_pdata = { .probe = dw_dma_probe, .remove = dw_dma_remove, + .m_master = 0, + .p_master = 1, }; static const struct dw_dma_platform_data idma32_pdata = { @@ -72,6 +76,8 @@ static __maybe_unused const struct dw_dma_chip_pdata idma32_chip_pdata = { .pdata = &idma32_pdata, .probe = idma32_dma_probe, .remove = idma32_dma_remove, + .m_master = 0, + .p_master = 0, }; static const struct dw_dma_platform_data xbar_pdata = { @@ -88,6 +94,8 @@ static __maybe_unused const struct dw_dma_chip_pdata xbar_chip_pdata = { .pdata = &xbar_pdata, .probe = idma32_dma_probe, .remove = idma32_dma_remove, + .m_master = 0, + .p_master = 0, }; #endif /* _DMA_DW_INTERNAL_H */ diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c index ad2d4d012cf7..a3aae3d1c093 100644 --- a/drivers/dma/dw/pci.c +++ b/drivers/dma/dw/pci.c @@ -56,10 +56,10 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) if (ret) return ret; - dw_dma_acpi_controller_register(chip->dw); - pci_set_drvdata(pdev, data); + dw_dma_acpi_controller_register(chip->dw); + return 0; } @@ -76,8 +76,6 @@ static void dw_pci_remove(struct pci_dev *pdev) dev_warn(&pdev->dev, "can't remove device properly: %d\n", ret); } -#ifdef CONFIG_PM_SLEEP - static int dw_pci_suspend_late(struct device *dev) { struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); @@ -94,10 +92,8 @@ static int dw_pci_resume_early(struct device *dev) return do_dw_dma_enable(chip); }; -#endif /* CONFIG_PM_SLEEP */ - static const struct dev_pm_ops dw_pci_dev_pm_ops = { - SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_pci_suspend_late, dw_pci_resume_early) + LATE_SYSTEM_SLEEP_PM_OPS(dw_pci_suspend_late, dw_pci_resume_early) }; static const struct pci_device_id dw_pci_id_table[] = { @@ -136,7 +132,7 @@ static struct pci_driver dw_pci_driver = { .probe = dw_pci_probe, .remove = dw_pci_remove, .driver = { - .pm = &dw_pci_dev_pm_ops, + .pm = pm_sleep_ptr(&dw_pci_dev_pm_ops), }, }; diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index 47f2292dba98..c63fa52036d7 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -21,15 +21,13 @@ #include "internal.h" -#define DRV_NAME "dw_dmac" - static int dw_probe(struct platform_device *pdev) { const struct dw_dma_chip_pdata *match; struct dw_dma_chip_pdata *data; struct dw_dma_chip *chip; struct device *dev = &pdev->dev; - int err; + int ret; match = device_get_match_data(dev); if (!match) @@ -51,9 +49,9 @@ static int dw_probe(struct platform_device *pdev) if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); - err = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (err) - return err; + ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; if (!data->pdata) data->pdata = dev_get_platdata(dev); @@ -69,14 +67,14 @@ static int dw_probe(struct platform_device *pdev) chip->clk = devm_clk_get_optional(chip->dev, "hclk"); if (IS_ERR(chip->clk)) return PTR_ERR(chip->clk); - err = clk_prepare_enable(chip->clk); - if (err) - return err; + ret = clk_prepare_enable(chip->clk); + if (ret) + return ret; pm_runtime_enable(&pdev->dev); - err = data->probe(chip); - if (err) + ret = data->probe(chip); + if (ret) goto err_dw_dma_probe; platform_set_drvdata(pdev, data); @@ -90,10 +88,10 @@ static int dw_probe(struct platform_device *pdev) err_dw_dma_probe: pm_runtime_disable(&pdev->dev); clk_disable_unprepare(chip->clk); - return err; + return ret; } -static int dw_remove(struct platform_device *pdev) +static void dw_remove(struct platform_device *pdev) { struct dw_dma_chip_pdata *data = platform_get_drvdata(pdev); struct dw_dma_chip *chip = data->chip; @@ -109,8 +107,6 @@ static int dw_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); clk_disable_unprepare(chip->clk); - - return 0; } static void dw_shutdown(struct platform_device *pdev) @@ -159,8 +155,6 @@ static const struct acpi_device_id dw_dma_acpi_id_table[] = { MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table); #endif -#ifdef CONFIG_PM_SLEEP - static int dw_suspend_late(struct device *dev) { struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); @@ -185,10 +179,8 @@ static int dw_resume_early(struct device *dev) return do_dw_dma_enable(chip); } -#endif /* CONFIG_PM_SLEEP */ - static const struct dev_pm_ops dw_dev_pm_ops = { - SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_suspend_late, dw_resume_early) + LATE_SYSTEM_SLEEP_PM_OPS(dw_suspend_late, dw_resume_early) }; static struct platform_driver dw_driver = { @@ -196,8 +188,8 @@ static struct platform_driver dw_driver = { .remove = dw_remove, .shutdown = dw_shutdown, .driver = { - .name = DRV_NAME, - .pm = &dw_dev_pm_ops, + .name = "dw_dmac", + .pm = pm_sleep_ptr(&dw_dev_pm_ops), .of_match_table = of_match_ptr(dw_dma_of_id_table), .acpi_match_table = ACPI_PTR(dw_dma_acpi_id_table), }, @@ -217,4 +209,3 @@ module_exit(dw_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller platform driver"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h index 76654bd13c1a..5969d9cc8d7a 100644 --- a/drivers/dma/dw/regs.h +++ b/drivers/dma/dw/regs.h @@ -327,7 +327,6 @@ struct dw_dma { void (*suspend_chan)(struct dw_dma_chan *dwc, bool drain); void (*resume_chan)(struct dw_dma_chan *dwc, bool drain); u32 (*prepare_ctllo)(struct dw_dma_chan *dwc); - void (*encode_maxburst)(struct dw_dma_chan *dwc, u32 *maxburst); u32 (*bytes2block)(struct dw_dma_chan *dwc, size_t bytes, unsigned int width, size_t *len); size_t (*block2bytes)(struct dw_dma_chan *dwc, u32 block, u32 width); diff --git a/drivers/dma/dw/rzn1-dmamux.c b/drivers/dma/dw/rzn1-dmamux.c index f9912c3dd4d7..deadf135681b 100644 --- a/drivers/dma/dw/rzn1-dmamux.c +++ b/drivers/dma/dw/rzn1-dmamux.c @@ -5,8 +5,10 @@ * Based on TI crossbar driver written by Peter Ujfalusi <peter.ujfalusi@ti.com> */ #include <linux/bitops.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/soc/renesas/r9a06g032-sysctrl.h> #include <linux/types.h> @@ -46,12 +48,16 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, u32 mask; int ret; - if (dma_spec->args_count != RNZ1_DMAMUX_NCELLS) - return ERR_PTR(-EINVAL); + if (dma_spec->args_count != RNZ1_DMAMUX_NCELLS) { + ret = -EINVAL; + goto put_device; + } map = kzalloc(sizeof(*map), GFP_KERNEL); - if (!map) - return ERR_PTR(-ENOMEM); + if (!map) { + ret = -ENOMEM; + goto put_device; + } chan = dma_spec->args[0]; map->req_idx = dma_spec->args[4]; @@ -92,12 +98,15 @@ static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, if (ret) goto clear_bitmap; + put_device(&pdev->dev); return map; clear_bitmap: clear_bit(map->req_idx, dmamux->used_chans); free_map: kfree(map); +put_device: + put_device(&pdev->dev); return ERR_PTR(ret); } diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 5338a94f1a69..e424bb5c40e7 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -17,14 +17,15 @@ #include <linux/clk.h> #include <linux/init.h> #include <linux/interrupt.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/module.h> #include <linux/mod_devicetable.h> +#include <linux/of_dma.h> +#include <linux/overflow.h> #include <linux/platform_device.h> #include <linux/slab.h> -#include <linux/platform_data/dma-ep93xx.h> - #include "dmaengine.h" /* M2P registers */ @@ -104,6 +105,31 @@ #define DMA_MAX_CHAN_BYTES 0xffff #define DMA_MAX_CHAN_DESCRIPTORS 32 +/* + * M2P channels. + * + * Note that these values are also directly used for setting the PPALLOC + * register. + */ +#define EP93XX_DMA_I2S1 0 +#define EP93XX_DMA_I2S2 1 +#define EP93XX_DMA_AAC1 2 +#define EP93XX_DMA_AAC2 3 +#define EP93XX_DMA_AAC3 4 +#define EP93XX_DMA_I2S3 5 +#define EP93XX_DMA_UART1 6 +#define EP93XX_DMA_UART2 7 +#define EP93XX_DMA_UART3 8 +#define EP93XX_DMA_IRDA 9 +/* M2M channels */ +#define EP93XX_DMA_SSP 10 +#define EP93XX_DMA_IDE 11 + +enum ep93xx_dma_type { + M2P_DMA, + M2M_DMA, +}; + struct ep93xx_dma_engine; static int ep93xx_dma_slave_config_write(struct dma_chan *chan, enum dma_transfer_direction dir, @@ -129,11 +155,17 @@ struct ep93xx_dma_desc { struct list_head node; }; +struct ep93xx_dma_chan_cfg { + u8 port; + enum dma_transfer_direction dir; +}; + /** * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel * @chan: dmaengine API channel * @edma: pointer to the engine device * @regs: memory mapped registers + * @dma_cfg: channel number, direction * @irq: interrupt number of the channel * @clk: clock used by this channel * @tasklet: channel specific tasklet used for callbacks @@ -157,14 +189,12 @@ struct ep93xx_dma_desc { * descriptor in the chain. When a descriptor is moved to the @active queue, * the first and chained descriptors are flattened into a single list. * - * @chan.private holds pointer to &struct ep93xx_dma_data which contains - * necessary channel configuration information. For memcpy channels this must - * be %NULL. */ struct ep93xx_dma_chan { struct dma_chan chan; const struct ep93xx_dma_engine *edma; void __iomem *regs; + struct ep93xx_dma_chan_cfg dma_cfg; int irq; struct clk *clk; struct tasklet_struct tasklet; @@ -213,7 +243,12 @@ struct ep93xx_dma_engine { #define INTERRUPT_NEXT_BUFFER 2 size_t num_channels; - struct ep93xx_dma_chan channels[]; + struct ep93xx_dma_chan channels[] __counted_by(num_channels); +}; + +struct ep93xx_edma_data { + u32 id; + size_t num_channels; }; static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac) @@ -226,6 +261,31 @@ static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan) return container_of(chan, struct ep93xx_dma_chan, chan); } +static inline bool ep93xx_dma_chan_is_m2p(struct dma_chan *chan) +{ + if (device_is_compatible(chan->device->dev, "cirrus,ep9301-dma-m2p")) + return true; + + return !strcmp(dev_name(chan->device->dev), "ep93xx-dma-m2p"); +} + +/* + * ep93xx_dma_chan_direction - returns direction the channel can be used + * + * This function can be used in filter functions to find out whether the + * channel supports given DMA direction. Only M2P channels have such + * limitation, for M2M channels the direction is configurable. + */ +static inline enum dma_transfer_direction +ep93xx_dma_chan_direction(struct dma_chan *chan) +{ + if (!ep93xx_dma_chan_is_m2p(chan)) + return DMA_TRANS_NONE; + + /* even channels are for TX, odd for RX */ + return (chan->chan_id % 2 == 0) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; +} + /** * ep93xx_dma_set_active - set new active descriptor chain * @edmac: channel @@ -318,10 +378,9 @@ static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control) static int m2p_hw_setup(struct ep93xx_dma_chan *edmac) { - struct ep93xx_dma_data *data = edmac->chan.private; u32 control; - writel(data->port & 0xf, edmac->regs + M2P_PPALLOC); + writel(edmac->dma_cfg.port & 0xf, edmac->regs + M2P_PPALLOC); control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE | M2P_CONTROL_ENABLE; @@ -458,16 +517,15 @@ static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac) static int m2m_hw_setup(struct ep93xx_dma_chan *edmac) { - const struct ep93xx_dma_data *data = edmac->chan.private; u32 control = 0; - if (!data) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_MEM) { /* This is memcpy channel, nothing to configure */ writel(control, edmac->regs + M2M_CONTROL); return 0; } - switch (data->port) { + switch (edmac->dma_cfg.port) { case EP93XX_DMA_SSP: /* * This was found via experimenting - anything less than 5 @@ -477,7 +535,7 @@ static int m2m_hw_setup(struct ep93xx_dma_chan *edmac) control = (5 << M2M_CONTROL_PWSC_SHIFT); control |= M2M_CONTROL_NO_HDSK; - if (data->direction == DMA_MEM_TO_DEV) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_DEV) { control |= M2M_CONTROL_DAH; control |= M2M_CONTROL_TM_TX; control |= M2M_CONTROL_RSS_SSPTX; @@ -493,7 +551,7 @@ static int m2m_hw_setup(struct ep93xx_dma_chan *edmac) * This IDE part is totally untested. Values below are taken * from the EP93xx Users's Guide and might not be correct. */ - if (data->direction == DMA_MEM_TO_DEV) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_DEV) { /* Worst case from the UG */ control = (3 << M2M_CONTROL_PWSC_SHIFT); control |= M2M_CONTROL_DAH; @@ -548,7 +606,6 @@ static void m2m_fill_desc(struct ep93xx_dma_chan *edmac) static void m2m_hw_submit(struct ep93xx_dma_chan *edmac) { - struct ep93xx_dma_data *data = edmac->chan.private; u32 control = readl(edmac->regs + M2M_CONTROL); /* @@ -574,7 +631,7 @@ static void m2m_hw_submit(struct ep93xx_dma_chan *edmac) control |= M2M_CONTROL_ENABLE; writel(control, edmac->regs + M2M_CONTROL); - if (!data) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_MEM) { /* * For memcpy channels the software trigger must be asserted * in order to start the memcpy operation. @@ -636,7 +693,7 @@ static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac) */ if (ep93xx_dma_advance_active(edmac)) { m2m_fill_desc(edmac); - if (done && !edmac->chan.private) { + if (done && edmac->dma_cfg.dir == DMA_MEM_TO_MEM) { /* Software trigger for memcpy channel */ control = readl(edmac->regs + M2M_CONTROL); control |= M2M_CONTROL_START; @@ -841,7 +898,7 @@ static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx) desc = container_of(tx, struct ep93xx_dma_desc, txd); /* - * If nothing is currently prosessed, we push this descriptor + * If nothing is currently processed, we push this descriptor * directly to the hardware. Otherwise we put the descriptor * to the pending queue. */ @@ -867,25 +924,21 @@ static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx) static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan) { struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); - struct ep93xx_dma_data *data = chan->private; const char *name = dma_chan_name(chan); int ret, i; /* Sanity check the channel parameters */ if (!edmac->edma->m2m) { - if (!data) - return -EINVAL; - if (data->port < EP93XX_DMA_I2S1 || - data->port > EP93XX_DMA_IRDA) + if (edmac->dma_cfg.port > EP93XX_DMA_IRDA) return -EINVAL; - if (data->direction != ep93xx_dma_chan_direction(chan)) + if (edmac->dma_cfg.dir != ep93xx_dma_chan_direction(chan)) return -EINVAL; } else { - if (data) { - switch (data->port) { + if (edmac->dma_cfg.dir != DMA_MEM_TO_MEM) { + switch (edmac->dma_cfg.port) { case EP93XX_DMA_SSP: case EP93XX_DMA_IDE: - if (!is_slave_direction(data->direction)) + if (!is_slave_direction(edmac->dma_cfg.dir)) return -EINVAL; break; default: @@ -894,9 +947,6 @@ static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan) } } - if (data && data->name) - name = data->name; - ret = clk_prepare_enable(edmac->clk); if (ret) return ret; @@ -1025,7 +1075,7 @@ fail: * @chan: channel * @sgl: list of buffers to transfer * @sg_len: number of entries in @sgl - * @dir: direction of tha DMA transfer + * @dir: direction of the DMA transfer * @flags: flags for the descriptor * @context: operation context (ignored) * @@ -1315,38 +1365,56 @@ static void ep93xx_dma_issue_pending(struct dma_chan *chan) ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan)); } -static int __init ep93xx_dma_probe(struct platform_device *pdev) +static struct ep93xx_dma_engine *ep93xx_dma_of_probe(struct platform_device *pdev) { - struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev); + const struct ep93xx_edma_data *data; + struct device *dev = &pdev->dev; struct ep93xx_dma_engine *edma; struct dma_device *dma_dev; - size_t edma_size; - int ret, i; + char dma_clk_name[5]; + int i; - edma_size = pdata->num_channels * sizeof(struct ep93xx_dma_chan); - edma = kzalloc(sizeof(*edma) + edma_size, GFP_KERNEL); + data = device_get_match_data(dev); + if (!data) + return ERR_PTR(dev_err_probe(dev, -ENODEV, "No device match found\n")); + + edma = devm_kzalloc(dev, struct_size(edma, channels, data->num_channels), + GFP_KERNEL); if (!edma) - return -ENOMEM; + return ERR_PTR(-ENOMEM); + edma->m2m = data->id; + edma->num_channels = data->num_channels; dma_dev = &edma->dma_dev; - edma->m2m = platform_get_device_id(pdev)->driver_data; - edma->num_channels = pdata->num_channels; INIT_LIST_HEAD(&dma_dev->channels); - for (i = 0; i < pdata->num_channels; i++) { - const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i]; + for (i = 0; i < edma->num_channels; i++) { struct ep93xx_dma_chan *edmac = &edma->channels[i]; + int len; edmac->chan.device = dma_dev; - edmac->regs = cdata->base; - edmac->irq = cdata->irq; + edmac->regs = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(edmac->regs)) + return ERR_CAST(edmac->regs); + + edmac->irq = fwnode_irq_get(dev_fwnode(dev), i); + if (edmac->irq < 0) + return ERR_PTR(edmac->irq); + edmac->edma = edma; - edmac->clk = clk_get(NULL, cdata->name); + if (edma->m2m) + len = snprintf(dma_clk_name, sizeof(dma_clk_name), "m2m%u", i); + else + len = snprintf(dma_clk_name, sizeof(dma_clk_name), "m2p%u", i); + if (len >= sizeof(dma_clk_name)) + return ERR_PTR(-ENOBUFS); + + edmac->clk = devm_clk_get(dev, dma_clk_name); if (IS_ERR(edmac->clk)) { - dev_warn(&pdev->dev, "failed to get clock for %s\n", - cdata->name); - continue; + dev_err_probe(dev, PTR_ERR(edmac->clk), + "no %s clock found\n", dma_clk_name); + return ERR_CAST(edmac->clk); } spin_lock_init(&edmac->lock); @@ -1359,6 +1427,90 @@ static int __init ep93xx_dma_probe(struct platform_device *pdev) &dma_dev->channels); } + return edma; +} + +static bool ep93xx_m2p_dma_filter(struct dma_chan *chan, void *filter_param) +{ + struct ep93xx_dma_chan *echan = to_ep93xx_dma_chan(chan); + struct ep93xx_dma_chan_cfg *cfg = filter_param; + + if (cfg->dir != ep93xx_dma_chan_direction(chan)) + return false; + + echan->dma_cfg = *cfg; + return true; +} + +static struct dma_chan *ep93xx_m2p_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct ep93xx_dma_engine *edma = ofdma->of_dma_data; + dma_cap_mask_t mask = edma->dma_dev.cap_mask; + struct ep93xx_dma_chan_cfg dma_cfg; + u8 port = dma_spec->args[0]; + u8 direction = dma_spec->args[1]; + + if (port > EP93XX_DMA_IRDA) + return NULL; + + if (!is_slave_direction(direction)) + return NULL; + + dma_cfg.port = port; + dma_cfg.dir = direction; + + return __dma_request_channel(&mask, ep93xx_m2p_dma_filter, &dma_cfg, ofdma->of_node); +} + +static bool ep93xx_m2m_dma_filter(struct dma_chan *chan, void *filter_param) +{ + struct ep93xx_dma_chan *echan = to_ep93xx_dma_chan(chan); + struct ep93xx_dma_chan_cfg *cfg = filter_param; + + echan->dma_cfg = *cfg; + + return true; +} + +static struct dma_chan *ep93xx_m2m_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct ep93xx_dma_engine *edma = ofdma->of_dma_data; + dma_cap_mask_t mask = edma->dma_dev.cap_mask; + struct ep93xx_dma_chan_cfg dma_cfg; + u8 port = dma_spec->args[0]; + u8 direction = dma_spec->args[1]; + + if (!is_slave_direction(direction)) + return NULL; + + switch (port) { + case EP93XX_DMA_SSP: + case EP93XX_DMA_IDE: + break; + default: + return NULL; + } + + dma_cfg.port = port; + dma_cfg.dir = direction; + + return __dma_request_channel(&mask, ep93xx_m2m_dma_filter, &dma_cfg, ofdma->of_node); +} + +static int ep93xx_dma_probe(struct platform_device *pdev) +{ + struct ep93xx_dma_engine *edma; + struct dma_device *dma_dev; + int ret; + + edma = ep93xx_dma_of_probe(pdev); + if (IS_ERR(edma)) + return PTR_ERR(edma); + + dma_dev = &edma->dma_dev; + dma_cap_zero(dma_dev->cap_mask); dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); @@ -1395,21 +1547,46 @@ static int __init ep93xx_dma_probe(struct platform_device *pdev) } ret = dma_async_device_register(dma_dev); - if (unlikely(ret)) { - for (i = 0; i < edma->num_channels; i++) { - struct ep93xx_dma_chan *edmac = &edma->channels[i]; - if (!IS_ERR_OR_NULL(edmac->clk)) - clk_put(edmac->clk); - } - kfree(edma); + if (ret) + return ret; + + if (edma->m2m) { + ret = of_dma_controller_register(pdev->dev.of_node, ep93xx_m2m_dma_of_xlate, + edma); } else { - dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n", - edma->m2m ? "M" : "P"); + ret = of_dma_controller_register(pdev->dev.of_node, ep93xx_m2p_dma_of_xlate, + edma); } + if (ret) + goto err_dma_unregister; + + dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n", edma->m2m ? "M" : "P"); + + return 0; + +err_dma_unregister: + dma_async_device_unregister(dma_dev); return ret; } +static const struct ep93xx_edma_data edma_m2p = { + .id = M2P_DMA, + .num_channels = 10, +}; + +static const struct ep93xx_edma_data edma_m2m = { + .id = M2M_DMA, + .num_channels = 2, +}; + +static const struct of_device_id ep93xx_dma_of_ids[] = { + { .compatible = "cirrus,ep9301-dma-m2p", .data = &edma_m2p }, + { .compatible = "cirrus,ep9301-dma-m2m", .data = &edma_m2m }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ep93xx_dma_of_ids); + static const struct platform_device_id ep93xx_dma_driver_ids[] = { { "ep93xx-dma-m2p", 0 }, { "ep93xx-dma-m2m", 1 }, @@ -1419,15 +1596,13 @@ static const struct platform_device_id ep93xx_dma_driver_ids[] = { static struct platform_driver ep93xx_dma_driver = { .driver = { .name = "ep93xx-dma", + .of_match_table = ep93xx_dma_of_ids, }, .id_table = ep93xx_dma_driver_ids, + .probe = ep93xx_dma_probe, }; -static int __init ep93xx_dma_module_init(void) -{ - return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe); -} -subsys_initcall(ep93xx_dma_module_init); +module_platform_driver(ep93xx_dma_driver); MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>"); MODULE_DESCRIPTION("EP93xx DMA driver"); diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c index a42a37634881..36384d019263 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c @@ -38,15 +38,17 @@ static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan) if (!dpaa2_chan->fd_pool) goto err; - dpaa2_chan->fl_pool = dma_pool_create("fl_pool", dev, - sizeof(struct dpaa2_fl_entry), - sizeof(struct dpaa2_fl_entry), 0); + dpaa2_chan->fl_pool = + dma_pool_create("fl_pool", dev, + sizeof(struct dpaa2_fl_entry) * 3, + sizeof(struct dpaa2_fl_entry), 0); + if (!dpaa2_chan->fl_pool) goto err_fd; dpaa2_chan->sdd_pool = dma_pool_create("sdd_pool", dev, - sizeof(struct dpaa2_qdma_sd_d), + sizeof(struct dpaa2_qdma_sd_d) * 2, sizeof(struct dpaa2_qdma_sd_d), 0); if (!dpaa2_chan->sdd_pool) goto err_fl; @@ -360,7 +362,7 @@ static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) for (i = 0; i < priv->num_pairs; i++) { err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, - i, &priv->rx_queue_attr[i]); + i, 0, &priv->rx_queue_attr[i]); if (err) { dev_err(dev, "dpdmai_get_rx_queue() failed\n"); goto exit; @@ -368,13 +370,13 @@ static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, - i, &priv->tx_fqid[i]); + i, 0, &priv->tx_queue_attr[i]); if (err) { dev_err(dev, "dpdmai_get_tx_queue() failed\n"); goto exit; } - ppriv->req_fqid = priv->tx_fqid[i]; - ppriv->prio = i; + ppriv->req_fqid = priv->tx_queue_attr[i].fqid; + ppriv->prio = DPAA2_QDMA_DEFAULT_PRIORITY; ppriv->priv = priv; ppriv++; } @@ -540,7 +542,7 @@ static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; rx_queue_cfg.dest_cfg.priority = ppriv->prio; err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, - rx_queue_cfg.dest_cfg.priority, + rx_queue_cfg.dest_cfg.priority, 0, &rx_queue_cfg); if (err) { dev_err(dev, "dpdmai_set_rx_queue() failed\n"); @@ -640,7 +642,7 @@ static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma) for (i = 0; i < dpaa2_qdma->n_chans; i++) { dpaa2_chan = &dpaa2_qdma->chans[i]; dpaa2_chan->qdma = dpaa2_qdma; - dpaa2_chan->fqid = priv->tx_fqid[i % num]; + dpaa2_chan->fqid = priv->tx_queue_attr[i % num].fqid; dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev); spin_lock_init(&dpaa2_chan->queue_lock); @@ -800,7 +802,7 @@ static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); dpaa2_dpdmai_dpio_unbind(priv); dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); - dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle); + dpdmai_destroy(priv->mc_io, 0, priv->dpqdma_id, ls_dev->mc_handle); } static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { @@ -814,7 +816,6 @@ static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { static struct fsl_mc_driver dpaa2_qdma_driver = { .driver = { .name = "dpaa2-qdma", - .owner = THIS_MODULE, }, .probe = dpaa2_qdma_probe, .remove = dpaa2_qdma_remove, diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h index 03e2f4e0baca..36c284a3d184 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h @@ -6,13 +6,14 @@ #define DPAA2_QDMA_STORE_SIZE 16 #define NUM_CH 8 +#define DPAA2_QDMA_DEFAULT_PRIORITY 0 struct dpaa2_qdma_sd_d { u32 rsv:32; union { struct { - u32 ssd:12; /* souce stride distance */ - u32 sss:12; /* souce stride size */ + u32 ssd:12; /* source stride distance */ + u32 sss:12; /* source stride size */ u32 rsv1:8; } sdf; struct { @@ -47,7 +48,7 @@ struct dpaa2_qdma_sd_d { #define QDMA_SER_DISABLE (8) /* no notification */ #define QDMA_SER_CTX BIT(8) /* notification by FQD_CTX[fqid] */ #define QDMA_SER_DEST (2 << 8) /* notification by destination desc */ -#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */ +#define QDMA_SER_BOTH (3 << 8) /* source and dest notification */ #define QDMA_FD_SPF_ENALBE BIT(30) /* source prefetch enable */ #define QMAN_FD_VA_ENABLE BIT(14) /* Address used is virtual address */ @@ -122,8 +123,8 @@ struct dpaa2_qdma_priv { struct dpaa2_qdma_engine *dpaa2_qdma; struct dpaa2_qdma_priv_per_prio *ppriv; - struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM]; - u32 tx_fqid[DPDMAI_PRIO_NUM]; + struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_MAX_QUEUE_NUM]; + struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_MAX_QUEUE_NUM]; }; struct dpaa2_qdma_priv_per_prio { diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c index 878662aaa1c2..4be81db24a19 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c @@ -1,52 +1,52 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright 2019 NXP +#include <linux/bitfield.h> #include <linux/module.h> #include <linux/types.h> #include <linux/io.h> #include <linux/fsl/mc.h> #include "dpdmai.h" +#define DEST_TYPE_MASK 0xF + struct dpdmai_rsp_get_attributes { __le32 id; u8 num_of_priorities; - u8 pad0[3]; + u8 num_of_queues; + u8 pad0[2]; __le16 major; __le16 minor; }; struct dpdmai_cmd_queue { __le32 dest_id; - u8 priority; - u8 queue; + u8 dest_priority; + union { + u8 queue; + u8 pri; + }; u8 dest_type; - u8 pad; + u8 queue_idx; __le64 user_ctx; union { __le32 options; __le32 fqid; }; -}; +} __packed; struct dpdmai_rsp_get_tx_queue { __le64 pad; __le32 fqid; }; -#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \ - ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg)) - -/* cmd, param, offset, width, type, arg_name */ -#define DPDMAI_CMD_CREATE(cmd, cfg) \ -do { \ - MC_CMD_OP(cmd, 0, 8, 8, u8, (cfg)->priorities[0]);\ - MC_CMD_OP(cmd, 0, 16, 8, u8, (cfg)->priorities[1]);\ -} while (0) +struct dpdmai_cmd_open { + __le32 dpdmai_id; +} __packed; -static inline u64 mc_enc(int lsoffset, int width, u64 val) -{ - return (val & MAKE_UMASK64(width)) << lsoffset; -} +struct dpdmai_cmd_destroy { + __le32 dpdmai_id; +} __packed; /** * dpdmai_open() - Open a control session for the specified object @@ -68,16 +68,16 @@ static inline u64 mc_enc(int lsoffset, int width, u64 val) int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token) { + struct dpdmai_cmd_open *cmd_params; struct fsl_mc_command cmd = { 0 }; - __le64 *cmd_dpdmai_id; int err; /* prepare command */ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN, cmd_flags, 0); - cmd_dpdmai_id = cmd.params; - *cmd_dpdmai_id = cpu_to_le32(dpdmai_id); + cmd_params = (struct dpdmai_cmd_open *)&cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -116,65 +116,26 @@ int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) EXPORT_SYMBOL_GPL(dpdmai_close); /** - * dpdmai_create() - Create the DPDMAI object - * @mc_io: Pointer to MC portal's I/O object - * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' - * @cfg: Configuration structure - * @token: Returned token; use in subsequent API calls - * - * Create the DPDMAI object, allocate required resources and - * perform required initialization. - * - * The object can be created either by declaring it in the - * DPL file, or by calling this function. - * - * This function returns a unique authentication token, - * associated with the specific object ID and the specific MC - * portal; this token must be used in all subsequent calls to - * this specific object. For objects that are created using the - * DPL file, call dpdmai_open() function to get an authentication - * token first. - * - * Return: '0' on Success; Error code otherwise. - */ -int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, - const struct dpdmai_cfg *cfg, u16 *token) -{ - struct fsl_mc_command cmd = { 0 }; - int err; - - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE, - cmd_flags, 0); - DPDMAI_CMD_CREATE(cmd, cfg); - - /* send command to mc*/ - err = mc_send_command(mc_io, &cmd); - if (err) - return err; - - /* retrieve response parameters */ - *token = mc_cmd_hdr_read_token(&cmd); - - return 0; -} - -/** * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpdmai_id: The object id; it must be a valid id within the container that created this object; * @token: Token of DPDMAI object * * Return: '0' on Success; error code otherwise. */ -int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u32 dpdmai_id, u16 token) { + struct dpdmai_cmd_destroy *cmd_params; struct fsl_mc_command cmd = { 0 }; /* prepare command */ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, cmd_flags, token); + cmd_params = (struct dpdmai_cmd_destroy *)&cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } @@ -274,6 +235,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, attr->version.major = le16_to_cpu(rsp_params->major); attr->version.minor = le16_to_cpu(rsp_params->minor); attr->num_of_priorities = rsp_params->num_of_priorities; + attr->num_of_queues = rsp_params->num_of_queues; return 0; } @@ -284,13 +246,14 @@ EXPORT_SYMBOL_GPL(dpdmai_get_attributes); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @cfg: Rx queue configuration * * Return: '0' on Success; Error code otherwise. */ -int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 queue_idx, u8 priority, const struct dpdmai_rx_queue_cfg *cfg) { struct dpdmai_cmd_queue *cmd_params; @@ -302,11 +265,12 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); - cmd_params->priority = cfg->dest_cfg.priority; - cmd_params->queue = priority; + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->pri = priority; cmd_params->dest_type = cfg->dest_cfg.dest_type; cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); cmd_params->options = cpu_to_le32(cfg->options); + cmd_params->queue_idx = queue_idx; /* send command to mc*/ return mc_send_command(mc_io, &cmd); @@ -318,13 +282,14 @@ EXPORT_SYMBOL_GPL(dpdmai_set_rx_queue); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA Queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @attr: Returned Rx queue attributes * * Return: '0' on Success; Error code otherwise. */ -int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 queue_idx, u8 priority, struct dpdmai_rx_queue_attr *attr) { struct dpdmai_cmd_queue *cmd_params; @@ -337,6 +302,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->queue = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -345,8 +311,8 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, /* retrieve response parameters */ attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); - attr->dest_cfg.priority = cmd_params->priority; - attr->dest_cfg.dest_type = cmd_params->dest_type; + attr->dest_cfg.priority = cmd_params->dest_priority; + attr->dest_cfg.dest_type = FIELD_GET(DEST_TYPE_MASK, cmd_params->dest_type); attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); attr->fqid = le32_to_cpu(cmd_params->fqid); @@ -359,14 +325,15 @@ EXPORT_SYMBOL_GPL(dpdmai_get_rx_queue); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation - * @fqid: Returned Tx queue + * @attr: Returned DMA Tx queue attributes * * Return: '0' on Success; Error code otherwise. */ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, - u16 token, u8 priority, u32 *fqid) + u16 token, u8 queue_idx, u8 priority, struct dpdmai_tx_queue_attr *attr) { struct dpdmai_rsp_get_tx_queue *rsp_params; struct dpdmai_cmd_queue *cmd_params; @@ -379,6 +346,7 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->queue = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -388,10 +356,11 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, /* retrieve response parameters */ rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params; - *fqid = le32_to_cpu(rsp_params->fqid); + attr->fqid = le32_to_cpu(rsp_params->fqid); return 0; } EXPORT_SYMBOL_GPL(dpdmai_get_tx_queue); +MODULE_DESCRIPTION("NXP DPAA2 QDMA driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h index b13b9bf0c003..3fe7d8327366 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h @@ -5,14 +5,19 @@ #define __FSL_DPDMAI_H /* DPDMAI Version */ -#define DPDMAI_VER_MAJOR 2 -#define DPDMAI_VER_MINOR 2 +#define DPDMAI_VER_MAJOR 3 +#define DPDMAI_VER_MINOR 3 -#define DPDMAI_CMD_BASE_VERSION 0 +#define DPDMAI_CMD_BASE_VERSION 1 #define DPDMAI_CMD_ID_OFFSET 4 -#define DPDMAI_CMDID_FORMAT(x) (((x) << DPDMAI_CMD_ID_OFFSET) | \ - DPDMAI_CMD_BASE_VERSION) +/* + * Maximum number of Tx/Rx queues per DPDMAI object + */ +#define DPDMAI_MAX_QUEUE_NUM 8 + +#define DPDMAI_CMDID_FORMAT_V(x, v) (((x) << DPDMAI_CMD_ID_OFFSET) | (v)) +#define DPDMAI_CMDID_FORMAT(x) DPDMAI_CMDID_FORMAT_V(x, DPDMAI_CMD_BASE_VERSION) /* Command IDs */ #define DPDMAI_CMDID_CLOSE DPDMAI_CMDID_FORMAT(0x800) @@ -26,18 +31,9 @@ #define DPDMAI_CMDID_RESET DPDMAI_CMDID_FORMAT(0x005) #define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMDID_FORMAT(0x006) -#define DPDMAI_CMDID_SET_IRQ DPDMAI_CMDID_FORMAT(0x010) -#define DPDMAI_CMDID_GET_IRQ DPDMAI_CMDID_FORMAT(0x011) -#define DPDMAI_CMDID_SET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x012) -#define DPDMAI_CMDID_GET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x013) -#define DPDMAI_CMDID_SET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x014) -#define DPDMAI_CMDID_GET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x015) -#define DPDMAI_CMDID_GET_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x016) -#define DPDMAI_CMDID_CLEAR_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x017) - -#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A0) -#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A1) -#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMDID_FORMAT(0x1A2) +#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A0, 2) +#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A1, 2) +#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A2, 2) #define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */ #define MC_CMD_HDR_TOKEN_S 16 /* Token field size */ @@ -49,30 +45,32 @@ * Contains initialization APIs and runtime control APIs for DPDMAI */ -/** +/* * Maximum number of Tx/Rx priorities per DPDMAI object */ #define DPDMAI_PRIO_NUM 2 /* DPDMAI queue modification options */ -/** +/* * Select to modify the user's context associated with the queue */ #define DPDMAI_QUEUE_OPT_USER_CTX 0x1 -/** +/* * Select to modify the queue's destination */ #define DPDMAI_QUEUE_OPT_DEST 0x2 /** * struct dpdmai_cfg - Structure representing DPDMAI configuration + * @num_queues: Number of the DMA queues * @priorities: Priorities for the DMA hardware processing; valid priorities are * configured with values 1-8; the entry following last valid entry * should be configured with 0 */ struct dpdmai_cfg { + u8 num_queues; u8 priorities[DPDMAI_PRIO_NUM]; }; @@ -80,20 +78,19 @@ struct dpdmai_cfg { * struct dpdmai_attr - Structure representing DPDMAI attributes * @id: DPDMAI object ID * @version: DPDMAI version + * @version.major: DPDMAI major version + * @version.minor: DPDMAI minor version * @num_of_priorities: number of priorities + * @num_of_queues: number of the DMA queues */ struct dpdmai_attr { int id; - /** - * struct version - DPDMAI version - * @major: DPDMAI major version - * @minor: DPDMAI minor version - */ struct { u16 major; u16 minor; } version; u8 num_of_priorities; + u8 num_of_queues; }; /** @@ -158,22 +155,24 @@ struct dpdmai_rx_queue_attr { u32 fqid; }; +struct dpdmai_tx_queue_attr { + u32 fqid; +}; + int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token); int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, - const struct dpdmai_cfg *cfg, u16 *token); +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u32 dpdmai_id, u16 token); int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, struct dpdmai_attr *attr); int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, - u8 priority, const struct dpdmai_rx_queue_cfg *cfg); + u8 queue_idx, u8 priority, const struct dpdmai_rx_queue_cfg *cfg); int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, - u8 priority, struct dpdmai_rx_queue_attr *attr); + u8 queue_idx, u8 priority, struct dpdmai_rx_queue_attr *attr); int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, - u16 token, u8 priority, u32 *fqid); + u16 token, u8 queue_idx, u8 priority, struct dpdmai_tx_queue_attr *attr); #endif /* __FSL_DPDMAI_H */ diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index a06a1575a2a5..a59212758029 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -3,10 +3,14 @@ // Copyright (c) 2013-2014 Freescale Semiconductor, Inc // Copyright (c) 2017 Sysam, Angelo Dureghello <angelo@sysam.it> +#include <linux/cleanup.h> +#include <linux/clk.h> #include <linux/dmapool.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/pm_domain.h> #include "fsl-edma-common.h" @@ -40,14 +44,70 @@ #define EDMA64_ERRH 0x28 #define EDMA64_ERRL 0x2c -#define EDMA_TCD 0x1000 +void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan) +{ + spin_lock(&fsl_chan->vchan.lock); + + if (!fsl_chan->edesc) { + /* terminate_all called before */ + spin_unlock(&fsl_chan->vchan.lock); + return; + } + + if (!fsl_chan->edesc->iscyclic) { + list_del(&fsl_chan->edesc->vdesc.node); + vchan_cookie_complete(&fsl_chan->edesc->vdesc); + fsl_chan->edesc = NULL; + fsl_chan->status = DMA_COMPLETE; + } else { + vchan_cyclic_callback(&fsl_chan->edesc->vdesc); + } + + if (!fsl_chan->edesc) + fsl_edma_xfer_desc(fsl_chan); + + spin_unlock(&fsl_chan->vchan.lock); +} + +static void fsl_edma3_enable_request(struct fsl_edma_chan *fsl_chan) +{ + u32 val, flags; + + flags = fsl_edma_drvflags(fsl_chan); + val = edma_readl_chreg(fsl_chan, ch_sbr); + if (fsl_chan->is_rxchan) + val |= EDMA_V3_CH_SBR_RD; + else + val |= EDMA_V3_CH_SBR_WR; + + if (fsl_chan->is_remote) + val &= ~(EDMA_V3_CH_SBR_RD | EDMA_V3_CH_SBR_WR); + + edma_writel_chreg(fsl_chan, val, ch_sbr); + + if (flags & FSL_EDMA_DRV_HAS_CHMUX) { + /* + * ch_mux: With the exception of 0, attempts to write a value + * already in use will be forced to 0. + */ + if (!edma_readl(fsl_chan->edma, fsl_chan->mux_addr)) + edma_writel(fsl_chan->edma, fsl_chan->srcid, fsl_chan->mux_addr); + } + + val = edma_readl_chreg(fsl_chan, ch_csr); + val |= EDMA_V3_CH_CSR_ERQ | EDMA_V3_CH_CSR_EEI; + edma_writel_chreg(fsl_chan, val, ch_csr); +} static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) { struct edma_regs *regs = &fsl_chan->edma->regs; u32 ch = fsl_chan->vchan.chan.chan_id; - if (fsl_chan->edma->drvdata->version == v1) { + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_SPLIT_REG) + return fsl_edma3_enable_request(fsl_chan); + + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_WRAP_IO) { edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), regs->seei); edma_writeb(fsl_chan->edma, ch, regs->serq); } else { @@ -59,12 +119,29 @@ static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) } } +static void fsl_edma3_disable_request(struct fsl_edma_chan *fsl_chan) +{ + u32 val = edma_readl_chreg(fsl_chan, ch_csr); + u32 flags; + + flags = fsl_edma_drvflags(fsl_chan); + + if (flags & FSL_EDMA_DRV_HAS_CHMUX) + edma_writel(fsl_chan->edma, 0, fsl_chan->mux_addr); + + val &= ~EDMA_V3_CH_CSR_ERQ; + edma_writel_chreg(fsl_chan, val, ch_csr); +} + void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) { struct edma_regs *regs = &fsl_chan->edma->regs; u32 ch = fsl_chan->vchan.chan.chan_id; - if (fsl_chan->edma->drvdata->version == v1) { + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_SPLIT_REG) + return fsl_edma3_disable_request(fsl_chan); + + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_WRAP_IO) { edma_writeb(fsl_chan->edma, ch, regs->cerq); edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), regs->ceei); } else { @@ -75,7 +152,6 @@ void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) iowrite8(EDMA_CEEI_CEEI(ch), regs->ceei); } } -EXPORT_SYMBOL_GPL(fsl_edma_disable_request); static void mux_configure8(struct fsl_edma_chan *fsl_chan, void __iomem *addr, u32 off, u32 slot, bool enable) @@ -112,36 +188,37 @@ void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, int endian_diff[4] = {3, 1, -1, -3}; u32 dmamux_nr = fsl_chan->edma->drvdata->dmamuxs; + if (!dmamux_nr) + return; + chans_per_mux = fsl_chan->edma->n_chans / dmamux_nr; ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; - if (fsl_chan->edma->drvdata->mux_swap) + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_MUX_SWAP) ch_off += endian_diff[ch_off % 4]; muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; slot = EDMAMUX_CHCFG_SOURCE(slot); - if (fsl_chan->edma->drvdata->version == v3) + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_CONFIG32) mux_configure32(fsl_chan, muxaddr, ch_off, slot, enable); else mux_configure8(fsl_chan, muxaddr, ch_off, slot, enable); } -EXPORT_SYMBOL_GPL(fsl_edma_chan_mux); -static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) +static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth src_addr_width, + enum dma_slave_buswidth dst_addr_width) { - switch (addr_width) { - case 1: - return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; - case 2: - return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; - case 4: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - case 8: - return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; - default: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - } + u32 src_val, dst_val; + + if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + + src_val = ffs(src_addr_width) - 1; + dst_val = ffs(dst_addr_width) - 1; + return dst_val | (src_val << 8); } void fsl_edma_free_desc(struct virt_dma_desc *vdesc) @@ -155,7 +232,6 @@ void fsl_edma_free_desc(struct virt_dma_desc *vdesc) fsl_desc->tcd[i].ptcd); kfree(fsl_desc); } -EXPORT_SYMBOL_GPL(fsl_edma_free_desc); int fsl_edma_terminate_all(struct dma_chan *chan) { @@ -166,13 +242,16 @@ int fsl_edma_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&fsl_chan->vchan.lock, flags); fsl_edma_disable_request(fsl_chan); fsl_chan->edesc = NULL; - fsl_chan->idle = true; + fsl_chan->status = DMA_COMPLETE; vchan_get_all_descriptors(&fsl_chan->vchan, &head); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_PD) + pm_runtime_allow(fsl_chan->pd_dev); + return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_terminate_all); int fsl_edma_pause(struct dma_chan *chan) { @@ -183,12 +262,10 @@ int fsl_edma_pause(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma_disable_request(fsl_chan); fsl_chan->status = DMA_PAUSED; - fsl_chan->idle = true; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_pause); int fsl_edma_resume(struct dma_chan *chan) { @@ -199,12 +276,10 @@ int fsl_edma_resume(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_resume); static void fsl_edma_unprep_slave_dma(struct fsl_edma_chan *fsl_chan) { @@ -265,40 +340,51 @@ int fsl_edma_slave_config(struct dma_chan *chan, return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_slave_config); static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, struct virt_dma_desc *vdesc, bool in_progress) { struct fsl_edma_desc *edesc = fsl_chan->edesc; - struct edma_regs *regs = &fsl_chan->edma->regs; - u32 ch = fsl_chan->vchan.chan.chan_id; enum dma_transfer_direction dir = edesc->dirn; - dma_addr_t cur_addr, dma_addr; + dma_addr_t cur_addr, dma_addr, old_addr; size_t len, size; + u32 nbytes = 0; int i; /* calculate the total size in this desc */ - for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) - len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); + for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) { + nbytes = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, nbytes); + if (nbytes & (EDMA_V3_TCD_NBYTES_DMLOE | EDMA_V3_TCD_NBYTES_SMLOE)) + nbytes = EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(nbytes); + len += nbytes * fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, biter); + } if (!in_progress) return len; - if (dir == DMA_MEM_TO_DEV) - cur_addr = edma_readl(fsl_chan->edma, ®s->tcd[ch].saddr); - else - cur_addr = edma_readl(fsl_chan->edma, ®s->tcd[ch].daddr); + /* 64bit read is not atomic, need read retry when high 32bit changed */ + do { + if (dir == DMA_MEM_TO_DEV) { + old_addr = edma_read_tcdreg(fsl_chan, saddr); + cur_addr = edma_read_tcdreg(fsl_chan, saddr); + } else { + old_addr = edma_read_tcdreg(fsl_chan, daddr); + cur_addr = edma_read_tcdreg(fsl_chan, daddr); + } + } while (upper_32_bits(cur_addr) != upper_32_bits(old_addr)); /* figure out the finished and calculate the residue */ for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { - size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); + nbytes = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, nbytes); + if (nbytes & (EDMA_V3_TCD_NBYTES_DMLOE | EDMA_V3_TCD_NBYTES_SMLOE)) + nbytes = EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(nbytes); + + size = nbytes * fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, biter); + if (dir == DMA_MEM_TO_DEV) - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); + dma_addr = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, saddr); else - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); + dma_addr = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, daddr); len -= size; if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { @@ -340,14 +426,9 @@ enum dma_status fsl_edma_tx_status(struct dma_chan *chan, return fsl_chan->status; } -EXPORT_SYMBOL_GPL(fsl_edma_tx_status); -static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, - struct fsl_edma_hw_tcd *tcd) +static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, void *tcd) { - struct fsl_edma_engine *edma = fsl_chan->edma; - struct edma_regs *regs = &fsl_chan->edma->regs; - u32 ch = fsl_chan->vchan.chan.chan_id; u16 csr = 0; /* @@ -356,39 +437,54 @@ static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, * big- or little-endian obeying the eDMA engine model endian, * and this is performed from specific edma_write functions */ - edma_writew(edma, 0, ®s->tcd[ch].csr); + edma_write_tcdreg(fsl_chan, 0, csr); + + edma_cp_tcd_to_reg(fsl_chan, tcd, saddr); + edma_cp_tcd_to_reg(fsl_chan, tcd, daddr); - edma_writel(edma, (s32)tcd->saddr, ®s->tcd[ch].saddr); - edma_writel(edma, (s32)tcd->daddr, ®s->tcd[ch].daddr); + edma_cp_tcd_to_reg(fsl_chan, tcd, attr); + edma_cp_tcd_to_reg(fsl_chan, tcd, soff); - edma_writew(edma, (s16)tcd->attr, ®s->tcd[ch].attr); - edma_writew(edma, tcd->soff, ®s->tcd[ch].soff); + edma_cp_tcd_to_reg(fsl_chan, tcd, nbytes); + edma_cp_tcd_to_reg(fsl_chan, tcd, slast); - edma_writel(edma, (s32)tcd->nbytes, ®s->tcd[ch].nbytes); - edma_writel(edma, (s32)tcd->slast, ®s->tcd[ch].slast); + edma_cp_tcd_to_reg(fsl_chan, tcd, citer); + edma_cp_tcd_to_reg(fsl_chan, tcd, biter); + edma_cp_tcd_to_reg(fsl_chan, tcd, doff); - edma_writew(edma, (s16)tcd->citer, ®s->tcd[ch].citer); - edma_writew(edma, (s16)tcd->biter, ®s->tcd[ch].biter); - edma_writew(edma, (s16)tcd->doff, ®s->tcd[ch].doff); + edma_cp_tcd_to_reg(fsl_chan, tcd, dlast_sga); - edma_writel(edma, (s32)tcd->dlast_sga, - ®s->tcd[ch].dlast_sga); + csr = fsl_edma_get_tcd_to_cpu(fsl_chan, tcd, csr); if (fsl_chan->is_sw) { - csr = le16_to_cpu(tcd->csr); csr |= EDMA_TCD_CSR_START; - tcd->csr = cpu_to_le16(csr); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, csr, csr); } - edma_writew(edma, (s16)tcd->csr, ®s->tcd[ch].csr); + /* + * Must clear CHn_CSR[DONE] bit before enable TCDn_CSR[ESG] at EDMAv3 + * eDMAv4 have not such requirement. + * Change MLINK need clear CHn_CSR[DONE] for both eDMAv3 and eDMAv4. + */ + if (((fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_CLEAR_DONE_E_SG) && + (csr & EDMA_TCD_CSR_E_SG)) || + ((fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_CLEAR_DONE_E_LINK) && + (csr & EDMA_TCD_CSR_E_LINK))) + edma_writel_chreg(fsl_chan, edma_readl_chreg(fsl_chan, ch_csr), ch_csr); + + + edma_cp_tcd_to_reg(fsl_chan, tcd, csr); } static inline -void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, - u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, - u16 biter, u16 doff, u32 dlast_sga, bool major_int, +void fsl_edma_fill_tcd(struct fsl_edma_chan *fsl_chan, + struct fsl_edma_hw_tcd *tcd, dma_addr_t src, dma_addr_t dst, + u16 attr, u16 soff, u32 nbytes, dma_addr_t slast, u16 citer, + u16 biter, u16 doff, dma_addr_t dlast_sga, bool major_int, bool disable_req, bool enable_sg) { + struct dma_slave_config *cfg = &fsl_chan->cfg; + u32 burst = 0; u16 csr = 0; /* @@ -397,22 +493,52 @@ void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, * So we put the value in little endian in memory, waiting * for fsl_edma_set_tcd_regs doing the swap. */ - tcd->saddr = cpu_to_le32(src); - tcd->daddr = cpu_to_le32(dst); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, src, saddr); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, dst, daddr); + + fsl_edma_set_tcd_to_le(fsl_chan, tcd, attr, attr); - tcd->attr = cpu_to_le16(attr); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, soff, soff); - tcd->soff = cpu_to_le16(soff); + /* If we expect to have either multi_fifo or a port window size, + * we will use minor loop offset, meaning bits 29-10 will be used for + * address offset, while bits 9-0 will be used to tell DMA how much + * data to read from addr. + * If we don't have either of those, will use a major loop reading from addr + * nbytes (29bits). + */ + if (cfg->direction == DMA_MEM_TO_DEV) { + if (fsl_chan->is_multi_fifo) + burst = cfg->dst_maxburst * 4; + if (cfg->dst_port_window_size) + burst = cfg->dst_port_window_size * cfg->dst_addr_width; + if (burst) { + nbytes |= EDMA_V3_TCD_NBYTES_MLOFF(-burst); + nbytes |= EDMA_V3_TCD_NBYTES_DMLOE; + nbytes &= ~EDMA_V3_TCD_NBYTES_SMLOE; + } + } else { + if (fsl_chan->is_multi_fifo) + burst = cfg->src_maxburst * 4; + if (cfg->src_port_window_size) + burst = cfg->src_port_window_size * cfg->src_addr_width; + if (burst) { + nbytes |= EDMA_V3_TCD_NBYTES_MLOFF(-burst); + nbytes |= EDMA_V3_TCD_NBYTES_SMLOE; + nbytes &= ~EDMA_V3_TCD_NBYTES_DMLOE; + } + } + + fsl_edma_set_tcd_to_le(fsl_chan, tcd, nbytes, nbytes); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, slast, slast); - tcd->nbytes = cpu_to_le32(nbytes); - tcd->slast = cpu_to_le32(slast); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, EDMA_TCD_CITER_CITER(citer), citer); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, doff, doff); - tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); - tcd->doff = cpu_to_le16(doff); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, dlast_sga, dlast_sga); - tcd->dlast_sga = cpu_to_le32(dlast_sga); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, EDMA_TCD_BITER_BITER(biter), biter); - tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); if (major_int) csr |= EDMA_TCD_CSR_INT_MAJOR; @@ -422,7 +548,15 @@ void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, if (enable_sg) csr |= EDMA_TCD_CSR_E_SG; - tcd->csr = cpu_to_le16(csr); + if (fsl_chan->is_rxchan) + csr |= EDMA_TCD_CSR_ACTIVE; + + if (fsl_chan->is_sw) + csr |= EDMA_TCD_CSR_START; + + fsl_edma_set_tcd_to_le(fsl_chan, tcd, csr, csr); + + trace_edma_fill_tcd(fsl_chan, tcd); } static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, @@ -461,9 +595,11 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); struct fsl_edma_desc *fsl_desc; dma_addr_t dma_buf_next; + bool major_int = true; int sg_len, i; - u32 src_addr, dst_addr, last_sg, nbytes; + dma_addr_t src_addr, dst_addr, last_sg; u16 soff, doff, iter; + u32 nbytes; if (!is_slave_direction(direction)) return NULL; @@ -480,13 +616,19 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( dma_buf_next = dma_addr; if (direction == DMA_MEM_TO_DEV) { + if (!fsl_chan->cfg.src_addr_width) + fsl_chan->cfg.src_addr_width = fsl_chan->cfg.dst_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.dst_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.dst_addr_width * fsl_chan->cfg.dst_maxburst; } else { + if (!fsl_chan->cfg.dst_addr_width) + fsl_chan->cfg.dst_addr_width = fsl_chan->cfg.src_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.src_addr_width * fsl_chan->cfg.src_maxburst; } @@ -504,23 +646,32 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( src_addr = dma_buf_next; dst_addr = fsl_chan->dma_dev_addr; soff = fsl_chan->cfg.dst_addr_width; - doff = 0; - } else { + doff = fsl_chan->is_multi_fifo ? 4 : 0; + if (fsl_chan->cfg.dst_port_window_size) + doff = fsl_chan->cfg.dst_addr_width; + } else if (direction == DMA_DEV_TO_MEM) { src_addr = fsl_chan->dma_dev_addr; dst_addr = dma_buf_next; - soff = 0; + soff = fsl_chan->is_multi_fifo ? 4 : 0; doff = fsl_chan->cfg.src_addr_width; + if (fsl_chan->cfg.src_port_window_size) + soff = fsl_chan->cfg.src_addr_width; + } else { + /* DMA_DEV_TO_DEV */ + src_addr = fsl_chan->cfg.src_addr; + dst_addr = fsl_chan->cfg.dst_addr; + soff = doff = 0; + major_int = false; } - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, dst_addr, fsl_chan->attr, soff, nbytes, 0, iter, - iter, doff, last_sg, true, false, true); + iter, doff, last_sg, major_int, false, true); dma_buf_next += period_len; } return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); } -EXPORT_SYMBOL_GPL(fsl_edma_prep_dma_cyclic); struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, @@ -530,8 +681,9 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); struct fsl_edma_desc *fsl_desc; struct scatterlist *sg; - u32 src_addr, dst_addr, last_sg, nbytes; + dma_addr_t src_addr, dst_addr, last_sg; u16 soff, doff, iter; + u32 nbytes; int i; if (!is_slave_direction(direction)) @@ -547,13 +699,19 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( fsl_desc->dirn = direction; if (direction == DMA_MEM_TO_DEV) { + if (!fsl_chan->cfg.src_addr_width) + fsl_chan->cfg.src_addr_width = fsl_chan->cfg.dst_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.dst_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.dst_addr_width * fsl_chan->cfg.dst_maxburst; } else { + if (!fsl_chan->cfg.dst_addr_width) + fsl_chan->cfg.dst_addr_width = fsl_chan->cfg.src_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.src_addr_width * fsl_chan->cfg.src_maxburst; } @@ -564,23 +722,51 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( dst_addr = fsl_chan->dma_dev_addr; soff = fsl_chan->cfg.dst_addr_width; doff = 0; - } else { + } else if (direction == DMA_DEV_TO_MEM) { src_addr = fsl_chan->dma_dev_addr; dst_addr = sg_dma_address(sg); soff = 0; doff = fsl_chan->cfg.src_addr_width; + } else { + /* DMA_DEV_TO_DEV */ + src_addr = fsl_chan->cfg.src_addr; + dst_addr = fsl_chan->cfg.dst_addr; + soff = 0; + doff = 0; } + /* + * Choose the suitable burst length if sg_dma_len is not + * multiple of burst length so that the whole transfer length is + * multiple of minor loop(burst length). + */ + if (sg_dma_len(sg) % nbytes) { + u32 width = (direction == DMA_DEV_TO_MEM) ? doff : soff; + u32 burst = (direction == DMA_DEV_TO_MEM) ? + fsl_chan->cfg.src_maxburst : + fsl_chan->cfg.dst_maxburst; + int j; + + for (j = burst; j > 1; j--) { + if (!(sg_dma_len(sg) % (j * width))) { + nbytes = j * width; + break; + } + } + /* Set burst size as 1 if there's no suitable one */ + if (j == 1) + nbytes = width; + } iter = sg_dma_len(sg) / nbytes; if (i < sg_len - 1) { last_sg = fsl_desc->tcd[(i + 1)].ptcd; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, dst_addr, fsl_chan->attr, soff, nbytes, 0, iter, iter, doff, last_sg, false, false, true); } else { last_sg = 0; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, dst_addr, fsl_chan->attr, soff, nbytes, 0, iter, iter, doff, last_sg, true, true, false); @@ -589,7 +775,6 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); } -EXPORT_SYMBOL_GPL(fsl_edma_prep_slave_sg); struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(struct dma_chan *chan, dma_addr_t dma_dst, dma_addr_t dma_src, @@ -597,6 +782,10 @@ struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(struct dma_chan *chan, { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); struct fsl_edma_desc *fsl_desc; + u32 src_bus_width, dst_bus_width; + + src_bus_width = min_t(u32, DMA_SLAVE_BUSWIDTH_32_BYTES, 1 << (ffs(dma_src) - 1)); + dst_bus_width = min_t(u32, DMA_SLAVE_BUSWIDTH_32_BYTES, 1 << (ffs(dma_dst) - 1)); fsl_desc = fsl_edma_alloc_desc(fsl_chan, 1); if (!fsl_desc) @@ -604,15 +793,17 @@ struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(struct dma_chan *chan, fsl_desc->iscyclic = false; fsl_chan->is_sw = true; + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_MEM_REMOTE) + fsl_chan->is_remote = true; /* To match with copy_align and max_seg_size so 1 tcd is enough */ - fsl_edma_fill_tcd(fsl_desc->tcd[0].vtcd, dma_src, dma_dst, - EDMA_TCD_ATTR_SSIZE_32BYTE | EDMA_TCD_ATTR_DSIZE_32BYTE, - 32, len, 0, 1, 1, 32, 0, true, true, false); + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[0].vtcd, dma_src, dma_dst, + fsl_edma_get_tcd_attr(src_bus_width, dst_bus_width), + src_bus_width, len, 0, 1, 1, dst_bus_width, 0, true, + true, false); return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); } -EXPORT_SYMBOL_GPL(fsl_edma_prep_memcpy); void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) { @@ -627,9 +818,7 @@ void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); fsl_edma_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; } -EXPORT_SYMBOL_GPL(fsl_edma_xfer_desc); void fsl_edma_issue_pending(struct dma_chan *chan) { @@ -649,18 +838,44 @@ void fsl_edma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); } -EXPORT_SYMBOL_GPL(fsl_edma_issue_pending); int fsl_edma_alloc_chan_resources(struct dma_chan *chan) { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + int ret = 0; + + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) + clk_prepare_enable(fsl_chan->clk); fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, - sizeof(struct fsl_edma_hw_tcd), + fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_TCD64 ? + sizeof(struct fsl_edma_hw_tcd64) : sizeof(struct fsl_edma_hw_tcd), 32, 0); + + if (fsl_chan->txirq) + ret = request_irq(fsl_chan->txirq, fsl_chan->irq_handler, IRQF_SHARED, + fsl_chan->chan_name, fsl_chan); + + if (ret) + goto err_txirq; + + if (fsl_chan->errirq > 0) + ret = request_irq(fsl_chan->errirq, fsl_chan->errirq_handler, IRQF_SHARED, + fsl_chan->errirq_name, fsl_chan); + + if (ret) + goto err_errirq; + return 0; + +err_errirq: + if (fsl_chan->txirq) + free_irq(fsl_chan->txirq, fsl_chan); +err_txirq: + dma_pool_destroy(fsl_chan->tcd_pool); + + return ret; } -EXPORT_SYMBOL_GPL(fsl_edma_alloc_chan_resources); void fsl_edma_free_chan_resources(struct dma_chan *chan) { @@ -678,12 +893,20 @@ void fsl_edma_free_chan_resources(struct dma_chan *chan) fsl_edma_unprep_slave_dma(fsl_chan); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + if (fsl_chan->txirq) + free_irq(fsl_chan->txirq, fsl_chan); + if (fsl_chan->errirq) + free_irq(fsl_chan->errirq, fsl_chan); + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); dma_pool_destroy(fsl_chan->tcd_pool); fsl_chan->tcd_pool = NULL; fsl_chan->is_sw = false; + fsl_chan->srcid = 0; + fsl_chan->is_remote = false; + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) + clk_disable_unprepare(fsl_chan->clk); } -EXPORT_SYMBOL_GPL(fsl_edma_free_chan_resources); void fsl_edma_cleanup_vchan(struct dma_device *dmadev) { @@ -695,12 +918,10 @@ void fsl_edma_cleanup_vchan(struct dma_device *dmadev) tasklet_kill(&chan->vchan.task); } } -EXPORT_SYMBOL_GPL(fsl_edma_cleanup_vchan); /* - * On the 32 channels Vybrid/mpc577x edma version (here called "v1"), - * register offsets are different compared to ColdFire mcf5441x 64 channels - * edma (here called "v2"). + * On the 32 channels Vybrid/mpc577x edma version, register offsets are + * different compared to ColdFire mcf5441x 64 channels edma. * * This function sets up register offsets as per proper declared version * so must be called in xxx_edma_probe() just after setting the @@ -708,41 +929,30 @@ EXPORT_SYMBOL_GPL(fsl_edma_cleanup_vchan); */ void fsl_edma_setup_regs(struct fsl_edma_engine *edma) { + bool is64 = !!(edma->drvdata->flags & FSL_EDMA_DRV_EDMA64); + edma->regs.cr = edma->membase + EDMA_CR; edma->regs.es = edma->membase + EDMA_ES; edma->regs.erql = edma->membase + EDMA_ERQ; edma->regs.eeil = edma->membase + EDMA_EEI; - edma->regs.serq = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_SERQ : EDMA_SERQ); - edma->regs.cerq = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CERQ : EDMA_CERQ); - edma->regs.seei = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_SEEI : EDMA_SEEI); - edma->regs.ceei = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CEEI : EDMA_CEEI); - edma->regs.cint = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CINT : EDMA_CINT); - edma->regs.cerr = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CERR : EDMA_CERR); - edma->regs.ssrt = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_SSRT : EDMA_SSRT); - edma->regs.cdne = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CDNE : EDMA_CDNE); - edma->regs.intl = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_INTL : EDMA_INTR); - edma->regs.errl = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_ERRL : EDMA_ERR); - - if (edma->drvdata->version == v2) { + edma->regs.serq = edma->membase + (is64 ? EDMA64_SERQ : EDMA_SERQ); + edma->regs.cerq = edma->membase + (is64 ? EDMA64_CERQ : EDMA_CERQ); + edma->regs.seei = edma->membase + (is64 ? EDMA64_SEEI : EDMA_SEEI); + edma->regs.ceei = edma->membase + (is64 ? EDMA64_CEEI : EDMA_CEEI); + edma->regs.cint = edma->membase + (is64 ? EDMA64_CINT : EDMA_CINT); + edma->regs.cerr = edma->membase + (is64 ? EDMA64_CERR : EDMA_CERR); + edma->regs.ssrt = edma->membase + (is64 ? EDMA64_SSRT : EDMA_SSRT); + edma->regs.cdne = edma->membase + (is64 ? EDMA64_CDNE : EDMA_CDNE); + edma->regs.intl = edma->membase + (is64 ? EDMA64_INTL : EDMA_INTR); + edma->regs.errl = edma->membase + (is64 ? EDMA64_ERRL : EDMA_ERR); + + if (is64) { edma->regs.erqh = edma->membase + EDMA64_ERQH; edma->regs.eeih = edma->membase + EDMA64_EEIH; edma->regs.errh = edma->membase + EDMA64_ERRH; edma->regs.inth = edma->membase + EDMA64_INTH; } - - edma->regs.tcd = edma->membase + EDMA_TCD; } -EXPORT_SYMBOL_GPL(fsl_edma_setup_regs); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index 004ec4a6bc86..205a96489094 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -29,19 +29,10 @@ #define EDMA_TCD_ATTR_DMOD(x) (((x) & GENMASK(4, 0)) << 3) #define EDMA_TCD_ATTR_SSIZE(x) (((x) & GENMASK(2, 0)) << 8) #define EDMA_TCD_ATTR_SMOD(x) (((x) & GENMASK(4, 0)) << 11) -#define EDMA_TCD_ATTR_DSIZE_8BIT 0 -#define EDMA_TCD_ATTR_DSIZE_16BIT BIT(0) -#define EDMA_TCD_ATTR_DSIZE_32BIT BIT(1) -#define EDMA_TCD_ATTR_DSIZE_64BIT (BIT(0) | BIT(1)) -#define EDMA_TCD_ATTR_DSIZE_32BYTE (BIT(2) | BIT(0)) -#define EDMA_TCD_ATTR_SSIZE_8BIT 0 -#define EDMA_TCD_ATTR_SSIZE_16BIT (EDMA_TCD_ATTR_DSIZE_16BIT << 8) -#define EDMA_TCD_ATTR_SSIZE_32BIT (EDMA_TCD_ATTR_DSIZE_32BIT << 8) -#define EDMA_TCD_ATTR_SSIZE_64BIT (EDMA_TCD_ATTR_DSIZE_64BIT << 8) -#define EDMA_TCD_ATTR_SSIZE_32BYTE (EDMA_TCD_ATTR_DSIZE_32BYTE << 8) - -#define EDMA_TCD_CITER_CITER(x) ((x) & GENMASK(14, 0)) -#define EDMA_TCD_BITER_BITER(x) ((x) & GENMASK(14, 0)) + +#define EDMA_TCD_ITER_MASK GENMASK(14, 0) +#define EDMA_TCD_CITER_CITER(x) ((x) & EDMA_TCD_ITER_MASK) +#define EDMA_TCD_BITER_BITER(x) ((x) & EDMA_TCD_ITER_MASK) #define EDMA_TCD_CSR_START BIT(0) #define EDMA_TCD_CSR_INT_MAJOR BIT(1) @@ -52,16 +43,46 @@ #define EDMA_TCD_CSR_ACTIVE BIT(6) #define EDMA_TCD_CSR_DONE BIT(7) +#define EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(x) ((x) & GENMASK(9, 0)) +#define EDMA_V3_TCD_NBYTES_MLOFF(x) (x << 10) +#define EDMA_V3_TCD_NBYTES_DMLOE (1 << 30) +#define EDMA_V3_TCD_NBYTES_SMLOE (1 << 31) + #define EDMAMUX_CHCFG_DIS 0x0 #define EDMAMUX_CHCFG_ENBL 0x80 #define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) #define DMAMUX_NR 2 +#define EDMA_TCD 0x1000 + #define FSL_EDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +#define EDMA_V3_CH_SBR_RD BIT(22) +#define EDMA_V3_CH_SBR_WR BIT(21) +#define EDMA_V3_CH_CSR_ERQ BIT(0) +#define EDMA_V3_CH_CSR_EARQ BIT(1) +#define EDMA_V3_CH_CSR_EEI BIT(2) +#define EDMA_V3_CH_CSR_DONE BIT(30) +#define EDMA_V3_CH_CSR_ACTIVE BIT(31) +#define EDMA_V3_CH_ES_ERR BIT(31) +#define EDMA_V3_MP_ES_VLD BIT(31) + +#define EDMA_V3_CH_ERR_DBE BIT(0) +#define EDMA_V3_CH_ERR_SBE BIT(1) +#define EDMA_V3_CH_ERR_SGE BIT(2) +#define EDMA_V3_CH_ERR_NCE BIT(3) +#define EDMA_V3_CH_ERR_DOE BIT(4) +#define EDMA_V3_CH_ERR_DAE BIT(5) +#define EDMA_V3_CH_ERR_SOE BIT(6) +#define EDMA_V3_CH_ERR_SAE BIT(7) +#define EDMA_V3_CH_ERR_ECX BIT(8) +#define EDMA_V3_CH_ERR_UCE BIT(9) +#define EDMA_V3_CH_ERR BIT(31) + enum fsl_edma_pm_state { RUNNING = 0, SUSPENDED, @@ -81,6 +102,35 @@ struct fsl_edma_hw_tcd { __le16 biter; }; +struct fsl_edma_hw_tcd64 { + __le64 saddr; + __le16 soff; + __le16 attr; + __le32 nbytes; + __le64 slast; + __le64 daddr; + __le64 dlast_sga; + __le16 doff; + __le16 citer; + __le16 csr; + __le16 biter; +} __packed; + +struct fsl_edma3_ch_reg { + __le32 ch_csr; + __le32 ch_es; + __le32 ch_int; + __le32 ch_sbr; + __le32 ch_pri; + __le32 ch_mux; + __le32 ch_mattr; /* edma4, reserved for edma3 */ + __le32 ch_reserved; + union { + struct fsl_edma_hw_tcd tcd; + struct fsl_edma_hw_tcd64 tcd64; + }; +} __packed; + /* * These are iomem pointers, for both v32 and v64. */ @@ -103,20 +153,17 @@ struct edma_regs { void __iomem *intl; void __iomem *errh; void __iomem *errl; - struct fsl_edma_hw_tcd __iomem *tcd; }; struct fsl_edma_sw_tcd { dma_addr_t ptcd; - struct fsl_edma_hw_tcd *vtcd; + void *vtcd; }; struct fsl_edma_chan { struct virt_dma_chan vchan; enum dma_status status; enum fsl_edma_pm_state pm_state; - bool idle; - u32 slave_id; struct fsl_edma_engine *edma; struct fsl_edma_desc *edesc; struct dma_slave_config cfg; @@ -126,7 +173,26 @@ struct fsl_edma_chan { dma_addr_t dma_dev_addr; u32 dma_dev_size; enum dma_data_direction dma_dir; - char chan_name[16]; + char chan_name[32]; + char errirq_name[36]; + void __iomem *tcd; + void __iomem *mux_addr; + u32 real_count; + struct work_struct issue_worker; + struct platform_device *pdev; + struct device *pd_dev; + struct device_link *pd_dev_link; + u32 srcid; + struct clk *clk; + int priority; + int hw_chanid; + int txirq; + int errirq; + irqreturn_t (*irq_handler)(int irq, void *dev_id); + irqreturn_t (*errirq_handler)(int irq, void *dev_id); + bool is_rxchan; + bool is_remote; + bool is_multi_fifo; }; struct fsl_edma_desc { @@ -138,17 +204,49 @@ struct fsl_edma_desc { struct fsl_edma_sw_tcd tcd[]; }; -enum edma_version { - v1, /* 32ch, Vybrid, mpc57x, etc */ - v2, /* 64ch Coldfire */ - v3, /* 32ch, i.mx7ulp */ -}; +#define FSL_EDMA_DRV_HAS_DMACLK BIT(0) +#define FSL_EDMA_DRV_MUX_SWAP BIT(1) +#define FSL_EDMA_DRV_CONFIG32 BIT(2) +#define FSL_EDMA_DRV_WRAP_IO BIT(3) +#define FSL_EDMA_DRV_EDMA64 BIT(4) +#define FSL_EDMA_DRV_HAS_PD BIT(5) +#define FSL_EDMA_DRV_HAS_CHCLK BIT(6) +#define FSL_EDMA_DRV_HAS_CHMUX BIT(7) +#define FSL_EDMA_DRV_MEM_REMOTE BIT(8) +/* control and status register is in tcd address space, edma3 reg layout */ +#define FSL_EDMA_DRV_SPLIT_REG BIT(9) +#define FSL_EDMA_DRV_BUS_8BYTE BIT(10) +#define FSL_EDMA_DRV_DEV_TO_DEV BIT(11) +#define FSL_EDMA_DRV_ALIGN_64BYTE BIT(12) +/* Need clean CHn_CSR DONE before enable TCD's ESG */ +#define FSL_EDMA_DRV_CLEAR_DONE_E_SG BIT(13) +/* Need clean CHn_CSR DONE before enable TCD's MAJORELINK */ +#define FSL_EDMA_DRV_CLEAR_DONE_E_LINK BIT(14) +#define FSL_EDMA_DRV_TCD64 BIT(15) +/* All channel ERR IRQ share one IRQ line */ +#define FSL_EDMA_DRV_ERRIRQ_SHARE BIT(16) + + +#define FSL_EDMA_DRV_EDMA3 (FSL_EDMA_DRV_SPLIT_REG | \ + FSL_EDMA_DRV_BUS_8BYTE | \ + FSL_EDMA_DRV_DEV_TO_DEV | \ + FSL_EDMA_DRV_ALIGN_64BYTE | \ + FSL_EDMA_DRV_CLEAR_DONE_E_SG | \ + FSL_EDMA_DRV_CLEAR_DONE_E_LINK) + +#define FSL_EDMA_DRV_EDMA4 (FSL_EDMA_DRV_SPLIT_REG | \ + FSL_EDMA_DRV_BUS_8BYTE | \ + FSL_EDMA_DRV_DEV_TO_DEV | \ + FSL_EDMA_DRV_ALIGN_64BYTE | \ + FSL_EDMA_DRV_CLEAR_DONE_E_LINK) struct fsl_edma_drvdata { - enum edma_version version; - u32 dmamuxs; - bool has_dmaclk; - bool mux_swap; + u32 dmamuxs; /* only used before v3 */ + u32 chreg_off; + u32 chreg_space_sz; + u32 flags; + u32 mux_off; /* channel mux register offset */ + u32 mux_skip; /* how much skip for each channel */ int (*setup_irq)(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma); }; @@ -163,24 +261,158 @@ struct fsl_edma_engine { const struct fsl_edma_drvdata *drvdata; u32 n_chans; int txirq; + int txirq_16_31; int errirq; bool big_endian; struct edma_regs regs; - struct fsl_edma_chan chans[]; + u64 chan_masked; + struct fsl_edma_chan chans[] __counted_by(n_chans); }; +static inline u32 fsl_edma_drvflags(struct fsl_edma_chan *fsl_chan) +{ + return fsl_chan->edma->drvdata->flags; +} + +#define edma_read_tcdreg_c(chan, _tcd, __name) \ +_Generic(((_tcd)->__name), \ + __iomem __le64 : edma_readq(chan->edma, &(_tcd)->__name), \ + __iomem __le32 : edma_readl(chan->edma, &(_tcd)->__name), \ + __iomem __le16 : edma_readw(chan->edma, &(_tcd)->__name) \ + ) + +#define edma_read_tcdreg(chan, __name) \ +((fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) ? \ + edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd64 __iomem *)chan->tcd), __name) : \ + edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd __iomem *)chan->tcd), __name) \ +) + +#define edma_write_tcdreg_c(chan, _tcd, _val, __name) \ +_Generic((_tcd->__name), \ + __iomem __le64 : edma_writeq(chan->edma, (u64 __force)(_val), &_tcd->__name), \ + __iomem __le32 : edma_writel(chan->edma, (u32 __force)(_val), &_tcd->__name), \ + __iomem __le16 : edma_writew(chan->edma, (u16 __force)(_val), &_tcd->__name), \ + __iomem u8 : edma_writeb(chan->edma, _val, &_tcd->__name) \ + ) + +#define edma_write_tcdreg(chan, val, __name) \ +do { \ + struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \ + struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd; \ + \ + if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) \ + edma_write_tcdreg_c(chan, tcd64_r, val, __name); \ + else \ + edma_write_tcdreg_c(chan, tcd_r, val, __name); \ +} while (0) + +#define edma_cp_tcd_to_reg(chan, __tcd, __name) \ +do { \ + struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \ + struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd; \ + struct fsl_edma_hw_tcd64 *tcd64_m = (struct fsl_edma_hw_tcd64 *)__tcd; \ + struct fsl_edma_hw_tcd *tcd_m = (struct fsl_edma_hw_tcd *)__tcd; \ + \ + if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) \ + edma_write_tcdreg_c(chan, tcd64_r, tcd64_m->__name, __name); \ + else \ + edma_write_tcdreg_c(chan, tcd_r, tcd_m->__name, __name); \ +} while (0) + +#define edma_readl_chreg(chan, __name) \ + edma_readl(chan->edma, \ + (void __iomem *)&(container_of(((__force void *)chan->tcd),\ + struct fsl_edma3_ch_reg, tcd)->__name)) + +#define edma_writel_chreg(chan, val, __name) \ + edma_writel(chan->edma, val, \ + (void __iomem *)&(container_of(((__force void *)chan->tcd),\ + struct fsl_edma3_ch_reg, tcd)->__name)) + +#define fsl_edma_get_tcd(_chan, _tcd, _field) \ +(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? (((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \ + (((struct fsl_edma_hw_tcd *)_tcd)->_field)) + +#define fsl_edma_le_to_cpu(x) \ +_Generic((x), \ + __le64 : le64_to_cpu((x)), \ + __le32 : le32_to_cpu((x)), \ + __le16 : le16_to_cpu((x)) \ +) + +#define fsl_edma_get_tcd_to_cpu(_chan, _tcd, _field) \ +(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? \ + fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \ + fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd *)_tcd)->_field)) + +#define fsl_edma_set_tcd_to_le_c(_tcd, _val, _field) \ +_Generic(((_tcd)->_field), \ + __le64 : (_tcd)->_field = cpu_to_le64(_val), \ + __le32 : (_tcd)->_field = cpu_to_le32(_val), \ + __le16 : (_tcd)->_field = cpu_to_le16(_val) \ +) + +#define fsl_edma_set_tcd_to_le(_chan, _tcd, _val, _field) \ +do { \ + if (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64) \ + fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd64 *)_tcd, _val, _field); \ + else \ + fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd *)_tcd, _val, _field); \ +} while (0) + +/* Need after struct defination */ +#include "fsl-edma-trace.h" + /* * R/W functions for big- or little-endian registers: * The eDMA controller's endian is independent of the CPU core's endian. * For the big-endian IP module, the offset for 8-bit or 16-bit registers * should also be swapped opposite to that in little-endian IP. */ +static inline u64 edma_readq(struct fsl_edma_engine *edma, void __iomem *addr) +{ + u64 l, h; + + if (edma->big_endian) { + l = ioread32be(addr); + h = ioread32be(addr + 4); + } else { + l = ioread32(addr); + h = ioread32(addr + 4); + } + + trace_edma_readl(edma, addr, l); + trace_edma_readl(edma, addr + 4, h); + + return (h << 32) | l; +} + static inline u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) { + u32 val; + if (edma->big_endian) - return ioread32be(addr); + val = ioread32be(addr); else - return ioread32(addr); + val = ioread32(addr); + + trace_edma_readl(edma, addr, val); + + return val; +} + +static inline u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr) +{ + u16 val; + + if (edma->big_endian) + val = ioread16be(addr); + else + val = ioread16(addr); + + trace_edma_readw(edma, addr, val); + + return val; } static inline void edma_writeb(struct fsl_edma_engine *edma, @@ -191,6 +423,8 @@ static inline void edma_writeb(struct fsl_edma_engine *edma, iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); else iowrite8(val, addr); + + trace_edma_writeb(edma, addr, val); } static inline void edma_writew(struct fsl_edma_engine *edma, @@ -201,6 +435,8 @@ static inline void edma_writew(struct fsl_edma_engine *edma, iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); else iowrite16(val, addr); + + trace_edma_writew(edma, addr, val); } static inline void edma_writel(struct fsl_edma_engine *edma, @@ -210,6 +446,23 @@ static inline void edma_writel(struct fsl_edma_engine *edma, iowrite32be(val, addr); else iowrite32(val, addr); + + trace_edma_writel(edma, addr, val); +} + +static inline void edma_writeq(struct fsl_edma_engine *edma, + u64 val, void __iomem *addr) +{ + if (edma->big_endian) { + iowrite32be(val & 0xFFFFFFFF, addr); + iowrite32be(val >> 32, addr + 4); + } else { + iowrite32(val & 0xFFFFFFFF, addr); + iowrite32(val >> 32, addr + 4); + } + + trace_edma_writel(edma, addr, val & 0xFFFFFFFF); + trace_edma_writel(edma, addr + 4, val >> 32); } static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) @@ -222,6 +475,12 @@ static inline struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) return container_of(vd, struct fsl_edma_desc, vdesc); } +static inline void fsl_edma_err_chan_handler(struct fsl_edma_chan *fsl_chan) +{ + fsl_chan->status = DMA_ERROR; +} + +void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan); void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan); void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, unsigned int slot, bool enable); diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c new file mode 100644 index 000000000000..a753b7cbfa7a --- /dev/null +++ b/drivers/dma/fsl-edma-main.c @@ -0,0 +1,1003 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * drivers/dma/fsl-edma.c + * + * Copyright 2013-2014 Freescale Semiconductor, Inc. + * Copyright 2024 NXP + * + * Driver for the Freescale eDMA engine with flexible channel multiplexing + * capability for DMA request sources. The eDMA block can be found on some + * Vybrid, Layerscape and S32G SoCs. + */ + +#include <dt-bindings/dma/fsl-edma.h> +#include <linux/bitfield.h> +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/clk.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/pm_domain.h> +#include <linux/property.h> + +#include "fsl-edma-common.h" + +static void fsl_edma_synchronize(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + + vchan_synchronize(&fsl_chan->vchan); +} + +static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int intr, ch; + struct edma_regs *regs = &fsl_edma->regs; + + intr = edma_readl(fsl_edma, regs->intl); + if (!intr) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (intr & (0x1 << ch)) { + edma_writeb(fsl_edma, EDMA_CINT_CINT(ch), regs->cint); + fsl_edma_tx_chan_handler(&fsl_edma->chans[ch]); + } + } + return IRQ_HANDLED; +} + +static void fsl_edma3_err_check(struct fsl_edma_chan *fsl_chan) +{ + unsigned int ch_err; + u32 val; + + scoped_guard(spinlock, &fsl_chan->vchan.lock) { + ch_err = edma_readl_chreg(fsl_chan, ch_es); + if (!(ch_err & EDMA_V3_CH_ERR)) + return; + + edma_writel_chreg(fsl_chan, EDMA_V3_CH_ERR, ch_es); + val = edma_readl_chreg(fsl_chan, ch_csr); + val &= ~EDMA_V3_CH_CSR_ERQ; + edma_writel_chreg(fsl_chan, val, ch_csr); + } + + /* Ignore this interrupt since channel has been disabled already */ + if (!fsl_chan->edesc) + return; + + if (ch_err & EDMA_V3_CH_ERR_DBE) + dev_err(&fsl_chan->pdev->dev, "Destination Bus Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SBE) + dev_err(&fsl_chan->pdev->dev, "Source Bus Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SGE) + dev_err(&fsl_chan->pdev->dev, "Scatter/Gather Configuration Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_NCE) + dev_err(&fsl_chan->pdev->dev, "NBYTES/CITER Configuration Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_DOE) + dev_err(&fsl_chan->pdev->dev, "Destination Offset Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_DAE) + dev_err(&fsl_chan->pdev->dev, "Destination Address Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SOE) + dev_err(&fsl_chan->pdev->dev, "Source Offset Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SAE) + dev_err(&fsl_chan->pdev->dev, "Source Address Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_ECX) + dev_err(&fsl_chan->pdev->dev, "Transfer Canceled interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_UCE) + dev_err(&fsl_chan->pdev->dev, "Uncorrectable TCD error during channel execution interrupt.\n"); + + fsl_chan->status = DMA_ERROR; +} + +static irqreturn_t fsl_edma3_err_handler_per_chan(int irq, void *dev_id) +{ + struct fsl_edma_chan *fsl_chan = dev_id; + + fsl_edma3_err_check(fsl_chan); + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma3_err_handler_shared(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int ch; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (fsl_edma->chan_masked & BIT(ch)) + continue; + + fsl_edma3_err_check(&fsl_edma->chans[ch]); + } + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) +{ + struct fsl_edma_chan *fsl_chan = dev_id; + unsigned int intr; + + intr = edma_readl_chreg(fsl_chan, ch_int); + if (!intr) + return IRQ_NONE; + + edma_writel_chreg(fsl_chan, 1, ch_int); + + fsl_edma_tx_chan_handler(fsl_chan); + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma2_tx_handler(int irq, void *devi_id) +{ + struct fsl_edma_chan *fsl_chan = devi_id; + + return fsl_edma_tx_handler(irq, fsl_chan->edma); +} + +static irqreturn_t fsl_edma3_or_tx_handler(int irq, void *dev_id, + u8 start, u8 end) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + struct fsl_edma_chan *chan; + int i; + + end = min(end, fsl_edma->n_chans); + + for (i = start; i < end; i++) { + chan = &fsl_edma->chans[i]; + + fsl_edma3_tx_handler(irq, chan); + } + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma3_tx_0_15_handler(int irq, void *dev_id) +{ + return fsl_edma3_or_tx_handler(irq, dev_id, 0, 16); +} + +static irqreturn_t fsl_edma3_tx_16_31_handler(int irq, void *dev_id) +{ + return fsl_edma3_or_tx_handler(irq, dev_id, 16, 32); +} + +static irqreturn_t fsl_edma3_or_err_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + struct edma_regs *regs = &fsl_edma->regs; + unsigned int err, ch, ch_es; + struct fsl_edma_chan *chan; + + err = edma_readl(fsl_edma, regs->es); + if (!(err & EDMA_V3_MP_ES_VLD)) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + chan = &fsl_edma->chans[ch]; + + ch_es = edma_readl_chreg(chan, ch_es); + if (!(ch_es & EDMA_V3_CH_ES_ERR)) + continue; + + edma_writel_chreg(chan, EDMA_V3_CH_ES_ERR, ch_es); + fsl_edma_disable_request(chan); + fsl_edma->chans[ch].status = DMA_ERROR; + } + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int err, ch; + struct edma_regs *regs = &fsl_edma->regs; + + err = edma_readl(fsl_edma, regs->errl); + if (!err) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (err & (0x1 << ch)) { + fsl_edma_disable_request(&fsl_edma->chans[ch]); + edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), regs->cerr); + fsl_edma_err_chan_handler(&fsl_edma->chans[ch]); + } + } + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) +{ + if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED) + return IRQ_HANDLED; + + return fsl_edma_err_handler(irq, dev_id); +} + +static bool fsl_edma_srcid_in_use(struct fsl_edma_engine *fsl_edma, u32 srcid) +{ + struct fsl_edma_chan *fsl_chan; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + + if (fsl_chan->srcid && srcid == fsl_chan->srcid) { + dev_err(&fsl_chan->pdev->dev, "The srcid is in use, can't use!\n"); + return true; + } + } + return false; +} + +static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; + struct dma_chan *chan, *_chan; + struct fsl_edma_chan *fsl_chan; + u32 dmamux_nr = fsl_edma->drvdata->dmamuxs; + unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr; + + if (dma_spec->args_count != 2) + return NULL; + + guard(mutex)(&fsl_edma->fsl_edma_mutex); + + list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { + if (chan->client_count) + continue; + + if (fsl_edma_srcid_in_use(fsl_edma, dma_spec->args[1])) + return NULL; + + if ((chan->chan_id / chans_per_mux) == dma_spec->args[0]) { + chan = dma_get_slave_channel(chan); + if (chan) { + chan->device->privatecnt++; + fsl_chan = to_fsl_edma_chan(chan); + fsl_chan->srcid = dma_spec->args[1]; + + if (!fsl_chan->srcid) { + dev_err(&fsl_chan->pdev->dev, "Invalidate srcid %d\n", + fsl_chan->srcid); + return NULL; + } + + fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid, + true); + return chan; + } + } + } + return NULL; +} + +static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; + struct dma_chan *chan, *_chan; + struct fsl_edma_chan *fsl_chan; + bool b_chmux; + int i; + + if (dma_spec->args_count != 3) + return NULL; + + b_chmux = !!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHMUX); + + guard(mutex)(&fsl_edma->fsl_edma_mutex); + list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, + device_node) { + + if (chan->client_count) + continue; + + fsl_chan = to_fsl_edma_chan(chan); + if (fsl_edma_srcid_in_use(fsl_edma, dma_spec->args[0])) + return NULL; + i = fsl_chan - fsl_edma->chans; + + fsl_chan->priority = dma_spec->args[1]; + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + + if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) + continue; + + if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) + continue; + + if (!b_chmux && i == dma_spec->args[0]) { + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + return chan; + } else if (b_chmux && !fsl_chan->srcid) { + /* if controller support channel mux, choose a free channel */ + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + fsl_chan->srcid = dma_spec->args[0]; + return chan; + } + } + return NULL; +} + +static int +fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + int ret; + + edma_writel(fsl_edma, ~0, fsl_edma->regs.intl); + + fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx"); + if (fsl_edma->txirq < 0) + return fsl_edma->txirq; + + fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err"); + if (fsl_edma->errirq < 0) + return fsl_edma->errirq; + + if (fsl_edma->txirq == fsl_edma->errirq) { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma_irq_handler, 0, "eDMA", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA IRQ.\n"); + return ret; + } + } else { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n"); + return ret; + } + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, + fsl_edma_err_handler, 0, "eDMA err", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n"); + return ret; + } + } + + return 0; +} + +static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + char *errirq_name; + int i, ret; + + for (i = 0; i < fsl_edma->n_chans; i++) { + + struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; + + if (fsl_edma->chan_masked & BIT(i)) + continue; + + /* request channel irq */ + fsl_chan->txirq = platform_get_irq(pdev, i); + if (fsl_chan->txirq < 0) + return -EINVAL; + + fsl_chan->irq_handler = fsl_edma3_tx_handler; + + if (!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_ERRIRQ_SHARE)) { + fsl_chan->errirq = fsl_chan->txirq; + fsl_chan->errirq_handler = fsl_edma3_err_handler_per_chan; + } + } + + /* All channel err use one irq number */ + if (fsl_edma->drvdata->flags & FSL_EDMA_DRV_ERRIRQ_SHARE) { + /* last one is error irq */ + fsl_edma->errirq = platform_get_irq_optional(pdev, fsl_edma->n_chans); + if (fsl_edma->errirq < 0) + return 0; /* dts miss err irq, treat as no err irq case */ + + errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err", + dev_name(&pdev->dev)); + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared, + 0, errirq_name, fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Can't register eDMA err IRQ.\n"); + } + + return 0; +} + +static int fsl_edma3_or_irq_init(struct platform_device *pdev, + struct fsl_edma_engine *fsl_edma) +{ + int ret; + + fsl_edma->txirq = platform_get_irq_byname(pdev, "tx-0-15"); + if (fsl_edma->txirq < 0) + return fsl_edma->txirq; + + fsl_edma->txirq_16_31 = platform_get_irq_byname(pdev, "tx-16-31"); + if (fsl_edma->txirq_16_31 < 0) + return fsl_edma->txirq_16_31; + + fsl_edma->errirq = platform_get_irq_byname(pdev, "err"); + if (fsl_edma->errirq < 0) + return fsl_edma->errirq; + + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma3_tx_0_15_handler, 0, "eDMA tx0_15", + fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA tx0_15 IRQ.\n"); + + if (fsl_edma->n_chans > 16) { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq_16_31, + fsl_edma3_tx_16_31_handler, 0, + "eDMA tx16_31", fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA tx16_31 IRQ.\n"); + } + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, + fsl_edma3_or_err_handler, 0, "eDMA err", + fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA err IRQ.\n"); + + return 0; +} + +static int +fsl_edma2_irq_init(struct platform_device *pdev, + struct fsl_edma_engine *fsl_edma) +{ + int i, ret, irq; + int count; + + edma_writel(fsl_edma, ~0, fsl_edma->regs.intl); + + count = platform_irq_count(pdev); + dev_dbg(&pdev->dev, "%s Found %d interrupts\r\n", __func__, count); + if (count <= 2) { + dev_err(&pdev->dev, "Interrupts in DTS not correct.\n"); + return -EINVAL; + } + /* + * 16 channel independent interrupts + 1 error interrupt on i.mx7ulp. + * 2 channel share one interrupt, for example, ch0/ch16, ch1/ch17... + * For now, just simply request irq without IRQF_SHARED flag, since 16 + * channels are enough on i.mx7ulp whose M4 domain own some peripherals. + */ + for (i = 0; i < count; i++) { + irq = platform_get_irq(pdev, i); + ret = 0; + if (irq < 0) + return -ENXIO; + + /* The last IRQ is for eDMA err */ + if (i == count - 1) { + fsl_edma->errirq = irq; + ret = devm_request_irq(&pdev->dev, irq, + fsl_edma_err_handler, + 0, "eDMA2-ERR", fsl_edma); + } else { + fsl_edma->chans[i].txirq = irq; + fsl_edma->chans[i].irq_handler = fsl_edma2_tx_handler; + } + + if (ret) + return ret; + } + + return 0; +} + +static void fsl_edma_irq_exit( + struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + if (fsl_edma->txirq == fsl_edma->errirq) { + if (fsl_edma->txirq >= 0) + devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); + } else { + if (fsl_edma->txirq >= 0) + devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); + if (fsl_edma->errirq >= 0) + devm_free_irq(&pdev->dev, fsl_edma->errirq, fsl_edma); + } +} + +static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks) +{ + int i; + + for (i = 0; i < nr_clocks; i++) + clk_disable_unprepare(fsl_edma->muxclk[i]); +} + +static struct fsl_edma_drvdata vf610_data = { + .dmamuxs = DMAMUX_NR, + .flags = FSL_EDMA_DRV_WRAP_IO, + .chreg_off = EDMA_TCD, + .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), + .setup_irq = fsl_edma_irq_init, +}; + +static struct fsl_edma_drvdata ls1028a_data = { + .dmamuxs = DMAMUX_NR, + .flags = FSL_EDMA_DRV_MUX_SWAP | FSL_EDMA_DRV_WRAP_IO, + .chreg_off = EDMA_TCD, + .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), + .setup_irq = fsl_edma_irq_init, +}; + +static struct fsl_edma_drvdata imx7ulp_data = { + .dmamuxs = 1, + .chreg_off = EDMA_TCD, + .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), + .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_CONFIG32, + .setup_irq = fsl_edma2_irq_init, +}; + +static struct fsl_edma_drvdata imx8qm_data = { + .flags = FSL_EDMA_DRV_HAS_PD | FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_MEM_REMOTE + | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x10000, + .chreg_off = 0x10000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx8ulp_data = { + .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_CHCLK | FSL_EDMA_DRV_HAS_DMACLK | + FSL_EDMA_DRV_EDMA3, + .chreg_space_sz = 0x10000, + .chreg_off = 0x10000, + .mux_off = 0x10000 + offsetof(struct fsl_edma3_ch_reg, ch_mux), + .mux_skip = 0x10000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx93_data3 = { + .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x10000, + .chreg_off = 0x10000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx93_data4 = { + .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA4 + | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x8000, + .chreg_off = 0x10000, + .mux_off = 0x10000 + offsetof(struct fsl_edma3_ch_reg, ch_mux), + .mux_skip = 0x8000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx95_data5 = { + .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA4 | + FSL_EDMA_DRV_TCD64 | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x8000, + .chreg_off = 0x10000, + .mux_off = 0x200, + .mux_skip = sizeof(u32), + .setup_irq = fsl_edma3_irq_init, +}; + +static const struct fsl_edma_drvdata s32g2_data = { + .dmamuxs = DMAMUX_NR, + .chreg_space_sz = EDMA_TCD, + .chreg_off = 0x4000, + .flags = FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_MUX_SWAP, + .setup_irq = fsl_edma3_or_irq_init, +}; + +static const struct of_device_id fsl_edma_dt_ids[] = { + { .compatible = "fsl,vf610-edma", .data = &vf610_data}, + { .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data}, + { .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data}, + { .compatible = "fsl,imx8qm-edma", .data = &imx8qm_data}, + { .compatible = "fsl,imx8ulp-edma", .data = &imx8ulp_data}, + { .compatible = "fsl,imx93-edma3", .data = &imx93_data3}, + { .compatible = "fsl,imx93-edma4", .data = &imx93_data4}, + { .compatible = "fsl,imx95-edma5", .data = &imx95_data5}, + { .compatible = "nxp,s32g2-edma", .data = &s32g2_data}, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); + +static void fsl_edma3_detach_pd(struct fsl_edma_engine *fsl_edma) +{ + struct fsl_edma_chan *fsl_chan; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + if (fsl_edma->chan_masked & BIT(i)) + continue; + fsl_chan = &fsl_edma->chans[i]; + if (fsl_chan->pd_dev_link) + device_link_del(fsl_chan->pd_dev_link); + if (fsl_chan->pd_dev) { + dev_pm_domain_detach(fsl_chan->pd_dev, false); + pm_runtime_dont_use_autosuspend(fsl_chan->pd_dev); + pm_runtime_set_suspended(fsl_chan->pd_dev); + } + } +} + +static void devm_fsl_edma3_detach_pd(void *data) +{ + fsl_edma3_detach_pd(data); +} + +static int fsl_edma3_attach_pd(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + struct fsl_edma_chan *fsl_chan; + struct device *pd_chan; + struct device *dev; + int i; + + dev = &pdev->dev; + + for (i = 0; i < fsl_edma->n_chans; i++) { + if (fsl_edma->chan_masked & BIT(i)) + continue; + + fsl_chan = &fsl_edma->chans[i]; + + pd_chan = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR_OR_NULL(pd_chan)) { + dev_err(dev, "Failed attach pd %d\n", i); + goto detach; + } + + fsl_chan->pd_dev_link = device_link_add(dev, pd_chan, DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!fsl_chan->pd_dev_link) { + dev_err(dev, "Failed to add device_link to %d\n", i); + dev_pm_domain_detach(pd_chan, false); + goto detach; + } + + fsl_chan->pd_dev = pd_chan; + + pm_runtime_use_autosuspend(fsl_chan->pd_dev); + pm_runtime_set_autosuspend_delay(fsl_chan->pd_dev, 200); + pm_runtime_set_active(fsl_chan->pd_dev); + } + + return 0; + +detach: + fsl_edma3_detach_pd(fsl_edma); + return -EINVAL; +} + +static int fsl_edma_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma_engine *fsl_edma; + const struct fsl_edma_drvdata *drvdata = NULL; + u32 chan_mask[2] = {0, 0}; + char clk_name[36]; + struct edma_regs *regs; + int chans; + int ret, i; + + drvdata = device_get_match_data(&pdev->dev); + if (!drvdata) { + dev_err(&pdev->dev, "unable to find driver data\n"); + return -EINVAL; + } + + ret = of_property_read_u32(np, "dma-channels", &chans); + if (ret) { + dev_err(&pdev->dev, "Can't get dma-channels.\n"); + return ret; + } + + fsl_edma = devm_kzalloc(&pdev->dev, struct_size(fsl_edma, chans, chans), + GFP_KERNEL); + if (!fsl_edma) + return -ENOMEM; + + fsl_edma->errirq = -EINVAL; + fsl_edma->txirq = -EINVAL; + fsl_edma->drvdata = drvdata; + fsl_edma->n_chans = chans; + mutex_init(&fsl_edma->fsl_edma_mutex); + + fsl_edma->membase = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(fsl_edma->membase)) + return PTR_ERR(fsl_edma->membase); + + if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) { + fsl_edma_setup_regs(fsl_edma); + regs = &fsl_edma->regs; + } + + if (drvdata->flags & FSL_EDMA_DRV_HAS_DMACLK) { + fsl_edma->dmaclk = devm_clk_get_enabled(&pdev->dev, "dma"); + if (IS_ERR(fsl_edma->dmaclk)) { + dev_err(&pdev->dev, "Missing DMA block clock.\n"); + return PTR_ERR(fsl_edma->dmaclk); + } + } + + ret = of_property_read_variable_u32_array(np, "dma-channel-mask", chan_mask, 1, 2); + + if (ret > 0) { + fsl_edma->chan_masked = chan_mask[1]; + fsl_edma->chan_masked <<= 32; + fsl_edma->chan_masked |= chan_mask[0]; + } + + for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) { + char clkname[32]; + + fsl_edma->muxbase[i] = devm_platform_ioremap_resource(pdev, + 1 + i); + if (IS_ERR(fsl_edma->muxbase[i])) { + /* on error: disable all previously enabled clks */ + fsl_disable_clocks(fsl_edma, i); + return PTR_ERR(fsl_edma->muxbase[i]); + } + + sprintf(clkname, "dmamux%d", i); + fsl_edma->muxclk[i] = devm_clk_get_enabled(&pdev->dev, clkname); + if (IS_ERR(fsl_edma->muxclk[i])) { + dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); + /* on error: disable all previously enabled clks */ + return PTR_ERR(fsl_edma->muxclk[i]); + } + } + + fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); + + if (drvdata->flags & FSL_EDMA_DRV_HAS_PD) { + ret = fsl_edma3_attach_pd(pdev, fsl_edma); + if (ret) + return ret; + ret = devm_add_action_or_reset(&pdev->dev, devm_fsl_edma3_detach_pd, fsl_edma); + if (ret) + return ret; + } + + if (drvdata->flags & FSL_EDMA_DRV_TCD64) + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + + INIT_LIST_HEAD(&fsl_edma->dma_dev.channels); + for (i = 0; i < fsl_edma->n_chans; i++) { + struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; + int len; + + if (fsl_edma->chan_masked & BIT(i)) + continue; + + snprintf(fsl_chan->chan_name, sizeof(fsl_chan->chan_name), "%s-CH%02d", + dev_name(&pdev->dev), i); + + snprintf(fsl_chan->errirq_name, sizeof(fsl_chan->errirq_name), + "%s-CH%02d-err", dev_name(&pdev->dev), i); + + fsl_chan->edma = fsl_edma; + fsl_chan->pm_state = RUNNING; + fsl_chan->srcid = 0; + fsl_chan->dma_dir = DMA_NONE; + fsl_chan->vchan.desc_free = fsl_edma_free_desc; + + len = (drvdata->flags & FSL_EDMA_DRV_SPLIT_REG) ? + offsetof(struct fsl_edma3_ch_reg, tcd) : 0; + fsl_chan->tcd = fsl_edma->membase + + i * drvdata->chreg_space_sz + drvdata->chreg_off + len; + fsl_chan->mux_addr = fsl_edma->membase + drvdata->mux_off + i * drvdata->mux_skip; + + if (drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) { + snprintf(clk_name, sizeof(clk_name), "ch%02d", i); + fsl_chan->clk = devm_clk_get_enabled(&pdev->dev, + (const char *)clk_name); + + if (IS_ERR(fsl_chan->clk)) + return PTR_ERR(fsl_chan->clk); + } + fsl_chan->pdev = pdev; + vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); + + edma_write_tcdreg(fsl_chan, cpu_to_le32(0), csr); + fsl_edma_chan_mux(fsl_chan, 0, false); + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) + clk_disable_unprepare(fsl_chan->clk); + } + + ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma); + if (ret) + return ret; + + dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_MEMCPY, fsl_edma->dma_dev.cap_mask); + + fsl_edma->dma_dev.dev = &pdev->dev; + fsl_edma->dma_dev.device_alloc_chan_resources + = fsl_edma_alloc_chan_resources; + fsl_edma->dma_dev.device_free_chan_resources + = fsl_edma_free_chan_resources; + fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; + fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; + fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; + fsl_edma->dma_dev.device_prep_dma_memcpy = fsl_edma_prep_memcpy; + fsl_edma->dma_dev.device_config = fsl_edma_slave_config; + fsl_edma->dma_dev.device_pause = fsl_edma_pause; + fsl_edma->dma_dev.device_resume = fsl_edma_resume; + fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all; + fsl_edma->dma_dev.device_synchronize = fsl_edma_synchronize; + fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending; + + fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; + fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; + + if (drvdata->flags & FSL_EDMA_DRV_BUS_8BYTE) { + fsl_edma->dma_dev.src_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + fsl_edma->dma_dev.dst_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + } + + fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + if (drvdata->flags & FSL_EDMA_DRV_DEV_TO_DEV) + fsl_edma->dma_dev.directions |= BIT(DMA_DEV_TO_DEV); + + fsl_edma->dma_dev.copy_align = drvdata->flags & FSL_EDMA_DRV_ALIGN_64BYTE ? + DMAENGINE_ALIGN_64_BYTES : + DMAENGINE_ALIGN_32_BYTES; + + /* Per worst case 'nbytes = 1' take CITER as the max_seg_size */ + dma_set_max_seg_size(fsl_edma->dma_dev.dev, + FIELD_GET(EDMA_TCD_ITER_MASK, EDMA_TCD_ITER_MASK)); + + fsl_edma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + + platform_set_drvdata(pdev, fsl_edma); + + ret = dma_async_device_register(&fsl_edma->dma_dev); + if (ret) { + dev_err(&pdev->dev, + "Can't register Freescale eDMA engine. (%d)\n", ret); + return ret; + } + + ret = of_dma_controller_register(np, + drvdata->dmamuxs ? fsl_edma_xlate : fsl_edma3_xlate, + fsl_edma); + if (ret) { + dev_err(&pdev->dev, + "Can't register Freescale eDMA of_dma. (%d)\n", ret); + dma_async_device_unregister(&fsl_edma->dma_dev); + return ret; + } + + /* enable round robin arbitration */ + if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) + edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); + + return 0; +} + +static void fsl_edma_remove(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); + + fsl_edma_irq_exit(pdev, fsl_edma); + of_dma_controller_free(np); + dma_async_device_unregister(&fsl_edma->dma_dev); + fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); + fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); +} + +static int fsl_edma_suspend_late(struct device *dev) +{ + struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma_chan *fsl_chan; + unsigned long flags; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + if (fsl_edma->chan_masked & BIT(i)) + continue; + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + /* Make sure chan is idle or will force disable. */ + if (unlikely(fsl_chan->status == DMA_IN_PROGRESS)) { + dev_warn(dev, "WARN: There is non-idle channel.\n"); + fsl_edma_disable_request(fsl_chan); + fsl_edma_chan_mux(fsl_chan, 0, false); + } + + fsl_chan->pm_state = SUSPENDED; + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + } + + return 0; +} + +static int fsl_edma_resume_early(struct device *dev) +{ + struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma_chan *fsl_chan; + struct edma_regs *regs = &fsl_edma->regs; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + if (fsl_edma->chan_masked & BIT(i)) + continue; + fsl_chan->pm_state = RUNNING; + edma_write_tcdreg(fsl_chan, 0, csr); + if (fsl_chan->srcid != 0) + fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid, true); + } + + if (!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) + edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); + + return 0; +} + +/* + * eDMA provides the service to others, so it should be suspend late + * and resume early. When eDMA suspend, all of the clients should stop + * the DMA data transmission and let the channel idle. + */ +static const struct dev_pm_ops fsl_edma_pm_ops = { + .suspend_late = fsl_edma_suspend_late, + .resume_early = fsl_edma_resume_early, +}; + +static struct platform_driver fsl_edma_driver = { + .driver = { + .name = "fsl-edma", + .of_match_table = fsl_edma_dt_ids, + .pm = &fsl_edma_pm_ops, + }, + .probe = fsl_edma_probe, + .remove = fsl_edma_remove, +}; + +static int __init fsl_edma_init(void) +{ + return platform_driver_register(&fsl_edma_driver); +} +subsys_initcall(fsl_edma_init); + +static void __exit fsl_edma_exit(void) +{ + platform_driver_unregister(&fsl_edma_driver); +} +module_exit(fsl_edma_exit); + +MODULE_DESCRIPTION("Freescale eDMA engine driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-edma-trace.c b/drivers/dma/fsl-edma-trace.c new file mode 100644 index 000000000000..28300ad80bb7 --- /dev/null +++ b/drivers/dma/fsl-edma-trace.c @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define CREATE_TRACE_POINTS +#include "fsl-edma-common.h" diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h new file mode 100644 index 000000000000..d3541301a247 --- /dev/null +++ b/drivers/dma/fsl-edma-trace.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright 2023 NXP. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM fsl_edma + +#if !defined(__LINUX_FSL_EDMA_TRACE) || defined(TRACE_HEADER_MULTI_READ) +#define __LINUX_FSL_EDMA_TRACE + +#include <linux/types.h> +#include <linux/tracepoint.h> + +DECLARE_EVENT_CLASS(edma_log_io, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value), + TP_STRUCT__entry( + __field(struct fsl_edma_engine *, edma) + __field(void __iomem *, addr) + __field(u32, value) + ), + TP_fast_assign( + __entry->edma = edma; + __entry->addr = addr; + __entry->value = value; + ), + TP_printk("offset %08x: value %08x", + (u32)(__entry->addr - __entry->edma->membase), __entry->value) +); + +DEFINE_EVENT(edma_log_io, edma_readl, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writel, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_readw, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writew, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_readb, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writeb, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DECLARE_EVENT_CLASS(edma_log_tcd, + TP_PROTO(struct fsl_edma_chan *chan, void *tcd), + TP_ARGS(chan, tcd), + TP_STRUCT__entry( + __field(u64, saddr) + __field(u16, soff) + __field(u16, attr) + __field(u32, nbytes) + __field(u64, slast) + __field(u64, daddr) + __field(u16, doff) + __field(u16, citer) + __field(u64, dlast_sga) + __field(u16, csr) + __field(u16, biter) + + ), + TP_fast_assign( + __entry->saddr = fsl_edma_get_tcd_to_cpu(chan, tcd, saddr), + __entry->soff = fsl_edma_get_tcd_to_cpu(chan, tcd, soff), + __entry->attr = fsl_edma_get_tcd_to_cpu(chan, tcd, attr), + __entry->nbytes = fsl_edma_get_tcd_to_cpu(chan, tcd, nbytes), + __entry->slast = fsl_edma_get_tcd_to_cpu(chan, tcd, slast), + __entry->daddr = fsl_edma_get_tcd_to_cpu(chan, tcd, daddr), + __entry->doff = fsl_edma_get_tcd_to_cpu(chan, tcd, doff), + __entry->citer = fsl_edma_get_tcd_to_cpu(chan, tcd, citer), + __entry->dlast_sga = fsl_edma_get_tcd_to_cpu(chan, tcd, dlast_sga), + __entry->csr = fsl_edma_get_tcd_to_cpu(chan, tcd, csr), + __entry->biter = fsl_edma_get_tcd_to_cpu(chan, tcd, biter); + ), + TP_printk("\n==== TCD =====\n" + " saddr: 0x%016llx\n" + " soff: 0x%04x\n" + " attr: 0x%04x\n" + " nbytes: 0x%08x\n" + " slast: 0x%016llx\n" + " daddr: 0x%016llx\n" + " doff: 0x%04x\n" + " citer: 0x%04x\n" + " dlast: 0x%016llx\n" + " csr: 0x%04x\n" + " biter: 0x%04x\n", + __entry->saddr, + __entry->soff, + __entry->attr, + __entry->nbytes, + __entry->slast, + __entry->daddr, + __entry->doff, + __entry->citer, + __entry->dlast_sga, + __entry->csr, + __entry->biter) +); + +DEFINE_EVENT(edma_log_tcd, edma_fill_tcd, + TP_PROTO(struct fsl_edma_chan *chan, void *tcd), + TP_ARGS(chan, tcd) +); + +#endif + +/* this part must be outside header guard */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE fsl-edma-trace + +#include <trace/define_trace.h> diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c deleted file mode 100644 index e40769666e39..000000000000 --- a/drivers/dma/fsl-edma.c +++ /dev/null @@ -1,517 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * drivers/dma/fsl-edma.c - * - * Copyright 2013-2014 Freescale Semiconductor, Inc. - * - * Driver for the Freescale eDMA engine with flexible channel multiplexing - * capability for DMA request sources. The eDMA block can be found on some - * Vybrid and Layerscape SoCs. - */ - -#include <linux/module.h> -#include <linux/interrupt.h> -#include <linux/clk.h> -#include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_dma.h> -#include <linux/dma-mapping.h> - -#include "fsl-edma-common.h" - -static void fsl_edma_synchronize(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - - vchan_synchronize(&fsl_chan->vchan); -} - -static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) -{ - struct fsl_edma_engine *fsl_edma = dev_id; - unsigned int intr, ch; - struct edma_regs *regs = &fsl_edma->regs; - struct fsl_edma_chan *fsl_chan; - - intr = edma_readl(fsl_edma, regs->intl); - if (!intr) - return IRQ_NONE; - - for (ch = 0; ch < fsl_edma->n_chans; ch++) { - if (intr & (0x1 << ch)) { - edma_writeb(fsl_edma, EDMA_CINT_CINT(ch), regs->cint); - - fsl_chan = &fsl_edma->chans[ch]; - - spin_lock(&fsl_chan->vchan.lock); - - if (!fsl_chan->edesc) { - /* terminate_all called before */ - spin_unlock(&fsl_chan->vchan.lock); - continue; - } - - if (!fsl_chan->edesc->iscyclic) { - list_del(&fsl_chan->edesc->vdesc.node); - vchan_cookie_complete(&fsl_chan->edesc->vdesc); - fsl_chan->edesc = NULL; - fsl_chan->status = DMA_COMPLETE; - fsl_chan->idle = true; - } else { - vchan_cyclic_callback(&fsl_chan->edesc->vdesc); - } - - if (!fsl_chan->edesc) - fsl_edma_xfer_desc(fsl_chan); - - spin_unlock(&fsl_chan->vchan.lock); - } - } - return IRQ_HANDLED; -} - -static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) -{ - struct fsl_edma_engine *fsl_edma = dev_id; - unsigned int err, ch; - struct edma_regs *regs = &fsl_edma->regs; - - err = edma_readl(fsl_edma, regs->errl); - if (!err) - return IRQ_NONE; - - for (ch = 0; ch < fsl_edma->n_chans; ch++) { - if (err & (0x1 << ch)) { - fsl_edma_disable_request(&fsl_edma->chans[ch]); - edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), regs->cerr); - fsl_edma->chans[ch].status = DMA_ERROR; - fsl_edma->chans[ch].idle = true; - } - } - return IRQ_HANDLED; -} - -static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) -{ - if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED) - return IRQ_HANDLED; - - return fsl_edma_err_handler(irq, dev_id); -} - -static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; - struct dma_chan *chan, *_chan; - struct fsl_edma_chan *fsl_chan; - u32 dmamux_nr = fsl_edma->drvdata->dmamuxs; - unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr; - - if (dma_spec->args_count != 2) - return NULL; - - mutex_lock(&fsl_edma->fsl_edma_mutex); - list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { - if (chan->client_count) - continue; - if ((chan->chan_id / chans_per_mux) == dma_spec->args[0]) { - chan = dma_get_slave_channel(chan); - if (chan) { - chan->device->privatecnt++; - fsl_chan = to_fsl_edma_chan(chan); - fsl_chan->slave_id = dma_spec->args[1]; - fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, - true); - mutex_unlock(&fsl_edma->fsl_edma_mutex); - return chan; - } - } - } - mutex_unlock(&fsl_edma->fsl_edma_mutex); - return NULL; -} - -static int -fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) -{ - int ret; - - fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx"); - if (fsl_edma->txirq < 0) - return fsl_edma->txirq; - - fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err"); - if (fsl_edma->errirq < 0) - return fsl_edma->errirq; - - if (fsl_edma->txirq == fsl_edma->errirq) { - ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, - fsl_edma_irq_handler, 0, "eDMA", fsl_edma); - if (ret) { - dev_err(&pdev->dev, "Can't register eDMA IRQ.\n"); - return ret; - } - } else { - ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, - fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma); - if (ret) { - dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n"); - return ret; - } - - ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, - fsl_edma_err_handler, 0, "eDMA err", fsl_edma); - if (ret) { - dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n"); - return ret; - } - } - - return 0; -} - -static int -fsl_edma2_irq_init(struct platform_device *pdev, - struct fsl_edma_engine *fsl_edma) -{ - int i, ret, irq; - int count; - - count = platform_irq_count(pdev); - dev_dbg(&pdev->dev, "%s Found %d interrupts\r\n", __func__, count); - if (count <= 2) { - dev_err(&pdev->dev, "Interrupts in DTS not correct.\n"); - return -EINVAL; - } - /* - * 16 channel independent interrupts + 1 error interrupt on i.mx7ulp. - * 2 channel share one interrupt, for example, ch0/ch16, ch1/ch17... - * For now, just simply request irq without IRQF_SHARED flag, since 16 - * channels are enough on i.mx7ulp whose M4 domain own some peripherals. - */ - for (i = 0; i < count; i++) { - irq = platform_get_irq(pdev, i); - if (irq < 0) - return -ENXIO; - - sprintf(fsl_edma->chans[i].chan_name, "eDMA2-CH%02d", i); - - /* The last IRQ is for eDMA err */ - if (i == count - 1) - ret = devm_request_irq(&pdev->dev, irq, - fsl_edma_err_handler, - 0, "eDMA2-ERR", fsl_edma); - else - ret = devm_request_irq(&pdev->dev, irq, - fsl_edma_tx_handler, 0, - fsl_edma->chans[i].chan_name, - fsl_edma); - if (ret) - return ret; - } - - return 0; -} - -static void fsl_edma_irq_exit( - struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) -{ - if (fsl_edma->txirq == fsl_edma->errirq) { - devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); - } else { - devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); - devm_free_irq(&pdev->dev, fsl_edma->errirq, fsl_edma); - } -} - -static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks) -{ - int i; - - for (i = 0; i < nr_clocks; i++) - clk_disable_unprepare(fsl_edma->muxclk[i]); -} - -static struct fsl_edma_drvdata vf610_data = { - .version = v1, - .dmamuxs = DMAMUX_NR, - .setup_irq = fsl_edma_irq_init, -}; - -static struct fsl_edma_drvdata ls1028a_data = { - .version = v1, - .dmamuxs = DMAMUX_NR, - .mux_swap = true, - .setup_irq = fsl_edma_irq_init, -}; - -static struct fsl_edma_drvdata imx7ulp_data = { - .version = v3, - .dmamuxs = 1, - .has_dmaclk = true, - .setup_irq = fsl_edma2_irq_init, -}; - -static const struct of_device_id fsl_edma_dt_ids[] = { - { .compatible = "fsl,vf610-edma", .data = &vf610_data}, - { .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data}, - { .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data}, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); - -static int fsl_edma_probe(struct platform_device *pdev) -{ - const struct of_device_id *of_id = - of_match_device(fsl_edma_dt_ids, &pdev->dev); - struct device_node *np = pdev->dev.of_node; - struct fsl_edma_engine *fsl_edma; - const struct fsl_edma_drvdata *drvdata = NULL; - struct fsl_edma_chan *fsl_chan; - struct edma_regs *regs; - int len, chans; - int ret, i; - - if (of_id) - drvdata = of_id->data; - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } - - ret = of_property_read_u32(np, "dma-channels", &chans); - if (ret) { - dev_err(&pdev->dev, "Can't get dma-channels.\n"); - return ret; - } - - len = sizeof(*fsl_edma) + sizeof(*fsl_chan) * chans; - fsl_edma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); - if (!fsl_edma) - return -ENOMEM; - - fsl_edma->drvdata = drvdata; - fsl_edma->n_chans = chans; - mutex_init(&fsl_edma->fsl_edma_mutex); - - fsl_edma->membase = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(fsl_edma->membase)) - return PTR_ERR(fsl_edma->membase); - - fsl_edma_setup_regs(fsl_edma); - regs = &fsl_edma->regs; - - if (drvdata->has_dmaclk) { - fsl_edma->dmaclk = devm_clk_get(&pdev->dev, "dma"); - if (IS_ERR(fsl_edma->dmaclk)) { - dev_err(&pdev->dev, "Missing DMA block clock.\n"); - return PTR_ERR(fsl_edma->dmaclk); - } - - ret = clk_prepare_enable(fsl_edma->dmaclk); - if (ret) { - dev_err(&pdev->dev, "DMA clk block failed.\n"); - return ret; - } - } - - for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) { - char clkname[32]; - - fsl_edma->muxbase[i] = devm_platform_ioremap_resource(pdev, - 1 + i); - if (IS_ERR(fsl_edma->muxbase[i])) { - /* on error: disable all previously enabled clks */ - fsl_disable_clocks(fsl_edma, i); - return PTR_ERR(fsl_edma->muxbase[i]); - } - - sprintf(clkname, "dmamux%d", i); - fsl_edma->muxclk[i] = devm_clk_get(&pdev->dev, clkname); - if (IS_ERR(fsl_edma->muxclk[i])) { - dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); - /* on error: disable all previously enabled clks */ - fsl_disable_clocks(fsl_edma, i); - return PTR_ERR(fsl_edma->muxclk[i]); - } - - ret = clk_prepare_enable(fsl_edma->muxclk[i]); - if (ret) - /* on error: disable all previously enabled clks */ - fsl_disable_clocks(fsl_edma, i); - - } - - fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); - - INIT_LIST_HEAD(&fsl_edma->dma_dev.channels); - for (i = 0; i < fsl_edma->n_chans; i++) { - struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; - - fsl_chan->edma = fsl_edma; - fsl_chan->pm_state = RUNNING; - fsl_chan->slave_id = 0; - fsl_chan->idle = true; - fsl_chan->dma_dir = DMA_NONE; - fsl_chan->vchan.desc_free = fsl_edma_free_desc; - vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); - - edma_writew(fsl_edma, 0x0, ®s->tcd[i].csr); - fsl_edma_chan_mux(fsl_chan, 0, false); - } - - edma_writel(fsl_edma, ~0, regs->intl); - ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma); - if (ret) - return ret; - - dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask); - dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask); - dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask); - dma_cap_set(DMA_MEMCPY, fsl_edma->dma_dev.cap_mask); - - fsl_edma->dma_dev.dev = &pdev->dev; - fsl_edma->dma_dev.device_alloc_chan_resources - = fsl_edma_alloc_chan_resources; - fsl_edma->dma_dev.device_free_chan_resources - = fsl_edma_free_chan_resources; - fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; - fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; - fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; - fsl_edma->dma_dev.device_prep_dma_memcpy = fsl_edma_prep_memcpy; - fsl_edma->dma_dev.device_config = fsl_edma_slave_config; - fsl_edma->dma_dev.device_pause = fsl_edma_pause; - fsl_edma->dma_dev.device_resume = fsl_edma_resume; - fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all; - fsl_edma->dma_dev.device_synchronize = fsl_edma_synchronize; - fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending; - - fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; - fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; - fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - - fsl_edma->dma_dev.copy_align = DMAENGINE_ALIGN_32_BYTES; - /* Per worst case 'nbytes = 1' take CITER as the max_seg_size */ - dma_set_max_seg_size(fsl_edma->dma_dev.dev, 0x3fff); - - platform_set_drvdata(pdev, fsl_edma); - - ret = dma_async_device_register(&fsl_edma->dma_dev); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA engine. (%d)\n", ret); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); - return ret; - } - - ret = of_dma_controller_register(np, fsl_edma_xlate, fsl_edma); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA of_dma. (%d)\n", ret); - dma_async_device_unregister(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); - return ret; - } - - /* enable round robin arbitration */ - edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); - - return 0; -} - -static int fsl_edma_remove(struct platform_device *pdev) -{ - struct device_node *np = pdev->dev.of_node; - struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); - - fsl_edma_irq_exit(pdev, fsl_edma); - fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); - of_dma_controller_free(np); - dma_async_device_unregister(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); - - return 0; -} - -static int fsl_edma_suspend_late(struct device *dev) -{ - struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); - struct fsl_edma_chan *fsl_chan; - unsigned long flags; - int i; - - for (i = 0; i < fsl_edma->n_chans; i++) { - fsl_chan = &fsl_edma->chans[i]; - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - /* Make sure chan is idle or will force disable. */ - if (unlikely(!fsl_chan->idle)) { - dev_warn(dev, "WARN: There is non-idle channel."); - fsl_edma_disable_request(fsl_chan); - fsl_edma_chan_mux(fsl_chan, 0, false); - } - - fsl_chan->pm_state = SUSPENDED; - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - } - - return 0; -} - -static int fsl_edma_resume_early(struct device *dev) -{ - struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); - struct fsl_edma_chan *fsl_chan; - struct edma_regs *regs = &fsl_edma->regs; - int i; - - for (i = 0; i < fsl_edma->n_chans; i++) { - fsl_chan = &fsl_edma->chans[i]; - fsl_chan->pm_state = RUNNING; - edma_writew(fsl_edma, 0x0, ®s->tcd[i].csr); - if (fsl_chan->slave_id != 0) - fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, true); - } - - edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); - - return 0; -} - -/* - * eDMA provides the service to others, so it should be suspend late - * and resume early. When eDMA suspend, all of the clients should stop - * the DMA data transmission and let the channel idle. - */ -static const struct dev_pm_ops fsl_edma_pm_ops = { - .suspend_late = fsl_edma_suspend_late, - .resume_early = fsl_edma_resume_early, -}; - -static struct platform_driver fsl_edma_driver = { - .driver = { - .name = "fsl-edma", - .of_match_table = fsl_edma_dt_ids, - .pm = &fsl_edma_pm_ops, - }, - .probe = fsl_edma_probe, - .remove = fsl_edma_remove, -}; - -static int __init fsl_edma_init(void) -{ - return platform_driver_register(&fsl_edma_driver); -} -subsys_initcall(fsl_edma_init); - -static void __exit fsl_edma_exit(void) -{ - platform_driver_unregister(&fsl_edma_driver); -} -module_exit(fsl_edma_exit); - -MODULE_ALIAS("platform:fsl-edma"); -MODULE_DESCRIPTION("Freescale eDMA engine driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c index eddb2688f234..6ace5bf80c40 100644 --- a/drivers/dma/fsl-qdma.c +++ b/drivers/dma/fsl-qdma.c @@ -13,10 +13,10 @@ #include <linux/module.h> #include <linux/delay.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <linux/dma-mapping.h> +#include <linux/platform_device.h> #include "virt-dma.h" #include "fsldma.h" @@ -109,6 +109,7 @@ #define FSL_QDMA_CMD_WTHROTL_OFFSET 20 #define FSL_QDMA_CMD_DSEN_OFFSET 19 #define FSL_QDMA_CMD_LWC_OFFSET 16 +#define FSL_QDMA_CMD_PF BIT(17) /* Field definition for Descriptor status */ #define QDMA_CCDF_STATUS_RTE BIT(5) @@ -147,6 +148,9 @@ * @__reserved1: Reserved field. * @cfg8b_w1: Compound descriptor command queue origin produced * by qDMA and dynamic debug field. + * @__reserved2: Reserved field. + * @cmd: Command for QDMA (see FSL_QDMA_CMD_RWTTYPE and + * others). * @data: Pointer to the memory 40-bit address, describes DMA * source information and DMA destination information. */ @@ -160,6 +164,10 @@ struct fsl_qdma_format { u8 __reserved1[2]; u8 cfg8b_w1; } __packed; + struct { + __le32 __reserved2; + __le32 cmd; + } __packed; __le64 data; }; } __packed; @@ -354,7 +362,6 @@ static void fsl_qdma_free_chan_resources(struct dma_chan *chan) static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, dma_addr_t dst, dma_addr_t src, u32 len) { - u32 cmd; struct fsl_qdma_format *sdf, *ddf; struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest; @@ -383,14 +390,11 @@ static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, /* This entry is the last entry. */ qdma_csgf_set_f(csgf_dest, len); /* Descriptor Buffer */ - cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE << - FSL_QDMA_CMD_RWTTYPE_OFFSET); - sdf->data = QDMA_SDDF_CMD(cmd); - - cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE << - FSL_QDMA_CMD_RWTTYPE_OFFSET); - cmd |= cpu_to_le32(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET); - ddf->data = QDMA_SDDF_CMD(cmd); + sdf->cmd = cpu_to_le32((FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET) | + FSL_QDMA_CMD_PF); + + ddf->cmd = cpu_to_le32((FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET) | + (FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET)); } /* @@ -514,11 +518,11 @@ static struct fsl_qdma_queue queue_temp = queue_head + i + (j * queue_num); queue_temp->cq = - dma_alloc_coherent(&pdev->dev, - sizeof(struct fsl_qdma_format) * - queue_size[i], - &queue_temp->bus_addr, - GFP_KERNEL); + dmam_alloc_coherent(&pdev->dev, + sizeof(struct fsl_qdma_format) * + queue_size[i], + &queue_temp->bus_addr, + GFP_KERNEL); if (!queue_temp->cq) return NULL; queue_temp->block_base = fsl_qdma->block_base + @@ -563,15 +567,14 @@ static struct fsl_qdma_queue /* * Buffer for queue command */ - status_head->cq = dma_alloc_coherent(&pdev->dev, - sizeof(struct fsl_qdma_format) * - status_size, - &status_head->bus_addr, - GFP_KERNEL); - if (!status_head->cq) { - devm_kfree(&pdev->dev, status_head); + status_head->cq = dmam_alloc_coherent(&pdev->dev, + sizeof(struct fsl_qdma_format) * + status_size, + &status_head->bus_addr, + GFP_KERNEL); + if (!status_head->cq) return NULL; - } + status_head->n_cq = status_size; status_head->virt_head = status_head->cq; status_head->virt_tail = status_head->cq; @@ -625,7 +628,7 @@ static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma) static int fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma, - void *block, + __iomem void *block, int id) { bool duplicate; @@ -805,7 +808,7 @@ fsl_qdma_irq_init(struct platform_device *pdev, int i; int cpu; int ret; - char irq_name[20]; + char irq_name[32]; fsl_qdma->error_irq = platform_get_irq_byname(pdev, "qdma-error"); @@ -1197,10 +1200,6 @@ static int fsl_qdma_probe(struct platform_device *pdev) if (!fsl_qdma->queue) return -ENOMEM; - ret = fsl_qdma_irq_init(pdev, fsl_qdma); - if (ret) - return ret; - fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0"); if (fsl_qdma->irq_base < 0) return fsl_qdma->irq_base; @@ -1239,16 +1238,19 @@ static int fsl_qdma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, fsl_qdma); - ret = dma_async_device_register(&fsl_qdma->dma_dev); + ret = fsl_qdma_reg_init(fsl_qdma); if (ret) { - dev_err(&pdev->dev, - "Can't register NXP Layerscape qDMA engine.\n"); + dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); return ret; } - ret = fsl_qdma_reg_init(fsl_qdma); + ret = fsl_qdma_irq_init(pdev, fsl_qdma); + if (ret) + return ret; + + ret = dma_async_device_register(&fsl_qdma->dma_dev); if (ret) { - dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); + dev_err(&pdev->dev, "Can't register NXP Layerscape qDMA engine.\n"); return ret; } @@ -1266,10 +1268,8 @@ static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev) } } -static int fsl_qdma_remove(struct platform_device *pdev) +static void fsl_qdma_remove(struct platform_device *pdev) { - int i; - struct fsl_qdma_queue *status; struct device_node *np = pdev->dev.of_node; struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev); @@ -1277,13 +1277,6 @@ static int fsl_qdma_remove(struct platform_device *pdev) fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev); of_dma_controller_free(np); dma_async_device_unregister(&fsl_qdma->dma_dev); - - for (i = 0; i < fsl_qdma->block_number; i++) { - status = fsl_qdma->status[i]; - dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_format) * - status->n_cq, status->cq, status->bus_addr); - } - return 0; } static const struct of_device_id fsl_qdma_dt_ids[] = { @@ -1303,6 +1296,5 @@ static struct platform_driver fsl_qdma_driver = { module_platform_driver(fsl_qdma_driver); -MODULE_ALIAS("platform:fsl-qdma"); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver"); diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index fdf3500d96a9..6aa97e258a55 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -60,9 +60,10 @@ */ #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/of_irq.h> -#include <linux/of_address.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> #include <linux/dmaengine.h> @@ -856,7 +857,7 @@ static void fsl_re_remove_chan(struct fsl_re_chan *chan) chan->oub_phys_addr); } -static int fsl_re_remove(struct platform_device *ofdev) +static void fsl_re_remove(struct platform_device *ofdev) { struct fsl_re_drv_private *re_priv; struct device *dev; @@ -871,8 +872,6 @@ static int fsl_re_remove(struct platform_device *ofdev) /* Unregister the driver */ dma_async_device_unregister(&re_priv->dma_dev); - - return 0; } static const struct of_device_id fsl_re_ids[] = { diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index f8459cc5315d..9b126a260267 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -28,9 +28,10 @@ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> +#include <linux/of.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/fsldma.h> #include "dmaengine.h" #include "fsldma.h" @@ -1225,6 +1226,8 @@ static int fsldma_of_probe(struct platform_device *op) fdev->dev = &op->dev; INIT_LIST_HEAD(&fdev->common.channels); + /* The DMA address bits supported for this device. */ + fdev->addr_bits = (long)device_get_match_data(fdev->dev); /* ioremap the registers for use */ fdev->regs = of_iomap(op->dev.of_node, 0); @@ -1253,7 +1256,7 @@ static int fsldma_of_probe(struct platform_device *op) fdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); fdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - dma_set_mask(&(op->dev), DMA_BIT_MASK(36)); + dma_set_mask(&(op->dev), DMA_BIT_MASK(fdev->addr_bits)); platform_set_drvdata(op, fdev); @@ -1305,7 +1308,7 @@ out_return: return err; } -static int fsldma_of_remove(struct platform_device *op) +static void fsldma_of_remove(struct platform_device *op) { struct fsldma_device *fdev; unsigned int i; @@ -1323,8 +1326,6 @@ static int fsldma_of_remove(struct platform_device *op) iounmap(fdev->regs); kfree(fdev); - - return 0; } #ifdef CONFIG_PM @@ -1388,10 +1389,20 @@ static const struct dev_pm_ops fsldma_pm_ops = { }; #endif +/* The .data field is used for dma-bit-mask. */ static const struct of_device_id fsldma_of_ids[] = { - { .compatible = "fsl,elo3-dma", }, - { .compatible = "fsl,eloplus-dma", }, - { .compatible = "fsl,elo-dma", }, + { + .compatible = "fsl,elo3-dma", + .data = (void *)40, + }, + { + .compatible = "fsl,eloplus-dma", + .data = (void *)36, + }, + { + .compatible = "fsl,elo-dma", + .data = (void *)32, + }, {} }; MODULE_DEVICE_TABLE(of, fsldma_of_ids); diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h index 308bed0a560a..d7b7a3138b85 100644 --- a/drivers/dma/fsldma.h +++ b/drivers/dma/fsldma.h @@ -124,6 +124,7 @@ struct fsldma_device { struct fsldma_chan *chan[FSL_DMA_MAX_CHANS_PER_DEVICE]; u32 feature; /* The same as DMA channels */ int irq; /* Channel IRQ */ + int addr_bits; /* DMA addressing bits supported */ }; /* Define macros for fsldma_chan->feature property */ diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index c1350a36fddd..25a4134be36b 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -163,7 +163,7 @@ struct hisi_dma_dev { u32 chan_depth; enum hisi_dma_reg_layout reg_layout; void __iomem *queue_base; /* queue region start of register */ - struct hisi_dma_chan chan[]; + struct hisi_dma_chan chan[] __counted_by(chan_num); }; #ifdef CONFIG_DEBUG_FS @@ -677,7 +677,7 @@ static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index) writel_relaxed(tmp, addr); /* - * 0 - dma should process FLR whith CPU. + * 0 - dma should process FLR with CPU. * 1 - dma not process FLR, only cpu process FLR. */ addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE + diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c index 0ac634a51c5e..d147353d47ab 100644 --- a/drivers/dma/idma64.c +++ b/drivers/dma/idma64.c @@ -171,6 +171,10 @@ static irqreturn_t idma64_irq(int irq, void *dev) u32 status_err; unsigned short i; + /* Since IRQ may be shared, check if DMA controller is powered on */ + if (status == GENMASK(31, 0)) + return IRQ_NONE; + dev_vdbg(idma64->dma.dev, "%s: status=%#x\n", __func__, status); /* Check if we have any interrupt from the DMA controller */ @@ -286,7 +290,7 @@ static void idma64_desc_fill(struct idma64_chan *idma64c, desc->length += hw->len; } while (i); - /* Trigger an interrupt after the last block is transfered */ + /* Trigger an interrupt after the last block is transferred */ lli->ctllo |= IDMA64C_CTLL_INT_EN; /* Disable LLP transfer in the last block */ @@ -360,7 +364,7 @@ static size_t idma64_active_desc_size(struct idma64_chan *idma64c) if (!i) return bytes; - /* The current chunk is not fully transfered yet */ + /* The current chunk is not fully transferred yet */ bytes += desc->hw[--i].len; return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi); @@ -660,13 +664,11 @@ static int idma64_platform_probe(struct platform_device *pdev) return 0; } -static int idma64_platform_remove(struct platform_device *pdev) +static void idma64_platform_remove(struct platform_device *pdev) { struct idma64_chip *chip = platform_get_drvdata(pdev); idma64_remove(chip); - - return 0; } static int __maybe_unused idma64_pm_suspend(struct device *dev) diff --git a/drivers/dma/idxd/Makefile b/drivers/dma/idxd/Makefile index dc096839ac63..9ff9d7b87b64 100644 --- a/drivers/dma/idxd/Makefile +++ b/drivers/dma/idxd/Makefile @@ -1,12 +1,12 @@ -ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=IDXD +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE='"IDXD"' + +obj-$(CONFIG_INTEL_IDXD_BUS) += idxd_bus.o +idxd_bus-y := bus.o obj-$(CONFIG_INTEL_IDXD) += idxd.o -idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o debugfs.o +idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o debugfs.o defaults.o idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o -obj-$(CONFIG_INTEL_IDXD_BUS) += idxd_bus.o -idxd_bus-y := bus.o - obj-$(CONFIG_INTEL_IDXD_COMPAT) += idxd_compat.o idxd_compat-y := compat.o diff --git a/drivers/dma/idxd/bus.c b/drivers/dma/idxd/bus.c index 6f84621053c6..e647a684485d 100644 --- a/drivers/dma/idxd/bus.c +++ b/drivers/dma/idxd/bus.c @@ -33,10 +33,10 @@ void idxd_driver_unregister(struct idxd_device_driver *idxd_drv) EXPORT_SYMBOL_GPL(idxd_driver_unregister); static int idxd_config_bus_match(struct device *dev, - struct device_driver *drv) + const struct device_driver *drv) { - struct idxd_device_driver *idxd_drv = - container_of(drv, struct idxd_device_driver, drv); + const struct idxd_device_driver *idxd_drv = + container_of_const(drv, struct idxd_device_driver, drv); struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); int i = 0; @@ -67,11 +67,17 @@ static void idxd_config_bus_remove(struct device *dev) idxd_drv->remove(idxd_dev); } -struct bus_type dsa_bus_type = { +static int idxd_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + return add_uevent_var(env, "MODALIAS=" IDXD_DEVICES_MODALIAS_FMT, 0); +} + +const struct bus_type dsa_bus_type = { .name = "dsa", .match = idxd_config_bus_match, .probe = idxd_config_bus_probe, .remove = idxd_config_bus_remove, + .uevent = idxd_bus_uevent, }; EXPORT_SYMBOL_GPL(dsa_bus_type); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index d32deb9b4e3d..7e4715f92773 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -28,7 +28,6 @@ struct idxd_cdev_context { * global to avoid conflict file names. */ static DEFINE_IDA(file_ida); -static DEFINE_MUTEX(ida_lock); /* * ictx is an array based off of accelerator types. enum idxd_type @@ -123,9 +122,7 @@ static void idxd_file_dev_release(struct device *dev) struct idxd_device *idxd = wq->idxd; int rc; - mutex_lock(&ida_lock); ida_free(&file_ida, ctx->id); - mutex_unlock(&ida_lock); /* Wait for in-flight operations to complete. */ if (wq_shared(wq)) { @@ -152,7 +149,7 @@ static void idxd_file_dev_release(struct device *dev) mutex_unlock(&wq->wq_lock); } -static struct device_type idxd_cdev_file_type = { +static const struct device_type idxd_cdev_file_type = { .name = "idxd_file", .release = idxd_file_dev_release, .groups = cdev_file_attribute_groups, @@ -165,11 +162,11 @@ static void idxd_cdev_dev_release(struct device *dev) struct idxd_wq *wq = idxd_cdev->wq; cdev_ctx = &ictx[wq->idxd->data->type]; - ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); kfree(idxd_cdev); } -static struct device_type idxd_cdev_device_type = { +static const struct device_type idxd_cdev_device_type = { .name = "idxd_cdev", .release = idxd_cdev_dev_release, }; @@ -225,7 +222,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) struct idxd_wq *wq; struct device *dev, *fdev; int rc = 0; - struct iommu_sva *sva; + struct iommu_sva *sva = NULL; unsigned int pasid; struct idxd_cdev *idxd_cdev; @@ -284,9 +281,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) } idxd_cdev = wq->idxd_cdev; - mutex_lock(&ida_lock); ctx->id = ida_alloc(&file_ida, GFP_KERNEL); - mutex_unlock(&ida_lock); if (ctx->id < 0) { dev_warn(dev, "ida alloc failure\n"); goto failed_ida; @@ -322,7 +317,7 @@ failed_set_pasid: if (device_user_pasid_enabled(idxd)) idxd_xa_pasid_remove(ctx); failed_get_pasid: - if (device_user_pasid_enabled(idxd)) + if (device_user_pasid_enabled(idxd) && !IS_ERR_OR_NULL(sva)) iommu_sva_unbind_device(sva); failed: mutex_unlock(&wq->wq_lock); @@ -342,10 +337,10 @@ static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) if (!evl) return; - spin_lock(&evl->lock); + mutex_lock(&evl->lock); status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); t = status.tail; - h = evl->head; + h = status.head; size = evl->size; while (h != t) { @@ -354,9 +349,10 @@ static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) set_bit(h, evl->bmap); h = (h + 1) % size; } - spin_unlock(&evl->lock); + if (wq->wq) + drain_workqueue(wq->wq); - drain_workqueue(wq->wq); + mutex_unlock(&evl->lock); } static int idxd_cdev_release(struct inode *node, struct file *filep) @@ -401,6 +397,21 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) int rc; dev_dbg(&pdev->dev, "%s called\n", __func__); + + /* + * Due to an erratum in some of the devices supported by the driver, + * direct user submission to the device can be unsafe. + * (See the INTEL-SA-01084 security advisory) + * + * For the devices that exhibit this behavior, require that the user + * has CAP_SYS_RAWIO capabilities. + */ + if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO)) + return -EPERM; + + if (current->mm != ctx->mm) + return -EPERM; + rc = check_vma(wq, vma, __func__); if (rc < 0) return rc; @@ -415,6 +426,75 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_page_prot); } +static int idxd_submit_user_descriptor(struct idxd_user_context *ctx, + struct dsa_hw_desc __user *udesc) +{ + struct idxd_wq *wq = ctx->wq; + struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev; + const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40; + void __iomem *portal = idxd_wq_portal_addr(wq); + struct dsa_hw_desc descriptor __aligned(64); + int rc; + + rc = copy_from_user(&descriptor, udesc, sizeof(descriptor)); + if (rc) + return -EFAULT; + + /* + * DSA devices are capable of indirect ("batch") command submission. + * On devices where direct user submissions are not safe, we cannot + * allow this since there is no good way for us to verify these + * indirect commands. Narrow the restriction of operations with the + * BATCH opcode to only DSA version 1 devices. + */ + if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH && + wq->idxd->hw.version == DEVICE_VERSION_1 && + !wq->idxd->user_submission_safe) + return -EINVAL; + /* + * As per the programming specification, the completion address must be + * aligned to 32 or 64 bytes. If this is violated the hardware + * engine can get very confused (security issue). + */ + if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align)) + return -EINVAL; + + if (wq_dedicated(wq)) + iosubmit_cmds512(portal, &descriptor, 1); + else { + descriptor.priv = 0; + descriptor.pasid = ctx->pasid; + rc = idxd_enqcmds(wq, portal, &descriptor); + if (rc < 0) + return rc; + } + + return 0; +} + +static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len, + loff_t *unused) +{ + struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf; + struct idxd_user_context *ctx = filp->private_data; + ssize_t written = 0; + int i; + + if (current->mm != ctx->mm) + return -EPERM; + + for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) { + int rc = idxd_submit_user_descriptor(ctx, udesc + i); + + if (rc) + return written ? written : rc; + + written += sizeof(struct dsa_hw_desc); + } + + return written; +} + static __poll_t idxd_cdev_poll(struct file *filp, struct poll_table_struct *wait) { @@ -423,6 +503,9 @@ static __poll_t idxd_cdev_poll(struct file *filp, struct idxd_device *idxd = wq->idxd; __poll_t out = 0; + if (current->mm != ctx->mm) + return POLLNVAL; + poll_wait(filp, &wq->err_queue, wait); spin_lock(&idxd->dev_lock); if (idxd->sw_err.valid) @@ -437,6 +520,7 @@ static const struct file_operations idxd_cdev_fops = { .open = idxd_cdev_open, .release = idxd_cdev_release, .mmap = idxd_cdev_mmap, + .write = idxd_cdev_write, .poll = idxd_cdev_poll, }; @@ -463,7 +547,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) cdev = &idxd_cdev->cdev; dev = cdev_dev(idxd_cdev); cdev_ctx = &ictx[wq->idxd->data->type]; - minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); + minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL); if (minor < 0) { kfree(idxd_cdev); return minor; @@ -501,7 +585,6 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) struct idxd_cdev *idxd_cdev; idxd_cdev = wq->idxd_cdev; - ida_destroy(&file_ida); wq->idxd_cdev = NULL; cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); put_device(cdev_dev(idxd_cdev)); @@ -509,6 +592,7 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) { + struct device *dev = &idxd_dev->conf_dev; struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); struct idxd_device *idxd = wq->idxd; int rc; @@ -516,6 +600,14 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) if (idxd->state != IDXD_DEV_ENABLED) return -ENXIO; + mutex_lock(&wq->wq_lock); + + if (!idxd_wq_driver_name_match(wq, dev)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + rc = -ENODEV; + goto wq_err; + } + /* * User type WQ is enabled only when SVA is enabled for two reasons: * - If no IOMMU or IOMMU Passthrough without SVA, userspace @@ -531,11 +623,10 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) dev_dbg(&idxd->pdev->dev, "User type WQ cannot be enabled without SVA.\n"); - return -EOPNOTSUPP; + rc = -EOPNOTSUPP; + goto wq_err; } - mutex_lock(&wq->wq_lock); - wq->wq = create_workqueue(dev_name(wq_confdev(wq))); if (!wq->wq) { rc = -ENOMEM; @@ -543,7 +634,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) } wq->type = IDXD_WQT_USER; - rc = drv_enable_wq(wq); + rc = idxd_drv_enable_wq(wq); if (rc < 0) goto err; @@ -558,7 +649,7 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) return 0; err_cdev: - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); err: destroy_workqueue(wq->wq); wq->type = IDXD_WQT_NONE; @@ -573,7 +664,7 @@ static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) mutex_lock(&wq->wq_lock); idxd_wq_del_cdev(wq); - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); wq->type = IDXD_WQT_NONE; destroy_workqueue(wq->wq); wq->wq = NULL; diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c index 5fd38d1b9d28..eff9943f1a42 100644 --- a/drivers/dma/idxd/compat.c +++ b/drivers/dma/idxd/compat.c @@ -7,7 +7,6 @@ #include <linux/device/bus.h> #include "idxd.h" -extern int device_driver_attach(struct device_driver *drv, struct device *dev); extern void device_driver_detach(struct device *dev); #define DRIVER_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \ @@ -104,4 +103,4 @@ struct idxd_device_driver dsa_drv = { }; module_idxd_driver(dsa_drv); -MODULE_IMPORT_NS(IDXD); +MODULE_IMPORT_NS("IDXD"); diff --git a/drivers/dma/idxd/debugfs.c b/drivers/dma/idxd/debugfs.c index 9cfbd9b14c4c..ad4245cb301d 100644 --- a/drivers/dma/idxd/debugfs.c +++ b/drivers/dma/idxd/debugfs.c @@ -66,11 +66,11 @@ static int debugfs_evl_show(struct seq_file *s, void *d) if (!evl || !evl->log) return 0; - spin_lock(&evl->lock); + mutex_lock(&evl->lock); - h = evl->head; evl_status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); t = evl_status.tail; + h = evl_status.head; evl_size = evl->size; seq_printf(s, "Event Log head %u tail %u interrupt pending %u\n\n", @@ -87,7 +87,7 @@ static int debugfs_evl_show(struct seq_file *s, void *d) dump_event_entry(idxd, s, i, &count, processed); } - spin_unlock(&evl->lock); + mutex_unlock(&evl->lock); return 0; } diff --git a/drivers/dma/idxd/defaults.c b/drivers/dma/idxd/defaults.c new file mode 100644 index 000000000000..2bbbcd02a0da --- /dev/null +++ b/drivers/dma/idxd/defaults.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2023 Intel Corporation. All rights rsvd. */ +#include <linux/kernel.h> +#include "idxd.h" + +int idxd_load_iaa_device_defaults(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct idxd_group *group; + struct idxd_wq *wq; + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return 0; + + wq = idxd->wqs[0]; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + /* set mode to "dedicated" */ + set_bit(WQ_FLAG_DEDICATED, &wq->flags); + wq->threshold = 0; + + /* only setting up 1 wq, so give it all the wq space */ + wq->size = idxd->max_wq_size; + + /* set priority to 10 */ + wq->priority = 10; + + /* set type to "kernel" */ + wq->type = IDXD_WQT_KERNEL; + + /* set wq group to 0 */ + group = idxd->groups[0]; + wq->group = group; + group->num_wqs++; + + /* set name to "iaa_crypto" */ + strscpy_pad(wq->name, "iaa_crypto"); + + /* set driver_name to "crypto" */ + strscpy_pad(wq->driver_name, "crypto"); + + engine = idxd->engines[0]; + + /* set engine group to 0 */ + engine->group = idxd->groups[0]; + engine->group->num_engines++; + + return 0; +} diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 9a15f0d12c79..c2cdf41b6e57 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -16,6 +16,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, u32 *status); static void idxd_device_wqs_clear_state(struct idxd_device *idxd); static void idxd_wq_disable_cleanup(struct idxd_wq *wq); +static int idxd_wq_config_write(struct idxd_wq *wq); /* Interrupt control bits */ void idxd_unmask_error_interrupts(struct idxd_device *idxd) @@ -161,6 +162,7 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) free_hw_descs(wq); return rc; } +EXPORT_SYMBOL_NS_GPL(idxd_wq_alloc_resources, "IDXD"); void idxd_wq_free_resources(struct idxd_wq *wq) { @@ -174,6 +176,7 @@ void idxd_wq_free_resources(struct idxd_wq *wq) dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); sbitmap_queue_free(&wq->sbq); } +EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, "IDXD"); int idxd_wq_enable(struct idxd_wq *wq) { @@ -213,14 +216,28 @@ int idxd_wq_disable(struct idxd_wq *wq, bool reset_config) return 0; } + /* + * Disable WQ does not drain address translations, if WQ attributes are + * changed before translations are drained, pending translations can + * be issued using updated WQ attibutes, resulting in invalid + * translations being cached in the device translation cache. + * + * To make sure pending translations are drained before WQ + * attributes are changed, we use a WQ Drain followed by WQ Reset and + * then restore the WQ configuration. + */ + idxd_wq_drain(wq); + operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); - idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status); + idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, &status); if (status != IDXD_CMDSTS_SUCCESS) { - dev_dbg(dev, "WQ disable failed: %#x\n", status); + dev_dbg(dev, "WQ reset failed: %#x\n", status); return -ENXIO; } + idxd_wq_config_write(wq); + if (reset_config) idxd_wq_disable_cleanup(wq); clear_bit(wq->id, idxd->wq_enable_map); @@ -299,21 +316,6 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd) } } -static void __idxd_wq_set_priv_locked(struct idxd_wq *wq, int priv) -{ - struct idxd_device *idxd = wq->idxd; - union wqcfg wqcfg; - unsigned int offset; - - offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PRIVL_IDX); - spin_lock(&idxd->dev_lock); - wqcfg.bits[WQCFG_PRIVL_IDX] = ioread32(idxd->reg_base + offset); - wqcfg.priv = priv; - wq->wqcfg->bits[WQCFG_PRIVL_IDX] = wqcfg.bits[WQCFG_PRIVL_IDX]; - iowrite32(wqcfg.bits[WQCFG_PRIVL_IDX], idxd->reg_base + offset); - spin_unlock(&idxd->dev_lock); -} - static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid) { struct idxd_device *idxd = wq->idxd; @@ -420,6 +422,7 @@ int idxd_wq_init_percpu_ref(struct idxd_wq *wq) reinit_completion(&wq->wq_resurrect); return 0; } +EXPORT_SYMBOL_NS_GPL(idxd_wq_init_percpu_ref, "IDXD"); void __idxd_wq_quiesce(struct idxd_wq *wq) { @@ -429,6 +432,7 @@ void __idxd_wq_quiesce(struct idxd_wq *wq) complete_all(&wq->wq_resurrect); wait_for_completion(&wq->wq_dead); } +EXPORT_SYMBOL_NS_GPL(__idxd_wq_quiesce, "IDXD"); void idxd_wq_quiesce(struct idxd_wq *wq) { @@ -436,6 +440,7 @@ void idxd_wq_quiesce(struct idxd_wq *wq) __idxd_wq_quiesce(wq); mutex_unlock(&wq->wq_lock); } +EXPORT_SYMBOL_NS_GPL(idxd_wq_quiesce, "IDXD"); /* Device control bits */ static inline bool idxd_is_enabled(struct idxd_device *idxd) @@ -492,6 +497,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, union idxd_command_reg cmd; DECLARE_COMPLETION_ONSTACK(done); u32 stat; + unsigned long flags; if (idxd_device_is_halted(idxd)) { dev_warn(&idxd->pdev->dev, "Device is HALTED!\n"); @@ -505,7 +511,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, cmd.operand = operand; cmd.int_req = 1; - spin_lock(&idxd->cmd_lock); + spin_lock_irqsave(&idxd->cmd_lock, flags); wait_event_lock_irq(idxd->cmd_waitq, !test_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags), idxd->cmd_lock); @@ -522,7 +528,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, * After command submitted, release lock and go to sleep until * the command completes via interrupt. */ - spin_unlock(&idxd->cmd_lock); + spin_unlock_irqrestore(&idxd->cmd_lock, flags); wait_for_completion(&done); stat = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); spin_lock(&idxd->cmd_lock); @@ -784,9 +790,7 @@ static int idxd_device_evl_setup(struct idxd_device *idxd) goto err_alloc; } - memset(addr, 0, size); - - spin_lock(&evl->lock); + mutex_lock(&evl->lock); evl->log = addr; evl->dma = dma_addr; evl->log_size = size; @@ -807,7 +811,7 @@ static int idxd_device_evl_setup(struct idxd_device *idxd) gencfg.evl_en = 1; iowrite32(gencfg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); - spin_unlock(&evl->lock); + mutex_unlock(&evl->lock); return 0; err_alloc: @@ -818,6 +822,9 @@ err_bmap: static void idxd_device_evl_free(struct idxd_device *idxd) { + void *evl_log; + unsigned int evl_log_size; + dma_addr_t evl_dma; union gencfg_reg gencfg; union genctrl_reg genctrl; struct device *dev = &idxd->pdev->dev; @@ -827,7 +834,7 @@ static void idxd_device_evl_free(struct idxd_device *idxd) if (!gencfg.evl_en) return; - spin_lock(&evl->lock); + mutex_lock(&evl->lock); gencfg.evl_en = 0; iowrite32(gencfg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); @@ -838,11 +845,15 @@ static void idxd_device_evl_free(struct idxd_device *idxd) iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET); iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET + 8); - dma_free_coherent(dev, evl->log_size, evl->log, evl->dma); bitmap_free(evl->bmap); + evl_log = evl->log; + evl_log_size = evl->log_size; + evl_dma = evl->dma; evl->log = NULL; evl->size = IDXD_EVL_SIZE_MIN; - spin_unlock(&evl->lock); + mutex_unlock(&evl->lock); + + dma_free_coherent(dev, evl_log_size, evl_log, evl_dma); } static void idxd_group_config_write(struct idxd_group *group) @@ -1282,7 +1293,7 @@ static void idxd_flush_pending_descs(struct idxd_irq_entry *ie) tx = &desc->txd; tx->callback = NULL; tx->callback_result = NULL; - idxd_dma_complete_txd(desc, ctype, true); + idxd_dma_complete_txd(desc, ctype, true, NULL, NULL); } } @@ -1366,7 +1377,7 @@ err_irq: return rc; } -int drv_enable_wq(struct idxd_wq *wq) +int idxd_drv_enable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; @@ -1421,15 +1432,14 @@ int drv_enable_wq(struct idxd_wq *wq) } /* - * In the event that the WQ is configurable for pasid and priv bits. - * For kernel wq, the driver should setup the pasid, pasid_en, and priv bit. - * However, for non-kernel wq, the driver should only set the pasid_en bit for - * shared wq. A dedicated wq that is not 'kernel' type will configure pasid and + * In the event that the WQ is configurable for pasid, the driver + * should setup the pasid, pasid_en bit. This is true for both kernel + * and user shared workqueues. There is no need to setup priv bit in + * that in-kernel DMA will also do user privileged requests. + * A dedicated wq that is not 'kernel' type will configure pasid and * pasid_en later on so there is no need to setup. */ if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { - int priv = 0; - if (wq_pasid_enabled(wq)) { if (is_idxd_wq_kernel(wq) || wq_shared(wq)) { u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0; @@ -1437,10 +1447,6 @@ int drv_enable_wq(struct idxd_wq *wq) __idxd_wq_set_pasid_locked(wq, pasid); } } - - if (is_idxd_wq_kernel(wq)) - priv = 1; - __idxd_wq_set_priv_locked(wq, priv); } rc = 0; @@ -1503,8 +1509,9 @@ err_map_portal: err: return rc; } +EXPORT_SYMBOL_NS_GPL(idxd_drv_enable_wq, "IDXD"); -void drv_disable_wq(struct idxd_wq *wq) +void idxd_drv_disable_wq(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; @@ -1524,6 +1531,7 @@ void drv_disable_wq(struct idxd_wq *wq) wq->type = IDXD_WQT_NONE; wq->client_count = 0; } +EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, "IDXD"); int idxd_device_drv_probe(struct idxd_dev *idxd_dev) { @@ -1548,6 +1556,15 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev) if (rc < 0) return -ENXIO; + /* + * System PASID is preserved across device disable/enable cycle, but + * genconfig register content gets cleared during device reset. We + * need to re-enable user interrupts for kernel work queue completion + * IRQ to function. + */ + if (idxd->pasid != IOMMU_PASID_INVALID) + idxd_set_user_intr(idxd, 1); + rc = idxd_device_evl_setup(idxd); if (rc < 0) { idxd->cmd_status = IDXD_SCMD_DEV_EVL_ERR; diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index eb35ca313684..dbecd699237e 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -22,7 +22,7 @@ static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c) void idxd_dma_complete_txd(struct idxd_desc *desc, enum idxd_complete_type comp_type, - bool free_desc) + bool free_desc, void *ctx, u32 *status) { struct idxd_device *idxd = desc->wq->idxd; struct dma_async_tx_descriptor *tx; @@ -75,9 +75,10 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq, hw->xfer_size = len; /* * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv - * field instead. This field should be set to 1 for kernel descriptors. + * field instead. This field should be set to 0 for kernel descriptors + * since kernel DMA on VT-d supports "user" privilege only. */ - hw->priv = 1; + hw->priv = 0; hw->completion_addr = compl; } @@ -268,7 +269,7 @@ static int idxd_register_dma_channel(struct idxd_wq *wq) desc->txd.tx_submit = idxd_dma_tx_submit; } - rc = dma_async_device_channel_register(dma, chan); + rc = dma_async_device_channel_register(dma, chan, NULL); if (rc < 0) { kfree(idxd_chan); return rc; @@ -305,9 +306,15 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) return -ENXIO; mutex_lock(&wq->wq_lock); + if (!idxd_wq_driver_name_match(wq, dev)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + rc = -ENODEV; + goto err; + } + wq->type = IDXD_WQT_KERNEL; - rc = drv_enable_wq(wq); + rc = idxd_drv_enable_wq(wq); if (rc < 0) { dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc); rc = -ENXIO; @@ -326,7 +333,7 @@ static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) return 0; err_dma: - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); err: wq->type = IDXD_WQT_NONE; mutex_unlock(&wq->wq_lock); @@ -340,7 +347,7 @@ static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) mutex_lock(&wq->wq_lock); __idxd_wq_quiesce(wq); idxd_unregister_dma_channel(wq); - drv_disable_wq(wq); + idxd_drv_disable_wq(wq); mutex_unlock(&wq->wq_lock); } @@ -352,6 +359,7 @@ static enum idxd_dev_type dev_types[] = { struct idxd_device_driver idxd_dmaengine_drv = { .probe = idxd_dmaengine_drv_probe, .remove = idxd_dmaengine_drv_remove, + .desc_complete = idxd_dma_complete_txd, .name = "dmaengine", .type = dev_types, }; diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 5428a2e1b1ec..74e6695881e6 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -13,12 +13,12 @@ #include <linux/bitmap.h> #include <linux/perf_event.h> #include <linux/iommu.h> +#include <linux/crypto.h> #include <uapi/linux/idxd.h> #include "registers.h" #define IDXD_DRIVER_VERSION "1.00" -extern struct kmem_cache *idxd_desc_pool; extern bool tc_override; struct idxd_wq; @@ -57,11 +57,23 @@ enum idxd_type { #define IDXD_ENQCMDS_RETRIES 32 #define IDXD_ENQCMDS_MAX_RETRIES 64 +enum idxd_complete_type { + IDXD_COMPLETE_NORMAL = 0, + IDXD_COMPLETE_ABORT, + IDXD_COMPLETE_DEV_FAIL, +}; + +struct idxd_desc; + struct idxd_device_driver { const char *name; enum idxd_dev_type *type; int (*probe)(struct idxd_dev *idxd_dev); void (*remove)(struct idxd_dev *idxd_dev); + void (*desc_complete)(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc, + void *ctx, u32 *status); struct device_driver drv; }; @@ -111,7 +123,6 @@ struct idxd_pmu { struct pmu pmu; char name[IDXD_NAME_SIZE]; - int cpu; int n_counters; int counter_width; @@ -122,8 +133,6 @@ struct idxd_pmu { unsigned long supported_filters; int n_filters; - - struct hlist_node cpuhp_node; }; #define IDXD_MAX_PRIORITY 0xf @@ -159,7 +168,8 @@ struct idxd_cdev { int minor; }; -#define IDXD_ALLOCATED_BATCH_SIZE 128U +#define DRIVER_NAME_SIZE 128 + #define WQ_NAME_SIZE 1024 #define WQ_TYPE_SIZE 10 @@ -172,12 +182,6 @@ enum idxd_op_type { IDXD_OP_NONBLOCK = 1, }; -enum idxd_complete_type { - IDXD_COMPLETE_NORMAL = 0, - IDXD_COMPLETE_ABORT, - IDXD_COMPLETE_DEV_FAIL, -}; - struct idxd_dma_chan { struct dma_chan chan; struct idxd_wq *wq; @@ -227,6 +231,8 @@ struct idxd_wq { /* Lock to protect upasid_xa access. */ struct mutex uc_lock; struct xarray upasid_xa; + + char driver_name[DRIVER_NAME_SIZE + 1]; }; struct idxd_engine { @@ -266,27 +272,30 @@ struct idxd_dma_dev { struct dma_device dma; }; +typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd); + struct idxd_driver_data { const char *name_prefix; enum idxd_type type; - struct device_type *dev_type; + const struct device_type *dev_type; int compl_size; int align; int evl_cr_off; int cr_status_off; int cr_result_off; + bool user_submission_safe; + load_device_defaults_fn_t load_device_defaults; }; struct idxd_evl { /* Lock to protect event log access. */ - spinlock_t lock; + struct mutex lock; void *log; dma_addr_t dma; /* Total size of event log = number of entries * entry size. */ unsigned int log_size; /* The number of entries in the event log. */ u16 size; - u16 head; unsigned long *bmap; bool batch_fail[IDXD_MAX_BATCH_IDENT]; }; @@ -361,6 +370,19 @@ struct idxd_device { struct dentry *dbgfs_dir; struct dentry *dbgfs_evl_file; + + bool user_submission_safe; + + struct idxd_saved_states *idxd_saved; +}; + +struct idxd_saved_states { + struct idxd_device saved_idxd; + struct idxd_evl saved_evl; + struct idxd_engine **saved_engines; + struct idxd_wq **saved_wqs; + struct idxd_group **saved_groups; + unsigned long *saved_wq_enable_map; }; static inline unsigned int evl_ent_size(struct idxd_device *idxd) @@ -374,6 +396,14 @@ static inline unsigned int evl_size(struct idxd_device *idxd) return idxd->evl->size * evl_ent_size(idxd); } +struct crypto_ctx { + struct acomp_req *req; + struct crypto_tfm *tfm; + dma_addr_t src_addr; + dma_addr_t dst_addr; + bool compress; +}; + /* IDXD software descriptor */ struct idxd_desc { union { @@ -386,7 +416,10 @@ struct idxd_desc { struct iax_completion_record *iax_completion; }; dma_addr_t compl_dma; - struct dma_async_tx_descriptor txd; + union { + struct dma_async_tx_descriptor txd; + struct crypto_ctx crypto; + }; struct llist_node llnode; struct list_head list; int id; @@ -413,6 +446,15 @@ enum idxd_completion_status { #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) +static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq) +{ + struct device *dev = wq_confdev(wq); + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + + return idxd_drv; +} + static inline struct idxd_device *confdev_to_idxd(struct device *dev) { struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); @@ -473,15 +515,24 @@ static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) return container_of(ie, struct idxd_device, ie); } -extern struct bus_type dsa_bus_type; +static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable) +{ + union gencfg_reg reg; + + reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); + reg.user_int_en = enable; + iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); +} + +extern const struct bus_type dsa_bus_type; extern bool support_enqcmd; extern struct ida idxd_ida; -extern struct device_type dsa_device_type; -extern struct device_type iax_device_type; -extern struct device_type idxd_wq_device_type; -extern struct device_type idxd_engine_device_type; -extern struct device_type idxd_group_device_type; +extern const struct device_type dsa_device_type; +extern const struct device_type iax_device_type; +extern const struct device_type idxd_wq_device_type; +extern const struct device_type idxd_engine_device_type; +extern const struct device_type idxd_group_device_type; static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) { @@ -605,6 +656,16 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; +static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private) +{ + dev_set_drvdata(wq_confdev(wq), private); +} + +static inline void *idxd_wq_get_private(struct idxd_wq *wq) +{ + return dev_get_drvdata(wq_confdev(wq)); +} + /* * Intel IAA does not support batch processing. * The max batch size of device, max batch size of wq and @@ -637,6 +698,14 @@ static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wq wqcfg->max_batch_shift = max_batch_shift; } +static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev) +{ + return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); +} + +#define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*") +#define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d" + int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *module, const char *mod_name); #define idxd_driver_register(driver) \ @@ -647,15 +716,30 @@ void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); #define module_idxd_driver(__idxd_driver) \ module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) -int idxd_register_bus_type(void); -void idxd_unregister_bus_type(void); +void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); +void idxd_dma_complete_txd(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc, void *ctx, u32 *status); + +static inline void idxd_desc_complete(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc) +{ + struct idxd_device_driver *drv; + u32 status; + + drv = wq_to_idxd_drv(desc->wq); + if (drv->desc_complete) + drv->desc_complete(desc, comp_type, free_desc, + &desc->txd, &status); +} + int idxd_register_devices(struct idxd_device *idxd); void idxd_unregister_devices(struct idxd_device *idxd); -int idxd_register_driver(void); -void idxd_unregister_driver(void); void idxd_wqs_quiesce(struct idxd_device *idxd); bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count); +int idxd_load_iaa_device_defaults(struct idxd_device *idxd); /* device interrupt control */ irqreturn_t idxd_misc_thread(int vec, void *data); @@ -664,12 +748,12 @@ void idxd_mask_error_interrupts(struct idxd_device *idxd); void idxd_unmask_error_interrupts(struct idxd_device *idxd); /* device control */ -int idxd_register_idxd_drv(void); -void idxd_unregister_idxd_drv(void); int idxd_device_drv_probe(struct idxd_dev *idxd_dev); +int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, + const struct pci_device_id *id); void idxd_device_drv_remove(struct idxd_dev *idxd_dev); -int drv_enable_wq(struct idxd_wq *wq); -void drv_disable_wq(struct idxd_wq *wq); +int idxd_drv_enable_wq(struct idxd_wq *wq); +void idxd_drv_disable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); int idxd_device_enable(struct idxd_device *idxd); int idxd_device_disable(struct idxd_device *idxd); @@ -704,15 +788,11 @@ int idxd_wq_request_irq(struct idxd_wq *wq); /* submission */ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); -void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); /* dmaengine */ int idxd_register_dma_device(struct idxd_device *idxd); void idxd_unregister_dma_device(struct idxd_device *idxd); -void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); -void idxd_dma_complete_txd(struct idxd_desc *desc, - enum idxd_complete_type comp_type, bool free_desc); /* cdev */ int idxd_cdev_register(void); @@ -729,14 +809,10 @@ void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index); int perfmon_pmu_init(struct idxd_device *idxd); void perfmon_pmu_remove(struct idxd_device *idxd); void perfmon_counter_overflow(struct idxd_device *idxd); -void perfmon_init(void); -void perfmon_exit(void); #else static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} -static inline void perfmon_init(void) {} -static inline void perfmon_exit(void) {} #endif /* debugfs */ diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 1aa823974cda..2acc34b3daff 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -22,9 +22,10 @@ #include "perfmon.h" MODULE_VERSION(IDXD_DRIVER_VERSION); +MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver"); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Intel Corporation"); -MODULE_IMPORT_NS(IDXD); +MODULE_IMPORT_NS("IDXD"); static bool sva = true; module_param(sva, bool, 0644); @@ -47,6 +48,7 @@ static struct idxd_driver_data idxd_driver_data[] = { .align = 32, .dev_type = &dsa_device_type, .evl_cr_off = offsetof(struct dsa_evl_entry, cr), + .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ .cr_status_off = offsetof(struct dsa_completion_record, status), .cr_result_off = offsetof(struct dsa_completion_record, result), }, @@ -57,17 +59,29 @@ static struct idxd_driver_data idxd_driver_data[] = { .align = 64, .dev_type = &iax_device_type, .evl_cr_off = offsetof(struct iax_evl_entry, cr), + .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ .cr_status_off = offsetof(struct iax_completion_record, status), .cr_result_off = offsetof(struct iax_completion_record, error_code), + .load_device_defaults = idxd_load_iaa_device_defaults, }, }; static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, + /* DSA on GNR-D platforms */ + { PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) }, + /* DSA on DMR platforms */ + { PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) }, /* IAX ver 1.0 platforms */ { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, + /* IAA on DMR platforms */ + { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) }, + /* IAA PTL platforms */ + { PCI_DEVICE_DATA(INTEL, IAA_PTL, &idxd_driver_data[IDXD_TYPE_IAX]) }, + /* IAA WCL platforms */ + { PCI_DEVICE_DATA(INTEL, IAA_WCL, &idxd_driver_data[IDXD_TYPE_IAX]) }, { 0, } }; MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); @@ -143,6 +157,25 @@ static void idxd_cleanup_interrupts(struct idxd_device *idxd) pci_free_irq_vectors(pdev); } +static void idxd_clean_wqs(struct idxd_device *idxd) +{ + struct idxd_wq *wq; + struct device *conf_dev; + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + wq = idxd->wqs[i]; + if (idxd->hw.wq_cap.op_config) + bitmap_free(wq->opcap_bmap); + kfree(wq->wqcfg); + conf_dev = wq_confdev(wq); + put_device(conf_dev); + kfree(wq); + } + bitmap_free(idxd->wq_enable_map); + kfree(idxd->wqs); +} + static int idxd_setup_wqs(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -157,29 +190,30 @@ static int idxd_setup_wqs(struct idxd_device *idxd) idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); if (!idxd->wq_enable_map) { - kfree(idxd->wqs); - return -ENOMEM; + rc = -ENOMEM; + goto err_free_wqs; } for (i = 0; i < idxd->max_wqs; i++) { wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); if (!wq) { rc = -ENOMEM; - goto err; + goto err_unwind; } idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); conf_dev = wq_confdev(wq); wq->id = i; wq->idxd = idxd; - device_initialize(wq_confdev(wq)); + device_initialize(conf_dev); conf_dev->parent = idxd_confdev(idxd); conf_dev->bus = &dsa_bus_type; conf_dev->type = &idxd_wq_device_type; rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); if (rc < 0) { put_device(conf_dev); - goto err; + kfree(wq); + goto err_unwind; } mutex_init(&wq->wq_lock); @@ -192,16 +226,19 @@ static int idxd_setup_wqs(struct idxd_device *idxd) wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); if (!wq->wqcfg) { put_device(conf_dev); + kfree(wq); rc = -ENOMEM; - goto err; + goto err_unwind; } if (idxd->hw.wq_cap.op_config) { wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); if (!wq->opcap_bmap) { + kfree(wq->wqcfg); put_device(conf_dev); + kfree(wq); rc = -ENOMEM; - goto err; + goto err_unwind; } bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); } @@ -212,15 +249,39 @@ static int idxd_setup_wqs(struct idxd_device *idxd) return 0; - err: +err_unwind: while (--i >= 0) { wq = idxd->wqs[i]; + if (idxd->hw.wq_cap.op_config) + bitmap_free(wq->opcap_bmap); + kfree(wq->wqcfg); conf_dev = wq_confdev(wq); put_device(conf_dev); + kfree(wq); } + bitmap_free(idxd->wq_enable_map); + +err_free_wqs: + kfree(idxd->wqs); + return rc; } +static void idxd_clean_engines(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct device *conf_dev; + int i; + + for (i = 0; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + conf_dev = engine_confdev(engine); + put_device(conf_dev); + kfree(engine); + } + kfree(idxd->engines); +} + static int idxd_setup_engines(struct idxd_device *idxd) { struct idxd_engine *engine; @@ -251,6 +312,7 @@ static int idxd_setup_engines(struct idxd_device *idxd) rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); if (rc < 0) { put_device(conf_dev); + kfree(engine); goto err; } @@ -264,10 +326,26 @@ static int idxd_setup_engines(struct idxd_device *idxd) engine = idxd->engines[i]; conf_dev = engine_confdev(engine); put_device(conf_dev); + kfree(engine); } + kfree(idxd->engines); + return rc; } +static void idxd_clean_groups(struct idxd_device *idxd) +{ + struct idxd_group *group; + int i; + + for (i = 0; i < idxd->max_groups; i++) { + group = idxd->groups[i]; + put_device(group_confdev(group)); + kfree(group); + } + kfree(idxd->groups); +} + static int idxd_setup_groups(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -298,6 +376,7 @@ static int idxd_setup_groups(struct idxd_device *idxd) rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); if (rc < 0) { put_device(conf_dev); + kfree(group); goto err; } @@ -322,27 +401,27 @@ static int idxd_setup_groups(struct idxd_device *idxd) while (--i >= 0) { group = idxd->groups[i]; put_device(group_confdev(group)); + kfree(group); } + kfree(idxd->groups); + return rc; } static void idxd_cleanup_internals(struct idxd_device *idxd) { - int i; - - for (i = 0; i < idxd->max_groups; i++) - put_device(group_confdev(idxd->groups[i])); - for (i = 0; i < idxd->max_engines; i++) - put_device(engine_confdev(idxd->engines[i])); - for (i = 0; i < idxd->max_wqs; i++) - put_device(wq_confdev(idxd->wqs[i])); + idxd_clean_groups(idxd); + idxd_clean_engines(idxd); + idxd_clean_wqs(idxd); destroy_workqueue(idxd->wq); } static int idxd_init_evl(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; + unsigned int evl_cache_size; struct idxd_evl *evl; + const char *idxd_name; if (idxd->hw.gen_cap.evl_support == 0) return 0; @@ -351,12 +430,19 @@ static int idxd_init_evl(struct idxd_device *idxd) if (!evl) return -ENOMEM; - spin_lock_init(&evl->lock); + mutex_init(&evl->lock); evl->size = IDXD_EVL_SIZE_MIN; - idxd->evl_cache = kmem_cache_create(dev_name(idxd_confdev(idxd)), - sizeof(struct idxd_evl_fault) + evl_ent_size(idxd), - 0, 0, NULL); + idxd_name = dev_name(idxd_confdev(idxd)); + evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); + /* + * Since completion record in evl_cache will be copied to user + * when handling completion record page fault, need to create + * the cache suitable for user copy. + */ + idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, + 0, 0, 0, evl_cache_size, + NULL); if (!idxd->evl_cache) { kfree(evl); return -ENOMEM; @@ -369,7 +455,7 @@ static int idxd_init_evl(struct idxd_device *idxd) static int idxd_setup_internals(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; - int rc, i; + int rc; init_waitqueue_head(&idxd->cmd_waitq); @@ -400,14 +486,11 @@ static int idxd_setup_internals(struct idxd_device *idxd) err_evl: destroy_workqueue(idxd->wq); err_wkq_create: - for (i = 0; i < idxd->max_groups; i++) - put_device(group_confdev(idxd->groups[i])); + idxd_clean_groups(idxd); err_group: - for (i = 0; i < idxd->max_engines; i++) - put_device(engine_confdev(idxd->engines[i])); + idxd_clean_engines(idxd); err_engine: - for (i = 0; i < idxd->max_wqs; i++) - put_device(wq_confdev(idxd->wqs[i])); + idxd_clean_wqs(idxd); err_wqs: return rc; } @@ -507,6 +590,17 @@ static void idxd_read_caps(struct idxd_device *idxd) idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); } +static void idxd_free(struct idxd_device *idxd) +{ + if (!idxd) + return; + + put_device(idxd_confdev(idxd)); + bitmap_free(idxd->opcap_bmap); + ida_free(&idxd_ida, idxd->id); + kfree(idxd); +} + static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) { struct device *dev = &pdev->dev; @@ -524,61 +618,91 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); if (idxd->id < 0) - return NULL; + goto err_ida; idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); - if (!idxd->opcap_bmap) { - ida_free(&idxd_ida, idxd->id); - return NULL; - } + if (!idxd->opcap_bmap) + goto err_opcap; device_initialize(conf_dev); conf_dev->parent = dev; conf_dev->bus = &dsa_bus_type; conf_dev->type = idxd->data->dev_type; rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); - if (rc < 0) { - put_device(conf_dev); - return NULL; - } + if (rc < 0) + goto err_name; spin_lock_init(&idxd->dev_lock); spin_lock_init(&idxd->cmd_lock); return idxd; -} - -static int idxd_enable_system_pasid(struct idxd_device *idxd) -{ - return -EOPNOTSUPP; -} -static void idxd_disable_system_pasid(struct idxd_device *idxd) -{ +err_name: + put_device(conf_dev); + bitmap_free(idxd->opcap_bmap); +err_opcap: + ida_free(&idxd_ida, idxd->id); +err_ida: + kfree(idxd); - iommu_sva_unbind_device(idxd->sva); - idxd->sva = NULL; + return NULL; } -static int idxd_enable_sva(struct pci_dev *pdev) +static int idxd_enable_system_pasid(struct idxd_device *idxd) { + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iommu_domain *domain; + ioasid_t pasid; int ret; - ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); - if (ret) + /* + * Attach a global PASID to the DMA domain so that we can use ENQCMDS + * to submit work on buffers mapped by DMA API. + */ + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -EPERM; + + pasid = iommu_alloc_global_pasid(dev); + if (pasid == IOMMU_PASID_INVALID) + return -ENOSPC; + + /* + * DMA domain is owned by the driver, it should support all valid + * types such as DMA-FQ, identity, etc. + */ + ret = iommu_attach_device_pasid(domain, dev, pasid, NULL); + if (ret) { + dev_err(dev, "failed to attach device pasid %d, domain type %d", + pasid, domain->type); + iommu_free_global_pasid(pasid); return ret; + } - ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); - if (ret) - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); + /* Since we set user privilege for kernel DMA, enable completion IRQ */ + idxd_set_user_intr(idxd, 1); + idxd->pasid = pasid; return ret; } -static void idxd_disable_sva(struct pci_dev *pdev) +static void idxd_disable_system_pasid(struct idxd_device *idxd) { - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iommu_domain *domain; + + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return; + + iommu_detach_device_pasid(domain, dev, idxd->pasid); + iommu_free_global_pasid(idxd->pasid); + + idxd_set_user_intr(idxd, 0); + idxd->sva = NULL; + idxd->pasid = IOMMU_PASID_INVALID; } static int idxd_probe(struct idxd_device *idxd) @@ -595,16 +719,13 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD reset complete\n"); if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { - if (idxd_enable_sva(pdev)) { - dev_warn(dev, "Unable to turn on user SVA feature.\n"); - } else { - set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); + set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); - if (idxd_enable_system_pasid(idxd)) - dev_warn(dev, "No in-kernel DMA with PASID.\n"); - else - set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); - } + rc = idxd_enable_system_pasid(idxd); + if (rc) + dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); + else + set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); } else if (!sva) { dev_warn(dev, "User forced SVA off via module param.\n"); } @@ -642,8 +763,6 @@ static int idxd_probe(struct idxd_device *idxd) err: if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - if (device_user_pasid_enabled(idxd)) - idxd_disable_sva(pdev); return rc; } @@ -654,64 +773,465 @@ static void idxd_cleanup(struct idxd_device *idxd) idxd_cleanup_internals(idxd); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - if (device_user_pasid_enabled(idxd)) - idxd_disable_sva(idxd->pdev); } -static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +/* + * Attach IDXD device to IDXD driver. + */ +static int idxd_bind(struct device_driver *drv, const char *buf) { - struct device *dev = &pdev->dev; - struct idxd_device *idxd; - struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; + const struct bus_type *bus = drv->bus; + struct device *dev; + int err = -ENODEV; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev) + err = device_driver_attach(drv, dev); + + put_device(dev); + + return err; +} + +/* + * Detach IDXD device from driver. + */ +static void idxd_unbind(struct device_driver *drv, const char *buf) +{ + const struct bus_type *bus = drv->bus; + struct device *dev; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev && dev->driver == drv) + device_release_driver(dev); + + put_device(dev); +} + +#define idxd_free_saved_configs(saved_configs, count) \ + do { \ + int i; \ + \ + for (i = 0; i < (count); i++) \ + kfree(saved_configs[i]); \ + } while (0) + +static void idxd_free_saved(struct idxd_group **saved_groups, + struct idxd_engine **saved_engines, + struct idxd_wq **saved_wqs, + struct idxd_device *idxd) +{ + if (saved_groups) + idxd_free_saved_configs(saved_groups, idxd->max_groups); + if (saved_engines) + idxd_free_saved_configs(saved_engines, idxd->max_engines); + if (saved_wqs) + idxd_free_saved_configs(saved_wqs, idxd->max_wqs); +} + +/* + * Save IDXD device configurations including engines, groups, wqs etc. + * The saved configurations can be restored when needed. + */ +static int idxd_device_config_save(struct idxd_device *idxd, + struct idxd_saved_states *idxd_saved) +{ + struct device *dev = &idxd->pdev->dev; + int i; + + memcpy(&idxd_saved->saved_idxd, idxd, sizeof(*idxd)); + + if (idxd->evl) { + memcpy(&idxd_saved->saved_evl, idxd->evl, + sizeof(struct idxd_evl)); + } + + struct idxd_group **saved_groups __free(kfree) = + kcalloc_node(idxd->max_groups, + sizeof(struct idxd_group *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_groups) + return -ENOMEM; + + for (i = 0; i < idxd->max_groups; i++) { + struct idxd_group *saved_group __free(kfree) = + kzalloc_node(sizeof(*saved_group), GFP_KERNEL, + dev_to_node(dev)); + + if (!saved_group) { + /* Free saved groups */ + idxd_free_saved(saved_groups, NULL, NULL, idxd); + + return -ENOMEM; + } + + memcpy(saved_group, idxd->groups[i], sizeof(*saved_group)); + saved_groups[i] = no_free_ptr(saved_group); + } + + struct idxd_engine **saved_engines = + kcalloc_node(idxd->max_engines, + sizeof(struct idxd_engine *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_engines) { + /* Free saved groups */ + idxd_free_saved(saved_groups, NULL, NULL, idxd); + + return -ENOMEM; + } + for (i = 0; i < idxd->max_engines; i++) { + struct idxd_engine *saved_engine __free(kfree) = + kzalloc_node(sizeof(*saved_engine), GFP_KERNEL, + dev_to_node(dev)); + if (!saved_engine) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, + idxd); + + return -ENOMEM; + } + + memcpy(saved_engine, idxd->engines[i], sizeof(*saved_engine)); + saved_engines[i] = no_free_ptr(saved_engine); + } + + unsigned long *saved_wq_enable_map __free(bitmap) = + bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, + dev_to_node(dev)); + if (!saved_wq_enable_map) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, idxd); + + return -ENOMEM; + } + + bitmap_copy(saved_wq_enable_map, idxd->wq_enable_map, idxd->max_wqs); + + struct idxd_wq **saved_wqs __free(kfree) = + kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_wqs) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, idxd); + + return -ENOMEM; + } + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *saved_wq __free(kfree) = + kzalloc_node(sizeof(*saved_wq), GFP_KERNEL, + dev_to_node(dev)); + struct idxd_wq *wq; + + if (!saved_wq) { + /* Free saved groups, engines, and wqs */ + idxd_free_saved(saved_groups, saved_engines, saved_wqs, + idxd); + + return -ENOMEM; + } + + if (!test_bit(i, saved_wq_enable_map)) + continue; + + wq = idxd->wqs[i]; + mutex_lock(&wq->wq_lock); + memcpy(saved_wq, wq, sizeof(*saved_wq)); + saved_wqs[i] = no_free_ptr(saved_wq); + mutex_unlock(&wq->wq_lock); + } + + /* Save configurations */ + idxd_saved->saved_groups = no_free_ptr(saved_groups); + idxd_saved->saved_engines = no_free_ptr(saved_engines); + idxd_saved->saved_wq_enable_map = no_free_ptr(saved_wq_enable_map); + idxd_saved->saved_wqs = no_free_ptr(saved_wqs); + + return 0; +} + +/* + * Restore IDXD device configurations including engines, groups, wqs etc + * that were saved before. + */ +static void idxd_device_config_restore(struct idxd_device *idxd, + struct idxd_saved_states *idxd_saved) +{ + struct idxd_evl *saved_evl = &idxd_saved->saved_evl; + int i; + + idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; + + idxd->evl->size = saved_evl->size; + + for (i = 0; i < idxd->max_groups; i++) { + struct idxd_group *saved_group, *group; + + saved_group = idxd_saved->saved_groups[i]; + group = idxd->groups[i]; + + group->rdbufs_allowed = saved_group->rdbufs_allowed; + group->rdbufs_reserved = saved_group->rdbufs_reserved; + group->tc_a = saved_group->tc_a; + group->tc_b = saved_group->tc_b; + group->use_rdbuf_limit = saved_group->use_rdbuf_limit; + + kfree(saved_group); + } + kfree(idxd_saved->saved_groups); + + for (i = 0; i < idxd->max_engines; i++) { + struct idxd_engine *saved_engine, *engine; + + saved_engine = idxd_saved->saved_engines[i]; + engine = idxd->engines[i]; + + engine->group = saved_engine->group; + + kfree(saved_engine); + } + kfree(idxd_saved->saved_engines); + + bitmap_copy(idxd->wq_enable_map, idxd_saved->saved_wq_enable_map, + idxd->max_wqs); + bitmap_free(idxd_saved->saved_wq_enable_map); + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *saved_wq, *wq; + size_t len; + + if (!test_bit(i, idxd->wq_enable_map)) + continue; + + saved_wq = idxd_saved->saved_wqs[i]; + wq = idxd->wqs[i]; + + mutex_lock(&wq->wq_lock); + + wq->group = saved_wq->group; + wq->flags = saved_wq->flags; + wq->threshold = saved_wq->threshold; + wq->size = saved_wq->size; + wq->priority = saved_wq->priority; + wq->type = saved_wq->type; + len = strlen(saved_wq->name) + 1; + strscpy(wq->name, saved_wq->name, len); + wq->max_xfer_bytes = saved_wq->max_xfer_bytes; + wq->max_batch_size = saved_wq->max_batch_size; + wq->enqcmds_retries = saved_wq->enqcmds_retries; + wq->descs = saved_wq->descs; + wq->idxd_chan = saved_wq->idxd_chan; + len = strlen(saved_wq->driver_name) + 1; + strscpy(wq->driver_name, saved_wq->driver_name, len); + + mutex_unlock(&wq->wq_lock); + + kfree(saved_wq); + } + + kfree(idxd_saved->saved_wqs); +} + +static void idxd_reset_prepare(struct pci_dev *pdev) +{ + struct idxd_device *idxd = pci_get_drvdata(pdev); + struct device *dev = &idxd->pdev->dev; + const char *idxd_name; int rc; - rc = pci_enable_device(pdev); - if (rc) - return rc; + idxd_name = dev_name(idxd_confdev(idxd)); - dev_dbg(dev, "Alloc IDXD context\n"); - idxd = idxd_alloc(pdev, data); - if (!idxd) { - rc = -ENOMEM; - goto err_idxd_alloc; + struct idxd_saved_states *idxd_saved __free(kfree) = + kzalloc_node(sizeof(*idxd_saved), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!idxd_saved) { + dev_err(dev, "HALT: no memory\n"); + + return; } - dev_dbg(dev, "Mapping BARs\n"); - idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); - if (!idxd->reg_base) { - rc = -ENOMEM; - goto err_iomap; + /* Save IDXD configurations. */ + rc = idxd_device_config_save(idxd, idxd_saved); + if (rc < 0) { + dev_err(dev, "HALT: cannot save %s configs\n", idxd_name); + + return; + } + + idxd->idxd_saved = no_free_ptr(idxd_saved); + + /* Save PCI device state. */ + pci_save_state(idxd->pdev); +} + +static void idxd_reset_done(struct pci_dev *pdev) +{ + struct idxd_device *idxd = pci_get_drvdata(pdev); + const char *idxd_name; + struct device *dev; + int rc, i; + + if (!idxd->idxd_saved) + return; + + dev = &idxd->pdev->dev; + idxd_name = dev_name(idxd_confdev(idxd)); + + /* Restore PCI device state. */ + pci_restore_state(idxd->pdev); + + /* Unbind idxd device from driver. */ + idxd_unbind(&idxd_drv.drv, idxd_name); + + /* + * Probe PCI device without allocating or changing + * idxd software data which keeps the same as before FLR. + */ + idxd_pci_probe_alloc(idxd, NULL, NULL); + + /* Restore IDXD configurations. */ + idxd_device_config_restore(idxd, idxd->idxd_saved); + + /* Re-configure IDXD device if allowed. */ + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + rc = idxd_device_config(idxd); + if (rc < 0) { + dev_err(dev, "HALT: %s config fails\n", idxd_name); + goto out; + } } - dev_dbg(dev, "Set DMA masks\n"); - rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + /* Bind IDXD device to driver. */ + rc = idxd_bind(&idxd_drv.drv, idxd_name); + if (rc < 0) { + dev_err(dev, "HALT: binding %s to driver fails\n", idxd_name); + goto out; + } + + /* Bind enabled wq in the IDXD device to driver. */ + for (i = 0; i < idxd->max_wqs; i++) { + if (test_bit(i, idxd->wq_enable_map)) { + struct idxd_wq *wq = idxd->wqs[i]; + char wq_name[32]; + + wq->state = IDXD_WQ_DISABLED; + sprintf(wq_name, "wq%d.%d", idxd->id, wq->id); + /* + * Bind to user driver depending on wq type. + * + * Currently only support user type WQ. Will support + * kernel type WQ in the future. + */ + if (wq->type == IDXD_WQT_USER) + rc = idxd_bind(&idxd_user_drv.drv, wq_name); + else + rc = -EINVAL; + if (rc < 0) { + clear_bit(i, idxd->wq_enable_map); + dev_err(dev, + "HALT: unable to re-enable wq %s\n", + dev_name(wq_confdev(wq))); + } + } + } +out: + kfree(idxd->idxd_saved); +} + +static const struct pci_error_handlers idxd_error_handler = { + .reset_prepare = idxd_reset_prepare, + .reset_done = idxd_reset_done, +}; + +/* + * Probe idxd PCI device. + * If idxd is not given, need to allocate idxd and set up its data. + * + * If idxd is given, idxd was allocated and setup already. Just need to + * configure device without re-allocating and re-configuring idxd data. + * This is useful for recovering from FLR. + */ +int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, + const struct pci_device_id *id) +{ + bool alloc_idxd = idxd ? false : true; + struct idxd_driver_data *data; + struct device *dev; + int rc; + + pdev = idxd ? idxd->pdev : pdev; + dev = &pdev->dev; + data = id ? (struct idxd_driver_data *)id->driver_data : NULL; + rc = pci_enable_device(pdev); if (rc) - goto err; + return rc; + + if (alloc_idxd) { + dev_dbg(dev, "Alloc IDXD context\n"); + idxd = idxd_alloc(pdev, data); + if (!idxd) { + rc = -ENOMEM; + goto err_idxd_alloc; + } + + dev_dbg(dev, "Mapping BARs\n"); + idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); + if (!idxd->reg_base) { + rc = -ENOMEM; + goto err_iomap; + } + + dev_dbg(dev, "Set DMA masks\n"); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + goto err; + } dev_dbg(dev, "Set PCI master\n"); pci_set_master(pdev); pci_set_drvdata(pdev, idxd); - idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); - rc = idxd_probe(idxd); - if (rc) { - dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); - goto err; - } + if (alloc_idxd) { + idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); + rc = idxd_probe(idxd); + if (rc) { + dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); + goto err; + } + + if (data->load_device_defaults) { + rc = data->load_device_defaults(idxd); + if (rc) + dev_warn(dev, "IDXD loading device defaults failed\n"); + } + + rc = idxd_register_devices(idxd); + if (rc) { + dev_err(dev, "IDXD sysfs setup failed\n"); + goto err_dev_register; + } - rc = idxd_register_devices(idxd); - if (rc) { - dev_err(dev, "IDXD sysfs setup failed\n"); - goto err_dev_register; + rc = idxd_device_init_debugfs(idxd); + if (rc) + dev_warn(dev, "IDXD debugfs failed to setup\n"); } - rc = idxd_device_init_debugfs(idxd); - if (rc) - dev_warn(dev, "IDXD debugfs failed to setup\n"); + if (!alloc_idxd) { + /* Release interrupts in the IDXD device. */ + idxd_cleanup_interrupts(idxd); + + /* Re-enable interrupts in the IDXD device. */ + rc = idxd_setup_interrupts(idxd); + if (rc) + dev_warn(dev, "IDXD interrupts failed to setup\n"); + } dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", idxd->hw.version); + if (data) + idxd->user_submission_safe = data->user_submission_safe; + return 0; err_dev_register: @@ -719,12 +1239,17 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err: pci_iounmap(pdev, idxd->reg_base); err_iomap: - put_device(idxd_confdev(idxd)); + idxd_free(idxd); err_idxd_alloc: pci_disable_device(pdev); return rc; } +static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + return idxd_pci_probe_alloc(NULL, pdev, id); +} + void idxd_wqs_quiesce(struct idxd_device *idxd) { struct idxd_wq *wq; @@ -756,7 +1281,6 @@ static void idxd_shutdown(struct pci_dev *pdev) static void idxd_remove(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); - struct idxd_irq_entry *irq_entry; idxd_unregister_devices(idxd); /* @@ -769,20 +1293,14 @@ static void idxd_remove(struct pci_dev *pdev) get_device(idxd_confdev(idxd)); device_unregister(idxd_confdev(idxd)); idxd_shutdown(pdev); + idxd_device_remove_debugfs(idxd); + perfmon_pmu_remove(idxd); + idxd_cleanup_interrupts(idxd); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - idxd_device_remove_debugfs(idxd); - - irq_entry = idxd_get_ie(idxd, 0); - free_irq(irq_entry->vector, irq_entry); - pci_free_irq_vectors(pdev); pci_iounmap(pdev, idxd->reg_base); - if (device_user_pasid_enabled(idxd)) - idxd_disable_sva(pdev); - pci_disable_device(pdev); - destroy_workqueue(idxd->wq); - perfmon_pmu_remove(idxd); put_device(idxd_confdev(idxd)); + pci_disable_device(pdev); } static struct pci_driver idxd_pci_driver = { @@ -791,6 +1309,7 @@ static struct pci_driver idxd_pci_driver = { .probe = idxd_pci_probe, .remove = idxd_remove, .shutdown = idxd_shutdown, + .err_handler = &idxd_error_handler, }; static int __init idxd_init_module(void) @@ -811,8 +1330,6 @@ static int __init idxd_init_module(void) else support_enqcmd = true; - perfmon_init(); - err = idxd_driver_register(&idxd_drv); if (err < 0) goto err_idxd_driver_register; @@ -861,7 +1378,6 @@ static void __exit idxd_exit_module(void) idxd_driver_unregister(&idxd_drv); pci_unregister_driver(&idxd_pci_driver); idxd_cdev_remove(); - perfmon_exit(); idxd_remove_debugfs(); } module_exit(idxd_exit_module); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index b501320a9c7a..1107db3ce0a3 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -123,7 +123,7 @@ static void idxd_abort_invalid_int_handle_descs(struct idxd_irq_entry *ie) list_for_each_entry_safe(d, t, &flist, list) { list_del(&d->list); - idxd_dma_complete_txd(d, IDXD_COMPLETE_ABORT, true); + idxd_desc_complete(d, IDXD_COMPLETE_ABORT, true); } } @@ -363,13 +363,13 @@ static void process_evl_entries(struct idxd_device *idxd) evl_status.bits = 0; evl_status.int_pending = 1; - spin_lock(&evl->lock); + mutex_lock(&evl->lock); /* Clear interrupt pending bit */ iowrite32(evl_status.bits_upper32, idxd->reg_base + IDXD_EVLSTATUS_OFFSET + sizeof(u32)); - h = evl->head; evl_status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); t = evl_status.tail; + h = evl_status.head; size = idxd->evl->size; while (h != t) { @@ -378,10 +378,61 @@ static void process_evl_entries(struct idxd_device *idxd) h = (h + 1) % size; } - evl->head = h; evl_status.head = h; iowrite32(evl_status.bits_lower32, idxd->reg_base + IDXD_EVLSTATUS_OFFSET); - spin_unlock(&evl->lock); + mutex_unlock(&evl->lock); +} + +static void idxd_device_flr(struct work_struct *work) +{ + struct idxd_device *idxd = container_of(work, struct idxd_device, work); + int rc; + + /* + * IDXD device requires a Function Level Reset (FLR). + * pci_reset_function() will reset the device with FLR. + */ + rc = pci_reset_function(idxd->pdev); + if (rc) + dev_err(&idxd->pdev->dev, "FLR failed\n"); +} + +static irqreturn_t idxd_halt(struct idxd_device *idxd) +{ + union gensts_reg gensts; + + gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); + if (gensts.state == IDXD_DEVICE_STATE_HALT) { + idxd->state = IDXD_DEV_HALTED; + if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { + /* + * If we need a software reset, we will throw the work + * on a system workqueue in order to allow interrupts + * for the device command completions. + */ + INIT_WORK(&idxd->work, idxd_device_reinit); + queue_work(idxd->wq, &idxd->work); + } else if (gensts.reset_type == IDXD_DEVICE_RESET_FLR) { + idxd->state = IDXD_DEV_HALTED; + idxd_mask_error_interrupts(idxd); + dev_dbg(&idxd->pdev->dev, + "idxd halted, doing FLR. After FLR, configs are restored\n"); + INIT_WORK(&idxd->work, idxd_device_flr); + queue_work(idxd->wq, &idxd->work); + + } else { + idxd->state = IDXD_DEV_HALTED; + idxd_wqs_quiesce(idxd); + idxd_wqs_unmap_portal(idxd); + idxd_device_clear_state(idxd); + dev_err(&idxd->pdev->dev, + "idxd halted, need system reset"); + + return -ENXIO; + } + } + + return IRQ_HANDLED; } irqreturn_t idxd_misc_thread(int vec, void *data) @@ -389,10 +440,8 @@ irqreturn_t idxd_misc_thread(int vec, void *data) struct idxd_irq_entry *irq_entry = data; struct idxd_device *idxd = ie_to_idxd(irq_entry); struct device *dev = &idxd->pdev->dev; - union gensts_reg gensts; u32 val = 0; int i; - bool err = false; u32 cause; cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); @@ -402,7 +451,7 @@ irqreturn_t idxd_misc_thread(int vec, void *data) iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); if (cause & IDXD_INTC_HALT_STATE) - goto halt; + return idxd_halt(idxd); if (cause & IDXD_INTC_ERR) { spin_lock(&idxd->dev_lock); @@ -434,9 +483,8 @@ irqreturn_t idxd_misc_thread(int vec, void *data) val |= IDXD_INTC_ERR; for (i = 0; i < 4; i++) - dev_warn(dev, "err[%d]: %#16.16llx\n", - i, idxd->sw_err.bits[i]); - err = true; + dev_warn_ratelimited(dev, "err[%d]: %#16.16llx\n", + i, idxd->sw_err.bits[i]); } if (cause & IDXD_INTC_INT_HANDLE_REVOKED) { @@ -481,34 +529,6 @@ irqreturn_t idxd_misc_thread(int vec, void *data) dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", val); - if (!err) - goto out; - -halt: - gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); - if (gensts.state == IDXD_DEVICE_STATE_HALT) { - idxd->state = IDXD_DEV_HALTED; - if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { - /* - * If we need a software reset, we will throw the work - * on a system workqueue in order to allow interrupts - * for the device command completions. - */ - INIT_WORK(&idxd->work, idxd_device_reinit); - queue_work(idxd->wq, &idxd->work); - } else { - idxd->state = IDXD_DEV_HALTED; - idxd_wqs_quiesce(idxd); - idxd_wqs_unmap_portal(idxd); - idxd_device_clear_state(idxd); - dev_err(&idxd->pdev->dev, - "idxd halted, need %s.\n", - gensts.reset_type == IDXD_DEVICE_RESET_FLR ? - "FLR" : "system reset"); - } - } - -out: return IRQ_HANDLED; } @@ -534,7 +554,7 @@ static void idxd_int_handle_resubmit_work(struct work_struct *work) */ if (rc != -EAGAIN) { desc->completion->status = IDXD_COMP_DESC_ABORT; - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, false); + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, false); } idxd_free_desc(wq, desc); } @@ -575,11 +595,11 @@ static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry) * and 0xff, which DSA_COMP_STATUS_MASK can mask out. */ if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, true); + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true); continue; } - idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL, true); + idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true); } else { spin_lock(&irq_entry->list_lock); list_add_tail(&desc->list, @@ -612,17 +632,19 @@ static void irq_process_work_list(struct idxd_irq_entry *irq_entry) spin_unlock(&irq_entry->list_lock); - list_for_each_entry(desc, &flist, list) { + list_for_each_entry_safe(desc, n, &flist, list) { /* * Check against the original status as ABORT is software defined * and 0xff, which DSA_COMP_STATUS_MASK can mask out. */ + list_del(&desc->list); + if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT, true); + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true); continue; } - idxd_dma_complete_txd(desc, IDXD_COMPLETE_NORMAL, true); + idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true); } } diff --git a/drivers/dma/idxd/perfmon.c b/drivers/dma/idxd/perfmon.c index d73004f47cf4..4b6af2f15d8a 100644 --- a/drivers/dma/idxd/perfmon.c +++ b/drivers/dma/idxd/perfmon.c @@ -6,29 +6,6 @@ #include "idxd.h" #include "perfmon.h" -static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, - char *buf); - -static cpumask_t perfmon_dsa_cpu_mask; -static bool cpuhp_set_up; -static enum cpuhp_state cpuhp_slot; - -/* - * perf userspace reads this attribute to determine which cpus to open - * counters on. It's connected to perfmon_dsa_cpu_mask, which is - * maintained by the cpu hotplug handlers. - */ -static DEVICE_ATTR_RO(cpumask); - -static struct attribute *perfmon_cpumask_attrs[] = { - &dev_attr_cpumask.attr, - NULL, -}; - -static struct attribute_group cpumask_attr_group = { - .attrs = perfmon_cpumask_attrs, -}; - /* * These attributes specify the bits in the config word that the perf * syscall uses to pass the event ids and categories to perfmon. @@ -67,16 +44,9 @@ static struct attribute_group perfmon_format_attr_group = { static const struct attribute_group *perfmon_attr_groups[] = { &perfmon_format_attr_group, - &cpumask_attr_group, NULL, }; -static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - return cpumap_print_to_pagebuf(true, buf, &perfmon_dsa_cpu_mask); -} - static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event) { return &idxd_pmu->pmu == event->pmu; @@ -217,7 +187,6 @@ static int perfmon_pmu_event_init(struct perf_event *event) return -EINVAL; event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd)); - event->cpu = idxd->idxd_pmu->cpu; event->hw.config = event->attr.config; if (event->group_leader != event) @@ -245,12 +214,11 @@ static void perfmon_pmu_event_update(struct perf_event *event) int shift = 64 - idxd->idxd_pmu->counter_width; struct hw_perf_event *hwc = &event->hw; + prev_raw_count = local64_read(&hwc->prev_count); do { - prev_raw_count = local64_read(&hwc->prev_count); new_raw_count = perfmon_pmu_read_counter(event); - } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, - new_raw_count) != prev_raw_count); - + } while (!local64_try_cmpxchg(&hwc->prev_count, + &prev_raw_count, new_raw_count)); n = (new_raw_count << shift); p = (prev_raw_count << shift); @@ -481,14 +449,15 @@ static void idxd_pmu_init(struct idxd_pmu *idxd_pmu) idxd_pmu->pmu.attr_groups = perfmon_attr_groups; idxd_pmu->pmu.task_ctx_nr = perf_invalid_context; idxd_pmu->pmu.event_init = perfmon_pmu_event_init; - idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable, - idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable, + idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable; + idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable; idxd_pmu->pmu.add = perfmon_pmu_event_add; idxd_pmu->pmu.del = perfmon_pmu_event_del; idxd_pmu->pmu.start = perfmon_pmu_event_start; idxd_pmu->pmu.stop = perfmon_pmu_event_stop; idxd_pmu->pmu.read = perfmon_pmu_event_update; idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; + idxd_pmu->pmu.scope = PERF_PMU_SCOPE_SYS_WIDE; idxd_pmu->pmu.module = THIS_MODULE; } @@ -497,50 +466,11 @@ void perfmon_pmu_remove(struct idxd_device *idxd) if (!idxd->idxd_pmu) return; - cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node); perf_pmu_unregister(&idxd->idxd_pmu->pmu); kfree(idxd->idxd_pmu); idxd->idxd_pmu = NULL; } -static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node) -{ - struct idxd_pmu *idxd_pmu; - - idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); - - /* select the first online CPU as the designated reader */ - if (cpumask_empty(&perfmon_dsa_cpu_mask)) { - cpumask_set_cpu(cpu, &perfmon_dsa_cpu_mask); - idxd_pmu->cpu = cpu; - } - - return 0; -} - -static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node) -{ - struct idxd_pmu *idxd_pmu; - unsigned int target; - - idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); - - if (!cpumask_test_and_clear_cpu(cpu, &perfmon_dsa_cpu_mask)) - return 0; - - target = cpumask_any_but(cpu_online_mask, cpu); - - /* migrate events if there is a valid target */ - if (target < nr_cpu_ids) - cpumask_set_cpu(target, &perfmon_dsa_cpu_mask); - else - target = -1; - - perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target); - - return 0; -} - int perfmon_pmu_init(struct idxd_device *idxd) { union idxd_perfcap perfcap; @@ -548,12 +478,6 @@ int perfmon_pmu_init(struct idxd_device *idxd) int rc = -ENODEV; /* - * perfmon module initialization failed, nothing to do - */ - if (!cpuhp_set_up) - return -ENODEV; - - /* * If perfmon_offset or num_counters is 0, it means perfmon is * not supported on this hardware. */ @@ -628,11 +552,6 @@ int perfmon_pmu_init(struct idxd_device *idxd) if (rc) goto free; - rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node); - if (rc) { - perf_pmu_unregister(&idxd->idxd_pmu->pmu); - goto free; - } out: return rc; free: @@ -641,22 +560,3 @@ free: goto out; } - -void __init perfmon_init(void) -{ - int rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, - "driver/dma/idxd/perf:online", - perf_event_cpu_online, - perf_event_cpu_offline); - if (WARN_ON(rc < 0)) - return; - - cpuhp_slot = rc; - cpuhp_set_up = true; -} - -void __exit perfmon_exit(void) -{ - if (cpuhp_set_up) - cpuhp_remove_multi_state(cpuhp_slot); -} diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index 7b54a3939ea1..8dc2e8bca779 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -3,11 +3,18 @@ #ifndef _IDXD_REGISTERS_H_ #define _IDXD_REGISTERS_H_ +#ifdef __KERNEL__ #include <uapi/linux/idxd.h> +#else +#include <linux/idxd.h> +#endif /* PCI Config */ -#define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25 -#define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe +#define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb +#define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 +#define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 +#define PCI_DEVICE_ID_INTEL_IAA_PTL 0xb02d +#define PCI_DEVICE_ID_INTEL_IAA_WCL 0xfd2d #define DEVICE_VERSION_1 0x100 #define DEVICE_VERSION_2 0x200 @@ -43,7 +50,7 @@ union gen_cap_reg { u64 rsvd3:32; }; u64 bits; -} __packed; +}; #define IDXD_GENCAP_OFFSET 0x10 union wq_cap_reg { @@ -63,7 +70,7 @@ union wq_cap_reg { u64 rsvd4:8; }; u64 bits; -} __packed; +}; #define IDXD_WQCAP_OFFSET 0x20 #define IDXD_WQCFG_MIN 5 @@ -77,7 +84,7 @@ union group_cap_reg { u64 rsvd:45; }; u64 bits; -} __packed; +}; #define IDXD_GRPCAP_OFFSET 0x30 union engine_cap_reg { @@ -86,7 +93,7 @@ union engine_cap_reg { u64 rsvd:56; }; u64 bits; -} __packed; +}; #define IDXD_ENGCAP_OFFSET 0x38 @@ -112,7 +119,7 @@ union offsets_reg { u64 rsvd:48; }; u64 bits[2]; -} __packed; +}; #define IDXD_TABLE_MULT 0x100 @@ -126,7 +133,7 @@ union gencfg_reg { u32 rsvd2:18; }; u32 bits; -} __packed; +}; #define IDXD_GENCTRL_OFFSET 0x88 union genctrl_reg { @@ -137,7 +144,7 @@ union genctrl_reg { u32 rsvd:29; }; u32 bits; -} __packed; +}; #define IDXD_GENSTATS_OFFSET 0x90 union gensts_reg { @@ -147,7 +154,7 @@ union gensts_reg { u32 rsvd:28; }; u32 bits; -} __packed; +}; enum idxd_device_status_state { IDXD_DEVICE_STATE_DISABLED = 0, @@ -181,7 +188,7 @@ union idxd_command_reg { u32 int_req:1; }; u32 bits; -} __packed; +}; enum idxd_cmd { IDXD_CMD_ENABLE_DEVICE = 1, @@ -211,7 +218,7 @@ union cmdsts_reg { u8 active:1; }; u32 bits; -} __packed; +}; #define IDXD_CMDSTS_ACTIVE 0x80000000 #define IDXD_CMDSTS_ERR_MASK 0xff #define IDXD_CMDSTS_RES_SHIFT 8 @@ -282,7 +289,7 @@ union sw_err_reg { u64 rsvd5; }; u64 bits[4]; -} __packed; +}; union iaa_cap_reg { struct { @@ -301,7 +308,7 @@ union iaa_cap_reg { u64 rsvd:52; }; u64 bits; -} __packed; +}; #define IDXD_IAACAP_OFFSET 0x180 @@ -318,7 +325,7 @@ union evlcfg_reg { u64 rsvd2:28; }; u64 bits[2]; -} __packed; +}; #define IDXD_EVL_SIZE_MIN 0x0040 #define IDXD_EVL_SIZE_MAX 0xffff @@ -332,7 +339,7 @@ union msix_perm { u32 pasid:20; }; u32 bits; -} __packed; +}; union group_flags { struct { @@ -350,13 +357,13 @@ union group_flags { u64 rsvd5:26; }; u64 bits; -} __packed; +}; struct grpcfg { u64 wqs[4]; u64 engines; union group_flags flags; -} __packed; +}; union wqcfg { struct { @@ -408,7 +415,7 @@ union wqcfg { u64 op_config[4]; }; u32 bits[16]; -} __packed; +}; #define WQCFG_PASID_IDX 2 #define WQCFG_PRIVL_IDX 2 @@ -440,12 +447,14 @@ union wqcfg { /* * This macro calculates the offset into the GRPCFG register * idxd - struct idxd * - * n - wq id - * ofs - the index of the 32b dword for the config register + * n - group id + * ofs - the index of the 64b qword for the config register * - * The WQCFG register block is divided into groups per each wq. The n index - * allows us to move to the register group that's for that particular wq. - * Each register is 32bits. The ofs gives us the number of register to access. + * The GRPCFG register block is divided into three sub-registers, which + * are GRPWQCFG, GRPENGCFG and GRPFLGCFG. The n index allows us to move + * to the register block that contains the three sub-registers. + * Each register block is 64bits. And the ofs gives us the offset + * within the GRPWQCFG register to access. */ #define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\ (n) * GRPCFG_SIZE + sizeof(u64) * (ofs)) @@ -470,7 +479,7 @@ union idxd_perfcap { u64 rsvd3:8; }; u64 bits; -} __packed; +}; #define IDXD_EVNTCAP_OFFSET 0x80 union idxd_evntcap { @@ -479,7 +488,7 @@ union idxd_evntcap { u64 rsvd:36; }; u64 bits; -} __packed; +}; struct idxd_event { union { @@ -489,7 +498,7 @@ struct idxd_event { }; u32 val; }; -} __packed; +}; #define IDXD_CNTRCAP_OFFSET 0x800 struct idxd_cntrcap { @@ -502,7 +511,7 @@ struct idxd_cntrcap { u32 val; }; struct idxd_event events[]; -} __packed; +}; #define IDXD_PERFRST_OFFSET 0x10 union idxd_perfrst { @@ -512,7 +521,7 @@ union idxd_perfrst { u32 rsvd:30; }; u32 val; -} __packed; +}; #define IDXD_OVFSTATUS_OFFSET 0x30 #define IDXD_PERFFRZ_OFFSET 0x20 @@ -529,7 +538,7 @@ union idxd_cntrcfg { u64 rsvd3:4; }; u64 val; -} __packed; +}; #define IDXD_FLTCFG_OFFSET 0x300 @@ -539,7 +548,7 @@ union idxd_cntrdata { u64 event_count_value; }; u64 val; -} __packed; +}; union event_cfg { struct { @@ -547,7 +556,7 @@ union event_cfg { u64 event_enc:28; }; u64 val; -} __packed; +}; union filter_cfg { struct { @@ -558,7 +567,7 @@ union filter_cfg { u64 eng:8; }; u64 val; -} __packed; +}; #define IDXD_EVLSTATUS_OFFSET 0xf0 @@ -576,7 +585,7 @@ union evl_status_reg { u32 bits_upper32; }; u64 bits; -} __packed; +}; #define IDXD_MAX_BATCH_IDENT 256 @@ -616,17 +625,17 @@ struct __evl_entry { }; u64 fault_addr; u64 rsvd5; -} __packed; +}; struct dsa_evl_entry { struct __evl_entry e; struct dsa_completion_record cr; -} __packed; +}; struct iax_evl_entry { struct __evl_entry e; u64 rsvd[4]; struct iax_completion_record cr; -} __packed; +}; #endif diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index c01db23e3333..6db1c5fcedc5 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -61,6 +61,7 @@ struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype) return __get_desc(wq, idx, cpu); } +EXPORT_SYMBOL_NS_GPL(idxd_alloc_desc, "IDXD"); void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) { @@ -69,6 +70,7 @@ void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) desc->cpu = -1; sbitmap_queue_clear(&wq->sbq, desc->id, cpu); } +EXPORT_SYMBOL_NS_GPL(idxd_free_desc, "IDXD"); static struct idxd_desc *list_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, struct idxd_desc *desc) @@ -125,17 +127,19 @@ static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, spin_unlock(&ie->list_lock); if (found) - idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, false); + idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, false, + NULL, NULL); /* * completing the descriptor will return desc to allocator and * the desc can be acquired by a different process and the * desc->list can be modified. Delete desc from list so the - * list trasversing does not get corrupted by the other process. + * list traversing does not get corrupted by the other process. */ list_for_each_entry_safe(d, t, &flist, list) { list_del_init(&d->list); - idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true); + idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true, + NULL, NULL); } } @@ -183,13 +187,6 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) portal = idxd_wq_portal_addr(wq); /* - * The wmb() flushes writes to coherent DMA data before - * possibly triggering a DMA read. The wmb() is necessary - * even on UP because the recipient is a device. - */ - wmb(); - - /* * Pending the descriptor to the lockless list for the irq_entry * that we designated the descriptor to. */ @@ -199,6 +196,13 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) llist_add(&desc->llnode, &ie->pending_llist); } + /* + * The wmb() flushes writes to coherent DMA data before + * possibly triggering a DMA read. The wmb() is necessary + * even on UP because the recipient is a device. + */ + wmb(); + if (wq_dedicated(wq)) { iosubmit_cmds512(portal, desc->hw, 1); } else { @@ -215,3 +219,4 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) percpu_ref_put(&wq->wq_active); return 0; } +EXPORT_SYMBOL_NS_GPL(idxd_submit_desc, "IDXD"); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 293739ac5596..9f0701021af0 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -91,7 +91,7 @@ static void idxd_conf_engine_release(struct device *dev) kfree(engine); } -struct device_type idxd_engine_device_type = { +const struct device_type idxd_engine_device_type = { .name = "engine", .release = idxd_conf_engine_release, .groups = idxd_engine_attribute_groups, @@ -577,7 +577,7 @@ static void idxd_conf_group_release(struct device *dev) kfree(group); } -struct device_type idxd_group_device_type = { +const struct device_type idxd_group_device_type = { .name = "group", .release = idxd_conf_group_release, .groups = idxd_group_attribute_groups, @@ -948,13 +948,6 @@ static ssize_t wq_name_store(struct device *dev, if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0) return -EINVAL; - /* - * This is temporarily placed here until we have SVM support for - * dmaengine. - */ - if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd)) - return -EOPNOTSUPP; - input = kstrndup(buf, count, GFP_KERNEL); if (!input) return -ENOMEM; @@ -1095,8 +1088,8 @@ static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute if (wq->state != IDXD_WQ_DISABLED) return -EPERM; - if (!idxd->hw.wq_cap.wq_ats_support) - return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; rc = kstrtobool(buf, &ats_dis); if (rc < 0) @@ -1131,8 +1124,8 @@ static ssize_t wq_prs_disable_store(struct device *dev, struct device_attribute if (wq->state != IDXD_WQ_DISABLED) return -EPERM; - if (!idxd->hw.wq_cap.wq_prs_support) - return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; rc = kstrtobool(buf, &prs_dis); if (rc < 0) @@ -1204,12 +1197,37 @@ static ssize_t wq_enqcmds_retries_store(struct device *dev, struct device_attrib static struct device_attribute dev_attr_wq_enqcmds_retries = __ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store); +static ssize_t op_cap_show_common(struct device *dev, char *buf, unsigned long *opcap_bmap) +{ + ssize_t pos; + int i; + + pos = 0; + for (i = IDXD_MAX_OPCAP_BITS/64 - 1; i >= 0; i--) { + unsigned long val = opcap_bmap[i]; + + /* On systems where direct user submissions are not safe, we need to clear out + * the BATCH capability from the capability mask in sysfs since we cannot support + * that command on such systems. Narrow the restriction of operations with the + * BATCH opcode to only DSA version 1 devices. + */ + if (i == DSA_OPCODE_BATCH/64 && !confdev_to_idxd(dev)->user_submission_safe && + confdev_to_idxd(dev)->hw.version == DEVICE_VERSION_1) + clear_bit(DSA_OPCODE_BATCH % 64, &val); + + pos += sysfs_emit_at(buf, pos, "%*pb", 64, &val); + pos += sysfs_emit_at(buf, pos, "%c", i == 0 ? '\n' : ','); + } + + return pos; +} + static ssize_t wq_op_config_show(struct device *dev, struct device_attribute *attr, char *buf) { struct idxd_wq *wq = confdev_to_wq(dev); - return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, wq->opcap_bmap); + return op_cap_show_common(dev, buf, wq->opcap_bmap); } static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask) @@ -1266,6 +1284,39 @@ err: static struct device_attribute dev_attr_wq_op_config = __ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store); +static ssize_t wq_driver_name_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + + return sysfs_emit(buf, "%s\n", wq->driver_name); +} + +static ssize_t wq_driver_name_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + char *input, *pos; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + if (strlen(buf) > DRIVER_NAME_SIZE || strlen(buf) == 0) + return -EINVAL; + + input = kstrndup(buf, count, GFP_KERNEL); + if (!input) + return -ENOMEM; + + pos = strim(input); + memset(wq->driver_name, 0, DRIVER_NAME_SIZE + 1); + sprintf(wq->driver_name, "%s", pos); + kfree(input); + return count; +} + +static struct device_attribute dev_attr_wq_driver_name = + __ATTR(driver_name, 0644, wq_driver_name_show, wq_driver_name_store); + static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_clients.attr, &dev_attr_wq_state.attr, @@ -1285,15 +1336,13 @@ static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_occupancy.attr, &dev_attr_wq_enqcmds_retries.attr, &dev_attr_wq_op_config.attr, + &dev_attr_wq_driver_name.attr, NULL, }; -static bool idxd_wq_attr_op_config_invisible(struct attribute *attr, - struct idxd_device *idxd) -{ - return attr == &dev_attr_wq_op_config.attr && - !idxd->hw.wq_cap.op_config; -} +/* A WQ attr is invisible if the feature is not supported in WQCAP. */ +#define idxd_wq_attr_invisible(name, cap_field, a, idxd) \ + ((a) == &dev_attr_wq_##name.attr && !(idxd)->hw.wq_cap.cap_field) static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr, struct idxd_device *idxd) @@ -1303,13 +1352,6 @@ static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr, idxd->data->type == IDXD_TYPE_IAX; } -static bool idxd_wq_attr_wq_prs_disable_invisible(struct attribute *attr, - struct idxd_device *idxd) -{ - return attr == &dev_attr_wq_prs_disable.attr && - !idxd->hw.wq_cap.wq_prs_support; -} - static umode_t idxd_wq_attr_visible(struct kobject *kobj, struct attribute *attr, int n) { @@ -1317,13 +1359,16 @@ static umode_t idxd_wq_attr_visible(struct kobject *kobj, struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; - if (idxd_wq_attr_op_config_invisible(attr, idxd)) + if (idxd_wq_attr_invisible(op_config, op_config, attr, idxd)) return 0; if (idxd_wq_attr_max_batch_size_invisible(attr, idxd)) return 0; - if (idxd_wq_attr_wq_prs_disable_invisible(attr, idxd)) + if (idxd_wq_attr_invisible(prs_disable, wq_prs_support, attr, idxd)) + return 0; + + if (idxd_wq_attr_invisible(ats_disable, wq_ats_support, attr, idxd)) return 0; return attr->mode; @@ -1349,7 +1394,7 @@ static void idxd_conf_wq_release(struct device *dev) kfree(wq); } -struct device_type idxd_wq_device_type = { +const struct device_type idxd_wq_device_type = { .name = "wq", .release = idxd_conf_wq_release, .groups = idxd_wq_attribute_groups, @@ -1435,7 +1480,7 @@ static ssize_t op_cap_show(struct device *dev, { struct idxd_device *idxd = confdev_to_idxd(dev); - return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, idxd->opcap_bmap); + return op_cap_show_common(dev, buf, idxd->opcap_bmap); } static DEVICE_ATTR_RO(op_cap); @@ -1480,7 +1525,7 @@ static ssize_t pasid_enabled_show(struct device *dev, { struct idxd_device *idxd = confdev_to_idxd(dev); - return sysfs_emit(buf, "%u\n", device_pasid_enabled(idxd)); + return sysfs_emit(buf, "%u\n", device_user_pasid_enabled(idxd)); } static DEVICE_ATTR_RO(pasid_enabled); @@ -1778,13 +1823,13 @@ static void idxd_conf_device_release(struct device *dev) kfree(idxd); } -struct device_type dsa_device_type = { +const struct device_type dsa_device_type = { .name = "dsa", .release = idxd_conf_device_release, .groups = idxd_attribute_groups, }; -struct device_type iax_device_type = { +const struct device_type iax_device_type = { .name = "iax", .release = idxd_conf_device_release, .groups = idxd_attribute_groups, @@ -1936,13 +1981,3 @@ void idxd_unregister_devices(struct idxd_device *idxd) device_unregister(group_confdev(group)); } } - -int idxd_register_bus_type(void) -{ - return bus_register(&dsa_bus_type); -} - -void idxd_unregister_bus_type(void) -{ - bus_unregister(&dsa_bus_type); -} diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c index ad084552640f..fd55bcd060ab 100644 --- a/drivers/dma/img-mdc-dma.c +++ b/drivers/dma/img-mdc-dma.c @@ -17,7 +17,6 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -1018,7 +1017,7 @@ suspend: return ret; } -static int mdc_dma_remove(struct platform_device *pdev) +static void mdc_dma_remove(struct platform_device *pdev) { struct mdc_dma *mdma = platform_get_drvdata(pdev); struct mdc_chan *mchan, *next; @@ -1038,8 +1037,6 @@ static int mdc_dma_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) img_mdc_runtime_suspend(&pdev->dev); - - return 0; } #ifdef CONFIG_PM_SLEEP @@ -1076,7 +1073,7 @@ static struct platform_driver mdc_dma_driver = { .driver = { .name = "img-mdc-dma", .pm = &img_mdc_pm_ops, - .of_match_table = of_match_ptr(mdc_dma_of_match), + .of_match_table = mdc_dma_of_match, }, .probe = mdc_dma_probe, .remove = mdc_dma_remove, diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index f040751690af..ba434657059a 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -17,11 +17,12 @@ #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/platform_device.h> #include <linux/clk.h> #include <linux/dmaengine.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <asm/irq.h> @@ -167,7 +168,6 @@ struct imxdma_channel { enum imx_dma_type { IMX1_DMA, - IMX21_DMA, IMX27_DMA, }; @@ -195,8 +195,6 @@ static const struct of_device_id imx_dma_of_dev_id[] = { { .compatible = "fsl,imx1-dma", .data = (const void *)IMX1_DMA, }, { - .compatible = "fsl,imx21-dma", .data = (const void *)IMX21_DMA, - }, { .compatible = "fsl,imx27-dma", .data = (const void *)IMX27_DMA, }, { /* sentinel */ @@ -326,7 +324,7 @@ static void imxdma_disable_hw(struct imxdma_channel *imxdmac) dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel); if (imxdma_hw_chain(imxdmac)) - del_timer(&imxdmac->watchdog); + timer_delete(&imxdmac->watchdog); local_irq_save(flags); imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) | @@ -339,7 +337,8 @@ static void imxdma_disable_hw(struct imxdma_channel *imxdmac) static void imxdma_watchdog(struct timer_list *t) { - struct imxdma_channel *imxdmac = from_timer(imxdmac, t, watchdog); + struct imxdma_channel *imxdmac = timer_container_of(imxdmac, t, + watchdog); struct imxdma_engine *imxdma = imxdmac->imxdma; int channel = imxdmac->channel; @@ -456,7 +455,7 @@ static void dma_irq_handle_channel(struct imxdma_channel *imxdmac) } if (imxdma_hw_chain(imxdmac)) { - del_timer(&imxdmac->watchdog); + timer_delete(&imxdmac->watchdog); return; } } @@ -945,7 +944,7 @@ static struct dma_async_tx_descriptor *imxdma_prep_dma_interleaved( " src_sgl=%s dst_sgl=%s numf=%zu frame_size=%zu\n", __func__, imxdmac->channel, (unsigned long long)xt->src_start, (unsigned long long) xt->dst_start, - xt->src_sgl ? "true" : "false", xt->dst_sgl ? "true" : "false", + str_true_false(xt->src_sgl), str_true_false(xt->dst_sgl), xt->numf, xt->frame_size); if (list_empty(&imxdmac->ld_free) || @@ -1216,7 +1215,7 @@ static void imxdma_free_irq(struct platform_device *pdev, struct imxdma_engine * } } -static int imxdma_remove(struct platform_device *pdev) +static void imxdma_remove(struct platform_device *pdev) { struct imxdma_engine *imxdma = platform_get_drvdata(pdev); @@ -1229,8 +1228,6 @@ static int imxdma_remove(struct platform_device *pdev) clk_disable_unprepare(imxdma->dma_ipg); clk_disable_unprepare(imxdma->dma_ahb); - - return 0; } static struct platform_driver imxdma_driver = { diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 7a912f90c2a9..ed9e56de5a9b 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -24,6 +24,7 @@ #include <linux/semaphore.h> #include <linux/spinlock.h> #include <linux/device.h> +#include <linux/genalloc.h> #include <linux/dma-mapping.h> #include <linux/firmware.h> #include <linux/slab.h> @@ -31,7 +32,6 @@ #include <linux/dmaengine.h> #include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/workqueue.h> @@ -138,7 +138,11 @@ * 0: Source on AIPS * 12 Destination Bit(DP) 1: Destination on SPBA * 0: Destination on AIPS - * 13-15 --------- MUST BE 0 + * 13 Source FIFO 1: Source is dual FIFO + * 0: Source is single FIFO + * 14 Destination FIFO 1: Destination is dual FIFO + * 0: Destination is single FIFO + * 15 --------- MUST BE 0 * 16-23 Higher WML HWML * 24-27 N Total number of samples after * which Pad adding/Swallowing @@ -169,6 +173,8 @@ #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) #define SDMA_WATERMARK_LEVEL_SP BIT(11) #define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_SD BIT(13) +#define SDMA_WATERMARK_LEVEL_DD BIT(14) #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) #define SDMA_WATERMARK_LEVEL_LWE BIT(28) #define SDMA_WATERMARK_LEVEL_HWE BIT(29) @@ -176,6 +182,7 @@ #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ @@ -233,20 +240,23 @@ struct sdma_script_start_addrs { s32 utra_addr; s32 ram_code_start_addr; /* End of v1 array */ - s32 mcu_2_ssish_addr; + union { s32 v1_end; s32 mcu_2_ssish_addr; }; s32 ssish_2_mcu_addr; s32 hdmi_dma_addr; /* End of v2 array */ - s32 zcanfd_2_mcu_addr; + union { s32 v2_end; s32 zcanfd_2_mcu_addr; }; s32 zqspi_2_mcu_addr; s32 mcu_2_ecspi_addr; s32 mcu_2_sai_addr; s32 sai_2_mcu_addr; s32 uart_2_mcu_rom_addr; s32 uartsh_2_mcu_rom_addr; + s32 i2c_2_mcu_addr; + s32 mcu_2_i2c_addr; /* End of v3 array */ - s32 mcu_2_zqspi_addr; + union { s32 v3_end; s32 mcu_2_zqspi_addr; }; /* End of v4 array */ + s32 v4_end[]; }; /* @@ -422,9 +432,7 @@ struct sdma_desc { * @shp_addr: value for gReg[6] * @per_addr: value for gReg[2] * @status: status of dma channel - * @context_loaded: ensure context is only loaded once * @data: specific sdma interface structure - * @bd_pool: dma_pool for bd * @terminate_worker: used to call back into terminate work function * @terminated: terminated list * @is_ram_script: flag for script in ram @@ -487,8 +495,6 @@ struct sdma_channel { * @num_script_addrs: Number of script addresses in this image * @ram_code_start: offset of SDMA ram image in this firmware image * @ram_code_size: size of SDMA ram image - * @script_addrs: Stores the start address of the SDMA scripts - * (in SDMA memory space) */ struct sdma_firmware_header { u32 magic; @@ -536,6 +542,7 @@ struct sdma_engine { /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ bool clk_ratio; bool fw_loaded; + struct gen_pool *iram_pool; }; static int sdma_config_write(struct dma_chan *chan, @@ -1077,6 +1084,11 @@ static int sdma_get_pc(struct sdma_channel *sdmac, per_2_emi = sdma->script_addrs->sai_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_sai_addr; break; + case IMX_DMATYPE_I2C: + per_2_emi = sdma->script_addrs->i2c_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_i2c_addr; + sdmac->is_ram_script = true; + break; case IMX_DMATYPE_HDMI: emi_2_per = sdma->script_addrs->hdmi_dma_addr; sdmac->is_ram_script = true; @@ -1260,6 +1272,16 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; + + /* + * Limitation: The p2p script support dual fifos in maximum, + * So when fifo number is larger than 1, force enable dual + * fifos. + */ + if (sdmac->n_fifos_src > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD; + if (sdmac->n_fifos_dst > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; } static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) @@ -1363,8 +1385,14 @@ static int sdma_request_channel0(struct sdma_engine *sdma) { int ret = -EBUSY; - sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, - GFP_NOWAIT); + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); if (!sdma->bd0) { ret = -ENOMEM; goto out; @@ -1384,10 +1412,14 @@ out: static int sdma_alloc_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; int ret = 0; - desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, - &desc->bd_phys, GFP_NOWAIT); + if (sdma->iram_pool) + desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size, &desc->bd_phys); + else + desc->bd = dma_alloc_coherent(sdma->dev, bd_size, &desc->bd_phys, GFP_NOWAIT); + if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1399,9 +1431,12 @@ out: static void sdma_free_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; - dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, - desc->bd_phys); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, bd_size); + else + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1424,9 +1459,8 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) * dmatest, thus create 'struct imx_dma_data mem_data' for this case. * Please note in any other slave case, you have to setup chan->private * with 'struct imx_dma_data' in your own filter function if you want to - * request dma channel by dma_request_channel() rather than - * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear - * to warn you to correct your filter function. + * request DMA channel by dma_request_channel(), otherwise, 'MEMCPY in + * case?' will appear to warn you to correct your filter function. */ if (!data) { dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n"); @@ -1648,6 +1682,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (count & 3 || sg->dma_address & 3) goto err_bd_out; break; + case DMA_SLAVE_BUSWIDTH_3_BYTES: + bd->mode.command = 3; + break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) @@ -1885,10 +1922,17 @@ static void sdma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&sdmac->vc.lock, flags); } -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 45 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 46 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 \ +(offsetof(struct sdma_script_start_addrs, v1_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 \ +(offsetof(struct sdma_script_start_addrs, v2_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 \ +(offsetof(struct sdma_script_start_addrs, v3_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 \ +(offsetof(struct sdma_script_start_addrs, v4_end) / sizeof(s32)) static void sdma_add_scripts(struct sdma_engine *sdma, const struct sdma_script_start_addrs *addr) @@ -2062,9 +2106,8 @@ static int sdma_get_firmware(struct sdma_engine *sdma, { int ret; - ret = request_firmware_nowait(THIS_MODULE, - FW_ACTION_UEVENT, fw_name, sdma->dev, - GFP_KERNEL, sdma, sdma_load_firmware); + ret = firmware_request_nowait_nowarn(THIS_MODULE, fw_name, sdma->dev, + GFP_KERNEL, sdma, sdma_load_firmware); return ret; } @@ -2073,6 +2116,7 @@ static int sdma_init(struct sdma_engine *sdma) { int i, ret; dma_addr_t ccb_phys; + int ccbsize; ret = clk_enable(sdma->clk_ipg); if (ret) @@ -2088,10 +2132,14 @@ static int sdma_init(struct sdma_engine *sdma) /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - sdma->channel_control = dma_alloc_coherent(sdma->dev, - MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control) + - sizeof(struct sdma_context_data), - &ccb_phys, GFP_KERNEL); + ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data)); + + if (sdma->iram_pool) + sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, ccbsize, &ccb_phys); + else + sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, &ccb_phys, + GFP_KERNEL); if (!sdma->channel_control) { ret = -ENOMEM; @@ -2277,6 +2325,12 @@ static int sdma_probe(struct platform_device *pdev) vchan_init(&sdmac->vc, &sdma->dma_device); } + if (np) { + sdma->iram_pool = of_gen_pool_get(np, "iram", 0); + if (sdma->iram_pool) + dev_info(&pdev->dev, "alloc bd from iram.\n"); + } + ret = sdma_init(sdma); if (ret) goto err_init; @@ -2359,7 +2413,7 @@ err_clk: return ret; } -static int sdma_remove(struct platform_device *pdev) +static void sdma_remove(struct platform_device *pdev) { struct sdma_engine *sdma = platform_get_drvdata(pdev); int i; @@ -2378,7 +2432,6 @@ static int sdma_remove(struct platform_device *pdev) } platform_set_drvdata(pdev, NULL); - return 0; } static struct platform_driver sdma_driver = { diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index 289c59ed74b9..5d3c0ae6b342 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -10,6 +10,8 @@ #include <linux/interrupt.h> #include <linux/dca.h> +#include <asm/cpuid/api.h> + /* either a kernel change is needed, or we need something like this in kernel */ #ifndef CONFIG_SMP #include <asm/smp.h> @@ -51,18 +53,18 @@ /* pack PCI B/D/F into a u16 */ static inline u16 dcaid_from_pcidev(struct pci_dev *pci) { - return (pci->bus->number << 8) | pci->devfn; + return pci_dev_id(pci); } static int dca_enabled_in_bios(struct pci_dev *pdev) { /* CPUID level 9 returns DCA configuration */ /* Bit 0 indicates DCA enabled by the BIOS */ - unsigned long cpuid_level_9; + u32 eax; int res; - cpuid_level_9 = cpuid_eax(9); - res = test_bit(0, &cpuid_level_9); + eax = cpuid_eax(CPUID_LEAF_DCA); + res = eax & BIT(0); if (!res) dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n"); diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 79d8957f9e60..b8fff8333aef 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -159,7 +159,7 @@ void ioat_stop(struct ioatdma_chan *ioat_chan) } /* flush inflight timers */ - del_timer_sync(&ioat_chan->timer); + timer_delete_sync(&ioat_chan->timer); /* flush inflight tasklet runs */ tasklet_kill(&ioat_chan->cleanup_task); @@ -901,7 +901,8 @@ static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan) void ioat_timer_event(struct timer_list *t) { - struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer); + struct ioatdma_chan *ioat_chan = timer_container_of(ioat_chan, t, + timer); dma_addr_t phys_complete; u64 status; diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 35e06b382603..12a4a4860a74 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -19,6 +19,8 @@ #define IOAT_DMA_DCA_ANY_CPU ~0 +int system_has_dca_enabled(struct pci_dev *pdev); + #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, dma_dev) #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev) #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev) @@ -74,6 +76,7 @@ struct ioatdma_device { struct dca_provider *dca; enum ioat_irq_mode irq_mode; u32 cap; + int chancnt; /* shadow version for CB3.3 chan reset errata workaround */ u64 msixtba0; diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index 79e4e4c09c18..0373c48520c9 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -63,9 +63,6 @@ #define IOAT_VER_3_3 0x33 /* Version 3.3 */ #define IOAT_VER_3_4 0x34 /* Version 3.4 */ - -int system_has_dca_enabled(struct pci_dev *pdev); - #define IOAT_DESC_SZ 64 struct ioat_dma_descriptor { diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index c4602bfc9c74..227398673b73 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -23,6 +23,7 @@ #include "../dmaengine.h" MODULE_VERSION(IOAT_DMA_VERSION); +MODULE_DESCRIPTION("Intel I/OAT DMA Linux driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel Corporation"); @@ -420,7 +421,7 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma) msix: /* The number of MSI-X vectors should equal the number of channels */ - msixcnt = ioat_dma->dma_dev.chancnt; + msixcnt = ioat_dma->chancnt; for (i = 0; i < msixcnt; i++) ioat_dma->msix_entries[i].entry = i; @@ -511,7 +512,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) dma_cap_set(DMA_MEMCPY, dma->cap_mask); dma->dev = &pdev->dev; - if (!dma->chancnt) { + if (!ioat_dma->chancnt) { dev_err(dev, "channel enumeration error\n"); goto err_setup_interrupts; } @@ -534,18 +535,6 @@ err_out: return err; } -static int ioat_register(struct ioatdma_device *ioat_dma) -{ - int err = dma_async_device_register(&ioat_dma->dma_dev); - - if (err) { - ioat_disable_interrupts(ioat_dma); - dma_pool_destroy(ioat_dma->completion_pool); - } - - return err; -} - static void ioat_dma_remove(struct ioatdma_device *ioat_dma) { struct dma_device *dma = &ioat_dma->dma_dev; @@ -567,15 +556,16 @@ static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) struct device *dev = &ioat_dma->pdev->dev; struct dma_device *dma = &ioat_dma->dma_dev; u8 xfercap_log; + int chancnt; int i; INIT_LIST_HEAD(&dma->channels); - dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); - dma->chancnt &= 0x1f; /* bits [4:0] valid */ - if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) { + chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); + chancnt &= 0x1f; /* bits [4:0] valid */ + if (chancnt > ARRAY_SIZE(ioat_dma->idx)) { dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", - dma->chancnt, ARRAY_SIZE(ioat_dma->idx)); - dma->chancnt = ARRAY_SIZE(ioat_dma->idx); + chancnt, ARRAY_SIZE(ioat_dma->idx)); + chancnt = ARRAY_SIZE(ioat_dma->idx); } xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET); xfercap_log &= 0x1f; /* bits [4:0] valid */ @@ -583,7 +573,7 @@ static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) return; dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); - for (i = 0; i < dma->chancnt; i++) { + for (i = 0; i < chancnt; i++) { ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); if (!ioat_chan) break; @@ -596,7 +586,7 @@ static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) break; } } - dma->chancnt = i; + ioat_dma->chancnt = i; } /** @@ -915,7 +905,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) op = IOAT_OP_XOR_VAL; - /* validate the sources with the destintation page */ + /* validate the sources with the destination page */ for (i = 0; i < IOAT_NUM_SRC_TEST; i++) xor_val_srcs[i] = xor_srcs[i]; xor_val_srcs[i] = dest; @@ -1180,9 +1170,9 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) ioat_chan->reg_base + IOAT_DCACTRL_OFFSET); } - err = ioat_register(ioat_dma); + err = dma_async_device_register(&ioat_dma->dma_dev); if (err) - return err; + goto err_disable_interrupts; ioat_kobject_add(ioat_dma, &ioat_ktype); @@ -1191,20 +1181,29 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) /* disable relaxed ordering */ err = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &val16); - if (err) - return pcibios_err_to_errno(err); + if (err) { + err = pcibios_err_to_errno(err); + goto err_disable_interrupts; + } /* clear relaxed ordering enable */ val16 &= ~PCI_EXP_DEVCTL_RELAX_EN; err = pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, val16); - if (err) - return pcibios_err_to_errno(err); + if (err) { + err = pcibios_err_to_errno(err); + goto err_disable_interrupts; + } if (ioat_dma->cap & IOAT_CAP_DPS) writeb(ioat_pending_level + 1, ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET); return 0; + +err_disable_interrupts: + ioat_disable_interrupts(ioat_dma); + dma_pool_destroy(ioat_dma->completion_pool); + return err; } static void ioat_shutdown(struct pci_dev *pdev) @@ -1225,12 +1224,12 @@ static void ioat_shutdown(struct pci_dev *pdev) set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); spin_unlock_bh(&ioat_chan->prep_lock); /* - * Synchronization rule for del_timer_sync(): + * Synchronization rule for timer_delete_sync(): * - The caller must not hold locks which would prevent * completion of the timer's handler. * So prep_lock cannot be held before calling it. */ - del_timer_sync(&ioat_chan->timer); + timer_delete_sync(&ioat_chan->timer); /* this should quiesce then reset */ ioat_reset_hw(ioat_chan); @@ -1287,7 +1286,6 @@ static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_wake_from_d3(pdev, false); } @@ -1349,6 +1347,8 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) void __iomem * const *iomap; struct device *dev = &pdev->dev; struct ioatdma_device *device; + unsigned int i; + u8 version; int err; err = pcim_enable_device(pdev); @@ -1362,6 +1362,10 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!iomap) return -ENOMEM; + version = readb(iomap[IOAT_MMIO_BAR] + IOAT_VER_OFFSET); + if (version < IOAT_VER_3_0) + return -ENODEV; + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (err) return err; @@ -1372,17 +1376,18 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_master(pdev); pci_set_drvdata(pdev, device); - device->version = readb(device->reg_base + IOAT_VER_OFFSET); + device->version = version; if (device->version >= IOAT_VER_3_4) ioat_dca_enabled = 0; - if (device->version >= IOAT_VER_3_0) { - if (is_skx_ioat(pdev)) - device->version = IOAT_VER_3_2; - err = ioat3_dma_probe(device, ioat_dca_enabled); - } else - return -ENODEV; + if (is_skx_ioat(pdev)) + device->version = IOAT_VER_3_2; + + err = ioat3_dma_probe(device, ioat_dca_enabled); if (err) { + for (i = 0; i < IOAT_MAX_CHANS; i++) + kfree(device->idx[i]); + kfree(device); dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n"); return -ENODEV; } @@ -1444,6 +1449,7 @@ module_init(ioat_init_module); static void __exit ioat_exit_module(void) { pci_unregister_driver(&ioat_pci_driver); + kmem_cache_destroy(ioat_sed_cache); kmem_cache_destroy(ioat_cache); } module_exit(ioat_exit_module); diff --git a/drivers/dma/ipu/Makefile b/drivers/dma/ipu/Makefile deleted file mode 100644 index c79ff116daf6..000000000000 --- a/drivers/dma/ipu/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-y += ipu_irq.o ipu_idmac.o diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c deleted file mode 100644 index d799b99c18bd..000000000000 --- a/drivers/dma/ipu/ipu_idmac.c +++ /dev/null @@ -1,1801 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2008 - * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de> - * - * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved. - */ - -#include <linux/dma-mapping.h> -#include <linux/init.h> -#include <linux/platform_device.h> -#include <linux/err.h> -#include <linux/spinlock.h> -#include <linux/delay.h> -#include <linux/list.h> -#include <linux/clk.h> -#include <linux/vmalloc.h> -#include <linux/string.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/module.h> -#include <linux/dma/ipu-dma.h> - -#include "../dmaengine.h" -#include "ipu_intern.h" - -#define FS_VF_IN_VALID 0x00000002 -#define FS_ENC_IN_VALID 0x00000001 - -static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan, - bool wait_for_stop); - -/* - * There can be only one, we could allocate it dynamically, but then we'd have - * to add an extra parameter to some functions, and use something as ugly as - * struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device)); - * in the ISR - */ -static struct ipu ipu_data; - -#define to_ipu(id) container_of(id, struct ipu, idmac) - -static u32 __idmac_read_icreg(struct ipu *ipu, unsigned long reg) -{ - return __raw_readl(ipu->reg_ic + reg); -} - -#define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF) - -static void __idmac_write_icreg(struct ipu *ipu, u32 value, unsigned long reg) -{ - __raw_writel(value, ipu->reg_ic + reg); -} - -#define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF) - -static u32 idmac_read_ipureg(struct ipu *ipu, unsigned long reg) -{ - return __raw_readl(ipu->reg_ipu + reg); -} - -static void idmac_write_ipureg(struct ipu *ipu, u32 value, unsigned long reg) -{ - __raw_writel(value, ipu->reg_ipu + reg); -} - -/***************************************************************************** - * IPU / IC common functions - */ -static void dump_idmac_reg(struct ipu *ipu) -{ - dev_dbg(ipu->dev, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, " - "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n", - idmac_read_icreg(ipu, IDMAC_CONF), - idmac_read_icreg(ipu, IC_CONF), - idmac_read_icreg(ipu, IDMAC_CHA_EN), - idmac_read_icreg(ipu, IDMAC_CHA_PRI), - idmac_read_icreg(ipu, IDMAC_CHA_BUSY)); - dev_dbg(ipu->dev, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, " - "DB_MODE 0x%x, TASKS_STAT 0x%x\n", - idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY), - idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY), - idmac_read_ipureg(ipu, IPU_CHA_CUR_BUF), - idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL), - idmac_read_ipureg(ipu, IPU_TASKS_STAT)); -} - -static uint32_t bytes_per_pixel(enum pixel_fmt fmt) -{ - switch (fmt) { - case IPU_PIX_FMT_GENERIC: /* generic data */ - case IPU_PIX_FMT_RGB332: - case IPU_PIX_FMT_YUV420P: - case IPU_PIX_FMT_YUV422P: - default: - return 1; - case IPU_PIX_FMT_RGB565: - case IPU_PIX_FMT_YUYV: - case IPU_PIX_FMT_UYVY: - return 2; - case IPU_PIX_FMT_BGR24: - case IPU_PIX_FMT_RGB24: - return 3; - case IPU_PIX_FMT_GENERIC_32: /* generic data */ - case IPU_PIX_FMT_BGR32: - case IPU_PIX_FMT_RGB32: - case IPU_PIX_FMT_ABGR32: - return 4; - } -} - -/* Enable direct write to memory by the Camera Sensor Interface */ -static void ipu_ic_enable_task(struct ipu *ipu, enum ipu_channel channel) -{ - uint32_t ic_conf, mask; - - switch (channel) { - case IDMAC_IC_0: - mask = IC_CONF_PRPENC_EN; - break; - case IDMAC_IC_7: - mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN; - break; - default: - return; - } - ic_conf = idmac_read_icreg(ipu, IC_CONF) | mask; - idmac_write_icreg(ipu, ic_conf, IC_CONF); -} - -/* Called under spin_lock_irqsave(&ipu_data.lock) */ -static void ipu_ic_disable_task(struct ipu *ipu, enum ipu_channel channel) -{ - uint32_t ic_conf, mask; - - switch (channel) { - case IDMAC_IC_0: - mask = IC_CONF_PRPENC_EN; - break; - case IDMAC_IC_7: - mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN; - break; - default: - return; - } - ic_conf = idmac_read_icreg(ipu, IC_CONF) & ~mask; - idmac_write_icreg(ipu, ic_conf, IC_CONF); -} - -static uint32_t ipu_channel_status(struct ipu *ipu, enum ipu_channel channel) -{ - uint32_t stat = TASK_STAT_IDLE; - uint32_t task_stat_reg = idmac_read_ipureg(ipu, IPU_TASKS_STAT); - - switch (channel) { - case IDMAC_IC_7: - stat = (task_stat_reg & TSTAT_CSI2MEM_MASK) >> - TSTAT_CSI2MEM_OFFSET; - break; - case IDMAC_IC_0: - case IDMAC_SDC_0: - case IDMAC_SDC_1: - default: - break; - } - return stat; -} - -struct chan_param_mem_planar { - /* Word 0 */ - u32 xv:10; - u32 yv:10; - u32 xb:12; - - u32 yb:12; - u32 res1:2; - u32 nsb:1; - u32 lnpb:6; - u32 ubo_l:11; - - u32 ubo_h:15; - u32 vbo_l:17; - - u32 vbo_h:9; - u32 res2:3; - u32 fw:12; - u32 fh_l:8; - - u32 fh_h:4; - u32 res3:28; - - /* Word 1 */ - u32 eba0; - - u32 eba1; - - u32 bpp:3; - u32 sl:14; - u32 pfs:3; - u32 bam:3; - u32 res4:2; - u32 npb:6; - u32 res5:1; - - u32 sat:2; - u32 res6:30; -} __attribute__ ((packed)); - -struct chan_param_mem_interleaved { - /* Word 0 */ - u32 xv:10; - u32 yv:10; - u32 xb:12; - - u32 yb:12; - u32 sce:1; - u32 res1:1; - u32 nsb:1; - u32 lnpb:6; - u32 sx:10; - u32 sy_l:1; - - u32 sy_h:9; - u32 ns:10; - u32 sm:10; - u32 sdx_l:3; - - u32 sdx_h:2; - u32 sdy:5; - u32 sdrx:1; - u32 sdry:1; - u32 sdr1:1; - u32 res2:2; - u32 fw:12; - u32 fh_l:8; - - u32 fh_h:4; - u32 res3:28; - - /* Word 1 */ - u32 eba0; - - u32 eba1; - - u32 bpp:3; - u32 sl:14; - u32 pfs:3; - u32 bam:3; - u32 res4:2; - u32 npb:6; - u32 res5:1; - - u32 sat:2; - u32 scc:1; - u32 ofs0:5; - u32 ofs1:5; - u32 ofs2:5; - u32 ofs3:5; - u32 wid0:3; - u32 wid1:3; - u32 wid2:3; - - u32 wid3:3; - u32 dec_sel:1; - u32 res6:28; -} __attribute__ ((packed)); - -union chan_param_mem { - struct chan_param_mem_planar pp; - struct chan_param_mem_interleaved ip; -}; - -static void ipu_ch_param_set_plane_offset(union chan_param_mem *params, - u32 u_offset, u32 v_offset) -{ - params->pp.ubo_l = u_offset & 0x7ff; - params->pp.ubo_h = u_offset >> 11; - params->pp.vbo_l = v_offset & 0x1ffff; - params->pp.vbo_h = v_offset >> 17; -} - -static void ipu_ch_param_set_size(union chan_param_mem *params, - uint32_t pixel_fmt, uint16_t width, - uint16_t height, uint16_t stride) -{ - u32 u_offset; - u32 v_offset; - - params->pp.fw = width - 1; - params->pp.fh_l = height - 1; - params->pp.fh_h = (height - 1) >> 8; - params->pp.sl = stride - 1; - - switch (pixel_fmt) { - case IPU_PIX_FMT_GENERIC: - /*Represents 8-bit Generic data */ - params->pp.bpp = 3; - params->pp.pfs = 7; - params->pp.npb = 31; - params->pp.sat = 2; /* SAT = use 32-bit access */ - break; - case IPU_PIX_FMT_GENERIC_32: - /*Represents 32-bit Generic data */ - params->pp.bpp = 0; - params->pp.pfs = 7; - params->pp.npb = 7; - params->pp.sat = 2; /* SAT = use 32-bit access */ - break; - case IPU_PIX_FMT_RGB565: - params->ip.bpp = 2; - params->ip.pfs = 4; - params->ip.npb = 15; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 0; /* Red bit offset */ - params->ip.ofs1 = 5; /* Green bit offset */ - params->ip.ofs2 = 11; /* Blue bit offset */ - params->ip.ofs3 = 16; /* Alpha bit offset */ - params->ip.wid0 = 4; /* Red bit width - 1 */ - params->ip.wid1 = 5; /* Green bit width - 1 */ - params->ip.wid2 = 4; /* Blue bit width - 1 */ - break; - case IPU_PIX_FMT_BGR24: - params->ip.bpp = 1; /* 24 BPP & RGB PFS */ - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 0; /* Red bit offset */ - params->ip.ofs1 = 8; /* Green bit offset */ - params->ip.ofs2 = 16; /* Blue bit offset */ - params->ip.ofs3 = 24; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - break; - case IPU_PIX_FMT_RGB24: - params->ip.bpp = 1; /* 24 BPP & RGB PFS */ - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 16; /* Red bit offset */ - params->ip.ofs1 = 8; /* Green bit offset */ - params->ip.ofs2 = 0; /* Blue bit offset */ - params->ip.ofs3 = 24; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - break; - case IPU_PIX_FMT_BGRA32: - case IPU_PIX_FMT_BGR32: - case IPU_PIX_FMT_ABGR32: - params->ip.bpp = 0; - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 8; /* Red bit offset */ - params->ip.ofs1 = 16; /* Green bit offset */ - params->ip.ofs2 = 24; /* Blue bit offset */ - params->ip.ofs3 = 0; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - params->ip.wid3 = 7; /* Alpha bit width - 1 */ - break; - case IPU_PIX_FMT_RGBA32: - case IPU_PIX_FMT_RGB32: - params->ip.bpp = 0; - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 24; /* Red bit offset */ - params->ip.ofs1 = 16; /* Green bit offset */ - params->ip.ofs2 = 8; /* Blue bit offset */ - params->ip.ofs3 = 0; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - params->ip.wid3 = 7; /* Alpha bit width - 1 */ - break; - case IPU_PIX_FMT_UYVY: - params->ip.bpp = 2; - params->ip.pfs = 6; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - break; - case IPU_PIX_FMT_YUV420P2: - case IPU_PIX_FMT_YUV420P: - params->ip.bpp = 3; - params->ip.pfs = 3; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - u_offset = stride * height; - v_offset = u_offset + u_offset / 4; - ipu_ch_param_set_plane_offset(params, u_offset, v_offset); - break; - case IPU_PIX_FMT_YVU422P: - params->ip.bpp = 3; - params->ip.pfs = 2; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - v_offset = stride * height; - u_offset = v_offset + v_offset / 2; - ipu_ch_param_set_plane_offset(params, u_offset, v_offset); - break; - case IPU_PIX_FMT_YUV422P: - params->ip.bpp = 3; - params->ip.pfs = 2; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - u_offset = stride * height; - v_offset = u_offset + u_offset / 2; - ipu_ch_param_set_plane_offset(params, u_offset, v_offset); - break; - default: - dev_err(ipu_data.dev, - "mx3 ipu: unimplemented pixel format %d\n", pixel_fmt); - break; - } - - params->pp.nsb = 1; -} - -static void ipu_ch_param_set_buffer(union chan_param_mem *params, - dma_addr_t buf0, dma_addr_t buf1) -{ - params->pp.eba0 = buf0; - params->pp.eba1 = buf1; -} - -static void ipu_ch_param_set_rotation(union chan_param_mem *params, - enum ipu_rotate_mode rotate) -{ - params->pp.bam = rotate; -} - -static void ipu_write_param_mem(uint32_t addr, uint32_t *data, - uint32_t num_words) -{ - for (; num_words > 0; num_words--) { - dev_dbg(ipu_data.dev, - "write param mem - addr = 0x%08X, data = 0x%08X\n", - addr, *data); - idmac_write_ipureg(&ipu_data, addr, IPU_IMA_ADDR); - idmac_write_ipureg(&ipu_data, *data++, IPU_IMA_DATA); - addr++; - if ((addr & 0x7) == 5) { - addr &= ~0x7; /* set to word 0 */ - addr += 8; /* increment to next row */ - } - } -} - -static int calc_resize_coeffs(uint32_t in_size, uint32_t out_size, - uint32_t *resize_coeff, - uint32_t *downsize_coeff) -{ - uint32_t temp_size; - uint32_t temp_downsize; - - *resize_coeff = 1 << 13; - *downsize_coeff = 1 << 13; - - /* Cannot downsize more than 8:1 */ - if (out_size << 3 < in_size) - return -EINVAL; - - /* compute downsizing coefficient */ - temp_downsize = 0; - temp_size = in_size; - while (temp_size >= out_size * 2 && temp_downsize < 2) { - temp_size >>= 1; - temp_downsize++; - } - *downsize_coeff = temp_downsize; - - /* - * compute resizing coefficient using the following formula: - * resize_coeff = M*(SI -1)/(SO - 1) - * where M = 2^13, SI - input size, SO - output size - */ - *resize_coeff = (8192L * (temp_size - 1)) / (out_size - 1); - if (*resize_coeff >= 16384L) { - dev_err(ipu_data.dev, "Warning! Overflow on resize coeff.\n"); - *resize_coeff = 0x3FFF; - } - - dev_dbg(ipu_data.dev, "resizing from %u -> %u pixels, " - "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size, out_size, - *downsize_coeff, *resize_coeff >= 8192L ? 1 : 0, - ((*resize_coeff & 0x1FFF) * 10000L) / 8192L, *resize_coeff); - - return 0; -} - -static enum ipu_color_space format_to_colorspace(enum pixel_fmt fmt) -{ - switch (fmt) { - case IPU_PIX_FMT_RGB565: - case IPU_PIX_FMT_BGR24: - case IPU_PIX_FMT_RGB24: - case IPU_PIX_FMT_BGR32: - case IPU_PIX_FMT_RGB32: - return IPU_COLORSPACE_RGB; - default: - return IPU_COLORSPACE_YCBCR; - } -} - -static int ipu_ic_init_prpenc(struct ipu *ipu, - union ipu_channel_param *params, bool src_is_csi) -{ - uint32_t reg, ic_conf; - uint32_t downsize_coeff, resize_coeff; - enum ipu_color_space in_fmt, out_fmt; - - /* Setup vertical resizing */ - calc_resize_coeffs(params->video.in_height, - params->video.out_height, - &resize_coeff, &downsize_coeff); - reg = (downsize_coeff << 30) | (resize_coeff << 16); - - /* Setup horizontal resizing */ - calc_resize_coeffs(params->video.in_width, - params->video.out_width, - &resize_coeff, &downsize_coeff); - reg |= (downsize_coeff << 14) | resize_coeff; - - /* Setup color space conversion */ - in_fmt = format_to_colorspace(params->video.in_pixel_fmt); - out_fmt = format_to_colorspace(params->video.out_pixel_fmt); - - /* - * Colourspace conversion unsupported yet - see _init_csc() in - * Freescale sources - */ - if (in_fmt != out_fmt) { - dev_err(ipu->dev, "Colourspace conversion unsupported!\n"); - return -EOPNOTSUPP; - } - - idmac_write_icreg(ipu, reg, IC_PRP_ENC_RSC); - - ic_conf = idmac_read_icreg(ipu, IC_CONF); - - if (src_is_csi) - ic_conf &= ~IC_CONF_RWS_EN; - else - ic_conf |= IC_CONF_RWS_EN; - - idmac_write_icreg(ipu, ic_conf, IC_CONF); - - return 0; -} - -static uint32_t dma_param_addr(uint32_t dma_ch) -{ - /* Channel Parameter Memory */ - return 0x10000 | (dma_ch << 4); -} - -static void ipu_channel_set_priority(struct ipu *ipu, enum ipu_channel channel, - bool prio) -{ - u32 reg = idmac_read_icreg(ipu, IDMAC_CHA_PRI); - - if (prio) - reg |= 1UL << channel; - else - reg &= ~(1UL << channel); - - idmac_write_icreg(ipu, reg, IDMAC_CHA_PRI); - - dump_idmac_reg(ipu); -} - -static uint32_t ipu_channel_conf_mask(enum ipu_channel channel) -{ - uint32_t mask; - - switch (channel) { - case IDMAC_IC_0: - case IDMAC_IC_7: - mask = IPU_CONF_CSI_EN | IPU_CONF_IC_EN; - break; - case IDMAC_SDC_0: - case IDMAC_SDC_1: - mask = IPU_CONF_SDC_EN | IPU_CONF_DI_EN; - break; - default: - mask = 0; - break; - } - - return mask; -} - -/** - * ipu_enable_channel() - enable an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: IDMAC channel. - * @return: 0 on success or negative error code on failure. - */ -static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan) -{ - struct ipu *ipu = to_ipu(idmac); - enum ipu_channel channel = ichan->dma_chan.chan_id; - uint32_t reg; - unsigned long flags; - - spin_lock_irqsave(&ipu->lock, flags); - - /* Reset to buffer 0 */ - idmac_write_ipureg(ipu, 1UL << channel, IPU_CHA_CUR_BUF); - ichan->active_buffer = 0; - ichan->status = IPU_CHANNEL_ENABLED; - - switch (channel) { - case IDMAC_SDC_0: - case IDMAC_SDC_1: - case IDMAC_IC_7: - ipu_channel_set_priority(ipu, channel, true); - break; - default: - break; - } - - reg = idmac_read_icreg(ipu, IDMAC_CHA_EN); - - idmac_write_icreg(ipu, reg | (1UL << channel), IDMAC_CHA_EN); - - ipu_ic_enable_task(ipu, channel); - - spin_unlock_irqrestore(&ipu->lock, flags); - return 0; -} - -/** - * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel. - * @ichan: IDMAC channel. - * @pixel_fmt: pixel format of buffer. Pixel format is a FOURCC ASCII code. - * @width: width of buffer in pixels. - * @height: height of buffer in pixels. - * @stride: stride length of buffer in pixels. - * @rot_mode: rotation mode of buffer. A rotation setting other than - * IPU_ROTATE_VERT_FLIP should only be used for input buffers of - * rotation channels. - * @phyaddr_0: buffer 0 physical address. - * @phyaddr_1: buffer 1 physical address. Setting this to a value other than - * NULL enables double buffering mode. - * @return: 0 on success or negative error code on failure. - */ -static int ipu_init_channel_buffer(struct idmac_channel *ichan, - enum pixel_fmt pixel_fmt, - uint16_t width, uint16_t height, - uint32_t stride, - enum ipu_rotate_mode rot_mode, - dma_addr_t phyaddr_0, dma_addr_t phyaddr_1) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - struct idmac *idmac = to_idmac(ichan->dma_chan.device); - struct ipu *ipu = to_ipu(idmac); - union chan_param_mem params = {}; - unsigned long flags; - uint32_t reg; - uint32_t stride_bytes; - - stride_bytes = stride * bytes_per_pixel(pixel_fmt); - - if (stride_bytes % 4) { - dev_err(ipu->dev, - "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n", - stride, stride_bytes); - return -EINVAL; - } - - /* IC channel's stride must be a multiple of 8 pixels */ - if ((channel <= IDMAC_IC_13) && (stride % 8)) { - dev_err(ipu->dev, "Stride must be 8 pixel multiple\n"); - return -EINVAL; - } - - /* Build parameter memory data for DMA channel */ - ipu_ch_param_set_size(¶ms, pixel_fmt, width, height, stride_bytes); - ipu_ch_param_set_buffer(¶ms, phyaddr_0, phyaddr_1); - ipu_ch_param_set_rotation(¶ms, rot_mode); - - spin_lock_irqsave(&ipu->lock, flags); - - ipu_write_param_mem(dma_param_addr(channel), (uint32_t *)¶ms, 10); - - reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL); - - if (phyaddr_1) - reg |= 1UL << channel; - else - reg &= ~(1UL << channel); - - idmac_write_ipureg(ipu, reg, IPU_CHA_DB_MODE_SEL); - - ichan->status = IPU_CHANNEL_READY; - - spin_unlock_irqrestore(&ipu->lock, flags); - - return 0; -} - -/** - * ipu_select_buffer() - mark a channel's buffer as ready. - * @channel: channel ID. - * @buffer_n: buffer number to mark ready. - */ -static void ipu_select_buffer(enum ipu_channel channel, int buffer_n) -{ - /* No locking - this is a write-one-to-set register, cleared by IPU */ - if (buffer_n == 0) - /* Mark buffer 0 as ready. */ - idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF0_RDY); - else - /* Mark buffer 1 as ready. */ - idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF1_RDY); -} - -/** - * ipu_update_channel_buffer() - update physical address of a channel buffer. - * @ichan: IDMAC channel. - * @buffer_n: buffer number to update. - * 0 or 1 are the only valid values. - * @phyaddr: buffer physical address. - */ -/* Called under spin_lock(_irqsave)(&ichan->lock) */ -static void ipu_update_channel_buffer(struct idmac_channel *ichan, - int buffer_n, dma_addr_t phyaddr) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - uint32_t reg; - unsigned long flags; - - spin_lock_irqsave(&ipu_data.lock, flags); - - if (buffer_n == 0) { - reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY); - if (reg & (1UL << channel)) { - ipu_ic_disable_task(&ipu_data, channel); - ichan->status = IPU_CHANNEL_READY; - } - - /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */ - idmac_write_ipureg(&ipu_data, dma_param_addr(channel) + - 0x0008UL, IPU_IMA_ADDR); - idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA); - } else { - reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY); - if (reg & (1UL << channel)) { - ipu_ic_disable_task(&ipu_data, channel); - ichan->status = IPU_CHANNEL_READY; - } - - /* Check if double-buffering is already enabled */ - reg = idmac_read_ipureg(&ipu_data, IPU_CHA_DB_MODE_SEL); - - if (!(reg & (1UL << channel))) - idmac_write_ipureg(&ipu_data, reg | (1UL << channel), - IPU_CHA_DB_MODE_SEL); - - /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */ - idmac_write_ipureg(&ipu_data, dma_param_addr(channel) + - 0x0009UL, IPU_IMA_ADDR); - idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA); - } - - spin_unlock_irqrestore(&ipu_data.lock, flags); -} - -/* Called under spin_lock_irqsave(&ichan->lock) */ -static int ipu_submit_buffer(struct idmac_channel *ichan, - struct idmac_tx_desc *desc, struct scatterlist *sg, int buf_idx) -{ - unsigned int chan_id = ichan->dma_chan.chan_id; - struct device *dev = &ichan->dma_chan.dev->device; - - if (async_tx_test_ack(&desc->txd)) - return -EINTR; - - /* - * On first invocation this shouldn't be necessary, the call to - * ipu_init_channel_buffer() above will set addresses for us, so we - * could make it conditional on status >= IPU_CHANNEL_ENABLED, but - * doing it again shouldn't hurt either. - */ - ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg)); - - ipu_select_buffer(chan_id, buf_idx); - dev_dbg(dev, "Updated sg %p on channel 0x%x buffer %d\n", - sg, chan_id, buf_idx); - - return 0; -} - -/* Called under spin_lock_irqsave(&ichan->lock) */ -static int ipu_submit_channel_buffers(struct idmac_channel *ichan, - struct idmac_tx_desc *desc) -{ - struct scatterlist *sg; - int i, ret = 0; - - for (i = 0, sg = desc->sg; i < 2 && sg; i++) { - if (!ichan->sg[i]) { - ichan->sg[i] = sg; - - ret = ipu_submit_buffer(ichan, desc, sg, i); - if (ret < 0) - return ret; - - sg = sg_next(sg); - } - } - - return ret; -} - -static dma_cookie_t idmac_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct idmac_tx_desc *desc = to_tx_desc(tx); - struct idmac_channel *ichan = to_idmac_chan(tx->chan); - struct idmac *idmac = to_idmac(tx->chan->device); - struct ipu *ipu = to_ipu(idmac); - struct device *dev = &ichan->dma_chan.dev->device; - dma_cookie_t cookie; - unsigned long flags; - int ret; - - /* Sanity check */ - if (!list_empty(&desc->list)) { - /* The descriptor doesn't belong to client */ - dev_err(dev, "Descriptor %p not prepared!\n", tx); - return -EBUSY; - } - - mutex_lock(&ichan->chan_mutex); - - async_tx_clear_ack(tx); - - if (ichan->status < IPU_CHANNEL_READY) { - struct idmac_video_param *video = &ichan->params.video; - /* - * Initial buffer assignment - the first two sg-entries from - * the descriptor will end up in the IDMAC buffers - */ - dma_addr_t dma_1 = sg_is_last(desc->sg) ? 0 : - sg_dma_address(&desc->sg[1]); - - WARN_ON(ichan->sg[0] || ichan->sg[1]); - - cookie = ipu_init_channel_buffer(ichan, - video->out_pixel_fmt, - video->out_width, - video->out_height, - video->out_stride, - IPU_ROTATE_NONE, - sg_dma_address(&desc->sg[0]), - dma_1); - if (cookie < 0) - goto out; - } - - dev_dbg(dev, "Submitting sg %p\n", &desc->sg[0]); - - cookie = dma_cookie_assign(tx); - - /* ipu->lock can be taken under ichan->lock, but not v.v. */ - spin_lock_irqsave(&ichan->lock, flags); - - list_add_tail(&desc->list, &ichan->queue); - /* submit_buffers() atomically verifies and fills empty sg slots */ - ret = ipu_submit_channel_buffers(ichan, desc); - - spin_unlock_irqrestore(&ichan->lock, flags); - - if (ret < 0) { - cookie = ret; - goto dequeue; - } - - if (ichan->status < IPU_CHANNEL_ENABLED) { - ret = ipu_enable_channel(idmac, ichan); - if (ret < 0) { - cookie = ret; - goto dequeue; - } - } - - dump_idmac_reg(ipu); - -dequeue: - if (cookie < 0) { - spin_lock_irqsave(&ichan->lock, flags); - list_del_init(&desc->list); - spin_unlock_irqrestore(&ichan->lock, flags); - tx->cookie = cookie; - ichan->dma_chan.cookie = cookie; - } - -out: - mutex_unlock(&ichan->chan_mutex); - - return cookie; -} - -/* Called with ichan->chan_mutex held */ -static int idmac_desc_alloc(struct idmac_channel *ichan, int n) -{ - struct idmac_tx_desc *desc = - vmalloc(array_size(n, sizeof(struct idmac_tx_desc))); - struct idmac *idmac = to_idmac(ichan->dma_chan.device); - - if (!desc) - return -ENOMEM; - - /* No interrupts, just disable the tasklet for a moment */ - tasklet_disable(&to_ipu(idmac)->tasklet); - - ichan->n_tx_desc = n; - ichan->desc = desc; - INIT_LIST_HEAD(&ichan->queue); - INIT_LIST_HEAD(&ichan->free_list); - - while (n--) { - struct dma_async_tx_descriptor *txd = &desc->txd; - - memset(txd, 0, sizeof(*txd)); - dma_async_tx_descriptor_init(txd, &ichan->dma_chan); - txd->tx_submit = idmac_tx_submit; - - list_add(&desc->list, &ichan->free_list); - - desc++; - } - - tasklet_enable(&to_ipu(idmac)->tasklet); - - return 0; -} - -/** - * ipu_init_channel() - initialize an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: pointer to the channel object. - * @return 0 on success or negative error code on failure. - */ -static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan) -{ - union ipu_channel_param *params = &ichan->params; - uint32_t ipu_conf; - enum ipu_channel channel = ichan->dma_chan.chan_id; - unsigned long flags; - uint32_t reg; - struct ipu *ipu = to_ipu(idmac); - int ret = 0, n_desc = 0; - - dev_dbg(ipu->dev, "init channel = %d\n", channel); - - if (channel != IDMAC_SDC_0 && channel != IDMAC_SDC_1 && - channel != IDMAC_IC_7) - return -EINVAL; - - spin_lock_irqsave(&ipu->lock, flags); - - switch (channel) { - case IDMAC_IC_7: - n_desc = 16; - reg = idmac_read_icreg(ipu, IC_CONF); - idmac_write_icreg(ipu, reg & ~IC_CONF_CSI_MEM_WR_EN, IC_CONF); - break; - case IDMAC_IC_0: - n_desc = 16; - reg = idmac_read_ipureg(ipu, IPU_FS_PROC_FLOW); - idmac_write_ipureg(ipu, reg & ~FS_ENC_IN_VALID, IPU_FS_PROC_FLOW); - ret = ipu_ic_init_prpenc(ipu, params, true); - break; - case IDMAC_SDC_0: - case IDMAC_SDC_1: - n_desc = 4; - break; - default: - break; - } - - ipu->channel_init_mask |= 1L << channel; - - /* Enable IPU sub module */ - ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) | - ipu_channel_conf_mask(channel); - idmac_write_ipureg(ipu, ipu_conf, IPU_CONF); - - spin_unlock_irqrestore(&ipu->lock, flags); - - if (n_desc && !ichan->desc) - ret = idmac_desc_alloc(ichan, n_desc); - - dump_idmac_reg(ipu); - - return ret; -} - -/** - * ipu_uninit_channel() - uninitialize an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: pointer to the channel object. - */ -static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - unsigned long flags; - uint32_t reg; - unsigned long chan_mask = 1UL << channel; - uint32_t ipu_conf; - struct ipu *ipu = to_ipu(idmac); - - spin_lock_irqsave(&ipu->lock, flags); - - if (!(ipu->channel_init_mask & chan_mask)) { - dev_err(ipu->dev, "Channel already uninitialized %d\n", - channel); - spin_unlock_irqrestore(&ipu->lock, flags); - return; - } - - /* Reset the double buffer */ - reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL); - idmac_write_ipureg(ipu, reg & ~chan_mask, IPU_CHA_DB_MODE_SEL); - - ichan->sec_chan_en = false; - - switch (channel) { - case IDMAC_IC_7: - reg = idmac_read_icreg(ipu, IC_CONF); - idmac_write_icreg(ipu, reg & ~(IC_CONF_RWS_EN | IC_CONF_PRPENC_EN), - IC_CONF); - break; - case IDMAC_IC_0: - reg = idmac_read_icreg(ipu, IC_CONF); - idmac_write_icreg(ipu, reg & ~(IC_CONF_PRPENC_EN | IC_CONF_PRPENC_CSC1), - IC_CONF); - break; - case IDMAC_SDC_0: - case IDMAC_SDC_1: - default: - break; - } - - ipu->channel_init_mask &= ~(1L << channel); - - ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) & - ~ipu_channel_conf_mask(channel); - idmac_write_ipureg(ipu, ipu_conf, IPU_CONF); - - spin_unlock_irqrestore(&ipu->lock, flags); - - ichan->n_tx_desc = 0; - vfree(ichan->desc); - ichan->desc = NULL; -} - -/** - * ipu_disable_channel() - disable an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: channel object pointer. - * @wait_for_stop: flag to set whether to wait for channel end of frame or - * return immediately. - * @return: 0 on success or negative error code on failure. - */ -static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan, - bool wait_for_stop) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - struct ipu *ipu = to_ipu(idmac); - uint32_t reg; - unsigned long flags; - unsigned long chan_mask = 1UL << channel; - unsigned int timeout; - - if (wait_for_stop && channel != IDMAC_SDC_1 && channel != IDMAC_SDC_0) { - timeout = 40; - /* This waiting always fails. Related to spurious irq problem */ - while ((idmac_read_icreg(ipu, IDMAC_CHA_BUSY) & chan_mask) || - (ipu_channel_status(ipu, channel) == TASK_STAT_ACTIVE)) { - timeout--; - msleep(10); - - if (!timeout) { - dev_dbg(ipu->dev, - "Warning: timeout waiting for channel %u to " - "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, " - "busy = 0x%08X, tstat = 0x%08X\n", channel, - idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY), - idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY), - idmac_read_icreg(ipu, IDMAC_CHA_BUSY), - idmac_read_ipureg(ipu, IPU_TASKS_STAT)); - break; - } - } - dev_dbg(ipu->dev, "timeout = %d * 10ms\n", 40 - timeout); - } - /* SDC BG and FG must be disabled before DMA is disabled */ - if (wait_for_stop && (channel == IDMAC_SDC_0 || - channel == IDMAC_SDC_1)) { - for (timeout = 5; - timeout && !ipu_irq_status(ichan->eof_irq); timeout--) - msleep(5); - } - - spin_lock_irqsave(&ipu->lock, flags); - - /* Disable IC task */ - ipu_ic_disable_task(ipu, channel); - - /* Disable DMA channel(s) */ - reg = idmac_read_icreg(ipu, IDMAC_CHA_EN); - idmac_write_icreg(ipu, reg & ~chan_mask, IDMAC_CHA_EN); - - spin_unlock_irqrestore(&ipu->lock, flags); - - return 0; -} - -static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan, - struct idmac_tx_desc **desc, struct scatterlist *sg) -{ - struct scatterlist *sgnew = sg ? sg_next(sg) : NULL; - - if (sgnew) - /* next sg-element in this list */ - return sgnew; - - if ((*desc)->list.next == &ichan->queue) - /* No more descriptors on the queue */ - return NULL; - - /* Fetch next descriptor */ - *desc = list_entry((*desc)->list.next, struct idmac_tx_desc, list); - return (*desc)->sg; -} - -/* - * We have several possibilities here: - * current BUF next BUF - * - * not last sg next not last sg - * not last sg next last sg - * last sg first sg from next descriptor - * last sg NULL - * - * Besides, the descriptor queue might be empty or not. We process all these - * cases carefully. - */ -static irqreturn_t idmac_interrupt(int irq, void *dev_id) -{ - struct idmac_channel *ichan = dev_id; - struct device *dev = &ichan->dma_chan.dev->device; - unsigned int chan_id = ichan->dma_chan.chan_id; - struct scatterlist **sg, *sgnext, *sgnew = NULL; - /* Next transfer descriptor */ - struct idmac_tx_desc *desc, *descnew; - bool done = false; - u32 ready0, ready1, curbuf, err; - struct dmaengine_desc_callback cb; - - /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */ - - dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer); - - spin_lock(&ipu_data.lock); - - ready0 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY); - ready1 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY); - curbuf = idmac_read_ipureg(&ipu_data, IPU_CHA_CUR_BUF); - err = idmac_read_ipureg(&ipu_data, IPU_INT_STAT_4); - - if (err & (1 << chan_id)) { - idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4); - spin_unlock(&ipu_data.lock); - /* - * Doing this - * ichan->sg[0] = ichan->sg[1] = NULL; - * you can force channel re-enable on the next tx_submit(), but - * this is dirty - think about descriptors with multiple - * sg elements. - */ - dev_warn(dev, "NFB4EOF on channel %d, ready %x, %x, cur %x\n", - chan_id, ready0, ready1, curbuf); - return IRQ_HANDLED; - } - spin_unlock(&ipu_data.lock); - - /* Other interrupts do not interfere with this channel */ - spin_lock(&ichan->lock); - if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) || - (!ichan->active_buffer && (ready0 >> chan_id) & 1) - )) { - spin_unlock(&ichan->lock); - dev_dbg(dev, - "IRQ with active buffer still ready on channel %x, " - "active %d, ready %x, %x!\n", chan_id, - ichan->active_buffer, ready0, ready1); - return IRQ_NONE; - } - - if (unlikely(list_empty(&ichan->queue))) { - ichan->sg[ichan->active_buffer] = NULL; - spin_unlock(&ichan->lock); - dev_err(dev, - "IRQ without queued buffers on channel %x, active %d, " - "ready %x, %x!\n", chan_id, - ichan->active_buffer, ready0, ready1); - return IRQ_NONE; - } - - /* - * active_buffer is a software flag, it shows which buffer we are - * currently expecting back from the hardware, IDMAC should be - * processing the other buffer already - */ - sg = &ichan->sg[ichan->active_buffer]; - sgnext = ichan->sg[!ichan->active_buffer]; - - if (!*sg) { - spin_unlock(&ichan->lock); - return IRQ_HANDLED; - } - - desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list); - descnew = desc; - - dev_dbg(dev, "IDMAC irq %d, dma %#llx, next dma %#llx, current %d, curbuf %#x\n", - irq, (u64)sg_dma_address(*sg), - sgnext ? (u64)sg_dma_address(sgnext) : 0, - ichan->active_buffer, curbuf); - - /* Find the descriptor of sgnext */ - sgnew = idmac_sg_next(ichan, &descnew, *sg); - if (sgnext != sgnew) - dev_err(dev, "Submitted buffer %p, next buffer %p\n", sgnext, sgnew); - - /* - * if sgnext == NULL sg must be the last element in a scatterlist and - * queue must be empty - */ - if (unlikely(!sgnext)) { - if (!WARN_ON(sg_next(*sg))) - dev_dbg(dev, "Underrun on channel %x\n", chan_id); - ichan->sg[!ichan->active_buffer] = sgnew; - - if (unlikely(sgnew)) { - ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer); - } else { - spin_lock(&ipu_data.lock); - ipu_ic_disable_task(&ipu_data, chan_id); - spin_unlock(&ipu_data.lock); - ichan->status = IPU_CHANNEL_READY; - /* Continue to check for complete descriptor */ - } - } - - /* Calculate and submit the next sg element */ - sgnew = idmac_sg_next(ichan, &descnew, sgnew); - - if (unlikely(!sg_next(*sg)) || !sgnext) { - /* - * Last element in scatterlist done, remove from the queue, - * _init for debugging - */ - list_del_init(&desc->list); - done = true; - } - - *sg = sgnew; - - if (likely(sgnew) && - ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) { - dmaengine_desc_get_callback(&descnew->txd, &cb); - - list_del_init(&descnew->list); - spin_unlock(&ichan->lock); - - dmaengine_desc_callback_invoke(&cb, NULL); - spin_lock(&ichan->lock); - } - - /* Flip the active buffer - even if update above failed */ - ichan->active_buffer = !ichan->active_buffer; - if (done) - dma_cookie_complete(&desc->txd); - - dmaengine_desc_get_callback(&desc->txd, &cb); - - spin_unlock(&ichan->lock); - - if (done && (desc->txd.flags & DMA_PREP_INTERRUPT)) - dmaengine_desc_callback_invoke(&cb, NULL); - - return IRQ_HANDLED; -} - -static void ipu_gc_tasklet(struct tasklet_struct *t) -{ - struct ipu *ipu = from_tasklet(ipu, t, tasklet); - int i; - - for (i = 0; i < IPU_CHANNELS_NUM; i++) { - struct idmac_channel *ichan = ipu->channel + i; - struct idmac_tx_desc *desc; - unsigned long flags; - struct scatterlist *sg; - int j, k; - - for (j = 0; j < ichan->n_tx_desc; j++) { - desc = ichan->desc + j; - spin_lock_irqsave(&ichan->lock, flags); - if (async_tx_test_ack(&desc->txd)) { - list_move(&desc->list, &ichan->free_list); - for_each_sg(desc->sg, sg, desc->sg_len, k) { - if (ichan->sg[0] == sg) - ichan->sg[0] = NULL; - else if (ichan->sg[1] == sg) - ichan->sg[1] = NULL; - } - async_tx_clear_ack(&desc->txd); - } - spin_unlock_irqrestore(&ichan->lock, flags); - } - } -} - -/* Allocate and initialise a transfer descriptor. */ -static struct dma_async_tx_descriptor *idmac_prep_slave_sg(struct dma_chan *chan, - struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction direction, unsigned long tx_flags, - void *context) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac_tx_desc *desc = NULL; - struct dma_async_tx_descriptor *txd = NULL; - unsigned long flags; - - /* We only can handle these three channels so far */ - if (chan->chan_id != IDMAC_SDC_0 && chan->chan_id != IDMAC_SDC_1 && - chan->chan_id != IDMAC_IC_7) - return NULL; - - if (!is_slave_direction(direction)) { - dev_err(chan->device->dev, "Invalid DMA direction %d!\n", direction); - return NULL; - } - - mutex_lock(&ichan->chan_mutex); - - spin_lock_irqsave(&ichan->lock, flags); - if (!list_empty(&ichan->free_list)) { - desc = list_entry(ichan->free_list.next, - struct idmac_tx_desc, list); - - list_del_init(&desc->list); - - desc->sg_len = sg_len; - desc->sg = sgl; - txd = &desc->txd; - txd->flags = tx_flags; - } - spin_unlock_irqrestore(&ichan->lock, flags); - - mutex_unlock(&ichan->chan_mutex); - - tasklet_schedule(&to_ipu(to_idmac(chan->device))->tasklet); - - return txd; -} - -/* Re-select the current buffer and re-activate the channel */ -static void idmac_issue_pending(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - struct ipu *ipu = to_ipu(idmac); - unsigned long flags; - - /* This is not always needed, but doesn't hurt either */ - spin_lock_irqsave(&ipu->lock, flags); - ipu_select_buffer(chan->chan_id, ichan->active_buffer); - spin_unlock_irqrestore(&ipu->lock, flags); - - /* - * Might need to perform some parts of initialisation from - * ipu_enable_channel(), but not all, we do not want to reset to buffer - * 0, don't need to set priority again either, but re-enabling the task - * and the channel might be a good idea. - */ -} - -static int idmac_pause(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - struct ipu *ipu = to_ipu(idmac); - struct list_head *list, *tmp; - unsigned long flags; - - mutex_lock(&ichan->chan_mutex); - - spin_lock_irqsave(&ipu->lock, flags); - ipu_ic_disable_task(ipu, chan->chan_id); - - /* Return all descriptors into "prepared" state */ - list_for_each_safe(list, tmp, &ichan->queue) - list_del_init(list); - - ichan->sg[0] = NULL; - ichan->sg[1] = NULL; - - spin_unlock_irqrestore(&ipu->lock, flags); - - ichan->status = IPU_CHANNEL_INITIALIZED; - - mutex_unlock(&ichan->chan_mutex); - - return 0; -} - -static int __idmac_terminate_all(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - struct ipu *ipu = to_ipu(idmac); - unsigned long flags; - int i; - - ipu_disable_channel(idmac, ichan, - ichan->status >= IPU_CHANNEL_ENABLED); - - tasklet_disable(&ipu->tasklet); - - /* ichan->queue is modified in ISR, have to spinlock */ - spin_lock_irqsave(&ichan->lock, flags); - list_splice_init(&ichan->queue, &ichan->free_list); - - if (ichan->desc) - for (i = 0; i < ichan->n_tx_desc; i++) { - struct idmac_tx_desc *desc = ichan->desc + i; - if (list_empty(&desc->list)) - /* Descriptor was prepared, but not submitted */ - list_add(&desc->list, &ichan->free_list); - - async_tx_clear_ack(&desc->txd); - } - - ichan->sg[0] = NULL; - ichan->sg[1] = NULL; - spin_unlock_irqrestore(&ichan->lock, flags); - - tasklet_enable(&ipu->tasklet); - - ichan->status = IPU_CHANNEL_INITIALIZED; - - return 0; -} - -static int idmac_terminate_all(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - int ret; - - mutex_lock(&ichan->chan_mutex); - - ret = __idmac_terminate_all(chan); - - mutex_unlock(&ichan->chan_mutex); - - return ret; -} - -#ifdef DEBUG -static irqreturn_t ic_sof_irq(int irq, void *dev_id) -{ - struct idmac_channel *ichan = dev_id; - printk(KERN_DEBUG "Got SOF IRQ %d on Channel %d\n", - irq, ichan->dma_chan.chan_id); - disable_irq_nosync(irq); - return IRQ_HANDLED; -} - -static irqreturn_t ic_eof_irq(int irq, void *dev_id) -{ - struct idmac_channel *ichan = dev_id; - printk(KERN_DEBUG "Got EOF IRQ %d on Channel %d\n", - irq, ichan->dma_chan.chan_id); - disable_irq_nosync(irq); - return IRQ_HANDLED; -} - -static int ic_sof = -EINVAL, ic_eof = -EINVAL; -#endif - -static int idmac_alloc_chan_resources(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - int ret; - - /* dmaengine.c now guarantees to only offer free channels */ - BUG_ON(chan->client_count > 1); - WARN_ON(ichan->status != IPU_CHANNEL_FREE); - - dma_cookie_init(chan); - - ret = ipu_irq_map(chan->chan_id); - if (ret < 0) - goto eimap; - - ichan->eof_irq = ret; - - /* - * Important to first disable the channel, because maybe someone - * used it before us, e.g., the bootloader - */ - ipu_disable_channel(idmac, ichan, true); - - ret = ipu_init_channel(idmac, ichan); - if (ret < 0) - goto eichan; - - ret = request_irq(ichan->eof_irq, idmac_interrupt, 0, - ichan->eof_name, ichan); - if (ret < 0) - goto erirq; - -#ifdef DEBUG - if (chan->chan_id == IDMAC_IC_7) { - ic_sof = ipu_irq_map(69); - if (ic_sof > 0) { - ret = request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan); - if (ret) - dev_err(&chan->dev->device, "request irq failed for IC SOF"); - } - ic_eof = ipu_irq_map(70); - if (ic_eof > 0) { - ret = request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan); - if (ret) - dev_err(&chan->dev->device, "request irq failed for IC EOF"); - } - } -#endif - - ichan->status = IPU_CHANNEL_INITIALIZED; - - dev_dbg(&chan->dev->device, "Found channel 0x%x, irq %d\n", - chan->chan_id, ichan->eof_irq); - - return ret; - -erirq: - ipu_uninit_channel(idmac, ichan); -eichan: - ipu_irq_unmap(chan->chan_id); -eimap: - return ret; -} - -static void idmac_free_chan_resources(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - - mutex_lock(&ichan->chan_mutex); - - __idmac_terminate_all(chan); - - if (ichan->status > IPU_CHANNEL_FREE) { -#ifdef DEBUG - if (chan->chan_id == IDMAC_IC_7) { - if (ic_sof > 0) { - free_irq(ic_sof, ichan); - ipu_irq_unmap(69); - ic_sof = -EINVAL; - } - if (ic_eof > 0) { - free_irq(ic_eof, ichan); - ipu_irq_unmap(70); - ic_eof = -EINVAL; - } - } -#endif - free_irq(ichan->eof_irq, ichan); - ipu_irq_unmap(chan->chan_id); - } - - ichan->status = IPU_CHANNEL_FREE; - - ipu_uninit_channel(idmac, ichan); - - mutex_unlock(&ichan->chan_mutex); - - tasklet_schedule(&to_ipu(idmac)->tasklet); -} - -static enum dma_status idmac_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, struct dma_tx_state *txstate) -{ - return dma_cookie_status(chan, cookie, txstate); -} - -static int __init ipu_idmac_init(struct ipu *ipu) -{ - struct idmac *idmac = &ipu->idmac; - struct dma_device *dma = &idmac->dma; - int i; - - dma_cap_set(DMA_SLAVE, dma->cap_mask); - dma_cap_set(DMA_PRIVATE, dma->cap_mask); - - /* Compulsory common fields */ - dma->dev = ipu->dev; - dma->device_alloc_chan_resources = idmac_alloc_chan_resources; - dma->device_free_chan_resources = idmac_free_chan_resources; - dma->device_tx_status = idmac_tx_status; - dma->device_issue_pending = idmac_issue_pending; - - /* Compulsory for DMA_SLAVE fields */ - dma->device_prep_slave_sg = idmac_prep_slave_sg; - dma->device_pause = idmac_pause; - dma->device_terminate_all = idmac_terminate_all; - - INIT_LIST_HEAD(&dma->channels); - for (i = 0; i < IPU_CHANNELS_NUM; i++) { - struct idmac_channel *ichan = ipu->channel + i; - struct dma_chan *dma_chan = &ichan->dma_chan; - - spin_lock_init(&ichan->lock); - mutex_init(&ichan->chan_mutex); - - ichan->status = IPU_CHANNEL_FREE; - ichan->sec_chan_en = false; - snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i); - - dma_chan->device = &idmac->dma; - dma_cookie_init(dma_chan); - dma_chan->chan_id = i; - list_add_tail(&dma_chan->device_node, &dma->channels); - } - - idmac_write_icreg(ipu, 0x00000070, IDMAC_CONF); - - return dma_async_device_register(&idmac->dma); -} - -static void ipu_idmac_exit(struct ipu *ipu) -{ - int i; - struct idmac *idmac = &ipu->idmac; - - for (i = 0; i < IPU_CHANNELS_NUM; i++) { - struct idmac_channel *ichan = ipu->channel + i; - - idmac_terminate_all(&ichan->dma_chan); - } - - dma_async_device_unregister(&idmac->dma); -} - -/***************************************************************************** - * IPU common probe / remove - */ - -static int __init ipu_probe(struct platform_device *pdev) -{ - struct resource *mem_ipu, *mem_ic; - int ret; - - spin_lock_init(&ipu_data.lock); - - mem_ipu = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mem_ic = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!mem_ipu || !mem_ic) - return -EINVAL; - - ipu_data.dev = &pdev->dev; - - platform_set_drvdata(pdev, &ipu_data); - - ret = platform_get_irq(pdev, 0); - if (ret < 0) - goto err_noirq; - - ipu_data.irq_fn = ret; - ret = platform_get_irq(pdev, 1); - if (ret < 0) - goto err_noirq; - - ipu_data.irq_err = ret; - - dev_dbg(&pdev->dev, "fn irq %u, err irq %u\n", - ipu_data.irq_fn, ipu_data.irq_err); - - /* Remap IPU common registers */ - ipu_data.reg_ipu = ioremap(mem_ipu->start, resource_size(mem_ipu)); - if (!ipu_data.reg_ipu) { - ret = -ENOMEM; - goto err_ioremap_ipu; - } - - /* Remap Image Converter and Image DMA Controller registers */ - ipu_data.reg_ic = ioremap(mem_ic->start, resource_size(mem_ic)); - if (!ipu_data.reg_ic) { - ret = -ENOMEM; - goto err_ioremap_ic; - } - - /* Get IPU clock */ - ipu_data.ipu_clk = clk_get(&pdev->dev, NULL); - if (IS_ERR(ipu_data.ipu_clk)) { - ret = PTR_ERR(ipu_data.ipu_clk); - goto err_clk_get; - } - - /* Make sure IPU HSP clock is running */ - clk_prepare_enable(ipu_data.ipu_clk); - - /* Disable all interrupts */ - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_1); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_2); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_3); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_4); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_5); - - dev_dbg(&pdev->dev, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev->name, - (unsigned long)mem_ipu->start, ipu_data.irq_fn, ipu_data.irq_err); - - ret = ipu_irq_attach_irq(&ipu_data, pdev); - if (ret < 0) - goto err_attach_irq; - - /* Initialize DMA engine */ - ret = ipu_idmac_init(&ipu_data); - if (ret < 0) - goto err_idmac_init; - - tasklet_setup(&ipu_data.tasklet, ipu_gc_tasklet); - - ipu_data.dev = &pdev->dev; - - dev_dbg(ipu_data.dev, "IPU initialized\n"); - - return 0; - -err_idmac_init: -err_attach_irq: - ipu_irq_detach_irq(&ipu_data, pdev); - clk_disable_unprepare(ipu_data.ipu_clk); - clk_put(ipu_data.ipu_clk); -err_clk_get: - iounmap(ipu_data.reg_ic); -err_ioremap_ic: - iounmap(ipu_data.reg_ipu); -err_ioremap_ipu: -err_noirq: - dev_err(&pdev->dev, "Failed to probe IPU: %d\n", ret); - return ret; -} - -static int ipu_remove(struct platform_device *pdev) -{ - struct ipu *ipu = platform_get_drvdata(pdev); - - ipu_idmac_exit(ipu); - ipu_irq_detach_irq(ipu, pdev); - clk_disable_unprepare(ipu->ipu_clk); - clk_put(ipu->ipu_clk); - iounmap(ipu->reg_ic); - iounmap(ipu->reg_ipu); - tasklet_kill(&ipu->tasklet); - - return 0; -} - -/* - * We need two MEM resources - with IPU-common and Image Converter registers, - * including PF_CONF and IDMAC_* registers, and two IRQs - function and error - */ -static struct platform_driver ipu_platform_driver = { - .driver = { - .name = "ipu-core", - }, - .remove = ipu_remove, -}; - -static int __init ipu_init(void) -{ - return platform_driver_probe(&ipu_platform_driver, ipu_probe); -} -subsys_initcall(ipu_init); - -MODULE_DESCRIPTION("IPU core driver"); -MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>"); -MODULE_ALIAS("platform:ipu-core"); diff --git a/drivers/dma/ipu/ipu_intern.h b/drivers/dma/ipu/ipu_intern.h deleted file mode 100644 index e7ec1dec3edf..000000000000 --- a/drivers/dma/ipu/ipu_intern.h +++ /dev/null @@ -1,173 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2008 - * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de> - * - * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved. - */ - -#ifndef _IPU_INTERN_H_ -#define _IPU_INTERN_H_ - -#include <linux/dmaengine.h> -#include <linux/platform_device.h> -#include <linux/interrupt.h> - -/* IPU Common registers */ -#define IPU_CONF 0x00 -#define IPU_CHA_BUF0_RDY 0x04 -#define IPU_CHA_BUF1_RDY 0x08 -#define IPU_CHA_DB_MODE_SEL 0x0C -#define IPU_CHA_CUR_BUF 0x10 -#define IPU_FS_PROC_FLOW 0x14 -#define IPU_FS_DISP_FLOW 0x18 -#define IPU_TASKS_STAT 0x1C -#define IPU_IMA_ADDR 0x20 -#define IPU_IMA_DATA 0x24 -#define IPU_INT_CTRL_1 0x28 -#define IPU_INT_CTRL_2 0x2C -#define IPU_INT_CTRL_3 0x30 -#define IPU_INT_CTRL_4 0x34 -#define IPU_INT_CTRL_5 0x38 -#define IPU_INT_STAT_1 0x3C -#define IPU_INT_STAT_2 0x40 -#define IPU_INT_STAT_3 0x44 -#define IPU_INT_STAT_4 0x48 -#define IPU_INT_STAT_5 0x4C -#define IPU_BRK_CTRL_1 0x50 -#define IPU_BRK_CTRL_2 0x54 -#define IPU_BRK_STAT 0x58 -#define IPU_DIAGB_CTRL 0x5C - -/* IPU_CONF Register bits */ -#define IPU_CONF_CSI_EN 0x00000001 -#define IPU_CONF_IC_EN 0x00000002 -#define IPU_CONF_ROT_EN 0x00000004 -#define IPU_CONF_PF_EN 0x00000008 -#define IPU_CONF_SDC_EN 0x00000010 -#define IPU_CONF_ADC_EN 0x00000020 -#define IPU_CONF_DI_EN 0x00000040 -#define IPU_CONF_DU_EN 0x00000080 -#define IPU_CONF_PXL_ENDIAN 0x00000100 - -/* Image Converter Registers */ -#define IC_CONF 0x88 -#define IC_PRP_ENC_RSC 0x8C -#define IC_PRP_VF_RSC 0x90 -#define IC_PP_RSC 0x94 -#define IC_CMBP_1 0x98 -#define IC_CMBP_2 0x9C -#define PF_CONF 0xA0 -#define IDMAC_CONF 0xA4 -#define IDMAC_CHA_EN 0xA8 -#define IDMAC_CHA_PRI 0xAC -#define IDMAC_CHA_BUSY 0xB0 - -/* Image Converter Register bits */ -#define IC_CONF_PRPENC_EN 0x00000001 -#define IC_CONF_PRPENC_CSC1 0x00000002 -#define IC_CONF_PRPENC_ROT_EN 0x00000004 -#define IC_CONF_PRPVF_EN 0x00000100 -#define IC_CONF_PRPVF_CSC1 0x00000200 -#define IC_CONF_PRPVF_CSC2 0x00000400 -#define IC_CONF_PRPVF_CMB 0x00000800 -#define IC_CONF_PRPVF_ROT_EN 0x00001000 -#define IC_CONF_PP_EN 0x00010000 -#define IC_CONF_PP_CSC1 0x00020000 -#define IC_CONF_PP_CSC2 0x00040000 -#define IC_CONF_PP_CMB 0x00080000 -#define IC_CONF_PP_ROT_EN 0x00100000 -#define IC_CONF_IC_GLB_LOC_A 0x10000000 -#define IC_CONF_KEY_COLOR_EN 0x20000000 -#define IC_CONF_RWS_EN 0x40000000 -#define IC_CONF_CSI_MEM_WR_EN 0x80000000 - -#define IDMA_CHAN_INVALID 0x000000FF -#define IDMA_IC_0 0x00000001 -#define IDMA_IC_1 0x00000002 -#define IDMA_IC_2 0x00000004 -#define IDMA_IC_3 0x00000008 -#define IDMA_IC_4 0x00000010 -#define IDMA_IC_5 0x00000020 -#define IDMA_IC_6 0x00000040 -#define IDMA_IC_7 0x00000080 -#define IDMA_IC_8 0x00000100 -#define IDMA_IC_9 0x00000200 -#define IDMA_IC_10 0x00000400 -#define IDMA_IC_11 0x00000800 -#define IDMA_IC_12 0x00001000 -#define IDMA_IC_13 0x00002000 -#define IDMA_SDC_BG 0x00004000 -#define IDMA_SDC_FG 0x00008000 -#define IDMA_SDC_MASK 0x00010000 -#define IDMA_SDC_PARTIAL 0x00020000 -#define IDMA_ADC_SYS1_WR 0x00040000 -#define IDMA_ADC_SYS2_WR 0x00080000 -#define IDMA_ADC_SYS1_CMD 0x00100000 -#define IDMA_ADC_SYS2_CMD 0x00200000 -#define IDMA_ADC_SYS1_RD 0x00400000 -#define IDMA_ADC_SYS2_RD 0x00800000 -#define IDMA_PF_QP 0x01000000 -#define IDMA_PF_BSP 0x02000000 -#define IDMA_PF_Y_IN 0x04000000 -#define IDMA_PF_U_IN 0x08000000 -#define IDMA_PF_V_IN 0x10000000 -#define IDMA_PF_Y_OUT 0x20000000 -#define IDMA_PF_U_OUT 0x40000000 -#define IDMA_PF_V_OUT 0x80000000 - -#define TSTAT_PF_H264_PAUSE 0x00000001 -#define TSTAT_CSI2MEM_MASK 0x0000000C -#define TSTAT_CSI2MEM_OFFSET 2 -#define TSTAT_VF_MASK 0x00000600 -#define TSTAT_VF_OFFSET 9 -#define TSTAT_VF_ROT_MASK 0x000C0000 -#define TSTAT_VF_ROT_OFFSET 18 -#define TSTAT_ENC_MASK 0x00000180 -#define TSTAT_ENC_OFFSET 7 -#define TSTAT_ENC_ROT_MASK 0x00030000 -#define TSTAT_ENC_ROT_OFFSET 16 -#define TSTAT_PP_MASK 0x00001800 -#define TSTAT_PP_OFFSET 11 -#define TSTAT_PP_ROT_MASK 0x00300000 -#define TSTAT_PP_ROT_OFFSET 20 -#define TSTAT_PF_MASK 0x00C00000 -#define TSTAT_PF_OFFSET 22 -#define TSTAT_ADCSYS1_MASK 0x03000000 -#define TSTAT_ADCSYS1_OFFSET 24 -#define TSTAT_ADCSYS2_MASK 0x0C000000 -#define TSTAT_ADCSYS2_OFFSET 26 - -#define TASK_STAT_IDLE 0 -#define TASK_STAT_ACTIVE 1 -#define TASK_STAT_WAIT4READY 2 - -struct idmac { - struct dma_device dma; -}; - -struct ipu { - void __iomem *reg_ipu; - void __iomem *reg_ic; - unsigned int irq_fn; /* IPU Function IRQ to the CPU */ - unsigned int irq_err; /* IPU Error IRQ to the CPU */ - unsigned int irq_base; /* Beginning of the IPU IRQ range */ - unsigned long channel_init_mask; - spinlock_t lock; - struct clk *ipu_clk; - struct device *dev; - struct idmac idmac; - struct idmac_channel channel[IPU_CHANNELS_NUM]; - struct tasklet_struct tasklet; -}; - -#define to_idmac(d) container_of(d, struct idmac, dma) - -extern int ipu_irq_attach_irq(struct ipu *ipu, struct platform_device *dev); -extern void ipu_irq_detach_irq(struct ipu *ipu, struct platform_device *dev); - -extern bool ipu_irq_status(uint32_t irq); -extern int ipu_irq_map(unsigned int source); -extern int ipu_irq_unmap(unsigned int source); - -#endif diff --git a/drivers/dma/ipu/ipu_irq.c b/drivers/dma/ipu/ipu_irq.c deleted file mode 100644 index 97d9a6f04f2a..000000000000 --- a/drivers/dma/ipu/ipu_irq.c +++ /dev/null @@ -1,367 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2008 - * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de> - */ - -#include <linux/init.h> -#include <linux/err.h> -#include <linux/spinlock.h> -#include <linux/delay.h> -#include <linux/clk.h> -#include <linux/irq.h> -#include <linux/io.h> -#include <linux/module.h> -#include <linux/dma/ipu-dma.h> - -#include "ipu_intern.h" - -/* - * Register read / write - shall be inlined by the compiler - */ -static u32 ipu_read_reg(struct ipu *ipu, unsigned long reg) -{ - return __raw_readl(ipu->reg_ipu + reg); -} - -static void ipu_write_reg(struct ipu *ipu, u32 value, unsigned long reg) -{ - __raw_writel(value, ipu->reg_ipu + reg); -} - - -/* - * IPU IRQ chip driver - */ - -#define IPU_IRQ_NR_FN_BANKS 3 -#define IPU_IRQ_NR_ERR_BANKS 2 -#define IPU_IRQ_NR_BANKS (IPU_IRQ_NR_FN_BANKS + IPU_IRQ_NR_ERR_BANKS) - -struct ipu_irq_bank { - unsigned int control; - unsigned int status; - struct ipu *ipu; -}; - -static struct ipu_irq_bank irq_bank[IPU_IRQ_NR_BANKS] = { - /* 3 groups of functional interrupts */ - { - .control = IPU_INT_CTRL_1, - .status = IPU_INT_STAT_1, - }, { - .control = IPU_INT_CTRL_2, - .status = IPU_INT_STAT_2, - }, { - .control = IPU_INT_CTRL_3, - .status = IPU_INT_STAT_3, - }, - /* 2 groups of error interrupts */ - { - .control = IPU_INT_CTRL_4, - .status = IPU_INT_STAT_4, - }, { - .control = IPU_INT_CTRL_5, - .status = IPU_INT_STAT_5, - }, -}; - -struct ipu_irq_map { - unsigned int irq; - int source; - struct ipu_irq_bank *bank; - struct ipu *ipu; -}; - -static struct ipu_irq_map irq_map[CONFIG_MX3_IPU_IRQS]; -/* Protects allocations from the above array of maps */ -static DEFINE_MUTEX(map_lock); -/* Protects register accesses and individual mappings */ -static DEFINE_RAW_SPINLOCK(bank_lock); - -static struct ipu_irq_map *src2map(unsigned int src) -{ - int i; - - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) - if (irq_map[i].source == src) - return irq_map + i; - - return NULL; -} - -static void ipu_irq_unmask(struct irq_data *d) -{ - struct ipu_irq_map *map = irq_data_get_irq_chip_data(d); - struct ipu_irq_bank *bank; - uint32_t reg; - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - - bank = map->bank; - if (!bank) { - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - pr_err("IPU: %s(%u) - unmapped!\n", __func__, d->irq); - return; - } - - reg = ipu_read_reg(bank->ipu, bank->control); - reg |= (1UL << (map->source & 31)); - ipu_write_reg(bank->ipu, reg, bank->control); - - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); -} - -static void ipu_irq_mask(struct irq_data *d) -{ - struct ipu_irq_map *map = irq_data_get_irq_chip_data(d); - struct ipu_irq_bank *bank; - uint32_t reg; - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - - bank = map->bank; - if (!bank) { - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - pr_err("IPU: %s(%u) - unmapped!\n", __func__, d->irq); - return; - } - - reg = ipu_read_reg(bank->ipu, bank->control); - reg &= ~(1UL << (map->source & 31)); - ipu_write_reg(bank->ipu, reg, bank->control); - - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); -} - -static void ipu_irq_ack(struct irq_data *d) -{ - struct ipu_irq_map *map = irq_data_get_irq_chip_data(d); - struct ipu_irq_bank *bank; - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - - bank = map->bank; - if (!bank) { - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - pr_err("IPU: %s(%u) - unmapped!\n", __func__, d->irq); - return; - } - - ipu_write_reg(bank->ipu, 1UL << (map->source & 31), bank->status); - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); -} - -/** - * ipu_irq_status() - returns the current interrupt status of the specified IRQ. - * @irq: interrupt line to get status for. - * @return: true if the interrupt is pending/asserted or false if the - * interrupt is not pending. - */ -bool ipu_irq_status(unsigned int irq) -{ - struct ipu_irq_map *map = irq_get_chip_data(irq); - struct ipu_irq_bank *bank; - unsigned long lock_flags; - bool ret; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - bank = map->bank; - ret = bank && ipu_read_reg(bank->ipu, bank->status) & - (1UL << (map->source & 31)); - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - - return ret; -} - -/** - * ipu_irq_map() - map an IPU interrupt source to an IRQ number - * @source: interrupt source bit position (see below) - * @return: mapped IRQ number or negative error code - * - * The source parameter has to be explained further. On i.MX31 IPU has 137 IRQ - * sources, they are broken down in 5 32-bit registers, like 32, 32, 24, 32, 17. - * However, the source argument of this function is not the sequence number of - * the possible IRQ, but rather its bit position. So, first interrupt in fourth - * register has source number 96, and not 88. This makes calculations easier, - * and also provides forward compatibility with any future IPU implementations - * with any interrupt bit assignments. - */ -int ipu_irq_map(unsigned int source) -{ - int i, ret = -ENOMEM; - struct ipu_irq_map *map; - - might_sleep(); - - mutex_lock(&map_lock); - map = src2map(source); - if (map) { - pr_err("IPU: Source %u already mapped to IRQ %u\n", source, map->irq); - ret = -EBUSY; - goto out; - } - - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) { - if (irq_map[i].source < 0) { - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - irq_map[i].source = source; - irq_map[i].bank = irq_bank + source / 32; - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - - ret = irq_map[i].irq; - pr_debug("IPU: mapped source %u to IRQ %u\n", - source, ret); - break; - } - } -out: - mutex_unlock(&map_lock); - - if (ret < 0) - pr_err("IPU: couldn't map source %u: %d\n", source, ret); - - return ret; -} - -/** - * ipu_irq_unmap() - unmap an IPU interrupt source - * @source: interrupt source bit position (see ipu_irq_map()) - * @return: 0 or negative error code - */ -int ipu_irq_unmap(unsigned int source) -{ - int i, ret = -EINVAL; - - might_sleep(); - - mutex_lock(&map_lock); - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) { - if (irq_map[i].source == source) { - unsigned long lock_flags; - - pr_debug("IPU: unmapped source %u from IRQ %u\n", - source, irq_map[i].irq); - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - irq_map[i].source = -EINVAL; - irq_map[i].bank = NULL; - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - - ret = 0; - break; - } - } - mutex_unlock(&map_lock); - - return ret; -} - -/* Chained IRQ handler for IPU function and error interrupt */ -static void ipu_irq_handler(struct irq_desc *desc) -{ - struct ipu *ipu = irq_desc_get_handler_data(desc); - u32 status; - int i, line; - - for (i = 0; i < IPU_IRQ_NR_BANKS; i++) { - struct ipu_irq_bank *bank = irq_bank + i; - - raw_spin_lock(&bank_lock); - status = ipu_read_reg(ipu, bank->status); - /* - * Don't think we have to clear all interrupts here, they will - * be acked by ->handle_irq() (handle_level_irq). However, we - * might want to clear unhandled interrupts after the loop... - */ - status &= ipu_read_reg(ipu, bank->control); - raw_spin_unlock(&bank_lock); - while ((line = ffs(status))) { - struct ipu_irq_map *map; - unsigned int irq; - - line--; - status &= ~(1UL << line); - - raw_spin_lock(&bank_lock); - map = src2map(32 * i + line); - if (!map) { - raw_spin_unlock(&bank_lock); - pr_err("IPU: Interrupt on unmapped source %u bank %d\n", - line, i); - continue; - } - irq = map->irq; - raw_spin_unlock(&bank_lock); - generic_handle_irq(irq); - } - } -} - -static struct irq_chip ipu_irq_chip = { - .name = "ipu_irq", - .irq_ack = ipu_irq_ack, - .irq_mask = ipu_irq_mask, - .irq_unmask = ipu_irq_unmask, -}; - -/* Install the IRQ handler */ -int __init ipu_irq_attach_irq(struct ipu *ipu, struct platform_device *dev) -{ - unsigned int irq, i; - int irq_base = irq_alloc_descs(-1, 0, CONFIG_MX3_IPU_IRQS, - numa_node_id()); - - if (irq_base < 0) - return irq_base; - - for (i = 0; i < IPU_IRQ_NR_BANKS; i++) - irq_bank[i].ipu = ipu; - - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) { - int ret; - - irq = irq_base + i; - ret = irq_set_chip(irq, &ipu_irq_chip); - if (ret < 0) - return ret; - ret = irq_set_chip_data(irq, irq_map + i); - if (ret < 0) - return ret; - irq_map[i].ipu = ipu; - irq_map[i].irq = irq; - irq_map[i].source = -EINVAL; - irq_set_handler(irq, handle_level_irq); - irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE); - } - - irq_set_chained_handler_and_data(ipu->irq_fn, ipu_irq_handler, ipu); - - irq_set_chained_handler_and_data(ipu->irq_err, ipu_irq_handler, ipu); - - ipu->irq_base = irq_base; - - return 0; -} - -void ipu_irq_detach_irq(struct ipu *ipu, struct platform_device *dev) -{ - unsigned int irq, irq_base; - - irq_base = ipu->irq_base; - - irq_set_chained_handler_and_data(ipu->irq_fn, NULL, NULL); - - irq_set_chained_handler_and_data(ipu->irq_err, NULL, NULL); - - for (irq = irq_base; irq < irq_base + CONFIG_MX3_IPU_IRQS; irq++) { - irq_set_status_flags(irq, IRQ_NOREQUEST); - irq_set_chip(irq, NULL); - irq_set_chip_data(irq, NULL); - } -} diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index ecdaada95120..0f9cd7815f88 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -15,7 +15,6 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/of_device.h> #include <linux/of.h> #include <linux/clk.h> #include <linux/of_dma.h> @@ -839,7 +838,6 @@ static int k3_dma_probe(struct platform_device *op) { const struct k3dma_soc_data *soc_data; struct k3_dma_dev *d; - const struct of_device_id *of_id; int i, ret, irq = 0; d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); @@ -854,19 +852,16 @@ static int k3_dma_probe(struct platform_device *op) if (IS_ERR(d->base)) return PTR_ERR(d->base); - of_id = of_match_device(k3_pdma_dt_ids, &op->dev); - if (of_id) { - of_property_read_u32((&op->dev)->of_node, - "dma-channels", &d->dma_channels); - of_property_read_u32((&op->dev)->of_node, - "dma-requests", &d->dma_requests); - ret = of_property_read_u32((&op->dev)->of_node, - "dma-channel-mask", &d->dma_channel_mask); - if (ret) { - dev_warn(&op->dev, - "dma-channel-mask doesn't exist, considering all as available.\n"); - d->dma_channel_mask = (u32)~0UL; - } + of_property_read_u32((&op->dev)->of_node, + "dma-channels", &d->dma_channels); + of_property_read_u32((&op->dev)->of_node, + "dma-requests", &d->dma_requests); + ret = of_property_read_u32((&op->dev)->of_node, + "dma-channel-mask", &d->dma_channel_mask); + if (ret) { + dev_warn(&op->dev, + "dma-channel-mask doesn't exist, considering all as available.\n"); + d->dma_channel_mask = (u32)~0UL; } if (!(soc_data->flags & K3_FLAG_NOCLK)) { @@ -974,7 +969,7 @@ dma_async_register_fail: return ret; } -static int k3_dma_remove(struct platform_device *op) +static void k3_dma_remove(struct platform_device *op) { struct k3_dma_chan *c, *cn; struct k3_dma_dev *d = platform_get_drvdata(op); @@ -990,7 +985,6 @@ static int k3_dma_remove(struct platform_device *op) } tasklet_kill(&d->task); clk_disable_unprepare(d->clk); - return 0; } #ifdef CONFIG_PM_SLEEP @@ -1040,5 +1034,4 @@ static struct platform_driver k3_pdma_driver = { module_platform_driver(k3_pdma_driver); MODULE_DESCRIPTION("HiSilicon k3 DMA Driver"); -MODULE_ALIAS("platform:k3dma"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/lgm/lgm-dma.c b/drivers/dma/lgm/lgm-dma.c index 1709d159af7e..8173c3f1075a 100644 --- a/drivers/dma/lgm/lgm-dma.c +++ b/drivers/dma/lgm/lgm-dma.c @@ -107,7 +107,7 @@ * If header mode is set in DMA descriptor, * If bit 30 is disabled, HDR_LEN must be configured according to channel * requirement. - * If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to + * If bit 30 is enabled(checksum with header mode), HDR_LEN has no need to * be configured. It will enable check sum for switch * If header mode is not set in DMA descriptor, * This register setting doesn't matter @@ -1732,9 +1732,4 @@ static struct platform_driver intel_ldma_driver = { * registered DMA channels and DMA capabilities to clients before their * initialization. */ -static int __init intel_ldma_init(void) -{ - return platform_driver_register(&intel_ldma_driver); -} - -device_initcall(intel_ldma_init); +builtin_platform_driver(intel_ldma_driver); diff --git a/drivers/dma/loongson1-apb-dma.c b/drivers/dma/loongson1-apb-dma.c new file mode 100644 index 000000000000..255fe7eca212 --- /dev/null +++ b/drivers/dma/loongson1-apb-dma.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for Loongson-1 APB DMA Controller + * + * Copyright (C) 2015-2024 Keguang Zhang <keguang.zhang@gmail.com> + */ + +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +/* Loongson-1 DMA Control Register */ +#define LS1X_DMA_CTRL 0x0 + +/* DMA Control Register Bits */ +#define LS1X_DMA_STOP BIT(4) +#define LS1X_DMA_START BIT(3) +#define LS1X_DMA_ASK_VALID BIT(2) + +/* DMA Next Field Bits */ +#define LS1X_DMA_NEXT_VALID BIT(0) + +/* DMA Command Field Bits */ +#define LS1X_DMA_RAM2DEV BIT(12) +#define LS1X_DMA_INT BIT(1) +#define LS1X_DMA_INT_MASK BIT(0) + +#define LS1X_DMA_LLI_ALIGNMENT 64 +#define LS1X_DMA_LLI_ADDR_MASK GENMASK(31, __ffs(LS1X_DMA_LLI_ALIGNMENT)) +#define LS1X_DMA_MAX_CHANNELS 3 + +enum ls1x_dmadesc_offsets { + LS1X_DMADESC_NEXT = 0, + LS1X_DMADESC_SADDR, + LS1X_DMADESC_DADDR, + LS1X_DMADESC_LENGTH, + LS1X_DMADESC_STRIDE, + LS1X_DMADESC_CYCLES, + LS1X_DMADESC_CMD, + LS1X_DMADESC_SIZE +}; + +struct ls1x_dma_lli { + unsigned int hw[LS1X_DMADESC_SIZE]; + dma_addr_t phys; + struct list_head node; +} __aligned(LS1X_DMA_LLI_ALIGNMENT); + +struct ls1x_dma_desc { + struct virt_dma_desc vd; + struct list_head lli_list; +}; + +struct ls1x_dma_chan { + struct virt_dma_chan vc; + struct dma_pool *lli_pool; + phys_addr_t src_addr; + phys_addr_t dst_addr; + enum dma_slave_buswidth src_addr_width; + enum dma_slave_buswidth dst_addr_width; + unsigned int bus_width; + void __iomem *reg_base; + int irq; + bool is_cyclic; + struct ls1x_dma_lli *curr_lli; +}; + +struct ls1x_dma { + struct dma_device ddev; + unsigned int nr_chans; + struct ls1x_dma_chan chan[]; +}; + +static irqreturn_t ls1x_dma_irq_handler(int irq, void *data); + +#define to_ls1x_dma_chan(dchan) \ + container_of(dchan, struct ls1x_dma_chan, vc.chan) + +#define to_ls1x_dma_desc(d) \ + container_of(d, struct ls1x_dma_desc, vd) + +static inline struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static inline int ls1x_dma_query(struct ls1x_dma_chan *chan, + dma_addr_t *lli_phys) +{ + struct dma_chan *dchan = &chan->vc.chan; + int val, ret; + + val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK; + val |= LS1X_DMA_ASK_VALID; + val |= dchan->chan_id; + writel(val, chan->reg_base + LS1X_DMA_CTRL); + ret = readl_poll_timeout_atomic(chan->reg_base + LS1X_DMA_CTRL, val, + !(val & LS1X_DMA_ASK_VALID), 0, 3000); + if (ret) + dev_err(chan2dev(dchan), "failed to query DMA\n"); + + return ret; +} + +static inline int ls1x_dma_start(struct ls1x_dma_chan *chan, + dma_addr_t *lli_phys) +{ + struct dma_chan *dchan = &chan->vc.chan; + struct device *dev = chan2dev(dchan); + int val, ret; + + val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK; + val |= LS1X_DMA_START; + val |= dchan->chan_id; + writel(val, chan->reg_base + LS1X_DMA_CTRL); + ret = readl_poll_timeout(chan->reg_base + LS1X_DMA_CTRL, val, + !(val & LS1X_DMA_START), 0, 1000); + if (!ret) + dev_dbg(dev, "start DMA with lli_phys=%pad\n", lli_phys); + else + dev_err(dev, "failed to start DMA\n"); + + return ret; +} + +static inline void ls1x_dma_stop(struct ls1x_dma_chan *chan) +{ + int val = readl(chan->reg_base + LS1X_DMA_CTRL); + + writel(val | LS1X_DMA_STOP, chan->reg_base + LS1X_DMA_CTRL); +} + +static void ls1x_dma_free_chan_resources(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct device *dev = chan2dev(dchan); + + dma_free_coherent(dev, sizeof(struct ls1x_dma_lli), + chan->curr_lli, chan->curr_lli->phys); + dma_pool_destroy(chan->lli_pool); + chan->lli_pool = NULL; + devm_free_irq(dev, chan->irq, chan); + vchan_free_chan_resources(&chan->vc); +} + +static int ls1x_dma_alloc_chan_resources(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct device *dev = chan2dev(dchan); + dma_addr_t phys; + int ret; + + ret = devm_request_irq(dev, chan->irq, ls1x_dma_irq_handler, + IRQF_SHARED, dma_chan_name(dchan), chan); + if (ret) { + dev_err(dev, "failed to request IRQ %d\n", chan->irq); + return ret; + } + + chan->lli_pool = dma_pool_create(dma_chan_name(dchan), dev, + sizeof(struct ls1x_dma_lli), + __alignof__(struct ls1x_dma_lli), 0); + if (!chan->lli_pool) + return -ENOMEM; + + /* allocate memory for querying the current lli */ + dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + chan->curr_lli = dma_alloc_coherent(dev, sizeof(struct ls1x_dma_lli), + &phys, GFP_KERNEL); + if (!chan->curr_lli) { + dma_pool_destroy(chan->lli_pool); + return -ENOMEM; + } + chan->curr_lli->phys = phys; + + return 0; +} + +static void ls1x_dma_free_desc(struct virt_dma_desc *vd) +{ + struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd); + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(vd->tx.chan); + struct ls1x_dma_lli *lli, *_lli; + + list_for_each_entry_safe(lli, _lli, &desc->lli_list, node) { + list_del(&lli->node); + dma_pool_free(chan->lli_pool, lli, lli->phys); + } + + kfree(desc); +} + +static struct ls1x_dma_desc *ls1x_dma_alloc_desc(void) +{ + struct ls1x_dma_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + INIT_LIST_HEAD(&desc->lli_list); + + return desc; +} + +static int ls1x_dma_prep_lli(struct dma_chan *dchan, struct ls1x_dma_desc *desc, + struct scatterlist *sgl, unsigned int sg_len, + enum dma_transfer_direction dir, bool is_cyclic) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct ls1x_dma_lli *lli, *prev = NULL, *first = NULL; + struct device *dev = chan2dev(dchan); + struct list_head *pos = NULL; + struct scatterlist *sg; + unsigned int dev_addr, cmd, i; + + switch (dir) { + case DMA_MEM_TO_DEV: + dev_addr = chan->dst_addr; + chan->bus_width = chan->dst_addr_width; + cmd = LS1X_DMA_RAM2DEV | LS1X_DMA_INT; + break; + case DMA_DEV_TO_MEM: + dev_addr = chan->src_addr; + chan->bus_width = chan->src_addr_width; + cmd = LS1X_DMA_INT; + break; + default: + dev_err(dev, "unsupported DMA direction: %s\n", + dmaengine_get_direction_text(dir)); + return -EINVAL; + } + + for_each_sg(sgl, sg, sg_len, i) { + dma_addr_t buf_addr = sg_dma_address(sg); + size_t buf_len = sg_dma_len(sg); + dma_addr_t phys; + + if (!is_dma_copy_aligned(dchan->device, buf_addr, 0, buf_len)) { + dev_err(dev, "buffer is not aligned\n"); + return -EINVAL; + } + + /* allocate HW descriptors */ + lli = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, &phys); + if (!lli) { + dev_err(dev, "failed to alloc lli %u\n", i); + return -ENOMEM; + } + + /* setup HW descriptors */ + lli->phys = phys; + lli->hw[LS1X_DMADESC_SADDR] = buf_addr; + lli->hw[LS1X_DMADESC_DADDR] = dev_addr; + lli->hw[LS1X_DMADESC_LENGTH] = buf_len / chan->bus_width; + lli->hw[LS1X_DMADESC_STRIDE] = 0; + lli->hw[LS1X_DMADESC_CYCLES] = 1; + lli->hw[LS1X_DMADESC_CMD] = cmd; + + if (prev) + prev->hw[LS1X_DMADESC_NEXT] = + lli->phys | LS1X_DMA_NEXT_VALID; + prev = lli; + + if (!first) + first = lli; + + list_add_tail(&lli->node, &desc->lli_list); + } + + if (is_cyclic) { + lli->hw[LS1X_DMADESC_NEXT] = first->phys | LS1X_DMA_NEXT_VALID; + chan->is_cyclic = is_cyclic; + } + + list_for_each(pos, &desc->lli_list) { + lli = list_entry(pos, struct ls1x_dma_lli, node); + print_hex_dump_debug("LLI: ", DUMP_PREFIX_OFFSET, 16, 4, + lli, sizeof(*lli), false); + } + + return 0; +} + +static struct dma_async_tx_descriptor * +ls1x_dma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct ls1x_dma_desc *desc; + + dev_dbg(chan2dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n", + sg_len, flags, dmaengine_get_direction_text(dir)); + + desc = ls1x_dma_alloc_desc(); + if (!desc) + return NULL; + + if (ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, false)) { + ls1x_dma_free_desc(&desc->vd); + return NULL; + } + + return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags); +} + +static struct dma_async_tx_descriptor * +ls1x_dma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction dir, unsigned long flags) +{ + struct ls1x_dma_desc *desc; + struct scatterlist *sgl; + unsigned int sg_len; + unsigned int i; + int ret; + + dev_dbg(chan2dev(dchan), + "buf_len=%zu period_len=%zu flags=0x%lx dir=%s\n", + buf_len, period_len, flags, dmaengine_get_direction_text(dir)); + + desc = ls1x_dma_alloc_desc(); + if (!desc) + return NULL; + + /* allocate the scatterlist */ + sg_len = buf_len / period_len; + sgl = kmalloc_array(sg_len, sizeof(*sgl), GFP_NOWAIT); + if (!sgl) + return NULL; + + sg_init_table(sgl, sg_len); + for (i = 0; i < sg_len; ++i) { + sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(buf_addr)), + period_len, offset_in_page(buf_addr)); + sg_dma_address(&sgl[i]) = buf_addr; + sg_dma_len(&sgl[i]) = period_len; + buf_addr += period_len; + } + + ret = ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, true); + kfree(sgl); + if (ret) { + ls1x_dma_free_desc(&desc->vd); + return NULL; + } + + return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags); +} + +static int ls1x_dma_slave_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + + chan->src_addr = config->src_addr; + chan->src_addr_width = config->src_addr_width; + chan->dst_addr = config->dst_addr; + chan->dst_addr_width = config->dst_addr_width; + + return 0; +} + +static int ls1x_dma_pause(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + int ret; + + guard(spinlock_irqsave)(&chan->vc.lock); + /* save the current lli */ + ret = ls1x_dma_query(chan, &chan->curr_lli->phys); + if (!ret) + ls1x_dma_stop(chan); + + return ret; +} + +static int ls1x_dma_resume(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + + guard(spinlock_irqsave)(&chan->vc.lock); + + return ls1x_dma_start(chan, &chan->curr_lli->phys); +} + +static int ls1x_dma_terminate_all(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct virt_dma_desc *vd; + LIST_HEAD(head); + + ls1x_dma_stop(chan); + + scoped_guard(spinlock_irqsave, &chan->vc.lock) { + vd = vchan_next_desc(&chan->vc); + if (vd) + vchan_terminate_vdesc(vd); + + vchan_get_all_descriptors(&chan->vc, &head); + } + + vchan_dma_desc_free_list(&chan->vc, &head); + + return 0; +} + +static void ls1x_dma_synchronize(struct dma_chan *dchan) +{ + vchan_synchronize(to_virt_chan(dchan)); +} + +static enum dma_status ls1x_dma_tx_status(struct dma_chan *dchan, + dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct virt_dma_desc *vd; + enum dma_status status; + size_t bytes = 0; + + status = dma_cookie_status(dchan, cookie, state); + if (status == DMA_COMPLETE) + return status; + + scoped_guard(spinlock_irqsave, &chan->vc.lock) { + vd = vchan_find_desc(&chan->vc, cookie); + if (vd) { + struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd); + struct ls1x_dma_lli *lli; + dma_addr_t next_phys; + + /* get the current lli */ + if (ls1x_dma_query(chan, &chan->curr_lli->phys)) + return status; + + /* locate the current lli */ + next_phys = chan->curr_lli->hw[LS1X_DMADESC_NEXT]; + list_for_each_entry(lli, &desc->lli_list, node) + if (lli->hw[LS1X_DMADESC_NEXT] == next_phys) + break; + + dev_dbg(chan2dev(dchan), "current lli_phys=%pad", + &lli->phys); + + /* count the residues */ + list_for_each_entry_from(lli, &desc->lli_list, node) + bytes += lli->hw[LS1X_DMADESC_LENGTH] * + chan->bus_width; + } + } + + dma_set_residue(state, bytes); + + return status; +} + +static void ls1x_dma_issue_pending(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + + guard(spinlock_irqsave)(&chan->vc.lock); + + if (vchan_issue_pending(&chan->vc)) { + struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); + + if (vd) { + struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd); + struct ls1x_dma_lli *lli; + + lli = list_first_entry(&desc->lli_list, + struct ls1x_dma_lli, node); + ls1x_dma_start(chan, &lli->phys); + } + } +} + +static irqreturn_t ls1x_dma_irq_handler(int irq, void *data) +{ + struct ls1x_dma_chan *chan = data; + struct dma_chan *dchan = &chan->vc.chan; + struct device *dev = chan2dev(dchan); + struct virt_dma_desc *vd; + + scoped_guard(spinlock, &chan->vc.lock) { + vd = vchan_next_desc(&chan->vc); + if (!vd) { + dev_warn(dev, + "IRQ %d with no active desc on channel %d\n", + irq, dchan->chan_id); + return IRQ_NONE; + } + + if (chan->is_cyclic) { + vchan_cyclic_callback(vd); + } else { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + } + + dev_dbg(dev, "DMA IRQ %d on channel %d\n", irq, dchan->chan_id); + + return IRQ_HANDLED; +} + +static int ls1x_dma_chan_probe(struct platform_device *pdev, + struct ls1x_dma *dma) +{ + void __iomem *reg_base; + int id; + + reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg_base)) + return PTR_ERR(reg_base); + + for (id = 0; id < dma->nr_chans; id++) { + struct ls1x_dma_chan *chan = &dma->chan[id]; + char pdev_irqname[16]; + + snprintf(pdev_irqname, sizeof(pdev_irqname), "ch%d", id); + chan->irq = platform_get_irq_byname(pdev, pdev_irqname); + if (chan->irq < 0) + return dev_err_probe(&pdev->dev, chan->irq, + "failed to get IRQ for ch%d\n", + id); + + chan->reg_base = reg_base; + chan->vc.desc_free = ls1x_dma_free_desc; + vchan_init(&chan->vc, &dma->ddev); + } + + return 0; +} + +static void ls1x_dma_chan_remove(struct ls1x_dma *dma) +{ + int id; + + for (id = 0; id < dma->nr_chans; id++) { + struct ls1x_dma_chan *chan = &dma->chan[id]; + + if (chan->vc.chan.device == &dma->ddev) { + list_del(&chan->vc.chan.device_node); + tasklet_kill(&chan->vc.task); + } + } +} + +static int ls1x_dma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dma_device *ddev; + struct ls1x_dma *dma; + int ret; + + ret = platform_irq_count(pdev); + if (ret <= 0 || ret > LS1X_DMA_MAX_CHANNELS) + return dev_err_probe(dev, -EINVAL, + "Invalid number of IRQ channels: %d\n", + ret); + + dma = devm_kzalloc(dev, struct_size(dma, chan, ret), GFP_KERNEL); + if (!dma) + return -ENOMEM; + dma->nr_chans = ret; + + /* initialize DMA device */ + ddev = &dma->ddev; + ddev->dev = dev; + ddev->copy_align = DMAENGINE_ALIGN_4_BYTES; + ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + ddev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + ddev->device_alloc_chan_resources = ls1x_dma_alloc_chan_resources; + ddev->device_free_chan_resources = ls1x_dma_free_chan_resources; + ddev->device_prep_slave_sg = ls1x_dma_prep_slave_sg; + ddev->device_prep_dma_cyclic = ls1x_dma_prep_dma_cyclic; + ddev->device_config = ls1x_dma_slave_config; + ddev->device_pause = ls1x_dma_pause; + ddev->device_resume = ls1x_dma_resume; + ddev->device_terminate_all = ls1x_dma_terminate_all; + ddev->device_synchronize = ls1x_dma_synchronize; + ddev->device_tx_status = ls1x_dma_tx_status; + ddev->device_issue_pending = ls1x_dma_issue_pending; + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + INIT_LIST_HEAD(&ddev->channels); + + /* initialize DMA channels */ + ret = ls1x_dma_chan_probe(pdev, dma); + if (ret) + goto err; + + ret = dmaenginem_async_device_register(ddev); + if (ret) { + dev_err(dev, "failed to register DMA device\n"); + goto err; + } + + ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, + ddev); + if (ret) { + dev_err(dev, "failed to register DMA controller\n"); + goto err; + } + + platform_set_drvdata(pdev, dma); + dev_info(dev, "Loongson1 DMA driver registered\n"); + + return 0; + +err: + ls1x_dma_chan_remove(dma); + + return ret; +} + +static void ls1x_dma_remove(struct platform_device *pdev) +{ + struct ls1x_dma *dma = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + ls1x_dma_chan_remove(dma); +} + +static const struct of_device_id ls1x_dma_match[] = { + { .compatible = "loongson,ls1b-apbdma" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ls1x_dma_match); + +static struct platform_driver ls1x_dma_driver = { + .probe = ls1x_dma_probe, + .remove = ls1x_dma_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = ls1x_dma_match, + }, +}; + +module_platform_driver(ls1x_dma_driver); + +MODULE_AUTHOR("Keguang Zhang <keguang.zhang@gmail.com>"); +MODULE_DESCRIPTION("Loongson-1 APB DMA Controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/loongson2-apb-dma.c b/drivers/dma/loongson2-apb-dma.c new file mode 100644 index 000000000000..c528f02b9f84 --- /dev/null +++ b/drivers/dma/loongson2-apb-dma.c @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for the Loongson-2 APB DMA Controller + * + * Copyright (C) 2017-2023 Loongson Corporation + */ + +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +/* Global Configuration Register */ +#define LDMA_ORDER_ERG 0x0 + +/* Bitfield definitions */ + +/* Bitfields in Global Configuration Register */ +#define LDMA_64BIT_EN BIT(0) /* 1: 64 bit support */ +#define LDMA_UNCOHERENT_EN BIT(1) /* 0: cache, 1: uncache */ +#define LDMA_ASK_VALID BIT(2) +#define LDMA_START BIT(3) /* DMA start operation */ +#define LDMA_STOP BIT(4) /* DMA stop operation */ +#define LDMA_CONFIG_MASK GENMASK_ULL(4, 0) /* DMA controller config bits mask */ + +/* Bitfields in ndesc_addr field of HW descriptor */ +#define LDMA_DESC_EN BIT(0) /*1: The next descriptor is valid */ +#define LDMA_DESC_ADDR_LOW GENMASK(31, 1) + +/* Bitfields in cmd field of HW descriptor */ +#define LDMA_INT BIT(1) /* Enable DMA interrupts */ +#define LDMA_DATA_DIRECTION BIT(12) /* 1: write to device, 0: read from device */ + +#define LDMA_SLAVE_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +#define LDMA_MAX_TRANS_LEN U32_MAX + +/*-- descriptors -----------------------------------------------------*/ + +/* + * struct ls2x_dma_hw_desc - DMA HW descriptor + * @ndesc_addr: the next descriptor low address. + * @mem_addr: memory low address. + * @apb_addr: device buffer address. + * @len: length of a piece of carried content, in words. + * @step_len: length between two moved memory data blocks. + * @step_times: number of blocks to be carried in a single DMA operation. + * @cmd: descriptor command or state. + * @stats: DMA status. + * @high_ndesc_addr: the next descriptor high address. + * @high_mem_addr: memory high address. + * @reserved: reserved + */ +struct ls2x_dma_hw_desc { + u32 ndesc_addr; + u32 mem_addr; + u32 apb_addr; + u32 len; + u32 step_len; + u32 step_times; + u32 cmd; + u32 stats; + u32 high_ndesc_addr; + u32 high_mem_addr; + u32 reserved[2]; +} __packed; + +/* + * struct ls2x_dma_sg - ls2x dma scatter gather entry + * @hw: the pointer to DMA HW descriptor. + * @llp: physical address of the DMA HW descriptor. + * @phys: destination or source address(mem). + * @len: number of Bytes to read. + */ +struct ls2x_dma_sg { + struct ls2x_dma_hw_desc *hw; + dma_addr_t llp; + dma_addr_t phys; + u32 len; +}; + +/* + * struct ls2x_dma_desc - software descriptor + * @vdesc: pointer to the virtual dma descriptor. + * @cyclic: flag to dma cyclic + * @burst_size: burst size of transaction, in words. + * @desc_num: number of sg entries. + * @direction: transfer direction, to or from device. + * @status: dma controller status. + * @sg: array of sgs. + */ +struct ls2x_dma_desc { + struct virt_dma_desc vdesc; + bool cyclic; + size_t burst_size; + u32 desc_num; + enum dma_transfer_direction direction; + enum dma_status status; + struct ls2x_dma_sg sg[] __counted_by(desc_num); +}; + +/*-- Channels --------------------------------------------------------*/ + +/* + * struct ls2x_dma_chan - internal representation of an LS2X APB DMA channel + * @vchan: virtual dma channel entry. + * @desc: pointer to the ls2x sw dma descriptor. + * @pool: hw desc table + * @irq: irq line + * @sconfig: configuration for slave transfers, passed via .device_config + */ +struct ls2x_dma_chan { + struct virt_dma_chan vchan; + struct ls2x_dma_desc *desc; + void *pool; + int irq; + struct dma_slave_config sconfig; +}; + +/*-- Controller ------------------------------------------------------*/ + +/* + * struct ls2x_dma_priv - LS2X APB DMAC specific information + * @ddev: dmaengine dma_device object members + * @dma_clk: DMAC clock source + * @regs: memory mapped register base + * @lchan: channel to store ls2x_dma_chan structures + */ +struct ls2x_dma_priv { + struct dma_device ddev; + struct clk *dma_clk; + void __iomem *regs; + struct ls2x_dma_chan lchan; +}; + +/*-- Helper functions ------------------------------------------------*/ + +static inline struct ls2x_dma_desc *to_ldma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct ls2x_dma_desc, vdesc); +} + +static inline struct ls2x_dma_chan *to_ldma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct ls2x_dma_chan, vchan.chan); +} + +static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev) +{ + return container_of(ddev, struct ls2x_dma_priv, ddev); +} + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan); + struct ls2x_dma_desc *desc = to_ldma_desc(vdesc); + int i; + + for (i = 0; i < desc->desc_num; i++) { + if (desc->sg[i].hw) + dma_pool_free(lchan->pool, desc->sg[i].hw, + desc->sg[i].llp); + } + + kfree(desc); +} + +static void ls2x_dma_write_cmd(struct ls2x_dma_chan *lchan, bool cmd) +{ + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); + u64 val; + + val = lo_hi_readq(priv->regs + LDMA_ORDER_ERG) & ~LDMA_CONFIG_MASK; + val |= LDMA_64BIT_EN | cmd; + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); +} + +static void ls2x_dma_start_transfer(struct ls2x_dma_chan *lchan) +{ + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); + struct ls2x_dma_sg *ldma_sg; + struct virt_dma_desc *vdesc; + u64 val; + + /* Get the next descriptor */ + vdesc = vchan_next_desc(&lchan->vchan); + if (!vdesc) { + lchan->desc = NULL; + return; + } + + list_del(&vdesc->node); + lchan->desc = to_ldma_desc(vdesc); + ldma_sg = &lchan->desc->sg[0]; + + /* Start DMA */ + lo_hi_writeq(0, priv->regs + LDMA_ORDER_ERG); + val = (ldma_sg->llp & ~LDMA_CONFIG_MASK) | LDMA_64BIT_EN | LDMA_START; + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); +} + +static size_t ls2x_dmac_detect_burst(struct ls2x_dma_chan *lchan) +{ + u32 maxburst, buswidth; + + /* Reject definitely invalid configurations */ + if ((lchan->sconfig.src_addr_width & LDMA_SLAVE_BUSWIDTHS) && + (lchan->sconfig.dst_addr_width & LDMA_SLAVE_BUSWIDTHS)) + return 0; + + if (lchan->sconfig.direction == DMA_MEM_TO_DEV) { + maxburst = lchan->sconfig.dst_maxburst; + buswidth = lchan->sconfig.dst_addr_width; + } else { + maxburst = lchan->sconfig.src_maxburst; + buswidth = lchan->sconfig.src_addr_width; + } + + /* If maxburst is zero, fallback to LDMA_MAX_TRANS_LEN */ + return maxburst ? (maxburst * buswidth) >> 2 : LDMA_MAX_TRANS_LEN; +} + +static void ls2x_dma_fill_desc(struct ls2x_dma_chan *lchan, u32 sg_index, + struct ls2x_dma_desc *desc) +{ + struct ls2x_dma_sg *ldma_sg = &desc->sg[sg_index]; + u32 num_segments, segment_size; + + if (desc->direction == DMA_MEM_TO_DEV) { + ldma_sg->hw->cmd = LDMA_INT | LDMA_DATA_DIRECTION; + ldma_sg->hw->apb_addr = lchan->sconfig.dst_addr; + } else { + ldma_sg->hw->cmd = LDMA_INT; + ldma_sg->hw->apb_addr = lchan->sconfig.src_addr; + } + + ldma_sg->hw->mem_addr = lower_32_bits(ldma_sg->phys); + ldma_sg->hw->high_mem_addr = upper_32_bits(ldma_sg->phys); + + /* Split into multiple equally sized segments if necessary */ + num_segments = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, desc->burst_size); + segment_size = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, num_segments); + + /* Word count register takes input in words */ + ldma_sg->hw->len = segment_size; + ldma_sg->hw->step_times = num_segments; + ldma_sg->hw->step_len = 0; + + /* lets make a link list */ + if (sg_index) { + desc->sg[sg_index - 1].hw->ndesc_addr = ldma_sg->llp | LDMA_DESC_EN; + desc->sg[sg_index - 1].hw->high_ndesc_addr = upper_32_bits(ldma_sg->llp); + } +} + +/*-- DMA Engine API --------------------------------------------------*/ + +/* + * ls2x_dma_alloc_chan_resources - allocate resources for DMA channel + * @chan: allocate descriptor resources for this channel + * + * return - the number of allocated descriptors + */ +static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + /* Create a pool of consistent memory blocks for hardware descriptors */ + lchan->pool = dma_pool_create(dev_name(chan2dev(chan)), + chan->device->dev, PAGE_SIZE, + __alignof__(struct ls2x_dma_hw_desc), 0); + if (!lchan->pool) { + dev_err(chan2dev(chan), "No memory for descriptors\n"); + return -ENOMEM; + } + + return 1; +} + +/* + * ls2x_dma_free_chan_resources - free all channel resources + * @chan: DMA channel + */ +static void ls2x_dma_free_chan_resources(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + vchan_free_chan_resources(to_virt_chan(chan)); + dma_pool_destroy(lchan->pool); + lchan->pool = NULL; +} + +/* + * ls2x_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction + * @chan: DMA channel + * @sgl: scatterlist to transfer to/from + * @sg_len: number of entries in @scatterlist + * @direction: DMA direction + * @flags: tx descriptor status flags + * @context: transaction context (ignored) + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +ls2x_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + u32 sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + struct ls2x_dma_desc *desc; + struct scatterlist *sg; + size_t burst_size; + int i; + + if (unlikely(!sg_len || !is_slave_direction(direction))) + return NULL; + + burst_size = ls2x_dmac_detect_burst(lchan); + if (!burst_size) + return NULL; + + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->desc_num = sg_len; + desc->direction = direction; + desc->burst_size = burst_size; + + for_each_sg(sgl, sg, sg_len, i) { + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; + + /* Allocate DMA capable memory for hardware descriptor */ + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); + if (!ldma_sg->hw) { + desc->desc_num = i; + ls2x_dma_desc_free(&desc->vdesc); + return NULL; + } + + ldma_sg->phys = sg_dma_address(sg); + ldma_sg->len = sg_dma_len(sg); + + ls2x_dma_fill_desc(lchan, i, desc); + } + + /* Setting the last descriptor enable bit */ + desc->sg[sg_len - 1].hw->ndesc_addr &= ~LDMA_DESC_EN; + desc->status = DMA_IN_PROGRESS; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +/* + * ls2x_dma_prep_dma_cyclic - prepare the cyclic DMA transfer + * @chan: the DMA channel to prepare + * @buf_addr: physical DMA address where the buffer starts + * @buf_len: total number of bytes for the entire buffer + * @period_len: number of bytes for each period + * @direction: transfer direction, to or from device + * @flags: tx descriptor status flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +ls2x_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + struct ls2x_dma_desc *desc; + size_t burst_size; + u32 num_periods; + int i; + + if (unlikely(!buf_len || !period_len)) + return NULL; + + if (unlikely(!is_slave_direction(direction))) + return NULL; + + burst_size = ls2x_dmac_detect_burst(lchan); + if (!burst_size) + return NULL; + + num_periods = buf_len / period_len; + desc = kzalloc(struct_size(desc, sg, num_periods), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->desc_num = num_periods; + desc->direction = direction; + desc->burst_size = burst_size; + + /* Build cyclic linked list */ + for (i = 0; i < num_periods; i++) { + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; + + /* Allocate DMA capable memory for hardware descriptor */ + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); + if (!ldma_sg->hw) { + desc->desc_num = i; + ls2x_dma_desc_free(&desc->vdesc); + return NULL; + } + + ldma_sg->phys = buf_addr + period_len * i; + ldma_sg->len = period_len; + + ls2x_dma_fill_desc(lchan, i, desc); + } + + /* Lets make a cyclic list */ + desc->sg[num_periods - 1].hw->ndesc_addr = desc->sg[0].llp | LDMA_DESC_EN; + desc->sg[num_periods - 1].hw->high_ndesc_addr = upper_32_bits(desc->sg[0].llp); + desc->cyclic = true; + desc->status = DMA_IN_PROGRESS; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +/* + * ls2x_slave_config - set slave configuration for channel + * @chan: dma channel + * @cfg: slave configuration + * + * Sets slave configuration for channel + */ +static int ls2x_dma_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + memcpy(&lchan->sconfig, config, sizeof(*config)); + return 0; +} + +/* + * ls2x_dma_issue_pending - push pending transactions to the hardware + * @chan: channel + * + * When this function is called, all pending transactions are pushed to the + * hardware and executed. + */ +static void ls2x_dma_issue_pending(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) + ls2x_dma_start_transfer(lchan); + spin_unlock_irqrestore(&lchan->vchan.lock, flags); +} + +/* + * ls2x_dma_terminate_all - terminate all transactions + * @chan: channel + * + * Stops all DMA transactions. + */ +static int ls2x_dma_terminate_all(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&lchan->vchan.lock, flags); + /* Setting stop cmd */ + ls2x_dma_write_cmd(lchan, LDMA_STOP); + if (lchan->desc) { + vchan_terminate_vdesc(&lchan->desc->vdesc); + lchan->desc = NULL; + } + + vchan_get_all_descriptors(&lchan->vchan, &head); + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + vchan_dma_desc_free_list(&lchan->vchan, &head); + return 0; +} + +/* + * ls2x_dma_synchronize - Synchronizes the termination of transfers to the + * current context. + * @chan: channel + */ +static void ls2x_dma_synchronize(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + vchan_synchronize(&lchan->vchan); +} + +static int ls2x_dma_pause(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) { + ls2x_dma_write_cmd(lchan, LDMA_STOP); + lchan->desc->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + return 0; +} + +static int ls2x_dma_resume(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (lchan->desc && lchan->desc->status == DMA_PAUSED) { + lchan->desc->status = DMA_IN_PROGRESS; + ls2x_dma_write_cmd(lchan, LDMA_START); + } + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + return 0; +} + +/* + * ls2x_dma_isr - LS2X DMA Interrupt handler + * @irq: IRQ number + * @dev_id: Pointer to ls2x_dma_chan + * + * Return: IRQ_HANDLED/IRQ_NONE + */ +static irqreturn_t ls2x_dma_isr(int irq, void *dev_id) +{ + struct ls2x_dma_chan *lchan = dev_id; + struct ls2x_dma_desc *desc; + + spin_lock(&lchan->vchan.lock); + desc = lchan->desc; + if (desc) { + if (desc->cyclic) { + vchan_cyclic_callback(&desc->vdesc); + } else { + desc->status = DMA_COMPLETE; + vchan_cookie_complete(&desc->vdesc); + ls2x_dma_start_transfer(lchan); + } + + /* ls2x_dma_start_transfer() updates lchan->desc */ + if (!lchan->desc) + ls2x_dma_write_cmd(lchan, LDMA_STOP); + } + spin_unlock(&lchan->vchan.lock); + + return IRQ_HANDLED; +} + +static int ls2x_dma_chan_init(struct platform_device *pdev, + struct ls2x_dma_priv *priv) +{ + struct ls2x_dma_chan *lchan = &priv->lchan; + struct device *dev = &pdev->dev; + int ret; + + lchan->irq = platform_get_irq(pdev, 0); + if (lchan->irq < 0) + return lchan->irq; + + ret = devm_request_irq(dev, lchan->irq, ls2x_dma_isr, IRQF_TRIGGER_RISING, + dev_name(&pdev->dev), lchan); + if (ret) + return ret; + + /* Initialize channels related values */ + INIT_LIST_HEAD(&priv->ddev.channels); + lchan->vchan.desc_free = ls2x_dma_desc_free; + vchan_init(&lchan->vchan, &priv->ddev); + + return 0; +} + +/* + * ls2x_dma_probe - Driver probe function + * @pdev: Pointer to the platform_device structure + * + * Return: '0' on success and failure value on error + */ +static int ls2x_dma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ls2x_dma_priv *priv; + struct dma_device *ddev; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->regs)) + return dev_err_probe(dev, PTR_ERR(priv->regs), + "devm_platform_ioremap_resource failed.\n"); + + priv->dma_clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(priv->dma_clk)) + return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "devm_clk_get failed.\n"); + + ret = clk_prepare_enable(priv->dma_clk); + if (ret) + return dev_err_probe(dev, ret, "clk_prepare_enable failed.\n"); + + ret = ls2x_dma_chan_init(pdev, priv); + if (ret) + goto disable_clk; + + ddev = &priv->ddev; + ddev->dev = dev; + dma_cap_zero(ddev->cap_mask); + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); + + ddev->device_alloc_chan_resources = ls2x_dma_alloc_chan_resources; + ddev->device_free_chan_resources = ls2x_dma_free_chan_resources; + ddev->device_tx_status = dma_cookie_status; + ddev->device_issue_pending = ls2x_dma_issue_pending; + ddev->device_prep_slave_sg = ls2x_dma_prep_slave_sg; + ddev->device_prep_dma_cyclic = ls2x_dma_prep_dma_cyclic; + ddev->device_config = ls2x_dma_slave_config; + ddev->device_terminate_all = ls2x_dma_terminate_all; + ddev->device_synchronize = ls2x_dma_synchronize; + ddev->device_pause = ls2x_dma_pause; + ddev->device_resume = ls2x_dma_resume; + + ddev->src_addr_widths = LDMA_SLAVE_BUSWIDTHS; + ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS; + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + + ret = dma_async_device_register(&priv->ddev); + if (ret < 0) + goto disable_clk; + + ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv); + if (ret < 0) + goto unregister_dmac; + + platform_set_drvdata(pdev, priv); + + dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n"); + return 0; + +unregister_dmac: + dma_async_device_unregister(&priv->ddev); +disable_clk: + clk_disable_unprepare(priv->dma_clk); + + return ret; +} + +/* + * ls2x_dma_remove - Driver remove function + * @pdev: Pointer to the platform_device structure + */ +static void ls2x_dma_remove(struct platform_device *pdev) +{ + struct ls2x_dma_priv *priv = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&priv->ddev); + clk_disable_unprepare(priv->dma_clk); +} + +static const struct of_device_id ls2x_dma_of_match_table[] = { + { .compatible = "loongson,ls2k1000-apbdma" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ls2x_dma_of_match_table); + +static struct platform_driver ls2x_dmac_driver = { + .probe = ls2x_dma_probe, + .remove = ls2x_dma_remove, + .driver = { + .name = "ls2x-apbdma", + .of_match_table = ls2x_dma_of_match_table, + }, +}; +module_platform_driver(ls2x_dmac_driver); + +MODULE_DESCRIPTION("Loongson-2 APB DMA Controller driver"); +MODULE_AUTHOR("Loongson Technology Corporation Limited"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/lpc18xx-dmamux.c b/drivers/dma/lpc18xx-dmamux.c index df98cae8792b..2b6436f4b193 100644 --- a/drivers/dma/lpc18xx-dmamux.c +++ b/drivers/dma/lpc18xx-dmamux.c @@ -12,8 +12,10 @@ #include <linux/err.h> #include <linux/init.h> #include <linux/mfd/syscon.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/spinlock.h> diff --git a/drivers/dma/lpc32xx-dmamux.c b/drivers/dma/lpc32xx-dmamux.c new file mode 100644 index 000000000000..351d7e23e615 --- /dev/null +++ b/drivers/dma/lpc32xx-dmamux.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Copyright 2024 Timesys Corporation <piotr.wojtaszczyk@timesys.com> +// +// Based on TI DMA Crossbar driver by: +// Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com +// Author: Peter Ujfalusi <peter.ujfalusi@ti.com> + +#include <linux/err.h> +#include <linux/init.h> +#include <linux/mfd/syscon.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/spinlock.h> + +#define LPC32XX_SSP_CLK_CTRL 0x78 +#define LPC32XX_I2S_CLK_CTRL 0x7c + +struct lpc32xx_dmamux { + int signal; + char *name_sel0; + char *name_sel1; + int muxval; + int muxreg; + int bit; + bool busy; +}; + +struct lpc32xx_dmamux_data { + struct dma_router dmarouter; + struct regmap *reg; + spinlock_t lock; /* protects busy status flag */ +}; + +/* From LPC32x0 User manual "3.2.1 DMA request signals" */ +static struct lpc32xx_dmamux lpc32xx_muxes[] = { + { + .signal = 3, + .name_sel0 = "spi2-rx-tx", + .name_sel1 = "ssp1-rx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 5, + }, + { + .signal = 10, + .name_sel0 = "uart7-rx", + .name_sel1 = "i2s1-dma1", + .muxreg = LPC32XX_I2S_CLK_CTRL, + .bit = 4, + }, + { + .signal = 11, + .name_sel0 = "spi1-rx-tx", + .name_sel1 = "ssp1-tx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 4, + }, + { + .signal = 14, + .name_sel0 = "none", + .name_sel1 = "ssp0-rx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 3, + }, + { + .signal = 15, + .name_sel0 = "none", + .name_sel1 = "ssp0-tx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 2, + }, +}; + +static void lpc32xx_dmamux_release(struct device *dev, void *route_data) +{ + struct lpc32xx_dmamux_data *dmamux = dev_get_drvdata(dev); + struct lpc32xx_dmamux *mux = route_data; + + dev_dbg(dev, "releasing dma request signal %d routed to %s\n", + mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); + + guard(spinlock)(&dmamux->lock); + + mux->busy = false; +} + +static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct device *dev = &pdev->dev; + struct lpc32xx_dmamux_data *dmamux = platform_get_drvdata(pdev); + unsigned long flags; + struct lpc32xx_dmamux *mux = NULL; + int i; + + if (dma_spec->args_count != 3) { + dev_err(&pdev->dev, "invalid number of dma mux args\n"); + return ERR_PTR(-EINVAL); + } + + for (i = 0; i < ARRAY_SIZE(lpc32xx_muxes); i++) { + if (lpc32xx_muxes[i].signal == dma_spec->args[0]) { + mux = &lpc32xx_muxes[i]; + break; + } + } + if (!mux) { + dev_err(&pdev->dev, "invalid mux request number: %d\n", + dma_spec->args[0]); + return ERR_PTR(-EINVAL); + } + + if (dma_spec->args[2] > 1) { + dev_err(&pdev->dev, "invalid dma mux value: %d\n", + dma_spec->args[1]); + return ERR_PTR(-EINVAL); + } + + /* The of_node_put() will be done in the core for the node */ + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); + if (!dma_spec->np) { + dev_err(&pdev->dev, "can't get dma master\n"); + return ERR_PTR(-EINVAL); + } + + spin_lock_irqsave(&dmamux->lock, flags); + if (mux->busy) { + spin_unlock_irqrestore(&dmamux->lock, flags); + dev_err(dev, "dma request signal %d busy, routed to %s\n", + mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); + of_node_put(dma_spec->np); + return ERR_PTR(-EBUSY); + } + + mux->busy = true; + mux->muxval = dma_spec->args[2] ? BIT(mux->bit) : 0; + + regmap_update_bits(dmamux->reg, mux->muxreg, BIT(mux->bit), mux->muxval); + spin_unlock_irqrestore(&dmamux->lock, flags); + + dma_spec->args[2] = 0; + dma_spec->args_count = 2; + + dev_dbg(dev, "dma request signal %d routed to %s\n", + mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); + + return mux; +} + +static int lpc32xx_dmamux_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct lpc32xx_dmamux_data *dmamux; + + dmamux = devm_kzalloc(&pdev->dev, sizeof(*dmamux), GFP_KERNEL); + if (!dmamux) + return -ENOMEM; + + dmamux->reg = syscon_node_to_regmap(np->parent); + if (IS_ERR(dmamux->reg)) { + dev_err(&pdev->dev, "syscon lookup failed\n"); + return PTR_ERR(dmamux->reg); + } + + spin_lock_init(&dmamux->lock); + platform_set_drvdata(pdev, dmamux); + dmamux->dmarouter.dev = &pdev->dev; + dmamux->dmarouter.route_free = lpc32xx_dmamux_release; + + return of_dma_router_register(np, lpc32xx_dmamux_reserve, + &dmamux->dmarouter); +} + +static const struct of_device_id lpc32xx_dmamux_match[] = { + { .compatible = "nxp,lpc3220-dmamux" }, + {}, +}; + +static struct platform_driver lpc32xx_dmamux_driver = { + .probe = lpc32xx_dmamux_probe, + .driver = { + .name = "lpc32xx-dmamux", + .of_match_table = lpc32xx_dmamux_match, + }, +}; + +static int __init lpc32xx_dmamux_init(void) +{ + return platform_driver_register(&lpc32xx_dmamux_driver); +} +arch_initcall(lpc32xx_dmamux_init); diff --git a/drivers/dma/mcf-edma.c b/drivers/dma/mcf-edma-main.c index 9413fad08a60..9e1c6400c77b 100644 --- a/drivers/dma/mcf-edma.c +++ b/drivers/dma/mcf-edma-main.c @@ -19,7 +19,6 @@ static irqreturn_t mcf_edma_tx_handler(int irq, void *dev_id) struct fsl_edma_engine *mcf_edma = dev_id; struct edma_regs *regs = &mcf_edma->regs; unsigned int ch; - struct fsl_edma_chan *mcf_chan; u64 intmap; intmap = ioread32(regs->inth); @@ -31,31 +30,7 @@ static irqreturn_t mcf_edma_tx_handler(int irq, void *dev_id) for (ch = 0; ch < mcf_edma->n_chans; ch++) { if (intmap & BIT(ch)) { iowrite8(EDMA_MASK_CH(ch), regs->cint); - - mcf_chan = &mcf_edma->chans[ch]; - - spin_lock(&mcf_chan->vchan.lock); - - if (!mcf_chan->edesc) { - /* terminate_all called before */ - spin_unlock(&mcf_chan->vchan.lock); - continue; - } - - if (!mcf_chan->edesc->iscyclic) { - list_del(&mcf_chan->edesc->vdesc.node); - vchan_cookie_complete(&mcf_chan->edesc->vdesc); - mcf_chan->edesc = NULL; - mcf_chan->status = DMA_COMPLETE; - mcf_chan->idle = true; - } else { - vchan_cyclic_callback(&mcf_chan->edesc->vdesc); - } - - if (!mcf_chan->edesc) - fsl_edma_xfer_desc(mcf_chan); - - spin_unlock(&mcf_chan->vchan.lock); + fsl_edma_tx_chan_handler(&mcf_edma->chans[ch]); } } @@ -76,8 +51,7 @@ static irqreturn_t mcf_edma_err_handler(int irq, void *dev_id) if (err & BIT(ch)) { fsl_edma_disable_request(&mcf_edma->chans[ch]); iowrite8(EDMA_CERR_CERR(ch), regs->cerr); - mcf_edma->chans[ch].status = DMA_ERROR; - mcf_edma->chans[ch].idle = true; + fsl_edma_err_chan_handler(&mcf_edma->chans[ch]); } } @@ -90,7 +64,6 @@ static irqreturn_t mcf_edma_err_handler(int irq, void *dev_id) fsl_edma_disable_request(&mcf_edma->chans[ch]); iowrite8(EDMA_CERR_CERR(ch), regs->cerr); mcf_edma->chans[ch].status = DMA_ERROR; - mcf_edma->chans[ch].idle = true; } } @@ -172,7 +145,7 @@ static void mcf_edma_irq_free(struct platform_device *pdev, } static struct fsl_edma_drvdata mcf_data = { - .version = v2, + .flags = FSL_EDMA_DRV_EDMA64, .setup_irq = mcf_edma_irq_init, }; @@ -180,9 +153,8 @@ static int mcf_edma_probe(struct platform_device *pdev) { struct mcf_edma_platform_data *pdata; struct fsl_edma_engine *mcf_edma; - struct fsl_edma_chan *mcf_chan; struct edma_regs *regs; - int ret, i, len, chans; + int ret, i, chans; pdata = dev_get_platdata(&pdev->dev); if (!pdata) { @@ -197,8 +169,8 @@ static int mcf_edma_probe(struct platform_device *pdev) chans = pdata->dma_channels; } - len = sizeof(*mcf_edma) + sizeof(*mcf_chan) * chans; - mcf_edma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + mcf_edma = devm_kzalloc(&pdev->dev, struct_size(mcf_edma, chans, chans), + GFP_KERNEL); if (!mcf_edma) return -ENOMEM; @@ -222,12 +194,13 @@ static int mcf_edma_probe(struct platform_device *pdev) struct fsl_edma_chan *mcf_chan = &mcf_edma->chans[i]; mcf_chan->edma = mcf_edma; - mcf_chan->slave_id = i; - mcf_chan->idle = true; + mcf_chan->srcid = i; mcf_chan->dma_dir = DMA_NONE; mcf_chan->vchan.desc_free = fsl_edma_free_desc; vchan_init(&mcf_chan->vchan, &mcf_edma->dma_dev); - iowrite32(0x0, ®s->tcd[i].csr); + mcf_chan->tcd = mcf_edma->membase + EDMA_TCD + + i * sizeof(struct fsl_edma_hw_tcd); + edma_write_tcdreg(mcf_chan, cpu_to_le32(0), csr); } iowrite32(~0, regs->inth); @@ -280,15 +253,13 @@ static int mcf_edma_probe(struct platform_device *pdev) return 0; } -static int mcf_edma_remove(struct platform_device *pdev) +static void mcf_edma_remove(struct platform_device *pdev) { struct fsl_edma_engine *mcf_edma = platform_get_drvdata(pdev); mcf_edma_irq_free(pdev, mcf_edma); fsl_edma_cleanup_vchan(&mcf_edma->dma_dev); dma_async_device_unregister(&mcf_edma->dma_dev); - - return 0; } static struct platform_driver mcf_edma_driver = { @@ -304,7 +275,7 @@ bool mcf_edma_filter_fn(struct dma_chan *chan, void *param) if (chan->device->dev->driver == &mcf_edma_driver.driver) { struct fsl_edma_chan *mcf_chan = to_fsl_edma_chan(chan); - return (mcf_chan->slave_id == (uintptr_t)param); + return (mcf_chan->srcid == (uintptr_t)param); } return false; diff --git a/drivers/dma/mediatek/mtk-cqdma.c b/drivers/dma/mediatek/mtk-cqdma.c index 9ae92b8940ef..9f0c41ca7770 100644 --- a/drivers/dma/mediatek/mtk-cqdma.c +++ b/drivers/dma/mediatek/mtk-cqdma.c @@ -18,7 +18,6 @@ #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -421,15 +420,11 @@ static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, { struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); struct virt_dma_desc *vd; - unsigned long flags; - spin_lock_irqsave(&cvc->pc->lock, flags); list_for_each_entry(vd, &cvc->pc->queue, node) if (vd->tx.cookie == cookie) { - spin_unlock_irqrestore(&cvc->pc->lock, flags); return vd; } - spin_unlock_irqrestore(&cvc->pc->lock, flags); list_for_each_entry(vd, &cvc->vc.desc_issued, node) if (vd->tx.cookie == cookie) @@ -453,9 +448,11 @@ static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, if (ret == DMA_COMPLETE || !txstate) return ret; - spin_lock_irqsave(&cvc->vc.lock, flags); + spin_lock_irqsave(&cvc->pc->lock, flags); + spin_lock(&cvc->vc.lock); vd = mtk_cqdma_find_active_desc(c, cookie); - spin_unlock_irqrestore(&cvc->vc.lock, flags); + spin_unlock(&cvc->vc.lock); + spin_unlock_irqrestore(&cvc->pc->lock, flags); if (vd) { cvd = to_cqdma_vdesc(vd); @@ -519,7 +516,7 @@ mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, /* setup dma channel */ cvd[i]->ch = c; - /* setup sourece, destination, and length */ + /* setup source, destination, and length */ tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len; cvd[i]->len = tlen; cvd[i]->src = src; @@ -618,7 +615,7 @@ static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c) u32 i, min_refcnt = U32_MAX, refcnt; unsigned long flags; - /* allocate PC with the minimun refcount */ + /* allocate PC with the minimum refcount */ for (i = 0; i < cqdma->dma_channels; ++i) { refcnt = refcount_read(&cqdma->pc[i]->refcnt); if (refcnt < min_refcnt) { @@ -886,7 +883,7 @@ err_unregister: return err; } -static int mtk_cqdma_remove(struct platform_device *pdev) +static void mtk_cqdma_remove(struct platform_device *pdev) { struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev); struct mtk_cqdma_vchan *vc; @@ -919,8 +916,6 @@ static int mtk_cqdma_remove(struct platform_device *pdev) dma_async_device_unregister(&cqdma->ddev); of_dma_controller_free(pdev->dev.of_node); - - return 0; } static struct platform_driver mtk_cqdma_driver = { diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c index 69cc61c0b262..fa77bb24a430 100644 --- a/drivers/dma/mediatek/mtk-hsdma.c +++ b/drivers/dma/mediatek/mtk-hsdma.c @@ -17,7 +17,6 @@ #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -227,7 +226,7 @@ struct mtk_hsdma_soc { * @pc_refcnt: Track how many VCs are using the PC * @lock: Lock protect agaisting multiple VCs access PC * @soc: The pointer to area holding differences among - * vaious platform + * various platform */ struct mtk_hsdma_device { struct dma_device ddev; @@ -1010,7 +1009,7 @@ err_unregister: return err; } -static int mtk_hsdma_remove(struct platform_device *pdev) +static void mtk_hsdma_remove(struct platform_device *pdev) { struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev); struct mtk_hsdma_vchan *vc; @@ -1035,8 +1034,6 @@ static int mtk_hsdma_remove(struct platform_device *pdev) dma_async_device_unregister(&hsdma->ddev); of_dma_controller_free(pdev->dev.of_node); - - return 0; } static struct platform_driver mtk_hsdma_driver = { diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index a1517ef1f4a0..08e15177427b 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -16,7 +16,6 @@ #include <linux/kernel.h> #include <linux/list.h> #include <linux/module.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -451,9 +450,8 @@ static int mtk_uart_apdma_device_pause(struct dma_chan *chan) mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); - synchronize_irq(c->irq); - spin_unlock_irqrestore(&c->vc.lock, flags); + synchronize_irq(c->irq); return 0; } @@ -551,7 +549,6 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev) } pm_runtime_enable(&pdev->dev); - pm_runtime_set_active(&pdev->dev); rc = dma_async_device_register(&mtkd->ddev); if (rc) @@ -575,7 +572,7 @@ err_no_dma: return rc; } -static int mtk_uart_apdma_remove(struct platform_device *pdev) +static void mtk_uart_apdma_remove(struct platform_device *pdev) { struct mtk_uart_apdmadev *mtkd = platform_get_drvdata(pdev); @@ -586,8 +583,6 @@ static int mtk_uart_apdma_remove(struct platform_device *pdev) dma_async_device_unregister(&mtkd->ddev); pm_runtime_disable(&pdev->dev); - - return 0; } #ifdef CONFIG_PM_SLEEP @@ -642,7 +637,7 @@ static const struct dev_pm_ops mtk_uart_apdma_pm_ops = { static struct platform_driver mtk_uart_apdma_driver = { .probe = mtk_uart_apdma_probe, - .remove = mtk_uart_apdma_remove, + .remove = mtk_uart_apdma_remove, .driver = { .name = KBUILD_MODNAME, .pm = &mtk_uart_apdma_pm_ops, diff --git a/drivers/dma/milbeaut-hdmac.c b/drivers/dma/milbeaut-hdmac.c index 1b0a95892627..9a5ec247ed6d 100644 --- a/drivers/dma/milbeaut-hdmac.c +++ b/drivers/dma/milbeaut-hdmac.c @@ -531,7 +531,7 @@ disable_clk: return ret; } -static int milbeaut_hdmac_remove(struct platform_device *pdev) +static void milbeaut_hdmac_remove(struct platform_device *pdev) { struct milbeaut_hdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -546,16 +546,21 @@ static int milbeaut_hdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } milbeaut_hdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&mdev->ddev); clk_disable_unprepare(mdev->clk); - - return 0; } static const struct of_device_id milbeaut_hdmac_match[] = { diff --git a/drivers/dma/milbeaut-xdmac.c b/drivers/dma/milbeaut-xdmac.c index d29d01e730aa..58d4fd6df0bf 100644 --- a/drivers/dma/milbeaut-xdmac.c +++ b/drivers/dma/milbeaut-xdmac.c @@ -368,7 +368,7 @@ disable_xdmac: return ret; } -static int milbeaut_xdmac_remove(struct platform_device *pdev) +static void milbeaut_xdmac_remove(struct platform_device *pdev) { struct milbeaut_xdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -383,8 +383,15 @@ static int milbeaut_xdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } milbeaut_xdmac_free_chan_resources(chan); } @@ -392,8 +399,6 @@ static int milbeaut_xdmac_remove(struct platform_device *pdev) dma_async_device_unregister(&mdev->ddev); disable_xdmac(mdev); - - return 0; } static const struct of_device_id milbeaut_xdmac_match[] = { diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index ebdfdcbb4f7a..d07229a74886 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -15,7 +15,8 @@ #include <linux/device.h> #include <linux/platform_data/mmp_dma.h> #include <linux/dmapool.h> -#include <linux/of_device.h> +#include <linux/clk.h> +#include <linux/reset.h> #include <linux/of_dma.h> #include <linux/of.h> @@ -24,9 +25,12 @@ #define DCSR 0x0000 #define DALGN 0x00a0 #define DINT 0x00f0 -#define DDADR 0x0200 +#define DDADR(n) (0x0200 + ((n) << 4)) #define DSADR(n) (0x0204 + ((n) << 4)) #define DTADR(n) (0x0208 + ((n) << 4)) +#define DDADRH(n) (0x0300 + ((n) << 4)) +#define DSADRH(n) (0x0304 + ((n) << 4)) +#define DTADRH(n) (0x0308 + ((n) << 4)) #define DCMD 0x020c #define DCSR_RUN BIT(31) /* Run Bit (read / write) */ @@ -43,6 +47,7 @@ #define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ #define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ #define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ +#define DCSR_LPAEEN BIT(21) /* Long Physical Address Extension Enable */ #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ #define DCSR_EORINTR BIT(9) /* The end of Receive */ @@ -75,6 +80,16 @@ struct mmp_pdma_desc_hw { u32 dsadr; /* DSADR value for the current transfer */ u32 dtadr; /* DTADR value for the current transfer */ u32 dcmd; /* DCMD value for the current transfer */ + /* + * The following 32-bit words are only used in the 64-bit, ie. + * LPAE (Long Physical Address Extension) mode. + * They are used to specify the high 32 bits of the descriptor's + * addresses. + */ + u32 ddadrh; /* High 32-bit of DDADR */ + u32 dsadrh; /* High 32-bit of DSADR */ + u32 dtadrh; /* High 32-bit of DTADR */ + u32 rsvd; /* reserved */ } __aligned(32); struct mmp_pdma_desc_sw { @@ -119,12 +134,55 @@ struct mmp_pdma_phy { struct mmp_pdma_chan *vchan; }; +/** + * struct mmp_pdma_ops - Operations for the MMP PDMA controller + * + * Hardware Register Operations (read/write hardware registers): + * @write_next_addr: Function to program address of next descriptor into + * DDADR/DDADRH + * @read_src_addr: Function to read the source address from DSADR/DSADRH + * @read_dst_addr: Function to read the destination address from DTADR/DTADRH + * + * Descriptor Memory Operations (manipulate descriptor structs in memory): + * @set_desc_next_addr: Function to set next descriptor address in descriptor + * @set_desc_src_addr: Function to set the source address in descriptor + * @set_desc_dst_addr: Function to set the destination address in descriptor + * @get_desc_src_addr: Function to get the source address from descriptor + * @get_desc_dst_addr: Function to get the destination address from descriptor + * + * Controller Configuration: + * @run_bits: Control bits in DCSR register for channel start/stop + * @dma_mask: DMA addressing capability of controller. 0 to use OF/platform + * settings, or explicit mask like DMA_BIT_MASK(32/64) + */ +struct mmp_pdma_ops { + /* Hardware Register Operations */ + void (*write_next_addr)(struct mmp_pdma_phy *phy, dma_addr_t addr); + u64 (*read_src_addr)(struct mmp_pdma_phy *phy); + u64 (*read_dst_addr)(struct mmp_pdma_phy *phy); + + /* Descriptor Memory Operations */ + void (*set_desc_next_addr)(struct mmp_pdma_desc_hw *desc, + dma_addr_t addr); + void (*set_desc_src_addr)(struct mmp_pdma_desc_hw *desc, + dma_addr_t addr); + void (*set_desc_dst_addr)(struct mmp_pdma_desc_hw *desc, + dma_addr_t addr); + u64 (*get_desc_src_addr)(const struct mmp_pdma_desc_hw *desc); + u64 (*get_desc_dst_addr)(const struct mmp_pdma_desc_hw *desc); + + /* Controller Configuration */ + u32 run_bits; + u64 dma_mask; +}; + struct mmp_pdma_device { int dma_channels; void __iomem *base; struct device *dev; struct dma_device device; struct mmp_pdma_phy *phy; + const struct mmp_pdma_ops *ops; spinlock_t phy_lock; /* protect alloc/free phy channels */ }; @@ -137,24 +195,112 @@ struct mmp_pdma_device { #define to_mmp_pdma_dev(dmadev) \ container_of(dmadev, struct mmp_pdma_device, device) -static int mmp_pdma_config_write(struct dma_chan *dchan, - struct dma_slave_config *cfg, - enum dma_transfer_direction direction); +/* For 32-bit PDMA */ +static void write_next_addr_32(struct mmp_pdma_phy *phy, dma_addr_t addr) +{ + writel(addr, phy->base + DDADR(phy->idx)); +} + +static u64 read_src_addr_32(struct mmp_pdma_phy *phy) +{ + return readl(phy->base + DSADR(phy->idx)); +} + +static u64 read_dst_addr_32(struct mmp_pdma_phy *phy) +{ + return readl(phy->base + DTADR(phy->idx)); +} + +static void set_desc_next_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->ddadr = addr; +} + +static void set_desc_src_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dsadr = addr; +} + +static void set_desc_dst_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dtadr = addr; +} + +static u64 get_desc_src_addr_32(const struct mmp_pdma_desc_hw *desc) +{ + return desc->dsadr; +} + +static u64 get_desc_dst_addr_32(const struct mmp_pdma_desc_hw *desc) +{ + return desc->dtadr; +} + +/* For 64-bit PDMA */ +static void write_next_addr_64(struct mmp_pdma_phy *phy, dma_addr_t addr) +{ + writel(lower_32_bits(addr), phy->base + DDADR(phy->idx)); + writel(upper_32_bits(addr), phy->base + DDADRH(phy->idx)); +} + +static u64 read_src_addr_64(struct mmp_pdma_phy *phy) +{ + u32 low = readl(phy->base + DSADR(phy->idx)); + u32 high = readl(phy->base + DSADRH(phy->idx)); + + return ((u64)high << 32) | low; +} -static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr) +static u64 read_dst_addr_64(struct mmp_pdma_phy *phy) { - u32 reg = (phy->idx << 4) + DDADR; + u32 low = readl(phy->base + DTADR(phy->idx)); + u32 high = readl(phy->base + DTADRH(phy->idx)); - writel(addr, phy->base + reg); + return ((u64)high << 32) | low; } +static void set_desc_next_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->ddadr = lower_32_bits(addr); + desc->ddadrh = upper_32_bits(addr); +} + +static void set_desc_src_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dsadr = lower_32_bits(addr); + desc->dsadrh = upper_32_bits(addr); +} + +static void set_desc_dst_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dtadr = lower_32_bits(addr); + desc->dtadrh = upper_32_bits(addr); +} + +static u64 get_desc_src_addr_64(const struct mmp_pdma_desc_hw *desc) +{ + return ((u64)desc->dsadrh << 32) | desc->dsadr; +} + +static u64 get_desc_dst_addr_64(const struct mmp_pdma_desc_hw *desc) +{ + return ((u64)desc->dtadrh << 32) | desc->dtadr; +} + +static int mmp_pdma_config_write(struct dma_chan *dchan, + struct dma_slave_config *cfg, + enum dma_transfer_direction direction); + static void enable_chan(struct mmp_pdma_phy *phy) { u32 reg, dalgn; + struct mmp_pdma_device *pdev; if (!phy->vchan) return; + pdev = to_mmp_pdma_dev(phy->vchan->chan.device); + reg = DRCMR(phy->vchan->drcmr); writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); @@ -166,18 +312,29 @@ static void enable_chan(struct mmp_pdma_phy *phy) writel(dalgn, phy->base + DALGN); reg = (phy->idx << 2) + DCSR; - writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg); + writel(readl(phy->base + reg) | pdev->ops->run_bits, + phy->base + reg); } static void disable_chan(struct mmp_pdma_phy *phy) { - u32 reg; + u32 reg, dcsr; if (!phy) return; reg = (phy->idx << 2) + DCSR; - writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg); + dcsr = readl(phy->base + reg); + + if (phy->vchan) { + struct mmp_pdma_device *pdev; + + pdev = to_mmp_pdma_dev(phy->vchan->chan.device); + writel(dcsr & ~pdev->ops->run_bits, phy->base + reg); + } else { + /* If no vchan, just clear the RUN bit */ + writel(dcsr & ~DCSR_RUN, phy->base + reg); + } } static int clear_chan_irq(struct mmp_pdma_phy *phy) @@ -296,6 +453,7 @@ static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) static void start_pending_queue(struct mmp_pdma_chan *chan) { struct mmp_pdma_desc_sw *desc; + struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device); /* still in running, irq will start the pending list */ if (!chan->idle) { @@ -330,7 +488,7 @@ static void start_pending_queue(struct mmp_pdma_chan *chan) * Program the descriptor's address into the DMA controller, * then start the DMA transaction */ - set_desc(chan->phy, desc->async_tx.phys); + pdev->ops->write_next_addr(chan->phy, desc->async_tx.phys); enable_chan(chan->phy); chan->idle = false; } @@ -446,15 +604,14 @@ mmp_pdma_prep_memcpy(struct dma_chan *dchan, size_t len, unsigned long flags) { struct mmp_pdma_chan *chan; + struct mmp_pdma_device *pdev; struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; size_t copy = 0; - if (!dchan) - return NULL; - - if (!len) + if (!dchan || !len) return NULL; + pdev = to_mmp_pdma_dev(dchan->device); chan = to_mmp_pdma_chan(dchan); chan->byte_align = false; @@ -477,13 +634,14 @@ mmp_pdma_prep_memcpy(struct dma_chan *dchan, chan->byte_align = true; new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy); - new->desc.dsadr = dma_src; - new->desc.dtadr = dma_dst; + pdev->ops->set_desc_src_addr(&new->desc, dma_src); + pdev->ops->set_desc_dst_addr(&new->desc, dma_dst); if (!first) first = new; else - prev->desc.ddadr = new->async_tx.phys; + pdev->ops->set_desc_next_addr(&prev->desc, + new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -527,6 +685,7 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, unsigned long flags, void *context) { struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); + struct mmp_pdma_device *pdev = to_mmp_pdma_dev(dchan->device); struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; size_t len, avail; struct scatterlist *sg; @@ -558,17 +717,18 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len); if (dir == DMA_MEM_TO_DEV) { - new->desc.dsadr = addr; + pdev->ops->set_desc_src_addr(&new->desc, addr); new->desc.dtadr = chan->dev_addr; } else { new->desc.dsadr = chan->dev_addr; - new->desc.dtadr = addr; + pdev->ops->set_desc_dst_addr(&new->desc, addr); } if (!first) first = new; else - prev->desc.ddadr = new->async_tx.phys; + pdev->ops->set_desc_next_addr(&prev->desc, + new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -608,12 +768,15 @@ mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, unsigned long flags) { struct mmp_pdma_chan *chan; + struct mmp_pdma_device *pdev; struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; dma_addr_t dma_src, dma_dst; if (!dchan || !len || !period_len) return NULL; + pdev = to_mmp_pdma_dev(dchan->device); + /* the buffer length must be a multiple of period_len */ if (len % period_len != 0) return NULL; @@ -650,13 +813,14 @@ mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | (DCMD_LENGTH & period_len)); - new->desc.dsadr = dma_src; - new->desc.dtadr = dma_dst; + pdev->ops->set_desc_src_addr(&new->desc, dma_src); + pdev->ops->set_desc_dst_addr(&new->desc, dma_dst); if (!first) first = new; else - prev->desc.ddadr = new->async_tx.phys; + pdev->ops->set_desc_next_addr(&prev->desc, + new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -677,7 +841,7 @@ mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, first->async_tx.cookie = -EBUSY; /* make the cyclic link */ - new->desc.ddadr = first->async_tx.phys; + pdev->ops->set_desc_next_addr(&new->desc, first->async_tx.phys); chan->cyclic_first = first; return &first->async_tx; @@ -763,7 +927,9 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, dma_cookie_t cookie) { struct mmp_pdma_desc_sw *sw; - u32 curr, residue = 0; + struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device); + u64 curr; + u32 residue = 0; bool passed = false; bool cyclic = chan->cyclic_first != NULL; @@ -775,17 +941,18 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, return 0; if (chan->dir == DMA_DEV_TO_MEM) - curr = readl(chan->phy->base + DTADR(chan->phy->idx)); + curr = pdev->ops->read_dst_addr(chan->phy); else - curr = readl(chan->phy->base + DSADR(chan->phy->idx)); + curr = pdev->ops->read_src_addr(chan->phy); list_for_each_entry(sw, &chan->chain_running, node) { - u32 start, end, len; + u64 start, end; + u32 len; if (chan->dir == DMA_DEV_TO_MEM) - start = sw->desc.dtadr; + start = pdev->ops->get_desc_dst_addr(&sw->desc); else - start = sw->desc.dsadr; + start = pdev->ops->get_desc_src_addr(&sw->desc); len = sw->desc.dcmd & DCMD_LENGTH; end = start + len; @@ -801,7 +968,7 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, if (passed) { residue += len; } else if (curr >= start && curr <= end) { - residue += end - curr; + residue += (u32)(end - curr); passed = true; } @@ -932,7 +1099,7 @@ static void dma_do_tasklet(struct tasklet_struct *t) } } -static int mmp_pdma_remove(struct platform_device *op) +static void mmp_pdma_remove(struct platform_device *op) { struct mmp_pdma_device *pdev = platform_get_drvdata(op); struct mmp_pdma_phy *phy; @@ -958,7 +1125,6 @@ static int mmp_pdma_remove(struct platform_device *op) } dma_async_device_unregister(&pdev->device); - return 0; } static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) @@ -996,9 +1162,42 @@ static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) return 0; } +static const struct mmp_pdma_ops marvell_pdma_v1_ops = { + .write_next_addr = write_next_addr_32, + .read_src_addr = read_src_addr_32, + .read_dst_addr = read_dst_addr_32, + .set_desc_next_addr = set_desc_next_addr_32, + .set_desc_src_addr = set_desc_src_addr_32, + .set_desc_dst_addr = set_desc_dst_addr_32, + .get_desc_src_addr = get_desc_src_addr_32, + .get_desc_dst_addr = get_desc_dst_addr_32, + .run_bits = (DCSR_RUN), + .dma_mask = 0, /* let OF/platform set DMA mask */ +}; + +static const struct mmp_pdma_ops spacemit_k1_pdma_ops = { + .write_next_addr = write_next_addr_64, + .read_src_addr = read_src_addr_64, + .read_dst_addr = read_dst_addr_64, + .set_desc_next_addr = set_desc_next_addr_64, + .set_desc_src_addr = set_desc_src_addr_64, + .set_desc_dst_addr = set_desc_dst_addr_64, + .get_desc_src_addr = get_desc_src_addr_64, + .get_desc_dst_addr = get_desc_dst_addr_64, + .run_bits = (DCSR_RUN | DCSR_LPAEEN), + .dma_mask = DMA_BIT_MASK(64), /* force 64-bit DMA addr capability */ +}; + static const struct of_device_id mmp_pdma_dt_ids[] = { - { .compatible = "marvell,pdma-1.0", }, - {} + { + .compatible = "marvell,pdma-1.0", + .data = &marvell_pdma_v1_ops + }, { + .compatible = "spacemit,k1-pdma", + .data = &spacemit_k1_pdma_ops + }, { + /* sentinel */ + } }; MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids); @@ -1020,8 +1219,9 @@ static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, static int mmp_pdma_probe(struct platform_device *op) { struct mmp_pdma_device *pdev; - const struct of_device_id *of_id; struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); + struct clk *clk; + struct reset_control *rst; int i, ret, irq = 0; int dma_channels = 0, irq_num = 0; const enum dma_slave_buswidth widths = @@ -1040,8 +1240,20 @@ static int mmp_pdma_probe(struct platform_device *op) if (IS_ERR(pdev->base)) return PTR_ERR(pdev->base); - of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev); - if (of_id) { + clk = devm_clk_get_optional_enabled(pdev->dev, NULL); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + rst = devm_reset_control_get_optional_exclusive_deasserted(pdev->dev, + NULL); + if (IS_ERR(rst)) + return PTR_ERR(rst); + + pdev->ops = of_device_get_match_data(&op->dev); + if (!pdev->ops) + return -ENODEV; + + if (pdev->dev->of_node) { /* Parse new and deprecated dma-channels properties */ if (of_property_read_u32(pdev->dev->of_node, "dma-channels", &dma_channels)) @@ -1102,7 +1314,10 @@ static int mmp_pdma_probe(struct platform_device *op) pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - if (pdev->dev->coherent_dma_mask) + /* Set DMA mask based on ops->dma_mask, or OF/platform */ + if (pdev->ops->dma_mask) + dma_set_mask(pdev->dev, pdev->ops->dma_mask); + else if (pdev->dev->coherent_dma_mask) dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask); else dma_set_mask(pdev->dev, DMA_BIT_MASK(64)); diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index d49fa6bc6775..ba03321eeff7 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -14,9 +14,9 @@ #include <linux/slab.h> #include <linux/dmaengine.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <linux/device.h> #include <linux/genalloc.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include "dmaengine.h" @@ -552,12 +552,9 @@ static void mmp_tdma_issue_pending(struct dma_chan *chan) mmp_tdma_enable_chan(tdmac); } -static int mmp_tdma_remove(struct platform_device *pdev) +static void mmp_tdma_remove(struct platform_device *pdev) { - if (pdev->dev.of_node) - of_dma_controller_free(pdev->dev.of_node); - - return 0; + of_dma_controller_free(pdev->dev.of_node); } static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev, @@ -637,18 +634,13 @@ MODULE_DEVICE_TABLE(of, mmp_tdma_dt_ids); static int mmp_tdma_probe(struct platform_device *pdev) { enum mmp_tdma_type type; - const struct of_device_id *of_id; struct mmp_tdma_device *tdev; int i, ret; int irq = 0, irq_num = 0; int chan_num = TDMA_CHANNEL_NUM; struct gen_pool *pool = NULL; - of_id = of_match_device(mmp_tdma_dt_ids, &pdev->dev); - if (of_id) - type = (enum mmp_tdma_type) of_id->data; - else - type = platform_get_device_id(pdev)->driver_data; + type = (kernel_ulong_t)device_get_match_data(&pdev->dev); /* always have couple channels */ tdev = devm_kzalloc(&pdev->dev, sizeof(*tdev), GFP_KERNEL); @@ -726,32 +718,22 @@ static int mmp_tdma_probe(struct platform_device *pdev) return ret; } - if (pdev->dev.of_node) { - ret = of_dma_controller_register(pdev->dev.of_node, - mmp_tdma_xlate, tdev); - if (ret) { - dev_err(tdev->device.dev, - "failed to register controller\n"); - return ret; - } + ret = of_dma_controller_register(pdev->dev.of_node, + mmp_tdma_xlate, tdev); + if (ret) { + dev_err(tdev->device.dev, "failed to register controller\n"); + return ret; } dev_info(tdev->device.dev, "initialized\n"); return 0; } -static const struct platform_device_id mmp_tdma_id_table[] = { - { "mmp-adma", MMP_AUD_TDMA }, - { "pxa910-squ", PXA910_SQU }, - { }, -}; - static struct platform_driver mmp_tdma_driver = { .driver = { .name = "mmp-tdma", .of_match_table = mmp_tdma_dt_ids, }, - .id_table = mmp_tdma_id_table, .probe = mmp_tdma_probe, .remove = mmp_tdma_remove, }; @@ -760,6 +742,5 @@ module_platform_driver(mmp_tdma_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("MMP Two-Channel DMA Driver"); -MODULE_ALIAS("platform:mmp-tdma"); MODULE_AUTHOR("Leo Yan <leoy@marvell.com>"); MODULE_AUTHOR("Zhangfei Gao <zhangfei.gao@marvell.com>"); diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c index 7565ad98ba66..de09e1ab7767 100644 --- a/drivers/dma/moxart-dma.c +++ b/drivers/dma/moxart-dma.c @@ -124,7 +124,7 @@ struct moxart_desc { unsigned int dma_cycles; struct virt_dma_desc vd; uint8_t es; - struct moxart_sg sg[]; + struct moxart_sg sg[] __counted_by(sglen); }; struct moxart_chan { @@ -148,11 +148,6 @@ struct moxart_dmadev { unsigned int irq; }; -struct moxart_filter_data { - struct moxart_dmadev *mdc; - struct of_phandle_args *dma_spec; -}; - static const unsigned int es_bytes[] = { [MOXART_DMA_DATA_TYPE_S8] = 1, [MOXART_DMA_DATA_TYPE_S16] = 2, @@ -309,6 +304,7 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg( d = kzalloc(struct_size(d, sg, sg_len), GFP_ATOMIC); if (!d) return NULL; + d->sglen = sg_len; d->dma_dir = dir; d->dev_addr = dev_addr; @@ -319,8 +315,6 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg( d->sg[i].len = sg_dma_len(sgent); } - d->sglen = sg_len; - ch->error = 0; return vchan_tx_prep(&ch->vc, &d->vd, tx_flags); @@ -630,7 +624,7 @@ static int moxart_probe(struct platform_device *pdev) return 0; } -static int moxart_remove(struct platform_device *pdev) +static void moxart_remove(struct platform_device *pdev) { struct moxart_dmadev *m = platform_get_drvdata(pdev); @@ -640,8 +634,6 @@ static int moxart_remove(struct platform_device *pdev) if (pdev->dev.of_node) of_dma_controller_free(pdev->dev.of_node); - - return 0; } static const struct of_device_id moxart_dma_match[] = { @@ -652,7 +644,7 @@ MODULE_DEVICE_TABLE(of, moxart_dma_match); static struct platform_driver moxart_driver = { .probe = moxart_probe, - .remove = moxart_remove, + .remove = moxart_remove, .driver = { .name = "moxart-dma-engine", .of_match_table = moxart_dma_match, diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index 4a51fdbf5aa9..bf131cb5db66 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -36,11 +36,11 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/slab.h> +#include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/of_dma.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/random.h> @@ -1084,7 +1084,7 @@ err: return retval; } -static int mpc_dma_remove(struct platform_device *op) +static void mpc_dma_remove(struct platform_device *op) { struct device *dev = &op->dev; struct mpc_dma *mdma = dev_get_drvdata(dev); @@ -1099,8 +1099,6 @@ static int mpc_dma_remove(struct platform_device *op) free_irq(mdma->irq, mdma); irq_dispose_mapping(mdma->irq); tasklet_kill(&mdma->tasklet); - - return 0; } static const struct of_device_id mpc_dma_match[] = { diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 23b232b57518..5e8386296046 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -10,8 +10,8 @@ #include <linux/dma-mapping.h> #include <linux/spinlock.h> #include <linux/interrupt.h> -#include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <linux/memory.h> #include <linux/clk.h> #include <linux/of.h> @@ -414,7 +414,7 @@ mv_xor_tx_submit(struct dma_async_tx_descriptor *tx) if (!mv_chan_is_busy(mv_chan)) { u32 current_desc = mv_chan_get_current_desc(mv_chan); /* - * and the curren desc is the end of the chain before + * and the current desc is the end of the chain before * the append, then we need to start the channel */ if (current_desc == old_chain_tail->async_tx.phys) @@ -1013,7 +1013,7 @@ static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan) dma_async_device_unregister(&mv_chan->dmadev); - dma_free_coherent(dev, MV_XOR_POOL_SIZE, + dma_free_wc(dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); dma_unmap_single(dev, mv_chan->dummy_src_addr, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); @@ -1061,8 +1061,16 @@ mv_xor_channel_add(struct mv_xor_device *xordev, */ mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev, mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); + if (dma_mapping_error(dma_dev->dev, mv_chan->dummy_src_addr)) + return ERR_PTR(-ENOMEM); + mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev, mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev->dev, mv_chan->dummy_dst_addr)) { + ret = -ENOMEM; + goto err_unmap_src; + } + /* allocate coherent memory for hardware descriptors * note: writecombine gives slightly better performance, but @@ -1071,10 +1079,12 @@ mv_xor_channel_add(struct mv_xor_device *xordev, mv_chan->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool, GFP_KERNEL); - if (!mv_chan->dma_desc_pool_virt) - return ERR_PTR(-ENOMEM); + if (!mv_chan->dma_desc_pool_virt) { + ret = -ENOMEM; + goto err_unmap_dst; + } - /* discover transaction capabilites from the platform data */ + /* discover transaction capabilities from the platform data */ dma_dev->cap_mask = cap_mask; INIT_LIST_HEAD(&dma_dev->channels); @@ -1153,8 +1163,15 @@ mv_xor_channel_add(struct mv_xor_device *xordev, err_free_irq: free_irq(mv_chan->irq, mv_chan); err_free_dma: - dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE, + dma_free_wc(&pdev->dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); +err_unmap_dst: + dma_unmap_single(dma_dev->dev, mv_chan->dummy_dst_addr, + MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); +err_unmap_src: + dma_unmap_single(dma_dev->dev, mv_chan->dummy_src_addr, + MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); + return ERR_PTR(ret); } @@ -1328,13 +1345,8 @@ static int mv_xor_probe(struct platform_device *pdev) * setting up. In non-dt case it can only be the legacy one. */ xordev->xor_type = XOR_ORION; - if (pdev->dev.of_node) { - const struct of_device_id *of_id = - of_match_device(mv_xor_dt_ids, - &pdev->dev); - - xordev->xor_type = (uintptr_t)of_id->data; - } + if (pdev->dev.of_node) + xordev->xor_type = (uintptr_t)device_get_match_data(&pdev->dev); /* * (Re-)program MBUS remapping windows if we are asked to. @@ -1374,10 +1386,9 @@ static int mv_xor_probe(struct platform_device *pdev) return 0; if (pdev->dev.of_node) { - struct device_node *np; int i = 0; - for_each_child_of_node(pdev->dev.of_node, np) { + for_each_child_of_node_scoped(pdev->dev.of_node, np) { struct mv_xor_chan *chan; dma_cap_mask_t cap_mask; int irq; diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h index d86086b05b0e..c87cefd38a07 100644 --- a/drivers/dma/mv_xor.h +++ b/drivers/dma/mv_xor.h @@ -99,7 +99,7 @@ struct mv_xor_device { * @common: common dmaengine channel object members * @slots_allocated: records the actual size of the descriptor slot pool * @irq_tasklet: bottom half where mv_xor_slot_cleanup runs - * @op_in_desc: new mode of driver, each op is writen to descriptor. + * @op_in_desc: new mode of driver, each op is written to descriptor. */ struct mv_xor_chan { int pending; diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c index 0e1e9ca1c005..cad4d4fb51ac 100644 --- a/drivers/dma/mv_xor_v2.c +++ b/drivers/dma/mv_xor_v2.c @@ -175,7 +175,7 @@ struct mv_xor_v2_device { * struct mv_xor_v2_sw_desc - implements a xor SW descriptor * @idx: descriptor index * @async_tx: support for the async_tx api - * @hw_desc: assosiated HW descriptor + * @hw_desc: associated HW descriptor * @free_list: node of the free SW descriprots list */ struct mv_xor_v2_sw_desc { @@ -635,7 +635,7 @@ static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev) writel(MV_XOR_V2_DESC_NUM, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF); - /* write the DESQ address to the DMA enngine*/ + /* write the DESQ address to the DMA engine*/ writel(lower_32_bits(xor_dev->hw_desq), xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF); writel(upper_32_bits(xor_dev->hw_desq), @@ -747,8 +747,8 @@ static int mv_xor_v2_probe(struct platform_device *pdev) if (IS_ERR(xor_dev->clk)) return PTR_ERR(xor_dev->clk); - ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1, - mv_xor_v2_set_msi_msg); + ret = platform_device_msi_init_and_alloc_irqs(&pdev->dev, 1, + mv_xor_v2_set_msi_msg); if (ret) return ret; @@ -851,11 +851,11 @@ free_hw_desq: xor_dev->desc_size * MV_XOR_V2_DESC_NUM, xor_dev->hw_desq_virt, xor_dev->hw_desq); free_msi_irqs: - platform_msi_domain_free_irqs(&pdev->dev); + platform_device_msi_free_irqs_all(&pdev->dev); return ret; } -static int mv_xor_v2_remove(struct platform_device *pdev) +static void mv_xor_v2_remove(struct platform_device *pdev) { struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev); @@ -867,11 +867,9 @@ static int mv_xor_v2_remove(struct platform_device *pdev) devm_free_irq(&pdev->dev, xor_dev->irq, xor_dev); - platform_msi_domain_free_irqs(&pdev->dev); + platform_device_msi_free_irqs_all(&pdev->dev); tasklet_kill(&xor_dev->irq_tasklet); - - return 0; } #ifdef CONFIG_OF diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index acc4d53e4630..cfb9962417ef 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -21,7 +21,6 @@ #include <linux/module.h> #include <linux/stmp_device.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/list.h> #include <linux/dma/mxs-dma.h> diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index e72e8c10355e..334425faac00 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -15,7 +15,6 @@ #include <linux/log2.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -712,6 +711,9 @@ static int nbpf_desc_page_alloc(struct nbpf_channel *chan) list_add_tail(&ldesc->node, &lhead); ldesc->hwdesc_dma_addr = dma_map_single(dchan->device->dev, hwdesc, sizeof(*hwdesc), DMA_TO_DEVICE); + if (dma_mapping_error(dchan->device->dev, + ldesc->hwdesc_dma_addr)) + goto unmap_error; dev_dbg(dev, "%s(): mapped 0x%p to %pad\n", __func__, hwdesc, &ldesc->hwdesc_dma_addr); @@ -738,6 +740,16 @@ static int nbpf_desc_page_alloc(struct nbpf_channel *chan) spin_unlock_irq(&chan->lock); return ARRAY_SIZE(dpage->desc); + +unmap_error: + while (i--) { + ldesc--; hwdesc--; + + dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr, + sizeof(hwdesc), DMA_TO_DEVICE); + } + + return -ENOMEM; } static void nbpf_desc_put(struct nbpf_desc *desc) @@ -898,7 +910,7 @@ static int nbpf_config(struct dma_chan *dchan, /* * We could check config->slave_id to match chan->terminal here, * but with DT they would be coming from the same source, so - * such a check would be superflous + * such a check would be superfluous */ chan->slave_dst_addr = config->dst_addr; @@ -1352,7 +1364,7 @@ static int nbpf_probe(struct platform_device *pdev) if (irqs == 1) { eirq = irqbuf[0]; - for (i = 0; i <= num_channels; i++) + for (i = 0; i < num_channels; i++) nbpf->chan[i].irq = irqbuf[0]; } else { eirq = platform_get_irq_byname(pdev, "error"); @@ -1362,16 +1374,15 @@ static int nbpf_probe(struct platform_device *pdev) if (irqs == num_channels + 1) { struct nbpf_channel *chan; - for (i = 0, chan = nbpf->chan; i <= num_channels; + for (i = 0, chan = nbpf->chan; i < num_channels; i++, chan++) { /* Skip the error IRQ */ if (irqbuf[i] == eirq) i++; + if (i >= ARRAY_SIZE(irqbuf)) + return -EINVAL; chan->irq = irqbuf[i]; } - - if (chan != nbpf->chan + num_channels) - return -EINVAL; } else { /* 2 IRQs and more than one channel */ if (irqbuf[0] == eirq) @@ -1379,7 +1390,7 @@ static int nbpf_probe(struct platform_device *pdev) else irq = irqbuf[0]; - for (i = 0; i <= num_channels; i++) + for (i = 0; i < num_channels; i++) nbpf->chan[i].irq = irq; } } @@ -1455,7 +1466,7 @@ e_clk_off: return ret; } -static int nbpf_remove(struct platform_device *pdev) +static void nbpf_remove(struct platform_device *pdev) { struct nbpf_device *nbpf = platform_get_drvdata(pdev); int i; @@ -1473,8 +1484,6 @@ static int nbpf_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&nbpf->dma_dev); clk_disable_unprepare(nbpf->clk); - - return 0; } static const struct platform_device_id nbpf_ids[] = { @@ -1491,7 +1500,6 @@ static const struct platform_device_id nbpf_ids[] = { }; MODULE_DEVICE_TABLE(platform, nbpf_ids); -#ifdef CONFIG_PM static int nbpf_runtime_suspend(struct device *dev) { struct nbpf_device *nbpf = dev_get_drvdata(dev); @@ -1504,17 +1512,16 @@ static int nbpf_runtime_resume(struct device *dev) struct nbpf_device *nbpf = dev_get_drvdata(dev); return clk_prepare_enable(nbpf->clk); } -#endif static const struct dev_pm_ops nbpf_pm_ops = { - SET_RUNTIME_PM_OPS(nbpf_runtime_suspend, nbpf_runtime_resume, NULL) + RUNTIME_PM_OPS(nbpf_runtime_suspend, nbpf_runtime_resume, NULL) }; static struct platform_driver nbpf_driver = { .driver = { .name = "dma-nbpf", .of_match_table = nbpf_match, - .pm = &nbpf_pm_ops, + .pm = pm_ptr(&nbpf_pm_ops), }, .id_table = nbpf_ids, .probe = nbpf_probe, diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c index 775a7f408b9a..423442e55d36 100644 --- a/drivers/dma/of-dma.c +++ b/drivers/dma/of-dma.c @@ -26,10 +26,10 @@ static DEFINE_MUTEX(of_dma_lock); * * Finds a DMA controller with matching device node and number for dma cells * in a list of registered DMA controllers. If a match is found a valid pointer - * to the DMA data stored is retuned. A NULL pointer is returned if no match is + * to the DMA data stored is returned. A NULL pointer is returned if no match is * found. */ -static struct of_dma *of_dma_find_controller(struct of_phandle_args *dma_spec) +static struct of_dma *of_dma_find_controller(const struct of_phandle_args *dma_spec) { struct of_dma *ofdma; @@ -342,7 +342,7 @@ EXPORT_SYMBOL_GPL(of_dma_simple_xlate); * * This function can be used as the of xlate callback for DMA driver which wants * to match the channel based on the channel id. When using this xlate function - * the #dma-cells propety of the DMA controller dt node needs to be set to 1. + * the #dma-cells property of the DMA controller dt node needs to be set to 1. * The data parameter of of_dma_controller_register must be a pointer to the * dma_device struct the function should match upon. * diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c index b6e0ac8314e5..57cec757d8f5 100644 --- a/drivers/dma/owl-dma.c +++ b/drivers/dma/owl-dma.c @@ -20,8 +20,9 @@ #include <linux/io.h> #include <linux/mm.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include "virt-dma.h" @@ -249,7 +250,7 @@ static void pchan_update(struct owl_dma_pchan *pchan, u32 reg, else regval &= ~val; - writel(val, pchan->base + reg); + writel(regval, pchan->base + reg); } static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data) @@ -273,7 +274,7 @@ static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state) else regval &= ~val; - writel(val, od->base + reg); + writel(regval, od->base + reg); } static void dma_writel(struct owl_dma *od, u32 reg, u32 data) @@ -1116,7 +1117,7 @@ static int owl_dma_probe(struct platform_device *pdev) dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n", nr_channels, nr_requests); - od->devid = (enum owl_dma_id)of_device_get_match_data(&pdev->dev); + od->devid = (uintptr_t)of_device_get_match_data(&pdev->dev); od->nr_pchans = nr_channels; od->nr_vchans = nr_requests; @@ -1155,7 +1156,7 @@ static int owl_dma_probe(struct platform_device *pdev) } /* - * Eventhough the DMA controller is capable of generating 4 + * Even though the DMA controller is capable of generating 4 * IRQ's for DMA priority feature, we only use 1 IRQ for * simplification. */ @@ -1230,7 +1231,7 @@ err_pool_free: return ret; } -static int owl_dma_remove(struct platform_device *pdev) +static void owl_dma_remove(struct platform_device *pdev) { struct owl_dma *od = platform_get_drvdata(pdev); @@ -1247,13 +1248,11 @@ static int owl_dma_remove(struct platform_device *pdev) clk_disable_unprepare(od->clk); dma_pool_destroy(od->lli_pool); - - return 0; } static struct platform_driver owl_dma_driver = { .probe = owl_dma_probe, - .remove = owl_dma_remove, + .remove = owl_dma_remove, .driver = { .name = "dma-owl", .of_match_table = of_match_ptr(owl_dma_match), diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index c359decc07a3..6b2793b07694 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -155,11 +155,6 @@ static inline struct device *chan2dev(struct dma_chan *chan) return &chan->dev->device; } -static inline struct device *chan2parent(struct dma_chan *chan) -{ - return chan->dev->device.parent; -} - static inline struct pch_dma_desc *pdc_first_active(struct pch_dma_chan *pd_chan) { diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 3cf0b38387ae..82a9fe88ad54 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -2585,6 +2585,7 @@ static struct dma_pl330_desc *pluck_desc(struct list_head *pool, desc->status = PREP; desc->txd.callback = NULL; + desc->txd.callback_result = NULL; } spin_unlock_irqrestore(lock, flags); @@ -3162,10 +3163,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) * This is the limit for transfers with a buswidth of 1, larger * buswidths will have larger limits. */ - ret = dma_set_max_seg_size(&adev->dev, 1900800); - if (ret) - dev_err(&adev->dev, "unable to set the seg size\n"); - + dma_set_max_seg_size(&adev->dev, 1900800); init_pl330_debugfs(pl330); dev_info(&adev->dev, @@ -3261,7 +3259,6 @@ MODULE_DEVICE_TABLE(amba, pl330_ids); static struct amba_driver pl330_driver = { .drv = { - .owner = THIS_MODULE, .name = "dma-pl330", .pm = &pl330_pm, }, diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index 686c270ef710..61500ad7c850 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -9,7 +9,7 @@ */ /* - * This driver supports the asynchrounous DMA copy and RAID engines available + * This driver supports the asynchronous DMA copy and RAID engines available * on the AMCC PPC440SPe Processors. * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) * ADMA driver written by D.Williams. @@ -28,7 +28,7 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <asm/dcr.h> #include <asm/dcr-regs.h> #include "adma.h" @@ -874,7 +874,7 @@ static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", __func__, src_cnt, state, addr_count, order); for (i = 0; i < src_cnt; i++) - pr_err("\t[%d] 0x%llx \n", i, srcs[i]); + pr_err("\t[%d] 0x%llx\n", i, srcs[i]); BUG(); } @@ -3636,7 +3636,7 @@ static void ppc440spe_adma_issue_pending(struct dma_chan *chan) ppc440spe_chan = to_ppc440spe_adma_chan(chan); dev_dbg(ppc440spe_chan->device->common.dev, - "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, + "ppc440spe adma%d: %s %d\n", ppc440spe_chan->device->id, __func__, ppc440spe_chan->pending); if (ppc440spe_chan->pending) { @@ -4230,7 +4230,7 @@ out: /** * ppc440spe_adma_remove - remove the asynch device */ -static int ppc440spe_adma_remove(struct platform_device *ofdev) +static void ppc440spe_adma_remove(struct platform_device *ofdev) { struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); struct device_node *np = ofdev->dev.of_node; @@ -4278,7 +4278,6 @@ static int ppc440spe_adma_remove(struct platform_device *ofdev) of_address_to_resource(np, 0, &res); release_mem_region(res.start, resource_size(&res)); kfree(adev); - return 0; } /* diff --git a/drivers/dma/ppc4xx/dma.h b/drivers/dma/ppc4xx/dma.h index 1ff4be23db0f..b5725481bfa6 100644 --- a/drivers/dma/ppc4xx/dma.h +++ b/drivers/dma/ppc4xx/dma.h @@ -14,7 +14,7 @@ /* Number of elements in the array with statical CDBs */ #define MAX_STAT_DMA_CDBS 16 -/* Number of DMA engines available on the contoller */ +/* Number of DMA engines available on the controller */ #define DMA_ENGINES_NUM 2 /* Maximum h/w supported number of destinations */ diff --git a/drivers/dma/ptdma/Kconfig b/drivers/dma/ptdma/Kconfig deleted file mode 100644 index b430edd709f9..000000000000 --- a/drivers/dma/ptdma/Kconfig +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -config AMD_PTDMA - tristate "AMD PassThru DMA Engine" - depends on X86_64 && PCI - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the AMD PTDMA controller. This controller - provides DMA capabilities to perform high bandwidth memory to - memory and IO copy operations. It performs DMA transfer through - queue-based descriptor management. This DMA controller is intended - to be used with AMD Non-Transparent Bridge devices and not for - general purpose peripheral DMA. diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index 1b046d9a3a26..249296389771 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -10,14 +10,14 @@ #include <linux/interrupt.h> #include <linux/dma-mapping.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/dmaengine.h> #include <linux/platform_device.h> #include <linux/device.h> #include <linux/platform_data/mmp_dma.h> #include <linux/dmapool.h> -#include <linux/of_device.h> -#include <linux/of_dma.h> #include <linux/of.h> +#include <linux/of_dma.h> #include <linux/wait.h> #include <linux/dma/pxa-dma.h> @@ -91,7 +91,8 @@ struct pxad_desc_sw { bool cyclic; struct dma_pool *desc_pool; /* Channel's used allocator */ - struct pxad_desc_hw *hw_desc[]; /* DMA coherent descriptors */ + struct pxad_desc_hw *hw_desc[] __counted_by(nb_desc); + /* DMA coherent descriptors */ }; struct pxad_phy { @@ -277,8 +278,7 @@ static int chan_state_show(struct seq_file *s, void *p) seq_printf(s, "\tPriority : %s\n", str_prio[(phy->idx & 0xf) / 4]); seq_printf(s, "\tUnaligned transfer bit: %s\n", - _phy_readl_relaxed(phy, DALGN) & BIT(phy->idx) ? - "yes" : "no"); + str_yes_no(_phy_readl_relaxed(phy, DALGN) & BIT(phy->idx))); seq_printf(s, "\tDCSR = %08x (%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", dcsr, PXA_DCSR_STR(RUN), PXA_DCSR_STR(NODESC), PXA_DCSR_STR(STOPIRQEN), PXA_DCSR_STR(EORIRQEN), @@ -722,7 +722,6 @@ static void pxad_free_desc(struct virt_dma_desc *vd) dma_addr_t dma; struct pxad_desc_sw *sw_desc = to_pxad_sw_desc(vd); - BUG_ON(sw_desc->nb_desc == 0); for (i = sw_desc->nb_desc - 1; i >= 0; i--) { if (i > 0) dma = sw_desc->hw_desc[i - 1]->ddadr; @@ -740,6 +739,7 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) { struct pxad_desc_sw *sw_desc; dma_addr_t dma; + void *desc; int i; sw_desc = kzalloc(struct_size(sw_desc, hw_desc, nb_hw_desc), @@ -749,20 +749,21 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) sw_desc->desc_pool = chan->desc_pool; for (i = 0; i < nb_hw_desc; i++) { - sw_desc->hw_desc[i] = dma_pool_alloc(sw_desc->desc_pool, - GFP_NOWAIT, &dma); - if (!sw_desc->hw_desc[i]) { + desc = dma_pool_alloc(sw_desc->desc_pool, GFP_NOWAIT, &dma); + if (!desc) { dev_err(&chan->vc.chan.dev->device, "%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n", __func__, i, sw_desc->desc_pool); goto err; } + sw_desc->nb_desc++; + sw_desc->hw_desc[i] = desc; + if (i == 0) sw_desc->first = dma; else sw_desc->hw_desc[i - 1]->ddadr = dma; - sw_desc->nb_desc++; } return sw_desc; @@ -1221,13 +1222,12 @@ static void pxad_free_channels(struct dma_device *dmadev) } } -static int pxad_remove(struct platform_device *op) +static void pxad_remove(struct platform_device *op) { struct pxad_device *pdev = platform_get_drvdata(op); pxad_cleanup_debugfs(pdev); pxad_free_channels(&pdev->slave); - return 0; } static int pxad_init_phys(struct platform_device *op, @@ -1343,7 +1343,6 @@ static int pxad_init_dmadev(struct platform_device *op, static int pxad_probe(struct platform_device *op) { struct pxad_device *pdev; - const struct of_device_id *of_id; const struct dma_slave_map *slave_map = NULL; struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); int ret, dma_channels = 0, nb_requestors = 0, slave_map_cnt = 0; @@ -1361,8 +1360,7 @@ static int pxad_probe(struct platform_device *op) if (IS_ERR(pdev->base)) return PTR_ERR(pdev->base); - of_id = of_match_device(pxad_dt_ids, &op->dev); - if (of_id) { + if (op->dev.of_node) { /* Parse new and deprecated dma-channels properties */ if (of_property_read_u32(op->dev.of_node, "dma-channels", &dma_channels)) diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 4c3eb972039d..2cf060174795 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -74,7 +74,7 @@ struct bam_async_desc { struct list_head desc_node; enum dma_transfer_direction dir; size_t length; - struct bam_desc_hw desc[]; + struct bam_desc_hw desc[] __counted_by(num_desc); }; enum bam_reg { @@ -440,7 +440,7 @@ static void bam_reset(struct bam_device *bdev) val |= BAM_EN; writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); - /* set descriptor threshhold, start with 4 bytes */ + /* set descriptor threshold, start with 4 bytes */ writel_relaxed(DEFAULT_CNT_THRSHLD, bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); @@ -667,7 +667,7 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, for_each_sg(sgl, sg, sg_len, i) num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE); - /* allocate enough room to accomodate the number of entries */ + /* allocate enough room to accommodate the number of entries */ async_desc = kzalloc(struct_size(async_desc, desc, num_alloc), GFP_NOWAIT); @@ -1283,13 +1283,17 @@ static int bam_dma_probe(struct platform_device *pdev) if (!bdev->bamclk) { ret = of_property_read_u32(pdev->dev.of_node, "num-channels", &bdev->num_channels); - if (ret) + if (ret) { dev_err(bdev->dev, "num-channels unspecified in dt\n"); + return ret; + } ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees", &bdev->num_ees); - if (ret) + if (ret) { dev_err(bdev->dev, "num-ees unspecified in dt\n"); + return ret; + } } ret = clk_prepare_enable(bdev->bamclk); @@ -1325,11 +1329,7 @@ static int bam_dma_probe(struct platform_device *pdev) /* set max dma segment size */ bdev->common.dev = bdev->dev; - ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE); - if (ret) { - dev_err(bdev->dev, "cannot set maximum segment size\n"); - goto err_bam_channel_exit; - } + dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE); platform_set_drvdata(pdev, bdev); @@ -1386,7 +1386,7 @@ err_disable_clk: return ret; } -static int bam_dma_remove(struct platform_device *pdev) +static void bam_dma_remove(struct platform_device *pdev) { struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; @@ -1416,8 +1416,6 @@ static int bam_dma_remove(struct platform_device *pdev) tasklet_kill(&bdev->task); clk_disable_unprepare(bdev->bamclk); - - return 0; } static int __maybe_unused bam_dma_runtime_suspend(struct device *dev) diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 932628b319c8..66bfea1f156d 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -18,6 +18,7 @@ #include "../virt-dma.h" #define TRE_TYPE_DMA 0x10 +#define TRE_TYPE_IMMEDIATE_DMA 0x11 #define TRE_TYPE_GO 0x20 #define TRE_TYPE_CONFIG0 0x22 @@ -64,6 +65,7 @@ /* DMA TRE */ #define TRE_DMA_LEN GENMASK(23, 0) +#define TRE_DMA_IMMEDIATE_LEN GENMASK(3, 0) /* Register offsets from gpi-top */ #define GPII_n_CH_k_CNTXT_0_OFFS(n, k) (0x20000 + (0x4000 * (n)) + (0x80 * (k))) @@ -476,12 +478,6 @@ struct gpi_dev { struct gpii *gpiis; }; -struct reg_info { - char *name; - u32 offset; - u32 val; -}; - struct gchan { struct virt_dma_chan vc; u32 chid; @@ -573,17 +569,6 @@ static inline void gpi_write_reg(struct gpii *gpii, void __iomem *addr, u32 val) writel_relaxed(val, addr); } -/* gpi_write_reg_field - write to specific bit field */ -static inline void gpi_write_reg_field(struct gpii *gpii, void __iomem *addr, - u32 mask, u32 shift, u32 val) -{ - u32 tmp = gpi_read_reg(gpii, addr); - - tmp &= ~mask; - val = tmp | ((val << shift) & mask); - gpi_write_reg(gpii, addr, val); -} - static __always_inline void gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val) { @@ -1197,7 +1182,6 @@ static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd) { struct gpii *gpii = gchan->gpii; struct gpi_ring *ch_ring = &gchan->ch_ring; - unsigned long flags; LIST_HEAD(list); int ret; @@ -1220,9 +1204,9 @@ static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd) gpi_mark_stale_events(gchan); /* remove all async descriptors */ - spin_lock_irqsave(&gchan->vc.lock, flags); + spin_lock(&gchan->vc.lock); vchan_get_all_descriptors(&gchan->vc, &list); - spin_unlock_irqrestore(&gchan->vc.lock, flags); + spin_unlock(&gchan->vc.lock); write_unlock_irq(&gpii->pm_lock); vchan_dma_desc_free_list(&gchan->vc, &list); @@ -1635,7 +1619,8 @@ gpi_peripheral_config(struct dma_chan *chan, struct dma_slave_config *config) } static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, - struct scatterlist *sgl, enum dma_transfer_direction direction) + struct scatterlist *sgl, enum dma_transfer_direction direction, + unsigned long flags) { struct gpi_i2c_config *i2c = chan->config; struct device *dev = chan->gpii->gpi_dev->dev; @@ -1700,6 +1685,9 @@ static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + + if (!(flags & DMA_PREP_INTERRUPT)) + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_BEI); } for (i = 0; i < tre_idx; i++) @@ -1718,6 +1706,7 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, dma_addr_t address; struct gpi_tre *tre; unsigned int i; + int len; /* first create config tre if applicable */ if (direction == DMA_MEM_TO_DEV && spi->set_config) { @@ -1770,14 +1759,30 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, tre_idx++; address = sg_dma_address(sgl); - tre->dword[0] = lower_32_bits(address); - tre->dword[1] = upper_32_bits(address); + len = sg_dma_len(sgl); - tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN); + /* Support Immediate dma for write transfers for data length up to 8 bytes */ + if (direction == DMA_MEM_TO_DEV && len <= 2 * sizeof(tre->dword[0])) { + /* + * For Immediate dma, data length may not always be length of 8 bytes, + * it can be length less than 8, hence initialize both dword's with 0 + */ + tre->dword[0] = 0; + tre->dword[1] = 0; + memcpy(&tre->dword[0], sg_virt(sgl), len); - tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); - if (direction == DMA_MEM_TO_DEV) - tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + tre->dword[2] = u32_encode_bits(len, TRE_DMA_IMMEDIATE_LEN); + tre->dword[3] = u32_encode_bits(TRE_TYPE_IMMEDIATE_DMA, TRE_FLAGS_TYPE); + } else { + tre->dword[0] = lower_32_bits(address); + tre->dword[1] = upper_32_bits(address); + + tre->dword[2] = u32_encode_bits(len, TRE_DMA_LEN); + tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); + } + + tre->dword[3] |= u32_encode_bits(direction == DMA_MEM_TO_DEV, + TRE_FLAGS_IEOT); for (i = 0; i < tre_idx; i++) dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], @@ -1826,6 +1831,9 @@ gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return NULL; } + if (!(flags & DMA_PREP_INTERRUPT) && (nr - nr_tre < 2)) + return NULL; + gpi_desc = kzalloc(sizeof(*gpi_desc), GFP_NOWAIT); if (!gpi_desc) return NULL; @@ -1834,7 +1842,7 @@ gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (gchan->protocol == QCOM_GPI_SPI) { i = gpi_create_spi_tre(gchan, gpi_desc, sgl, direction); } else if (gchan->protocol == QCOM_GPI_I2C) { - i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction); + i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction, flags); } else { dev_err(dev, "invalid peripheral: %d\n", gchan->protocol); kfree(gpi_desc); @@ -1863,7 +1871,7 @@ static void gpi_issue_pending(struct dma_chan *chan) read_lock_irqsave(&gpii->pm_lock, pm_lock_flags); - /* move all submitted discriptors to issued list */ + /* move all submitted descriptors to issued list */ spin_lock_irqsave(&gchan->vc.lock, flags); if (vchan_issue_pending(&gchan->vc)) vd = list_last_entry(&gchan->vc.desc_issued, @@ -2160,8 +2168,7 @@ static int gpi_probe(struct platform_device *pdev) return -ENOMEM; gpi_dev->dev = &pdev->dev; - gpi_dev->res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - gpi_dev->regs = devm_ioremap_resource(gpi_dev->dev, gpi_dev->res); + gpi_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &gpi_dev->res); if (IS_ERR(gpi_dev->regs)) return PTR_ERR(gpi_dev->regs); gpi_dev->ee_base = gpi_dev->regs; diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 344525c3a32f..c2b3e4452e71 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -45,12 +45,11 @@ #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/list.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/of_dma.h> -#include <linux/of_device.h> #include <linux/property.h> #include <linux/delay.h> #include <linux/acpi.h> @@ -696,7 +695,7 @@ static void hidma_free_msis(struct hidma_dev *dmadev) devm_free_irq(dev, virq, &dmadev->lldev); } - platform_msi_domain_free_irqs(dev); + platform_device_msi_free_irqs_all(dev); #endif } @@ -706,8 +705,8 @@ static int hidma_request_msi(struct hidma_dev *dmadev, #ifdef CONFIG_GENERIC_MSI_IRQ int rc, i, virq; - rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, - hidma_write_msi_msg); + rc = platform_device_msi_init_and_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, + hidma_write_msi_msg); if (rc) return rc; @@ -745,7 +744,7 @@ static bool hidma_test_capability(struct device *dev, enum hidma_cap test_cap) { enum hidma_cap cap; - cap = (enum hidma_cap) device_get_match_data(dev); + cap = (uintptr_t) device_get_match_data(dev); return cap ? ((cap & test_cap) > 0) : 0; } @@ -765,17 +764,15 @@ static int hidma_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - trca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); - trca = devm_ioremap_resource(&pdev->dev, trca_resource); + trca = devm_platform_get_and_ioremap_resource(pdev, 0, &trca_resource); if (IS_ERR(trca)) { - rc = -ENOMEM; + rc = PTR_ERR(trca); goto bailout; } - evca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 1); - evca = devm_ioremap_resource(&pdev->dev, evca_resource); + evca = devm_platform_get_and_ioremap_resource(pdev, 1, &evca_resource); if (IS_ERR(evca)) { - rc = -ENOMEM; + rc = PTR_ERR(evca); goto bailout; } @@ -785,7 +782,7 @@ static int hidma_probe(struct platform_device *pdev) */ chirq = platform_get_irq(pdev, 0); if (chirq < 0) { - rc = -ENODEV; + rc = chirq; goto bailout; } @@ -917,7 +914,7 @@ static void hidma_shutdown(struct platform_device *pdev) } -static int hidma_remove(struct platform_device *pdev) +static void hidma_remove(struct platform_device *pdev) { struct hidma_dev *dmadev = platform_get_drvdata(pdev); @@ -937,8 +934,6 @@ static int hidma_remove(struct platform_device *pdev) dev_info(&pdev->dev, "HI-DMA engine removed\n"); pm_runtime_put_sync_suspend(&pdev->dev); pm_runtime_disable(&pdev->dev); - - return 0; } #if IS_ENABLED(CONFIG_ACPI) @@ -951,25 +946,16 @@ static const struct acpi_device_id hidma_acpi_ids[] = { MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids); #endif -static const struct of_device_id hidma_match[] = { - {.compatible = "qcom,hidma-1.0",}, - {.compatible = "qcom,hidma-1.1", .data = (void *)(HIDMA_MSI_CAP),}, - {.compatible = "qcom,hidma-1.2", - .data = (void *)(HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP),}, - {}, -}; -MODULE_DEVICE_TABLE(of, hidma_match); - static struct platform_driver hidma_driver = { .probe = hidma_probe, .remove = hidma_remove, .shutdown = hidma_shutdown, .driver = { .name = "hidma", - .of_match_table = hidma_match, .acpi_match_table = ACPI_PTR(hidma_acpi_ids), }, }; module_platform_driver(hidma_driver); +MODULE_DESCRIPTION("Qualcomm Technologies HIDMA Channel support"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c index 05e96b31d871..4805ce390ffa 100644 --- a/drivers/dma/qcom/hidma_mgmt.c +++ b/drivers/dma/qcom/hidma_mgmt.c @@ -7,12 +7,7 @@ #include <linux/dmaengine.h> #include <linux/acpi.h> -#include <linux/of.h> #include <linux/property.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/module.h> #include <linux/uaccess.h> @@ -176,10 +171,9 @@ static int hidma_mgmt_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); pm_runtime_get_sync(&pdev->dev); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - virtaddr = devm_ioremap_resource(&pdev->dev, res); + virtaddr = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(virtaddr)) { - rc = -ENOMEM; + rc = PTR_ERR(virtaddr); goto out; } @@ -328,115 +322,14 @@ static const struct acpi_device_id hidma_mgmt_acpi_ids[] = { MODULE_DEVICE_TABLE(acpi, hidma_mgmt_acpi_ids); #endif -static const struct of_device_id hidma_mgmt_match[] = { - {.compatible = "qcom,hidma-mgmt-1.0",}, - {}, -}; -MODULE_DEVICE_TABLE(of, hidma_mgmt_match); - static struct platform_driver hidma_mgmt_driver = { .probe = hidma_mgmt_probe, .driver = { .name = "hidma-mgmt", - .of_match_table = hidma_mgmt_match, .acpi_match_table = ACPI_PTR(hidma_mgmt_acpi_ids), }, }; -#if defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) -static int object_counter; - -static int __init hidma_mgmt_of_populate_channels(struct device_node *np) -{ - struct platform_device *pdev_parent = of_find_device_by_node(np); - struct platform_device_info pdevinfo; - struct device_node *child; - struct resource *res; - int ret = 0; - - /* allocate a resource array */ - res = kcalloc(3, sizeof(*res), GFP_KERNEL); - if (!res) - return -ENOMEM; - - for_each_available_child_of_node(np, child) { - struct platform_device *new_pdev; - - ret = of_address_to_resource(child, 0, &res[0]); - if (!ret) - goto out; - - ret = of_address_to_resource(child, 1, &res[1]); - if (!ret) - goto out; - - ret = of_irq_to_resource(child, 0, &res[2]); - if (ret <= 0) - goto out; - - memset(&pdevinfo, 0, sizeof(pdevinfo)); - pdevinfo.fwnode = &child->fwnode; - pdevinfo.parent = pdev_parent ? &pdev_parent->dev : NULL; - pdevinfo.name = child->name; - pdevinfo.id = object_counter++; - pdevinfo.res = res; - pdevinfo.num_res = 3; - pdevinfo.data = NULL; - pdevinfo.size_data = 0; - pdevinfo.dma_mask = DMA_BIT_MASK(64); - new_pdev = platform_device_register_full(&pdevinfo); - if (IS_ERR(new_pdev)) { - ret = PTR_ERR(new_pdev); - goto out; - } - new_pdev->dev.of_node = child; - of_dma_configure(&new_pdev->dev, child, true); - /* - * It is assumed that calling of_msi_configure is safe on - * platforms with or without MSI support. - */ - of_msi_configure(&new_pdev->dev, child); - } - - kfree(res); - - return ret; - -out: - of_node_put(child); - kfree(res); - - return ret; -} -#endif - -static int __init hidma_mgmt_init(void) -{ -#if defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) - struct device_node *child; - - for_each_matching_node(child, hidma_mgmt_match) { - /* device tree based firmware here */ - hidma_mgmt_of_populate_channels(child); - } -#endif - /* - * We do not check for return value here, as it is assumed that - * platform_driver_register must not fail. The reason for this is that - * the (potential) hidma_mgmt_of_populate_channels calls above are not - * cleaned up if it does fail, and to do this work is quite - * complicated. In particular, various calls of of_address_to_resource, - * of_irq_to_resource, platform_device_register_full, of_dma_configure, - * and of_msi_configure which then call other functions and so on, must - * be cleaned up - this is not a trivial exercise. - * - * Currently, this module is not intended to be unloaded, and there is - * no module_exit function defined which does the needed cleanup. For - * this reason, we have to assume success here. - */ - platform_driver_register(&hidma_mgmt_driver); - - return 0; -} -module_init(hidma_mgmt_init); +module_platform_driver(hidma_mgmt_driver); +MODULE_DESCRIPTION("Qualcomm Technologies HIDMA DMA engine interface"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/qcom/qcom_adm.c b/drivers/dma/qcom/qcom_adm.c index d56caf1681ff..6be54fddcee1 100644 --- a/drivers/dma/qcom/qcom_adm.c +++ b/drivers/dma/qcom/qcom_adm.c @@ -650,7 +650,7 @@ static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie, /* * residue is either the full length if it is in the issued list, or 0 * if it is in progress. We have no reliable way of determining - * anything inbetween + * anything in between */ dma_set_residue(txstate, residue); @@ -904,7 +904,7 @@ err_disable_core_clk: return ret; } -static int adm_dma_remove(struct platform_device *pdev) +static void adm_dma_remove(struct platform_device *pdev) { struct adm_device *adev = platform_get_drvdata(pdev); struct adm_chan *achan; @@ -927,8 +927,6 @@ static int adm_dma_remove(struct platform_device *pdev) clk_disable_unprepare(adev->core_clk); clk_disable_unprepare(adev->iface_clk); - - return 0; } static const struct of_device_id adm_of_match[] = { diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index a29c13cae716..dc1a9a05252e 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -78,7 +78,7 @@ struct sa11x0_dma_desc { bool cyclic; unsigned sglen; - struct sa11x0_dma_sg sg[]; + struct sa11x0_dma_sg sg[] __counted_by(sglen); }; struct sa11x0_dma_phy; @@ -558,6 +558,7 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg( dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc); return NULL; } + txd->sglen = j; j = 0; for_each_sg(sg, sgent, sglen, i) { @@ -593,7 +594,6 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg( txd->ddar = c->ddar; txd->size = size; - txd->sglen = j; dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n", &c->vc, &txd->vd, txd->size, txd->sglen); @@ -628,6 +628,7 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic( dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc); return NULL; } + txd->sglen = sglen; for (i = k = 0; i < size / period; i++) { size_t tlen, len = period; @@ -653,7 +654,6 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic( txd->ddar = c->ddar; txd->size = size; - txd->sglen = sglen; txd->cyclic = 1; txd->period = sgperiod; @@ -984,7 +984,7 @@ static int sa11x0_dma_probe(struct platform_device *pdev) return ret; } -static int sa11x0_dma_remove(struct platform_device *pdev) +static void sa11x0_dma_remove(struct platform_device *pdev) { struct sa11x0_dma_dev *d = platform_get_drvdata(pdev); unsigned pch; @@ -997,8 +997,6 @@ static int sa11x0_dma_remove(struct platform_device *pdev) tasklet_kill(&d->task); iounmap(d->base); kfree(d); - - return 0; } static __maybe_unused int sa11x0_dma_suspend(struct device *dev) diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index d1c6956af452..7ad3c29be146 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -20,10 +20,13 @@ #include <linux/mod_devicetable.h> #include <linux/dma-mapping.h> #include <linux/of.h> +#include <linux/of_dma.h> #include <linux/slab.h> #include "sf-pdma.h" +#define PDMA_QUIRK_NO_STRICT_ORDERING BIT(0) + #ifndef readq static inline unsigned long long readq(void __iomem *addr) { @@ -65,7 +68,7 @@ static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan) static void sf_pdma_fill_desc(struct sf_pdma_desc *desc, u64 dst, u64 src, u64 size) { - desc->xfer_type = PDMA_FULL_SPEED; + desc->xfer_type = desc->chan->pdma->transfer_type; desc->xfer_size = size; desc->dst_addr = dst; desc->src_addr = src; @@ -351,7 +354,7 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id) if (!residue) { tasklet_hi_schedule(&chan->done_tasklet); } else { - /* submit next trascatioin if possible */ + /* submit next transaction if possible */ struct sf_pdma_desc *desc = chan->desc; desc->src_addr += desc->xfer_size - residue; @@ -492,6 +495,7 @@ static void sf_pdma_setup_chans(struct sf_pdma *pdma) static int sf_pdma_probe(struct platform_device *pdev) { + const struct sf_pdma_driver_platdata *ddata; struct sf_pdma *pdma; int ret, n_chans; const enum dma_slave_buswidth widths = @@ -517,6 +521,14 @@ static int sf_pdma_probe(struct platform_device *pdev) pdma->n_chans = n_chans; + pdma->transfer_type = PDMA_FULL_SPEED | PDMA_STRICT_ORDERING; + + ddata = device_get_match_data(&pdev->dev); + if (ddata) { + if (ddata->quirks & PDMA_QUIRK_NO_STRICT_ORDERING) + pdma->transfer_type &= ~PDMA_STRICT_ORDERING; + } + pdma->membase = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pdma->membase)) return PTR_ERR(pdma->membase); @@ -563,10 +575,23 @@ static int sf_pdma_probe(struct platform_device *pdev) return ret; } + ret = of_dma_controller_register(pdev->dev.of_node, + of_dma_xlate_by_chan_id, pdma); + if (ret < 0) { + dev_err(&pdev->dev, + "Can't register SiFive Platform OF_DMA. (%d)\n", ret); + goto err_unregister; + } + return 0; + +err_unregister: + dma_async_device_unregister(&pdma->dma_dev); + + return ret; } -static int sf_pdma_remove(struct platform_device *pdev) +static void sf_pdma_remove(struct platform_device *pdev) { struct sf_pdma *pdma = platform_get_drvdata(pdev); struct sf_pdma_chan *ch; @@ -583,14 +608,25 @@ static int sf_pdma_remove(struct platform_device *pdev) tasklet_kill(&ch->err_tasklet); } - dma_async_device_unregister(&pdma->dma_dev); + if (pdev->dev.of_node) + of_dma_controller_free(pdev->dev.of_node); - return 0; + dma_async_device_unregister(&pdma->dma_dev); } +static const struct sf_pdma_driver_platdata mpfs_pdma = { + .quirks = PDMA_QUIRK_NO_STRICT_ORDERING, +}; + static const struct of_device_id sf_pdma_dt_ids[] = { - { .compatible = "sifive,fu540-c000-pdma" }, - { .compatible = "sifive,pdma0" }, + { + .compatible = "sifive,fu540-c000-pdma", + }, { + .compatible = "sifive,pdma0", + }, { + .compatible = "microchip,mpfs-pdma", + .data = &mpfs_pdma, + }, {}, }; MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids); diff --git a/drivers/dma/sf-pdma/sf-pdma.h b/drivers/dma/sf-pdma/sf-pdma.h index 5c398a83b491..215e07183d7e 100644 --- a/drivers/dma/sf-pdma/sf-pdma.h +++ b/drivers/dma/sf-pdma/sf-pdma.h @@ -48,7 +48,8 @@ #define PDMA_ERR_STATUS_MASK GENMASK(31, 31) /* Transfer Type */ -#define PDMA_FULL_SPEED 0xFF000008 +#define PDMA_FULL_SPEED 0xFF000000 +#define PDMA_STRICT_ORDERING BIT(3) /* Error Recovery */ #define MAX_RETRY 1 @@ -112,8 +113,13 @@ struct sf_pdma { struct dma_device dma_dev; void __iomem *membase; void __iomem *mappedbase; + u32 transfer_type; u32 n_chans; - struct sf_pdma_chan chans[]; + struct sf_pdma_chan chans[] __counted_by(n_chans); +}; + +struct sf_pdma_driver_platdata { + u32 quirks; }; #endif /* _SF_PDMA_H */ diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig index c0b2997ab7fd..a16c7e83bd14 100644 --- a/drivers/dma/sh/Kconfig +++ b/drivers/dma/sh/Kconfig @@ -16,7 +16,7 @@ config SH_DMAE_BASE depends on SUPERH || COMPILE_TEST depends on !SUPERH || SH_DMA depends on !SH_DMA_API - default y + default SUPERH || SH_DMA select RENESAS_DMA help Enable support for the Renesas SuperH DMA controllers. @@ -49,10 +49,10 @@ config RENESAS_USB_DMAC SoCs. config RZ_DMAC - tristate "Renesas RZ/{G2L,V2L} DMA Controller" - depends on ARCH_RZG2L || COMPILE_TEST + tristate "Renesas RZ DMA Controller" + depends on ARCH_RENESAS || COMPILE_TEST select RENESAS_DMA select DMA_VIRTUAL_CHANNELS help - This driver supports the general purpose DMA controller found in the - Renesas RZ/{G2L,V2L} SoC variants. + This driver supports the general purpose DMA controller typically + found in the Renesas RZ SoC variants. diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 641d689d17ff..475a347cae1b 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -1728,19 +1728,12 @@ static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, * Power management */ -#ifdef CONFIG_PM -static int rcar_dmac_runtime_suspend(struct device *dev) -{ - return 0; -} - static int rcar_dmac_runtime_resume(struct device *dev) { struct rcar_dmac *dmac = dev_get_drvdata(dev); return rcar_dmac_init(dmac); } -#endif static const struct dev_pm_ops rcar_dmac_pm = { /* @@ -1748,10 +1741,9 @@ static const struct dev_pm_ops rcar_dmac_pm = { * - Wait for the current transfer to complete and stop the device, * - Resume transfers, if any. */ - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume, - NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + RUNTIME_PM_OPS(NULL, rcar_dmac_runtime_resume, NULL) }; /* ----------------------------------------------------------------------------- @@ -1868,9 +1860,7 @@ static int rcar_dmac_probe(struct platform_device *pdev) dmac->dev = &pdev->dev; platform_set_drvdata(pdev, dmac); - ret = dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK); - if (ret) - return ret; + dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK); ret = dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); if (ret) @@ -1990,7 +1980,7 @@ err_pm_disable: return ret; } -static int rcar_dmac_remove(struct platform_device *pdev) +static void rcar_dmac_remove(struct platform_device *pdev) { struct rcar_dmac *dmac = platform_get_drvdata(pdev); @@ -1998,8 +1988,6 @@ static int rcar_dmac_remove(struct platform_device *pdev) dma_async_device_unregister(&dmac->engine); pm_runtime_disable(&pdev->dev); - - return 0; } static void rcar_dmac_shutdown(struct platform_device *pdev) @@ -2027,6 +2015,10 @@ static const struct of_device_id rcar_dmac_of_ids[] = { .compatible = "renesas,rcar-gen4-dmac", .data = &rcar_gen4_dmac_data, }, { + /* + * Backward compatibility for between v5.12 - v5.19 + * which didn't combined with "renesas,rcar-gen4-dmac" + */ .compatible = "renesas,dmac-r8a779a0", .data = &rcar_gen4_dmac_data, }, @@ -2036,7 +2028,7 @@ MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); static struct platform_driver rcar_dmac_driver = { .driver = { - .pm = &rcar_dmac_pm, + .pm = pm_ptr(&rcar_dmac_pm), .name = "rcar-dmac", .of_match_table = rcar_dmac_of_ids, }, diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index 9479f29692d3..1f687b08d6b8 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -9,10 +9,12 @@ * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com> */ +#include <linux/bitfield.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/interrupt.h> #include <linux/iopoll.h> +#include <linux/irqchip/irq-renesas-rzv2h.h> #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> @@ -88,8 +90,14 @@ struct rz_dmac_chan { #define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan) +struct rz_dmac_icu { + struct platform_device *pdev; + u8 dmac_index; +}; + struct rz_dmac { struct dma_device engine; + struct rz_dmac_icu icu; struct device *dev; struct reset_control *rstc; void __iomem *base; @@ -98,6 +106,8 @@ struct rz_dmac { unsigned int n_channels; struct rz_dmac_chan *channels; + bool has_icu; + DECLARE_BITMAP(modules, 1024); }; @@ -145,8 +155,8 @@ struct rz_dmac { #define CHCFG_REQD BIT(3) #define CHCFG_SEL(bits) ((bits) & 0x07) #define CHCFG_MEM_COPY (0x80400008) -#define CHCFG_FILL_DDS(a) (((a) << 16) & GENMASK(19, 16)) -#define CHCFG_FILL_SDS(a) (((a) << 12) & GENMASK(15, 12)) +#define CHCFG_FILL_DDS_MASK GENMASK(19, 16) +#define CHCFG_FILL_SDS_MASK GENMASK(15, 12) #define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22) #define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6) #define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5) @@ -166,6 +176,9 @@ struct rz_dmac { #define RZ_DMAC_MAX_CHANNELS 16 #define DMAC_NR_LMDESC 64 +/* RZ/V2H ICU related */ +#define RZV2H_MAX_DMAC_INDEX 4 + /* * ----------------------------------------------------------------------------- * Device access @@ -323,7 +336,13 @@ static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) lmdesc->chext = 0; lmdesc->header = HEADER_LV; - rz_dmac_set_dmars_register(dmac, channel->index, 0); + if (dmac->has_icu) { + rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + channel->index, + RZV2H_ICU_DMAC_REQ_NO_DEFAULT); + } else { + rz_dmac_set_dmars_register(dmac, channel->index, 0); + } channel->chcfg = chcfg; channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; @@ -374,7 +393,13 @@ static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) channel->lmdesc.tail = lmdesc; - rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); + if (dmac->has_icu) { + rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + channel->index, channel->mid_rid); + } else { + rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); + } + channel->chctrl = CHCTRL_SETEN; } @@ -539,8 +564,8 @@ static int rz_dmac_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&channel->vc.lock, flags); list_splice_tail_init(&channel->ld_active, &channel->ld_free); list_splice_tail_init(&channel->ld_queue, &channel->ld_free); - spin_unlock_irqrestore(&channel->vc.lock, flags); vchan_get_all_descriptors(&channel->vc, &head); + spin_unlock_irqrestore(&channel->vc.lock, flags); vchan_dma_desc_free_list(&channel->vc, &head); return 0; @@ -600,20 +625,25 @@ static int rz_dmac_config(struct dma_chan *chan, struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); u32 val; - channel->src_per_address = config->src_addr; channel->dst_per_address = config->dst_addr; + channel->chcfg &= ~CHCFG_FILL_DDS_MASK; + if (channel->dst_per_address) { + val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); + if (val == CHCFG_DS_INVALID) + return -EINVAL; - val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); - if (val == CHCFG_DS_INVALID) - return -EINVAL; - - channel->chcfg |= CHCFG_FILL_DDS(val); + channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val); + } - val = rz_dmac_ds_to_val_mapping(config->src_addr_width); - if (val == CHCFG_DS_INVALID) - return -EINVAL; + channel->src_per_address = config->src_addr; + channel->chcfg &= ~CHCFG_FILL_SDS_MASK; + if (channel->src_per_address) { + val = rz_dmac_ds_to_val_mapping(config->src_addr_width); + if (val == CHCFG_DS_INVALID) + return -EINVAL; - channel->chcfg |= CHCFG_FILL_SDS(val); + channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val); + } return 0; } @@ -641,7 +671,13 @@ static void rz_dmac_device_synchronize(struct dma_chan *chan) if (ret < 0) dev_warn(dmac->dev, "DMA Timeout"); - rz_dmac_set_dmars_register(dmac, channel->index, 0); + if (dmac->has_icu) { + rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + channel->index, + RZV2H_ICU_DMAC_REQ_NO_DEFAULT); + } else { + rz_dmac_set_dmars_register(dmac, channel->index, 0); + } } /* @@ -742,7 +778,8 @@ static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); - return dma_request_channel(mask, rz_dmac_chan_filter, dma_spec); + return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec, + ofdma->of_node); } /* @@ -752,11 +789,11 @@ static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, static int rz_dmac_chan_probe(struct rz_dmac *dmac, struct rz_dmac_chan *channel, - unsigned int index) + u8 index) { struct platform_device *pdev = to_platform_device(dmac->dev); struct rz_lmdesc *lmdesc; - char pdev_irqname[5]; + char pdev_irqname[6]; char *irqname; int ret; @@ -764,7 +801,7 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, channel->mid_rid = -EINVAL; /* Request the channel interrupt. */ - sprintf(pdev_irqname, "ch%u", index); + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); channel->irq = platform_get_irq_byname(pdev, pdev_irqname); if (channel->irq < 0) return channel->irq; @@ -817,6 +854,38 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, return 0; } +static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) +{ + struct device_node *np = dev->of_node; + struct of_phandle_args args; + uint32_t dmac_index; + int ret; + + ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); + if (ret == -ENOENT) + return 0; + if (ret) + return ret; + + dmac->has_icu = true; + + dmac->icu.pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + if (!dmac->icu.pdev) { + dev_err(dev, "ICU device not found.\n"); + return -ENODEV; + } + + dmac_index = args.args[0]; + if (dmac_index > RZV2H_MAX_DMAC_INDEX) { + dev_err(dev, "DMAC index %u invalid.\n", dmac_index); + return -EINVAL; + } + dmac->icu.dmac_index = dmac_index; + + return 0; +} + static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) { struct device_node *np = dev->of_node; @@ -833,7 +902,7 @@ static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) return -EINVAL; } - return 0; + return rz_dmac_parse_of_icu(dev, dmac); } static int rz_dmac_probe(struct platform_device *pdev) @@ -842,9 +911,9 @@ static int rz_dmac_probe(struct platform_device *pdev) struct dma_device *engine; struct rz_dmac *dmac; int channel_num; - unsigned int i; int ret; int irq; + u8 i; dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); if (!dmac) @@ -867,9 +936,11 @@ static int rz_dmac_probe(struct platform_device *pdev) if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); - dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); - if (IS_ERR(dmac->ext_base)) - return PTR_ERR(dmac->ext_base); + if (!dmac->has_icu) { + dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(dmac->ext_base)) + return PTR_ERR(dmac->ext_base); + } /* Register interrupt handler for error */ irq = platform_get_irq_byname(pdev, irqname); @@ -887,7 +958,7 @@ static int rz_dmac_probe(struct platform_device *pdev) /* Initialize the channels. */ INIT_LIST_HEAD(&dmac->engine.channels); - dmac->rstc = devm_reset_control_array_get_exclusive(&pdev->dev); + dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev); if (IS_ERR(dmac->rstc)) return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), "failed to get resets\n"); @@ -947,7 +1018,6 @@ static int rz_dmac_probe(struct platform_device *pdev) dma_register_err: of_dma_controller_free(pdev->dev.of_node); err: - reset_control_assert(dmac->rstc); channel_num = i ? i - 1 : 0; for (i = 0; i < channel_num; i++) { struct rz_dmac_chan *channel = &dmac->channels[i]; @@ -958,6 +1028,7 @@ err: channel->lmdesc.base_dma); } + reset_control_assert(dmac->rstc); err_pm_runtime_put: pm_runtime_put(&pdev->dev); err_pm_disable: @@ -966,11 +1037,13 @@ err_pm_disable: return ret; } -static int rz_dmac_remove(struct platform_device *pdev) +static void rz_dmac_remove(struct platform_device *pdev) { struct rz_dmac *dmac = platform_get_drvdata(pdev); unsigned int i; + dma_async_device_unregister(&dmac->engine); + of_dma_controller_free(pdev->dev.of_node); for (i = 0; i < dmac->n_channels; i++) { struct rz_dmac_chan *channel = &dmac->channels[i]; @@ -979,16 +1052,15 @@ static int rz_dmac_remove(struct platform_device *pdev) channel->lmdesc.base, channel->lmdesc.base_dma); } - of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&dmac->engine); reset_control_assert(dmac->rstc); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); - return 0; + platform_device_put(dmac->icu.pdev); } static const struct of_device_id of_rz_dmac_match[] = { + { .compatible = "renesas,r9a09g057-dmac", }, { .compatible = "renesas,rz-dmac", }, { /* Sentinel */ } }; diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index 588c5f409a80..834741adadaa 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -129,12 +129,25 @@ static dma_cookie_t shdma_tx_submit(struct dma_async_tx_descriptor *tx) const struct shdma_ops *ops = sdev->ops; dev_dbg(schan->dev, "Bring up channel %d\n", schan->id); - /* - * TODO: .xfer_setup() might fail on some platforms. - * Make it int then, on error remove chunks from the - * queue again - */ - ops->setup_xfer(schan, schan->slave_id); + + ret = ops->setup_xfer(schan, schan->slave_id); + if (ret < 0) { + dev_err(schan->dev, "setup_xfer failed: %d\n", ret); + + /* Remove chunks from the queue and mark them as idle */ + list_for_each_entry_safe(chunk, c, &schan->ld_queue, node) { + if (chunk->cookie == cookie) { + chunk->mark = DESC_IDLE; + list_move(&chunk->node, &schan->ld_free); + } + } + + schan->pm_state = SHDMA_PM_ESTABLISHED; + ret = pm_runtime_put(schan->dev); + + spin_unlock_irq(&schan->chan_lock); + return ret; + } if (schan->pm_state == SHDMA_PM_PENDING) shdma_chan_xfer_ld_queue(schan); @@ -725,7 +738,7 @@ static struct dma_async_tx_descriptor *shdma_prep_dma_cyclic( slave_addr = ops->slave_addr(schan); /* - * Allocate the sg list dynamically as it would consumer too much stack + * Allocate the sg list dynamically as it would consume too much stack * space. */ sgl = kmalloc_array(sg_len, sizeof(*sgl), GFP_KERNEL); @@ -961,7 +974,7 @@ void shdma_chan_probe(struct shdma_dev *sdev, spin_lock_init(&schan->chan_lock); - /* Init descripter manage list */ + /* Init descriptor manage list */ INIT_LIST_HEAD(&schan->ld_queue); INIT_LIST_HEAD(&schan->ld_free); diff --git a/drivers/dma/sh/shdma.h b/drivers/dma/sh/shdma.h index 9c121a4b33ad..f97d80343aea 100644 --- a/drivers/dma/sh/shdma.h +++ b/drivers/dma/sh/shdma.h @@ -25,7 +25,7 @@ struct sh_dmae_chan { const struct sh_dmae_slave_config *config; /* Slave DMA configuration */ int xmit_shift; /* log_2(bytes_per_xfer) */ void __iomem *base; - char dev_id[16]; /* unique name per DMAC of channel */ + char dev_id[32]; /* unique name per DMAC of channel */ int pm_error; dma_addr_t slave_addr; }; diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 5aafe548ca5f..603e15102e45 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -23,7 +23,6 @@ #include <linux/module.h> #include <linux/notifier.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/rculist.h> @@ -301,25 +300,34 @@ static bool sh_dmae_channel_busy(struct shdma_chan *schan) return dmae_is_busy(sh_chan); } -static void sh_dmae_setup_xfer(struct shdma_chan *schan, - int slave_id) +static int sh_dmae_setup_xfer(struct shdma_chan *schan, int slave_id) { struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan, shdma_chan); + int ret = 0; if (slave_id >= 0) { const struct sh_dmae_slave_config *cfg = sh_chan->config; - dmae_set_dmars(sh_chan, cfg->mid_rid); - dmae_set_chcr(sh_chan, cfg->chcr); + ret = dmae_set_dmars(sh_chan, cfg->mid_rid); + if (ret < 0) + goto END; + + ret = dmae_set_chcr(sh_chan, cfg->chcr); + if (ret < 0) + goto END; + } else { dmae_init(sh_chan); } + +END: + return ret; } /* - * Find a slave channel configuration from the contoller list by either a slave + * Find a slave channel configuration from the controller list by either a slave * ID in the non-DT case, or by a MID/RID value in the DT case */ static const struct sh_dmae_slave_config *dmae_find_slave( @@ -678,7 +686,7 @@ static int sh_dmae_probe(struct platform_device *pdev) int err, errirq, i, irq_cnt = 0, irqres = 0, irq_cap = 0; struct sh_dmae_device *shdev; struct dma_device *dma_dev; - struct resource *chan, *dmars, *errirq_res, *chanirq_res; + struct resource *dmars, *errirq_res, *chanirq_res; if (pdev->dev.of_node) pdata = of_device_get_match_data(&pdev->dev); @@ -689,7 +697,6 @@ static int sh_dmae_probe(struct platform_device *pdev) if (!pdata || !pdata->channel_num) return -ENODEV; - chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); /* DMARS area is optional */ dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); /* @@ -709,7 +716,7 @@ static int sh_dmae_probe(struct platform_device *pdev) * requested with the IRQF_SHARED flag */ errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!chan || !errirq_res) + if (!errirq_res) return -ENODEV; shdev = devm_kzalloc(&pdev->dev, sizeof(struct sh_dmae_device), @@ -719,7 +726,7 @@ static int sh_dmae_probe(struct platform_device *pdev) dma_dev = &shdev->shdma_dev.dma_dev; - shdev->chan_reg = devm_ioremap_resource(&pdev->dev, chan); + shdev->chan_reg = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(shdev->chan_reg)) return PTR_ERR(shdev->chan_reg); if (dmars) { @@ -884,7 +891,7 @@ eshdma: return err; } -static int sh_dmae_remove(struct platform_device *pdev) +static void sh_dmae_remove(struct platform_device *pdev) { struct sh_dmae_device *shdev = platform_get_drvdata(pdev); struct dma_device *dma_dev = &shdev->shdma_dev.dma_dev; @@ -901,8 +908,6 @@ static int sh_dmae_remove(struct platform_device *pdev) shdma_cleanup(&shdev->shdma_dev); synchronize_rcu(); - - return 0; } static struct platform_driver sh_dmae_driver = { diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c index b14cf350b669..b42e5a66fd95 100644 --- a/drivers/dma/sh/usb-dmac.c +++ b/drivers/dma/sh/usb-dmac.c @@ -57,7 +57,7 @@ struct usb_dmac_desc { u32 residue; struct list_head node; dma_cookie_t done_cookie; - struct usb_dmac_sg sg[]; + struct usb_dmac_sg sg[] __counted_by(sg_allocated_len); }; #define to_usb_dmac_desc(vd) container_of(vd, struct usb_dmac_desc, vd) @@ -301,7 +301,7 @@ static struct usb_dmac_desc *usb_dmac_desc_get(struct usb_dmac_chan *chan, struct usb_dmac_desc *desc = NULL; unsigned long flags; - /* Get a freed descritpor */ + /* Get a freed descriptor */ spin_lock_irqsave(&chan->vc.lock, flags); list_for_each_entry(desc, &chan->desc_freed, node) { if (sg_len <= desc->sg_allocated_len) { @@ -670,7 +670,6 @@ static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec, * Power management */ -#ifdef CONFIG_PM static int usb_dmac_runtime_suspend(struct device *dev) { struct usb_dmac *dmac = dev_get_drvdata(dev); @@ -691,13 +690,11 @@ static int usb_dmac_runtime_resume(struct device *dev) return usb_dmac_init(dmac); } -#endif /* CONFIG_PM */ static const struct dev_pm_ops usb_dmac_pm = { - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(usb_dmac_runtime_suspend, usb_dmac_runtime_resume, - NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + RUNTIME_PM_OPS(usb_dmac_runtime_suspend, usb_dmac_runtime_resume, NULL) }; /* ----------------------------------------------------------------------------- @@ -706,10 +703,10 @@ static const struct dev_pm_ops usb_dmac_pm = { static int usb_dmac_chan_probe(struct usb_dmac *dmac, struct usb_dmac_chan *uchan, - unsigned int index) + u8 index) { struct platform_device *pdev = to_platform_device(dmac->dev); - char pdev_irqname[5]; + char pdev_irqname[6]; char *irqname; int ret; @@ -717,7 +714,7 @@ static int usb_dmac_chan_probe(struct usb_dmac *dmac, uchan->iomem = dmac->iomem + USB_DMAC_CHAN_OFFSET(index); /* Request the channel interrupt. */ - sprintf(pdev_irqname, "ch%u", index); + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); uchan->irq = platform_get_irq_byname(pdev, pdev_irqname); if (uchan->irq < 0) return -ENODEV; @@ -768,8 +765,8 @@ static int usb_dmac_probe(struct platform_device *pdev) const enum dma_slave_buswidth widths = USB_DMAC_SLAVE_BUSWIDTH; struct dma_device *engine; struct usb_dmac *dmac; - unsigned int i; int ret; + u8 i; dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); if (!dmac) @@ -866,10 +863,10 @@ static void usb_dmac_chan_remove(struct usb_dmac *dmac, devm_free_irq(dmac->dev, uchan->irq, uchan); } -static int usb_dmac_remove(struct platform_device *pdev) +static void usb_dmac_remove(struct platform_device *pdev) { struct usb_dmac *dmac = platform_get_drvdata(pdev); - int i; + u8 i; for (i = 0; i < dmac->n_channels; ++i) usb_dmac_chan_remove(dmac, &dmac->channels[i]); @@ -877,8 +874,6 @@ static int usb_dmac_remove(struct platform_device *pdev) dma_async_device_unregister(&dmac->engine); pm_runtime_disable(&pdev->dev); - - return 0; } static void usb_dmac_shutdown(struct platform_device *pdev) @@ -896,7 +891,7 @@ MODULE_DEVICE_TABLE(of, usb_dmac_of_ids); static struct platform_driver usb_dmac_driver = { .driver = { - .pm = &usb_dmac_pm, + .pm = pm_ptr(&usb_dmac_pm), .name = "usb-dmac", .of_match_table = usb_dmac_of_ids, }, diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index 2b639adb48ba..6207e0b185e1 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -15,7 +15,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> @@ -212,7 +212,7 @@ struct sprd_dma_dev { struct clk *ashb_clk; int irq; u32 total_chns; - struct sprd_dma_chn channels[]; + struct sprd_dma_chn channels[] __counted_by(total_chns); }; static void sprd_dma_free_desc(struct virt_dma_desc *vd); @@ -572,8 +572,7 @@ static void sprd_dma_stop(struct sprd_dma_chn *schan) schan->cur_desc = NULL; } -static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc, - enum sprd_dma_int_type int_type, +static bool sprd_dma_check_trans_done(enum sprd_dma_int_type int_type, enum sprd_dma_req_mode req_mode) { if (int_type == SPRD_DMA_NO_INT) @@ -619,8 +618,7 @@ static irqreturn_t dma_irq_handle(int irq, void *dev_id) vchan_cyclic_callback(&sdesc->vd); } else { /* Check if the dma request descriptor is done. */ - trans_done = sprd_dma_check_trans_done(sdesc, int_type, - req_type); + trans_done = sprd_dma_check_trans_done(int_type, req_type); if (trans_done == true) { vchan_cookie_complete(&sdesc->vd); schan->cur_desc = NULL; @@ -1117,6 +1115,15 @@ static int sprd_dma_probe(struct platform_device *pdev) u32 chn_count; int ret, i; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); + if (ret) { + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(&pdev->dev, "unable to set coherent mask to 32\n"); + return ret; + } + } + /* Parse new and deprecated dma-channels properties */ ret = device_property_read_u32(&pdev->dev, "dma-channels", &chn_count); if (ret) @@ -1232,7 +1239,7 @@ err_rpm: return ret; } -static int sprd_dma_remove(struct platform_device *pdev) +static void sprd_dma_remove(struct platform_device *pdev) { struct sprd_dma_dev *sdev = platform_get_drvdata(pdev); struct sprd_dma_chn *c, *cn; @@ -1255,7 +1262,6 @@ static int sprd_dma_remove(struct platform_device *pdev) pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); - return 0; } static const struct of_device_id sprd_dma_match[] = { @@ -1305,4 +1311,3 @@ MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("DMA driver for Spreadtrum"); MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>"); MODULE_AUTHOR("Eric Long <eric.long@spreadtrum.com>"); -MODULE_ALIAS("platform:sprd-dma"); diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c index d95c421877fb..dc2ab7d16cf2 100644 --- a/drivers/dma/st_fdma.c +++ b/drivers/dma/st_fdma.c @@ -10,9 +10,10 @@ #include <linux/init.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <linux/interrupt.h> #include <linux/remoteproc.h> #include <linux/slab.h> @@ -739,18 +740,11 @@ static void st_fdma_free(struct st_fdma_dev *fdev) static int st_fdma_probe(struct platform_device *pdev) { struct st_fdma_dev *fdev; - const struct of_device_id *match; struct device_node *np = pdev->dev.of_node; const struct st_fdma_driverdata *drvdata; int ret, i; - match = of_match_device((st_fdma_match), &pdev->dev); - if (!match || !match->data) { - dev_err(&pdev->dev, "No device match found\n"); - return -ENODEV; - } - - drvdata = match->data; + drvdata = device_get_match_data(&pdev->dev); fdev = devm_kzalloc(&pdev->dev, sizeof(*fdev), GFP_KERNEL); if (!fdev) @@ -849,15 +843,13 @@ err: return ret; } -static int st_fdma_remove(struct platform_device *pdev) +static void st_fdma_remove(struct platform_device *pdev) { struct st_fdma_dev *fdev = platform_get_drvdata(pdev); devm_free_irq(&pdev->dev, fdev->irq, fdev); st_slim_rproc_put(fdev->slim_rproc); of_dma_controller_free(pdev->dev.of_node); - - return 0; } static struct platform_driver st_fdma_platform_driver = { @@ -874,4 +866,3 @@ MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("STMicroelectronics FDMA engine driver"); MODULE_AUTHOR("Ludovic.barre <Ludovic.barre@st.com>"); MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>"); -MODULE_ALIAS("platform:" DRIVER_NAME); diff --git a/drivers/dma/st_fdma.h b/drivers/dma/st_fdma.h index fa15b97a3bab..f296412e96b6 100644 --- a/drivers/dma/st_fdma.h +++ b/drivers/dma/st_fdma.h @@ -97,7 +97,7 @@ struct st_fdma_desc { struct st_fdma_chan *fchan; bool iscyclic; unsigned int n_nodes; - struct st_fdma_sw_node node[]; + struct st_fdma_sw_node node[] __counted_by(n_nodes); }; enum st_fdma_type { diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 825001bde42c..d52e1685aed5 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -31,13 +31,11 @@ /** * struct stedma40_platform_data - Configuration struct for the dma device. * - * @dev_tx: mapping between destination event line and io address - * @dev_rx: mapping between source event line and io address * @disabled_channels: A vector, ending with -1, that marks physical channels * that are for different reasons not available for the driver. * @soft_lli_chans: A vector, that marks physical channels will use LLI by SW * which avoids HW bug that exists in some versions of the controller. - * SoftLLI introduces relink overhead that could impact performace for + * SoftLLI introduces relink overhead that could impact performance for * certain use cases. * @num_of_soft_lli_chans: The number of channels that needs to be configured * to use SoftLLI. @@ -184,7 +182,7 @@ static __maybe_unused u32 d40_backup_regs[] = { /* * since 9540 and 8540 has the same HW revision - * use v4a for 9540 or ealier + * use v4a for 9540 or earlier * use v4b for 8540 or later * HW revision: * DB8500ed has revision 0 @@ -411,7 +409,7 @@ struct d40_desc { * * @base: The virtual address of LCLA. 18 bit aligned. * @dma_addr: DMA address, if mapped - * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used. + * @base_unaligned: The original kmalloc pointer, if kmalloc is used. * This pointer is only there for clean-up on error. * @pages: The number of pages needed for all physical channels. * Only used later for clean-up on error @@ -1655,7 +1653,7 @@ static void dma_tasklet(struct tasklet_struct *t) return; check_pending_tx: - /* Rescue manouver if receiving double interrupts */ + /* Rescue maneuver if receiving double interrupts */ if (d40c->pending_tx > 0) d40c->pending_tx--; spin_unlock_irqrestore(&d40c->lock, flags); @@ -3412,7 +3410,7 @@ static int __init d40_lcla_allocate(struct d40_base *base) base->lcla_pool.base = (void *)page_list[i]; } else { /* - * After many attempts and no succees with finding the correct + * After many attempts and no success with finding the correct * alignment, try with allocating a big buffer. */ dev_warn(base->dev, @@ -3590,6 +3588,10 @@ static int __init d40_probe(struct platform_device *pdev) spin_lock_init(&base->lcla_pool.lock); base->irq = platform_get_irq(pdev, 0); + if (base->irq < 0) { + ret = base->irq; + goto destroy_cache; + } ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); if (ret) { @@ -3630,11 +3632,7 @@ static int __init d40_probe(struct platform_device *pdev) if (ret) goto destroy_cache; - ret = dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE); - if (ret) { - d40_err(dev, "Failed to set dma max seg size\n"); - goto destroy_cache; - } + dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE); d40_hw_init(base); @@ -3664,6 +3662,7 @@ static int __init d40_probe(struct platform_device *pdev) regulator_disable(base->lcpa_regulator); regulator_put(base->lcpa_regulator); } + pm_runtime_disable(base->dev); report_failure: d40_err(dev, "probe failed\n"); diff --git a/drivers/dma/ste_dma40.h b/drivers/dma/ste_dma40.h index c697bfe16a01..a90c786acc1f 100644 --- a/drivers/dma/ste_dma40.h +++ b/drivers/dma/ste_dma40.h @@ -4,7 +4,7 @@ #define STE_DMA40_H /* - * Maxium size for a single dma descriptor + * Maximum size for a single dma descriptor * Size is limited to 16 bits. * Size is in the units of addr-widths (1,2,4,8 bytes) * Larger transfers will be split up to multiple linked desc diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h index c504e855eb02..2e30e9a94a1e 100644 --- a/drivers/dma/ste_dma40_ll.h +++ b/drivers/dma/ste_dma40_ll.h @@ -369,7 +369,7 @@ struct d40_phy_lli_bidir { * @lcsp02: Either maps to register lcsp0 if src or lcsp2 if dst. * @lcsp13: Either maps to register lcsp1 if src or lcsp3 if dst. * - * This struct must be 8 bytes aligned since it will be accessed directy by + * This struct must be 8 bytes aligned since it will be accessed directly by * the DMA. Never add any none hw mapped registers to this struct. */ diff --git a/drivers/dma/stm32/Kconfig b/drivers/dma/stm32/Kconfig new file mode 100644 index 000000000000..4d8d8063133b --- /dev/null +++ b/drivers/dma/stm32/Kconfig @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# STM32 DMA controllers drivers +# +if ARCH_STM32 || COMPILE_TEST + +config STM32_DMA + bool "STMicroelectronics STM32 DMA support" + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the on-chip DMA controller on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA controller + and want to use DMA say Y here. + +config STM32_DMAMUX + bool "STMicroelectronics STM32 DMA multiplexer support" + depends on STM32_DMA + help + Enable support for the on-chip DMA multiplexer on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA multiplexer + and want to use DMAMUX say Y here. + +config STM32_MDMA + bool "STMicroelectronics STM32 master DMA support" + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the on-chip MDMA controller on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA controller + and want to use MDMA say Y here. + +config STM32_DMA3 + tristate "STMicroelectronics STM32 DMA3 support" + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the on-chip DMA3 controller on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA3 controller + and want to use DMA3, say Y here. + +endif diff --git a/drivers/dma/stm32/Makefile b/drivers/dma/stm32/Makefile new file mode 100644 index 000000000000..5082db4b4c1c --- /dev/null +++ b/drivers/dma/stm32/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_STM32_DMA) += stm32-dma.o +obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o +obj-$(CONFIG_STM32_MDMA) += stm32-mdma.o +obj-$(CONFIG_STM32_DMA3) += stm32-dma3.o diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32/stm32-dma.c index 37674029cb42..04389936c8a6 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32/stm32-dma.c @@ -21,7 +21,6 @@ #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -29,7 +28,7 @@ #include <linux/sched.h> #include <linux/slab.h> -#include "virt-dma.h" +#include "../virt-dma.h" #define STM32_DMA_LISR 0x0000 /* DMA Low Int Status Reg */ #define STM32_DMA_HISR 0x0004 /* DMA High Int Status Reg */ @@ -191,7 +190,7 @@ struct stm32_dma_desc { struct virt_dma_desc vdesc; bool cyclic; u32 num_sgs; - struct stm32_dma_sg_req sg_req[]; + struct stm32_dma_sg_req sg_req[] __counted_by(num_sgs); }; /** @@ -614,7 +613,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) reg->dma_scr |= STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); - dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); } static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) @@ -677,7 +676,7 @@ static void stm32_dma_handle_chan_paused(struct stm32_dma_chan *chan) chan->status = DMA_PAUSED; - dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan); } static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan) @@ -729,7 +728,7 @@ static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan) dma_scr |= STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); - dev_dbg(chan2dev(chan), "vchan %pK: reconfigured after pause/resume\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: reconfigured after pause/resume\n", &chan->vchan); } static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr) @@ -745,7 +744,7 @@ static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr) /* cyclic while CIRC/DBM disable => post resume reconfiguration needed */ if (!(scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM))) stm32_dma_post_resume_reconfigure(chan); - else if (scr & STM32_DMA_SCR_DBM) + else if (scr & STM32_DMA_SCR_DBM && chan->desc->num_sgs > 2) stm32_dma_configure_next_sg(chan); } else { chan->busy = false; @@ -821,7 +820,7 @@ static void stm32_dma_issue_pending(struct dma_chan *c) spin_lock_irqsave(&chan->vchan.lock, flags); if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) { - dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); stm32_dma_start_transfer(chan); } @@ -923,7 +922,7 @@ static int stm32_dma_resume(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); - dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan); return 0; } @@ -1105,6 +1104,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( desc = kzalloc(struct_size(desc, sg_req, sg_len), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = sg_len; /* Set peripheral flow controller */ if (chan->dma_sconfig.device_fc) @@ -1113,8 +1113,10 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; /* Activate Double Buffer Mode if DMA triggers STM32 MDMA and more than 1 sg */ - if (chan->trig_mdma && sg_len > 1) + if (chan->trig_mdma && sg_len > 1) { chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM; + chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT; + } for_each_sg(sgl, sg, sg_len, i) { ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, @@ -1141,8 +1143,6 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( desc->sg_req[i].chan_reg.dma_sm1ar += sg_dma_len(sg); desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; } - - desc->num_sgs = sg_len; desc->cyclic = false; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -1216,6 +1216,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( desc = kzalloc(struct_size(desc, sg_req, num_periods), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = num_periods; for (i = 0; i < num_periods; i++) { desc->sg_req[i].len = period_len; @@ -1232,8 +1233,6 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( if (!chan->trig_mdma) buf_addr += period_len; } - - desc->num_sgs = num_periods; desc->cyclic = true; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -1247,13 +1246,14 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( enum dma_slave_buswidth max_width; struct stm32_dma_desc *desc; size_t xfer_count, offset; - u32 num_sgs, best_burst, dma_burst, threshold; - int i; + u32 num_sgs, best_burst, threshold; + int dma_burst, i; num_sgs = DIV_ROUND_UP(len, STM32_DMA_ALIGNED_MAX_DATA_ITEMS); desc = kzalloc(struct_size(desc, sg_req, num_sgs), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = num_sgs; threshold = chan->threshold; @@ -1266,6 +1266,10 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( best_burst = stm32_dma_get_best_burst(len, STM32_DMA_MAX_BURST, threshold, max_width); dma_burst = stm32_dma_get_burst(chan, best_burst); + if (dma_burst < 0) { + kfree(desc); + return NULL; + } stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); desc->sg_req[i].chan_reg.dma_scr = @@ -1283,8 +1287,6 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( desc->sg_req[i].chan_reg.dma_sndtr = xfer_count; desc->sg_req[i].len = xfer_count; } - - desc->num_sgs = num_sgs; desc->cyclic = false; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -1387,11 +1389,12 @@ static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, residue = stm32_dma_get_remaining_bytes(chan); - if (chan->desc->cyclic && !stm32_dma_is_current_sg(chan)) { + if ((chan->desc->cyclic || chan->trig_mdma) && !stm32_dma_is_current_sg(chan)) { n_sg++; if (n_sg == chan->desc->num_sgs) n_sg = 0; - residue = sg_req->len; + if (!chan->trig_mdma) + residue = sg_req->len; } /* @@ -1401,7 +1404,7 @@ static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, * residue = remaining bytes from NDTR + remaining * periods/sg to be transferred */ - if (!chan->desc->cyclic || n_sg != 0) + if ((!chan->desc->cyclic && !chan->trig_mdma) || n_sg != 0) for (i = n_sg; i < desc->num_sgs; i++) residue += desc->sg_req[i].len; @@ -1564,25 +1567,17 @@ static int stm32_dma_probe(struct platform_device *pdev) struct stm32_dma_chan *chan; struct stm32_dma_device *dmadev; struct dma_device *dd; - const struct of_device_id *match; struct resource *res; struct reset_control *rst; int i, ret; - match = of_match_device(stm32_dma_of_match, &pdev->dev); - if (!match) { - dev_err(&pdev->dev, "Error: No device match found\n"); - return -ENODEV; - } - dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); if (!dmadev) return -ENOMEM; dd = &dmadev->ddev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dmadev->base = devm_ioremap_resource(&pdev->dev, res); + dmadev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(dmadev->base)) return PTR_ERR(dmadev->base); diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c new file mode 100644 index 000000000000..50e7106c5cb7 --- /dev/null +++ b/drivers/dma/stm32/stm32-dma3.c @@ -0,0 +1,1926 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * STM32 DMA3 controller driver + * + * Copyright (C) STMicroelectronics 2024 + * Author(s): Amelie Delaunay <amelie.delaunay@foss.st.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> +#include <linux/slab.h> + +#include "../virt-dma.h" + +#define STM32_DMA3_SECCFGR 0x00 +#define STM32_DMA3_PRIVCFGR 0x04 +#define STM32_DMA3_RCFGLOCKR 0x08 +#define STM32_DMA3_MISR 0x0c +#define STM32_DMA3_SMISR 0x10 + +#define STM32_DMA3_CLBAR(x) (0x50 + 0x80 * (x)) +#define STM32_DMA3_CCIDCFGR(x) (0x54 + 0x80 * (x)) +#define STM32_DMA3_CSEMCR(x) (0x58 + 0x80 * (x)) +#define STM32_DMA3_CFCR(x) (0x5c + 0x80 * (x)) +#define STM32_DMA3_CSR(x) (0x60 + 0x80 * (x)) +#define STM32_DMA3_CCR(x) (0x64 + 0x80 * (x)) +#define STM32_DMA3_CTR1(x) (0x90 + 0x80 * (x)) +#define STM32_DMA3_CTR2(x) (0x94 + 0x80 * (x)) +#define STM32_DMA3_CBR1(x) (0x98 + 0x80 * (x)) +#define STM32_DMA3_CSAR(x) (0x9c + 0x80 * (x)) +#define STM32_DMA3_CDAR(x) (0xa0 + 0x80 * (x)) +#define STM32_DMA3_CLLR(x) (0xcc + 0x80 * (x)) + +#define STM32_DMA3_HWCFGR13 0xfc0 /* G_PER_CTRL(X) x=8..15 */ +#define STM32_DMA3_HWCFGR12 0xfc4 /* G_PER_CTRL(X) x=0..7 */ +#define STM32_DMA3_HWCFGR4 0xfe4 /* G_FIFO_SIZE(X) x=8..15 */ +#define STM32_DMA3_HWCFGR3 0xfe8 /* G_FIFO_SIZE(X) x=0..7 */ +#define STM32_DMA3_HWCFGR2 0xfec /* G_MAX_REQ_ID */ +#define STM32_DMA3_HWCFGR1 0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */ +#define STM32_DMA3_VERR 0xff4 + +/* SECCFGR DMA secure configuration register */ +#define SECCFGR_SEC(x) BIT(x) + +/* MISR DMA non-secure/secure masked interrupt status register */ +#define MISR_MIS(x) BIT(x) + +/* CxLBAR DMA channel x linked_list base address register */ +#define CLBAR_LBA GENMASK(31, 16) + +/* CxCIDCFGR DMA channel x CID register */ +#define CCIDCFGR_CFEN BIT(0) +#define CCIDCFGR_SEM_EN BIT(1) +#define CCIDCFGR_SCID GENMASK(5, 4) +#define CCIDCFGR_SEM_WLIST_CID0 BIT(16) +#define CCIDCFGR_SEM_WLIST_CID1 BIT(17) +#define CCIDCFGR_SEM_WLIST_CID2 BIT(18) + +enum ccidcfgr_cid { + CCIDCFGR_CID0, + CCIDCFGR_CID1, + CCIDCFGR_CID2, +}; + +/* CxSEMCR DMA channel x semaphore control register */ +#define CSEMCR_SEM_MUTEX BIT(0) +#define CSEMCR_SEM_CCID GENMASK(5, 4) + +/* CxFCR DMA channel x flag clear register */ +#define CFCR_TCF BIT(8) +#define CFCR_HTF BIT(9) +#define CFCR_DTEF BIT(10) +#define CFCR_ULEF BIT(11) +#define CFCR_USEF BIT(12) +#define CFCR_SUSPF BIT(13) + +/* CxSR DMA channel x status register */ +#define CSR_IDLEF BIT(0) +#define CSR_TCF BIT(8) +#define CSR_HTF BIT(9) +#define CSR_DTEF BIT(10) +#define CSR_ULEF BIT(11) +#define CSR_USEF BIT(12) +#define CSR_SUSPF BIT(13) +#define CSR_ALL_F GENMASK(13, 8) +#define CSR_FIFOL GENMASK(24, 16) + +/* CxCR DMA channel x control register */ +#define CCR_EN BIT(0) +#define CCR_RESET BIT(1) +#define CCR_SUSP BIT(2) +#define CCR_TCIE BIT(8) +#define CCR_HTIE BIT(9) +#define CCR_DTEIE BIT(10) +#define CCR_ULEIE BIT(11) +#define CCR_USEIE BIT(12) +#define CCR_SUSPIE BIT(13) +#define CCR_ALLIE GENMASK(13, 8) +#define CCR_LSM BIT(16) +#define CCR_LAP BIT(17) +#define CCR_PRIO GENMASK(23, 22) + +enum ccr_prio { + CCR_PRIO_LOW, + CCR_PRIO_MID, + CCR_PRIO_HIGH, + CCR_PRIO_VERY_HIGH, +}; + +/* CxTR1 DMA channel x transfer register 1 */ +#define CTR1_SINC BIT(3) +#define CTR1_SBL_1 GENMASK(9, 4) +#define CTR1_DINC BIT(19) +#define CTR1_DBL_1 GENMASK(25, 20) +#define CTR1_SDW_LOG2 GENMASK(1, 0) +#define CTR1_PAM GENMASK(12, 11) +#define CTR1_SAP BIT(14) +#define CTR1_DDW_LOG2 GENMASK(17, 16) +#define CTR1_DAP BIT(30) + +enum ctr1_dw { + CTR1_DW_BYTE, + CTR1_DW_HWORD, + CTR1_DW_WORD, + CTR1_DW_DWORD, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */ +}; + +enum ctr1_pam { + CTR1_PAM_0S_LT, /* if DDW > SDW, padded with 0s else left-truncated */ + CTR1_PAM_SE_RT, /* if DDW > SDW, sign extended else right-truncated */ + CTR1_PAM_PACK_UNPACK, /* FIFO queued */ +}; + +/* CxTR2 DMA channel x transfer register 2 */ +#define CTR2_REQSEL GENMASK(7, 0) +#define CTR2_SWREQ BIT(9) +#define CTR2_DREQ BIT(10) +#define CTR2_BREQ BIT(11) +#define CTR2_PFREQ BIT(12) +#define CTR2_TCEM GENMASK(31, 30) + +enum ctr2_tcem { + CTR2_TCEM_BLOCK, + CTR2_TCEM_REPEAT_BLOCK, + CTR2_TCEM_LLI, + CTR2_TCEM_CHANNEL, +}; + +/* CxBR1 DMA channel x block register 1 */ +#define CBR1_BNDT GENMASK(15, 0) + +/* CxLLR DMA channel x linked-list address register */ +#define CLLR_LA GENMASK(15, 2) +#define CLLR_ULL BIT(16) +#define CLLR_UDA BIT(27) +#define CLLR_USA BIT(28) +#define CLLR_UB1 BIT(29) +#define CLLR_UT2 BIT(30) +#define CLLR_UT1 BIT(31) + +/* HWCFGR13 DMA hardware configuration register 13 x=8..15 */ +/* HWCFGR12 DMA hardware configuration register 12 x=0..7 */ +#define G_PER_CTRL(x) (ULL(0x1) << (4 * (x))) + +/* HWCFGR4 DMA hardware configuration register 4 x=8..15 */ +/* HWCFGR3 DMA hardware configuration register 3 x=0..7 */ +#define G_FIFO_SIZE(x) (ULL(0x7) << (4 * (x))) + +#define get_chan_hwcfg(x, mask, reg) (((reg) & (mask)) >> (4 * (x))) + +/* HWCFGR2 DMA hardware configuration register 2 */ +#define G_MAX_REQ_ID GENMASK(7, 0) + +/* HWCFGR1 DMA hardware configuration register 1 */ +#define G_MASTER_PORTS GENMASK(2, 0) +#define G_NUM_CHANNELS GENMASK(12, 8) +#define G_M0_DATA_WIDTH_ENC GENMASK(25, 24) +#define G_M1_DATA_WIDTH_ENC GENMASK(29, 28) + +enum stm32_dma3_master_ports { + AXI64, /* 1x AXI: 64-bit port 0 */ + AHB32, /* 1x AHB: 32-bit port 0 */ + AHB32_AHB32, /* 2x AHB: 32-bit port 0 and 32-bit port 1 */ + AXI64_AHB32, /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */ + AXI64_AXI64, /* 2x AXI: 64-bit port 0 and 64-bit port 1 */ + AXI128_AHB32, /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */ +}; + +enum stm32_dma3_port_data_width { + DW_32, /* 32-bit, for AHB */ + DW_64, /* 64-bit, for AXI */ + DW_128, /* 128-bit, for AXI */ + DW_INVALID, +}; + +/* VERR DMA version register */ +#define VERR_MINREV GENMASK(3, 0) +#define VERR_MAJREV GENMASK(7, 4) + +/* Device tree */ +/* struct stm32_dma3_dt_conf */ +/* .ch_conf */ +#define STM32_DMA3_DT_PRIO GENMASK(1, 0) /* CCR_PRIO */ +#define STM32_DMA3_DT_FIFO GENMASK(7, 4) +/* .tr_conf */ +#define STM32_DMA3_DT_SINC BIT(0) /* CTR1_SINC */ +#define STM32_DMA3_DT_SAP BIT(1) /* CTR1_SAP */ +#define STM32_DMA3_DT_DINC BIT(4) /* CTR1_DINC */ +#define STM32_DMA3_DT_DAP BIT(5) /* CTR1_DAP */ +#define STM32_DMA3_DT_BREQ BIT(8) /* CTR2_BREQ */ +#define STM32_DMA3_DT_PFREQ BIT(9) /* CTR2_PFREQ */ +#define STM32_DMA3_DT_TCEM GENMASK(13, 12) /* CTR2_TCEM */ +#define STM32_DMA3_DT_NOPACK BIT(16) /* CTR1_PAM */ +#define STM32_DMA3_DT_NOREFACT BIT(17) + +/* struct stm32_dma3_chan .config_set bitfield */ +#define STM32_DMA3_CFG_SET_DT BIT(0) +#define STM32_DMA3_CFG_SET_DMA BIT(1) +#define STM32_DMA3_CFG_SET_BOTH (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA) + +#define STM32_DMA3_MAX_BLOCK_SIZE ALIGN_DOWN(CBR1_BNDT, 64) +#define STM32_DMA3_MAX_BURST_LEN (1 + min_t(u32, FIELD_MAX(CTR1_SBL_1), \ + FIELD_MAX(CTR1_DBL_1))) +#define port_is_ahb(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \ + ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); }) +#define port_is_axi(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \ + ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); }) +#define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) || \ + (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \ + DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES) + +/* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */ +struct stm32_dma3_hwdesc { + u32 ctr1; + u32 ctr2; + u32 cbr1; + u32 csar; + u32 cdar; + u32 cllr; +} __packed __aligned(32); + +/* + * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed + * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte + * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents + * the sg size limitation. + */ +#define STM32_DMA3_MAX_SEG_SIZE ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE) + +/* + * Linked-list items + */ +struct stm32_dma3_lli { + struct stm32_dma3_hwdesc *hwdesc; + dma_addr_t hwdesc_addr; +}; + +struct stm32_dma3_swdesc { + struct virt_dma_desc vdesc; + u32 ccr; + bool cyclic; + u32 lli_size; + struct stm32_dma3_lli lli[] __counted_by(lli_size); +}; + +struct stm32_dma3_dt_conf { + u32 ch_id; + u32 req_line; + u32 ch_conf; + u32 tr_conf; +}; + +struct stm32_dma3_chan { + struct virt_dma_chan vchan; + u32 id; + int irq; + u32 fifo_size; + u32 max_burst; + bool semaphore_mode; + struct stm32_dma3_dt_conf dt_config; + struct dma_slave_config dma_config; + u8 config_set; + struct dma_pool *lli_pool; + struct stm32_dma3_swdesc *swdesc; + enum ctr2_tcem tcem; + u32 dma_status; +}; + +struct stm32_dma3_pdata { + u32 axi_max_burst_len; +}; + +struct stm32_dma3_ddata { + struct dma_device dma_dev; + void __iomem *base; + struct clk *clk; + struct stm32_dma3_chan *chans; + u32 dma_channels; + u32 dma_requests; + enum stm32_dma3_port_data_width ports_max_dw[2]; + u32 axi_max_burst_len; +}; + +static inline struct stm32_dma3_ddata *to_stm32_dma3_ddata(struct stm32_dma3_chan *chan) +{ + return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev); +} + +static inline struct stm32_dma3_chan *to_stm32_dma3_chan(struct dma_chan *c) +{ + return container_of(c, struct stm32_dma3_chan, vchan.chan); +} + +static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct stm32_dma3_swdesc, vdesc); +} + +static struct device *chan2dev(struct stm32_dma3_chan *chan) +{ + return &chan->vchan.chan.dev->device; +} + +static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct device *dev = chan2dev(chan); + u32 id = chan->id, offset; + + offset = STM32_DMA3_SECCFGR; + dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_PRIVCFGR; + dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CCIDCFGR(id); + dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CSEMCR(id); + dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CSR(id); + dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CCR(id); + dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CTR1(id); + dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CTR2(id); + dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CBR1(id); + dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CSAR(id); + dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CDAR(id); + dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CLLR(id); + dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CLBAR(id); + dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); +} + +static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc) +{ + struct stm32_dma3_hwdesc *hwdesc; + int i; + + for (i = 0; i < swdesc->lli_size; i++) { + hwdesc = swdesc->lli[i].hwdesc; + if (i) + dev_dbg(chan2dev(chan), "V\n"); + dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr); + dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1); + dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2); + dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1); + dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar); + dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar); + dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr); + } + + if (swdesc->cyclic) { + dev_dbg(chan2dev(chan), "|\n"); + dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr); + } else { + dev_dbg(chan2dev(chan), "X\n"); + } +} + +static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan *chan, u32 count) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct stm32_dma3_swdesc *swdesc; + int i; + + /* + * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes + * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be + * addressed, so abort the allocation. + */ + if ((count * 32) > CLLR_LA) { + dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE); + return NULL; + } + + swdesc = kzalloc(struct_size(swdesc, lli, count), GFP_NOWAIT); + if (!swdesc) + return NULL; + swdesc->lli_size = count; + + for (i = 0; i < count; i++) { + swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, + &swdesc->lli[i].hwdesc_addr); + if (!swdesc->lli[i].hwdesc) + goto err_pool_free; + } + swdesc->ccr = 0; + + /* Set LL base address */ + writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA, + ddata->base + STM32_DMA3_CLBAR(chan->id)); + + /* Set LL allocated port */ + swdesc->ccr &= ~CCR_LAP; + + return swdesc; + +err_pool_free: + dev_err(chan2dev(chan), "Failed to alloc descriptors\n"); + while (--i >= 0) + dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); + kfree(swdesc); + + return NULL; +} + +static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc) +{ + int i; + + for (i = 0; i < swdesc->lli_size; i++) + dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); + + kfree(swdesc); +} + +static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc) +{ + struct stm32_dma3_swdesc *swdesc = to_stm32_dma3_swdesc(vdesc); + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan); + + stm32_dma3_chan_desc_free(chan, swdesc); +} + +static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct device *dev = chan2dev(chan); + u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id)); + u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); + u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id)); + u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); + u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); + u32 bndt = FIELD_GET(CBR1_BNDT, cbr1); + u32 sdw = 1 << FIELD_GET(CTR1_SDW_LOG2, ctr1); + u32 ddw = 1 << FIELD_GET(CTR1_DDW_LOG2, ctr1); + u32 sap = FIELD_GET(CTR1_SAP, ctr1); + u32 dap = FIELD_GET(CTR1_DAP, ctr1); + + if (!bndt && !FIELD_GET(CLLR_UB1, cllr)) + dev_err(dev, "null source block size and no update of this value\n"); + if (bndt % sdw) + dev_err(dev, "source block size not multiple of src data width\n"); + if (FIELD_GET(CTR1_PAM, ctr1) == CTR1_PAM_PACK_UNPACK && bndt % ddw) + dev_err(dev, "(un)packing mode w/ src block size not multiple of dst data width\n"); + if (csar % sdw) + dev_err(dev, "unaligned source address not multiple of src data width\n"); + if (cdar % ddw) + dev_err(dev, "unaligned destination address not multiple of dst data width\n"); + if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap])) + dev_err(dev, "double-word source data width not supported on port %u\n", sap); + if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap])) + dev_err(dev, "double-word destination data width not supported on port %u\n", dap); +} + +static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc, + u32 curr, dma_addr_t src, dma_addr_t dst, u32 len, + u32 ctr1, u32 ctr2, bool is_last, bool is_cyclic) +{ + struct stm32_dma3_hwdesc *hwdesc; + dma_addr_t next_lli; + u32 next = curr + 1; + + hwdesc = swdesc->lli[curr].hwdesc; + hwdesc->ctr1 = ctr1; + hwdesc->ctr2 = ctr2; + hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len); + hwdesc->csar = src; + hwdesc->cdar = dst; + + if (is_last) { + if (is_cyclic) + next_lli = swdesc->lli[0].hwdesc_addr; + else + next_lli = 0; + } else { + next_lli = swdesc->lli[next].hwdesc_addr; + } + + hwdesc->cllr = 0; + if (next_lli) { + hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1; + hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL; + hwdesc->cllr |= (next_lli & CLLR_LA); + } + + /* + * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read + * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel + * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors + * are reused (DMA_CTRL_REUSE). + */ + if (is_last) + dma_wmb(); +} + +static enum dma_slave_buswidth stm32_dma3_get_max_dw(u32 chan_max_burst, + enum stm32_dma3_port_data_width port_max_dw, + u32 len, dma_addr_t addr) +{ + enum dma_slave_buswidth max_dw = get_chan_max_dw(port_max_dw, chan_max_burst); + + /* len and addr must be a multiple of dw */ + return 1 << __ffs(len | addr | max_dw); +} + +static u32 stm32_dma3_get_max_burst(u32 len, enum dma_slave_buswidth dw, + u32 chan_max_burst, u32 bus_max_burst) +{ + u32 max_burst = chan_max_burst ? chan_max_burst / dw : 1; + + /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */ + if (len < chan_max_burst) + max_burst = len / dw; + + /* + * HW doesn't modify the burst if burst size <= half of the fifo size. + * If len is not a multiple of burst size, last burst is shortened by HW. + * Take care of maximum burst supported on interconnect bus. + */ + return min_t(u32, max_burst, bus_max_burst); +} + +static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transfer_direction dir, + u32 *ccr, u32 *ctr1, u32 *ctr2, + dma_addr_t src_addr, dma_addr_t dst_addr, u32 len) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct dma_device dma_device = ddata->dma_dev; + u32 src_max_burst = STM32_DMA3_MAX_BURST_LEN, dst_max_burst = STM32_DMA3_MAX_BURST_LEN; + u32 sdw, ddw, sbl_max, dbl_max, tcem, init_dw, init_bl_max; + u32 _ctr1 = 0, _ctr2 = 0; + u32 ch_conf = chan->dt_config.ch_conf; + u32 tr_conf = chan->dt_config.tr_conf; + u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw; + u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw; + + dev_dbg(chan2dev(chan), "%s from %pad to %pad\n", + dmaengine_get_direction_text(dir), &src_addr, &dst_addr); + + sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst); + ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst); + sbl_max = chan->dma_config.src_maxburst ? : 1; + dbl_max = chan->dma_config.dst_maxburst ? : 1; + + /* Following conditions would raise User Setting Error interrupt */ + if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) { + dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw); + return -EINVAL; + } + + if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) { + dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n"); + return -EINVAL; + } + + sap_max_dw = ddata->ports_max_dw[sap]; + dap_max_dw = ddata->ports_max_dw[dap]; + if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) || + (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) { + dev_err(chan2dev(chan), + "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n", + sdw, ddw, sap, dap); + return -EINVAL; + } + + if (FIELD_GET(STM32_DMA3_DT_SINC, tr_conf)) + _ctr1 |= CTR1_SINC; + if (sap) + _ctr1 |= CTR1_SAP; + if (port_is_axi(sap_max_dw)) /* AXI - apply axi maximum burst limitation */ + src_max_burst = ddata->axi_max_burst_len; + if (FIELD_GET(STM32_DMA3_DT_DINC, tr_conf)) + _ctr1 |= CTR1_DINC; + if (dap) + _ctr1 |= CTR1_DAP; + if (port_is_axi(dap_max_dw)) /* AXI - apply axi maximum burst limitation */ + dst_max_burst = ddata->axi_max_burst_len; + + _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ; + if (FIELD_GET(STM32_DMA3_DT_BREQ, tr_conf)) + _ctr2 |= CTR2_BREQ; + if (dir == DMA_DEV_TO_MEM && FIELD_GET(STM32_DMA3_DT_PFREQ, tr_conf)) + _ctr2 |= CTR2_PFREQ; + tcem = FIELD_GET(STM32_DMA3_DT_TCEM, tr_conf); + _ctr2 |= FIELD_PREP(CTR2_TCEM, tcem); + + /* Store TCEM to know on which event TC flag occurred */ + chan->tcem = tcem; + /* Store direction for residue computation */ + chan->dma_config.direction = dir; + + switch (dir) { + case DMA_MEM_TO_DEV: + /* Set destination (device) data width and burst */ + ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, + len, dst_addr)); + dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst, + dst_max_burst)); + + /* Set source (memory) data width and burst */ + sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); + sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst); + if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf)) { + sdw = ddw; + sbl_max = dbl_max; + } + + _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); + _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); + _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); + _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); + + if (ddw != sdw) { + _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); + /* Should never reach this case as ddw is clamped down */ + if (len & (ddw - 1)) { + dev_err(chan2dev(chan), + "Packing mode is enabled and len is not multiple of ddw"); + return -EINVAL; + } + } + + /* dst = dev */ + _ctr2 |= CTR2_DREQ; + + break; + + case DMA_DEV_TO_MEM: + /* Set source (device) data width and burst */ + sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, + len, src_addr)); + sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst, + src_max_burst)); + + /* Set destination (memory) data width and burst */ + ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); + dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst); + if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf) || + ((_ctr2 & CTR2_PFREQ) && ddw > sdw)) { /* Packing to wider ddw not supported */ + ddw = sdw; + dbl_max = sbl_max; + } + + _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); + _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); + _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); + _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); + + if (ddw != sdw) { + _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); + /* Should never reach this case as ddw is clamped down */ + if (len & (ddw - 1)) { + dev_err(chan2dev(chan), + "Packing mode is enabled and len is not multiple of ddw\n"); + return -EINVAL; + } + } + + /* dst = mem */ + _ctr2 &= ~CTR2_DREQ; + + break; + + case DMA_MEM_TO_MEM: + /* Set source (memory) data width and burst */ + init_dw = sdw; + init_bl_max = sbl_max; + sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); + sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst); + if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { + sdw = min_t(u32, init_dw, sdw); + sbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, sdw, + chan->max_burst, + src_max_burst)); + } + + /* Set destination (memory) data width and burst */ + init_dw = ddw; + init_bl_max = dbl_max; + ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); + dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst); + if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { + ddw = min_t(u32, init_dw, ddw); + dbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, ddw, + chan->max_burst, + dst_max_burst)); + } + + _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); + _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); + _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); + _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); + + if (ddw != sdw) { + _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); + /* Should never reach this case as ddw is clamped down */ + if (len & (ddw - 1)) { + dev_err(chan2dev(chan), + "Packing mode is enabled and len is not multiple of ddw"); + return -EINVAL; + } + } + + /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */ + _ctr2 |= CTR2_SWREQ; + + break; + + default: + dev_err(chan2dev(chan), "Direction %s not supported\n", + dmaengine_get_direction_text(dir)); + return -EINVAL; + } + + *ccr |= FIELD_PREP(CCR_PRIO, FIELD_GET(STM32_DMA3_DT_PRIO, ch_conf)); + *ctr1 = _ctr1; + *ctr2 = _ctr2; + + dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n", + __func__, sdw, sbl_max, ddw, dbl_max); + + return 0; +} + +static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct virt_dma_desc *vdesc; + struct stm32_dma3_hwdesc *hwdesc; + u32 id = chan->id; + u32 csr, ccr; + + vdesc = vchan_next_desc(&chan->vchan); + if (!vdesc) { + chan->swdesc = NULL; + return; + } + list_del(&vdesc->node); + + chan->swdesc = to_stm32_dma3_swdesc(vdesc); + hwdesc = chan->swdesc->lli[0].hwdesc; + + stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc); + + writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id)); + writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id)); + writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id)); + writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id)); + writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id)); + writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id)); + writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id)); + + /* Clear any pending interrupts */ + csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id)); + if (csr & CSR_ALL_F) + writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id)); + + stm32_dma3_chan_dump_reg(chan); + + ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id)); + writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id)); + + chan->dma_status = DMA_IN_PROGRESS; + + dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); +} + +static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; + int ret = 0; + + if (susp) + ccr |= CCR_SUSP; + else + ccr &= ~CCR_SUSP; + + writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id)); + + if (susp) { + ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, + csr & CSR_SUSPF, 1, 10); + if (!ret) + writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); + + stm32_dma3_chan_dump_reg(chan); + } + + return ret; +} + +static void stm32_dma3_chan_reset(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; + + writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id)); +} + +static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc *swdesc, u32 cllr, u32 *residue) +{ + u32 i, lli_offset, next_lli_offset = cllr & CLLR_LA; + + /* If cllr is null, it means it is either the last or single item */ + if (!cllr) + return swdesc->lli_size - 1; + + /* In cyclic mode, go fast and first check we are not on the last item */ + if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA)) + return swdesc->lli_size - 1; + + /* As transfer is in progress, look backward from the last item */ + for (i = swdesc->lli_size - 1; i > 0; i--) { + *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1); + lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA; + if (lli_offset == next_lli_offset) + return i - 1; + } + + return -EINVAL; +} + +static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc, + struct dma_tx_state *txstate) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct device *dev = chan2dev(chan); + struct stm32_dma3_hwdesc *hwdesc; + u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol; + bool pack_unpack; + int ret; + + csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); + if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) { + /* Suspend current transfer to read registers for a snapshot */ + writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id)); + ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, + csr & (CSR_SUSPF | CSR_IDLEF), 1, 10); + + if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) { + writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); + writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); + if (ret) + dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr); + } + } + + /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */ + if (!(csr & CSR_IDLEF)) + dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr); + + /* + * If channel is not suspended, but Idle and Transfer Complete are set, + * linked-list is over, no residue + */ + if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF)) + return; + + /* Read registers to have a snapshot */ + cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); + cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); + cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); + + /* Resume current transfer */ + if (csr & CSR_SUSPF) { + writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); + writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); + } + + /* Add current BNDT */ + bndt = FIELD_GET(CBR1_BNDT, cbr1); + residue = bndt; + + /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */ + ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue); + if (ret < 0) { + dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n"); + return; + } + curr_lli = ret; + + /* Read current FIFO level - in units of programmed destination data width */ + hwdesc = swdesc->lli[curr_lli].hwdesc; + fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1)); + /* If the FIFO contains as many bytes as its size, it can't contain more */ + if (fifol == (1 << (chan->fifo_size + 1))) + goto skip_fifol_update; + + /* + * In case of PACKING (Destination burst length > Source burst length) or UNPACKING + * (Source burst length > Destination burst length), bytes could be pending in the FIFO + * (to be packed up to Destination burst length or unpacked into Destination burst length + * chunks). + * BNDT is not reliable, as it reflects the number of bytes read from the source but not the + * number of bytes written to the destination. + * FIFOL is also not sufficient, because it reflects the number of available write beats in + * units of Destination data width but not the bytes not yet packed or unpacked. + * In case of Destination increment DINC, it is possible to compute the number of bytes in + * the FIFO: + * fifol_in_bytes = bytes_read - bytes_written. + */ + pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK); + if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) { + int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt; + int bytes_written = cdar - hwdesc->cdar; + + if (bytes_read > 0) + fifol = bytes_read - bytes_written; + } + +skip_fifol_update: + if (fifol) { + dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol); + dma_set_in_flight_bytes(txstate, fifol); + /* + * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from + * the source memory buffer, so just need to add fifol to residue in case of + * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination + * memory buffer. + */ + if (chan->dma_config.direction == DMA_DEV_TO_MEM) + residue += fifol; + } + dma_set_residue(txstate, residue); +} + +static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 ccr; + int ret = 0; + + chan->dma_status = DMA_COMPLETE; + + /* Disable interrupts */ + ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)); + writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id)); + + if (!(ccr & CCR_SUSP) && (ccr & CCR_EN)) { + /* Suspend the channel */ + ret = stm32_dma3_chan_suspend(chan, true); + if (ret) + dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__); + } + + /* + * Reset the channel: this causes the reset of the FIFO and the reset of the channel + * internal state, the reset of CCR_EN and CCR_SUSP bits. + */ + stm32_dma3_chan_reset(chan); + + return ret; +} + +static void stm32_dma3_chan_complete(struct stm32_dma3_chan *chan) +{ + if (!chan->swdesc) + return; + + vchan_cookie_complete(&chan->swdesc->vdesc); + chan->swdesc = NULL; + stm32_dma3_chan_start(chan); +} + +static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid) +{ + struct stm32_dma3_chan *chan = devid; + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 misr, csr, ccr; + + spin_lock(&chan->vchan.lock); + + misr = readl_relaxed(ddata->base + STM32_DMA3_MISR); + if (!(misr & MISR_MIS(chan->id))) { + spin_unlock(&chan->vchan.lock); + return IRQ_NONE; + } + + csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); + ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE; + + if (csr & CSR_TCF && ccr & CCR_TCIE) { + if (chan->swdesc->cyclic) + vchan_cyclic_callback(&chan->swdesc->vdesc); + else + stm32_dma3_chan_complete(chan); + } + + if (csr & CSR_USEF && ccr & CCR_USEIE) { + dev_err(chan2dev(chan), "User setting error\n"); + chan->dma_status = DMA_ERROR; + /* CCR.EN automatically cleared by HW */ + stm32_dma3_check_user_setting(chan); + stm32_dma3_chan_reset(chan); + } + + if (csr & CSR_ULEF && ccr & CCR_ULEIE) { + dev_err(chan2dev(chan), "Update link transfer error\n"); + chan->dma_status = DMA_ERROR; + /* CCR.EN automatically cleared by HW */ + stm32_dma3_chan_reset(chan); + } + + if (csr & CSR_DTEF && ccr & CCR_DTEIE) { + dev_err(chan2dev(chan), "Data transfer error\n"); + chan->dma_status = DMA_ERROR; + /* CCR.EN automatically cleared by HW */ + stm32_dma3_chan_reset(chan); + } + + /* + * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set, + * ensure HTF flag to be cleared, with other flags. + */ + csr &= (ccr | CCR_HTIE); + + if (csr) + writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id)); + + spin_unlock(&chan->vchan.lock); + + return IRQ_HANDLED; +} + +static int stm32_dma3_alloc_chan_resources(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 id = chan->id, csemcr, ccid; + int ret; + + ret = pm_runtime_resume_and_get(ddata->dma_dev.dev); + if (ret < 0) + return ret; + + /* Ensure the channel is free */ + if (chan->semaphore_mode && + readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) { + ret = -EBUSY; + goto err_put_sync; + } + + chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev, + sizeof(struct stm32_dma3_hwdesc), + __alignof__(struct stm32_dma3_hwdesc), SZ_64K); + if (!chan->lli_pool) { + dev_err(chan2dev(chan), "Failed to create LLI pool\n"); + ret = -ENOMEM; + goto err_put_sync; + } + + /* Take the channel semaphore */ + if (chan->semaphore_mode) { + writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id)); + csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id)); + ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr); + /* Check that the channel is well taken */ + if (ccid != CCIDCFGR_CID1) { + dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid); + ret = -EPERM; + goto err_pool_destroy; + } + dev_dbg(chan2dev(chan), "Under CID1 control (semcr=0x%08x)\n", csemcr); + } + + return 0; + +err_pool_destroy: + dmam_pool_destroy(chan->lli_pool); + chan->lli_pool = NULL; + +err_put_sync: + pm_runtime_put_sync(ddata->dma_dev.dev); + + return ret; +} + +static void stm32_dma3_free_chan_resources(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + unsigned long flags; + + /* Ensure channel is in idle state */ + spin_lock_irqsave(&chan->vchan.lock, flags); + stm32_dma3_chan_stop(chan); + chan->swdesc = NULL; + spin_unlock_irqrestore(&chan->vchan.lock, flags); + + vchan_free_chan_resources(to_virt_chan(c)); + + dmam_pool_destroy(chan->lli_pool); + chan->lli_pool = NULL; + + /* Release the channel semaphore */ + if (chan->semaphore_mode) + writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id)); + + pm_runtime_put_sync(ddata->dma_dev.dev); + + /* Reset configuration */ + memset(&chan->dt_config, 0, sizeof(chan->dt_config)); + memset(&chan->dma_config, 0, sizeof(chan->dma_config)); + chan->config_set = 0; +} + +static u32 stm32_dma3_get_ll_count(struct stm32_dma3_chan *chan, size_t len, bool prevent_refactor) +{ + u32 count; + + if (prevent_refactor) + return DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE); + + count = len / STM32_DMA3_MAX_BLOCK_SIZE; + len -= (len / STM32_DMA3_MAX_BLOCK_SIZE) * STM32_DMA3_MAX_BLOCK_SIZE; + + if (len >= chan->max_burst) { + count += 1; /* len < STM32_DMA3_MAX_BLOCK_SIZE here, so it fits in one item */ + len -= (len / chan->max_burst) * chan->max_burst; + } + + /* Unaligned remainder fits in one extra item */ + if (len > 0) + count += 1; + + return count; +} + +static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan *chan, + dma_addr_t dst, dma_addr_t src) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */ + u32 burst = chan->max_burst / dw; + + /* Initialize dt_config if channel not pre-configured through DT */ + if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) { + chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH); + chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size); + chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC; + chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL); + } + + /* Initialize dma_config if dmaengine_slave_config() not used */ + if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) { + chan->dma_config.src_addr_width = dw; + chan->dma_config.dst_addr_width = dw; + chan->dma_config.src_maxburst = burst; + chan->dma_config.dst_maxburst = burst; + chan->dma_config.src_addr = src; + chan->dma_config.dst_addr = dst; + } +} + +static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_chan *c, + dma_addr_t dst, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc; + size_t next_size, offset; + u32 count, i, ctr1, ctr2; + bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) || + !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf); + + count = stm32_dma3_get_ll_count(chan, len, prevent_refactor); + + swdesc = stm32_dma3_chan_desc_alloc(chan, count); + if (!swdesc) + return NULL; + + if (chan->config_set != STM32_DMA3_CFG_SET_BOTH) + stm32_dma3_init_chan_config_for_memcpy(chan, dst, src); + + for (i = 0, offset = 0; offset < len; i++, offset += next_size) { + size_t remaining; + int ret; + + remaining = len - offset; + next_size = min_t(size_t, remaining, STM32_DMA3_MAX_BLOCK_SIZE); + + if (!prevent_refactor && + (next_size < STM32_DMA3_MAX_BLOCK_SIZE && next_size >= chan->max_burst)) + next_size = chan->max_burst * (remaining / chan->max_burst); + + ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, + src + offset, dst + offset, next_size); + if (ret) + goto err_desc_free; + + stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src + offset, dst + offset, next_size, + ctr1, ctr2, next_size == remaining, false); + } + + /* Enable Errors interrupts */ + swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; + /* Enable Transfer state interrupts */ + swdesc->ccr |= CCR_TCIE; + + swdesc->cyclic = false; + + return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); + +err_desc_free: + stm32_dma3_chan_desc_free(chan, swdesc); + + return NULL; +} + +static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan *c, + struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc; + struct scatterlist *sg; + size_t len; + dma_addr_t sg_addr, dev_addr, src, dst; + u32 i, j, count, ctr1, ctr2; + bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) || + !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf); + int ret; + + count = 0; + for_each_sg(sgl, sg, sg_len, i) + count += stm32_dma3_get_ll_count(chan, sg_dma_len(sg), prevent_refactor); + + swdesc = stm32_dma3_chan_desc_alloc(chan, count); + if (!swdesc) + return NULL; + + /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */ + j = 0; + for_each_sg(sgl, sg, sg_len, i) { + sg_addr = sg_dma_address(sg); + dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr : + chan->dma_config.src_addr; + len = sg_dma_len(sg); + + do { + size_t chunk = min_t(size_t, len, STM32_DMA3_MAX_BLOCK_SIZE); + + if (!prevent_refactor && + (chunk < STM32_DMA3_MAX_BLOCK_SIZE && chunk >= chan->max_burst)) + chunk = chan->max_burst * (len / chan->max_burst); + + if (dir == DMA_MEM_TO_DEV) { + src = sg_addr; + dst = dev_addr; + + ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, + src, dst, chunk); + + if (FIELD_GET(CTR1_DINC, ctr1)) + dev_addr += chunk; + } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */ + src = dev_addr; + dst = sg_addr; + + ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, + src, dst, chunk); + + if (FIELD_GET(CTR1_SINC, ctr1)) + dev_addr += chunk; + } + + if (ret) + goto err_desc_free; + + stm32_dma3_chan_prep_hwdesc(chan, swdesc, j, src, dst, chunk, + ctr1, ctr2, j == (count - 1), false); + + sg_addr += chunk; + len -= chunk; + j++; + } while (len); + } + + if (count != sg_len && chan->tcem != CTR2_TCEM_CHANNEL) + dev_warn(chan2dev(chan), "Linked-list refactored, %d items instead of %d\n", + count, sg_len); + + /* Enable Error interrupts */ + swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; + /* Enable Transfer state interrupts */ + swdesc->ccr |= CCR_TCIE; + + swdesc->cyclic = false; + + return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); + +err_desc_free: + stm32_dma3_chan_desc_free(chan, swdesc); + + return NULL; +} + +static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_chan *c, + dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction dir, + unsigned long flags) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc; + dma_addr_t src, dst; + u32 count, i, ctr1, ctr2; + int ret; + + if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) { + dev_err(chan2dev(chan), "Invalid buffer/period length\n"); + return NULL; + } + + if (buf_len % period_len) { + dev_err(chan2dev(chan), "Buffer length not multiple of period length\n"); + return NULL; + } + + count = buf_len / period_len; + swdesc = stm32_dma3_chan_desc_alloc(chan, count); + if (!swdesc) + return NULL; + + if (dir == DMA_MEM_TO_DEV) { + src = buf_addr; + dst = chan->dma_config.dst_addr; + + ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2, + src, dst, period_len); + } else if (dir == DMA_DEV_TO_MEM) { + src = chan->dma_config.src_addr; + dst = buf_addr; + + ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, + src, dst, period_len); + } else { + dev_err(chan2dev(chan), "Invalid direction\n"); + ret = -EINVAL; + } + + if (ret) + goto err_desc_free; + + for (i = 0; i < count; i++) { + if (dir == DMA_MEM_TO_DEV) { + src = buf_addr + i * period_len; + dst = chan->dma_config.dst_addr; + } else { /* (dir == DMA_DEV_TO_MEM) */ + src = chan->dma_config.src_addr; + dst = buf_addr + i * period_len; + } + + stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src, dst, period_len, + ctr1, ctr2, i == (count - 1), true); + } + + /* Enable Error interrupts */ + swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; + /* Enable Transfer state interrupts */ + swdesc->ccr |= CCR_TCIE; + + swdesc->cyclic = true; + + return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); + +err_desc_free: + stm32_dma3_chan_desc_free(chan, swdesc); + + return NULL; +} + +static void stm32_dma3_caps(struct dma_chan *c, struct dma_slave_caps *caps) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + if (!chan->fifo_size) { + caps->max_burst = 0; + caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + } else { + /* Burst transfer should not exceed half of the fifo size */ + caps->max_burst = chan->max_burst; + if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) { + caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + } + } +} + +static int stm32_dma3_config(struct dma_chan *c, struct dma_slave_config *config) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + memcpy(&chan->dma_config, config, sizeof(*config)); + chan->config_set |= STM32_DMA3_CFG_SET_DMA; + + return 0; +} + +static int stm32_dma3_pause(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + int ret; + + ret = stm32_dma3_chan_suspend(chan, true); + if (ret) + return ret; + + chan->dma_status = DMA_PAUSED; + + dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan); + + return 0; +} + +static int stm32_dma3_resume(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + stm32_dma3_chan_suspend(chan, false); + + chan->dma_status = DMA_IN_PROGRESS; + + dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan); + + return 0; +} + +static int stm32_dma3_terminate_all(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&chan->vchan.lock, flags); + + if (chan->swdesc) { + vchan_terminate_vdesc(&chan->swdesc->vdesc); + chan->swdesc = NULL; + } + + stm32_dma3_chan_stop(chan); + + vchan_get_all_descriptors(&chan->vchan, &head); + + spin_unlock_irqrestore(&chan->vchan.lock, flags); + vchan_dma_desc_free_list(&chan->vchan, &head); + + dev_dbg(chan2dev(chan), "vchan %p: terminated\n", &chan->vchan); + + return 0; +} + +static void stm32_dma3_synchronize(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + vchan_synchronize(&chan->vchan); +} + +static enum dma_status stm32_dma3_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc = NULL; + enum dma_status status; + unsigned long flags; + struct virt_dma_desc *vd; + + status = dma_cookie_status(c, cookie, txstate); + if (status == DMA_COMPLETE) + return status; + + if (!txstate) + return chan->dma_status; + + spin_lock_irqsave(&chan->vchan.lock, flags); + + vd = vchan_find_desc(&chan->vchan, cookie); + if (vd) + swdesc = to_stm32_dma3_swdesc(vd); + else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie) + swdesc = chan->swdesc; + + /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */ + if (swdesc) + stm32_dma3_chan_set_residue(chan, swdesc, txstate); + + spin_unlock_irqrestore(&chan->vchan.lock, flags); + + return chan->dma_status; +} + +static void stm32_dma3_issue_pending(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + unsigned long flags; + + spin_lock_irqsave(&chan->vchan.lock, flags); + + if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) { + dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); + stm32_dma3_chan_start(chan); + } + + spin_unlock_irqrestore(&chan->vchan.lock, flags); +} + +static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct stm32_dma3_dt_conf *conf = fn_param; + u32 mask, semcr; + int ret; + + dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n", + __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf); + + if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask)) + if (!(mask & BIT(chan->id))) + return false; + + ret = pm_runtime_resume_and_get(ddata->dma_dev.dev); + if (ret < 0) + return false; + semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)); + pm_runtime_put_sync(ddata->dma_dev.dev); + + /* Check if chan is free */ + if (semcr & CSEMCR_SEM_MUTEX) + return false; + + /* Check if chan fifo fits well */ + if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size) + return false; + + return true; +} + +static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) +{ + struct stm32_dma3_ddata *ddata = ofdma->of_dma_data; + dma_cap_mask_t mask = ddata->dma_dev.cap_mask; + struct stm32_dma3_dt_conf conf; + struct stm32_dma3_chan *chan; + struct dma_chan *c; + + if (dma_spec->args_count < 3) { + dev_err(ddata->dma_dev.dev, "Invalid args count\n"); + return NULL; + } + + conf.req_line = dma_spec->args[0]; + conf.ch_conf = dma_spec->args[1]; + conf.tr_conf = dma_spec->args[2]; + + if (conf.req_line >= ddata->dma_requests) { + dev_err(ddata->dma_dev.dev, "Invalid request line\n"); + return NULL; + } + + /* Request dma channel among the generic dma controller list */ + c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf); + if (!c) { + dev_err(ddata->dma_dev.dev, "No suitable channel found\n"); + return NULL; + } + + chan = to_stm32_dma3_chan(c); + chan->dt_config = conf; + chan->config_set |= STM32_DMA3_CFG_SET_DT; + + return c; +} + +static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata) +{ + u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0; + + /* Reserve Secure channels */ + chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR); + + /* + * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of + * the processor which is configuring and using the given channel. + * In case CID filtering is not configured, dma-channel-mask property can be used to + * specify available DMA channels to the kernel. + */ + of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask); + + /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */ + for (i = 0; i < ddata->dma_channels; i++) { + ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i)); + + if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */ + invalid_cid |= BIT(i); + if (!(mask & BIT(i))) /* Not in dma-channel-mask */ + chan_reserved |= BIT(i); + } else { /* CID-filtered */ + if (!(ccidcfgr & CCIDCFGR_SEM_EN)) { /* Static CID mode */ + if (FIELD_GET(CCIDCFGR_SCID, ccidcfgr) != CCIDCFGR_CID1) + chan_reserved |= BIT(i); + } else { /* Semaphore mode */ + if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1, ccidcfgr)) + chan_reserved |= BIT(i); + ddata->chans[i].semaphore_mode = true; + } + } + dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i, + !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" : + ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID", + (chan_reserved & BIT(i)) ? "denied" : + mask & BIT(i) ? "force allowed" : "allowed"); + } + + if (invalid_cid) + dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n", + ddata->dma_channels, &invalid_cid); + + return chan_reserved; +} + +static struct stm32_dma3_pdata stm32mp25_pdata = { + .axi_max_burst_len = 16, +}; + +static const struct of_device_id stm32_dma3_of_match[] = { + { .compatible = "st,stm32mp25-dma3", .data = &stm32mp25_pdata, }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, stm32_dma3_of_match); + +static int stm32_dma3_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct stm32_dma3_pdata *pdata; + struct stm32_dma3_ddata *ddata; + struct reset_control *reset; + struct stm32_dma3_chan *chan; + struct dma_device *dma_dev; + u32 master_ports, chan_reserved, i, verr; + u64 hwcfgr; + int ret; + + ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL); + if (!ddata) + return -ENOMEM; + platform_set_drvdata(pdev, ddata); + + dma_dev = &ddata->dma_dev; + + ddata->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ddata->base)) + return PTR_ERR(ddata->base); + + ddata->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(ddata->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n"); + + reset = devm_reset_control_get_optional(&pdev->dev, NULL); + if (IS_ERR(reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n"); + + ret = clk_prepare_enable(ddata->clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n"); + + reset_control_reset(reset); + + INIT_LIST_HEAD(&dma_dev->channels); + + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_dev->dev = &pdev->dev; + /* + * This controller supports up to 8-byte buswidth depending on the port used and the + * channel, and can only access address at even boundaries, multiple of the buswidth. + */ + dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES; + dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM); + + dma_dev->descriptor_reuse = true; + dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE; + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources; + dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources; + dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy; + dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg; + dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic; + dma_dev->device_caps = stm32_dma3_caps; + dma_dev->device_config = stm32_dma3_config; + dma_dev->device_pause = stm32_dma3_pause; + dma_dev->device_resume = stm32_dma3_resume; + dma_dev->device_terminate_all = stm32_dma3_terminate_all; + dma_dev->device_synchronize = stm32_dma3_synchronize; + dma_dev->device_tx_status = stm32_dma3_tx_status; + dma_dev->device_issue_pending = stm32_dma3_issue_pending; + + /* if dma_channels is not modified, get it from hwcfgr1 */ + if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) { + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); + ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr); + } + + /* if dma_requests is not modified, get it from hwcfgr2 */ + if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) { + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2); + ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1; + } + + /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */ + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); + master_ports = FIELD_GET(G_MASTER_PORTS, hwcfgr); + + ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr); + if (master_ports == AXI64 || master_ports == AHB32) /* Single master port */ + ddata->ports_max_dw[1] = DW_INVALID; + else /* Dual master ports */ + ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr); + + /* axi_max_burst_len is optional, if not defined, use STM32_DMA3_MAX_BURST_LEN */ + ddata->axi_max_burst_len = STM32_DMA3_MAX_BURST_LEN; + pdata = device_get_match_data(&pdev->dev); + if (pdata && pdata->axi_max_burst_len) { + ddata->axi_max_burst_len = min_t(u32, pdata->axi_max_burst_len, + STM32_DMA3_MAX_BURST_LEN); + dev_dbg(&pdev->dev, "Burst is limited to %u beats through AXI port\n", + ddata->axi_max_burst_len); + } + + ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans), + GFP_KERNEL); + if (!ddata->chans) { + ret = -ENOMEM; + goto err_clk_disable; + } + + chan_reserved = stm32_dma3_check_rif(ddata); + + if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) { + ret = -ENODEV; + dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n"); + goto err_clk_disable; + } + + /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */ + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3); + hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32; + + for (i = 0; i < ddata->dma_channels; i++) { + if (chan_reserved & BIT(i)) + continue; + + chan = &ddata->chans[i]; + chan->id = i; + chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr); + /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */ + chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0; + } + + ret = dmaenginem_async_device_register(dma_dev); + if (ret) + goto err_clk_disable; + + for (i = 0; i < ddata->dma_channels; i++) { + char name[12]; + + if (chan_reserved & BIT(i)) + continue; + + chan = &ddata->chans[i]; + snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id); + + chan->vchan.desc_free = stm32_dma3_chan_vdesc_free; + vchan_init(&chan->vchan, dma_dev); + + ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name); + if (ret) { + dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name); + goto err_clk_disable; + } + + ret = platform_get_irq(pdev, i); + if (ret < 0) + goto err_clk_disable; + chan->irq = ret; + + ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0, + dev_name(chan2dev(chan)), chan); + if (ret) { + dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n", + dev_name(chan2dev(chan))); + goto err_clk_disable; + } + } + + ret = of_dma_controller_register(np, stm32_dma3_of_xlate, ddata); + if (ret) { + dev_err_probe(&pdev->dev, ret, "Failed to register controller\n"); + goto err_clk_disable; + } + + verr = readl_relaxed(ddata->base + STM32_DMA3_VERR); + + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_put(&pdev->dev); + + dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n", + FIELD_GET(VERR_MAJREV, verr), FIELD_GET(VERR_MINREV, verr)); + + return 0; + +err_clk_disable: + clk_disable_unprepare(ddata->clk); + + return ret; +} + +static void stm32_dma3_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); +} + +static int stm32_dma3_runtime_suspend(struct device *dev) +{ + struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); + + clk_disable_unprepare(ddata->clk); + + return 0; +} + +static int stm32_dma3_runtime_resume(struct device *dev) +{ + struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(ddata->clk); + if (ret) + dev_err(dev, "Failed to enable clk: %d\n", ret); + + return ret; +} + +static const struct dev_pm_ops stm32_dma3_pm_ops = { + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL) +}; + +static struct platform_driver stm32_dma3_driver = { + .probe = stm32_dma3_probe, + .remove = stm32_dma3_remove, + .driver = { + .name = "stm32-dma3", + .of_match_table = stm32_dma3_of_match, + .pm = pm_ptr(&stm32_dma3_pm_ops), + }, +}; + +static int __init stm32_dma3_init(void) +{ + return platform_driver_register(&stm32_dma3_driver); +} + +subsys_initcall(stm32_dma3_init); + +MODULE_DESCRIPTION("STM32 DMA3 controller driver"); +MODULE_AUTHOR("Amelie Delaunay <amelie.delaunay@foss.st.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32/stm32-dmamux.c index e415bd9f4f2b..8d77e2a7939a 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32/stm32-dmamux.c @@ -15,8 +15,10 @@ #include <linux/err.h> #include <linux/init.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32/stm32-mdma.c index 1d0e9dd72ab3..080c1c725216 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32/stm32-mdma.c @@ -24,14 +24,13 @@ #include <linux/log2.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> -#include "virt-dma.h" +#include "../virt-dma.h" #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */ @@ -225,7 +224,7 @@ struct stm32_mdma_desc { u32 ccr; bool cyclic; u32 count; - struct stm32_mdma_desc_node node[]; + struct stm32_mdma_desc_node node[] __counted_by(count); }; struct stm32_mdma_dma_config { @@ -257,7 +256,7 @@ struct stm32_mdma_device { u32 nr_ahb_addr_masks; u32 chan_reserved; struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS]; - u32 ahb_addr_masks[]; + u32 ahb_addr_masks[] __counted_by(nr_ahb_addr_masks); }; static struct stm32_mdma_device *stm32_mdma_get_dev( @@ -322,6 +321,7 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc( desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT); if (!desc) return NULL; + desc->count = count; for (i = 0; i < count; i++) { desc->node[i].hwdesc = @@ -331,8 +331,6 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc( goto err; } - desc->count = count; - return desc; err: @@ -490,7 +488,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, src_maxburst = chan->dma_config.src_maxburst; dst_maxburst = chan->dma_config.dst_maxburst; - ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN; ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)); ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)); @@ -778,8 +776,6 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, /* Enable interrupts */ ccr &= ~STM32_MDMA_CCR_IRQ_MASK; ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE; - if (sg_len > 1) - ccr |= STM32_MDMA_CCR_BTIE; desc->ccr = ccr; return 0; @@ -968,7 +964,7 @@ stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src, if (!desc) return NULL; - ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN; ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)); ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)); cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)); @@ -1191,7 +1187,7 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan) chan->busy = true; - dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); } static void stm32_mdma_issue_pending(struct dma_chan *c) @@ -1204,7 +1200,7 @@ static void stm32_mdma_issue_pending(struct dma_chan *c) if (!vchan_issue_pending(&chan->vchan)) goto end; - dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); if (!chan->desc && !chan->busy) stm32_mdma_start_transfer(chan); @@ -1224,7 +1220,7 @@ static int stm32_mdma_pause(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); if (!ret) - dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: pause\n", &chan->vchan); return ret; } @@ -1237,6 +1233,10 @@ static int stm32_mdma_resume(struct dma_chan *c) unsigned long flags; u32 status, reg; + /* Transfer can be terminated */ + if (!chan->desc || (stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & STM32_MDMA_CCR_EN)) + return -EPERM; + hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc; spin_lock_irqsave(&chan->vchan.lock, flags); @@ -1261,7 +1261,7 @@ static int stm32_mdma_resume(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); - dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: resume\n", &chan->vchan); return 0; } @@ -1317,21 +1317,35 @@ static int stm32_mdma_slave_config(struct dma_chan *c, static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan, struct stm32_mdma_desc *desc, - u32 curr_hwdesc) + u32 curr_hwdesc, + struct dma_tx_state *state) { struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); struct stm32_mdma_hwdesc *hwdesc; - u32 cbndtr, residue, modulo, burst_size; + u32 cisr, clar, cbndtr, residue, modulo, burst_size; int i; + cisr = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id)); + residue = 0; - for (i = curr_hwdesc + 1; i < desc->count; i++) { + /* Get the next hw descriptor to process from current transfer */ + clar = stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)); + for (i = desc->count - 1; i >= 0; i--) { hwdesc = desc->node[i].hwdesc; + + if (hwdesc->clar == clar) + break;/* Current transfer found, stop cumulating */ + + /* Cumulate residue of unprocessed hw descriptors */ residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr); } cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)); residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK; + state->in_flight_bytes = 0; + if (chan->chan_config.m2m_hw && (cisr & STM32_MDMA_CISR_CRQA)) + state->in_flight_bytes = cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK; + if (!chan->mem_burst) return residue; @@ -1361,11 +1375,10 @@ static enum dma_status stm32_mdma_tx_status(struct dma_chan *c, vdesc = vchan_find_desc(&chan->vchan, cookie); if (chan->desc && cookie == chan->desc->vdesc.tx.cookie) - residue = stm32_mdma_desc_residue(chan, chan->desc, - chan->curr_hwdesc); + residue = stm32_mdma_desc_residue(chan, chan->desc, chan->curr_hwdesc, state); else if (vdesc) - residue = stm32_mdma_desc_residue(chan, - to_stm32_mdma_desc(vdesc), 0); + residue = stm32_mdma_desc_residue(chan, to_stm32_mdma_desc(vdesc), 0, state); + dma_set_residue(state, residue); spin_unlock_irqrestore(&chan->vchan.lock, flags); @@ -1613,13 +1626,13 @@ static int stm32_mdma_probe(struct platform_device *pdev) GFP_KERNEL); if (!dmadev) return -ENOMEM; + dmadev->nr_ahb_addr_masks = count; dmadev->nr_channels = nr_channels; dmadev->nr_requests = nr_requests; device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks", dmadev->ahb_addr_masks, count); - dmadev->nr_ahb_addr_masks = count; dmadev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dmadev->base)) diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index e86c8829513a..00d2fd38d17f 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -13,7 +13,9 @@ #include <linux/interrupt.h> #include <linux/module.h> #include <linux/of_dma.h> +#include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/reset.h> #include <linux/slab.h> #include <linux/spinlock.h> @@ -31,12 +33,21 @@ #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5) #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type) +#define SUNIV_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 24) +#define SUNIV_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 8) + +#define SUN4I_MAX_BURST 8 +#define SUNIV_MAX_BURST 4 + /** Normal DMA register values **/ /* Normal DMA source/destination data request type values */ #define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16 #define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1) +#define SUNIV_NDMA_DRQ_TYPE_SDRAM 0x11 +#define SUNIV_NDMA_DRQ_TYPE_LIMIT (0x17 + 1) + /** Normal DMA register layout **/ /* Dedicated DMA source/destination address mode values */ @@ -50,6 +61,9 @@ #define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15) #define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6) +#define SUNIV_NDMA_CFG_CONT_MODE BIT(29) +#define SUNIV_NDMA_CFG_WAIT_STATE(n) ((n) << 26) + /** Dedicated DMA register values **/ /* Dedicated DMA source/destination address mode values */ @@ -62,6 +76,9 @@ #define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1 #define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1) +#define SUNIV_DDMA_DRQ_TYPE_SDRAM 0x1 +#define SUNIV_DDMA_DRQ_TYPE_LIMIT (0x9 + 1) + /** Dedicated DMA register layout **/ /* Dedicated DMA configuration register layout */ @@ -115,6 +132,11 @@ #define SUN4I_DMA_NR_MAX_VCHANS \ (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS) +#define SUNIV_NDMA_NR_MAX_CHANNELS 4 +#define SUNIV_DDMA_NR_MAX_CHANNELS 4 +#define SUNIV_NDMA_NR_MAX_VCHANS (24 * 2 - 1) +#define SUNIV_DDMA_NR_MAX_VCHANS 10 + /* This set of SUN4I_DDMA timing parameters were found experimentally while * working with the SPI driver and seem to make it behave correctly */ #define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \ @@ -132,6 +154,33 @@ #define SUN4I_DDMA_MAX_SEG_SIZE SZ_16M #define SUN4I_DMA_MAX_SEG_SIZE SUN4I_NDMA_MAX_SEG_SIZE +/* + * Hardware channels / ports representation + * + * The hardware is used in several SoCs, with differing numbers + * of channels and endpoints. This structure ties those numbers + * to a certain compatible string. + */ +struct sun4i_dma_config { + u32 ndma_nr_max_channels; + u32 ndma_nr_max_vchans; + + u32 ddma_nr_max_channels; + u32 ddma_nr_max_vchans; + + u32 dma_nr_max_channels; + + void (*set_dst_data_width)(u32 *p_cfg, s8 data_width); + void (*set_src_data_width)(u32 *p_cfg, s8 data_width); + int (*convert_burst)(u32 maxburst); + + u8 ndma_drq_sdram; + u8 ddma_drq_sdram; + + u8 max_burst; + bool has_reset; +}; + struct sun4i_dma_pchan { /* Register base of channel */ void __iomem *base; @@ -170,7 +219,7 @@ struct sun4i_dma_contract { }; struct sun4i_dma_dev { - DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS); + unsigned long *pchans_used; struct dma_device slave; struct sun4i_dma_pchan *pchans; struct sun4i_dma_vchan *vchans; @@ -178,6 +227,8 @@ struct sun4i_dma_dev { struct clk *clk; int irq; spinlock_t lock; + const struct sun4i_dma_config *cfg; + struct reset_control *rst; }; static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev) @@ -200,7 +251,27 @@ static struct device *chan2dev(struct dma_chan *chan) return &chan->dev->device; } -static int convert_burst(u32 maxburst) +static void set_dst_data_width_a10(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(data_width); +} + +static void set_src_data_width_a10(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(data_width); +} + +static void set_dst_data_width_f1c100s(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUNIV_DMA_CFG_DST_DATA_WIDTH(data_width); +} + +static void set_src_data_width_f1c100s(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUNIV_DMA_CFG_SRC_DATA_WIDTH(data_width); +} + +static int convert_burst_a10(u32 maxburst) { if (maxburst > 8) return -EINVAL; @@ -209,6 +280,15 @@ static int convert_burst(u32 maxburst) return (maxburst >> 2); } +static int convert_burst_f1c100s(u32 maxburst) +{ + if (maxburst > 4) + return -EINVAL; + + /* 1 -> 0, 4 -> 1 */ + return (maxburst >> 2); +} + static int convert_buswidth(enum dma_slave_buswidth addr_width) { if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES) @@ -233,15 +313,15 @@ static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv, int i, max; /* - * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and - * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones + * pchans 0-priv->cfg->ndma_nr_max_channels are normal, and + * priv->cfg->ndma_nr_max_channels+ are dedicated ones */ if (vchan->is_dedicated) { - i = SUN4I_NDMA_NR_MAX_CHANNELS; - max = SUN4I_DMA_NR_MAX_CHANNELS; + i = priv->cfg->ndma_nr_max_channels; + max = priv->cfg->dma_nr_max_channels; } else { i = 0; - max = SUN4I_NDMA_NR_MAX_CHANNELS; + max = priv->cfg->ndma_nr_max_channels; } spin_lock_irqsave(&priv->lock, flags); @@ -444,6 +524,7 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, size_t len, struct dma_slave_config *sconfig, enum dma_transfer_direction direction) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_promise *promise; int ret; @@ -467,13 +548,13 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, sconfig->src_addr_width, sconfig->dst_addr_width); /* Source burst */ - ret = convert_burst(sconfig->src_maxburst); + ret = priv->cfg->convert_burst(sconfig->src_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ - ret = convert_burst(sconfig->dst_maxburst); + ret = priv->cfg->convert_burst(sconfig->dst_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); @@ -482,13 +563,13 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, ret = convert_buswidth(sconfig->src_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + priv->cfg->set_src_data_width(&promise->cfg, ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + priv->cfg->set_dst_data_width(&promise->cfg, ret); return promise; @@ -510,6 +591,7 @@ static struct sun4i_dma_promise * generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, size_t len, struct dma_slave_config *sconfig) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_promise *promise; int ret; @@ -524,13 +606,13 @@ generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN; /* Source burst */ - ret = convert_burst(sconfig->src_maxburst); + ret = priv->cfg->convert_burst(sconfig->src_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ - ret = convert_burst(sconfig->dst_maxburst); + ret = priv->cfg->convert_burst(sconfig->dst_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); @@ -539,13 +621,13 @@ generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, ret = convert_buswidth(sconfig->src_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + priv->cfg->set_src_data_width(&promise->cfg, ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + priv->cfg->set_dst_data_width(&promise->cfg, ret); return promise; @@ -622,6 +704,7 @@ static struct dma_async_tx_descriptor * sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -638,8 +721,8 @@ sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, */ sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - sconfig->src_maxburst = 8; - sconfig->dst_maxburst = 8; + sconfig->src_maxburst = priv->cfg->max_burst; + sconfig->dst_maxburst = priv->cfg->max_burst; if (vchan->is_dedicated) promise = generate_ddma_promise(chan, src, dest, len, sconfig); @@ -654,11 +737,13 @@ sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, /* Configure memcpy mode */ if (vchan->is_dedicated) { - promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) | - SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM); + promise->cfg |= + SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ddma_drq_sdram) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ddma_drq_sdram); } else { - promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) | - SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM); + promise->cfg |= + SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ndma_drq_sdram) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ndma_drq_sdram); } /* Fill the contract with our only promise */ @@ -673,6 +758,7 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, size_t period_len, enum dma_transfer_direction dir, unsigned long flags) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -696,11 +782,11 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, if (vchan->is_dedicated) { io_mode = SUN4I_DDMA_ADDR_MODE_IO; linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ddma_drq_sdram; } else { io_mode = SUN4I_NDMA_ADDR_MODE_IO; linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ndma_drq_sdram; } if (dir == DMA_MEM_TO_DEV) { @@ -793,6 +879,7 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -818,11 +905,11 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (vchan->is_dedicated) { io_mode = SUN4I_DDMA_ADDR_MODE_IO; linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ddma_drq_sdram; } else { io_mode = SUN4I_NDMA_ADDR_MODE_IO; linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ndma_drq_sdram; } if (dir == DMA_MEM_TO_DEV) @@ -1150,6 +1237,10 @@ static int sun4i_dma_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + priv->cfg = of_device_get_match_data(&pdev->dev); + if (!priv->cfg) + return -ENODEV; + priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); @@ -1158,10 +1249,16 @@ static int sun4i_dma_probe(struct platform_device *pdev) if (priv->irq < 0) return priv->irq; - priv->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(priv->clk)) { - dev_err(&pdev->dev, "No clock specified\n"); - return PTR_ERR(priv->clk); + priv->clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(priv->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk), + "Couldn't start the clock\n"); + + if (priv->cfg->has_reset) { + priv->rst = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL); + if (IS_ERR(priv->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->rst), + "Failed to get reset control\n"); } platform_set_drvdata(pdev, priv); @@ -1197,23 +1294,26 @@ static int sun4i_dma_probe(struct platform_device *pdev) priv->slave.dev = &pdev->dev; - priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS, + priv->pchans = devm_kcalloc(&pdev->dev, priv->cfg->dma_nr_max_channels, sizeof(struct sun4i_dma_pchan), GFP_KERNEL); priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS, sizeof(struct sun4i_dma_vchan), GFP_KERNEL); - if (!priv->vchans || !priv->pchans) + priv->pchans_used = devm_kcalloc(&pdev->dev, + BITS_TO_LONGS(priv->cfg->dma_nr_max_channels), + sizeof(unsigned long), GFP_KERNEL); + if (!priv->vchans || !priv->pchans || !priv->pchans_used) return -ENOMEM; /* - * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and - * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are + * [0..priv->cfg->ndma_nr_max_channels) are normal pchans, and + * [priv->cfg->ndma_nr_max_channels..priv->cfg->dma_nr_max_channels) are * dedicated ones */ - for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++) + for (i = 0; i < priv->cfg->ndma_nr_max_channels; i++) priv->pchans[i].base = priv->base + SUN4I_NDMA_CHANNEL_REG_BASE(i); - for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) { + for (j = 0; i < priv->cfg->dma_nr_max_channels; i++, j++) { priv->pchans[i].base = priv->base + SUN4I_DDMA_CHANNEL_REG_BASE(j); priv->pchans[i].is_dedicated = 1; @@ -1227,12 +1327,6 @@ static int sun4i_dma_probe(struct platform_device *pdev) vchan_init(&vchan->vc, &priv->slave); } - ret = clk_prepare_enable(priv->clk); - if (ret) { - dev_err(&pdev->dev, "Couldn't enable the clock\n"); - return ret; - } - /* * Make sure the IRQs are all disabled and accounted for. The bootloader * likes to leave these dirty @@ -1242,36 +1336,26 @@ static int sun4i_dma_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt, 0, dev_name(&pdev->dev), priv); - if (ret) { - dev_err(&pdev->dev, "Cannot request IRQ\n"); - goto err_clk_disable; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "Cannot request IRQ\n"); - ret = dma_async_device_register(&priv->slave); - if (ret) { - dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); - goto err_clk_disable; - } + ret = dmaenginem_async_device_register(&priv->slave); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to register DMA engine device\n"); ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate, priv); - if (ret) { - dev_err(&pdev->dev, "of_dma_controller_register failed\n"); - goto err_dma_unregister; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to register translation function\n"); dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n"); return 0; - -err_dma_unregister: - dma_async_device_unregister(&priv->slave); -err_clk_disable: - clk_disable_unprepare(priv->clk); - return ret; } -static int sun4i_dma_remove(struct platform_device *pdev) +static void sun4i_dma_remove(struct platform_device *pdev) { struct sun4i_dma_dev *priv = platform_get_drvdata(pdev); @@ -1279,22 +1363,60 @@ static int sun4i_dma_remove(struct platform_device *pdev) disable_irq(priv->irq); of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&priv->slave); +} - clk_disable_unprepare(priv->clk); +static struct sun4i_dma_config sun4i_a10_dma_cfg = { + .ndma_nr_max_channels = SUN4I_NDMA_NR_MAX_CHANNELS, + .ndma_nr_max_vchans = SUN4I_NDMA_NR_MAX_VCHANS, - return 0; -} + .ddma_nr_max_channels = SUN4I_DDMA_NR_MAX_CHANNELS, + .ddma_nr_max_vchans = SUN4I_DDMA_NR_MAX_VCHANS, + + .dma_nr_max_channels = SUN4I_DMA_NR_MAX_CHANNELS, + + .set_dst_data_width = set_dst_data_width_a10, + .set_src_data_width = set_src_data_width_a10, + .convert_burst = convert_burst_a10, + + .ndma_drq_sdram = SUN4I_NDMA_DRQ_TYPE_SDRAM, + .ddma_drq_sdram = SUN4I_DDMA_DRQ_TYPE_SDRAM, + + .max_burst = SUN4I_MAX_BURST, + .has_reset = false, +}; + +static struct sun4i_dma_config suniv_f1c100s_dma_cfg = { + .ndma_nr_max_channels = SUNIV_NDMA_NR_MAX_CHANNELS, + .ndma_nr_max_vchans = SUNIV_NDMA_NR_MAX_VCHANS, + + .ddma_nr_max_channels = SUNIV_DDMA_NR_MAX_CHANNELS, + .ddma_nr_max_vchans = SUNIV_DDMA_NR_MAX_VCHANS, + + .dma_nr_max_channels = SUNIV_NDMA_NR_MAX_CHANNELS + + SUNIV_DDMA_NR_MAX_CHANNELS, + + .set_dst_data_width = set_dst_data_width_f1c100s, + .set_src_data_width = set_src_data_width_f1c100s, + .convert_burst = convert_burst_f1c100s, + + .ndma_drq_sdram = SUNIV_NDMA_DRQ_TYPE_SDRAM, + .ddma_drq_sdram = SUNIV_DDMA_DRQ_TYPE_SDRAM, + + .max_burst = SUNIV_MAX_BURST, + .has_reset = true, +}; static const struct of_device_id sun4i_dma_match[] = { - { .compatible = "allwinner,sun4i-a10-dma" }, + { .compatible = "allwinner,sun4i-a10-dma", .data = &sun4i_a10_dma_cfg }, + { .compatible = "allwinner,suniv-f1c100s-dma", + .data = &suniv_f1c100s_dma_cfg }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, sun4i_dma_match); static struct platform_driver sun4i_dma_driver = { .probe = sun4i_dma_probe, - .remove = sun4i_dma_remove, + .remove = sun4i_dma_remove, .driver = { .name = "sun4i-dma", .of_match_table = sun4i_dma_match, diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index ebfd29888b2f..2215ff877bf7 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -14,11 +14,12 @@ #include <linux/dmapool.h> #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/reset.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/types.h> #include "virt-dma.h" @@ -553,7 +554,7 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) continue; dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n", - i ? "high" : "low", status); + str_high_low(i), status); writel(status, sdev->base + DMA_IRQ_STAT(i)); @@ -1470,7 +1471,7 @@ err_chan_free: return ret; } -static int sun6i_dma_remove(struct platform_device *pdev) +static void sun6i_dma_remove(struct platform_device *pdev) { struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev); @@ -1484,8 +1485,6 @@ static int sun6i_dma_remove(struct platform_device *pdev) reset_control_assert(sdc->rstc); sun6i_dma_free(sdc); - - return 0; } static struct platform_driver sun6i_dma_driver = { diff --git a/drivers/dma/tegra186-gpc-dma.c b/drivers/dma/tegra186-gpc-dma.c index 8f67f453a492..4d6fe0efa76e 100644 --- a/drivers/dma/tegra186-gpc-dma.c +++ b/drivers/dma/tegra186-gpc-dma.c @@ -13,7 +13,7 @@ #include <linux/iopoll.h> #include <linux/minmax.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/reset.h> @@ -221,7 +221,7 @@ struct tegra_dma_desc { unsigned int sg_count; struct virt_dma_desc vd; struct tegra_dma_channel *tdc; - struct tegra_dma_sg_req sg_req[]; + struct tegra_dma_sg_req sg_req[] __counted_by(sg_count); }; /* @@ -231,6 +231,7 @@ struct tegra_dma_channel { bool config_init; char name[30]; enum dma_transfer_direction sid_dir; + enum dma_status status; int id; int irq; int slave_id; @@ -393,6 +394,8 @@ static int tegra_dma_pause(struct tegra_dma_channel *tdc) tegra_dma_dump_chan_regs(tdc); } + tdc->status = DMA_PAUSED; + return ret; } @@ -419,6 +422,8 @@ static void tegra_dma_resume(struct tegra_dma_channel *tdc) val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE); val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE; tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val); + + tdc->status = DMA_IN_PROGRESS; } static int tegra_dma_device_resume(struct dma_chan *dc) @@ -544,6 +549,7 @@ static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc) tegra_dma_sid_free(tdc); tdc->dma_desc = NULL; + tdc->status = DMA_COMPLETE; } static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc, @@ -716,6 +722,7 @@ static int tegra_dma_terminate_all(struct dma_chan *dc) tdc->dma_desc = NULL; } + tdc->status = DMA_COMPLETE; tegra_dma_sid_free(tdc); vchan_get_all_descriptors(&tdc->vc, &head); spin_unlock_irqrestore(&tdc->vc.lock, flags); @@ -746,6 +753,9 @@ static int tegra_dma_get_residual(struct tegra_dma_channel *tdc) bytes_xfer = dma_desc->bytes_xfer + sg_req[dma_desc->sg_idx].len - (wcount * 4); + if (dma_desc->bytes_req == bytes_xfer) + return 0; + residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req); return residual; @@ -766,6 +776,9 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, if (ret == DMA_COMPLETE) return ret; + if (tdc->status == DMA_PAUSED) + ret = DMA_PAUSED; + spin_lock_irqsave(&tdc->vc.lock, flags); vd = vchan_find_desc(&tdc->vc, cookie); if (vd) { @@ -1348,8 +1361,8 @@ static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id) static int tegra_dma_probe(struct platform_device *pdev) { const struct tegra_dma_chip_data *cdata = NULL; - struct iommu_fwspec *iommu_spec; - unsigned int stream_id, i; + unsigned int i; + u32 stream_id; struct tegra_dma *tdma; int ret; @@ -1378,12 +1391,10 @@ static int tegra_dma_probe(struct platform_device *pdev) tdma->dma_dev.dev = &pdev->dev; - iommu_spec = dev_iommu_fwspec_get(&pdev->dev); - if (!iommu_spec) { + if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) { dev_err(&pdev->dev, "Missing iommu stream-id\n"); return -EINVAL; } - stream_id = iommu_spec->ids[0] & 0xffff; ret = device_property_read_u32(&pdev->dev, "dma-channel-mask", &tdma->chan_mask); @@ -1473,14 +1484,12 @@ static int tegra_dma_probe(struct platform_device *pdev) return 0; } -static int tegra_dma_remove(struct platform_device *pdev) +static void tegra_dma_remove(struct platform_device *pdev) { struct tegra_dma *tdma = platform_get_drvdata(pdev); of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&tdma->dma_dev); - - return 0; } static int __maybe_unused tegra_dma_pm_suspend(struct device *dev) diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index cc6b91f48979..14a61e53a41b 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -17,7 +17,6 @@ #include <linux/mm.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm.h> @@ -464,7 +463,7 @@ static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, /* * If interrupt is pending then do nothing as the ISR will handle - * the programing for new request. + * the programming for new request. */ if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { dev_err(tdc2dev(tdc), @@ -1582,7 +1581,7 @@ err_clk_unprepare: return ret; } -static int tegra_dma_remove(struct platform_device *pdev) +static void tegra_dma_remove(struct platform_device *pdev) { struct tegra_dma *tdma = platform_get_drvdata(pdev); @@ -1590,8 +1589,6 @@ static int tegra_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&tdma->dma_dev); pm_runtime_disable(&pdev->dev); clk_unprepare(tdma->dma_clk); - - return 0; } static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev) diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index b97004036071..d0e8bb27a03b 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -8,9 +8,10 @@ #include <linux/clk.h> #include <linux/iopoll.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> @@ -26,10 +27,10 @@ #define ADMA_CH_INT_CLEAR 0x1c #define ADMA_CH_CTRL 0x24 -#define ADMA_CH_CTRL_DIR(val) (((val) & 0xf) << 12) +#define ADMA_CH_CTRL_DIR(val, mask, shift) (((val) & (mask)) << (shift)) #define ADMA_CH_CTRL_DIR_AHUB2MEM 2 #define ADMA_CH_CTRL_DIR_MEM2AHUB 4 -#define ADMA_CH_CTRL_MODE_CONTINUOUS (2 << 8) +#define ADMA_CH_CTRL_MODE_CONTINUOUS(shift) (2 << (shift)) #define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1) #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 0 @@ -40,11 +41,27 @@ #define ADMA_CH_CONFIG_MAX_BURST_SIZE 16 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf) #define ADMA_CH_CONFIG_MAX_BUFS 8 -#define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) (reqs << 4) +#define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) ((reqs) << 4) + +#define ADMA_GLOBAL_CH_CONFIG 0x400 +#define ADMA_GLOBAL_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0x7) +#define ADMA_GLOBAL_CH_CONFIG_OUTSTANDING_REQS(reqs) ((reqs) << 8) + +#define TEGRA186_ADMA_GLOBAL_PAGE_CHGRP 0x30 +#define TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ 0x70 +#define TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ 0x84 +#define TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0 0x44 +#define TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1 0x48 +#define TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0 0x100 +#define TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1 0x104 +#define TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0 0x180 +#define TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1 0x184 +#define TEGRA264_ADMA_GLOBAL_PAGE_OFFSET 0x8 #define ADMA_CH_FIFO_CTRL 0x2c #define ADMA_CH_TX_FIFO_SIZE_SHIFT 8 #define ADMA_CH_RX_FIFO_SIZE_SHIFT 0 +#define ADMA_GLOBAL_CH_FIFO_CTRL 0x300 #define ADMA_CH_LOWER_SRC_ADDR 0x34 #define ADMA_CH_LOWER_TRG_ADDR 0x3c @@ -68,33 +85,49 @@ struct tegra_adma; * @adma_get_burst_config: Function callback used to set DMA burst size. * @global_reg_offset: Register offset of DMA global register. * @global_int_clear: Register offset of DMA global interrupt clear. + * @global_ch_fifo_base: Global channel fifo ctrl base offset + * @global_ch_config_base: Global channel config base offset * @ch_req_tx_shift: Register offset for AHUB transmit channel select. * @ch_req_rx_shift: Register offset for AHUB receive channel select. + * @ch_dir_shift: Channel direction bit position. + * @ch_mode_shift: Channel mode bit position. * @ch_base_offset: Register offset of DMA channel registers. + * @ch_tc_offset_diff: From TC register onwards offset differs for Tegra264 * @ch_fifo_ctrl: Default value for channel FIFO CTRL register. + * @ch_config: Outstanding and WRR config values * @ch_req_mask: Mask for Tx or Rx channel select. + * @ch_dir_mask: Mask for channel direction. * @ch_req_max: Maximum number of Tx or Rx channels available. * @ch_reg_size: Size of DMA channel register space. * @nr_channels: Number of DMA channels available. * @ch_fifo_size_mask: Mask for FIFO size field. * @sreq_index_offset: Slave channel index offset. - * @has_outstanding_reqs: If DMA channel can have outstanding requests. + * @max_page: Maximum ADMA Channel Page. + * @set_global_pg_config: Global page programming. */ struct tegra_adma_chip_data { unsigned int (*adma_get_burst_config)(unsigned int burst_size); unsigned int global_reg_offset; unsigned int global_int_clear; + unsigned int global_ch_fifo_base; + unsigned int global_ch_config_base; unsigned int ch_req_tx_shift; unsigned int ch_req_rx_shift; + unsigned int ch_dir_shift; + unsigned int ch_mode_shift; unsigned int ch_base_offset; + unsigned int ch_tc_offset_diff; unsigned int ch_fifo_ctrl; + unsigned int ch_config; unsigned int ch_req_mask; + unsigned int ch_dir_mask; unsigned int ch_req_max; unsigned int ch_reg_size; unsigned int nr_channels; unsigned int ch_fifo_size_mask; unsigned int sreq_index_offset; - bool has_outstanding_reqs; + unsigned int max_page; + void (*set_global_pg_config)(struct tegra_adma *tdma); }; /* @@ -103,6 +136,7 @@ struct tegra_adma_chip_data { struct tegra_adma_chan_regs { unsigned int ctrl; unsigned int config; + unsigned int global_config; unsigned int src_addr; unsigned int trg_addr; unsigned int fifo_ctrl; @@ -141,6 +175,9 @@ struct tegra_adma_chan { /* Transfer count and position info */ unsigned int tx_buf_count; unsigned int tx_buf_pos; + + unsigned int global_ch_fifo_offset; + unsigned int global_ch_config_offset; }; /* @@ -150,18 +187,21 @@ struct tegra_adma { struct dma_device dma_dev; struct device *dev; void __iomem *base_addr; + void __iomem *ch_base_addr; struct clk *ahub_clk; unsigned int nr_channels; + unsigned long *dma_chan_mask; unsigned long rx_requests_reserved; unsigned long tx_requests_reserved; /* Used to store global command register state when suspending */ unsigned int global_cmd; + unsigned int ch_page_no; const struct tegra_adma_chip_data *cdata; /* Last member of the structure */ - struct tegra_adma_chan channels[]; + struct tegra_adma_chan channels[] __counted_by(nr_channels); }; static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) @@ -174,6 +214,11 @@ static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg) return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg); } +static inline void tdma_ch_global_write(struct tegra_adma *tdma, u32 reg, u32 val) +{ + writel(val, tdma->ch_base_addr + tdma->cdata->global_reg_offset + reg); +} + static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val) { writel(val, tdc->chan_addr + reg); @@ -215,13 +260,53 @@ static int tegra_adma_slave_config(struct dma_chan *dc, return 0; } +static void tegra186_adma_global_page_config(struct tegra_adma *tdma) +{ + /* + * Clear the default page1 channel group configs and program + * the global registers based on the actual page usage + */ + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP + (tdma->ch_page_no * 0x4), 0xff); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ + (tdma->ch_page_no * 0x4), 0x1ffffff); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ + (tdma->ch_page_no * 0x4), 0xffffff); +} + +static void tegra264_adma_global_page_config(struct tegra_adma *tdma) +{ + u32 global_page_offset = tdma->ch_page_no * TEGRA264_ADMA_GLOBAL_PAGE_OFFSET; + + /* If the default page (page1) is not used, then clear page1 registers */ + if (tdma->ch_page_no) { + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1, 0); + } + + /* Program global registers for selected page */ + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1 + global_page_offset, 0x1); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1 + global_page_offset, 0x1); +} + static int tegra_adma_init(struct tegra_adma *tdma) { u32 status; int ret; - /* Clear any interrupts */ - tdma_write(tdma, tdma->cdata->ch_base_offset + tdma->cdata->global_int_clear, 0x1); + /* Clear any channels group global interrupts */ + tdma_ch_global_write(tdma, tdma->cdata->global_int_clear, 0x1); + + if (!tdma->base_addr) + return 0; /* Assert soft reset */ tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1); @@ -235,6 +320,9 @@ static int tegra_adma_init(struct tegra_adma *tdma) if (ret) return ret; + if (tdma->cdata->set_global_pg_config) + tdma->cdata->set_global_pg_config(tdma); + /* Enable global ADMA registers */ tdma_write(tdma, ADMA_GLOBAL_CMD, 1); @@ -367,11 +455,21 @@ static void tegra_adma_start(struct tegra_adma_chan *tdc) tdc->tx_buf_pos = 0; tdc->tx_buf_count = 0; - tdma_ch_write(tdc, ADMA_CH_TC, ch_regs->tc); + tdma_ch_write(tdc, ADMA_CH_TC - tdc->tdma->cdata->ch_tc_offset_diff, ch_regs->tc); tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); - tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_regs->src_addr); - tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_regs->trg_addr); - tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl); + tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR - tdc->tdma->cdata->ch_tc_offset_diff, + ch_regs->src_addr); + tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR - tdc->tdma->cdata->ch_tc_offset_diff, + ch_regs->trg_addr); + + if (!tdc->tdma->cdata->global_ch_fifo_base) + tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl); + else if (tdc->global_ch_fifo_offset) + tdma_write(tdc->tdma, tdc->global_ch_fifo_offset, ch_regs->fifo_ctrl); + + if (tdc->global_ch_config_offset) + tdma_write(tdc->tdma, tdc->global_ch_config_offset, ch_regs->global_config); + tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config); /* Start ADMA */ @@ -384,7 +482,8 @@ static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc) { struct tegra_adma_desc *desc = tdc->desc; unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1; - unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS); + unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS - + tdc->tdma->cdata->ch_tc_offset_diff); unsigned int periods_remaining; /* @@ -590,13 +689,16 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, return -EINVAL; } - ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) | - ADMA_CH_CTRL_MODE_CONTINUOUS | + ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir, cdata->ch_dir_mask, + cdata->ch_dir_shift) | + ADMA_CH_CTRL_MODE_CONTINUOUS(cdata->ch_mode_shift) | ADMA_CH_CTRL_FLOWCTRL_EN; ch_regs->config |= cdata->adma_get_burst_config(burst_size); - ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1); - if (cdata->has_outstanding_reqs) - ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8); + + if (cdata->global_ch_config_base) + ch_regs->global_config |= cdata->ch_config; + else + ch_regs->config |= cdata->ch_config; /* * 'sreq_index' represents the current ADMAIF channel number and as per @@ -734,23 +836,40 @@ static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev) struct tegra_adma_chan *tdc; int i; - tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); + if (tdma->base_addr) + tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); + if (!tdma->global_cmd) goto clk_disable; for (i = 0; i < tdma->nr_channels; i++) { tdc = &tdma->channels[i]; + /* skip for reserved channels */ + if (!tdc->tdma) + continue; + ch_reg = &tdc->ch_regs; ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); /* skip if channel is not active */ if (!ch_reg->cmd) continue; - ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC); - ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR); - ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR); + ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC - tdma->cdata->ch_tc_offset_diff); + ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR - + tdma->cdata->ch_tc_offset_diff); + ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR - + tdma->cdata->ch_tc_offset_diff); ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); - ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); + + if (tdc->global_ch_config_offset) + ch_reg->global_config = tdma_read(tdc->tdma, tdc->global_ch_config_offset); + + if (!tdc->tdma->cdata->global_ch_fifo_base) + ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); + else if (tdc->global_ch_fifo_offset) + ch_reg->fifo_ctrl = tdma_read(tdc->tdma, tdc->global_ch_fifo_offset); + ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG); + } clk_disable: @@ -771,23 +890,41 @@ static int __maybe_unused tegra_adma_runtime_resume(struct device *dev) dev_err(dev, "ahub clk_enable failed: %d\n", ret); return ret; } - tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); + if (tdma->base_addr) { + tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); + if (tdma->cdata->set_global_pg_config) + tdma->cdata->set_global_pg_config(tdma); + } if (!tdma->global_cmd) return 0; for (i = 0; i < tdma->nr_channels; i++) { tdc = &tdma->channels[i]; + /* skip for reserved channels */ + if (!tdc->tdma) + continue; ch_reg = &tdc->ch_regs; /* skip if channel was not active earlier */ if (!ch_reg->cmd) continue; - tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc); - tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr); - tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr); + tdma_ch_write(tdc, ADMA_CH_TC - tdma->cdata->ch_tc_offset_diff, ch_reg->tc); + tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR - tdma->cdata->ch_tc_offset_diff, + ch_reg->src_addr); + tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR - tdma->cdata->ch_tc_offset_diff, + ch_reg->trg_addr); tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl); - tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl); + + if (!tdc->tdma->cdata->global_ch_fifo_base) + tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl); + else if (tdc->global_ch_fifo_offset) + tdma_write(tdc->tdma, tdc->global_ch_fifo_offset, ch_reg->fifo_ctrl); + + if (tdc->global_ch_config_offset) + tdma_write(tdc->tdma, tdc->global_ch_config_offset, ch_reg->global_config); + tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config); + tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd); } @@ -798,37 +935,80 @@ static const struct tegra_adma_chip_data tegra210_chip_data = { .adma_get_burst_config = tegra210_adma_get_burst_config, .global_reg_offset = 0xc00, .global_int_clear = 0x20, + .global_ch_fifo_base = 0, + .global_ch_config_base = 0, .ch_req_tx_shift = 28, .ch_req_rx_shift = 24, + .ch_dir_shift = 12, + .ch_mode_shift = 8, .ch_base_offset = 0, + .ch_tc_offset_diff = 0, + .ch_config = ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1), .ch_req_mask = 0xf, + .ch_dir_mask = 0xf, .ch_req_max = 10, .ch_reg_size = 0x80, .nr_channels = 22, .ch_fifo_size_mask = 0xf, .sreq_index_offset = 2, - .has_outstanding_reqs = false, + .max_page = 0, + .set_global_pg_config = NULL, }; static const struct tegra_adma_chip_data tegra186_chip_data = { .adma_get_burst_config = tegra186_adma_get_burst_config, .global_reg_offset = 0, .global_int_clear = 0x402c, + .global_ch_fifo_base = 0, + .global_ch_config_base = 0, .ch_req_tx_shift = 27, .ch_req_rx_shift = 22, + .ch_dir_shift = 12, + .ch_mode_shift = 8, .ch_base_offset = 0x10000, + .ch_tc_offset_diff = 0, + .ch_config = ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1) | + TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8), .ch_req_mask = 0x1f, + .ch_dir_mask = 0xf, .ch_req_max = 20, .ch_reg_size = 0x100, .nr_channels = 32, .ch_fifo_size_mask = 0x1f, .sreq_index_offset = 4, - .has_outstanding_reqs = true, + .max_page = 4, + .set_global_pg_config = tegra186_adma_global_page_config, +}; + +static const struct tegra_adma_chip_data tegra264_chip_data = { + .adma_get_burst_config = tegra186_adma_get_burst_config, + .global_reg_offset = 0, + .global_int_clear = 0x800c, + .global_ch_fifo_base = ADMA_GLOBAL_CH_FIFO_CTRL, + .global_ch_config_base = ADMA_GLOBAL_CH_CONFIG, + .ch_req_tx_shift = 26, + .ch_req_rx_shift = 20, + .ch_dir_shift = 10, + .ch_mode_shift = 7, + .ch_base_offset = 0x10000, + .ch_tc_offset_diff = 4, + .ch_config = ADMA_GLOBAL_CH_CONFIG_WEIGHT_FOR_WRR(1) | + ADMA_GLOBAL_CH_CONFIG_OUTSTANDING_REQS(8), + .ch_req_mask = 0x3f, + .ch_dir_mask = 7, + .ch_req_max = 32, + .ch_reg_size = 0x100, + .nr_channels = 64, + .ch_fifo_size_mask = 0x7f, + .sreq_index_offset = 0, + .max_page = 10, + .set_global_pg_config = tegra264_adma_global_page_config, }; static const struct of_device_id tegra_adma_of_match[] = { { .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data }, { .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data }, + { .compatible = "nvidia,tegra264-adma", .data = &tegra264_chip_data }, { }, }; MODULE_DEVICE_TABLE(of, tegra_adma_of_match); @@ -837,6 +1017,7 @@ static int tegra_adma_probe(struct platform_device *pdev) { const struct tegra_adma_chip_data *cdata; struct tegra_adma *tdma; + struct resource *res_page, *res_base; int ret, i; cdata = of_device_get_match_data(&pdev->dev); @@ -856,9 +1037,46 @@ static int tegra_adma_probe(struct platform_device *pdev) tdma->nr_channels = cdata->nr_channels; platform_set_drvdata(pdev, tdma); - tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(tdma->base_addr)) - return PTR_ERR(tdma->base_addr); + res_page = platform_get_resource_byname(pdev, IORESOURCE_MEM, "page"); + if (res_page) { + tdma->ch_base_addr = devm_ioremap_resource(&pdev->dev, res_page); + if (IS_ERR(tdma->ch_base_addr)) + return PTR_ERR(tdma->ch_base_addr); + + res_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "global"); + if (res_base) { + resource_size_t page_offset, page_no; + unsigned int ch_base_offset; + + if (res_page->start < res_base->start) + return -EINVAL; + page_offset = res_page->start - res_base->start; + ch_base_offset = cdata->ch_base_offset; + if (!ch_base_offset) + return -EINVAL; + + page_no = div_u64(page_offset, ch_base_offset); + if (!page_no || page_no > INT_MAX) + return -EINVAL; + + tdma->ch_page_no = page_no - 1; + tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); + } + } else { + /* If no 'page' property found, then reg DT binding would be legacy */ + res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res_base) { + tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); + } else { + return -ENODEV; + } + + tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset; + } tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio"); if (IS_ERR(tdma->ahub_clk)) { @@ -866,12 +1084,41 @@ static int tegra_adma_probe(struct platform_device *pdev) return PTR_ERR(tdma->ahub_clk); } + tdma->dma_chan_mask = devm_kzalloc(&pdev->dev, + BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long), + GFP_KERNEL); + if (!tdma->dma_chan_mask) + return -ENOMEM; + + /* Enable all channels by default */ + bitmap_fill(tdma->dma_chan_mask, tdma->nr_channels); + + ret = of_property_read_u32_array(pdev->dev.of_node, "dma-channel-mask", + (u32 *)tdma->dma_chan_mask, + BITS_TO_U32(tdma->nr_channels)); + if (ret < 0 && (ret != -EINVAL)) { + dev_err(&pdev->dev, "dma-channel-mask is not complete.\n"); + return ret; + } + INIT_LIST_HEAD(&tdma->dma_dev.channels); for (i = 0; i < tdma->nr_channels; i++) { struct tegra_adma_chan *tdc = &tdma->channels[i]; - tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset - + (cdata->ch_reg_size * i); + /* skip for reserved channels */ + if (!test_bit(i, tdma->dma_chan_mask)) + continue; + + tdc->chan_addr = tdma->ch_base_addr + (cdata->ch_reg_size * i); + + if (tdma->base_addr) { + if (cdata->global_ch_fifo_base) + tdc->global_ch_fifo_offset = cdata->global_ch_fifo_base + (4 * i); + + if (cdata->global_ch_config_base) + tdc->global_ch_config_offset = + cdata->global_ch_config_base + (4 * i); + } tdc->irq = of_irq_get(pdev->dev.of_node, i); if (tdc->irq <= 0) { @@ -948,7 +1195,7 @@ irq_dispose: return ret; } -static int tegra_adma_remove(struct platform_device *pdev) +static void tegra_adma_remove(struct platform_device *pdev) { struct tegra_adma *tdma = platform_get_drvdata(pdev); int i; @@ -956,12 +1203,12 @@ static int tegra_adma_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&tdma->dma_dev); - for (i = 0; i < tdma->nr_channels; ++i) - irq_dispose_mapping(tdma->channels[i].irq); + for (i = 0; i < tdma->nr_channels; ++i) { + if (tdma->channels[i].irq) + irq_dispose_mapping(tdma->channels[i].irq); + } pm_runtime_disable(&pdev->dev); - - return 0; } static const struct dev_pm_ops tegra_adma_dev_pm_ops = { @@ -983,7 +1230,6 @@ static struct platform_driver tegra_admac_driver = { module_platform_driver(tegra_admac_driver); -MODULE_ALIAS("platform:tegra210-adma"); MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver"); MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>"); MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>"); diff --git a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig index 2adc2cca10e9..dbf168146d35 100644 --- a/drivers/dma/ti/Kconfig +++ b/drivers/dma/ti/Kconfig @@ -17,7 +17,7 @@ config TI_EDMA select DMA_ENGINE select DMA_VIRTUAL_CHANNELS select TI_DMA_CROSSBAR if (ARCH_OMAP || COMPILE_TEST) - default y + default ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE help Enable support for the TI EDMA (Enhanced DMA) controller. This DMA engine is found on TI DaVinci, AM33xx, AM43xx, DRA7xx and Keystone 2 @@ -29,7 +29,7 @@ config DMA_OMAP select DMA_ENGINE select DMA_VIRTUAL_CHANNELS select TI_DMA_CROSSBAR if (SOC_DRA7XX || COMPILE_TEST) - default y + default ARCH_OMAP help Enable support for the TI sDMA (System DMA or DMA4) controller. This DMA engine is found on OMAP and DRA7xx parts. diff --git a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile index acc950bf609c..d376c117cecf 100644 --- a/drivers/dma/ti/Makefile +++ b/drivers/dma/ti/Makefile @@ -12,6 +12,7 @@ k3-psil-lib-objs := k3-psil.o \ k3-psil-j721s2.o \ k3-psil-am62.o \ k3-psil-am62a.o \ - k3-psil-j784s4.o + k3-psil-j784s4.o \ + k3-psil-am62p.o obj-$(CONFIG_TI_K3_PSIL) += k3-psil-lib.o obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o diff --git a/drivers/dma/ti/cppi41.c b/drivers/dma/ti/cppi41.c index c3555cfb0681..8d8c3d6038fc 100644 --- a/drivers/dma/ti/cppi41.c +++ b/drivers/dma/ti/cppi41.c @@ -1156,7 +1156,7 @@ err_get_sync: return ret; } -static int cppi41_dma_remove(struct platform_device *pdev) +static void cppi41_dma_remove(struct platform_device *pdev) { struct cppi41_dd *cdd = platform_get_drvdata(pdev); int error; @@ -1173,7 +1173,6 @@ static int cppi41_dma_remove(struct platform_device *pdev) pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); - return 0; } static int __maybe_unused cppi41_suspend(struct device *dev) @@ -1253,5 +1252,6 @@ static struct platform_driver cpp41_dma_driver = { }; module_platform_driver(cpp41_dma_driver); +MODULE_DESCRIPTION("Texas Instruments CPPI 4.1 DMA support"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>"); diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c index f744ddbbbad7..7f17ee87a6dc 100644 --- a/drivers/dma/ti/dma-crossbar.c +++ b/drivers/dma/ti/dma-crossbar.c @@ -3,14 +3,15 @@ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> */ +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/err.h> #include <linux/init.h> #include <linux/list.h> #include <linux/io.h> -#include <linux/of_address.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_platform.h> #define TI_XBAR_DRA7 0 #define TI_XBAR_AM335X 1 diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 9ea91c640c32..552be71db6c4 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -16,11 +16,11 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/string_choices.h> #include <linux/of.h> #include <linux/of_dma.h> #include <linux/of_irq.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/pm_runtime.h> #include <linux/platform_data/edma.h> @@ -203,13 +203,12 @@ struct edma_desc { u32 residue; u32 residue_stat; - struct edma_pset pset[]; + struct edma_pset pset[] __counted_by(pset_nr); }; struct edma_cc; struct edma_tc { - struct device_node *node; u16 id; }; @@ -2049,7 +2048,7 @@ static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata, dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels); dev_dbg(dev, "num_slots: %u\n", ecc->num_slots); dev_dbg(dev, "num_tc: %u\n", ecc->num_tc); - dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no"); + dev_dbg(dev, "chmap_exist: %s\n", str_yes_no(ecc->chmap_exist)); /* Nothing need to be done if queue priority is provided */ if (pdata->queue_priority_mapping) @@ -2065,8 +2064,8 @@ static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata, * priority. So Q0 is the highest priority queue and the last queue has * the lowest priority. */ - queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8), - GFP_KERNEL); + queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, + sizeof(*queue_priority_map), GFP_KERNEL); if (!queue_priority_map) return -ENOMEM; @@ -2260,8 +2259,12 @@ static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec, return NULL; out: - /* The channel is going to be used as HW synchronized */ - echan->hw_triggered = true; + /* + * The channel is going to be HW synchronized, unless it was + * reserved as a memcpy channel + */ + echan->hw_triggered = + !edma_is_memcpy_channel(i, ecc->info->memcpy_channels); return dma_get_slave_channel(chan); } #else @@ -2402,9 +2405,14 @@ static int edma_probe(struct platform_device *pdev) if (irq < 0 && node) irq = irq_of_parse_and_map(node, 0); - if (irq >= 0) { + if (irq > 0) { irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint", dev_name(dev)); + if (!irq_name) { + ret = -ENOMEM; + goto err_disable_pm; + } + ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name, ecc); if (ret) { @@ -2418,9 +2426,14 @@ static int edma_probe(struct platform_device *pdev) if (irq < 0 && node) irq = irq_of_parse_and_map(node, 2); - if (irq >= 0) { + if (irq > 0) { irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint", dev_name(dev)); + if (!irq_name) { + ret = -ENOMEM; + goto err_disable_pm; + } + ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name, ecc); if (ret) { @@ -2451,19 +2464,19 @@ static int edma_probe(struct platform_device *pdev) goto err_reg1; } - for (i = 0;; i++) { + for (i = 0; i < ecc->num_tc; i++) { ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs", 1, i, &tc_args); - if (ret || i == ecc->num_tc) + if (ret) break; - ecc->tc_list[i].node = tc_args.np; ecc->tc_list[i].id = i; queue_priority_mapping[i][1] = tc_args.args[0]; if (queue_priority_mapping[i][1] > lowest_priority) { lowest_priority = queue_priority_mapping[i][1]; info->default_queue = i; } + of_node_put(tc_args.np); } /* See if we have optional dma-channel-mask array */ @@ -2551,7 +2564,7 @@ static void edma_cleanupp_vchan(struct dma_device *dmadev) } } -static int edma_remove(struct platform_device *pdev) +static void edma_remove(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct edma_cc *ecc = dev_get_drvdata(dev); @@ -2569,8 +2582,6 @@ static int edma_remove(struct platform_device *pdev) edma_free_slot(ecc, ecc->dummy_slot); pm_runtime_put_sync(dev); pm_runtime_disable(dev); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/dma/ti/k3-psil-am62.c b/drivers/dma/ti/k3-psil-am62.c index 2b6fd6e37c61..1272b1541f61 100644 --- a/drivers/dma/ti/k3-psil-am62.c +++ b/drivers/dma/ti/k3-psil-am62.c @@ -74,7 +74,9 @@ static struct psil_ep am62_src_ep_map[] = { PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), - /* PDMA_MAIN0 - SPI0-3 */ + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), PSIL_PDMA_XY_PKT(0x4302), PSIL_PDMA_XY_PKT(0x4303), PSIL_PDMA_XY_PKT(0x4304), @@ -85,8 +87,6 @@ static struct psil_ep am62_src_ep_map[] = { PSIL_PDMA_XY_PKT(0x4309), PSIL_PDMA_XY_PKT(0x430a), PSIL_PDMA_XY_PKT(0x430b), - PSIL_PDMA_XY_PKT(0x430c), - PSIL_PDMA_XY_PKT(0x430d), /* PDMA_MAIN1 - UART0-6 */ PSIL_PDMA_XY_PKT(0x4400), PSIL_PDMA_XY_PKT(0x4401), @@ -141,7 +141,9 @@ static struct psil_ep am62_dst_ep_map[] = { /* SAUL */ PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), - /* PDMA_MAIN0 - SPI0-3 */ + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0xc300), + PSIL_PDMA_XY_PKT(0xc301), PSIL_PDMA_XY_PKT(0xc302), PSIL_PDMA_XY_PKT(0xc303), PSIL_PDMA_XY_PKT(0xc304), @@ -152,8 +154,6 @@ static struct psil_ep am62_dst_ep_map[] = { PSIL_PDMA_XY_PKT(0xc309), PSIL_PDMA_XY_PKT(0xc30a), PSIL_PDMA_XY_PKT(0xc30b), - PSIL_PDMA_XY_PKT(0xc30c), - PSIL_PDMA_XY_PKT(0xc30d), /* PDMA_MAIN1 - UART0-6 */ PSIL_PDMA_XY_PKT(0xc400), PSIL_PDMA_XY_PKT(0xc401), diff --git a/drivers/dma/ti/k3-psil-am62a.c b/drivers/dma/ti/k3-psil-am62a.c index ca9d71f91422..4cf9123b0e93 100644 --- a/drivers/dma/ti/k3-psil-am62a.c +++ b/drivers/dma/ti/k3-psil-am62a.c @@ -84,7 +84,9 @@ static struct psil_ep am62a_src_ep_map[] = { PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), - /* PDMA_MAIN0 - SPI0-3 */ + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), PSIL_PDMA_XY_PKT(0x4302), PSIL_PDMA_XY_PKT(0x4303), PSIL_PDMA_XY_PKT(0x4304), @@ -95,8 +97,6 @@ static struct psil_ep am62a_src_ep_map[] = { PSIL_PDMA_XY_PKT(0x4309), PSIL_PDMA_XY_PKT(0x430a), PSIL_PDMA_XY_PKT(0x430b), - PSIL_PDMA_XY_PKT(0x430c), - PSIL_PDMA_XY_PKT(0x430d), /* PDMA_MAIN1 - UART0-6 */ PSIL_PDMA_XY_PKT(0x4400), PSIL_PDMA_XY_PKT(0x4401), @@ -151,7 +151,9 @@ static struct psil_ep am62a_dst_ep_map[] = { /* SAUL */ PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), - /* PDMA_MAIN0 - SPI0-3 */ + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0xc300), + PSIL_PDMA_XY_PKT(0xc301), PSIL_PDMA_XY_PKT(0xc302), PSIL_PDMA_XY_PKT(0xc303), PSIL_PDMA_XY_PKT(0xc304), @@ -162,8 +164,6 @@ static struct psil_ep am62a_dst_ep_map[] = { PSIL_PDMA_XY_PKT(0xc309), PSIL_PDMA_XY_PKT(0xc30a), PSIL_PDMA_XY_PKT(0xc30b), - PSIL_PDMA_XY_PKT(0xc30c), - PSIL_PDMA_XY_PKT(0xc30d), /* PDMA_MAIN1 - UART0-6 */ PSIL_PDMA_XY_PKT(0xc400), PSIL_PDMA_XY_PKT(0xc401), diff --git a/drivers/dma/ti/k3-psil-am62p.c b/drivers/dma/ti/k3-psil-am62p.c new file mode 100644 index 000000000000..0f338e16d971 --- /dev/null +++ b/drivers/dma/ti/k3-psil-am62p.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/kernel.h> + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_TR(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + }, \ + } + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = flow_base, \ + }, \ + } + +#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = default_flow, \ + .notdpkt = tx, \ + }, \ + } + +#define PSIL_PDMA_MCASP(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pdma_acc32 = 1, \ + .pdma_burst = 1, \ + }, \ + } + +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep am62p_src_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0x7504, 20, 35, 8, 35, 0), + PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), + PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), + PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), + PSIL_PDMA_XY_PKT(0x4302), + PSIL_PDMA_XY_PKT(0x4303), + PSIL_PDMA_XY_PKT(0x4304), + PSIL_PDMA_XY_PKT(0x4305), + PSIL_PDMA_XY_PKT(0x4306), + PSIL_PDMA_XY_PKT(0x4307), + PSIL_PDMA_XY_PKT(0x4308), + PSIL_PDMA_XY_PKT(0x4309), + PSIL_PDMA_XY_PKT(0x430a), + PSIL_PDMA_XY_PKT(0x430b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0x4400), + PSIL_PDMA_XY_PKT(0x4401), + PSIL_PDMA_XY_PKT(0x4402), + PSIL_PDMA_XY_PKT(0x4403), + PSIL_PDMA_XY_PKT(0x4404), + PSIL_PDMA_XY_PKT(0x4405), + PSIL_PDMA_XY_PKT(0x4406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0x4500), + PSIL_PDMA_MCASP(0x4501), + PSIL_PDMA_MCASP(0x4502), + /* CPSW3G */ + PSIL_ETHERNET(0x4600, 19, 19, 16), + /* CSI2RX */ + PSIL_CSI2RX(0x5000), + PSIL_CSI2RX(0x5001), + PSIL_CSI2RX(0x5002), + PSIL_CSI2RX(0x5003), + PSIL_CSI2RX(0x5004), + PSIL_CSI2RX(0x5005), + PSIL_CSI2RX(0x5006), + PSIL_CSI2RX(0x5007), + PSIL_CSI2RX(0x5008), + PSIL_CSI2RX(0x5009), + PSIL_CSI2RX(0x500a), + PSIL_CSI2RX(0x500b), + PSIL_CSI2RX(0x500c), + PSIL_CSI2RX(0x500d), + PSIL_CSI2RX(0x500e), + PSIL_CSI2RX(0x500f), + PSIL_CSI2RX(0x5010), + PSIL_CSI2RX(0x5011), + PSIL_CSI2RX(0x5012), + PSIL_CSI2RX(0x5013), + PSIL_CSI2RX(0x5014), + PSIL_CSI2RX(0x5015), + PSIL_CSI2RX(0x5016), + PSIL_CSI2RX(0x5017), + PSIL_CSI2RX(0x5018), + PSIL_CSI2RX(0x5019), + PSIL_CSI2RX(0x501a), + PSIL_CSI2RX(0x501b), + PSIL_CSI2RX(0x501c), + PSIL_CSI2RX(0x501d), + PSIL_CSI2RX(0x501e), + PSIL_CSI2RX(0x501f), + PSIL_CSI2RX(0x5000), + PSIL_CSI2RX(0x5001), + PSIL_CSI2RX(0x5002), + PSIL_CSI2RX(0x5003), + PSIL_CSI2RX(0x5004), + PSIL_CSI2RX(0x5005), + PSIL_CSI2RX(0x5006), + PSIL_CSI2RX(0x5007), + PSIL_CSI2RX(0x5008), + PSIL_CSI2RX(0x5009), + PSIL_CSI2RX(0x500a), + PSIL_CSI2RX(0x500b), + PSIL_CSI2RX(0x500c), + PSIL_CSI2RX(0x500d), + PSIL_CSI2RX(0x500e), + PSIL_CSI2RX(0x500f), + PSIL_CSI2RX(0x5010), + PSIL_CSI2RX(0x5011), + PSIL_CSI2RX(0x5012), + PSIL_CSI2RX(0x5013), + PSIL_CSI2RX(0x5014), + PSIL_CSI2RX(0x5015), + PSIL_CSI2RX(0x5016), + PSIL_CSI2RX(0x5017), + PSIL_CSI2RX(0x5018), + PSIL_CSI2RX(0x5019), + PSIL_CSI2RX(0x501a), + PSIL_CSI2RX(0x501b), + PSIL_CSI2RX(0x501c), + PSIL_CSI2RX(0x501d), + PSIL_CSI2RX(0x501e), + PSIL_CSI2RX(0x501f), + /* CSIRX 1-3 (only for J722S) */ + PSIL_CSI2RX(0x5100), + PSIL_CSI2RX(0x5101), + PSIL_CSI2RX(0x5102), + PSIL_CSI2RX(0x5103), + PSIL_CSI2RX(0x5104), + PSIL_CSI2RX(0x5105), + PSIL_CSI2RX(0x5106), + PSIL_CSI2RX(0x5107), + PSIL_CSI2RX(0x5108), + PSIL_CSI2RX(0x5109), + PSIL_CSI2RX(0x510a), + PSIL_CSI2RX(0x510b), + PSIL_CSI2RX(0x510c), + PSIL_CSI2RX(0x510d), + PSIL_CSI2RX(0x510e), + PSIL_CSI2RX(0x510f), + PSIL_CSI2RX(0x5110), + PSIL_CSI2RX(0x5111), + PSIL_CSI2RX(0x5112), + PSIL_CSI2RX(0x5113), + PSIL_CSI2RX(0x5114), + PSIL_CSI2RX(0x5115), + PSIL_CSI2RX(0x5116), + PSIL_CSI2RX(0x5117), + PSIL_CSI2RX(0x5118), + PSIL_CSI2RX(0x5119), + PSIL_CSI2RX(0x511a), + PSIL_CSI2RX(0x511b), + PSIL_CSI2RX(0x511c), + PSIL_CSI2RX(0x511d), + PSIL_CSI2RX(0x511e), + PSIL_CSI2RX(0x511f), + PSIL_CSI2RX(0x5200), + PSIL_CSI2RX(0x5201), + PSIL_CSI2RX(0x5202), + PSIL_CSI2RX(0x5203), + PSIL_CSI2RX(0x5204), + PSIL_CSI2RX(0x5205), + PSIL_CSI2RX(0x5206), + PSIL_CSI2RX(0x5207), + PSIL_CSI2RX(0x5208), + PSIL_CSI2RX(0x5209), + PSIL_CSI2RX(0x520a), + PSIL_CSI2RX(0x520b), + PSIL_CSI2RX(0x520c), + PSIL_CSI2RX(0x520d), + PSIL_CSI2RX(0x520e), + PSIL_CSI2RX(0x520f), + PSIL_CSI2RX(0x5210), + PSIL_CSI2RX(0x5211), + PSIL_CSI2RX(0x5212), + PSIL_CSI2RX(0x5213), + PSIL_CSI2RX(0x5214), + PSIL_CSI2RX(0x5215), + PSIL_CSI2RX(0x5216), + PSIL_CSI2RX(0x5217), + PSIL_CSI2RX(0x5218), + PSIL_CSI2RX(0x5219), + PSIL_CSI2RX(0x521a), + PSIL_CSI2RX(0x521b), + PSIL_CSI2RX(0x521c), + PSIL_CSI2RX(0x521d), + PSIL_CSI2RX(0x521e), + PSIL_CSI2RX(0x521f), + PSIL_CSI2RX(0x5300), + PSIL_CSI2RX(0x5301), + PSIL_CSI2RX(0x5302), + PSIL_CSI2RX(0x5303), + PSIL_CSI2RX(0x5304), + PSIL_CSI2RX(0x5305), + PSIL_CSI2RX(0x5306), + PSIL_CSI2RX(0x5307), + PSIL_CSI2RX(0x5308), + PSIL_CSI2RX(0x5309), + PSIL_CSI2RX(0x530a), + PSIL_CSI2RX(0x530b), + PSIL_CSI2RX(0x530c), + PSIL_CSI2RX(0x530d), + PSIL_CSI2RX(0x530e), + PSIL_CSI2RX(0x530f), + PSIL_CSI2RX(0x5310), + PSIL_CSI2RX(0x5311), + PSIL_CSI2RX(0x5312), + PSIL_CSI2RX(0x5313), + PSIL_CSI2RX(0x5314), + PSIL_CSI2RX(0x5315), + PSIL_CSI2RX(0x5316), + PSIL_CSI2RX(0x5317), + PSIL_CSI2RX(0x5318), + PSIL_CSI2RX(0x5319), + PSIL_CSI2RX(0x531a), + PSIL_CSI2RX(0x531b), + PSIL_CSI2RX(0x531c), + PSIL_CSI2RX(0x531d), + PSIL_CSI2RX(0x531e), + PSIL_CSI2RX(0x531f), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep am62p_dst_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), + PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0xc300), + PSIL_PDMA_XY_PKT(0xc301), + PSIL_PDMA_XY_PKT(0xc302), + PSIL_PDMA_XY_PKT(0xc303), + PSIL_PDMA_XY_PKT(0xc304), + PSIL_PDMA_XY_PKT(0xc305), + PSIL_PDMA_XY_PKT(0xc306), + PSIL_PDMA_XY_PKT(0xc307), + PSIL_PDMA_XY_PKT(0xc308), + PSIL_PDMA_XY_PKT(0xc309), + PSIL_PDMA_XY_PKT(0xc30a), + PSIL_PDMA_XY_PKT(0xc30b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0xc400), + PSIL_PDMA_XY_PKT(0xc401), + PSIL_PDMA_XY_PKT(0xc402), + PSIL_PDMA_XY_PKT(0xc403), + PSIL_PDMA_XY_PKT(0xc404), + PSIL_PDMA_XY_PKT(0xc405), + PSIL_PDMA_XY_PKT(0xc406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0xc500), + PSIL_PDMA_MCASP(0xc501), + PSIL_PDMA_MCASP(0xc502), + /* CPSW3G */ + PSIL_ETHERNET(0xc600, 19, 19, 8), + PSIL_ETHERNET(0xc601, 20, 27, 8), + PSIL_ETHERNET(0xc602, 21, 35, 8), + PSIL_ETHERNET(0xc603, 22, 43, 8), + PSIL_ETHERNET(0xc604, 23, 51, 8), + PSIL_ETHERNET(0xc605, 24, 59, 8), + PSIL_ETHERNET(0xc606, 25, 67, 8), + PSIL_ETHERNET(0xc607, 26, 75, 8), +}; + +struct psil_ep_map am62p_ep_map = { + .name = "am62p", + .src = am62p_src_ep_map, + .src_count = ARRAY_SIZE(am62p_src_ep_map), + .dst = am62p_dst_ep_map, + .dst_count = ARRAY_SIZE(am62p_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-j721s2.c b/drivers/dma/ti/k3-psil-j721s2.c index 1d5430fc5724..ba08bdcdcd2b 100644 --- a/drivers/dma/ti/k3-psil-j721s2.c +++ b/drivers/dma/ti/k3-psil-j721s2.c @@ -57,6 +57,14 @@ }, \ } +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + /* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ static struct psil_ep j721s2_src_ep_map[] = { /* PDMA_MCASP - McASP0-4 */ @@ -114,6 +122,71 @@ static struct psil_ep j721s2_src_ep_map[] = { PSIL_PDMA_XY_PKT(0x4707), PSIL_PDMA_XY_PKT(0x4708), PSIL_PDMA_XY_PKT(0x4709), + /* CSI2RX */ + PSIL_CSI2RX(0x4940), + PSIL_CSI2RX(0x4941), + PSIL_CSI2RX(0x4942), + PSIL_CSI2RX(0x4943), + PSIL_CSI2RX(0x4944), + PSIL_CSI2RX(0x4945), + PSIL_CSI2RX(0x4946), + PSIL_CSI2RX(0x4947), + PSIL_CSI2RX(0x4948), + PSIL_CSI2RX(0x4949), + PSIL_CSI2RX(0x494a), + PSIL_CSI2RX(0x494b), + PSIL_CSI2RX(0x494c), + PSIL_CSI2RX(0x494d), + PSIL_CSI2RX(0x494e), + PSIL_CSI2RX(0x494f), + PSIL_CSI2RX(0x4950), + PSIL_CSI2RX(0x4951), + PSIL_CSI2RX(0x4952), + PSIL_CSI2RX(0x4953), + PSIL_CSI2RX(0x4954), + PSIL_CSI2RX(0x4955), + PSIL_CSI2RX(0x4956), + PSIL_CSI2RX(0x4957), + PSIL_CSI2RX(0x4958), + PSIL_CSI2RX(0x4959), + PSIL_CSI2RX(0x495a), + PSIL_CSI2RX(0x495b), + PSIL_CSI2RX(0x495c), + PSIL_CSI2RX(0x495d), + PSIL_CSI2RX(0x495e), + PSIL_CSI2RX(0x495f), + PSIL_CSI2RX(0x4960), + PSIL_CSI2RX(0x4961), + PSIL_CSI2RX(0x4962), + PSIL_CSI2RX(0x4963), + PSIL_CSI2RX(0x4964), + PSIL_CSI2RX(0x4965), + PSIL_CSI2RX(0x4966), + PSIL_CSI2RX(0x4967), + PSIL_CSI2RX(0x4968), + PSIL_CSI2RX(0x4969), + PSIL_CSI2RX(0x496a), + PSIL_CSI2RX(0x496b), + PSIL_CSI2RX(0x496c), + PSIL_CSI2RX(0x496d), + PSIL_CSI2RX(0x496e), + PSIL_CSI2RX(0x496f), + PSIL_CSI2RX(0x4970), + PSIL_CSI2RX(0x4971), + PSIL_CSI2RX(0x4972), + PSIL_CSI2RX(0x4973), + PSIL_CSI2RX(0x4974), + PSIL_CSI2RX(0x4975), + PSIL_CSI2RX(0x4976), + PSIL_CSI2RX(0x4977), + PSIL_CSI2RX(0x4978), + PSIL_CSI2RX(0x4979), + PSIL_CSI2RX(0x497a), + PSIL_CSI2RX(0x497b), + PSIL_CSI2RX(0x497c), + PSIL_CSI2RX(0x497d), + PSIL_CSI2RX(0x497e), + PSIL_CSI2RX(0x497f), /* MAIN SA2UL */ PSIL_SA2UL(0x4a40, 0), PSIL_SA2UL(0x4a41, 0), diff --git a/drivers/dma/ti/k3-psil-priv.h b/drivers/dma/ti/k3-psil-priv.h index c383723d1c8f..a577be97e344 100644 --- a/drivers/dma/ti/k3-psil-priv.h +++ b/drivers/dma/ti/k3-psil-priv.h @@ -45,5 +45,6 @@ extern struct psil_ep_map j721s2_ep_map; extern struct psil_ep_map am62_ep_map; extern struct psil_ep_map am62a_ep_map; extern struct psil_ep_map j784s4_ep_map; +extern struct psil_ep_map am62p_ep_map; #endif /* K3_PSIL_PRIV_H_ */ diff --git a/drivers/dma/ti/k3-psil.c b/drivers/dma/ti/k3-psil.c index c11389d67a3f..c4b6f0df4686 100644 --- a/drivers/dma/ti/k3-psil.c +++ b/drivers/dma/ti/k3-psil.c @@ -26,6 +26,8 @@ static const struct soc_device_attribute k3_soc_devices[] = { { .family = "AM62X", .data = &am62_ep_map }, { .family = "AM62AX", .data = &am62a_ep_map }, { .family = "J784S4", .data = &j784s4_ep_map }, + { .family = "AM62PX", .data = &am62p_ep_map }, + { .family = "J722S", .data = &am62p_ep_map }, { /* sentinel */ } }; @@ -104,4 +106,5 @@ int psil_set_new_ep_config(struct device *dev, const char *name, return 0; } EXPORT_SYMBOL_GPL(psil_set_new_ep_config); +MODULE_DESCRIPTION("K3 PSI-L endpoint configuration"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 789193ed0386..f87d244cc2d6 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -84,6 +84,7 @@ struct k3_udma_glue_rx_channel { struct k3_udma_glue_rx_flow *flows; u32 flow_num; u32 flows_ready; + bool single_fdq; /* one FDQ for all flows */ }; static void k3_udma_chan_dev_release(struct device *dev) @@ -111,6 +112,35 @@ static int of_k3_udma_glue_parse(struct device_node *udmax_np, return 0; } +static int of_k3_udma_glue_parse_chn_common(struct k3_udma_glue_common *common, u32 thread_id, + bool tx_chn) +{ + if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) + return -EINVAL; + + if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) + return -EINVAL; + + /* get psil endpoint config */ + common->ep_config = psil_get_ep_config(thread_id); + if (IS_ERR(common->ep_config)) { + dev_err(common->dev, + "No configuration for psi-l thread 0x%04x\n", + thread_id); + return PTR_ERR(common->ep_config); + } + + common->epib = common->ep_config->needs_epib; + common->psdata_size = common->ep_config->psd_size; + + if (tx_chn) + common->dst_thread = thread_id; + else + common->src_thread = thread_id; + + return 0; +} + static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, const char *name, struct k3_udma_glue_common *common, bool tx_chn) @@ -153,38 +183,29 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, common->atype_asel = dma_spec.args[1]; } - if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { - ret = -EINVAL; - goto out_put_spec; - } + ret = of_k3_udma_glue_parse_chn_common(common, thread_id, tx_chn); - if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { - ret = -EINVAL; - goto out_put_spec; - } +out_put_spec: + of_node_put(dma_spec.np); + return ret; +} - /* get psil endpoint config */ - common->ep_config = psil_get_ep_config(thread_id); - if (IS_ERR(common->ep_config)) { - dev_err(common->dev, - "No configuration for psi-l thread 0x%04x\n", - thread_id); - ret = PTR_ERR(common->ep_config); - goto out_put_spec; - } +static int +of_k3_udma_glue_parse_chn_by_id(struct device_node *udmax_np, struct k3_udma_glue_common *common, + bool tx_chn, u32 thread_id) +{ + int ret = 0; - common->epib = common->ep_config->needs_epib; - common->psdata_size = common->ep_config->psd_size; + if (unlikely(!udmax_np)) + return -EINVAL; - if (tx_chn) - common->dst_thread = thread_id; - else - common->src_thread = thread_id; + ret = of_k3_udma_glue_parse(udmax_np, common); + if (ret) + return ret; -out_put_spec: - of_node_put(dma_spec.np); + ret = of_k3_udma_glue_parse_chn_common(common, thread_id, tx_chn); return ret; -}; +} static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) { @@ -251,29 +272,13 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); } -struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, - const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) +static int +k3_udma_glue_request_tx_chn_common(struct device *dev, + struct k3_udma_glue_tx_channel *tx_chn, + struct k3_udma_glue_tx_channel_cfg *cfg) { - struct k3_udma_glue_tx_channel *tx_chn; int ret; - tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); - if (!tx_chn) - return ERR_PTR(-ENOMEM); - - tx_chn->common.dev = dev; - tx_chn->common.swdata_size = cfg->swdata_size; - tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; - tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; - tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; - tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; - - /* parse of udmap channel */ - ret = of_k3_udma_glue_parse_chn(dev->of_node, name, - &tx_chn->common, true); - if (ret) - goto err; - tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, tx_chn->common.psdata_size, tx_chn->common.swdata_size); @@ -289,7 +294,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, if (IS_ERR(tx_chn->udma_tchanx)) { ret = PTR_ERR(tx_chn->udma_tchanx); dev_err(dev, "UDMAX tchanx get err %d\n", ret); - goto err; + return ret; } tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); @@ -302,7 +307,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, dev_err(dev, "Channel Device registration failed %d\n", ret); put_device(&tx_chn->common.chan_dev); tx_chn->common.chan_dev.parent = NULL; - goto err; + return ret; } if (xudma_is_pktdma(tx_chn->common.udmax)) { @@ -326,7 +331,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, &tx_chn->ringtxcq); if (ret) { dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret); - goto err; + return ret; } /* Set the dma_dev for the rings to be configured */ @@ -342,13 +347,13 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); - goto err; + return ret; } ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); - goto err; + return ret; } /* request and cfg psi-l */ @@ -359,11 +364,42 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = k3_udma_glue_cfg_tx_chn(tx_chn); if (ret) { dev_err(dev, "Failed to cfg tchan %d\n", ret); - goto err; + return ret; } k3_udma_glue_dump_tx_chn(tx_chn); + return 0; +} + +struct k3_udma_glue_tx_channel * +k3_udma_glue_request_tx_chn(struct device *dev, const char *name, + struct k3_udma_glue_tx_channel_cfg *cfg) +{ + struct k3_udma_glue_tx_channel *tx_chn; + int ret; + + tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); + if (!tx_chn) + return ERR_PTR(-ENOMEM); + + tx_chn->common.dev = dev; + tx_chn->common.swdata_size = cfg->swdata_size; + tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; + tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; + tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; + tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; + + /* parse of udmap channel */ + ret = of_k3_udma_glue_parse_chn(dev->of_node, name, + &tx_chn->common, true); + if (ret) + goto err; + + ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); + if (ret) + goto err; + return tx_chn; err: @@ -372,6 +408,41 @@ err: } EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); +struct k3_udma_glue_tx_channel * +k3_udma_glue_request_tx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_tx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id) +{ + struct k3_udma_glue_tx_channel *tx_chn; + int ret; + + tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); + if (!tx_chn) + return ERR_PTR(-ENOMEM); + + tx_chn->common.dev = dev; + tx_chn->common.swdata_size = cfg->swdata_size; + tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; + tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; + tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; + tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; + + ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, &tx_chn->common, true, thread_id); + if (ret) + goto err; + + ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); + if (ret) + goto err; + + return tx_chn; + +err: + k3_udma_glue_release_tx_chn(tx_chn); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn_for_thread_id); + void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) { if (tx_chn->psil_paired) { @@ -558,6 +629,9 @@ int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); } + if (!tx_chn->virq) + return -ENXIO; + return tx_chn->virq; } EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); @@ -897,10 +971,13 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, ep_cfg = rx_chn->common.ep_config; - if (xudma_is_pktdma(rx_chn->common.udmax)) + if (xudma_is_pktdma(rx_chn->common.udmax)) { rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id; - else + rx_chn->single_fdq = false; + } else { rx_chn->udma_rchan_id = -1; + rx_chn->single_fdq = true; + } /* request and cfg UDMAP RX channel */ rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, @@ -997,12 +1074,62 @@ err: return ERR_PTR(ret); } +static int +k3_udma_glue_request_remote_rx_chn_common(struct k3_udma_glue_rx_channel *rx_chn, + struct k3_udma_glue_rx_channel_cfg *cfg, + struct device *dev) +{ + int ret, i; + + rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, + rx_chn->common.psdata_size, + rx_chn->common.swdata_size); + + rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, + sizeof(*rx_chn->flows), GFP_KERNEL); + if (!rx_chn->flows) + return -ENOMEM; + + rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; + rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); + dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x-0x%02x", + rx_chn->common.src_thread, rx_chn->flow_id_base); + ret = device_register(&rx_chn->common.chan_dev); + if (ret) { + dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&rx_chn->common.chan_dev); + rx_chn->common.chan_dev.parent = NULL; + return ret; + } + + if (xudma_is_pktdma(rx_chn->common.udmax)) { + /* prepare the channel device as coherent */ + rx_chn->common.chan_dev.dma_coherent = true; + dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, + DMA_BIT_MASK(48)); + rx_chn->single_fdq = false; + } else { + rx_chn->single_fdq = true; + } + + ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); + if (ret) + return ret; + + for (i = 0; i < rx_chn->flow_num; i++) + rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; + + k3_udma_glue_dump_rx_chn(rx_chn); + + return 0; +} + static struct k3_udma_glue_rx_channel * k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, struct k3_udma_glue_rx_channel_cfg *cfg) { struct k3_udma_glue_rx_channel *rx_chn; - int ret, i; + int ret; if (cfg->flow_id_num <= 0 || cfg->flow_id_use_rxchan_id || @@ -1033,44 +1160,55 @@ k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, if (ret) goto err; - rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, - rx_chn->common.psdata_size, - rx_chn->common.swdata_size); - - rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, - sizeof(*rx_chn->flows), GFP_KERNEL); - if (!rx_chn->flows) { - ret = -ENOMEM; + ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); + if (ret) goto err; - } - rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; - rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); - dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x", - rx_chn->common.src_thread); - ret = device_register(&rx_chn->common.chan_dev); - if (ret) { - dev_err(dev, "Channel Device registration failed %d\n", ret); - put_device(&rx_chn->common.chan_dev); - rx_chn->common.chan_dev.parent = NULL; - goto err; - } + return rx_chn; - if (xudma_is_pktdma(rx_chn->common.udmax)) { - /* prepare the channel device as coherent */ - rx_chn->common.chan_dev.dma_coherent = true; - dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, - DMA_BIT_MASK(48)); - } +err: + k3_udma_glue_release_rx_chn(rx_chn); + return ERR_PTR(ret); +} - ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); +struct k3_udma_glue_rx_channel * +k3_udma_glue_request_remote_rx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_rx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id) +{ + struct k3_udma_glue_rx_channel *rx_chn; + int ret; + + if (cfg->flow_id_num <= 0 || + cfg->flow_id_use_rxchan_id || + cfg->def_flow_cfg || + cfg->flow_id_base < 0) + return ERR_PTR(-EINVAL); + + /* + * Remote RX channel is under control of Remote CPU core, so + * Linux can only request and manipulate by dedicated RX flows + */ + + rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); + if (!rx_chn) + return ERR_PTR(-ENOMEM); + + rx_chn->common.dev = dev; + rx_chn->common.swdata_size = cfg->swdata_size; + rx_chn->remote = true; + rx_chn->udma_rchan_id = -1; + rx_chn->flow_num = cfg->flow_id_num; + rx_chn->flow_id_base = cfg->flow_id_base; + rx_chn->psil_paired = false; + + ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, &rx_chn->common, false, thread_id); if (ret) goto err; - for (i = 0; i < rx_chn->flow_num; i++) - rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; - - k3_udma_glue_dump_rx_chn(rx_chn); + ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); + if (ret) + goto err; return rx_chn; @@ -1078,6 +1216,7 @@ err: k3_udma_glue_release_rx_chn(rx_chn); return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(k3_udma_glue_request_remote_rx_chn_for_thread_id); struct k3_udma_glue_rx_channel * k3_udma_glue_request_rx_chn(struct device *dev, const char *name, @@ -1321,7 +1460,7 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, u32 flow_num, void *data, - void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) + void (*cleanup)(void *data, dma_addr_t desc_dma)) { struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; struct device *dev = rx_chn->common.dev; @@ -1333,7 +1472,7 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); /* Skip RX FDQ in case one FDQ is used for the set of flows */ - if (skip_fdq) + if (rx_chn->single_fdq && flow_num) goto do_reset; /* @@ -1396,6 +1535,9 @@ int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); } + if (!flow->virq) + return -ENXIO; + return flow->virq; } EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); @@ -1439,4 +1581,5 @@ static int __init k3_udma_glue_class_init(void) } module_init(k3_udma_glue_class_init); +MODULE_DESCRIPTION("TI K3 NAVSS DMA glue interface"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index 85e00701473c..05228bf00033 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -3,6 +3,8 @@ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> */ +#include <linux/of.h> +#include <linux/of_platform.h> int xudma_navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) { diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index eb4dc5fffe64..aa2dc762140f 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -20,7 +20,6 @@ #include <linux/sys_soc.h> #include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/workqueue.h> #include <linux/completion.h> @@ -1092,8 +1091,11 @@ static void udma_check_tx_completion(struct work_struct *work) u32 residue_diff; ktime_t time_diff; unsigned long delay; + unsigned long flags; while (1) { + spin_lock_irqsave(&uc->vc.lock, flags); + if (uc->desc) { /* Get previous residue and time stamp */ residue_diff = uc->tx_drain.residue; @@ -1128,6 +1130,8 @@ static void udma_check_tx_completion(struct work_struct *work) break; } + spin_unlock_irqrestore(&uc->vc.lock, flags); + usleep_range(ktime_to_us(delay), ktime_to_us(delay) + 10); continue; @@ -1144,6 +1148,8 @@ static void udma_check_tx_completion(struct work_struct *work) break; } + + spin_unlock_irqrestore(&uc->vc.lock, flags); } static irqreturn_t udma_ring_irq_handler(int irq, void *data) @@ -3186,27 +3192,40 @@ static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, d->static_tr.elcnt = elcnt; - /* - * PDMA must to close the packet when the channel is in packet mode. - * For TR mode when the channel is not cyclic we also need PDMA to close - * the packet otherwise the transfer will stall because PDMA holds on - * the data it has received from the peripheral. - */ if (uc->config.pkt_mode || !uc->cyclic) { + /* + * PDMA must close the packet when the channel is in packet mode. + * For TR mode when the channel is not cyclic we also need PDMA + * to close the packet otherwise the transfer will stall because + * PDMA holds on the data it has received from the peripheral. + */ unsigned int div = dev_width * elcnt; if (uc->cyclic) d->static_tr.bstcnt = d->residue / d->sglen / div; else d->static_tr.bstcnt = d->residue / div; + } else if (uc->ud->match_data->type == DMA_TYPE_BCDMA && + uc->config.dir == DMA_DEV_TO_MEM && + uc->cyclic) { + /* + * For cyclic mode with BCDMA we have to set EOP in each TR to + * prevent short packet errors seen on channel teardown. So the + * PDMA must close the packet after every TR transfer by setting + * burst count equal to the number of bytes transferred. + */ + struct cppi5_tr_type1_t *tr_req = d->hwdesc[0].tr_req_base; - if (uc->config.dir == DMA_DEV_TO_MEM && - d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) - return -EINVAL; + d->static_tr.bstcnt = + (tr_req->icnt0 * tr_req->icnt1) / dev_width; } else { d->static_tr.bstcnt = 0; } + if (uc->config.dir == DMA_DEV_TO_MEM && + d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) + return -EINVAL; + return 0; } @@ -3451,8 +3470,9 @@ udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, /* static TR for remote PDMA */ if (udma_configure_statictr(uc, d, dev_width, burst)) { dev_err(uc->ud->dev, - "%s: StaticTR Z is limited to maximum 4095 (%u)\n", - __func__, d->static_tr.bstcnt); + "%s: StaticTR Z is limited to maximum %u (%u)\n", + __func__, uc->ud->match_data->statictr_z_mask, + d->static_tr.bstcnt); udma_free_hwdesc(uc, d); kfree(d); @@ -3477,6 +3497,7 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; unsigned int i; int num_tr; + u32 period_csf = 0; num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); @@ -3499,6 +3520,20 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, period_addr = buf_addr | ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT); + /* + * For BCDMA <-> PDMA transfers, the EOP flag needs to be set on the + * last TR of a descriptor, to mark the packet as complete. + * This is required for getting the teardown completion message in case + * of TX, and to avoid short-packet error in case of RX. + * + * As we are in cyclic mode, we do not know which period might be the + * last one, so set the flag for each period. + */ + if (uc->config.ep_type == PSIL_EP_PDMA_XY && + uc->ud->match_data->type == DMA_TYPE_BCDMA) { + period_csf = CPPI5_TR_CSF_EOP; + } + for (i = 0; i < periods; i++) { int tr_idx = i * num_tr; @@ -3526,8 +3561,10 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, } if (!(flags & DMA_PREP_INTERRUPT)) - cppi5_tr_csf_set(&tr_req[tr_idx].flags, - CPPI5_TR_CSF_SUPR_EVT); + period_csf |= CPPI5_TR_CSF_SUPR_EVT; + + if (period_csf) + cppi5_tr_csf_set(&tr_req[tr_idx].flags, period_csf); period_addr += period_len; } @@ -3656,8 +3693,9 @@ udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, /* static TR for remote PDMA */ if (udma_configure_statictr(uc, d, dev_width, burst)) { dev_err(uc->ud->dev, - "%s: StaticTR Z is limited to maximum 4095 (%u)\n", - __func__, d->static_tr.bstcnt); + "%s: StaticTR Z is limited to maximum %u (%u)\n", + __func__, uc->ud->match_data->statictr_z_mask, + d->static_tr.bstcnt); udma_free_hwdesc(uc, d); kfree(d); @@ -3969,6 +4007,7 @@ static void udma_desc_pre_callback(struct virt_dma_chan *vc, { struct udma_chan *uc = to_udma_chan(&vc->chan); struct udma_desc *d; + u8 status; if (!vd) return; @@ -3978,12 +4017,12 @@ static void udma_desc_pre_callback(struct virt_dma_chan *vc, if (d->metadata_size) udma_fetch_epib(uc, d); - /* Provide residue information for the client */ if (result) { void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); if (cppi5_desc_get_type(desc_vaddr) == CPPI5_INFO0_DESC_TYPE_VAL_HOST) { + /* Provide residue information for the client */ result->residue = d->residue - cppi5_hdesc_get_pktlen(desc_vaddr); if (result->residue) @@ -3992,7 +4031,12 @@ static void udma_desc_pre_callback(struct virt_dma_chan *vc, result->result = DMA_TRANS_NOERROR; } else { result->residue = 0; - result->result = DMA_TRANS_NOERROR; + /* Propagate TR Response errors to the client */ + status = d->hwdesc[0].tr_resp_base->status; + if (status) + result->result = DMA_TRANS_ABORTED; + else + result->result = DMA_TRANS_NOERROR; } } } @@ -4209,7 +4253,6 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct udma_dev *ud = ofdma->of_dma_data; - dma_cap_mask_t mask = ud->ddev.cap_mask; struct udma_filter_param filter_param; struct dma_chan *chan; @@ -4241,7 +4284,7 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, } } - chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, + chan = __dma_request_channel(&ud->ddev.cap_mask, udma_dma_filter_fn, &filter_param, ofdma->of_node); if (!chan) { dev_err(ud->dev, "get channel fail in %s.\n", __func__); @@ -4367,6 +4410,18 @@ static struct udma_match_data j721s2_bcdma_csi_data = { .soc_data = &j721s2_bcdma_csi_soc_data, }; +static struct udma_match_data j722s_bcdma_csi_data = { + .type = DMA_TYPE_BCDMA, + .psil_base = 0x3100, + .enable_memcpy_support = false, + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, + .soc_data = &j721s2_bcdma_csi_soc_data, +}; + static const struct of_device_id udma_of_match[] = { { .compatible = "ti,am654-navss-main-udmap", @@ -4398,8 +4453,13 @@ static const struct of_device_id udma_of_match[] = { .compatible = "ti,j721s2-dmss-bcdma-csi", .data = &j721s2_bcdma_csi_data, }, + { + .compatible = "ti,j722s-dmss-bcdma-csi", + .data = &j722s_bcdma_csi_data, + }, { /* Sentinel */ }, }; +MODULE_DEVICE_TABLE(of, udma_of_match); static struct udma_soc_data am654_soc_data = { .oes = { @@ -4442,6 +4502,8 @@ static const struct soc_device_attribute k3_soc_devices[] = { { .family = "AM62X", .data = &am64_soc_data }, { .family = "AM62AX", .data = &am64_soc_data }, { .family = "J784S4", .data = &j721e_soc_data }, + { .family = "AM62PX", .data = &am64_soc_data }, + { .family = "J722S", .data = &am64_soc_data }, { /* sentinel */ } }; @@ -4465,7 +4527,9 @@ static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); break; case DMA_TYPE_BCDMA: - ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); + ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2) + + BCDMA_CAP3_HBCHAN_CNT(cap3) + + BCDMA_CAP3_UBCHAN_CNT(cap3); ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); ud->rflow_cnt = ud->rchan_cnt; @@ -4828,6 +4892,12 @@ static int bcdma_setup_resources(struct udma_dev *ud) irq_res.desc[i].start = rm_res->desc[i].start + oes->bcdma_bchan_ring; irq_res.desc[i].num = rm_res->desc[i].num; + + if (rm_res->desc[i].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[i].start_sec + + oes->bcdma_bchan_ring; + irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; + } } } } else { @@ -4851,6 +4921,15 @@ static int bcdma_setup_resources(struct udma_dev *ud) irq_res.desc[i + 1].start = rm_res->desc[j].start + oes->bcdma_tchan_ring; irq_res.desc[i + 1].num = rm_res->desc[j].num; + + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_tchan_data; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + irq_res.desc[i + 1].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_tchan_ring; + irq_res.desc[i + 1].num_sec = rm_res->desc[j].num_sec; + } } } } @@ -4871,6 +4950,15 @@ static int bcdma_setup_resources(struct udma_dev *ud) irq_res.desc[i + 1].start = rm_res->desc[j].start + oes->bcdma_rchan_ring; irq_res.desc[i + 1].num = rm_res->desc[j].num; + + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_rchan_data; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + irq_res.desc[i + 1].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_rchan_ring; + irq_res.desc[i + 1].num_sec = rm_res->desc[j].num_sec; + } } } } @@ -5005,6 +5093,12 @@ static int pktdma_setup_resources(struct udma_dev *ud) irq_res.desc[i].start = rm_res->desc[i].start + oes->pktdma_tchan_flow; irq_res.desc[i].num = rm_res->desc[i].num; + + if (rm_res->desc[i].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[i].start_sec + + oes->pktdma_tchan_flow; + irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; + } } } rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; @@ -5016,6 +5110,12 @@ static int pktdma_setup_resources(struct udma_dev *ud) irq_res.desc[i].start = rm_res->desc[j].start + oes->pktdma_rchan_flow; irq_res.desc[i].num = rm_res->desc[j].num; + + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + oes->pktdma_rchan_flow; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + } } } ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); @@ -5524,7 +5624,8 @@ static int udma_probe(struct platform_device *pdev) uc->config.dir = DMA_MEM_TO_MEM; uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", dev_name(dev), i); - + if (!uc->name) + return -ENOMEM; vchan_init(&uc->vc, &ud->ddev); /* Use custom vchan completion handling */ tasklet_setup(&uc->vc.task, udma_vchan_complete); @@ -5614,6 +5715,7 @@ static struct platform_driver udma_driver = { }; module_platform_driver(udma_driver); +MODULE_DESCRIPTION("Texas Instruments UDMA support"); MODULE_LICENSE("GPL v2"); /* Private interfaces to UDMA */ diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index d349c6d482ae..9062a237cd16 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -131,7 +131,6 @@ int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread, struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property); struct device *xudma_get_device(struct udma_dev *ud); struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud); -void xudma_dev_put(struct udma_dev *ud); u32 xudma_dev_get_psil_base(struct udma_dev *ud); struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud); diff --git a/drivers/dma/ti/omap-dma.c b/drivers/dma/ti/omap-dma.c index 02e1c08c596d..8c023c6e623a 100644 --- a/drivers/dma/ti/omap-dma.c +++ b/drivers/dma/ti/omap-dma.c @@ -16,8 +16,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> #include "../virt-dma.h" @@ -124,7 +124,7 @@ struct omap_desc { uint32_t csdp; /* CSDP value */ unsigned sglen; - struct omap_sg sg[]; + struct omap_sg sg[] __counted_by(sglen); }; enum { @@ -1005,6 +1005,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC); if (!d) return NULL; + d->sglen = sglen; d->dir = dir; d->dev_addr = dev_addr; @@ -1120,8 +1121,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( } } - d->sglen = sglen; - /* Release the dma_pool entries if one allocation failed */ if (ll_failed) { for (i = 0; i < d->sglen; i++) { @@ -1187,10 +1186,10 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->dev_addr = dev_addr; d->fi = burst; d->es = es; + d->sglen = 1; d->sg[0].addr = buf_addr; d->sg[0].en = period_len / es_bytes[es]; d->sg[0].fn = buf_len / period_len; - d->sglen = 1; d->ccr = c->ccr; if (dir == DMA_DEV_TO_MEM) @@ -1259,10 +1258,10 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy( d->dev_addr = src; d->fi = 0; d->es = data_type; + d->sglen = 1; d->sg[0].en = len / BIT(data_type); d->sg[0].fn = 1; d->sg[0].addr = dest; - d->sglen = 1; d->ccr = c->ccr; d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC; @@ -1310,6 +1309,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved( if (data_type > CSDP_DATA_TYPE_32) data_type = CSDP_DATA_TYPE_32; + d->sglen = 1; sg = &d->sg[0]; d->dir = DMA_MEM_TO_MEM; d->dev_addr = xt->src_start; @@ -1317,7 +1317,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved( sg->en = xt->sgl[0].size / BIT(data_type); sg->fn = xt->numf; sg->addr = xt->dst_start; - d->sglen = 1; d->ccr = c->ccr; src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); @@ -1844,7 +1843,7 @@ static int omap_dma_probe(struct platform_device *pdev) return rc; } -static int omap_dma_remove(struct platform_device *pdev) +static void omap_dma_remove(struct platform_device *pdev) { struct omap_dmadev *od = platform_get_drvdata(pdev); int irq; @@ -1869,8 +1868,6 @@ static int omap_dma_remove(struct platform_device *pdev) dma_pool_destroy(od->desc_pool); omap_dma_free(od); - - return 0; } static const struct omap_dma_config omap2420_data = { @@ -1918,7 +1915,7 @@ MODULE_DEVICE_TABLE(of, omap_dma_match); static struct platform_driver omap_dma_driver = { .probe = omap_dma_probe, - .remove = omap_dma_remove, + .remove = omap_dma_remove, .driver = { .name = "omap-dma-engine", .of_match_table = omap_dma_match, @@ -1953,4 +1950,5 @@ static void __exit omap_dma_exit(void) module_exit(omap_dma_exit); MODULE_AUTHOR("Russell King"); +MODULE_DESCRIPTION("Texas Instruments sDMA DMAengine support"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index 3f524be69efb..ecaf002558af 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -740,7 +740,7 @@ err_release_region: } -static int td_remove(struct platform_device *pdev) +static void td_remove(struct platform_device *pdev) { struct timb_dma *td = platform_get_drvdata(pdev); struct resource *iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -754,7 +754,6 @@ static int td_remove(struct platform_device *pdev) release_mem_region(iomem->start, resource_size(iomem)); dev_dbg(&pdev->dev, "Removed...\n"); - return 0; } static struct platform_driver td_driver = { @@ -762,7 +761,7 @@ static struct platform_driver td_driver = { .name = DRIVER_NAME, }, .probe = td_probe, - .remove = td_remove, + .remove = td_remove, }; module_platform_driver(td_driver); diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 5b6b375a257e..35d5221683b2 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -1151,7 +1151,7 @@ static int __init txx9dmac_chan_probe(struct platform_device *pdev) return 0; } -static int txx9dmac_chan_remove(struct platform_device *pdev) +static void txx9dmac_chan_remove(struct platform_device *pdev) { struct txx9dmac_chan *dc = platform_get_drvdata(pdev); @@ -1162,7 +1162,6 @@ static int txx9dmac_chan_remove(struct platform_device *pdev) tasklet_kill(&dc->tasklet); } dc->ddev->chan[pdev->id % TXX9_DMA_MAX_NR_CHANNELS] = NULL; - return 0; } static int __init txx9dmac_probe(struct platform_device *pdev) @@ -1215,7 +1214,7 @@ static int __init txx9dmac_probe(struct platform_device *pdev) return 0; } -static int txx9dmac_remove(struct platform_device *pdev) +static void txx9dmac_remove(struct platform_device *pdev) { struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); @@ -1224,7 +1223,6 @@ static int txx9dmac_remove(struct platform_device *pdev) devm_free_irq(&pdev->dev, ddev->irq, ddev); tasklet_kill(&ddev->tasklet); } - return 0; } static void txx9dmac_shutdown(struct platform_device *pdev) diff --git a/drivers/dma/uniphier-mdmac.c b/drivers/dma/uniphier-mdmac.c index 618839df0748..7a99f86ecb5a 100644 --- a/drivers/dma/uniphier-mdmac.c +++ b/drivers/dma/uniphier-mdmac.c @@ -453,7 +453,7 @@ disable_clk: return ret; } -static int uniphier_mdmac_remove(struct platform_device *pdev) +static void uniphier_mdmac_remove(struct platform_device *pdev) { struct uniphier_mdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -468,16 +468,21 @@ static int uniphier_mdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } uniphier_mdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&mdev->ddev); clk_disable_unprepare(mdev->clk); - - return 0; } static const struct of_device_id uniphier_mdmac_match[] = { diff --git a/drivers/dma/uniphier-xdmac.c b/drivers/dma/uniphier-xdmac.c index 290836b7e1be..ceeb6171c9d1 100644 --- a/drivers/dma/uniphier-xdmac.c +++ b/drivers/dma/uniphier-xdmac.c @@ -80,7 +80,7 @@ struct uniphier_xdmac_desc { unsigned int nr_node; unsigned int cur_node; enum dma_transfer_direction dir; - struct uniphier_xdmac_desc_node nodes[]; + struct uniphier_xdmac_desc_node nodes[] __counted_by(nr_node); }; struct uniphier_xdmac_chan { @@ -97,7 +97,7 @@ struct uniphier_xdmac_device { struct dma_device ddev; void __iomem *reg_base; int nr_chans; - struct uniphier_xdmac_chan channels[]; + struct uniphier_xdmac_chan channels[] __counted_by(nr_chans); }; static struct uniphier_xdmac_chan * @@ -295,6 +295,7 @@ uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, xd = kzalloc(struct_size(xd, nodes, nr), GFP_NOWAIT); if (!xd) return NULL; + xd->nr_node = nr; for (i = 0; i < nr; i++) { burst_size = min_t(size_t, len, XDMAC_MAX_WORD_SIZE); @@ -309,7 +310,6 @@ uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, } xd->dir = DMA_MEM_TO_MEM; - xd->nr_node = nr; xd->cur_node = 0; return vchan_tx_prep(vc, &xd->vd, flags); @@ -351,6 +351,7 @@ uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, xd = kzalloc(struct_size(xd, nodes, sg_len), GFP_NOWAIT); if (!xd) return NULL; + xd->nr_node = sg_len; for_each_sg(sgl, sg, sg_len, i) { xd->nodes[i].src = (direction == DMA_DEV_TO_MEM) @@ -385,7 +386,6 @@ uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, } xd->dir = direction; - xd->nr_node = sg_len; xd->cur_node = 0; return vchan_tx_prep(vc, &xd->vd, flags); @@ -563,7 +563,7 @@ out_unregister_dmac: return ret; } -static int uniphier_xdmac_remove(struct platform_device *pdev) +static void uniphier_xdmac_remove(struct platform_device *pdev) { struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev); struct dma_device *ddev = &xdev->ddev; @@ -579,15 +579,20 @@ static int uniphier_xdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &ddev->channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *xdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } uniphier_xdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(ddev); - - return 0; } static const struct of_device_id uniphier_xdmac_match[] = { diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c index a6f4265be0c9..7961172a780d 100644 --- a/drivers/dma/virt-dma.c +++ b/drivers/dma/virt-dma.c @@ -139,4 +139,5 @@ void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev) EXPORT_SYMBOL_GPL(vchan_init); MODULE_AUTHOR("Russell King"); +MODULE_DESCRIPTION("Virtual DMA channel support for DMAengine"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index e9f5250fbe4d..59d9eabc8b67 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -81,6 +81,8 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan */ static inline bool vchan_issue_pending(struct virt_dma_chan *vc) { + lockdep_assert_held(&vc->lock); + list_splice_tail_init(&vc->desc_submitted, &vc->desc_issued); return !list_empty(&vc->desc_issued); } @@ -96,6 +98,8 @@ static inline void vchan_cookie_complete(struct virt_dma_desc *vd) struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); dma_cookie_t cookie; + lockdep_assert_held(&vc->lock); + cookie = vd->tx.cookie; dma_cookie_complete(&vd->tx); dev_vdbg(vc->chan.device->dev, "txd %p[%x]: marked complete\n", @@ -146,6 +150,8 @@ static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd) { struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); + lockdep_assert_held(&vc->lock); + list_add_tail(&vd->node, &vc->desc_terminated); if (vc->cyclic == vd) @@ -160,6 +166,8 @@ static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd) */ static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) { + lockdep_assert_held(&vc->lock); + return list_first_entry_or_null(&vc->desc_issued, struct virt_dma_desc, node); } @@ -177,6 +185,8 @@ static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) static inline void vchan_get_all_descriptors(struct virt_dma_chan *vc, struct list_head *head) { + lockdep_assert_held(&vc->lock); + list_splice_tail_init(&vc->desc_allocated, head); list_splice_tail_init(&vc->desc_submitted, head); list_splice_tail_init(&vc->desc_issued, head); diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c index 3589b4ef50b8..f64624ea44ad 100644 --- a/drivers/dma/xgene-dma.c +++ b/drivers/dma/xgene-dma.c @@ -18,8 +18,9 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include "dmaengine.h" @@ -1741,7 +1742,7 @@ static int xgene_dma_probe(struct platform_device *pdev) /* Initialize DMA channels software state */ xgene_dma_init_channels(pdma); - /* Configue DMA rings */ + /* Configure DMA rings */ ret = xgene_dma_init_rings(pdma); if (ret) goto err_clk_enable; @@ -1775,7 +1776,7 @@ err_clk_enable: return ret; } -static int xgene_dma_remove(struct platform_device *pdev) +static void xgene_dma_remove(struct platform_device *pdev) { struct xgene_dma *pdma = platform_get_drvdata(pdev); struct xgene_dma_chan *chan; @@ -1796,8 +1797,6 @@ static int xgene_dma_remove(struct platform_device *pdev) if (!IS_ERR(pdma->clk)) clk_disable_unprepare(pdma->clk); - - return 0; } #ifdef CONFIG_ACPI diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h index dd98b4526b90..6ad08878e938 100644 --- a/drivers/dma/xilinx/xdma-regs.h +++ b/drivers/dma/xilinx/xdma-regs.h @@ -44,6 +44,8 @@ FIELD_PREP(XDMA_DESC_FLAGS_BITS, (flag))) #define XDMA_DESC_CONTROL_LAST \ XDMA_DESC_CONTROL(1, XDMA_DESC_STOPPED | XDMA_DESC_COMPLETED) +#define XDMA_DESC_CONTROL_CYCLIC \ + XDMA_DESC_CONTROL(1, XDMA_DESC_COMPLETED) /* * Descriptor for a single contiguous memory block transfer. @@ -62,9 +64,10 @@ struct xdma_hw_desc { __le64 next_desc; }; -#define XDMA_DESC_SIZE sizeof(struct xdma_hw_desc) -#define XDMA_DESC_BLOCK_SIZE (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT) -#define XDMA_DESC_BLOCK_ALIGN 4096 +#define XDMA_DESC_SIZE sizeof(struct xdma_hw_desc) +#define XDMA_DESC_BLOCK_SIZE (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT) +#define XDMA_DESC_BLOCK_ALIGN 32 +#define XDMA_DESC_BLOCK_BOUNDARY 4096 /* * Channel registers @@ -74,6 +77,7 @@ struct xdma_hw_desc { #define XDMA_CHAN_CONTROL_W1S 0x8 #define XDMA_CHAN_CONTROL_W1C 0xc #define XDMA_CHAN_STATUS 0x40 +#define XDMA_CHAN_STATUS_RC 0x44 #define XDMA_CHAN_COMPLETED_DESC 0x48 #define XDMA_CHAN_ALIGNMENTS 0x4c #define XDMA_CHAN_INTR_ENABLE 0x90 @@ -99,6 +103,7 @@ struct xdma_hw_desc { #define CHAN_CTRL_IE_MAGIC_STOPPED BIT(4) #define CHAN_CTRL_IE_IDLE_STOPPED BIT(6) #define CHAN_CTRL_IE_READ_ERROR GENMASK(13, 9) +#define CHAN_CTRL_IE_WRITE_ERROR GENMASK(18, 14) #define CHAN_CTRL_IE_DESC_ERROR GENMASK(23, 19) #define CHAN_CTRL_NON_INCR_ADDR BIT(25) #define CHAN_CTRL_POLL_MODE_WB BIT(26) @@ -109,8 +114,20 @@ struct xdma_hw_desc { CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ CHAN_CTRL_IE_MAGIC_STOPPED | \ CHAN_CTRL_IE_READ_ERROR | \ + CHAN_CTRL_IE_WRITE_ERROR | \ CHAN_CTRL_IE_DESC_ERROR) +/* bits of the channel status register */ +#define XDMA_CHAN_STATUS_BUSY BIT(0) + +#define XDMA_CHAN_STATUS_MASK CHAN_CTRL_START + +#define XDMA_CHAN_ERROR_MASK (CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ + CHAN_CTRL_IE_MAGIC_STOPPED | \ + CHAN_CTRL_IE_READ_ERROR | \ + CHAN_CTRL_IE_WRITE_ERROR | \ + CHAN_CTRL_IE_DESC_ERROR) + /* bits of the channel interrupt enable mask */ #define CHAN_IM_DESC_ERROR BIT(19) #define CHAN_IM_READ_ERROR BIT(9) @@ -132,18 +149,6 @@ struct xdma_hw_desc { #define XDMA_SGDMA_DESC_ADJ 0x4088 #define XDMA_SGDMA_DESC_CREDIT 0x408c -/* bits of the SG DMA control register */ -#define XDMA_CTRL_RUN_STOP BIT(0) -#define XDMA_CTRL_IE_DESC_STOPPED BIT(1) -#define XDMA_CTRL_IE_DESC_COMPLETED BIT(2) -#define XDMA_CTRL_IE_DESC_ALIGN_MISMATCH BIT(3) -#define XDMA_CTRL_IE_MAGIC_STOPPED BIT(4) -#define XDMA_CTRL_IE_IDLE_STOPPED BIT(6) -#define XDMA_CTRL_IE_READ_ERROR GENMASK(13, 9) -#define XDMA_CTRL_IE_DESC_ERROR GENMASK(23, 19) -#define XDMA_CTRL_NON_INCR_ADDR BIT(25) -#define XDMA_CTRL_POLL_MODE_WB BIT(26) - /* * interrupt registers */ diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c index e0bfd129d563..0d88b1a670e1 100644 --- a/drivers/dma/xilinx/xdma.c +++ b/drivers/dma/xilinx/xdma.c @@ -71,6 +71,8 @@ struct xdma_chan { enum dma_transfer_direction dir; struct dma_slave_config cfg; u32 irq; + struct completion last_interrupt; + bool stop_requested; }; /** @@ -78,21 +80,31 @@ struct xdma_chan { * @vdesc: Virtual DMA descriptor * @chan: DMA channel pointer * @dir: Transferring direction of the request - * @dev_addr: Physical address on DMA device side * @desc_blocks: Hardware descriptor blocks * @dblk_num: Number of hardware descriptor blocks * @desc_num: Number of hardware descriptors * @completed_desc_num: Completed hardware descriptors + * @cyclic: Cyclic transfer vs. scatter-gather + * @interleaved_dma: Interleaved DMA transfer + * @periods: Number of periods in the cyclic transfer + * @period_size: Size of a period in bytes in cyclic transfers + * @frames_left: Number of frames left in interleaved DMA transfer + * @error: tx error flag */ struct xdma_desc { struct virt_dma_desc vdesc; struct xdma_chan *chan; enum dma_transfer_direction dir; - u64 dev_addr; struct xdma_desc_block *desc_blocks; u32 dblk_num; u32 desc_num; u32 completed_desc_num; + bool cyclic; + bool interleaved_dma; + u32 periods; + u32 period_size; + u32 frames_left; + bool error; }; #define XDMA_DEV_STATUS_REG_DMA BIT(0) @@ -137,10 +149,10 @@ static inline void *xdma_blk_last_desc(struct xdma_desc_block *block) } /** - * xdma_link_desc_blocks - Link descriptor blocks for DMA transfer + * xdma_link_sg_desc_blocks - Link SG descriptor blocks for DMA transfer * @sw_desc: Tx descriptor pointer */ -static void xdma_link_desc_blocks(struct xdma_desc *sw_desc) +static void xdma_link_sg_desc_blocks(struct xdma_desc *sw_desc) { struct xdma_desc_block *block; u32 last_blk_desc, desc_control; @@ -174,6 +186,25 @@ static void xdma_link_desc_blocks(struct xdma_desc *sw_desc) desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST); } +/** + * xdma_link_cyclic_desc_blocks - Link cyclic descriptor blocks for DMA transfer + * @sw_desc: Tx descriptor pointer + */ +static void xdma_link_cyclic_desc_blocks(struct xdma_desc *sw_desc) +{ + struct xdma_desc_block *block; + struct xdma_hw_desc *desc; + int i; + + block = sw_desc->desc_blocks; + for (i = 0; i < sw_desc->desc_num - 1; i++) { + desc = block->virt_addr + i * XDMA_DESC_SIZE; + desc->next_desc = cpu_to_le64(block->dma_addr + ((i + 1) * XDMA_DESC_SIZE)); + } + desc = block->virt_addr + i * XDMA_DESC_SIZE; + desc->next_desc = cpu_to_le64(block->dma_addr); +} + static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan) { return container_of(chan, struct xdma_chan, vchan.chan); @@ -231,14 +262,16 @@ static void xdma_free_desc(struct virt_dma_desc *vdesc) * xdma_alloc_desc - Allocate descriptor * @chan: DMA channel pointer * @desc_num: Number of hardware descriptors + * @cyclic: Whether this is a cyclic transfer */ static struct xdma_desc * -xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num) +xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num, bool cyclic) { struct xdma_desc *sw_desc; struct xdma_hw_desc *desc; dma_addr_t dma_addr; u32 dblk_num; + u32 control; void *addr; int i, j; @@ -248,12 +281,19 @@ xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num) sw_desc->chan = chan; sw_desc->desc_num = desc_num; + sw_desc->cyclic = cyclic; + sw_desc->error = false; dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT); sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks), GFP_NOWAIT); if (!sw_desc->desc_blocks) goto failed; + if (cyclic) + control = XDMA_DESC_CONTROL_CYCLIC; + else + control = XDMA_DESC_CONTROL(1, 0); + sw_desc->dblk_num = dblk_num; for (i = 0; i < sw_desc->dblk_num; i++) { addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr); @@ -263,10 +303,13 @@ xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num) sw_desc->desc_blocks[i].virt_addr = addr; sw_desc->desc_blocks[i].dma_addr = dma_addr; for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++) - desc[j].control = cpu_to_le32(XDMA_DESC_CONTROL(1, 0)); + desc[j].control = cpu_to_le32(control); } - xdma_link_desc_blocks(sw_desc); + if (cyclic) + xdma_link_cyclic_desc_blocks(sw_desc); + else + xdma_link_sg_desc_blocks(sw_desc); return sw_desc; @@ -335,10 +378,26 @@ static int xdma_xfer_start(struct xdma_chan *xchan) return ret; xchan->busy = true; + xchan->stop_requested = false; + reinit_completion(&xchan->last_interrupt); + return 0; } /** + * xdma_xfer_stop - Stop DMA transfer + * @xchan: DMA channel pointer + */ +static int xdma_xfer_stop(struct xdma_chan *xchan) +{ + struct xdma_device *xdev = xchan->xdev_hdl; + + /* clear run stop bit to prevent any further auto-triggering */ + return regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, + CHAN_CTRL_RUN_STOP); +} + +/** * xdma_alloc_channels - Detect and allocate DMA channels * @xdev: DMA device pointer * @dir: Channel direction @@ -408,6 +467,8 @@ static int xdma_alloc_channels(struct xdma_device *xdev, xchan->xdev_hdl = xdev; xchan->base = base + i * XDMA_CHAN_STRIDE; xchan->dir = dir; + xchan->stop_requested = false; + init_completion(&xchan->last_interrupt); ret = xdma_channel_init(xchan); if (ret) @@ -440,6 +501,94 @@ static void xdma_issue_pending(struct dma_chan *chan) } /** + * xdma_terminate_all - Terminate all transactions + * @chan: DMA channel pointer + */ +static int xdma_terminate_all(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct virt_dma_desc *vd; + unsigned long flags; + LIST_HEAD(head); + + xdma_xfer_stop(xdma_chan); + + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); + + xdma_chan->busy = false; + xdma_chan->stop_requested = true; + vd = vchan_next_desc(&xdma_chan->vchan); + if (vd) { + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); + } + vchan_get_all_descriptors(&xdma_chan->vchan, &head); + list_splice_tail(&head, &xdma_chan->vchan.desc_terminated); + + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); + + return 0; +} + +/** + * xdma_synchronize - Synchronize terminated transactions + * @chan: DMA channel pointer + */ +static void xdma_synchronize(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_device *xdev = xdma_chan->xdev_hdl; + int st = 0; + + /* If the engine continues running, wait for the last interrupt */ + regmap_read(xdev->rmap, xdma_chan->base + XDMA_CHAN_STATUS, &st); + if (st & XDMA_CHAN_STATUS_BUSY) + wait_for_completion_timeout(&xdma_chan->last_interrupt, msecs_to_jiffies(1000)); + + vchan_synchronize(&xdma_chan->vchan); +} + +/** + * xdma_fill_descs() - Fill hardware descriptors for one contiguous memory chunk. + * More than one descriptor will be used if the size is bigger + * than XDMA_DESC_BLEN_MAX. + * @sw_desc: Descriptor container + * @src_addr: First value for the ->src_addr field + * @dst_addr: First value for the ->dst_addr field + * @size: Size of the contiguous memory block + * @filled_descs_num: Index of the first descriptor to take care of in @sw_desc + */ +static inline u32 xdma_fill_descs(struct xdma_desc *sw_desc, u64 src_addr, + u64 dst_addr, u32 size, u32 filled_descs_num) +{ + u32 left = size, len, desc_num = filled_descs_num; + struct xdma_desc_block *dblk; + struct xdma_hw_desc *desc; + + dblk = sw_desc->desc_blocks + (desc_num / XDMA_DESC_ADJACENT); + desc = dblk->virt_addr; + desc += desc_num & XDMA_DESC_ADJACENT_MASK; + do { + len = min_t(u32, left, XDMA_DESC_BLEN_MAX); + /* set hardware descriptor */ + desc->bytes = cpu_to_le32(len); + desc->src_addr = cpu_to_le64(src_addr); + desc->dst_addr = cpu_to_le64(dst_addr); + if (!(++desc_num & XDMA_DESC_ADJACENT_MASK)) + desc = (++dblk)->virt_addr; + else + desc++; + + src_addr += len; + dst_addr += len; + left -= len; + } while (left); + + return desc_num - filled_descs_num; +} + +/** * xdma_prep_device_sg - prepare a descriptor for a DMA transaction * @chan: DMA channel pointer * @sgl: Transfer scatter gather list @@ -455,21 +604,20 @@ xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, { struct xdma_chan *xdma_chan = to_xdma_chan(chan); struct dma_async_tx_descriptor *tx_desc; - u32 desc_num = 0, i, len, rest; - struct xdma_desc_block *dblk; - struct xdma_hw_desc *desc; struct xdma_desc *sw_desc; - u64 dev_addr, *src, *dst; + u32 desc_num = 0, i; + u64 addr, dev_addr, *src, *dst; struct scatterlist *sg; - u64 addr; for_each_sg(sgl, sg, sg_len, i) desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX); - sw_desc = xdma_alloc_desc(xdma_chan, desc_num); + sw_desc = xdma_alloc_desc(xdma_chan, desc_num, false); if (!sw_desc) return NULL; sw_desc->dir = dir; + sw_desc->cyclic = false; + sw_desc->interleaved_dma = false; if (dir == DMA_MEM_TO_DEV) { dev_addr = xdma_chan->cfg.dst_addr; @@ -481,32 +629,87 @@ xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, dst = &addr; } - dblk = sw_desc->desc_blocks; - desc = dblk->virt_addr; - desc_num = 1; + desc_num = 0; for_each_sg(sgl, sg, sg_len, i) { addr = sg_dma_address(sg); - rest = sg_dma_len(sg); + desc_num += xdma_fill_descs(sw_desc, *src, *dst, sg_dma_len(sg), desc_num); + dev_addr += sg_dma_len(sg); + } + + tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); + if (!tx_desc) + goto failed; + + return tx_desc; + +failed: + xdma_free_desc(&sw_desc->vdesc); + + return NULL; +} + +/** + * xdma_prep_dma_cyclic - prepare for cyclic DMA transactions + * @chan: DMA channel pointer + * @address: Device DMA address to access + * @size: Total length to transfer + * @period_size: Period size to use for each transfer + * @dir: Transfer direction + * @flags: Transfer ack flags + */ +static struct dma_async_tx_descriptor * +xdma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t address, + size_t size, size_t period_size, + enum dma_transfer_direction dir, + unsigned long flags) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_device *xdev = xdma_chan->xdev_hdl; + unsigned int periods = size / period_size; + struct dma_async_tx_descriptor *tx_desc; + struct xdma_desc *sw_desc; + u64 addr, dev_addr, *src, *dst; + u32 desc_num; + unsigned int i; - do { - len = min_t(u32, rest, XDMA_DESC_BLEN_MAX); - /* set hardware descriptor */ - desc->bytes = cpu_to_le32(len); - desc->src_addr = cpu_to_le64(*src); - desc->dst_addr = cpu_to_le64(*dst); + /* + * Simplify the whole logic by preventing an abnormally high number of + * periods and periods size. + */ + if (period_size > XDMA_DESC_BLEN_MAX) { + xdma_err(xdev, "period size limited to %lu bytes\n", XDMA_DESC_BLEN_MAX); + return NULL; + } + + if (periods > XDMA_DESC_ADJACENT) { + xdma_err(xdev, "number of periods limited to %u\n", XDMA_DESC_ADJACENT); + return NULL; + } + + sw_desc = xdma_alloc_desc(xdma_chan, periods, true); + if (!sw_desc) + return NULL; - if (!(desc_num & XDMA_DESC_ADJACENT_MASK)) { - dblk++; - desc = dblk->virt_addr; - } else { - desc++; - } + sw_desc->periods = periods; + sw_desc->period_size = period_size; + sw_desc->dir = dir; + sw_desc->interleaved_dma = false; - desc_num++; - dev_addr += len; - addr += len; - rest -= len; - } while (rest); + addr = address; + if (dir == DMA_MEM_TO_DEV) { + dev_addr = xdma_chan->cfg.dst_addr; + src = &addr; + dst = &dev_addr; + } else { + dev_addr = xdma_chan->cfg.src_addr; + src = &dev_addr; + dst = &addr; + } + + desc_num = 0; + for (i = 0; i < periods; i++) { + desc_num += xdma_fill_descs(sw_desc, *src, *dst, period_size, desc_num); + addr += period_size; } tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); @@ -522,6 +725,57 @@ failed: } /** + * xdma_prep_interleaved_dma - Prepare virtual descriptor for interleaved DMA transfers + * @chan: DMA channel + * @xt: DMA transfer template + * @flags: tx flags + */ +static struct dma_async_tx_descriptor * +xdma_prep_interleaved_dma(struct dma_chan *chan, + struct dma_interleaved_template *xt, + unsigned long flags) +{ + int i; + u32 desc_num = 0, period_size = 0; + struct dma_async_tx_descriptor *tx_desc; + struct xdma_chan *xchan = to_xdma_chan(chan); + struct xdma_desc *sw_desc; + u64 src_addr, dst_addr; + + for (i = 0; i < xt->frame_size; ++i) + desc_num += DIV_ROUND_UP(xt->sgl[i].size, XDMA_DESC_BLEN_MAX); + + sw_desc = xdma_alloc_desc(xchan, desc_num, false); + if (!sw_desc) + return NULL; + sw_desc->dir = xt->dir; + sw_desc->interleaved_dma = true; + sw_desc->cyclic = flags & DMA_PREP_REPEAT; + sw_desc->frames_left = xt->numf; + sw_desc->periods = xt->numf; + + desc_num = 0; + src_addr = xt->src_start; + dst_addr = xt->dst_start; + for (i = 0; i < xt->frame_size; ++i) { + desc_num += xdma_fill_descs(sw_desc, src_addr, dst_addr, xt->sgl[i].size, desc_num); + src_addr += dmaengine_get_src_icg(xt, &xt->sgl[i]) + (xt->src_inc ? + xt->sgl[i].size : 0); + dst_addr += dmaengine_get_dst_icg(xt, &xt->sgl[i]) + (xt->dst_inc ? + xt->sgl[i].size : 0); + period_size += xt->sgl[i].size; + } + sw_desc->period_size = period_size; + + tx_desc = vchan_tx_prep(&xchan->vchan, &sw_desc->vdesc, flags); + if (tx_desc) + return tx_desc; + + xdma_free_desc(&sw_desc->vdesc); + return NULL; +} + +/** * xdma_device_config - Configure the DMA channel * @chan: DMA channel * @cfg: channel configuration @@ -566,9 +820,8 @@ static int xdma_alloc_chan_resources(struct dma_chan *chan) return -EINVAL; } - xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), - dev, XDMA_DESC_BLOCK_SIZE, - XDMA_DESC_BLOCK_ALIGN, 0); + xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), dev, XDMA_DESC_BLOCK_SIZE, + XDMA_DESC_BLOCK_ALIGN, XDMA_DESC_BLOCK_BOUNDARY); if (!xdma_chan->desc_pool) { xdma_err(xdev, "unable to allocate descriptor pool"); return -ENOMEM; @@ -577,6 +830,41 @@ static int xdma_alloc_chan_resources(struct dma_chan *chan) return 0; } +static enum dma_status xdma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_desc *desc = NULL; + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; + unsigned int period_idx; + u32 residue = 0; + + ret = dma_cookie_status(chan, cookie, state); + if (ret == DMA_COMPLETE) + return ret; + + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); + + vd = vchan_find_desc(&xdma_chan->vchan, cookie); + if (!vd) + goto out; + + desc = to_xdma_desc(vd); + if (desc->error) { + ret = DMA_ERROR; + } else if (desc->cyclic) { + period_idx = desc->completed_desc_num % desc->periods; + residue = (desc->periods - period_idx) * desc->period_size; + dma_set_residue(state, residue); + } +out: + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); + + return ret; +} + /** * xdma_channel_isr - XDMA channel interrupt handler * @irq: IRQ number @@ -586,43 +874,92 @@ static irqreturn_t xdma_channel_isr(int irq, void *dev_id) { struct xdma_chan *xchan = dev_id; u32 complete_desc_num = 0; - struct xdma_device *xdev; - struct virt_dma_desc *vd; + struct xdma_device *xdev = xchan->xdev_hdl; + struct virt_dma_desc *vd, *next_vd; struct xdma_desc *desc; int ret; + u32 st; + bool repeat_tx; spin_lock(&xchan->vchan.lock); + if (xchan->stop_requested) + complete(&xchan->last_interrupt); + /* get submitted request */ vd = vchan_next_desc(&xchan->vchan); if (!vd) goto out; - xchan->busy = false; + /* Clear-on-read the status register */ + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &st); + if (ret) + goto out; + desc = to_xdma_desc(vd); - xdev = xchan->xdev_hdl; + + st &= XDMA_CHAN_STATUS_MASK; + if ((st & XDMA_CHAN_ERROR_MASK) || + !(st & (CHAN_CTRL_IE_DESC_COMPLETED | CHAN_CTRL_IE_DESC_STOPPED))) { + desc->error = true; + xdma_err(xdev, "channel error, status register value: 0x%x", st); + goto out; + } ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_COMPLETED_DESC, &complete_desc_num); if (ret) goto out; - desc->completed_desc_num += complete_desc_num; - /* - * if all data blocks are transferred, remove and complete the request - */ - if (desc->completed_desc_num == desc->desc_num) { - list_del(&vd->node); - vchan_cookie_complete(vd); - goto out; - } + if (desc->interleaved_dma) { + xchan->busy = false; + desc->completed_desc_num += complete_desc_num; + if (complete_desc_num == XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) { + xdma_xfer_start(xchan); + goto out; + } - if (desc->completed_desc_num > desc->desc_num || - complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) - goto out; + /* last desc of any frame */ + desc->frames_left--; + if (desc->frames_left) + goto out; + + /* last desc of the last frame */ + repeat_tx = vd->tx.flags & DMA_PREP_REPEAT; + next_vd = list_first_entry_or_null(&vd->node, struct virt_dma_desc, node); + if (next_vd) + repeat_tx = repeat_tx && !(next_vd->tx.flags & DMA_PREP_LOAD_EOT); + if (repeat_tx) { + desc->frames_left = desc->periods; + desc->completed_desc_num = 0; + vchan_cyclic_callback(vd); + } else { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + /* start (or continue) the tx of a first desc on the vc.desc_issued list, if any */ + xdma_xfer_start(xchan); + } else if (!desc->cyclic) { + xchan->busy = false; + desc->completed_desc_num += complete_desc_num; + + /* if all data blocks are transferred, remove and complete the request */ + if (desc->completed_desc_num == desc->desc_num) { + list_del(&vd->node); + vchan_cookie_complete(vd); + goto out; + } - /* transfer the rest of data */ - xdma_xfer_start(xchan); + if (desc->completed_desc_num > desc->desc_num || + complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) + goto out; + + /* transfer the rest of data */ + xdma_xfer_start(xchan); + } else { + desc->completed_desc_num = complete_desc_num; + vchan_cyclic_callback(vd); + } out: spin_unlock(&xchan->vchan.lock); @@ -841,7 +1178,7 @@ EXPORT_SYMBOL(xdma_get_user_irq); * xdma_remove - Driver remove function * @pdev: Pointer to the platform_device structure */ -static int xdma_remove(struct platform_device *pdev) +static void xdma_remove(struct platform_device *pdev) { struct xdma_device *xdev = platform_get_drvdata(pdev); @@ -850,8 +1187,6 @@ static int xdma_remove(struct platform_device *pdev) if (xdev->status & XDMA_DEV_STATUS_REG_DMA) dma_async_device_unregister(&xdev->dma_dev); - - return 0; } /** @@ -885,7 +1220,7 @@ static int xdma_probe(struct platform_device *pdev) goto failed; } xdev->irq_start = res->start; - xdev->irq_num = res->end - res->start + 1; + xdev->irq_num = resource_size(res); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!res) { @@ -921,17 +1256,26 @@ static int xdma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask); dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_INTERLEAVE, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_REPEAT, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_LOAD_EOT, xdev->dma_dev.cap_mask); xdev->dma_dev.dev = &pdev->dev; + xdev->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; xdev->dma_dev.device_free_chan_resources = xdma_free_chan_resources; xdev->dma_dev.device_alloc_chan_resources = xdma_alloc_chan_resources; - xdev->dma_dev.device_tx_status = dma_cookie_status; + xdev->dma_dev.device_tx_status = xdma_tx_status; xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg; xdev->dma_dev.device_config = xdma_device_config; xdev->dma_dev.device_issue_pending = xdma_issue_pending; + xdev->dma_dev.device_terminate_all = xdma_terminate_all; + xdev->dma_dev.device_synchronize = xdma_synchronize; xdev->dma_dev.filter.map = pdata->device_map; xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt; xdev->dma_dev.filter.fn = xdma_filter_fn; + xdev->dma_dev.device_prep_dma_cyclic = xdma_prep_dma_cyclic; + xdev->dma_dev.device_prep_interleaved_dma = xdma_prep_interleaved_dma; ret = dma_async_device_register(&xdev->dma_dev); if (ret) { @@ -959,6 +1303,7 @@ static const struct platform_device_id xdma_id_table[] = { { "xdma", 0}, { }, }; +MODULE_DEVICE_TABLE(platform, xdma_id_table); static struct platform_driver xdma_driver = { .driver = { diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index ac09f0e5f58d..fabff602065f 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -41,11 +41,12 @@ #include <linux/io.h> #include <linux/iopoll.h> #include <linux/module.h> -#include <linux/of_address.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_platform.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/clk.h> #include <linux/io-64-nonatomic-lo-hi.h> @@ -112,7 +113,9 @@ /* Register Direct Mode Registers */ #define XILINX_DMA_REG_VSIZE 0x0000 +#define XILINX_DMA_VSIZE_MASK GENMASK(12, 0) #define XILINX_DMA_REG_HSIZE 0x0004 +#define XILINX_DMA_HSIZE_MASK GENMASK(15, 0) #define XILINX_DMA_REG_FRMDLY_STRIDE 0x0008 #define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT 24 @@ -173,12 +176,15 @@ #define XILINX_DMA_MAX_TRANS_LEN_MAX 23 #define XILINX_DMA_V2_MAX_TRANS_LEN_MAX 26 #define XILINX_DMA_CR_COALESCE_MAX GENMASK(23, 16) +#define XILINX_DMA_CR_DELAY_MAX GENMASK(31, 24) #define XILINX_DMA_CR_CYCLIC_BD_EN_MASK BIT(4) #define XILINX_DMA_CR_COALESCE_SHIFT 16 +#define XILINX_DMA_CR_DELAY_SHIFT 24 #define XILINX_DMA_BD_SOP BIT(27) #define XILINX_DMA_BD_EOP BIT(26) +#define XILINX_DMA_BD_COMP_MASK BIT(31) #define XILINX_DMA_COALESCE_MAX 255 -#define XILINX_DMA_NUM_DESCS 255 +#define XILINX_DMA_NUM_DESCS 512 #define XILINX_DMA_NUM_APP_WORDS 5 /* AXI CDMA Specific Registers/Offsets */ @@ -410,6 +416,7 @@ struct xilinx_dma_tx_descriptor { * @stop_transfer: Differentiate b/w DMA IP's quiesce * @tdest: TDEST value for mcdma * @has_vflip: S2MM vertical flip + * @irq_delay: Interrupt delay timeout */ struct xilinx_dma_chan { struct xilinx_dma_device *xdev; @@ -448,6 +455,7 @@ struct xilinx_dma_chan { int (*stop_transfer)(struct xilinx_dma_chan *chan); u16 tdest; bool has_vflip; + u8 irq_delay; }; /** @@ -493,6 +501,7 @@ struct xilinx_dma_config { * @s2mm_chan_id: DMA s2mm channel identifier * @mm2s_chan_id: DMA mm2s channel identifier * @max_buffer_len: Max buffer length + * @has_axistream_connected: AXI DMA connected to AXI Stream IP */ struct xilinx_dma_device { void __iomem *regs; @@ -511,6 +520,7 @@ struct xilinx_dma_device { u32 s2mm_chan_id; u32 mm2s_chan_id; u32 max_buffer_len; + bool has_axistream_connected; }; /* Macros */ @@ -623,6 +633,29 @@ static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan, } } +/** + * xilinx_dma_get_metadata_ptr- Populate metadata pointer and payload length + * @tx: async transaction descriptor + * @payload_len: metadata payload length + * @max_len: metadata max length + * Return: The app field pointer. + */ +static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx, + size_t *payload_len, size_t *max_len) +{ + struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); + struct xilinx_axidma_tx_segment *seg; + + *max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS; + seg = list_first_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + return seg->hw.app; +} + +static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { + .get_ptr = xilinx_dma_get_metadata_ptr, +}; + /* ----------------------------------------------------------------------------- * Descriptors and segments alloc and free */ @@ -1372,16 +1405,18 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); - j = chan->desc_submitcount; - reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); - if (chan->direction == DMA_MEM_TO_DEV) { - reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; - } else { - reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + if (config->park) { + j = chan->desc_submitcount; + reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); + if (chan->direction == DMA_MEM_TO_DEV) { + reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; + } else { + reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + } + dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); } - dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); /* Start the hardware */ xilinx_dma_start(chan); @@ -1535,6 +1570,9 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) if (chan->has_sg) xilinx_write(chan, XILINX_DMA_REG_CURDESC, head_desc->async_tx.phys); + reg &= ~XILINX_DMA_CR_DELAY_MAX; + reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); xilinx_dma_start(chan); @@ -1683,6 +1721,14 @@ static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan) return; list_for_each_entry_safe(desc, next, &chan->active_list, node) { + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + struct xilinx_axidma_tx_segment *seg; + + seg = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg) + break; + } if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA) desc->residue = xilinx_dma_get_residue(chan, desc); @@ -1816,7 +1862,7 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) spin_unlock(&chan->lock); } - tasklet_schedule(&chan->tasklet); + tasklet_hi_schedule(&chan->tasklet); return IRQ_HANDLED; } @@ -1864,15 +1910,8 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) } } - if (status & XILINX_DMA_DMASR_DLY_CNT_IRQ) { - /* - * Device takes too long to do the transfer when user requires - * responsiveness. - */ - dev_dbg(chan->dev, "Inter-packet latency too long\n"); - } - - if (status & XILINX_DMA_DMASR_FRM_CNT_IRQ) { + if (status & (XILINX_DMA_DMASR_FRM_CNT_IRQ | + XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); chan->idle = true; @@ -2016,6 +2055,10 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, if (!xt->numf || !xt->sgl[0].size) return NULL; + if (xt->numf & ~XILINX_DMA_VSIZE_MASK || + xt->sgl[0].size & ~XILINX_DMA_HSIZE_MASK) + return NULL; + if (xt->frame_size != 1) return NULL; @@ -2130,6 +2173,99 @@ error: } /** + * xilinx_dma_prep_peripheral_dma_vec - prepare descriptors for a DMA_SLAVE + * transaction from DMA vectors + * @dchan: DMA channel + * @vecs: Array of DMA vectors that should be transferred + * @nb: number of entries in @vecs + * @direction: DMA direction + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor *xilinx_dma_prep_peripheral_dma_vec( + struct dma_chan *dchan, const struct dma_vec *vecs, size_t nb, + enum dma_transfer_direction direction, unsigned long flags) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_axidma_tx_segment *segment, *head, *prev = NULL; + size_t copy; + size_t sg_used; + unsigned int i; + + if (!is_slave_direction(direction) || direction != chan->direction) + return NULL; + + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + + /* Build transactions using information from DMA vectors */ + for (i = 0; i < nb; i++) { + sg_used = 0; + + /* Loop until the entire dma_vec entry is used */ + while (sg_used < vecs[i].len) { + struct xilinx_axidma_desc_hw *hw; + + /* Get a free segment */ + segment = xilinx_axidma_alloc_tx_segment(chan); + if (!segment) + goto error; + + /* + * Calculate the maximum number of bytes to transfer, + * making sure it is less than the hw limit + */ + copy = xilinx_dma_calc_copysize(chan, vecs[i].len, + sg_used); + hw = &segment->hw; + + /* Fill in the descriptor */ + xilinx_axidma_buf(chan, hw, vecs[i].addr, sg_used, 0); + hw->control = copy; + + if (prev) + prev->hw.next_desc = segment->phys; + + prev = segment; + sg_used += copy; + + /* + * Insert the segment into the descriptor segments + * list. + */ + list_add_tail(&segment->node, &desc->segments); + } + } + + head = list_first_entry(&desc->segments, struct xilinx_axidma_tx_segment, node); + desc->async_tx.phys = head->phys; + + /* For the last DMA_MEM_TO_DEV transfer, set EOP */ + if (chan->direction == DMA_MEM_TO_DEV) { + segment->hw.control |= XILINX_DMA_BD_SOP; + segment = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, + node); + segment->hw.control |= XILINX_DMA_BD_EOP; + } + + if (chan->xdev->has_axistream_connected) + desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction * @dchan: DMA channel * @sgl: scatterlist to transfer to/from @@ -2221,6 +2357,9 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg( segment->hw.control |= XILINX_DMA_BD_EOP; } + if (chan->xdev->has_axistream_connected) + desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; + return &desc->async_tx; error: @@ -2796,6 +2935,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, /* Retrieve the channel properties from the device tree */ has_dre = of_property_read_bool(node, "xlnx,include-dre"); + of_property_read_u8(node, "xlnx,irq-delay", &chan->irq_delay); + chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); err = of_property_read_u32(node, "xlnx,datawidth", &value); @@ -2861,6 +3002,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, return -EINVAL; } + xdev->common.directions |= chan->direction; + /* Request the interrupt */ chan->irq = of_irq_get(node, chan->tdest); if (chan->irq < 0) @@ -2893,7 +3036,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, XILINX_DMA_DMASR_SG_MASK) chan->has_sg = true; dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id, - chan->has_sg ? "enabled" : "disabled"); + str_enabled_disabled(chan->has_sg)); } /* Initialize the tasklet */ @@ -3067,6 +3210,13 @@ static int xilinx_dma_probe(struct platform_device *pdev) } } + dma_set_max_seg_size(xdev->dev, xdev->max_buffer_len); + + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + xdev->has_axistream_connected = + of_property_read_bool(node, "xlnx,axistream-connected"); + } + if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { err = of_property_read_u32(node, "xlnx,num-fstores", &num_frames); @@ -3092,6 +3242,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) else xdev->ext_addr = false; + /* Set metadata mode */ + if (xdev->has_axistream_connected) + xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE; + /* Set the dma mask bits */ err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width)); if (err < 0) { @@ -3119,6 +3273,7 @@ static int xilinx_dma_probe(struct platform_device *pdev) xdev->common.device_config = xilinx_dma_device_config; if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask); + xdev->common.device_prep_peripheral_dma_vec = xilinx_dma_prep_peripheral_dma_vec; xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg; xdev->common.device_prep_dma_cyclic = xilinx_dma_prep_dma_cyclic; @@ -3194,10 +3349,8 @@ disable_clks: /** * xilinx_dma_remove - Driver remove function * @pdev: Pointer to the platform_device structure - * - * Return: Always '0' */ -static int xilinx_dma_remove(struct platform_device *pdev) +static void xilinx_dma_remove(struct platform_device *pdev) { struct xilinx_dma_device *xdev = platform_get_drvdata(pdev); int i; @@ -3211,8 +3364,6 @@ static int xilinx_dma_remove(struct platform_device *pdev) xilinx_dma_chan_remove(xdev->chan[i]); xdma_disable_allclks(xdev); - - return 0; } static struct platform_driver xilinx_vdma_driver = { diff --git a/drivers/dma/xilinx/xilinx_dpdma.c b/drivers/dma/xilinx/xilinx_dpdma.c index 84dc5240a807..ee5d9fdbfd7f 100644 --- a/drivers/dma/xilinx/xilinx_dpdma.c +++ b/drivers/dma/xilinx/xilinx_dpdma.c @@ -149,7 +149,7 @@ struct xilinx_dpdma_chan; * @addr_ext: upper 16 bit of 48 bit address (next_desc and src_addr) * @next_desc: next descriptor 32 bit address * @src_addr: payload source address (1st page, 32 LSB) - * @addr_ext_23: payload source address (3nd and 3rd pages, 16 LSBs) + * @addr_ext_23: payload source address (2nd and 3rd pages, 16 LSBs) * @addr_ext_45: payload source address (4th and 5th pages, 16 LSBs) * @src_addr2: payload source address (2nd page, 32 LSB) * @src_addr3: payload source address (3rd page, 32 LSB) @@ -210,11 +210,12 @@ struct xilinx_dpdma_tx_desc { * @vchan: virtual DMA channel * @reg: register base address * @id: channel ID - * @wait_to_stop: queue to wait for outstanding transacitons before stopping + * @wait_to_stop: queue to wait for outstanding transactions before stopping * @running: true if the channel is running * @first_frame: flag for the first frame of stream * @video_group: flag if multi-channel operation is needed for video channels - * @lock: lock to access struct xilinx_dpdma_chan + * @lock: lock to access struct xilinx_dpdma_chan. Must be taken before + * @vchan.lock, if both are to be held. * @desc_pool: descriptor allocation pool * @err_task: error IRQ bottom half handler * @desc: References to descriptors being processed @@ -309,7 +310,7 @@ static ssize_t xilinx_dpdma_debugfs_desc_done_irq_read(char *buf) out_str_len = strlen(XILINX_DPDMA_DEBUGFS_UINT16_MAX_STR); out_str_len = min_t(size_t, XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE, - out_str_len); + out_str_len + 1); snprintf(buf, out_str_len, "%d", dpdma_debugfs.xilinx_dpdma_irq_done_count); @@ -670,6 +671,84 @@ static void xilinx_dpdma_chan_free_tx_desc(struct virt_dma_desc *vdesc) } /** + * xilinx_dpdma_chan_prep_cyclic - Prepare a cyclic dma descriptor + * @chan: DPDMA channel + * @buf_addr: buffer address + * @buf_len: buffer length + * @period_len: number of periods + * @flags: tx flags argument passed in to prepare function + * + * Prepare a tx descriptor incudling internal software/hardware descriptors + * for the given cyclic transaction. + * + * Return: A dma async tx descriptor on success, or NULL. + */ +static struct dma_async_tx_descriptor * +xilinx_dpdma_chan_prep_cyclic(struct xilinx_dpdma_chan *chan, + dma_addr_t buf_addr, size_t buf_len, + size_t period_len, unsigned long flags) +{ + struct xilinx_dpdma_tx_desc *tx_desc; + struct xilinx_dpdma_sw_desc *sw_desc, *last = NULL; + unsigned int periods = buf_len / period_len; + unsigned int i; + + tx_desc = xilinx_dpdma_chan_alloc_tx_desc(chan); + if (!tx_desc) + return NULL; + + for (i = 0; i < periods; i++) { + struct xilinx_dpdma_hw_desc *hw_desc; + + if (!IS_ALIGNED(buf_addr, XILINX_DPDMA_ALIGN_BYTES)) { + dev_err(chan->xdev->dev, + "buffer should be aligned at %d B\n", + XILINX_DPDMA_ALIGN_BYTES); + goto error; + } + + sw_desc = xilinx_dpdma_chan_alloc_sw_desc(chan); + if (!sw_desc) + goto error; + + xilinx_dpdma_sw_desc_set_dma_addrs(chan->xdev, sw_desc, last, + &buf_addr, 1); + hw_desc = &sw_desc->hw; + hw_desc->xfer_size = period_len; + hw_desc->hsize_stride = + FIELD_PREP(XILINX_DPDMA_DESC_HSIZE_STRIDE_HSIZE_MASK, + period_len) | + FIELD_PREP(XILINX_DPDMA_DESC_HSIZE_STRIDE_STRIDE_MASK, + period_len); + hw_desc->control = XILINX_DPDMA_DESC_CONTROL_PREEMBLE | + XILINX_DPDMA_DESC_CONTROL_IGNORE_DONE | + XILINX_DPDMA_DESC_CONTROL_COMPLETE_INTR; + + list_add_tail(&sw_desc->node, &tx_desc->descriptors); + + buf_addr += period_len; + last = sw_desc; + } + + sw_desc = list_first_entry(&tx_desc->descriptors, + struct xilinx_dpdma_sw_desc, node); + last->hw.next_desc = lower_32_bits(sw_desc->dma_addr); + if (chan->xdev->ext_addr) + last->hw.addr_ext |= + FIELD_PREP(XILINX_DPDMA_DESC_ADDR_EXT_NEXT_ADDR_MASK, + upper_32_bits(sw_desc->dma_addr)); + + last->hw.control |= XILINX_DPDMA_DESC_CONTROL_LAST_OF_FRAME; + + return vchan_tx_prep(&chan->vchan, &tx_desc->vdesc, flags); + +error: + xilinx_dpdma_chan_free_tx_desc(&tx_desc->vdesc); + + return NULL; +} + +/** * xilinx_dpdma_chan_prep_interleaved_dma - Prepare an interleaved dma * descriptor * @chan: DPDMA channel @@ -1042,9 +1121,8 @@ static int xilinx_dpdma_chan_stop(struct xilinx_dpdma_chan *chan) static void xilinx_dpdma_chan_done_irq(struct xilinx_dpdma_chan *chan) { struct xilinx_dpdma_tx_desc *active; - unsigned long flags; - spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->lock); xilinx_dpdma_debugfs_desc_done_irq(chan); @@ -1056,7 +1134,7 @@ static void xilinx_dpdma_chan_done_irq(struct xilinx_dpdma_chan *chan) "chan%u: DONE IRQ with no active descriptor!\n", chan->id); - spin_unlock_irqrestore(&chan->lock, flags); + spin_unlock(&chan->lock); } /** @@ -1071,10 +1149,9 @@ static void xilinx_dpdma_chan_vsync_irq(struct xilinx_dpdma_chan *chan) { struct xilinx_dpdma_tx_desc *pending; struct xilinx_dpdma_sw_desc *sw_desc; - unsigned long flags; u32 desc_id; - spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->lock); pending = chan->desc.pending; if (!chan->running || !pending) @@ -1097,15 +1174,17 @@ static void xilinx_dpdma_chan_vsync_irq(struct xilinx_dpdma_chan *chan) * Complete the active descriptor, if any, promote the pending * descriptor to active, and queue the next transfer, if any. */ + spin_lock(&chan->vchan.lock); if (chan->desc.active) vchan_cookie_complete(&chan->desc.active->vdesc); chan->desc.active = pending; chan->desc.pending = NULL; xilinx_dpdma_chan_queue_transfer(chan); + spin_unlock(&chan->vchan.lock); out: - spin_unlock_irqrestore(&chan->lock, flags); + spin_unlock(&chan->lock); } /** @@ -1188,6 +1267,23 @@ out_unlock: /* ----------------------------------------------------------------------------- * DMA Engine Operations */ +static struct dma_async_tx_descriptor * +xilinx_dpdma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct xilinx_dpdma_chan *chan = to_xilinx_chan(dchan); + + if (direction != DMA_MEM_TO_DEV) + return NULL; + + if (buf_len % period_len) + return NULL; + + return xilinx_dpdma_chan_prep_cyclic(chan, buf_addr, buf_len, + period_len, flags); +} static struct dma_async_tx_descriptor * xilinx_dpdma_prep_interleaved_dma(struct dma_chan *dchan, @@ -1264,10 +1360,12 @@ static void xilinx_dpdma_issue_pending(struct dma_chan *dchan) struct xilinx_dpdma_chan *chan = to_xilinx_chan(dchan); unsigned long flags; - spin_lock_irqsave(&chan->vchan.lock, flags); + spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->vchan.lock); if (vchan_issue_pending(&chan->vchan)) xilinx_dpdma_chan_queue_transfer(chan); - spin_unlock_irqrestore(&chan->vchan.lock, flags); + spin_unlock(&chan->vchan.lock); + spin_unlock_irqrestore(&chan->lock, flags); } static int xilinx_dpdma_config(struct dma_chan *dchan, @@ -1495,7 +1593,9 @@ static void xilinx_dpdma_chan_err_task(struct tasklet_struct *t) XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id); spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->vchan.lock); xilinx_dpdma_chan_queue_transfer(chan); + spin_unlock(&chan->vchan.lock); spin_unlock_irqrestore(&chan->lock, flags); } @@ -1667,6 +1767,7 @@ static int xilinx_dpdma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, ddev->cap_mask); dma_cap_set(DMA_PRIVATE, ddev->cap_mask); + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); dma_cap_set(DMA_INTERLEAVE, ddev->cap_mask); dma_cap_set(DMA_REPEAT, ddev->cap_mask); dma_cap_set(DMA_LOAD_EOT, ddev->cap_mask); @@ -1674,6 +1775,7 @@ static int xilinx_dpdma_probe(struct platform_device *pdev) ddev->device_alloc_chan_resources = xilinx_dpdma_alloc_chan_resources; ddev->device_free_chan_resources = xilinx_dpdma_free_chan_resources; + ddev->device_prep_dma_cyclic = xilinx_dpdma_prep_dma_cyclic; ddev->device_prep_interleaved_dma = xilinx_dpdma_prep_interleaved_dma; /* TODO: Can we achieve better granularity ? */ ddev->device_tx_status = dma_cookie_status; @@ -1736,7 +1838,7 @@ error: return ret; } -static int xilinx_dpdma_remove(struct platform_device *pdev) +static void xilinx_dpdma_remove(struct platform_device *pdev) { struct xilinx_dpdma_device *xdev = platform_get_drvdata(pdev); unsigned int i; @@ -1751,8 +1853,6 @@ static int xilinx_dpdma_remove(struct platform_device *pdev) for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) xilinx_dpdma_chan_remove(xdev->chan[i]); - - return 0; } static const struct of_device_id xilinx_dpdma_of_match[] = { diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index 9360f43b8e0f..f7e584de4335 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -11,8 +11,9 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/clk.h> #include <linux/io-64-nonatomic-lo-hi.h> @@ -21,10 +22,10 @@ #include "../dmaengine.h" /* Register Offsets */ -#define ZYNQMP_DMA_ISR 0x100 -#define ZYNQMP_DMA_IMR 0x104 -#define ZYNQMP_DMA_IER 0x108 -#define ZYNQMP_DMA_IDS 0x10C +#define ZYNQMP_DMA_ISR (chan->irq_offset + 0x100) +#define ZYNQMP_DMA_IMR (chan->irq_offset + 0x104) +#define ZYNQMP_DMA_IER (chan->irq_offset + 0x108) +#define ZYNQMP_DMA_IDS (chan->irq_offset + 0x10c) #define ZYNQMP_DMA_CTRL0 0x110 #define ZYNQMP_DMA_CTRL1 0x114 #define ZYNQMP_DMA_DATA_ATTR 0x120 @@ -144,6 +145,9 @@ #define tx_to_desc(tx) container_of(tx, struct zynqmp_dma_desc_sw, \ async_tx) +/* IRQ Register offset for Versal Gen 2 */ +#define IRQ_REG_OFFSET 0x308 + /** * struct zynqmp_dma_desc_ll - Hw linked list descriptor * @addr: Buffer address @@ -210,6 +214,7 @@ struct zynqmp_dma_desc_sw { * @bus_width: Bus width * @src_burst_len: Source burst length * @dst_burst_len: Dest burst length + * @irq_offset: Irq register offset */ struct zynqmp_dma_chan { struct zynqmp_dma_device *zdev; @@ -234,6 +239,7 @@ struct zynqmp_dma_chan { u32 bus_width; u32 src_burst_len; u32 dst_burst_len; + u32 irq_offset; }; /** @@ -252,6 +258,14 @@ struct zynqmp_dma_device { struct clk *clk_apb; }; +struct zynqmp_dma_config { + u32 offset; +}; + +static const struct zynqmp_dma_config versal2_dma_config = { + .offset = IRQ_REG_OFFSET, +}; + static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg, u64 value) { @@ -352,7 +366,7 @@ static void zynqmp_dma_init(struct zynqmp_dma_chan *chan) } writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR); - /* Clearing the interrupt account rgisters */ + /* Clearing the interrupt account registers */ val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); @@ -891,6 +905,7 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, { struct zynqmp_dma_chan *chan; struct device_node *node = pdev->dev.of_node; + const struct zynqmp_dma_config *match_data; int err; chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL); @@ -918,6 +933,10 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, return -EINVAL; } + match_data = of_device_get_match_data(&pdev->dev); + if (match_data) + chan->irq_offset = match_data->offset; + chan->is_dmacoherent = of_property_read_bool(node, "dma-coherent"); zdev->chan = chan; tasklet_setup(&chan->tasklet, zynqmp_dma_do_tasklet); @@ -1146,7 +1165,7 @@ err_disable_pm: * * Return: Always '0' */ -static int zynqmp_dma_remove(struct platform_device *pdev) +static void zynqmp_dma_remove(struct platform_device *pdev) { struct zynqmp_dma_device *zdev = platform_get_drvdata(pdev); @@ -1154,14 +1173,13 @@ static int zynqmp_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&zdev->common); zynqmp_dma_chan_remove(zdev->chan); - pm_runtime_disable(zdev->dev); - if (!pm_runtime_enabled(zdev->dev)) + if (pm_runtime_active(zdev->dev)) zynqmp_dma_runtime_suspend(zdev->dev); - - return 0; + pm_runtime_disable(zdev->dev); } static const struct of_device_id zynqmp_dma_of_match[] = { + { .compatible = "amd,versal2-dma-1.0", .data = &versal2_dma_config }, { .compatible = "xlnx,zynqmp-dma-1.0", }, {} }; @@ -1175,6 +1193,7 @@ static struct platform_driver zynqmp_dma_driver = { }, .probe = zynqmp_dma_probe, .remove = zynqmp_dma_remove, + .shutdown = zynqmp_dma_remove, }; module_platform_driver(zynqmp_dma_driver); |