diff options
Diffstat (limited to 'drivers/dma')
218 files changed, 30831 insertions, 14447 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 39b5b46e880f..8bb0a119ecd4 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -85,16 +85,32 @@ config AMCC_PPC440SPE_ADMA help Enable support for the AMCC PPC440SPe RAID engines. +config APPLE_ADMAC + tristate "Apple ADMAC support" + depends on ARCH_APPLE || COMPILE_TEST + select DMA_ENGINE + help + Enable support for Audio DMA Controller found on Apple Silicon SoCs. + +config ARM_DMA350 + tristate "Arm DMA-350 support" + depends on ARM || ARM64 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the Arm DMA-350 controller. + config AT_HDMAC tristate "Atmel AHB DMA support" - depends on ARCH_AT91 + depends on ARCH_AT91 || COMPILE_TEST select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS help Support the Atmel AHB DMA controller. config AT_XDMAC tristate "Atmel XDMA support" - depends on ARCH_AT91 + depends on ARCH_MICROCHIP select DMA_ENGINE help Support the Atmel XDMA controller. @@ -127,7 +143,7 @@ config BCM_SBA_RAID config DMA_BCM2835 tristate "BCM2835 DMA engine support" - depends on ARCH_BCM2835 + depends on ARCH_BCM2835 || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -153,8 +169,8 @@ config DMA_SA11X0 config DMA_SUN4I tristate "Allwinner A10 DMA SoCs support" - depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I - default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I) + depends on MACH_SUN4I || MACH_SUN5I || MACH_SUN7I || MACH_SUNIV + default (MACH_SUN4I || MACH_SUN5I || MACH_SUN7I || MACH_SUNIV) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -163,7 +179,7 @@ config DMA_SUN4I config DMA_SUN6I tristate "Allwinner A31 SoCs DMA support" - depends on MACH_SUN6I || MACH_SUN8I || (ARM64 && ARCH_SUNXI) || COMPILE_TEST + depends on ARCH_SUNXI || COMPILE_TEST depends on RESET_CONTROLLER select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -172,7 +188,7 @@ config DMA_SUN6I config DW_AXI_DMAC tristate "Synopsys DesignWare AXI DMA support" - depends on OF || COMPILE_TEST + depends on OF depends on HAS_IOMEM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -202,6 +218,7 @@ config FSL_DMA config FSL_EDMA tristate "Freescale eDMA engine support" depends on OF + depends on HAS_IOMEM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -236,7 +253,7 @@ config FSL_RAID config HISI_DMA tristate "HiSilicon DMA Engine support" - depends on ARM64 || COMPILE_TEST + depends on ARCH_HISI || COMPILE_TEST depends on PCI_MSI select DMA_ENGINE select DMA_VIRTUAL_CHANNELS @@ -271,16 +288,22 @@ config IMX_SDMA config INTEL_IDMA64 tristate "Intel integrated DMA 64-bit support" + depends on HAS_IOMEM select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help Enable DMA support for Intel Low Power Subsystem such as found on Intel Skylake PCH. +config INTEL_IDXD_BUS + tristate + default INTEL_IDXD + config INTEL_IDXD tristate "Intel Data Accelerators support" - depends on PCI && X86_64 + depends on PCI && X86_64 && !UML depends on PCI_MSI + depends on PCI_PASID depends on SBITMAP select DMA_ENGINE help @@ -291,6 +314,23 @@ config INTEL_IDXD If unsure, say N. +config INTEL_IDXD_COMPAT + bool "Legacy behavior for idxd driver" + depends on PCI && X86_64 + select INTEL_IDXD_BUS + help + Compatible driver to support old /sys/bus/dsa/drivers/dsa behavior. + The old behavior performed driver bind/unbind for device and wq + devices all under the dsa driver. The compat driver will emulate + the legacy behavior in order to allow existing support apps (i.e. + accel-config) to continue function. It is expected that accel-config + v3.2 and earlier will need the compat mode. A distro with later + accel-config version can disable this compat config. + + Say Y if you have old applications that require such behavior. + + If unsure, say N. + # Config symbol that collects all the dependencies that's necessary to # support shared virtual memory for the devices supported by idxd. config INTEL_IDXD_SVM @@ -315,7 +355,7 @@ config INTEL_IDXD_PERFMON config INTEL_IOATDMA tristate "Intel I/OAT DMA support" - depends on PCI && X86_64 + depends on PCI && X86_64 && !UML select DMA_ENGINE select DMA_ENGINE_RAID select DCA @@ -327,23 +367,38 @@ config INTEL_IOATDMA If unsure, say N. -config INTEL_IOP_ADMA - tristate "Intel IOP32x ADMA support" - depends on ARCH_IOP32X || COMPILE_TEST - select DMA_ENGINE - select ASYNC_TX_ENABLE_CHANNEL_SWITCH - help - Enable support for the Intel(R) IOP Series RAID engines. - config K3_DMA tristate "Hisilicon K3 DMA support" - depends on ARCH_HI3xxx || ARCH_HISI || COMPILE_TEST + depends on ARCH_HISI || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help Support the DMA engine for Hisilicon K3 platform devices. +config LOONGSON1_APB_DMA + tristate "Loongson1 APB DMA support" + depends on MACH_LOONGSON32 || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + This selects support for the APB DMA controller in Loongson1 SoCs, + which is required by Loongson1 NAND and audio support. + +config LOONGSON2_APB_DMA + tristate "Loongson2 APB DMA support" + depends on LOONGARCH || COMPILE_TEST + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Loongson2 APB DMA controller driver. The + DMA controller is having single DMA channel which can be + configured for different peripherals like audio, nand, sdio + etc which is in APB bus. + + This DMA controller transfers data from memory to peripheral fifo. + It does not support memory to memory data transfer. + config LPC18XX_DMAMUX bool "NXP LPC18xx/43xx DMA MUX for PL080" depends on ARCH_LPC18XX || COMPILE_TEST @@ -353,9 +408,18 @@ config LPC18XX_DMAMUX Enable support for DMA on NXP LPC18xx/43xx platforms with PL080 and multiplexed DMA request lines. +config LPC32XX_DMAMUX + bool "NXP LPC32xx DMA MUX for PL080" + depends on ARCH_LPC32XX || COMPILE_TEST + depends on OF && AMBA_PL08X + select MFD_SYSCON + help + Support for PL080 multiplexed DMA request lines on + LPC32XX platrofm. + config MCF_EDMA tristate "Freescale eDMA engine support, ColdFire mcf5441x SoCs" - depends on M5441x || COMPILE_TEST + depends on M5441x || (COMPILE_TEST && FSL_EDMA=n) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -386,7 +450,7 @@ config MILBEAUT_XDMAC config MMP_PDMA tristate "MMP PDMA support" - depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST + depends on ARCH_MMP || ARCH_PXA || ARCH_SPACEMIT || COMPILE_TEST select DMA_ENGINE help Support the MMP PDMA engine for PXA and MMP platform. @@ -432,7 +496,7 @@ config MV_XOR_V2 select DMA_ENGINE select DMA_ENGINE_RAID select ASYNC_TX_ENABLE_CHANNEL_SWITCH - select GENERIC_MSI_IRQ_DOMAIN + select GENERIC_MSI_IRQ help Enable support for the Marvell version 2 XOR engine. @@ -449,25 +513,6 @@ config MXS_DMA Support the MXS DMA engine. This engine including APBH-DMA and APBX-DMA is integrated into some Freescale chips. -config MX3_IPU - bool "MX3x Image Processing Unit support" - depends on ARCH_MXC - select DMA_ENGINE - default y - help - If you plan to use the Image Processing unit in the i.MX3x, say - Y here. If unsure, select Y. - -config MX3_IPU_IRQS - int "Number of dynamically mapped interrupts for IPU" - depends on MX3_IPU - range 2 137 - default 4 - help - Out of 137 interrupt sources on i.MX31 IPU only very few are used. - To avoid bloating the irq_desc[] array we allocate a sufficient - number of IRQ slots and map them dynamically to specific sources. - config NBPFAXI_DMA tristate "Renesas Type-AXI NBPF DMA support" select DMA_ENGINE @@ -508,7 +553,7 @@ config PL330_DMA config PXA_DMA bool "PXA DMA support" - depends on (ARCH_MMP || ARCH_PXA) + depends on ARCH_MMP || ARCH_PXA || COMPILE_TEST select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help @@ -526,10 +571,20 @@ config PLX_DMA These are exposed via extra functions on the switch's upstream port. Each function exposes one DMA channel. +config SOPHGO_CV1800B_DMAMUX + tristate "Sophgo CV1800/SG2000 series SoC DMA multiplexer support" + depends on MFD_SYSCON + depends on ARCH_SOPHGO || COMPILE_TEST + help + Support for the DMA multiplexer on Sophgo CV1800/SG2000 + series SoCs. + Say Y here if your board have this soc. + config STE_DMA40 bool "ST-Ericsson DMA40 support" depends on ARCH_U8500 select DMA_ENGINE + select SRAM help Support for ST-Ericsson DMA40 controller @@ -547,38 +602,6 @@ config ST_FDMA Say Y here if you have such a chipset. If unsure, say N. -config STM32_DMA - bool "STMicroelectronics STM32 DMA support" - depends on ARCH_STM32 || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the on-chip DMA controller on STMicroelectronics - STM32 MCUs. - If you have a board based on such a MCU and wish to use DMA say Y - here. - -config STM32_DMAMUX - bool "STMicroelectronics STM32 dma multiplexer support" - depends on STM32_DMA || COMPILE_TEST - help - Enable support for the on-chip DMA multiplexer on STMicroelectronics - STM32 MCUs. - If you have a board based on such a MCU and wish to use DMAMUX say Y - here. - -config STM32_MDMA - bool "STMicroelectronics STM32 master dma support" - depends on ARCH_STM32 || COMPILE_TEST - depends on OF - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the on-chip MDMA controller on STMicroelectronics - STM32 platforms. - If you have a board based on STM32 SoC and wish to use the master DMA - say Y here. - config SPRD_DMA tristate "Spreadtrum DMA support" depends on ARCH_SPRD || COMPILE_TEST @@ -587,26 +610,27 @@ config SPRD_DMA help Enable support for the on-chip DMA controller on Spreadtrum platform. -config S3C24XX_DMAC - bool "Samsung S3C24XX DMA support" - depends on ARCH_S3C24XX || COMPILE_TEST - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Support for the Samsung S3C24XX DMA controller driver. The - DMA controller is having multiple DMA channels which can be - configured for different peripherals like audio, UART, SPI. - The DMA controller can transfer data from memory to peripheral, - periphal to memory, periphal to periphal and memory to memory. - config TXX9_DMAC tristate "Toshiba TXx9 SoC DMA support" - depends on MACH_TX49XX || MACH_TX39XX + depends on MACH_TX49XX select DMA_ENGINE help Support the TXx9 SoC internal DMA controller. This can be integrated in chips such as the Toshiba TX4927/38/39. +config TEGRA186_GPC_DMA + tristate "NVIDIA Tegra GPC DMA support" + depends on (ARCH_TEGRA || COMPILE_TEST) && ARCH_DMA_ADDR_T_64BIT + depends on IOMMU_API + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the NVIDIA Tegra General Purpose Central DMA controller. + The DMA controller has multiple DMA channels which can be configured + for different peripherals like UART, SPI, etc which are on APB bus. + This DMA controller transfers data from memory to peripheral FIFO + or vice versa. It also supports memory to memory data transfer. + config TEGRA20_APB_DMA tristate "NVIDIA Tegra20 APB DMA support" depends on ARCH_TEGRA || COMPILE_TEST @@ -621,16 +645,16 @@ config TEGRA20_APB_DMA config TEGRA210_ADMA tristate "NVIDIA Tegra210 ADMA support" - depends on (ARCH_TEGRA_210_SOC || COMPILE_TEST) + depends on (ARCH_TEGRA || COMPILE_TEST) select DMA_ENGINE select DMA_VIRTUAL_CHANNELS help - Support for the NVIDIA Tegra210 ADMA controller driver. The - DMA controller has multiple DMA channels and is used to service - various audio clients in the Tegra210 audio processing engine - (APE). This DMA controller transfers data from memory to - peripheral and vice versa. It does not support memory to - memory data transfer. + Support for the NVIDIA Tegra210/Tegra186/Tegra194/Tegra234 ADMA + controller driver. The DMA controller has multiple DMA channels + and is used to service various audio clients in the Tegra210 + audio processing engine (APE). This DMA controller transfers + data from memory to peripheral and vice versa. It does not + support memory to memory data transfer. config TIMB_DMA tristate "Timberdale FPGA DMA support" @@ -672,7 +696,7 @@ config XGENE_DMA config XILINX_DMA tristate "Xilinx AXI DMAS Engine" - depends on (ARCH_ZYNQ || MICROBLAZE || ARM64) + depends on HAS_IOMEM select DMA_ENGINE help Enable support for Xilinx AXI VDMA Soft IP. @@ -693,9 +717,23 @@ config XILINX_DMA the scatter gather interface with multiple channels independent configuration support. +config XILINX_XDMA + tristate "Xilinx DMA/Bridge Subsystem DMA Engine" + depends on HAS_IOMEM + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select REGMAP_MMIO + help + Enable support for Xilinx DMA/Bridge Subsystem DMA engine. The DMA + provides high performance block data movement between Host memory + and the DMA subsystem. These direct memory transfers can be both in + the Host to Card (H2C) and Card to Host (C2H) transfers. + The core also provides up to 16 user interrupt wires that generate + interrupts to the host. + config XILINX_ZYNQMP_DMA tristate "Xilinx ZynqMP DMA Engine" - depends on (ARCH_ZYNQ || MICROBLAZE || ARM64) + depends on ARCH_ZYNQ || MICROBLAZE || ARM64 || COMPILE_TEST select DMA_ENGINE help Enable support for Xilinx ZynqMP DMA controller. @@ -712,6 +750,8 @@ config XILINX_ZYNQMP_DPDMA display driver. # driver files +source "drivers/dma/amd/Kconfig" + source "drivers/dma/bestcomm/Kconfig" source "drivers/dma/mediatek/Kconfig" @@ -734,6 +774,8 @@ source "drivers/dma/fsl-dpaa2-qdma/Kconfig" source "drivers/dma/lgm/Kconfig" +source "drivers/dma/stm32/Kconfig" + # clients comment "DMA Clients" depends on DMA_ENGINE diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index aa69094e3547..a54d7688392b 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -16,6 +16,8 @@ obj-$(CONFIG_DMATEST) += dmatest.o obj-$(CONFIG_ALTERA_MSGDMA) += altera-msgdma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ +obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o +obj-$(CONFIG_ARM_DMA350) += arm-dma350.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o obj-$(CONFIG_AXI_DMAC) += dma-axi-dmac.o @@ -29,9 +31,13 @@ obj-$(CONFIG_DW_AXI_DMAC) += dw-axi-dmac/ obj-$(CONFIG_DW_DMAC_CORE) += dw/ obj-$(CONFIG_DW_EDMA) += dw-edma/ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o +fsl-edma-trace-$(CONFIG_TRACING) := fsl-edma-trace.o +CFLAGS_fsl-edma-trace.o := -I$(src) obj-$(CONFIG_FSL_DMA) += fsldma.o -obj-$(CONFIG_FSL_EDMA) += fsl-edma.o fsl-edma-common.o -obj-$(CONFIG_MCF_EDMA) += mcf-edma.o fsl-edma-common.o +fsl-edma-objs := fsl-edma-main.o fsl-edma-common.o ${fsl-edma-trace-y} +obj-$(CONFIG_FSL_EDMA) += fsl-edma.o +mcf-edma-objs := mcf-edma-main.o fsl-edma-common.o ${fsl-edma-trace-y} +obj-$(CONFIG_MCF_EDMA) += mcf-edma.o obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o obj-$(CONFIG_FSL_RAID) += fsl_raid.o obj-$(CONFIG_HISI_DMA) += hisi_dma.o @@ -41,10 +47,12 @@ obj-$(CONFIG_IMX_DMA) += imx-dma.o obj-$(CONFIG_IMX_SDMA) += imx-sdma.o obj-$(CONFIG_INTEL_IDMA64) += idma64.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ -obj-$(CONFIG_INTEL_IDXD) += idxd/ -obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o +obj-y += idxd/ obj-$(CONFIG_K3_DMA) += k3dma.o +obj-$(CONFIG_LOONGSON1_APB_DMA) += loongson1-apb-dma.o +obj-$(CONFIG_LOONGSON2_APB_DMA) += loongson2-apb-dma.o obj-$(CONFIG_LPC18XX_DMAMUX) += lpc18xx-dmamux.o +obj-$(CONFIG_LPC32XX_DMAMUX) += lpc32xx-dmamux.o obj-$(CONFIG_MILBEAUT_HDMAC) += milbeaut-hdmac.o obj-$(CONFIG_MILBEAUT_XDMAC) += milbeaut-xdmac.o obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o @@ -54,7 +62,6 @@ obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o obj-$(CONFIG_MV_XOR) += mv_xor.o obj-$(CONFIG_MV_XOR_V2) += mv_xor_v2.o obj-$(CONFIG_MXS_DMA) += mxs-dma.o -obj-$(CONFIG_MX3_IPU) += ipu/ obj-$(CONFIG_NBPFAXI_DMA) += nbpfaxi.o obj-$(CONFIG_OWL_DMA) += owl-dma.o obj-$(CONFIG_PCH_DMA) += pch_dma.o @@ -64,13 +71,11 @@ obj-$(CONFIG_PPC_BESTCOMM) += bestcomm/ obj-$(CONFIG_PXA_DMA) += pxa_dma.o obj-$(CONFIG_RENESAS_DMA) += sh/ obj-$(CONFIG_SF_PDMA) += sf-pdma/ +obj-$(CONFIG_SOPHGO_CV1800B_DMAMUX) += cv1800b-dmamux.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o -obj-$(CONFIG_STM32_DMA) += stm32-dma.o -obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o -obj-$(CONFIG_STM32_MDMA) += stm32-mdma.o obj-$(CONFIG_SPRD_DMA) += sprd-dma.o -obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o +obj-$(CONFIG_TEGRA186_GPC_DMA) += tegra186-gpc-dma.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o obj-$(CONFIG_TEGRA210_ADMA) += tegra210-adma.o obj-$(CONFIG_TIMB_DMA) += timb_dma.o @@ -81,7 +86,9 @@ obj-$(CONFIG_ST_FDMA) += st_fdma.o obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma/ obj-$(CONFIG_INTEL_LDMA) += lgm/ +obj-y += amd/ obj-y += mediatek/ obj-y += qcom/ +obj-y += stm32/ obj-y += ti/ obj-y += xilinx/ diff --git a/drivers/dma/acpi-dma.c b/drivers/dma/acpi-dma.c index 235f1396f968..2abbe11e797e 100644 --- a/drivers/dma/acpi-dma.c +++ b/drivers/dma/acpi-dma.c @@ -9,18 +9,21 @@ * Mika Westerberg <mika.westerberg@linux.intel.com> */ +#include <linux/acpi.h> +#include <linux/acpi_dma.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/err.h> -#include <linux/module.h> +#include <linux/errno.h> +#include <linux/export.h> +#include <linux/ioport.h> #include <linux/kernel.h> #include <linux/list.h> #include <linux/mutex.h> -#include <linux/slab.h> -#include <linux/ioport.h> -#include <linux/acpi.h> -#include <linux/acpi_dma.h> #include <linux/property.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/types.h> static LIST_HEAD(acpi_dma_list); static DEFINE_MUTEX(acpi_dma_lock); @@ -70,10 +73,22 @@ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp, si = (const struct acpi_csrt_shared_info *)&grp[1]; - /* Match device by MMIO and IRQ */ + /* Match device by MMIO */ if (si->mmio_base_low != lower_32_bits(mem) || - si->mmio_base_high != upper_32_bits(mem) || - si->gsi_interrupt != irq) + si->mmio_base_high != upper_32_bits(mem)) + return 0; + + /* + * acpi_gsi_to_irq() can't be used because some platforms do not save + * registered IRQs in the MP table. Instead we just try to register + * the GSI, which is the core part of the above mentioned function. + */ + ret = acpi_register_gsi(NULL, si->gsi_interrupt, si->interrupt_mode, si->interrupt_polarity); + if (ret < 0) + return 0; + + /* Match device by Linux vIRQ */ + if (ret != irq) return 0; dev_dbg(&adev->dev, "matches with %.4s%04X (rev %u)\n", @@ -100,7 +115,7 @@ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp, } /** - * acpi_dma_parse_csrt - parse CSRT to exctract additional DMA resources + * acpi_dma_parse_csrt - parse CSRT to extract additional DMA resources * @adev: ACPI device to match with * @adma: struct acpi_dma of the given DMA controller * @@ -224,7 +239,7 @@ int acpi_dma_controller_free(struct device *dev) } EXPORT_SYMBOL_GPL(acpi_dma_controller_free); -static void devm_acpi_dma_release(struct device *dev, void *res) +static void devm_acpi_dma_free(void *dev) { acpi_dma_controller_free(dev); } @@ -247,37 +262,15 @@ int devm_acpi_dma_controller_register(struct device *dev, (struct acpi_dma_spec *, struct acpi_dma *), void *data) { - void *res; int ret; - res = devres_alloc(devm_acpi_dma_release, 0, GFP_KERNEL); - if (!res) - return -ENOMEM; - ret = acpi_dma_controller_register(dev, acpi_dma_xlate, data); - if (ret) { - devres_free(res); + if (ret) return ret; - } - devres_add(dev, res); - return 0; -} -EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_register); -/** - * devm_acpi_dma_controller_free - resource managed acpi_dma_controller_free() - * @dev: device that is unregistering as DMA controller - * - * Unregister a DMA controller registered with - * devm_acpi_dma_controller_register(). Normally this function will not need to - * be called and the resource management code will ensure that the resource is - * freed. - */ -void devm_acpi_dma_controller_free(struct device *dev) -{ - WARN_ON(devres_release(dev, devm_acpi_dma_release, NULL, NULL)); + return devm_add_action_or_reset(dev, devm_acpi_dma_free, dev); } -EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_free); +EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_register); /** * acpi_dma_update_dma_spec - prepare dma specifier to pass to translation function @@ -293,7 +286,7 @@ EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_free); * found. * * Return: - * 0, if no information is avaiable, -1 on mismatch, and 1 otherwise. + * 0, if no information is available, -1 on mismatch, and 1 otherwise. */ static int acpi_dma_update_dma_spec(struct acpi_dma *adma, struct acpi_dma_spec *dma_spec) diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c index 0fe0676f8e1d..a203fdd84950 100644 --- a/drivers/dma/altera-msgdma.c +++ b/drivers/dma/altera-msgdma.c @@ -153,7 +153,7 @@ struct msgdma_extended_desc { /** * struct msgdma_sw_desc - implements a sw descriptor * @async_tx: support for the async_tx api - * @hw_desc: assosiated HW descriptor + * @hw_desc: associated HW descriptor * @node: node to move from the free list to the tx list * @tx_list: transmit list node */ @@ -233,7 +233,7 @@ static void msgdma_free_descriptor(struct msgdma_device *mdev, struct msgdma_sw_desc *child, *next; mdev->desc_free_cnt++; - list_add_tail(&desc->node, &mdev->free_list); + list_move_tail(&desc->node, &mdev->free_list); list_for_each_entry_safe(child, next, &desc->tx_list, node) { mdev->desc_free_cnt++; list_move_tail(&child->node, &mdev->free_list); @@ -511,7 +511,7 @@ static void msgdma_copy_one(struct msgdma_device *mdev, * of the DMA controller. The descriptor will get flushed to the * FIFO, once the last word (control word) is written. Since we * are not 100% sure that memcpy() writes all word in the "correct" - * oder (address from low to high) on all architectures, we make + * order (address from low to high) on all architectures, we make * sure this control word is written last by single coding it and * adding some write-barriers here. */ @@ -583,24 +583,25 @@ static void msgdma_issue_pending(struct dma_chan *chan) static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev) { struct msgdma_sw_desc *desc, *next; + unsigned long irqflags; - list_for_each_entry_safe(desc, next, &mdev->done_list, node) { - dma_async_tx_callback callback; - void *callback_param; + spin_lock_irqsave(&mdev->lock, irqflags); - list_del(&desc->node); + list_for_each_entry_safe(desc, next, &mdev->done_list, node) { + struct dmaengine_desc_callback cb; - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; - if (callback) { - spin_unlock(&mdev->lock); - callback(callback_param); - spin_lock(&mdev->lock); + dmaengine_desc_get_callback(&desc->async_tx, &cb); + if (dmaengine_desc_callback_valid(&cb)) { + spin_unlock_irqrestore(&mdev->lock, irqflags); + dmaengine_desc_callback_invoke(&cb, NULL); + spin_lock_irqsave(&mdev->lock, irqflags); } /* Run any dependencies, then free the descriptor */ msgdma_free_descriptor(mdev, desc); } + + spin_unlock_irqrestore(&mdev->lock, irqflags); } /** @@ -691,10 +692,14 @@ static void msgdma_tasklet(struct tasklet_struct *t) spin_lock_irqsave(&mdev->lock, flags); - /* Read number of responses that are available */ - count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL); - dev_dbg(mdev->dev, "%s (%d): response count=%d\n", - __func__, __LINE__, count); + if (mdev->resp) { + /* Read number of responses that are available */ + count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL); + dev_dbg(mdev->dev, "%s (%d): response count=%d\n", + __func__, __LINE__, count); + } else { + count = 1; + } while (count--) { /* @@ -703,14 +708,19 @@ static void msgdma_tasklet(struct tasklet_struct *t) * have any real values, like transferred bytes or error * bits. So we need to just drop these values. */ - size = ioread32(mdev->resp + MSGDMA_RESP_BYTES_TRANSFERRED); - status = ioread32(mdev->resp + MSGDMA_RESP_STATUS); + if (mdev->resp) { + size = ioread32(mdev->resp + + MSGDMA_RESP_BYTES_TRANSFERRED); + status = ioread32(mdev->resp + + MSGDMA_RESP_STATUS); + } msgdma_complete_descriptor(mdev); - msgdma_chan_desc_cleanup(mdev); } spin_unlock_irqrestore(&mdev->lock, flags); + + msgdma_chan_desc_cleanup(mdev); } /** @@ -743,7 +753,7 @@ static irqreturn_t msgdma_irq_handler(int irq, void *data) } /** - * msgdma_chan_remove - Channel remove function + * msgdma_dev_remove() - Device remove function * @mdev: Pointer to the Altera mSGDMA device structure */ static void msgdma_dev_remove(struct msgdma_device *mdev) @@ -757,14 +767,21 @@ static void msgdma_dev_remove(struct msgdma_device *mdev) } static int request_and_map(struct platform_device *pdev, const char *name, - struct resource **res, void __iomem **ptr) + struct resource **res, void __iomem **ptr, + bool optional) { struct resource *region; struct device *device = &pdev->dev; *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); if (*res == NULL) { - dev_err(device, "resource %s not defined\n", name); + if (optional) { + *ptr = NULL; + dev_info(device, "optional resource %s not defined\n", + name); + return 0; + } + dev_err(device, "mandatory resource %s not defined\n", name); return -ENODEV; } @@ -805,17 +822,17 @@ static int msgdma_probe(struct platform_device *pdev) mdev->dev = &pdev->dev; /* Map CSR space */ - ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr); + ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr, false); if (ret) return ret; /* Map (extended) descriptor space */ - ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc); + ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc, false); if (ret) return ret; /* Map response space */ - ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp); + ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp, true); if (ret) return ret; @@ -878,9 +895,7 @@ static int msgdma_probe(struct platform_device *pdev) ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (ret) { dev_warn(&pdev->dev, "unable to set coherent mask to 64"); - ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (ret) - goto fail; + goto fail; } msgdma_reset(mdev); @@ -907,12 +922,12 @@ fail: } /** - * msgdma_dma_remove - Driver remove function + * msgdma_remove() - Driver remove function * @pdev: Pointer to the platform_device structure * * Return: Always '0' */ -static int msgdma_remove(struct platform_device *pdev) +static void msgdma_remove(struct platform_device *pdev) { struct msgdma_device *mdev = platform_get_drvdata(pdev); @@ -922,8 +937,6 @@ static int msgdma_remove(struct platform_device *pdev) msgdma_dev_remove(mdev); dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n"); - - return 0; } #ifdef CONFIG_OF diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index a24882ba3764..38cdbca59485 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -2,7 +2,7 @@ /* * Copyright (c) 2006 ARM Ltd. * Copyright (c) 2010 ST-Ericsson SA - * Copyirght (c) 2017 Linaro Ltd. + * Copyright (c) 2017 Linaro Ltd. * * Author: Peter Pearse <peter.pearse@arm.com> * Author: Linus Walleij <linus.walleij@linaro.org> @@ -231,7 +231,7 @@ enum pl08x_dma_chan_state { /** * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel - * @vc: wrappped virtual channel + * @vc: wrapped virtual channel * @phychan: the physical channel utilized by this channel, if there is one * @name: name of channel * @cd: channel platform data @@ -1535,14 +1535,6 @@ static void pl08x_free_chan_resources(struct dma_chan *chan) vchan_free_chan_resources(to_virt_chan(chan)); } -static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt( - struct dma_chan *chan, unsigned long flags) -{ - struct dma_async_tx_descriptor *retval = NULL; - - return retval; -} - /* * Code accessing dma_async_is_complete() in a tight loop may give problems. * If slaves are relying on interrupts to signal completion this function @@ -2247,7 +2239,7 @@ static int pl08x_resume(struct dma_chan *chan) bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) { struct pl08x_dma_chan *plchan; - char *name = chan_id; + const char *name = chan_id; /* Reject channels for devices not bound to this driver */ if (chan->device->dev->driver != &pl08x_amba_driver.drv) @@ -2375,7 +2367,7 @@ static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, INIT_LIST_HEAD(&dmadev->channels); /* - * Register as many many memcpy as we have physical channels, + * Register as many memcpy as we have physical channels, * we won't always be able to use all but the code will have * to cope with that situation. */ @@ -2760,7 +2752,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) pl08x->memcpy.dev = &adev->dev; pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; - pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt; pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; pl08x->memcpy.device_issue_pending = pl08x_issue_pending; pl08x->memcpy.device_config = pl08x_config; @@ -2787,8 +2778,6 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) pl08x->slave.dev = &adev->dev; pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources; - pl08x->slave.device_prep_dma_interrupt = - pl08x_prep_dma_interrupt; pl08x->slave.device_tx_status = pl08x_dma_tx_status; pl08x->slave.device_issue_pending = pl08x_issue_pending; pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; @@ -2866,8 +2855,8 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) } /* Initialize physical channels */ - pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)), - GFP_KERNEL); + pl08x->phy_chans = kcalloc(vd->channels, sizeof(*pl08x->phy_chans), + GFP_KERNEL); if (!pl08x->phy_chans) { ret = -ENOMEM; goto out_no_phychans; diff --git a/drivers/dma/amd/Kconfig b/drivers/dma/amd/Kconfig new file mode 100644 index 000000000000..00d874872a8f --- /dev/null +++ b/drivers/dma/amd/Kconfig @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: GPL-2.0-only +# + +config AMD_AE4DMA + tristate "AMD AE4DMA Engine" + depends on (X86_64 || COMPILE_TEST) && PCI + depends on AMD_PTDMA + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD AE4DMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. + +config AMD_PTDMA + tristate "AMD PassThru DMA Engine" + depends on X86_64 && PCI + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the AMD PTDMA controller. This controller + provides DMA capabilities to perform high bandwidth memory to + memory and IO copy operations. It performs DMA transfer through + queue-based descriptor management. This DMA controller is intended + to be used with AMD Non-Transparent Bridge devices and not for + general purpose peripheral DMA. + +config AMD_QDMA + tristate "AMD Queue-based DMA" + depends on HAS_IOMEM + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + select REGMAP_MMIO + help + Enable support for the AMD Queue-based DMA subsystem. The primary + mechanism to transfer data using the QDMA is for the QDMA engine to + operate on instructions (descriptors) provided by the host operating + system. Using the descriptors, the QDMA can move data in either the + Host to Card (H2C) direction or the Card to Host (C2H) direction. diff --git a/drivers/dma/amd/Makefile b/drivers/dma/amd/Makefile new file mode 100644 index 000000000000..11278c06374d --- /dev/null +++ b/drivers/dma/amd/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_AMD_AE4DMA) += ae4dma/ +obj-$(CONFIG_AMD_PTDMA) += ptdma/ +obj-$(CONFIG_AMD_QDMA) += qdma/ diff --git a/drivers/dma/amd/ae4dma/Makefile b/drivers/dma/amd/ae4dma/Makefile new file mode 100644 index 000000000000..e918f85a80ec --- /dev/null +++ b/drivers/dma/amd/ae4dma/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# AMD AE4DMA driver +# + +obj-$(CONFIG_AMD_AE4DMA) += ae4dma.o + +ae4dma-objs := ae4dma-dev.o + +ae4dma-$(CONFIG_PCI) += ae4dma-pci.o diff --git a/drivers/dma/amd/ae4dma/ae4dma-dev.c b/drivers/dma/amd/ae4dma/ae4dma-dev.c new file mode 100644 index 000000000000..8de3bef41b58 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-dev.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com> + */ + +#include "ae4dma.h" + +static unsigned int max_hw_q = 1; +module_param(max_hw_q, uint, 0444); +MODULE_PARM_DESC(max_hw_q, "max hw queues supported by engine (any non-zero value, default: 1)"); + +static void ae4_pending_work(struct work_struct *work) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(work, struct ae4_cmd_queue, p_work.work); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct pt_cmd *cmd; + u32 cridx; + + for (;;) { + wait_event_interruptible(ae4cmd_q->q_w, + ((atomic64_read(&ae4cmd_q->done_cnt)) < + atomic64_read(&ae4cmd_q->intr_cnt))); + + atomic64_inc(&ae4cmd_q->done_cnt); + + mutex_lock(&ae4cmd_q->cmd_lock); + cridx = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); + while ((ae4cmd_q->dridx != cridx) && !list_empty(&ae4cmd_q->cmd)) { + cmd = list_first_entry(&ae4cmd_q->cmd, struct pt_cmd, entry); + list_del(&cmd->entry); + + ae4_check_status_error(ae4cmd_q, ae4cmd_q->dridx); + cmd->pt_cmd_callback(cmd->data, cmd->ret); + + ae4cmd_q->q_cmd_count--; + ae4cmd_q->dridx = (ae4cmd_q->dridx + 1) % CMD_Q_LEN; + + complete_all(&ae4cmd_q->cmp); + } + mutex_unlock(&ae4cmd_q->cmd_lock); + } +} + +static irqreturn_t ae4_core_irq_handler(int irq, void *data) +{ + struct ae4_cmd_queue *ae4cmd_q = data; + struct pt_cmd_queue *cmd_q; + struct pt_device *pt; + u32 status; + + cmd_q = &ae4cmd_q->cmd_q; + pt = cmd_q->pt; + + pt->total_interrupts++; + atomic64_inc(&ae4cmd_q->intr_cnt); + + status = readl(cmd_q->reg_control + AE4_INTR_STS_OFF); + if (status & BIT(0)) { + status &= GENMASK(31, 1); + writel(status, cmd_q->reg_control + AE4_INTR_STS_OFF); + } + + wake_up(&ae4cmd_q->q_w); + + return IRQ_HANDLED; +} + +void ae4_destroy_work(struct ae4_device *ae4) +{ + struct ae4_cmd_queue *ae4cmd_q; + int i; + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + if (!ae4cmd_q->pws) + break; + + cancel_delayed_work_sync(&ae4cmd_q->p_work); + destroy_workqueue(ae4cmd_q->pws); + } +} + +int ae4_core_init(struct ae4_device *ae4) +{ + struct pt_device *pt = &ae4->pt; + struct ae4_cmd_queue *ae4cmd_q; + struct device *dev = pt->dev; + struct pt_cmd_queue *cmd_q; + int i, ret = 0; + + writel(max_hw_q, pt->io_regs); + + for (i = 0; i < max_hw_q; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + ae4cmd_q->id = ae4->cmd_q_count; + ae4->cmd_q_count++; + + cmd_q = &ae4cmd_q->cmd_q; + cmd_q->pt = pt; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + ret = devm_request_irq(dev, ae4->ae4_irq[i], ae4_core_irq_handler, 0, + dev_name(pt->dev), ae4cmd_q); + if (ret) + return ret; + + cmd_q->qsize = Q_SIZE(sizeof(struct ae4dma_desc)); + + cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) + return -ENOMEM; + } + + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + + cmd_q = &ae4cmd_q->cmd_q; + + cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ); + + /* Update the device registers with queue information. */ + writel(CMD_Q_LEN, cmd_q->reg_control + AE4_MAX_IDX_OFF); + + cmd_q->qdma_tail = cmd_q->qbase_dma; + writel(lower_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_L_OFF); + writel(upper_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_H_OFF); + + INIT_LIST_HEAD(&ae4cmd_q->cmd); + init_waitqueue_head(&ae4cmd_q->q_w); + + ae4cmd_q->pws = alloc_ordered_workqueue("ae4dma_%d", WQ_MEM_RECLAIM, ae4cmd_q->id); + if (!ae4cmd_q->pws) { + ae4_destroy_work(ae4); + return -ENOMEM; + } + INIT_DELAYED_WORK(&ae4cmd_q->p_work, ae4_pending_work); + queue_delayed_work(ae4cmd_q->pws, &ae4cmd_q->p_work, usecs_to_jiffies(100)); + + init_completion(&ae4cmd_q->cmp); + } + + ret = pt_dmaengine_register(pt); + if (ret) + ae4_destroy_work(ae4); + else + ptdma_debugfs_setup(pt); + + return ret; +} diff --git a/drivers/dma/amd/ae4dma/ae4dma-pci.c b/drivers/dma/amd/ae4dma/ae4dma-pci.c new file mode 100644 index 000000000000..2c63907db228 --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma-pci.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com> + */ + +#include "ae4dma.h" + +static int ae4_get_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + int i, v, ret; + + pdev = to_pci_dev(dev); + + for (v = 0; v < ARRAY_SIZE(ae4_msix->msix_entry); v++) + ae4_msix->msix_entry[v].entry = v; + + ret = pci_alloc_irq_vectors(pdev, v, v, PCI_IRQ_MSIX); + if (ret != v) { + if (ret > 0) + pci_free_irq_vectors(pdev); + + dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + dev_err(dev, "could not enable MSI (%d)\n", ret); + return ret; + } + + ret = pci_irq_vector(pdev, 0); + if (ret < 0) { + pci_free_irq_vectors(pdev); + return ret; + } + + for (i = 0; i < MAX_AE4_HW_QUEUES; i++) + ae4->ae4_irq[i] = ret; + + } else { + ae4_msix->msix_count = ret; + for (i = 0; i < ae4_msix->msix_count; i++) + ae4->ae4_irq[i] = pci_irq_vector(pdev, i); + } + + return ret; +} + +static void ae4_free_irqs(struct ae4_device *ae4) +{ + struct ae4_msix *ae4_msix = ae4->ae4_msix; + struct pt_device *pt = &ae4->pt; + struct device *dev = pt->dev; + struct pci_dev *pdev; + + pdev = to_pci_dev(dev); + + if (ae4_msix && (ae4_msix->msix_count || ae4->ae4_irq[MAX_AE4_HW_QUEUES - 1])) + pci_free_irq_vectors(pdev); +} + +static void ae4_deinit(struct ae4_device *ae4) +{ + ae4_free_irqs(ae4); +} + +static int ae4_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct device *dev = &pdev->dev; + struct ae4_device *ae4; + struct pt_device *pt; + int bar_mask; + int ret = 0; + + ae4 = devm_kzalloc(dev, sizeof(*ae4), GFP_KERNEL); + if (!ae4) + return -ENOMEM; + + ae4->ae4_msix = devm_kzalloc(dev, sizeof(struct ae4_msix), GFP_KERNEL); + if (!ae4->ae4_msix) + return -ENOMEM; + + ret = pcim_enable_device(pdev); + if (ret) + goto ae4_error; + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ae4dma"); + if (ret) + goto ae4_error; + + pt = &ae4->pt; + pt->dev = dev; + pt->ver = AE4_DMA_VERSION; + + pt->io_regs = pcim_iomap_table(pdev)[0]; + if (!pt->io_regs) { + ret = -ENOMEM; + goto ae4_error; + } + + ret = ae4_get_irqs(ae4); + if (ret < 0) + goto ae4_error; + + pci_set_master(pdev); + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + + dev_set_drvdata(dev, ae4); + + ret = ae4_core_init(ae4); + if (ret) + goto ae4_error; + + return 0; + +ae4_error: + ae4_deinit(ae4); + + return ret; +} + +static void ae4_pci_remove(struct pci_dev *pdev) +{ + struct ae4_device *ae4 = dev_get_drvdata(&pdev->dev); + + ae4_destroy_work(ae4); + ae4_deinit(ae4); +} + +static const struct pci_device_id ae4_pci_table[] = { + { PCI_VDEVICE(AMD, 0x149B), }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, ae4_pci_table); + +static struct pci_driver ae4_pci_driver = { + .name = "ae4dma", + .id_table = ae4_pci_table, + .probe = ae4_pci_probe, + .remove = ae4_pci_remove, +}; + +module_pci_driver(ae4_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD AE4DMA driver"); diff --git a/drivers/dma/amd/ae4dma/ae4dma.h b/drivers/dma/amd/ae4dma/ae4dma.h new file mode 100644 index 000000000000..57f6048726bb --- /dev/null +++ b/drivers/dma/amd/ae4dma/ae4dma.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD AE4DMA driver + * + * Copyright (c) 2024, Advanced Micro Devices, Inc. + * All Rights Reserved. + * + * Author: Basavaraj Natikar <Basavaraj.Natikar@amd.com> + */ +#ifndef __AE4DMA_H__ +#define __AE4DMA_H__ + +#include <linux/device.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/list.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/wait.h> + +#include "../ptdma/ptdma.h" +#include "../../virt-dma.h" + +#define MAX_AE4_HW_QUEUES 16 + +#define AE4_DESC_COMPLETED 0x03 + +#define AE4_MAX_IDX_OFF 0x08 +#define AE4_RD_IDX_OFF 0x0c +#define AE4_WR_IDX_OFF 0x10 +#define AE4_INTR_STS_OFF 0x14 +#define AE4_Q_BASE_L_OFF 0x18 +#define AE4_Q_BASE_H_OFF 0x1c +#define AE4_Q_SZ 0x20 + +#define AE4_DMA_VERSION 4 +#define CMD_AE4_DESC_DW0_VAL 2 + +#define AE4_TIME_OUT 5000 + +struct ae4_msix { + int msix_count; + struct msix_entry msix_entry[MAX_AE4_HW_QUEUES]; +}; + +struct ae4_cmd_queue { + struct ae4_device *ae4; + struct pt_cmd_queue cmd_q; + struct list_head cmd; + /* protect command operations */ + struct mutex cmd_lock; + struct delayed_work p_work; + struct workqueue_struct *pws; + struct completion cmp; + wait_queue_head_t q_w; + atomic64_t intr_cnt; + atomic64_t done_cnt; + u64 q_cmd_count; + u32 dridx; + u32 tail_wi; + u32 id; +}; + +union dwou { + u32 dw0; + struct dword0 { + u8 byte0; + u8 byte1; + u16 timestamp; + } dws; +}; + +struct dword1 { + u8 status; + u8 err_code; + u16 desc_id; +}; + +struct ae4dma_desc { + union dwou dwouv; + struct dword1 dw1; + u32 length; + u32 rsvd; + u32 src_hi; + u32 src_lo; + u32 dst_hi; + u32 dst_lo; +}; + +struct ae4_device { + struct pt_device pt; + struct ae4_msix *ae4_msix; + struct ae4_cmd_queue ae4cmd_q[MAX_AE4_HW_QUEUES]; + unsigned int ae4_irq[MAX_AE4_HW_QUEUES]; + unsigned int cmd_q_count; +}; + +int ae4_core_init(struct ae4_device *ae4); +void ae4_destroy_work(struct ae4_device *ae4); +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx); +#endif diff --git a/drivers/dma/amd/ptdma/Makefile b/drivers/dma/amd/ptdma/Makefile new file mode 100644 index 000000000000..ce5410268a9a --- /dev/null +++ b/drivers/dma/amd/ptdma/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# AMD Passthru DMA driver +# + +obj-$(CONFIG_AMD_PTDMA) += ptdma.o + +ptdma-objs := ptdma-dev.o ptdma-dmaengine.o ptdma-debugfs.o + +ptdma-$(CONFIG_PCI) += ptdma-pci.o diff --git a/drivers/dma/amd/ptdma/ptdma-debugfs.c b/drivers/dma/amd/ptdma/ptdma-debugfs.c new file mode 100644 index 000000000000..c7c90bbf6fd8 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-debugfs.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthrough DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta <sanju.mehta@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + */ + +#include <linux/debugfs.h> +#include <linux/seq_file.h> + +#include "ptdma.h" +#include "../ae4dma/ae4dma.h" + +/* DebugFS helpers */ +#define RI_VERSION_NUM 0x0000003F + +#define RI_NUM_VQM 0x00078000 +#define RI_NVQM_SHIFT 15 + +static int pt_debugfs_info_show(struct seq_file *s, void *p) +{ + struct pt_device *pt = s->private; + struct ae4_device *ae4; + unsigned int regval; + + seq_printf(s, "Device name: %s\n", dev_name(pt->dev)); + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + seq_printf(s, " # Queues: %d\n", ae4->cmd_q_count); + seq_printf(s, " # Cmds per queue: %d\n", CMD_Q_LEN); + } else { + seq_printf(s, " # Queues: %d\n", 1); + seq_printf(s, " # Cmds: %d\n", pt->cmd_count); + } + + regval = ioread32(pt->io_regs + CMD_PT_VERSION); + + seq_printf(s, " Version: %d\n", regval & RI_VERSION_NUM); + seq_puts(s, " Engines:"); + seq_puts(s, "\n"); + seq_printf(s, " Queues: %d\n", (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT); + + return 0; +} + +/* + * Return a formatted buffer containing the current + * statistics of queue for PTDMA + */ +static int pt_debugfs_stats_show(struct seq_file *s, void *p) +{ + struct pt_device *pt = s->private; + + seq_printf(s, "Total Interrupts Handled: %ld\n", pt->total_interrupts); + + return 0; +} + +static int pt_debugfs_queue_show(struct seq_file *s, void *p) +{ + struct pt_cmd_queue *cmd_q = s->private; + struct pt_device *pt; + unsigned int regval; + + if (!cmd_q) + return 0; + + seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops); + + pt = cmd_q->pt; + if (pt->ver == AE4_DMA_VERSION) { + regval = readl(cmd_q->reg_control + 0x4); + seq_printf(s, " Enabled Interrupts:: status 0x%x\n", regval); + } else { + regval = ioread32(cmd_q->reg_control + 0x000C); + + seq_puts(s, " Enabled Interrupts:"); + if (regval & INT_EMPTY_QUEUE) + seq_puts(s, " EMPTY"); + if (regval & INT_QUEUE_STOPPED) + seq_puts(s, " STOPPED"); + if (regval & INT_ERROR) + seq_puts(s, " ERROR"); + if (regval & INT_COMPLETION) + seq_puts(s, " COMPLETION"); + seq_puts(s, "\n"); + } + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_info); +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_queue); +DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats); + +void ptdma_debugfs_setup(struct pt_device *pt) +{ + struct dentry *debugfs_q_instance; + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + char name[30]; + int i; + + if (!debugfs_initialized()) + return; + + debugfs_create_file("info", 0400, pt->dma_dev.dbg_dev_root, pt, + &pt_debugfs_info_fops); + + debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt, + &pt_debugfs_stats_fops); + + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + for (i = 0; i < ae4->cmd_q_count; i++) { + ae4cmd_q = &ae4->ae4cmd_q[i]; + cmd_q = &ae4cmd_q->cmd_q; + + memset(name, 0, sizeof(name)); + snprintf(name, 29, "q%d", ae4cmd_q->id); + + debugfs_q_instance = + debugfs_create_dir(name, pt->dma_dev.dbg_dev_root); + + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } + } else { + debugfs_q_instance = + debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); + cmd_q = &pt->cmd_q; + debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, + &pt_debugfs_queue_fops); + } +} +EXPORT_SYMBOL_GPL(ptdma_debugfs_setup); diff --git a/drivers/dma/amd/ptdma/ptdma-dev.c b/drivers/dma/amd/ptdma/ptdma-dev.c new file mode 100644 index 000000000000..a2bf13ff18b6 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-dev.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta <sanju.mehta@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + */ + +#include <linux/bitfield.h> +#include <linux/dma-mapping.h> +#include <linux/debugfs.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> + +#include "ptdma.h" + +/* Human-readable error strings */ +static char *pt_error_codes[] = { + "", + "ERR 01: ILLEGAL_ENGINE", + "ERR 03: ILLEGAL_FUNCTION_TYPE", + "ERR 04: ILLEGAL_FUNCTION_MODE", + "ERR 06: ILLEGAL_FUNCTION_SIZE", + "ERR 08: ILLEGAL_FUNCTION_RSVD", + "ERR 09: ILLEGAL_BUFFER_LENGTH", + "ERR 10: VLSB_FAULT", + "ERR 11: ILLEGAL_MEM_ADDR", + "ERR 12: ILLEGAL_MEM_SEL", + "ERR 13: ILLEGAL_CONTEXT_ID", + "ERR 15: 0xF Reserved", + "ERR 18: CMD_TIMEOUT", + "ERR 19: IDMA0_AXI_SLVERR", + "ERR 20: IDMA0_AXI_DECERR", + "ERR 21: 0x15 Reserved", + "ERR 22: IDMA1_AXI_SLAVE_FAULT", + "ERR 23: IDMA1_AIXI_DECERR", + "ERR 24: 0x18 Reserved", + "ERR 27: 0x1B Reserved", + "ERR 38: ODMA0_AXI_SLVERR", + "ERR 39: ODMA0_AXI_DECERR", + "ERR 40: 0x28 Reserved", + "ERR 41: ODMA1_AXI_SLVERR", + "ERR 42: ODMA1_AXI_DECERR", + "ERR 43: LSB_PARITY_ERR", +}; + +static void pt_log_error(struct pt_device *d, int e) +{ + dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e); +} + +void pt_start_queue(struct pt_cmd_queue *cmd_q) +{ + /* Turn on the run bit */ + iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control); +} + +void pt_stop_queue(struct pt_cmd_queue *cmd_q) +{ + /* Turn off the run bit */ + iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control); +} + +static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q) +{ + bool soc = FIELD_GET(DWORD0_SOC, desc->dw0); + u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx]; + u32 tail; + unsigned long flags; + + if (soc) { + desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0); + desc->dw0 &= ~DWORD0_SOC; + } + spin_lock_irqsave(&cmd_q->q_lock, flags); + + /* Copy 32-byte command descriptor to hw queue. */ + memcpy(q_desc, desc, 32); + cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN; + + /* The data used by this command must be flushed to memory */ + wmb(); + + /* Write the new tail address back to the queue register */ + tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); + iowrite32(tail, cmd_q->reg_control + 0x0004); + + /* Turn the queue back on using our cached control register */ + pt_start_queue(cmd_q); + spin_unlock_irqrestore(&cmd_q->q_lock, flags); + + return 0; +} + +int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine) +{ + struct ptdma_desc desc; + struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q); + + cmd_q->cmd_error = 0; + cmd_q->total_pt_ops++; + memset(&desc, 0, sizeof(desc)); + desc.dw0 = CMD_DESC_DW0_VAL; + desc.length = pt_engine->src_len; + desc.src_lo = lower_32_bits(pt_engine->src_dma); + desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma); + desc.dst_lo = lower_32_bits(pt_engine->dst_dma); + desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma); + + if (cmd_q->int_en) + pt_core_enable_queue_interrupts(pt); + else + pt_core_disable_queue_interrupts(pt); + + return pt_core_execute_cmd(&desc, cmd_q); +} + +static void pt_do_cmd_complete(unsigned long data) +{ + struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data; + struct pt_cmd *cmd = tdata->cmd; + struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q; + u32 tail; + + if (cmd_q->cmd_error) { + /* + * Log the error and flush the queue by + * moving the head pointer + */ + tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); + pt_log_error(cmd_q->pt, cmd_q->cmd_error); + iowrite32(tail, cmd_q->reg_control + 0x0008); + } + + cmd->pt_cmd_callback(cmd->data, cmd->ret); +} + +void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q) +{ + u32 status; + + status = ioread32(cmd_q->reg_control + 0x0010); + if (status) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100); + cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104); + + /* On error, only save the first error value */ + if ((status & INT_ERROR) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + /* Acknowledge the completion */ + iowrite32(status, cmd_q->reg_control + 0x0010); + pt_do_cmd_complete((ulong)&pt->tdata); + } +} + +static irqreturn_t pt_core_irq_handler(int irq, void *data) +{ + struct pt_device *pt = data; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + + pt_core_disable_queue_interrupts(pt); + pt->total_interrupts++; + pt_check_status_trans(pt, cmd_q); + pt_core_enable_queue_interrupts(pt); + return IRQ_HANDLED; +} + +int pt_core_init(struct pt_device *pt) +{ + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + u32 dma_addr_lo, dma_addr_hi; + struct device *dev = pt->dev; + struct dma_pool *dma_pool; + int ret; + + /* Allocate a dma pool for the queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev)); + + dma_pool = dma_pool_create(dma_pool_name, dev, + PT_DMAPOOL_MAX_SIZE, + PT_DMAPOOL_ALIGN, 0); + if (!dma_pool) + return -ENOMEM; + + /* ptdma core initialisation */ + iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET); + iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET); + iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET); + iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET); + iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET); + + cmd_q->pt = pt; + cmd_q->dma_pool = dma_pool; + spin_lock_init(&cmd_q->q_lock); + + /* Page alignment satisfies our needs for N <= 128 */ + cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); + cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, + &cmd_q->qbase_dma, + GFP_KERNEL); + if (!cmd_q->qbase) { + dev_err(dev, "unable to allocate command queue\n"); + ret = -ENOMEM; + goto e_destroy_pool; + } + + cmd_q->qidx = 0; + + /* Preset some register values */ + cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR; + + /* Turn off the queues and disable interrupts until ready */ + pt_core_disable_queue_interrupts(pt); + + cmd_q->qcontrol = 0; /* Start with nothing */ + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + ioread32(cmd_q->reg_control + 0x0104); + ioread32(cmd_q->reg_control + 0x0100); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + + /* Request an irq */ + ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_free_dma; + } + + /* Update the device registers with queue information. */ + cmd_q->qcontrol &= ~CMD_Q_SIZE; + cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL); + + cmd_q->qdma_tail = cmd_q->qbase_dma; + dma_addr_lo = lower_32_bits(cmd_q->qdma_tail); + iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004); + iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008); + + dma_addr_hi = upper_32_bits(cmd_q->qdma_tail); + cmd_q->qcontrol |= (dma_addr_hi << 16); + iowrite32(cmd_q->qcontrol, cmd_q->reg_control); + + pt_core_enable_queue_interrupts(pt); + + /* Register the DMA engine support */ + ret = pt_dmaengine_register(pt); + if (ret) + goto e_free_irq; + + /* Set up debugfs entries */ + ptdma_debugfs_setup(pt); + + return 0; + +e_free_irq: + free_irq(pt->pt_irq, pt); + +e_free_dma: + dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma); + +e_destroy_pool: + dma_pool_destroy(pt->cmd_q.dma_pool); + + return ret; +} + +void pt_core_destroy(struct pt_device *pt) +{ + struct device *dev = pt->dev; + struct pt_cmd_queue *cmd_q = &pt->cmd_q; + struct pt_cmd *cmd; + + /* Unregister the DMA engine */ + pt_dmaengine_unregister(pt); + + /* Disable and clear interrupts */ + pt_core_disable_queue_interrupts(pt); + + /* Turn off the run bit */ + pt_stop_queue(cmd_q); + + /* Clear the interrupt status */ + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + ioread32(cmd_q->reg_control + 0x0104); + ioread32(cmd_q->reg_control + 0x0100); + + free_irq(pt->pt_irq, pt); + + dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, + cmd_q->qbase_dma); + + /* Flush the cmd queue */ + while (!list_empty(&pt->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry); + list_del(&cmd->entry); + cmd->pt_cmd_callback(cmd->data, -ENODEV); + } +} diff --git a/drivers/dma/amd/ptdma/ptdma-dmaengine.c b/drivers/dma/amd/ptdma/ptdma-dmaengine.c new file mode 100644 index 000000000000..628c49ce5de9 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-dmaengine.c @@ -0,0 +1,659 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthrough DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta <sanju.mehta@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + */ + +#include <linux/bitfield.h> +#include "ptdma.h" +#include "../ae4dma/ae4dma.h" +#include "../../dmaengine.h" + +static char *ae4_error_codes[] = { + "", + "ERR 01: INVALID HEADER DW0", + "ERR 02: INVALID STATUS", + "ERR 03: INVALID LENGTH - 4 BYTE ALIGNMENT", + "ERR 04: INVALID SRC ADDR - 4 BYTE ALIGNMENT", + "ERR 05: INVALID DST ADDR - 4 BYTE ALIGNMENT", + "ERR 06: INVALID ALIGNMENT", + "ERR 07: INVALID DESCRIPTOR", +}; + +static void ae4_log_error(struct pt_device *d, int e) +{ + /* ERR 01 - 07 represents Invalid AE4 errors */ + if (e <= 7) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", ae4_error_codes[e], e); + /* ERR 08 - 15 represents Invalid Descriptor errors */ + else if (e > 7 && e <= 15) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "INVALID DESCRIPTOR", e); + /* ERR 16 - 31 represents Firmware errors */ + else if (e > 15 && e <= 31) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FIRMWARE ERROR", e); + /* ERR 32 - 63 represents Fatal errors */ + else if (e > 31 && e <= 63) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "FATAL ERROR", e); + /* ERR 64 - 255 represents PTE errors */ + else if (e > 63 && e <= 255) + dev_info(d->dev, "AE4DMA error: %s (0x%x)\n", "PTE ERROR", e); + else + dev_info(d->dev, "Unknown AE4DMA error"); +} + +void ae4_check_status_error(struct ae4_cmd_queue *ae4cmd_q, int idx) +{ + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + struct ae4dma_desc desc; + u8 status; + + memcpy(&desc, &cmd_q->qbase[idx], sizeof(struct ae4dma_desc)); + status = desc.dw1.status; + if (status && status != AE4_DESC_COMPLETED) { + cmd_q->cmd_error = desc.dw1.err_code; + if (cmd_q->cmd_error) + ae4_log_error(cmd_q->pt, cmd_q->cmd_error); + } +} +EXPORT_SYMBOL_GPL(ae4_check_status_error); + +static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) +{ + return container_of(dma_chan, struct pt_dma_chan, vc.chan); +} + +static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct pt_dma_desc, vd); +} + +static void pt_free_chan_resources(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + + vchan_free_chan_resources(&chan->vc); +} + +static void pt_synchronize(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + + vchan_synchronize(&chan->vc); +} + +static void pt_do_cleanup(struct virt_dma_desc *vd) +{ + struct pt_dma_desc *desc = to_pt_desc(vd); + struct pt_device *pt = desc->pt; + + kmem_cache_free(pt->dma_desc_cache, desc); +} + +static struct pt_cmd_queue *pt_get_cmd_queue(struct pt_device *pt, struct pt_dma_chan *chan) +{ + struct ae4_cmd_queue *ae4cmd_q; + struct pt_cmd_queue *cmd_q; + struct ae4_device *ae4; + + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + cmd_q = &ae4cmd_q->cmd_q; + } else { + cmd_q = &pt->cmd_q; + } + + return cmd_q; +} + +static int ae4_core_execute_cmd(struct ae4dma_desc *desc, struct ae4_cmd_queue *ae4cmd_q) +{ + bool soc = FIELD_GET(DWORD0_SOC, desc->dwouv.dw0); + struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q; + + if (soc) { + desc->dwouv.dw0 |= FIELD_PREP(DWORD0_IOC, desc->dwouv.dw0); + desc->dwouv.dw0 &= ~DWORD0_SOC; + } + + mutex_lock(&ae4cmd_q->cmd_lock); + memcpy(&cmd_q->qbase[ae4cmd_q->tail_wi], desc, sizeof(struct ae4dma_desc)); + ae4cmd_q->q_cmd_count++; + ae4cmd_q->tail_wi = (ae4cmd_q->tail_wi + 1) % CMD_Q_LEN; + writel(ae4cmd_q->tail_wi, cmd_q->reg_control + AE4_WR_IDX_OFF); + mutex_unlock(&ae4cmd_q->cmd_lock); + + wake_up(&ae4cmd_q->q_w); + + return 0; +} + +static int pt_core_perform_passthru_ae4(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + struct ae4dma_desc desc; + + cmd_q->cmd_error = 0; + cmd_q->total_pt_ops++; + memset(&desc, 0, sizeof(desc)); + desc.dwouv.dws.byte0 = CMD_AE4_DESC_DW0_VAL; + + desc.dw1.status = 0; + desc.dw1.err_code = 0; + desc.dw1.desc_id = 0; + + desc.length = pt_engine->src_len; + + desc.src_lo = upper_32_bits(pt_engine->src_dma); + desc.src_hi = lower_32_bits(pt_engine->src_dma); + desc.dst_lo = upper_32_bits(pt_engine->dst_dma); + desc.dst_hi = lower_32_bits(pt_engine->dst_dma); + + return ae4_core_execute_cmd(&desc, ae4cmd_q); +} + +static int pt_dma_start_desc(struct pt_dma_desc *desc, struct pt_dma_chan *chan) +{ + struct pt_passthru_engine *pt_engine; + struct pt_device *pt; + struct pt_cmd *pt_cmd; + struct pt_cmd_queue *cmd_q; + + desc->issued_to_hw = 1; + + pt_cmd = &desc->pt_cmd; + pt = pt_cmd->pt; + + cmd_q = pt_get_cmd_queue(pt, chan); + + pt_engine = &pt_cmd->passthru; + + pt->tdata.cmd = pt_cmd; + + /* Execute the command */ + if (pt->ver == AE4_DMA_VERSION) + pt_cmd->ret = pt_core_perform_passthru_ae4(cmd_q, pt_engine); + else + pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); + + return 0; +} + +static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan) +{ + /* Get the next DMA descriptor on the active list */ + struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); + + return vd ? to_pt_desc(vd) : NULL; +} + +static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, + struct pt_dma_desc *desc) +{ + struct dma_async_tx_descriptor *tx_desc; + struct virt_dma_desc *vd; + struct pt_device *pt; + unsigned long flags; + + pt = chan->pt; + /* Loop over descriptors until one is found with commands */ + do { + if (desc) { + if (!desc->issued_to_hw) { + /* No errors, keep going */ + if (desc->status != DMA_ERROR) + return desc; + } + + tx_desc = &desc->vd.tx; + vd = &desc->vd; + } else { + tx_desc = NULL; + } + + spin_lock_irqsave(&chan->vc.lock, flags); + + if (pt->ver != AE4_DMA_VERSION && desc) { + if (desc->status != DMA_COMPLETE) { + if (desc->status != DMA_ERROR) + desc->status = DMA_COMPLETE; + + dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); + list_del(&desc->vd.node); + } else { + /* Don't handle it twice */ + tx_desc = NULL; + } + } + + desc = pt_next_dma_desc(chan); + + spin_unlock_irqrestore(&chan->vc.lock, flags); + + if (pt->ver != AE4_DMA_VERSION && tx_desc) { + dmaengine_desc_get_callback_invoke(tx_desc, NULL); + dma_run_dependencies(tx_desc); + vchan_vdesc_fini(vd); + } + } while (desc); + + return NULL; +} + +static inline bool ae4_core_queue_full(struct pt_cmd_queue *cmd_q) +{ + u32 front_wi = readl(cmd_q->reg_control + AE4_WR_IDX_OFF); + u32 rear_ri = readl(cmd_q->reg_control + AE4_RD_IDX_OFF); + + if (((MAX_CMD_QLEN + front_wi - rear_ri) % MAX_CMD_QLEN) >= (MAX_CMD_QLEN - 1)) + return true; + + return false; +} + +static void pt_cmd_callback(void *data, int err) +{ + struct pt_dma_desc *desc = data; + struct ae4_cmd_queue *ae4cmd_q; + struct dma_chan *dma_chan; + struct pt_dma_chan *chan; + struct ae4_device *ae4; + struct pt_device *pt; + int ret; + + if (err == -EINPROGRESS) + return; + + dma_chan = desc->vd.tx.chan; + chan = to_pt_chan(dma_chan); + pt = chan->pt; + + if (err) + desc->status = DMA_ERROR; + + while (true) { + if (pt->ver == AE4_DMA_VERSION) { + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + + if (ae4cmd_q->q_cmd_count >= (CMD_Q_LEN - 1) || + ae4_core_queue_full(&ae4cmd_q->cmd_q)) { + wake_up(&ae4cmd_q->q_w); + + if (wait_for_completion_timeout(&ae4cmd_q->cmp, + msecs_to_jiffies(AE4_TIME_OUT)) + == 0) { + dev_err(pt->dev, "TIMEOUT %d:\n", ae4cmd_q->id); + break; + } + + reinit_completion(&ae4cmd_q->cmp); + continue; + } + } + + /* Check for DMA descriptor completion */ + desc = pt_handle_active_desc(chan, desc); + + /* Don't submit cmd if no descriptor or DMA is paused */ + if (!desc) + break; + + ret = pt_dma_start_desc(desc, chan); + if (!ret) + break; + + desc->status = DMA_ERROR; + } +} + +static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, + unsigned long flags) +{ + struct pt_dma_desc *desc; + + desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT); + if (!desc) + return NULL; + + vchan_tx_prep(&chan->vc, &desc->vd, flags); + + desc->pt = chan->pt; + desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT); + desc->issued_to_hw = 0; + desc->status = DMA_IN_PROGRESS; + + return desc; +} + +static void pt_cmd_callback_work(void *data, int err) +{ + struct dma_async_tx_descriptor *tx_desc; + struct pt_dma_desc *desc = data; + struct dma_chan *dma_chan; + struct virt_dma_desc *vd; + struct pt_dma_chan *chan; + unsigned long flags; + + if (!desc) + return; + + dma_chan = desc->vd.tx.chan; + chan = to_pt_chan(dma_chan); + + if (err == -EINPROGRESS) + return; + + tx_desc = &desc->vd.tx; + vd = &desc->vd; + + if (err) + desc->status = DMA_ERROR; + + spin_lock_irqsave(&chan->vc.lock, flags); + if (desc->status != DMA_COMPLETE) { + if (desc->status != DMA_ERROR) + desc->status = DMA_COMPLETE; + + dma_cookie_complete(tx_desc); + dma_descriptor_unmap(tx_desc); + } else { + tx_desc = NULL; + } + spin_unlock_irqrestore(&chan->vc.lock, flags); + + if (tx_desc) { + dmaengine_desc_get_callback_invoke(tx_desc, NULL); + dma_run_dependencies(tx_desc); + list_del(&desc->vd.node); + vchan_vdesc_fini(vd); + } +} + +static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, + dma_addr_t dst, + dma_addr_t src, + unsigned int len, + unsigned long flags) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_passthru_engine *pt_engine; + struct pt_device *pt = chan->pt; + struct ae4_cmd_queue *ae4cmd_q; + struct pt_dma_desc *desc; + struct ae4_device *ae4; + struct pt_cmd *pt_cmd; + + desc = pt_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + pt_cmd = &desc->pt_cmd; + pt_cmd->pt = pt; + pt_engine = &pt_cmd->passthru; + pt_cmd->engine = PT_ENGINE_PASSTHRU; + pt_engine->src_dma = src; + pt_engine->dst_dma = dst; + pt_engine->src_len = len; + pt_cmd->pt_cmd_callback = pt_cmd_callback; + pt_cmd->data = desc; + + desc->len = len; + + if (pt->ver == AE4_DMA_VERSION) { + pt_cmd->pt_cmd_callback = pt_cmd_callback_work; + ae4 = container_of(pt, struct ae4_device, pt); + ae4cmd_q = &ae4->ae4cmd_q[chan->id]; + mutex_lock(&ae4cmd_q->cmd_lock); + list_add_tail(&pt_cmd->entry, &ae4cmd_q->cmd); + mutex_unlock(&ae4cmd_q->cmd_lock); + } + + return desc; +} + +static struct dma_async_tx_descriptor * +pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct pt_dma_desc *desc; + + desc = pt_create_desc(dma_chan, dst, src, len, flags); + if (!desc) + return NULL; + + return &desc->vd.tx; +} + +static struct dma_async_tx_descriptor * +pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc; + + desc = pt_alloc_dma_desc(chan, flags); + if (!desc) + return NULL; + + return &desc->vd.tx; +} + +static void pt_issue_pending(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc; + struct pt_device *pt; + unsigned long flags; + bool engine_is_idle = true; + + pt = chan->pt; + + spin_lock_irqsave(&chan->vc.lock, flags); + + desc = pt_next_dma_desc(chan); + if (desc && pt->ver != AE4_DMA_VERSION) + engine_is_idle = false; + + vchan_issue_pending(&chan->vc); + + desc = pt_next_dma_desc(chan); + + spin_unlock_irqrestore(&chan->vc.lock, flags); + + /* If there was nothing active, start processing */ + if (engine_is_idle && desc) + pt_cmd_callback(desc, 0); +} + +static void pt_check_status_trans_ae4(struct pt_device *pt, struct pt_cmd_queue *cmd_q) +{ + struct ae4_cmd_queue *ae4cmd_q = container_of(cmd_q, struct ae4_cmd_queue, cmd_q); + int i; + + for (i = 0; i < CMD_Q_LEN; i++) + ae4_check_status_error(ae4cmd_q, i); +} + +static enum dma_status +pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct pt_dma_chan *chan = to_pt_chan(c); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + + cmd_q = pt_get_cmd_queue(pt, chan); + + if (pt->ver == AE4_DMA_VERSION) + pt_check_status_trans_ae4(pt, cmd_q); + else + pt_check_status_trans(pt, cmd_q); + + return dma_cookie_status(c, cookie, txstate); +} + +static int pt_pause(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_stop_queue(cmd_q); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + return 0; +} + +static int pt_resume(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_dma_desc *desc = NULL; + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + unsigned long flags; + + spin_lock_irqsave(&chan->vc.lock, flags); + cmd_q = pt_get_cmd_queue(pt, chan); + pt_start_queue(cmd_q); + desc = pt_next_dma_desc(chan); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + /* If there was something active, re-start */ + if (desc) + pt_cmd_callback(desc, 0); + + return 0; +} + +static int pt_terminate_all(struct dma_chan *dma_chan) +{ + struct pt_dma_chan *chan = to_pt_chan(dma_chan); + struct pt_device *pt = chan->pt; + struct pt_cmd_queue *cmd_q; + unsigned long flags; + LIST_HEAD(head); + + cmd_q = pt_get_cmd_queue(pt, chan); + if (pt->ver == AE4_DMA_VERSION) + pt_stop_queue(cmd_q); + else + iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); + + spin_lock_irqsave(&chan->vc.lock, flags); + vchan_get_all_descriptors(&chan->vc, &head); + spin_unlock_irqrestore(&chan->vc.lock, flags); + + vchan_dma_desc_free_list(&chan->vc, &head); + vchan_free_chan_resources(&chan->vc); + + return 0; +} + +int pt_dmaengine_register(struct pt_device *pt) +{ + struct dma_device *dma_dev = &pt->dma_dev; + struct ae4_cmd_queue *ae4cmd_q = NULL; + struct ae4_device *ae4 = NULL; + struct pt_dma_chan *chan; + char *desc_cache_name; + int ret, i; + + if (pt->ver == AE4_DMA_VERSION) + ae4 = container_of(pt, struct ae4_device, pt); + + if (ae4) + pt->pt_dma_chan = devm_kcalloc(pt->dev, ae4->cmd_q_count, + sizeof(*pt->pt_dma_chan), GFP_KERNEL); + else + pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), + GFP_KERNEL); + + if (!pt->pt_dma_chan) + return -ENOMEM; + + desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, + "%s-dmaengine-desc-cache", + dev_name(pt->dev)); + if (!desc_cache_name) + return -ENOMEM; + + pt->dma_desc_cache = kmem_cache_create(desc_cache_name, + sizeof(struct pt_dma_desc), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!pt->dma_desc_cache) + return -ENOMEM; + + dma_dev->dev = pt->dev; + dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; + dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; + dma_dev->directions = DMA_MEM_TO_MEM; + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); + + /* + * PTDMA is intended to be used with the AMD NTB devices, hence + * marking it as DMA_PRIVATE. + */ + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + + INIT_LIST_HEAD(&dma_dev->channels); + + /* Set base and prep routines */ + dma_dev->device_free_chan_resources = pt_free_chan_resources; + dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; + dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt; + dma_dev->device_issue_pending = pt_issue_pending; + dma_dev->device_tx_status = pt_tx_status; + dma_dev->device_pause = pt_pause; + dma_dev->device_resume = pt_resume; + dma_dev->device_terminate_all = pt_terminate_all; + dma_dev->device_synchronize = pt_synchronize; + + if (ae4) { + for (i = 0; i < ae4->cmd_q_count; i++) { + chan = pt->pt_dma_chan + i; + ae4cmd_q = &ae4->ae4cmd_q[i]; + chan->id = ae4cmd_q->id; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } + } else { + chan = pt->pt_dma_chan; + chan->pt = pt; + chan->vc.desc_free = pt_do_cleanup; + vchan_init(&chan->vc, dma_dev); + } + + ret = dma_async_device_register(dma_dev); + if (ret) + goto err_reg; + + return 0; + +err_reg: + kmem_cache_destroy(pt->dma_desc_cache); + + return ret; +} +EXPORT_SYMBOL_GPL(pt_dmaengine_register); + +void pt_dmaengine_unregister(struct pt_device *pt) +{ + struct dma_device *dma_dev = &pt->dma_dev; + + dma_async_device_unregister(dma_dev); + + kmem_cache_destroy(pt->dma_desc_cache); +} diff --git a/drivers/dma/amd/ptdma/ptdma-pci.c b/drivers/dma/amd/ptdma/ptdma-pci.c new file mode 100644 index 000000000000..22739ff0c3c5 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma-pci.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta <sanju.mehta@amd.com> + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + */ + +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +#include <linux/module.h> +#include <linux/pci_ids.h> +#include <linux/pci.h> +#include <linux/spinlock.h> + +#include "ptdma.h" + +struct pt_msix { + int msix_count; + struct msix_entry msix_entry; +}; + +/* + * pt_alloc_struct - allocate and initialize the pt_device struct + * + * @dev: device struct of the PTDMA + */ +static struct pt_device *pt_alloc_struct(struct device *dev) +{ + struct pt_device *pt; + + pt = devm_kzalloc(dev, sizeof(*pt), GFP_KERNEL); + + if (!pt) + return NULL; + pt->dev = dev; + + INIT_LIST_HEAD(&pt->cmd); + + return pt; +} + +static int pt_get_msix_irqs(struct pt_device *pt) +{ + struct pt_msix *pt_msix = pt->pt_msix; + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + pt_msix->msix_entry.entry = 0; + + ret = pci_enable_msix_range(pdev, &pt_msix->msix_entry, 1, 1); + if (ret < 0) + return ret; + + pt_msix->msix_count = ret; + + pt->pt_irq = pt_msix->msix_entry.vector; + + return 0; +} + +static int pt_get_msi_irq(struct pt_device *pt) +{ + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + int ret; + + ret = pci_enable_msi(pdev); + if (ret) + return ret; + + pt->pt_irq = pdev->irq; + + return 0; +} + +static int pt_get_irqs(struct pt_device *pt) +{ + struct device *dev = pt->dev; + int ret; + + ret = pt_get_msix_irqs(pt); + if (!ret) + return 0; + + /* Couldn't get MSI-X vectors, try MSI */ + dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = pt_get_msi_irq(pt); + if (!ret) + return 0; + + /* Couldn't get MSI interrupt */ + dev_err(dev, "could not enable MSI (%d)\n", ret); + + return ret; +} + +static void pt_free_irqs(struct pt_device *pt) +{ + struct pt_msix *pt_msix = pt->pt_msix; + struct device *dev = pt->dev; + struct pci_dev *pdev = to_pci_dev(dev); + + if (pt_msix->msix_count) + pci_disable_msix(pdev); + else if (pt->pt_irq) + pci_disable_msi(pdev); + + pt->pt_irq = 0; +} + +static int pt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct pt_device *pt; + struct pt_msix *pt_msix; + struct device *dev = &pdev->dev; + void __iomem * const *iomap_table; + int bar_mask; + int ret = -ENOMEM; + + pt = pt_alloc_struct(dev); + if (!pt) + goto e_err; + + pt_msix = devm_kzalloc(dev, sizeof(*pt_msix), GFP_KERNEL); + if (!pt_msix) + goto e_err; + + pt->pt_msix = pt_msix; + pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data; + if (!pt->dev_vdata) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } + + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(dev, "pcim_enable_device failed (%d)\n", ret); + goto e_err; + } + + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + ret = pcim_iomap_regions(pdev, bar_mask, "ptdma"); + if (ret) { + dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret); + goto e_err; + } + + iomap_table = pcim_iomap_table(pdev); + if (!iomap_table) { + dev_err(dev, "pcim_iomap_table failed\n"); + ret = -ENOMEM; + goto e_err; + } + + pt->io_regs = iomap_table[pt->dev_vdata->bar]; + if (!pt->io_regs) { + dev_err(dev, "ioremap failed\n"); + ret = -ENOMEM; + goto e_err; + } + + ret = pt_get_irqs(pt); + if (ret) + goto e_err; + + pci_set_master(pdev); + + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); + if (ret) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", + ret); + goto e_err; + } + } + + dev_set_drvdata(dev, pt); + + if (pt->dev_vdata) + ret = pt_core_init(pt); + + if (ret) + goto e_err; + + return 0; + +e_err: + dev_err(dev, "initialization failed ret = %d\n", ret); + + return ret; +} + +static void pt_pci_remove(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct pt_device *pt = dev_get_drvdata(dev); + + if (!pt) + return; + + if (pt->dev_vdata) + pt_core_destroy(pt); + + pt_free_irqs(pt); +} + +static const struct pt_dev_vdata dev_vdata[] = { + { + .bar = 2, + }, +}; + +static const struct pci_device_id pt_pci_table[] = { + { PCI_VDEVICE(AMD, 0x1498), (kernel_ulong_t)&dev_vdata[0] }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, pt_pci_table); + +static struct pci_driver pt_pci_driver = { + .name = "ptdma", + .id_table = pt_pci_table, + .probe = pt_pci_probe, + .remove = pt_pci_remove, +}; + +module_pci_driver(pt_pci_driver); + +MODULE_AUTHOR("Sanjay R Mehta <sanju.mehta@amd.com>"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AMD PassThru DMA driver"); diff --git a/drivers/dma/amd/ptdma/ptdma.h b/drivers/dma/amd/ptdma/ptdma.h new file mode 100644 index 000000000000..ef3f55632107 --- /dev/null +++ b/drivers/dma/amd/ptdma/ptdma.h @@ -0,0 +1,338 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AMD Passthru DMA device driver + * -- Based on the CCP driver + * + * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. + * + * Author: Sanjay R Mehta <sanju.mehta@amd.com> + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * Author: Gary R Hook <gary.hook@amd.com> + */ + +#ifndef __PT_DEV_H__ +#define __PT_DEV_H__ + +#include <linux/device.h> +#include <linux/dmaengine.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/dmapool.h> + +#include "../../virt-dma.h" + +#define MAX_PT_NAME_LEN 16 +#define MAX_DMAPOOL_NAME_LEN 32 + +#define MAX_HW_QUEUES 1 +#define MAX_CMD_QLEN 100 + +#define PT_ENGINE_PASSTHRU 5 + +/* Register Mappings */ +#define IRQ_MASK_REG 0x040 +#define IRQ_STATUS_REG 0x200 + +#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) + +#define CMD_QUEUE_PRIO_OFFSET 0x00 +#define CMD_REQID_CONFIG_OFFSET 0x04 +#define CMD_TIMEOUT_OFFSET 0x08 +#define CMD_PT_VERSION 0x10 + +#define CMD_Q_CONTROL_BASE 0x0000 +#define CMD_Q_TAIL_LO_BASE 0x0004 +#define CMD_Q_HEAD_LO_BASE 0x0008 +#define CMD_Q_INT_ENABLE_BASE 0x000C +#define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 + +#define CMD_Q_STATUS_BASE 0x0100 +#define CMD_Q_INT_STATUS_BASE 0x0104 +#define CMD_Q_DMA_STATUS_BASE 0x0108 +#define CMD_Q_DMA_READ_STATUS_BASE 0x010C +#define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110 +#define CMD_Q_ABORT_BASE 0x0114 +#define CMD_Q_AX_CACHE_BASE 0x0118 + +#define CMD_CONFIG_OFFSET 0x1120 +#define CMD_CLK_GATE_CTL_OFFSET 0x6004 + +#define CMD_DESC_DW0_VAL 0x500012 + +/* Address offset for virtual queue registers */ +#define CMD_Q_STATUS_INCR 0x1000 + +/* Bit masks */ +#define CMD_CONFIG_REQID 0 +#define CMD_TIMEOUT_DISABLE 0 +#define CMD_CLK_DYN_GATING_DIS 0 +#define CMD_CLK_SW_GATE_MODE 0 +#define CMD_CLK_GATE_CTL 0 +#define CMD_QUEUE_PRIO GENMASK(2, 1) +#define CMD_CONFIG_VHB_EN BIT(0) +#define CMD_CLK_DYN_GATING_EN BIT(0) +#define CMD_CLK_HW_GATE_MODE BIT(0) +#define CMD_CLK_GATE_ON_DELAY BIT(12) +#define CMD_CLK_GATE_OFF_DELAY BIT(12) + +#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \ + CMD_CLK_HW_GATE_MODE | \ + CMD_CLK_GATE_ON_DELAY | \ + CMD_CLK_DYN_GATING_EN | \ + CMD_CLK_GATE_OFF_DELAY) + +#define CMD_Q_LEN 32 +#define CMD_Q_RUN BIT(0) +#define CMD_Q_HALT BIT(1) +#define CMD_Q_MEM_LOCATION BIT(2) +#define CMD_Q_SIZE_MASK GENMASK(4, 0) +#define CMD_Q_SIZE GENMASK(7, 3) +#define CMD_Q_SHIFT GENMASK(1, 0) +#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \ + CMD_Q_SIZE_MASK) +#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) +#define Q_DESC_SIZE sizeof(struct ptdma_desc) +#define Q_SIZE(n) (CMD_Q_LEN * (n)) + +#define INT_COMPLETION BIT(0) +#define INT_ERROR BIT(1) +#define INT_QUEUE_STOPPED BIT(2) +#define INT_EMPTY_QUEUE BIT(3) +#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) + +/****** Local Storage Block ******/ +#define LSB_START 0 +#define LSB_END 127 +#define LSB_COUNT (LSB_END - LSB_START + 1) + +#define PT_DMAPOOL_MAX_SIZE 64 +#define PT_DMAPOOL_ALIGN BIT(5) + +#define PT_PASSTHRU_BLOCKSIZE 512 + +struct pt_device; + +struct pt_tasklet_data { + struct completion completion; + struct pt_cmd *cmd; +}; + +/* + * struct pt_passthru_engine - pass-through operation + * without performing DMA mapping + * @mask: mask to be applied to data + * @mask_len: length in bytes of mask + * @src_dma: data to be used for this operation + * @dst_dma: data produced by this operation + * @src_len: length in bytes of data used for this operation + * + * Variables required to be set when calling pt_enqueue_cmd(): + * - bit_mod, byte_swap, src, dst, src_len + * - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP + */ +struct pt_passthru_engine { + dma_addr_t mask; + u32 mask_len; /* In bytes */ + + dma_addr_t src_dma, dst_dma; + u64 src_len; /* In bytes */ +}; + +/* + * struct pt_cmd - PTDMA operation request + * @entry: list element + * @work: work element used for callbacks + * @pt: PT device to be run on + * @ret: operation return code + * @flags: cmd processing flags + * @engine: PTDMA operation to perform (passthru) + * @engine_error: PT engine return code + * @passthru: engine specific structures, refer to specific engine struct below + * @callback: operation completion callback function + * @data: parameter value to be supplied to the callback function + * + * Variables required to be set when calling pt_enqueue_cmd(): + * - engine, callback + * - See the operation structures below for what is required for each + * operation. + */ +struct pt_cmd { + struct list_head entry; + struct work_struct work; + struct pt_device *pt; + int ret; + u32 engine; + u32 engine_error; + struct pt_passthru_engine passthru; + /* Completion callback support */ + void (*pt_cmd_callback)(void *data, int err); + void *data; +}; + +struct pt_dma_desc { + struct virt_dma_desc vd; + struct pt_device *pt; + enum dma_status status; + size_t len; + bool issued_to_hw; + struct pt_cmd pt_cmd; +}; + +struct pt_dma_chan { + struct virt_dma_chan vc; + struct pt_device *pt; + u32 id; +}; + +struct pt_cmd_queue { + struct pt_device *pt; + + /* Queue dma pool */ + struct dma_pool *dma_pool; + + /* Queue base address (not necessarily aligned)*/ + struct ptdma_desc *qbase; + + /* Aligned queue start address (per requirement) */ + spinlock_t q_lock ____cacheline_aligned; + unsigned int qidx; + + unsigned int qsize; + dma_addr_t qbase_dma; + dma_addr_t qdma_tail; + + unsigned int active; + unsigned int suspended; + + /* Interrupt flag */ + bool int_en; + + /* Register addresses for queue */ + void __iomem *reg_control; + u32 qcontrol; /* Cached control register */ + + /* Status values from job */ + u32 int_status; + u32 q_status; + u32 q_int_status; + u32 cmd_error; + /* Queue Statistics */ + unsigned long total_pt_ops; +} ____cacheline_aligned; + +struct pt_device { + struct list_head entry; + + unsigned int ord; + char name[MAX_PT_NAME_LEN]; + + struct device *dev; + + /* Bus specific device information */ + struct pt_msix *pt_msix; + + struct pt_dev_vdata *dev_vdata; + + unsigned int pt_irq; + + /* I/O area used for device communication */ + void __iomem *io_regs; + + spinlock_t cmd_lock ____cacheline_aligned; + unsigned int cmd_count; + struct list_head cmd; + + /* + * The command queue. This represent the queue available on the + * PTDMA that are available for processing cmds + */ + struct pt_cmd_queue cmd_q; + + /* Support for the DMA Engine capabilities */ + struct dma_device dma_dev; + struct pt_dma_chan *pt_dma_chan; + struct kmem_cache *dma_desc_cache; + + wait_queue_head_t lsb_queue; + + /* Device Statistics */ + unsigned long total_interrupts; + + struct pt_tasklet_data tdata; + int ver; +}; + +/* + * descriptor for PTDMA commands + * 8 32-bit words: + * word 0: function; engine; control bits + * word 1: length of source data + * word 2: low 32 bits of source pointer + * word 3: upper 16 bits of source pointer; source memory type + * word 4: low 32 bits of destination pointer + * word 5: upper 16 bits of destination pointer; destination memory type + * word 6: reserved 32 bits + * word 7: reserved 32 bits + */ + +#define DWORD0_SOC BIT(0) +#define DWORD0_IOC BIT(1) + +struct dword3 { + unsigned int src_hi:16; + unsigned int src_mem:2; + unsigned int lsb_cxt_id:8; + unsigned int rsvd1:5; + unsigned int fixed:1; +}; + +struct dword5 { + unsigned int dst_hi:16; + unsigned int dst_mem:2; + unsigned int rsvd1:13; + unsigned int fixed:1; +}; + +struct ptdma_desc { + u32 dw0; + u32 length; + u32 src_lo; + struct dword3 dw3; + u32 dst_lo; + struct dword5 dw5; + __le32 rsvd1; + __le32 rsvd2; +}; + +/* Structure to hold PT device data */ +struct pt_dev_vdata { + const unsigned int bar; +}; + +int pt_dmaengine_register(struct pt_device *pt); +void pt_dmaengine_unregister(struct pt_device *pt); + +void ptdma_debugfs_setup(struct pt_device *pt); +int pt_core_init(struct pt_device *pt); +void pt_core_destroy(struct pt_device *pt); + +int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, + struct pt_passthru_engine *pt_engine); + +void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q); +void pt_start_queue(struct pt_cmd_queue *cmd_q); +void pt_stop_queue(struct pt_cmd_queue *cmd_q); + +static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) +{ + iowrite32(0, pt->cmd_q.reg_control + 0x000C); +} + +static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) +{ + iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); +} +#endif diff --git a/drivers/dma/amd/qdma/Makefile b/drivers/dma/amd/qdma/Makefile new file mode 100644 index 000000000000..011268fef377 --- /dev/null +++ b/drivers/dma/amd/qdma/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_AMD_QDMA) += amd-qdma.o + +amd-qdma-$(CONFIG_AMD_QDMA) := qdma.o qdma-comm-regs.o diff --git a/drivers/dma/amd/qdma/qdma-comm-regs.c b/drivers/dma/amd/qdma/qdma-comm-regs.c new file mode 100644 index 000000000000..9162f9d367cc --- /dev/null +++ b/drivers/dma/amd/qdma/qdma-comm-regs.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef __QDMA_REGS_DEF_H +#define __QDMA_REGS_DEF_H + +#include "qdma.h" + +const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX] = { + [QDMA_REGO_CTXT_DATA] = QDMA_REGO(0x804, 8), + [QDMA_REGO_CTXT_CMD] = QDMA_REGO(0x844, 1), + [QDMA_REGO_CTXT_MASK] = QDMA_REGO(0x824, 8), + [QDMA_REGO_MM_H2C_CTRL] = QDMA_REGO(0x1004, 1), + [QDMA_REGO_MM_C2H_CTRL] = QDMA_REGO(0x1204, 1), + [QDMA_REGO_QUEUE_COUNT] = QDMA_REGO(0x120, 1), + [QDMA_REGO_RING_SIZE] = QDMA_REGO(0x204, 1), + [QDMA_REGO_H2C_PIDX] = QDMA_REGO(0x18004, 1), + [QDMA_REGO_C2H_PIDX] = QDMA_REGO(0x18008, 1), + [QDMA_REGO_INTR_CIDX] = QDMA_REGO(0x18000, 1), + [QDMA_REGO_FUNC_ID] = QDMA_REGO(0x12c, 1), + [QDMA_REGO_ERR_INT] = QDMA_REGO(0xb04, 1), + [QDMA_REGO_ERR_STAT] = QDMA_REGO(0x248, 1), +}; + +const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX] = { + /* QDMA_REGO_CTXT_DATA fields */ + [QDMA_REGF_IRQ_ENABLE] = QDMA_REGF(53, 53), + [QDMA_REGF_WBK_ENABLE] = QDMA_REGF(52, 52), + [QDMA_REGF_WBI_CHECK] = QDMA_REGF(34, 34), + [QDMA_REGF_IRQ_ARM] = QDMA_REGF(16, 16), + [QDMA_REGF_IRQ_VEC] = QDMA_REGF(138, 128), + [QDMA_REGF_IRQ_AGG] = QDMA_REGF(139, 139), + [QDMA_REGF_WBI_INTVL_ENABLE] = QDMA_REGF(35, 35), + [QDMA_REGF_MRKR_DISABLE] = QDMA_REGF(62, 62), + [QDMA_REGF_QUEUE_ENABLE] = QDMA_REGF(32, 32), + [QDMA_REGF_QUEUE_MODE] = QDMA_REGF(63, 63), + [QDMA_REGF_DESC_BASE] = QDMA_REGF(127, 64), + [QDMA_REGF_DESC_SIZE] = QDMA_REGF(49, 48), + [QDMA_REGF_RING_ID] = QDMA_REGF(47, 44), + [QDMA_REGF_QUEUE_BASE] = QDMA_REGF(11, 0), + [QDMA_REGF_QUEUE_MAX] = QDMA_REGF(44, 32), + [QDMA_REGF_FUNCTION_ID] = QDMA_REGF(24, 17), + [QDMA_REGF_INTR_AGG_BASE] = QDMA_REGF(66, 15), + [QDMA_REGF_INTR_VECTOR] = QDMA_REGF(11, 1), + [QDMA_REGF_INTR_SIZE] = QDMA_REGF(69, 67), + [QDMA_REGF_INTR_VALID] = QDMA_REGF(0, 0), + [QDMA_REGF_INTR_COLOR] = QDMA_REGF(14, 14), + [QDMA_REGF_INTR_FUNCTION_ID] = QDMA_REGF(125, 114), + /* QDMA_REGO_CTXT_CMD fields */ + [QDMA_REGF_CMD_INDX] = QDMA_REGF(19, 7), + [QDMA_REGF_CMD_CMD] = QDMA_REGF(6, 5), + [QDMA_REGF_CMD_TYPE] = QDMA_REGF(4, 1), + [QDMA_REGF_CMD_BUSY] = QDMA_REGF(0, 0), + /* QDMA_REGO_QUEUE_COUNT fields */ + [QDMA_REGF_QUEUE_COUNT] = QDMA_REGF(11, 0), + /* QDMA_REGO_ERR_INT fields */ + [QDMA_REGF_ERR_INT_FUNC] = QDMA_REGF(11, 0), + [QDMA_REGF_ERR_INT_VEC] = QDMA_REGF(22, 12), + [QDMA_REGF_ERR_INT_ARM] = QDMA_REGF(24, 24), +}; + +#endif /* __QDMA_REGS_DEF_H */ diff --git a/drivers/dma/amd/qdma/qdma.c b/drivers/dma/amd/qdma/qdma.c new file mode 100644 index 000000000000..8fb2d5e1df20 --- /dev/null +++ b/drivers/dma/amd/qdma/qdma.c @@ -0,0 +1,1143 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DMA driver for AMD Queue-based DMA Subsystem + * + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/platform_device.h> +#include <linux/platform_data/amd_qdma.h> +#include <linux/regmap.h> + +#include "qdma.h" + +#define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H") +#define QDMA_REG_OFF(d, r) ((d)->roffs[r].off) + +/* MMIO regmap config for all QDMA registers */ +static const struct regmap_config qdma_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static inline struct qdma_queue *to_qdma_queue(struct dma_chan *chan) +{ + return container_of(chan, struct qdma_queue, vchan.chan); +} + +static inline struct qdma_mm_vdesc *to_qdma_vdesc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct qdma_mm_vdesc, vdesc); +} + +static inline u32 qdma_get_intr_ring_idx(struct qdma_device *qdev) +{ + u32 idx; + + idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx; + qdev->qintr_ring_idx %= qdev->qintr_ring_num; + + return idx; +} + +static u64 qdma_get_field(const struct qdma_device *qdev, const u32 *data, + enum qdma_reg_fields field) +{ + const struct qdma_reg_field *f = &qdev->rfields[field]; + u16 low_pos, hi_pos, low_bit, hi_bit; + u64 value = 0, mask; + + low_pos = f->lsb / BITS_PER_TYPE(*data); + hi_pos = f->msb / BITS_PER_TYPE(*data); + + if (low_pos == hi_pos) { + low_bit = f->lsb % BITS_PER_TYPE(*data); + hi_bit = f->msb % BITS_PER_TYPE(*data); + mask = GENMASK(hi_bit, low_bit); + value = (data[low_pos] & mask) >> low_bit; + } else if (hi_pos == low_pos + 1) { + low_bit = f->lsb % BITS_PER_TYPE(*data); + hi_bit = low_bit + (f->msb - f->lsb); + value = ((u64)data[hi_pos] << BITS_PER_TYPE(*data)) | + data[low_pos]; + mask = GENMASK_ULL(hi_bit, low_bit); + value = (value & mask) >> low_bit; + } else { + hi_bit = f->msb % BITS_PER_TYPE(*data); + mask = GENMASK(hi_bit, 0); + value = data[hi_pos] & mask; + low_bit = f->msb - f->lsb - hi_bit; + value <<= low_bit; + low_bit -= 32; + value |= (u64)data[hi_pos - 1] << low_bit; + mask = GENMASK(31, 32 - low_bit); + value |= (data[hi_pos - 2] & mask) >> low_bit; + } + + return value; +} + +static void qdma_set_field(const struct qdma_device *qdev, u32 *data, + enum qdma_reg_fields field, u64 value) +{ + const struct qdma_reg_field *f = &qdev->rfields[field]; + u16 low_pos, hi_pos, low_bit; + + low_pos = f->lsb / BITS_PER_TYPE(*data); + hi_pos = f->msb / BITS_PER_TYPE(*data); + low_bit = f->lsb % BITS_PER_TYPE(*data); + + data[low_pos++] |= value << low_bit; + if (low_pos <= hi_pos) + data[low_pos++] |= (u32)(value >> (32 - low_bit)); + if (low_pos <= hi_pos) + data[low_pos] |= (u32)(value >> (64 - low_bit)); +} + +static inline int qdma_reg_write(const struct qdma_device *qdev, + const u32 *data, enum qdma_regs reg) +{ + const struct qdma_reg *r = &qdev->roffs[reg]; + int ret; + + if (r->count > 1) + ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count); + else + ret = regmap_write(qdev->regmap, r->off, *data); + + return ret; +} + +static inline int qdma_reg_read(const struct qdma_device *qdev, u32 *data, + enum qdma_regs reg) +{ + const struct qdma_reg *r = &qdev->roffs[reg]; + int ret; + + if (r->count > 1) + ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count); + else + ret = regmap_read(qdev->regmap, r->off, data); + + return ret; +} + +static int qdma_context_cmd_execute(const struct qdma_device *qdev, + enum qdma_ctxt_type type, + enum qdma_ctxt_cmd cmd, u16 index) +{ + u32 value = 0; + int ret; + + qdma_set_field(qdev, &value, QDMA_REGF_CMD_INDX, index); + qdma_set_field(qdev, &value, QDMA_REGF_CMD_CMD, cmd); + qdma_set_field(qdev, &value, QDMA_REGF_CMD_TYPE, type); + + ret = qdma_reg_write(qdev, &value, QDMA_REGO_CTXT_CMD); + if (ret) + return ret; + + ret = regmap_read_poll_timeout(qdev->regmap, + QDMA_REG_OFF(qdev, QDMA_REGO_CTXT_CMD), + value, + !qdma_get_field(qdev, &value, + QDMA_REGF_CMD_BUSY), + QDMA_POLL_INTRVL_US, + QDMA_POLL_TIMEOUT_US); + if (ret) { + qdma_err(qdev, "Context command execution timed out"); + return ret; + } + + return 0; +} + +static int qdma_context_write_data(const struct qdma_device *qdev, + const u32 *data) +{ + u32 mask[QDMA_CTXT_REGMAP_LEN]; + int ret; + + memset(mask, ~0, sizeof(mask)); + + ret = qdma_reg_write(qdev, mask, QDMA_REGO_CTXT_MASK); + if (ret) + return ret; + + ret = qdma_reg_write(qdev, data, QDMA_REGO_CTXT_DATA); + if (ret) + return ret; + + return 0; +} + +static void qdma_prep_sw_desc_context(const struct qdma_device *qdev, + const struct qdma_ctxt_sw_desc *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec); + qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid); + + qdma_set_field(qdev, data, QDMA_REGF_DESC_SIZE, QDMA_DESC_SIZE_32B); + qdma_set_field(qdev, data, QDMA_REGF_RING_ID, QDMA_DEFAULT_RING_ID); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MODE, QDMA_QUEUE_OP_MM); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBK_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBI_CHECK, 1); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_ARM, 1); + qdma_set_field(qdev, data, QDMA_REGF_IRQ_AGG, 1); + qdma_set_field(qdev, data, QDMA_REGF_WBI_INTVL_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_ENABLE, 1); + qdma_set_field(qdev, data, QDMA_REGF_MRKR_DISABLE, 1); +} + +static void qdma_prep_intr_context(const struct qdma_device *qdev, + const struct qdma_ctxt_intr *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base); + qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec); + qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size); + qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid); + qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color); + qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid); +} + +static void qdma_prep_fmap_context(const struct qdma_device *qdev, + const struct qdma_ctxt_fmap *ctxt, + u32 *data) +{ + memset(data, 0, QDMA_CTXT_REGMAP_LEN * sizeof(*data)); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase); + qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax); +} + +/* + * Program the indirect context register space + * + * Once the queue is enabled, context is dynamically updated by hardware. Any + * modification of the context through this API when the queue is enabled can + * result in unexpected behavior. Reading the context when the queue is enabled + * is not recommended as it can result in reduced performance. + */ +static int qdma_prog_context(struct qdma_device *qdev, enum qdma_ctxt_type type, + enum qdma_ctxt_cmd cmd, u16 index, u32 *ctxt) +{ + int ret; + + mutex_lock(&qdev->ctxt_lock); + if (cmd == QDMA_CTXT_WRITE) { + ret = qdma_context_write_data(qdev, ctxt); + if (ret) + goto failed; + } + + ret = qdma_context_cmd_execute(qdev, type, cmd, index); + if (ret) + goto failed; + + if (cmd == QDMA_CTXT_READ) { + ret = qdma_reg_read(qdev, ctxt, QDMA_REGO_CTXT_DATA); + if (ret) + goto failed; + } + +failed: + mutex_unlock(&qdev->ctxt_lock); + + return ret; +} + +static int qdma_check_queue_status(struct qdma_device *qdev, + enum dma_transfer_direction dir, u16 qid) +{ + u32 status, data[QDMA_CTXT_REGMAP_LEN] = {0}; + enum qdma_ctxt_type type; + int ret; + + if (dir == DMA_MEM_TO_DEV) + type = QDMA_CTXT_DESC_SW_H2C; + else + type = QDMA_CTXT_DESC_SW_C2H; + ret = qdma_prog_context(qdev, type, QDMA_CTXT_READ, qid, data); + if (ret) + return ret; + + status = qdma_get_field(qdev, data, QDMA_REGF_QUEUE_ENABLE); + if (status) { + qdma_err(qdev, "queue %d already in use", qid); + return -EBUSY; + } + + return 0; +} + +static int qdma_clear_queue_context(const struct qdma_queue *queue) +{ + static const enum qdma_ctxt_type h2c_types[] = { + QDMA_CTXT_DESC_SW_H2C, + QDMA_CTXT_DESC_HW_H2C, + QDMA_CTXT_DESC_CR_H2C, + QDMA_CTXT_PFTCH, + }; + static const enum qdma_ctxt_type c2h_types[] = { + QDMA_CTXT_DESC_SW_C2H, + QDMA_CTXT_DESC_HW_C2H, + QDMA_CTXT_DESC_CR_C2H, + QDMA_CTXT_PFTCH, + }; + struct qdma_device *qdev = queue->qdev; + const enum qdma_ctxt_type *type; + int ret, num, i; + + if (queue->dir == DMA_MEM_TO_DEV) { + type = h2c_types; + num = ARRAY_SIZE(h2c_types); + } else { + type = c2h_types; + num = ARRAY_SIZE(c2h_types); + } + for (i = 0; i < num; i++) { + ret = qdma_prog_context(qdev, type[i], QDMA_CTXT_CLEAR, + queue->qid, NULL); + if (ret) { + qdma_err(qdev, "Failed to clear ctxt %d", type[i]); + return ret; + } + } + + return 0; +} + +static int qdma_setup_fmap_context(struct qdma_device *qdev) +{ + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + struct qdma_ctxt_fmap fmap; + int ret; + + ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_CLEAR, + qdev->fid, NULL); + if (ret) { + qdma_err(qdev, "Failed clearing context"); + return ret; + } + + fmap.qbase = 0; + fmap.qmax = qdev->chan_num * 2; + qdma_prep_fmap_context(qdev, &fmap, ctxt); + ret = qdma_prog_context(qdev, QDMA_CTXT_FMAP, QDMA_CTXT_WRITE, + qdev->fid, ctxt); + if (ret) + qdma_err(qdev, "Failed setup fmap, ret %d", ret); + + return ret; +} + +static int qdma_setup_queue_context(struct qdma_device *qdev, + const struct qdma_ctxt_sw_desc *sw_desc, + enum dma_transfer_direction dir, u16 qid) +{ + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + enum qdma_ctxt_type type; + int ret; + + if (dir == DMA_MEM_TO_DEV) + type = QDMA_CTXT_DESC_SW_H2C; + else + type = QDMA_CTXT_DESC_SW_C2H; + + qdma_prep_sw_desc_context(qdev, sw_desc, ctxt); + /* Setup SW descriptor context */ + ret = qdma_prog_context(qdev, type, QDMA_CTXT_WRITE, qid, ctxt); + if (ret) + qdma_err(qdev, "Failed setup SW desc ctxt for queue: %d", qid); + + return ret; +} + +/* + * Enable or disable memory-mapped DMA engines + * 1: enable, 0: disable + */ +static int qdma_sgdma_control(struct qdma_device *qdev, u32 ctrl) +{ + int ret; + + ret = qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_H2C_CTRL); + ret |= qdma_reg_write(qdev, &ctrl, QDMA_REGO_MM_C2H_CTRL); + + return ret; +} + +static int qdma_get_hw_info(struct qdma_device *qdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); + u32 value = 0; + int ret; + + ret = qdma_reg_read(qdev, &value, QDMA_REGO_QUEUE_COUNT); + if (ret) + return ret; + + value = qdma_get_field(qdev, &value, QDMA_REGF_QUEUE_COUNT) + 1; + if (pdata->max_mm_channels * 2 > value) { + qdma_err(qdev, "not enough hw queues %d", value); + return -EINVAL; + } + qdev->chan_num = pdata->max_mm_channels; + + ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID); + if (ret) + return ret; + + qdma_info(qdev, "max channel %d, function id %d", + qdev->chan_num, qdev->fid); + + return 0; +} + +static inline int qdma_update_pidx(const struct qdma_queue *queue, u16 pidx) +{ + struct qdma_device *qdev = queue->qdev; + + return regmap_write(qdev->regmap, queue->pidx_reg, + pidx | QDMA_QUEUE_ARM_BIT); +} + +static inline int qdma_update_cidx(const struct qdma_queue *queue, + u16 ridx, u16 cidx) +{ + struct qdma_device *qdev = queue->qdev; + + return regmap_write(qdev->regmap, queue->cidx_reg, + ((u32)ridx << 16) | cidx); +} + +/** + * qdma_free_vdesc - Free descriptor + * @vdesc: Virtual DMA descriptor + */ +static void qdma_free_vdesc(struct virt_dma_desc *vdesc) +{ + struct qdma_mm_vdesc *vd = to_qdma_vdesc(vdesc); + + kfree(vd); +} + +static int qdma_alloc_queues(struct qdma_device *qdev, + enum dma_transfer_direction dir) +{ + struct qdma_queue *q, **queues; + u32 i, pidx_base; + int ret; + + if (dir == DMA_MEM_TO_DEV) { + queues = &qdev->h2c_queues; + pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_H2C_PIDX); + } else { + queues = &qdev->c2h_queues; + pidx_base = QDMA_REG_OFF(qdev, QDMA_REGO_C2H_PIDX); + } + + *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q), + GFP_KERNEL); + if (!*queues) + return -ENOMEM; + + for (i = 0; i < qdev->chan_num; i++) { + ret = qdma_check_queue_status(qdev, dir, i); + if (ret) + return ret; + + q = &(*queues)[i]; + q->ring_size = QDMA_DEFAULT_RING_SIZE; + q->idx_mask = q->ring_size - 2; + q->qdev = qdev; + q->dir = dir; + q->qid = i; + q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE; + q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) + + i * QDMA_DMAP_REG_STRIDE; + q->vchan.desc_free = qdma_free_vdesc; + vchan_init(&q->vchan, &qdev->dma_dev); + } + + return 0; +} + +static int qdma_device_verify(struct qdma_device *qdev) +{ + u32 value; + int ret; + + ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value); + if (ret) + return ret; + + value = FIELD_GET(QDMA_IDENTIFIER_MASK, value); + if (value != QDMA_IDENTIFIER) { + qdma_err(qdev, "Invalid identifier"); + return -ENODEV; + } + qdev->rfields = qdma_regfs_default; + qdev->roffs = qdma_regos_default; + + return 0; +} + +static int qdma_device_setup(struct qdma_device *qdev) +{ + u32 ring_sz = QDMA_DEFAULT_RING_SIZE; + int ret = 0; + + ret = qdma_setup_fmap_context(qdev); + if (ret) { + qdma_err(qdev, "Failed setup fmap context"); + return ret; + } + + /* Setup global ring buffer size at QDMA_DEFAULT_RING_ID index */ + ret = qdma_reg_write(qdev, &ring_sz, QDMA_REGO_RING_SIZE); + if (ret) { + qdma_err(qdev, "Failed to setup ring %d of size %ld", + QDMA_DEFAULT_RING_ID, QDMA_DEFAULT_RING_SIZE); + return ret; + } + + /* Enable memory-mapped DMA engine in both directions */ + ret = qdma_sgdma_control(qdev, 1); + if (ret) { + qdma_err(qdev, "Failed to SGDMA with error %d", ret); + return ret; + } + + ret = qdma_alloc_queues(qdev, DMA_MEM_TO_DEV); + if (ret) { + qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret); + return ret; + } + + ret = qdma_alloc_queues(qdev, DMA_DEV_TO_MEM); + if (ret) { + qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret); + return ret; + } + + return 0; +} + +/** + * qdma_free_queue_resources() - Free queue resources + * @chan: DMA channel + */ +static void qdma_free_queue_resources(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_device *qdev = queue->qdev; + struct qdma_platdata *pdata; + + qdma_clear_queue_context(queue); + vchan_free_chan_resources(&queue->vchan); + pdata = dev_get_platdata(&qdev->pdev->dev); + dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE, + queue->desc_base, queue->dma_desc_base); +} + +/** + * qdma_alloc_queue_resources() - Allocate queue resources + * @chan: DMA channel + */ +static int qdma_alloc_queue_resources(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_device *qdev = queue->qdev; + struct qdma_ctxt_sw_desc desc; + struct qdma_platdata *pdata; + size_t size; + int ret; + + ret = qdma_clear_queue_context(queue); + if (ret) + return ret; + + pdata = dev_get_platdata(&qdev->pdev->dev); + size = queue->ring_size * QDMA_MM_DESC_SIZE; + queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size, + &queue->dma_desc_base, + GFP_KERNEL); + if (!queue->desc_base) { + qdma_err(qdev, "Failed to allocate descriptor ring"); + return -ENOMEM; + } + + /* Setup SW descriptor queue context for DMA memory map */ + desc.vec = qdma_get_intr_ring_idx(qdev); + desc.desc_base = queue->dma_desc_base; + ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid); + if (ret) { + qdma_err(qdev, "Failed to setup SW desc ctxt for %s", + chan->name); + dma_free_coherent(pdata->dma_dev, size, queue->desc_base, + queue->dma_desc_base); + return ret; + } + + queue->pidx = 0; + queue->cidx = 0; + + return 0; +} + +static bool qdma_filter_fn(struct dma_chan *chan, void *param) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + struct qdma_queue_info *info = param; + + return info->dir == queue->dir; +} + +static int qdma_xfer_start(struct qdma_queue *queue) +{ + struct qdma_device *qdev = queue->qdev; + int ret; + + if (!vchan_next_desc(&queue->vchan)) + return 0; + + qdma_dbg(qdev, "Tnx kickoff with P: %d for %s%d", + queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid); + + ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx); + if (ret) { + qdma_err(qdev, "Failed to update PIDX to %d for %s queue: %d", + queue->pidx, CHAN_STR(queue), queue->qid); + } + + return ret; +} + +static void qdma_issue_pending(struct dma_chan *chan) +{ + struct qdma_queue *queue = to_qdma_queue(chan); + unsigned long flags; + + spin_lock_irqsave(&queue->vchan.lock, flags); + if (vchan_issue_pending(&queue->vchan)) { + if (queue->submitted_vdesc) { + queue->issued_vdesc = queue->submitted_vdesc; + queue->submitted_vdesc = NULL; + } + qdma_xfer_start(queue); + } + + spin_unlock_irqrestore(&queue->vchan.lock, flags); +} + +static struct qdma_mm_desc *qdma_get_desc(struct qdma_queue *q) +{ + struct qdma_mm_desc *desc; + + if (((q->pidx + 1) & q->idx_mask) == q->cidx) + return NULL; + + desc = q->desc_base + q->pidx; + q->pidx = (q->pidx + 1) & q->idx_mask; + + return desc; +} + +static int qdma_hw_enqueue(struct qdma_queue *q, struct qdma_mm_vdesc *vdesc) +{ + struct qdma_mm_desc *desc; + struct scatterlist *sg; + u64 addr, *src, *dst; + u32 rest, len; + int ret = 0; + u32 i; + + if (!vdesc->sg_len) + return 0; + + if (q->dir == DMA_MEM_TO_DEV) { + dst = &vdesc->dev_addr; + src = &addr; + } else { + dst = &addr; + src = &vdesc->dev_addr; + } + + for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) { + addr = sg_dma_address(sg) + vdesc->sg_off; + rest = sg_dma_len(sg) - vdesc->sg_off; + while (rest) { + len = min_t(u32, rest, QDMA_MM_DESC_MAX_LEN); + desc = qdma_get_desc(q); + if (!desc) { + ret = -EBUSY; + goto out; + } + + desc->src_addr = cpu_to_le64(*src); + desc->dst_addr = cpu_to_le64(*dst); + desc->len = cpu_to_le32(len); + + vdesc->dev_addr += len; + vdesc->sg_off += len; + vdesc->pending_descs++; + addr += len; + rest -= len; + } + vdesc->sg_off = 0; + } +out: + vdesc->sg_len -= i; + vdesc->pidx = q->pidx; + return ret; +} + +static void qdma_fill_pending_vdesc(struct qdma_queue *q) +{ + struct virt_dma_chan *vc = &q->vchan; + struct qdma_mm_vdesc *vdesc = NULL; + struct virt_dma_desc *vd; + int ret; + + if (!list_empty(&vc->desc_issued)) { + vd = &q->issued_vdesc->vdesc; + list_for_each_entry_from(vd, &vc->desc_issued, node) { + vdesc = to_qdma_vdesc(vd); + ret = qdma_hw_enqueue(q, vdesc); + if (ret) { + q->issued_vdesc = vdesc; + return; + } + } + q->issued_vdesc = vdesc; + } + + if (list_empty(&vc->desc_submitted)) + return; + + if (q->submitted_vdesc) + vd = &q->submitted_vdesc->vdesc; + else + vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node); + + list_for_each_entry_from(vd, &vc->desc_submitted, node) { + vdesc = to_qdma_vdesc(vd); + ret = qdma_hw_enqueue(q, vdesc); + if (ret) + break; + } + q->submitted_vdesc = vdesc; +} + +static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct virt_dma_chan *vc = to_virt_chan(tx->chan); + struct qdma_queue *q = to_qdma_queue(&vc->chan); + struct virt_dma_desc *vd; + unsigned long flags; + dma_cookie_t cookie; + + vd = container_of(tx, struct virt_dma_desc, tx); + spin_lock_irqsave(&vc->lock, flags); + cookie = dma_cookie_assign(tx); + + list_move_tail(&vd->node, &vc->desc_submitted); + qdma_fill_pending_vdesc(q); + spin_unlock_irqrestore(&vc->lock, flags); + + return cookie; +} + +static struct dma_async_tx_descriptor * +qdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct qdma_queue *q = to_qdma_queue(chan); + struct dma_async_tx_descriptor *tx; + struct qdma_mm_vdesc *vdesc; + + vdesc = kzalloc(sizeof(*vdesc), GFP_NOWAIT); + if (!vdesc) + return NULL; + vdesc->sgl = sgl; + vdesc->sg_len = sg_len; + if (dir == DMA_MEM_TO_DEV) + vdesc->dev_addr = q->cfg.dst_addr; + else + vdesc->dev_addr = q->cfg.src_addr; + + tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags); + tx->tx_submit = qdma_tx_submit; + + return tx; +} + +static int qdma_device_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct qdma_queue *q = to_qdma_queue(chan); + + memcpy(&q->cfg, cfg, sizeof(*cfg)); + + return 0; +} + +static int qdma_arm_err_intr(const struct qdma_device *qdev) +{ + u32 value = 0; + + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid); + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx); + qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_ARM, 1); + + return qdma_reg_write(qdev, &value, QDMA_REGO_ERR_INT); +} + +static irqreturn_t qdma_error_isr(int irq, void *data) +{ + struct qdma_device *qdev = data; + u32 err_stat = 0; + int ret; + + ret = qdma_reg_read(qdev, &err_stat, QDMA_REGO_ERR_STAT); + if (ret) { + qdma_err(qdev, "read error state failed, ret %d", ret); + goto out; + } + + qdma_err(qdev, "global error %d", err_stat); + ret = qdma_reg_write(qdev, &err_stat, QDMA_REGO_ERR_STAT); + if (ret) + qdma_err(qdev, "clear error state failed, ret %d", ret); + +out: + qdma_arm_err_intr(qdev); + return IRQ_HANDLED; +} + +static irqreturn_t qdma_queue_isr(int irq, void *data) +{ + struct qdma_intr_ring *intr = data; + struct qdma_queue *q = NULL; + struct qdma_device *qdev; + u32 index, comp_desc; + u64 intr_ent; + u8 color; + int ret; + u16 qid; + + qdev = intr->qdev; + index = intr->cidx; + while (1) { + struct virt_dma_desc *vd; + struct qdma_mm_vdesc *vdesc; + unsigned long flags; + u32 cidx; + + intr_ent = le64_to_cpu(intr->base[index]); + color = FIELD_GET(QDMA_INTR_MASK_COLOR, intr_ent); + if (color != intr->color) + break; + + qid = FIELD_GET(QDMA_INTR_MASK_QID, intr_ent); + if (FIELD_GET(QDMA_INTR_MASK_TYPE, intr_ent)) + q = qdev->c2h_queues; + else + q = qdev->h2c_queues; + q += qid; + + cidx = FIELD_GET(QDMA_INTR_MASK_CIDX, intr_ent); + + spin_lock_irqsave(&q->vchan.lock, flags); + comp_desc = (cidx - q->cidx) & q->idx_mask; + + vd = vchan_next_desc(&q->vchan); + if (!vd) + goto skip; + + vdesc = to_qdma_vdesc(vd); + while (comp_desc > vdesc->pending_descs) { + list_del(&vd->node); + vchan_cookie_complete(vd); + comp_desc -= vdesc->pending_descs; + vd = vchan_next_desc(&q->vchan); + vdesc = to_qdma_vdesc(vd); + } + vdesc->pending_descs -= comp_desc; + if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + q->cidx = cidx; + + qdma_fill_pending_vdesc(q); + qdma_xfer_start(q); + +skip: + spin_unlock_irqrestore(&q->vchan.lock, flags); + + /* + * Wrap the index value and flip the expected color value if + * interrupt aggregation PIDX has wrapped around. + */ + index++; + index &= QDMA_INTR_RING_IDX_MASK; + if (!index) + intr->color = !intr->color; + } + + /* + * Update the software interrupt aggregation ring CIDX if a valid entry + * was found. + */ + if (q) { + qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index); + + /* + * Record the last read index of status descriptor from the + * interrupt aggregation ring. + */ + intr->cidx = index; + + ret = qdma_update_cidx(q, intr->ridx, index); + if (ret) { + qdma_err(qdev, "Failed to update IRQ CIDX"); + return IRQ_NONE; + } + } + + return IRQ_HANDLED; +} + +static int qdma_init_error_irq(struct qdma_device *qdev) +{ + struct device *dev = &qdev->pdev->dev; + int ret; + u32 vec; + + vec = qdev->queue_irq_start - 1; + + ret = devm_request_threaded_irq(dev, vec, NULL, qdma_error_isr, + IRQF_ONESHOT, "amd-qdma-error", qdev); + if (ret) { + qdma_err(qdev, "Failed to request error IRQ vector: %d", vec); + return ret; + } + + ret = qdma_arm_err_intr(qdev); + if (ret) + qdma_err(qdev, "Failed to arm err interrupt, ret %d", ret); + + return ret; +} + +static int qdmam_alloc_qintr_rings(struct qdma_device *qdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); + struct device *dev = &qdev->pdev->dev; + u32 ctxt[QDMA_CTXT_REGMAP_LEN]; + struct qdma_intr_ring *ring; + struct qdma_ctxt_intr intr_ctxt; + u32 vector; + int ret, i; + + qdev->qintr_ring_num = qdev->queue_irq_num; + qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num, + sizeof(*qdev->qintr_rings), + GFP_KERNEL); + if (!qdev->qintr_rings) + return -ENOMEM; + + vector = qdev->queue_irq_start; + for (i = 0; i < qdev->qintr_ring_num; i++, vector++) { + ring = &qdev->qintr_rings[i]; + ring->qdev = qdev; + ring->msix_id = qdev->err_irq_idx + i + 1; + ring->ridx = i; + ring->color = 1; + ring->base = dmam_alloc_coherent(pdata->dma_dev, + QDMA_INTR_RING_SIZE, + &ring->dev_base, GFP_KERNEL); + if (!ring->base) { + qdma_err(qdev, "Failed to alloc intr ring %d", i); + return -ENOMEM; + } + intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base); + intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096; + intr_ctxt.vec = ring->msix_id; + intr_ctxt.valid = true; + intr_ctxt.color = true; + ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, + QDMA_CTXT_CLEAR, ring->ridx, NULL); + if (ret) { + qdma_err(qdev, "Failed clear intr ctx, ret %d", ret); + return ret; + } + + qdma_prep_intr_context(qdev, &intr_ctxt, ctxt); + ret = qdma_prog_context(qdev, QDMA_CTXT_INTR_COAL, + QDMA_CTXT_WRITE, ring->ridx, ctxt); + if (ret) { + qdma_err(qdev, "Failed setup intr ctx, ret %d", ret); + return ret; + } + + ret = devm_request_threaded_irq(dev, vector, NULL, + qdma_queue_isr, IRQF_ONESHOT, + "amd-qdma-queue", ring); + if (ret) { + qdma_err(qdev, "Failed to request irq %d", vector); + return ret; + } + } + + return 0; +} + +static int qdma_intr_init(struct qdma_device *qdev) +{ + int ret; + + ret = qdma_init_error_irq(qdev); + if (ret) { + qdma_err(qdev, "Failed to init error IRQs, ret %d", ret); + return ret; + } + + ret = qdmam_alloc_qintr_rings(qdev); + if (ret) { + qdma_err(qdev, "Failed to init queue IRQs, ret %d", ret); + return ret; + } + + return 0; +} + +static void amd_qdma_remove(struct platform_device *pdev) +{ + struct qdma_device *qdev = platform_get_drvdata(pdev); + + qdma_sgdma_control(qdev, 0); + dma_async_device_unregister(&qdev->dma_dev); + + mutex_destroy(&qdev->ctxt_lock); +} + +static int amd_qdma_probe(struct platform_device *pdev) +{ + struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev); + struct qdma_device *qdev; + struct resource *res; + void __iomem *regs; + int ret; + + qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL); + if (!qdev) + return -ENOMEM; + + platform_set_drvdata(pdev, qdev); + qdev->pdev = pdev; + mutex_init(&qdev->ctxt_lock); + + res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res) { + qdma_err(qdev, "Failed to get IRQ resource"); + ret = -ENODEV; + goto failed; + } + qdev->err_irq_idx = pdata->irq_index; + qdev->queue_irq_start = res->start + 1; + qdev->queue_irq_num = resource_size(res) - 1; + + regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(regs)) { + ret = PTR_ERR(regs); + qdma_err(qdev, "Failed to map IO resource, err %d", ret); + goto failed; + } + + qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs, + &qdma_regmap_config); + if (IS_ERR(qdev->regmap)) { + ret = PTR_ERR(qdev->regmap); + qdma_err(qdev, "Regmap init failed, err %d", ret); + goto failed; + } + + ret = qdma_device_verify(qdev); + if (ret) + goto failed; + + ret = qdma_get_hw_info(qdev); + if (ret) + goto failed; + + INIT_LIST_HEAD(&qdev->dma_dev.channels); + + ret = qdma_device_setup(qdev); + if (ret) + goto failed; + + ret = qdma_intr_init(qdev); + if (ret) { + qdma_err(qdev, "Failed to initialize IRQs %d", ret); + goto failed_disable_engine; + } + + dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask); + dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask); + + qdev->dma_dev.dev = &pdev->dev; + qdev->dma_dev.filter.map = pdata->device_map; + qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2; + qdev->dma_dev.filter.fn = qdma_filter_fn; + qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources; + qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources; + qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg; + qdev->dma_dev.device_config = qdma_device_config; + qdev->dma_dev.device_issue_pending = qdma_issue_pending; + qdev->dma_dev.device_tx_status = dma_cookie_status; + qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + + ret = dma_async_device_register(&qdev->dma_dev); + if (ret) { + qdma_err(qdev, "Failed to register AMD QDMA: %d", ret); + goto failed_disable_engine; + } + + return 0; + +failed_disable_engine: + qdma_sgdma_control(qdev, 0); +failed: + mutex_destroy(&qdev->ctxt_lock); + qdma_err(qdev, "Failed to probe AMD QDMA driver"); + return ret; +} + +static struct platform_driver amd_qdma_driver = { + .driver = { + .name = "amd-qdma", + }, + .probe = amd_qdma_probe, + .remove = amd_qdma_remove, +}; + +module_platform_driver(amd_qdma_driver); + +MODULE_DESCRIPTION("AMD QDMA driver"); +MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/amd/qdma/qdma.h b/drivers/dma/amd/qdma/qdma.h new file mode 100644 index 000000000000..94089f1f0c11 --- /dev/null +++ b/drivers/dma/amd/qdma/qdma.h @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * DMA header for AMD Queue-based DMA Subsystem + * + * Copyright (C) 2023-2024, Advanced Micro Devices, Inc. + */ + +#ifndef __QDMA_H +#define __QDMA_H + +#include <linux/bitfield.h> +#include <linux/dmaengine.h> +#include <linux/kernel.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include "../../virt-dma.h" + +#define DISABLE 0 +#define ENABLE 1 + +#define QDMA_MIN_IRQ 3 +#define QDMA_INTR_NAME_MAX_LEN 30 +#define QDMA_INTR_PREFIX "amd-qdma" + +#define QDMA_IDENTIFIER 0x1FD3 +#define QDMA_DEFAULT_RING_SIZE (BIT(10) + 1) +#define QDMA_DEFAULT_RING_ID 0 +#define QDMA_POLL_INTRVL_US 10 /* 10us */ +#define QDMA_POLL_TIMEOUT_US (500 * 1000) /* 500ms */ +#define QDMA_DMAP_REG_STRIDE 16 +#define QDMA_CTXT_REGMAP_LEN 8 /* 8 regs */ +#define QDMA_MM_DESC_SIZE 32 /* Bytes */ +#define QDMA_MM_DESC_LEN_BITS 28 +#define QDMA_MM_DESC_MAX_LEN (BIT(QDMA_MM_DESC_LEN_BITS) - 1) +#define QDMA_MIN_DMA_ALLOC_SIZE 4096 +#define QDMA_INTR_RING_SIZE BIT(13) +#define QDMA_INTR_RING_IDX_MASK GENMASK(9, 0) +#define QDMA_INTR_RING_BASE(_addr) ((_addr) >> 12) + +#define QDMA_IDENTIFIER_REGOFF 0x0 +#define QDMA_IDENTIFIER_MASK GENMASK(31, 16) +#define QDMA_QUEUE_ARM_BIT BIT(16) + +#define qdma_err(qdev, fmt, args...) \ + dev_err(&(qdev)->pdev->dev, fmt, ##args) + +#define qdma_dbg(qdev, fmt, args...) \ + dev_dbg(&(qdev)->pdev->dev, fmt, ##args) + +#define qdma_info(qdev, fmt, args...) \ + dev_info(&(qdev)->pdev->dev, fmt, ##args) + +enum qdma_reg_fields { + QDMA_REGF_IRQ_ENABLE, + QDMA_REGF_WBK_ENABLE, + QDMA_REGF_WBI_CHECK, + QDMA_REGF_IRQ_ARM, + QDMA_REGF_IRQ_VEC, + QDMA_REGF_IRQ_AGG, + QDMA_REGF_WBI_INTVL_ENABLE, + QDMA_REGF_MRKR_DISABLE, + QDMA_REGF_QUEUE_ENABLE, + QDMA_REGF_QUEUE_MODE, + QDMA_REGF_DESC_BASE, + QDMA_REGF_DESC_SIZE, + QDMA_REGF_RING_ID, + QDMA_REGF_CMD_INDX, + QDMA_REGF_CMD_CMD, + QDMA_REGF_CMD_TYPE, + QDMA_REGF_CMD_BUSY, + QDMA_REGF_QUEUE_COUNT, + QDMA_REGF_QUEUE_MAX, + QDMA_REGF_QUEUE_BASE, + QDMA_REGF_FUNCTION_ID, + QDMA_REGF_INTR_AGG_BASE, + QDMA_REGF_INTR_VECTOR, + QDMA_REGF_INTR_SIZE, + QDMA_REGF_INTR_VALID, + QDMA_REGF_INTR_COLOR, + QDMA_REGF_INTR_FUNCTION_ID, + QDMA_REGF_ERR_INT_FUNC, + QDMA_REGF_ERR_INT_VEC, + QDMA_REGF_ERR_INT_ARM, + QDMA_REGF_MAX +}; + +enum qdma_regs { + QDMA_REGO_CTXT_DATA, + QDMA_REGO_CTXT_CMD, + QDMA_REGO_CTXT_MASK, + QDMA_REGO_MM_H2C_CTRL, + QDMA_REGO_MM_C2H_CTRL, + QDMA_REGO_QUEUE_COUNT, + QDMA_REGO_RING_SIZE, + QDMA_REGO_H2C_PIDX, + QDMA_REGO_C2H_PIDX, + QDMA_REGO_INTR_CIDX, + QDMA_REGO_FUNC_ID, + QDMA_REGO_ERR_INT, + QDMA_REGO_ERR_STAT, + QDMA_REGO_MAX +}; + +struct qdma_reg_field { + u16 lsb; /* Least significant bit of field */ + u16 msb; /* Most significant bit of field */ +}; + +struct qdma_reg { + u32 off; + u32 count; +}; + +#define QDMA_REGF(_msb, _lsb) { \ + .lsb = (_lsb), \ + .msb = (_msb), \ +} + +#define QDMA_REGO(_off, _count) { \ + .off = (_off), \ + .count = (_count), \ +} + +enum qdma_desc_size { + QDMA_DESC_SIZE_8B, + QDMA_DESC_SIZE_16B, + QDMA_DESC_SIZE_32B, + QDMA_DESC_SIZE_64B, +}; + +enum qdma_queue_op_mode { + QDMA_QUEUE_OP_STREAM, + QDMA_QUEUE_OP_MM, +}; + +enum qdma_ctxt_type { + QDMA_CTXT_DESC_SW_C2H, + QDMA_CTXT_DESC_SW_H2C, + QDMA_CTXT_DESC_HW_C2H, + QDMA_CTXT_DESC_HW_H2C, + QDMA_CTXT_DESC_CR_C2H, + QDMA_CTXT_DESC_CR_H2C, + QDMA_CTXT_WRB, + QDMA_CTXT_PFTCH, + QDMA_CTXT_INTR_COAL, + QDMA_CTXT_RSVD, + QDMA_CTXT_HOST_PROFILE, + QDMA_CTXT_TIMER, + QDMA_CTXT_FMAP, + QDMA_CTXT_FNC_STS, +}; + +enum qdma_ctxt_cmd { + QDMA_CTXT_CLEAR, + QDMA_CTXT_WRITE, + QDMA_CTXT_READ, + QDMA_CTXT_INVALIDATE, + QDMA_CTXT_MAX +}; + +struct qdma_ctxt_sw_desc { + u64 desc_base; + u16 vec; +}; + +struct qdma_ctxt_intr { + u64 agg_base; + u16 vec; + u32 size; + bool valid; + bool color; +}; + +struct qdma_ctxt_fmap { + u16 qbase; + u16 qmax; +}; + +struct qdma_device; + +struct qdma_mm_desc { + __le64 src_addr; + __le32 len; + __le32 reserved1; + __le64 dst_addr; + __le64 reserved2; +} __packed; + +struct qdma_mm_vdesc { + struct virt_dma_desc vdesc; + struct qdma_queue *queue; + struct scatterlist *sgl; + u64 sg_off; + u32 sg_len; + u64 dev_addr; + u32 pidx; + u32 pending_descs; + struct dma_slave_config cfg; +}; + +#define QDMA_VDESC_QUEUED(vdesc) (!(vdesc)->sg_len) + +struct qdma_queue { + struct qdma_device *qdev; + struct virt_dma_chan vchan; + enum dma_transfer_direction dir; + struct dma_slave_config cfg; + struct qdma_mm_desc *desc_base; + struct qdma_mm_vdesc *submitted_vdesc; + struct qdma_mm_vdesc *issued_vdesc; + dma_addr_t dma_desc_base; + u32 pidx_reg; + u32 cidx_reg; + u32 ring_size; + u32 idx_mask; + u16 qid; + u32 pidx; + u32 cidx; +}; + +struct qdma_intr_ring { + struct qdma_device *qdev; + __le64 *base; + dma_addr_t dev_base; + char msix_name[QDMA_INTR_NAME_MAX_LEN]; + u32 msix_vector; + u16 msix_id; + u32 ring_size; + u16 ridx; + u16 cidx; + u8 color; +}; + +#define QDMA_INTR_MASK_PIDX GENMASK_ULL(15, 0) +#define QDMA_INTR_MASK_CIDX GENMASK_ULL(31, 16) +#define QDMA_INTR_MASK_DESC_COLOR GENMASK_ULL(32, 32) +#define QDMA_INTR_MASK_STATE GENMASK_ULL(34, 33) +#define QDMA_INTR_MASK_ERROR GENMASK_ULL(36, 35) +#define QDMA_INTR_MASK_TYPE GENMASK_ULL(38, 38) +#define QDMA_INTR_MASK_QID GENMASK_ULL(62, 39) +#define QDMA_INTR_MASK_COLOR GENMASK_ULL(63, 63) + +struct qdma_device { + struct platform_device *pdev; + struct dma_device dma_dev; + struct regmap *regmap; + struct mutex ctxt_lock; /* protect ctxt registers */ + const struct qdma_reg_field *rfields; + const struct qdma_reg *roffs; + struct qdma_queue *h2c_queues; + struct qdma_queue *c2h_queues; + struct qdma_intr_ring *qintr_rings; + u32 qintr_ring_num; + u32 qintr_ring_idx; + u32 chan_num; + u32 queue_irq_start; + u32 queue_irq_num; + u32 err_irq_idx; + u32 fid; +}; + +extern const struct qdma_reg qdma_regos_default[QDMA_REGO_MAX]; +extern const struct qdma_reg_field qdma_regfs_default[QDMA_REGF_MAX]; + +#endif /* __QDMA_H */ diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c new file mode 100644 index 000000000000..bd49f0374291 --- /dev/null +++ b/drivers/dma/apple-admac.c @@ -0,0 +1,956 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips + * + * Copyright (C) The Asahi Linux Contributors + */ + +#include <linux/bits.h> +#include <linux/bitfield.h> +#include <linux/device.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> + +#include "dmaengine.h" + +#define NCHANNELS_MAX 64 +#define IRQ_NOUTPUTS 4 + +/* + * For allocation purposes we split the cache + * memory into blocks of fixed size (given in bytes). + */ +#define SRAM_BLOCK 2048 + +#define RING_WRITE_SLOT GENMASK(1, 0) +#define RING_READ_SLOT GENMASK(5, 4) +#define RING_FULL BIT(9) +#define RING_EMPTY BIT(8) +#define RING_ERR BIT(10) + +#define STATUS_DESC_DONE BIT(0) +#define STATUS_ERR BIT(6) + +#define FLAG_DESC_NOTIFY BIT(16) + +#define REG_TX_START 0x0000 +#define REG_TX_STOP 0x0004 +#define REG_RX_START 0x0008 +#define REG_RX_STOP 0x000c +#define REG_IMPRINT 0x0090 +#define REG_TX_SRAM_SIZE 0x0094 +#define REG_RX_SRAM_SIZE 0x0098 + +#define REG_CHAN_CTL(ch) (0x8000 + (ch) * 0x200) +#define REG_CHAN_CTL_RST_RINGS BIT(0) + +#define REG_DESC_RING(ch) (0x8070 + (ch) * 0x200) +#define REG_REPORT_RING(ch) (0x8074 + (ch) * 0x200) + +#define REG_RESIDUE(ch) (0x8064 + (ch) * 0x200) + +#define REG_BUS_WIDTH(ch) (0x8040 + (ch) * 0x200) + +#define BUS_WIDTH_WORD_SIZE GENMASK(3, 0) +#define BUS_WIDTH_FRAME_SIZE GENMASK(7, 4) +#define BUS_WIDTH_8BIT 0x00 +#define BUS_WIDTH_16BIT 0x01 +#define BUS_WIDTH_32BIT 0x02 +#define BUS_WIDTH_FRAME_2_WORDS 0x10 +#define BUS_WIDTH_FRAME_4_WORDS 0x20 + +#define REG_CHAN_SRAM_CARVEOUT(ch) (0x8050 + (ch) * 0x200) +#define CHAN_SRAM_CARVEOUT_SIZE GENMASK(31, 16) +#define CHAN_SRAM_CARVEOUT_BASE GENMASK(15, 0) + +#define REG_CHAN_FIFOCTL(ch) (0x8054 + (ch) * 0x200) +#define CHAN_FIFOCTL_LIMIT GENMASK(31, 16) +#define CHAN_FIFOCTL_THRESHOLD GENMASK(15, 0) + +#define REG_DESC_WRITE(ch) (0x10000 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000) +#define REG_REPORT_READ(ch) (0x10100 + ((ch) / 2) * 0x4 + ((ch) & 1) * 0x4000) + +#define REG_TX_INTSTATE(idx) (0x0030 + (idx) * 4) +#define REG_RX_INTSTATE(idx) (0x0040 + (idx) * 4) +#define REG_GLOBAL_INTSTATE(idx) (0x0050 + (idx) * 4) +#define REG_CHAN_INTSTATUS(ch, idx) (0x8010 + (ch) * 0x200 + (idx) * 4) +#define REG_CHAN_INTMASK(ch, idx) (0x8020 + (ch) * 0x200 + (idx) * 4) + +struct admac_data; +struct admac_tx; + +struct admac_chan { + unsigned int no; + struct admac_data *host; + struct dma_chan chan; + struct tasklet_struct tasklet; + + u32 carveout; + + spinlock_t lock; + struct admac_tx *current_tx; + int nperiod_acks; + + /* + * We maintain a 'submitted' and 'issued' list mainly for interface + * correctness. Typical use of the driver (per channel) will be + * prepping, submitting and issuing a single cyclic transaction which + * will stay current until terminate_all is called. + */ + struct list_head submitted; + struct list_head issued; + + struct list_head to_free; +}; + +struct admac_sram { + u32 size; + /* + * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than + * 64K and a 32-bit bitfield over 2K blocks covers it. + */ + u32 allocated; +}; + +struct admac_data { + struct dma_device dma; + struct device *dev; + __iomem void *base; + struct reset_control *rstc; + + struct mutex cache_alloc_lock; + struct admac_sram txcache, rxcache; + + int irq; + int irq_index; + int nchannels; + struct admac_chan channels[] __counted_by(nchannels); +}; + +struct admac_tx { + struct dma_async_tx_descriptor tx; + bool cyclic; + dma_addr_t buf_addr; + dma_addr_t buf_end; + size_t buf_len; + size_t period_len; + + size_t submitted_pos; + size_t reclaimed_pos; + + struct list_head node; +}; + +static int admac_alloc_sram_carveout(struct admac_data *ad, + enum dma_transfer_direction dir, + u32 *out) +{ + struct admac_sram *sram; + int i, ret = 0, nblocks; + ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); + ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); + + if (dir == DMA_MEM_TO_DEV) + sram = &ad->txcache; + else + sram = &ad->rxcache; + + mutex_lock(&ad->cache_alloc_lock); + + nblocks = sram->size / SRAM_BLOCK; + for (i = 0; i < nblocks; i++) + if (!(sram->allocated & BIT(i))) + break; + + if (i < nblocks) { + *out = FIELD_PREP(CHAN_SRAM_CARVEOUT_BASE, i * SRAM_BLOCK) | + FIELD_PREP(CHAN_SRAM_CARVEOUT_SIZE, SRAM_BLOCK); + sram->allocated |= BIT(i); + } else { + ret = -EBUSY; + } + + mutex_unlock(&ad->cache_alloc_lock); + + return ret; +} + +static void admac_free_sram_carveout(struct admac_data *ad, + enum dma_transfer_direction dir, + u32 carveout) +{ + struct admac_sram *sram; + u32 base = FIELD_GET(CHAN_SRAM_CARVEOUT_BASE, carveout); + int i; + + if (dir == DMA_MEM_TO_DEV) + sram = &ad->txcache; + else + sram = &ad->rxcache; + + if (WARN_ON(base >= sram->size)) + return; + + mutex_lock(&ad->cache_alloc_lock); + i = base / SRAM_BLOCK; + sram->allocated &= ~BIT(i); + mutex_unlock(&ad->cache_alloc_lock); +} + +static void admac_modify(struct admac_data *ad, int reg, u32 mask, u32 val) +{ + void __iomem *addr = ad->base + reg; + u32 curr = readl_relaxed(addr); + + writel_relaxed((curr & ~mask) | (val & mask), addr); +} + +static struct admac_chan *to_admac_chan(struct dma_chan *chan) +{ + return container_of(chan, struct admac_chan, chan); +} + +static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct admac_tx, tx); +} + +static enum dma_transfer_direction admac_chan_direction(int channo) +{ + /* Channel directions are hardwired */ + return (channo & 1) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV; +} + +static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct admac_tx *adtx = to_admac_tx(tx); + struct admac_chan *adchan = to_admac_chan(tx->chan); + unsigned long flags; + dma_cookie_t cookie; + + spin_lock_irqsave(&adchan->lock, flags); + cookie = dma_cookie_assign(tx); + list_add_tail(&adtx->node, &adchan->submitted); + spin_unlock_irqrestore(&adchan->lock, flags); + + return cookie; +} + +static int admac_desc_free(struct dma_async_tx_descriptor *tx) +{ + kfree(to_admac_tx(tx)); + + return 0; +} + +static struct dma_async_tx_descriptor *admac_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct admac_chan *adchan = container_of(chan, struct admac_chan, chan); + struct admac_tx *adtx; + + if (direction != admac_chan_direction(adchan->no)) + return NULL; + + adtx = kzalloc(sizeof(*adtx), GFP_NOWAIT); + if (!adtx) + return NULL; + + adtx->cyclic = true; + + adtx->buf_addr = buf_addr; + adtx->buf_len = buf_len; + adtx->buf_end = buf_addr + buf_len; + adtx->period_len = period_len; + + adtx->submitted_pos = 0; + adtx->reclaimed_pos = 0; + + dma_async_tx_descriptor_init(&adtx->tx, chan); + adtx->tx.tx_submit = admac_tx_submit; + adtx->tx.desc_free = admac_desc_free; + + return &adtx->tx; +} + +/* + * Write one hardware descriptor for a dmaengine cyclic transaction. + */ +static void admac_cyclic_write_one_desc(struct admac_data *ad, int channo, + struct admac_tx *tx) +{ + dma_addr_t addr; + + addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len); + + /* If happens means we have buggy code */ + WARN_ON_ONCE(addr + tx->period_len > tx->buf_end); + + dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n", + channo, &addr, tx->period_len, FLAG_DESC_NOTIFY); + + writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); + writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); + writel_relaxed(tx->period_len, ad->base + REG_DESC_WRITE(channo)); + writel_relaxed(FLAG_DESC_NOTIFY, ad->base + REG_DESC_WRITE(channo)); + + tx->submitted_pos += tx->period_len; + tx->submitted_pos %= 2 * tx->buf_len; +} + +/* + * Write all the hardware descriptors for a dmaengine cyclic + * transaction there is space for. + */ +static void admac_cyclic_write_desc(struct admac_data *ad, int channo, + struct admac_tx *tx) +{ + int i; + + for (i = 0; i < 4; i++) { + if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL) + break; + admac_cyclic_write_one_desc(ad, channo, tx); + } +} + +static int admac_ring_noccupied_slots(int ringval) +{ + int wrslot = FIELD_GET(RING_WRITE_SLOT, ringval); + int rdslot = FIELD_GET(RING_READ_SLOT, ringval); + + if (wrslot != rdslot) { + return (wrslot + 4 - rdslot) % 4; + } else { + WARN_ON((ringval & (RING_FULL | RING_EMPTY)) == 0); + + if (ringval & RING_FULL) + return 4; + else + return 0; + } +} + +/* + * Read from hardware the residue of a cyclic dmaengine transaction. + */ +static u32 admac_cyclic_read_residue(struct admac_data *ad, int channo, + struct admac_tx *adtx) +{ + u32 ring1, ring2; + u32 residue1, residue2; + int nreports; + size_t pos; + + ring1 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); + residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo)); + ring2 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); + residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo)); + + if (residue2 > residue1) { + /* + * Controller must have loaded next descriptor between + * the two residue reads + */ + nreports = admac_ring_noccupied_slots(ring1) + 1; + } else { + /* No descriptor load between the two reads, ring2 is safe to use */ + nreports = admac_ring_noccupied_slots(ring2); + } + + pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2; + + return adtx->buf_len - pos % adtx->buf_len; +} + +static enum dma_status admac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct admac_chan *adchan = to_admac_chan(chan); + struct admac_data *ad = adchan->host; + struct admac_tx *adtx; + + enum dma_status ret; + size_t residue; + unsigned long flags; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&adchan->lock, flags); + adtx = adchan->current_tx; + + if (adtx && adtx->tx.cookie == cookie) { + ret = DMA_IN_PROGRESS; + residue = admac_cyclic_read_residue(ad, adchan->no, adtx); + } else { + ret = DMA_IN_PROGRESS; + residue = 0; + list_for_each_entry(adtx, &adchan->issued, node) { + if (adtx->tx.cookie == cookie) { + residue = adtx->buf_len; + break; + } + } + } + spin_unlock_irqrestore(&adchan->lock, flags); + + dma_set_residue(txstate, residue); + return ret; +} + +static void admac_start_chan(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + u32 startbit = 1 << (adchan->no / 2); + + writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, + ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index)); + writel_relaxed(STATUS_DESC_DONE | STATUS_ERR, + ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index)); + + switch (admac_chan_direction(adchan->no)) { + case DMA_MEM_TO_DEV: + writel_relaxed(startbit, ad->base + REG_TX_START); + break; + case DMA_DEV_TO_MEM: + writel_relaxed(startbit, ad->base + REG_RX_START); + break; + default: + break; + } + dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no); +} + +static void admac_stop_chan(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + u32 stopbit = 1 << (adchan->no / 2); + + switch (admac_chan_direction(adchan->no)) { + case DMA_MEM_TO_DEV: + writel_relaxed(stopbit, ad->base + REG_TX_STOP); + break; + case DMA_DEV_TO_MEM: + writel_relaxed(stopbit, ad->base + REG_RX_STOP); + break; + default: + break; + } + dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no); +} + +static void admac_reset_rings(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + + writel_relaxed(REG_CHAN_CTL_RST_RINGS, + ad->base + REG_CHAN_CTL(adchan->no)); + writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no)); +} + +static void admac_start_current_tx(struct admac_chan *adchan) +{ + struct admac_data *ad = adchan->host; + int ch = adchan->no; + + admac_reset_rings(adchan); + writel_relaxed(0, ad->base + REG_CHAN_CTL(ch)); + + admac_cyclic_write_one_desc(ad, ch, adchan->current_tx); + admac_start_chan(adchan); + admac_cyclic_write_desc(ad, ch, adchan->current_tx); +} + +static void admac_issue_pending(struct dma_chan *chan) +{ + struct admac_chan *adchan = to_admac_chan(chan); + struct admac_tx *tx; + unsigned long flags; + + spin_lock_irqsave(&adchan->lock, flags); + list_splice_tail_init(&adchan->submitted, &adchan->issued); + if (!list_empty(&adchan->issued) && !adchan->current_tx) { + tx = list_first_entry(&adchan->issued, struct admac_tx, node); + list_del(&tx->node); + + adchan->current_tx = tx; + adchan->nperiod_acks = 0; + admac_start_current_tx(adchan); + } + spin_unlock_irqrestore(&adchan->lock, flags); +} + +static int admac_pause(struct dma_chan *chan) +{ + struct admac_chan *adchan = to_admac_chan(chan); + + admac_stop_chan(adchan); + + return 0; +} + +static int admac_resume(struct dma_chan *chan) +{ + struct admac_chan *adchan = to_admac_chan(chan); + + admac_start_chan(adchan); + + return 0; +} + +static int admac_terminate_all(struct dma_chan *chan) +{ + struct admac_chan *adchan = to_admac_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&adchan->lock, flags); + admac_stop_chan(adchan); + admac_reset_rings(adchan); + + if (adchan->current_tx) { + list_add_tail(&adchan->current_tx->node, &adchan->to_free); + adchan->current_tx = NULL; + } + /* + * Descriptors can only be freed after the tasklet + * has been killed (in admac_synchronize). + */ + list_splice_tail_init(&adchan->submitted, &adchan->to_free); + list_splice_tail_init(&adchan->issued, &adchan->to_free); + spin_unlock_irqrestore(&adchan->lock, flags); + + return 0; +} + +static void admac_synchronize(struct dma_chan *chan) +{ + struct admac_chan *adchan = to_admac_chan(chan); + struct admac_tx *adtx, *_adtx; + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&adchan->lock, flags); + list_splice_tail_init(&adchan->to_free, &head); + spin_unlock_irqrestore(&adchan->lock, flags); + + tasklet_kill(&adchan->tasklet); + + list_for_each_entry_safe(adtx, _adtx, &head, node) { + list_del(&adtx->node); + admac_desc_free(&adtx->tx); + } +} + +static int admac_alloc_chan_resources(struct dma_chan *chan) +{ + struct admac_chan *adchan = to_admac_chan(chan); + struct admac_data *ad = adchan->host; + int ret; + + dma_cookie_init(&adchan->chan); + ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no), + &adchan->carveout); + if (ret < 0) + return ret; + + writel_relaxed(adchan->carveout, + ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no)); + return 0; +} + +static void admac_free_chan_resources(struct dma_chan *chan) +{ + struct admac_chan *adchan = to_admac_chan(chan); + + admac_terminate_all(chan); + admac_synchronize(chan); + admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no), + adchan->carveout); +} + +static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data; + unsigned int index; + + if (dma_spec->args_count != 1) + return NULL; + + index = dma_spec->args[0]; + + if (index >= ad->nchannels) { + dev_err(ad->dev, "channel index %u out of bounds\n", index); + return NULL; + } + + return dma_get_slave_channel(&ad->channels[index].chan); +} + +static int admac_drain_reports(struct admac_data *ad, int channo) +{ + int count; + + for (count = 0; count < 4; count++) { + u32 countval_hi, countval_lo, unk1, flags; + + if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY) + break; + + countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo)); + countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo)); + unk1 = readl_relaxed(ad->base + REG_REPORT_READ(channo)); + flags = readl_relaxed(ad->base + REG_REPORT_READ(channo)); + + dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n", + channo, ((u64) countval_hi) << 32 | countval_lo, unk1, flags); + } + + return count; +} + +static void admac_handle_status_err(struct admac_data *ad, int channo) +{ + bool handled = false; + + if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) { + writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo)); + dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo); + handled = true; + } + + if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) { + writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo)); + dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo); + handled = true; + } + + if (unlikely(!handled)) { + dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo); + admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index), + STATUS_ERR, 0); + } +} + +static void admac_handle_status_desc_done(struct admac_data *ad, int channo) +{ + struct admac_chan *adchan = &ad->channels[channo]; + unsigned long flags; + int nreports; + + writel_relaxed(STATUS_DESC_DONE, + ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index)); + + spin_lock_irqsave(&adchan->lock, flags); + nreports = admac_drain_reports(ad, channo); + + if (adchan->current_tx) { + struct admac_tx *tx = adchan->current_tx; + + adchan->nperiod_acks += nreports; + tx->reclaimed_pos += nreports * tx->period_len; + tx->reclaimed_pos %= 2 * tx->buf_len; + + admac_cyclic_write_desc(ad, channo, tx); + tasklet_schedule(&adchan->tasklet); + } + spin_unlock_irqrestore(&adchan->lock, flags); +} + +static void admac_handle_chan_int(struct admac_data *ad, int no) +{ + u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index)); + + if (cause & STATUS_ERR) + admac_handle_status_err(ad, no); + + if (cause & STATUS_DESC_DONE) + admac_handle_status_desc_done(ad, no); +} + +static irqreturn_t admac_interrupt(int irq, void *devid) +{ + struct admac_data *ad = devid; + u32 rx_intstate, tx_intstate, global_intstate; + int i; + + rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index)); + tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index)); + global_intstate = readl_relaxed(ad->base + REG_GLOBAL_INTSTATE(ad->irq_index)); + + if (!tx_intstate && !rx_intstate && !global_intstate) + return IRQ_NONE; + + for (i = 0; i < ad->nchannels; i += 2) { + if (tx_intstate & 1) + admac_handle_chan_int(ad, i); + tx_intstate >>= 1; + } + + for (i = 1; i < ad->nchannels; i += 2) { + if (rx_intstate & 1) + admac_handle_chan_int(ad, i); + rx_intstate >>= 1; + } + + if (global_intstate) { + dev_warn(ad->dev, "clearing unknown global interrupt flag: %x\n", + global_intstate); + writel_relaxed(~(u32) 0, ad->base + REG_GLOBAL_INTSTATE(ad->irq_index)); + } + + return IRQ_HANDLED; +} + +static void admac_chan_tasklet(struct tasklet_struct *t) +{ + struct admac_chan *adchan = from_tasklet(adchan, t, tasklet); + struct admac_tx *adtx; + struct dmaengine_desc_callback cb; + struct dmaengine_result tx_result; + int nacks; + + spin_lock_irq(&adchan->lock); + adtx = adchan->current_tx; + nacks = adchan->nperiod_acks; + adchan->nperiod_acks = 0; + spin_unlock_irq(&adchan->lock); + + if (!adtx || !nacks) + return; + + tx_result.result = DMA_TRANS_NOERROR; + tx_result.residue = 0; + + dmaengine_desc_get_callback(&adtx->tx, &cb); + while (nacks--) + dmaengine_desc_callback_invoke(&cb, &tx_result); +} + +static int admac_device_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct admac_chan *adchan = to_admac_chan(chan); + struct admac_data *ad = adchan->host; + bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV; + int wordsize = 0; + u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) & + ~(BUS_WIDTH_WORD_SIZE | BUS_WIDTH_FRAME_SIZE); + + switch (is_tx ? config->dst_addr_width : config->src_addr_width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + wordsize = 1; + bus_width |= BUS_WIDTH_8BIT; + break; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + wordsize = 2; + bus_width |= BUS_WIDTH_16BIT; + break; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + wordsize = 4; + bus_width |= BUS_WIDTH_32BIT; + break; + default: + return -EINVAL; + } + + /* + * We take port_window_size to be the number of words in a frame. + * + * The controller has some means of out-of-band signalling, to the peripheral, + * of words position in a frame. That's where the importance of this control + * comes from. + */ + switch (is_tx ? config->dst_port_window_size : config->src_port_window_size) { + case 0 ... 1: + break; + case 2: + bus_width |= BUS_WIDTH_FRAME_2_WORDS; + break; + case 4: + bus_width |= BUS_WIDTH_FRAME_4_WORDS; + break; + default: + return -EINVAL; + } + + writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no)); + + /* + * By FIFOCTL_LIMIT we seem to set the maximal number of bytes allowed to be + * held in controller's per-channel FIFO. Transfers seem to be triggered + * around the time FIFO occupancy touches FIFOCTL_THRESHOLD. + * + * The numbers we set are more or less arbitrary. + */ + writel_relaxed(FIELD_PREP(CHAN_FIFOCTL_LIMIT, 0x30 * wordsize) + | FIELD_PREP(CHAN_FIFOCTL_THRESHOLD, 0x18 * wordsize), + ad->base + REG_CHAN_FIFOCTL(adchan->no)); + + return 0; +} + +static int admac_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct admac_data *ad; + struct dma_device *dma; + int nchannels; + int err, irq, i; + + err = of_property_read_u32(np, "dma-channels", &nchannels); + if (err || nchannels > NCHANNELS_MAX) { + dev_err(&pdev->dev, "missing or invalid dma-channels property\n"); + return -EINVAL; + } + + ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL); + if (!ad) + return -ENOMEM; + + platform_set_drvdata(pdev, ad); + ad->dev = &pdev->dev; + ad->nchannels = nchannels; + mutex_init(&ad->cache_alloc_lock); + + /* + * The controller has 4 IRQ outputs. Try them all until + * we find one we can use. + */ + for (i = 0; i < IRQ_NOUTPUTS; i++) { + irq = platform_get_irq_optional(pdev, i); + if (irq >= 0) { + ad->irq_index = i; + break; + } + } + + if (irq < 0) + return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n"); + ad->irq = irq; + + ad->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ad->base)) + return dev_err_probe(&pdev->dev, PTR_ERR(ad->base), + "unable to obtain MMIO resource\n"); + + ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL); + if (IS_ERR(ad->rstc)) + return PTR_ERR(ad->rstc); + + dma = &ad->dma; + + dma_cap_set(DMA_PRIVATE, dma->cap_mask); + dma_cap_set(DMA_CYCLIC, dma->cap_mask); + + dma->dev = &pdev->dev; + dma->device_alloc_chan_resources = admac_alloc_chan_resources; + dma->device_free_chan_resources = admac_free_chan_resources; + dma->device_tx_status = admac_tx_status; + dma->device_issue_pending = admac_issue_pending; + dma->device_terminate_all = admac_terminate_all; + dma->device_synchronize = admac_synchronize; + dma->device_prep_dma_cyclic = admac_prep_dma_cyclic; + dma->device_config = admac_device_config; + dma->device_pause = admac_pause; + dma->device_resume = admac_resume; + + dma->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); + dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + + INIT_LIST_HEAD(&dma->channels); + for (i = 0; i < nchannels; i++) { + struct admac_chan *adchan = &ad->channels[i]; + + adchan->host = ad; + adchan->no = i; + adchan->chan.device = &ad->dma; + spin_lock_init(&adchan->lock); + INIT_LIST_HEAD(&adchan->submitted); + INIT_LIST_HEAD(&adchan->issued); + INIT_LIST_HEAD(&adchan->to_free); + list_add_tail(&adchan->chan.device_node, &dma->channels); + tasklet_setup(&adchan->tasklet, admac_chan_tasklet); + } + + err = reset_control_reset(ad->rstc); + if (err) + return dev_err_probe(&pdev->dev, err, + "unable to trigger reset\n"); + + err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad); + if (err) { + dev_err_probe(&pdev->dev, err, + "unable to register interrupt\n"); + goto free_reset; + } + + err = dma_async_device_register(&ad->dma); + if (err) { + dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); + goto free_irq; + } + + err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad); + if (err) { + dma_async_device_unregister(&ad->dma); + dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); + goto free_irq; + } + + dev_info(&pdev->dev, "Audio DMA Controller\n"); + + return 0; + +free_irq: + free_irq(ad->irq, ad); +free_reset: + reset_control_rearm(ad->rstc); + return err; +} + +static void admac_remove(struct platform_device *pdev) +{ + struct admac_data *ad = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&ad->dma); + free_irq(ad->irq, ad); + reset_control_rearm(ad->rstc); +} + +static const struct of_device_id admac_of_match[] = { + { .compatible = "apple,admac", }, + { } +}; +MODULE_DEVICE_TABLE(of, admac_of_match); + +static struct platform_driver apple_admac_driver = { + .driver = { + .name = "apple-admac", + .of_match_table = admac_of_match, + }, + .probe = admac_probe, + .remove = admac_remove, +}; +module_platform_driver(apple_admac_driver); + +MODULE_AUTHOR("Martin PoviÅ¡er <povik+lin@cutebit.org>"); +MODULE_DESCRIPTION("Driver for Audio DMA Controller (ADMAC) on Apple SoCs"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c new file mode 100644 index 000000000000..9efe2ca7d5ec --- /dev/null +++ b/drivers/dma/arm-dma350.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2024-2025 Arm Limited +// Arm DMA-350 driver + +#include <linux/bitfield.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/of.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +#define DMAINFO 0x0f00 + +#define DMA_BUILDCFG0 0xb0 +#define DMA_CFG_DATA_WIDTH GENMASK(18, 16) +#define DMA_CFG_ADDR_WIDTH GENMASK(15, 10) +#define DMA_CFG_NUM_CHANNELS GENMASK(9, 4) + +#define DMA_BUILDCFG1 0xb4 +#define DMA_CFG_NUM_TRIGGER_IN GENMASK(8, 0) + +#define IIDR 0xc8 +#define IIDR_PRODUCTID GENMASK(31, 20) +#define IIDR_VARIANT GENMASK(19, 16) +#define IIDR_REVISION GENMASK(15, 12) +#define IIDR_IMPLEMENTER GENMASK(11, 0) + +#define PRODUCTID_DMA350 0x3a0 +#define IMPLEMENTER_ARM 0x43b + +#define DMACH(n) (0x1000 + 0x0100 * (n)) + +#define CH_CMD 0x00 +#define CH_CMD_RESUME BIT(5) +#define CH_CMD_PAUSE BIT(4) +#define CH_CMD_STOP BIT(3) +#define CH_CMD_DISABLE BIT(2) +#define CH_CMD_CLEAR BIT(1) +#define CH_CMD_ENABLE BIT(0) + +#define CH_STATUS 0x04 +#define CH_STAT_RESUMEWAIT BIT(21) +#define CH_STAT_PAUSED BIT(20) +#define CH_STAT_STOPPED BIT(19) +#define CH_STAT_DISABLED BIT(18) +#define CH_STAT_ERR BIT(17) +#define CH_STAT_DONE BIT(16) +#define CH_STAT_INTR_ERR BIT(1) +#define CH_STAT_INTR_DONE BIT(0) + +#define CH_INTREN 0x08 +#define CH_INTREN_ERR BIT(1) +#define CH_INTREN_DONE BIT(0) + +#define CH_CTRL 0x0c +#define CH_CTRL_USEDESTRIGIN BIT(26) +#define CH_CTRL_USESRCTRIGIN BIT(26) +#define CH_CTRL_DONETYPE GENMASK(23, 21) +#define CH_CTRL_REGRELOADTYPE GENMASK(20, 18) +#define CH_CTRL_XTYPE GENMASK(11, 9) +#define CH_CTRL_TRANSIZE GENMASK(2, 0) + +#define CH_SRCADDR 0x10 +#define CH_SRCADDRHI 0x14 +#define CH_DESADDR 0x18 +#define CH_DESADDRHI 0x1c +#define CH_XSIZE 0x20 +#define CH_XSIZEHI 0x24 +#define CH_SRCTRANSCFG 0x28 +#define CH_DESTRANSCFG 0x2c +#define CH_CFG_MAXBURSTLEN GENMASK(19, 16) +#define CH_CFG_PRIVATTR BIT(11) +#define CH_CFG_SHAREATTR GENMASK(9, 8) +#define CH_CFG_MEMATTR GENMASK(7, 0) + +#define TRANSCFG_DEVICE \ + FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ + FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ + FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_DEVICE) +#define TRANSCFG_NC \ + FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ + FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_OSH) | \ + FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_NC) +#define TRANSCFG_WB \ + FIELD_PREP(CH_CFG_MAXBURSTLEN, 0xf) | \ + FIELD_PREP(CH_CFG_SHAREATTR, SHAREATTR_ISH) | \ + FIELD_PREP(CH_CFG_MEMATTR, MEMATTR_WB) + +#define CH_XADDRINC 0x30 +#define CH_XY_DES GENMASK(31, 16) +#define CH_XY_SRC GENMASK(15, 0) + +#define CH_FILLVAL 0x38 +#define CH_SRCTRIGINCFG 0x4c +#define CH_DESTRIGINCFG 0x50 +#define CH_LINKATTR 0x70 +#define CH_LINK_SHAREATTR GENMASK(9, 8) +#define CH_LINK_MEMATTR GENMASK(7, 0) + +#define CH_AUTOCFG 0x74 +#define CH_LINKADDR 0x78 +#define CH_LINKADDR_EN BIT(0) + +#define CH_LINKADDRHI 0x7c +#define CH_ERRINFO 0x90 +#define CH_ERRINFO_AXIRDPOISERR BIT(18) +#define CH_ERRINFO_AXIWRRESPERR BIT(17) +#define CH_ERRINFO_AXIRDRESPERR BIT(16) + +#define CH_BUILDCFG0 0xf8 +#define CH_CFG_INC_WIDTH GENMASK(29, 26) +#define CH_CFG_DATA_WIDTH GENMASK(24, 22) +#define CH_CFG_DATA_BUF_SIZE GENMASK(7, 0) + +#define CH_BUILDCFG1 0xfc +#define CH_CFG_HAS_CMDLINK BIT(8) +#define CH_CFG_HAS_TRIGSEL BIT(7) +#define CH_CFG_HAS_TRIGIN BIT(5) +#define CH_CFG_HAS_WRAP BIT(1) + + +#define LINK_REGCLEAR BIT(0) +#define LINK_INTREN BIT(2) +#define LINK_CTRL BIT(3) +#define LINK_SRCADDR BIT(4) +#define LINK_SRCADDRHI BIT(5) +#define LINK_DESADDR BIT(6) +#define LINK_DESADDRHI BIT(7) +#define LINK_XSIZE BIT(8) +#define LINK_XSIZEHI BIT(9) +#define LINK_SRCTRANSCFG BIT(10) +#define LINK_DESTRANSCFG BIT(11) +#define LINK_XADDRINC BIT(12) +#define LINK_FILLVAL BIT(14) +#define LINK_SRCTRIGINCFG BIT(19) +#define LINK_DESTRIGINCFG BIT(20) +#define LINK_AUTOCFG BIT(29) +#define LINK_LINKADDR BIT(30) +#define LINK_LINKADDRHI BIT(31) + + +enum ch_ctrl_donetype { + CH_CTRL_DONETYPE_NONE = 0, + CH_CTRL_DONETYPE_CMD = 1, + CH_CTRL_DONETYPE_CYCLE = 3 +}; + +enum ch_ctrl_xtype { + CH_CTRL_XTYPE_DISABLE = 0, + CH_CTRL_XTYPE_CONTINUE = 1, + CH_CTRL_XTYPE_WRAP = 2, + CH_CTRL_XTYPE_FILL = 3 +}; + +enum ch_cfg_shareattr { + SHAREATTR_NSH = 0, + SHAREATTR_OSH = 2, + SHAREATTR_ISH = 3 +}; + +enum ch_cfg_memattr { + MEMATTR_DEVICE = 0x00, + MEMATTR_NC = 0x44, + MEMATTR_WB = 0xff +}; + +struct d350_desc { + struct virt_dma_desc vd; + u32 command[16]; + u16 xsize; + u16 xsizehi; + u8 tsz; +}; + +struct d350_chan { + struct virt_dma_chan vc; + struct d350_desc *desc; + void __iomem *base; + int irq; + enum dma_status status; + dma_cookie_t cookie; + u32 residue; + u8 tsz; + bool has_trig; + bool has_wrap; + bool coherent; +}; + +struct d350 { + struct dma_device dma; + int nchan; + int nreq; + struct d350_chan channels[] __counted_by(nchan); +}; + +static inline struct d350_chan *to_d350_chan(struct dma_chan *chan) +{ + return container_of(chan, struct d350_chan, vc.chan); +} + +static inline struct d350_desc *to_d350_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct d350_desc, vd); +} + +static void d350_desc_free(struct virt_dma_desc *vd) +{ + kfree(to_d350_desc(vd)); +} + +static struct dma_async_tx_descriptor *d350_prep_memcpy(struct dma_chan *chan, + dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) +{ + struct d350_chan *dch = to_d350_chan(chan); + struct d350_desc *desc; + u32 *cmd; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->tsz = __ffs(len | dest | src | (1 << dch->tsz)); + desc->xsize = lower_16_bits(len >> desc->tsz); + desc->xsizehi = upper_16_bits(len >> desc->tsz); + + cmd = desc->command; + cmd[0] = LINK_CTRL | LINK_SRCADDR | LINK_SRCADDRHI | LINK_DESADDR | + LINK_DESADDRHI | LINK_XSIZE | LINK_XSIZEHI | LINK_SRCTRANSCFG | + LINK_DESTRANSCFG | LINK_XADDRINC | LINK_LINKADDR; + + cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | + FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_CONTINUE) | + FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); + + cmd[2] = lower_32_bits(src); + cmd[3] = upper_32_bits(src); + cmd[4] = lower_32_bits(dest); + cmd[5] = upper_32_bits(dest); + cmd[6] = FIELD_PREP(CH_XY_SRC, desc->xsize) | FIELD_PREP(CH_XY_DES, desc->xsize); + cmd[7] = FIELD_PREP(CH_XY_SRC, desc->xsizehi) | FIELD_PREP(CH_XY_DES, desc->xsizehi); + cmd[8] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; + cmd[9] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; + cmd[10] = FIELD_PREP(CH_XY_SRC, 1) | FIELD_PREP(CH_XY_DES, 1); + cmd[11] = 0; + + return vchan_tx_prep(&dch->vc, &desc->vd, flags); +} + +static struct dma_async_tx_descriptor *d350_prep_memset(struct dma_chan *chan, + dma_addr_t dest, int value, size_t len, unsigned long flags) +{ + struct d350_chan *dch = to_d350_chan(chan); + struct d350_desc *desc; + u32 *cmd; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->tsz = __ffs(len | dest | (1 << dch->tsz)); + desc->xsize = lower_16_bits(len >> desc->tsz); + desc->xsizehi = upper_16_bits(len >> desc->tsz); + + cmd = desc->command; + cmd[0] = LINK_CTRL | LINK_DESADDR | LINK_DESADDRHI | + LINK_XSIZE | LINK_XSIZEHI | LINK_DESTRANSCFG | + LINK_XADDRINC | LINK_FILLVAL | LINK_LINKADDR; + + cmd[1] = FIELD_PREP(CH_CTRL_TRANSIZE, desc->tsz) | + FIELD_PREP(CH_CTRL_XTYPE, CH_CTRL_XTYPE_FILL) | + FIELD_PREP(CH_CTRL_DONETYPE, CH_CTRL_DONETYPE_CMD); + + cmd[2] = lower_32_bits(dest); + cmd[3] = upper_32_bits(dest); + cmd[4] = FIELD_PREP(CH_XY_DES, desc->xsize); + cmd[5] = FIELD_PREP(CH_XY_DES, desc->xsizehi); + cmd[6] = dch->coherent ? TRANSCFG_WB : TRANSCFG_NC; + cmd[7] = FIELD_PREP(CH_XY_DES, 1); + cmd[8] = (u8)value * 0x01010101; + cmd[9] = 0; + + return vchan_tx_prep(&dch->vc, &desc->vd, flags); +} + +static int d350_pause(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dch->vc.lock, flags); + if (dch->status == DMA_IN_PROGRESS) { + writel_relaxed(CH_CMD_PAUSE, dch->base + CH_CMD); + dch->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&dch->vc.lock, flags); + + return 0; +} + +static int d350_resume(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dch->vc.lock, flags); + if (dch->status == DMA_PAUSED) { + writel_relaxed(CH_CMD_RESUME, dch->base + CH_CMD); + dch->status = DMA_IN_PROGRESS; + } + spin_unlock_irqrestore(&dch->vc.lock, flags); + + return 0; +} + +static u32 d350_get_residue(struct d350_chan *dch) +{ + u32 res, xsize, xsizehi, hi_new; + int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */ + + hi_new = readl_relaxed(dch->base + CH_XSIZEHI); + do { + xsizehi = hi_new; + xsize = readl_relaxed(dch->base + CH_XSIZE); + hi_new = readl_relaxed(dch->base + CH_XSIZEHI); + } while (xsizehi != hi_new && --retries); + + res = FIELD_GET(CH_XY_DES, xsize); + res |= FIELD_GET(CH_XY_DES, xsizehi) << 16; + + return res << dch->desc->tsz; +} + +static int d350_terminate_all(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + LIST_HEAD(list); + + spin_lock_irqsave(&dch->vc.lock, flags); + writel_relaxed(CH_CMD_STOP, dch->base + CH_CMD); + if (dch->desc) { + if (dch->status != DMA_ERROR) + vchan_terminate_vdesc(&dch->desc->vd); + dch->desc = NULL; + dch->status = DMA_COMPLETE; + } + vchan_get_all_descriptors(&dch->vc, &list); + list_splice_tail(&list, &dch->vc.desc_terminated); + spin_unlock_irqrestore(&dch->vc.lock, flags); + + return 0; +} + +static void d350_synchronize(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + + vchan_synchronize(&dch->vc); +} + +static u32 d350_desc_bytes(struct d350_desc *desc) +{ + return ((u32)desc->xsizehi << 16 | desc->xsize) << desc->tsz; +} + +static enum dma_status d350_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct d350_chan *dch = to_d350_chan(chan); + struct virt_dma_desc *vd; + enum dma_status status; + unsigned long flags; + u32 residue = 0; + + status = dma_cookie_status(chan, cookie, state); + + spin_lock_irqsave(&dch->vc.lock, flags); + if (cookie == dch->cookie) { + status = dch->status; + if (status == DMA_IN_PROGRESS || status == DMA_PAUSED) + dch->residue = d350_get_residue(dch); + residue = dch->residue; + } else if ((vd = vchan_find_desc(&dch->vc, cookie))) { + residue = d350_desc_bytes(to_d350_desc(vd)); + } else if (status == DMA_IN_PROGRESS) { + /* Somebody else terminated it? */ + status = DMA_ERROR; + } + spin_unlock_irqrestore(&dch->vc.lock, flags); + + dma_set_residue(state, residue); + return status; +} + +static void d350_start_next(struct d350_chan *dch) +{ + u32 hdr, *reg; + + dch->desc = to_d350_desc(vchan_next_desc(&dch->vc)); + if (!dch->desc) + return; + + list_del(&dch->desc->vd.node); + dch->status = DMA_IN_PROGRESS; + dch->cookie = dch->desc->vd.tx.cookie; + dch->residue = d350_desc_bytes(dch->desc); + + hdr = dch->desc->command[0]; + reg = &dch->desc->command[1]; + + if (hdr & LINK_INTREN) + writel_relaxed(*reg++, dch->base + CH_INTREN); + if (hdr & LINK_CTRL) + writel_relaxed(*reg++, dch->base + CH_CTRL); + if (hdr & LINK_SRCADDR) + writel_relaxed(*reg++, dch->base + CH_SRCADDR); + if (hdr & LINK_SRCADDRHI) + writel_relaxed(*reg++, dch->base + CH_SRCADDRHI); + if (hdr & LINK_DESADDR) + writel_relaxed(*reg++, dch->base + CH_DESADDR); + if (hdr & LINK_DESADDRHI) + writel_relaxed(*reg++, dch->base + CH_DESADDRHI); + if (hdr & LINK_XSIZE) + writel_relaxed(*reg++, dch->base + CH_XSIZE); + if (hdr & LINK_XSIZEHI) + writel_relaxed(*reg++, dch->base + CH_XSIZEHI); + if (hdr & LINK_SRCTRANSCFG) + writel_relaxed(*reg++, dch->base + CH_SRCTRANSCFG); + if (hdr & LINK_DESTRANSCFG) + writel_relaxed(*reg++, dch->base + CH_DESTRANSCFG); + if (hdr & LINK_XADDRINC) + writel_relaxed(*reg++, dch->base + CH_XADDRINC); + if (hdr & LINK_FILLVAL) + writel_relaxed(*reg++, dch->base + CH_FILLVAL); + if (hdr & LINK_SRCTRIGINCFG) + writel_relaxed(*reg++, dch->base + CH_SRCTRIGINCFG); + if (hdr & LINK_DESTRIGINCFG) + writel_relaxed(*reg++, dch->base + CH_DESTRIGINCFG); + if (hdr & LINK_AUTOCFG) + writel_relaxed(*reg++, dch->base + CH_AUTOCFG); + if (hdr & LINK_LINKADDR) + writel_relaxed(*reg++, dch->base + CH_LINKADDR); + if (hdr & LINK_LINKADDRHI) + writel_relaxed(*reg++, dch->base + CH_LINKADDRHI); + + writel(CH_CMD_ENABLE, dch->base + CH_CMD); +} + +static void d350_issue_pending(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&dch->vc.lock, flags); + if (vchan_issue_pending(&dch->vc) && !dch->desc) + d350_start_next(dch); + spin_unlock_irqrestore(&dch->vc.lock, flags); +} + +static irqreturn_t d350_irq(int irq, void *data) +{ + struct d350_chan *dch = data; + struct device *dev = dch->vc.chan.device->dev; + struct virt_dma_desc *vd = &dch->desc->vd; + u32 ch_status; + + ch_status = readl(dch->base + CH_STATUS); + if (!ch_status) + return IRQ_NONE; + + if (ch_status & CH_STAT_INTR_ERR) { + u32 errinfo = readl_relaxed(dch->base + CH_ERRINFO); + + if (errinfo & (CH_ERRINFO_AXIRDPOISERR | CH_ERRINFO_AXIRDRESPERR)) + vd->tx_result.result = DMA_TRANS_READ_FAILED; + else if (errinfo & CH_ERRINFO_AXIWRRESPERR) + vd->tx_result.result = DMA_TRANS_WRITE_FAILED; + else + vd->tx_result.result = DMA_TRANS_ABORTED; + + vd->tx_result.residue = d350_get_residue(dch); + } else if (!(ch_status & CH_STAT_INTR_DONE)) { + dev_warn(dev, "Unexpected IRQ source? 0x%08x\n", ch_status); + } + writel_relaxed(ch_status, dch->base + CH_STATUS); + + spin_lock(&dch->vc.lock); + vchan_cookie_complete(vd); + if (ch_status & CH_STAT_INTR_DONE) { + dch->status = DMA_COMPLETE; + dch->residue = 0; + d350_start_next(dch); + } else { + dch->status = DMA_ERROR; + dch->residue = vd->tx_result.residue; + } + spin_unlock(&dch->vc.lock); + + return IRQ_HANDLED; +} + +static int d350_alloc_chan_resources(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + int ret = request_irq(dch->irq, d350_irq, IRQF_SHARED, + dev_name(&dch->vc.chan.dev->device), dch); + if (!ret) + writel_relaxed(CH_INTREN_DONE | CH_INTREN_ERR, dch->base + CH_INTREN); + + return ret; +} + +static void d350_free_chan_resources(struct dma_chan *chan) +{ + struct d350_chan *dch = to_d350_chan(chan); + + writel_relaxed(0, dch->base + CH_INTREN); + free_irq(dch->irq, dch); + vchan_free_chan_resources(&dch->vc); +} + +static int d350_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct d350 *dmac; + void __iomem *base; + u32 reg; + int ret, nchan, dw, aw, r, p; + bool coherent, memset; + + base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(base)) + return PTR_ERR(base); + + reg = readl_relaxed(base + DMAINFO + IIDR); + r = FIELD_GET(IIDR_VARIANT, reg); + p = FIELD_GET(IIDR_REVISION, reg); + if (FIELD_GET(IIDR_IMPLEMENTER, reg) != IMPLEMENTER_ARM || + FIELD_GET(IIDR_PRODUCTID, reg) != PRODUCTID_DMA350) + return dev_err_probe(dev, -ENODEV, "Not a DMA-350!"); + + reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG0); + nchan = FIELD_GET(DMA_CFG_NUM_CHANNELS, reg) + 1; + dw = 1 << FIELD_GET(DMA_CFG_DATA_WIDTH, reg); + aw = FIELD_GET(DMA_CFG_ADDR_WIDTH, reg) + 1; + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aw)); + coherent = device_get_dma_attr(dev) == DEV_DMA_COHERENT; + + dmac = devm_kzalloc(dev, struct_size(dmac, channels, nchan), GFP_KERNEL); + if (!dmac) + return -ENOMEM; + + dmac->nchan = nchan; + + reg = readl_relaxed(base + DMAINFO + DMA_BUILDCFG1); + dmac->nreq = FIELD_GET(DMA_CFG_NUM_TRIGGER_IN, reg); + + dev_dbg(dev, "DMA-350 r%dp%d with %d channels, %d requests\n", r, p, dmac->nchan, dmac->nreq); + + dmac->dma.dev = dev; + for (int i = min(dw, 16); i > 0; i /= 2) { + dmac->dma.src_addr_widths |= BIT(i); + dmac->dma.dst_addr_widths |= BIT(i); + } + dmac->dma.directions = BIT(DMA_MEM_TO_MEM); + dmac->dma.descriptor_reuse = true; + dmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dmac->dma.device_alloc_chan_resources = d350_alloc_chan_resources; + dmac->dma.device_free_chan_resources = d350_free_chan_resources; + dma_cap_set(DMA_MEMCPY, dmac->dma.cap_mask); + dmac->dma.device_prep_dma_memcpy = d350_prep_memcpy; + dmac->dma.device_pause = d350_pause; + dmac->dma.device_resume = d350_resume; + dmac->dma.device_terminate_all = d350_terminate_all; + dmac->dma.device_synchronize = d350_synchronize; + dmac->dma.device_tx_status = d350_tx_status; + dmac->dma.device_issue_pending = d350_issue_pending; + INIT_LIST_HEAD(&dmac->dma.channels); + + /* Would be nice to have per-channel caps for this... */ + memset = true; + for (int i = 0; i < nchan; i++) { + struct d350_chan *dch = &dmac->channels[i]; + + dch->base = base + DMACH(i); + writel_relaxed(CH_CMD_CLEAR, dch->base + CH_CMD); + + reg = readl_relaxed(dch->base + CH_BUILDCFG1); + if (!(FIELD_GET(CH_CFG_HAS_CMDLINK, reg))) { + dev_warn(dev, "No command link support on channel %d\n", i); + continue; + } + dch->irq = platform_get_irq(pdev, i); + if (dch->irq < 0) + return dev_err_probe(dev, dch->irq, + "Failed to get IRQ for channel %d\n", i); + + dch->has_wrap = FIELD_GET(CH_CFG_HAS_WRAP, reg); + dch->has_trig = FIELD_GET(CH_CFG_HAS_TRIGIN, reg) & + FIELD_GET(CH_CFG_HAS_TRIGSEL, reg); + + /* Fill is a special case of Wrap */ + memset &= dch->has_wrap; + + reg = readl_relaxed(dch->base + CH_BUILDCFG0); + dch->tsz = FIELD_GET(CH_CFG_DATA_WIDTH, reg); + + reg = FIELD_PREP(CH_LINK_SHAREATTR, coherent ? SHAREATTR_ISH : SHAREATTR_OSH); + reg |= FIELD_PREP(CH_LINK_MEMATTR, coherent ? MEMATTR_WB : MEMATTR_NC); + writel_relaxed(reg, dch->base + CH_LINKATTR); + + dch->vc.desc_free = d350_desc_free; + vchan_init(&dch->vc, &dmac->dma); + } + + if (memset) { + dma_cap_set(DMA_MEMSET, dmac->dma.cap_mask); + dmac->dma.device_prep_dma_memset = d350_prep_memset; + } + + platform_set_drvdata(pdev, dmac); + + ret = dma_async_device_register(&dmac->dma); + if (ret) + return dev_err_probe(dev, ret, "Failed to register DMA device\n"); + + return 0; +} + +static void d350_remove(struct platform_device *pdev) +{ + struct d350 *dmac = platform_get_drvdata(pdev); + + dma_async_device_unregister(&dmac->dma); +} + +static const struct of_device_id d350_of_match[] __maybe_unused = { + { .compatible = "arm,dma-350" }, + {} +}; +MODULE_DEVICE_TABLE(of, d350_of_match); + +static struct platform_driver d350_driver = { + .driver = { + .name = "arm-dma350", + .of_match_table = of_match_ptr(d350_of_match), + }, + .probe = d350_probe, + .remove = d350_remove, +}; +module_platform_driver(d350_driver); + +MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>"); +MODULE_DESCRIPTION("Arm DMA-350 driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 30ae36124b1d..7d226453961f 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -3,6 +3,7 @@ * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems) * * Copyright (C) 2008 Atmel Corporation + * Copyright (C) 2022 Microchip Technology, Inc. and its subsidiaries * * This supports the Atmel AHB DMA Controller found in several Atmel SoCs. * The only Atmel DMA Controller that is not covered by this driver is the one @@ -10,20 +11,22 @@ */ #include <dt-bindings/dma/at91.h> +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/dmaengine.h> -#include <linux/dma-mapping.h> #include <linux/dmapool.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/module.h> -#include <linux/platform_device.h> -#include <linux/slab.h> #include <linux/of.h> -#include <linux/of_device.h> +#include <linux/overflow.h> +#include <linux/of_platform.h> #include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> -#include "at_hdmac_regs.h" #include "dmaengine.h" +#include "virt-dma.h" /* * Glossary @@ -34,9 +37,460 @@ * atc_ / atchan : ATmel DMA Channel entity related */ -#define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO) -#define ATC_DEFAULT_CTRLB (ATC_SIF(AT_DMA_MEM_IF) \ - |ATC_DIF(AT_DMA_MEM_IF)) +#define AT_DMA_MAX_NR_CHANNELS 8 + +/* Global Configuration Register */ +#define AT_DMA_GCFG 0x00 +#define AT_DMA_IF_BIGEND(i) BIT((i)) /* AHB-Lite Interface i in Big-endian mode */ +#define AT_DMA_ARB_CFG BIT(4) /* Arbiter mode. */ + +/* Controller Enable Register */ +#define AT_DMA_EN 0x04 +#define AT_DMA_ENABLE BIT(0) + +/* Software Single Request Register */ +#define AT_DMA_SREQ 0x08 +#define AT_DMA_SSREQ(x) BIT((x) << 1) /* Request a source single transfer on channel x */ +#define AT_DMA_DSREQ(x) BIT(1 + ((x) << 1)) /* Request a destination single transfer on channel x */ + +/* Software Chunk Transfer Request Register */ +#define AT_DMA_CREQ 0x0c +#define AT_DMA_SCREQ(x) BIT((x) << 1) /* Request a source chunk transfer on channel x */ +#define AT_DMA_DCREQ(x) BIT(1 + ((x) << 1)) /* Request a destination chunk transfer on channel x */ + +/* Software Last Transfer Flag Register */ +#define AT_DMA_LAST 0x10 +#define AT_DMA_SLAST(x) BIT((x) << 1) /* This src rq is last tx of buffer on channel x */ +#define AT_DMA_DLAST(x) BIT(1 + ((x) << 1)) /* This dst rq is last tx of buffer on channel x */ + +/* Request Synchronization Register */ +#define AT_DMA_SYNC 0x14 +#define AT_DMA_SYR(h) BIT((h)) /* Synchronize handshake line h */ + +/* Error, Chained Buffer transfer completed and Buffer transfer completed Interrupt registers */ +#define AT_DMA_EBCIER 0x18 /* Enable register */ +#define AT_DMA_EBCIDR 0x1c /* Disable register */ +#define AT_DMA_EBCIMR 0x20 /* Mask Register */ +#define AT_DMA_EBCISR 0x24 /* Status Register */ +#define AT_DMA_CBTC_OFFSET 8 +#define AT_DMA_ERR_OFFSET 16 +#define AT_DMA_BTC(x) BIT((x)) +#define AT_DMA_CBTC(x) BIT(AT_DMA_CBTC_OFFSET + (x)) +#define AT_DMA_ERR(x) BIT(AT_DMA_ERR_OFFSET + (x)) + +/* Channel Handler Enable Register */ +#define AT_DMA_CHER 0x28 +#define AT_DMA_ENA(x) BIT((x)) +#define AT_DMA_SUSP(x) BIT(8 + (x)) +#define AT_DMA_KEEP(x) BIT(24 + (x)) + +/* Channel Handler Disable Register */ +#define AT_DMA_CHDR 0x2c +#define AT_DMA_DIS(x) BIT(x) +#define AT_DMA_RES(x) BIT(8 + (x)) + +/* Channel Handler Status Register */ +#define AT_DMA_CHSR 0x30 +#define AT_DMA_EMPT(x) BIT(16 + (x)) +#define AT_DMA_STAL(x) BIT(24 + (x)) + +/* Channel registers base address */ +#define AT_DMA_CH_REGS_BASE 0x3c +#define ch_regs(x) (AT_DMA_CH_REGS_BASE + (x) * 0x28) /* Channel x base addr */ + +/* Hardware register offset for each channel */ +#define ATC_SADDR_OFFSET 0x00 /* Source Address Register */ +#define ATC_DADDR_OFFSET 0x04 /* Destination Address Register */ +#define ATC_DSCR_OFFSET 0x08 /* Descriptor Address Register */ +#define ATC_CTRLA_OFFSET 0x0c /* Control A Register */ +#define ATC_CTRLB_OFFSET 0x10 /* Control B Register */ +#define ATC_CFG_OFFSET 0x14 /* Configuration Register */ +#define ATC_SPIP_OFFSET 0x18 /* Src PIP Configuration Register */ +#define ATC_DPIP_OFFSET 0x1c /* Dst PIP Configuration Register */ + + +/* Bitfield definitions */ + +/* Bitfields in DSCR */ +#define ATC_DSCR_IF GENMASK(1, 0) /* Dsc feched via AHB-Lite Interface */ + +/* Bitfields in CTRLA */ +#define ATC_BTSIZE_MAX GENMASK(15, 0) /* Maximum Buffer Transfer Size */ +#define ATC_BTSIZE GENMASK(15, 0) /* Buffer Transfer Size */ +#define ATC_SCSIZE GENMASK(18, 16) /* Source Chunk Transfer Size */ +#define ATC_DCSIZE GENMASK(22, 20) /* Destination Chunk Transfer Size */ +#define ATC_SRC_WIDTH GENMASK(25, 24) /* Source Single Transfer Size */ +#define ATC_DST_WIDTH GENMASK(29, 28) /* Destination Single Transfer Size */ +#define ATC_DONE BIT(31) /* Tx Done (only written back in descriptor) */ + +/* Bitfields in CTRLB */ +#define ATC_SIF GENMASK(1, 0) /* Src tx done via AHB-Lite Interface i */ +#define ATC_DIF GENMASK(5, 4) /* Dst tx done via AHB-Lite Interface i */ +#define AT_DMA_MEM_IF 0x0 /* interface 0 as memory interface */ +#define AT_DMA_PER_IF 0x1 /* interface 1 as peripheral interface */ +#define ATC_SRC_PIP BIT(8) /* Source Picture-in-Picture enabled */ +#define ATC_DST_PIP BIT(12) /* Destination Picture-in-Picture enabled */ +#define ATC_SRC_DSCR_DIS BIT(16) /* Src Descriptor fetch disable */ +#define ATC_DST_DSCR_DIS BIT(20) /* Dst Descriptor fetch disable */ +#define ATC_FC GENMASK(23, 21) /* Choose Flow Controller */ +#define ATC_FC_MEM2MEM 0x0 /* Mem-to-Mem (DMA) */ +#define ATC_FC_MEM2PER 0x1 /* Mem-to-Periph (DMA) */ +#define ATC_FC_PER2MEM 0x2 /* Periph-to-Mem (DMA) */ +#define ATC_FC_PER2PER 0x3 /* Periph-to-Periph (DMA) */ +#define ATC_FC_PER2MEM_PER 0x4 /* Periph-to-Mem (Peripheral) */ +#define ATC_FC_MEM2PER_PER 0x5 /* Mem-to-Periph (Peripheral) */ +#define ATC_FC_PER2PER_SRCPER 0x6 /* Periph-to-Periph (Src Peripheral) */ +#define ATC_FC_PER2PER_DSTPER 0x7 /* Periph-to-Periph (Dst Peripheral) */ +#define ATC_SRC_ADDR_MODE GENMASK(25, 24) +#define ATC_SRC_ADDR_MODE_INCR 0x0 /* Incrementing Mode */ +#define ATC_SRC_ADDR_MODE_DECR 0x1 /* Decrementing Mode */ +#define ATC_SRC_ADDR_MODE_FIXED 0x2 /* Fixed Mode */ +#define ATC_DST_ADDR_MODE GENMASK(29, 28) +#define ATC_DST_ADDR_MODE_INCR 0x0 /* Incrementing Mode */ +#define ATC_DST_ADDR_MODE_DECR 0x1 /* Decrementing Mode */ +#define ATC_DST_ADDR_MODE_FIXED 0x2 /* Fixed Mode */ +#define ATC_IEN BIT(30) /* BTC interrupt enable (active low) */ +#define ATC_AUTO BIT(31) /* Auto multiple buffer tx enable */ + +/* Bitfields in CFG */ +#define ATC_SRC_PER GENMASK(3, 0) /* Channel src rq associated with periph handshaking ifc h */ +#define ATC_DST_PER GENMASK(7, 4) /* Channel dst rq associated with periph handshaking ifc h */ +#define ATC_SRC_REP BIT(8) /* Source Replay Mod */ +#define ATC_SRC_H2SEL BIT(9) /* Source Handshaking Mod */ +#define ATC_SRC_PER_MSB GENMASK(11, 10) /* Channel src rq (most significant bits) */ +#define ATC_DST_REP BIT(12) /* Destination Replay Mod */ +#define ATC_DST_H2SEL BIT(13) /* Destination Handshaking Mod */ +#define ATC_DST_PER_MSB GENMASK(15, 14) /* Channel dst rq (most significant bits) */ +#define ATC_SOD BIT(16) /* Stop On Done */ +#define ATC_LOCK_IF BIT(20) /* Interface Lock */ +#define ATC_LOCK_B BIT(21) /* AHB Bus Lock */ +#define ATC_LOCK_IF_L BIT(22) /* Master Interface Arbiter Lock */ +#define ATC_AHB_PROT GENMASK(26, 24) /* AHB Protection */ +#define ATC_FIFOCFG GENMASK(29, 28) /* FIFO Request Configuration */ +#define ATC_FIFOCFG_LARGESTBURST 0x0 +#define ATC_FIFOCFG_HALFFIFO 0x1 +#define ATC_FIFOCFG_ENOUGHSPACE 0x2 + +/* Bitfields in SPIP */ +#define ATC_SPIP_HOLE GENMASK(15, 0) +#define ATC_SPIP_BOUNDARY GENMASK(25, 16) + +/* Bitfields in DPIP */ +#define ATC_DPIP_HOLE GENMASK(15, 0) +#define ATC_DPIP_BOUNDARY GENMASK(25, 16) + +#define ATC_PER_MSB GENMASK(5, 4) /* Extract MSBs of a handshaking identifier */ +#define ATC_SRC_PER_ID(id) \ + ({ typeof(id) _id = (id); \ + FIELD_PREP(ATC_SRC_PER_MSB, FIELD_GET(ATC_PER_MSB, _id)) | \ + FIELD_PREP(ATC_SRC_PER, _id); }) +#define ATC_DST_PER_ID(id) \ + ({ typeof(id) _id = (id); \ + FIELD_PREP(ATC_DST_PER_MSB, FIELD_GET(ATC_PER_MSB, _id)) | \ + FIELD_PREP(ATC_DST_PER, _id); }) + + + +/*-- descriptors -----------------------------------------------------*/ + +/* LLI == Linked List Item; aka DMA buffer descriptor */ +struct at_lli { + /* values that are not changed by hardware */ + u32 saddr; + u32 daddr; + /* value that may get written back: */ + u32 ctrla; + /* more values that are not changed by hardware */ + u32 ctrlb; + u32 dscr; /* chain to next lli */ +}; + +/** + * struct atdma_sg - atdma scatter gather entry + * @len: length of the current Linked List Item. + * @lli: linked list item that is passed to the DMA controller + * @lli_phys: physical address of the LLI. + */ +struct atdma_sg { + unsigned int len; + struct at_lli *lli; + dma_addr_t lli_phys; +}; + +/** + * struct at_desc - software descriptor + * @vd: pointer to the virtual dma descriptor. + * @atchan: pointer to the atmel dma channel. + * @total_len: total transaction byte count + * @sglen: number of sg entries. + * @sg: array of sgs. + * @boundary: number of transfers to perform before the automatic address increment operation + * @dst_hole: value to add to the destination address when the boundary has been reached + * @src_hole: value to add to the source address when the boundary has been reached + * @memset_buffer: buffer used for the memset operation + * @memset_paddr: physical address of the buffer used for the memset operation + * @memset_vaddr: virtual address of the buffer used for the memset operation + */ +struct at_desc { + struct virt_dma_desc vd; + struct at_dma_chan *atchan; + size_t total_len; + unsigned int sglen; + /* Interleaved data */ + size_t boundary; + size_t dst_hole; + size_t src_hole; + + /* Memset temporary buffer */ + bool memset_buffer; + dma_addr_t memset_paddr; + int *memset_vaddr; + struct atdma_sg sg[] __counted_by(sglen); +}; + +/*-- Channels --------------------------------------------------------*/ + +/** + * enum atc_status - information bits stored in channel status flag + * + * @ATC_IS_PAUSED: If channel is pauses + * @ATC_IS_CYCLIC: If channel is cyclic + * + * Manipulated with atomic operations. + */ +enum atc_status { + ATC_IS_PAUSED = 1, + ATC_IS_CYCLIC = 24, +}; + +/** + * struct at_dma_chan - internal representation of an Atmel HDMAC channel + * @vc: virtual dma channel entry. + * @atdma: pointer to the driver data. + * @ch_regs: memory mapped register base + * @mask: channel index in a mask + * @per_if: peripheral interface + * @mem_if: memory interface + * @status: transmit status information from irq/prep* functions + * to tasklet (use atomic operations) + * @save_cfg: configuration register that is saved on suspend/resume cycle + * @save_dscr: for cyclic operations, preserve next descriptor address in + * the cyclic list on suspend/resume cycle + * @dma_sconfig: configuration for slave transfers, passed via + * .device_config + * @desc: pointer to the atmel dma descriptor. + */ +struct at_dma_chan { + struct virt_dma_chan vc; + struct at_dma *atdma; + void __iomem *ch_regs; + u8 mask; + u8 per_if; + u8 mem_if; + unsigned long status; + u32 save_cfg; + u32 save_dscr; + struct dma_slave_config dma_sconfig; + struct at_desc *desc; +}; + +#define channel_readl(atchan, name) \ + __raw_readl((atchan)->ch_regs + ATC_##name##_OFFSET) + +#define channel_writel(atchan, name, val) \ + __raw_writel((val), (atchan)->ch_regs + ATC_##name##_OFFSET) + +/* + * Fix sconfig's burst size according to at_hdmac. We need to convert them as: + * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3, 32 -> 4, 64 -> 5, 128 -> 6, 256 -> 7. + * + * This can be done by finding most significant bit set. + */ +static inline void convert_burst(u32 *maxburst) +{ + if (*maxburst > 1) + *maxburst = fls(*maxburst) - 2; + else + *maxburst = 0; +} + +/* + * Fix sconfig's bus width according to at_hdmac. + * 1 byte -> 0, 2 bytes -> 1, 4 bytes -> 2. + */ +static inline u8 convert_buswidth(enum dma_slave_buswidth addr_width) +{ + switch (addr_width) { + case DMA_SLAVE_BUSWIDTH_2_BYTES: + return 1; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + return 2; + default: + /* For 1 byte width or fallback */ + return 0; + } +} + +/*-- Controller ------------------------------------------------------*/ + +/** + * struct at_dma - internal representation of an Atmel HDMA Controller + * @dma_device: dmaengine dma_device object members + * @regs: memory mapped register base + * @clk: dma controller clock + * @save_imr: interrupt mask register that is saved on suspend/resume cycle + * @all_chan_mask: all channels available in a mask + * @lli_pool: hw lli table + * @memset_pool: hw memset pool + * @chan: channels table to store at_dma_chan structures + */ +struct at_dma { + struct dma_device dma_device; + void __iomem *regs; + struct clk *clk; + u32 save_imr; + + u8 all_chan_mask; + + struct dma_pool *lli_pool; + struct dma_pool *memset_pool; + /* AT THE END channels table */ + struct at_dma_chan chan[]; +}; + +#define dma_readl(atdma, name) \ + __raw_readl((atdma)->regs + AT_DMA_##name) +#define dma_writel(atdma, name, val) \ + __raw_writel((val), (atdma)->regs + AT_DMA_##name) + +static inline struct at_desc *to_atdma_desc(struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct at_desc, vd.tx); +} + +static inline struct at_dma_chan *to_at_dma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct at_dma_chan, vc.chan); +} + +static inline struct at_dma *to_at_dma(struct dma_device *ddev) +{ + return container_of(ddev, struct at_dma, dma_device); +} + + +/*-- Helper functions ------------------------------------------------*/ + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +#if defined(VERBOSE_DEBUG) +static void vdbg_dump_regs(struct at_dma_chan *atchan) +{ + struct at_dma *atdma = to_at_dma(atchan->vc.chan.device); + + dev_err(chan2dev(&atchan->vc.chan), + " channel %d : imr = 0x%x, chsr = 0x%x\n", + atchan->vc.chan.chan_id, + dma_readl(atdma, EBCIMR), + dma_readl(atdma, CHSR)); + + dev_err(chan2dev(&atchan->vc.chan), + " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n", + channel_readl(atchan, SADDR), + channel_readl(atchan, DADDR), + channel_readl(atchan, CTRLA), + channel_readl(atchan, CTRLB), + channel_readl(atchan, CFG), + channel_readl(atchan, DSCR)); +} +#else +static void vdbg_dump_regs(struct at_dma_chan *atchan) {} +#endif + +static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli) +{ + dev_crit(chan2dev(&atchan->vc.chan), + "desc: s%pad d%pad ctrl0x%x:0x%x l%pad\n", + &lli->saddr, &lli->daddr, + lli->ctrla, lli->ctrlb, &lli->dscr); +} + + +static void atc_setup_irq(struct at_dma *atdma, int chan_id, int on) +{ + u32 ebci; + + /* enable interrupts on buffer transfer completion & error */ + ebci = AT_DMA_BTC(chan_id) + | AT_DMA_ERR(chan_id); + if (on) + dma_writel(atdma, EBCIER, ebci); + else + dma_writel(atdma, EBCIDR, ebci); +} + +static void atc_enable_chan_irq(struct at_dma *atdma, int chan_id) +{ + atc_setup_irq(atdma, chan_id, 1); +} + +static void atc_disable_chan_irq(struct at_dma *atdma, int chan_id) +{ + atc_setup_irq(atdma, chan_id, 0); +} + + +/** + * atc_chan_is_enabled - test if given channel is enabled + * @atchan: channel we want to test status + */ +static inline int atc_chan_is_enabled(struct at_dma_chan *atchan) +{ + struct at_dma *atdma = to_at_dma(atchan->vc.chan.device); + + return !!(dma_readl(atdma, CHSR) & atchan->mask); +} + +/** + * atc_chan_is_paused - test channel pause/resume status + * @atchan: channel we want to test status + */ +static inline int atc_chan_is_paused(struct at_dma_chan *atchan) +{ + return test_bit(ATC_IS_PAUSED, &atchan->status); +} + +/** + * atc_chan_is_cyclic - test if given channel has cyclic property set + * @atchan: channel we want to test status + */ +static inline int atc_chan_is_cyclic(struct at_dma_chan *atchan) +{ + return test_bit(ATC_IS_CYCLIC, &atchan->status); +} + +/** + * set_lli_eol - set end-of-link to descriptor so it will end transfer + * @desc: descriptor, signle or at the end of a chain, to end chain on + * @i: index of the atmel scatter gather entry that is at the end of the chain. + */ +static void set_lli_eol(struct at_desc *desc, unsigned int i) +{ + u32 ctrlb = desc->sg[i].lli->ctrlb; + + ctrlb &= ~ATC_IEN; + ctrlb |= ATC_SRC_DSCR_DIS | ATC_DST_DSCR_DIS; + + desc->sg[i].lli->ctrlb = ctrlb; + desc->sg[i].lli->dscr = 0; +} + +#define ATC_DEFAULT_CFG FIELD_PREP(ATC_FIFOCFG, ATC_FIFOCFG_HALFFIFO) +#define ATC_DEFAULT_CTRLB (FIELD_PREP(ATC_SIF, AT_DMA_MEM_IF) | \ + FIELD_PREP(ATC_DIF, AT_DMA_MEM_IF)) #define ATC_DMA_BUSWIDTHS\ (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\ BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\ @@ -74,13 +528,6 @@ struct at_dma_slave { u32 cfg; }; -/* prototypes */ -static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx); -static void atc_issue_pending(struct dma_chan *chan); - - -/*----------------------------------------------------------------------*/ - static inline unsigned int atc_get_xfer_width(dma_addr_t src, dma_addr_t dst, size_t len) { @@ -96,192 +543,72 @@ static inline unsigned int atc_get_xfer_width(dma_addr_t src, dma_addr_t dst, return width; } -static struct at_desc *atc_first_active(struct at_dma_chan *atchan) -{ - return list_first_entry(&atchan->active_list, - struct at_desc, desc_node); -} - -static struct at_desc *atc_first_queued(struct at_dma_chan *atchan) -{ - return list_first_entry(&atchan->queue, - struct at_desc, desc_node); -} - -/** - * atc_alloc_descriptor - allocate and return an initialized descriptor - * @chan: the channel to allocate descriptors for - * @gfp_flags: GFP allocation flags - * - * Note: The ack-bit is positioned in the descriptor flag at creation time - * to make initial allocation more convenient. This bit will be cleared - * and control will be given to client at usage time (during - * preparation functions). - */ -static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan, - gfp_t gfp_flags) +static void atdma_lli_chain(struct at_desc *desc, unsigned int i) { - struct at_desc *desc = NULL; - struct at_dma *atdma = to_at_dma(chan->device); - dma_addr_t phys; + struct atdma_sg *atdma_sg = &desc->sg[i]; - desc = dma_pool_zalloc(atdma->dma_desc_pool, gfp_flags, &phys); - if (desc) { - INIT_LIST_HEAD(&desc->tx_list); - dma_async_tx_descriptor_init(&desc->txd, chan); - /* txd.flags will be overwritten in prep functions */ - desc->txd.flags = DMA_CTRL_ACK; - desc->txd.tx_submit = atc_tx_submit; - desc->txd.phys = phys; - } - - return desc; -} - -/** - * atc_desc_get - get an unused descriptor from free_list - * @atchan: channel we want a new descriptor for - */ -static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) -{ - struct at_desc *desc, *_desc; - struct at_desc *ret = NULL; - unsigned long flags; - unsigned int i = 0; - - spin_lock_irqsave(&atchan->lock, flags); - list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { - i++; - if (async_tx_test_ack(&desc->txd)) { - list_del(&desc->desc_node); - ret = desc; - break; - } - dev_dbg(chan2dev(&atchan->chan_common), - "desc %p not ACKed\n", desc); - } - spin_unlock_irqrestore(&atchan->lock, flags); - dev_vdbg(chan2dev(&atchan->chan_common), - "scanned %u descriptors on freelist\n", i); - - /* no more descriptor available in initial pool: create one more */ - if (!ret) - ret = atc_alloc_descriptor(&atchan->chan_common, GFP_NOWAIT); - - return ret; -} - -/** - * atc_desc_put - move a descriptor, including any children, to the free list - * @atchan: channel we work on - * @desc: descriptor, at the head of a chain, to move to free list - */ -static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) -{ - if (desc) { - struct at_desc *child; - unsigned long flags; - - spin_lock_irqsave(&atchan->lock, flags); - list_for_each_entry(child, &desc->tx_list, desc_node) - dev_vdbg(chan2dev(&atchan->chan_common), - "moving child desc %p to freelist\n", - child); - list_splice_init(&desc->tx_list, &atchan->free_list); - dev_vdbg(chan2dev(&atchan->chan_common), - "moving desc %p to freelist\n", desc); - list_add(&desc->desc_node, &atchan->free_list); - spin_unlock_irqrestore(&atchan->lock, flags); - } -} - -/** - * atc_desc_chain - build chain adding a descriptor - * @first: address of first descriptor of the chain - * @prev: address of previous descriptor of the chain - * @desc: descriptor to queue - * - * Called from prep_* functions - */ -static void atc_desc_chain(struct at_desc **first, struct at_desc **prev, - struct at_desc *desc) -{ - if (!(*first)) { - *first = desc; - } else { - /* inform the HW lli about chaining */ - (*prev)->lli.dscr = desc->txd.phys; - /* insert the link descriptor to the LD ring */ - list_add_tail(&desc->desc_node, - &(*first)->tx_list); - } - *prev = desc; + if (i) + desc->sg[i - 1].lli->dscr = atdma_sg->lli_phys; } /** * atc_dostart - starts the DMA engine for real * @atchan: the channel we want to start - * @first: first descriptor in the list we want to begin with - * - * Called with atchan->lock held and bh disabled */ -static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) +static void atc_dostart(struct at_dma_chan *atchan) { - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); + struct virt_dma_desc *vd = vchan_next_desc(&atchan->vc); + struct at_desc *desc; - /* ASSERT: channel is idle */ - if (atc_chan_is_enabled(atchan)) { - dev_err(chan2dev(&atchan->chan_common), - "BUG: Attempted to start non-idle channel\n"); - dev_err(chan2dev(&atchan->chan_common), - " channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n", - channel_readl(atchan, SADDR), - channel_readl(atchan, DADDR), - channel_readl(atchan, CTRLA), - channel_readl(atchan, CTRLB), - channel_readl(atchan, DSCR)); - - /* The tasklet will hopefully advance the queue... */ + if (!vd) { + atchan->desc = NULL; return; } vdbg_dump_regs(atchan); + list_del(&vd->node); + atchan->desc = desc = to_atdma_desc(&vd->tx); + channel_writel(atchan, SADDR, 0); channel_writel(atchan, DADDR, 0); channel_writel(atchan, CTRLA, 0); channel_writel(atchan, CTRLB, 0); - channel_writel(atchan, DSCR, first->txd.phys); - channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) | - ATC_SPIP_BOUNDARY(first->boundary)); - channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) | - ATC_DPIP_BOUNDARY(first->boundary)); - dma_writel(atdma, CHER, atchan->mask); + channel_writel(atchan, DSCR, desc->sg[0].lli_phys); + channel_writel(atchan, SPIP, + FIELD_PREP(ATC_SPIP_HOLE, desc->src_hole) | + FIELD_PREP(ATC_SPIP_BOUNDARY, desc->boundary)); + channel_writel(atchan, DPIP, + FIELD_PREP(ATC_DPIP_HOLE, desc->dst_hole) | + FIELD_PREP(ATC_DPIP_BOUNDARY, desc->boundary)); + + /* Don't allow CPU to reorder channel enable. */ + wmb(); + dma_writel(atchan->atdma, CHER, atchan->mask); vdbg_dump_regs(atchan); } -/* - * atc_get_desc_by_cookie - get the descriptor of a cookie - * @atchan: the DMA channel - * @cookie: the cookie to get the descriptor for - */ -static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, - dma_cookie_t cookie) +static void atdma_desc_free(struct virt_dma_desc *vd) { - struct at_desc *desc, *_desc; - - list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) { - if (desc->txd.cookie == cookie) - return desc; + struct at_dma *atdma = to_at_dma(vd->tx.chan->device); + struct at_desc *desc = to_atdma_desc(&vd->tx); + unsigned int i; + + for (i = 0; i < desc->sglen; i++) { + if (desc->sg[i].lli) + dma_pool_free(atdma->lli_pool, desc->sg[i].lli, + desc->sg[i].lli_phys); } - list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { - if (desc->txd.cookie == cookie) - return desc; + /* If the transfer was a memset, free our temporary buffer */ + if (desc->memset_buffer) { + dma_pool_free(atdma->memset_pool, desc->memset_vaddr, + desc->memset_paddr); + desc->memset_buffer = false; } - return NULL; + kfree(desc); } /** @@ -291,10 +618,10 @@ static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, * @current_len: the number of bytes left before reading CTRLA * @ctrla: the value of CTRLA */ -static inline int atc_calc_bytes_left(int current_len, u32 ctrla) +static inline u32 atc_calc_bytes_left(u32 current_len, u32 ctrla) { - u32 btsize = (ctrla & ATC_BTSIZE_MAX); - u32 src_width = ATC_REG_TO_SRC_WIDTH(ctrla); + u32 btsize = FIELD_GET(ATC_BTSIZE, ctrla); + u32 src_width = FIELD_GET(ATC_SRC_WIDTH, ctrla); /* * According to the datasheet, when reading the Control A Register @@ -306,277 +633,155 @@ static inline int atc_calc_bytes_left(int current_len, u32 ctrla) } /** - * atc_get_bytes_left - get the number of bytes residue for a cookie - * @chan: DMA channel - * @cookie: transaction identifier to check status of + * atc_get_llis_residue - Get residue for a hardware linked list transfer + * @atchan: pointer to an atmel hdmac channel. + * @desc: pointer to the descriptor for which the residue is calculated. + * @residue: residue to be set to dma_tx_state. + * + * Calculate the residue by removing the length of the Linked List Item (LLI) + * already transferred from the total length. To get the current LLI we can use + * the value of the channel's DSCR register and compare it against the DSCR + * value of each LLI. + * + * The CTRLA register provides us with the amount of data already read from the + * source for the LLI. So we can compute a more accurate residue by also + * removing the number of bytes corresponding to this amount of data. + * + * However, the DSCR and CTRLA registers cannot be read both atomically. Hence a + * race condition may occur: the first read register may refer to one LLI + * whereas the second read may refer to a later LLI in the list because of the + * DMA transfer progression inbetween the two reads. + * + * One solution could have been to pause the DMA transfer, read the DSCR and + * CTRLA then resume the DMA transfer. Nonetheless, this approach presents some + * drawbacks: + * - If the DMA transfer is paused, RX overruns or TX underruns are more likey + * to occur depending on the system latency. Taking the USART driver as an + * example, it uses a cyclic DMA transfer to read data from the Receive + * Holding Register (RHR) to avoid RX overruns since the RHR is not protected + * by any FIFO on most Atmel SoCs. So pausing the DMA transfer to compute the + * residue would break the USART driver design. + * - The atc_pause() function masks interrupts but we'd rather avoid to do so + * for system latency purpose. + * + * Then we'd rather use another solution: the DSCR is read a first time, the + * CTRLA is read in turn, next the DSCR is read a second time. If the two + * consecutive read values of the DSCR are the same then we assume both refers + * to the very same LLI as well as the CTRLA value read inbetween does. For + * cyclic transfers, the assumption is that a full loop is "not so fast". If the + * two DSCR values are different, we read again the CTRLA then the DSCR till two + * consecutive read values from DSCR are equal or till the maximum trials is + * reach. This algorithm is very unlikely not to find a stable value for DSCR. + * + * Returns: %0 on success, -errno otherwise. */ -static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) +static int atc_get_llis_residue(struct at_dma_chan *atchan, + struct at_desc *desc, u32 *residue) { - struct at_dma_chan *atchan = to_at_dma_chan(chan); - struct at_desc *desc_first = atc_first_active(atchan); - struct at_desc *desc; - int ret; - u32 ctrla, dscr, trials; - - /* - * If the cookie doesn't match to the currently running transfer then - * we can return the total length of the associated DMA transfer, - * because it is still queued. - */ - desc = atc_get_desc_by_cookie(atchan, cookie); - if (desc == NULL) - return -EINVAL; - else if (desc != desc_first) - return desc->total_len; + u32 len, ctrla, dscr; + unsigned int i; - /* cookie matches to the currently running transfer */ - ret = desc_first->total_len; + len = desc->total_len; + dscr = channel_readl(atchan, DSCR); + rmb(); /* ensure DSCR is read before CTRLA */ + ctrla = channel_readl(atchan, CTRLA); + for (i = 0; i < ATC_MAX_DSCR_TRIALS; ++i) { + u32 new_dscr; - if (desc_first->lli.dscr) { - /* hardware linked list transfer */ + rmb(); /* ensure DSCR is read after CTRLA */ + new_dscr = channel_readl(atchan, DSCR); /* - * Calculate the residue by removing the length of the child - * descriptors already transferred from the total length. - * To get the current child descriptor we can use the value of - * the channel's DSCR register and compare it against the value - * of the hardware linked list structure of each child - * descriptor. - * - * The CTRLA register provides us with the amount of data - * already read from the source for the current child - * descriptor. So we can compute a more accurate residue by also - * removing the number of bytes corresponding to this amount of - * data. - * - * However, the DSCR and CTRLA registers cannot be read both - * atomically. Hence a race condition may occur: the first read - * register may refer to one child descriptor whereas the second - * read may refer to a later child descriptor in the list - * because of the DMA transfer progression inbetween the two - * reads. - * - * One solution could have been to pause the DMA transfer, read - * the DSCR and CTRLA then resume the DMA transfer. Nonetheless, - * this approach presents some drawbacks: - * - If the DMA transfer is paused, RX overruns or TX underruns - * are more likey to occur depending on the system latency. - * Taking the USART driver as an example, it uses a cyclic DMA - * transfer to read data from the Receive Holding Register - * (RHR) to avoid RX overruns since the RHR is not protected - * by any FIFO on most Atmel SoCs. So pausing the DMA transfer - * to compute the residue would break the USART driver design. - * - The atc_pause() function masks interrupts but we'd rather - * avoid to do so for system latency purpose. - * - * Then we'd rather use another solution: the DSCR is read a - * first time, the CTRLA is read in turn, next the DSCR is read - * a second time. If the two consecutive read values of the DSCR - * are the same then we assume both refers to the very same - * child descriptor as well as the CTRLA value read inbetween - * does. For cyclic tranfers, the assumption is that a full loop - * is "not so fast". - * If the two DSCR values are different, we read again the CTRLA - * then the DSCR till two consecutive read values from DSCR are - * equal or till the maxium trials is reach. - * This algorithm is very unlikely not to find a stable value for - * DSCR. + * If the DSCR register value has not changed inside the DMA + * controller since the previous read, we assume that both the + * dscr and ctrla values refers to the very same descriptor. */ - - dscr = channel_readl(atchan, DSCR); - rmb(); /* ensure DSCR is read before CTRLA */ - ctrla = channel_readl(atchan, CTRLA); - for (trials = 0; trials < ATC_MAX_DSCR_TRIALS; ++trials) { - u32 new_dscr; - - rmb(); /* ensure DSCR is read after CTRLA */ - new_dscr = channel_readl(atchan, DSCR); - - /* - * If the DSCR register value has not changed inside the - * DMA controller since the previous read, we assume - * that both the dscr and ctrla values refers to the - * very same descriptor. - */ - if (likely(new_dscr == dscr)) - break; - - /* - * DSCR has changed inside the DMA controller, so the - * previouly read value of CTRLA may refer to an already - * processed descriptor hence could be outdated. - * We need to update ctrla to match the current - * descriptor. - */ - dscr = new_dscr; - rmb(); /* ensure DSCR is read before CTRLA */ - ctrla = channel_readl(atchan, CTRLA); - } - if (unlikely(trials >= ATC_MAX_DSCR_TRIALS)) - return -ETIMEDOUT; - - /* for the first descriptor we can be more accurate */ - if (desc_first->lli.dscr == dscr) - return atc_calc_bytes_left(ret, ctrla); - - ret -= desc_first->len; - list_for_each_entry(desc, &desc_first->tx_list, desc_node) { - if (desc->lli.dscr == dscr) - break; - - ret -= desc->len; - } + if (likely(new_dscr == dscr)) + break; /* - * For the current descriptor in the chain we can calculate - * the remaining bytes using the channel's register. + * DSCR has changed inside the DMA controller, so the previously + * read value of CTRLA may refer to an already processed + * descriptor hence could be outdated. We need to update ctrla + * to match the current descriptor. */ - ret = atc_calc_bytes_left(ret, ctrla); - } else { - /* single transfer */ + dscr = new_dscr; + rmb(); /* ensure DSCR is read before CTRLA */ ctrla = channel_readl(atchan, CTRLA); - ret = atc_calc_bytes_left(ret, ctrla); } + if (unlikely(i == ATC_MAX_DSCR_TRIALS)) + return -ETIMEDOUT; - return ret; -} - -/** - * atc_chain_complete - finish work for one transaction chain - * @atchan: channel we work on - * @desc: descriptor at the head of the chain we want do complete - */ -static void -atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) -{ - struct dma_async_tx_descriptor *txd = &desc->txd; - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); - unsigned long flags; - - dev_vdbg(chan2dev(&atchan->chan_common), - "descriptor %u complete\n", txd->cookie); - - spin_lock_irqsave(&atchan->lock, flags); - - /* mark the descriptor as complete for non cyclic cases only */ - if (!atc_chan_is_cyclic(atchan)) - dma_cookie_complete(txd); - - /* If the transfer was a memset, free our temporary buffer */ - if (desc->memset_buffer) { - dma_pool_free(atdma->memset_pool, desc->memset_vaddr, - desc->memset_paddr); - desc->memset_buffer = false; + /* For the first descriptor we can be more accurate. */ + if (desc->sg[0].lli->dscr == dscr) { + *residue = atc_calc_bytes_left(len, ctrla); + return 0; } + len -= desc->sg[0].len; - /* move children to free_list */ - list_splice_init(&desc->tx_list, &atchan->free_list); - /* move myself to free_list */ - list_move(&desc->desc_node, &atchan->free_list); - - spin_unlock_irqrestore(&atchan->lock, flags); - - dma_descriptor_unmap(txd); - /* for cyclic transfers, - * no need to replay callback function while stopping */ - if (!atc_chan_is_cyclic(atchan)) - dmaengine_desc_get_callback_invoke(txd, NULL); - - dma_run_dependencies(txd); -} - -/** - * atc_complete_all - finish work for all transactions - * @atchan: channel to complete transactions for - * - * Eventually submit queued descriptors if any - * - * Assume channel is idle while calling this function - * Called with atchan->lock held and bh disabled - */ -static void atc_complete_all(struct at_dma_chan *atchan) -{ - struct at_desc *desc, *_desc; - LIST_HEAD(list); - unsigned long flags; - - dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); - - spin_lock_irqsave(&atchan->lock, flags); + for (i = 1; i < desc->sglen; i++) { + if (desc->sg[i].lli && desc->sg[i].lli->dscr == dscr) + break; + len -= desc->sg[i].len; + } /* - * Submit queued descriptors ASAP, i.e. before we go through - * the completed ones. + * For the current LLI in the chain we can calculate the remaining bytes + * using the channel's CTRLA register. */ - if (!list_empty(&atchan->queue)) - atc_dostart(atchan, atc_first_queued(atchan)); - /* empty active_list now it is completed */ - list_splice_init(&atchan->active_list, &list); - /* empty queue list by moving descriptors (if any) to active_list */ - list_splice_init(&atchan->queue, &atchan->active_list); - - spin_unlock_irqrestore(&atchan->lock, flags); + *residue = atc_calc_bytes_left(len, ctrla); + return 0; - list_for_each_entry_safe(desc, _desc, &list, desc_node) - atc_chain_complete(atchan, desc); } /** - * atc_advance_work - at the end of a transaction, move forward - * @atchan: channel where the transaction ended + * atc_get_residue - get the number of bytes residue for a cookie. + * The residue is passed by address and updated on success. + * @chan: DMA channel + * @cookie: transaction identifier to check status of + * @residue: residue to be updated. + * + * Return: %0 on success, -errno otherwise. */ -static void atc_advance_work(struct at_dma_chan *atchan) +static int atc_get_residue(struct dma_chan *chan, dma_cookie_t cookie, + u32 *residue) { - unsigned long flags; - int ret; + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct virt_dma_desc *vd; + struct at_desc *desc = NULL; + u32 len, ctrla; - dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); + vd = vchan_find_desc(&atchan->vc, cookie); + if (vd) + desc = to_atdma_desc(&vd->tx); + else if (atchan->desc && atchan->desc->vd.tx.cookie == cookie) + desc = atchan->desc; - spin_lock_irqsave(&atchan->lock, flags); - ret = atc_chan_is_enabled(atchan); - spin_unlock_irqrestore(&atchan->lock, flags); - if (ret) - return; - - if (list_empty(&atchan->active_list) || - list_is_singular(&atchan->active_list)) - return atc_complete_all(atchan); + if (!desc) + return -EINVAL; - atc_chain_complete(atchan, atc_first_active(atchan)); + if (desc->sg[0].lli->dscr) + /* hardware linked list transfer */ + return atc_get_llis_residue(atchan, desc, residue); - /* advance work */ - spin_lock_irqsave(&atchan->lock, flags); - atc_dostart(atchan, atc_first_active(atchan)); - spin_unlock_irqrestore(&atchan->lock, flags); + /* single transfer */ + len = desc->total_len; + ctrla = channel_readl(atchan, CTRLA); + *residue = atc_calc_bytes_left(len, ctrla); + return 0; } - /** * atc_handle_error - handle errors reported by DMA controller - * @atchan: channel where error occurs + * @atchan: channel where error occurs. + * @i: channel index */ -static void atc_handle_error(struct at_dma_chan *atchan) +static void atc_handle_error(struct at_dma_chan *atchan, unsigned int i) { - struct at_desc *bad_desc; - struct at_desc *child; - unsigned long flags; + struct at_desc *desc = atchan->desc; - spin_lock_irqsave(&atchan->lock, flags); - /* - * The descriptor currently at the head of the active list is - * broked. Since we don't have any way to report errors, we'll - * just have to scream loudly and try to carry on. - */ - bad_desc = atc_first_active(atchan); - list_del_init(&bad_desc->desc_node); - - /* As we are stopped, take advantage to push queued descriptors - * in active_list */ - list_splice_init(&atchan->queue, atchan->active_list.prev); - - /* Try to restart the controller */ - if (!list_empty(&atchan->active_list)) - atc_dostart(atchan, atc_first_active(atchan)); + /* Disable channel on AHB error */ + dma_writel(atchan->atdma, CHDR, AT_DMA_RES(i) | atchan->mask); /* * KERN_CRITICAL may seem harsh, but since this only happens @@ -585,54 +790,42 @@ static void atc_handle_error(struct at_dma_chan *atchan) * controller flagged an error instead of scribbling over * random memory locations. */ - dev_crit(chan2dev(&atchan->chan_common), - "Bad descriptor submitted for DMA!\n"); - dev_crit(chan2dev(&atchan->chan_common), - " cookie: %d\n", bad_desc->txd.cookie); - atc_dump_lli(atchan, &bad_desc->lli); - list_for_each_entry(child, &bad_desc->tx_list, desc_node) - atc_dump_lli(atchan, &child->lli); - - spin_unlock_irqrestore(&atchan->lock, flags); - - /* Pretend the descriptor completed successfully */ - atc_chain_complete(atchan, bad_desc); -} - -/** - * atc_handle_cyclic - at the end of a period, run callback function - * @atchan: channel used for cyclic operations - */ -static void atc_handle_cyclic(struct at_dma_chan *atchan) -{ - struct at_desc *first = atc_first_active(atchan); - struct dma_async_tx_descriptor *txd = &first->txd; - - dev_vdbg(chan2dev(&atchan->chan_common), - "new cyclic period llp 0x%08x\n", - channel_readl(atchan, DSCR)); - - dmaengine_desc_get_callback_invoke(txd, NULL); + dev_crit(chan2dev(&atchan->vc.chan), "Bad descriptor submitted for DMA!\n"); + dev_crit(chan2dev(&atchan->vc.chan), "cookie: %d\n", + desc->vd.tx.cookie); + for (i = 0; i < desc->sglen; i++) + atc_dump_lli(atchan, desc->sg[i].lli); } -/*-- IRQ & Tasklet ---------------------------------------------------*/ - -static void atc_tasklet(struct tasklet_struct *t) +static void atdma_handle_chan_done(struct at_dma_chan *atchan, u32 pending, + unsigned int i) { - struct at_dma_chan *atchan = from_tasklet(atchan, t, tasklet); + struct at_desc *desc; - if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) - return atc_handle_error(atchan); + spin_lock(&atchan->vc.lock); + desc = atchan->desc; - if (atc_chan_is_cyclic(atchan)) - return atc_handle_cyclic(atchan); + if (desc) { + if (pending & AT_DMA_ERR(i)) { + atc_handle_error(atchan, i); + /* Pretend the descriptor completed successfully */ + } - atc_advance_work(atchan); + if (atc_chan_is_cyclic(atchan)) { + vchan_cyclic_callback(&desc->vd); + } else { + vchan_cookie_complete(&desc->vd); + atchan->desc = NULL; + if (!(atc_chan_is_enabled(atchan))) + atc_dostart(atchan); + } + } + spin_unlock(&atchan->vc.lock); } static irqreturn_t at_dma_interrupt(int irq, void *dev_id) { - struct at_dma *atdma = (struct at_dma *)dev_id; + struct at_dma *atdma = dev_id; struct at_dma_chan *atchan; int i; u32 status, pending, imr; @@ -646,23 +839,16 @@ static irqreturn_t at_dma_interrupt(int irq, void *dev_id) if (!pending) break; - dev_vdbg(atdma->dma_common.dev, + dev_vdbg(atdma->dma_device.dev, "interrupt: status = 0x%08x, 0x%08x, 0x%08x\n", status, imr, pending); - for (i = 0; i < atdma->dma_common.chancnt; i++) { + for (i = 0; i < atdma->dma_device.chancnt; i++) { atchan = &atdma->chan[i]; - if (pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i))) { - if (pending & AT_DMA_ERR(i)) { - /* Disable channel on AHB error */ - dma_writel(atdma, CHDR, - AT_DMA_RES(i) | atchan->mask); - /* Give information to tasklet */ - set_bit(ATC_IS_ERROR, &atchan->status); - } - tasklet_schedule(&atchan->tasklet); - ret = IRQ_HANDLED; - } + if (!(pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i)))) + continue; + atdma_handle_chan_done(atchan, pending, i); + ret = IRQ_HANDLED; } } while (pending); @@ -670,43 +856,7 @@ static irqreturn_t at_dma_interrupt(int irq, void *dev_id) return ret; } - /*-- DMA Engine API --------------------------------------------------*/ - -/** - * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine - * @tx: descriptor at the head of the transaction chain - * - * Queue chain if DMA engine is working already - * - * Cookie increment and adding to active_list or queue must be atomic - */ -static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct at_desc *desc = txd_to_at_desc(tx); - struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); - dma_cookie_t cookie; - unsigned long flags; - - spin_lock_irqsave(&atchan->lock, flags); - cookie = dma_cookie_assign(tx); - - if (list_empty(&atchan->active_list)) { - dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n", - desc->txd.cookie); - atc_dostart(atchan, desc); - list_add_tail(&desc->desc_node, &atchan->active_list); - } else { - dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n", - desc->txd.cookie); - list_add_tail(&desc->desc_node, &atchan->queue); - } - - spin_unlock_irqrestore(&atchan->lock, flags); - - return cookie; -} - /** * atc_prep_dma_interleaved - prepare memory to memory interleaved operation * @chan: the channel to prepare operation on @@ -718,9 +868,12 @@ atc_prep_dma_interleaved(struct dma_chan *chan, struct dma_interleaved_template *xt, unsigned long flags) { + struct at_dma *atdma = to_at_dma(chan->device); struct at_dma_chan *atchan = to_at_dma_chan(chan); struct data_chunk *first; - struct at_desc *desc = NULL; + struct atdma_sg *atdma_sg; + struct at_desc *desc; + struct at_lli *lli; size_t xfer_count; unsigned int dwidth; u32 ctrla; @@ -734,7 +887,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan, first = xt->sgl; dev_info(chan2dev(chan), - "%s: src=%pad, dest=%pad, numf=%d, frame_size=%d, flags=0x%lx\n", + "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n", __func__, &xt->src_start, &xt->dst_start, xt->numf, xt->frame_size, flags); @@ -759,8 +912,7 @@ atc_prep_dma_interleaved(struct dma_chan *chan, len += chunk->size; } - dwidth = atc_get_xfer_width(xt->src_start, - xt->dst_start, len); + dwidth = atc_get_xfer_width(xt->src_start, xt->dst_start, len); xfer_count = len >> dwidth; if (xfer_count > ATC_BTSIZE_MAX) { @@ -768,42 +920,43 @@ atc_prep_dma_interleaved(struct dma_chan *chan, return NULL; } - ctrla = ATC_SRC_WIDTH(dwidth) | - ATC_DST_WIDTH(dwidth); - - ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN - | ATC_SRC_ADDR_MODE_INCR - | ATC_DST_ADDR_MODE_INCR - | ATC_SRC_PIP - | ATC_DST_PIP - | ATC_FC_MEM2MEM; - - /* create the transfer */ - desc = atc_desc_get(atchan); - if (!desc) { - dev_err(chan2dev(chan), - "%s: couldn't allocate our descriptor\n", __func__); + ctrla = FIELD_PREP(ATC_SRC_WIDTH, dwidth) | + FIELD_PREP(ATC_DST_WIDTH, dwidth); + + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_INCR) | + FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | + ATC_SRC_PIP | ATC_DST_PIP | + FIELD_PREP(ATC_FC, ATC_FC_MEM2MEM); + + desc = kzalloc(struct_size(desc, sg, 1), GFP_ATOMIC); + if (!desc) + return NULL; + desc->sglen = 1; + + atdma_sg = desc->sg; + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_NOWAIT, + &atdma_sg->lli_phys); + if (!atdma_sg->lli) { + kfree(desc); return NULL; } + lli = atdma_sg->lli; - desc->lli.saddr = xt->src_start; - desc->lli.daddr = xt->dst_start; - desc->lli.ctrla = ctrla | xfer_count; - desc->lli.ctrlb = ctrlb; + lli->saddr = xt->src_start; + lli->daddr = xt->dst_start; + lli->ctrla = ctrla | xfer_count; + lli->ctrlb = ctrlb; desc->boundary = first->size >> dwidth; desc->dst_hole = (dmaengine_get_dst_icg(xt, first) >> dwidth) + 1; desc->src_hole = (dmaengine_get_src_icg(xt, first) >> dwidth) + 1; - desc->txd.cookie = -EBUSY; - desc->total_len = desc->len = len; - - /* set end-of-link to the last link descriptor of list*/ - set_desc_eol(desc); - - desc->txd.flags = flags; /* client is in control of this ack */ + atdma_sg->len = len; + desc->total_len = len; - return &desc->txd; + set_lli_eol(desc, 0); + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); } /** @@ -818,29 +971,36 @@ static struct dma_async_tx_descriptor * atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) { + struct at_dma *atdma = to_at_dma(chan->device); struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_desc *desc = NULL; - struct at_desc *first = NULL; - struct at_desc *prev = NULL; size_t xfer_count; size_t offset; + size_t sg_len; unsigned int src_width; unsigned int dst_width; + unsigned int i; u32 ctrla; u32 ctrlb; - dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n", - &dest, &src, len, flags); + dev_dbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n", + &dest, &src, len, flags); if (unlikely(!len)) { - dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); + dev_err(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); return NULL; } - ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN - | ATC_SRC_ADDR_MODE_INCR - | ATC_DST_ADDR_MODE_INCR - | ATC_FC_MEM2MEM; + sg_len = DIV_ROUND_UP(len, ATC_BTSIZE_MAX); + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC); + if (!desc) + return NULL; + desc->sglen = sg_len; + + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_INCR) | + FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | + FIELD_PREP(ATC_FC, ATC_FC_MEM2MEM); /* * We can be a lot more clever here, but this should take care @@ -848,82 +1008,78 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, */ src_width = dst_width = atc_get_xfer_width(src, dest, len); - ctrla = ATC_SRC_WIDTH(src_width) | - ATC_DST_WIDTH(dst_width); + ctrla = FIELD_PREP(ATC_SRC_WIDTH, src_width) | + FIELD_PREP(ATC_DST_WIDTH, dst_width); - for (offset = 0; offset < len; offset += xfer_count << src_width) { - xfer_count = min_t(size_t, (len - offset) >> src_width, - ATC_BTSIZE_MAX); + for (offset = 0, i = 0; offset < len; + offset += xfer_count << src_width, i++) { + struct atdma_sg *atdma_sg = &desc->sg[i]; + struct at_lli *lli; - desc = atc_desc_get(atchan); - if (!desc) + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_NOWAIT, + &atdma_sg->lli_phys); + if (!atdma_sg->lli) goto err_desc_get; + lli = atdma_sg->lli; - desc->lli.saddr = src + offset; - desc->lli.daddr = dest + offset; - desc->lli.ctrla = ctrla | xfer_count; - desc->lli.ctrlb = ctrlb; + xfer_count = min_t(size_t, (len - offset) >> src_width, + ATC_BTSIZE_MAX); + + lli->saddr = src + offset; + lli->daddr = dest + offset; + lli->ctrla = ctrla | xfer_count; + lli->ctrlb = ctrlb; - desc->txd.cookie = 0; - desc->len = xfer_count << src_width; + desc->sg[i].len = xfer_count << src_width; - atc_desc_chain(&first, &prev, desc); + atdma_lli_chain(desc, i); } - /* First descriptor of the chain embedds additional information */ - first->txd.cookie = -EBUSY; - first->total_len = len; + desc->total_len = len; /* set end-of-link to the last link descriptor of list*/ - set_desc_eol(desc); + set_lli_eol(desc, i - 1); - first->txd.flags = flags; /* client is in control of this ack */ - - return &first->txd; + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); err_desc_get: - atc_desc_put(atchan, first); + atdma_desc_free(&desc->vd); return NULL; } -static struct at_desc *atc_create_memset_desc(struct dma_chan *chan, - dma_addr_t psrc, - dma_addr_t pdst, - size_t len) +static int atdma_create_memset_lli(struct dma_chan *chan, + struct atdma_sg *atdma_sg, + dma_addr_t psrc, dma_addr_t pdst, size_t len) { - struct at_dma_chan *atchan = to_at_dma_chan(chan); - struct at_desc *desc; + struct at_dma *atdma = to_at_dma(chan->device); + struct at_lli *lli; size_t xfer_count; - - u32 ctrla = ATC_SRC_WIDTH(2) | ATC_DST_WIDTH(2); + u32 ctrla = FIELD_PREP(ATC_SRC_WIDTH, 2) | FIELD_PREP(ATC_DST_WIDTH, 2); u32 ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | - ATC_SRC_ADDR_MODE_FIXED | - ATC_DST_ADDR_MODE_INCR | - ATC_FC_MEM2MEM; + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_FIXED) | + FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | + FIELD_PREP(ATC_FC, ATC_FC_MEM2MEM); xfer_count = len >> 2; if (xfer_count > ATC_BTSIZE_MAX) { - dev_err(chan2dev(chan), "%s: buffer is too big\n", - __func__); - return NULL; + dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__); + return -EINVAL; } - desc = atc_desc_get(atchan); - if (!desc) { - dev_err(chan2dev(chan), "%s: can't get a descriptor\n", - __func__); - return NULL; - } + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_NOWAIT, + &atdma_sg->lli_phys); + if (!atdma_sg->lli) + return -ENOMEM; + lli = atdma_sg->lli; - desc->lli.saddr = psrc; - desc->lli.daddr = pdst; - desc->lli.ctrla = ctrla | xfer_count; - desc->lli.ctrlb = ctrlb; + lli->saddr = psrc; + lli->daddr = pdst; + lli->ctrla = ctrla | xfer_count; + lli->ctrlb = ctrlb; - desc->txd.cookie = 0; - desc->len = len; + atdma_sg->len = len; - return desc; + return 0; } /** @@ -938,10 +1094,13 @@ static struct dma_async_tx_descriptor * atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, size_t len, unsigned long flags) { + struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc; void __iomem *vaddr; dma_addr_t paddr; + char fill_pattern; + int ret; dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__, &dest, value, len, flags); @@ -963,29 +1122,37 @@ atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, __func__); return NULL; } - *(u32*)vaddr = value; - desc = atc_create_memset_desc(chan, paddr, dest, len); - if (!desc) { - dev_err(chan2dev(chan), "%s: couldn't get a descriptor\n", - __func__); + /* Only the first byte of value is to be used according to dmaengine */ + fill_pattern = (char)value; + + *(u32*)vaddr = (fill_pattern << 24) | + (fill_pattern << 16) | + (fill_pattern << 8) | + fill_pattern; + + desc = kzalloc(struct_size(desc, sg, 1), GFP_ATOMIC); + if (!desc) goto err_free_buffer; - } + desc->sglen = 1; + + ret = atdma_create_memset_lli(chan, desc->sg, paddr, dest, len); + if (ret) + goto err_free_desc; desc->memset_paddr = paddr; desc->memset_vaddr = vaddr; desc->memset_buffer = true; - desc->txd.cookie = -EBUSY; desc->total_len = len; /* set end-of-link on the descriptor */ - set_desc_eol(desc); - - desc->txd.flags = flags; + set_lli_eol(desc, 0); - return &desc->txd; + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); +err_free_desc: + kfree(desc); err_free_buffer: dma_pool_free(atdma->memset_pool, vaddr, paddr); return NULL; @@ -999,14 +1166,15 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); - struct at_desc *desc = NULL, *first = NULL, *prev = NULL; + struct at_desc *desc; struct scatterlist *sg; void __iomem *vaddr; dma_addr_t paddr; size_t total_len = 0; int i; + int ret; - dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%zx f0x%lx\n", __func__, + dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%x f0x%lx\n", __func__, value, sg_len, flags); if (unlikely(!sgl || !sg_len)) { @@ -1023,6 +1191,11 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, } *(u32*)vaddr = value; + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC); + if (!desc) + goto err_free_dma_buf; + desc->sglen = sg_len; + for_each_sg(sgl, sg, sg_len, i) { dma_addr_t dest = sg_dma_address(sg); size_t len = sg_dma_len(sg); @@ -1033,38 +1206,33 @@ atc_prep_dma_memset_sg(struct dma_chan *chan, if (!is_dma_fill_aligned(chan->device, dest, 0, len)) { dev_err(chan2dev(chan), "%s: buffer is not aligned\n", __func__); - goto err_put_desc; + goto err_free_desc; } - desc = atc_create_memset_desc(chan, paddr, dest, len); - if (!desc) - goto err_put_desc; - - atc_desc_chain(&first, &prev, desc); + ret = atdma_create_memset_lli(chan, &desc->sg[i], paddr, dest, + len); + if (ret) + goto err_free_desc; + atdma_lli_chain(desc, i); total_len += len; } - /* - * Only set the buffer pointers on the last descriptor to - * avoid free'ing while we have our transfer still going - */ desc->memset_paddr = paddr; desc->memset_vaddr = vaddr; desc->memset_buffer = true; - first->txd.cookie = -EBUSY; - first->total_len = total_len; + desc->total_len = total_len; /* set end-of-link on the descriptor */ - set_desc_eol(desc); - - first->txd.flags = flags; + set_lli_eol(desc, i - 1); - return &first->txd; + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); -err_put_desc: - atc_desc_put(atchan, first); +err_free_desc: + atdma_desc_free(&desc->vd); +err_free_dma_buf: + dma_pool_free(atdma->memset_pool, vaddr, paddr); return NULL; } @@ -1082,11 +1250,11 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context) { + struct at_dma *atdma = to_at_dma(chan->device); struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma_slave *atslave = chan->private; struct dma_slave_config *sconfig = &atchan->dma_sconfig; - struct at_desc *first = NULL; - struct at_desc *prev = NULL; + struct at_desc *desc; u32 ctrla; u32 ctrlb; dma_addr_t reg; @@ -1106,27 +1274,38 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return NULL; } - ctrla = ATC_SCSIZE(sconfig->src_maxburst) - | ATC_DCSIZE(sconfig->dst_maxburst); + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_ATOMIC); + if (!desc) + return NULL; + desc->sglen = sg_len; + + ctrla = FIELD_PREP(ATC_SCSIZE, sconfig->src_maxburst) | + FIELD_PREP(ATC_DCSIZE, sconfig->dst_maxburst); ctrlb = ATC_IEN; switch (direction) { case DMA_MEM_TO_DEV: reg_width = convert_buswidth(sconfig->dst_addr_width); - ctrla |= ATC_DST_WIDTH(reg_width); - ctrlb |= ATC_DST_ADDR_MODE_FIXED - | ATC_SRC_ADDR_MODE_INCR - | ATC_FC_MEM2PER - | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); + ctrla |= FIELD_PREP(ATC_DST_WIDTH, reg_width); + ctrlb |= FIELD_PREP(ATC_DST_ADDR_MODE, + ATC_DST_ADDR_MODE_FIXED) | + FIELD_PREP(ATC_SRC_ADDR_MODE, ATC_SRC_ADDR_MODE_INCR) | + FIELD_PREP(ATC_FC, ATC_FC_MEM2PER) | + FIELD_PREP(ATC_SIF, atchan->mem_if) | + FIELD_PREP(ATC_DIF, atchan->per_if); reg = sconfig->dst_addr; for_each_sg(sgl, sg, sg_len, i) { - struct at_desc *desc; + struct atdma_sg *atdma_sg = &desc->sg[i]; + struct at_lli *lli; u32 len; u32 mem; - desc = atc_desc_get(atchan); - if (!desc) + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, + GFP_NOWAIT, + &atdma_sg->lli_phys); + if (!atdma_sg->lli) goto err_desc_get; + lli = atdma_sg->lli; mem = sg_dma_address(sg); len = sg_dma_len(sg); @@ -1139,35 +1318,43 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (unlikely(mem & 3 || len & 3)) mem_width = 0; - desc->lli.saddr = mem; - desc->lli.daddr = reg; - desc->lli.ctrla = ctrla - | ATC_SRC_WIDTH(mem_width) - | len >> mem_width; - desc->lli.ctrlb = ctrlb; - desc->len = len; + lli->saddr = mem; + lli->daddr = reg; + lli->ctrla = ctrla | + FIELD_PREP(ATC_SRC_WIDTH, mem_width) | + len >> mem_width; + lli->ctrlb = ctrlb; - atc_desc_chain(&first, &prev, desc); + atdma_sg->len = len; total_len += len; + + desc->sg[i].len = len; + atdma_lli_chain(desc, i); } break; case DMA_DEV_TO_MEM: reg_width = convert_buswidth(sconfig->src_addr_width); - ctrla |= ATC_SRC_WIDTH(reg_width); - ctrlb |= ATC_DST_ADDR_MODE_INCR - | ATC_SRC_ADDR_MODE_FIXED - | ATC_FC_PER2MEM - | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); + ctrla |= FIELD_PREP(ATC_SRC_WIDTH, reg_width); + ctrlb |= FIELD_PREP(ATC_DST_ADDR_MODE, ATC_DST_ADDR_MODE_INCR) | + FIELD_PREP(ATC_SRC_ADDR_MODE, + ATC_SRC_ADDR_MODE_FIXED) | + FIELD_PREP(ATC_FC, ATC_FC_PER2MEM) | + FIELD_PREP(ATC_SIF, atchan->per_if) | + FIELD_PREP(ATC_DIF, atchan->mem_if); reg = sconfig->src_addr; for_each_sg(sgl, sg, sg_len, i) { - struct at_desc *desc; + struct atdma_sg *atdma_sg = &desc->sg[i]; + struct at_lli *lli; u32 len; u32 mem; - desc = atc_desc_get(atchan); - if (!desc) + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, + GFP_NOWAIT, + &atdma_sg->lli_phys); + if (!atdma_sg->lli) goto err_desc_get; + lli = atdma_sg->lli; mem = sg_dma_address(sg); len = sg_dma_len(sg); @@ -1180,16 +1367,17 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (unlikely(mem & 3 || len & 3)) mem_width = 0; - desc->lli.saddr = reg; - desc->lli.daddr = mem; - desc->lli.ctrla = ctrla - | ATC_DST_WIDTH(mem_width) - | len >> reg_width; - desc->lli.ctrlb = ctrlb; - desc->len = len; + lli->saddr = reg; + lli->daddr = mem; + lli->ctrla = ctrla | + FIELD_PREP(ATC_DST_WIDTH, mem_width) | + len >> reg_width; + lli->ctrlb = ctrlb; - atc_desc_chain(&first, &prev, desc); + desc->sg[i].len = len; total_len += len; + + atdma_lli_chain(desc, i); } break; default: @@ -1197,21 +1385,16 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, } /* set end-of-link to the last link descriptor of list*/ - set_desc_eol(prev); + set_lli_eol(desc, i - 1); - /* First descriptor of the chain embedds additional information */ - first->txd.cookie = -EBUSY; - first->total_len = total_len; + desc->total_len = total_len; - /* first link descriptor of list is responsible of flags */ - first->txd.flags = flags; /* client is in control of this ack */ - - return &first->txd; + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); err_desc_get: dev_err(chan2dev(chan), "not enough descriptors available\n"); err: - atc_desc_put(atchan, first); + atdma_desc_free(&desc->vd); return NULL; } @@ -1241,50 +1424,59 @@ err_out: */ static int atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc, - unsigned int period_index, dma_addr_t buf_addr, + unsigned int i, dma_addr_t buf_addr, unsigned int reg_width, size_t period_len, enum dma_transfer_direction direction) { + struct at_dma *atdma = to_at_dma(chan->device); struct at_dma_chan *atchan = to_at_dma_chan(chan); struct dma_slave_config *sconfig = &atchan->dma_sconfig; - u32 ctrla; + struct atdma_sg *atdma_sg = &desc->sg[i]; + struct at_lli *lli; - /* prepare common CRTLA value */ - ctrla = ATC_SCSIZE(sconfig->src_maxburst) - | ATC_DCSIZE(sconfig->dst_maxburst) - | ATC_DST_WIDTH(reg_width) - | ATC_SRC_WIDTH(reg_width) - | period_len >> reg_width; + atdma_sg->lli = dma_pool_alloc(atdma->lli_pool, GFP_ATOMIC, + &atdma_sg->lli_phys); + if (!atdma_sg->lli) + return -ENOMEM; + lli = atdma_sg->lli; switch (direction) { case DMA_MEM_TO_DEV: - desc->lli.saddr = buf_addr + (period_len * period_index); - desc->lli.daddr = sconfig->dst_addr; - desc->lli.ctrla = ctrla; - desc->lli.ctrlb = ATC_DST_ADDR_MODE_FIXED - | ATC_SRC_ADDR_MODE_INCR - | ATC_FC_MEM2PER - | ATC_SIF(atchan->mem_if) - | ATC_DIF(atchan->per_if); - desc->len = period_len; + lli->saddr = buf_addr + (period_len * i); + lli->daddr = sconfig->dst_addr; + lli->ctrlb = FIELD_PREP(ATC_DST_ADDR_MODE, + ATC_DST_ADDR_MODE_FIXED) | + FIELD_PREP(ATC_SRC_ADDR_MODE, + ATC_SRC_ADDR_MODE_INCR) | + FIELD_PREP(ATC_FC, ATC_FC_MEM2PER) | + FIELD_PREP(ATC_SIF, atchan->mem_if) | + FIELD_PREP(ATC_DIF, atchan->per_if); + break; case DMA_DEV_TO_MEM: - desc->lli.saddr = sconfig->src_addr; - desc->lli.daddr = buf_addr + (period_len * period_index); - desc->lli.ctrla = ctrla; - desc->lli.ctrlb = ATC_DST_ADDR_MODE_INCR - | ATC_SRC_ADDR_MODE_FIXED - | ATC_FC_PER2MEM - | ATC_SIF(atchan->per_if) - | ATC_DIF(atchan->mem_if); - desc->len = period_len; + lli->saddr = sconfig->src_addr; + lli->daddr = buf_addr + (period_len * i); + lli->ctrlb = FIELD_PREP(ATC_DST_ADDR_MODE, + ATC_DST_ADDR_MODE_INCR) | + FIELD_PREP(ATC_SRC_ADDR_MODE, + ATC_SRC_ADDR_MODE_FIXED) | + FIELD_PREP(ATC_FC, ATC_FC_PER2MEM) | + FIELD_PREP(ATC_SIF, atchan->per_if) | + FIELD_PREP(ATC_DIF, atchan->mem_if); break; default: return -EINVAL; } + lli->ctrla = FIELD_PREP(ATC_SCSIZE, sconfig->src_maxburst) | + FIELD_PREP(ATC_DCSIZE, sconfig->dst_maxburst) | + FIELD_PREP(ATC_DST_WIDTH, reg_width) | + FIELD_PREP(ATC_SRC_WIDTH, reg_width) | + period_len >> reg_width; + desc->sg[i].len = period_len; + return 0; } @@ -1305,14 +1497,13 @@ atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma_slave *atslave = chan->private; struct dma_slave_config *sconfig = &atchan->dma_sconfig; - struct at_desc *first = NULL; - struct at_desc *prev = NULL; + struct at_desc *desc; unsigned long was_cyclic; unsigned int reg_width; unsigned int periods = buf_len / period_len; unsigned int i; - dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%d/%d)\n", + dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%zu/%zu)\n", direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE", &buf_addr, periods, buf_len, period_len); @@ -1340,33 +1531,26 @@ atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, if (atc_dma_cyclic_check_values(reg_width, buf_addr, period_len)) goto err_out; + desc = kzalloc(struct_size(desc, sg, periods), GFP_ATOMIC); + if (!desc) + goto err_out; + desc->sglen = periods; + /* build cyclic linked list */ for (i = 0; i < periods; i++) { - struct at_desc *desc; - - desc = atc_desc_get(atchan); - if (!desc) - goto err_desc_get; - if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr, reg_width, period_len, direction)) - goto err_desc_get; - - atc_desc_chain(&first, &prev, desc); + goto err_fill_desc; + atdma_lli_chain(desc, i); } - + desc->total_len = buf_len; /* lets make a cyclic list */ - prev->lli.dscr = first->txd.phys; - - /* First descriptor of the chain embedds additional information */ - first->txd.cookie = -EBUSY; - first->total_len = buf_len; + desc->sg[i - 1].lli->dscr = desc->sg[0].lli_phys; - return &first->txd; + return vchan_tx_prep(&atchan->vc, &desc->vd, flags); -err_desc_get: - dev_err(chan2dev(chan), "not enough descriptors available\n"); - atc_desc_put(atchan, first); +err_fill_desc: + atdma_desc_free(&desc->vd); err_out: clear_bit(ATC_IS_CYCLIC, &atchan->status); return NULL; @@ -1395,17 +1579,17 @@ static int atc_pause(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); - int chan_id = atchan->chan_common.chan_id; + int chan_id = atchan->vc.chan.chan_id; unsigned long flags; dev_vdbg(chan2dev(chan), "%s\n", __func__); - spin_lock_irqsave(&atchan->lock, flags); + spin_lock_irqsave(&atchan->vc.lock, flags); dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id)); set_bit(ATC_IS_PAUSED, &atchan->status); - spin_unlock_irqrestore(&atchan->lock, flags); + spin_unlock_irqrestore(&atchan->vc.lock, flags); return 0; } @@ -1414,7 +1598,7 @@ static int atc_resume(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); - int chan_id = atchan->chan_common.chan_id; + int chan_id = atchan->vc.chan.chan_id; unsigned long flags; dev_vdbg(chan2dev(chan), "%s\n", __func__); @@ -1422,12 +1606,12 @@ static int atc_resume(struct dma_chan *chan) if (!atc_chan_is_paused(atchan)) return 0; - spin_lock_irqsave(&atchan->lock, flags); + spin_lock_irqsave(&atchan->vc.lock, flags); dma_writel(atdma, CHDR, AT_DMA_RES(chan_id)); clear_bit(ATC_IS_PAUSED, &atchan->status); - spin_unlock_irqrestore(&atchan->lock, flags); + spin_unlock_irqrestore(&atchan->vc.lock, flags); return 0; } @@ -1436,8 +1620,7 @@ static int atc_terminate_all(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); - int chan_id = atchan->chan_common.chan_id; - struct at_desc *desc, *_desc; + int chan_id = atchan->vc.chan.chan_id; unsigned long flags; LIST_HEAD(list); @@ -1450,7 +1633,7 @@ static int atc_terminate_all(struct dma_chan *chan) * channel. We still have to poll the channel enable bit due * to AHB/HSB limitations. */ - spin_lock_irqsave(&atchan->lock, flags); + spin_lock_irqsave(&atchan->vc.lock, flags); /* disabling channel: must also remove suspend state */ dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); @@ -1459,20 +1642,21 @@ static int atc_terminate_all(struct dma_chan *chan) while (dma_readl(atdma, CHSR) & atchan->mask) cpu_relax(); - /* active_list entries will end up before queued entries */ - list_splice_init(&atchan->queue, &list); - list_splice_init(&atchan->active_list, &list); - - spin_unlock_irqrestore(&atchan->lock, flags); + if (atchan->desc) { + vchan_terminate_vdesc(&atchan->desc->vd); + atchan->desc = NULL; + } - /* Flush all pending and queued descriptors */ - list_for_each_entry_safe(desc, _desc, &list, desc_node) - atc_chain_complete(atchan, desc); + vchan_get_all_descriptors(&atchan->vc, &list); clear_bit(ATC_IS_PAUSED, &atchan->status); /* if channel dedicated to cyclic operations, free it */ clear_bit(ATC_IS_CYCLIC, &atchan->status); + spin_unlock_irqrestore(&atchan->vc.lock, flags); + + vchan_dma_desc_free_list(&atchan->vc, &list); + return 0; } @@ -1493,69 +1677,56 @@ atc_tx_status(struct dma_chan *chan, { struct at_dma_chan *atchan = to_at_dma_chan(chan); unsigned long flags; - enum dma_status ret; - int bytes = 0; - - ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; - /* - * There's no point calculating the residue if there's - * no txstate to store the value. - */ - if (!txstate) - return DMA_ERROR; + enum dma_status dma_status; + u32 residue; + int ret; - spin_lock_irqsave(&atchan->lock, flags); + dma_status = dma_cookie_status(chan, cookie, txstate); + if (dma_status == DMA_COMPLETE || !txstate) + return dma_status; + spin_lock_irqsave(&atchan->vc.lock, flags); /* Get number of bytes left in the active transactions */ - bytes = atc_get_bytes_left(chan, cookie); + ret = atc_get_residue(chan, cookie, &residue); + spin_unlock_irqrestore(&atchan->vc.lock, flags); - spin_unlock_irqrestore(&atchan->lock, flags); - - if (unlikely(bytes < 0)) { + if (unlikely(ret < 0)) { dev_vdbg(chan2dev(chan), "get residual bytes error\n"); return DMA_ERROR; } else { - dma_set_residue(txstate, bytes); + dma_set_residue(txstate, residue); } - dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n", - ret, cookie, bytes); + dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %u\n", + dma_status, cookie, residue); - return ret; + return dma_status; } -/** - * atc_issue_pending - try to finish work - * @chan: target DMA channel - */ static void atc_issue_pending(struct dma_chan *chan) { - struct at_dma_chan *atchan = to_at_dma_chan(chan); - - dev_vdbg(chan2dev(chan), "issue_pending\n"); - - /* Not needed for cyclic transfers */ - if (atc_chan_is_cyclic(atchan)) - return; + struct at_dma_chan *atchan = to_at_dma_chan(chan); + unsigned long flags; - atc_advance_work(atchan); + spin_lock_irqsave(&atchan->vc.lock, flags); + if (vchan_issue_pending(&atchan->vc) && !atchan->desc) { + if (!(atc_chan_is_enabled(atchan))) + atc_dostart(atchan); + } + spin_unlock_irqrestore(&atchan->vc.lock, flags); } /** * atc_alloc_chan_resources - allocate resources for DMA channel * @chan: allocate descriptor resources for this channel * - * return - the number of allocated descriptors + * Return: the number of allocated descriptors */ static int atc_alloc_chan_resources(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); - struct at_desc *desc; struct at_dma_slave *atslave; - int i; u32 cfg; dev_vdbg(chan2dev(chan), "alloc_chan_resources\n"); @@ -1566,11 +1737,6 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) return -EIO; } - if (!list_empty(&atchan->free_list)) { - dev_dbg(chan2dev(chan), "can't allocate channel resources (channel not freed from a previous use)\n"); - return -EIO; - } - cfg = ATC_DEFAULT_CFG; atslave = chan->private; @@ -1579,33 +1745,17 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) * We need controller-specific data to set up slave * transfers. */ - BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev); + BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_device.dev); /* if cfg configuration specified take it instead of default */ if (atslave->cfg) cfg = atslave->cfg; } - /* Allocate initial pool of descriptors */ - for (i = 0; i < init_nr_desc_per_channel; i++) { - desc = atc_alloc_descriptor(chan, GFP_KERNEL); - if (!desc) { - dev_err(atdma->dma_common.dev, - "Only %d initial descriptors\n", i); - break; - } - list_add_tail(&desc->desc_node, &atchan->free_list); - } - - dma_cookie_init(chan); - /* channel parameters */ channel_writel(atchan, CFG, cfg); - dev_dbg(chan2dev(chan), - "alloc_chan_resources: allocated %d descriptors\n", i); - - return i; + return 0; } /** @@ -1615,22 +1765,10 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) static void atc_free_chan_resources(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); - struct at_dma *atdma = to_at_dma(chan->device); - struct at_desc *desc, *_desc; - LIST_HEAD(list); - /* ASSERT: channel is idle */ - BUG_ON(!list_empty(&atchan->active_list)); - BUG_ON(!list_empty(&atchan->queue)); BUG_ON(atc_chan_is_enabled(atchan)); - list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { - dev_vdbg(chan2dev(chan), " freeing descriptor %p\n", desc); - list_del(&desc->desc_node); - /* free link descriptor */ - dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys); - } - list_splice_init(&atchan->free_list, &list); + vchan_free_chan_resources(to_virt_chan(chan)); atchan->status = 0; /* @@ -1681,14 +1819,13 @@ static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec, return NULL; } - atslave->cfg = ATC_DST_H2SEL_HW | ATC_SRC_H2SEL_HW; + atslave->cfg = ATC_DST_H2SEL | ATC_SRC_H2SEL; /* * We can fill both SRC_PER and DST_PER, one of these fields will be * ignored depending on DMA transfer direction. */ per_id = dma_spec->args[1] & AT91_DMA_CFG_PER_ID_MASK; - atslave->cfg |= ATC_DST_PER_MSB(per_id) | ATC_DST_PER(per_id) - | ATC_SRC_PER_MSB(per_id) | ATC_SRC_PER(per_id); + atslave->cfg |= ATC_DST_PER_ID(per_id) | ATC_SRC_PER_ID(per_id); /* * We have to translate the value we get from the device tree since * the half FIFO configuration value had to be 0 to keep backward @@ -1696,14 +1833,16 @@ static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec, */ switch (dma_spec->args[1] & AT91_DMA_CFG_FIFOCFG_MASK) { case AT91_DMA_CFG_FIFOCFG_ALAP: - atslave->cfg |= ATC_FIFOCFG_LARGESTBURST; + atslave->cfg |= FIELD_PREP(ATC_FIFOCFG, + ATC_FIFOCFG_LARGESTBURST); break; case AT91_DMA_CFG_FIFOCFG_ASAP: - atslave->cfg |= ATC_FIFOCFG_ENOUGHSPACE; + atslave->cfg |= FIELD_PREP(ATC_FIFOCFG, + ATC_FIFOCFG_ENOUGHSPACE); break; case AT91_DMA_CFG_FIFOCFG_HALF: default: - atslave->cfg |= ATC_FIFOCFG_HALFFIFO; + atslave->cfg |= FIELD_PREP(ATC_FIFOCFG, ATC_FIFOCFG_HALFFIFO); } atslave->dma_dev = &dmac_pdev->dev; @@ -1798,9 +1937,7 @@ static void at_dma_off(struct at_dma *atdma) static int __init at_dma_probe(struct platform_device *pdev) { - struct resource *io; struct at_dma *atdma; - size_t size; int irq; int err; int i; @@ -1820,44 +1957,31 @@ static int __init at_dma_probe(struct platform_device *pdev) if (!plat_dat) return -ENODEV; - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!io) - return -EINVAL; + atdma = devm_kzalloc(&pdev->dev, + struct_size(atdma, chan, plat_dat->nr_channels), + GFP_KERNEL); + if (!atdma) + return -ENOMEM; + + atdma->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(atdma->regs)) + return PTR_ERR(atdma->regs); irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; - size = sizeof(struct at_dma); - size += plat_dat->nr_channels * sizeof(struct at_dma_chan); - atdma = kzalloc(size, GFP_KERNEL); - if (!atdma) - return -ENOMEM; - /* discover transaction capabilities */ - atdma->dma_common.cap_mask = plat_dat->cap_mask; + atdma->dma_device.cap_mask = plat_dat->cap_mask; atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1; - size = resource_size(io); - if (!request_mem_region(io->start, size, pdev->dev.driver->name)) { - err = -EBUSY; - goto err_kfree; - } + atdma->clk = devm_clk_get(&pdev->dev, "dma_clk"); + if (IS_ERR(atdma->clk)) + return PTR_ERR(atdma->clk); - atdma->regs = ioremap(io->start, size); - if (!atdma->regs) { - err = -ENOMEM; - goto err_release_r; - } - - atdma->clk = clk_get(&pdev->dev, "dma_clk"); - if (IS_ERR(atdma->clk)) { - err = PTR_ERR(atdma->clk); - goto err_clk; - } err = clk_prepare_enable(atdma->clk); if (err) - goto err_clk_prepare; + return err; /* force dma off, just in case */ at_dma_off(atdma); @@ -1869,11 +1993,11 @@ static int __init at_dma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, atdma); /* create a pool of consistent memory blocks for hardware descriptors */ - atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool", - &pdev->dev, sizeof(struct at_desc), - 4 /* word alignment */, 0); - if (!atdma->dma_desc_pool) { - dev_err(&pdev->dev, "No memory for descriptors dma pool\n"); + atdma->lli_pool = dma_pool_create("at_hdmac_lli_pool", + &pdev->dev, sizeof(struct at_lli), + 4 /* word alignment */, 0); + if (!atdma->lli_pool) { + dev_err(&pdev->dev, "Unable to allocate DMA LLI descriptor pool\n"); err = -ENOMEM; goto err_desc_pool_create; } @@ -1892,73 +2016,70 @@ static int __init at_dma_probe(struct platform_device *pdev) cpu_relax(); /* initialize channels related values */ - INIT_LIST_HEAD(&atdma->dma_common.channels); + INIT_LIST_HEAD(&atdma->dma_device.channels); for (i = 0; i < plat_dat->nr_channels; i++) { struct at_dma_chan *atchan = &atdma->chan[i]; atchan->mem_if = AT_DMA_MEM_IF; atchan->per_if = AT_DMA_PER_IF; - atchan->chan_common.device = &atdma->dma_common; - dma_cookie_init(&atchan->chan_common); - list_add_tail(&atchan->chan_common.device_node, - &atdma->dma_common.channels); atchan->ch_regs = atdma->regs + ch_regs(i); - spin_lock_init(&atchan->lock); atchan->mask = 1 << i; - INIT_LIST_HEAD(&atchan->active_list); - INIT_LIST_HEAD(&atchan->queue); - INIT_LIST_HEAD(&atchan->free_list); - - tasklet_setup(&atchan->tasklet, atc_tasklet); + atchan->atdma = atdma; + atchan->vc.desc_free = atdma_desc_free; + vchan_init(&atchan->vc, &atdma->dma_device); atc_enable_chan_irq(atdma, i); } /* set base routines */ - atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources; - atdma->dma_common.device_free_chan_resources = atc_free_chan_resources; - atdma->dma_common.device_tx_status = atc_tx_status; - atdma->dma_common.device_issue_pending = atc_issue_pending; - atdma->dma_common.dev = &pdev->dev; + atdma->dma_device.device_alloc_chan_resources = atc_alloc_chan_resources; + atdma->dma_device.device_free_chan_resources = atc_free_chan_resources; + atdma->dma_device.device_tx_status = atc_tx_status; + atdma->dma_device.device_issue_pending = atc_issue_pending; + atdma->dma_device.dev = &pdev->dev; /* set prep routines based on capability */ - if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask)) - atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved; + if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_device.cap_mask)) + atdma->dma_device.device_prep_interleaved_dma = atc_prep_dma_interleaved; - if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask)) - atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy; + if (dma_has_cap(DMA_MEMCPY, atdma->dma_device.cap_mask)) + atdma->dma_device.device_prep_dma_memcpy = atc_prep_dma_memcpy; - if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) { - atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset; - atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg; - atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES; + if (dma_has_cap(DMA_MEMSET, atdma->dma_device.cap_mask)) { + atdma->dma_device.device_prep_dma_memset = atc_prep_dma_memset; + atdma->dma_device.device_prep_dma_memset_sg = atc_prep_dma_memset_sg; + atdma->dma_device.fill_align = DMAENGINE_ALIGN_4_BYTES; } - if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { - atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; + if (dma_has_cap(DMA_SLAVE, atdma->dma_device.cap_mask)) { + atdma->dma_device.device_prep_slave_sg = atc_prep_slave_sg; /* controller can do slave DMA: can trigger cyclic transfers */ - dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask); - atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic; - atdma->dma_common.device_config = atc_config; - atdma->dma_common.device_pause = atc_pause; - atdma->dma_common.device_resume = atc_resume; - atdma->dma_common.device_terminate_all = atc_terminate_all; - atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS; - atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS; - atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dma_cap_set(DMA_CYCLIC, atdma->dma_device.cap_mask); + atdma->dma_device.device_prep_dma_cyclic = atc_prep_dma_cyclic; + atdma->dma_device.device_config = atc_config; + atdma->dma_device.device_pause = atc_pause; + atdma->dma_device.device_resume = atc_resume; + atdma->dma_device.device_terminate_all = atc_terminate_all; + atdma->dma_device.src_addr_widths = ATC_DMA_BUSWIDTHS; + atdma->dma_device.dst_addr_widths = ATC_DMA_BUSWIDTHS; + atdma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + atdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; } dma_writel(atdma, EN, AT_DMA_ENABLE); dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s), %d channels\n", - dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", - dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "", - dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", + dma_has_cap(DMA_MEMCPY, atdma->dma_device.cap_mask) ? "cpy " : "", + dma_has_cap(DMA_MEMSET, atdma->dma_device.cap_mask) ? "set " : "", + dma_has_cap(DMA_SLAVE, atdma->dma_device.cap_mask) ? "slave " : "", plat_dat->nr_channels); - dma_async_device_register(&atdma->dma_common); + err = dma_async_device_register(&atdma->dma_device); + if (err) { + dev_err(&pdev->dev, "Unable to register: %d.\n", err); + goto err_dma_async_device_register; + } /* * Do not return an error if the dmac node is not present in order to @@ -1977,64 +2098,40 @@ static int __init at_dma_probe(struct platform_device *pdev) return 0; err_of_dma_controller_register: - dma_async_device_unregister(&atdma->dma_common); + dma_async_device_unregister(&atdma->dma_device); +err_dma_async_device_register: dma_pool_destroy(atdma->memset_pool); err_memset_pool_create: - dma_pool_destroy(atdma->dma_desc_pool); + dma_pool_destroy(atdma->lli_pool); err_desc_pool_create: free_irq(platform_get_irq(pdev, 0), atdma); err_irq: clk_disable_unprepare(atdma->clk); -err_clk_prepare: - clk_put(atdma->clk); -err_clk: - iounmap(atdma->regs); - atdma->regs = NULL; -err_release_r: - release_mem_region(io->start, size); -err_kfree: - kfree(atdma); return err; } -static int at_dma_remove(struct platform_device *pdev) +static void at_dma_remove(struct platform_device *pdev) { struct at_dma *atdma = platform_get_drvdata(pdev); struct dma_chan *chan, *_chan; - struct resource *io; at_dma_off(atdma); if (pdev->dev.of_node) of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&atdma->dma_common); + dma_async_device_unregister(&atdma->dma_device); dma_pool_destroy(atdma->memset_pool); - dma_pool_destroy(atdma->dma_desc_pool); + dma_pool_destroy(atdma->lli_pool); free_irq(platform_get_irq(pdev, 0), atdma); - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, device_node) { - struct at_dma_chan *atchan = to_at_dma_chan(chan); - /* Disable interrupts */ atc_disable_chan_irq(atdma, chan->chan_id); - - tasklet_kill(&atchan->tasklet); list_del(&chan->device_node); } clk_disable_unprepare(atdma->clk); - clk_put(atdma->clk); - - iounmap(atdma->regs); - atdma->regs = NULL; - - io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(io->start, resource_size(io)); - - kfree(atdma); - - return 0; } static void at_dma_shutdown(struct platform_device *pdev) @@ -2050,7 +2147,7 @@ static int at_dma_prepare(struct device *dev) struct at_dma *atdma = dev_get_drvdata(dev); struct dma_chan *chan, *_chan; - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, device_node) { struct at_dma_chan *atchan = to_at_dma_chan(chan); /* wait for transaction completion (except in cyclic case) */ @@ -2062,7 +2159,7 @@ static int at_dma_prepare(struct device *dev) static void atc_suspend_cyclic(struct at_dma_chan *atchan) { - struct dma_chan *chan = &atchan->chan_common; + struct dma_chan *chan = &atchan->vc.chan; /* Channel should be paused by user * do it anyway even if it is not done already */ @@ -2085,7 +2182,7 @@ static int at_dma_suspend_noirq(struct device *dev) struct dma_chan *chan, *_chan; /* preserve data */ - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, device_node) { struct at_dma_chan *atchan = to_at_dma_chan(chan); @@ -2103,7 +2200,7 @@ static int at_dma_suspend_noirq(struct device *dev) static void atc_resume_cyclic(struct at_dma_chan *atchan) { - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); + struct at_dma *atdma = to_at_dma(atchan->vc.chan.device); /* restore channel status for cyclic descriptors list: * next descriptor in the cyclic list at the time of suspend */ @@ -2135,7 +2232,7 @@ static int at_dma_resume_noirq(struct device *dev) /* restore saved data */ dma_writel(atdma, EBCIER, atdma->save_imr); - list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + list_for_each_entry_safe(chan, _chan, &atdma->dma_device.channels, device_node) { struct at_dma_chan *atchan = to_at_dma_chan(chan); @@ -2146,7 +2243,7 @@ static int at_dma_resume_noirq(struct device *dev) return 0; } -static const struct dev_pm_ops at_dma_dev_pm_ops = { +static const struct dev_pm_ops __maybe_unused at_dma_dev_pm_ops = { .prepare = at_dma_prepare, .suspend_noirq = at_dma_suspend_noirq, .resume_noirq = at_dma_resume_noirq, @@ -2158,7 +2255,7 @@ static struct platform_driver at_dma_driver = { .id_table = atdma_devtypes, .driver = { .name = "at_hdmac", - .pm = &at_dma_dev_pm_ops, + .pm = pm_ptr(&at_dma_dev_pm_ops), .of_match_table = of_match_ptr(atmel_dma_dt_ids), }, }; @@ -2177,5 +2274,6 @@ module_exit(at_dma_exit); MODULE_DESCRIPTION("Atmel AHB DMA Controller driver"); MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>"); +MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>"); MODULE_LICENSE("GPL"); MODULE_ALIAS("platform:at_hdmac"); diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h deleted file mode 100644 index 4d1ebc040031..000000000000 --- a/drivers/dma/at_hdmac_regs.h +++ /dev/null @@ -1,478 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Header file for the Atmel AHB DMA Controller driver - * - * Copyright (C) 2008 Atmel Corporation - */ -#ifndef AT_HDMAC_REGS_H -#define AT_HDMAC_REGS_H - -#define AT_DMA_MAX_NR_CHANNELS 8 - - -#define AT_DMA_GCFG 0x00 /* Global Configuration Register */ -#define AT_DMA_IF_BIGEND(i) (0x1 << (i)) /* AHB-Lite Interface i in Big-endian mode */ -#define AT_DMA_ARB_CFG (0x1 << 4) /* Arbiter mode. */ -#define AT_DMA_ARB_CFG_FIXED (0x0 << 4) -#define AT_DMA_ARB_CFG_ROUND_ROBIN (0x1 << 4) - -#define AT_DMA_EN 0x04 /* Controller Enable Register */ -#define AT_DMA_ENABLE (0x1 << 0) - -#define AT_DMA_SREQ 0x08 /* Software Single Request Register */ -#define AT_DMA_SSREQ(x) (0x1 << ((x) << 1)) /* Request a source single transfer on channel x */ -#define AT_DMA_DSREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination single transfer on channel x */ - -#define AT_DMA_CREQ 0x0C /* Software Chunk Transfer Request Register */ -#define AT_DMA_SCREQ(x) (0x1 << ((x) << 1)) /* Request a source chunk transfer on channel x */ -#define AT_DMA_DCREQ(x) (0x1 << (1 + ((x) << 1))) /* Request a destination chunk transfer on channel x */ - -#define AT_DMA_LAST 0x10 /* Software Last Transfer Flag Register */ -#define AT_DMA_SLAST(x) (0x1 << ((x) << 1)) /* This src rq is last tx of buffer on channel x */ -#define AT_DMA_DLAST(x) (0x1 << (1 + ((x) << 1))) /* This dst rq is last tx of buffer on channel x */ - -#define AT_DMA_SYNC 0x14 /* Request Synchronization Register */ -#define AT_DMA_SYR(h) (0x1 << (h)) /* Synchronize handshake line h */ - -/* Error, Chained Buffer transfer completed and Buffer transfer completed Interrupt registers */ -#define AT_DMA_EBCIER 0x18 /* Enable register */ -#define AT_DMA_EBCIDR 0x1C /* Disable register */ -#define AT_DMA_EBCIMR 0x20 /* Mask Register */ -#define AT_DMA_EBCISR 0x24 /* Status Register */ -#define AT_DMA_CBTC_OFFSET 8 -#define AT_DMA_ERR_OFFSET 16 -#define AT_DMA_BTC(x) (0x1 << (x)) -#define AT_DMA_CBTC(x) (0x1 << (AT_DMA_CBTC_OFFSET + (x))) -#define AT_DMA_ERR(x) (0x1 << (AT_DMA_ERR_OFFSET + (x))) - -#define AT_DMA_CHER 0x28 /* Channel Handler Enable Register */ -#define AT_DMA_ENA(x) (0x1 << (x)) -#define AT_DMA_SUSP(x) (0x1 << ( 8 + (x))) -#define AT_DMA_KEEP(x) (0x1 << (24 + (x))) - -#define AT_DMA_CHDR 0x2C /* Channel Handler Disable Register */ -#define AT_DMA_DIS(x) (0x1 << (x)) -#define AT_DMA_RES(x) (0x1 << ( 8 + (x))) - -#define AT_DMA_CHSR 0x30 /* Channel Handler Status Register */ -#define AT_DMA_EMPT(x) (0x1 << (16 + (x))) -#define AT_DMA_STAL(x) (0x1 << (24 + (x))) - - -#define AT_DMA_CH_REGS_BASE 0x3C /* Channel registers base address */ -#define ch_regs(x) (AT_DMA_CH_REGS_BASE + (x) * 0x28) /* Channel x base addr */ - -/* Hardware register offset for each channel */ -#define ATC_SADDR_OFFSET 0x00 /* Source Address Register */ -#define ATC_DADDR_OFFSET 0x04 /* Destination Address Register */ -#define ATC_DSCR_OFFSET 0x08 /* Descriptor Address Register */ -#define ATC_CTRLA_OFFSET 0x0C /* Control A Register */ -#define ATC_CTRLB_OFFSET 0x10 /* Control B Register */ -#define ATC_CFG_OFFSET 0x14 /* Configuration Register */ -#define ATC_SPIP_OFFSET 0x18 /* Src PIP Configuration Register */ -#define ATC_DPIP_OFFSET 0x1C /* Dst PIP Configuration Register */ - - -/* Bitfield definitions */ - -/* Bitfields in DSCR */ -#define ATC_DSCR_IF(i) (0x3 & (i)) /* Dsc feched via AHB-Lite Interface i */ - -/* Bitfields in CTRLA */ -#define ATC_BTSIZE_MAX 0xFFFFUL /* Maximum Buffer Transfer Size */ -#define ATC_BTSIZE(x) (ATC_BTSIZE_MAX & (x)) /* Buffer Transfer Size */ -#define ATC_SCSIZE_MASK (0x7 << 16) /* Source Chunk Transfer Size */ -#define ATC_SCSIZE(x) (ATC_SCSIZE_MASK & ((x) << 16)) -#define ATC_SCSIZE_1 (0x0 << 16) -#define ATC_SCSIZE_4 (0x1 << 16) -#define ATC_SCSIZE_8 (0x2 << 16) -#define ATC_SCSIZE_16 (0x3 << 16) -#define ATC_SCSIZE_32 (0x4 << 16) -#define ATC_SCSIZE_64 (0x5 << 16) -#define ATC_SCSIZE_128 (0x6 << 16) -#define ATC_SCSIZE_256 (0x7 << 16) -#define ATC_DCSIZE_MASK (0x7 << 20) /* Destination Chunk Transfer Size */ -#define ATC_DCSIZE(x) (ATC_DCSIZE_MASK & ((x) << 20)) -#define ATC_DCSIZE_1 (0x0 << 20) -#define ATC_DCSIZE_4 (0x1 << 20) -#define ATC_DCSIZE_8 (0x2 << 20) -#define ATC_DCSIZE_16 (0x3 << 20) -#define ATC_DCSIZE_32 (0x4 << 20) -#define ATC_DCSIZE_64 (0x5 << 20) -#define ATC_DCSIZE_128 (0x6 << 20) -#define ATC_DCSIZE_256 (0x7 << 20) -#define ATC_SRC_WIDTH_MASK (0x3 << 24) /* Source Single Transfer Size */ -#define ATC_SRC_WIDTH(x) ((x) << 24) -#define ATC_SRC_WIDTH_BYTE (0x0 << 24) -#define ATC_SRC_WIDTH_HALFWORD (0x1 << 24) -#define ATC_SRC_WIDTH_WORD (0x2 << 24) -#define ATC_REG_TO_SRC_WIDTH(r) (((r) >> 24) & 0x3) -#define ATC_DST_WIDTH_MASK (0x3 << 28) /* Destination Single Transfer Size */ -#define ATC_DST_WIDTH(x) ((x) << 28) -#define ATC_DST_WIDTH_BYTE (0x0 << 28) -#define ATC_DST_WIDTH_HALFWORD (0x1 << 28) -#define ATC_DST_WIDTH_WORD (0x2 << 28) -#define ATC_DONE (0x1 << 31) /* Tx Done (only written back in descriptor) */ - -/* Bitfields in CTRLB */ -#define ATC_SIF(i) (0x3 & (i)) /* Src tx done via AHB-Lite Interface i */ -#define ATC_DIF(i) ((0x3 & (i)) << 4) /* Dst tx done via AHB-Lite Interface i */ - /* Specify AHB interfaces */ -#define AT_DMA_MEM_IF 0 /* interface 0 as memory interface */ -#define AT_DMA_PER_IF 1 /* interface 1 as peripheral interface */ - -#define ATC_SRC_PIP (0x1 << 8) /* Source Picture-in-Picture enabled */ -#define ATC_DST_PIP (0x1 << 12) /* Destination Picture-in-Picture enabled */ -#define ATC_SRC_DSCR_DIS (0x1 << 16) /* Src Descriptor fetch disable */ -#define ATC_DST_DSCR_DIS (0x1 << 20) /* Dst Descriptor fetch disable */ -#define ATC_FC_MASK (0x7 << 21) /* Choose Flow Controller */ -#define ATC_FC_MEM2MEM (0x0 << 21) /* Mem-to-Mem (DMA) */ -#define ATC_FC_MEM2PER (0x1 << 21) /* Mem-to-Periph (DMA) */ -#define ATC_FC_PER2MEM (0x2 << 21) /* Periph-to-Mem (DMA) */ -#define ATC_FC_PER2PER (0x3 << 21) /* Periph-to-Periph (DMA) */ -#define ATC_FC_PER2MEM_PER (0x4 << 21) /* Periph-to-Mem (Peripheral) */ -#define ATC_FC_MEM2PER_PER (0x5 << 21) /* Mem-to-Periph (Peripheral) */ -#define ATC_FC_PER2PER_SRCPER (0x6 << 21) /* Periph-to-Periph (Src Peripheral) */ -#define ATC_FC_PER2PER_DSTPER (0x7 << 21) /* Periph-to-Periph (Dst Peripheral) */ -#define ATC_SRC_ADDR_MODE_MASK (0x3 << 24) -#define ATC_SRC_ADDR_MODE_INCR (0x0 << 24) /* Incrementing Mode */ -#define ATC_SRC_ADDR_MODE_DECR (0x1 << 24) /* Decrementing Mode */ -#define ATC_SRC_ADDR_MODE_FIXED (0x2 << 24) /* Fixed Mode */ -#define ATC_DST_ADDR_MODE_MASK (0x3 << 28) -#define ATC_DST_ADDR_MODE_INCR (0x0 << 28) /* Incrementing Mode */ -#define ATC_DST_ADDR_MODE_DECR (0x1 << 28) /* Decrementing Mode */ -#define ATC_DST_ADDR_MODE_FIXED (0x2 << 28) /* Fixed Mode */ -#define ATC_IEN (0x1 << 30) /* BTC interrupt enable (active low) */ -#define ATC_AUTO (0x1 << 31) /* Auto multiple buffer tx enable */ - -/* Bitfields in CFG */ -#define ATC_PER_MSB(h) ((0x30U & (h)) >> 4) /* Extract most significant bits of a handshaking identifier */ - -#define ATC_SRC_PER(h) (0xFU & (h)) /* Channel src rq associated with periph handshaking ifc h */ -#define ATC_DST_PER(h) ((0xFU & (h)) << 4) /* Channel dst rq associated with periph handshaking ifc h */ -#define ATC_SRC_REP (0x1 << 8) /* Source Replay Mod */ -#define ATC_SRC_H2SEL (0x1 << 9) /* Source Handshaking Mod */ -#define ATC_SRC_H2SEL_SW (0x0 << 9) -#define ATC_SRC_H2SEL_HW (0x1 << 9) -#define ATC_SRC_PER_MSB(h) (ATC_PER_MSB(h) << 10) /* Channel src rq (most significant bits) */ -#define ATC_DST_REP (0x1 << 12) /* Destination Replay Mod */ -#define ATC_DST_H2SEL (0x1 << 13) /* Destination Handshaking Mod */ -#define ATC_DST_H2SEL_SW (0x0 << 13) -#define ATC_DST_H2SEL_HW (0x1 << 13) -#define ATC_DST_PER_MSB(h) (ATC_PER_MSB(h) << 14) /* Channel dst rq (most significant bits) */ -#define ATC_SOD (0x1 << 16) /* Stop On Done */ -#define ATC_LOCK_IF (0x1 << 20) /* Interface Lock */ -#define ATC_LOCK_B (0x1 << 21) /* AHB Bus Lock */ -#define ATC_LOCK_IF_L (0x1 << 22) /* Master Interface Arbiter Lock */ -#define ATC_LOCK_IF_L_CHUNK (0x0 << 22) -#define ATC_LOCK_IF_L_BUFFER (0x1 << 22) -#define ATC_AHB_PROT_MASK (0x7 << 24) /* AHB Protection */ -#define ATC_FIFOCFG_MASK (0x3 << 28) /* FIFO Request Configuration */ -#define ATC_FIFOCFG_LARGESTBURST (0x0 << 28) -#define ATC_FIFOCFG_HALFFIFO (0x1 << 28) -#define ATC_FIFOCFG_ENOUGHSPACE (0x2 << 28) - -/* Bitfields in SPIP */ -#define ATC_SPIP_HOLE(x) (0xFFFFU & (x)) -#define ATC_SPIP_BOUNDARY(x) ((0x3FF & (x)) << 16) - -/* Bitfields in DPIP */ -#define ATC_DPIP_HOLE(x) (0xFFFFU & (x)) -#define ATC_DPIP_BOUNDARY(x) ((0x3FF & (x)) << 16) - - -/*-- descriptors -----------------------------------------------------*/ - -/* LLI == Linked List Item; aka DMA buffer descriptor */ -struct at_lli { - /* values that are not changed by hardware */ - dma_addr_t saddr; - dma_addr_t daddr; - /* value that may get written back: */ - u32 ctrla; - /* more values that are not changed by hardware */ - u32 ctrlb; - dma_addr_t dscr; /* chain to next lli */ -}; - -/** - * struct at_desc - software descriptor - * @at_lli: hardware lli structure - * @txd: support for the async_tx api - * @desc_node: node on the channed descriptors list - * @len: descriptor byte count - * @total_len: total transaction byte count - */ -struct at_desc { - /* FIRST values the hardware uses */ - struct at_lli lli; - - /* THEN values for driver housekeeping */ - struct list_head tx_list; - struct dma_async_tx_descriptor txd; - struct list_head desc_node; - size_t len; - size_t total_len; - - /* Interleaved data */ - size_t boundary; - size_t dst_hole; - size_t src_hole; - - /* Memset temporary buffer */ - bool memset_buffer; - dma_addr_t memset_paddr; - int *memset_vaddr; -}; - -static inline struct at_desc * -txd_to_at_desc(struct dma_async_tx_descriptor *txd) -{ - return container_of(txd, struct at_desc, txd); -} - - -/*-- Channels --------------------------------------------------------*/ - -/** - * atc_status - information bits stored in channel status flag - * - * Manipulated with atomic operations. - */ -enum atc_status { - ATC_IS_ERROR = 0, - ATC_IS_PAUSED = 1, - ATC_IS_CYCLIC = 24, -}; - -/** - * struct at_dma_chan - internal representation of an Atmel HDMAC channel - * @chan_common: common dmaengine channel object members - * @device: parent device - * @ch_regs: memory mapped register base - * @mask: channel index in a mask - * @per_if: peripheral interface - * @mem_if: memory interface - * @status: transmit status information from irq/prep* functions - * to tasklet (use atomic operations) - * @tasklet: bottom half to finish transaction work - * @save_cfg: configuration register that is saved on suspend/resume cycle - * @save_dscr: for cyclic operations, preserve next descriptor address in - * the cyclic list on suspend/resume cycle - * @dma_sconfig: configuration for slave transfers, passed via - * .device_config - * @lock: serializes enqueue/dequeue operations to descriptors lists - * @active_list: list of descriptors dmaengine is being running on - * @queue: list of descriptors ready to be submitted to engine - * @free_list: list of descriptors usable by the channel - */ -struct at_dma_chan { - struct dma_chan chan_common; - struct at_dma *device; - void __iomem *ch_regs; - u8 mask; - u8 per_if; - u8 mem_if; - unsigned long status; - struct tasklet_struct tasklet; - u32 save_cfg; - u32 save_dscr; - struct dma_slave_config dma_sconfig; - - spinlock_t lock; - - /* these other elements are all protected by lock */ - struct list_head active_list; - struct list_head queue; - struct list_head free_list; -}; - -#define channel_readl(atchan, name) \ - __raw_readl((atchan)->ch_regs + ATC_##name##_OFFSET) - -#define channel_writel(atchan, name, val) \ - __raw_writel((val), (atchan)->ch_regs + ATC_##name##_OFFSET) - -static inline struct at_dma_chan *to_at_dma_chan(struct dma_chan *dchan) -{ - return container_of(dchan, struct at_dma_chan, chan_common); -} - -/* - * Fix sconfig's burst size according to at_hdmac. We need to convert them as: - * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3, 32 -> 4, 64 -> 5, 128 -> 6, 256 -> 7. - * - * This can be done by finding most significant bit set. - */ -static inline void convert_burst(u32 *maxburst) -{ - if (*maxburst > 1) - *maxburst = fls(*maxburst) - 2; - else - *maxburst = 0; -} - -/* - * Fix sconfig's bus width according to at_hdmac. - * 1 byte -> 0, 2 bytes -> 1, 4 bytes -> 2. - */ -static inline u8 convert_buswidth(enum dma_slave_buswidth addr_width) -{ - switch (addr_width) { - case DMA_SLAVE_BUSWIDTH_2_BYTES: - return 1; - case DMA_SLAVE_BUSWIDTH_4_BYTES: - return 2; - default: - /* For 1 byte width or fallback */ - return 0; - } -} - -/*-- Controller ------------------------------------------------------*/ - -/** - * struct at_dma - internal representation of an Atmel HDMA Controller - * @chan_common: common dmaengine dma_device object members - * @atdma_devtype: identifier of DMA controller compatibility - * @ch_regs: memory mapped register base - * @clk: dma controller clock - * @save_imr: interrupt mask register that is saved on suspend/resume cycle - * @all_chan_mask: all channels availlable in a mask - * @dma_desc_pool: base of DMA descriptor region (DMA address) - * @chan: channels table to store at_dma_chan structures - */ -struct at_dma { - struct dma_device dma_common; - void __iomem *regs; - struct clk *clk; - u32 save_imr; - - u8 all_chan_mask; - - struct dma_pool *dma_desc_pool; - struct dma_pool *memset_pool; - /* AT THE END channels table */ - struct at_dma_chan chan[]; -}; - -#define dma_readl(atdma, name) \ - __raw_readl((atdma)->regs + AT_DMA_##name) -#define dma_writel(atdma, name, val) \ - __raw_writel((val), (atdma)->regs + AT_DMA_##name) - -static inline struct at_dma *to_at_dma(struct dma_device *ddev) -{ - return container_of(ddev, struct at_dma, dma_common); -} - - -/*-- Helper functions ------------------------------------------------*/ - -static struct device *chan2dev(struct dma_chan *chan) -{ - return &chan->dev->device; -} - -#if defined(VERBOSE_DEBUG) -static void vdbg_dump_regs(struct at_dma_chan *atchan) -{ - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); - - dev_err(chan2dev(&atchan->chan_common), - " channel %d : imr = 0x%x, chsr = 0x%x\n", - atchan->chan_common.chan_id, - dma_readl(atdma, EBCIMR), - dma_readl(atdma, CHSR)); - - dev_err(chan2dev(&atchan->chan_common), - " channel: s0x%x d0x%x ctrl0x%x:0x%x cfg0x%x l0x%x\n", - channel_readl(atchan, SADDR), - channel_readl(atchan, DADDR), - channel_readl(atchan, CTRLA), - channel_readl(atchan, CTRLB), - channel_readl(atchan, CFG), - channel_readl(atchan, DSCR)); -} -#else -static void vdbg_dump_regs(struct at_dma_chan *atchan) {} -#endif - -static void atc_dump_lli(struct at_dma_chan *atchan, struct at_lli *lli) -{ - dev_crit(chan2dev(&atchan->chan_common), - "desc: s%pad d%pad ctrl0x%x:0x%x l%pad\n", - &lli->saddr, &lli->daddr, - lli->ctrla, lli->ctrlb, &lli->dscr); -} - - -static void atc_setup_irq(struct at_dma *atdma, int chan_id, int on) -{ - u32 ebci; - - /* enable interrupts on buffer transfer completion & error */ - ebci = AT_DMA_BTC(chan_id) - | AT_DMA_ERR(chan_id); - if (on) - dma_writel(atdma, EBCIER, ebci); - else - dma_writel(atdma, EBCIDR, ebci); -} - -static void atc_enable_chan_irq(struct at_dma *atdma, int chan_id) -{ - atc_setup_irq(atdma, chan_id, 1); -} - -static void atc_disable_chan_irq(struct at_dma *atdma, int chan_id) -{ - atc_setup_irq(atdma, chan_id, 0); -} - - -/** - * atc_chan_is_enabled - test if given channel is enabled - * @atchan: channel we want to test status - */ -static inline int atc_chan_is_enabled(struct at_dma_chan *atchan) -{ - struct at_dma *atdma = to_at_dma(atchan->chan_common.device); - - return !!(dma_readl(atdma, CHSR) & atchan->mask); -} - -/** - * atc_chan_is_paused - test channel pause/resume status - * @atchan: channel we want to test status - */ -static inline int atc_chan_is_paused(struct at_dma_chan *atchan) -{ - return test_bit(ATC_IS_PAUSED, &atchan->status); -} - -/** - * atc_chan_is_cyclic - test if given channel has cyclic property set - * @atchan: channel we want to test status - */ -static inline int atc_chan_is_cyclic(struct at_dma_chan *atchan) -{ - return test_bit(ATC_IS_CYCLIC, &atchan->status); -} - -/** - * set_desc_eol - set end-of-link to descriptor so it will end transfer - * @desc: descriptor, signle or at the end of a chain, to end chain on - */ -static void set_desc_eol(struct at_desc *desc) -{ - u32 ctrlb = desc->lli.ctrlb; - - ctrlb &= ~ATC_IEN; - ctrlb |= ATC_SRC_DSCR_DIS | ATC_DST_DSCR_DIS; - - desc->lli.ctrlb = ctrlb; - desc->lli.dscr = 0; -} - -#endif /* AT_HDMAC_REGS_H */ diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index 64a52bf4d737..3fbc74710a13 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -21,6 +21,7 @@ #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm.h> +#include <linux/pm_runtime.h> #include "dmaengine.h" @@ -99,6 +100,7 @@ #define AT_XDMAC_CNDC_NDE (0x1 << 0) /* Channel x Next Descriptor Enable */ #define AT_XDMAC_CNDC_NDSUP (0x1 << 1) /* Channel x Next Descriptor Source Update */ #define AT_XDMAC_CNDC_NDDUP (0x1 << 2) /* Channel x Next Descriptor Destination Update */ +#define AT_XDMAC_CNDC_NDVIEW_MASK GENMASK(28, 27) #define AT_XDMAC_CNDC_NDVIEW_NDV0 (0x0 << 3) /* Channel x Next Descriptor View 0 */ #define AT_XDMAC_CNDC_NDVIEW_NDV1 (0x1 << 3) /* Channel x Next Descriptor View 1 */ #define AT_XDMAC_CNDC_NDVIEW_NDV2 (0x2 << 3) /* Channel x Next Descriptor View 2 */ @@ -155,7 +157,7 @@ #define AT_XDMAC_CC_WRIP (0x1 << 23) /* Write in Progress (read only) */ #define AT_XDMAC_CC_WRIP_DONE (0x0 << 23) #define AT_XDMAC_CC_WRIP_IN_PROGRESS (0x1 << 23) -#define AT_XDMAC_CC_PERID(i) (0x7f & (i) << 24) /* Channel Peripheral Identifier */ +#define AT_XDMAC_CC_PERID(i) ((0x7f & (i)) << 24) /* Channel Peripheral Identifier */ #define AT_XDMAC_CDS_MSP 0x2C /* Channel Data Stride Memory Set Pattern */ #define AT_XDMAC_CSUS 0x30 /* Channel Source Microblock Stride */ #define AT_XDMAC_CDUS 0x34 /* Channel Destination Microblock Stride */ @@ -185,6 +187,7 @@ enum atc_status { AT_XDMAC_CHAN_IS_CYCLIC = 0, AT_XDMAC_CHAN_IS_PAUSED, + AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, }; struct at_xdmac_layout { @@ -239,9 +242,11 @@ struct at_xdmac_chan { struct at_xdmac { struct dma_device dma; void __iomem *regs; + struct device *dev; int irq; struct clk *clk; u32 save_gim; + u32 save_gs; struct dma_pool *at_xdmac_desc_pool; const struct at_xdmac_layout *layout; struct at_xdmac_chan chan[]; @@ -252,15 +257,15 @@ struct at_xdmac { /* Linked List Descriptor */ struct at_xdmac_lld { - dma_addr_t mbr_nda; /* Next Descriptor Member */ - u32 mbr_ubc; /* Microblock Control Member */ - dma_addr_t mbr_sa; /* Source Address Member */ - dma_addr_t mbr_da; /* Destination Address Member */ - u32 mbr_cfg; /* Configuration Register */ - u32 mbr_bc; /* Block Control Register */ - u32 mbr_ds; /* Data Stride Register */ - u32 mbr_sus; /* Source Microblock Stride Register */ - u32 mbr_dus; /* Destination Microblock Stride Register */ + u32 mbr_nda; /* Next Descriptor Member */ + u32 mbr_ubc; /* Microblock Control Member */ + u32 mbr_sa; /* Source Address Member */ + u32 mbr_da; /* Destination Address Member */ + u32 mbr_cfg; /* Configuration Register */ + u32 mbr_bc; /* Block Control Register */ + u32 mbr_ds; /* Data Stride Register */ + u32 mbr_sus; /* Source Microblock Stride Register */ + u32 mbr_dus; /* Destination Microblock Stride Register */ }; /* 64-bit alignment needed to update CNDA and CUBC registers in an atomic way. */ @@ -344,6 +349,11 @@ static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan) return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); } +static inline int at_xdmac_chan_is_paused_internal(struct at_xdmac_chan *atchan) +{ + return test_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status); +} + static inline bool at_xdmac_chan_is_peripheral_xfer(u32 cfg) { return cfg & AT_XDMAC_CC_TYPE_PER_TRAN; @@ -360,13 +370,65 @@ MODULE_PARM_DESC(init_nr_desc_per_channel, "initial descriptors per channel (default: 64)"); +static void at_xdmac_runtime_suspend_descriptors(struct at_xdmac_chan *atchan) +{ + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + struct at_xdmac_desc *desc, *_desc; + + list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { + if (!desc->active_xfer) + continue; + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + } +} + +static int at_xdmac_runtime_resume_descriptors(struct at_xdmac_chan *atchan) +{ + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + struct at_xdmac_desc *desc, *_desc; + int ret; + + list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { + if (!desc->active_xfer) + continue; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + } + + return 0; +} + static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan) { - return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask; + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return false; + + ret = !!(at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask); + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + + return ret; } -static void at_xdmac_off(struct at_xdmac *atxdmac) +static void at_xdmac_off(struct at_xdmac *atxdmac, bool suspend_descriptors) { + struct dma_chan *chan, *_chan; + struct at_xdmac_chan *atchan; + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return; + at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L); /* Wait that all chans are disabled. */ @@ -374,6 +436,18 @@ static void at_xdmac_off(struct at_xdmac *atxdmac) cpu_relax(); at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L); + + /* Decrement runtime PM ref counter for each active descriptor. */ + if (!list_empty(&atxdmac->dma.channels) && suspend_descriptors) { + list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, + device_node) { + atchan = to_at_xdmac_chan(chan); + at_xdmac_runtime_suspend_descriptors(atchan); + } + } + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); } /* Call with lock hold. */ @@ -382,12 +456,14 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, { struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); u32 reg; + int ret; - dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); - - if (at_xdmac_chan_is_enabled(atchan)) + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) return; + dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first); + /* Set transfer as active to not try to start it again. */ first->active_xfer = true; @@ -405,7 +481,8 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, */ if (at_xdmac_chan_is_cyclic(atchan)) reg = AT_XDMAC_CNDC_NDVIEW_NDV1; - else if (first->lld.mbr_ubc & AT_XDMAC_MBR_UBC_NDV3) + else if ((first->lld.mbr_ubc & + AT_XDMAC_CNDC_NDVIEW_MASK) == AT_XDMAC_MBR_UBC_NDV3) reg = AT_XDMAC_CNDC_NDVIEW_NDV3; else reg = AT_XDMAC_CNDC_NDVIEW_NDV2; @@ -463,7 +540,6 @@ static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan, at_xdmac_chan_read(atchan, AT_XDMAC_CSA), at_xdmac_chan_read(atchan, AT_XDMAC_CDA), at_xdmac_chan_read(atchan, AT_XDMAC_CUBC)); - } static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx) @@ -476,13 +552,12 @@ static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx) spin_lock_irqsave(&atchan->lock, irqflags); cookie = dma_cookie_assign(tx); + list_add_tail(&desc->xfer_node, &atchan->xfers_list); + spin_unlock_irqrestore(&atchan->lock, irqflags); + dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n", __func__, atchan, desc); - list_add_tail(&desc->xfer_node, &atchan->xfers_list); - if (list_is_singular(&atchan->xfers_list)) - at_xdmac_start_xfer(atchan, desc); - spin_unlock_irqrestore(&atchan->lock, irqflags); return cookie; } @@ -651,7 +726,7 @@ static int at_xdmac_compute_chan_conf(struct dma_chan *chan, } /* - * Only check that maxburst and addr width values are supported by the + * Only check that maxburst and addr width values are supported by * the controller but not that the configuration is good to perform the * transfer since we don't know the direction at this stage. */ @@ -733,7 +808,8 @@ at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!desc) { dev_err(chan2dev(chan), "can't get descriptor\n"); if (first) - list_splice_init(&first->descs_list, &atchan->free_descs_list); + list_splice_tail_init(&first->descs_list, + &atchan->free_descs_list); goto spin_unlock; } @@ -821,7 +897,8 @@ at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, if (!desc) { dev_err(chan2dev(chan), "can't get descriptor\n"); if (first) - list_splice_init(&first->descs_list, &atchan->free_descs_list); + list_splice_tail_init(&first->descs_list, + &atchan->free_descs_list); spin_unlock_irqrestore(&atchan->lock, irqflags); return NULL; } @@ -1025,6 +1102,8 @@ at_xdmac_prep_interleaved(struct dma_chan *chan, NULL, src_addr, dst_addr, xt, xt->sgl); + if (!first) + return NULL; /* Length of the block is (BLEN+1) microblocks. */ for (i = 0; i < xt->numf - 1; i++) @@ -1055,8 +1134,9 @@ at_xdmac_prep_interleaved(struct dma_chan *chan, src_addr, dst_addr, xt, chunk); if (!desc) { - list_splice_init(&first->descs_list, - &atchan->free_descs_list); + if (first) + list_splice_tail_init(&first->descs_list, + &atchan->free_descs_list); return NULL; } @@ -1136,7 +1216,8 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if (!desc) { dev_err(chan2dev(chan), "can't get descriptor\n"); if (first) - list_splice_init(&first->descs_list, &atchan->free_descs_list); + list_splice_tail_init(&first->descs_list, + &atchan->free_descs_list); return NULL; } @@ -1201,6 +1282,7 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, unsigned long flags; size_t ublen; u32 dwidth; + char pattern; /* * WARNING: The channel configuration is set here since there is no * dmaengine_slave_config call in this case. Moreover we don't know the @@ -1243,10 +1325,16 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth); + /* Only the first byte of value is to be used according to dmaengine */ + pattern = (char)value; + ublen = len >> dwidth; desc->lld.mbr_da = dst_addr; - desc->lld.mbr_ds = value; + desc->lld.mbr_ds = (pattern << 24) | + (pattern << 16) | + (pattern << 8) | + pattern; desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3 | AT_XDMAC_MBR_UBC_NDEN | AT_XDMAC_MBR_UBC_NSEN @@ -1275,6 +1363,8 @@ at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, return NULL; desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value); + if (!desc) + return NULL; list_add_tail(&desc->desc_node, &desc->descs_list); desc->tx_dma_desc.cookie = -EBUSY; @@ -1312,8 +1402,8 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, sg_dma_len(sg), value); if (!desc && first) - list_splice_init(&first->descs_list, - &atchan->free_descs_list); + list_splice_tail_init(&first->descs_list, + &atchan->free_descs_list); if (!first) first = desc; @@ -1448,25 +1538,26 @@ at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl, static enum dma_status at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, - struct dma_tx_state *txstate) + struct dma_tx_state *txstate) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); - struct at_xdmac_desc *desc, *_desc; + struct at_xdmac_desc *desc, *_desc, *iter; struct list_head *descs_list; enum dma_status ret; - int residue, retry; + int residue, retry, pm_status; u32 cur_nda, check_nda, cur_ubc, mask, value; u8 dwidth = 0; unsigned long flags; bool initd; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) + if (ret == DMA_COMPLETE || !txstate) return ret; - if (!txstate) - return ret; + pm_status = pm_runtime_resume_and_get(atxdmac->dev); + if (pm_status < 0) + return DMA_ERROR; spin_lock_irqsave(&atchan->lock, flags); @@ -1567,11 +1658,13 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, * microblock. */ descs_list = &desc->descs_list; - list_for_each_entry_safe(desc, _desc, descs_list, desc_node) { - dwidth = at_xdmac_get_dwidth(desc->lld.mbr_cfg); - residue -= (desc->lld.mbr_ubc & 0xffffff) << dwidth; - if ((desc->lld.mbr_nda & 0xfffffffc) == cur_nda) + list_for_each_entry_safe(iter, _desc, descs_list, desc_node) { + dwidth = at_xdmac_get_dwidth(iter->lld.mbr_cfg); + residue -= (iter->lld.mbr_ubc & 0xffffff) << dwidth; + if ((iter->lld.mbr_nda & 0xfffffffc) == cur_nda) { + desc = iter; break; + } } residue += cur_ubc << dwidth; @@ -1583,23 +1676,11 @@ at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, spin_unlock: spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); return ret; } -/* Call must be protected by lock. */ -static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan, - struct at_xdmac_desc *desc) -{ - dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); - - /* - * Remove the transfer from the transfer list then move the transfer - * descriptors into the free descriptors list. - */ - list_del(&desc->xfer_node); - list_splice_init(&desc->descs_list, &atchan->free_descs_list); -} - static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) { struct at_xdmac_desc *desc; @@ -1608,14 +1689,14 @@ static void at_xdmac_advance_work(struct at_xdmac_chan *atchan) * If channel is enabled, do nothing, advance_work will be triggered * after the interruption. */ - if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) { - desc = list_first_entry(&atchan->xfers_list, - struct at_xdmac_desc, - xfer_node); - dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); - if (!desc->active_xfer) - at_xdmac_start_xfer(atchan, desc); - } + if (at_xdmac_chan_is_enabled(atchan) || list_empty(&atchan->xfers_list)) + return; + + desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, + xfer_node); + dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); + if (!desc->active_xfer) + at_xdmac_start_xfer(atchan, desc); } static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) @@ -1623,20 +1704,31 @@ static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan) struct at_xdmac_desc *desc; struct dma_async_tx_descriptor *txd; - if (!list_empty(&atchan->xfers_list)) { - desc = list_first_entry(&atchan->xfers_list, - struct at_xdmac_desc, xfer_node); - txd = &desc->tx_dma_desc; - - if (txd->flags & DMA_PREP_INTERRUPT) - dmaengine_desc_get_callback_invoke(txd, NULL); + spin_lock_irq(&atchan->lock); + dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n", + __func__, atchan->irq_status); + if (list_empty(&atchan->xfers_list)) { + spin_unlock_irq(&atchan->lock); + return; } + desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, + xfer_node); + spin_unlock_irq(&atchan->lock); + txd = &desc->tx_dma_desc; + if (txd->flags & DMA_PREP_INTERRUPT) + dmaengine_desc_get_callback_invoke(txd, NULL); } +/* Called with atchan->lock held. */ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) { struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); struct at_xdmac_desc *bad_desc; + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return; /* * The descriptor currently at the head of the active list is @@ -1651,8 +1743,6 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) if (atchan->irq_status & AT_XDMAC_CIS_ROIS) dev_err(chan2dev(&atchan->chan), "request overflow error!!!"); - spin_lock_irq(&atchan->lock); - /* Channel must be disabled first as it's not done automatically */ at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) @@ -1662,65 +1752,78 @@ static void at_xdmac_handle_error(struct at_xdmac_chan *atchan) struct at_xdmac_desc, xfer_node); - spin_unlock_irq(&atchan->lock); - /* Print bad descriptor's details if needed */ dev_dbg(chan2dev(&atchan->chan), "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n", __func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da, bad_desc->lld.mbr_ubc); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + /* Then continue with usual descriptor management */ } static void at_xdmac_tasklet(struct tasklet_struct *t) { struct at_xdmac_chan *atchan = from_tasklet(atchan, t, tasklet); + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); struct at_xdmac_desc *desc; + struct dma_async_tx_descriptor *txd; u32 error_mask; + if (at_xdmac_chan_is_cyclic(atchan)) + return at_xdmac_handle_cyclic(atchan); + + error_mask = AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS | + AT_XDMAC_CIS_ROIS; + + spin_lock_irq(&atchan->lock); + dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n", __func__, atchan->irq_status); - error_mask = AT_XDMAC_CIS_RBEIS - | AT_XDMAC_CIS_WBEIS - | AT_XDMAC_CIS_ROIS; - - if (at_xdmac_chan_is_cyclic(atchan)) { - at_xdmac_handle_cyclic(atchan); - } else if ((atchan->irq_status & AT_XDMAC_CIS_LIS) - || (atchan->irq_status & error_mask)) { - struct dma_async_tx_descriptor *txd; - - if (atchan->irq_status & error_mask) - at_xdmac_handle_error(atchan); - - spin_lock_irq(&atchan->lock); - desc = list_first_entry(&atchan->xfers_list, - struct at_xdmac_desc, - xfer_node); - dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); - if (!desc->active_xfer) { - dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); - spin_unlock_irq(&atchan->lock); - return; - } + if (!(atchan->irq_status & AT_XDMAC_CIS_LIS) && + !(atchan->irq_status & error_mask)) { + spin_unlock_irq(&atchan->lock); + return; + } - txd = &desc->tx_dma_desc; + if (atchan->irq_status & error_mask) + at_xdmac_handle_error(atchan); - at_xdmac_remove_xfer(atchan, desc); + desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, + xfer_node); + dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc); + if (!desc->active_xfer) { + dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting"); spin_unlock_irq(&atchan->lock); + return; + } - dma_cookie_complete(txd); - if (txd->flags & DMA_PREP_INTERRUPT) - dmaengine_desc_get_callback_invoke(txd, NULL); + txd = &desc->tx_dma_desc; + dma_cookie_complete(txd); + /* Remove the transfer from the transfer list. */ + list_del(&desc->xfer_node); + spin_unlock_irq(&atchan->lock); - dma_run_dependencies(txd); + if (txd->flags & DMA_PREP_INTERRUPT) + dmaengine_desc_get_callback_invoke(txd, NULL); - spin_lock_irq(&atchan->lock); - at_xdmac_advance_work(atchan); - spin_unlock_irq(&atchan->lock); - } + dma_run_dependencies(txd); + + spin_lock_irq(&atchan->lock); + /* Move the xfer descriptors into the free descriptors list. */ + list_splice_tail_init(&desc->descs_list, &atchan->free_descs_list); + at_xdmac_advance_work(atchan); + spin_unlock_irq(&atchan->lock); + + /* + * Decrement runtime PM ref counter incremented in + * at_xdmac_start_xfer(). + */ + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); } static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id) @@ -1784,11 +1887,9 @@ static void at_xdmac_issue_pending(struct dma_chan *chan) dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__); - if (!at_xdmac_chan_is_cyclic(atchan)) { - spin_lock_irqsave(&atchan->lock, flags); - at_xdmac_advance_work(atchan); - spin_unlock_irqrestore(&atchan->lock, flags); - } + spin_lock_irqsave(&atchan->lock, flags); + at_xdmac_advance_work(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); return; } @@ -1809,46 +1910,98 @@ static int at_xdmac_device_config(struct dma_chan *chan, return ret; } +static void at_xdmac_device_pause_set(struct at_xdmac *atxdmac, + struct at_xdmac_chan *atchan) +{ + at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask); + while (at_xdmac_chan_read(atchan, AT_XDMAC_CC) & + (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP)) + cpu_relax(); +} + +static void at_xdmac_device_pause_internal(struct at_xdmac_chan *atchan) +{ + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + unsigned long flags; + + spin_lock_irqsave(&atchan->lock, flags); + set_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status); + at_xdmac_device_pause_set(atxdmac, atchan); + spin_unlock_irqrestore(&atchan->lock, flags); +} + static int at_xdmac_device_pause(struct dma_chan *chan) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); unsigned long flags; + int ret; dev_dbg(chan2dev(chan), "%s\n", __func__); if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status)) return 0; + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + spin_lock_irqsave(&atchan->lock, flags); - at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask); - while (at_xdmac_chan_read(atchan, AT_XDMAC_CC) - & (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP)) - cpu_relax(); + + at_xdmac_device_pause_set(atxdmac, atchan); + /* Decrement runtime PM ref counter for each active descriptor. */ + at_xdmac_runtime_suspend_descriptors(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + return 0; } +static void at_xdmac_device_resume_internal(struct at_xdmac_chan *atchan) +{ + struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); + unsigned long flags; + + spin_lock_irqsave(&atchan->lock, flags); + at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask); + clear_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status); + spin_unlock_irqrestore(&atchan->lock, flags); +} + static int at_xdmac_device_resume(struct dma_chan *chan) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); unsigned long flags; + int ret; dev_dbg(chan2dev(chan), "%s\n", __func__); + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + spin_lock_irqsave(&atchan->lock, flags); - if (!at_xdmac_chan_is_paused(atchan)) { - spin_unlock_irqrestore(&atchan->lock, flags); - return 0; - } + if (!at_xdmac_chan_is_paused(atchan)) + goto unlock; + + /* Increment runtime PM ref counter for each active descriptor. */ + ret = at_xdmac_runtime_resume_descriptors(atchan); + if (ret < 0) + goto unlock; at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask); clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); + +unlock: spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); - return 0; + return ret; } static int at_xdmac_device_terminate_all(struct dma_chan *chan) @@ -1857,22 +2010,40 @@ static int at_xdmac_device_terminate_all(struct dma_chan *chan) struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); struct at_xdmac *atxdmac = to_at_xdmac(atchan->chan.device); unsigned long flags; + int ret; dev_dbg(chan2dev(chan), "%s\n", __func__); + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; + spin_lock_irqsave(&atchan->lock, flags); at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask); while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask) cpu_relax(); /* Cancel all pending transfers. */ - list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) - at_xdmac_remove_xfer(atchan, desc); + list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) { + list_del(&desc->xfer_node); + list_splice_tail_init(&desc->descs_list, + &atchan->free_descs_list); + /* + * We incremented the runtime PM reference count on + * at_xdmac_start_xfer() for this descriptor. Now it's time + * to release it. + */ + if (desc->active_xfer) + pm_runtime_put_noidle(atxdmac->dev); + } clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status); clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status); spin_unlock_irqrestore(&atchan->lock, flags); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + return 0; } @@ -1897,6 +2068,11 @@ static int at_xdmac_alloc_chan_resources(struct dma_chan *chan) for (i = 0; i < init_nr_desc_per_channel; i++) { desc = at_xdmac_alloc_desc(chan, GFP_KERNEL); if (!desc) { + if (i == 0) { + dev_warn(chan2dev(chan), + "can't allocate any descriptors\n"); + return -EIO; + } dev_warn(chan2dev(chan), "only %d descriptors have been allocated\n", i); break; @@ -1926,8 +2102,31 @@ static void at_xdmac_free_chan_resources(struct dma_chan *chan) return; } -#ifdef CONFIG_PM -static int atmel_xdmac_prepare(struct device *dev) +static void at_xdmac_axi_config(struct platform_device *pdev) +{ + struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev); + bool dev_m2m = false; + u32 dma_requests; + + if (!atxdmac->layout->axi_config) + return; /* Not supported */ + + if (!of_property_read_u32(pdev->dev.of_node, "dma-requests", + &dma_requests)) { + dev_info(&pdev->dev, "controller in mem2mem mode.\n"); + dev_m2m = true; + } + + if (dev_m2m) { + at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M); + at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M); + } else { + at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M); + at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M); + } +} + +static int __maybe_unused atmel_xdmac_prepare(struct device *dev) { struct at_xdmac *atxdmac = dev_get_drvdata(dev); struct dma_chan *chan, *_chan; @@ -1941,47 +2140,60 @@ static int atmel_xdmac_prepare(struct device *dev) } return 0; } -#else -# define atmel_xdmac_prepare NULL -#endif -#ifdef CONFIG_PM_SLEEP -static int atmel_xdmac_suspend(struct device *dev) +static int __maybe_unused atmel_xdmac_suspend(struct device *dev) { struct at_xdmac *atxdmac = dev_get_drvdata(dev); struct dma_chan *chan, *_chan; + int ret; + + ret = pm_runtime_resume_and_get(atxdmac->dev); + if (ret < 0) + return ret; list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan); atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC); if (at_xdmac_chan_is_cyclic(atchan)) { - if (!at_xdmac_chan_is_paused(atchan)) - at_xdmac_device_pause(chan); + if (!at_xdmac_chan_is_paused(atchan)) { + dev_warn(chan2dev(chan), "%s: channel %d not paused\n", + __func__, chan->chan_id); + at_xdmac_device_pause_internal(atchan); + at_xdmac_runtime_suspend_descriptors(atchan); + } atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM); atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA); atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC); } } atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM); + atxdmac->save_gs = at_xdmac_read(atxdmac, AT_XDMAC_GS); - at_xdmac_off(atxdmac); + at_xdmac_off(atxdmac, false); + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_noidle(atxdmac->dev); clk_disable_unprepare(atxdmac->clk); + return 0; } -static int atmel_xdmac_resume(struct device *dev) +static int __maybe_unused atmel_xdmac_resume(struct device *dev) { struct at_xdmac *atxdmac = dev_get_drvdata(dev); struct at_xdmac_chan *atchan; struct dma_chan *chan, *_chan; - int i; - int ret; + struct platform_device *pdev = container_of(dev, struct platform_device, dev); + int i, ret; ret = clk_prepare_enable(atxdmac->clk); if (ret) return ret; + pm_runtime_get_noresume(atxdmac->dev); + + at_xdmac_axi_config(pdev); + /* Clear pending interrupts. */ for (i = 0; i < atxdmac->dma.chancnt; i++) { atchan = &atxdmac->chan[i]; @@ -1992,49 +2204,63 @@ static int atmel_xdmac_resume(struct device *dev) at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim); list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) { atchan = to_at_xdmac_chan(chan); + at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc); if (at_xdmac_chan_is_cyclic(atchan)) { - if (at_xdmac_chan_is_paused(atchan)) - at_xdmac_device_resume(chan); + /* + * Resume only channels not explicitly paused by + * consumers. + */ + if (at_xdmac_chan_is_paused_internal(atchan)) { + ret = at_xdmac_runtime_resume_descriptors(atchan); + if (ret < 0) + return ret; + at_xdmac_device_resume_internal(atchan); + } + + /* + * We may resume from a deep sleep state where power + * to DMA controller is cut-off. Thus, restore the + * suspend state of channels set though dmaengine API. + */ + else if (at_xdmac_chan_is_paused(atchan)) + at_xdmac_device_pause_set(atxdmac, atchan); + at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda); at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc); at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim); wmb(); - at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); + if (atxdmac->save_gs & atchan->mask) + at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask); } } + + pm_runtime_mark_last_busy(atxdmac->dev); + pm_runtime_put_autosuspend(atxdmac->dev); + return 0; } -#endif /* CONFIG_PM_SLEEP */ -static void at_xdmac_axi_config(struct platform_device *pdev) +static int __maybe_unused atmel_xdmac_runtime_suspend(struct device *dev) { - struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev); - bool dev_m2m = false; - u32 dma_requests; + struct at_xdmac *atxdmac = dev_get_drvdata(dev); - if (!atxdmac->layout->axi_config) - return; /* Not supported */ + clk_disable(atxdmac->clk); - if (!of_property_read_u32(pdev->dev.of_node, "dma-requests", - &dma_requests)) { - dev_info(&pdev->dev, "controller in mem2mem mode.\n"); - dev_m2m = true; - } + return 0; +} - if (dev_m2m) { - at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M); - at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M); - } else { - at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M); - at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M); - } +static int __maybe_unused atmel_xdmac_runtime_resume(struct device *dev) +{ + struct at_xdmac *atxdmac = dev_get_drvdata(dev); + + return clk_enable(atxdmac->clk); } static int at_xdmac_probe(struct platform_device *pdev) { struct at_xdmac *atxdmac; - int irq, size, nr_channels, i, ret; + int irq, nr_channels, i, ret; void __iomem *base; u32 reg; @@ -2059,9 +2285,9 @@ static int at_xdmac_probe(struct platform_device *pdev) return -EINVAL; } - size = sizeof(*atxdmac); - size += nr_channels * sizeof(struct at_xdmac_chan); - atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + atxdmac = devm_kzalloc(&pdev->dev, + struct_size(atxdmac, chan, nr_channels), + GFP_KERNEL); if (!atxdmac) { dev_err(&pdev->dev, "can't allocate at_xdmac structure\n"); return -ENOMEM; @@ -2069,6 +2295,7 @@ static int at_xdmac_probe(struct platform_device *pdev) atxdmac->regs = base; atxdmac->irq = irq; + atxdmac->dev = &pdev->dev; atxdmac->layout = of_device_get_match_data(&pdev->dev); if (!atxdmac->layout) @@ -2133,11 +2360,20 @@ static int at_xdmac_probe(struct platform_device *pdev) atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; - /* Disable all chans and interrupts. */ - at_xdmac_off(atxdmac); + platform_set_drvdata(pdev, atxdmac); + + pm_runtime_set_autosuspend_delay(&pdev->dev, 500); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); /* Init channels. */ INIT_LIST_HEAD(&atxdmac->dma.channels); + + /* Disable all chans and interrupts. */ + at_xdmac_off(atxdmac, true); + for (i = 0; i < nr_channels; i++) { struct at_xdmac_chan *atchan = &atxdmac->chan[i]; @@ -2157,12 +2393,11 @@ static int at_xdmac_probe(struct platform_device *pdev) while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS)) cpu_relax(); } - platform_set_drvdata(pdev, atxdmac); ret = dma_async_device_register(&atxdmac->dma); if (ret) { dev_err(&pdev->dev, "fail to register DMA engine device\n"); - goto err_clk_disable; + goto err_pm_disable; } ret = of_dma_controller_register(pdev->dev.of_node, @@ -2177,10 +2412,18 @@ static int at_xdmac_probe(struct platform_device *pdev) at_xdmac_axi_config(pdev); + pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_put_autosuspend(&pdev->dev); + return 0; err_dma_unregister: dma_async_device_unregister(&atxdmac->dma); +err_pm_disable: + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); err_clk_disable: clk_disable_unprepare(atxdmac->clk); err_free_irq: @@ -2188,14 +2431,17 @@ err_free_irq: return ret; } -static int at_xdmac_remove(struct platform_device *pdev) +static void at_xdmac_remove(struct platform_device *pdev) { struct at_xdmac *atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev); int i; - at_xdmac_off(atxdmac); + at_xdmac_off(atxdmac, true); of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&atxdmac->dma); + pm_runtime_disable(atxdmac->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); clk_disable_unprepare(atxdmac->clk); free_irq(atxdmac->irq, atxdmac); @@ -2206,13 +2452,13 @@ static int at_xdmac_remove(struct platform_device *pdev) tasklet_kill(&atchan->tasklet); at_xdmac_free_chan_resources(&atchan->chan); } - - return 0; } -static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = { +static const struct dev_pm_ops __maybe_unused atmel_xdmac_dev_pm_ops = { .prepare = atmel_xdmac_prepare, SET_LATE_SYSTEM_SLEEP_PM_OPS(atmel_xdmac_suspend, atmel_xdmac_resume) + SET_RUNTIME_PM_OPS(atmel_xdmac_runtime_suspend, + atmel_xdmac_runtime_resume, NULL) }; static const struct of_device_id atmel_xdmac_dt_ids[] = { @@ -2234,16 +2480,22 @@ static struct platform_driver at_xdmac_driver = { .driver = { .name = "at_xdmac", .of_match_table = of_match_ptr(atmel_xdmac_dt_ids), - .pm = &atmel_xdmac_dev_pm_ops, + .pm = pm_ptr(&atmel_xdmac_dev_pm_ops), } }; static int __init at_xdmac_init(void) { - return platform_driver_probe(&at_xdmac_driver, at_xdmac_probe); + return platform_driver_register(&at_xdmac_driver); } subsys_initcall(at_xdmac_init); +static void __exit at_xdmac_exit(void) +{ + platform_driver_unregister(&at_xdmac_driver); +} +module_exit(at_xdmac_exit); + MODULE_DESCRIPTION("Atmel Extended DMA Controller driver"); MODULE_AUTHOR("Ludovic Desroches <ludovic.desroches@atmel.com>"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index 64239da02e74..7f0e76439ce5 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -1,15 +1,5 @@ -/* - * Copyright (C) 2017 Broadcom - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2017 Broadcom /* * Broadcom SBA RAID Driver @@ -25,7 +15,7 @@ * number of hardware rings over one or more SBA hardware devices. By * design, the internal buffer size of SBA hardware device is limited * but all offload operations supported by SBA can be broken down into - * multiple small size requests and executed parallely on multiple SBA + * multiple small size requests and executed parallelly on multiple SBA * hardware devices for achieving high through-put. * * The Broadcom SBA RAID driver does not require any register programming @@ -45,7 +35,9 @@ #include <linux/mailbox_client.h> #include <linux/mailbox/brcm-message.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/raid/pq.h> @@ -143,7 +135,7 @@ struct sba_device { u32 max_xor_srcs; u32 max_resp_pool_size; u32 max_cmds_pool_size; - /* Maibox client and Mailbox channels */ + /* Mailbox client and Mailbox channels */ struct mbox_client client; struct mbox_chan *mchan; struct device *mbox_dev; @@ -1742,7 +1734,7 @@ fail_free_mchan: return ret; } -static int sba_remove(struct platform_device *pdev) +static void sba_remove(struct platform_device *pdev) { struct sba_device *sba = platform_get_drvdata(pdev); @@ -1753,8 +1745,6 @@ static int sba_remove(struct platform_device *pdev) sba_freeup_channel_resources(sba); mbox_free_channel(sba->mchan); - - return 0; } static const struct of_device_id sba_of_match[] = { diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c index 630dfbb01a40..321748e2983e 100644 --- a/drivers/dma/bcm2835-dma.c +++ b/drivers/dma/bcm2835-dma.c @@ -369,7 +369,7 @@ static struct bcm2835_desc *bcm2835_dma_create_cb_chain( /* the last frame requires extra flags */ d->cb_list[d->frames - 1].cb->info |= finalextrainfo; - /* detect a size missmatch */ + /* detect a size mismatch */ if (buf_len && (d->size != buf_len)) goto error_cb; @@ -875,10 +875,30 @@ static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, return chan; } +static int bcm2835_dma_suspend_late(struct device *dev) +{ + struct bcm2835_dmadev *od = dev_get_drvdata(dev); + struct bcm2835_chan *c, *next; + + list_for_each_entry_safe(c, next, &od->ddev.channels, + vc.chan.device_node) { + void __iomem *chan_base = c->chan_base; + + /* Check if DMA channel is busy */ + if (readl(chan_base + BCM2835_DMA_ADDR)) + return -EBUSY; + } + + return 0; +} + +static const struct dev_pm_ops bcm2835_dma_pm_ops = { + LATE_SYSTEM_SLEEP_PM_OPS(bcm2835_dma_suspend_late, NULL) +}; + static int bcm2835_dma_probe(struct platform_device *pdev) { struct bcm2835_dmadev *od; - struct resource *res; void __iomem *base; int rc; int i, j; @@ -902,8 +922,7 @@ static int bcm2835_dma_probe(struct platform_device *pdev) dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - base = devm_ioremap_resource(&pdev->dev, res); + base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); @@ -1021,28 +1040,26 @@ err_no_dma: return rc; } -static int bcm2835_dma_remove(struct platform_device *pdev) +static void bcm2835_dma_remove(struct platform_device *pdev) { struct bcm2835_dmadev *od = platform_get_drvdata(pdev); dma_async_device_unregister(&od->ddev); bcm2835_dma_free(od); - - return 0; } static struct platform_driver bcm2835_dma_driver = { .probe = bcm2835_dma_probe, - .remove = bcm2835_dma_remove, + .remove = bcm2835_dma_remove, .driver = { .name = "bcm2835-dma", .of_match_table = of_match_ptr(bcm2835_dma_of_match), + .pm = pm_ptr(&bcm2835_dma_pm_ops), }, }; module_platform_driver(bcm2835_dma_driver); -MODULE_ALIAS("platform:bcm2835-dma"); MODULE_DESCRIPTION("BCM2835 DMA engine driver"); MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/bestcomm/ata.c b/drivers/dma/bestcomm/ata.c index 2fd87f83cf90..502a45d76adc 100644 --- a/drivers/dma/bestcomm/ata.c +++ b/drivers/dma/bestcomm/ata.c @@ -1,16 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Bestcomm ATA task driver * - * * Patterned after bestcomm/fec.c by Dale Farnsworth <dfarnsworth@mvista.com> * 2003-2004 (c) MontaVista, Software, Inc. * * Copyright (C) 2006-2007 Sylvain Munaut <tnt@246tNt.com> * Copyright (C) 2006 Freescale - John Rigby - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. */ #include <linux/kernel.h> @@ -133,7 +129,7 @@ void bcom_ata_reset_bd(struct bcom_task *tsk) struct bcom_ata_var *var; /* Reset all BD */ - memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); + memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); tsk->index = 0; tsk->outdex = 0; @@ -154,4 +150,3 @@ EXPORT_SYMBOL_GPL(bcom_ata_release); MODULE_DESCRIPTION("BestComm ATA task driver"); MODULE_AUTHOR("John Rigby"); MODULE_LICENSE("GPL v2"); - diff --git a/drivers/dma/bestcomm/bestcomm.c b/drivers/dma/bestcomm/bestcomm.c index d91cbbe7a48f..6c4d655ffe77 100644 --- a/drivers/dma/bestcomm/bestcomm.c +++ b/drivers/dma/bestcomm/bestcomm.c @@ -1,24 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Driver for MPC52xx processor BestComm peripheral controller * - * * Copyright (C) 2006-2007 Sylvain Munaut <tnt@246tNt.com> * Copyright (C) 2005 Varma Electronics Oy, * ( by Andrey Volkov <avolkov@varma-el.com> ) * Copyright (C) 2003-2004 MontaVista, Software, Inc. * ( by Dale Farnsworth <dfarnsworth@mvista.com> ) - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. */ #include <linux/module.h> #include <linux/kernel.h> #include <linux/slab.h> #include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> #include <asm/io.h> #include <asm/irq.h> #include <asm/mpc52xx.h> @@ -95,7 +92,7 @@ bcom_task_alloc(int bd_count, int bd_size, int priv_size) tsk->bd = bcom_sram_alloc(bd_count * bd_size, 4, &tsk->bd_pa); if (!tsk->bd) goto error; - memset(tsk->bd, 0x00, bd_count * bd_size); + memset_io(tsk->bd, 0x00, bd_count * bd_size); tsk->num_bd = bd_count; tsk->bd_size = bd_size; @@ -186,16 +183,16 @@ bcom_load_image(int task, u32 *task_image) inc = bcom_task_inc(task); /* Clear & copy */ - memset(var, 0x00, BCOM_VAR_SIZE); - memset(inc, 0x00, BCOM_INC_SIZE); + memset_io(var, 0x00, BCOM_VAR_SIZE); + memset_io(inc, 0x00, BCOM_INC_SIZE); desc_src = (u32 *)(hdr + 1); var_src = desc_src + hdr->desc_size; inc_src = var_src + hdr->var_size; - memcpy(desc, desc_src, hdr->desc_size * sizeof(u32)); - memcpy(var + hdr->first_var, var_src, hdr->var_size * sizeof(u32)); - memcpy(inc, inc_src, hdr->inc_size * sizeof(u32)); + memcpy_toio(desc, desc_src, hdr->desc_size * sizeof(u32)); + memcpy_toio(var + hdr->first_var, var_src, hdr->var_size * sizeof(u32)); + memcpy_toio(inc, inc_src, hdr->inc_size * sizeof(u32)); return 0; } @@ -302,13 +299,13 @@ static int bcom_engine_init(void) return -ENOMEM; } - memset(bcom_eng->tdt, 0x00, tdt_size); - memset(bcom_eng->ctx, 0x00, ctx_size); - memset(bcom_eng->var, 0x00, var_size); - memset(bcom_eng->fdt, 0x00, fdt_size); + memset_io(bcom_eng->tdt, 0x00, tdt_size); + memset_io(bcom_eng->ctx, 0x00, ctx_size); + memset_io(bcom_eng->var, 0x00, var_size); + memset_io(bcom_eng->fdt, 0x00, fdt_size); /* Copy the FDT for the EU#3 */ - memcpy(&bcom_eng->fdt[48], fdt_ops, sizeof(fdt_ops)); + memcpy_toio(&bcom_eng->fdt[48], fdt_ops, sizeof(fdt_ops)); /* Initialize Task base structure */ for (task=0; task<BCOM_MAX_TASKS; task++) @@ -458,7 +455,7 @@ error_ofput: } -static int mpc52xx_bcom_remove(struct platform_device *op) +static void mpc52xx_bcom_remove(struct platform_device *op) { /* Clean up the engine */ bcom_engine_cleanup(); @@ -476,8 +473,6 @@ static int mpc52xx_bcom_remove(struct platform_device *op) /* Release memory */ kfree(bcom_eng); bcom_eng = NULL; - - return 0; } static const struct of_device_id mpc52xx_bcom_of_match[] = { @@ -526,4 +521,3 @@ MODULE_AUTHOR("Sylvain Munaut <tnt@246tNt.com>"); MODULE_AUTHOR("Andrey Volkov <avolkov@varma-el.com>"); MODULE_AUTHOR("Dale Farnsworth <dfarnsworth@mvista.com>"); MODULE_LICENSE("GPL v2"); - diff --git a/drivers/dma/bestcomm/fec.c b/drivers/dma/bestcomm/fec.c index 7f1fb1c999e4..3a4a2f7910c6 100644 --- a/drivers/dma/bestcomm/fec.c +++ b/drivers/dma/bestcomm/fec.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Bestcomm FEC tasks driver * - * * Copyright (C) 2006-2007 Sylvain Munaut <tnt@246tNt.com> * Copyright (C) 2003-2004 MontaVista, Software, Inc. * ( by Dale Farnsworth <dfarnsworth@mvista.com> ) - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. */ #include <linux/kernel.h> @@ -140,7 +136,7 @@ bcom_fec_rx_reset(struct bcom_task *tsk) tsk->index = 0; tsk->outdex = 0; - memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); + memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); /* Configure some stuff */ bcom_set_task_pragma(tsk->tasknum, BCOM_FEC_RX_BD_PRAGMA); @@ -241,7 +237,7 @@ bcom_fec_tx_reset(struct bcom_task *tsk) tsk->index = 0; tsk->outdex = 0; - memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); + memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); /* Configure some stuff */ bcom_set_task_pragma(tsk->tasknum, BCOM_FEC_TX_BD_PRAGMA); @@ -267,4 +263,3 @@ EXPORT_SYMBOL_GPL(bcom_fec_tx_release); MODULE_DESCRIPTION("BestComm FEC tasks driver"); MODULE_AUTHOR("Dale Farnsworth <dfarnsworth@mvista.com>"); MODULE_LICENSE("GPL v2"); - diff --git a/drivers/dma/bestcomm/gen_bd.c b/drivers/dma/bestcomm/gen_bd.c index 906ddba6a6f5..8a24a5cbc263 100644 --- a/drivers/dma/bestcomm/gen_bd.c +++ b/drivers/dma/bestcomm/gen_bd.c @@ -142,7 +142,7 @@ bcom_gen_bd_rx_reset(struct bcom_task *tsk) tsk->index = 0; tsk->outdex = 0; - memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); + memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); /* Configure some stuff */ bcom_set_task_pragma(tsk->tasknum, BCOM_GEN_RX_BD_PRAGMA); @@ -226,7 +226,7 @@ bcom_gen_bd_tx_reset(struct bcom_task *tsk) tsk->index = 0; tsk->outdex = 0; - memset(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); + memset_io(tsk->bd, 0x00, tsk->num_bd * tsk->bd_size); /* Configure some stuff */ bcom_set_task_pragma(tsk->tasknum, BCOM_GEN_TX_BD_PRAGMA); diff --git a/drivers/dma/bestcomm/sram.c b/drivers/dma/bestcomm/sram.c index 2074e0e3fa21..ad74d57cc3ab 100644 --- a/drivers/dma/bestcomm/sram.c +++ b/drivers/dma/bestcomm/sram.c @@ -1,14 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Simple memory allocator for on-board SRAM * - * * Maintainer : Sylvain Munaut <tnt@246tNt.com> * * Copyright (C) 2005 Sylvain Munaut <tnt@246tNt.com> - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of any - * kind, whether express or implied. */ #include <linux/err.h> @@ -42,7 +38,7 @@ int bcom_sram_init(struct device_node *sram_node, char *owner) { int rv; const u32 *regaddr_p; - u64 regaddr64, size64; + struct resource res; unsigned int psize; /* Create our state struct */ @@ -60,21 +56,18 @@ int bcom_sram_init(struct device_node *sram_node, char *owner) } /* Get address and size of the sram */ - regaddr_p = of_get_address(sram_node, 0, &size64, NULL); - if (!regaddr_p) { + rv = of_address_to_resource(sram_node, 0, &res); + if (rv) { printk(KERN_ERR "%s: bcom_sram_init: " "Invalid device node !\n", owner); - rv = -EINVAL; goto error_free; } - regaddr64 = of_translate_address(sram_node, regaddr_p); - - bcom_sram->base_phys = (phys_addr_t) regaddr64; - bcom_sram->size = (unsigned int) size64; + bcom_sram->base_phys = res.start; + bcom_sram->size = resource_size(&res); /* Request region */ - if (!request_mem_region(bcom_sram->base_phys, bcom_sram->size, owner)) { + if (!request_mem_region(res.start, resource_size(&res), owner)) { printk(KERN_ERR "%s: bcom_sram_init: " "Couldn't request region !\n", owner); rv = -EBUSY; @@ -83,7 +76,7 @@ int bcom_sram_init(struct device_node *sram_node, char *owner) /* Map SRAM */ /* sram is not really __iomem */ - bcom_sram->base_virt = (void*) ioremap(bcom_sram->base_phys, bcom_sram->size); + bcom_sram->base_virt = (void *)ioremap(res.start, resource_size(&res)); if (!bcom_sram->base_virt) { printk(KERN_ERR "%s: bcom_sram_init: " @@ -97,13 +90,8 @@ int bcom_sram_init(struct device_node *sram_node, char *owner) bcom_sram->rh = rh_create(4); /* Attach the free zones */ -#if 0 - /* Currently disabled ... for future use only */ - reg_addr_p = of_get_property(sram_node, "available", &psize); -#else regaddr_p = NULL; psize = 0; -#endif if (!regaddr_p || !psize) { /* Attach the whole zone */ @@ -124,7 +112,7 @@ int bcom_sram_init(struct device_node *sram_node, char *owner) return 0; error_release: - release_mem_region(bcom_sram->base_phys, bcom_sram->size); + release_mem_region(res.start, resource_size(&res)); error_free: kfree(bcom_sram); bcom_sram = NULL; @@ -176,4 +164,3 @@ void bcom_sram_free(void *ptr) spin_unlock(&bcom_sram->lock); } EXPORT_SYMBOL_GPL(bcom_sram_free); - diff --git a/drivers/dma/cv1800b-dmamux.c b/drivers/dma/cv1800b-dmamux.c new file mode 100644 index 000000000000..e900d6595617 --- /dev/null +++ b/drivers/dma/cv1800b-dmamux.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025 Inochi Amaoto <inochiama@gmail.com> + */ + +#include <linux/bitops.h> +#include <linux/cleanup.h> +#include <linux/module.h> +#include <linux/of_dma.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/llist.h> +#include <linux/regmap.h> +#include <linux/spinlock.h> +#include <linux/mfd/syscon.h> + +#define REG_DMA_CHANNEL_REMAP0 0x154 +#define REG_DMA_CHANNEL_REMAP1 0x158 +#define REG_DMA_INT_MUX 0x298 + +#define DMAMUX_NCELLS 2 +#define MAX_DMA_MAPPING_ID 42 +#define MAX_DMA_CPU_ID 2 +#define MAX_DMA_CH_ID 7 + +#define DMAMUX_INTMUX_REGISTER_LEN 4 +#define DMAMUX_NR_CH_PER_REGISTER 4 +#define DMAMUX_BIT_PER_CH 8 +#define DMAMUX_CH_MASk GENMASK(5, 0) +#define DMAMUX_INT_BIT_PER_CPU 10 +#define DMAMUX_CH_UPDATE_BIT BIT(31) + +#define DMAMUX_CH_REGPOS(chid) \ + ((chid) / DMAMUX_NR_CH_PER_REGISTER) +#define DMAMUX_CH_REGOFF(chid) \ + ((chid) % DMAMUX_NR_CH_PER_REGISTER) +#define DMAMUX_CH_REG(chid) \ + ((DMAMUX_CH_REGPOS(chid) * sizeof(u32)) + \ + REG_DMA_CHANNEL_REMAP0) +#define DMAMUX_CH_SET(chid, val) \ + (((val) << (DMAMUX_CH_REGOFF(chid) * DMAMUX_BIT_PER_CH)) | \ + DMAMUX_CH_UPDATE_BIT) +#define DMAMUX_CH_MASK(chid) \ + DMAMUX_CH_SET(chid, DMAMUX_CH_MASk) + +#define DMAMUX_INT_BIT(chid, cpuid) \ + BIT((cpuid) * DMAMUX_INT_BIT_PER_CPU + (chid)) +#define DMAMUX_INTEN_BIT(cpuid) \ + DMAMUX_INT_BIT(8, cpuid) +#define DMAMUX_INT_CH_BIT(chid, cpuid) \ + (DMAMUX_INT_BIT(chid, cpuid) | DMAMUX_INTEN_BIT(cpuid)) +#define DMAMUX_INT_MASK(chid) \ + (DMAMUX_INT_BIT(chid, 0) | \ + DMAMUX_INT_BIT(chid, 1) | \ + DMAMUX_INT_BIT(chid, 2)) +#define DMAMUX_INT_CH_MASK(chid, cpuid) \ + (DMAMUX_INT_MASK(chid) | DMAMUX_INTEN_BIT(cpuid)) + +struct cv1800_dmamux_data { + struct dma_router dmarouter; + struct regmap *regmap; + spinlock_t lock; + struct llist_head free_maps; + struct llist_head reserve_maps; + DECLARE_BITMAP(mapped_peripherals, MAX_DMA_MAPPING_ID); +}; + +struct cv1800_dmamux_map { + struct llist_node node; + unsigned int channel; + unsigned int peripheral; + unsigned int cpu; +}; + +static void cv1800_dmamux_free(struct device *dev, void *route_data) +{ + struct cv1800_dmamux_data *dmamux = dev_get_drvdata(dev); + struct cv1800_dmamux_map *map = route_data; + + guard(spinlock_irqsave)(&dmamux->lock); + + regmap_update_bits(dmamux->regmap, + DMAMUX_CH_REG(map->channel), + DMAMUX_CH_MASK(map->channel), + DMAMUX_CH_UPDATE_BIT); + + regmap_update_bits(dmamux->regmap, REG_DMA_INT_MUX, + DMAMUX_INT_CH_MASK(map->channel, map->cpu), + DMAMUX_INTEN_BIT(map->cpu)); + + dev_dbg(dev, "free channel %u for req %u (cpu %u)\n", + map->channel, map->peripheral, map->cpu); +} + +static void *cv1800_dmamux_route_allocate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct cv1800_dmamux_data *dmamux = platform_get_drvdata(pdev); + struct cv1800_dmamux_map *map; + struct llist_node *node; + unsigned long flags; + unsigned int chid, devid, cpuid; + int ret; + + if (dma_spec->args_count != DMAMUX_NCELLS) { + dev_err(&pdev->dev, "invalid number of dma mux args\n"); + return ERR_PTR(-EINVAL); + } + + devid = dma_spec->args[0]; + cpuid = dma_spec->args[1]; + dma_spec->args_count = 1; + + if (devid > MAX_DMA_MAPPING_ID) { + dev_err(&pdev->dev, "invalid device id: %u\n", devid); + return ERR_PTR(-EINVAL); + } + + if (cpuid > MAX_DMA_CPU_ID) { + dev_err(&pdev->dev, "invalid cpu id: %u\n", cpuid); + return ERR_PTR(-EINVAL); + } + + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); + if (!dma_spec->np) { + dev_err(&pdev->dev, "can't get dma master\n"); + return ERR_PTR(-EINVAL); + } + + spin_lock_irqsave(&dmamux->lock, flags); + + if (test_bit(devid, dmamux->mapped_peripherals)) { + llist_for_each_entry(map, dmamux->reserve_maps.first, node) { + if (map->peripheral == devid && map->cpu == cpuid) + goto found; + } + + ret = -EINVAL; + goto failed; + } else { + node = llist_del_first(&dmamux->free_maps); + if (!node) { + ret = -ENODEV; + goto failed; + } + + map = llist_entry(node, struct cv1800_dmamux_map, node); + llist_add(&map->node, &dmamux->reserve_maps); + set_bit(devid, dmamux->mapped_peripherals); + } + +found: + chid = map->channel; + map->peripheral = devid; + map->cpu = cpuid; + + regmap_set_bits(dmamux->regmap, + DMAMUX_CH_REG(chid), + DMAMUX_CH_SET(chid, devid)); + + regmap_update_bits(dmamux->regmap, REG_DMA_INT_MUX, + DMAMUX_INT_CH_MASK(chid, cpuid), + DMAMUX_INT_CH_BIT(chid, cpuid)); + + spin_unlock_irqrestore(&dmamux->lock, flags); + + dma_spec->args[0] = chid; + + dev_dbg(&pdev->dev, "register channel %u for req %u (cpu %u)\n", + chid, devid, cpuid); + + return map; + +failed: + spin_unlock_irqrestore(&dmamux->lock, flags); + of_node_put(dma_spec->np); + dev_err(&pdev->dev, "errno %d\n", ret); + return ERR_PTR(ret); +} + +static int cv1800_dmamux_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *mux_node = dev->of_node; + struct cv1800_dmamux_data *data; + struct cv1800_dmamux_map *tmp; + struct device *parent = dev->parent; + struct regmap *regmap = NULL; + unsigned int i; + + if (!parent) + return -ENODEV; + + regmap = device_node_to_regmap(parent->of_node); + if (IS_ERR(regmap)) + return PTR_ERR(regmap); + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + spin_lock_init(&data->lock); + init_llist_head(&data->free_maps); + init_llist_head(&data->reserve_maps); + + for (i = 0; i <= MAX_DMA_CH_ID; i++) { + tmp = devm_kmalloc(dev, sizeof(*tmp), GFP_KERNEL); + if (!tmp) { + /* It is OK for not allocating all channel */ + dev_warn(dev, "can not allocate channel %u\n", i); + continue; + } + + init_llist_node(&tmp->node); + tmp->channel = i; + llist_add(&tmp->node, &data->free_maps); + } + + /* if no channel is allocated, the probe must fail */ + if (llist_empty(&data->free_maps)) + return -ENOMEM; + + data->regmap = regmap; + data->dmarouter.dev = dev; + data->dmarouter.route_free = cv1800_dmamux_free; + + platform_set_drvdata(pdev, data); + + return of_dma_router_register(mux_node, + cv1800_dmamux_route_allocate, + &data->dmarouter); +} + +static void cv1800_dmamux_remove(struct platform_device *pdev) +{ + of_dma_controller_free(pdev->dev.of_node); +} + +static const struct of_device_id cv1800_dmamux_ids[] = { + { .compatible = "sophgo,cv1800b-dmamux", }, + { } +}; +MODULE_DEVICE_TABLE(of, cv1800_dmamux_ids); + +static struct platform_driver cv1800_dmamux_driver = { + .probe = cv1800_dmamux_probe, + .remove = cv1800_dmamux_remove, + .driver = { + .name = "cv1800-dmamux", + .of_match_table = cv1800_dmamux_ids, + }, +}; +module_platform_driver(cv1800_dmamux_driver); + +MODULE_AUTHOR("Inochi Amaoto <inochiama@gmail.com>"); +MODULE_DESCRIPTION("Sophgo CV1800/SG2000 Series SoC DMAMUX driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/dma-axi-dmac.c b/drivers/dma/dma-axi-dmac.c index 5161b73c30c4..5b06b0dc67ee 100644 --- a/drivers/dma/dma-axi-dmac.c +++ b/drivers/dma/dma-axi-dmac.c @@ -6,6 +6,7 @@ * Author: Lars-Peter Clausen <lars@metafoo.de> */ +#include <linux/adi-axi-common.h> #include <linux/bitfield.h> #include <linux/clk.h> #include <linux/device.h> @@ -18,10 +19,10 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_address.h> #include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/slab.h> -#include <linux/fpga/adi-axi-common.h> #include <dt-bindings/dma/axi-dmac.h> @@ -55,6 +56,9 @@ #define AXI_DMAC_DMA_DST_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x) #define AXI_DMAC_DMA_DST_WIDTH_MSK GENMASK(3, 0) #define AXI_DMAC_DMA_DST_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x) +#define AXI_DMAC_REG_COHERENCY_DESC 0x14 +#define AXI_DMAC_DST_COHERENT_MSK BIT(0) +#define AXI_DMAC_DST_COHERENT_GET(x) FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x) #define AXI_DMAC_REG_IRQ_MASK 0x80 #define AXI_DMAC_REG_IRQ_PENDING 0x84 @@ -77,9 +81,13 @@ #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 +#define AXI_DMAC_REG_CURRENT_SG_ID 0x454 +#define AXI_DMAC_REG_SG_ADDRESS 0x47c +#define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc #define AXI_DMAC_CTRL_ENABLE BIT(0) #define AXI_DMAC_CTRL_PAUSE BIT(1) +#define AXI_DMAC_CTRL_ENABLE_SG BIT(2) #define AXI_DMAC_IRQ_SOT BIT(0) #define AXI_DMAC_IRQ_EOT BIT(1) @@ -93,27 +101,42 @@ /* The maximum ID allocated by the hardware is 31 */ #define AXI_DMAC_SG_UNUSED 32U +/* Flags for axi_dmac_hw_desc.flags */ +#define AXI_DMAC_HW_FLAG_LAST BIT(0) +#define AXI_DMAC_HW_FLAG_IRQ BIT(1) + +struct axi_dmac_hw_desc { + u32 flags; + u32 id; + u64 dest_addr; + u64 src_addr; + u64 next_sg_addr; + u32 y_len; + u32 x_len; + u32 src_stride; + u32 dst_stride; + u64 __pad[2]; +}; + struct axi_dmac_sg { - dma_addr_t src_addr; - dma_addr_t dest_addr; - unsigned int x_len; - unsigned int y_len; - unsigned int dest_stride; - unsigned int src_stride; - unsigned int id; unsigned int partial_len; bool schedule_when_free; + + struct axi_dmac_hw_desc *hw; + dma_addr_t hw_phys; }; struct axi_dmac_desc { struct virt_dma_desc vdesc; + struct axi_dmac_chan *chan; + bool cyclic; bool have_partial_xfer; unsigned int num_submitted; unsigned int num_completed; unsigned int num_sgs; - struct axi_dmac_sg sg[]; + struct axi_dmac_sg sg[] __counted_by(num_sgs); }; struct axi_dmac_chan { @@ -135,6 +158,7 @@ struct axi_dmac_chan { bool hw_partial_xfer; bool hw_cyclic; bool hw_2d; + bool hw_sg; }; struct axi_dmac { @@ -209,9 +233,11 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) unsigned int flags = 0; unsigned int val; - val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); - if (val) /* Queue is full, wait for the next SOT IRQ */ - return; + if (!chan->hw_sg) { + val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); + if (val) /* Queue is full, wait for the next SOT IRQ */ + return; + } desc = chan->next_desc; @@ -225,14 +251,15 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) sg = &desc->sg[desc->num_submitted]; /* Already queued in cyclic mode. Wait for it to finish */ - if (sg->id != AXI_DMAC_SG_UNUSED) { + if (sg->hw->id != AXI_DMAC_SG_UNUSED) { sg->schedule_when_free = true; return; } - desc->num_submitted++; - if (desc->num_submitted == desc->num_sgs || - desc->have_partial_xfer) { + if (chan->hw_sg) { + chan->next_desc = NULL; + } else if (++desc->num_submitted == desc->num_sgs || + desc->have_partial_xfer) { if (desc->cyclic) desc->num_submitted = 0; /* Start again */ else @@ -242,32 +269,42 @@ static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) chan->next_desc = desc; } - sg->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); + sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); - if (axi_dmac_dest_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->dest_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->dest_stride); - } + if (!chan->hw_sg) { + if (axi_dmac_dest_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); + } - if (axi_dmac_src_is_mem(chan)) { - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->src_addr); - axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->src_stride); + if (axi_dmac_src_is_mem(chan)) { + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); + axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); + } } /* * If the hardware supports cyclic transfers and there is no callback to - * call and only a single segment, enable hw cyclic mode to avoid - * unnecessary interrupts. + * call, enable hw cyclic mode to avoid unnecessary interrupts. */ - if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback && - desc->num_sgs == 1) - flags |= AXI_DMAC_FLAG_CYCLIC; + if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { + if (chan->hw_sg) + desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; + else if (desc->num_sgs == 1) + flags |= AXI_DMAC_FLAG_CYCLIC; + } if (chan->hw_partial_xfer) flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; - axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->x_len - 1); - axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->y_len - 1); + if (chan->hw_sg) { + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys); + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH, + (u64)sg->hw_phys >> 32); + } else { + axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); + } axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); } @@ -282,9 +319,9 @@ static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, struct axi_dmac_sg *sg) { if (chan->hw_2d) - return sg->x_len * sg->y_len; + return (sg->hw->x_len + 1) * (sg->hw->y_len + 1); else - return sg->x_len; + return (sg->hw->x_len + 1); } static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) @@ -303,9 +340,9 @@ static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) list_for_each_entry(desc, &chan->active_descs, vdesc.node) { for (i = 0; i < desc->num_sgs; i++) { sg = &desc->sg[i]; - if (sg->id == AXI_DMAC_SG_UNUSED) + if (sg->hw->id == AXI_DMAC_SG_UNUSED) continue; - if (sg->id == id) { + if (sg->hw->id == id) { desc->have_partial_xfer = true; sg->partial_len = len; found_sg = true; @@ -344,6 +381,9 @@ static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, rslt->result = DMA_TRANS_NOERROR; rslt->residue = 0; + if (chan->hw_sg) + return; + /* * We get here if the last completed segment is partial, which * means we can compute the residue from that segment onwards @@ -370,36 +410,47 @@ static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) axi_dmac_dequeue_partial_xfers(chan); - do { - sg = &active->sg[active->num_completed]; - if (sg->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ - break; - if (!(BIT(sg->id) & completed_transfers)) - break; - active->num_completed++; - sg->id = AXI_DMAC_SG_UNUSED; - if (sg->schedule_when_free) { - sg->schedule_when_free = false; - start_next = true; + if (chan->hw_sg) { + if (active->cyclic) { + vchan_cyclic_callback(&active->vdesc); + } else { + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + active = axi_dmac_active_desc(chan); + start_next = !!active; } + } else { + do { + sg = &active->sg[active->num_completed]; + if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ + break; + if (!(BIT(sg->hw->id) & completed_transfers)) + break; + active->num_completed++; + sg->hw->id = AXI_DMAC_SG_UNUSED; + if (sg->schedule_when_free) { + sg->schedule_when_free = false; + start_next = true; + } - if (sg->partial_len) - axi_dmac_compute_residue(chan, active); + if (sg->partial_len) + axi_dmac_compute_residue(chan, active); - if (active->cyclic) - vchan_cyclic_callback(&active->vdesc); + if (active->cyclic) + vchan_cyclic_callback(&active->vdesc); - if (active->num_completed == active->num_sgs || - sg->partial_len) { - if (active->cyclic) { - active->num_completed = 0; /* wrap around */ - } else { - list_del(&active->vdesc.node); - vchan_cookie_complete(&active->vdesc); - active = axi_dmac_active_desc(chan); + if (active->num_completed == active->num_sgs || + sg->partial_len) { + if (active->cyclic) { + active->num_completed = 0; /* wrap around */ + } else { + list_del(&active->vdesc.node); + vchan_cookie_complete(&active->vdesc); + active = axi_dmac_active_desc(chan); + } } - } - } while (active); + } while (active); + } return start_next; } @@ -463,8 +514,12 @@ static void axi_dmac_issue_pending(struct dma_chan *c) struct axi_dmac_chan *chan = to_axi_dmac_chan(c); struct axi_dmac *dmac = chan_to_axi_dmac(chan); unsigned long flags; + u32 ctrl = AXI_DMAC_CTRL_ENABLE; + + if (chan->hw_sg) + ctrl |= AXI_DMAC_CTRL_ENABLE_SG; - axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); + axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl); spin_lock_irqsave(&chan->vchan.lock, flags); if (vchan_issue_pending(&chan->vchan)) @@ -472,23 +527,58 @@ static void axi_dmac_issue_pending(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); } -static struct axi_dmac_desc *axi_dmac_alloc_desc(unsigned int num_sgs) +static struct axi_dmac_desc * +axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) { + struct axi_dmac *dmac = chan_to_axi_dmac(chan); + struct device *dev = dmac->dma_dev.dev; + struct axi_dmac_hw_desc *hws; struct axi_dmac_desc *desc; + dma_addr_t hw_phys; unsigned int i; desc = kzalloc(struct_size(desc, sg, num_sgs), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = num_sgs; + desc->chan = chan; - for (i = 0; i < num_sgs; i++) - desc->sg[i].id = AXI_DMAC_SG_UNUSED; + hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)), + &hw_phys, GFP_ATOMIC); + if (!hws) { + kfree(desc); + return NULL; + } - desc->num_sgs = num_sgs; + for (i = 0; i < num_sgs; i++) { + desc->sg[i].hw = &hws[i]; + desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws); + + hws[i].id = AXI_DMAC_SG_UNUSED; + hws[i].flags = 0; + + /* Link hardware descriptors */ + hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws); + } + + /* The last hardware descriptor will trigger an interrupt */ + desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ; return desc; } +static void axi_dmac_free_desc(struct axi_dmac_desc *desc) +{ + struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan); + struct device *dev = dmac->dma_dev.dev; + struct axi_dmac_hw_desc *hw = desc->sg[0].hw; + dma_addr_t hw_phys = desc->sg[0].hw_phys; + + dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)), + hw, hw_phys); + kfree(desc); +} + static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, enum dma_transfer_direction direction, dma_addr_t addr, unsigned int num_periods, unsigned int period_len, @@ -505,26 +595,24 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; for (i = 0; i < num_periods; i++) { - len = period_len; - - while (len > segment_size) { + for (len = period_len; len > segment_size; sg++) { if (direction == DMA_DEV_TO_MEM) - sg->dest_addr = addr; + sg->hw->dest_addr = addr; else - sg->src_addr = addr; - sg->x_len = segment_size; - sg->y_len = 1; - sg++; + sg->hw->src_addr = addr; + sg->hw->x_len = segment_size - 1; + sg->hw->y_len = 0; + sg->hw->flags = 0; addr += segment_size; len -= segment_size; } if (direction == DMA_DEV_TO_MEM) - sg->dest_addr = addr; + sg->hw->dest_addr = addr; else - sg->src_addr = addr; - sg->x_len = len; - sg->y_len = 1; + sg->hw->src_addr = addr; + sg->hw->x_len = len - 1; + sg->hw->y_len = 0; sg++; addr += len; } @@ -532,6 +620,45 @@ static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, return sg; } +static struct dma_async_tx_descriptor * +axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, + size_t nb, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct axi_dmac_chan *chan = to_axi_dmac_chan(c); + struct axi_dmac_desc *desc; + unsigned int num_sgs = 0; + struct axi_dmac_sg *dsg; + size_t i; + + if (direction != chan->direction) + return NULL; + + for (i = 0; i < nb; i++) + num_sgs += DIV_ROUND_UP(vecs[i].len, chan->max_length); + + desc = axi_dmac_alloc_desc(chan, num_sgs); + if (!desc) + return NULL; + + dsg = desc->sg; + + for (i = 0; i < nb; i++) { + if (!axi_dmac_check_addr(chan, vecs[i].addr) || + !axi_dmac_check_len(chan, vecs[i].len)) { + kfree(desc); + return NULL; + } + + dsg = axi_dmac_fill_linear_sg(chan, direction, vecs[i].addr, 1, + vecs[i].len, dsg); + } + + desc->cyclic = false; + + return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); +} + static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( struct dma_chan *c, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, @@ -551,7 +678,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) num_sgs += DIV_ROUND_UP(sg_dma_len(sg), chan->max_length); - desc = axi_dmac_alloc_desc(num_sgs); + desc = axi_dmac_alloc_desc(chan, num_sgs); if (!desc) return NULL; @@ -560,7 +687,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) { if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || !axi_dmac_check_len(chan, sg_dma_len(sg))) { - kfree(desc); + axi_dmac_free_desc(desc); return NULL; } @@ -580,7 +707,7 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( { struct axi_dmac_chan *chan = to_axi_dmac_chan(c); struct axi_dmac_desc *desc; - unsigned int num_periods, num_segments; + unsigned int num_periods, num_segments, num_sgs; if (direction != chan->direction) return NULL; @@ -594,11 +721,16 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( num_periods = buf_len / period_len; num_segments = DIV_ROUND_UP(period_len, chan->max_length); + num_sgs = num_periods * num_segments; - desc = axi_dmac_alloc_desc(num_periods * num_segments); + desc = axi_dmac_alloc_desc(chan, num_sgs); if (!desc) return NULL; + /* Chain the last descriptor to the first, and remove its "last" flag */ + desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; + desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; + axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods, period_len, desc->sg); @@ -650,26 +782,26 @@ static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( return NULL; } - desc = axi_dmac_alloc_desc(1); + desc = axi_dmac_alloc_desc(chan, 1); if (!desc) return NULL; if (axi_dmac_src_is_mem(chan)) { - desc->sg[0].src_addr = xt->src_start; - desc->sg[0].src_stride = xt->sgl[0].size + src_icg; + desc->sg[0].hw->src_addr = xt->src_start; + desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg; } if (axi_dmac_dest_is_mem(chan)) { - desc->sg[0].dest_addr = xt->dst_start; - desc->sg[0].dest_stride = xt->sgl[0].size + dst_icg; + desc->sg[0].hw->dest_addr = xt->dst_start; + desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg; } if (chan->hw_2d) { - desc->sg[0].x_len = xt->sgl[0].size; - desc->sg[0].y_len = xt->numf; + desc->sg[0].hw->x_len = xt->sgl[0].size - 1; + desc->sg[0].hw->y_len = xt->numf - 1; } else { - desc->sg[0].x_len = xt->sgl[0].size * xt->numf; - desc->sg[0].y_len = 1; + desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1; + desc->sg[0].hw->y_len = 0; } if (flags & DMA_CYCLIC) @@ -685,7 +817,7 @@ static void axi_dmac_free_chan_resources(struct dma_chan *c) static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) { - kfree(container_of(vdesc, struct axi_dmac_desc, vdesc)); + axi_dmac_free_desc(to_axi_dmac_desc(vdesc)); } static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) @@ -711,6 +843,9 @@ static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) case AXI_DMAC_REG_CURRENT_DEST_ADDR: case AXI_DMAC_REG_PARTIAL_XFER_LEN: case AXI_DMAC_REG_PARTIAL_XFER_ID: + case AXI_DMAC_REG_CURRENT_SG_ID: + case AXI_DMAC_REG_SG_ADDRESS: + case AXI_DMAC_REG_SG_ADDRESS_HIGH: return true; default: return false; @@ -863,6 +998,10 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) chan->hw_cyclic = true; + axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff); + if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS)) + chan->hw_sg = true; + axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) chan->hw_2d = true; @@ -902,13 +1041,23 @@ static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) return 0; } +static void axi_dmac_tasklet_kill(void *task) +{ + tasklet_kill(task); +} + +static void axi_dmac_free_dma_controller(void *of_node) +{ + of_dma_controller_free(of_node); +} + static int axi_dmac_probe(struct platform_device *pdev) { struct dma_device *dma_dev; struct axi_dmac *dmac; - struct resource *res; struct regmap *regmap; unsigned int version; + u32 irq_mask = 0; int ret; dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); @@ -921,19 +1070,14 @@ static int axi_dmac_probe(struct platform_device *pdev) if (dmac->irq == 0) return -EINVAL; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dmac->base = devm_ioremap_resource(&pdev->dev, res); + dmac->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dmac->base)) return PTR_ERR(dmac->base); - dmac->clk = devm_clk_get(&pdev->dev, NULL); + dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL); if (IS_ERR(dmac->clk)) return PTR_ERR(dmac->clk); - ret = clk_prepare_enable(dmac->clk); - if (ret < 0) - return ret; - version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); if (version >= ADI_AXI_PCORE_VER(4, 3, 'a')) @@ -942,7 +1086,7 @@ static int axi_dmac_probe(struct platform_device *pdev) ret = axi_dmac_parse_dt(&pdev->dev, dmac); if (ret < 0) - goto err_clk_disable; + return ret; INIT_LIST_HEAD(&dmac->chan.active_descs); @@ -956,16 +1100,17 @@ static int axi_dmac_probe(struct platform_device *pdev) dma_dev->device_tx_status = dma_cookie_status; dma_dev->device_issue_pending = axi_dmac_issue_pending; dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg; + dma_dev->device_prep_peripheral_dma_vec = axi_dmac_prep_peripheral_dma_vec; dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic; dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved; dma_dev->device_terminate_all = axi_dmac_terminate_all; dma_dev->device_synchronize = axi_dmac_synchronize; dma_dev->dev = &pdev->dev; - dma_dev->chancnt = 1; dma_dev->src_addr_widths = BIT(dmac->chan.src_width); dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); dma_dev->directions = BIT(dmac->chan.direction); dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */ INIT_LIST_HEAD(&dma_dev->channels); dmac->chan.vchan.desc_free = axi_dmac_desc_free; @@ -973,60 +1118,58 @@ static int axi_dmac_probe(struct platform_device *pdev) ret = axi_dmac_detect_caps(dmac, version); if (ret) - goto err_clk_disable; + return ret; dma_dev->copy_align = (dmac->chan.address_align_mask + 1); - axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, 0x00); + if (dmac->chan.hw_sg) + irq_mask |= AXI_DMAC_IRQ_SOT; - ret = dma_async_device_register(dma_dev); + axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask); + + if (of_dma_is_coherent(pdev->dev.of_node)) { + ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC); + + if (version < ADI_AXI_PCORE_VER(4, 4, 'a') || + !AXI_DMAC_DST_COHERENT_GET(ret)) { + dev_err(dmac->dma_dev.dev, + "Coherent DMA not supported in hardware"); + return -EINVAL; + } + } + + ret = dmaenginem_async_device_register(dma_dev); if (ret) - goto err_clk_disable; + return ret; + + /* + * Put the action in here so it get's done before unregistering the DMA + * device. + */ + ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill, + &dmac->chan.vchan.task); + if (ret) + return ret; ret = of_dma_controller_register(pdev->dev.of_node, of_dma_xlate_by_chan_id, dma_dev); if (ret) - goto err_unregister_device; + return ret; - ret = request_irq(dmac->irq, axi_dmac_interrupt_handler, IRQF_SHARED, - dev_name(&pdev->dev), dmac); + ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller, + pdev->dev.of_node); if (ret) - goto err_unregister_of; + return ret; - platform_set_drvdata(pdev, dmac); + ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler, + IRQF_SHARED, dev_name(&pdev->dev), dmac); + if (ret) + return ret; regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base, &axi_dmac_regmap_config); - if (IS_ERR(regmap)) { - ret = PTR_ERR(regmap); - goto err_free_irq; - } - - return 0; - -err_free_irq: - free_irq(dmac->irq, dmac); -err_unregister_of: - of_dma_controller_free(pdev->dev.of_node); -err_unregister_device: - dma_async_device_unregister(&dmac->dma_dev); -err_clk_disable: - clk_disable_unprepare(dmac->clk); - return ret; -} - -static int axi_dmac_remove(struct platform_device *pdev) -{ - struct axi_dmac *dmac = platform_get_drvdata(pdev); - - of_dma_controller_free(pdev->dev.of_node); - free_irq(dmac->irq, dmac); - tasklet_kill(&dmac->chan.vchan.task); - dma_async_device_unregister(&dmac->dma_dev); - clk_disable_unprepare(dmac->clk); - - return 0; + return PTR_ERR_OR_ZERO(regmap); } static const struct of_device_id axi_dmac_of_match_table[] = { @@ -1041,7 +1184,6 @@ static struct platform_driver axi_dmac_driver = { .of_match_table = axi_dmac_of_match_table, }, .probe = axi_dmac_probe, - .remove = axi_dmac_remove, }; module_platform_driver(axi_dmac_driver); diff --git a/drivers/dma/dma-jz4780.c b/drivers/dma/dma-jz4780.c index ebee94dbd630..100057603fd4 100644 --- a/drivers/dma/dma-jz4780.c +++ b/drivers/dma/dma-jz4780.c @@ -8,11 +8,11 @@ #include <linux/clk.h> #include <linux/dmapool.h> +#include <linux/dma-mapping.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -104,10 +104,10 @@ * descriptor base address in the upper 8 bits. */ struct jz4780_dma_hwdesc { - uint32_t dcm; - uint32_t dsa; - uint32_t dta; - uint32_t dtc; + u32 dcm; + u32 dsa; + u32 dta; + u32 dtc; }; /* Size of allocations for hardware descriptor blocks. */ @@ -122,7 +122,8 @@ struct jz4780_dma_desc { dma_addr_t desc_phys; unsigned int count; enum dma_transaction_type type; - uint32_t status; + u32 transfer_type; + u32 status; }; struct jz4780_dma_chan { @@ -130,8 +131,8 @@ struct jz4780_dma_chan { unsigned int id; struct dma_pool *desc_pool; - uint32_t transfer_type; - uint32_t transfer_shift; + u32 transfer_type_tx, transfer_type_rx; + u32 transfer_shift; struct dma_slave_config config; struct jz4780_dma_desc *desc; @@ -152,12 +153,12 @@ struct jz4780_dma_dev { unsigned int irq; const struct jz4780_dma_soc_data *soc_data; - uint32_t chan_reserved; + u32 chan_reserved; struct jz4780_dma_chan chan[]; }; struct jz4780_dma_filter_data { - uint32_t transfer_type; + u32 transfer_type_tx, transfer_type_rx; int channel; }; @@ -179,26 +180,26 @@ static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( dma_device); } -static inline uint32_t jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma, +static inline u32 jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma, unsigned int chn, unsigned int reg) { return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); } static inline void jz4780_dma_chn_writel(struct jz4780_dma_dev *jzdma, - unsigned int chn, unsigned int reg, uint32_t val) + unsigned int chn, unsigned int reg, u32 val) { writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); } -static inline uint32_t jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma, +static inline u32 jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma, unsigned int reg) { return readl(jzdma->ctrl_base + reg); } static inline void jz4780_dma_ctrl_writel(struct jz4780_dma_dev *jzdma, - unsigned int reg, uint32_t val) + unsigned int reg, u32 val) { writel(val, jzdma->ctrl_base + reg); } @@ -226,9 +227,10 @@ static inline void jz4780_dma_chan_disable(struct jz4780_dma_dev *jzdma, jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DCKEC, BIT(chn)); } -static struct jz4780_dma_desc *jz4780_dma_desc_alloc( - struct jz4780_dma_chan *jzchan, unsigned int count, - enum dma_transaction_type type) +static struct jz4780_dma_desc * +jz4780_dma_desc_alloc(struct jz4780_dma_chan *jzchan, unsigned int count, + enum dma_transaction_type type, + enum dma_transfer_direction direction) { struct jz4780_dma_desc *desc; @@ -248,6 +250,12 @@ static struct jz4780_dma_desc *jz4780_dma_desc_alloc( desc->count = count; desc->type = type; + + if (direction == DMA_DEV_TO_MEM) + desc->transfer_type = jzchan->transfer_type_rx; + else + desc->transfer_type = jzchan->transfer_type_tx; + return desc; } @@ -260,8 +268,8 @@ static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) kfree(desc); } -static uint32_t jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan, - unsigned long val, uint32_t *shift) +static u32 jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan, + unsigned long val, u32 *shift) { struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); int ord = ffs(val) - 1; @@ -303,7 +311,7 @@ static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, enum dma_transfer_direction direction) { struct dma_slave_config *config = &jzchan->config; - uint32_t width, maxburst, tsz; + u32 width, maxburst, tsz; if (direction == DMA_MEM_TO_DEV) { desc->dcm = JZ_DMA_DCM_SAI; @@ -361,7 +369,7 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( unsigned int i; int err; - desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE); + desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE, direction); if (!desc) return NULL; @@ -379,7 +387,7 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( if (i != (sg_len - 1) && !(jzdma->soc_data->flags & JZ_SOC_DATA_BREAK_LINKS)) { - /* Automatically proceeed to the next descriptor. */ + /* Automatically proceed to the next descriptor. */ desc->desc[i].dcm |= JZ_DMA_DCM_LINK; /* @@ -410,7 +418,7 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( periods = buf_len / period_len; - desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC); + desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC, direction); if (!desc) return NULL; @@ -453,16 +461,16 @@ static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( { struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); struct jz4780_dma_desc *desc; - uint32_t tsz; + u32 tsz; - desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); + desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY, 0); if (!desc) return NULL; tsz = jz4780_dma_transfer_size(jzchan, dest | src | len, &jzchan->transfer_shift); - jzchan->transfer_type = JZ_DMA_DRT_AUTO; + desc->transfer_type = JZ_DMA_DRT_AUTO; desc->desc[0].dsa = src; desc->desc[0].dta = dest; @@ -528,7 +536,7 @@ static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) /* Set transfer type. */ jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT, - jzchan->transfer_type); + jzchan->desc->transfer_type); /* * Set the transfer count. This is redundant for a descriptor-driven @@ -670,7 +678,7 @@ static bool jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, { const unsigned int soc_flags = jzdma->soc_data->flags; struct jz4780_dma_desc *desc = jzchan->desc; - uint32_t dcs; + u32 dcs; bool ack = true; spin_lock(&jzchan->vchan.lock); @@ -727,7 +735,7 @@ static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) struct jz4780_dma_dev *jzdma = data; unsigned int nb_channels = jzdma->soc_data->nb_channels; unsigned long pending; - uint32_t dmac; + u32 dmac; int i; pending = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DIRQP); @@ -788,7 +796,8 @@ static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) return false; } - jzchan->transfer_type = data->transfer_type; + jzchan->transfer_type_tx = data->transfer_type_tx; + jzchan->transfer_type_rx = data->transfer_type_rx; return true; } @@ -800,11 +809,17 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, dma_cap_mask_t mask = jzdma->dma_device.cap_mask; struct jz4780_dma_filter_data data; - if (dma_spec->args_count != 2) + if (dma_spec->args_count == 2) { + data.transfer_type_tx = dma_spec->args[0]; + data.transfer_type_rx = dma_spec->args[0]; + data.channel = dma_spec->args[1]; + } else if (dma_spec->args_count == 3) { + data.transfer_type_tx = dma_spec->args[0]; + data.transfer_type_rx = dma_spec->args[1]; + data.channel = dma_spec->args[2]; + } else { return NULL; - - data.transfer_type = dma_spec->args[0]; - data.channel = dma_spec->args[1]; + } if (data.channel > -1) { if (data.channel >= jzdma->soc_data->nb_channels) { @@ -822,7 +837,8 @@ static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, return NULL; } - jzdma->chan[data.channel].transfer_type = data.transfer_type; + jzdma->chan[data.channel].transfer_type_tx = data.transfer_type_tx; + jzdma->chan[data.channel].transfer_type_rx = data.transfer_type_rx; return dma_get_slave_channel( &jzdma->chan[data.channel].vchan.chan); @@ -895,6 +911,14 @@ static int jz4780_dma_probe(struct platform_device *pdev) dd = &jzdma->dma_device; + /* + * The real segment size limit is dependent on the size unit selected + * for the transfer. Because the size unit is selected automatically + * and may be as small as 1 byte, use a safe limit of 2^24-1 bytes to + * ensure the 24-bit transfer count in the descriptor cannot overflow. + */ + dma_set_max_seg_size(dev, 0xffffff); + dma_cap_set(DMA_MEMCPY, dd->cap_mask); dma_cap_set(DMA_SLAVE, dd->cap_mask); dma_cap_set(DMA_CYCLIC, dd->cap_mask); @@ -915,6 +939,7 @@ static int jz4780_dma_probe(struct platform_device *pdev) dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dd->max_sg_burst = JZ_DMA_MAX_DESC; /* * Enable DMA controller, mark all channels as not programmable. @@ -937,6 +962,14 @@ static int jz4780_dma_probe(struct platform_device *pdev) jzchan->vchan.desc_free = jz4780_dma_desc_free; } + /* + * On JZ4760, chan0 won't enable properly the first time. + * Enabling then disabling chan1 will magically make chan0 work + * correctly. + */ + jz4780_dma_chan_enable(jzdma, 1); + jz4780_dma_chan_disable(jzdma, 1); + ret = platform_get_irq(pdev, 0); if (ret < 0) goto err_disable_clk; @@ -975,7 +1008,7 @@ err_disable_clk: return ret; } -static int jz4780_dma_remove(struct platform_device *pdev) +static void jz4780_dma_remove(struct platform_device *pdev) { struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); int i; @@ -987,8 +1020,6 @@ static int jz4780_dma_remove(struct platform_device *pdev) for (i = 0; i < jzdma->soc_data->nb_channels; i++) tasklet_kill(&jzdma->chan[i].vchan.task); - - return 0; } static const struct jz4780_dma_soc_data jz4740_dma_soc_data = { @@ -1004,18 +1035,49 @@ static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { JZ_SOC_DATA_BREAK_LINKS, }; +static const struct jz4780_dma_soc_data jz4755_dma_soc_data = { + .nb_channels = 4, + .transfer_ord_max = 5, + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC | + JZ_SOC_DATA_BREAK_LINKS, +}; + static const struct jz4780_dma_soc_data jz4760_dma_soc_data = { .nb_channels = 5, .transfer_ord_max = 6, .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, }; +static const struct jz4780_dma_soc_data jz4760_mdma_soc_data = { + .nb_channels = 2, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, +}; + +static const struct jz4780_dma_soc_data jz4760_bdma_soc_data = { + .nb_channels = 3, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, +}; + static const struct jz4780_dma_soc_data jz4760b_dma_soc_data = { .nb_channels = 5, .transfer_ord_max = 6, .flags = JZ_SOC_DATA_PER_CHAN_PM, }; +static const struct jz4780_dma_soc_data jz4760b_mdma_soc_data = { + .nb_channels = 2, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM, +}; + +static const struct jz4780_dma_soc_data jz4760b_bdma_soc_data = { + .nb_channels = 3, + .transfer_ord_max = 6, + .flags = JZ_SOC_DATA_PER_CHAN_PM, +}; + static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { .nb_channels = 6, .transfer_ord_max = 6, @@ -1043,8 +1105,13 @@ static const struct jz4780_dma_soc_data x1830_dma_soc_data = { static const struct of_device_id jz4780_dma_dt_match[] = { { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, + { .compatible = "ingenic,jz4755-dma", .data = &jz4755_dma_soc_data }, { .compatible = "ingenic,jz4760-dma", .data = &jz4760_dma_soc_data }, + { .compatible = "ingenic,jz4760-mdma", .data = &jz4760_mdma_soc_data }, + { .compatible = "ingenic,jz4760-bdma", .data = &jz4760_bdma_soc_data }, { .compatible = "ingenic,jz4760b-dma", .data = &jz4760b_dma_soc_data }, + { .compatible = "ingenic,jz4760b-mdma", .data = &jz4760b_mdma_soc_data }, + { .compatible = "ingenic,jz4760b-bdma", .data = &jz4760b_bdma_soc_data }, { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data }, diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index af3ee288bc11..ca13cd39330b 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -40,6 +40,8 @@ #include <linux/dmaengine.h> #include <linux/hardirq.h> #include <linux/spinlock.h> +#include <linux/of.h> +#include <linux/property.h> #include <linux/percpu.h> #include <linux/rcupdate.h> #include <linux/mutex.h> @@ -172,7 +174,7 @@ static ssize_t memcpy_count_show(struct device *dev, if (chan) { for_each_possible_cpu(i) count += per_cpu_ptr(chan->local, i)->memcpy_count; - err = sprintf(buf, "%lu\n", count); + err = sysfs_emit(buf, "%lu\n", count); } else err = -ENODEV; mutex_unlock(&dma_list_mutex); @@ -194,7 +196,7 @@ static ssize_t bytes_transferred_show(struct device *dev, if (chan) { for_each_possible_cpu(i) count += per_cpu_ptr(chan->local, i)->bytes_transferred; - err = sprintf(buf, "%lu\n", count); + err = sysfs_emit(buf, "%lu\n", count); } else err = -ENODEV; mutex_unlock(&dma_list_mutex); @@ -212,7 +214,7 @@ static ssize_t in_use_show(struct device *dev, struct device_attribute *attr, mutex_lock(&dma_list_mutex); chan = dev_to_dma_chan(dev); if (chan) - err = sprintf(buf, "%d\n", chan->client_count); + err = sysfs_emit(buf, "%d\n", chan->client_count); else err = -ENODEV; mutex_unlock(&dma_list_mutex); @@ -451,7 +453,8 @@ static int dma_chan_get(struct dma_chan *chan) /* The channel is already in use, update client count */ if (chan->client_count) { __module_get(owner); - goto out; + chan->client_count++; + return 0; } if (!try_module_get(owner)) @@ -470,11 +473,11 @@ static int dma_chan_get(struct dma_chan *chan) goto err_out; } + chan->client_count++; + if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask)) balance_ref_count(chan); -out: - chan->client_count++; return 0; err_out: @@ -695,13 +698,12 @@ static struct dma_chan *find_candidate(struct dma_device *device, */ struct dma_chan *dma_get_slave_channel(struct dma_chan *chan) { - int err = -EBUSY; - /* lock against __dma_request_channel */ mutex_lock(&dma_list_mutex); if (chan->client_count == 0) { struct dma_device *device = chan->device; + int err; dma_cap_set(DMA_PRIVATE, device->cap_mask); device->privatecnt++; @@ -812,15 +814,13 @@ static const struct dma_slave_map *dma_filter_match(struct dma_device *device, */ struct dma_chan *dma_request_chan(struct device *dev, const char *name) { + struct fwnode_handle *fwnode = dev_fwnode(dev); struct dma_device *d, *_d; struct dma_chan *chan = NULL; - /* If device-tree is present get slave info from here */ - if (dev->of_node) - chan = of_dma_request_slave_channel(dev->of_node, name); - - /* If device was enumerated by ACPI get slave info from here */ - if (has_acpi_companion(dev) && !chan) + if (is_of_node(fwnode)) + chan = of_dma_request_slave_channel(to_of_node(fwnode), name); + else if (is_acpi_device_node(fwnode)) chan = acpi_dma_request_slave_chan_by_name(dev, name); if (PTR_ERR(chan) == -EPROBE_DEFER) @@ -854,8 +854,8 @@ struct dma_chan *dma_request_chan(struct device *dev, const char *name) found: #ifdef CONFIG_DEBUG_FS - chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), - name); + chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), name); + /* No functional issue if it fails, users are supposed to test before use */ #endif chan->name = kasprintf(GFP_KERNEL, "dma:%s", name); @@ -926,6 +926,36 @@ void dma_release_channel(struct dma_chan *chan) } EXPORT_SYMBOL_GPL(dma_release_channel); +static void dmaenginem_release_channel(void *chan) +{ + dma_release_channel(chan); +} + +/** + * devm_dma_request_chan - try to allocate an exclusive slave channel + * @dev: pointer to client device structure + * @name: slave channel name + * + * Returns pointer to appropriate DMA channel on success or an error pointer. + * + * The operation is managed and will be undone on driver detach. + */ + +struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name) +{ + struct dma_chan *chan = dma_request_chan(dev, name); + int ret = 0; + + if (!IS_ERR(chan)) + ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan); + + if (ret) + return ERR_PTR(ret); + + return chan; +} +EXPORT_SYMBOL_GPL(devm_dma_request_chan); + /** * dmaengine_get - register interest in dma_channels */ @@ -1037,7 +1067,8 @@ static int get_dma_id(struct dma_device *device) } static int __dma_async_device_channel_register(struct dma_device *device, - struct dma_chan *chan) + struct dma_chan *chan, + const char *name) { int rc; @@ -1054,9 +1085,7 @@ static int __dma_async_device_channel_register(struct dma_device *device, * When the chan_id is a negative value, we are dynamically adding * the channel. Otherwise we are static enumerating. */ - mutex_lock(&device->chan_mutex); chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL); - mutex_unlock(&device->chan_mutex); if (chan->chan_id < 0) { pr_err("%s: unable to alloc ida for chan: %d\n", __func__, chan->chan_id); @@ -1068,8 +1097,10 @@ static int __dma_async_device_channel_register(struct dma_device *device, chan->dev->device.parent = device->dev; chan->dev->chan = chan; chan->dev->dev_id = device->dev_id; - dev_set_name(&chan->dev->device, "dma%dchan%d", - device->dev_id, chan->chan_id); + if (!name) + dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id); + else + dev_set_name(&chan->dev->device, "%s", name); rc = device_register(&chan->dev->device); if (rc) goto err_out_ida; @@ -1079,9 +1110,7 @@ static int __dma_async_device_channel_register(struct dma_device *device, return 0; err_out_ida: - mutex_lock(&device->chan_mutex); ida_free(&device->chan_ida, chan->chan_id); - mutex_unlock(&device->chan_mutex); err_free_dev: kfree(chan->dev); err_free_local: @@ -1091,11 +1120,12 @@ static int __dma_async_device_channel_register(struct dma_device *device, } int dma_async_device_channel_register(struct dma_device *device, - struct dma_chan *chan) + struct dma_chan *chan, + const char *name) { int rc; - rc = __dma_async_device_channel_register(device, chan); + rc = __dma_async_device_channel_register(device, chan, name); if (rc < 0) return rc; @@ -1107,6 +1137,9 @@ EXPORT_SYMBOL_GPL(dma_async_device_channel_register); static void __dma_async_device_channel_unregister(struct dma_device *device, struct dma_chan *chan) { + if (chan->local == NULL) + return; + WARN_ONCE(!device->device_release && chan->client_count, "%s called while %d clients hold a reference\n", __func__, chan->client_count); @@ -1114,9 +1147,7 @@ static void __dma_async_device_channel_unregister(struct dma_device *device, device->chancnt--; chan->dev->chan = NULL; mutex_unlock(&dma_list_mutex); - mutex_lock(&device->chan_mutex); ida_free(&device->chan_ida, chan->chan_id); - mutex_unlock(&device->chan_mutex); device_unregister(&chan->dev->device); free_percpu(chan->local); } @@ -1153,69 +1184,27 @@ int dma_async_device_register(struct dma_device *device) device->owner = device->dev->driver->owner; - if (dma_has_cap(DMA_MEMCPY, device->cap_mask) && !device->device_prep_dma_memcpy) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_MEMCPY"); - return -EIO; - } - - if (dma_has_cap(DMA_XOR, device->cap_mask) && !device->device_prep_dma_xor) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_XOR"); - return -EIO; - } - - if (dma_has_cap(DMA_XOR_VAL, device->cap_mask) && !device->device_prep_dma_xor_val) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_XOR_VAL"); - return -EIO; - } - - if (dma_has_cap(DMA_PQ, device->cap_mask) && !device->device_prep_dma_pq) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_PQ"); - return -EIO; - } - - if (dma_has_cap(DMA_PQ_VAL, device->cap_mask) && !device->device_prep_dma_pq_val) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_PQ_VAL"); - return -EIO; - } - - if (dma_has_cap(DMA_MEMSET, device->cap_mask) && !device->device_prep_dma_memset) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_MEMSET"); - return -EIO; - } - - if (dma_has_cap(DMA_INTERRUPT, device->cap_mask) && !device->device_prep_dma_interrupt) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_INTERRUPT"); - return -EIO; - } - - if (dma_has_cap(DMA_CYCLIC, device->cap_mask) && !device->device_prep_dma_cyclic) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_CYCLIC"); - return -EIO; - } +#define CHECK_CAP(_name, _type) \ +{ \ + if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) { \ + dev_err(device->dev, \ + "Device claims capability %s, but op is not defined\n", \ + __stringify(_type)); \ + return -EIO; \ + } \ +} - if (dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && !device->device_prep_interleaved_dma) { - dev_err(device->dev, - "Device claims capability %s, but op is not defined\n", - "DMA_INTERLEAVE"); - return -EIO; - } + CHECK_CAP(dma_memcpy, DMA_MEMCPY); + CHECK_CAP(dma_xor, DMA_XOR); + CHECK_CAP(dma_xor_val, DMA_XOR_VAL); + CHECK_CAP(dma_pq, DMA_PQ); + CHECK_CAP(dma_pq_val, DMA_PQ_VAL); + CHECK_CAP(dma_memset, DMA_MEMSET); + CHECK_CAP(dma_interrupt, DMA_INTERRUPT); + CHECK_CAP(dma_cyclic, DMA_CYCLIC); + CHECK_CAP(interleaved_dma, DMA_INTERLEAVE); +#undef CHECK_CAP if (!device->device_tx_status) { dev_err(device->dev, "Device tx_status is not defined\n"); @@ -1244,12 +1233,11 @@ int dma_async_device_register(struct dma_device *device) if (rc != 0) return rc; - mutex_init(&device->chan_mutex); ida_init(&device->chan_ida); /* represent channels in sysfs. Probably want devs too */ list_for_each_entry(chan, &device->channels, device_node) { - rc = __dma_async_device_channel_register(device, chan); + rc = __dma_async_device_channel_register(device, chan, NULL); if (rc < 0) goto err_out; } @@ -1330,11 +1318,8 @@ void dma_async_device_unregister(struct dma_device *device) } EXPORT_SYMBOL(dma_async_device_unregister); -static void dmam_device_release(struct device *dev, void *res) +static void dmaenginem_async_device_unregister(void *device) { - struct dma_device *device; - - device = *(struct dma_device **)res; dma_async_device_unregister(device); } @@ -1346,22 +1331,13 @@ static void dmam_device_release(struct device *dev, void *res) */ int dmaenginem_async_device_register(struct dma_device *device) { - void *p; int ret; - p = devres_alloc(dmam_device_release, sizeof(void *), GFP_KERNEL); - if (!p) - return -ENOMEM; - ret = dma_async_device_register(device); - if (!ret) { - *(struct dma_device **)p = device; - devres_add(device->dev, p); - } else { - devres_free(p); - } + if (ret) + return ret; - return ret; + return devm_add_action_or_reset(device->dev, dmaenginem_async_device_unregister, device); } EXPORT_SYMBOL(dmaenginem_async_device_register); diff --git a/drivers/dma/dmaengine.h b/drivers/dma/dmaengine.h index 1bfbd64b1371..53f16d3f0029 100644 --- a/drivers/dma/dmaengine.h +++ b/drivers/dma/dmaengine.h @@ -176,7 +176,7 @@ dmaengine_desc_get_callback_invoke(struct dma_async_tx_descriptor *tx, static inline bool dmaengine_desc_callback_valid(struct dmaengine_desc_callback *cb) { - return (cb->callback) ? true : false; + return cb->callback || cb->callback_result; } struct dma_chan *dma_get_slave_channel(struct dma_chan *chan); diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index f696246f57fd..91b2fbc0b864 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -21,52 +21,55 @@ #include <linux/slab.h> #include <linux/wait.h> +static bool nobounce; +module_param(nobounce, bool, 0644); +MODULE_PARM_DESC(nobounce, "Prevent using swiotlb buffer (default: use swiotlb buffer)"); + static unsigned int test_buf_size = 16384; -module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); +module_param(test_buf_size, uint, 0644); MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); static char test_device[32]; -module_param_string(device, test_device, sizeof(test_device), - S_IRUGO | S_IWUSR); +module_param_string(device, test_device, sizeof(test_device), 0644); MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); static unsigned int threads_per_chan = 1; -module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR); +module_param(threads_per_chan, uint, 0644); MODULE_PARM_DESC(threads_per_chan, "Number of threads to start per channel (default: 1)"); static unsigned int max_channels; -module_param(max_channels, uint, S_IRUGO | S_IWUSR); +module_param(max_channels, uint, 0644); MODULE_PARM_DESC(max_channels, "Maximum number of channels to use (default: all)"); static unsigned int iterations; -module_param(iterations, uint, S_IRUGO | S_IWUSR); +module_param(iterations, uint, 0644); MODULE_PARM_DESC(iterations, "Iterations before stopping test (default: infinite)"); static unsigned int dmatest; -module_param(dmatest, uint, S_IRUGO | S_IWUSR); +module_param(dmatest, uint, 0644); MODULE_PARM_DESC(dmatest, "dmatest 0-memcpy 1-memset (default: 0)"); static unsigned int xor_sources = 3; -module_param(xor_sources, uint, S_IRUGO | S_IWUSR); +module_param(xor_sources, uint, 0644); MODULE_PARM_DESC(xor_sources, "Number of xor source buffers (default: 3)"); static unsigned int pq_sources = 3; -module_param(pq_sources, uint, S_IRUGO | S_IWUSR); +module_param(pq_sources, uint, 0644); MODULE_PARM_DESC(pq_sources, "Number of p+q source buffers (default: 3)"); static int timeout = 3000; -module_param(timeout, int, S_IRUGO | S_IWUSR); +module_param(timeout, int, 0644); MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " "Pass -1 for infinite timeout"); static bool noverify; -module_param(noverify, bool, S_IRUGO | S_IWUSR); +module_param(noverify, bool, 0644); MODULE_PARM_DESC(noverify, "Disable data verification (default: verify)"); static bool norandom; @@ -74,7 +77,7 @@ module_param(norandom, bool, 0644); MODULE_PARM_DESC(norandom, "Disable random offset setup (default: random)"); static bool verbose; -module_param(verbose, bool, S_IRUGO | S_IWUSR); +module_param(verbose, bool, 0644); MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)"); static int alignment = -1; @@ -86,11 +89,12 @@ module_param(transfer_size, uint, 0644); MODULE_PARM_DESC(transfer_size, "Optional custom transfer size in bytes (default: not used (0))"); static bool polled; -module_param(polled, bool, S_IRUGO | S_IWUSR); +module_param(polled, bool, 0644); MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts"); /** * struct dmatest_params - test parameters. + * @nobounce: prevent using swiotlb buffer * @buf_size: size of the memcpy test buffer * @channel: bus ID of the channel to test * @device: bus ID of the DMA Engine to test @@ -107,6 +111,7 @@ MODULE_PARM_DESC(polled, "Use polling for completion instead of interrupts"); * @polled: use polling for completion instead of interrupts */ struct dmatest_params { + bool nobounce; unsigned int buf_size; char channel[20]; char device[32]; @@ -154,7 +159,7 @@ static const struct kernel_param_ops run_ops = { .get = dmatest_run_get, }; static bool dmatest_run; -module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR); +module_param_cb(run, &run_ops, &dmatest_run, 0644); MODULE_PARM_DESC(run, "Run the test (default: false)"); static int dmatest_chan_set(const char *val, const struct kernel_param *kp); @@ -216,6 +221,7 @@ struct dmatest_done { struct dmatest_data { u8 **raw; u8 **aligned; + gfp_t gfp_flags; unsigned int cnt; unsigned int off; }; @@ -290,7 +296,7 @@ static const struct kernel_param_ops wait_ops = { .get = dmatest_wait_get, .set = param_set_bool, }; -module_param_cb(wait, &wait_ops, &wait, S_IRUGO); +module_param_cb(wait, &wait_ops, &wait, 0444); MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)"); static bool dmatest_match_channel(struct dmatest_params *params, @@ -313,7 +319,7 @@ static unsigned long dmatest_random(void) { unsigned long buf; - prandom_bytes(&buf, sizeof(buf)); + get_random_bytes(&buf, sizeof(buf)); return buf; } @@ -494,7 +500,7 @@ static unsigned long long dmatest_persec(s64 runtime, unsigned int val) per_sec *= val; per_sec = INT_TO_FIXPT(per_sec); - do_div(per_sec, runtime); + do_div(per_sec, (u32)runtime); return per_sec; } @@ -534,7 +540,7 @@ static int dmatest_alloc_test_data(struct dmatest_data *d, goto err; for (i = 0; i < d->cnt; i++) { - d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); + d->raw[i] = kmalloc(buf_size + align, d->gfp_flags); if (!d->raw[i]) goto err; @@ -579,10 +585,10 @@ static int dmatest_func(void *data) unsigned int total_tests = 0; dma_cookie_t cookie; enum dma_status status; - enum dma_ctrl_flags flags; + enum dma_ctrl_flags flags; u8 *pq_coefs = NULL; int ret; - unsigned int buf_size; + unsigned int buf_size; struct dmatest_data *src; struct dmatest_data *dst; int i; @@ -656,6 +662,13 @@ static int dmatest_func(void *data) goto err_free_coefs; } + src->gfp_flags = GFP_KERNEL; + dst->gfp_flags = GFP_KERNEL; + if (params->nobounce) { + src->gfp_flags = GFP_DMA; + dst->gfp_flags = GFP_DMA; + } + if (dmatest_alloc_test_data(src, buf_size, align) < 0) goto err_free_coefs; @@ -1094,9 +1107,10 @@ static void add_threaded_test(struct dmatest_info *info) struct dmatest_params *params = &info->params; /* Copy test parameters */ + params->nobounce = nobounce; params->buf_size = test_buf_size; - strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); - strlcpy(params->device, strim(test_device), sizeof(params->device)); + strscpy(params->channel, strim(test_channel), sizeof(params->channel)); + strscpy(params->device, strim(test_device), sizeof(params->device)); params->threads_per_chan = threads_per_chan; params->max_channels = max_channels; params->iterations = iterations; @@ -1240,7 +1254,7 @@ static int dmatest_chan_set(const char *val, const struct kernel_param *kp) dtc = list_last_entry(&info->channels, struct dmatest_chan, node); - strlcpy(chan_reset_val, + strscpy(chan_reset_val, dma_chan_name(dtc->chan), sizeof(chan_reset_val)); ret = -EBUSY; @@ -1263,14 +1277,14 @@ static int dmatest_chan_set(const char *val, const struct kernel_param *kp) if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0) && (strcmp("", strim(test_channel)) != 0)) { ret = -EINVAL; - strlcpy(chan_reset_val, dma_chan_name(dtc->chan), + strscpy(chan_reset_val, dma_chan_name(dtc->chan), sizeof(chan_reset_val)); goto add_chan_err; } } else { /* Clear test_channel if no channels were added successfully */ - strlcpy(chan_reset_val, "", sizeof(chan_reset_val)); + strscpy(chan_reset_val, "", sizeof(chan_reset_val)); ret = -EBUSY; goto add_chan_err; } @@ -1295,7 +1309,7 @@ static int dmatest_chan_get(char *val, const struct kernel_param *kp) mutex_lock(&info->lock); if (!is_threaded_test_run(info) && !is_threaded_test_pending(info)) { stop_threaded_test(info); - strlcpy(test_channel, "", sizeof(test_channel)); + strscpy(test_channel, "", sizeof(test_channel)); } mutex_unlock(&info->lock); @@ -1358,4 +1372,5 @@ static void __exit dmatest_exit(void) module_exit(dmatest_exit); MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); +MODULE_DESCRIPTION("DMA Engine test module"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index d9e4ac3edb4e..b23536645ff7 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0 // (C) 2017-2018 Synopsys, Inc. (www.synopsys.com) /* @@ -25,6 +25,7 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/property.h> +#include <linux/reset.h> #include <linux/slab.h> #include <linux/types.h> @@ -35,7 +36,7 @@ /* * The set of bus widths supported by the DMA controller. DW AXI DMAC supports * master data bus width up to 512 bits (for both AXI master interfaces), but - * it depends on IP block configurarion. + * it depends on IP block configuration. */ #define AXI_DMA_BUSWIDTHS \ (DMA_SLAVE_BUSWIDTH_1_BYTE | \ @@ -46,6 +47,10 @@ DMA_SLAVE_BUSWIDTH_32_BYTES | \ DMA_SLAVE_BUSWIDTH_64_BYTES) +#define AXI_DMA_FLAG_HAS_APB_REGS BIT(0) +#define AXI_DMA_FLAG_HAS_RESETS BIT(1) +#define AXI_DMA_FLAG_USE_CFG2 BIT(2) + static inline void axi_dma_iowrite32(struct axi_dma_chip *chip, u32 reg, u32 val) { @@ -58,6 +63,17 @@ static inline u32 axi_dma_ioread32(struct axi_dma_chip *chip, u32 reg) } static inline void +axi_dma_iowrite64(struct axi_dma_chip *chip, u32 reg, u64 val) +{ + iowrite64(val, chip->regs + reg); +} + +static inline u64 axi_dma_ioread64(struct axi_dma_chip *chip, u32 reg) +{ + return ioread64(chip->regs + reg); +} + +static inline void axi_chan_iowrite32(struct axi_dma_chan *chan, u32 reg, u32 val) { iowrite32(val, chan->chan_regs + reg); @@ -79,6 +95,33 @@ axi_chan_iowrite64(struct axi_dma_chan *chan, u32 reg, u64 val) iowrite32(upper_32_bits(val), chan->chan_regs + reg + 4); } +static inline void axi_chan_config_write(struct axi_dma_chan *chan, + struct axi_dma_chan_config *config) +{ + u32 cfg_lo, cfg_hi; + + cfg_lo = (config->dst_multblk_type << CH_CFG_L_DST_MULTBLK_TYPE_POS | + config->src_multblk_type << CH_CFG_L_SRC_MULTBLK_TYPE_POS); + if (chan->chip->dw->hdata->reg_map_8_channels && + !chan->chip->dw->hdata->use_cfg2) { + cfg_hi = config->tt_fc << CH_CFG_H_TT_FC_POS | + config->hs_sel_src << CH_CFG_H_HS_SEL_SRC_POS | + config->hs_sel_dst << CH_CFG_H_HS_SEL_DST_POS | + config->src_per << CH_CFG_H_SRC_PER_POS | + config->dst_per << CH_CFG_H_DST_PER_POS | + config->prior << CH_CFG_H_PRIORITY_POS; + } else { + cfg_lo |= config->src_per << CH_CFG2_L_SRC_PER_POS | + config->dst_per << CH_CFG2_L_DST_PER_POS; + cfg_hi = config->tt_fc << CH_CFG2_H_TT_FC_POS | + config->hs_sel_src << CH_CFG2_H_HS_SEL_SRC_POS | + config->hs_sel_dst << CH_CFG2_H_HS_SEL_DST_POS | + config->prior << CH_CFG2_H_PRIORITY_POS; + } + axi_chan_iowrite32(chan, CH_CFG_L, cfg_lo); + axi_chan_iowrite32(chan, CH_CFG_H, cfg_hi); +} + static inline void axi_dma_disable(struct axi_dma_chip *chip) { u32 val; @@ -150,41 +193,87 @@ static inline u32 axi_chan_irq_read(struct axi_dma_chan *chan) static inline void axi_chan_disable(struct axi_dma_chan *chan) { - u32 val; - - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT); - val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + u64 val; + + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHEN); + if (chan->id >= DMAC_CHAN_16) { + val &= ~((u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN_SHIFT + DMAC_CHAN_BLOCK_SHIFT)); + val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT); + } else { + val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT); + val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; + } + axi_dma_iowrite64(chan->chip, DMAC_CHEN, val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + val &= ~(BIT(chan->id) << DMAC_CHAN_EN_SHIFT); + if (chan->chip->dw->hdata->reg_map_8_channels) + val |= BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; + else + val |= BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } } static inline void axi_chan_enable(struct axi_dma_chan *chan) { - u32 val; - - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | - BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + u64 val; + + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHEN); + if (chan->id >= DMAC_CHAN_16) { + val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN_SHIFT + DMAC_CHAN_BLOCK_SHIFT) | + (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_EN2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT); + } else { + val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | + BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; + } + axi_dma_iowrite64(chan->chip, DMAC_CHEN, val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + if (chan->chip->dw->hdata->reg_map_8_channels) { + val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | + BIT(chan->id) << DMAC_CHAN_EN_WE_SHIFT; + } else { + val |= BIT(chan->id) << DMAC_CHAN_EN_SHIFT | + BIT(chan->id) << DMAC_CHAN_EN2_WE_SHIFT; + } + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } } static inline bool axi_chan_is_hw_enable(struct axi_dma_chan *chan) { - u32 val; + u64 val; - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) + val = axi_dma_ioread64(chan->chip, DMAC_CHEN); + else + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - return !!(val & (BIT(chan->id) << DMAC_CHAN_EN_SHIFT)); + if (chan->id >= DMAC_CHAN_16) + return !!(val & ((u64)(BIT(chan->id) >> DMAC_CHAN_16) << DMAC_CHAN_BLOCK_SHIFT)); + else + return !!(val & (BIT(chan->id) << DMAC_CHAN_EN_SHIFT)); } static void axi_dma_hw_init(struct axi_dma_chip *chip) { + int ret; u32 i; for (i = 0; i < chip->dw->hdata->nr_channels; i++) { axi_chan_irq_disable(&chip->dw->chan[i], DWAXIDMAC_IRQ_ALL); axi_chan_disable(&chip->dw->chan[i]); } + ret = dma_set_mask_and_coherent(chip->dev, DMA_BIT_MASK(64)); + if (ret) + dev_warn(chip->dev, "Unable to set coherent mask\n"); } static u32 axi_chan_get_xfer_width(struct axi_dma_chan *chan, dma_addr_t src, @@ -213,6 +302,7 @@ static struct axi_dma_desc *axi_desc_alloc(u32 num) kfree(desc); return NULL; } + desc->nr_hw_descs = num; return desc; } @@ -239,7 +329,7 @@ static struct axi_dma_lli *axi_desc_get(struct axi_dma_chan *chan, static void axi_desc_put(struct axi_dma_desc *desc) { struct axi_dma_chan *chan = desc->chan; - int count = atomic_read(&chan->descs_allocated); + int count = desc->nr_hw_descs; struct axi_dma_hw_desc *hw_desc; int descs_put; @@ -288,8 +378,6 @@ dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, len = vd_to_axi_desc(vdesc)->hw_desc[0].len; completed_length = completed_blocks * len; bytes = length - completed_length; - } else { - bytes = vd_to_axi_desc(vdesc)->length; } spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -336,7 +424,8 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, struct axi_dma_desc *first) { u32 priority = chan->chip->dw->hdata->priority[chan->id]; - u32 reg, irq_mask; + struct axi_dma_chan_config config = {}; + u32 irq_mask; u8 lms = 0; /* Select AXI0 master for LLI fetching */ if (unlikely(axi_chan_is_hw_enable(chan))) { @@ -348,32 +437,36 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, axi_dma_enable(chan->chip); - reg = (DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_DST_MULTBLK_TYPE_POS | - DWAXIDMAC_MBLK_TYPE_LL << CH_CFG_L_SRC_MULTBLK_TYPE_POS); - axi_chan_iowrite32(chan, CH_CFG_L, reg); - - reg = (DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC << CH_CFG_H_TT_FC_POS | - priority << CH_CFG_H_PRIORITY_POS | - DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_DST_POS | - DWAXIDMAC_HS_SEL_HW << CH_CFG_H_HS_SEL_SRC_POS); + config.dst_multblk_type = DWAXIDMAC_MBLK_TYPE_LL; + config.src_multblk_type = DWAXIDMAC_MBLK_TYPE_LL; + config.tt_fc = DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC; + config.prior = priority; + config.hs_sel_dst = DWAXIDMAC_HS_SEL_HW; + config.hs_sel_src = DWAXIDMAC_HS_SEL_HW; switch (chan->direction) { case DMA_MEM_TO_DEV: dw_axi_dma_set_byte_halfword(chan, true); - reg |= (chan->config.device_fc ? - DWAXIDMAC_TT_FC_MEM_TO_PER_DST : - DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC) - << CH_CFG_H_TT_FC_POS; + config.tt_fc = chan->config.device_fc ? + DWAXIDMAC_TT_FC_MEM_TO_PER_DST : + DWAXIDMAC_TT_FC_MEM_TO_PER_DMAC; + if (chan->chip->apb_regs) + config.dst_per = chan->id; + else + config.dst_per = chan->hw_handshake_num; break; case DMA_DEV_TO_MEM: - reg |= (chan->config.device_fc ? - DWAXIDMAC_TT_FC_PER_TO_MEM_SRC : - DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC) - << CH_CFG_H_TT_FC_POS; + config.tt_fc = chan->config.device_fc ? + DWAXIDMAC_TT_FC_PER_TO_MEM_SRC : + DWAXIDMAC_TT_FC_PER_TO_MEM_DMAC; + if (chan->chip->apb_regs) + config.src_per = chan->id; + else + config.src_per = chan->hw_handshake_num; break; default: break; } - axi_chan_iowrite32(chan, CH_CFG_H, reg); + axi_chan_config_write(chan, &config); write_chan_llp(chan, first->hw_desc[0].llp | lms); @@ -470,18 +563,13 @@ static void dma_chan_free_chan_resources(struct dma_chan *dchan) pm_runtime_put(chan->chip->dev); } -static void dw_axi_dma_set_hw_channel(struct axi_dma_chip *chip, - u32 handshake_num, bool set) +static void dw_axi_dma_set_hw_channel(struct axi_dma_chan *chan, bool set) { - unsigned long start = 0; - unsigned long reg_value; - unsigned long reg_mask; - unsigned long reg_set; - unsigned long mask; - unsigned long val; + struct axi_dma_chip *chip = chan->chip; + unsigned long reg_value, val; if (!chip->apb_regs) { - dev_dbg(chip->dev, "apb_regs not initialized\n"); + dev_err(chip->dev, "apb_regs not initialized\n"); return; } @@ -490,26 +578,22 @@ static void dw_axi_dma_set_hw_channel(struct axi_dma_chip *chip, * Lock the DMA channel by assign a handshake number to the channel. * Unlock the DMA channel by assign 0x3F to the channel. */ - if (set) { - reg_set = UNUSED_CHANNEL; - val = handshake_num; - } else { - reg_set = handshake_num; + if (set) + val = chan->hw_handshake_num; + else val = UNUSED_CHANNEL; - } reg_value = lo_hi_readq(chip->apb_regs + DMAC_APB_HW_HS_SEL_0); - for_each_set_clump8(start, reg_mask, ®_value, 64) { - if (reg_mask == reg_set) { - mask = GENMASK_ULL(start + 7, start); - reg_value &= ~mask; - reg_value |= rol64(val, start); - lo_hi_writeq(reg_value, - chip->apb_regs + DMAC_APB_HW_HS_SEL_0); - break; - } - } + /* Channel is already allocated, set handshake as per channel ID */ + /* 64 bit write should handle for 8 channels */ + + reg_value &= ~(DMA_APB_HS_SEL_MASK << + (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); + reg_value |= (val << (chan->id * DMA_APB_HS_SEL_BIT_SIZE)); + lo_hi_writeq(reg_value, chip->apb_regs + DMAC_APB_HW_HS_SEL_0); + + return; } /* @@ -742,7 +826,7 @@ dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr, llp = hw_desc->llp; } while (total_segments); - dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + dw_axi_dma_set_hw_channel(chan, true); return vchan_tx_prep(&chan->vc, &desc->vd, flags); @@ -822,7 +906,7 @@ dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, llp = hw_desc->llp; } while (num_sgs); - dw_axi_dma_set_hw_channel(chan->chip, chan->hw_handshake_num, true); + dw_axi_dma_set_hw_channel(chan, true); return vchan_tx_prep(&chan->vc, &desc->vd, flags); @@ -949,6 +1033,11 @@ static int dw_axi_dma_chan_slave_config(struct dma_chan *dchan, static void axi_chan_dump_lli(struct axi_dma_chan *chan, struct axi_dma_hw_desc *desc) { + if (!desc->lli) { + dev_err(dchan2dev(&chan->vc.chan), "NULL LLI\n"); + return; + } + dev_err(dchan2dev(&chan->vc.chan), "SAR: 0x%llx DAR: 0x%llx LLP: 0x%llx BTS 0x%x CTL: 0x%x:%08x", le64_to_cpu(desc->lli->sar), @@ -980,6 +1069,11 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) /* The bad descriptor currently is in the head of vc list */ vd = vchan_next_desc(&chan->vc); + if (!vd) { + dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n", + axi_chan_name(chan)); + goto out; + } /* Remove the completed descriptor from issued list */ list_del(&vd->node); @@ -994,6 +1088,7 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) /* Try to restart the controller */ axi_chan_start_first_queued(chan); +out: spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -1016,6 +1111,11 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* The completed descriptor currently is in the head of vc list */ vd = vchan_next_desc(&chan->vc); + if (!vd) { + dev_err(chan2dev(chan), "BUG: %s, IRQ with no descriptors\n", + axi_chan_name(chan)); + goto out; + } if (chan->cyclic) { desc = vd_to_axi_desc(vd); @@ -1040,11 +1140,9 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) /* Remove the completed descriptor from issued list before completing */ list_del(&vd->node); vchan_cookie_complete(vd); - - /* Submit queued descriptors after processing the completed ones */ - axi_chan_start_first_queued(chan); } +out: spin_unlock_irqrestore(&chan->vc.lock, flags); } @@ -1056,10 +1154,10 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) u32 status, i; - /* Disable DMAC inerrupts. We'll enable them after processing chanels */ + /* Disable DMAC interrupts. We'll enable them after processing channels */ axi_dma_irq_disable(chip); - /* Poll, clear and process every chanel interrupt status */ + /* Poll, clear and process every channel interrupt status */ for (i = 0; i < dw->hdata->nr_channels; i++) { chan = &dw->chan[i]; status = axi_chan_irq_read(chan); @@ -1092,14 +1190,13 @@ static int dma_chan_terminate_all(struct dma_chan *dchan) axi_chan_disable(chan); ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val, - !(val & chan_active), 1000, 10000); + !(val & chan_active), 1000, 50000); if (ret == -ETIMEDOUT) dev_warn(dchan2dev(dchan), "%s failed to stop\n", axi_chan_name(chan)); if (chan->direction != DMA_MEM_TO_MEM) - dw_axi_dma_set_hw_channel(chan->chip, - chan->hw_handshake_num, false); + dw_axi_dma_set_hw_channel(chan, false); if (chan->direction == DMA_MEM_TO_DEV) dw_axi_dma_set_byte_halfword(chan, false); @@ -1122,14 +1219,35 @@ static int dma_chan_pause(struct dma_chan *dchan) struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); unsigned long flags; unsigned int timeout = 20; /* timeout iterations */ - u32 val; + u64 val; spin_lock_irqsave(&chan->vc.lock, flags); - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT | - BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT; - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHSUSPREG); + if (chan->id >= DMAC_CHAN_16) { + val |= (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_SHIFT + DMAC_CHAN_BLOCK_SHIFT) | + (u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT); + } else { + val |= BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT | + BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT; + } + axi_dma_iowrite64(chan->chip, DMAC_CHSUSPREG, val); + } else { + if (chan->chip->dw->hdata->reg_map_8_channels) { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + val |= BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT | + BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT; + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG); + val |= BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT | + BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT; + axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, (u32)val); + } + } do { if (axi_chan_irq_read(chan) & DWAXIDMAC_IRQ_SUSPENDED) @@ -1150,12 +1268,33 @@ static int dma_chan_pause(struct dma_chan *dchan) /* Called in chan locked context */ static inline void axi_chan_resume(struct axi_dma_chan *chan) { - u32 val; - - val = axi_dma_ioread32(chan->chip, DMAC_CHEN); - val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT); - val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT); - axi_dma_iowrite32(chan->chip, DMAC_CHEN, val); + u64 val; + + if (chan->chip->dw->hdata->nr_channels >= DMAC_CHAN_16) { + val = axi_dma_ioread64(chan->chip, DMAC_CHSUSPREG); + if (chan->id >= DMAC_CHAN_16) { + val &= ~((u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_SHIFT + DMAC_CHAN_BLOCK_SHIFT)); + val |= ((u64)(BIT(chan->id) >> DMAC_CHAN_16) + << (DMAC_CHAN_SUSP2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT)); + } else { + val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT); + val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT); + } + axi_dma_iowrite64(chan->chip, DMAC_CHSUSPREG, val); + } else { + if (chan->chip->dw->hdata->reg_map_8_channels) { + val = axi_dma_ioread32(chan->chip, DMAC_CHEN); + val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP_SHIFT); + val |= (BIT(chan->id) << DMAC_CHAN_SUSP_WE_SHIFT); + axi_dma_iowrite32(chan->chip, DMAC_CHEN, (u32)val); + } else { + val = axi_dma_ioread32(chan->chip, DMAC_CHSUSPREG); + val &= ~(BIT(chan->id) << DMAC_CHAN_SUSP2_SHIFT); + val |= (BIT(chan->id) << DMAC_CHAN_SUSP2_WE_SHIFT); + axi_dma_iowrite32(chan->chip, DMAC_CHSUSPREG, (u32)val); + } + } chan->is_paused = false; } @@ -1247,6 +1386,8 @@ static int parse_device_properties(struct axi_dma_chip *chip) return -EINVAL; chip->dw->hdata->nr_channels = tmp; + if (tmp <= DMA_REG_MAP_CH_REF) + chip->dw->hdata->reg_map_8_channels = true; ret = device_property_read_u32(dev, "snps,dma-masters", &tmp); if (ret) @@ -1296,7 +1437,25 @@ static int parse_device_properties(struct axi_dma_chip *chip) return -EINVAL; chip->dw->hdata->restrict_axi_burst_len = true; - chip->dw->hdata->axi_rw_burst_len = tmp - 1; + chip->dw->hdata->axi_rw_burst_len = tmp; + } + + return 0; +} + +static int axi_req_irqs(struct platform_device *pdev, struct axi_dma_chip *chip) +{ + int irq_count = platform_irq_count(pdev); + int ret; + + for (int i = 0; i < irq_count; i++) { + chip->irq[i] = platform_get_irq(pdev, i); + if (chip->irq[i] < 0) + return chip->irq[i]; + ret = devm_request_irq(chip->dev, chip->irq[i], dw_axi_dma_interrupt, + IRQF_SHARED, KBUILD_MODNAME, chip); + if (ret < 0) + return ret; } return 0; @@ -1304,11 +1463,11 @@ static int parse_device_properties(struct axi_dma_chip *chip) static int dw_probe(struct platform_device *pdev) { - struct device_node *node = pdev->dev.of_node; struct axi_dma_chip *chip; - struct resource *mem; struct dw_axi_dma *dw; struct dw_axi_dma_hcfg *hdata; + struct reset_control *resets; + unsigned int flags; u32 i; int ret; @@ -1328,21 +1487,29 @@ static int dw_probe(struct platform_device *pdev) chip->dev = &pdev->dev; chip->dw->hdata = hdata; - chip->irq = platform_get_irq(pdev, 0); - if (chip->irq < 0) - return chip->irq; - - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - chip->regs = devm_ioremap_resource(chip->dev, mem); + chip->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); - if (of_device_is_compatible(node, "intel,kmb-axi-dma")) { + flags = (uintptr_t)of_device_get_match_data(&pdev->dev); + if (flags & AXI_DMA_FLAG_HAS_APB_REGS) { chip->apb_regs = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(chip->apb_regs)) return PTR_ERR(chip->apb_regs); } + if (flags & AXI_DMA_FLAG_HAS_RESETS) { + resets = devm_reset_control_array_get_exclusive(&pdev->dev); + if (IS_ERR(resets)) + return PTR_ERR(resets); + + ret = reset_control_deassert(resets); + if (ret) + return ret; + } + + chip->dw->hdata->use_cfg2 = !!(flags & AXI_DMA_FLAG_USE_CFG2); + chip->core_clk = devm_clk_get(chip->dev, "core-clk"); if (IS_ERR(chip->core_clk)) return PTR_ERR(chip->core_clk); @@ -1360,12 +1527,10 @@ static int dw_probe(struct platform_device *pdev) if (!dw->chan) return -ENOMEM; - ret = devm_request_irq(chip->dev, chip->irq, dw_axi_dma_interrupt, - IRQF_SHARED, KBUILD_MODNAME, chip); + ret = axi_req_irqs(pdev, chip); if (ret) return ret; - INIT_LIST_HEAD(&dw->dma.channels); for (i = 0; i < hdata->nr_channels; i++) { struct axi_dma_chan *chan = &dw->chan[i]; @@ -1385,7 +1550,7 @@ static int dw_probe(struct platform_device *pdev) dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask); /* DMA capabilities */ - dw->dma.chancnt = hdata->nr_channels; + dw->dma.max_burst = hdata->axi_rw_burst_len; dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.directions = BIT(DMA_MEM_TO_MEM); @@ -1455,7 +1620,7 @@ err_pm_disable: return ret; } -static int dw_remove(struct platform_device *pdev) +static void dw_remove(struct platform_device *pdev) { struct axi_dma_chip *chip = platform_get_drvdata(pdev); struct dw_axi_dma *dw = chip->dw; @@ -1475,7 +1640,9 @@ static int dw_remove(struct platform_device *pdev) pm_runtime_disable(chip->dev); axi_dma_suspend(chip); - devm_free_irq(chip->dev, chip->irq, chip); + for (i = 0; i < DMAC_MAX_CHANNELS; i++) + if (chip->irq[i] > 0) + devm_free_irq(chip->dev, chip->irq[i], chip); of_dma_controller_free(chip->dev->of_node); @@ -1484,8 +1651,6 @@ static int dw_remove(struct platform_device *pdev) list_del(&chan->vc.chan.device_node); tasklet_kill(&chan->vc.task); } - - return 0; } static const struct dev_pm_ops dw_axi_dma_pm_ops = { @@ -1493,8 +1658,18 @@ static const struct dev_pm_ops dw_axi_dma_pm_ops = { }; static const struct of_device_id dw_dma_of_id_table[] = { - { .compatible = "snps,axi-dma-1.01a" }, - { .compatible = "intel,kmb-axi-dma" }, + { + .compatible = "snps,axi-dma-1.01a" + }, { + .compatible = "intel,kmb-axi-dma", + .data = (void *)AXI_DMA_FLAG_HAS_APB_REGS, + }, { + .compatible = "starfive,jh7110-axi-dma", + .data = (void *)(AXI_DMA_FLAG_HAS_RESETS | AXI_DMA_FLAG_USE_CFG2), + }, { + .compatible = "starfive,jh8100-axi-dma", + .data = (void *)AXI_DMA_FLAG_HAS_RESETS, + }, {} }; MODULE_DEVICE_TABLE(of, dw_dma_of_id_table); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index b69897887c76..b842e6a8d90d 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ // (C) 2017-2018 Synopsys, Inc. (www.synopsys.com) /* @@ -18,7 +18,7 @@ #include "../virt-dma.h" -#define DMAC_MAX_CHANNELS 8 +#define DMAC_MAX_CHANNELS 32 #define DMAC_MAX_MASTERS 2 #define DMAC_MAX_BLK_SIZE 0x200000 @@ -30,7 +30,10 @@ struct dw_axi_dma_hcfg { u32 priority[DMAC_MAX_CHANNELS]; /* maximum supported axi burst length */ u32 axi_rw_burst_len; + /* Register map for DMAX_NUM_CHANNELS <= 8 */ + bool reg_map_8_channels; bool restrict_axi_burst_len; + bool use_cfg2; }; struct axi_dma_chan { @@ -62,7 +65,7 @@ struct dw_axi_dma { struct axi_dma_chip { struct device *dev; - int irq; + int irq[DMAC_MAX_CHANNELS]; void __iomem *regs; void __iomem *apb_regs; struct clk *core_clk; @@ -101,6 +104,18 @@ struct axi_dma_desc { u32 completed_blocks; u32 length; u32 period_len; + u32 nr_hw_descs; +}; + +struct axi_dma_chan_config { + u8 dst_multblk_type; + u8 src_multblk_type; + u8 dst_per; + u8 src_per; + u8 tt_fc; + u8 prior; + u8 hs_sel_dst; + u8 hs_sel_src; }; static inline struct device *dchan2dev(struct dma_chan *dchan) @@ -139,6 +154,8 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define DMAC_CHEN 0x018 /* R/W DMAC Channel Enable */ #define DMAC_CHEN_L 0x018 /* R/W DMAC Channel Enable 00-31 */ #define DMAC_CHEN_H 0x01C /* R/W DMAC Channel Enable 32-63 */ +#define DMAC_CHSUSPREG 0x020 /* R/W DMAC Channel Suspend */ +#define DMAC_CHABORTREG 0x028 /* R/W DMAC Channel Abort */ #define DMAC_INTSTATUS 0x030 /* R DMAC Interrupt Status */ #define DMAC_COMMON_INTCLEAR 0x038 /* W DMAC Interrupt Clear */ #define DMAC_COMMON_INTSTATUS_ENA 0x040 /* R DMAC Interrupt Status Enable */ @@ -184,7 +201,10 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define DMAC_APB_HALFWORD_WR_CH_EN 0x020 /* DMAC Halfword write enables */ #define UNUSED_CHANNEL 0x3F /* Set unused DMA channel to 0x3F */ +#define DMA_APB_HS_SEL_BIT_SIZE 0x08 /* HW handshake bits per channel */ +#define DMA_APB_HS_SEL_MASK 0xFF /* HW handshake select masks */ #define MAX_BLOCK_SIZE 0x1000 /* 1024 blocks * 4 bytes data width */ +#define DMA_REG_MAP_CH_REF 0x08 /* Channel count to choose register map */ /* DMAC_CFG */ #define DMAC_EN_POS 0 @@ -193,12 +213,24 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan) #define INT_EN_POS 1 #define INT_EN_MASK BIT(INT_EN_POS) +/* DMAC_CHEN */ #define DMAC_CHAN_EN_SHIFT 0 #define DMAC_CHAN_EN_WE_SHIFT 8 #define DMAC_CHAN_SUSP_SHIFT 16 #define DMAC_CHAN_SUSP_WE_SHIFT 24 +/* DMAC_CHEN2 */ +#define DMAC_CHAN_EN2_WE_SHIFT 16 + +/* DMAC CHAN BLOCKS */ +#define DMAC_CHAN_BLOCK_SHIFT 32 +#define DMAC_CHAN_16 16 + +/* DMAC_CHSUSP */ +#define DMAC_CHAN_SUSP2_SHIFT 0 +#define DMAC_CHAN_SUSP2_WE_SHIFT 16 + /* CH_CTL_H */ #define CH_CTL_H_ARLEN_EN BIT(6) #define CH_CTL_H_ARLEN_POS 7 @@ -256,6 +288,8 @@ enum { /* CH_CFG_H */ #define CH_CFG_H_PRIORITY_POS 17 +#define CH_CFG_H_DST_PER_POS 12 +#define CH_CFG_H_SRC_PER_POS 7 #define CH_CFG_H_HS_SEL_DST_POS 4 #define CH_CFG_H_HS_SEL_SRC_POS 3 enum { @@ -285,6 +319,15 @@ enum { DWAXIDMAC_MBLK_TYPE_LL }; +/* CH_CFG2 */ +#define CH_CFG2_L_SRC_PER_POS 4 +#define CH_CFG2_L_DST_PER_POS 11 + +#define CH_CFG2_H_TT_FC_POS 0 +#define CH_CFG2_H_HS_SEL_SRC_POS 3 +#define CH_CFG2_H_HS_SEL_DST_POS 4 +#define CH_CFG2_H_PRIORITY_POS 20 + /** * DW AXI DMA channel interrupts * diff --git a/drivers/dma/dw-edma/Kconfig b/drivers/dma/dw-edma/Kconfig index 7ff17b2db6a1..2b6f2679508d 100644 --- a/drivers/dma/dw-edma/Kconfig +++ b/drivers/dma/dw-edma/Kconfig @@ -9,11 +9,14 @@ config DW_EDMA Support the Synopsys DesignWare eDMA controller, normally implemented on endpoints SoCs. +if DW_EDMA + config DW_EDMA_PCIE tristate "Synopsys DesignWare eDMA PCIe driver" depends on PCI && PCI_MSI - select DW_EDMA help Provides a glue-logic between the Synopsys DesignWare eDMA controller and an endpoint PCIe device. This also serves as a reference design to whom desires to use this IP. + +endif # DW_EDMA diff --git a/drivers/dma/dw-edma/Makefile b/drivers/dma/dw-edma/Makefile index 8d45c0d5689d..83ab58f87760 100644 --- a/drivers/dma/dw-edma/Makefile +++ b/drivers/dma/dw-edma/Makefile @@ -1,7 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_DW_EDMA) += dw-edma.o -dw-edma-$(CONFIG_DEBUG_FS) := dw-edma-v0-debugfs.o -dw-edma-objs := dw-edma-core.o \ - dw-edma-v0-core.o $(dw-edma-y) +dw-edma-$(CONFIG_DEBUG_FS) := dw-edma-v0-debugfs.o \ + dw-hdma-v0-debugfs.o +dw-edma-objs := dw-edma-core.o \ + dw-edma-v0-core.o \ + dw-hdma-v0-core.o $(dw-edma-y) obj-$(CONFIG_DW_EDMA_PCIE) += dw-edma-pcie.o diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 53289927dd0d..8e5f7defa6b6 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -9,35 +9,35 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/kernel.h> -#include <linux/pm_runtime.h> #include <linux/dmaengine.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/dma/edma.h> #include <linux/dma-mapping.h> +#include <linux/string_choices.h> #include "dw-edma-core.h" #include "dw-edma-v0-core.h" +#include "dw-hdma-v0-core.h" #include "../dmaengine.h" #include "../virt-dma.h" static inline -struct device *dchan2dev(struct dma_chan *dchan) +struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) { - return &dchan->dev->device; + return container_of(vd, struct dw_edma_desc, vd); } static inline -struct device *chan2dev(struct dw_edma_chan *chan) +u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) { - return &chan->vc.chan.dev->device; -} + struct dw_edma_chip *chip = chan->dw->chip; -static inline -struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) -{ - return container_of(vd, struct dw_edma_desc, vd); + if (chip->ops->pci_address) + return chip->ops->pci_address(chip->dev, cpu_addr); + + return cpu_addr; } static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) @@ -64,8 +64,8 @@ static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) { + struct dw_edma_chip *chip = desc->chan->dw->chip; struct dw_edma_chan *chan = desc->chan; - struct dw_edma *dw = chan->chip->dw; struct dw_edma_chunk *chunk; chunk = kzalloc(sizeof(*chunk), GFP_NOWAIT); @@ -82,11 +82,11 @@ static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) */ chunk->cb = !(desc->chunks_alloc % 2); if (chan->dir == EDMA_DIR_WRITE) { - chunk->ll_region.paddr = dw->ll_region_wr[chan->id].paddr; - chunk->ll_region.vaddr = dw->ll_region_wr[chan->id].vaddr; + chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; + chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; } else { - chunk->ll_region.paddr = dw->ll_region_rd[chan->id].paddr; - chunk->ll_region.vaddr = dw->ll_region_rd[chan->id].vaddr; + chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; + chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; } if (desc->chunk) { @@ -171,31 +171,52 @@ static void vchan_free_desc(struct virt_dma_desc *vdesc) dw_edma_free_desc(vd2dw_edma_desc(vdesc)); } -static void dw_edma_start_transfer(struct dw_edma_chan *chan) +static int dw_edma_start_transfer(struct dw_edma_chan *chan) { + struct dw_edma *dw = chan->dw; struct dw_edma_chunk *child; struct dw_edma_desc *desc; struct virt_dma_desc *vd; vd = vchan_next_desc(&chan->vc); if (!vd) - return; + return 0; desc = vd2dw_edma_desc(vd); if (!desc) - return; + return 0; child = list_first_entry_or_null(&desc->chunk->list, struct dw_edma_chunk, list); if (!child) - return; + return 0; - dw_edma_v0_core_start(child, !desc->xfer_sz); + dw_edma_core_start(dw, child, !desc->xfer_sz); desc->xfer_sz += child->ll_region.sz; dw_edma_free_burst(child); list_del(&child->list); kfree(child); desc->chunks_alloc--; + + return 1; +} + +static void dw_edma_device_caps(struct dma_chan *dchan, + struct dma_slave_caps *caps) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + if (chan->dir == EDMA_DIR_READ) + caps->directions = BIT(DMA_DEV_TO_MEM); + else + caps->directions = BIT(DMA_MEM_TO_DEV); + } else { + if (chan->dir == EDMA_DIR_WRITE) + caps->directions = BIT(DMA_DEV_TO_MEM); + else + caps->directions = BIT(DMA_MEM_TO_DEV); + } } static int dw_edma_device_config(struct dma_chan *dchan, @@ -249,7 +270,6 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) { struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; - LIST_HEAD(head); if (!chan->configured) { /* Do nothing */ @@ -258,7 +278,7 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) chan->configured = false; } else if (chan->status == EDMA_ST_IDLE) { chan->configured = false; - } else if (dw_edma_v0_core_ch_status(chan) == DMA_COMPLETE) { + } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { /* * The channel is in a false BUSY state, probably didn't * receive or lost an interrupt @@ -279,9 +299,12 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long flags; + if (!chan->configured) + return; + spin_lock_irqsave(&chan->vc.lock, flags); - if (chan->configured && chan->request == EDMA_REQ_NONE && - chan->status == EDMA_ST_IDLE && vchan_issue_pending(&chan->vc)) { + if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && + chan->status == EDMA_ST_IDLE) { chan->status = EDMA_ST_BUSY; dw_edma_start_transfer(chan); } @@ -329,32 +352,52 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) { struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); enum dma_transfer_direction dir = xfer->direction; - phys_addr_t src_addr, dst_addr; struct scatterlist *sg = NULL; struct dw_edma_chunk *chunk; struct dw_edma_burst *burst; struct dw_edma_desc *desc; + u64 src_addr, dst_addr; + size_t fsz = 0; u32 cnt = 0; int i; if (!chan->configured) return NULL; - switch (chan->config.direction) { - case DMA_DEV_TO_MEM: /* local DMA */ - if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_READ) - break; - return NULL; - case DMA_MEM_TO_DEV: /* local DMA */ - if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_WRITE) - break; - return NULL; - default: /* remote DMA */ - if (dir == DMA_MEM_TO_DEV && chan->dir == EDMA_DIR_READ) - break; - if (dir == DMA_DEV_TO_MEM && chan->dir == EDMA_DIR_WRITE) - break; - return NULL; + /* + * Local Root Port/End-point Remote End-point + * +-----------------------+ PCIe bus +----------------------+ + * | | +-+ | | + * | DEV_TO_MEM Rx Ch <----+ +---+ Tx Ch DEV_TO_MEM | + * | | | | | | + * | MEM_TO_DEV Tx Ch +----+ +---> Rx Ch MEM_TO_DEV | + * | | +-+ | | + * +-----------------------+ +----------------------+ + * + * 1. Normal logic: + * If eDMA is embedded into the DW PCIe RP/EP and controlled from the + * CPU/Application side, the Rx channel (EDMA_DIR_READ) will be used + * for the device read operations (DEV_TO_MEM) and the Tx channel + * (EDMA_DIR_WRITE) - for the write operations (MEM_TO_DEV). + * + * 2. Inverted logic: + * If eDMA is embedded into a Remote PCIe EP and is controlled by the + * MWr/MRd TLPs sent from the CPU's PCIe host controller, the Tx + * channel (EDMA_DIR_WRITE) will be used for the device read operations + * (DEV_TO_MEM) and the Rx channel (EDMA_DIR_READ) - for the write + * operations (MEM_TO_DEV). + * + * It is the client driver responsibility to choose a proper channel + * for the DMA transfers. + */ + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + if ((chan->dir == EDMA_DIR_READ && dir != DMA_DEV_TO_MEM) || + (chan->dir == EDMA_DIR_WRITE && dir != DMA_MEM_TO_DEV)) + return NULL; + } else { + if ((chan->dir == EDMA_DIR_WRITE && dir != DMA_DEV_TO_MEM) || + (chan->dir == EDMA_DIR_READ && dir != DMA_MEM_TO_DEV)) + return NULL; } if (xfer->type == EDMA_XFER_CYCLIC) { @@ -364,9 +407,9 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) if (xfer->xfer.sg.len < 1) return NULL; } else if (xfer->type == EDMA_XFER_INTERLEAVED) { - if (!xfer->xfer.il->numf) + if (!xfer->xfer.il->numf || xfer->xfer.il->frame_size < 1) return NULL; - if (xfer->xfer.il->numf > 0 && xfer->xfer.il->frame_size > 0) + if (!xfer->xfer.il->src_inc || !xfer->xfer.il->dst_inc) return NULL; } else { return NULL; @@ -388,16 +431,19 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) dst_addr = chan->config.dst_addr; } + if (dir == DMA_DEV_TO_MEM) + src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr); + else + dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr); + if (xfer->type == EDMA_XFER_CYCLIC) { cnt = xfer->xfer.cyclic.cnt; } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) { cnt = xfer->xfer.sg.len; sg = xfer->xfer.sg.sgl; } else if (xfer->type == EDMA_XFER_INTERLEAVED) { - if (xfer->xfer.il->numf > 0) - cnt = xfer->xfer.il->numf; - else - cnt = xfer->xfer.il->frame_size; + cnt = xfer->xfer.il->numf * xfer->xfer.il->frame_size; + fsz = xfer->xfer.il->frame_size; } for (i = 0; i < cnt; i++) { @@ -419,12 +465,12 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) else if (xfer->type == EDMA_XFER_SCATTER_GATHER) burst->sz = sg_dma_len(sg); else if (xfer->type == EDMA_XFER_INTERLEAVED) - burst->sz = xfer->xfer.il->sgl[i].size; + burst->sz = xfer->xfer.il->sgl[i % fsz].size; chunk->ll_region.sz += burst->sz; desc->alloc_sz += burst->sz; - if (chan->dir == EDMA_DIR_WRITE) { + if (dir == DMA_DEV_TO_MEM) { burst->sar = src_addr; if (xfer->type == EDMA_XFER_CYCLIC) { burst->dar = xfer->xfer.cyclic.paddr; @@ -438,6 +484,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) * and destination addresses are increased * by the same portion (data length) */ + } else if (xfer->type == EDMA_XFER_INTERLEAVED) { + burst->dar = dst_addr; } } else { burst->dar = dst_addr; @@ -453,25 +501,24 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) * and destination addresses are increased * by the same portion (data length) */ + } else if (xfer->type == EDMA_XFER_INTERLEAVED) { + burst->sar = src_addr; } } if (xfer->type == EDMA_XFER_SCATTER_GATHER) { sg = sg_next(sg); - } else if (xfer->type == EDMA_XFER_INTERLEAVED && - xfer->xfer.il->frame_size > 0) { + } else if (xfer->type == EDMA_XFER_INTERLEAVED) { struct dma_interleaved_template *il = xfer->xfer.il; - struct data_chunk *dc = &il->sgl[i]; + struct data_chunk *dc = &il->sgl[i % fsz]; - if (il->src_sgl) { - src_addr += burst->sz; + src_addr += burst->sz; + if (il->src_sgl) src_addr += dmaengine_get_src_icg(il, dc); - } - if (il->dst_sgl) { - dst_addr += burst->sz; + dst_addr += burst->sz; + if (il->dst_sgl) dst_addr += dmaengine_get_dst_icg(il, dc); - } } } @@ -537,28 +584,47 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, return dw_edma_device_transfer(&xfer); } +static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, + enum dmaengine_tx_result result) +{ + u32 residue = 0; + struct dw_edma_desc *desc; + struct dmaengine_result *res; + + if (!vd->tx.callback_result) + return; + + desc = vd2dw_edma_desc(vd); + if (desc) + residue = desc->alloc_sz - desc->xfer_sz; + + res = &vd->tx_result; + res->result = result; + res->residue = residue; +} + static void dw_edma_done_interrupt(struct dw_edma_chan *chan) { struct dw_edma_desc *desc; struct virt_dma_desc *vd; unsigned long flags; - dw_edma_v0_core_clear_done_int(chan); - spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); if (vd) { switch (chan->request) { case EDMA_REQ_NONE: desc = vd2dw_edma_desc(vd); - if (desc->chunks_alloc) { - chan->status = EDMA_ST_BUSY; - dw_edma_start_transfer(chan); - } else { + if (!desc->chunks_alloc) { + dw_hdma_set_callback_result(vd, + DMA_TRANS_NOERROR); list_del(&vd->node); vchan_cookie_complete(vd); - chan->status = EDMA_ST_IDLE; } + + /* Continue transferring if there are remaining chunks or issued requests. + */ + chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; break; case EDMA_REQ_STOP: @@ -585,11 +651,10 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) struct virt_dma_desc *vd; unsigned long flags; - dw_edma_v0_core_clear_abort_int(chan); - spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); if (vd) { + dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); } @@ -598,63 +663,32 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) chan->status = EDMA_ST_IDLE; } -static irqreturn_t dw_edma_interrupt(int irq, void *data, bool write) +static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) { struct dw_edma_irq *dw_irq = data; - struct dw_edma *dw = dw_irq->dw; - unsigned long total, pos, val; - unsigned long off; - u32 mask; - - if (write) { - total = dw->wr_ch_cnt; - off = 0; - mask = dw_irq->wr_mask; - } else { - total = dw->rd_ch_cnt; - off = dw->wr_ch_cnt; - mask = dw_irq->rd_mask; - } - - val = dw_edma_v0_core_status_done_int(dw, write ? - EDMA_DIR_WRITE : - EDMA_DIR_READ); - val &= mask; - for_each_set_bit(pos, &val, total) { - struct dw_edma_chan *chan = &dw->chan[pos + off]; - - dw_edma_done_interrupt(chan); - } - - val = dw_edma_v0_core_status_abort_int(dw, write ? - EDMA_DIR_WRITE : - EDMA_DIR_READ); - val &= mask; - for_each_set_bit(pos, &val, total) { - struct dw_edma_chan *chan = &dw->chan[pos + off]; - - dw_edma_abort_interrupt(chan); - } - return IRQ_HANDLED; -} - -static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) -{ - return dw_edma_interrupt(irq, data, true); + return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE, + dw_edma_done_interrupt, + dw_edma_abort_interrupt); } static inline irqreturn_t dw_edma_interrupt_read(int irq, void *data) { - return dw_edma_interrupt(irq, data, false); + struct dw_edma_irq *dw_irq = data; + + return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ, + dw_edma_done_interrupt, + dw_edma_abort_interrupt); } static irqreturn_t dw_edma_interrupt_common(int irq, void *data) { - dw_edma_interrupt(irq, data, true); - dw_edma_interrupt(irq, data, false); + irqreturn_t ret = IRQ_NONE; - return IRQ_HANDLED; + ret |= dw_edma_interrupt_write(irq, data); + ret |= dw_edma_interrupt_read(irq, data); + + return ret; } static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) @@ -664,15 +698,12 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) if (chan->status != EDMA_ST_IDLE) return -EBUSY; - pm_runtime_get(chan->chip->dev); - return 0; } static void dw_edma_free_chan_resources(struct dma_chan *dchan) { unsigned long timeout = jiffies + msecs_to_jiffies(5000); - struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int ret; while (time_before(jiffies, timeout)) { @@ -685,98 +716,81 @@ static void dw_edma_free_chan_resources(struct dma_chan *dchan) cpu_relax(); } - - pm_runtime_put(chan->chip->dev); } -static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, - u32 wr_alloc, u32 rd_alloc) +static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) { - struct dw_edma_region *dt_region; + struct dw_edma_chip *chip = dw->chip; struct device *dev = chip->dev; - struct dw_edma *dw = chip->dw; struct dw_edma_chan *chan; struct dw_edma_irq *irq; struct dma_device *dma; - u32 alloc, off_alloc; - u32 i, j, cnt; - int err = 0; + u32 i, ch_cnt; u32 pos; - if (write) { - i = 0; - cnt = dw->wr_ch_cnt; - dma = &dw->wr_edma; - alloc = wr_alloc; - off_alloc = 0; - } else { - i = dw->wr_ch_cnt; - cnt = dw->rd_ch_cnt; - dma = &dw->rd_edma; - alloc = rd_alloc; - off_alloc = wr_alloc; - } + ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; + dma = &dw->dma; INIT_LIST_HEAD(&dma->channels); - for (j = 0; (alloc || dw->nr_irqs == 1) && j < cnt; j++, i++) { + + for (i = 0; i < ch_cnt; i++) { chan = &dw->chan[i]; - dt_region = devm_kzalloc(dev, sizeof(*dt_region), GFP_KERNEL); - if (!dt_region) - return -ENOMEM; + chan->dw = dw; - chan->vc.chan.private = dt_region; + if (i < dw->wr_ch_cnt) { + chan->id = i; + chan->dir = EDMA_DIR_WRITE; + } else { + chan->id = i - dw->wr_ch_cnt; + chan->dir = EDMA_DIR_READ; + } - chan->chip = chip; - chan->id = j; - chan->dir = write ? EDMA_DIR_WRITE : EDMA_DIR_READ; chan->configured = false; chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; - if (write) - chan->ll_max = (dw->ll_region_wr[j].sz / EDMA_LL_SZ); + if (chan->dir == EDMA_DIR_WRITE) + chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); else - chan->ll_max = (dw->ll_region_rd[j].sz / EDMA_LL_SZ); + chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); chan->ll_max -= 1; dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", - write ? "write" : "read", j, chan->ll_max); + str_write_read(chan->dir == EDMA_DIR_WRITE), + chan->id, chan->ll_max); if (dw->nr_irqs == 1) pos = 0; + else if (chan->dir == EDMA_DIR_WRITE) + pos = chan->id % wr_alloc; else - pos = off_alloc + (j % alloc); + pos = wr_alloc + chan->id % rd_alloc; irq = &dw->irq[pos]; - if (write) - irq->wr_mask |= BIT(j); + if (chan->dir == EDMA_DIR_WRITE) + irq->wr_mask |= BIT(chan->id); else - irq->rd_mask |= BIT(j); + irq->rd_mask |= BIT(chan->id); irq->dw = dw; memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", - write ? "write" : "read", j, + str_write_read(chan->dir == EDMA_DIR_WRITE), + chan->id, chan->msi.address_hi, chan->msi.address_lo, chan->msi.data); chan->vc.desc_free = vchan_free_desc; - vchan_init(&chan->vc, dma); + chan->vc.chan.private = chan->dir == EDMA_DIR_WRITE ? + &dw->chip->dt_region_wr[chan->id] : + &dw->chip->dt_region_rd[chan->id]; - if (write) { - dt_region->paddr = dw->dt_region_wr[j].paddr; - dt_region->vaddr = dw->dt_region_wr[j].vaddr; - dt_region->sz = dw->dt_region_wr[j].sz; - } else { - dt_region->paddr = dw->dt_region_rd[j].paddr; - dt_region->vaddr = dw->dt_region_rd[j].vaddr; - dt_region->sz = dw->dt_region_rd[j].sz; - } + vchan_init(&chan->vc, dma); - dw_edma_v0_core_device_config(chan); + dw_edma_core_ch_config(chan); } /* Set DMA channel capabilities */ @@ -785,16 +799,16 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, dma_cap_set(DMA_CYCLIC, dma->cap_mask); dma_cap_set(DMA_PRIVATE, dma->cap_mask); dma_cap_set(DMA_INTERLEAVE, dma->cap_mask); - dma->directions = BIT(write ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV); + dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); dma->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - dma->chancnt = cnt; /* Set DMA channel callbacks */ dma->dev = chip->dev; dma->device_alloc_chan_resources = dw_edma_alloc_chan_resources; dma->device_free_chan_resources = dw_edma_free_chan_resources; + dma->device_caps = dw_edma_device_caps; dma->device_config = dw_edma_device_config; dma->device_pause = dw_edma_device_pause; dma->device_resume = dw_edma_device_resume; @@ -808,9 +822,7 @@ static int dw_edma_channel_setup(struct dw_edma_chip *chip, bool write, dma_set_max_seg_size(dma->dev, U32_MAX); /* Register DMA device */ - err = dma_async_device_register(dma); - - return err; + return dma_async_device_register(dma); } static inline void dw_edma_dec_irq_alloc(int *nr_irqs, u32 *alloc, u16 cnt) @@ -827,11 +839,11 @@ static inline void dw_edma_add_irq_mask(u32 *mask, u32 alloc, u16 cnt) (*mask)++; } -static int dw_edma_irq_request(struct dw_edma_chip *chip, +static int dw_edma_irq_request(struct dw_edma *dw, u32 *wr_alloc, u32 *rd_alloc) { - struct device *dev = chip->dev; - struct dw_edma *dw = chip->dw; + struct dw_edma_chip *chip = dw->chip; + struct device *dev = dw->chip->dev; u32 wr_mask = 1; u32 rd_mask = 1; int i, err = 0; @@ -840,12 +852,16 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip, ch_cnt = dw->wr_ch_cnt + dw->rd_ch_cnt; - if (dw->nr_irqs < 1) + if (chip->nr_irqs < 1 || !chip->ops->irq_vector) return -EINVAL; - if (dw->nr_irqs == 1) { + dw->irq = devm_kcalloc(dev, chip->nr_irqs, sizeof(*dw->irq), GFP_KERNEL); + if (!dw->irq) + return -ENOMEM; + + if (chip->nr_irqs == 1) { /* Common IRQ shared among all channels */ - irq = dw->ops->irq_vector(dev, 0); + irq = chip->ops->irq_vector(dev, 0); err = request_irq(irq, dw_edma_interrupt_common, IRQF_SHARED, dw->name, &dw->irq[0]); if (err) { @@ -855,9 +871,11 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip, if (irq_get_msi_desc(irq)) get_cached_msi_msg(irq, &dw->irq[0].msi); + + dw->nr_irqs = 1; } else { /* Distribute IRQs equally among all channels */ - int tmp = dw->nr_irqs; + int tmp = chip->nr_irqs; while (tmp && (*wr_alloc + *rd_alloc) < ch_cnt) { dw_edma_dec_irq_alloc(&tmp, wr_alloc, dw->wr_ch_cnt); @@ -868,17 +886,15 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip, dw_edma_add_irq_mask(&rd_mask, *rd_alloc, dw->rd_ch_cnt); for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { - irq = dw->ops->irq_vector(dev, i); + irq = chip->ops->irq_vector(dev, i); err = request_irq(irq, i < *wr_alloc ? dw_edma_interrupt_write : dw_edma_interrupt_read, IRQF_SHARED, dw->name, &dw->irq[i]); - if (err) { - dw->nr_irqs = i; - return err; - } + if (err) + goto err_irq_free; if (irq_get_msi_desc(irq)) get_cached_msi_msg(irq, &dw->irq[i].msi); @@ -887,6 +903,14 @@ static int dw_edma_irq_request(struct dw_edma_chip *chip, dw->nr_irqs = i; } + return 0; + +err_irq_free: + for (i--; i >= 0; i--) { + irq = chip->ops->irq_vector(dev, i); + free_irq(irq, &dw->irq[i]); + } + return err; } @@ -902,21 +926,28 @@ int dw_edma_probe(struct dw_edma_chip *chip) return -EINVAL; dev = chip->dev; - if (!dev) + if (!dev || !chip->ops) return -EINVAL; - dw = chip->dw; - if (!dw || !dw->irq || !dw->ops || !dw->ops->irq_vector) - return -EINVAL; + dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + dw->chip = chip; + + if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) + dw_hdma_v0_core_register(dw); + else + dw_edma_v0_core_register(dw); raw_spin_lock_init(&dw->lock); - dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, - dw_edma_v0_core_ch_count(dw, EDMA_DIR_WRITE)); + dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, + dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); - dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, - dw_edma_v0_core_ch_count(dw, EDMA_DIR_READ)); + dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, + dw_edma_core_ch_count(dw, EDMA_DIR_READ)); dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) @@ -931,39 +962,32 @@ int dw_edma_probe(struct dw_edma_chip *chip) if (!dw->chan) return -ENOMEM; - snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%d", chip->id); + snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", + dev_name(chip->dev)); /* Disable eDMA, only to establish the ideal initial conditions */ - dw_edma_v0_core_off(dw); + dw_edma_core_off(dw); /* Request IRQs */ - err = dw_edma_irq_request(chip, &wr_alloc, &rd_alloc); + err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); if (err) return err; - /* Setup write channels */ - err = dw_edma_channel_setup(chip, true, wr_alloc, rd_alloc); + /* Setup write/read channels */ + err = dw_edma_channel_setup(dw, wr_alloc, rd_alloc); if (err) goto err_irq_free; - /* Setup read channels */ - err = dw_edma_channel_setup(chip, false, wr_alloc, rd_alloc); - if (err) - goto err_irq_free; - - /* Power management */ - pm_runtime_enable(dev); - /* Turn debugfs on */ - dw_edma_v0_core_debugfs_on(chip); + dw_edma_core_debugfs_on(dw); + + chip->dw = dw; return 0; err_irq_free: for (i = (dw->nr_irqs - 1); i >= 0; i--) - free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); - - dw->nr_irqs = 0; + free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); return err; } @@ -976,34 +1000,25 @@ int dw_edma_remove(struct dw_edma_chip *chip) struct dw_edma *dw = chip->dw; int i; + /* Skip removal if no private data found */ + if (!dw) + return -ENODEV; + /* Disable eDMA */ - dw_edma_v0_core_off(dw); + dw_edma_core_off(dw); /* Free irqs */ for (i = (dw->nr_irqs - 1); i >= 0; i--) - free_irq(dw->ops->irq_vector(dev, i), &dw->irq[i]); - - /* Power management */ - pm_runtime_disable(dev); + free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); /* Deregister eDMA device */ - dma_async_device_unregister(&dw->wr_edma); - list_for_each_entry_safe(chan, _chan, &dw->wr_edma.channels, + dma_async_device_unregister(&dw->dma); + list_for_each_entry_safe(chan, _chan, &dw->dma.channels, vc.chan.device_node) { tasklet_kill(&chan->vc.task); list_del(&chan->vc.chan.device_node); } - dma_async_device_unregister(&dw->rd_edma); - list_for_each_entry_safe(chan, _chan, &dw->rd_edma.channels, - vc.chan.device_node) { - tasklet_kill(&chan->vc.task); - list_del(&chan->vc.chan.device_node); - } - - /* Turn debugfs off */ - dw_edma_v0_core_debugfs_off(chip); - return 0; } EXPORT_SYMBOL_GPL(dw_edma_remove); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 60316d408c3e..71894b9e0b15 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -15,20 +15,12 @@ #include "../virt-dma.h" #define EDMA_LL_SZ 24 -#define EDMA_MAX_WR_CH 8 -#define EDMA_MAX_RD_CH 8 enum dw_edma_dir { EDMA_DIR_WRITE = 0, EDMA_DIR_READ }; -enum dw_edma_map_format { - EDMA_MF_EDMA_LEGACY = 0x0, - EDMA_MF_EDMA_UNROLL = 0x1, - EDMA_MF_HDMA_COMPAT = 0x5 -}; - enum dw_edma_request { EDMA_REQ_NONE = 0, EDMA_REQ_STOP, @@ -57,12 +49,6 @@ struct dw_edma_burst { u32 sz; }; -struct dw_edma_region { - phys_addr_t paddr; - void __iomem *vaddr; - size_t sz; -}; - struct dw_edma_chunk { struct list_head list; struct dw_edma_chan *chan; @@ -87,7 +73,7 @@ struct dw_edma_desc { struct dw_edma_chan { struct virt_dma_chan vc; - struct dw_edma_chip *chip; + struct dw_edma *dw; int id; enum dw_edma_dir dir; @@ -109,37 +95,37 @@ struct dw_edma_irq { struct dw_edma *dw; }; -struct dw_edma_core_ops { - int (*irq_vector)(struct device *dev, unsigned int nr); -}; - struct dw_edma { - char name[20]; + char name[32]; - struct dma_device wr_edma; - u16 wr_ch_cnt; + struct dma_device dma; - struct dma_device rd_edma; + u16 wr_ch_cnt; u16 rd_ch_cnt; - struct dw_edma_region rg_region; /* Registers */ - struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH]; - struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH]; - struct dw_edma_irq *irq; int nr_irqs; - enum dw_edma_map_format mf; - struct dw_edma_chan *chan; - const struct dw_edma_core_ops *ops; raw_spinlock_t lock; /* Only for legacy */ -#ifdef CONFIG_DEBUG_FS - struct dentry *debugfs; -#endif /* CONFIG_DEBUG_FS */ + + struct dw_edma_chip *chip; + + const struct dw_edma_core_ops *core; +}; + +typedef void (*dw_edma_handler_t)(struct dw_edma_chan *); + +struct dw_edma_core_ops { + void (*off)(struct dw_edma *dw); + u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir); + enum dma_status (*ch_status)(struct dw_edma_chan *chan); + irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, + dw_edma_handler_t done, dw_edma_handler_t abort); + void (*start)(struct dw_edma_chunk *chunk, bool first); + void (*ch_config)(struct dw_edma_chan *chan); + void (*debugfs_on)(struct dw_edma *dw); }; struct dw_edma_sg { @@ -177,4 +163,47 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan) return vc2dw_edma_chan(to_virt_chan(dchan)); } +static inline +void dw_edma_core_off(struct dw_edma *dw) +{ + dw->core->off(dw); +} + +static inline +u16 dw_edma_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) +{ + return dw->core->ch_count(dw, dir); +} + +static inline +enum dma_status dw_edma_core_ch_status(struct dw_edma_chan *chan) +{ + return chan->dw->core->ch_status(chan); +} + +static inline irqreturn_t +dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, + dw_edma_handler_t done, dw_edma_handler_t abort) +{ + return dw_irq->dw->core->handle_int(dw_irq, dir, done, abort); +} + +static inline +void dw_edma_core_start(struct dw_edma *dw, struct dw_edma_chunk *chunk, bool first) +{ + dw->core->start(chunk, first); +} + +static inline +void dw_edma_core_ch_config(struct dw_edma_chan *chan) +{ + chan->dw->core->ch_config(chan); +} + +static inline +void dw_edma_core_debugfs_on(struct dw_edma *dw) +{ + dw->core->debugfs_on(dw); +} + #endif /* _DW_EDMA_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 44f6e09bdb53..3371e0a76d3c 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -95,8 +95,23 @@ static int dw_edma_pcie_irq_vector(struct device *dev, unsigned int nr) return pci_irq_vector(to_pci_dev(dev), nr); } -static const struct dw_edma_core_ops dw_edma_pcie_core_ops = { +static u64 dw_edma_pcie_address(struct device *dev, phys_addr_t cpu_addr) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_bus_region region; + struct resource res = { + .flags = IORESOURCE_MEM, + .start = cpu_addr, + .end = cpu_addr, + }; + + pcibios_resource_to_bus(pdev->bus, ®ion, &res); + return region.start; +} + +static const struct dw_edma_plat_ops dw_edma_pcie_plat_ops = { .irq_vector = dw_edma_pcie_irq_vector, + .pci_address = dw_edma_pcie_address, }; static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, @@ -121,7 +136,8 @@ static void dw_edma_pcie_get_vsec_dma_data(struct pci_dev *pdev, map = FIELD_GET(DW_PCIE_VSEC_DMA_MAP, val); if (map != EDMA_MF_EDMA_LEGACY && map != EDMA_MF_EDMA_UNROLL && - map != EDMA_MF_HDMA_COMPAT) + map != EDMA_MF_HDMA_COMPAT && + map != EDMA_MF_HDMA_NATIVE) return; pdata->mf = map; @@ -145,13 +161,16 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { struct dw_edma_pcie_data *pdata = (void *)pid->driver_data; - struct dw_edma_pcie_data vsec_data; + struct dw_edma_pcie_data *vsec_data __free(kfree) = NULL; struct device *dev = &pdev->dev; struct dw_edma_chip *chip; - struct dw_edma *dw; int err, nr_irqs; int i, mask; + vsec_data = kmalloc(sizeof(*vsec_data), GFP_KERNEL); + if (!vsec_data) + return -ENOMEM; + /* Enable PCI device */ err = pcim_enable_device(pdev); if (err) { @@ -159,23 +178,23 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return err; } - memcpy(&vsec_data, pdata, sizeof(struct dw_edma_pcie_data)); + memcpy(vsec_data, pdata, sizeof(struct dw_edma_pcie_data)); /* * Tries to find if exists a PCIe Vendor-Specific Extended Capability * for the DMA, if one exists, then reconfigures it. */ - dw_edma_pcie_get_vsec_dma_data(pdev, &vsec_data); + dw_edma_pcie_get_vsec_dma_data(pdev, vsec_data); /* Mapping PCI BAR regions */ - mask = BIT(vsec_data.rg.bar); - for (i = 0; i < vsec_data.wr_ch_cnt; i++) { - mask |= BIT(vsec_data.ll_wr[i].bar); - mask |= BIT(vsec_data.dt_wr[i].bar); + mask = BIT(vsec_data->rg.bar); + for (i = 0; i < vsec_data->wr_ch_cnt; i++) { + mask |= BIT(vsec_data->ll_wr[i].bar); + mask |= BIT(vsec_data->dt_wr[i].bar); } - for (i = 0; i < vsec_data.rd_ch_cnt; i++) { - mask |= BIT(vsec_data.ll_rd[i].bar); - mask |= BIT(vsec_data.dt_rd[i].bar); + for (i = 0; i < vsec_data->rd_ch_cnt; i++) { + mask |= BIT(vsec_data->ll_rd[i].bar); + mask |= BIT(vsec_data->dt_rd[i].bar); } err = pcim_iomap_regions(pdev, mask, pci_name(pdev)); if (err) { @@ -186,27 +205,10 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, pci_set_master(pdev); /* DMA configuration */ - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (!err) { - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) { - pci_err(pdev, "consistent DMA mask 64 set failed\n"); - return err; - } - } else { + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { pci_err(pdev, "DMA mask 64 set failed\n"); - - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) { - pci_err(pdev, "DMA mask 32 set failed\n"); - return err; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) { - pci_err(pdev, "consistent DMA mask 32 set failed\n"); - return err; - } + return err; } /* Data structure allocation */ @@ -214,12 +216,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, if (!chip) return -ENOMEM; - dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); - if (!dw) - return -ENOMEM; - /* IRQs allocation */ - nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data.irqs, + nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data->irqs, PCI_IRQ_MSI | PCI_IRQ_MSIX); if (nr_irqs < 1) { pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n", @@ -228,116 +226,111 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, } /* Data structure initialization */ - chip->dw = dw; chip->dev = dev; - chip->id = pdev->devfn; - chip->irq = pdev->irq; - dw->mf = vsec_data.mf; - dw->nr_irqs = nr_irqs; - dw->ops = &dw_edma_pcie_core_ops; - dw->wr_ch_cnt = vsec_data.wr_ch_cnt; - dw->rd_ch_cnt = vsec_data.rd_ch_cnt; + chip->mf = vsec_data->mf; + chip->nr_irqs = nr_irqs; + chip->ops = &dw_edma_pcie_plat_ops; - dw->rg_region.vaddr = pcim_iomap_table(pdev)[vsec_data.rg.bar]; - if (!dw->rg_region.vaddr) - return -ENOMEM; + chip->ll_wr_cnt = vsec_data->wr_ch_cnt; + chip->ll_rd_cnt = vsec_data->rd_ch_cnt; - dw->rg_region.vaddr += vsec_data.rg.off; - dw->rg_region.paddr = pdev->resource[vsec_data.rg.bar].start; - dw->rg_region.paddr += vsec_data.rg.off; - dw->rg_region.sz = vsec_data.rg.sz; + chip->reg_base = pcim_iomap_table(pdev)[vsec_data->rg.bar]; + if (!chip->reg_base) + return -ENOMEM; - for (i = 0; i < dw->wr_ch_cnt; i++) { - struct dw_edma_region *ll_region = &dw->ll_region_wr[i]; - struct dw_edma_region *dt_region = &dw->dt_region_wr[i]; - struct dw_edma_block *ll_block = &vsec_data.ll_wr[i]; - struct dw_edma_block *dt_block = &vsec_data.dt_wr[i]; + for (i = 0; i < chip->ll_wr_cnt; i++) { + struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; + struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; + struct dw_edma_block *ll_block = &vsec_data->ll_wr[i]; + struct dw_edma_block *dt_block = &vsec_data->dt_wr[i]; - ll_region->vaddr = pcim_iomap_table(pdev)[ll_block->bar]; - if (!ll_region->vaddr) + ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; + if (!ll_region->vaddr.io) return -ENOMEM; - ll_region->vaddr += ll_block->off; - ll_region->paddr = pdev->resource[ll_block->bar].start; + ll_region->vaddr.io += ll_block->off; + ll_region->paddr = pci_bus_address(pdev, ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; - dt_region->vaddr = pcim_iomap_table(pdev)[dt_block->bar]; - if (!dt_region->vaddr) + dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; + if (!dt_region->vaddr.io) return -ENOMEM; - dt_region->vaddr += dt_block->off; - dt_region->paddr = pdev->resource[dt_block->bar].start; + dt_region->vaddr.io += dt_block->off; + dt_region->paddr = pci_bus_address(pdev, dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } - for (i = 0; i < dw->rd_ch_cnt; i++) { - struct dw_edma_region *ll_region = &dw->ll_region_rd[i]; - struct dw_edma_region *dt_region = &dw->dt_region_rd[i]; - struct dw_edma_block *ll_block = &vsec_data.ll_rd[i]; - struct dw_edma_block *dt_block = &vsec_data.dt_rd[i]; + for (i = 0; i < chip->ll_rd_cnt; i++) { + struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; + struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; + struct dw_edma_block *ll_block = &vsec_data->ll_rd[i]; + struct dw_edma_block *dt_block = &vsec_data->dt_rd[i]; - ll_region->vaddr = pcim_iomap_table(pdev)[ll_block->bar]; - if (!ll_region->vaddr) + ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; + if (!ll_region->vaddr.io) return -ENOMEM; - ll_region->vaddr += ll_block->off; - ll_region->paddr = pdev->resource[ll_block->bar].start; + ll_region->vaddr.io += ll_block->off; + ll_region->paddr = pci_bus_address(pdev, ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; - dt_region->vaddr = pcim_iomap_table(pdev)[dt_block->bar]; - if (!dt_region->vaddr) + dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; + if (!dt_region->vaddr.io) return -ENOMEM; - dt_region->vaddr += dt_block->off; - dt_region->paddr = pdev->resource[dt_block->bar].start; + dt_region->vaddr.io += dt_block->off; + dt_region->paddr = pci_bus_address(pdev, dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } /* Debug info */ - if (dw->mf == EDMA_MF_EDMA_LEGACY) - pci_dbg(pdev, "Version:\teDMA Port Logic (0x%x)\n", dw->mf); - else if (dw->mf == EDMA_MF_EDMA_UNROLL) - pci_dbg(pdev, "Version:\teDMA Unroll (0x%x)\n", dw->mf); - else if (dw->mf == EDMA_MF_HDMA_COMPAT) - pci_dbg(pdev, "Version:\tHDMA Compatible (0x%x)\n", dw->mf); + if (chip->mf == EDMA_MF_EDMA_LEGACY) + pci_dbg(pdev, "Version:\teDMA Port Logic (0x%x)\n", chip->mf); + else if (chip->mf == EDMA_MF_EDMA_UNROLL) + pci_dbg(pdev, "Version:\teDMA Unroll (0x%x)\n", chip->mf); + else if (chip->mf == EDMA_MF_HDMA_COMPAT) + pci_dbg(pdev, "Version:\tHDMA Compatible (0x%x)\n", chip->mf); + else if (chip->mf == EDMA_MF_HDMA_NATIVE) + pci_dbg(pdev, "Version:\tHDMA Native (0x%x)\n", chip->mf); else - pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", dw->mf); + pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", chip->mf); - pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - vsec_data.rg.bar, vsec_data.rg.off, vsec_data.rg.sz, - dw->rg_region.vaddr, &dw->rg_region.paddr); + pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n", + vsec_data->rg.bar, vsec_data->rg.off, vsec_data->rg.sz, + chip->reg_base); - for (i = 0; i < dw->wr_ch_cnt; i++) { + for (i = 0; i < chip->ll_wr_cnt; i++) { pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.ll_wr[i].bar, - vsec_data.ll_wr[i].off, dw->ll_region_wr[i].sz, - dw->ll_region_wr[i].vaddr, &dw->ll_region_wr[i].paddr); + i, vsec_data->ll_wr[i].bar, + vsec_data->ll_wr[i].off, chip->ll_region_wr[i].sz, + chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr); pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.dt_wr[i].bar, - vsec_data.dt_wr[i].off, dw->dt_region_wr[i].sz, - dw->dt_region_wr[i].vaddr, &dw->dt_region_wr[i].paddr); + i, vsec_data->dt_wr[i].bar, + vsec_data->dt_wr[i].off, chip->dt_region_wr[i].sz, + chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr); } - for (i = 0; i < dw->rd_ch_cnt; i++) { + for (i = 0; i < chip->ll_rd_cnt; i++) { pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.ll_rd[i].bar, - vsec_data.ll_rd[i].off, dw->ll_region_rd[i].sz, - dw->ll_region_rd[i].vaddr, &dw->ll_region_rd[i].paddr); + i, vsec_data->ll_rd[i].bar, + vsec_data->ll_rd[i].off, chip->ll_region_rd[i].sz, + chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr); pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data.dt_rd[i].bar, - vsec_data.dt_rd[i].off, dw->dt_region_rd[i].sz, - dw->dt_region_rd[i].vaddr, &dw->dt_region_rd[i].paddr); + i, vsec_data->dt_rd[i].bar, + vsec_data->dt_rd[i].off, chip->dt_region_rd[i].sz, + chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr); } - pci_dbg(pdev, "Nr. IRQs:\t%u\n", dw->nr_irqs); + pci_dbg(pdev, "Nr. IRQs:\t%u\n", chip->nr_irqs); /* Validating if PCI interrupts were enabled */ if (!pci_dev_msi_enabled(pdev)) { @@ -345,10 +338,6 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -EPERM; } - dw->irq = devm_kcalloc(dev, nr_irqs, sizeof(*dw->irq), GFP_KERNEL); - if (!dw->irq) - return -ENOMEM; - /* Starting eDMA driver */ err = dw_edma_probe(chip); if (err) { diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 329fc2e57b70..b75fdaffad9a 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -7,6 +7,8 @@ */ #include <linux/bitfield.h> +#include <linux/irqreturn.h> +#include <linux/io-64-nonatomic-lo-hi.h> #include "dw-edma-core.h" #include "dw-edma-v0-core.h" @@ -25,7 +27,7 @@ enum dw_edma_control { static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) { - return dw->rg_region.vaddr; + return dw->chip->reg_base; } #define SET_32(dw, name, value) \ @@ -53,8 +55,6 @@ static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) SET_32(dw, rd_##name, value); \ } while (0) -#ifdef CONFIG_64BIT - #define SET_64(dw, name, value) \ writeq(value, &(__dw_regs(dw)->name)) @@ -80,8 +80,6 @@ static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) SET_64(dw, rd_##name, value); \ } while (0) -#endif /* CONFIG_64BIT */ - #define SET_COMPAT(dw, name, value) \ writel(value, &(__dw_regs(dw)->type.unroll.name)) @@ -96,7 +94,7 @@ static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) static inline struct dw_edma_v0_ch_regs __iomem * __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) { - if (dw->mf == EDMA_MF_EDMA_LEGACY) + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) return &(__dw_regs(dw)->type.legacy.ch); if (dir == EDMA_DIR_WRITE) @@ -108,7 +106,7 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, u32 value, void __iomem *addr) { - if (dw->mf == EDMA_MF_EDMA_LEGACY) { + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { u32 viewport_sel; unsigned long flags; @@ -133,7 +131,7 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, { u32 value; - if (dw->mf == EDMA_MF_EDMA_LEGACY) { + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { u32 viewport_sel; unsigned long flags; @@ -161,74 +159,8 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, #define GET_CH_32(dw, dir, ch, name) \ readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) -#define SET_LL_32(ll, value) \ - writel(value, ll) - -#ifdef CONFIG_64BIT - -static inline void writeq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, - u64 value, void __iomem *addr) -{ - if (dw->mf == EDMA_MF_EDMA_LEGACY) { - u32 viewport_sel; - unsigned long flags; - - raw_spin_lock_irqsave(&dw->lock, flags); - - viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); - if (dir == EDMA_DIR_READ) - viewport_sel |= BIT(31); - - writel(viewport_sel, - &(__dw_regs(dw)->type.legacy.viewport_sel)); - writeq(value, addr); - - raw_spin_unlock_irqrestore(&dw->lock, flags); - } else { - writeq(value, addr); - } -} - -static inline u64 readq_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, - const void __iomem *addr) -{ - u32 value; - - if (dw->mf == EDMA_MF_EDMA_LEGACY) { - u32 viewport_sel; - unsigned long flags; - - raw_spin_lock_irqsave(&dw->lock, flags); - - viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); - if (dir == EDMA_DIR_READ) - viewport_sel |= BIT(31); - - writel(viewport_sel, - &(__dw_regs(dw)->type.legacy.viewport_sel)); - value = readq(addr); - - raw_spin_unlock_irqrestore(&dw->lock, flags); - } else { - value = readq(addr); - } - - return value; -} - -#define SET_CH_64(dw, dir, ch, name, value) \ - writeq_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name)) - -#define GET_CH_64(dw, dir, ch, name) \ - readq_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) - -#define SET_LL_64(ll, value) \ - writeq(value, ll) - -#endif /* CONFIG_64BIT */ - /* eDMA management callbacks */ -void dw_edma_v0_core_off(struct dw_edma *dw) +static void dw_edma_v0_core_off(struct dw_edma *dw) { SET_BOTH_32(dw, int_mask, EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); @@ -237,7 +169,7 @@ void dw_edma_v0_core_off(struct dw_edma *dw) SET_BOTH_32(dw, engine_en, 0); } -u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) +static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { u32 num_ch; @@ -254,9 +186,9 @@ u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) return (u16)num_ch; } -enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) +static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; u32 tmp; tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK, @@ -270,94 +202,168 @@ enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) return DMA_ERROR; } -void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan) +static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; SET_RW_32(dw, chan->dir, int_clear, FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id))); } -void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) +static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; SET_RW_32(dw, chan->dir, int_clear, FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id))); } -u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir) +static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir) { return FIELD_GET(EDMA_V0_DONE_INT_MASK, GET_RW_32(dw, dir, int_status)); } -u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) +static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) { return FIELD_GET(EDMA_V0_ABORT_INT_MASK, GET_RW_32(dw, dir, int_status)); } +static irqreturn_t +dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, + dw_edma_handler_t done, dw_edma_handler_t abort) +{ + struct dw_edma *dw = dw_irq->dw; + unsigned long total, pos, val; + irqreturn_t ret = IRQ_NONE; + struct dw_edma_chan *chan; + unsigned long off; + u32 mask; + + if (dir == EDMA_DIR_WRITE) { + total = dw->wr_ch_cnt; + off = 0; + mask = dw_irq->wr_mask; + } else { + total = dw->rd_ch_cnt; + off = dw->wr_ch_cnt; + mask = dw_irq->rd_mask; + } + + val = dw_edma_v0_core_status_done_int(dw, dir); + val &= mask; + for_each_set_bit(pos, &val, total) { + chan = &dw->chan[pos + off]; + + dw_edma_v0_core_clear_done_int(chan); + done(chan); + + ret = IRQ_HANDLED; + } + + val = dw_edma_v0_core_status_abort_int(dw, dir); + val &= mask; + for_each_set_bit(pos, &val, total) { + chan = &dw->chan[pos + off]; + + dw_edma_v0_core_clear_abort_int(chan); + abort(chan); + + ret = IRQ_HANDLED; + } + + return ret; +} + +static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, + u32 control, u32 size, u64 sar, u64 dar) +{ + ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); + + if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; + + lli->control = control; + lli->transfer_size = size; + lli->sar.reg = sar; + lli->dar.reg = dar; + } else { + struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; + + writel(control, &lli->control); + writel(size, &lli->transfer_size); + writeq(sar, &lli->sar.reg); + writeq(dar, &lli->dar.reg); + } +} + +static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk, + int i, u32 control, u64 pointer) +{ + ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); + + if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; + + llp->control = control; + llp->llp.reg = pointer; + } else { + struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; + + writel(control, &llp->control); + writeq(pointer, &llp->llp.reg); + } +} + static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) { struct dw_edma_burst *child; - struct dw_edma_v0_lli __iomem *lli; - struct dw_edma_v0_llp __iomem *llp; + struct dw_edma_chan *chan = chunk->chan; u32 control = 0, i = 0; int j; - lli = chunk->ll_region.vaddr; - if (chunk->cb) control = DW_EDMA_V0_CB; j = chunk->bursts_alloc; list_for_each_entry(child, &chunk->burst->list, list) { j--; - if (!j) - control |= (DW_EDMA_V0_LIE | DW_EDMA_V0_RIE); + if (!j) { + control |= DW_EDMA_V0_LIE; + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + control |= DW_EDMA_V0_RIE; + } - /* Channel control */ - SET_LL_32(&lli[i].control, control); - /* Transfer size */ - SET_LL_32(&lli[i].transfer_size, child->sz); - /* SAR */ - #ifdef CONFIG_64BIT - SET_LL_64(&lli[i].sar.reg, child->sar); - #else /* CONFIG_64BIT */ - SET_LL_32(&lli[i].sar.lsb, lower_32_bits(child->sar)); - SET_LL_32(&lli[i].sar.msb, upper_32_bits(child->sar)); - #endif /* CONFIG_64BIT */ - /* DAR */ - #ifdef CONFIG_64BIT - SET_LL_64(&lli[i].dar.reg, child->dar); - #else /* CONFIG_64BIT */ - SET_LL_32(&lli[i].dar.lsb, lower_32_bits(child->dar)); - SET_LL_32(&lli[i].dar.msb, upper_32_bits(child->dar)); - #endif /* CONFIG_64BIT */ - i++; + dw_edma_v0_write_ll_data(chunk, i++, control, child->sz, + child->sar, child->dar); } - llp = (void __iomem *)&lli[i]; control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; if (!chunk->cb) control |= DW_EDMA_V0_CB; - /* Channel control */ - SET_LL_32(&llp->control, control); - /* Linked list */ - #ifdef CONFIG_64BIT - SET_LL_64(&llp->llp.reg, chunk->ll_region.paddr); - #else /* CONFIG_64BIT */ - SET_LL_32(&llp->llp.lsb, lower_32_bits(chunk->ll_region.paddr)); - SET_LL_32(&llp->llp.msb, upper_32_bits(chunk->ll_region.paddr)); - #endif /* CONFIG_64BIT */ + dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); +} + +static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +{ + /* + * In case of remote eDMA engine setup, the DW PCIe RP/EP internal + * configuration registers and application memory are normally accessed + * over different buses. Ensure LL-data reaches the memory before the + * doorbell register is toggled by issuing the dummy-read from the remote + * LL memory in a hope that the MRd TLP will return only after the + * last MWr TLP is completed + */ + if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chunk->ll_region.vaddr.io); } -void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) +static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; u32 tmp; dw_edma_v0_core_write_chunk(chunk); @@ -365,7 +371,7 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) if (first) { /* Enable engine */ SET_RW_32(dw, chan->dir, engine_en, BIT(0)); - if (dw->mf == EDMA_MF_HDMA_COMPAT) { + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { switch (chan->id) { case 0: SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, @@ -414,24 +420,23 @@ void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) SET_CH_32(dw, chan->dir, chan->id, ch_control1, (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); /* Linked list */ - #ifdef CONFIG_64BIT - SET_CH_64(dw, chan->dir, chan->id, llp.reg, - chunk->ll_region.paddr); - #else /* CONFIG_64BIT */ - SET_CH_32(dw, chan->dir, chan->id, llp.lsb, - lower_32_bits(chunk->ll_region.paddr)); - SET_CH_32(dw, chan->dir, chan->id, llp.msb, - upper_32_bits(chunk->ll_region.paddr)); - #endif /* CONFIG_64BIT */ + /* llp is not aligned on 64bit -> keep 32bit accesses */ + SET_CH_32(dw, chan->dir, chan->id, llp.lsb, + lower_32_bits(chunk->ll_region.paddr)); + SET_CH_32(dw, chan->dir, chan->id, llp.msb, + upper_32_bits(chunk->ll_region.paddr)); } + + dw_edma_v0_sync_ll_data(chunk); + /* Doorbell */ SET_RW_32(dw, chan->dir, doorbell, FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); } -int dw_edma_v0_core_device_config(struct dw_edma_chan *chan) +static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->chip->dw; + struct dw_edma *dw = chan->dw; u32 tmp = 0; /* MSI done addr - low, high */ @@ -496,17 +501,25 @@ int dw_edma_v0_core_device_config(struct dw_edma_chan *chan) SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp); break; } - - return 0; } /* eDMA debugfs callbacks */ -void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip) +static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) { - dw_edma_v0_debugfs_on(chip); + dw_edma_v0_debugfs_on(dw); } -void dw_edma_v0_core_debugfs_off(struct dw_edma_chip *chip) +static const struct dw_edma_core_ops dw_edma_v0_core = { + .off = dw_edma_v0_core_off, + .ch_count = dw_edma_v0_core_ch_count, + .ch_status = dw_edma_v0_core_ch_status, + .handle_int = dw_edma_v0_core_handle_int, + .start = dw_edma_v0_core_start, + .ch_config = dw_edma_v0_core_ch_config, + .debugfs_on = dw_edma_v0_core_debugfs_on, +}; + +void dw_edma_v0_core_register(struct dw_edma *dw) { - dw_edma_v0_debugfs_off(chip); + dw->core = &dw_edma_v0_core; } diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.h b/drivers/dma/dw-edma/dw-edma-v0-core.h index 2afa626b8300..04a882222f99 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.h +++ b/drivers/dma/dw-edma/dw-edma-v0-core.h @@ -11,18 +11,7 @@ #include <linux/dma/edma.h> -/* eDMA management callbacks */ -void dw_edma_v0_core_off(struct dw_edma *chan); -u16 dw_edma_v0_core_ch_count(struct dw_edma *chan, enum dw_edma_dir dir); -enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan); -void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan); -void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan); -u32 dw_edma_v0_core_status_done_int(struct dw_edma *chan, enum dw_edma_dir dir); -u32 dw_edma_v0_core_status_abort_int(struct dw_edma *chan, enum dw_edma_dir dir); -void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first); -int dw_edma_v0_core_device_config(struct dw_edma_chan *chan); -/* eDMA debug fs callbacks */ -void dw_edma_v0_core_debugfs_on(struct dw_edma_chip *chip); -void dw_edma_v0_core_debugfs_off(struct dw_edma_chip *chip); +/* eDMA core register */ +void dw_edma_v0_core_register(struct dw_edma *dw); #endif /* _DW_EDMA_V0_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c index 4b3bcffd15ef..406f169b09a7 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-debugfs.c +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.c @@ -13,76 +13,79 @@ #include "dw-edma-v0-regs.h" #include "dw-edma-core.h" -#define REGS_ADDR(name) \ - ((void __force *)®s->name) -#define REGISTER(name) \ - { #name, REGS_ADDR(name) } - -#define WR_REGISTER(name) \ - { #name, REGS_ADDR(wr_##name) } -#define RD_REGISTER(name) \ - { #name, REGS_ADDR(rd_##name) } - -#define WR_REGISTER_LEGACY(name) \ - { #name, REGS_ADDR(type.legacy.wr_##name) } +#define REGS_ADDR(dw, name) \ + ({ \ + struct dw_edma_v0_regs __iomem *__regs = (dw)->chip->reg_base; \ + \ + (void __iomem *)&__regs->name; \ + }) + +#define REGS_CH_ADDR(dw, name, _dir, _ch) \ + ({ \ + struct dw_edma_v0_ch_regs __iomem *__ch_regs; \ + \ + if ((dw)->chip->mf == EDMA_MF_EDMA_LEGACY) \ + __ch_regs = REGS_ADDR(dw, type.legacy.ch); \ + else if (_dir == EDMA_DIR_READ) \ + __ch_regs = REGS_ADDR(dw, type.unroll.ch[_ch].rd); \ + else \ + __ch_regs = REGS_ADDR(dw, type.unroll.ch[_ch].wr); \ + \ + (void __iomem *)&__ch_regs->name; \ + }) + +#define REGISTER(dw, name) \ + { dw, #name, REGS_ADDR(dw, name) } + +#define CTX_REGISTER(dw, name, dir, ch) \ + { dw, #name, REGS_CH_ADDR(dw, name, dir, ch), dir, ch } + +#define WR_REGISTER(dw, name) \ + { dw, #name, REGS_ADDR(dw, wr_##name) } +#define RD_REGISTER(dw, name) \ + { dw, #name, REGS_ADDR(dw, rd_##name) } + +#define WR_REGISTER_LEGACY(dw, name) \ + { dw, #name, REGS_ADDR(dw, type.legacy.wr_##name) } #define RD_REGISTER_LEGACY(name) \ - { #name, REGS_ADDR(type.legacy.rd_##name) } + { dw, #name, REGS_ADDR(dw, type.legacy.rd_##name) } -#define WR_REGISTER_UNROLL(name) \ - { #name, REGS_ADDR(type.unroll.wr_##name) } -#define RD_REGISTER_UNROLL(name) \ - { #name, REGS_ADDR(type.unroll.rd_##name) } +#define WR_REGISTER_UNROLL(dw, name) \ + { dw, #name, REGS_ADDR(dw, type.unroll.wr_##name) } +#define RD_REGISTER_UNROLL(dw, name) \ + { dw, #name, REGS_ADDR(dw, type.unroll.rd_##name) } #define WRITE_STR "write" #define READ_STR "read" #define CHANNEL_STR "channel" #define REGISTERS_STR "registers" -static struct dw_edma *dw; -static struct dw_edma_v0_regs __iomem *regs; - -static struct { - void __iomem *start; - void __iomem *end; -} lim[2][EDMA_V0_MAX_NR_CH]; - -struct debugfs_entries { +struct dw_edma_debugfs_entry { + struct dw_edma *dw; const char *name; - dma_addr_t *reg; + void __iomem *reg; + enum dw_edma_dir dir; + u16 ch; }; static int dw_edma_debugfs_u32_get(void *data, u64 *val) { - void __iomem *reg = (void __force __iomem *)data; - if (dw->mf == EDMA_MF_EDMA_LEGACY && - reg >= (void __iomem *)®s->type.legacy.ch) { - void __iomem *ptr = ®s->type.legacy.ch; - u32 viewport_sel = 0; + struct dw_edma_debugfs_entry *entry = data; + struct dw_edma *dw = entry->dw; + void __iomem *reg = entry->reg; + + if (dw->chip->mf == EDMA_MF_EDMA_LEGACY && + reg >= REGS_ADDR(dw, type.legacy.ch)) { unsigned long flags; - u16 ch; - - for (ch = 0; ch < dw->wr_ch_cnt; ch++) - if (lim[0][ch].start >= reg && reg < lim[0][ch].end) { - ptr += (reg - lim[0][ch].start); - goto legacy_sel_wr; - } - - for (ch = 0; ch < dw->rd_ch_cnt; ch++) - if (lim[1][ch].start >= reg && reg < lim[1][ch].end) { - ptr += (reg - lim[1][ch].start); - goto legacy_sel_rd; - } - - return 0; -legacy_sel_rd: - viewport_sel = BIT(31); -legacy_sel_wr: - viewport_sel |= FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); + u32 viewport_sel; + + viewport_sel = entry->dir == EDMA_DIR_READ ? BIT(31) : 0; + viewport_sel |= FIELD_PREP(EDMA_V0_VIEWPORT_MASK, entry->ch); raw_spin_lock_irqsave(&dw->lock, flags); - writel(viewport_sel, ®s->type.legacy.viewport_sel); - *val = readl(ptr); + writel(viewport_sel, REGS_ADDR(dw, type.legacy.viewport_sel)); + *val = readl(reg); raw_spin_unlock_irqrestore(&dw->lock, flags); } else { @@ -93,222 +96,197 @@ legacy_sel_wr: } DEFINE_DEBUGFS_ATTRIBUTE(fops_x32, dw_edma_debugfs_u32_get, NULL, "0x%08llx\n"); -static void dw_edma_debugfs_create_x32(const struct debugfs_entries entries[], - int nr_entries, struct dentry *dir) +static void dw_edma_debugfs_create_x32(struct dw_edma *dw, + const struct dw_edma_debugfs_entry ini[], + int nr_entries, struct dentry *dent) { + struct dw_edma_debugfs_entry *entries; int i; + entries = devm_kcalloc(dw->chip->dev, nr_entries, sizeof(*entries), + GFP_KERNEL); + if (!entries) + return; + for (i = 0; i < nr_entries; i++) { - if (!debugfs_create_file_unsafe(entries[i].name, 0444, dir, - entries[i].reg, &fops_x32)) - break; + entries[i] = ini[i]; + + debugfs_create_file_unsafe(entries[i].name, 0444, dent, + &entries[i], &fops_x32); } } -static void dw_edma_debugfs_regs_ch(struct dw_edma_v0_ch_regs __iomem *regs, - struct dentry *dir) +static void dw_edma_debugfs_regs_ch(struct dw_edma *dw, enum dw_edma_dir dir, + u16 ch, struct dentry *dent) { - int nr_entries; - const struct debugfs_entries debugfs_regs[] = { - REGISTER(ch_control1), - REGISTER(ch_control2), - REGISTER(transfer_size), - REGISTER(sar.lsb), - REGISTER(sar.msb), - REGISTER(dar.lsb), - REGISTER(dar.msb), - REGISTER(llp.lsb), - REGISTER(llp.msb), + struct dw_edma_debugfs_entry debugfs_regs[] = { + CTX_REGISTER(dw, ch_control1, dir, ch), + CTX_REGISTER(dw, ch_control2, dir, ch), + CTX_REGISTER(dw, transfer_size, dir, ch), + CTX_REGISTER(dw, sar.lsb, dir, ch), + CTX_REGISTER(dw, sar.msb, dir, ch), + CTX_REGISTER(dw, dar.lsb, dir, ch), + CTX_REGISTER(dw, dar.msb, dir, ch), + CTX_REGISTER(dw, llp.lsb, dir, ch), + CTX_REGISTER(dw, llp.msb, dir, ch), }; + int nr_entries; nr_entries = ARRAY_SIZE(debugfs_regs); - dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, dir); + dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, dent); } -static void dw_edma_debugfs_regs_wr(struct dentry *dir) +static noinline_for_stack void +dw_edma_debugfs_regs_wr(struct dw_edma *dw, struct dentry *dent) { - const struct debugfs_entries debugfs_regs[] = { + const struct dw_edma_debugfs_entry debugfs_regs[] = { /* eDMA global registers */ - WR_REGISTER(engine_en), - WR_REGISTER(doorbell), - WR_REGISTER(ch_arb_weight.lsb), - WR_REGISTER(ch_arb_weight.msb), + WR_REGISTER(dw, engine_en), + WR_REGISTER(dw, doorbell), + WR_REGISTER(dw, ch_arb_weight.lsb), + WR_REGISTER(dw, ch_arb_weight.msb), /* eDMA interrupts registers */ - WR_REGISTER(int_status), - WR_REGISTER(int_mask), - WR_REGISTER(int_clear), - WR_REGISTER(err_status), - WR_REGISTER(done_imwr.lsb), - WR_REGISTER(done_imwr.msb), - WR_REGISTER(abort_imwr.lsb), - WR_REGISTER(abort_imwr.msb), - WR_REGISTER(ch01_imwr_data), - WR_REGISTER(ch23_imwr_data), - WR_REGISTER(ch45_imwr_data), - WR_REGISTER(ch67_imwr_data), - WR_REGISTER(linked_list_err_en), + WR_REGISTER(dw, int_status), + WR_REGISTER(dw, int_mask), + WR_REGISTER(dw, int_clear), + WR_REGISTER(dw, err_status), + WR_REGISTER(dw, done_imwr.lsb), + WR_REGISTER(dw, done_imwr.msb), + WR_REGISTER(dw, abort_imwr.lsb), + WR_REGISTER(dw, abort_imwr.msb), + WR_REGISTER(dw, ch01_imwr_data), + WR_REGISTER(dw, ch23_imwr_data), + WR_REGISTER(dw, ch45_imwr_data), + WR_REGISTER(dw, ch67_imwr_data), + WR_REGISTER(dw, linked_list_err_en), }; - const struct debugfs_entries debugfs_unroll_regs[] = { + const struct dw_edma_debugfs_entry debugfs_unroll_regs[] = { /* eDMA channel context grouping */ - WR_REGISTER_UNROLL(engine_chgroup), - WR_REGISTER_UNROLL(engine_hshake_cnt.lsb), - WR_REGISTER_UNROLL(engine_hshake_cnt.msb), - WR_REGISTER_UNROLL(ch0_pwr_en), - WR_REGISTER_UNROLL(ch1_pwr_en), - WR_REGISTER_UNROLL(ch2_pwr_en), - WR_REGISTER_UNROLL(ch3_pwr_en), - WR_REGISTER_UNROLL(ch4_pwr_en), - WR_REGISTER_UNROLL(ch5_pwr_en), - WR_REGISTER_UNROLL(ch6_pwr_en), - WR_REGISTER_UNROLL(ch7_pwr_en), + WR_REGISTER_UNROLL(dw, engine_chgroup), + WR_REGISTER_UNROLL(dw, engine_hshake_cnt.lsb), + WR_REGISTER_UNROLL(dw, engine_hshake_cnt.msb), + WR_REGISTER_UNROLL(dw, ch0_pwr_en), + WR_REGISTER_UNROLL(dw, ch1_pwr_en), + WR_REGISTER_UNROLL(dw, ch2_pwr_en), + WR_REGISTER_UNROLL(dw, ch3_pwr_en), + WR_REGISTER_UNROLL(dw, ch4_pwr_en), + WR_REGISTER_UNROLL(dw, ch5_pwr_en), + WR_REGISTER_UNROLL(dw, ch6_pwr_en), + WR_REGISTER_UNROLL(dw, ch7_pwr_en), }; - struct dentry *regs_dir, *ch_dir; + struct dentry *regs_dent, *ch_dent; int nr_entries, i; - char name[16]; + char name[32]; - regs_dir = debugfs_create_dir(WRITE_STR, dir); - if (!regs_dir) - return; + regs_dent = debugfs_create_dir(WRITE_STR, dent); nr_entries = ARRAY_SIZE(debugfs_regs); - dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); + dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, regs_dent); - if (dw->mf == EDMA_MF_HDMA_COMPAT) { + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { nr_entries = ARRAY_SIZE(debugfs_unroll_regs); - dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries, - regs_dir); + dw_edma_debugfs_create_x32(dw, debugfs_unroll_regs, nr_entries, + regs_dent); } for (i = 0; i < dw->wr_ch_cnt; i++) { snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i); - ch_dir = debugfs_create_dir(name, regs_dir); - if (!ch_dir) - return; - - dw_edma_debugfs_regs_ch(®s->type.unroll.ch[i].wr, ch_dir); + ch_dent = debugfs_create_dir(name, regs_dent); - lim[0][i].start = ®s->type.unroll.ch[i].wr; - lim[0][i].end = ®s->type.unroll.ch[i].padding_1[0]; + dw_edma_debugfs_regs_ch(dw, EDMA_DIR_WRITE, i, ch_dent); } } -static void dw_edma_debugfs_regs_rd(struct dentry *dir) +static noinline_for_stack void dw_edma_debugfs_regs_rd(struct dw_edma *dw, + struct dentry *dent) { - const struct debugfs_entries debugfs_regs[] = { + const struct dw_edma_debugfs_entry debugfs_regs[] = { /* eDMA global registers */ - RD_REGISTER(engine_en), - RD_REGISTER(doorbell), - RD_REGISTER(ch_arb_weight.lsb), - RD_REGISTER(ch_arb_weight.msb), + RD_REGISTER(dw, engine_en), + RD_REGISTER(dw, doorbell), + RD_REGISTER(dw, ch_arb_weight.lsb), + RD_REGISTER(dw, ch_arb_weight.msb), /* eDMA interrupts registers */ - RD_REGISTER(int_status), - RD_REGISTER(int_mask), - RD_REGISTER(int_clear), - RD_REGISTER(err_status.lsb), - RD_REGISTER(err_status.msb), - RD_REGISTER(linked_list_err_en), - RD_REGISTER(done_imwr.lsb), - RD_REGISTER(done_imwr.msb), - RD_REGISTER(abort_imwr.lsb), - RD_REGISTER(abort_imwr.msb), - RD_REGISTER(ch01_imwr_data), - RD_REGISTER(ch23_imwr_data), - RD_REGISTER(ch45_imwr_data), - RD_REGISTER(ch67_imwr_data), + RD_REGISTER(dw, int_status), + RD_REGISTER(dw, int_mask), + RD_REGISTER(dw, int_clear), + RD_REGISTER(dw, err_status.lsb), + RD_REGISTER(dw, err_status.msb), + RD_REGISTER(dw, linked_list_err_en), + RD_REGISTER(dw, done_imwr.lsb), + RD_REGISTER(dw, done_imwr.msb), + RD_REGISTER(dw, abort_imwr.lsb), + RD_REGISTER(dw, abort_imwr.msb), + RD_REGISTER(dw, ch01_imwr_data), + RD_REGISTER(dw, ch23_imwr_data), + RD_REGISTER(dw, ch45_imwr_data), + RD_REGISTER(dw, ch67_imwr_data), }; - const struct debugfs_entries debugfs_unroll_regs[] = { + const struct dw_edma_debugfs_entry debugfs_unroll_regs[] = { /* eDMA channel context grouping */ - RD_REGISTER_UNROLL(engine_chgroup), - RD_REGISTER_UNROLL(engine_hshake_cnt.lsb), - RD_REGISTER_UNROLL(engine_hshake_cnt.msb), - RD_REGISTER_UNROLL(ch0_pwr_en), - RD_REGISTER_UNROLL(ch1_pwr_en), - RD_REGISTER_UNROLL(ch2_pwr_en), - RD_REGISTER_UNROLL(ch3_pwr_en), - RD_REGISTER_UNROLL(ch4_pwr_en), - RD_REGISTER_UNROLL(ch5_pwr_en), - RD_REGISTER_UNROLL(ch6_pwr_en), - RD_REGISTER_UNROLL(ch7_pwr_en), + RD_REGISTER_UNROLL(dw, engine_chgroup), + RD_REGISTER_UNROLL(dw, engine_hshake_cnt.lsb), + RD_REGISTER_UNROLL(dw, engine_hshake_cnt.msb), + RD_REGISTER_UNROLL(dw, ch0_pwr_en), + RD_REGISTER_UNROLL(dw, ch1_pwr_en), + RD_REGISTER_UNROLL(dw, ch2_pwr_en), + RD_REGISTER_UNROLL(dw, ch3_pwr_en), + RD_REGISTER_UNROLL(dw, ch4_pwr_en), + RD_REGISTER_UNROLL(dw, ch5_pwr_en), + RD_REGISTER_UNROLL(dw, ch6_pwr_en), + RD_REGISTER_UNROLL(dw, ch7_pwr_en), }; - struct dentry *regs_dir, *ch_dir; + struct dentry *regs_dent, *ch_dent; int nr_entries, i; - char name[16]; + char name[32]; - regs_dir = debugfs_create_dir(READ_STR, dir); - if (!regs_dir) - return; + regs_dent = debugfs_create_dir(READ_STR, dent); nr_entries = ARRAY_SIZE(debugfs_regs); - dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); + dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, regs_dent); - if (dw->mf == EDMA_MF_HDMA_COMPAT) { + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { nr_entries = ARRAY_SIZE(debugfs_unroll_regs); - dw_edma_debugfs_create_x32(debugfs_unroll_regs, nr_entries, - regs_dir); + dw_edma_debugfs_create_x32(dw, debugfs_unroll_regs, nr_entries, + regs_dent); } for (i = 0; i < dw->rd_ch_cnt; i++) { snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i); - ch_dir = debugfs_create_dir(name, regs_dir); - if (!ch_dir) - return; - - dw_edma_debugfs_regs_ch(®s->type.unroll.ch[i].rd, ch_dir); + ch_dent = debugfs_create_dir(name, regs_dent); - lim[1][i].start = ®s->type.unroll.ch[i].rd; - lim[1][i].end = ®s->type.unroll.ch[i].padding_2[0]; + dw_edma_debugfs_regs_ch(dw, EDMA_DIR_READ, i, ch_dent); } } -static void dw_edma_debugfs_regs(void) +static void dw_edma_debugfs_regs(struct dw_edma *dw) { - const struct debugfs_entries debugfs_regs[] = { - REGISTER(ctrl_data_arb_prior), - REGISTER(ctrl), + const struct dw_edma_debugfs_entry debugfs_regs[] = { + REGISTER(dw, ctrl_data_arb_prior), + REGISTER(dw, ctrl), }; - struct dentry *regs_dir; + struct dentry *regs_dent; int nr_entries; - regs_dir = debugfs_create_dir(REGISTERS_STR, dw->debugfs); - if (!regs_dir) - return; + regs_dent = debugfs_create_dir(REGISTERS_STR, dw->dma.dbg_dev_root); nr_entries = ARRAY_SIZE(debugfs_regs); - dw_edma_debugfs_create_x32(debugfs_regs, nr_entries, regs_dir); + dw_edma_debugfs_create_x32(dw, debugfs_regs, nr_entries, regs_dent); - dw_edma_debugfs_regs_wr(regs_dir); - dw_edma_debugfs_regs_rd(regs_dir); + dw_edma_debugfs_regs_wr(dw, regs_dent); + dw_edma_debugfs_regs_rd(dw, regs_dent); } -void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip) +void dw_edma_v0_debugfs_on(struct dw_edma *dw) { - dw = chip->dw; - if (!dw) - return; - - regs = dw->rg_region.vaddr; - if (!regs) - return; - - dw->debugfs = debugfs_create_dir(dw->name, NULL); - if (!dw->debugfs) + if (!debugfs_initialized()) return; - debugfs_create_u32("mf", 0444, dw->debugfs, &dw->mf); - debugfs_create_u16("wr_ch_cnt", 0444, dw->debugfs, &dw->wr_ch_cnt); - debugfs_create_u16("rd_ch_cnt", 0444, dw->debugfs, &dw->rd_ch_cnt); - - dw_edma_debugfs_regs(); -} - -void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip) -{ - dw = chip->dw; - if (!dw) - return; + debugfs_create_u32("mf", 0444, dw->dma.dbg_dev_root, &dw->chip->mf); + debugfs_create_u16("wr_ch_cnt", 0444, dw->dma.dbg_dev_root, &dw->wr_ch_cnt); + debugfs_create_u16("rd_ch_cnt", 0444, dw->dma.dbg_dev_root, &dw->rd_ch_cnt); - debugfs_remove_recursive(dw->debugfs); - dw->debugfs = NULL; + dw_edma_debugfs_regs(dw); } diff --git a/drivers/dma/dw-edma/dw-edma-v0-debugfs.h b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h index d0ff25a9ea5c..fb3342d97d6d 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-debugfs.h +++ b/drivers/dma/dw-edma/dw-edma-v0-debugfs.h @@ -12,14 +12,9 @@ #include <linux/dma/edma.h> #ifdef CONFIG_DEBUG_FS -void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip); -void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip); +void dw_edma_v0_debugfs_on(struct dw_edma *dw); #else -static inline void dw_edma_v0_debugfs_on(struct dw_edma_chip *chip) -{ -} - -static inline void dw_edma_v0_debugfs_off(struct dw_edma_chip *chip) +static inline void dw_edma_v0_debugfs_on(struct dw_edma *dw) { } #endif /* CONFIG_DEBUG_FS */ diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c new file mode 100644 index 000000000000..e3f8db4fe909 --- /dev/null +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023 Cai Huoqing + * Synopsys DesignWare HDMA v0 core + */ + +#include <linux/bitfield.h> +#include <linux/irqreturn.h> +#include <linux/io-64-nonatomic-lo-hi.h> + +#include "dw-edma-core.h" +#include "dw-hdma-v0-core.h" +#include "dw-hdma-v0-regs.h" +#include "dw-hdma-v0-debugfs.h" + +enum dw_hdma_control { + DW_HDMA_V0_CB = BIT(0), + DW_HDMA_V0_TCB = BIT(1), + DW_HDMA_V0_LLP = BIT(2), + DW_HDMA_V0_LWIE = BIT(3), + DW_HDMA_V0_RWIE = BIT(4), + DW_HDMA_V0_CCS = BIT(8), + DW_HDMA_V0_LLE = BIT(9), +}; + +static inline struct dw_hdma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) +{ + return dw->chip->reg_base; +} + +static inline struct dw_hdma_v0_ch_regs __iomem * +__dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) +{ + if (dir == EDMA_DIR_WRITE) + return &(__dw_regs(dw)->ch[ch].wr); + else + return &(__dw_regs(dw)->ch[ch].rd); +} + +#define SET_CH_32(dw, dir, ch, name, value) \ + writel(value, &(__dw_ch_regs(dw, dir, ch)->name)) + +#define GET_CH_32(dw, dir, ch, name) \ + readl(&(__dw_ch_regs(dw, dir, ch)->name)) + +#define SET_BOTH_CH_32(dw, ch, name, value) \ + do { \ + writel(value, &(__dw_ch_regs(dw, EDMA_DIR_WRITE, ch)->name)); \ + writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name)); \ + } while (0) + +/* HDMA management callbacks */ +static void dw_hdma_v0_core_off(struct dw_edma *dw) +{ + int id; + + for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) { + SET_BOTH_CH_32(dw, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_BOTH_CH_32(dw, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_BOTH_CH_32(dw, id, ch_en, 0); + } +} + +static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) +{ + /* + * The HDMA IP have no way to know the number of hardware channels + * available, we set it to maximum channels and let the platform + * set the right number of channels. + */ + return HDMA_V0_MAX_NR_CH; +} + +static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + u32 tmp; + + tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK, + GET_CH_32(dw, chan->id, chan->dir, ch_stat)); + + if (tmp == 1) + return DMA_IN_PROGRESS; + else if (tmp == 3) + return DMA_COMPLETE; + else + return DMA_ERROR; +} + +static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK); +} + +static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_ABORT_INT_MASK); +} + +static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + return GET_CH_32(dw, chan->dir, chan->id, int_stat); +} + +static irqreturn_t +dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, + dw_edma_handler_t done, dw_edma_handler_t abort) +{ + struct dw_edma *dw = dw_irq->dw; + unsigned long total, pos, val; + irqreturn_t ret = IRQ_NONE; + struct dw_edma_chan *chan; + unsigned long off, mask; + + if (dir == EDMA_DIR_WRITE) { + total = dw->wr_ch_cnt; + off = 0; + mask = dw_irq->wr_mask; + } else { + total = dw->rd_ch_cnt; + off = dw->wr_ch_cnt; + mask = dw_irq->rd_mask; + } + + for_each_set_bit(pos, &mask, total) { + chan = &dw->chan[pos + off]; + + val = dw_hdma_v0_core_status_int(chan); + if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) { + dw_hdma_v0_core_clear_done_int(chan); + done(chan); + + ret = IRQ_HANDLED; + } + + if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) { + dw_hdma_v0_core_clear_abort_int(chan); + abort(chan); + + ret = IRQ_HANDLED; + } + } + + return ret; +} + +static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, + u32 control, u32 size, u64 sar, u64 dar) +{ + ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); + + if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; + + lli->control = control; + lli->transfer_size = size; + lli->sar.reg = sar; + lli->dar.reg = dar; + } else { + struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; + + writel(control, &lli->control); + writel(size, &lli->transfer_size); + writeq(sar, &lli->sar.reg); + writeq(dar, &lli->dar.reg); + } +} + +static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, + int i, u32 control, u64 pointer) +{ + ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); + + if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; + + llp->control = control; + llp->llp.reg = pointer; + } else { + struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; + + writel(control, &llp->control); + writeq(pointer, &llp->llp.reg); + } +} + +static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) +{ + struct dw_edma_burst *child; + u32 control = 0, i = 0; + + if (chunk->cb) + control = DW_HDMA_V0_CB; + + list_for_each_entry(child, &chunk->burst->list, list) + dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz, + child->sar, child->dar); + + control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; + if (!chunk->cb) + control |= DW_HDMA_V0_CB; + + dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); +} + +static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +{ + /* + * In case of remote HDMA engine setup, the DW PCIe RP/EP internal + * configuration registers and application memory are normally accessed + * over different buses. Ensure LL-data reaches the memory before the + * doorbell register is toggled by issuing the dummy-read from the remote + * LL memory in a hope that the MRd TLP will return only after the + * last MWr TLP is completed + */ + if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chunk->ll_region.vaddr.io); +} + +static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) +{ + struct dw_edma_chan *chan = chunk->chan; + struct dw_edma *dw = chan->dw; + u32 tmp; + + dw_hdma_v0_core_write_chunk(chunk); + + if (first) { + /* Enable engine */ + SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); + /* Interrupt unmask - stop, abort */ + tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); + tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + /* Interrupt enable - stop, abort */ + tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; + if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN; + SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); + /* Channel control */ + SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); + /* Linked list */ + /* llp is not aligned on 64bit -> keep 32bit accesses */ + SET_CH_32(dw, chan->dir, chan->id, llp.lsb, + lower_32_bits(chunk->ll_region.paddr)); + SET_CH_32(dw, chan->dir, chan->id, llp.msb, + upper_32_bits(chunk->ll_region.paddr)); + } + /* Set consumer cycle */ + SET_CH_32(dw, chan->dir, chan->id, cycle_sync, + HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); + + dw_hdma_v0_sync_ll_data(chunk); + + /* Doorbell */ + SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); +} + +static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + /* MSI done addr - low, high */ + SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo); + SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi); + /* MSI abort addr - low, high */ + SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo); + SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi); + /* config MSI data */ + SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data); +} + +/* HDMA debugfs callbacks */ +static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw) +{ + dw_hdma_v0_debugfs_on(dw); +} + +static const struct dw_edma_core_ops dw_hdma_v0_core = { + .off = dw_hdma_v0_core_off, + .ch_count = dw_hdma_v0_core_ch_count, + .ch_status = dw_hdma_v0_core_ch_status, + .handle_int = dw_hdma_v0_core_handle_int, + .start = dw_hdma_v0_core_start, + .ch_config = dw_hdma_v0_core_ch_config, + .debugfs_on = dw_hdma_v0_core_debugfs_on, +}; + +void dw_hdma_v0_core_register(struct dw_edma *dw) +{ + dw->core = &dw_hdma_v0_core; +} diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.h b/drivers/dma/dw-edma/dw-hdma-v0-core.h new file mode 100644 index 000000000000..c373b4f0bd8a --- /dev/null +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2023 Cai Huoqing + * Synopsys DesignWare HDMA v0 core + * + * Author: Cai Huoqing <cai.huoqing@linux.dev> + */ + +#ifndef _DW_HDMA_V0_CORE_H +#define _DW_HDMA_V0_CORE_H + +#include <linux/dma/edma.h> + +/* HDMA core register */ +void dw_hdma_v0_core_register(struct dw_edma *dw); + +#endif /* _DW_HDMA_V0_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c new file mode 100644 index 000000000000..dcdc57fe976c --- /dev/null +++ b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023 Cai Huoqing + * Synopsys DesignWare HDMA v0 debugfs + * + * Author: Cai Huoqing <cai.huoqing@linux.dev> + */ + +#include <linux/debugfs.h> +#include <linux/bitfield.h> + +#include "dw-hdma-v0-debugfs.h" +#include "dw-hdma-v0-regs.h" +#include "dw-edma-core.h" + +#define REGS_ADDR(dw, name) \ + ({ \ + struct dw_hdma_v0_regs __iomem *__regs = (dw)->chip->reg_base; \ + \ + (void __iomem *)&__regs->name; \ + }) + +#define REGS_CH_ADDR(dw, name, _dir, _ch) \ + ({ \ + struct dw_hdma_v0_ch_regs __iomem *__ch_regs; \ + \ + if (_dir == EDMA_DIR_READ) \ + __ch_regs = REGS_ADDR(dw, ch[_ch].rd); \ + else \ + __ch_regs = REGS_ADDR(dw, ch[_ch].wr); \ + \ + (void __iomem *)&__ch_regs->name; \ + }) + +#define CTX_REGISTER(dw, name, dir, ch) \ + {#name, REGS_CH_ADDR(dw, name, dir, ch)} + +#define WRITE_STR "write" +#define READ_STR "read" +#define CHANNEL_STR "channel" +#define REGISTERS_STR "registers" + +struct dw_hdma_debugfs_entry { + const char *name; + void __iomem *reg; +}; + +static int dw_hdma_debugfs_u32_get(void *data, u64 *val) +{ + struct dw_hdma_debugfs_entry *entry = data; + void __iomem *reg = entry->reg; + + *val = readl(reg); + + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_x32, dw_hdma_debugfs_u32_get, NULL, "0x%08llx\n"); + +static void dw_hdma_debugfs_create_x32(struct dw_edma *dw, + const struct dw_hdma_debugfs_entry ini[], + int nr_entries, struct dentry *dent) +{ + struct dw_hdma_debugfs_entry *entries; + int i; + + entries = devm_kcalloc(dw->chip->dev, nr_entries, sizeof(*entries), + GFP_KERNEL); + if (!entries) + return; + + for (i = 0; i < nr_entries; i++) { + entries[i] = ini[i]; + + debugfs_create_file_unsafe(entries[i].name, 0444, dent, + &entries[i], &fops_x32); + } +} + +static void dw_hdma_debugfs_regs_ch(struct dw_edma *dw, enum dw_edma_dir dir, + u16 ch, struct dentry *dent) +{ + const struct dw_hdma_debugfs_entry debugfs_regs[] = { + CTX_REGISTER(dw, ch_en, dir, ch), + CTX_REGISTER(dw, doorbell, dir, ch), + CTX_REGISTER(dw, prefetch, dir, ch), + CTX_REGISTER(dw, handshake, dir, ch), + CTX_REGISTER(dw, llp.lsb, dir, ch), + CTX_REGISTER(dw, llp.msb, dir, ch), + CTX_REGISTER(dw, cycle_sync, dir, ch), + CTX_REGISTER(dw, transfer_size, dir, ch), + CTX_REGISTER(dw, sar.lsb, dir, ch), + CTX_REGISTER(dw, sar.msb, dir, ch), + CTX_REGISTER(dw, dar.lsb, dir, ch), + CTX_REGISTER(dw, dar.msb, dir, ch), + CTX_REGISTER(dw, watermark_en, dir, ch), + CTX_REGISTER(dw, control1, dir, ch), + CTX_REGISTER(dw, func_num, dir, ch), + CTX_REGISTER(dw, qos, dir, ch), + CTX_REGISTER(dw, ch_stat, dir, ch), + CTX_REGISTER(dw, int_stat, dir, ch), + CTX_REGISTER(dw, int_setup, dir, ch), + CTX_REGISTER(dw, int_clear, dir, ch), + CTX_REGISTER(dw, msi_stop.lsb, dir, ch), + CTX_REGISTER(dw, msi_stop.msb, dir, ch), + CTX_REGISTER(dw, msi_watermark.lsb, dir, ch), + CTX_REGISTER(dw, msi_watermark.msb, dir, ch), + CTX_REGISTER(dw, msi_abort.lsb, dir, ch), + CTX_REGISTER(dw, msi_abort.msb, dir, ch), + CTX_REGISTER(dw, msi_msgdata, dir, ch), + }; + int nr_entries = ARRAY_SIZE(debugfs_regs); + + dw_hdma_debugfs_create_x32(dw, debugfs_regs, nr_entries, dent); +} + +static void dw_hdma_debugfs_regs_wr(struct dw_edma *dw, struct dentry *dent) +{ + struct dentry *regs_dent, *ch_dent; + char name[32]; + int i; + + regs_dent = debugfs_create_dir(WRITE_STR, dent); + + for (i = 0; i < dw->wr_ch_cnt; i++) { + snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i); + + ch_dent = debugfs_create_dir(name, regs_dent); + + dw_hdma_debugfs_regs_ch(dw, EDMA_DIR_WRITE, i, ch_dent); + } +} + +static void dw_hdma_debugfs_regs_rd(struct dw_edma *dw, struct dentry *dent) +{ + struct dentry *regs_dent, *ch_dent; + char name[32]; + int i; + + regs_dent = debugfs_create_dir(READ_STR, dent); + + for (i = 0; i < dw->rd_ch_cnt; i++) { + snprintf(name, sizeof(name), "%s:%d", CHANNEL_STR, i); + + ch_dent = debugfs_create_dir(name, regs_dent); + + dw_hdma_debugfs_regs_ch(dw, EDMA_DIR_READ, i, ch_dent); + } +} + +static void dw_hdma_debugfs_regs(struct dw_edma *dw) +{ + struct dentry *regs_dent; + + regs_dent = debugfs_create_dir(REGISTERS_STR, dw->dma.dbg_dev_root); + + dw_hdma_debugfs_regs_wr(dw, regs_dent); + dw_hdma_debugfs_regs_rd(dw, regs_dent); +} + +void dw_hdma_v0_debugfs_on(struct dw_edma *dw) +{ + if (!debugfs_initialized()) + return; + + debugfs_create_u32("mf", 0444, dw->dma.dbg_dev_root, &dw->chip->mf); + debugfs_create_u16("wr_ch_cnt", 0444, dw->dma.dbg_dev_root, &dw->wr_ch_cnt); + debugfs_create_u16("rd_ch_cnt", 0444, dw->dma.dbg_dev_root, &dw->rd_ch_cnt); + + dw_hdma_debugfs_regs(dw); +} diff --git a/drivers/dma/dw-edma/dw-hdma-v0-debugfs.h b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.h new file mode 100644 index 000000000000..e6842c83777d --- /dev/null +++ b/drivers/dma/dw-edma/dw-hdma-v0-debugfs.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2023 Cai Huoqing + * Synopsys DesignWare HDMA v0 debugfs + * + * Author: Cai Huoqing <cai.huoqing@linux.dev> + */ + +#ifndef _DW_HDMA_V0_DEBUG_FS_H +#define _DW_HDMA_V0_DEBUG_FS_H + +#include <linux/dma/edma.h> + +#ifdef CONFIG_DEBUG_FS +void dw_hdma_v0_debugfs_on(struct dw_edma *dw); +#else +static inline void dw_hdma_v0_debugfs_on(struct dw_edma *dw) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#endif /* _DW_HDMA_V0_DEBUG_FS_H */ diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h new file mode 100644 index 000000000000..eab5fd7177e5 --- /dev/null +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2023 Cai Huoqing + * Synopsys DesignWare HDMA v0 reg + * + * Author: Cai Huoqing <cai.huoqing@linux.dev> + */ + +#ifndef _DW_HDMA_V0_REGS_H +#define _DW_HDMA_V0_REGS_H + +#include <linux/dmaengine.h> + +#define HDMA_V0_MAX_NR_CH 8 +#define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6) +#define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5) +#define HDMA_V0_LOCAL_STOP_INT_EN BIT(4) +#define HDMA_V0_REMOTE_STOP_INT_EN BIT(3) +#define HDMA_V0_ABORT_INT_MASK BIT(2) +#define HDMA_V0_STOP_INT_MASK BIT(0) +#define HDMA_V0_LINKLIST_EN BIT(0) +#define HDMA_V0_CONSUMER_CYCLE_STAT BIT(1) +#define HDMA_V0_CONSUMER_CYCLE_BIT BIT(0) +#define HDMA_V0_DOORBELL_START BIT(0) +#define HDMA_V0_CH_STATUS_MASK GENMASK(1, 0) + +struct dw_hdma_v0_ch_regs { + u32 ch_en; /* 0x0000 */ + u32 doorbell; /* 0x0004 */ + u32 prefetch; /* 0x0008 */ + u32 handshake; /* 0x000c */ + union { + u64 reg; /* 0x0010..0x0014 */ + struct { + u32 lsb; /* 0x0010 */ + u32 msb; /* 0x0014 */ + }; + } llp; + u32 cycle_sync; /* 0x0018 */ + u32 transfer_size; /* 0x001c */ + union { + u64 reg; /* 0x0020..0x0024 */ + struct { + u32 lsb; /* 0x0020 */ + u32 msb; /* 0x0024 */ + }; + } sar; + union { + u64 reg; /* 0x0028..0x002c */ + struct { + u32 lsb; /* 0x0028 */ + u32 msb; /* 0x002c */ + }; + } dar; + u32 watermark_en; /* 0x0030 */ + u32 control1; /* 0x0034 */ + u32 func_num; /* 0x0038 */ + u32 qos; /* 0x003c */ + u32 padding_1[16]; /* 0x0040..0x007c */ + u32 ch_stat; /* 0x0080 */ + u32 int_stat; /* 0x0084 */ + u32 int_setup; /* 0x0088 */ + u32 int_clear; /* 0x008c */ + union { + u64 reg; /* 0x0090..0x0094 */ + struct { + u32 lsb; /* 0x0090 */ + u32 msb; /* 0x0094 */ + }; + } msi_stop; + union { + u64 reg; /* 0x0098..0x009c */ + struct { + u32 lsb; /* 0x0098 */ + u32 msb; /* 0x009c */ + }; + } msi_watermark; + union { + u64 reg; /* 0x00a0..0x00a4 */ + struct { + u32 lsb; /* 0x00a0 */ + u32 msb; /* 0x00a4 */ + }; + } msi_abort; + u32 msi_msgdata; /* 0x00a8 */ + u32 padding_2[21]; /* 0x00ac..0x00fc */ +} __packed; + +struct dw_hdma_v0_ch { + struct dw_hdma_v0_ch_regs wr; /* 0x0000 */ + struct dw_hdma_v0_ch_regs rd; /* 0x0100 */ +} __packed; + +struct dw_hdma_v0_regs { + struct dw_hdma_v0_ch ch[HDMA_V0_MAX_NR_CH]; /* 0x0000..0x0fa8 */ +} __packed; + +struct dw_hdma_v0_lli { + u32 control; + u32 transfer_size; + union { + u64 reg; + struct { + u32 lsb; + u32 msb; + }; + } sar; + union { + u64 reg; + struct { + u32 lsb; + u32 msb; + }; + } dar; +} __packed; + +struct dw_hdma_v0_llp { + u32 control; + u32 reserved; + union { + u64 reg; + struct { + u32 lsb; + u32 msb; + }; + } llp; +} __packed; + +#endif /* _DW_HDMA_V0_REGS_H */ diff --git a/drivers/dma/dw/Kconfig b/drivers/dma/dw/Kconfig index db25f9b7778c..a9828ddd6d06 100644 --- a/drivers/dma/dw/Kconfig +++ b/drivers/dma/dw/Kconfig @@ -16,6 +16,15 @@ config DW_DMAC Support the Synopsys DesignWare AHB DMA controller. This can be integrated in chips such as the Intel Cherrytrail. +config RZN1_DMAMUX + tristate "Renesas RZ/N1 DMAMUX driver" + depends on DW_DMAC + depends on ARCH_RZN1 || COMPILE_TEST + help + Support the Renesas RZ/N1 DMAMUX which is located in front of + the Synopsys DesignWare AHB DMA controller located on Renesas + SoCs. + config DW_DMAC_PCI tristate "Synopsys DesignWare AHB DMA PCI driver" depends on PCI diff --git a/drivers/dma/dw/Makefile b/drivers/dma/dw/Makefile index a6f358ad8591..e1796015f213 100644 --- a/drivers/dma/dw/Makefile +++ b/drivers/dma/dw/Makefile @@ -9,3 +9,5 @@ dw_dmac-$(CONFIG_OF) += of.o obj-$(CONFIG_DW_DMAC_PCI) += dw_dmac_pci.o dw_dmac_pci-y := pci.o + +obj-$(CONFIG_RZN1_DMAMUX) += rzn1-dmamux.o diff --git a/drivers/dma/dw/acpi.c b/drivers/dma/dw/acpi.c index c510c109d2c3..b6452fffa657 100644 --- a/drivers/dma/dw/acpi.c +++ b/drivers/dma/dw/acpi.c @@ -8,13 +8,15 @@ static bool dw_dma_acpi_filter(struct dma_chan *chan, void *param) { + struct dw_dma *dw = to_dw_dma(chan->device); + struct dw_dma_chip_pdata *data = dev_get_drvdata(dw->dma.dev); struct acpi_dma_spec *dma_spec = param; struct dw_dma_slave slave = { .dma_dev = dma_spec->dev, .src_id = dma_spec->slave_id, .dst_id = dma_spec->slave_id, - .m_master = 0, - .p_master = 1, + .m_master = data->m_master, + .p_master = data->p_master, }; return dw_dma_filter(chan, &slave); diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index 7ab83fe601ed..dd75f97a33b3 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -16,6 +16,7 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/log2.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/slab.h> @@ -29,9 +30,6 @@ * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all * of which use ARM any more). See the "Databook" from Synopsys for * information beyond what licensees probably provide. - * - * The driver has been tested with the Atmel AT32AP7000, which does not - * support descriptor writeback. */ /* The set of bus widths supported by the DMA controller */ @@ -624,12 +622,10 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct dw_desc *prev; struct dw_desc *first; u32 ctllo, ctlhi; - u8 m_master = dwc->dws.m_master; - u8 lms = DWC_LLP_LMS(m_master); + u8 lms = DWC_LLP_LMS(dwc->dws.m_master); dma_addr_t reg; unsigned int reg_width; unsigned int mem_width; - unsigned int data_width = dw->pdata->data_width[m_master]; unsigned int i; struct scatterlist *sg; size_t total_len = 0; @@ -663,7 +659,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, mem = sg_dma_address(sg); len = sg_dma_len(sg); - mem_width = __ffs(data_width | mem | len); + mem_width = __ffs(sconfig->src_addr_width | mem | len); slave_sg_todev_fill_desc: desc = dwc_desc_get(dwc); @@ -723,7 +719,7 @@ slave_sg_fromdev_fill_desc: lli_write(desc, sar, reg); lli_write(desc, dar, mem); lli_write(desc, ctlhi, ctlhi); - mem_width = __ffs(data_width | mem); + mem_width = __ffs(sconfig->dst_addr_width | mem); lli_write(desc, ctllo, ctllo | DWC_CTLL_DST_WIDTH(mem_width)); desc->len = dlen; @@ -783,20 +779,108 @@ bool dw_dma_filter(struct dma_chan *chan, void *param) } EXPORT_SYMBOL_GPL(dw_dma_filter); -static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) +static int dwc_verify_maxburst(struct dma_chan *chan) { struct dw_dma_chan *dwc = to_dw_dma_chan(chan); - struct dw_dma *dw = to_dw_dma(chan->device); - memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); + dwc->dma_sconfig.src_maxburst = + clamp(dwc->dma_sconfig.src_maxburst, 1U, dwc->max_burst); + dwc->dma_sconfig.dst_maxburst = + clamp(dwc->dma_sconfig.dst_maxburst, 1U, dwc->max_burst); dwc->dma_sconfig.src_maxburst = - clamp(dwc->dma_sconfig.src_maxburst, 0U, dwc->max_burst); + rounddown_pow_of_two(dwc->dma_sconfig.src_maxburst); dwc->dma_sconfig.dst_maxburst = - clamp(dwc->dma_sconfig.dst_maxburst, 0U, dwc->max_burst); + rounddown_pow_of_two(dwc->dma_sconfig.dst_maxburst); + + return 0; +} + +static int dwc_verify_p_buswidth(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + u32 reg_width, max_width; + + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) + reg_width = dwc->dma_sconfig.dst_addr_width; + else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) + reg_width = dwc->dma_sconfig.src_addr_width; + else /* DMA_MEM_TO_MEM */ + return 0; + + max_width = dw->pdata->data_width[dwc->dws.p_master]; + + /* Fall-back to 1-byte transfer width if undefined */ + if (reg_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + reg_width = DMA_SLAVE_BUSWIDTH_1_BYTE; + else if (!is_power_of_2(reg_width) || reg_width > max_width) + return -EINVAL; + else /* bus width is valid */ + return 0; + + /* Update undefined addr width value */ + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) + dwc->dma_sconfig.dst_addr_width = reg_width; + else /* DMA_DEV_TO_MEM */ + dwc->dma_sconfig.src_addr_width = reg_width; + + return 0; +} + +static int dwc_verify_m_buswidth(struct dma_chan *chan) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + struct dw_dma *dw = to_dw_dma(chan->device); + u32 reg_width, reg_burst, mem_width; + + mem_width = dw->pdata->data_width[dwc->dws.m_master]; + + /* + * It's possible to have a data portion locked in the DMA FIFO in case + * of the channel suspension. Subsequent channel disabling will cause + * that data silent loss. In order to prevent that maintain the src and + * dst transfer widths coherency by means of the relation: + * (CTLx.SRC_TR_WIDTH * CTLx.SRC_MSIZE >= CTLx.DST_TR_WIDTH) + * Look for the details in the commit message that brings this change. + * + * Note the DMA configs utilized in the calculations below must have + * been verified to have correct values by this method call. + */ + if (dwc->dma_sconfig.direction == DMA_MEM_TO_DEV) { + reg_width = dwc->dma_sconfig.dst_addr_width; + if (mem_width < reg_width) + return -EINVAL; + + dwc->dma_sconfig.src_addr_width = mem_width; + } else if (dwc->dma_sconfig.direction == DMA_DEV_TO_MEM) { + reg_width = dwc->dma_sconfig.src_addr_width; + reg_burst = dwc->dma_sconfig.src_maxburst; + + dwc->dma_sconfig.dst_addr_width = min(mem_width, reg_width * reg_burst); + } + + return 0; +} + +static int dwc_config(struct dma_chan *chan, struct dma_slave_config *sconfig) +{ + struct dw_dma_chan *dwc = to_dw_dma_chan(chan); + int ret; - dw->encode_maxburst(dwc, &dwc->dma_sconfig.src_maxburst); - dw->encode_maxburst(dwc, &dwc->dma_sconfig.dst_maxburst); + memcpy(&dwc->dma_sconfig, sconfig, sizeof(*sconfig)); + + ret = dwc_verify_maxburst(chan); + if (ret) + return ret; + + ret = dwc_verify_p_buswidth(chan); + if (ret) + return ret; + + ret = dwc_verify_m_buswidth(chan); + if (ret) + return ret; return 0; } @@ -892,7 +976,8 @@ static struct dw_desc *dwc_find_desc(struct dw_dma_chan *dwc, dma_cookie_t c) return NULL; } -static u32 dwc_get_residue(struct dw_dma_chan *dwc, dma_cookie_t cookie) +static u32 dwc_get_residue_and_status(struct dw_dma_chan *dwc, dma_cookie_t cookie, + enum dma_status *status) { struct dw_desc *desc; unsigned long flags; @@ -906,6 +991,8 @@ static u32 dwc_get_residue(struct dw_dma_chan *dwc, dma_cookie_t cookie) residue = desc->residue; if (test_bit(DW_DMA_IS_SOFT_LLP, &dwc->flags) && residue) residue -= dwc_get_sent(dwc); + if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags)) + *status = DMA_PAUSED; } else { residue = desc->total_len; } @@ -935,11 +1022,7 @@ dwc_tx_status(struct dma_chan *chan, if (ret == DMA_COMPLETE) return ret; - dma_set_residue(txstate, dwc_get_residue(dwc, cookie)); - - if (test_bit(DW_DMA_IS_PAUSED, &dwc->flags) && ret == DMA_IN_PROGRESS) - return DMA_PAUSED; - + dma_set_residue(txstate, dwc_get_residue_and_status(dwc, cookie, &ret)); return ret; } @@ -1072,7 +1155,7 @@ int do_dma_probe(struct dw_dma_chip *chip) bool autocfg = false; unsigned int dw_params; unsigned int i; - int err; + int ret; dw->pdata = devm_kzalloc(chip->dev, sizeof(*dw->pdata), GFP_KERNEL); if (!dw->pdata) @@ -1088,7 +1171,7 @@ int do_dma_probe(struct dw_dma_chip *chip) autocfg = dw_params >> DW_PARAMS_EN & 1; if (!autocfg) { - err = -EINVAL; + ret = -EINVAL; goto err_pdata; } @@ -1108,7 +1191,7 @@ int do_dma_probe(struct dw_dma_chip *chip) pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; pdata->chan_priority = CHAN_PRIORITY_ASCENDING; } else if (chip->pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { - err = -EINVAL; + ret = -EINVAL; goto err_pdata; } else { memcpy(dw->pdata, chip->pdata, sizeof(*dw->pdata)); @@ -1120,7 +1203,7 @@ int do_dma_probe(struct dw_dma_chip *chip) dw->chan = devm_kcalloc(chip->dev, pdata->nr_channels, sizeof(*dw->chan), GFP_KERNEL); if (!dw->chan) { - err = -ENOMEM; + ret = -ENOMEM; goto err_pdata; } @@ -1138,15 +1221,15 @@ int do_dma_probe(struct dw_dma_chip *chip) sizeof(struct dw_desc), 4, 0); if (!dw->desc_pool) { dev_err(chip->dev, "No memory for descriptors dma pool\n"); - err = -ENOMEM; + ret = -ENOMEM; goto err_pdata; } tasklet_setup(&dw->tasklet, dw_dma_tasklet); - err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, + ret = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, dw->name, dw); - if (err) + if (ret) goto err_pdata; INIT_LIST_HEAD(&dw->dma.channels); @@ -1258,8 +1341,8 @@ int do_dma_probe(struct dw_dma_chip *chip) */ dma_set_max_seg_size(dw->dma.dev, dw->chan[0].block_size); - err = dma_async_device_register(&dw->dma); - if (err) + ret = dma_async_device_register(&dw->dma); + if (ret) goto err_dma_register; dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", @@ -1273,7 +1356,7 @@ err_dma_register: free_irq(chip->irq, dw); err_pdata: pm_runtime_put_sync_suspend(chip->dev); - return err; + return ret; } int do_dma_remove(struct dw_dma_chip *chip) diff --git a/drivers/dma/dw/dw.c b/drivers/dma/dw/dw.c index a4862263ff14..6766142884b6 100644 --- a/drivers/dma/dw/dw.c +++ b/drivers/dma/dw/dw.c @@ -64,30 +64,39 @@ static size_t dw_dma_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) return DWC_CTLH_BLOCK_TS(block) << width; } +static inline u8 dw_dma_encode_maxburst(u32 maxburst) +{ + /* + * Fix burst size according to dw_dmac. We need to convert them as: + * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. + */ + return maxburst > 1 ? fls(maxburst) - 2 : 0; +} + static u32 dw_dma_prepare_ctllo(struct dw_dma_chan *dwc) { struct dma_slave_config *sconfig = &dwc->dma_sconfig; - u8 smsize = (dwc->direction == DMA_DEV_TO_MEM) ? sconfig->src_maxburst : 0; - u8 dmsize = (dwc->direction == DMA_MEM_TO_DEV) ? sconfig->dst_maxburst : 0; - u8 p_master = dwc->dws.p_master; - u8 m_master = dwc->dws.m_master; - u8 dms = (dwc->direction == DMA_MEM_TO_DEV) ? p_master : m_master; - u8 sms = (dwc->direction == DMA_DEV_TO_MEM) ? p_master : m_master; + u8 smsize = 0, dmsize = 0; + u8 sms, dms; + + if (dwc->direction == DMA_MEM_TO_DEV) { + sms = dwc->dws.m_master; + dms = dwc->dws.p_master; + dmsize = dw_dma_encode_maxburst(sconfig->dst_maxburst); + } else if (dwc->direction == DMA_DEV_TO_MEM) { + sms = dwc->dws.p_master; + dms = dwc->dws.m_master; + smsize = dw_dma_encode_maxburst(sconfig->src_maxburst); + } else /* DMA_MEM_TO_MEM */ { + sms = dwc->dws.m_master; + dms = dwc->dws.m_master; + } return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize) | DWC_CTLL_DMS(dms) | DWC_CTLL_SMS(sms); } -static void dw_dma_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) -{ - /* - * Fix burst size according to dw_dmac. We need to convert them as: - * 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3. - */ - *maxburst = *maxburst > 1 ? fls(*maxburst) - 2 : 0; -} - static void dw_dma_set_device_name(struct dw_dma *dw, int id) { snprintf(dw->name, sizeof(dw->name), "dw:dmac%d", id); @@ -116,7 +125,6 @@ int dw_dma_probe(struct dw_dma_chip *chip) dw->suspend_chan = dw_dma_suspend_chan; dw->resume_chan = dw_dma_resume_chan; dw->prepare_ctllo = dw_dma_prepare_ctllo; - dw->encode_maxburst = dw_dma_encode_maxburst; dw->bytes2block = dw_dma_bytes2block; dw->block2bytes = dw_dma_block2bytes; diff --git a/drivers/dma/dw/idma32.c b/drivers/dma/dw/idma32.c index 3ce44de25d33..dac617c183e6 100644 --- a/drivers/dma/dw/idma32.c +++ b/drivers/dma/dw/idma32.c @@ -1,15 +1,144 @@ // SPDX-License-Identifier: GPL-2.0 -// Copyright (C) 2013,2018 Intel Corporation +// Copyright (C) 2013,2018,2020-2021 Intel Corporation #include <linux/bitops.h> #include <linux/dmaengine.h> #include <linux/errno.h> +#include <linux/io.h> +#include <linux/pci.h> #include <linux/slab.h> #include <linux/types.h> #include "internal.h" -static void idma32_initialize_chan(struct dw_dma_chan *dwc) +#define DMA_CTL_CH(x) (0x1000 + (x) * 4) +#define DMA_SRC_ADDR_FILLIN(x) (0x1100 + (x) * 4) +#define DMA_DST_ADDR_FILLIN(x) (0x1200 + (x) * 4) +#define DMA_XBAR_SEL(x) (0x1300 + (x) * 4) +#define DMA_REGACCESS_CHID_CFG (0x1400) + +#define CTL_CH_TRANSFER_MODE_MASK GENMASK(1, 0) +#define CTL_CH_TRANSFER_MODE_S2S 0 +#define CTL_CH_TRANSFER_MODE_S2D 1 +#define CTL_CH_TRANSFER_MODE_D2S 2 +#define CTL_CH_TRANSFER_MODE_D2D 3 +#define CTL_CH_RD_RS_MASK GENMASK(4, 3) +#define CTL_CH_WR_RS_MASK GENMASK(6, 5) +#define CTL_CH_RD_NON_SNOOP_BIT BIT(8) +#define CTL_CH_WR_NON_SNOOP_BIT BIT(9) + +#define XBAR_SEL_DEVID_MASK GENMASK(15, 0) +#define XBAR_SEL_RX_TX_BIT BIT(16) +#define XBAR_SEL_RX_TX_SHIFT 16 + +#define REGACCESS_CHID_MASK GENMASK(2, 0) + +static unsigned int idma32_get_slave_devfn(struct dw_dma_chan *dwc) +{ + struct device *slave = dwc->chan.slave; + + if (!slave || !dev_is_pci(slave)) + return 0; + + return to_pci_dev(slave)->devfn; +} + +static void idma32_initialize_chan_xbar(struct dw_dma_chan *dwc) +{ + struct dw_dma *dw = to_dw_dma(dwc->chan.device); + void __iomem *misc = __dw_regs(dw); + u32 cfghi = 0, cfglo = 0; + u8 dst_id, src_id; + u32 value; + + /* DMA Channel ID Configuration register must be programmed first */ + value = readl(misc + DMA_REGACCESS_CHID_CFG); + + value &= ~REGACCESS_CHID_MASK; + value |= dwc->chan.chan_id; + + writel(value, misc + DMA_REGACCESS_CHID_CFG); + + /* Configure channel attributes */ + value = readl(misc + DMA_CTL_CH(dwc->chan.chan_id)); + + value &= ~(CTL_CH_RD_NON_SNOOP_BIT | CTL_CH_WR_NON_SNOOP_BIT); + value &= ~(CTL_CH_RD_RS_MASK | CTL_CH_WR_RS_MASK); + value &= ~CTL_CH_TRANSFER_MODE_MASK; + + switch (dwc->direction) { + case DMA_MEM_TO_DEV: + value |= CTL_CH_TRANSFER_MODE_D2S; + value |= CTL_CH_WR_NON_SNOOP_BIT; + break; + case DMA_DEV_TO_MEM: + value |= CTL_CH_TRANSFER_MODE_S2D; + value |= CTL_CH_RD_NON_SNOOP_BIT; + break; + default: + /* + * Memory-to-Memory and Device-to-Device are ignored for now. + * + * For Memory-to-Memory transfers we would need to set mode + * and disable snooping on both sides. + */ + return; + } + + writel(value, misc + DMA_CTL_CH(dwc->chan.chan_id)); + + /* Configure crossbar selection */ + value = readl(misc + DMA_XBAR_SEL(dwc->chan.chan_id)); + + /* DEVFN selection */ + value &= ~XBAR_SEL_DEVID_MASK; + value |= idma32_get_slave_devfn(dwc); + + switch (dwc->direction) { + case DMA_MEM_TO_DEV: + value |= XBAR_SEL_RX_TX_BIT; + break; + case DMA_DEV_TO_MEM: + value &= ~XBAR_SEL_RX_TX_BIT; + break; + default: + /* Memory-to-Memory and Device-to-Device are ignored for now */ + return; + } + + writel(value, misc + DMA_XBAR_SEL(dwc->chan.chan_id)); + + /* Configure DMA channel low and high registers */ + switch (dwc->direction) { + case DMA_MEM_TO_DEV: + dst_id = dwc->chan.chan_id; + src_id = dwc->dws.src_id; + break; + case DMA_DEV_TO_MEM: + dst_id = dwc->dws.dst_id; + src_id = dwc->chan.chan_id; + break; + default: + /* Memory-to-Memory and Device-to-Device are ignored for now */ + return; + } + + /* Set default burst alignment */ + cfglo |= IDMA32C_CFGL_DST_BURST_ALIGN | IDMA32C_CFGL_SRC_BURST_ALIGN; + + /* Low 4 bits of the request lines */ + cfghi |= IDMA32C_CFGH_DST_PER(dst_id & 0xf); + cfghi |= IDMA32C_CFGH_SRC_PER(src_id & 0xf); + + /* Request line extension (2 bits) */ + cfghi |= IDMA32C_CFGH_DST_PER_EXT(dst_id >> 4 & 0x3); + cfghi |= IDMA32C_CFGH_SRC_PER_EXT(src_id >> 4 & 0x3); + + channel_writel(dwc, CFG_LO, cfglo); + channel_writel(dwc, CFG_HI, cfghi); +} + +static void idma32_initialize_chan_generic(struct dw_dma_chan *dwc) { u32 cfghi = 0; u32 cfglo = 0; @@ -70,21 +199,25 @@ static size_t idma32_block2bytes(struct dw_dma_chan *dwc, u32 block, u32 width) return IDMA32C_CTLH_BLOCK_TS(block); } +static inline u8 idma32_encode_maxburst(u32 maxburst) +{ + return maxburst > 1 ? fls(maxburst) - 1 : 0; +} + static u32 idma32_prepare_ctllo(struct dw_dma_chan *dwc) { struct dma_slave_config *sconfig = &dwc->dma_sconfig; - u8 smsize = (dwc->direction == DMA_DEV_TO_MEM) ? sconfig->src_maxburst : 0; - u8 dmsize = (dwc->direction == DMA_MEM_TO_DEV) ? sconfig->dst_maxburst : 0; + u8 smsize = 0, dmsize = 0; + + if (dwc->direction == DMA_MEM_TO_DEV) + dmsize = idma32_encode_maxburst(sconfig->dst_maxburst); + else if (dwc->direction == DMA_DEV_TO_MEM) + smsize = idma32_encode_maxburst(sconfig->src_maxburst); return DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN | DWC_CTLL_DST_MSIZE(dmsize) | DWC_CTLL_SRC_MSIZE(smsize); } -static void idma32_encode_maxburst(struct dw_dma_chan *dwc, u32 *maxburst) -{ - *maxburst = *maxburst > 1 ? fls(*maxburst) - 1 : 0; -} - static void idma32_set_device_name(struct dw_dma *dw, int id) { snprintf(dw->name, sizeof(dw->name), "idma32:dmac%d", id); @@ -134,11 +267,13 @@ int idma32_dma_probe(struct dw_dma_chip *chip) return -ENOMEM; /* Channel operations */ - dw->initialize_chan = idma32_initialize_chan; + if (chip->pdata->quirks & DW_DMA_QUIRK_XBAR_PRESENT) + dw->initialize_chan = idma32_initialize_chan_xbar; + else + dw->initialize_chan = idma32_initialize_chan_generic; dw->suspend_chan = idma32_suspend_chan; dw->resume_chan = idma32_resume_chan; dw->prepare_ctllo = idma32_prepare_ctllo; - dw->encode_maxburst = idma32_encode_maxburst; dw->bytes2block = idma32_bytes2block; dw->block2bytes = idma32_block2bytes; diff --git a/drivers/dma/dw/internal.h b/drivers/dma/dw/internal.h index 2e1c52eefdeb..f1bd06a20cd6 100644 --- a/drivers/dma/dw/internal.h +++ b/drivers/dma/dw/internal.h @@ -51,11 +51,15 @@ struct dw_dma_chip_pdata { int (*probe)(struct dw_dma_chip *chip); int (*remove)(struct dw_dma_chip *chip); struct dw_dma_chip *chip; + u8 m_master; + u8 p_master; }; static __maybe_unused const struct dw_dma_chip_pdata dw_dma_chip_pdata = { .probe = dw_dma_probe, .remove = dw_dma_remove, + .m_master = 0, + .p_master = 1, }; static const struct dw_dma_platform_data idma32_pdata = { @@ -72,6 +76,26 @@ static __maybe_unused const struct dw_dma_chip_pdata idma32_chip_pdata = { .pdata = &idma32_pdata, .probe = idma32_dma_probe, .remove = idma32_dma_remove, + .m_master = 0, + .p_master = 0, +}; + +static const struct dw_dma_platform_data xbar_pdata = { + .nr_channels = 8, + .chan_allocation_order = CHAN_ALLOCATION_ASCENDING, + .chan_priority = CHAN_PRIORITY_ASCENDING, + .block_size = 131071, + .nr_masters = 1, + .data_width = {4}, + .quirks = DW_DMA_QUIRK_XBAR_PRESENT, +}; + +static __maybe_unused const struct dw_dma_chip_pdata xbar_chip_pdata = { + .pdata = &xbar_pdata, + .probe = idma32_dma_probe, + .remove = idma32_dma_remove, + .m_master = 0, + .p_master = 0, }; #endif /* _DMA_DW_INTERNAL_H */ diff --git a/drivers/dma/dw/of.c b/drivers/dma/dw/of.c index c1cf7675b9d1..523ca806837c 100644 --- a/drivers/dma/dw/of.c +++ b/drivers/dma/dw/of.c @@ -50,15 +50,10 @@ struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct dw_dma_platform_data *pdata; - u32 tmp, arr[DW_DMA_MAX_NR_MASTERS], mb[DW_DMA_MAX_NR_CHANNELS]; + u32 tmp, arr[DW_DMA_MAX_NR_MASTERS]; u32 nr_masters; u32 nr_channels; - if (!np) { - dev_err(&pdev->dev, "Missing DT data\n"); - return NULL; - } - if (of_property_read_u32(np, "dma-masters", &nr_masters)) return NULL; if (nr_masters < 1 || nr_masters > DW_DMA_MAX_NR_MASTERS) @@ -76,41 +71,29 @@ struct dw_dma_platform_data *dw_dma_parse_dt(struct platform_device *pdev) pdata->nr_masters = nr_masters; pdata->nr_channels = nr_channels; - if (!of_property_read_u32(np, "chan_allocation_order", &tmp)) - pdata->chan_allocation_order = (unsigned char)tmp; + of_property_read_u32(np, "chan_allocation_order", &pdata->chan_allocation_order); + of_property_read_u32(np, "chan_priority", &pdata->chan_priority); - if (!of_property_read_u32(np, "chan_priority", &tmp)) - pdata->chan_priority = tmp; + of_property_read_u32(np, "block_size", &pdata->block_size); - if (!of_property_read_u32(np, "block_size", &tmp)) - pdata->block_size = tmp; - - if (!of_property_read_u32_array(np, "data-width", arr, nr_masters)) { - for (tmp = 0; tmp < nr_masters; tmp++) - pdata->data_width[tmp] = arr[tmp]; - } else if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) { + /* Try deprecated property first */ + if (!of_property_read_u32_array(np, "data_width", arr, nr_masters)) { for (tmp = 0; tmp < nr_masters; tmp++) pdata->data_width[tmp] = BIT(arr[tmp] & 0x07); } - if (!of_property_read_u32_array(np, "multi-block", mb, nr_channels)) { - for (tmp = 0; tmp < nr_channels; tmp++) - pdata->multi_block[tmp] = mb[tmp]; - } else { - for (tmp = 0; tmp < nr_channels; tmp++) - pdata->multi_block[tmp] = 1; - } + /* If "data_width" and "data-width" both provided use the latter one */ + of_property_read_u32_array(np, "data-width", pdata->data_width, nr_masters); - if (of_property_read_u32_array(np, "snps,max-burst-len", pdata->max_burst, - nr_channels)) { - memset32(pdata->max_burst, DW_DMA_MAX_BURST, nr_channels); - } + memset32(pdata->multi_block, 1, nr_channels); + of_property_read_u32_array(np, "multi-block", pdata->multi_block, nr_channels); - if (!of_property_read_u32(np, "snps,dma-protection-control", &tmp)) { - if (tmp > CHAN_PROTCTL_MASK) - return NULL; - pdata->protctl = tmp; - } + memset32(pdata->max_burst, DW_DMA_MAX_BURST, nr_channels); + of_property_read_u32_array(np, "snps,max-burst-len", pdata->max_burst, nr_channels); + + of_property_read_u32(np, "snps,dma-protection-control", &pdata->protctl); + if (pdata->protctl > CHAN_PROTCTL_MASK) + return NULL; return pdata; } diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c index 1142aa6f8c4a..a3aae3d1c093 100644 --- a/drivers/dma/dw/pci.c +++ b/drivers/dma/dw/pci.c @@ -32,11 +32,7 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) pci_set_master(pdev); pci_try_set_mwi(pdev); - ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (ret) - return ret; - - ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) return ret; @@ -60,10 +56,10 @@ static int dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pid) if (ret) return ret; - dw_dma_acpi_controller_register(chip->dw); - pci_set_drvdata(pdev, data); + dw_dma_acpi_controller_register(chip->dw); + return 0; } @@ -80,8 +76,6 @@ static void dw_pci_remove(struct pci_dev *pdev) dev_warn(&pdev->dev, "can't remove device properly: %d\n", ret); } -#ifdef CONFIG_PM_SLEEP - static int dw_pci_suspend_late(struct device *dev) { struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); @@ -98,10 +92,8 @@ static int dw_pci_resume_early(struct device *dev) return do_dw_dma_enable(chip); }; -#endif /* CONFIG_PM_SLEEP */ - static const struct dev_pm_ops dw_pci_dev_pm_ops = { - SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_pci_suspend_late, dw_pci_resume_early) + LATE_SYSTEM_SLEEP_PM_OPS(dw_pci_suspend_late, dw_pci_resume_early) }; static const struct pci_device_id dw_pci_id_table[] = { @@ -120,9 +112,9 @@ static const struct pci_device_id dw_pci_id_table[] = { { PCI_VDEVICE(INTEL, 0x22c0), (kernel_ulong_t)&dw_dma_chip_pdata }, /* Elkhart Lake iDMA 32-bit (PSE DMA) */ - { PCI_VDEVICE(INTEL, 0x4bb4), (kernel_ulong_t)&idma32_chip_pdata }, - { PCI_VDEVICE(INTEL, 0x4bb5), (kernel_ulong_t)&idma32_chip_pdata }, - { PCI_VDEVICE(INTEL, 0x4bb6), (kernel_ulong_t)&idma32_chip_pdata }, + { PCI_VDEVICE(INTEL, 0x4bb4), (kernel_ulong_t)&xbar_chip_pdata }, + { PCI_VDEVICE(INTEL, 0x4bb5), (kernel_ulong_t)&xbar_chip_pdata }, + { PCI_VDEVICE(INTEL, 0x4bb6), (kernel_ulong_t)&xbar_chip_pdata }, /* Haswell */ { PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_dma_chip_pdata }, @@ -140,7 +132,7 @@ static struct pci_driver dw_pci_driver = { .probe = dw_pci_probe, .remove = dw_pci_remove, .driver = { - .pm = &dw_pci_dev_pm_ops, + .pm = pm_sleep_ptr(&dw_pci_dev_pm_ops), }, }; diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index 0585d749d935..c63fa52036d7 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -21,15 +21,13 @@ #include "internal.h" -#define DRV_NAME "dw_dmac" - static int dw_probe(struct platform_device *pdev) { const struct dw_dma_chip_pdata *match; struct dw_dma_chip_pdata *data; struct dw_dma_chip *chip; struct device *dev = &pdev->dev; - int err; + int ret; match = device_get_match_data(dev); if (!match) @@ -51,9 +49,9 @@ static int dw_probe(struct platform_device *pdev) if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); - err = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (err) - return err; + ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; if (!data->pdata) data->pdata = dev_get_platdata(dev); @@ -69,14 +67,14 @@ static int dw_probe(struct platform_device *pdev) chip->clk = devm_clk_get_optional(chip->dev, "hclk"); if (IS_ERR(chip->clk)) return PTR_ERR(chip->clk); - err = clk_prepare_enable(chip->clk); - if (err) - return err; + ret = clk_prepare_enable(chip->clk); + if (ret) + return ret; pm_runtime_enable(&pdev->dev); - err = data->probe(chip); - if (err) + ret = data->probe(chip); + if (ret) goto err_dw_dma_probe; platform_set_drvdata(pdev, data); @@ -90,10 +88,10 @@ static int dw_probe(struct platform_device *pdev) err_dw_dma_probe: pm_runtime_disable(&pdev->dev); clk_disable_unprepare(chip->clk); - return err; + return ret; } -static int dw_remove(struct platform_device *pdev) +static void dw_remove(struct platform_device *pdev) { struct dw_dma_chip_pdata *data = platform_get_drvdata(pdev); struct dw_dma_chip *chip = data->chip; @@ -109,8 +107,6 @@ static int dw_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); clk_disable_unprepare(chip->clk); - - return 0; } static void dw_shutdown(struct platform_device *pdev) @@ -137,6 +133,7 @@ static void dw_shutdown(struct platform_device *pdev) #ifdef CONFIG_OF static const struct of_device_id dw_dma_of_id_table[] = { { .compatible = "snps,dma-spear1340", .data = &dw_dma_chip_pdata }, + { .compatible = "renesas,rzn1-dma", .data = &dw_dma_chip_pdata }, {} }; MODULE_DEVICE_TABLE(of, dw_dma_of_id_table); @@ -149,17 +146,15 @@ static const struct acpi_device_id dw_dma_acpi_id_table[] = { { "808622C0", (kernel_ulong_t)&dw_dma_chip_pdata }, /* Elkhart Lake iDMA 32-bit (PSE DMA) */ - { "80864BB4", (kernel_ulong_t)&idma32_chip_pdata }, - { "80864BB5", (kernel_ulong_t)&idma32_chip_pdata }, - { "80864BB6", (kernel_ulong_t)&idma32_chip_pdata }, + { "80864BB4", (kernel_ulong_t)&xbar_chip_pdata }, + { "80864BB5", (kernel_ulong_t)&xbar_chip_pdata }, + { "80864BB6", (kernel_ulong_t)&xbar_chip_pdata }, { } }; MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table); #endif -#ifdef CONFIG_PM_SLEEP - static int dw_suspend_late(struct device *dev) { struct dw_dma_chip_pdata *data = dev_get_drvdata(dev); @@ -184,10 +179,8 @@ static int dw_resume_early(struct device *dev) return do_dw_dma_enable(chip); } -#endif /* CONFIG_PM_SLEEP */ - static const struct dev_pm_ops dw_dev_pm_ops = { - SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_suspend_late, dw_resume_early) + LATE_SYSTEM_SLEEP_PM_OPS(dw_suspend_late, dw_resume_early) }; static struct platform_driver dw_driver = { @@ -195,8 +188,8 @@ static struct platform_driver dw_driver = { .remove = dw_remove, .shutdown = dw_shutdown, .driver = { - .name = DRV_NAME, - .pm = &dw_dev_pm_ops, + .name = "dw_dmac", + .pm = pm_sleep_ptr(&dw_dev_pm_ops), .of_match_table = of_match_ptr(dw_dma_of_id_table), .acpi_match_table = ACPI_PTR(dw_dma_acpi_id_table), }, @@ -216,4 +209,3 @@ module_exit(dw_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller platform driver"); -MODULE_ALIAS("platform:" DRV_NAME); diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h index 76654bd13c1a..5969d9cc8d7a 100644 --- a/drivers/dma/dw/regs.h +++ b/drivers/dma/dw/regs.h @@ -327,7 +327,6 @@ struct dw_dma { void (*suspend_chan)(struct dw_dma_chan *dwc, bool drain); void (*resume_chan)(struct dw_dma_chan *dwc, bool drain); u32 (*prepare_ctllo)(struct dw_dma_chan *dwc); - void (*encode_maxburst)(struct dw_dma_chan *dwc, u32 *maxburst); u32 (*bytes2block)(struct dw_dma_chan *dwc, size_t bytes, unsigned int width, size_t *len); size_t (*block2bytes)(struct dw_dma_chan *dwc, u32 block, u32 width); diff --git a/drivers/dma/dw/rzn1-dmamux.c b/drivers/dma/dw/rzn1-dmamux.c new file mode 100644 index 000000000000..deadf135681b --- /dev/null +++ b/drivers/dma/dw/rzn1-dmamux.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2022 Schneider-Electric + * Author: Miquel Raynal <miquel.raynal@bootlin.com + * Based on TI crossbar driver written by Peter Ujfalusi <peter.ujfalusi@ti.com> + */ +#include <linux/bitops.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/soc/renesas/r9a06g032-sysctrl.h> +#include <linux/types.h> + +#define RNZ1_DMAMUX_NCELLS 6 +#define RZN1_DMAMUX_MAX_LINES 64 +#define RZN1_DMAMUX_LINES_PER_CTLR 16 + +struct rzn1_dmamux_data { + struct dma_router dmarouter; + DECLARE_BITMAP(used_chans, 2 * RZN1_DMAMUX_LINES_PER_CTLR); +}; + +struct rzn1_dmamux_map { + unsigned int req_idx; +}; + +static void rzn1_dmamux_free(struct device *dev, void *route_data) +{ + struct rzn1_dmamux_data *dmamux = dev_get_drvdata(dev); + struct rzn1_dmamux_map *map = route_data; + + dev_dbg(dev, "Unmapping DMAMUX request %u\n", map->req_idx); + + clear_bit(map->req_idx, dmamux->used_chans); + + kfree(map); +} + +static void *rzn1_dmamux_route_allocate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct rzn1_dmamux_data *dmamux = platform_get_drvdata(pdev); + struct rzn1_dmamux_map *map; + unsigned int dmac_idx, chan, val; + u32 mask; + int ret; + + if (dma_spec->args_count != RNZ1_DMAMUX_NCELLS) { + ret = -EINVAL; + goto put_device; + } + + map = kzalloc(sizeof(*map), GFP_KERNEL); + if (!map) { + ret = -ENOMEM; + goto put_device; + } + + chan = dma_spec->args[0]; + map->req_idx = dma_spec->args[4]; + val = dma_spec->args[5]; + dma_spec->args_count -= 2; + + if (chan >= RZN1_DMAMUX_LINES_PER_CTLR) { + dev_err(&pdev->dev, "Invalid DMA request line: %u\n", chan); + ret = -EINVAL; + goto free_map; + } + + if (map->req_idx >= RZN1_DMAMUX_MAX_LINES || + (map->req_idx % RZN1_DMAMUX_LINES_PER_CTLR) != chan) { + dev_err(&pdev->dev, "Invalid MUX request line: %u\n", map->req_idx); + ret = -EINVAL; + goto free_map; + } + + dmac_idx = map->req_idx >= RZN1_DMAMUX_LINES_PER_CTLR ? 1 : 0; + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", dmac_idx); + if (!dma_spec->np) { + dev_err(&pdev->dev, "Can't get DMA master\n"); + ret = -EINVAL; + goto free_map; + } + + dev_dbg(&pdev->dev, "Mapping DMAMUX request %u to DMAC%u request %u\n", + map->req_idx, dmac_idx, chan); + + if (test_and_set_bit(map->req_idx, dmamux->used_chans)) { + ret = -EBUSY; + goto free_map; + } + + mask = BIT(map->req_idx); + ret = r9a06g032_sysctrl_set_dmamux(mask, val ? mask : 0); + if (ret) + goto clear_bitmap; + + put_device(&pdev->dev); + return map; + +clear_bitmap: + clear_bit(map->req_idx, dmamux->used_chans); +free_map: + kfree(map); +put_device: + put_device(&pdev->dev); + + return ERR_PTR(ret); +} + +#ifdef CONFIG_OF +static const struct of_device_id rzn1_dmac_match[] = { + { .compatible = "renesas,rzn1-dma" }, + {} +}; +#endif + +static int rzn1_dmamux_probe(struct platform_device *pdev) +{ + struct device_node *mux_node = pdev->dev.of_node; + const struct of_device_id *match; + struct device_node *dmac_node; + struct rzn1_dmamux_data *dmamux; + + dmamux = devm_kzalloc(&pdev->dev, sizeof(*dmamux), GFP_KERNEL); + if (!dmamux) + return -ENOMEM; + + dmac_node = of_parse_phandle(mux_node, "dma-masters", 0); + if (!dmac_node) + return dev_err_probe(&pdev->dev, -ENODEV, "Can't get DMA master node\n"); + + match = of_match_node(rzn1_dmac_match, dmac_node); + of_node_put(dmac_node); + if (!match) + return dev_err_probe(&pdev->dev, -EINVAL, "DMA master is not supported\n"); + + dmamux->dmarouter.dev = &pdev->dev; + dmamux->dmarouter.route_free = rzn1_dmamux_free; + + platform_set_drvdata(pdev, dmamux); + + return of_dma_router_register(mux_node, rzn1_dmamux_route_allocate, + &dmamux->dmarouter); +} + +static const struct of_device_id rzn1_dmamux_match[] = { + { .compatible = "renesas,rzn1-dmamux" }, + {} +}; +MODULE_DEVICE_TABLE(of, rzn1_dmamux_match); + +static struct platform_driver rzn1_dmamux_driver = { + .driver = { + .name = "renesas,rzn1-dmamux", + .of_match_table = rzn1_dmamux_match, + }, + .probe = rzn1_dmamux_probe, +}; +module_platform_driver(rzn1_dmamux_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com"); +MODULE_DESCRIPTION("Renesas RZ/N1 DMAMUX driver"); diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 01027779beb8..e424bb5c40e7 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -17,14 +17,15 @@ #include <linux/clk.h> #include <linux/init.h> #include <linux/interrupt.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/module.h> #include <linux/mod_devicetable.h> +#include <linux/of_dma.h> +#include <linux/overflow.h> #include <linux/platform_device.h> #include <linux/slab.h> -#include <linux/platform_data/dma-ep93xx.h> - #include "dmaengine.h" /* M2P registers */ @@ -104,6 +105,31 @@ #define DMA_MAX_CHAN_BYTES 0xffff #define DMA_MAX_CHAN_DESCRIPTORS 32 +/* + * M2P channels. + * + * Note that these values are also directly used for setting the PPALLOC + * register. + */ +#define EP93XX_DMA_I2S1 0 +#define EP93XX_DMA_I2S2 1 +#define EP93XX_DMA_AAC1 2 +#define EP93XX_DMA_AAC2 3 +#define EP93XX_DMA_AAC3 4 +#define EP93XX_DMA_I2S3 5 +#define EP93XX_DMA_UART1 6 +#define EP93XX_DMA_UART2 7 +#define EP93XX_DMA_UART3 8 +#define EP93XX_DMA_IRDA 9 +/* M2M channels */ +#define EP93XX_DMA_SSP 10 +#define EP93XX_DMA_IDE 11 + +enum ep93xx_dma_type { + M2P_DMA, + M2M_DMA, +}; + struct ep93xx_dma_engine; static int ep93xx_dma_slave_config_write(struct dma_chan *chan, enum dma_transfer_direction dir, @@ -129,11 +155,17 @@ struct ep93xx_dma_desc { struct list_head node; }; +struct ep93xx_dma_chan_cfg { + u8 port; + enum dma_transfer_direction dir; +}; + /** * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel * @chan: dmaengine API channel - * @edma: pointer to to the engine device + * @edma: pointer to the engine device * @regs: memory mapped registers + * @dma_cfg: channel number, direction * @irq: interrupt number of the channel * @clk: clock used by this channel * @tasklet: channel specific tasklet used for callbacks @@ -157,14 +189,12 @@ struct ep93xx_dma_desc { * descriptor in the chain. When a descriptor is moved to the @active queue, * the first and chained descriptors are flattened into a single list. * - * @chan.private holds pointer to &struct ep93xx_dma_data which contains - * necessary channel configuration information. For memcpy channels this must - * be %NULL. */ struct ep93xx_dma_chan { struct dma_chan chan; const struct ep93xx_dma_engine *edma; void __iomem *regs; + struct ep93xx_dma_chan_cfg dma_cfg; int irq; struct clk *clk; struct tasklet_struct tasklet; @@ -213,7 +243,12 @@ struct ep93xx_dma_engine { #define INTERRUPT_NEXT_BUFFER 2 size_t num_channels; - struct ep93xx_dma_chan channels[]; + struct ep93xx_dma_chan channels[] __counted_by(num_channels); +}; + +struct ep93xx_edma_data { + u32 id; + size_t num_channels; }; static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac) @@ -226,6 +261,31 @@ static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan) return container_of(chan, struct ep93xx_dma_chan, chan); } +static inline bool ep93xx_dma_chan_is_m2p(struct dma_chan *chan) +{ + if (device_is_compatible(chan->device->dev, "cirrus,ep9301-dma-m2p")) + return true; + + return !strcmp(dev_name(chan->device->dev), "ep93xx-dma-m2p"); +} + +/* + * ep93xx_dma_chan_direction - returns direction the channel can be used + * + * This function can be used in filter functions to find out whether the + * channel supports given DMA direction. Only M2P channels have such + * limitation, for M2M channels the direction is configurable. + */ +static inline enum dma_transfer_direction +ep93xx_dma_chan_direction(struct dma_chan *chan) +{ + if (!ep93xx_dma_chan_is_m2p(chan)) + return DMA_TRANS_NONE; + + /* even channels are for TX, odd for RX */ + return (chan->chan_id % 2 == 0) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; +} + /** * ep93xx_dma_set_active - set new active descriptor chain * @edmac: channel @@ -318,10 +378,9 @@ static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control) static int m2p_hw_setup(struct ep93xx_dma_chan *edmac) { - struct ep93xx_dma_data *data = edmac->chan.private; u32 control; - writel(data->port & 0xf, edmac->regs + M2P_PPALLOC); + writel(edmac->dma_cfg.port & 0xf, edmac->regs + M2P_PPALLOC); control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE | M2P_CONTROL_ENABLE; @@ -458,16 +517,15 @@ static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac) static int m2m_hw_setup(struct ep93xx_dma_chan *edmac) { - const struct ep93xx_dma_data *data = edmac->chan.private; u32 control = 0; - if (!data) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_MEM) { /* This is memcpy channel, nothing to configure */ writel(control, edmac->regs + M2M_CONTROL); return 0; } - switch (data->port) { + switch (edmac->dma_cfg.port) { case EP93XX_DMA_SSP: /* * This was found via experimenting - anything less than 5 @@ -477,7 +535,7 @@ static int m2m_hw_setup(struct ep93xx_dma_chan *edmac) control = (5 << M2M_CONTROL_PWSC_SHIFT); control |= M2M_CONTROL_NO_HDSK; - if (data->direction == DMA_MEM_TO_DEV) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_DEV) { control |= M2M_CONTROL_DAH; control |= M2M_CONTROL_TM_TX; control |= M2M_CONTROL_RSS_SSPTX; @@ -493,7 +551,7 @@ static int m2m_hw_setup(struct ep93xx_dma_chan *edmac) * This IDE part is totally untested. Values below are taken * from the EP93xx Users's Guide and might not be correct. */ - if (data->direction == DMA_MEM_TO_DEV) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_DEV) { /* Worst case from the UG */ control = (3 << M2M_CONTROL_PWSC_SHIFT); control |= M2M_CONTROL_DAH; @@ -548,7 +606,6 @@ static void m2m_fill_desc(struct ep93xx_dma_chan *edmac) static void m2m_hw_submit(struct ep93xx_dma_chan *edmac) { - struct ep93xx_dma_data *data = edmac->chan.private; u32 control = readl(edmac->regs + M2M_CONTROL); /* @@ -574,7 +631,7 @@ static void m2m_hw_submit(struct ep93xx_dma_chan *edmac) control |= M2M_CONTROL_ENABLE; writel(control, edmac->regs + M2M_CONTROL); - if (!data) { + if (edmac->dma_cfg.dir == DMA_MEM_TO_MEM) { /* * For memcpy channels the software trigger must be asserted * in order to start the memcpy operation. @@ -636,7 +693,7 @@ static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac) */ if (ep93xx_dma_advance_active(edmac)) { m2m_fill_desc(edmac); - if (done && !edmac->chan.private) { + if (done && edmac->dma_cfg.dir == DMA_MEM_TO_MEM) { /* Software trigger for memcpy channel */ control = readl(edmac->regs + M2M_CONTROL); control |= M2M_CONTROL_START; @@ -841,7 +898,7 @@ static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx) desc = container_of(tx, struct ep93xx_dma_desc, txd); /* - * If nothing is currently prosessed, we push this descriptor + * If nothing is currently processed, we push this descriptor * directly to the hardware. Otherwise we put the descriptor * to the pending queue. */ @@ -867,25 +924,21 @@ static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx) static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan) { struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan); - struct ep93xx_dma_data *data = chan->private; const char *name = dma_chan_name(chan); int ret, i; /* Sanity check the channel parameters */ if (!edmac->edma->m2m) { - if (!data) - return -EINVAL; - if (data->port < EP93XX_DMA_I2S1 || - data->port > EP93XX_DMA_IRDA) + if (edmac->dma_cfg.port > EP93XX_DMA_IRDA) return -EINVAL; - if (data->direction != ep93xx_dma_chan_direction(chan)) + if (edmac->dma_cfg.dir != ep93xx_dma_chan_direction(chan)) return -EINVAL; } else { - if (data) { - switch (data->port) { + if (edmac->dma_cfg.dir != DMA_MEM_TO_MEM) { + switch (edmac->dma_cfg.port) { case EP93XX_DMA_SSP: case EP93XX_DMA_IDE: - if (!is_slave_direction(data->direction)) + if (!is_slave_direction(edmac->dma_cfg.dir)) return -EINVAL; break; default: @@ -894,10 +947,7 @@ static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan) } } - if (data && data->name) - name = data->name; - - ret = clk_enable(edmac->clk); + ret = clk_prepare_enable(edmac->clk); if (ret) return ret; @@ -936,7 +986,7 @@ static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan) fail_free_irq: free_irq(edmac->irq, edmac); fail_clk_disable: - clk_disable(edmac->clk); + clk_disable_unprepare(edmac->clk); return ret; } @@ -969,7 +1019,7 @@ static void ep93xx_dma_free_chan_resources(struct dma_chan *chan) list_for_each_entry_safe(desc, d, &list, node) kfree(desc); - clk_disable(edmac->clk); + clk_disable_unprepare(edmac->clk); free_irq(edmac->irq, edmac); } @@ -1025,7 +1075,7 @@ fail: * @chan: channel * @sgl: list of buffers to transfer * @sg_len: number of entries in @sgl - * @dir: direction of tha DMA transfer + * @dir: direction of the DMA transfer * @flags: flags for the descriptor * @context: operation context (ignored) * @@ -1183,7 +1233,7 @@ fail: * * Synchronizes the DMA channel termination to the current context. When this * function returns it is guaranteed that all transfers for previously issued - * descriptors have stopped and and it is safe to free the memory associated + * descriptors have stopped and it is safe to free the memory associated * with them. Furthermore it is guaranteed that all complete callback functions * for a previously submitted descriptor have finished running and it is safe to * free resources accessed from within the complete callbacks. @@ -1315,38 +1365,56 @@ static void ep93xx_dma_issue_pending(struct dma_chan *chan) ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan)); } -static int __init ep93xx_dma_probe(struct platform_device *pdev) +static struct ep93xx_dma_engine *ep93xx_dma_of_probe(struct platform_device *pdev) { - struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev); + const struct ep93xx_edma_data *data; + struct device *dev = &pdev->dev; struct ep93xx_dma_engine *edma; struct dma_device *dma_dev; - size_t edma_size; - int ret, i; + char dma_clk_name[5]; + int i; - edma_size = pdata->num_channels * sizeof(struct ep93xx_dma_chan); - edma = kzalloc(sizeof(*edma) + edma_size, GFP_KERNEL); + data = device_get_match_data(dev); + if (!data) + return ERR_PTR(dev_err_probe(dev, -ENODEV, "No device match found\n")); + + edma = devm_kzalloc(dev, struct_size(edma, channels, data->num_channels), + GFP_KERNEL); if (!edma) - return -ENOMEM; + return ERR_PTR(-ENOMEM); + edma->m2m = data->id; + edma->num_channels = data->num_channels; dma_dev = &edma->dma_dev; - edma->m2m = platform_get_device_id(pdev)->driver_data; - edma->num_channels = pdata->num_channels; INIT_LIST_HEAD(&dma_dev->channels); - for (i = 0; i < pdata->num_channels; i++) { - const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i]; + for (i = 0; i < edma->num_channels; i++) { struct ep93xx_dma_chan *edmac = &edma->channels[i]; + int len; edmac->chan.device = dma_dev; - edmac->regs = cdata->base; - edmac->irq = cdata->irq; + edmac->regs = devm_platform_ioremap_resource(pdev, i); + if (IS_ERR(edmac->regs)) + return ERR_CAST(edmac->regs); + + edmac->irq = fwnode_irq_get(dev_fwnode(dev), i); + if (edmac->irq < 0) + return ERR_PTR(edmac->irq); + edmac->edma = edma; - edmac->clk = clk_get(NULL, cdata->name); + if (edma->m2m) + len = snprintf(dma_clk_name, sizeof(dma_clk_name), "m2m%u", i); + else + len = snprintf(dma_clk_name, sizeof(dma_clk_name), "m2p%u", i); + if (len >= sizeof(dma_clk_name)) + return ERR_PTR(-ENOBUFS); + + edmac->clk = devm_clk_get(dev, dma_clk_name); if (IS_ERR(edmac->clk)) { - dev_warn(&pdev->dev, "failed to get clock for %s\n", - cdata->name); - continue; + dev_err_probe(dev, PTR_ERR(edmac->clk), + "no %s clock found\n", dma_clk_name); + return ERR_CAST(edmac->clk); } spin_lock_init(&edmac->lock); @@ -1359,6 +1427,90 @@ static int __init ep93xx_dma_probe(struct platform_device *pdev) &dma_dev->channels); } + return edma; +} + +static bool ep93xx_m2p_dma_filter(struct dma_chan *chan, void *filter_param) +{ + struct ep93xx_dma_chan *echan = to_ep93xx_dma_chan(chan); + struct ep93xx_dma_chan_cfg *cfg = filter_param; + + if (cfg->dir != ep93xx_dma_chan_direction(chan)) + return false; + + echan->dma_cfg = *cfg; + return true; +} + +static struct dma_chan *ep93xx_m2p_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct ep93xx_dma_engine *edma = ofdma->of_dma_data; + dma_cap_mask_t mask = edma->dma_dev.cap_mask; + struct ep93xx_dma_chan_cfg dma_cfg; + u8 port = dma_spec->args[0]; + u8 direction = dma_spec->args[1]; + + if (port > EP93XX_DMA_IRDA) + return NULL; + + if (!is_slave_direction(direction)) + return NULL; + + dma_cfg.port = port; + dma_cfg.dir = direction; + + return __dma_request_channel(&mask, ep93xx_m2p_dma_filter, &dma_cfg, ofdma->of_node); +} + +static bool ep93xx_m2m_dma_filter(struct dma_chan *chan, void *filter_param) +{ + struct ep93xx_dma_chan *echan = to_ep93xx_dma_chan(chan); + struct ep93xx_dma_chan_cfg *cfg = filter_param; + + echan->dma_cfg = *cfg; + + return true; +} + +static struct dma_chan *ep93xx_m2m_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct ep93xx_dma_engine *edma = ofdma->of_dma_data; + dma_cap_mask_t mask = edma->dma_dev.cap_mask; + struct ep93xx_dma_chan_cfg dma_cfg; + u8 port = dma_spec->args[0]; + u8 direction = dma_spec->args[1]; + + if (!is_slave_direction(direction)) + return NULL; + + switch (port) { + case EP93XX_DMA_SSP: + case EP93XX_DMA_IDE: + break; + default: + return NULL; + } + + dma_cfg.port = port; + dma_cfg.dir = direction; + + return __dma_request_channel(&mask, ep93xx_m2m_dma_filter, &dma_cfg, ofdma->of_node); +} + +static int ep93xx_dma_probe(struct platform_device *pdev) +{ + struct ep93xx_dma_engine *edma; + struct dma_device *dma_dev; + int ret; + + edma = ep93xx_dma_of_probe(pdev); + if (IS_ERR(edma)) + return PTR_ERR(edma); + + dma_dev = &edma->dma_dev; + dma_cap_zero(dma_dev->cap_mask); dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); @@ -1395,21 +1547,46 @@ static int __init ep93xx_dma_probe(struct platform_device *pdev) } ret = dma_async_device_register(dma_dev); - if (unlikely(ret)) { - for (i = 0; i < edma->num_channels; i++) { - struct ep93xx_dma_chan *edmac = &edma->channels[i]; - if (!IS_ERR_OR_NULL(edmac->clk)) - clk_put(edmac->clk); - } - kfree(edma); + if (ret) + return ret; + + if (edma->m2m) { + ret = of_dma_controller_register(pdev->dev.of_node, ep93xx_m2m_dma_of_xlate, + edma); } else { - dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n", - edma->m2m ? "M" : "P"); + ret = of_dma_controller_register(pdev->dev.of_node, ep93xx_m2p_dma_of_xlate, + edma); } + if (ret) + goto err_dma_unregister; + + dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n", edma->m2m ? "M" : "P"); + + return 0; + +err_dma_unregister: + dma_async_device_unregister(dma_dev); return ret; } +static const struct ep93xx_edma_data edma_m2p = { + .id = M2P_DMA, + .num_channels = 10, +}; + +static const struct ep93xx_edma_data edma_m2m = { + .id = M2M_DMA, + .num_channels = 2, +}; + +static const struct of_device_id ep93xx_dma_of_ids[] = { + { .compatible = "cirrus,ep9301-dma-m2p", .data = &edma_m2p }, + { .compatible = "cirrus,ep9301-dma-m2m", .data = &edma_m2m }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ep93xx_dma_of_ids); + static const struct platform_device_id ep93xx_dma_driver_ids[] = { { "ep93xx-dma-m2p", 0 }, { "ep93xx-dma-m2m", 1 }, @@ -1419,16 +1596,13 @@ static const struct platform_device_id ep93xx_dma_driver_ids[] = { static struct platform_driver ep93xx_dma_driver = { .driver = { .name = "ep93xx-dma", + .of_match_table = ep93xx_dma_of_ids, }, .id_table = ep93xx_dma_driver_ids, + .probe = ep93xx_dma_probe, }; -static int __init ep93xx_dma_module_init(void) -{ - return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe); -} -subsys_initcall(ep93xx_dma_module_init); +module_platform_driver(ep93xx_dma_driver); MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>"); MODULE_DESCRIPTION("EP93xx DMA driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c index 4ae057922ef1..36384d019263 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.c @@ -38,15 +38,17 @@ static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan) if (!dpaa2_chan->fd_pool) goto err; - dpaa2_chan->fl_pool = dma_pool_create("fl_pool", dev, - sizeof(struct dpaa2_fl_entry), - sizeof(struct dpaa2_fl_entry), 0); + dpaa2_chan->fl_pool = + dma_pool_create("fl_pool", dev, + sizeof(struct dpaa2_fl_entry) * 3, + sizeof(struct dpaa2_fl_entry), 0); + if (!dpaa2_chan->fl_pool) goto err_fd; dpaa2_chan->sdd_pool = dma_pool_create("sdd_pool", dev, - sizeof(struct dpaa2_qdma_sd_d), + sizeof(struct dpaa2_qdma_sd_d) * 2, sizeof(struct dpaa2_qdma_sd_d), 0); if (!dpaa2_chan->sdd_pool) goto err_fl; @@ -291,9 +293,8 @@ static void dpaa2_qdma_issue_pending(struct dma_chan *chan) err = dpaa2_io_service_enqueue_fq(NULL, dpaa2_chan->fqid, fd); if (err) { - list_del(&dpaa2_comp->list); - list_add_tail(&dpaa2_comp->list, - &dpaa2_chan->comp_free); + list_move_tail(&dpaa2_comp->list, + &dpaa2_chan->comp_free); } } err_enqueue: @@ -361,7 +362,7 @@ static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) for (i = 0; i < priv->num_pairs; i++) { err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, - i, &priv->rx_queue_attr[i]); + i, 0, &priv->rx_queue_attr[i]); if (err) { dev_err(dev, "dpdmai_get_rx_queue() failed\n"); goto exit; @@ -369,13 +370,13 @@ static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, - i, &priv->tx_fqid[i]); + i, 0, &priv->tx_queue_attr[i]); if (err) { dev_err(dev, "dpdmai_get_tx_queue() failed\n"); goto exit; } - ppriv->req_fqid = priv->tx_fqid[i]; - ppriv->prio = i; + ppriv->req_fqid = priv->tx_queue_attr[i].fqid; + ppriv->prio = DPAA2_QDMA_DEFAULT_PRIORITY; ppriv->priv = priv; ppriv++; } @@ -541,7 +542,7 @@ static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; rx_queue_cfg.dest_cfg.priority = ppriv->prio; err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, - rx_queue_cfg.dest_cfg.priority, + rx_queue_cfg.dest_cfg.priority, 0, &rx_queue_cfg); if (err) { dev_err(dev, "dpdmai_set_rx_queue() failed\n"); @@ -626,8 +627,7 @@ static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc) dpaa2_comp = to_fsl_qdma_comp(vdesc); qchan = dpaa2_comp->qchan; spin_lock_irqsave(&qchan->queue_lock, flags); - list_del(&dpaa2_comp->list); - list_add_tail(&dpaa2_comp->list, &qchan->comp_free); + list_move_tail(&dpaa2_comp->list, &qchan->comp_free); spin_unlock_irqrestore(&qchan->queue_lock, flags); } @@ -642,7 +642,7 @@ static int dpaa2_dpdmai_init_channels(struct dpaa2_qdma_engine *dpaa2_qdma) for (i = 0; i < dpaa2_qdma->n_chans; i++) { dpaa2_chan = &dpaa2_qdma->chans[i]; dpaa2_chan->qdma = dpaa2_qdma; - dpaa2_chan->fqid = priv->tx_fqid[i % num]; + dpaa2_chan->fqid = priv->tx_queue_attr[i % num].fqid; dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev); spin_lock_init(&dpaa2_chan->queue_lock); @@ -703,7 +703,7 @@ static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev) /* DPDMAI enable */ err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle); if (err) { - dev_err(dev, "dpdmai_enable() faile\n"); + dev_err(dev, "dpdmai_enable() failed\n"); goto err_enable; } @@ -767,7 +767,7 @@ err_mcportal: return err; } -static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) +static void dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) { struct dpaa2_qdma_engine *dpaa2_qdma; struct dpaa2_qdma_priv *priv; @@ -789,8 +789,6 @@ static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) dma_async_device_unregister(&dpaa2_qdma->dma_dev); kfree(priv); kfree(dpaa2_qdma); - - return 0; } static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) @@ -804,7 +802,7 @@ static void dpaa2_qdma_shutdown(struct fsl_mc_device *ls_dev) dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); dpaa2_dpdmai_dpio_unbind(priv); dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); - dpdmai_destroy(priv->mc_io, 0, ls_dev->mc_handle); + dpdmai_destroy(priv->mc_io, 0, priv->dpqdma_id, ls_dev->mc_handle); } static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { @@ -818,7 +816,6 @@ static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { static struct fsl_mc_driver dpaa2_qdma_driver = { .driver = { .name = "dpaa2-qdma", - .owner = THIS_MODULE, }, .probe = dpaa2_qdma_probe, .remove = dpaa2_qdma_remove, diff --git a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h index 7d571849c569..36c284a3d184 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpaa2-qdma.h @@ -6,13 +6,14 @@ #define DPAA2_QDMA_STORE_SIZE 16 #define NUM_CH 8 +#define DPAA2_QDMA_DEFAULT_PRIORITY 0 struct dpaa2_qdma_sd_d { u32 rsv:32; union { struct { - u32 ssd:12; /* souce stride distance */ - u32 sss:12; /* souce stride size */ + u32 ssd:12; /* source stride distance */ + u32 sss:12; /* source stride size */ u32 rsv1:8; } sdf; struct { @@ -47,7 +48,7 @@ struct dpaa2_qdma_sd_d { #define QDMA_SER_DISABLE (8) /* no notification */ #define QDMA_SER_CTX BIT(8) /* notification by FQD_CTX[fqid] */ #define QDMA_SER_DEST (2 << 8) /* notification by destination desc */ -#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */ +#define QDMA_SER_BOTH (3 << 8) /* source and dest notification */ #define QDMA_FD_SPF_ENALBE BIT(30) /* source prefetch enable */ #define QMAN_FD_VA_ENABLE BIT(14) /* Address used is virtual address */ @@ -122,8 +123,8 @@ struct dpaa2_qdma_priv { struct dpaa2_qdma_engine *dpaa2_qdma; struct dpaa2_qdma_priv_per_prio *ppriv; - struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM]; - u32 tx_fqid[DPDMAI_PRIO_NUM]; + struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_MAX_QUEUE_NUM]; + struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_MAX_QUEUE_NUM]; }; struct dpaa2_qdma_priv_per_prio { @@ -139,7 +140,7 @@ struct dpaa2_qdma_priv_per_prio { static struct soc_device_attribute soc_fixup_tuning[] = { { .family = "QorIQ LX2160A"}, - { }, + { /* sentinel */ } }; /* FD pool size: one FD + 3 Frame list + 2 source/destination descriptor */ diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c index 878662aaa1c2..4be81db24a19 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.c +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.c @@ -1,52 +1,52 @@ // SPDX-License-Identifier: GPL-2.0 // Copyright 2019 NXP +#include <linux/bitfield.h> #include <linux/module.h> #include <linux/types.h> #include <linux/io.h> #include <linux/fsl/mc.h> #include "dpdmai.h" +#define DEST_TYPE_MASK 0xF + struct dpdmai_rsp_get_attributes { __le32 id; u8 num_of_priorities; - u8 pad0[3]; + u8 num_of_queues; + u8 pad0[2]; __le16 major; __le16 minor; }; struct dpdmai_cmd_queue { __le32 dest_id; - u8 priority; - u8 queue; + u8 dest_priority; + union { + u8 queue; + u8 pri; + }; u8 dest_type; - u8 pad; + u8 queue_idx; __le64 user_ctx; union { __le32 options; __le32 fqid; }; -}; +} __packed; struct dpdmai_rsp_get_tx_queue { __le64 pad; __le32 fqid; }; -#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \ - ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg)) - -/* cmd, param, offset, width, type, arg_name */ -#define DPDMAI_CMD_CREATE(cmd, cfg) \ -do { \ - MC_CMD_OP(cmd, 0, 8, 8, u8, (cfg)->priorities[0]);\ - MC_CMD_OP(cmd, 0, 16, 8, u8, (cfg)->priorities[1]);\ -} while (0) +struct dpdmai_cmd_open { + __le32 dpdmai_id; +} __packed; -static inline u64 mc_enc(int lsoffset, int width, u64 val) -{ - return (val & MAKE_UMASK64(width)) << lsoffset; -} +struct dpdmai_cmd_destroy { + __le32 dpdmai_id; +} __packed; /** * dpdmai_open() - Open a control session for the specified object @@ -68,16 +68,16 @@ static inline u64 mc_enc(int lsoffset, int width, u64 val) int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token) { + struct dpdmai_cmd_open *cmd_params; struct fsl_mc_command cmd = { 0 }; - __le64 *cmd_dpdmai_id; int err; /* prepare command */ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN, cmd_flags, 0); - cmd_dpdmai_id = cmd.params; - *cmd_dpdmai_id = cpu_to_le32(dpdmai_id); + cmd_params = (struct dpdmai_cmd_open *)&cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -116,65 +116,26 @@ int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) EXPORT_SYMBOL_GPL(dpdmai_close); /** - * dpdmai_create() - Create the DPDMAI object - * @mc_io: Pointer to MC portal's I/O object - * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' - * @cfg: Configuration structure - * @token: Returned token; use in subsequent API calls - * - * Create the DPDMAI object, allocate required resources and - * perform required initialization. - * - * The object can be created either by declaring it in the - * DPL file, or by calling this function. - * - * This function returns a unique authentication token, - * associated with the specific object ID and the specific MC - * portal; this token must be used in all subsequent calls to - * this specific object. For objects that are created using the - * DPL file, call dpdmai_open() function to get an authentication - * token first. - * - * Return: '0' on Success; Error code otherwise. - */ -int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, - const struct dpdmai_cfg *cfg, u16 *token) -{ - struct fsl_mc_command cmd = { 0 }; - int err; - - /* prepare command */ - cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE, - cmd_flags, 0); - DPDMAI_CMD_CREATE(cmd, cfg); - - /* send command to mc*/ - err = mc_send_command(mc_io, &cmd); - if (err) - return err; - - /* retrieve response parameters */ - *token = mc_cmd_hdr_read_token(&cmd); - - return 0; -} - -/** * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpdmai_id: The object id; it must be a valid id within the container that created this object; * @token: Token of DPDMAI object * * Return: '0' on Success; error code otherwise. */ -int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u32 dpdmai_id, u16 token) { + struct dpdmai_cmd_destroy *cmd_params; struct fsl_mc_command cmd = { 0 }; /* prepare command */ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, cmd_flags, token); + cmd_params = (struct dpdmai_cmd_destroy *)&cmd.params; + cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); + /* send command to mc*/ return mc_send_command(mc_io, &cmd); } @@ -274,6 +235,7 @@ int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, attr->version.major = le16_to_cpu(rsp_params->major); attr->version.minor = le16_to_cpu(rsp_params->minor); attr->num_of_priorities = rsp_params->num_of_priorities; + attr->num_of_queues = rsp_params->num_of_queues; return 0; } @@ -284,13 +246,14 @@ EXPORT_SYMBOL_GPL(dpdmai_get_attributes); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @cfg: Rx queue configuration * * Return: '0' on Success; Error code otherwise. */ -int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, +int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 queue_idx, u8 priority, const struct dpdmai_rx_queue_cfg *cfg) { struct dpdmai_cmd_queue *cmd_params; @@ -302,11 +265,12 @@ int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); - cmd_params->priority = cfg->dest_cfg.priority; - cmd_params->queue = priority; + cmd_params->dest_priority = cfg->dest_cfg.priority; + cmd_params->pri = priority; cmd_params->dest_type = cfg->dest_cfg.dest_type; cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); cmd_params->options = cpu_to_le32(cfg->options); + cmd_params->queue_idx = queue_idx; /* send command to mc*/ return mc_send_command(mc_io, &cmd); @@ -318,13 +282,14 @@ EXPORT_SYMBOL_GPL(dpdmai_set_rx_queue); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA Queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation * @attr: Returned Rx queue attributes * * Return: '0' on Success; Error code otherwise. */ -int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, +int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 queue_idx, u8 priority, struct dpdmai_rx_queue_attr *attr) { struct dpdmai_cmd_queue *cmd_params; @@ -337,6 +302,7 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->queue = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -345,8 +311,8 @@ int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, /* retrieve response parameters */ attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); - attr->dest_cfg.priority = cmd_params->priority; - attr->dest_cfg.dest_type = cmd_params->dest_type; + attr->dest_cfg.priority = cmd_params->dest_priority; + attr->dest_cfg.dest_type = FIELD_GET(DEST_TYPE_MASK, cmd_params->dest_type); attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); attr->fqid = le32_to_cpu(cmd_params->fqid); @@ -359,14 +325,15 @@ EXPORT_SYMBOL_GPL(dpdmai_get_rx_queue); * @mc_io: Pointer to MC portal's I/O object * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' * @token: Token of DPDMAI object + * @queue_idx: DMA queue index * @priority: Select the queue relative to number of * priorities configured at DPDMAI creation - * @fqid: Returned Tx queue + * @attr: Returned DMA Tx queue attributes * * Return: '0' on Success; Error code otherwise. */ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, - u16 token, u8 priority, u32 *fqid) + u16 token, u8 queue_idx, u8 priority, struct dpdmai_tx_queue_attr *attr) { struct dpdmai_rsp_get_tx_queue *rsp_params; struct dpdmai_cmd_queue *cmd_params; @@ -379,6 +346,7 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, cmd_params = (struct dpdmai_cmd_queue *)cmd.params; cmd_params->queue = priority; + cmd_params->queue_idx = queue_idx; /* send command to mc*/ err = mc_send_command(mc_io, &cmd); @@ -388,10 +356,11 @@ int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, /* retrieve response parameters */ rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params; - *fqid = le32_to_cpu(rsp_params->fqid); + attr->fqid = le32_to_cpu(rsp_params->fqid); return 0; } EXPORT_SYMBOL_GPL(dpdmai_get_tx_queue); +MODULE_DESCRIPTION("NXP DPAA2 QDMA driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h index b13b9bf0c003..3fe7d8327366 100644 --- a/drivers/dma/fsl-dpaa2-qdma/dpdmai.h +++ b/drivers/dma/fsl-dpaa2-qdma/dpdmai.h @@ -5,14 +5,19 @@ #define __FSL_DPDMAI_H /* DPDMAI Version */ -#define DPDMAI_VER_MAJOR 2 -#define DPDMAI_VER_MINOR 2 +#define DPDMAI_VER_MAJOR 3 +#define DPDMAI_VER_MINOR 3 -#define DPDMAI_CMD_BASE_VERSION 0 +#define DPDMAI_CMD_BASE_VERSION 1 #define DPDMAI_CMD_ID_OFFSET 4 -#define DPDMAI_CMDID_FORMAT(x) (((x) << DPDMAI_CMD_ID_OFFSET) | \ - DPDMAI_CMD_BASE_VERSION) +/* + * Maximum number of Tx/Rx queues per DPDMAI object + */ +#define DPDMAI_MAX_QUEUE_NUM 8 + +#define DPDMAI_CMDID_FORMAT_V(x, v) (((x) << DPDMAI_CMD_ID_OFFSET) | (v)) +#define DPDMAI_CMDID_FORMAT(x) DPDMAI_CMDID_FORMAT_V(x, DPDMAI_CMD_BASE_VERSION) /* Command IDs */ #define DPDMAI_CMDID_CLOSE DPDMAI_CMDID_FORMAT(0x800) @@ -26,18 +31,9 @@ #define DPDMAI_CMDID_RESET DPDMAI_CMDID_FORMAT(0x005) #define DPDMAI_CMDID_IS_ENABLED DPDMAI_CMDID_FORMAT(0x006) -#define DPDMAI_CMDID_SET_IRQ DPDMAI_CMDID_FORMAT(0x010) -#define DPDMAI_CMDID_GET_IRQ DPDMAI_CMDID_FORMAT(0x011) -#define DPDMAI_CMDID_SET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x012) -#define DPDMAI_CMDID_GET_IRQ_ENABLE DPDMAI_CMDID_FORMAT(0x013) -#define DPDMAI_CMDID_SET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x014) -#define DPDMAI_CMDID_GET_IRQ_MASK DPDMAI_CMDID_FORMAT(0x015) -#define DPDMAI_CMDID_GET_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x016) -#define DPDMAI_CMDID_CLEAR_IRQ_STATUS DPDMAI_CMDID_FORMAT(0x017) - -#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A0) -#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMDID_FORMAT(0x1A1) -#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMDID_FORMAT(0x1A2) +#define DPDMAI_CMDID_SET_RX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A0, 2) +#define DPDMAI_CMDID_GET_RX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A1, 2) +#define DPDMAI_CMDID_GET_TX_QUEUE DPDMAI_CMDID_FORMAT_V(0x1A2, 2) #define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */ #define MC_CMD_HDR_TOKEN_S 16 /* Token field size */ @@ -49,30 +45,32 @@ * Contains initialization APIs and runtime control APIs for DPDMAI */ -/** +/* * Maximum number of Tx/Rx priorities per DPDMAI object */ #define DPDMAI_PRIO_NUM 2 /* DPDMAI queue modification options */ -/** +/* * Select to modify the user's context associated with the queue */ #define DPDMAI_QUEUE_OPT_USER_CTX 0x1 -/** +/* * Select to modify the queue's destination */ #define DPDMAI_QUEUE_OPT_DEST 0x2 /** * struct dpdmai_cfg - Structure representing DPDMAI configuration + * @num_queues: Number of the DMA queues * @priorities: Priorities for the DMA hardware processing; valid priorities are * configured with values 1-8; the entry following last valid entry * should be configured with 0 */ struct dpdmai_cfg { + u8 num_queues; u8 priorities[DPDMAI_PRIO_NUM]; }; @@ -80,20 +78,19 @@ struct dpdmai_cfg { * struct dpdmai_attr - Structure representing DPDMAI attributes * @id: DPDMAI object ID * @version: DPDMAI version + * @version.major: DPDMAI major version + * @version.minor: DPDMAI minor version * @num_of_priorities: number of priorities + * @num_of_queues: number of the DMA queues */ struct dpdmai_attr { int id; - /** - * struct version - DPDMAI version - * @major: DPDMAI major version - * @minor: DPDMAI minor version - */ struct { u16 major; u16 minor; } version; u8 num_of_priorities; + u8 num_of_queues; }; /** @@ -158,22 +155,24 @@ struct dpdmai_rx_queue_attr { u32 fqid; }; +struct dpdmai_tx_queue_attr { + u32 fqid; +}; + int dpdmai_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpdmai_id, u16 *token); int dpdmai_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -int dpdmai_create(struct fsl_mc_io *mc_io, u32 cmd_flags, - const struct dpdmai_cfg *cfg, u16 *token); +int dpdmai_destroy(struct fsl_mc_io *mc_io, u32 cmd_flags, u32 dpdmai_id, u16 token); int dpdmai_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); int dpdmai_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, struct dpdmai_attr *attr); int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, - u8 priority, const struct dpdmai_rx_queue_cfg *cfg); + u8 queue_idx, u8 priority, const struct dpdmai_rx_queue_cfg *cfg); int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, - u8 priority, struct dpdmai_rx_queue_attr *attr); + u8 queue_idx, u8 priority, struct dpdmai_rx_queue_attr *attr); int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, - u16 token, u8 priority, u32 *fqid); + u16 token, u8 queue_idx, u8 priority, struct dpdmai_tx_queue_attr *attr); #endif /* __FSL_DPDMAI_H */ diff --git a/drivers/dma/fsl-edma-common.c b/drivers/dma/fsl-edma-common.c index 930ae268c497..a59212758029 100644 --- a/drivers/dma/fsl-edma-common.c +++ b/drivers/dma/fsl-edma-common.c @@ -3,10 +3,14 @@ // Copyright (c) 2013-2014 Freescale Semiconductor, Inc // Copyright (c) 2017 Sysam, Angelo Dureghello <angelo@sysam.it> +#include <linux/cleanup.h> +#include <linux/clk.h> #include <linux/dmapool.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/pm_domain.h> #include "fsl-edma-common.h" @@ -40,14 +44,70 @@ #define EDMA64_ERRH 0x28 #define EDMA64_ERRL 0x2c -#define EDMA_TCD 0x1000 +void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan) +{ + spin_lock(&fsl_chan->vchan.lock); + + if (!fsl_chan->edesc) { + /* terminate_all called before */ + spin_unlock(&fsl_chan->vchan.lock); + return; + } + + if (!fsl_chan->edesc->iscyclic) { + list_del(&fsl_chan->edesc->vdesc.node); + vchan_cookie_complete(&fsl_chan->edesc->vdesc); + fsl_chan->edesc = NULL; + fsl_chan->status = DMA_COMPLETE; + } else { + vchan_cyclic_callback(&fsl_chan->edesc->vdesc); + } + + if (!fsl_chan->edesc) + fsl_edma_xfer_desc(fsl_chan); + + spin_unlock(&fsl_chan->vchan.lock); +} + +static void fsl_edma3_enable_request(struct fsl_edma_chan *fsl_chan) +{ + u32 val, flags; + + flags = fsl_edma_drvflags(fsl_chan); + val = edma_readl_chreg(fsl_chan, ch_sbr); + if (fsl_chan->is_rxchan) + val |= EDMA_V3_CH_SBR_RD; + else + val |= EDMA_V3_CH_SBR_WR; + + if (fsl_chan->is_remote) + val &= ~(EDMA_V3_CH_SBR_RD | EDMA_V3_CH_SBR_WR); + + edma_writel_chreg(fsl_chan, val, ch_sbr); + + if (flags & FSL_EDMA_DRV_HAS_CHMUX) { + /* + * ch_mux: With the exception of 0, attempts to write a value + * already in use will be forced to 0. + */ + if (!edma_readl(fsl_chan->edma, fsl_chan->mux_addr)) + edma_writel(fsl_chan->edma, fsl_chan->srcid, fsl_chan->mux_addr); + } + + val = edma_readl_chreg(fsl_chan, ch_csr); + val |= EDMA_V3_CH_CSR_ERQ | EDMA_V3_CH_CSR_EEI; + edma_writel_chreg(fsl_chan, val, ch_csr); +} static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) { struct edma_regs *regs = &fsl_chan->edma->regs; u32 ch = fsl_chan->vchan.chan.chan_id; - if (fsl_chan->edma->drvdata->version == v1) { + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_SPLIT_REG) + return fsl_edma3_enable_request(fsl_chan); + + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_WRAP_IO) { edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), regs->seei); edma_writeb(fsl_chan->edma, ch, regs->serq); } else { @@ -59,12 +119,29 @@ static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) } } +static void fsl_edma3_disable_request(struct fsl_edma_chan *fsl_chan) +{ + u32 val = edma_readl_chreg(fsl_chan, ch_csr); + u32 flags; + + flags = fsl_edma_drvflags(fsl_chan); + + if (flags & FSL_EDMA_DRV_HAS_CHMUX) + edma_writel(fsl_chan->edma, 0, fsl_chan->mux_addr); + + val &= ~EDMA_V3_CH_CSR_ERQ; + edma_writel_chreg(fsl_chan, val, ch_csr); +} + void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) { struct edma_regs *regs = &fsl_chan->edma->regs; u32 ch = fsl_chan->vchan.chan.chan_id; - if (fsl_chan->edma->drvdata->version == v1) { + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_SPLIT_REG) + return fsl_edma3_disable_request(fsl_chan); + + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_WRAP_IO) { edma_writeb(fsl_chan->edma, ch, regs->cerq); edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), regs->ceei); } else { @@ -75,7 +152,6 @@ void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) iowrite8(EDMA_CEEI_CEEI(ch), regs->ceei); } } -EXPORT_SYMBOL_GPL(fsl_edma_disable_request); static void mux_configure8(struct fsl_edma_chan *fsl_chan, void __iomem *addr, u32 off, u32 slot, bool enable) @@ -112,36 +188,37 @@ void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, int endian_diff[4] = {3, 1, -1, -3}; u32 dmamux_nr = fsl_chan->edma->drvdata->dmamuxs; + if (!dmamux_nr) + return; + chans_per_mux = fsl_chan->edma->n_chans / dmamux_nr; ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; - if (fsl_chan->edma->drvdata->mux_swap) + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_MUX_SWAP) ch_off += endian_diff[ch_off % 4]; muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux]; slot = EDMAMUX_CHCFG_SOURCE(slot); - if (fsl_chan->edma->drvdata->version == v3) + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_CONFIG32) mux_configure32(fsl_chan, muxaddr, ch_off, slot, enable); else mux_configure8(fsl_chan, muxaddr, ch_off, slot, enable); } -EXPORT_SYMBOL_GPL(fsl_edma_chan_mux); -static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) +static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth src_addr_width, + enum dma_slave_buswidth dst_addr_width) { - switch (addr_width) { - case 1: - return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; - case 2: - return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; - case 4: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - case 8: - return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; - default: - return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; - } + u32 src_val, dst_val; + + if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) + dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + + src_val = ffs(src_addr_width) - 1; + dst_val = ffs(dst_addr_width) - 1; + return dst_val | (src_val << 8); } void fsl_edma_free_desc(struct virt_dma_desc *vdesc) @@ -155,7 +232,6 @@ void fsl_edma_free_desc(struct virt_dma_desc *vdesc) fsl_desc->tcd[i].ptcd); kfree(fsl_desc); } -EXPORT_SYMBOL_GPL(fsl_edma_free_desc); int fsl_edma_terminate_all(struct dma_chan *chan) { @@ -166,13 +242,16 @@ int fsl_edma_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&fsl_chan->vchan.lock, flags); fsl_edma_disable_request(fsl_chan); fsl_chan->edesc = NULL; - fsl_chan->idle = true; + fsl_chan->status = DMA_COMPLETE; vchan_get_all_descriptors(&fsl_chan->vchan, &head); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_PD) + pm_runtime_allow(fsl_chan->pd_dev); + return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_terminate_all); int fsl_edma_pause(struct dma_chan *chan) { @@ -183,12 +262,10 @@ int fsl_edma_pause(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma_disable_request(fsl_chan); fsl_chan->status = DMA_PAUSED; - fsl_chan->idle = true; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_pause); int fsl_edma_resume(struct dma_chan *chan) { @@ -199,12 +276,10 @@ int fsl_edma_resume(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_resume); static void fsl_edma_unprep_slave_dma(struct fsl_edma_chan *fsl_chan) { @@ -265,40 +340,51 @@ int fsl_edma_slave_config(struct dma_chan *chan, return 0; } -EXPORT_SYMBOL_GPL(fsl_edma_slave_config); static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, struct virt_dma_desc *vdesc, bool in_progress) { struct fsl_edma_desc *edesc = fsl_chan->edesc; - struct edma_regs *regs = &fsl_chan->edma->regs; - u32 ch = fsl_chan->vchan.chan.chan_id; enum dma_transfer_direction dir = edesc->dirn; - dma_addr_t cur_addr, dma_addr; + dma_addr_t cur_addr, dma_addr, old_addr; size_t len, size; + u32 nbytes = 0; int i; /* calculate the total size in this desc */ - for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) - len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); + for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) { + nbytes = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, nbytes); + if (nbytes & (EDMA_V3_TCD_NBYTES_DMLOE | EDMA_V3_TCD_NBYTES_SMLOE)) + nbytes = EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(nbytes); + len += nbytes * fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, biter); + } if (!in_progress) return len; - if (dir == DMA_MEM_TO_DEV) - cur_addr = edma_readl(fsl_chan->edma, ®s->tcd[ch].saddr); - else - cur_addr = edma_readl(fsl_chan->edma, ®s->tcd[ch].daddr); + /* 64bit read is not atomic, need read retry when high 32bit changed */ + do { + if (dir == DMA_MEM_TO_DEV) { + old_addr = edma_read_tcdreg(fsl_chan, saddr); + cur_addr = edma_read_tcdreg(fsl_chan, saddr); + } else { + old_addr = edma_read_tcdreg(fsl_chan, daddr); + cur_addr = edma_read_tcdreg(fsl_chan, daddr); + } + } while (upper_32_bits(cur_addr) != upper_32_bits(old_addr)); /* figure out the finished and calculate the residue */ for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { - size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes) - * le16_to_cpu(edesc->tcd[i].vtcd->biter); + nbytes = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, nbytes); + if (nbytes & (EDMA_V3_TCD_NBYTES_DMLOE | EDMA_V3_TCD_NBYTES_SMLOE)) + nbytes = EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(nbytes); + + size = nbytes * fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, biter); + if (dir == DMA_MEM_TO_DEV) - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr); + dma_addr = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, saddr); else - dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr); + dma_addr = fsl_edma_get_tcd_to_cpu(fsl_chan, edesc->tcd[i].vtcd, daddr); len -= size; if (cur_addr >= dma_addr && cur_addr < dma_addr + size) { @@ -340,14 +426,10 @@ enum dma_status fsl_edma_tx_status(struct dma_chan *chan, return fsl_chan->status; } -EXPORT_SYMBOL_GPL(fsl_edma_tx_status); -static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, - struct fsl_edma_hw_tcd *tcd) +static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, void *tcd) { - struct fsl_edma_engine *edma = fsl_chan->edma; - struct edma_regs *regs = &fsl_chan->edma->regs; - u32 ch = fsl_chan->vchan.chan.chan_id; + u16 csr = 0; /* * TCD parameters are stored in struct fsl_edma_hw_tcd in little @@ -355,33 +437,54 @@ static void fsl_edma_set_tcd_regs(struct fsl_edma_chan *fsl_chan, * big- or little-endian obeying the eDMA engine model endian, * and this is performed from specific edma_write functions */ - edma_writew(edma, 0, ®s->tcd[ch].csr); + edma_write_tcdreg(fsl_chan, 0, csr); + + edma_cp_tcd_to_reg(fsl_chan, tcd, saddr); + edma_cp_tcd_to_reg(fsl_chan, tcd, daddr); - edma_writel(edma, (s32)tcd->saddr, ®s->tcd[ch].saddr); - edma_writel(edma, (s32)tcd->daddr, ®s->tcd[ch].daddr); + edma_cp_tcd_to_reg(fsl_chan, tcd, attr); + edma_cp_tcd_to_reg(fsl_chan, tcd, soff); - edma_writew(edma, (s16)tcd->attr, ®s->tcd[ch].attr); - edma_writew(edma, tcd->soff, ®s->tcd[ch].soff); + edma_cp_tcd_to_reg(fsl_chan, tcd, nbytes); + edma_cp_tcd_to_reg(fsl_chan, tcd, slast); - edma_writel(edma, (s32)tcd->nbytes, ®s->tcd[ch].nbytes); - edma_writel(edma, (s32)tcd->slast, ®s->tcd[ch].slast); + edma_cp_tcd_to_reg(fsl_chan, tcd, citer); + edma_cp_tcd_to_reg(fsl_chan, tcd, biter); + edma_cp_tcd_to_reg(fsl_chan, tcd, doff); - edma_writew(edma, (s16)tcd->citer, ®s->tcd[ch].citer); - edma_writew(edma, (s16)tcd->biter, ®s->tcd[ch].biter); - edma_writew(edma, (s16)tcd->doff, ®s->tcd[ch].doff); + edma_cp_tcd_to_reg(fsl_chan, tcd, dlast_sga); - edma_writel(edma, (s32)tcd->dlast_sga, - ®s->tcd[ch].dlast_sga); + csr = fsl_edma_get_tcd_to_cpu(fsl_chan, tcd, csr); - edma_writew(edma, (s16)tcd->csr, ®s->tcd[ch].csr); + if (fsl_chan->is_sw) { + csr |= EDMA_TCD_CSR_START; + fsl_edma_set_tcd_to_le(fsl_chan, tcd, csr, csr); + } + + /* + * Must clear CHn_CSR[DONE] bit before enable TCDn_CSR[ESG] at EDMAv3 + * eDMAv4 have not such requirement. + * Change MLINK need clear CHn_CSR[DONE] for both eDMAv3 and eDMAv4. + */ + if (((fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_CLEAR_DONE_E_SG) && + (csr & EDMA_TCD_CSR_E_SG)) || + ((fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_CLEAR_DONE_E_LINK) && + (csr & EDMA_TCD_CSR_E_LINK))) + edma_writel_chreg(fsl_chan, edma_readl_chreg(fsl_chan, ch_csr), ch_csr); + + + edma_cp_tcd_to_reg(fsl_chan, tcd, csr); } static inline -void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, - u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, - u16 biter, u16 doff, u32 dlast_sga, bool major_int, +void fsl_edma_fill_tcd(struct fsl_edma_chan *fsl_chan, + struct fsl_edma_hw_tcd *tcd, dma_addr_t src, dma_addr_t dst, + u16 attr, u16 soff, u32 nbytes, dma_addr_t slast, u16 citer, + u16 biter, u16 doff, dma_addr_t dlast_sga, bool major_int, bool disable_req, bool enable_sg) { + struct dma_slave_config *cfg = &fsl_chan->cfg; + u32 burst = 0; u16 csr = 0; /* @@ -390,22 +493,52 @@ void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, * So we put the value in little endian in memory, waiting * for fsl_edma_set_tcd_regs doing the swap. */ - tcd->saddr = cpu_to_le32(src); - tcd->daddr = cpu_to_le32(dst); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, src, saddr); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, dst, daddr); + + fsl_edma_set_tcd_to_le(fsl_chan, tcd, attr, attr); - tcd->attr = cpu_to_le16(attr); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, soff, soff); + + /* If we expect to have either multi_fifo or a port window size, + * we will use minor loop offset, meaning bits 29-10 will be used for + * address offset, while bits 9-0 will be used to tell DMA how much + * data to read from addr. + * If we don't have either of those, will use a major loop reading from addr + * nbytes (29bits). + */ + if (cfg->direction == DMA_MEM_TO_DEV) { + if (fsl_chan->is_multi_fifo) + burst = cfg->dst_maxburst * 4; + if (cfg->dst_port_window_size) + burst = cfg->dst_port_window_size * cfg->dst_addr_width; + if (burst) { + nbytes |= EDMA_V3_TCD_NBYTES_MLOFF(-burst); + nbytes |= EDMA_V3_TCD_NBYTES_DMLOE; + nbytes &= ~EDMA_V3_TCD_NBYTES_SMLOE; + } + } else { + if (fsl_chan->is_multi_fifo) + burst = cfg->src_maxburst * 4; + if (cfg->src_port_window_size) + burst = cfg->src_port_window_size * cfg->src_addr_width; + if (burst) { + nbytes |= EDMA_V3_TCD_NBYTES_MLOFF(-burst); + nbytes |= EDMA_V3_TCD_NBYTES_SMLOE; + nbytes &= ~EDMA_V3_TCD_NBYTES_DMLOE; + } + } - tcd->soff = cpu_to_le16(soff); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, nbytes, nbytes); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, slast, slast); - tcd->nbytes = cpu_to_le32(nbytes); - tcd->slast = cpu_to_le32(slast); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, EDMA_TCD_CITER_CITER(citer), citer); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, doff, doff); - tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer)); - tcd->doff = cpu_to_le16(doff); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, dlast_sga, dlast_sga); - tcd->dlast_sga = cpu_to_le32(dlast_sga); + fsl_edma_set_tcd_to_le(fsl_chan, tcd, EDMA_TCD_BITER_BITER(biter), biter); - tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter)); if (major_int) csr |= EDMA_TCD_CSR_INT_MAJOR; @@ -415,7 +548,15 @@ void fsl_edma_fill_tcd(struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, if (enable_sg) csr |= EDMA_TCD_CSR_E_SG; - tcd->csr = cpu_to_le16(csr); + if (fsl_chan->is_rxchan) + csr |= EDMA_TCD_CSR_ACTIVE; + + if (fsl_chan->is_sw) + csr |= EDMA_TCD_CSR_START; + + fsl_edma_set_tcd_to_le(fsl_chan, tcd, csr, csr); + + trace_edma_fill_tcd(fsl_chan, tcd); } static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, @@ -454,9 +595,11 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); struct fsl_edma_desc *fsl_desc; dma_addr_t dma_buf_next; + bool major_int = true; int sg_len, i; - u32 src_addr, dst_addr, last_sg, nbytes; + dma_addr_t src_addr, dst_addr, last_sg; u16 soff, doff, iter; + u32 nbytes; if (!is_slave_direction(direction)) return NULL; @@ -473,13 +616,19 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( dma_buf_next = dma_addr; if (direction == DMA_MEM_TO_DEV) { + if (!fsl_chan->cfg.src_addr_width) + fsl_chan->cfg.src_addr_width = fsl_chan->cfg.dst_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.dst_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.dst_addr_width * fsl_chan->cfg.dst_maxburst; } else { + if (!fsl_chan->cfg.dst_addr_width) + fsl_chan->cfg.dst_addr_width = fsl_chan->cfg.src_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.src_addr_width * fsl_chan->cfg.src_maxburst; } @@ -497,23 +646,32 @@ struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( src_addr = dma_buf_next; dst_addr = fsl_chan->dma_dev_addr; soff = fsl_chan->cfg.dst_addr_width; - doff = 0; - } else { + doff = fsl_chan->is_multi_fifo ? 4 : 0; + if (fsl_chan->cfg.dst_port_window_size) + doff = fsl_chan->cfg.dst_addr_width; + } else if (direction == DMA_DEV_TO_MEM) { src_addr = fsl_chan->dma_dev_addr; dst_addr = dma_buf_next; - soff = 0; + soff = fsl_chan->is_multi_fifo ? 4 : 0; doff = fsl_chan->cfg.src_addr_width; + if (fsl_chan->cfg.src_port_window_size) + soff = fsl_chan->cfg.src_addr_width; + } else { + /* DMA_DEV_TO_DEV */ + src_addr = fsl_chan->cfg.src_addr; + dst_addr = fsl_chan->cfg.dst_addr; + soff = doff = 0; + major_int = false; } - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr, + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, dst_addr, fsl_chan->attr, soff, nbytes, 0, iter, - iter, doff, last_sg, true, false, true); + iter, doff, last_sg, major_int, false, true); dma_buf_next += period_len; } return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); } -EXPORT_SYMBOL_GPL(fsl_edma_prep_dma_cyclic); struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, @@ -523,8 +681,9 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); struct fsl_edma_desc *fsl_desc; struct scatterlist *sg; - u32 src_addr, dst_addr, last_sg, nbytes; + dma_addr_t src_addr, dst_addr, last_sg; u16 soff, doff, iter; + u32 nbytes; int i; if (!is_slave_direction(direction)) @@ -540,43 +699,74 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( fsl_desc->dirn = direction; if (direction == DMA_MEM_TO_DEV) { + if (!fsl_chan->cfg.src_addr_width) + fsl_chan->cfg.src_addr_width = fsl_chan->cfg.dst_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.dst_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.dst_addr_width * fsl_chan->cfg.dst_maxburst; } else { + if (!fsl_chan->cfg.dst_addr_width) + fsl_chan->cfg.dst_addr_width = fsl_chan->cfg.src_addr_width; fsl_chan->attr = - fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width); + fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width, + fsl_chan->cfg.dst_addr_width); nbytes = fsl_chan->cfg.src_addr_width * fsl_chan->cfg.src_maxburst; } for_each_sg(sgl, sg, sg_len, i) { - /* get next sg's physical address */ - last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; - if (direction == DMA_MEM_TO_DEV) { src_addr = sg_dma_address(sg); dst_addr = fsl_chan->dma_dev_addr; soff = fsl_chan->cfg.dst_addr_width; doff = 0; - } else { + } else if (direction == DMA_DEV_TO_MEM) { src_addr = fsl_chan->dma_dev_addr; dst_addr = sg_dma_address(sg); soff = 0; doff = fsl_chan->cfg.src_addr_width; + } else { + /* DMA_DEV_TO_DEV */ + src_addr = fsl_chan->cfg.src_addr; + dst_addr = fsl_chan->cfg.dst_addr; + soff = 0; + doff = 0; } + /* + * Choose the suitable burst length if sg_dma_len is not + * multiple of burst length so that the whole transfer length is + * multiple of minor loop(burst length). + */ + if (sg_dma_len(sg) % nbytes) { + u32 width = (direction == DMA_DEV_TO_MEM) ? doff : soff; + u32 burst = (direction == DMA_DEV_TO_MEM) ? + fsl_chan->cfg.src_maxburst : + fsl_chan->cfg.dst_maxburst; + int j; + + for (j = burst; j > 1; j--) { + if (!(sg_dma_len(sg) % (j * width))) { + nbytes = j * width; + break; + } + } + /* Set burst size as 1 if there's no suitable one */ + if (j == 1) + nbytes = width; + } iter = sg_dma_len(sg) / nbytes; if (i < sg_len - 1) { last_sg = fsl_desc->tcd[(i + 1)].ptcd; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, dst_addr, fsl_chan->attr, soff, nbytes, 0, iter, iter, doff, last_sg, false, false, true); } else { last_sg = 0; - fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[i].vtcd, src_addr, dst_addr, fsl_chan->attr, soff, nbytes, 0, iter, iter, doff, last_sg, true, true, false); @@ -585,7 +775,35 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); } -EXPORT_SYMBOL_GPL(fsl_edma_prep_slave_sg); + +struct dma_async_tx_descriptor *fsl_edma_prep_memcpy(struct dma_chan *chan, + dma_addr_t dma_dst, dma_addr_t dma_src, + size_t len, unsigned long flags) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_desc *fsl_desc; + u32 src_bus_width, dst_bus_width; + + src_bus_width = min_t(u32, DMA_SLAVE_BUSWIDTH_32_BYTES, 1 << (ffs(dma_src) - 1)); + dst_bus_width = min_t(u32, DMA_SLAVE_BUSWIDTH_32_BYTES, 1 << (ffs(dma_dst) - 1)); + + fsl_desc = fsl_edma_alloc_desc(fsl_chan, 1); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = false; + + fsl_chan->is_sw = true; + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_MEM_REMOTE) + fsl_chan->is_remote = true; + + /* To match with copy_align and max_seg_size so 1 tcd is enough */ + fsl_edma_fill_tcd(fsl_chan, fsl_desc->tcd[0].vtcd, dma_src, dma_dst, + fsl_edma_get_tcd_attr(src_bus_width, dst_bus_width), + src_bus_width, len, 0, 1, 1, dst_bus_width, 0, true, + true, false); + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) { @@ -600,9 +818,7 @@ void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); fsl_edma_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; - fsl_chan->idle = false; } -EXPORT_SYMBOL_GPL(fsl_edma_xfer_desc); void fsl_edma_issue_pending(struct dma_chan *chan) { @@ -622,38 +838,75 @@ void fsl_edma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); } -EXPORT_SYMBOL_GPL(fsl_edma_issue_pending); int fsl_edma_alloc_chan_resources(struct dma_chan *chan) { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + int ret = 0; + + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) + clk_prepare_enable(fsl_chan->clk); fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, - sizeof(struct fsl_edma_hw_tcd), + fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_TCD64 ? + sizeof(struct fsl_edma_hw_tcd64) : sizeof(struct fsl_edma_hw_tcd), 32, 0); + + if (fsl_chan->txirq) + ret = request_irq(fsl_chan->txirq, fsl_chan->irq_handler, IRQF_SHARED, + fsl_chan->chan_name, fsl_chan); + + if (ret) + goto err_txirq; + + if (fsl_chan->errirq > 0) + ret = request_irq(fsl_chan->errirq, fsl_chan->errirq_handler, IRQF_SHARED, + fsl_chan->errirq_name, fsl_chan); + + if (ret) + goto err_errirq; + return 0; + +err_errirq: + if (fsl_chan->txirq) + free_irq(fsl_chan->txirq, fsl_chan); +err_txirq: + dma_pool_destroy(fsl_chan->tcd_pool); + + return ret; } -EXPORT_SYMBOL_GPL(fsl_edma_alloc_chan_resources); void fsl_edma_free_chan_resources(struct dma_chan *chan) { struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_engine *edma = fsl_chan->edma; unsigned long flags; LIST_HEAD(head); spin_lock_irqsave(&fsl_chan->vchan.lock, flags); fsl_edma_disable_request(fsl_chan); - fsl_edma_chan_mux(fsl_chan, 0, false); + if (edma->drvdata->dmamuxs) + fsl_edma_chan_mux(fsl_chan, 0, false); fsl_chan->edesc = NULL; vchan_get_all_descriptors(&fsl_chan->vchan, &head); fsl_edma_unprep_slave_dma(fsl_chan); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + if (fsl_chan->txirq) + free_irq(fsl_chan->txirq, fsl_chan); + if (fsl_chan->errirq) + free_irq(fsl_chan->errirq, fsl_chan); + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); dma_pool_destroy(fsl_chan->tcd_pool); fsl_chan->tcd_pool = NULL; + fsl_chan->is_sw = false; + fsl_chan->srcid = 0; + fsl_chan->is_remote = false; + if (fsl_edma_drvflags(fsl_chan) & FSL_EDMA_DRV_HAS_CHCLK) + clk_disable_unprepare(fsl_chan->clk); } -EXPORT_SYMBOL_GPL(fsl_edma_free_chan_resources); void fsl_edma_cleanup_vchan(struct dma_device *dmadev) { @@ -665,12 +918,10 @@ void fsl_edma_cleanup_vchan(struct dma_device *dmadev) tasklet_kill(&chan->vchan.task); } } -EXPORT_SYMBOL_GPL(fsl_edma_cleanup_vchan); /* - * On the 32 channels Vybrid/mpc577x edma version (here called "v1"), - * register offsets are different compared to ColdFire mcf5441x 64 channels - * edma (here called "v2"). + * On the 32 channels Vybrid/mpc577x edma version, register offsets are + * different compared to ColdFire mcf5441x 64 channels edma. * * This function sets up register offsets as per proper declared version * so must be called in xxx_edma_probe() just after setting the @@ -678,41 +929,30 @@ EXPORT_SYMBOL_GPL(fsl_edma_cleanup_vchan); */ void fsl_edma_setup_regs(struct fsl_edma_engine *edma) { + bool is64 = !!(edma->drvdata->flags & FSL_EDMA_DRV_EDMA64); + edma->regs.cr = edma->membase + EDMA_CR; edma->regs.es = edma->membase + EDMA_ES; edma->regs.erql = edma->membase + EDMA_ERQ; edma->regs.eeil = edma->membase + EDMA_EEI; - edma->regs.serq = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_SERQ : EDMA_SERQ); - edma->regs.cerq = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CERQ : EDMA_CERQ); - edma->regs.seei = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_SEEI : EDMA_SEEI); - edma->regs.ceei = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CEEI : EDMA_CEEI); - edma->regs.cint = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CINT : EDMA_CINT); - edma->regs.cerr = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CERR : EDMA_CERR); - edma->regs.ssrt = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_SSRT : EDMA_SSRT); - edma->regs.cdne = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_CDNE : EDMA_CDNE); - edma->regs.intl = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_INTL : EDMA_INTR); - edma->regs.errl = edma->membase + ((edma->drvdata->version == v2) ? - EDMA64_ERRL : EDMA_ERR); - - if (edma->drvdata->version == v2) { + edma->regs.serq = edma->membase + (is64 ? EDMA64_SERQ : EDMA_SERQ); + edma->regs.cerq = edma->membase + (is64 ? EDMA64_CERQ : EDMA_CERQ); + edma->regs.seei = edma->membase + (is64 ? EDMA64_SEEI : EDMA_SEEI); + edma->regs.ceei = edma->membase + (is64 ? EDMA64_CEEI : EDMA_CEEI); + edma->regs.cint = edma->membase + (is64 ? EDMA64_CINT : EDMA_CINT); + edma->regs.cerr = edma->membase + (is64 ? EDMA64_CERR : EDMA_CERR); + edma->regs.ssrt = edma->membase + (is64 ? EDMA64_SSRT : EDMA_SSRT); + edma->regs.cdne = edma->membase + (is64 ? EDMA64_CDNE : EDMA_CDNE); + edma->regs.intl = edma->membase + (is64 ? EDMA64_INTL : EDMA_INTR); + edma->regs.errl = edma->membase + (is64 ? EDMA64_ERRL : EDMA_ERR); + + if (is64) { edma->regs.erqh = edma->membase + EDMA64_ERQH; edma->regs.eeih = edma->membase + EDMA64_EEIH; edma->regs.errh = edma->membase + EDMA64_ERRH; edma->regs.inth = edma->membase + EDMA64_INTH; } - - edma->regs.tcd = edma->membase + EDMA_TCD; } -EXPORT_SYMBOL_GPL(fsl_edma_setup_regs); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-edma-common.h b/drivers/dma/fsl-edma-common.h index ec1169741de1..205a96489094 100644 --- a/drivers/dma/fsl-edma-common.h +++ b/drivers/dma/fsl-edma-common.h @@ -29,19 +29,10 @@ #define EDMA_TCD_ATTR_DMOD(x) (((x) & GENMASK(4, 0)) << 3) #define EDMA_TCD_ATTR_SSIZE(x) (((x) & GENMASK(2, 0)) << 8) #define EDMA_TCD_ATTR_SMOD(x) (((x) & GENMASK(4, 0)) << 11) -#define EDMA_TCD_ATTR_DSIZE_8BIT 0 -#define EDMA_TCD_ATTR_DSIZE_16BIT BIT(0) -#define EDMA_TCD_ATTR_DSIZE_32BIT BIT(1) -#define EDMA_TCD_ATTR_DSIZE_64BIT (BIT(0) | BIT(1)) -#define EDMA_TCD_ATTR_DSIZE_32BYTE (BIT(2) | BIT(0)) -#define EDMA_TCD_ATTR_SSIZE_8BIT 0 -#define EDMA_TCD_ATTR_SSIZE_16BIT (EDMA_TCD_ATTR_DSIZE_16BIT << 8) -#define EDMA_TCD_ATTR_SSIZE_32BIT (EDMA_TCD_ATTR_DSIZE_32BIT << 8) -#define EDMA_TCD_ATTR_SSIZE_64BIT (EDMA_TCD_ATTR_DSIZE_64BIT << 8) -#define EDMA_TCD_ATTR_SSIZE_32BYTE (EDMA_TCD_ATTR_DSIZE_32BYTE << 8) - -#define EDMA_TCD_CITER_CITER(x) ((x) & GENMASK(14, 0)) -#define EDMA_TCD_BITER_BITER(x) ((x) & GENMASK(14, 0)) + +#define EDMA_TCD_ITER_MASK GENMASK(14, 0) +#define EDMA_TCD_CITER_CITER(x) ((x) & EDMA_TCD_ITER_MASK) +#define EDMA_TCD_BITER_BITER(x) ((x) & EDMA_TCD_ITER_MASK) #define EDMA_TCD_CSR_START BIT(0) #define EDMA_TCD_CSR_INT_MAJOR BIT(1) @@ -52,16 +43,46 @@ #define EDMA_TCD_CSR_ACTIVE BIT(6) #define EDMA_TCD_CSR_DONE BIT(7) +#define EDMA_V3_TCD_NBYTES_MLOFF_NBYTES(x) ((x) & GENMASK(9, 0)) +#define EDMA_V3_TCD_NBYTES_MLOFF(x) (x << 10) +#define EDMA_V3_TCD_NBYTES_DMLOE (1 << 30) +#define EDMA_V3_TCD_NBYTES_SMLOE (1 << 31) + #define EDMAMUX_CHCFG_DIS 0x0 #define EDMAMUX_CHCFG_ENBL 0x80 #define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) #define DMAMUX_NR 2 +#define EDMA_TCD 0x1000 + #define FSL_EDMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +#define EDMA_V3_CH_SBR_RD BIT(22) +#define EDMA_V3_CH_SBR_WR BIT(21) +#define EDMA_V3_CH_CSR_ERQ BIT(0) +#define EDMA_V3_CH_CSR_EARQ BIT(1) +#define EDMA_V3_CH_CSR_EEI BIT(2) +#define EDMA_V3_CH_CSR_DONE BIT(30) +#define EDMA_V3_CH_CSR_ACTIVE BIT(31) +#define EDMA_V3_CH_ES_ERR BIT(31) +#define EDMA_V3_MP_ES_VLD BIT(31) + +#define EDMA_V3_CH_ERR_DBE BIT(0) +#define EDMA_V3_CH_ERR_SBE BIT(1) +#define EDMA_V3_CH_ERR_SGE BIT(2) +#define EDMA_V3_CH_ERR_NCE BIT(3) +#define EDMA_V3_CH_ERR_DOE BIT(4) +#define EDMA_V3_CH_ERR_DAE BIT(5) +#define EDMA_V3_CH_ERR_SOE BIT(6) +#define EDMA_V3_CH_ERR_SAE BIT(7) +#define EDMA_V3_CH_ERR_ECX BIT(8) +#define EDMA_V3_CH_ERR_UCE BIT(9) +#define EDMA_V3_CH_ERR BIT(31) + enum fsl_edma_pm_state { RUNNING = 0, SUSPENDED, @@ -81,6 +102,35 @@ struct fsl_edma_hw_tcd { __le16 biter; }; +struct fsl_edma_hw_tcd64 { + __le64 saddr; + __le16 soff; + __le16 attr; + __le32 nbytes; + __le64 slast; + __le64 daddr; + __le64 dlast_sga; + __le16 doff; + __le16 citer; + __le16 csr; + __le16 biter; +} __packed; + +struct fsl_edma3_ch_reg { + __le32 ch_csr; + __le32 ch_es; + __le32 ch_int; + __le32 ch_sbr; + __le32 ch_pri; + __le32 ch_mux; + __le32 ch_mattr; /* edma4, reserved for edma3 */ + __le32 ch_reserved; + union { + struct fsl_edma_hw_tcd tcd; + struct fsl_edma_hw_tcd64 tcd64; + }; +} __packed; + /* * These are iomem pointers, for both v32 and v64. */ @@ -103,29 +153,46 @@ struct edma_regs { void __iomem *intl; void __iomem *errh; void __iomem *errl; - struct fsl_edma_hw_tcd __iomem *tcd; }; struct fsl_edma_sw_tcd { dma_addr_t ptcd; - struct fsl_edma_hw_tcd *vtcd; + void *vtcd; }; struct fsl_edma_chan { struct virt_dma_chan vchan; enum dma_status status; enum fsl_edma_pm_state pm_state; - bool idle; - u32 slave_id; struct fsl_edma_engine *edma; struct fsl_edma_desc *edesc; struct dma_slave_config cfg; u32 attr; + bool is_sw; struct dma_pool *tcd_pool; dma_addr_t dma_dev_addr; u32 dma_dev_size; enum dma_data_direction dma_dir; - char chan_name[16]; + char chan_name[32]; + char errirq_name[36]; + void __iomem *tcd; + void __iomem *mux_addr; + u32 real_count; + struct work_struct issue_worker; + struct platform_device *pdev; + struct device *pd_dev; + struct device_link *pd_dev_link; + u32 srcid; + struct clk *clk; + int priority; + int hw_chanid; + int txirq; + int errirq; + irqreturn_t (*irq_handler)(int irq, void *dev_id); + irqreturn_t (*errirq_handler)(int irq, void *dev_id); + bool is_rxchan; + bool is_remote; + bool is_multi_fifo; }; struct fsl_edma_desc { @@ -137,17 +204,49 @@ struct fsl_edma_desc { struct fsl_edma_sw_tcd tcd[]; }; -enum edma_version { - v1, /* 32ch, Vybrid, mpc57x, etc */ - v2, /* 64ch Coldfire */ - v3, /* 32ch, i.mx7ulp */ -}; +#define FSL_EDMA_DRV_HAS_DMACLK BIT(0) +#define FSL_EDMA_DRV_MUX_SWAP BIT(1) +#define FSL_EDMA_DRV_CONFIG32 BIT(2) +#define FSL_EDMA_DRV_WRAP_IO BIT(3) +#define FSL_EDMA_DRV_EDMA64 BIT(4) +#define FSL_EDMA_DRV_HAS_PD BIT(5) +#define FSL_EDMA_DRV_HAS_CHCLK BIT(6) +#define FSL_EDMA_DRV_HAS_CHMUX BIT(7) +#define FSL_EDMA_DRV_MEM_REMOTE BIT(8) +/* control and status register is in tcd address space, edma3 reg layout */ +#define FSL_EDMA_DRV_SPLIT_REG BIT(9) +#define FSL_EDMA_DRV_BUS_8BYTE BIT(10) +#define FSL_EDMA_DRV_DEV_TO_DEV BIT(11) +#define FSL_EDMA_DRV_ALIGN_64BYTE BIT(12) +/* Need clean CHn_CSR DONE before enable TCD's ESG */ +#define FSL_EDMA_DRV_CLEAR_DONE_E_SG BIT(13) +/* Need clean CHn_CSR DONE before enable TCD's MAJORELINK */ +#define FSL_EDMA_DRV_CLEAR_DONE_E_LINK BIT(14) +#define FSL_EDMA_DRV_TCD64 BIT(15) +/* All channel ERR IRQ share one IRQ line */ +#define FSL_EDMA_DRV_ERRIRQ_SHARE BIT(16) + + +#define FSL_EDMA_DRV_EDMA3 (FSL_EDMA_DRV_SPLIT_REG | \ + FSL_EDMA_DRV_BUS_8BYTE | \ + FSL_EDMA_DRV_DEV_TO_DEV | \ + FSL_EDMA_DRV_ALIGN_64BYTE | \ + FSL_EDMA_DRV_CLEAR_DONE_E_SG | \ + FSL_EDMA_DRV_CLEAR_DONE_E_LINK) + +#define FSL_EDMA_DRV_EDMA4 (FSL_EDMA_DRV_SPLIT_REG | \ + FSL_EDMA_DRV_BUS_8BYTE | \ + FSL_EDMA_DRV_DEV_TO_DEV | \ + FSL_EDMA_DRV_ALIGN_64BYTE | \ + FSL_EDMA_DRV_CLEAR_DONE_E_LINK) struct fsl_edma_drvdata { - enum edma_version version; - u32 dmamuxs; - bool has_dmaclk; - bool mux_swap; + u32 dmamuxs; /* only used before v3 */ + u32 chreg_off; + u32 chreg_space_sz; + u32 flags; + u32 mux_off; /* channel mux register offset */ + u32 mux_skip; /* how much skip for each channel */ int (*setup_irq)(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma); }; @@ -162,24 +261,158 @@ struct fsl_edma_engine { const struct fsl_edma_drvdata *drvdata; u32 n_chans; int txirq; + int txirq_16_31; int errirq; bool big_endian; struct edma_regs regs; - struct fsl_edma_chan chans[]; + u64 chan_masked; + struct fsl_edma_chan chans[] __counted_by(n_chans); }; +static inline u32 fsl_edma_drvflags(struct fsl_edma_chan *fsl_chan) +{ + return fsl_chan->edma->drvdata->flags; +} + +#define edma_read_tcdreg_c(chan, _tcd, __name) \ +_Generic(((_tcd)->__name), \ + __iomem __le64 : edma_readq(chan->edma, &(_tcd)->__name), \ + __iomem __le32 : edma_readl(chan->edma, &(_tcd)->__name), \ + __iomem __le16 : edma_readw(chan->edma, &(_tcd)->__name) \ + ) + +#define edma_read_tcdreg(chan, __name) \ +((fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) ? \ + edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd64 __iomem *)chan->tcd), __name) : \ + edma_read_tcdreg_c(chan, ((struct fsl_edma_hw_tcd __iomem *)chan->tcd), __name) \ +) + +#define edma_write_tcdreg_c(chan, _tcd, _val, __name) \ +_Generic((_tcd->__name), \ + __iomem __le64 : edma_writeq(chan->edma, (u64 __force)(_val), &_tcd->__name), \ + __iomem __le32 : edma_writel(chan->edma, (u32 __force)(_val), &_tcd->__name), \ + __iomem __le16 : edma_writew(chan->edma, (u16 __force)(_val), &_tcd->__name), \ + __iomem u8 : edma_writeb(chan->edma, _val, &_tcd->__name) \ + ) + +#define edma_write_tcdreg(chan, val, __name) \ +do { \ + struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \ + struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd; \ + \ + if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) \ + edma_write_tcdreg_c(chan, tcd64_r, val, __name); \ + else \ + edma_write_tcdreg_c(chan, tcd_r, val, __name); \ +} while (0) + +#define edma_cp_tcd_to_reg(chan, __tcd, __name) \ +do { \ + struct fsl_edma_hw_tcd64 __iomem *tcd64_r = (struct fsl_edma_hw_tcd64 __iomem *)chan->tcd; \ + struct fsl_edma_hw_tcd __iomem *tcd_r = (struct fsl_edma_hw_tcd __iomem *)chan->tcd; \ + struct fsl_edma_hw_tcd64 *tcd64_m = (struct fsl_edma_hw_tcd64 *)__tcd; \ + struct fsl_edma_hw_tcd *tcd_m = (struct fsl_edma_hw_tcd *)__tcd; \ + \ + if (fsl_edma_drvflags(chan) & FSL_EDMA_DRV_TCD64) \ + edma_write_tcdreg_c(chan, tcd64_r, tcd64_m->__name, __name); \ + else \ + edma_write_tcdreg_c(chan, tcd_r, tcd_m->__name, __name); \ +} while (0) + +#define edma_readl_chreg(chan, __name) \ + edma_readl(chan->edma, \ + (void __iomem *)&(container_of(((__force void *)chan->tcd),\ + struct fsl_edma3_ch_reg, tcd)->__name)) + +#define edma_writel_chreg(chan, val, __name) \ + edma_writel(chan->edma, val, \ + (void __iomem *)&(container_of(((__force void *)chan->tcd),\ + struct fsl_edma3_ch_reg, tcd)->__name)) + +#define fsl_edma_get_tcd(_chan, _tcd, _field) \ +(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? (((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \ + (((struct fsl_edma_hw_tcd *)_tcd)->_field)) + +#define fsl_edma_le_to_cpu(x) \ +_Generic((x), \ + __le64 : le64_to_cpu((x)), \ + __le32 : le32_to_cpu((x)), \ + __le16 : le16_to_cpu((x)) \ +) + +#define fsl_edma_get_tcd_to_cpu(_chan, _tcd, _field) \ +(fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64 ? \ + fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd64 *)_tcd)->_field) : \ + fsl_edma_le_to_cpu(((struct fsl_edma_hw_tcd *)_tcd)->_field)) + +#define fsl_edma_set_tcd_to_le_c(_tcd, _val, _field) \ +_Generic(((_tcd)->_field), \ + __le64 : (_tcd)->_field = cpu_to_le64(_val), \ + __le32 : (_tcd)->_field = cpu_to_le32(_val), \ + __le16 : (_tcd)->_field = cpu_to_le16(_val) \ +) + +#define fsl_edma_set_tcd_to_le(_chan, _tcd, _val, _field) \ +do { \ + if (fsl_edma_drvflags(_chan) & FSL_EDMA_DRV_TCD64) \ + fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd64 *)_tcd, _val, _field); \ + else \ + fsl_edma_set_tcd_to_le_c((struct fsl_edma_hw_tcd *)_tcd, _val, _field); \ +} while (0) + +/* Need after struct defination */ +#include "fsl-edma-trace.h" + /* * R/W functions for big- or little-endian registers: * The eDMA controller's endian is independent of the CPU core's endian. * For the big-endian IP module, the offset for 8-bit or 16-bit registers * should also be swapped opposite to that in little-endian IP. */ +static inline u64 edma_readq(struct fsl_edma_engine *edma, void __iomem *addr) +{ + u64 l, h; + + if (edma->big_endian) { + l = ioread32be(addr); + h = ioread32be(addr + 4); + } else { + l = ioread32(addr); + h = ioread32(addr + 4); + } + + trace_edma_readl(edma, addr, l); + trace_edma_readl(edma, addr + 4, h); + + return (h << 32) | l; +} + static inline u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) { + u32 val; + if (edma->big_endian) - return ioread32be(addr); + val = ioread32be(addr); else - return ioread32(addr); + val = ioread32(addr); + + trace_edma_readl(edma, addr, val); + + return val; +} + +static inline u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr) +{ + u16 val; + + if (edma->big_endian) + val = ioread16be(addr); + else + val = ioread16(addr); + + trace_edma_readw(edma, addr, val); + + return val; } static inline void edma_writeb(struct fsl_edma_engine *edma, @@ -190,6 +423,8 @@ static inline void edma_writeb(struct fsl_edma_engine *edma, iowrite8(val, (void __iomem *)((unsigned long)addr ^ 0x3)); else iowrite8(val, addr); + + trace_edma_writeb(edma, addr, val); } static inline void edma_writew(struct fsl_edma_engine *edma, @@ -200,6 +435,8 @@ static inline void edma_writew(struct fsl_edma_engine *edma, iowrite16be(val, (void __iomem *)((unsigned long)addr ^ 0x2)); else iowrite16(val, addr); + + trace_edma_writew(edma, addr, val); } static inline void edma_writel(struct fsl_edma_engine *edma, @@ -209,6 +446,23 @@ static inline void edma_writel(struct fsl_edma_engine *edma, iowrite32be(val, addr); else iowrite32(val, addr); + + trace_edma_writel(edma, addr, val); +} + +static inline void edma_writeq(struct fsl_edma_engine *edma, + u64 val, void __iomem *addr) +{ + if (edma->big_endian) { + iowrite32be(val & 0xFFFFFFFF, addr); + iowrite32be(val >> 32, addr + 4); + } else { + iowrite32(val & 0xFFFFFFFF, addr); + iowrite32(val >> 32, addr + 4); + } + + trace_edma_writel(edma, addr, val & 0xFFFFFFFF); + trace_edma_writel(edma, addr + 4, val >> 32); } static inline struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) @@ -221,6 +475,12 @@ static inline struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) return container_of(vd, struct fsl_edma_desc, vdesc); } +static inline void fsl_edma_err_chan_handler(struct fsl_edma_chan *fsl_chan) +{ + fsl_chan->status = DMA_ERROR; +} + +void fsl_edma_tx_chan_handler(struct fsl_edma_chan *fsl_chan); void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan); void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, unsigned int slot, bool enable); @@ -240,6 +500,9 @@ struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context); +struct dma_async_tx_descriptor *fsl_edma_prep_memcpy( + struct dma_chan *chan, dma_addr_t dma_dst, dma_addr_t dma_src, + size_t len, unsigned long flags); void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan); void fsl_edma_issue_pending(struct dma_chan *chan); int fsl_edma_alloc_chan_resources(struct dma_chan *chan); diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c new file mode 100644 index 000000000000..a753b7cbfa7a --- /dev/null +++ b/drivers/dma/fsl-edma-main.c @@ -0,0 +1,1003 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * drivers/dma/fsl-edma.c + * + * Copyright 2013-2014 Freescale Semiconductor, Inc. + * Copyright 2024 NXP + * + * Driver for the Freescale eDMA engine with flexible channel multiplexing + * capability for DMA request sources. The eDMA block can be found on some + * Vybrid, Layerscape and S32G SoCs. + */ + +#include <dt-bindings/dma/fsl-edma.h> +#include <linux/bitfield.h> +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/clk.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/dma-mapping.h> +#include <linux/pm_runtime.h> +#include <linux/pm_domain.h> +#include <linux/property.h> + +#include "fsl-edma-common.h" + +static void fsl_edma_synchronize(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + + vchan_synchronize(&fsl_chan->vchan); +} + +static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int intr, ch; + struct edma_regs *regs = &fsl_edma->regs; + + intr = edma_readl(fsl_edma, regs->intl); + if (!intr) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (intr & (0x1 << ch)) { + edma_writeb(fsl_edma, EDMA_CINT_CINT(ch), regs->cint); + fsl_edma_tx_chan_handler(&fsl_edma->chans[ch]); + } + } + return IRQ_HANDLED; +} + +static void fsl_edma3_err_check(struct fsl_edma_chan *fsl_chan) +{ + unsigned int ch_err; + u32 val; + + scoped_guard(spinlock, &fsl_chan->vchan.lock) { + ch_err = edma_readl_chreg(fsl_chan, ch_es); + if (!(ch_err & EDMA_V3_CH_ERR)) + return; + + edma_writel_chreg(fsl_chan, EDMA_V3_CH_ERR, ch_es); + val = edma_readl_chreg(fsl_chan, ch_csr); + val &= ~EDMA_V3_CH_CSR_ERQ; + edma_writel_chreg(fsl_chan, val, ch_csr); + } + + /* Ignore this interrupt since channel has been disabled already */ + if (!fsl_chan->edesc) + return; + + if (ch_err & EDMA_V3_CH_ERR_DBE) + dev_err(&fsl_chan->pdev->dev, "Destination Bus Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SBE) + dev_err(&fsl_chan->pdev->dev, "Source Bus Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SGE) + dev_err(&fsl_chan->pdev->dev, "Scatter/Gather Configuration Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_NCE) + dev_err(&fsl_chan->pdev->dev, "NBYTES/CITER Configuration Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_DOE) + dev_err(&fsl_chan->pdev->dev, "Destination Offset Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_DAE) + dev_err(&fsl_chan->pdev->dev, "Destination Address Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SOE) + dev_err(&fsl_chan->pdev->dev, "Source Offset Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_SAE) + dev_err(&fsl_chan->pdev->dev, "Source Address Error interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_ECX) + dev_err(&fsl_chan->pdev->dev, "Transfer Canceled interrupt.\n"); + + if (ch_err & EDMA_V3_CH_ERR_UCE) + dev_err(&fsl_chan->pdev->dev, "Uncorrectable TCD error during channel execution interrupt.\n"); + + fsl_chan->status = DMA_ERROR; +} + +static irqreturn_t fsl_edma3_err_handler_per_chan(int irq, void *dev_id) +{ + struct fsl_edma_chan *fsl_chan = dev_id; + + fsl_edma3_err_check(fsl_chan); + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma3_err_handler_shared(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int ch; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (fsl_edma->chan_masked & BIT(ch)) + continue; + + fsl_edma3_err_check(&fsl_edma->chans[ch]); + } + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma3_tx_handler(int irq, void *dev_id) +{ + struct fsl_edma_chan *fsl_chan = dev_id; + unsigned int intr; + + intr = edma_readl_chreg(fsl_chan, ch_int); + if (!intr) + return IRQ_NONE; + + edma_writel_chreg(fsl_chan, 1, ch_int); + + fsl_edma_tx_chan_handler(fsl_chan); + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma2_tx_handler(int irq, void *devi_id) +{ + struct fsl_edma_chan *fsl_chan = devi_id; + + return fsl_edma_tx_handler(irq, fsl_chan->edma); +} + +static irqreturn_t fsl_edma3_or_tx_handler(int irq, void *dev_id, + u8 start, u8 end) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + struct fsl_edma_chan *chan; + int i; + + end = min(end, fsl_edma->n_chans); + + for (i = start; i < end; i++) { + chan = &fsl_edma->chans[i]; + + fsl_edma3_tx_handler(irq, chan); + } + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma3_tx_0_15_handler(int irq, void *dev_id) +{ + return fsl_edma3_or_tx_handler(irq, dev_id, 0, 16); +} + +static irqreturn_t fsl_edma3_tx_16_31_handler(int irq, void *dev_id) +{ + return fsl_edma3_or_tx_handler(irq, dev_id, 16, 32); +} + +static irqreturn_t fsl_edma3_or_err_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + struct edma_regs *regs = &fsl_edma->regs; + unsigned int err, ch, ch_es; + struct fsl_edma_chan *chan; + + err = edma_readl(fsl_edma, regs->es); + if (!(err & EDMA_V3_MP_ES_VLD)) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + chan = &fsl_edma->chans[ch]; + + ch_es = edma_readl_chreg(chan, ch_es); + if (!(ch_es & EDMA_V3_CH_ES_ERR)) + continue; + + edma_writel_chreg(chan, EDMA_V3_CH_ES_ERR, ch_es); + fsl_edma_disable_request(chan); + fsl_edma->chans[ch].status = DMA_ERROR; + } + + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int err, ch; + struct edma_regs *regs = &fsl_edma->regs; + + err = edma_readl(fsl_edma, regs->errl); + if (!err) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (err & (0x1 << ch)) { + fsl_edma_disable_request(&fsl_edma->chans[ch]); + edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), regs->cerr); + fsl_edma_err_chan_handler(&fsl_edma->chans[ch]); + } + } + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) +{ + if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED) + return IRQ_HANDLED; + + return fsl_edma_err_handler(irq, dev_id); +} + +static bool fsl_edma_srcid_in_use(struct fsl_edma_engine *fsl_edma, u32 srcid) +{ + struct fsl_edma_chan *fsl_chan; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + + if (fsl_chan->srcid && srcid == fsl_chan->srcid) { + dev_err(&fsl_chan->pdev->dev, "The srcid is in use, can't use!\n"); + return true; + } + } + return false; +} + +static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; + struct dma_chan *chan, *_chan; + struct fsl_edma_chan *fsl_chan; + u32 dmamux_nr = fsl_edma->drvdata->dmamuxs; + unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr; + + if (dma_spec->args_count != 2) + return NULL; + + guard(mutex)(&fsl_edma->fsl_edma_mutex); + + list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { + if (chan->client_count) + continue; + + if (fsl_edma_srcid_in_use(fsl_edma, dma_spec->args[1])) + return NULL; + + if ((chan->chan_id / chans_per_mux) == dma_spec->args[0]) { + chan = dma_get_slave_channel(chan); + if (chan) { + chan->device->privatecnt++; + fsl_chan = to_fsl_edma_chan(chan); + fsl_chan->srcid = dma_spec->args[1]; + + if (!fsl_chan->srcid) { + dev_err(&fsl_chan->pdev->dev, "Invalidate srcid %d\n", + fsl_chan->srcid); + return NULL; + } + + fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid, + true); + return chan; + } + } + } + return NULL; +} + +static struct dma_chan *fsl_edma3_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; + struct dma_chan *chan, *_chan; + struct fsl_edma_chan *fsl_chan; + bool b_chmux; + int i; + + if (dma_spec->args_count != 3) + return NULL; + + b_chmux = !!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHMUX); + + guard(mutex)(&fsl_edma->fsl_edma_mutex); + list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, + device_node) { + + if (chan->client_count) + continue; + + fsl_chan = to_fsl_edma_chan(chan); + if (fsl_edma_srcid_in_use(fsl_edma, dma_spec->args[0])) + return NULL; + i = fsl_chan - fsl_edma->chans; + + fsl_chan->priority = dma_spec->args[1]; + fsl_chan->is_rxchan = dma_spec->args[2] & FSL_EDMA_RX; + fsl_chan->is_remote = dma_spec->args[2] & FSL_EDMA_REMOTE; + fsl_chan->is_multi_fifo = dma_spec->args[2] & FSL_EDMA_MULTI_FIFO; + + if ((dma_spec->args[2] & FSL_EDMA_EVEN_CH) && (i & 0x1)) + continue; + + if ((dma_spec->args[2] & FSL_EDMA_ODD_CH) && !(i & 0x1)) + continue; + + if (!b_chmux && i == dma_spec->args[0]) { + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + return chan; + } else if (b_chmux && !fsl_chan->srcid) { + /* if controller support channel mux, choose a free channel */ + chan = dma_get_slave_channel(chan); + chan->device->privatecnt++; + fsl_chan->srcid = dma_spec->args[0]; + return chan; + } + } + return NULL; +} + +static int +fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + int ret; + + edma_writel(fsl_edma, ~0, fsl_edma->regs.intl); + + fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx"); + if (fsl_edma->txirq < 0) + return fsl_edma->txirq; + + fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err"); + if (fsl_edma->errirq < 0) + return fsl_edma->errirq; + + if (fsl_edma->txirq == fsl_edma->errirq) { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma_irq_handler, 0, "eDMA", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA IRQ.\n"); + return ret; + } + } else { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n"); + return ret; + } + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, + fsl_edma_err_handler, 0, "eDMA err", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n"); + return ret; + } + } + + return 0; +} + +static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + char *errirq_name; + int i, ret; + + for (i = 0; i < fsl_edma->n_chans; i++) { + + struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; + + if (fsl_edma->chan_masked & BIT(i)) + continue; + + /* request channel irq */ + fsl_chan->txirq = platform_get_irq(pdev, i); + if (fsl_chan->txirq < 0) + return -EINVAL; + + fsl_chan->irq_handler = fsl_edma3_tx_handler; + + if (!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_ERRIRQ_SHARE)) { + fsl_chan->errirq = fsl_chan->txirq; + fsl_chan->errirq_handler = fsl_edma3_err_handler_per_chan; + } + } + + /* All channel err use one irq number */ + if (fsl_edma->drvdata->flags & FSL_EDMA_DRV_ERRIRQ_SHARE) { + /* last one is error irq */ + fsl_edma->errirq = platform_get_irq_optional(pdev, fsl_edma->n_chans); + if (fsl_edma->errirq < 0) + return 0; /* dts miss err irq, treat as no err irq case */ + + errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err", + dev_name(&pdev->dev)); + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared, + 0, errirq_name, fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Can't register eDMA err IRQ.\n"); + } + + return 0; +} + +static int fsl_edma3_or_irq_init(struct platform_device *pdev, + struct fsl_edma_engine *fsl_edma) +{ + int ret; + + fsl_edma->txirq = platform_get_irq_byname(pdev, "tx-0-15"); + if (fsl_edma->txirq < 0) + return fsl_edma->txirq; + + fsl_edma->txirq_16_31 = platform_get_irq_byname(pdev, "tx-16-31"); + if (fsl_edma->txirq_16_31 < 0) + return fsl_edma->txirq_16_31; + + fsl_edma->errirq = platform_get_irq_byname(pdev, "err"); + if (fsl_edma->errirq < 0) + return fsl_edma->errirq; + + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma3_tx_0_15_handler, 0, "eDMA tx0_15", + fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA tx0_15 IRQ.\n"); + + if (fsl_edma->n_chans > 16) { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq_16_31, + fsl_edma3_tx_16_31_handler, 0, + "eDMA tx16_31", fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA tx16_31 IRQ.\n"); + } + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, + fsl_edma3_or_err_handler, 0, "eDMA err", + fsl_edma); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Can't register eDMA err IRQ.\n"); + + return 0; +} + +static int +fsl_edma2_irq_init(struct platform_device *pdev, + struct fsl_edma_engine *fsl_edma) +{ + int i, ret, irq; + int count; + + edma_writel(fsl_edma, ~0, fsl_edma->regs.intl); + + count = platform_irq_count(pdev); + dev_dbg(&pdev->dev, "%s Found %d interrupts\r\n", __func__, count); + if (count <= 2) { + dev_err(&pdev->dev, "Interrupts in DTS not correct.\n"); + return -EINVAL; + } + /* + * 16 channel independent interrupts + 1 error interrupt on i.mx7ulp. + * 2 channel share one interrupt, for example, ch0/ch16, ch1/ch17... + * For now, just simply request irq without IRQF_SHARED flag, since 16 + * channels are enough on i.mx7ulp whose M4 domain own some peripherals. + */ + for (i = 0; i < count; i++) { + irq = platform_get_irq(pdev, i); + ret = 0; + if (irq < 0) + return -ENXIO; + + /* The last IRQ is for eDMA err */ + if (i == count - 1) { + fsl_edma->errirq = irq; + ret = devm_request_irq(&pdev->dev, irq, + fsl_edma_err_handler, + 0, "eDMA2-ERR", fsl_edma); + } else { + fsl_edma->chans[i].txirq = irq; + fsl_edma->chans[i].irq_handler = fsl_edma2_tx_handler; + } + + if (ret) + return ret; + } + + return 0; +} + +static void fsl_edma_irq_exit( + struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + if (fsl_edma->txirq == fsl_edma->errirq) { + if (fsl_edma->txirq >= 0) + devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); + } else { + if (fsl_edma->txirq >= 0) + devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); + if (fsl_edma->errirq >= 0) + devm_free_irq(&pdev->dev, fsl_edma->errirq, fsl_edma); + } +} + +static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks) +{ + int i; + + for (i = 0; i < nr_clocks; i++) + clk_disable_unprepare(fsl_edma->muxclk[i]); +} + +static struct fsl_edma_drvdata vf610_data = { + .dmamuxs = DMAMUX_NR, + .flags = FSL_EDMA_DRV_WRAP_IO, + .chreg_off = EDMA_TCD, + .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), + .setup_irq = fsl_edma_irq_init, +}; + +static struct fsl_edma_drvdata ls1028a_data = { + .dmamuxs = DMAMUX_NR, + .flags = FSL_EDMA_DRV_MUX_SWAP | FSL_EDMA_DRV_WRAP_IO, + .chreg_off = EDMA_TCD, + .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), + .setup_irq = fsl_edma_irq_init, +}; + +static struct fsl_edma_drvdata imx7ulp_data = { + .dmamuxs = 1, + .chreg_off = EDMA_TCD, + .chreg_space_sz = sizeof(struct fsl_edma_hw_tcd), + .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_CONFIG32, + .setup_irq = fsl_edma2_irq_init, +}; + +static struct fsl_edma_drvdata imx8qm_data = { + .flags = FSL_EDMA_DRV_HAS_PD | FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_MEM_REMOTE + | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x10000, + .chreg_off = 0x10000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx8ulp_data = { + .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_CHCLK | FSL_EDMA_DRV_HAS_DMACLK | + FSL_EDMA_DRV_EDMA3, + .chreg_space_sz = 0x10000, + .chreg_off = 0x10000, + .mux_off = 0x10000 + offsetof(struct fsl_edma3_ch_reg, ch_mux), + .mux_skip = 0x10000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx93_data3 = { + .flags = FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x10000, + .chreg_off = 0x10000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx93_data4 = { + .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA4 + | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x8000, + .chreg_off = 0x10000, + .mux_off = 0x10000 + offsetof(struct fsl_edma3_ch_reg, ch_mux), + .mux_skip = 0x8000, + .setup_irq = fsl_edma3_irq_init, +}; + +static struct fsl_edma_drvdata imx95_data5 = { + .flags = FSL_EDMA_DRV_HAS_CHMUX | FSL_EDMA_DRV_HAS_DMACLK | FSL_EDMA_DRV_EDMA4 | + FSL_EDMA_DRV_TCD64 | FSL_EDMA_DRV_ERRIRQ_SHARE, + .chreg_space_sz = 0x8000, + .chreg_off = 0x10000, + .mux_off = 0x200, + .mux_skip = sizeof(u32), + .setup_irq = fsl_edma3_irq_init, +}; + +static const struct fsl_edma_drvdata s32g2_data = { + .dmamuxs = DMAMUX_NR, + .chreg_space_sz = EDMA_TCD, + .chreg_off = 0x4000, + .flags = FSL_EDMA_DRV_EDMA3 | FSL_EDMA_DRV_MUX_SWAP, + .setup_irq = fsl_edma3_or_irq_init, +}; + +static const struct of_device_id fsl_edma_dt_ids[] = { + { .compatible = "fsl,vf610-edma", .data = &vf610_data}, + { .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data}, + { .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data}, + { .compatible = "fsl,imx8qm-edma", .data = &imx8qm_data}, + { .compatible = "fsl,imx8ulp-edma", .data = &imx8ulp_data}, + { .compatible = "fsl,imx93-edma3", .data = &imx93_data3}, + { .compatible = "fsl,imx93-edma4", .data = &imx93_data4}, + { .compatible = "fsl,imx95-edma5", .data = &imx95_data5}, + { .compatible = "nxp,s32g2-edma", .data = &s32g2_data}, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); + +static void fsl_edma3_detach_pd(struct fsl_edma_engine *fsl_edma) +{ + struct fsl_edma_chan *fsl_chan; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + if (fsl_edma->chan_masked & BIT(i)) + continue; + fsl_chan = &fsl_edma->chans[i]; + if (fsl_chan->pd_dev_link) + device_link_del(fsl_chan->pd_dev_link); + if (fsl_chan->pd_dev) { + dev_pm_domain_detach(fsl_chan->pd_dev, false); + pm_runtime_dont_use_autosuspend(fsl_chan->pd_dev); + pm_runtime_set_suspended(fsl_chan->pd_dev); + } + } +} + +static void devm_fsl_edma3_detach_pd(void *data) +{ + fsl_edma3_detach_pd(data); +} + +static int fsl_edma3_attach_pd(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + struct fsl_edma_chan *fsl_chan; + struct device *pd_chan; + struct device *dev; + int i; + + dev = &pdev->dev; + + for (i = 0; i < fsl_edma->n_chans; i++) { + if (fsl_edma->chan_masked & BIT(i)) + continue; + + fsl_chan = &fsl_edma->chans[i]; + + pd_chan = dev_pm_domain_attach_by_id(dev, i); + if (IS_ERR_OR_NULL(pd_chan)) { + dev_err(dev, "Failed attach pd %d\n", i); + goto detach; + } + + fsl_chan->pd_dev_link = device_link_add(dev, pd_chan, DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!fsl_chan->pd_dev_link) { + dev_err(dev, "Failed to add device_link to %d\n", i); + dev_pm_domain_detach(pd_chan, false); + goto detach; + } + + fsl_chan->pd_dev = pd_chan; + + pm_runtime_use_autosuspend(fsl_chan->pd_dev); + pm_runtime_set_autosuspend_delay(fsl_chan->pd_dev, 200); + pm_runtime_set_active(fsl_chan->pd_dev); + } + + return 0; + +detach: + fsl_edma3_detach_pd(fsl_edma); + return -EINVAL; +} + +static int fsl_edma_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma_engine *fsl_edma; + const struct fsl_edma_drvdata *drvdata = NULL; + u32 chan_mask[2] = {0, 0}; + char clk_name[36]; + struct edma_regs *regs; + int chans; + int ret, i; + + drvdata = device_get_match_data(&pdev->dev); + if (!drvdata) { + dev_err(&pdev->dev, "unable to find driver data\n"); + return -EINVAL; + } + + ret = of_property_read_u32(np, "dma-channels", &chans); + if (ret) { + dev_err(&pdev->dev, "Can't get dma-channels.\n"); + return ret; + } + + fsl_edma = devm_kzalloc(&pdev->dev, struct_size(fsl_edma, chans, chans), + GFP_KERNEL); + if (!fsl_edma) + return -ENOMEM; + + fsl_edma->errirq = -EINVAL; + fsl_edma->txirq = -EINVAL; + fsl_edma->drvdata = drvdata; + fsl_edma->n_chans = chans; + mutex_init(&fsl_edma->fsl_edma_mutex); + + fsl_edma->membase = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(fsl_edma->membase)) + return PTR_ERR(fsl_edma->membase); + + if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) { + fsl_edma_setup_regs(fsl_edma); + regs = &fsl_edma->regs; + } + + if (drvdata->flags & FSL_EDMA_DRV_HAS_DMACLK) { + fsl_edma->dmaclk = devm_clk_get_enabled(&pdev->dev, "dma"); + if (IS_ERR(fsl_edma->dmaclk)) { + dev_err(&pdev->dev, "Missing DMA block clock.\n"); + return PTR_ERR(fsl_edma->dmaclk); + } + } + + ret = of_property_read_variable_u32_array(np, "dma-channel-mask", chan_mask, 1, 2); + + if (ret > 0) { + fsl_edma->chan_masked = chan_mask[1]; + fsl_edma->chan_masked <<= 32; + fsl_edma->chan_masked |= chan_mask[0]; + } + + for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) { + char clkname[32]; + + fsl_edma->muxbase[i] = devm_platform_ioremap_resource(pdev, + 1 + i); + if (IS_ERR(fsl_edma->muxbase[i])) { + /* on error: disable all previously enabled clks */ + fsl_disable_clocks(fsl_edma, i); + return PTR_ERR(fsl_edma->muxbase[i]); + } + + sprintf(clkname, "dmamux%d", i); + fsl_edma->muxclk[i] = devm_clk_get_enabled(&pdev->dev, clkname); + if (IS_ERR(fsl_edma->muxclk[i])) { + dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); + /* on error: disable all previously enabled clks */ + return PTR_ERR(fsl_edma->muxclk[i]); + } + } + + fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); + + if (drvdata->flags & FSL_EDMA_DRV_HAS_PD) { + ret = fsl_edma3_attach_pd(pdev, fsl_edma); + if (ret) + return ret; + ret = devm_add_action_or_reset(&pdev->dev, devm_fsl_edma3_detach_pd, fsl_edma); + if (ret) + return ret; + } + + if (drvdata->flags & FSL_EDMA_DRV_TCD64) + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + + INIT_LIST_HEAD(&fsl_edma->dma_dev.channels); + for (i = 0; i < fsl_edma->n_chans; i++) { + struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; + int len; + + if (fsl_edma->chan_masked & BIT(i)) + continue; + + snprintf(fsl_chan->chan_name, sizeof(fsl_chan->chan_name), "%s-CH%02d", + dev_name(&pdev->dev), i); + + snprintf(fsl_chan->errirq_name, sizeof(fsl_chan->errirq_name), + "%s-CH%02d-err", dev_name(&pdev->dev), i); + + fsl_chan->edma = fsl_edma; + fsl_chan->pm_state = RUNNING; + fsl_chan->srcid = 0; + fsl_chan->dma_dir = DMA_NONE; + fsl_chan->vchan.desc_free = fsl_edma_free_desc; + + len = (drvdata->flags & FSL_EDMA_DRV_SPLIT_REG) ? + offsetof(struct fsl_edma3_ch_reg, tcd) : 0; + fsl_chan->tcd = fsl_edma->membase + + i * drvdata->chreg_space_sz + drvdata->chreg_off + len; + fsl_chan->mux_addr = fsl_edma->membase + drvdata->mux_off + i * drvdata->mux_skip; + + if (drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) { + snprintf(clk_name, sizeof(clk_name), "ch%02d", i); + fsl_chan->clk = devm_clk_get_enabled(&pdev->dev, + (const char *)clk_name); + + if (IS_ERR(fsl_chan->clk)) + return PTR_ERR(fsl_chan->clk); + } + fsl_chan->pdev = pdev; + vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); + + edma_write_tcdreg(fsl_chan, cpu_to_le32(0), csr); + fsl_edma_chan_mux(fsl_chan, 0, false); + if (fsl_chan->edma->drvdata->flags & FSL_EDMA_DRV_HAS_CHCLK) + clk_disable_unprepare(fsl_chan->clk); + } + + ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma); + if (ret) + return ret; + + dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_MEMCPY, fsl_edma->dma_dev.cap_mask); + + fsl_edma->dma_dev.dev = &pdev->dev; + fsl_edma->dma_dev.device_alloc_chan_resources + = fsl_edma_alloc_chan_resources; + fsl_edma->dma_dev.device_free_chan_resources + = fsl_edma_free_chan_resources; + fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; + fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; + fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; + fsl_edma->dma_dev.device_prep_dma_memcpy = fsl_edma_prep_memcpy; + fsl_edma->dma_dev.device_config = fsl_edma_slave_config; + fsl_edma->dma_dev.device_pause = fsl_edma_pause; + fsl_edma->dma_dev.device_resume = fsl_edma_resume; + fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all; + fsl_edma->dma_dev.device_synchronize = fsl_edma_synchronize; + fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending; + + fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; + fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; + + if (drvdata->flags & FSL_EDMA_DRV_BUS_8BYTE) { + fsl_edma->dma_dev.src_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + fsl_edma->dma_dev.dst_addr_widths |= BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + } + + fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + if (drvdata->flags & FSL_EDMA_DRV_DEV_TO_DEV) + fsl_edma->dma_dev.directions |= BIT(DMA_DEV_TO_DEV); + + fsl_edma->dma_dev.copy_align = drvdata->flags & FSL_EDMA_DRV_ALIGN_64BYTE ? + DMAENGINE_ALIGN_64_BYTES : + DMAENGINE_ALIGN_32_BYTES; + + /* Per worst case 'nbytes = 1' take CITER as the max_seg_size */ + dma_set_max_seg_size(fsl_edma->dma_dev.dev, + FIELD_GET(EDMA_TCD_ITER_MASK, EDMA_TCD_ITER_MASK)); + + fsl_edma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + + platform_set_drvdata(pdev, fsl_edma); + + ret = dma_async_device_register(&fsl_edma->dma_dev); + if (ret) { + dev_err(&pdev->dev, + "Can't register Freescale eDMA engine. (%d)\n", ret); + return ret; + } + + ret = of_dma_controller_register(np, + drvdata->dmamuxs ? fsl_edma_xlate : fsl_edma3_xlate, + fsl_edma); + if (ret) { + dev_err(&pdev->dev, + "Can't register Freescale eDMA of_dma. (%d)\n", ret); + dma_async_device_unregister(&fsl_edma->dma_dev); + return ret; + } + + /* enable round robin arbitration */ + if (!(drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) + edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); + + return 0; +} + +static void fsl_edma_remove(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); + + fsl_edma_irq_exit(pdev, fsl_edma); + of_dma_controller_free(np); + dma_async_device_unregister(&fsl_edma->dma_dev); + fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); + fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); +} + +static int fsl_edma_suspend_late(struct device *dev) +{ + struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma_chan *fsl_chan; + unsigned long flags; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + if (fsl_edma->chan_masked & BIT(i)) + continue; + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + /* Make sure chan is idle or will force disable. */ + if (unlikely(fsl_chan->status == DMA_IN_PROGRESS)) { + dev_warn(dev, "WARN: There is non-idle channel.\n"); + fsl_edma_disable_request(fsl_chan); + fsl_edma_chan_mux(fsl_chan, 0, false); + } + + fsl_chan->pm_state = SUSPENDED; + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + } + + return 0; +} + +static int fsl_edma_resume_early(struct device *dev) +{ + struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma_chan *fsl_chan; + struct edma_regs *regs = &fsl_edma->regs; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + if (fsl_edma->chan_masked & BIT(i)) + continue; + fsl_chan->pm_state = RUNNING; + edma_write_tcdreg(fsl_chan, 0, csr); + if (fsl_chan->srcid != 0) + fsl_edma_chan_mux(fsl_chan, fsl_chan->srcid, true); + } + + if (!(fsl_edma->drvdata->flags & FSL_EDMA_DRV_SPLIT_REG)) + edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); + + return 0; +} + +/* + * eDMA provides the service to others, so it should be suspend late + * and resume early. When eDMA suspend, all of the clients should stop + * the DMA data transmission and let the channel idle. + */ +static const struct dev_pm_ops fsl_edma_pm_ops = { + .suspend_late = fsl_edma_suspend_late, + .resume_early = fsl_edma_resume_early, +}; + +static struct platform_driver fsl_edma_driver = { + .driver = { + .name = "fsl-edma", + .of_match_table = fsl_edma_dt_ids, + .pm = &fsl_edma_pm_ops, + }, + .probe = fsl_edma_probe, + .remove = fsl_edma_remove, +}; + +static int __init fsl_edma_init(void) +{ + return platform_driver_register(&fsl_edma_driver); +} +subsys_initcall(fsl_edma_init); + +static void __exit fsl_edma_exit(void) +{ + platform_driver_unregister(&fsl_edma_driver); +} +module_exit(fsl_edma_exit); + +MODULE_DESCRIPTION("Freescale eDMA engine driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-edma-trace.c b/drivers/dma/fsl-edma-trace.c new file mode 100644 index 000000000000..28300ad80bb7 --- /dev/null +++ b/drivers/dma/fsl-edma-trace.c @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define CREATE_TRACE_POINTS +#include "fsl-edma-common.h" diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h new file mode 100644 index 000000000000..d3541301a247 --- /dev/null +++ b/drivers/dma/fsl-edma-trace.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright 2023 NXP. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM fsl_edma + +#if !defined(__LINUX_FSL_EDMA_TRACE) || defined(TRACE_HEADER_MULTI_READ) +#define __LINUX_FSL_EDMA_TRACE + +#include <linux/types.h> +#include <linux/tracepoint.h> + +DECLARE_EVENT_CLASS(edma_log_io, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value), + TP_STRUCT__entry( + __field(struct fsl_edma_engine *, edma) + __field(void __iomem *, addr) + __field(u32, value) + ), + TP_fast_assign( + __entry->edma = edma; + __entry->addr = addr; + __entry->value = value; + ), + TP_printk("offset %08x: value %08x", + (u32)(__entry->addr - __entry->edma->membase), __entry->value) +); + +DEFINE_EVENT(edma_log_io, edma_readl, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writel, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_readw, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writew, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_readb, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DEFINE_EVENT(edma_log_io, edma_writeb, + TP_PROTO(struct fsl_edma_engine *edma, void __iomem *addr, u32 value), + TP_ARGS(edma, addr, value) +); + +DECLARE_EVENT_CLASS(edma_log_tcd, + TP_PROTO(struct fsl_edma_chan *chan, void *tcd), + TP_ARGS(chan, tcd), + TP_STRUCT__entry( + __field(u64, saddr) + __field(u16, soff) + __field(u16, attr) + __field(u32, nbytes) + __field(u64, slast) + __field(u64, daddr) + __field(u16, doff) + __field(u16, citer) + __field(u64, dlast_sga) + __field(u16, csr) + __field(u16, biter) + + ), + TP_fast_assign( + __entry->saddr = fsl_edma_get_tcd_to_cpu(chan, tcd, saddr), + __entry->soff = fsl_edma_get_tcd_to_cpu(chan, tcd, soff), + __entry->attr = fsl_edma_get_tcd_to_cpu(chan, tcd, attr), + __entry->nbytes = fsl_edma_get_tcd_to_cpu(chan, tcd, nbytes), + __entry->slast = fsl_edma_get_tcd_to_cpu(chan, tcd, slast), + __entry->daddr = fsl_edma_get_tcd_to_cpu(chan, tcd, daddr), + __entry->doff = fsl_edma_get_tcd_to_cpu(chan, tcd, doff), + __entry->citer = fsl_edma_get_tcd_to_cpu(chan, tcd, citer), + __entry->dlast_sga = fsl_edma_get_tcd_to_cpu(chan, tcd, dlast_sga), + __entry->csr = fsl_edma_get_tcd_to_cpu(chan, tcd, csr), + __entry->biter = fsl_edma_get_tcd_to_cpu(chan, tcd, biter); + ), + TP_printk("\n==== TCD =====\n" + " saddr: 0x%016llx\n" + " soff: 0x%04x\n" + " attr: 0x%04x\n" + " nbytes: 0x%08x\n" + " slast: 0x%016llx\n" + " daddr: 0x%016llx\n" + " doff: 0x%04x\n" + " citer: 0x%04x\n" + " dlast: 0x%016llx\n" + " csr: 0x%04x\n" + " biter: 0x%04x\n", + __entry->saddr, + __entry->soff, + __entry->attr, + __entry->nbytes, + __entry->slast, + __entry->daddr, + __entry->doff, + __entry->citer, + __entry->dlast_sga, + __entry->csr, + __entry->biter) +); + +DEFINE_EVENT(edma_log_tcd, edma_fill_tcd, + TP_PROTO(struct fsl_edma_chan *chan, void *tcd), + TP_ARGS(chan, tcd) +); + +#endif + +/* this part must be outside header guard */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE fsl-edma-trace + +#include <trace/define_trace.h> diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c deleted file mode 100644 index 90bb72af306c..000000000000 --- a/drivers/dma/fsl-edma.c +++ /dev/null @@ -1,512 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * drivers/dma/fsl-edma.c - * - * Copyright 2013-2014 Freescale Semiconductor, Inc. - * - * Driver for the Freescale eDMA engine with flexible channel multiplexing - * capability for DMA request sources. The eDMA block can be found on some - * Vybrid and Layerscape SoCs. - */ - -#include <linux/module.h> -#include <linux/interrupt.h> -#include <linux/clk.h> -#include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_dma.h> - -#include "fsl-edma-common.h" - -static void fsl_edma_synchronize(struct dma_chan *chan) -{ - struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); - - vchan_synchronize(&fsl_chan->vchan); -} - -static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) -{ - struct fsl_edma_engine *fsl_edma = dev_id; - unsigned int intr, ch; - struct edma_regs *regs = &fsl_edma->regs; - struct fsl_edma_chan *fsl_chan; - - intr = edma_readl(fsl_edma, regs->intl); - if (!intr) - return IRQ_NONE; - - for (ch = 0; ch < fsl_edma->n_chans; ch++) { - if (intr & (0x1 << ch)) { - edma_writeb(fsl_edma, EDMA_CINT_CINT(ch), regs->cint); - - fsl_chan = &fsl_edma->chans[ch]; - - spin_lock(&fsl_chan->vchan.lock); - - if (!fsl_chan->edesc) { - /* terminate_all called before */ - spin_unlock(&fsl_chan->vchan.lock); - continue; - } - - if (!fsl_chan->edesc->iscyclic) { - list_del(&fsl_chan->edesc->vdesc.node); - vchan_cookie_complete(&fsl_chan->edesc->vdesc); - fsl_chan->edesc = NULL; - fsl_chan->status = DMA_COMPLETE; - fsl_chan->idle = true; - } else { - vchan_cyclic_callback(&fsl_chan->edesc->vdesc); - } - - if (!fsl_chan->edesc) - fsl_edma_xfer_desc(fsl_chan); - - spin_unlock(&fsl_chan->vchan.lock); - } - } - return IRQ_HANDLED; -} - -static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) -{ - struct fsl_edma_engine *fsl_edma = dev_id; - unsigned int err, ch; - struct edma_regs *regs = &fsl_edma->regs; - - err = edma_readl(fsl_edma, regs->errl); - if (!err) - return IRQ_NONE; - - for (ch = 0; ch < fsl_edma->n_chans; ch++) { - if (err & (0x1 << ch)) { - fsl_edma_disable_request(&fsl_edma->chans[ch]); - edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), regs->cerr); - fsl_edma->chans[ch].status = DMA_ERROR; - fsl_edma->chans[ch].idle = true; - } - } - return IRQ_HANDLED; -} - -static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) -{ - if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED) - return IRQ_HANDLED; - - return fsl_edma_err_handler(irq, dev_id); -} - -static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, - struct of_dma *ofdma) -{ - struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; - struct dma_chan *chan, *_chan; - struct fsl_edma_chan *fsl_chan; - u32 dmamux_nr = fsl_edma->drvdata->dmamuxs; - unsigned long chans_per_mux = fsl_edma->n_chans / dmamux_nr; - - if (dma_spec->args_count != 2) - return NULL; - - mutex_lock(&fsl_edma->fsl_edma_mutex); - list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { - if (chan->client_count) - continue; - if ((chan->chan_id / chans_per_mux) == dma_spec->args[0]) { - chan = dma_get_slave_channel(chan); - if (chan) { - chan->device->privatecnt++; - fsl_chan = to_fsl_edma_chan(chan); - fsl_chan->slave_id = dma_spec->args[1]; - fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, - true); - mutex_unlock(&fsl_edma->fsl_edma_mutex); - return chan; - } - } - } - mutex_unlock(&fsl_edma->fsl_edma_mutex); - return NULL; -} - -static int -fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) -{ - int ret; - - fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx"); - if (fsl_edma->txirq < 0) - return fsl_edma->txirq; - - fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err"); - if (fsl_edma->errirq < 0) - return fsl_edma->errirq; - - if (fsl_edma->txirq == fsl_edma->errirq) { - ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, - fsl_edma_irq_handler, 0, "eDMA", fsl_edma); - if (ret) { - dev_err(&pdev->dev, "Can't register eDMA IRQ.\n"); - return ret; - } - } else { - ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, - fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma); - if (ret) { - dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n"); - return ret; - } - - ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, - fsl_edma_err_handler, 0, "eDMA err", fsl_edma); - if (ret) { - dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n"); - return ret; - } - } - - return 0; -} - -static int -fsl_edma2_irq_init(struct platform_device *pdev, - struct fsl_edma_engine *fsl_edma) -{ - int i, ret, irq; - int count; - - count = platform_irq_count(pdev); - dev_dbg(&pdev->dev, "%s Found %d interrupts\r\n", __func__, count); - if (count <= 2) { - dev_err(&pdev->dev, "Interrupts in DTS not correct.\n"); - return -EINVAL; - } - /* - * 16 channel independent interrupts + 1 error interrupt on i.mx7ulp. - * 2 channel share one interrupt, for example, ch0/ch16, ch1/ch17... - * For now, just simply request irq without IRQF_SHARED flag, since 16 - * channels are enough on i.mx7ulp whose M4 domain own some peripherals. - */ - for (i = 0; i < count; i++) { - irq = platform_get_irq(pdev, i); - if (irq < 0) - return -ENXIO; - - sprintf(fsl_edma->chans[i].chan_name, "eDMA2-CH%02d", i); - - /* The last IRQ is for eDMA err */ - if (i == count - 1) - ret = devm_request_irq(&pdev->dev, irq, - fsl_edma_err_handler, - 0, "eDMA2-ERR", fsl_edma); - else - ret = devm_request_irq(&pdev->dev, irq, - fsl_edma_tx_handler, 0, - fsl_edma->chans[i].chan_name, - fsl_edma); - if (ret) - return ret; - } - - return 0; -} - -static void fsl_edma_irq_exit( - struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) -{ - if (fsl_edma->txirq == fsl_edma->errirq) { - devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); - } else { - devm_free_irq(&pdev->dev, fsl_edma->txirq, fsl_edma); - devm_free_irq(&pdev->dev, fsl_edma->errirq, fsl_edma); - } -} - -static void fsl_disable_clocks(struct fsl_edma_engine *fsl_edma, int nr_clocks) -{ - int i; - - for (i = 0; i < nr_clocks; i++) - clk_disable_unprepare(fsl_edma->muxclk[i]); -} - -static struct fsl_edma_drvdata vf610_data = { - .version = v1, - .dmamuxs = DMAMUX_NR, - .setup_irq = fsl_edma_irq_init, -}; - -static struct fsl_edma_drvdata ls1028a_data = { - .version = v1, - .dmamuxs = DMAMUX_NR, - .mux_swap = true, - .setup_irq = fsl_edma_irq_init, -}; - -static struct fsl_edma_drvdata imx7ulp_data = { - .version = v3, - .dmamuxs = 1, - .has_dmaclk = true, - .setup_irq = fsl_edma2_irq_init, -}; - -static const struct of_device_id fsl_edma_dt_ids[] = { - { .compatible = "fsl,vf610-edma", .data = &vf610_data}, - { .compatible = "fsl,ls1028a-edma", .data = &ls1028a_data}, - { .compatible = "fsl,imx7ulp-edma", .data = &imx7ulp_data}, - { /* sentinel */ } -}; -MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); - -static int fsl_edma_probe(struct platform_device *pdev) -{ - const struct of_device_id *of_id = - of_match_device(fsl_edma_dt_ids, &pdev->dev); - struct device_node *np = pdev->dev.of_node; - struct fsl_edma_engine *fsl_edma; - const struct fsl_edma_drvdata *drvdata = NULL; - struct fsl_edma_chan *fsl_chan; - struct edma_regs *regs; - struct resource *res; - int len, chans; - int ret, i; - - if (of_id) - drvdata = of_id->data; - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } - - ret = of_property_read_u32(np, "dma-channels", &chans); - if (ret) { - dev_err(&pdev->dev, "Can't get dma-channels.\n"); - return ret; - } - - len = sizeof(*fsl_edma) + sizeof(*fsl_chan) * chans; - fsl_edma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); - if (!fsl_edma) - return -ENOMEM; - - fsl_edma->drvdata = drvdata; - fsl_edma->n_chans = chans; - mutex_init(&fsl_edma->fsl_edma_mutex); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - fsl_edma->membase = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(fsl_edma->membase)) - return PTR_ERR(fsl_edma->membase); - - fsl_edma_setup_regs(fsl_edma); - regs = &fsl_edma->regs; - - if (drvdata->has_dmaclk) { - fsl_edma->dmaclk = devm_clk_get(&pdev->dev, "dma"); - if (IS_ERR(fsl_edma->dmaclk)) { - dev_err(&pdev->dev, "Missing DMA block clock.\n"); - return PTR_ERR(fsl_edma->dmaclk); - } - - ret = clk_prepare_enable(fsl_edma->dmaclk); - if (ret) { - dev_err(&pdev->dev, "DMA clk block failed.\n"); - return ret; - } - } - - for (i = 0; i < fsl_edma->drvdata->dmamuxs; i++) { - char clkname[32]; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); - fsl_edma->muxbase[i] = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(fsl_edma->muxbase[i])) { - /* on error: disable all previously enabled clks */ - fsl_disable_clocks(fsl_edma, i); - return PTR_ERR(fsl_edma->muxbase[i]); - } - - sprintf(clkname, "dmamux%d", i); - fsl_edma->muxclk[i] = devm_clk_get(&pdev->dev, clkname); - if (IS_ERR(fsl_edma->muxclk[i])) { - dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); - /* on error: disable all previously enabled clks */ - fsl_disable_clocks(fsl_edma, i); - return PTR_ERR(fsl_edma->muxclk[i]); - } - - ret = clk_prepare_enable(fsl_edma->muxclk[i]); - if (ret) - /* on error: disable all previously enabled clks */ - fsl_disable_clocks(fsl_edma, i); - - } - - fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); - - INIT_LIST_HEAD(&fsl_edma->dma_dev.channels); - for (i = 0; i < fsl_edma->n_chans; i++) { - struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; - - fsl_chan->edma = fsl_edma; - fsl_chan->pm_state = RUNNING; - fsl_chan->slave_id = 0; - fsl_chan->idle = true; - fsl_chan->dma_dir = DMA_NONE; - fsl_chan->vchan.desc_free = fsl_edma_free_desc; - vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); - - edma_writew(fsl_edma, 0x0, ®s->tcd[i].csr); - fsl_edma_chan_mux(fsl_chan, 0, false); - } - - edma_writel(fsl_edma, ~0, regs->intl); - ret = fsl_edma->drvdata->setup_irq(pdev, fsl_edma); - if (ret) - return ret; - - dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask); - dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask); - dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask); - - fsl_edma->dma_dev.dev = &pdev->dev; - fsl_edma->dma_dev.device_alloc_chan_resources - = fsl_edma_alloc_chan_resources; - fsl_edma->dma_dev.device_free_chan_resources - = fsl_edma_free_chan_resources; - fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; - fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; - fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; - fsl_edma->dma_dev.device_config = fsl_edma_slave_config; - fsl_edma->dma_dev.device_pause = fsl_edma_pause; - fsl_edma->dma_dev.device_resume = fsl_edma_resume; - fsl_edma->dma_dev.device_terminate_all = fsl_edma_terminate_all; - fsl_edma->dma_dev.device_synchronize = fsl_edma_synchronize; - fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending; - - fsl_edma->dma_dev.src_addr_widths = FSL_EDMA_BUSWIDTHS; - fsl_edma->dma_dev.dst_addr_widths = FSL_EDMA_BUSWIDTHS; - fsl_edma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); - - platform_set_drvdata(pdev, fsl_edma); - - ret = dma_async_device_register(&fsl_edma->dma_dev); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA engine. (%d)\n", ret); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); - return ret; - } - - ret = of_dma_controller_register(np, fsl_edma_xlate, fsl_edma); - if (ret) { - dev_err(&pdev->dev, - "Can't register Freescale eDMA of_dma. (%d)\n", ret); - dma_async_device_unregister(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); - return ret; - } - - /* enable round robin arbitration */ - edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); - - return 0; -} - -static int fsl_edma_remove(struct platform_device *pdev) -{ - struct device_node *np = pdev->dev.of_node; - struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); - - fsl_edma_irq_exit(pdev, fsl_edma); - fsl_edma_cleanup_vchan(&fsl_edma->dma_dev); - of_dma_controller_free(np); - dma_async_device_unregister(&fsl_edma->dma_dev); - fsl_disable_clocks(fsl_edma, fsl_edma->drvdata->dmamuxs); - - return 0; -} - -static int fsl_edma_suspend_late(struct device *dev) -{ - struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); - struct fsl_edma_chan *fsl_chan; - unsigned long flags; - int i; - - for (i = 0; i < fsl_edma->n_chans; i++) { - fsl_chan = &fsl_edma->chans[i]; - spin_lock_irqsave(&fsl_chan->vchan.lock, flags); - /* Make sure chan is idle or will force disable. */ - if (unlikely(!fsl_chan->idle)) { - dev_warn(dev, "WARN: There is non-idle channel."); - fsl_edma_disable_request(fsl_chan); - fsl_edma_chan_mux(fsl_chan, 0, false); - } - - fsl_chan->pm_state = SUSPENDED; - spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); - } - - return 0; -} - -static int fsl_edma_resume_early(struct device *dev) -{ - struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); - struct fsl_edma_chan *fsl_chan; - struct edma_regs *regs = &fsl_edma->regs; - int i; - - for (i = 0; i < fsl_edma->n_chans; i++) { - fsl_chan = &fsl_edma->chans[i]; - fsl_chan->pm_state = RUNNING; - edma_writew(fsl_edma, 0x0, ®s->tcd[i].csr); - if (fsl_chan->slave_id != 0) - fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, true); - } - - edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, regs->cr); - - return 0; -} - -/* - * eDMA provides the service to others, so it should be suspend late - * and resume early. When eDMA suspend, all of the clients should stop - * the DMA data transmission and let the channel idle. - */ -static const struct dev_pm_ops fsl_edma_pm_ops = { - .suspend_late = fsl_edma_suspend_late, - .resume_early = fsl_edma_resume_early, -}; - -static struct platform_driver fsl_edma_driver = { - .driver = { - .name = "fsl-edma", - .of_match_table = fsl_edma_dt_ids, - .pm = &fsl_edma_pm_ops, - }, - .probe = fsl_edma_probe, - .remove = fsl_edma_remove, -}; - -static int __init fsl_edma_init(void) -{ - return platform_driver_register(&fsl_edma_driver); -} -subsys_initcall(fsl_edma_init); - -static void __exit fsl_edma_exit(void) -{ - platform_driver_unregister(&fsl_edma_driver); -} -module_exit(fsl_edma_exit); - -MODULE_ALIAS("platform:fsl-edma"); -MODULE_DESCRIPTION("Freescale eDMA engine driver"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsl-qdma.c b/drivers/dma/fsl-qdma.c index 045ead46ec8f..6ace5bf80c40 100644 --- a/drivers/dma/fsl-qdma.c +++ b/drivers/dma/fsl-qdma.c @@ -13,10 +13,10 @@ #include <linux/module.h> #include <linux/delay.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <linux/dma-mapping.h> +#include <linux/platform_device.h> #include "virt-dma.h" #include "fsldma.h" @@ -109,6 +109,7 @@ #define FSL_QDMA_CMD_WTHROTL_OFFSET 20 #define FSL_QDMA_CMD_DSEN_OFFSET 19 #define FSL_QDMA_CMD_LWC_OFFSET 16 +#define FSL_QDMA_CMD_PF BIT(17) /* Field definition for Descriptor status */ #define QDMA_CCDF_STATUS_RTE BIT(5) @@ -147,6 +148,9 @@ * @__reserved1: Reserved field. * @cfg8b_w1: Compound descriptor command queue origin produced * by qDMA and dynamic debug field. + * @__reserved2: Reserved field. + * @cmd: Command for QDMA (see FSL_QDMA_CMD_RWTTYPE and + * others). * @data: Pointer to the memory 40-bit address, describes DMA * source information and DMA destination information. */ @@ -160,6 +164,10 @@ struct fsl_qdma_format { u8 __reserved1[2]; u8 cfg8b_w1; } __packed; + struct { + __le32 __reserved2; + __le32 cmd; + } __packed; __le64 data; }; } __packed; @@ -354,7 +362,6 @@ static void fsl_qdma_free_chan_resources(struct dma_chan *chan) static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, dma_addr_t dst, dma_addr_t src, u32 len) { - u32 cmd; struct fsl_qdma_format *sdf, *ddf; struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest; @@ -383,14 +390,11 @@ static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, /* This entry is the last entry. */ qdma_csgf_set_f(csgf_dest, len); /* Descriptor Buffer */ - cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE << - FSL_QDMA_CMD_RWTTYPE_OFFSET); - sdf->data = QDMA_SDDF_CMD(cmd); - - cmd = cpu_to_le32(FSL_QDMA_CMD_RWTTYPE << - FSL_QDMA_CMD_RWTTYPE_OFFSET); - cmd |= cpu_to_le32(FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET); - ddf->data = QDMA_SDDF_CMD(cmd); + sdf->cmd = cpu_to_le32((FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET) | + FSL_QDMA_CMD_PF); + + ddf->cmd = cpu_to_le32((FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET) | + (FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET)); } /* @@ -514,11 +518,11 @@ static struct fsl_qdma_queue queue_temp = queue_head + i + (j * queue_num); queue_temp->cq = - dma_alloc_coherent(&pdev->dev, - sizeof(struct fsl_qdma_format) * - queue_size[i], - &queue_temp->bus_addr, - GFP_KERNEL); + dmam_alloc_coherent(&pdev->dev, + sizeof(struct fsl_qdma_format) * + queue_size[i], + &queue_temp->bus_addr, + GFP_KERNEL); if (!queue_temp->cq) return NULL; queue_temp->block_base = fsl_qdma->block_base + @@ -563,15 +567,14 @@ static struct fsl_qdma_queue /* * Buffer for queue command */ - status_head->cq = dma_alloc_coherent(&pdev->dev, - sizeof(struct fsl_qdma_format) * - status_size, - &status_head->bus_addr, - GFP_KERNEL); - if (!status_head->cq) { - devm_kfree(&pdev->dev, status_head); + status_head->cq = dmam_alloc_coherent(&pdev->dev, + sizeof(struct fsl_qdma_format) * + status_size, + &status_head->bus_addr, + GFP_KERNEL); + if (!status_head->cq) return NULL; - } + status_head->n_cq = status_size; status_head->virt_head = status_head->cq; status_head->virt_tail = status_head->cq; @@ -625,7 +628,7 @@ static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma) static int fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma, - void *block, + __iomem void *block, int id) { bool duplicate; @@ -805,7 +808,7 @@ fsl_qdma_irq_init(struct platform_device *pdev, int i; int cpu; int ret; - char irq_name[20]; + char irq_name[32]; fsl_qdma->error_irq = platform_get_irq_byname(pdev, "qdma-error"); @@ -1119,7 +1122,6 @@ static int fsl_qdma_probe(struct platform_device *pdev) int ret, i; int blk_num, blk_off; u32 len, chans, queues; - struct resource *res; struct fsl_qdma_chan *fsl_chan; struct fsl_qdma_engine *fsl_qdma; struct device_node *np = pdev->dev.of_node; @@ -1183,28 +1185,21 @@ static int fsl_qdma_probe(struct platform_device *pdev) if (!fsl_qdma->status[i]) return -ENOMEM; } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res); + fsl_qdma->ctrl_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(fsl_qdma->ctrl_base)) return PTR_ERR(fsl_qdma->ctrl_base); - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res); + fsl_qdma->status_base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(fsl_qdma->status_base)) return PTR_ERR(fsl_qdma->status_base); - res = platform_get_resource(pdev, IORESOURCE_MEM, 2); - fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res); + fsl_qdma->block_base = devm_platform_ioremap_resource(pdev, 2); if (IS_ERR(fsl_qdma->block_base)) return PTR_ERR(fsl_qdma->block_base); fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, fsl_qdma); if (!fsl_qdma->queue) return -ENOMEM; - ret = fsl_qdma_irq_init(pdev, fsl_qdma); - if (ret) - return ret; - fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0"); if (fsl_qdma->irq_base < 0) return fsl_qdma->irq_base; @@ -1243,16 +1238,19 @@ static int fsl_qdma_probe(struct platform_device *pdev) platform_set_drvdata(pdev, fsl_qdma); - ret = dma_async_device_register(&fsl_qdma->dma_dev); + ret = fsl_qdma_reg_init(fsl_qdma); if (ret) { - dev_err(&pdev->dev, - "Can't register NXP Layerscape qDMA engine.\n"); + dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); return ret; } - ret = fsl_qdma_reg_init(fsl_qdma); + ret = fsl_qdma_irq_init(pdev, fsl_qdma); + if (ret) + return ret; + + ret = dma_async_device_register(&fsl_qdma->dma_dev); if (ret) { - dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); + dev_err(&pdev->dev, "Can't register NXP Layerscape qDMA engine.\n"); return ret; } @@ -1270,10 +1268,8 @@ static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev) } } -static int fsl_qdma_remove(struct platform_device *pdev) +static void fsl_qdma_remove(struct platform_device *pdev) { - int i; - struct fsl_qdma_queue *status; struct device_node *np = pdev->dev.of_node; struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev); @@ -1281,13 +1277,6 @@ static int fsl_qdma_remove(struct platform_device *pdev) fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev); of_dma_controller_free(np); dma_async_device_unregister(&fsl_qdma->dma_dev); - - for (i = 0; i < fsl_qdma->block_number; i++) { - status = fsl_qdma->status[i]; - dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_format) * - status->n_cq, status->cq, status->bus_addr); - } - return 0; } static const struct of_device_id fsl_qdma_dt_ids[] = { @@ -1307,6 +1296,5 @@ static struct platform_driver fsl_qdma_driver = { module_platform_driver(fsl_qdma_driver); -MODULE_ALIAS("platform:fsl-qdma"); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver"); diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index fdf3500d96a9..6aa97e258a55 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -60,9 +60,10 @@ */ #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/of_irq.h> -#include <linux/of_address.h> #include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> #include <linux/dmaengine.h> @@ -856,7 +857,7 @@ static void fsl_re_remove_chan(struct fsl_re_chan *chan) chan->oub_phys_addr); } -static int fsl_re_remove(struct platform_device *ofdev) +static void fsl_re_remove(struct platform_device *ofdev) { struct fsl_re_drv_private *re_priv; struct device *dev; @@ -871,8 +872,6 @@ static int fsl_re_remove(struct platform_device *ofdev) /* Unregister the driver */ dma_async_device_unregister(&re_priv->dma_dev); - - return 0; } static const struct of_device_id fsl_re_ids[] = { diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index f8459cc5315d..9b126a260267 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -28,9 +28,10 @@ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> +#include <linux/of.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/fsldma.h> #include "dmaengine.h" #include "fsldma.h" @@ -1225,6 +1226,8 @@ static int fsldma_of_probe(struct platform_device *op) fdev->dev = &op->dev; INIT_LIST_HEAD(&fdev->common.channels); + /* The DMA address bits supported for this device. */ + fdev->addr_bits = (long)device_get_match_data(fdev->dev); /* ioremap the registers for use */ fdev->regs = of_iomap(op->dev.of_node, 0); @@ -1253,7 +1256,7 @@ static int fsldma_of_probe(struct platform_device *op) fdev->common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); fdev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - dma_set_mask(&(op->dev), DMA_BIT_MASK(36)); + dma_set_mask(&(op->dev), DMA_BIT_MASK(fdev->addr_bits)); platform_set_drvdata(op, fdev); @@ -1305,7 +1308,7 @@ out_return: return err; } -static int fsldma_of_remove(struct platform_device *op) +static void fsldma_of_remove(struct platform_device *op) { struct fsldma_device *fdev; unsigned int i; @@ -1323,8 +1326,6 @@ static int fsldma_of_remove(struct platform_device *op) iounmap(fdev->regs); kfree(fdev); - - return 0; } #ifdef CONFIG_PM @@ -1388,10 +1389,20 @@ static const struct dev_pm_ops fsldma_pm_ops = { }; #endif +/* The .data field is used for dma-bit-mask. */ static const struct of_device_id fsldma_of_ids[] = { - { .compatible = "fsl,elo3-dma", }, - { .compatible = "fsl,eloplus-dma", }, - { .compatible = "fsl,elo-dma", }, + { + .compatible = "fsl,elo3-dma", + .data = (void *)40, + }, + { + .compatible = "fsl,eloplus-dma", + .data = (void *)36, + }, + { + .compatible = "fsl,elo-dma", + .data = (void *)32, + }, {} }; MODULE_DEVICE_TABLE(of, fsldma_of_ids); diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h index 308bed0a560a..d7b7a3138b85 100644 --- a/drivers/dma/fsldma.h +++ b/drivers/dma/fsldma.h @@ -124,6 +124,7 @@ struct fsldma_device { struct fsldma_chan *chan[FSL_DMA_MAX_CHANS_PER_DEVICE]; u32 feature; /* The same as DMA channels */ int irq; /* Channel IRQ */ + int addr_bits; /* DMA addressing bits supported */ }; /* Define macros for fsldma_chan->feature property */ diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index a259ee010e9b..25a4134be36b 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2019 HiSilicon Limited. */ +/* Copyright(c) 2019-2022 HiSilicon Limited. */ + #include <linux/bitfield.h> #include <linux/dmaengine.h> #include <linux/init.h> @@ -9,32 +10,87 @@ #include <linux/spinlock.h> #include "virt-dma.h" -#define HISI_DMA_SQ_BASE_L 0x0 -#define HISI_DMA_SQ_BASE_H 0x4 -#define HISI_DMA_SQ_DEPTH 0x8 -#define HISI_DMA_SQ_TAIL_PTR 0xc -#define HISI_DMA_CQ_BASE_L 0x10 -#define HISI_DMA_CQ_BASE_H 0x14 -#define HISI_DMA_CQ_DEPTH 0x18 -#define HISI_DMA_CQ_HEAD_PTR 0x1c -#define HISI_DMA_CTRL0 0x20 -#define HISI_DMA_CTRL0_QUEUE_EN_S 0 -#define HISI_DMA_CTRL0_QUEUE_PAUSE_S 4 -#define HISI_DMA_CTRL1 0x24 -#define HISI_DMA_CTRL1_QUEUE_RESET_S 0 -#define HISI_DMA_Q_FSM_STS 0x30 -#define HISI_DMA_FSM_STS_MASK GENMASK(3, 0) -#define HISI_DMA_INT_STS 0x40 -#define HISI_DMA_INT_STS_MASK GENMASK(12, 0) -#define HISI_DMA_INT_MSK 0x44 -#define HISI_DMA_MODE 0x217c -#define HISI_DMA_OFFSET 0x100 - -#define HISI_DMA_MSI_NUM 30 -#define HISI_DMA_CHAN_NUM 30 -#define HISI_DMA_Q_DEPTH_VAL 1024 - -#define PCI_BAR_2 2 +/* HiSilicon DMA register common field define */ +#define HISI_DMA_Q_SQ_BASE_L 0x0 +#define HISI_DMA_Q_SQ_BASE_H 0x4 +#define HISI_DMA_Q_SQ_DEPTH 0x8 +#define HISI_DMA_Q_SQ_TAIL_PTR 0xc +#define HISI_DMA_Q_CQ_BASE_L 0x10 +#define HISI_DMA_Q_CQ_BASE_H 0x14 +#define HISI_DMA_Q_CQ_DEPTH 0x18 +#define HISI_DMA_Q_CQ_HEAD_PTR 0x1c +#define HISI_DMA_Q_CTRL0 0x20 +#define HISI_DMA_Q_CTRL0_QUEUE_EN BIT(0) +#define HISI_DMA_Q_CTRL0_QUEUE_PAUSE BIT(4) +#define HISI_DMA_Q_CTRL1 0x24 +#define HISI_DMA_Q_CTRL1_QUEUE_RESET BIT(0) +#define HISI_DMA_Q_FSM_STS 0x30 +#define HISI_DMA_Q_FSM_STS_MASK GENMASK(3, 0) +#define HISI_DMA_Q_ERR_INT_NUM0 0x84 +#define HISI_DMA_Q_ERR_INT_NUM1 0x88 +#define HISI_DMA_Q_ERR_INT_NUM2 0x8c + +/* HiSilicon IP08 DMA register and field define */ +#define HISI_DMA_HIP08_MODE 0x217C +#define HISI_DMA_HIP08_Q_BASE 0x0 +#define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN BIT(2) +#define HISI_DMA_HIP08_Q_INT_STS 0x40 +#define HISI_DMA_HIP08_Q_INT_MSK 0x44 +#define HISI_DMA_HIP08_Q_INT_STS_MASK GENMASK(14, 0) +#define HISI_DMA_HIP08_Q_ERR_INT_NUM3 0x90 +#define HISI_DMA_HIP08_Q_ERR_INT_NUM4 0x94 +#define HISI_DMA_HIP08_Q_ERR_INT_NUM5 0x98 +#define HISI_DMA_HIP08_Q_ERR_INT_NUM6 0x48 +#define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT BIT(24) + +/* HiSilicon IP09 DMA register and field define */ +#define HISI_DMA_HIP09_DMA_FLR_DISABLE 0xA00 +#define HISI_DMA_HIP09_DMA_FLR_DISABLE_B BIT(0) +#define HISI_DMA_HIP09_Q_BASE 0x2000 +#define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN GENMASK(31, 28) +#define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT BIT(26) +#define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT BIT(27) +#define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE BIT(2) +#define HISI_DMA_HIP09_Q_INT_STS 0x40 +#define HISI_DMA_HIP09_Q_INT_MSK 0x44 +#define HISI_DMA_HIP09_Q_INT_STS_MASK 0x1 +#define HISI_DMA_HIP09_Q_ERR_INT_STS 0x48 +#define HISI_DMA_HIP09_Q_ERR_INT_MSK 0x4C +#define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK GENMASK(18, 1) +#define HISI_DMA_HIP09_PORT_CFG_REG(port_id) (0x800 + \ + (port_id) * 0x20) +#define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B BIT(16) + +#define HISI_DMA_HIP09_MAX_PORT_NUM 16 + +#define HISI_DMA_HIP08_MSI_NUM 32 +#define HISI_DMA_HIP08_CHAN_NUM 30 +#define HISI_DMA_HIP09_MSI_NUM 4 +#define HISI_DMA_HIP09_CHAN_NUM 4 +#define HISI_DMA_REVISION_HIP08B 0x21 +#define HISI_DMA_REVISION_HIP09A 0x30 + +#define HISI_DMA_Q_OFFSET 0x100 +#define HISI_DMA_Q_DEPTH_VAL 1024 + +#define PCI_BAR_2 2 + +#define HISI_DMA_POLL_Q_STS_DELAY_US 10 +#define HISI_DMA_POLL_Q_STS_TIME_OUT_US 1000 + +#define HISI_DMA_MAX_DIR_NAME_LEN 128 + +/* + * The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they + * have the same pci device id but different pci revision. + * Unfortunately, they have different register layouts, so two layout + * enumerations are defined. + */ +enum hisi_dma_reg_layout { + HISI_DMA_REG_LAYOUT_INVALID = 0, + HISI_DMA_REG_LAYOUT_HIP08, + HISI_DMA_REG_LAYOUT_HIP09 +}; enum hisi_dma_mode { EP = 0, @@ -105,9 +161,162 @@ struct hisi_dma_dev { struct dma_device dma_dev; u32 chan_num; u32 chan_depth; - struct hisi_dma_chan chan[]; + enum hisi_dma_reg_layout reg_layout; + void __iomem *queue_base; /* queue region start of register */ + struct hisi_dma_chan chan[] __counted_by(chan_num); }; +#ifdef CONFIG_DEBUG_FS + +static const struct debugfs_reg32 hisi_dma_comm_chan_regs[] = { + {"DMA_QUEUE_SQ_DEPTH ", 0x0008ull}, + {"DMA_QUEUE_SQ_TAIL_PTR ", 0x000Cull}, + {"DMA_QUEUE_CQ_DEPTH ", 0x0018ull}, + {"DMA_QUEUE_CQ_HEAD_PTR ", 0x001Cull}, + {"DMA_QUEUE_CTRL0 ", 0x0020ull}, + {"DMA_QUEUE_CTRL1 ", 0x0024ull}, + {"DMA_QUEUE_FSM_STS ", 0x0030ull}, + {"DMA_QUEUE_SQ_STS ", 0x0034ull}, + {"DMA_QUEUE_CQ_TAIL_PTR ", 0x003Cull}, + {"DMA_QUEUE_INT_STS ", 0x0040ull}, + {"DMA_QUEUE_INT_MSK ", 0x0044ull}, + {"DMA_QUEUE_INT_RO ", 0x006Cull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip08_chan_regs[] = { + {"DMA_QUEUE_BYTE_CNT ", 0x0038ull}, + {"DMA_ERR_INT_NUM6 ", 0x0048ull}, + {"DMA_QUEUE_DESP0 ", 0x0050ull}, + {"DMA_QUEUE_DESP1 ", 0x0054ull}, + {"DMA_QUEUE_DESP2 ", 0x0058ull}, + {"DMA_QUEUE_DESP3 ", 0x005Cull}, + {"DMA_QUEUE_DESP4 ", 0x0074ull}, + {"DMA_QUEUE_DESP5 ", 0x0078ull}, + {"DMA_QUEUE_DESP6 ", 0x007Cull}, + {"DMA_QUEUE_DESP7 ", 0x0080ull}, + {"DMA_ERR_INT_NUM0 ", 0x0084ull}, + {"DMA_ERR_INT_NUM1 ", 0x0088ull}, + {"DMA_ERR_INT_NUM2 ", 0x008Cull}, + {"DMA_ERR_INT_NUM3 ", 0x0090ull}, + {"DMA_ERR_INT_NUM4 ", 0x0094ull}, + {"DMA_ERR_INT_NUM5 ", 0x0098ull}, + {"DMA_QUEUE_SQ_STS2 ", 0x00A4ull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip09_chan_regs[] = { + {"DMA_QUEUE_ERR_INT_STS ", 0x0048ull}, + {"DMA_QUEUE_ERR_INT_MSK ", 0x004Cull}, + {"DFX_SQ_READ_ERR_PTR ", 0x0068ull}, + {"DFX_DMA_ERR_INT_NUM0 ", 0x0084ull}, + {"DFX_DMA_ERR_INT_NUM1 ", 0x0088ull}, + {"DFX_DMA_ERR_INT_NUM2 ", 0x008Cull}, + {"DFX_DMA_QUEUE_SQ_STS2 ", 0x00A4ull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip08_comm_regs[] = { + {"DMA_ECC_ERR_ADDR ", 0x2004ull}, + {"DMA_ECC_ECC_CNT ", 0x2014ull}, + {"COMMON_AND_CH_ERR_STS ", 0x2030ull}, + {"LOCAL_CPL_ID_STS_0 ", 0x20E0ull}, + {"LOCAL_CPL_ID_STS_1 ", 0x20E4ull}, + {"LOCAL_CPL_ID_STS_2 ", 0x20E8ull}, + {"LOCAL_CPL_ID_STS_3 ", 0x20ECull}, + {"LOCAL_TLP_NUM ", 0x2158ull}, + {"SQCQ_TLP_NUM ", 0x2164ull}, + {"CPL_NUM ", 0x2168ull}, + {"INF_BACK_PRESS_STS ", 0x2170ull}, + {"DMA_CH_RAS_LEVEL ", 0x2184ull}, + {"DMA_CM_RAS_LEVEL ", 0x2188ull}, + {"DMA_CH_ERR_STS ", 0x2190ull}, + {"DMA_CH_DONE_STS ", 0x2194ull}, + {"DMA_SQ_TAG_STS_0 ", 0x21A0ull}, + {"DMA_SQ_TAG_STS_1 ", 0x21A4ull}, + {"DMA_SQ_TAG_STS_2 ", 0x21A8ull}, + {"DMA_SQ_TAG_STS_3 ", 0x21ACull}, + {"LOCAL_P_ID_STS_0 ", 0x21B0ull}, + {"LOCAL_P_ID_STS_1 ", 0x21B4ull}, + {"LOCAL_P_ID_STS_2 ", 0x21B8ull}, + {"LOCAL_P_ID_STS_3 ", 0x21BCull}, + {"DMA_PREBUFF_INFO_0 ", 0x2200ull}, + {"DMA_CM_TABLE_INFO_0 ", 0x2220ull}, + {"DMA_CM_CE_RO ", 0x2244ull}, + {"DMA_CM_NFE_RO ", 0x2248ull}, + {"DMA_CM_FE_RO ", 0x224Cull}, +}; + +static const struct debugfs_reg32 hisi_dma_hip09_comm_regs[] = { + {"COMMON_AND_CH_ERR_STS ", 0x0030ull}, + {"DMA_PORT_IDLE_STS ", 0x0150ull}, + {"DMA_CH_RAS_LEVEL ", 0x0184ull}, + {"DMA_CM_RAS_LEVEL ", 0x0188ull}, + {"DMA_CM_CE_RO ", 0x0244ull}, + {"DMA_CM_NFE_RO ", 0x0248ull}, + {"DMA_CM_FE_RO ", 0x024Cull}, + {"DFX_INF_BACK_PRESS_STS0 ", 0x1A40ull}, + {"DFX_INF_BACK_PRESS_STS1 ", 0x1A44ull}, + {"DFX_INF_BACK_PRESS_STS2 ", 0x1A48ull}, + {"DFX_DMA_WRR_DISABLE ", 0x1A4Cull}, + {"DFX_PA_REQ_TLP_NUM ", 0x1C00ull}, + {"DFX_PA_BACK_TLP_NUM ", 0x1C04ull}, + {"DFX_PA_RETRY_TLP_NUM ", 0x1C08ull}, + {"DFX_LOCAL_NP_TLP_NUM ", 0x1C0Cull}, + {"DFX_LOCAL_CPL_HEAD_TLP_NUM ", 0x1C10ull}, + {"DFX_LOCAL_CPL_DATA_TLP_NUM ", 0x1C14ull}, + {"DFX_LOCAL_CPL_EXT_DATA_TLP_NUM ", 0x1C18ull}, + {"DFX_LOCAL_P_HEAD_TLP_NUM ", 0x1C1Cull}, + {"DFX_LOCAL_P_ACK_TLP_NUM ", 0x1C20ull}, + {"DFX_BUF_ALOC_PORT_REQ_NUM ", 0x1C24ull}, + {"DFX_BUF_ALOC_PORT_RESULT_NUM ", 0x1C28ull}, + {"DFX_BUF_FAIL_SIZE_NUM ", 0x1C2Cull}, + {"DFX_BUF_ALOC_SIZE_NUM ", 0x1C30ull}, + {"DFX_BUF_NP_RELEASE_SIZE_NUM ", 0x1C34ull}, + {"DFX_BUF_P_RELEASE_SIZE_NUM ", 0x1C38ull}, + {"DFX_BUF_PORT_RELEASE_SIZE_NUM ", 0x1C3Cull}, + {"DFX_DMA_PREBUF_MEM0_ECC_ERR_ADDR ", 0x1CA8ull}, + {"DFX_DMA_PREBUF_MEM0_ECC_CNT ", 0x1CACull}, + {"DFX_DMA_LOC_NP_OSTB_ECC_ERR_ADDR ", 0x1CB0ull}, + {"DFX_DMA_LOC_NP_OSTB_ECC_CNT ", 0x1CB4ull}, + {"DFX_DMA_PREBUF_MEM1_ECC_ERR_ADDR ", 0x1CC0ull}, + {"DFX_DMA_PREBUF_MEM1_ECC_CNT ", 0x1CC4ull}, + {"DMA_CH_DONE_STS ", 0x02E0ull}, + {"DMA_CH_ERR_STS ", 0x0320ull}, +}; +#endif /* CONFIG_DEBUG_FS*/ + +static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_REG_LAYOUT_HIP08; + else if (pdev->revision >= HISI_DMA_REVISION_HIP09A) + return HISI_DMA_REG_LAYOUT_HIP09; + + return HISI_DMA_REG_LAYOUT_INVALID; +} + +static u32 hisi_dma_get_chan_num(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_HIP08_CHAN_NUM; + + return HISI_DMA_HIP09_CHAN_NUM; +} + +static u32 hisi_dma_get_msi_num(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_HIP08_MSI_NUM; + + return HISI_DMA_HIP09_MSI_NUM; +} + +static u32 hisi_dma_get_queue_base(struct pci_dev *pdev) +{ + if (pdev->revision == HISI_DMA_REVISION_HIP08B) + return HISI_DMA_HIP08_Q_BASE; + + return HISI_DMA_HIP09_Q_BASE; +} + static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c) { return container_of(c, struct hisi_dma_chan, vc.chan); @@ -121,7 +330,7 @@ static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd) static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index, u32 val) { - writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET); + writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET); } static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val) @@ -129,75 +338,103 @@ static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val) u32 tmp; tmp = readl_relaxed(addr); - tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos); + tmp = val ? tmp | pos : tmp & ~pos; writel_relaxed(tmp, addr); } -static void hisi_dma_free_irq_vectors(void *data) -{ - pci_free_irq_vectors(data); -} - static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index, bool pause) { - void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index * - HISI_DMA_OFFSET; + void __iomem *addr; - hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause); + addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 + + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause); } static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index, bool enable) { - void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index * - HISI_DMA_OFFSET; + void __iomem *addr; - hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable); + addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 + + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable); } static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) { - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index, - HISI_DMA_INT_STS_MASK); + void __iomem *q_base = hdma_dev->queue_base; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK, + qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK); + else { + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK, + qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK, + qp_index, + HISI_DMA_HIP09_Q_ERR_INT_STS_MASK); + } } static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index) { - void __iomem *base = hdma_dev->base; - - hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index, - HISI_DMA_INT_STS_MASK); - hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0); + void __iomem *q_base = hdma_dev->queue_base; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS, + qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK, + qp_index, 0); + } else { + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS, + qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS, + qp_index, + HISI_DMA_HIP09_Q_ERR_INT_STS_MASK); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK, + qp_index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK, + qp_index, 0); + } } static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index) { - void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index * - HISI_DMA_OFFSET; + void __iomem *addr; - hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1); + addr = hdma_dev->queue_base + + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1); } static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index) { - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0); - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0); + void __iomem *q_base = hdma_dev->queue_base; + + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0); } -static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan) +static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan, + bool disable) { struct hisi_dma_dev *hdma_dev = chan->hdma_dev; u32 index = chan->qp_num, tmp; + void __iomem *addr; int ret; hisi_dma_pause_dma(hdma_dev, index, true); hisi_dma_enable_dma(hdma_dev, index, false); hisi_dma_mask_irq(hdma_dev, index); - ret = readl_relaxed_poll_timeout(hdma_dev->base + - HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp, - FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN, 10, 1000); + addr = hdma_dev->queue_base + + HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET; + + ret = readl_relaxed_poll_timeout(addr, tmp, + FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN, + HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US); if (ret) { dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n"); WARN_ON(1); @@ -206,12 +443,15 @@ static void hisi_dma_reset_hw_chan(struct hisi_dma_chan *chan) hisi_dma_do_reset(hdma_dev, index); hisi_dma_reset_qp_point(hdma_dev, index); hisi_dma_pause_dma(hdma_dev, index, false); - hisi_dma_enable_dma(hdma_dev, index, true); - hisi_dma_unmask_irq(hdma_dev, index); - ret = readl_relaxed_poll_timeout(hdma_dev->base + - HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET, tmp, - FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE, 10, 1000); + if (!disable) { + hisi_dma_enable_dma(hdma_dev, index, true); + hisi_dma_unmask_irq(hdma_dev, index); + } + + ret = readl_relaxed_poll_timeout(addr, tmp, + FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE, + HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US); if (ret) { dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n"); WARN_ON(1); @@ -223,7 +463,7 @@ static void hisi_dma_free_chan_resources(struct dma_chan *c) struct hisi_dma_chan *chan = to_hisi_dma_chan(c); struct hisi_dma_dev *hdma_dev = chan->hdma_dev; - hisi_dma_reset_hw_chan(chan); + hisi_dma_reset_or_disable_hw_chan(chan, false); vchan_free_chan_resources(&chan->vc); memset(chan->sq, 0, sizeof(struct hisi_dma_sqe) * hdma_dev->chan_depth); @@ -272,7 +512,6 @@ static void hisi_dma_start_transfer(struct hisi_dma_chan *chan) vd = vchan_next_desc(&chan->vc); if (!vd) { - dev_err(&hdma_dev->pdev->dev, "no issued task!\n"); chan->desc = NULL; return; } @@ -293,8 +532,8 @@ static void hisi_dma_start_transfer(struct hisi_dma_chan *chan) chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth; /* update sq_tail to trigger a new task */ - hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num, - chan->sq_tail); + hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR, + chan->qp_num, chan->sq_tail); } static void hisi_dma_issue_pending(struct dma_chan *c) @@ -304,7 +543,7 @@ static void hisi_dma_issue_pending(struct dma_chan *c) spin_lock_irqsave(&chan->vc.lock, flags); - if (vchan_issue_pending(&chan->vc)) + if (vchan_issue_pending(&chan->vc) && !chan->desc) hisi_dma_start_transfer(chan); spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -368,26 +607,86 @@ static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev) static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index) { struct hisi_dma_chan *chan = &hdma_dev->chan[index]; + void __iomem *q_base = hdma_dev->queue_base; u32 hw_depth = hdma_dev->chan_depth - 1; - void __iomem *base = hdma_dev->base; + void __iomem *addr; + u32 tmp; /* set sq, cq base */ - hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index, lower_32_bits(chan->sq_dma)); - hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index, upper_32_bits(chan->sq_dma)); - hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index, lower_32_bits(chan->cq_dma)); - hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index, + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index, upper_32_bits(chan->cq_dma)); /* set sq, cq depth */ - hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth); - hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth); + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth); + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth); /* init sq tail and cq head */ - hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0); - hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0); + + /* init error interrupt stats */ + hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0); + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3, + index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4, + index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5, + index, 0); + hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6, + index, 0); + /* + * init SQ/CQ direction selecting register. + * "0" is to local side and "1" is to remote side. + */ + addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0); + + /* + * 0 - Continue to next descriptor if error occurs. + * 1 - Abort the DMA queue if error occurs. + */ + hisi_dma_update_bit(addr, + HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0); + } else { + addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET; + + /* + * init SQ/CQ direction selecting register. + * "0" is to local side and "1" is to remote side. + */ + hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0); + hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0); + + /* + * 0 - Continue to next descriptor if error occurs. + * 1 - Abort the DMA queue if error occurs. + */ + + tmp = readl_relaxed(addr); + tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN; + writel_relaxed(tmp, addr); + + /* + * 0 - dma should process FLR with CPU. + * 1 - dma not process FLR, only cpu process FLR. + */ + addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE + + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0); + + addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET; + hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1); + } } static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) @@ -399,7 +698,7 @@ static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) static void hisi_dma_disable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index) { - hisi_dma_reset_hw_chan(&hdma_dev->chan[qp_index]); + hisi_dma_reset_or_disable_hw_chan(&hdma_dev->chan[qp_index], true); } static void hisi_dma_enable_qps(struct hisi_dma_dev *hdma_dev) @@ -431,24 +730,23 @@ static irqreturn_t hisi_dma_irq(int irq, void *data) struct hisi_dma_dev *hdma_dev = chan->hdma_dev; struct hisi_dma_desc *desc; struct hisi_dma_cqe *cqe; + void __iomem *q_base; spin_lock(&chan->vc.lock); desc = chan->desc; cqe = chan->cq + chan->cq_head; + q_base = hdma_dev->queue_base; if (desc) { + chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth; + hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, + chan->qp_num, chan->cq_head); if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) { - chan->cq_head = (chan->cq_head + 1) % - hdma_dev->chan_depth; - hisi_dma_chan_write(hdma_dev->base, - HISI_DMA_CQ_HEAD_PTR, chan->qp_num, - chan->cq_head); vchan_cookie_complete(&desc->vd); + hisi_dma_start_transfer(chan); } else { dev_err(&hdma_dev->pdev->dev, "task error!\n"); } - - chan->desc = NULL; } spin_unlock(&chan->vc.lock); @@ -502,16 +800,169 @@ static void hisi_dma_disable_hw_channels(void *data) static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev, enum hisi_dma_mode mode) { - writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE); + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + writel_relaxed(mode == RC ? 1 : 0, + hdma_dev->base + HISI_DMA_HIP08_MODE); +} + +static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev) +{ + void __iomem *addr; + int i; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) { + for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) { + addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i); + hisi_dma_update_bit(addr, + HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1); + } + } +} + +static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev) +{ + struct dma_device *dma_dev; + + dma_dev = &hdma_dev->dma_dev; + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources; + dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy; + dma_dev->device_tx_status = hisi_dma_tx_status; + dma_dev->device_issue_pending = hisi_dma_issue_pending; + dma_dev->device_terminate_all = hisi_dma_terminate_all; + dma_dev->device_synchronize = hisi_dma_synchronize; + dma_dev->directions = BIT(DMA_MEM_TO_MEM); + dma_dev->dev = &hdma_dev->pdev->dev; + INIT_LIST_HEAD(&dma_dev->channels); +} + +/* --- debugfs implementation --- */ +#ifdef CONFIG_DEBUG_FS +#include <linux/debugfs.h> +static struct debugfs_reg32 *hisi_dma_get_ch_regs(struct hisi_dma_dev *hdma_dev, + u32 *regs_sz) +{ + struct device *dev = &hdma_dev->pdev->dev; + struct debugfs_reg32 *regs; + u32 regs_sz_comm; + + regs_sz_comm = ARRAY_SIZE(hisi_dma_comm_chan_regs); + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip08_chan_regs); + else + *regs_sz = regs_sz_comm + ARRAY_SIZE(hisi_dma_hip09_chan_regs); + + regs = devm_kcalloc(dev, *regs_sz, sizeof(struct debugfs_reg32), + GFP_KERNEL); + if (!regs) + return NULL; + memcpy(regs, hisi_dma_comm_chan_regs, sizeof(hisi_dma_comm_chan_regs)); + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) + memcpy(regs + regs_sz_comm, hisi_dma_hip08_chan_regs, + sizeof(hisi_dma_hip08_chan_regs)); + else + memcpy(regs + regs_sz_comm, hisi_dma_hip09_chan_regs, + sizeof(hisi_dma_hip09_chan_regs)); + + return regs; +} + +static int hisi_dma_create_chan_dir(struct hisi_dma_dev *hdma_dev) +{ + char dir_name[HISI_DMA_MAX_DIR_NAME_LEN]; + struct debugfs_regset32 *regsets; + struct debugfs_reg32 *regs; + struct dentry *chan_dir; + struct device *dev; + u32 regs_sz; + int ret; + int i; + + dev = &hdma_dev->pdev->dev; + + regsets = devm_kcalloc(dev, hdma_dev->chan_num, + sizeof(*regsets), GFP_KERNEL); + if (!regsets) + return -ENOMEM; + + regs = hisi_dma_get_ch_regs(hdma_dev, ®s_sz); + if (!regs) + return -ENOMEM; + + for (i = 0; i < hdma_dev->chan_num; i++) { + regsets[i].regs = regs; + regsets[i].nregs = regs_sz; + regsets[i].base = hdma_dev->queue_base + i * HISI_DMA_Q_OFFSET; + regsets[i].dev = dev; + + memset(dir_name, 0, HISI_DMA_MAX_DIR_NAME_LEN); + ret = sprintf(dir_name, "channel%d", i); + if (ret < 0) + return ret; + + chan_dir = debugfs_create_dir(dir_name, + hdma_dev->dma_dev.dbg_dev_root); + debugfs_create_regset32("regs", 0444, chan_dir, ®sets[i]); + } + + return 0; +} + +static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) +{ + struct debugfs_regset32 *regset; + struct device *dev; + int ret; + + dev = &hdma_dev->pdev->dev; + + if (hdma_dev->dma_dev.dbg_dev_root == NULL) + return; + + regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL); + if (!regset) + return; + + if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) { + regset->regs = hisi_dma_hip08_comm_regs; + regset->nregs = ARRAY_SIZE(hisi_dma_hip08_comm_regs); + } else { + regset->regs = hisi_dma_hip09_comm_regs; + regset->nregs = ARRAY_SIZE(hisi_dma_hip09_comm_regs); + } + regset->base = hdma_dev->base; + regset->dev = dev; + + debugfs_create_regset32("regs", 0444, + hdma_dev->dma_dev.dbg_dev_root, regset); + + ret = hisi_dma_create_chan_dir(hdma_dev); + if (ret < 0) + dev_info(&hdma_dev->pdev->dev, "fail to create debugfs for channels!\n"); } +#else +static void hisi_dma_create_debugfs(struct hisi_dma_dev *hdma_dev) { } +#endif /* CONFIG_DEBUG_FS*/ +/* --- debugfs implementation --- */ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + enum hisi_dma_reg_layout reg_layout; struct device *dev = &pdev->dev; struct hisi_dma_dev *hdma_dev; struct dma_device *dma_dev; + u32 chan_num; + u32 msi_num; int ret; + reg_layout = hisi_dma_get_reg_layout(pdev); + if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) { + dev_err(dev, "unsupported device!\n"); + return -EINVAL; + } + ret = pcim_enable_device(pdev); if (ret) { dev_err(dev, "failed to enable device mem!\n"); @@ -524,51 +975,41 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) return ret; } - ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (ret) return ret; - ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (ret) - return ret; - - hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL); + chan_num = hisi_dma_get_chan_num(pdev); + hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), + GFP_KERNEL); if (!hdma_dev) return -EINVAL; hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; hdma_dev->pdev = pdev; - hdma_dev->chan_num = HISI_DMA_CHAN_NUM; hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL; + hdma_dev->chan_num = chan_num; + hdma_dev->reg_layout = reg_layout; + hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev); pci_set_drvdata(pdev, hdma_dev); pci_set_master(pdev); - ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM, - PCI_IRQ_MSI); + msi_num = hisi_dma_get_msi_num(pdev); + + /* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */ + ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI); if (ret < 0) { dev_err(dev, "Failed to allocate MSI vectors!\n"); return ret; } - ret = devm_add_action_or_reset(dev, hisi_dma_free_irq_vectors, pdev); - if (ret) - return ret; - - dma_dev = &hdma_dev->dma_dev; - dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); - dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources; - dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy; - dma_dev->device_tx_status = hisi_dma_tx_status; - dma_dev->device_issue_pending = hisi_dma_issue_pending; - dma_dev->device_terminate_all = hisi_dma_terminate_all; - dma_dev->device_synchronize = hisi_dma_synchronize; - dma_dev->directions = BIT(DMA_MEM_TO_MEM); - dma_dev->dev = dev; - INIT_LIST_HEAD(&dma_dev->channels); + hisi_dma_init_dma_dev(hdma_dev); hisi_dma_set_mode(hdma_dev, RC); + hisi_dma_init_hw(hdma_dev); + ret = hisi_dma_enable_hw_channels(hdma_dev); if (ret < 0) { dev_err(dev, "failed to enable hw channel!\n"); @@ -580,11 +1021,16 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) return ret; + dma_dev = &hdma_dev->dma_dev; ret = dmaenginem_async_device_register(dma_dev); - if (ret < 0) + if (ret < 0) { dev_err(dev, "failed to register device!\n"); + return ret; + } - return ret; + hisi_dma_create_debugfs(hdma_dev); + + return 0; } static const struct pci_device_id hisi_dma_pci_tbl[] = { diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c index 92caae55aece..af5a2e252c25 100644 --- a/drivers/dma/hsu/hsu.c +++ b/drivers/dma/hsu/hsu.c @@ -16,12 +16,20 @@ * port 3, and so on. */ +#include <linux/bits.h> #include <linux/delay.h> +#include <linux/device.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/list.h> #include <linux/module.h> +#include <linux/percpu-defs.h> +#include <linux/scatterlist.h> #include <linux/slab.h> +#include <linux/string.h> +#include <linux/spinlock.h> #include "hsu.h" diff --git a/drivers/dma/hsu/hsu.h b/drivers/dma/hsu/hsu.h index 9e5956345748..3bca577b98a1 100644 --- a/drivers/dma/hsu/hsu.h +++ b/drivers/dma/hsu/hsu.h @@ -10,7 +10,11 @@ #ifndef __DMA_HSU_H__ #define __DMA_HSU_H__ -#include <linux/spinlock.h> +#include <linux/bits.h> +#include <linux/container_of.h> +#include <linux/io.h> +#include <linux/types.h> + #include <linux/dma/hsu.h> #include "../virt-dma.h" @@ -36,11 +40,11 @@ /* Bits in HSU_CH_SR */ #define HSU_CH_SR_DESCTO(x) BIT(8 + (x)) -#define HSU_CH_SR_DESCTO_ANY (BIT(11) | BIT(10) | BIT(9) | BIT(8)) +#define HSU_CH_SR_DESCTO_ANY GENMASK(11, 8) #define HSU_CH_SR_CHE BIT(15) #define HSU_CH_SR_DESCE(x) BIT(16 + (x)) -#define HSU_CH_SR_DESCE_ANY (BIT(19) | BIT(18) | BIT(17) | BIT(16)) -#define HSU_CH_SR_CDESC_ANY (BIT(31) | BIT(30)) +#define HSU_CH_SR_DESCE_ANY GENMASK(19, 16) +#define HSU_CH_SR_CDESC_ANY GENMASK(31, 30) /* Bits in HSU_CH_CR */ #define HSU_CH_CR_CHA BIT(0) diff --git a/drivers/dma/hsu/pci.c b/drivers/dma/hsu/pci.c index 9045a6f7f589..0fcc0c0c22fc 100644 --- a/drivers/dma/hsu/pci.c +++ b/drivers/dma/hsu/pci.c @@ -10,6 +10,7 @@ #include <linux/bitops.h> #include <linux/device.h> +#include <linux/interrupt.h> #include <linux/module.h> #include <linux/pci.h> @@ -26,29 +27,32 @@ static irqreturn_t hsu_pci_irq(int irq, void *dev) { struct hsu_dma_chip *chip = dev; - u32 dmaisr; - u32 status; + unsigned long dmaisr; unsigned short i; + u32 status; int ret = 0; int err; dmaisr = readl(chip->regs + HSU_PCI_DMAISR); - for (i = 0; i < chip->hsu->nr_channels; i++) { - if (dmaisr & 0x1) { - err = hsu_dma_get_status(chip, i, &status); - if (err > 0) - ret |= 1; - else if (err == 0) - ret |= hsu_dma_do_irq(chip, i, status); - } - dmaisr >>= 1; + for_each_set_bit(i, &dmaisr, chip->hsu->nr_channels) { + err = hsu_dma_get_status(chip, i, &status); + if (err > 0) + ret |= 1; + else if (err == 0) + ret |= hsu_dma_do_irq(chip, i, status); } return IRQ_RETVAL(ret); } +static void hsu_pci_dma_remove(void *chip) +{ + hsu_dma_remove(chip); +} + static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { + struct device *dev = &pdev->dev; struct hsu_dma_chip *chip; int ret; @@ -65,11 +69,7 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_master(pdev); pci_try_set_mwi(pdev); - ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (ret) - return ret; - - ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) return ret; @@ -91,9 +91,13 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) return ret; - ret = request_irq(chip->irq, hsu_pci_irq, 0, "hsu_dma_pci", chip); + ret = devm_add_action_or_reset(dev, hsu_pci_dma_remove, chip); + if (ret) + return ret; + + ret = devm_request_irq(dev, chip->irq, hsu_pci_irq, 0, "hsu_dma_pci", chip); if (ret) - goto err_register_irq; + return ret; /* * On Intel Tangier B0 and Anniedale the interrupt line, disregarding @@ -109,18 +113,6 @@ static int hsu_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_drvdata(pdev, chip); return 0; - -err_register_irq: - hsu_dma_remove(chip); - return ret; -} - -static void hsu_pci_remove(struct pci_dev *pdev) -{ - struct hsu_dma_chip *chip = pci_get_drvdata(pdev); - - free_irq(chip->irq, chip); - hsu_dma_remove(chip); } static const struct pci_device_id hsu_pci_id_table[] = { @@ -134,7 +126,6 @@ static struct pci_driver hsu_pci_driver = { .name = "hsu_dma_pci", .id_table = hsu_pci_id_table, .probe = hsu_pci_probe, - .remove = hsu_pci_remove, }; module_pci_driver(hsu_pci_driver); diff --git a/drivers/dma/idma64.c b/drivers/dma/idma64.c index f4c07ad3be15..d147353d47ab 100644 --- a/drivers/dma/idma64.c +++ b/drivers/dma/idma64.c @@ -137,8 +137,11 @@ static void idma64_chan_irq(struct idma64 *idma64, unsigned short c, u32 status_err, u32 status_xfer) { struct idma64_chan *idma64c = &idma64->chan[c]; + struct dma_chan_percpu *stat; struct idma64_desc *desc; + stat = this_cpu_ptr(idma64c->vchan.chan.local); + spin_lock(&idma64c->vchan.lock); desc = idma64c->desc; if (desc) { @@ -149,6 +152,7 @@ static void idma64_chan_irq(struct idma64 *idma64, unsigned short c, dma_writel(idma64, CLEAR(XFER), idma64c->mask); desc->status = DMA_COMPLETE; vchan_cookie_complete(&desc->vdesc); + stat->bytes_transferred += desc->length; idma64_start_transfer(idma64c); } @@ -167,6 +171,10 @@ static irqreturn_t idma64_irq(int irq, void *dev) u32 status_err; unsigned short i; + /* Since IRQ may be shared, check if DMA controller is powered on */ + if (status == GENMASK(31, 0)) + return IRQ_NONE; + dev_vdbg(idma64->dma.dev, "%s: status=%#x\n", __func__, status); /* Check if we have any interrupt from the DMA controller */ @@ -282,7 +290,7 @@ static void idma64_desc_fill(struct idma64_chan *idma64c, desc->length += hw->len; } while (i); - /* Trigger an interrupt after the last block is transfered */ + /* Trigger an interrupt after the last block is transferred */ lli->ctllo |= IDMA64C_CTLL_INT_EN; /* Disable LLP transfer in the last block */ @@ -356,7 +364,7 @@ static size_t idma64_active_desc_size(struct idma64_chan *idma64c) if (!i) return bytes; - /* The current chunk is not fully transfered yet */ + /* The current chunk is not fully transferred yet */ bytes += desc->hw[--i].len; return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi); @@ -600,7 +608,7 @@ static int idma64_probe(struct idma64_chip *chip) return 0; } -static int idma64_remove(struct idma64_chip *chip) +static void idma64_remove(struct idma64_chip *chip) { struct idma64 *idma64 = chip->idma64; unsigned short i; @@ -618,8 +626,6 @@ static int idma64_remove(struct idma64_chip *chip) tasklet_kill(&idma64c->vchan.task); } - - return 0; } /* ---------------------------------------------------------------------- */ @@ -629,7 +635,6 @@ static int idma64_platform_probe(struct platform_device *pdev) struct idma64_chip *chip; struct device *dev = &pdev->dev; struct device *sysdev = dev->parent; - struct resource *mem; int ret; chip = devm_kzalloc(dev, sizeof(*chip), GFP_KERNEL); @@ -640,8 +645,7 @@ static int idma64_platform_probe(struct platform_device *pdev) if (chip->irq < 0) return chip->irq; - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - chip->regs = devm_ioremap_resource(dev, mem); + chip->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); @@ -660,11 +664,11 @@ static int idma64_platform_probe(struct platform_device *pdev) return 0; } -static int idma64_platform_remove(struct platform_device *pdev) +static void idma64_platform_remove(struct platform_device *pdev) { struct idma64_chip *chip = platform_get_drvdata(pdev); - return idma64_remove(chip); + idma64_remove(chip); } static int __maybe_unused idma64_pm_suspend(struct device *dev) diff --git a/drivers/dma/idxd/Makefile b/drivers/dma/idxd/Makefile index 6d11558756f8..9ff9d7b87b64 100644 --- a/drivers/dma/idxd/Makefile +++ b/drivers/dma/idxd/Makefile @@ -1,4 +1,12 @@ +ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE='"IDXD"' + +obj-$(CONFIG_INTEL_IDXD_BUS) += idxd_bus.o +idxd_bus-y := bus.o + obj-$(CONFIG_INTEL_IDXD) += idxd.o -idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o +idxd-y := init.o irq.o device.o sysfs.o submit.o dma.o cdev.o debugfs.o defaults.o idxd-$(CONFIG_INTEL_IDXD_PERFMON) += perfmon.o + +obj-$(CONFIG_INTEL_IDXD_COMPAT) += idxd_compat.o +idxd_compat-y := compat.o diff --git a/drivers/dma/idxd/bus.c b/drivers/dma/idxd/bus.c new file mode 100644 index 000000000000..e647a684485d --- /dev/null +++ b/drivers/dma/idxd/bus.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/device.h> +#include "idxd.h" + + +int __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *owner, + const char *mod_name) +{ + struct device_driver *drv = &idxd_drv->drv; + + if (!idxd_drv->type) { + pr_debug("driver type not set (%ps)\n", __builtin_return_address(0)); + return -EINVAL; + } + + drv->name = idxd_drv->name; + drv->bus = &dsa_bus_type; + drv->owner = owner; + drv->mod_name = mod_name; + + return driver_register(drv); +} +EXPORT_SYMBOL_GPL(__idxd_driver_register); + +void idxd_driver_unregister(struct idxd_device_driver *idxd_drv) +{ + driver_unregister(&idxd_drv->drv); +} +EXPORT_SYMBOL_GPL(idxd_driver_unregister); + +static int idxd_config_bus_match(struct device *dev, + const struct device_driver *drv) +{ + const struct idxd_device_driver *idxd_drv = + container_of_const(drv, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + int i = 0; + + while (idxd_drv->type[i] != IDXD_DEV_NONE) { + if (idxd_dev->type == idxd_drv->type[i]) + return 1; + i++; + } + + return 0; +} + +static int idxd_config_bus_probe(struct device *dev) +{ + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return idxd_drv->probe(idxd_dev); +} + +static void idxd_config_bus_remove(struct device *dev) +{ + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + idxd_drv->remove(idxd_dev); +} + +static int idxd_bus_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + return add_uevent_var(env, "MODALIAS=" IDXD_DEVICES_MODALIAS_FMT, 0); +} + +const struct bus_type dsa_bus_type = { + .name = "dsa", + .match = idxd_config_bus_match, + .probe = idxd_config_bus_probe, + .remove = idxd_config_bus_remove, + .uevent = idxd_bus_uevent, +}; +EXPORT_SYMBOL_GPL(dsa_bus_type); + +static int __init dsa_bus_init(void) +{ + return bus_register(&dsa_bus_type); +} +module_init(dsa_bus_init); + +static void __exit dsa_bus_exit(void) +{ + bus_unregister(&dsa_bus_type); +} +module_exit(dsa_bus_exit); + +MODULE_DESCRIPTION("IDXD driver dsa_bus_type driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index e9def577c697..7e4715f92773 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -6,13 +6,14 @@ #include <linux/pci.h> #include <linux/device.h> #include <linux/sched/task.h> -#include <linux/intel-svm.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/cdev.h> #include <linux/fs.h> #include <linux/poll.h> #include <linux/iommu.h> +#include <linux/highmem.h> #include <uapi/linux/idxd.h> +#include <linux/xarray.h> #include "registers.h" #include "idxd.h" @@ -23,6 +24,12 @@ struct idxd_cdev_context { }; /* + * Since user file names are global in DSA devices, define their ida's as + * global to avoid conflict file names. + */ +static DEFINE_IDA(file_ida); + +/* * ictx is an array based off of accelerator types. enum idxd_type * is used as index */ @@ -35,22 +42,131 @@ struct idxd_user_context { struct idxd_wq *wq; struct task_struct *task; unsigned int pasid; + struct mm_struct *mm; unsigned int flags; struct iommu_sva *sva; + struct idxd_dev idxd_dev; + u64 counters[COUNTER_MAX]; + int id; + pid_t pid; +}; + +static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid); +static void idxd_xa_pasid_remove(struct idxd_user_context *ctx); + +static inline struct idxd_user_context *dev_to_uctx(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_user_context, idxd_dev); +} + +static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_user_context *ctx = dev_to_uctx(dev); + + return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]); +} +static DEVICE_ATTR_RO(cr_faults); + +static ssize_t cr_fault_failures_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_user_context *ctx = dev_to_uctx(dev); + + return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]); +} +static DEVICE_ATTR_RO(cr_fault_failures); + +static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_user_context *ctx = dev_to_uctx(dev); + + return sysfs_emit(buf, "%u\n", ctx->pid); +} +static DEVICE_ATTR_RO(pid); + +static struct attribute *cdev_file_attributes[] = { + &dev_attr_cr_faults.attr, + &dev_attr_cr_fault_failures.attr, + &dev_attr_pid.attr, + NULL +}; + +static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = container_of(kobj, typeof(*dev), kobj); + struct idxd_user_context *ctx = dev_to_uctx(dev); + struct idxd_wq *wq = ctx->wq; + + if (!wq_pasid_enabled(wq)) + return 0; + + return a->mode; +} + +static const struct attribute_group cdev_file_attribute_group = { + .attrs = cdev_file_attributes, + .is_visible = cdev_file_attr_visible, +}; + +static const struct attribute_group *cdev_file_attribute_groups[] = { + &cdev_file_attribute_group, + NULL +}; + +static void idxd_file_dev_release(struct device *dev) +{ + struct idxd_user_context *ctx = dev_to_uctx(dev); + struct idxd_wq *wq = ctx->wq; + struct idxd_device *idxd = wq->idxd; + int rc; + + ida_free(&file_ida, ctx->id); + + /* Wait for in-flight operations to complete. */ + if (wq_shared(wq)) { + idxd_device_drain_pasid(idxd, ctx->pasid); + } else { + if (device_user_pasid_enabled(idxd)) { + /* The wq disable in the disable pasid function will drain the wq */ + rc = idxd_wq_disable_pasid(wq); + if (rc < 0) + dev_err(dev, "wq disable pasid failed.\n"); + } else { + idxd_wq_drain(wq); + } + } + + if (ctx->sva) { + idxd_cdev_evl_drain_pasid(wq, ctx->pasid); + iommu_sva_unbind_device(ctx->sva); + idxd_xa_pasid_remove(ctx); + } + kfree(ctx); + mutex_lock(&wq->wq_lock); + idxd_wq_put(wq); + mutex_unlock(&wq->wq_lock); +} + +static const struct device_type idxd_cdev_file_type = { + .name = "idxd_file", + .release = idxd_file_dev_release, + .groups = cdev_file_attribute_groups, }; static void idxd_cdev_dev_release(struct device *dev) { - struct idxd_cdev *idxd_cdev = container_of(dev, struct idxd_cdev, dev); + struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); struct idxd_cdev_context *cdev_ctx; struct idxd_wq *wq = idxd_cdev->wq; cdev_ctx = &ictx[wq->idxd->data->type]; - ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); + ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); kfree(idxd_cdev); } -static struct device_type idxd_cdev_device_type = { +static const struct device_type idxd_cdev_device_type = { .name = "idxd_cdev", .release = idxd_cdev_dev_release, }; @@ -69,15 +185,46 @@ static inline struct idxd_wq *inode_wq(struct inode *inode) return idxd_cdev->wq; } +static void idxd_xa_pasid_remove(struct idxd_user_context *ctx) +{ + struct idxd_wq *wq = ctx->wq; + void *ptr; + + mutex_lock(&wq->uc_lock); + ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL); + if (ptr != (void *)ctx) + dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n", + ctx->pasid); + mutex_unlock(&wq->uc_lock); +} + +void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index) +{ + struct idxd_user_context *ctx; + + if (index >= COUNTER_MAX) + return; + + mutex_lock(&wq->uc_lock); + ctx = xa_load(&wq->upasid_xa, pasid); + if (!ctx) { + mutex_unlock(&wq->uc_lock); + return; + } + ctx->counters[index]++; + mutex_unlock(&wq->uc_lock); +} + static int idxd_cdev_open(struct inode *inode, struct file *filp) { struct idxd_user_context *ctx; struct idxd_device *idxd; struct idxd_wq *wq; - struct device *dev; + struct device *dev, *fdev; int rc = 0; - struct iommu_sva *sva; + struct iommu_sva *sva = NULL; unsigned int pasid; + struct idxd_cdev *idxd_cdev; wq = inode_wq(inode); idxd = wq->idxd; @@ -98,9 +245,10 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) ctx->wq = wq; filp->private_data = ctx; + ctx->pid = current->pid; - if (device_pasid_enabled(idxd)) { - sva = iommu_sva_bind_device(dev, current->mm, NULL); + if (device_user_pasid_enabled(idxd)) { + sva = iommu_sva_bind_device(dev, current->mm); if (IS_ERR(sva)) { rc = PTR_ERR(sva); dev_err(dev, "pasid allocation failed: %d\n", rc); @@ -109,65 +257,116 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) pasid = iommu_sva_get_pasid(sva); if (pasid == IOMMU_PASID_INVALID) { - iommu_sva_unbind_device(sva); rc = -EINVAL; - goto failed; + goto failed_get_pasid; } ctx->sva = sva; ctx->pasid = pasid; + ctx->mm = current->mm; + + mutex_lock(&wq->uc_lock); + rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL); + mutex_unlock(&wq->uc_lock); + if (rc < 0) + dev_warn(dev, "PASID entry already exist in xarray.\n"); if (wq_dedicated(wq)) { rc = idxd_wq_set_pasid(wq, pasid); if (rc < 0) { - iommu_sva_unbind_device(sva); dev_err(dev, "wq set pasid failed: %d\n", rc); - goto failed; + goto failed_set_pasid; } } } + idxd_cdev = wq->idxd_cdev; + ctx->id = ida_alloc(&file_ida, GFP_KERNEL); + if (ctx->id < 0) { + dev_warn(dev, "ida alloc failure\n"); + goto failed_ida; + } + ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE; + fdev = user_ctx_dev(ctx); + device_initialize(fdev); + fdev->parent = cdev_dev(idxd_cdev); + fdev->bus = &dsa_bus_type; + fdev->type = &idxd_cdev_file_type; + + rc = dev_set_name(fdev, "file%d", ctx->id); + if (rc < 0) { + dev_warn(dev, "set name failure\n"); + goto failed_dev_name; + } + + rc = device_add(fdev); + if (rc < 0) { + dev_warn(dev, "file device add failure\n"); + goto failed_dev_add; + } + idxd_wq_get(wq); mutex_unlock(&wq->wq_lock); return 0; - failed: +failed_dev_add: +failed_dev_name: + put_device(fdev); +failed_ida: +failed_set_pasid: + if (device_user_pasid_enabled(idxd)) + idxd_xa_pasid_remove(ctx); +failed_get_pasid: + if (device_user_pasid_enabled(idxd) && !IS_ERR_OR_NULL(sva)) + iommu_sva_unbind_device(sva); +failed: mutex_unlock(&wq->wq_lock); kfree(ctx); return rc; } +static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) +{ + struct idxd_device *idxd = wq->idxd; + struct idxd_evl *evl = idxd->evl; + union evl_status_reg status; + u16 h, t, size; + int ent_size = evl_ent_size(idxd); + struct __evl_entry *entry_head; + + if (!evl) + return; + + mutex_lock(&evl->lock); + status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); + t = status.tail; + h = status.head; + size = evl->size; + + while (h != t) { + entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); + if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id) + set_bit(h, evl->bmap); + h = (h + 1) % size; + } + if (wq->wq) + drain_workqueue(wq->wq); + + mutex_unlock(&evl->lock); +} + static int idxd_cdev_release(struct inode *node, struct file *filep) { struct idxd_user_context *ctx = filep->private_data; struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; - int rc; dev_dbg(dev, "%s called\n", __func__); filep->private_data = NULL; - /* Wait for in-flight operations to complete. */ - if (wq_shared(wq)) { - idxd_device_drain_pasid(idxd, ctx->pasid); - } else { - if (device_pasid_enabled(idxd)) { - /* The wq disable in the disable pasid function will drain the wq */ - rc = idxd_wq_disable_pasid(wq); - if (rc < 0) - dev_err(dev, "wq disable pasid failed.\n"); - } else { - idxd_wq_drain(wq); - } - } + device_unregister(user_ctx_dev(ctx)); - if (ctx->sva) - iommu_sva_unbind_device(ctx->sva); - kfree(ctx); - mutex_lock(&wq->wq_lock); - idxd_wq_put(wq); - mutex_unlock(&wq->wq_lock); return 0; } @@ -198,11 +397,26 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) int rc; dev_dbg(&pdev->dev, "%s called\n", __func__); + + /* + * Due to an erratum in some of the devices supported by the driver, + * direct user submission to the device can be unsafe. + * (See the INTEL-SA-01084 security advisory) + * + * For the devices that exhibit this behavior, require that the user + * has CAP_SYS_RAWIO capabilities. + */ + if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO)) + return -EPERM; + + if (current->mm != ctx->mm) + return -EPERM; + rc = check_vma(wq, vma, __func__); if (rc < 0) return rc; - vma->vm_flags |= VM_DONTCOPY; + vm_flags_set(vma, VM_DONTCOPY); pfn = (base + idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); @@ -212,20 +426,91 @@ static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) vma->vm_page_prot); } +static int idxd_submit_user_descriptor(struct idxd_user_context *ctx, + struct dsa_hw_desc __user *udesc) +{ + struct idxd_wq *wq = ctx->wq; + struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev; + const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40; + void __iomem *portal = idxd_wq_portal_addr(wq); + struct dsa_hw_desc descriptor __aligned(64); + int rc; + + rc = copy_from_user(&descriptor, udesc, sizeof(descriptor)); + if (rc) + return -EFAULT; + + /* + * DSA devices are capable of indirect ("batch") command submission. + * On devices where direct user submissions are not safe, we cannot + * allow this since there is no good way for us to verify these + * indirect commands. Narrow the restriction of operations with the + * BATCH opcode to only DSA version 1 devices. + */ + if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH && + wq->idxd->hw.version == DEVICE_VERSION_1 && + !wq->idxd->user_submission_safe) + return -EINVAL; + /* + * As per the programming specification, the completion address must be + * aligned to 32 or 64 bytes. If this is violated the hardware + * engine can get very confused (security issue). + */ + if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align)) + return -EINVAL; + + if (wq_dedicated(wq)) + iosubmit_cmds512(portal, &descriptor, 1); + else { + descriptor.priv = 0; + descriptor.pasid = ctx->pasid; + rc = idxd_enqcmds(wq, portal, &descriptor); + if (rc < 0) + return rc; + } + + return 0; +} + +static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len, + loff_t *unused) +{ + struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf; + struct idxd_user_context *ctx = filp->private_data; + ssize_t written = 0; + int i; + + if (current->mm != ctx->mm) + return -EPERM; + + for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) { + int rc = idxd_submit_user_descriptor(ctx, udesc + i); + + if (rc) + return written ? written : rc; + + written += sizeof(struct dsa_hw_desc); + } + + return written; +} + static __poll_t idxd_cdev_poll(struct file *filp, struct poll_table_struct *wait) { struct idxd_user_context *ctx = filp->private_data; struct idxd_wq *wq = ctx->wq; struct idxd_device *idxd = wq->idxd; - unsigned long flags; __poll_t out = 0; + if (current->mm != ctx->mm) + return POLLNVAL; + poll_wait(filp, &wq->err_queue, wait); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); if (idxd->sw_err.valid) out = EPOLLIN | EPOLLRDNORM; - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); return out; } @@ -235,6 +520,7 @@ static const struct file_operations idxd_cdev_fops = { .open = idxd_cdev_open, .release = idxd_cdev_release, .mmap = idxd_cdev_mmap, + .write = idxd_cdev_write, .poll = idxd_cdev_poll, }; @@ -256,11 +542,12 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) if (!idxd_cdev) return -ENOMEM; + idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; idxd_cdev->wq = wq; cdev = &idxd_cdev->cdev; - dev = &idxd_cdev->dev; + dev = cdev_dev(idxd_cdev); cdev_ctx = &ictx[wq->idxd->data->type]; - minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); + minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL); if (minor < 0) { kfree(idxd_cdev); return minor; @@ -268,7 +555,7 @@ int idxd_wq_add_cdev(struct idxd_wq *wq) idxd_cdev->minor = minor; device_initialize(dev); - dev->parent = &wq->conf_dev; + dev->parent = wq_confdev(wq); dev->bus = &dsa_bus_type; dev->type = &idxd_cdev_device_type; dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); @@ -299,10 +586,104 @@ void idxd_wq_del_cdev(struct idxd_wq *wq) idxd_cdev = wq->idxd_cdev; wq->idxd_cdev = NULL; - cdev_device_del(&idxd_cdev->cdev, &idxd_cdev->dev); - put_device(&idxd_cdev->dev); + cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); + put_device(cdev_dev(idxd_cdev)); } +static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) +{ + struct device *dev = &idxd_dev->conf_dev; + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + int rc; + + if (idxd->state != IDXD_DEV_ENABLED) + return -ENXIO; + + mutex_lock(&wq->wq_lock); + + if (!idxd_wq_driver_name_match(wq, dev)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + rc = -ENODEV; + goto wq_err; + } + + /* + * User type WQ is enabled only when SVA is enabled for two reasons: + * - If no IOMMU or IOMMU Passthrough without SVA, userspace + * can directly access physical address through the WQ. + * - The IDXD cdev driver does not provide any ways to pin + * user pages and translate the address from user VA to IOVA or + * PA without IOMMU SVA. Therefore the application has no way + * to instruct the device to perform DMA function. This makes + * the cdev not usable for normal application usage. + */ + if (!device_user_pasid_enabled(idxd)) { + idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; + dev_dbg(&idxd->pdev->dev, + "User type WQ cannot be enabled without SVA.\n"); + + rc = -EOPNOTSUPP; + goto wq_err; + } + + wq->wq = create_workqueue(dev_name(wq_confdev(wq))); + if (!wq->wq) { + rc = -ENOMEM; + goto wq_err; + } + + wq->type = IDXD_WQT_USER; + rc = idxd_drv_enable_wq(wq); + if (rc < 0) + goto err; + + rc = idxd_wq_add_cdev(wq); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_CDEV_ERR; + goto err_cdev; + } + + idxd->cmd_status = 0; + mutex_unlock(&wq->wq_lock); + return 0; + +err_cdev: + idxd_drv_disable_wq(wq); +err: + destroy_workqueue(wq->wq); + wq->type = IDXD_WQT_NONE; +wq_err: + mutex_unlock(&wq->wq_lock); + return rc; +} + +static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + + mutex_lock(&wq->wq_lock); + idxd_wq_del_cdev(wq); + idxd_drv_disable_wq(wq); + wq->type = IDXD_WQT_NONE; + destroy_workqueue(wq->wq); + wq->wq = NULL; + mutex_unlock(&wq->wq_lock); +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_WQ, + IDXD_DEV_NONE, +}; + +struct idxd_device_driver idxd_user_drv = { + .probe = idxd_user_drv_probe, + .remove = idxd_user_drv_remove, + .name = "user", + .type = dev_types, +}; +EXPORT_SYMBOL_GPL(idxd_user_drv); + int idxd_cdev_register(void) { int rc, i; @@ -312,10 +693,16 @@ int idxd_cdev_register(void) rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, ictx[i].name); if (rc) - return rc; + goto err_free_chrdev_region; } return 0; + +err_free_chrdev_region: + for (i--; i >= 0; i--) + unregister_chrdev_region(ictx[i].devt, MINORMASK); + + return rc; } void idxd_cdev_remove(void) @@ -327,3 +714,70 @@ void idxd_cdev_remove(void) ida_destroy(&ictx[i].minor_ida); } } + +/** + * idxd_copy_cr - copy completion record to user address space found by wq and + * PASID + * @wq: work queue + * @pasid: PASID + * @addr: user fault address to write + * @cr: completion record + * @len: number of bytes to copy + * + * This is called by a work that handles completion record fault. + * + * Return: number of bytes copied. + */ +int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, + void *cr, int len) +{ + struct device *dev = &wq->idxd->pdev->dev; + int left = len, status_size = 1; + struct idxd_user_context *ctx; + struct mm_struct *mm; + + mutex_lock(&wq->uc_lock); + + ctx = xa_load(&wq->upasid_xa, pasid); + if (!ctx) { + dev_warn(dev, "No user context\n"); + goto out; + } + + mm = ctx->mm; + /* + * The completion record fault handling work is running in kernel + * thread context. It temporarily switches to the mm to copy cr + * to addr in the mm. + */ + kthread_use_mm(mm); + left = copy_to_user((void __user *)addr + status_size, cr + status_size, + len - status_size); + /* + * Copy status only after the rest of completion record is copied + * successfully so that the user gets the complete completion record + * when a non-zero status is polled. + */ + if (!left) { + u8 status; + + /* + * Ensure that the completion record's status field is written + * after the rest of the completion record has been written. + * This ensures that the user receives the correct completion + * record information once polling for a non-zero status. + */ + wmb(); + status = *(u8 *)cr; + if (put_user(status, (u8 __user *)addr)) + left += status_size; + } else { + left += status_size; + } + kthread_unuse_mm(mm); + +out: + mutex_unlock(&wq->uc_lock); + + return len - left; +} diff --git a/drivers/dma/idxd/compat.c b/drivers/dma/idxd/compat.c new file mode 100644 index 000000000000..eff9943f1a42 --- /dev/null +++ b/drivers/dma/idxd/compat.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/device.h> +#include <linux/device/bus.h> +#include "idxd.h" + +extern void device_driver_detach(struct device *dev); + +#define DRIVER_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \ + struct driver_attribute driver_attr_##_name = \ + __ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) + +static ssize_t unbind_store(struct device_driver *drv, const char *buf, size_t count) +{ + const struct bus_type *bus = drv->bus; + struct device *dev; + int rc = -ENODEV; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev && dev->driver) { + device_driver_detach(dev); + rc = count; + } + + return rc; +} +static DRIVER_ATTR_IGNORE_LOCKDEP(unbind, 0200, NULL, unbind_store); + +static ssize_t bind_store(struct device_driver *drv, const char *buf, size_t count) +{ + const struct bus_type *bus = drv->bus; + struct device *dev; + struct device_driver *alt_drv = NULL; + int rc = -ENODEV; + struct idxd_dev *idxd_dev; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (!dev || dev->driver || drv != &dsa_drv.drv) + return -ENODEV; + + idxd_dev = confdev_to_idxd_dev(dev); + if (is_idxd_dev(idxd_dev)) { + alt_drv = driver_find("idxd", bus); + } else if (is_idxd_wq_dev(idxd_dev)) { + struct idxd_wq *wq = confdev_to_wq(dev); + + if (is_idxd_wq_kernel(wq)) + alt_drv = driver_find("dmaengine", bus); + else if (is_idxd_wq_user(wq)) + alt_drv = driver_find("user", bus); + } + if (!alt_drv) + return -ENODEV; + + rc = device_driver_attach(alt_drv, dev); + if (rc < 0) + return rc; + + return count; +} +static DRIVER_ATTR_IGNORE_LOCKDEP(bind, 0200, NULL, bind_store); + +static struct attribute *dsa_drv_compat_attrs[] = { + &driver_attr_bind.attr, + &driver_attr_unbind.attr, + NULL, +}; + +static const struct attribute_group dsa_drv_compat_attr_group = { + .attrs = dsa_drv_compat_attrs, +}; + +static const struct attribute_group *dsa_drv_compat_groups[] = { + &dsa_drv_compat_attr_group, + NULL, +}; + +static int idxd_dsa_drv_probe(struct idxd_dev *idxd_dev) +{ + return -ENODEV; +} + +static void idxd_dsa_drv_remove(struct idxd_dev *idxd_dev) +{ +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_NONE, +}; + +struct idxd_device_driver dsa_drv = { + .name = "dsa", + .probe = idxd_dsa_drv_probe, + .remove = idxd_dsa_drv_remove, + .type = dev_types, + .drv = { + .suppress_bind_attrs = true, + .groups = dsa_drv_compat_groups, + }, +}; + +module_idxd_driver(dsa_drv); +MODULE_IMPORT_NS("IDXD"); diff --git a/drivers/dma/idxd/debugfs.c b/drivers/dma/idxd/debugfs.c new file mode 100644 index 000000000000..ad4245cb301d --- /dev/null +++ b/drivers/dma/idxd/debugfs.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/debugfs.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <uapi/linux/idxd.h> +#include "idxd.h" +#include "registers.h" + +static struct dentry *idxd_debugfs_dir; + +static void dump_event_entry(struct idxd_device *idxd, struct seq_file *s, + u16 index, int *count, bool processed) +{ + struct idxd_evl *evl = idxd->evl; + struct dsa_evl_entry *entry; + struct dsa_completion_record *cr; + u64 *raw; + int i; + int evl_strides = evl_ent_size(idxd) / sizeof(u64); + + entry = (struct dsa_evl_entry *)evl->log + index; + + if (!entry->e.desc_valid) + return; + + seq_printf(s, "Event Log entry %d (real index %u) processed: %u\n", + *count, index, processed); + + seq_printf(s, "desc valid %u wq idx valid %u\n" + "batch %u fault rw %u priv %u error 0x%x\n" + "wq idx %u op %#x pasid %u batch idx %u\n" + "fault addr %#llx\n", + entry->e.desc_valid, entry->e.wq_idx_valid, + entry->e.batch, entry->e.fault_rw, entry->e.priv, + entry->e.error, entry->e.wq_idx, entry->e.operation, + entry->e.pasid, entry->e.batch_idx, entry->e.fault_addr); + + cr = &entry->cr; + seq_printf(s, "status %#x result %#x fault_info %#x bytes_completed %u\n" + "fault addr %#llx inv flags %#x\n\n", + cr->status, cr->result, cr->fault_info, cr->bytes_completed, + cr->fault_addr, cr->invalid_flags); + + raw = (u64 *)entry; + + for (i = 0; i < evl_strides; i++) + seq_printf(s, "entry[%d] = %#llx\n", i, raw[i]); + + seq_puts(s, "\n"); + *count += 1; +} + +static int debugfs_evl_show(struct seq_file *s, void *d) +{ + struct idxd_device *idxd = s->private; + struct idxd_evl *evl = idxd->evl; + union evl_status_reg evl_status; + u16 h, t, evl_size, i; + int count = 0; + bool processed = true; + + if (!evl || !evl->log) + return 0; + + mutex_lock(&evl->lock); + + evl_status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); + t = evl_status.tail; + h = evl_status.head; + evl_size = evl->size; + + seq_printf(s, "Event Log head %u tail %u interrupt pending %u\n\n", + evl_status.head, evl_status.tail, evl_status.int_pending); + + i = t; + while (1) { + i = (i + 1) % evl_size; + if (i == t) + break; + + if (processed && i == h) + processed = false; + dump_event_entry(idxd, s, i, &count, processed); + } + + mutex_unlock(&evl->lock); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(debugfs_evl); + +int idxd_device_init_debugfs(struct idxd_device *idxd) +{ + if (IS_ERR_OR_NULL(idxd_debugfs_dir)) + return 0; + + idxd->dbgfs_dir = debugfs_create_dir(dev_name(idxd_confdev(idxd)), idxd_debugfs_dir); + if (IS_ERR(idxd->dbgfs_dir)) + return PTR_ERR(idxd->dbgfs_dir); + + if (idxd->evl) { + idxd->dbgfs_evl_file = debugfs_create_file("event_log", 0400, + idxd->dbgfs_dir, idxd, + &debugfs_evl_fops); + if (IS_ERR(idxd->dbgfs_evl_file)) { + debugfs_remove_recursive(idxd->dbgfs_dir); + idxd->dbgfs_dir = NULL; + return PTR_ERR(idxd->dbgfs_evl_file); + } + } + + return 0; +} + +void idxd_device_remove_debugfs(struct idxd_device *idxd) +{ + debugfs_remove_recursive(idxd->dbgfs_dir); +} + +int idxd_init_debugfs(void) +{ + if (!debugfs_initialized()) + return 0; + + idxd_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + if (IS_ERR(idxd_debugfs_dir)) + return PTR_ERR(idxd_debugfs_dir); + return 0; +} + +void idxd_remove_debugfs(void) +{ + debugfs_remove_recursive(idxd_debugfs_dir); +} diff --git a/drivers/dma/idxd/defaults.c b/drivers/dma/idxd/defaults.c new file mode 100644 index 000000000000..2bbbcd02a0da --- /dev/null +++ b/drivers/dma/idxd/defaults.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2023 Intel Corporation. All rights rsvd. */ +#include <linux/kernel.h> +#include "idxd.h" + +int idxd_load_iaa_device_defaults(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct idxd_group *group; + struct idxd_wq *wq; + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return 0; + + wq = idxd->wqs[0]; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + /* set mode to "dedicated" */ + set_bit(WQ_FLAG_DEDICATED, &wq->flags); + wq->threshold = 0; + + /* only setting up 1 wq, so give it all the wq space */ + wq->size = idxd->max_wq_size; + + /* set priority to 10 */ + wq->priority = 10; + + /* set type to "kernel" */ + wq->type = IDXD_WQT_KERNEL; + + /* set wq group to 0 */ + group = idxd->groups[0]; + wq->group = group; + group->num_wqs++; + + /* set name to "iaa_crypto" */ + strscpy_pad(wq->name, "iaa_crypto"); + + /* set driver_name to "crypto" */ + strscpy_pad(wq->driver_name, "crypto"); + + engine = idxd->engines[0]; + + /* set engine group to 0 */ + engine->group = idxd->groups[0]; + engine->group->num_engines++; + + return 0; +} diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 420b93fe5feb..c2cdf41b6e57 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -7,7 +7,6 @@ #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/dmaengine.h> #include <linux/irq.h> -#include <linux/msi.h> #include <uapi/linux/idxd.h> #include "../dmaengine.h" #include "idxd.h" @@ -15,32 +14,11 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, u32 *status); +static void idxd_device_wqs_clear_state(struct idxd_device *idxd); +static void idxd_wq_disable_cleanup(struct idxd_wq *wq); +static int idxd_wq_config_write(struct idxd_wq *wq); /* Interrupt control bits */ -void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id) -{ - struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); - - pci_msi_mask_irq(data); -} - -void idxd_mask_msix_vectors(struct idxd_device *idxd) -{ - struct pci_dev *pdev = idxd->pdev; - int msixcnt = pci_msix_vec_count(pdev); - int i; - - for (i = 0; i < msixcnt; i++) - idxd_mask_msix_vector(idxd, i); -} - -void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id) -{ - struct irq_data *data = irq_get_irq_data(idxd->irq_entries[vec_id].vector); - - pci_msi_unmask_irq(data); -} - void idxd_unmask_error_interrupts(struct idxd_device *idxd) { union genctrl_reg genctrl; @@ -133,34 +111,24 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; int rc, num_descs, i; - int align; - u64 tmp; if (wq->type != IDXD_WQT_KERNEL) return 0; - wq->num_descs = wq->size; - num_descs = wq->size; + num_descs = wq_dedicated(wq) ? wq->size : wq->threshold; + wq->num_descs = num_descs; rc = alloc_hw_descs(wq, num_descs); if (rc < 0) return rc; - align = idxd->data->align; - wq->compls_size = num_descs * idxd->data->compl_size + align; - wq->compls_raw = dma_alloc_coherent(dev, wq->compls_size, - &wq->compls_addr_raw, GFP_KERNEL); - if (!wq->compls_raw) { + wq->compls_size = num_descs * idxd->data->compl_size; + wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL); + if (!wq->compls) { rc = -ENOMEM; goto fail_alloc_compls; } - /* Adjust alignment */ - wq->compls_addr = (wq->compls_addr_raw + (align - 1)) & ~(align - 1); - tmp = (u64)wq->compls_raw; - tmp = (tmp + (align - 1)) & ~(align - 1); - wq->compls = (struct dsa_completion_record *)tmp; - rc = alloc_descs(wq, num_descs); if (rc < 0) goto fail_alloc_descs; @@ -189,12 +157,12 @@ int idxd_wq_alloc_resources(struct idxd_wq *wq) fail_sbitmap_init: free_descs(wq); fail_alloc_descs: - dma_free_coherent(dev, wq->compls_size, wq->compls_raw, - wq->compls_addr_raw); + dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); fail_alloc_compls: free_hw_descs(wq); return rc; } +EXPORT_SYMBOL_NS_GPL(idxd_wq_alloc_resources, "IDXD"); void idxd_wq_free_resources(struct idxd_wq *wq) { @@ -205,10 +173,10 @@ void idxd_wq_free_resources(struct idxd_wq *wq) free_hw_descs(wq); free_descs(wq); - dma_free_coherent(dev, wq->compls_size, wq->compls_raw, - wq->compls_addr_raw); + dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr); sbitmap_queue_free(&wq->sbq); } +EXPORT_SYMBOL_NS_GPL(idxd_wq_free_resources, "IDXD"); int idxd_wq_enable(struct idxd_wq *wq) { @@ -218,7 +186,7 @@ int idxd_wq_enable(struct idxd_wq *wq) if (wq->state == IDXD_WQ_ENABLED) { dev_dbg(dev, "WQ %d already enabled\n", wq->id); - return -ENXIO; + return 0; } idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status); @@ -230,11 +198,12 @@ int idxd_wq_enable(struct idxd_wq *wq) } wq->state = IDXD_WQ_ENABLED; + set_bit(wq->id, idxd->wq_enable_map); dev_dbg(dev, "WQ %d enabled\n", wq->id); return 0; } -int idxd_wq_disable(struct idxd_wq *wq) +int idxd_wq_disable(struct idxd_wq *wq, bool reset_config) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; @@ -247,14 +216,31 @@ int idxd_wq_disable(struct idxd_wq *wq) return 0; } + /* + * Disable WQ does not drain address translations, if WQ attributes are + * changed before translations are drained, pending translations can + * be issued using updated WQ attibutes, resulting in invalid + * translations being cached in the device translation cache. + * + * To make sure pending translations are drained before WQ + * attributes are changed, we use a WQ Drain followed by WQ Reset and + * then restore the WQ configuration. + */ + idxd_wq_drain(wq); + operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); - idxd_cmd_exec(idxd, IDXD_CMD_DISABLE_WQ, operand, &status); + idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, &status); if (status != IDXD_CMDSTS_SUCCESS) { - dev_dbg(dev, "WQ disable failed: %#x\n", status); + dev_dbg(dev, "WQ reset failed: %#x\n", status); return -ENXIO; } + idxd_wq_config_write(wq); + + if (reset_config) + idxd_wq_disable_cleanup(wq); + clear_bit(wq->id, idxd->wq_enable_map); wq->state = IDXD_WQ_DISABLED; dev_dbg(dev, "WQ %d disabled\n", wq->id); return 0; @@ -289,7 +275,7 @@ void idxd_wq_reset(struct idxd_wq *wq) operand = BIT(wq->id % 16) | ((wq->id / 16) << 16); idxd_cmd_exec(idxd, IDXD_CMD_RESET_WQ, operand, NULL); - wq->state = IDXD_WQ_DISABLED; + idxd_wq_disable_cleanup(wq); } int idxd_wq_map_portal(struct idxd_wq *wq) @@ -315,6 +301,7 @@ void idxd_wq_unmap_portal(struct idxd_wq *wq) devm_iounmap(dev, wq->portal); wq->portal = NULL; + wq->portal_offset = 0; } void idxd_wqs_unmap_portal(struct idxd_device *idxd) @@ -329,25 +316,31 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd) } } -int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) +static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid) { struct idxd_device *idxd = wq->idxd; - int rc; union wqcfg wqcfg; unsigned int offset; - unsigned long flags; - - rc = idxd_wq_disable(wq); - if (rc < 0) - return rc; offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); wqcfg.pasid_en = 1; wqcfg.pasid = pasid; + wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX]; iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); +} + +int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid) +{ + int rc; + + rc = idxd_wq_disable(wq, false); + if (rc < 0) + return rc; + + __idxd_wq_set_pasid_locked(wq, pasid); rc = idxd_wq_enable(wq); if (rc < 0) @@ -362,19 +355,18 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq) int rc; union wqcfg wqcfg; unsigned int offset; - unsigned long flags; - rc = idxd_wq_disable(wq); + rc = idxd_wq_disable(wq, false); if (rc < 0) return rc; offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX); - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); wqcfg.bits[WQCFG_PASID_IDX] = ioread32(idxd->reg_base + offset); wqcfg.pasid_en = 0; wqcfg.pasid = 0; iowrite32(wqcfg.bits[WQCFG_PASID_IDX], idxd->reg_base + offset); - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); rc = idxd_wq_enable(wq); if (rc < 0) @@ -383,20 +375,31 @@ int idxd_wq_disable_pasid(struct idxd_wq *wq) return 0; } -void idxd_wq_disable_cleanup(struct idxd_wq *wq) +static void idxd_wq_disable_cleanup(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; - lockdep_assert_held(&idxd->dev_lock); + lockdep_assert_held(&wq->wq_lock); + wq->state = IDXD_WQ_DISABLED; memset(wq->wqcfg, 0, idxd->wqcfg_size); wq->type = IDXD_WQT_NONE; - wq->size = 0; - wq->group = NULL; wq->threshold = 0; wq->priority = 0; - wq->ats_dis = 0; - clear_bit(WQ_FLAG_DEDICATED, &wq->flags); + wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; + wq->flags = 0; memset(wq->name, 0, WQ_NAME_SIZE); + wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; + idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); + if (wq->opcap_bmap) + bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); +} + +static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq) +{ + lockdep_assert_held(&wq->wq_lock); + + wq->size = 0; + wq->group = NULL; } static void idxd_wq_ref_release(struct percpu_ref *ref) @@ -411,19 +414,33 @@ int idxd_wq_init_percpu_ref(struct idxd_wq *wq) int rc; memset(&wq->wq_active, 0, sizeof(wq->wq_active)); - rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release, 0, GFP_KERNEL); + rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release, + PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); if (rc < 0) return rc; reinit_completion(&wq->wq_dead); + reinit_completion(&wq->wq_resurrect); return 0; } +EXPORT_SYMBOL_NS_GPL(idxd_wq_init_percpu_ref, "IDXD"); -void idxd_wq_quiesce(struct idxd_wq *wq) +void __idxd_wq_quiesce(struct idxd_wq *wq) { + lockdep_assert_held(&wq->wq_lock); + reinit_completion(&wq->wq_resurrect); percpu_ref_kill(&wq->wq_active); + complete_all(&wq->wq_resurrect); wait_for_completion(&wq->wq_dead); - percpu_ref_exit(&wq->wq_active); } +EXPORT_SYMBOL_NS_GPL(__idxd_wq_quiesce, "IDXD"); + +void idxd_wq_quiesce(struct idxd_wq *wq) +{ + mutex_lock(&wq->wq_lock); + __idxd_wq_quiesce(wq); + mutex_unlock(&wq->wq_lock); +} +EXPORT_SYMBOL_NS_GPL(idxd_wq_quiesce, "IDXD"); /* Device control bits */ static inline bool idxd_is_enabled(struct idxd_device *idxd) @@ -455,7 +472,6 @@ int idxd_device_init_reset(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; union idxd_command_reg cmd; - unsigned long flags; if (idxd_device_is_halted(idxd)) { dev_warn(&idxd->pdev->dev, "Device is HALTED!\n"); @@ -465,13 +481,13 @@ int idxd_device_init_reset(struct idxd_device *idxd) memset(&cmd, 0, sizeof(cmd)); cmd.cmd = IDXD_CMD_RESET_DEVICE; dev_dbg(dev, "%s: sending reset for init.\n", __func__); - spin_lock_irqsave(&idxd->cmd_lock, flags); + spin_lock(&idxd->cmd_lock); iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) cpu_relax(); - spin_unlock_irqrestore(&idxd->cmd_lock, flags); + spin_unlock(&idxd->cmd_lock); return 0; } @@ -480,6 +496,7 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, { union idxd_command_reg cmd; DECLARE_COMPLETION_ONSTACK(done); + u32 stat; unsigned long flags; if (idxd_device_is_halted(idxd)) { @@ -513,16 +530,16 @@ static void idxd_cmd_exec(struct idxd_device *idxd, int cmd_code, u32 operand, */ spin_unlock_irqrestore(&idxd->cmd_lock, flags); wait_for_completion(&done); - spin_lock_irqsave(&idxd->cmd_lock, flags); - if (status) { - *status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); - idxd->cmd_status = *status & GENMASK(7, 0); - } + stat = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); + spin_lock(&idxd->cmd_lock); + if (status) + *status = stat; + idxd->cmd_status = stat & GENMASK(7, 0); __clear_bit(IDXD_FLAG_CMD_RUNNING, &idxd->flags); /* Wake up other pending commands */ wake_up(&idxd->cmd_waitq); - spin_unlock_irqrestore(&idxd->cmd_lock, flags); + spin_unlock(&idxd->cmd_lock); } int idxd_device_enable(struct idxd_device *idxd) @@ -548,27 +565,10 @@ int idxd_device_enable(struct idxd_device *idxd) return 0; } -void idxd_device_wqs_clear_state(struct idxd_device *idxd) -{ - int i; - - lockdep_assert_held(&idxd->dev_lock); - - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - if (wq->state == IDXD_WQ_ENABLED) { - idxd_wq_disable_cleanup(wq); - wq->state = IDXD_WQ_DISABLED; - } - } -} - int idxd_device_disable(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; u32 status; - unsigned long flags; if (!idxd_is_enabled(idxd)) { dev_dbg(dev, "Device is not enabled\n"); @@ -584,22 +584,17 @@ int idxd_device_disable(struct idxd_device *idxd) return -ENXIO; } - spin_lock_irqsave(&idxd->dev_lock, flags); - idxd_device_wqs_clear_state(idxd); - idxd->state = IDXD_DEV_CONF_READY; - spin_unlock_irqrestore(&idxd->dev_lock, flags); + idxd_device_clear_state(idxd); return 0; } void idxd_device_reset(struct idxd_device *idxd) { - unsigned long flags; - idxd_cmd_exec(idxd, IDXD_CMD_RESET_DEVICE, 0, NULL); - spin_lock_irqsave(&idxd->dev_lock, flags); - idxd_device_wqs_clear_state(idxd); - idxd->state = IDXD_DEV_CONF_READY; - spin_unlock_irqrestore(&idxd->dev_lock, flags); + idxd_device_clear_state(idxd); + spin_lock(&idxd->dev_lock); + idxd_unmask_error_interrupts(idxd); + spin_unlock(&idxd->dev_lock); } void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid) @@ -649,7 +644,6 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, struct device *dev = &idxd->pdev->dev; u32 operand, status; union idxd_command_reg cmd; - unsigned long flags; if (!(idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE))) return -EOPNOTSUPP; @@ -667,13 +661,13 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, dev_dbg(dev, "cmd: %u operand: %#x\n", IDXD_CMD_RELEASE_INT_HANDLE, operand); - spin_lock_irqsave(&idxd->cmd_lock, flags); + spin_lock(&idxd->cmd_lock); iowrite32(cmd.bits, idxd->reg_base + IDXD_CMD_OFFSET); while (ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET) & IDXD_CMDSTS_ACTIVE) cpu_relax(); status = ioread32(idxd->reg_base + IDXD_CMDSTS_OFFSET); - spin_unlock_irqrestore(&idxd->cmd_lock, flags); + spin_unlock(&idxd->cmd_lock); if ((status & IDXD_CMDSTS_ERR_MASK) != IDXD_CMDSTS_SUCCESS) { dev_dbg(dev, "release int handle failed: %#x\n", status); @@ -685,34 +679,181 @@ int idxd_device_release_int_handle(struct idxd_device *idxd, int handle, } /* Device configuration bits */ -void idxd_msix_perm_setup(struct idxd_device *idxd) +static void idxd_engines_clear_state(struct idxd_device *idxd) { - union msix_perm mperm; - int i, msixcnt; + struct idxd_engine *engine; + int i; - msixcnt = pci_msix_vec_count(idxd->pdev); - if (msixcnt < 0) - return; + lockdep_assert_held(&idxd->dev_lock); + for (i = 0; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + engine->group = NULL; + } +} - mperm.bits = 0; - mperm.pasid = idxd->pasid; - mperm.pasid_en = device_pasid_enabled(idxd); - for (i = 1; i < msixcnt; i++) - iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); +static void idxd_groups_clear_state(struct idxd_device *idxd) +{ + struct idxd_group *group; + int i; + + lockdep_assert_held(&idxd->dev_lock); + for (i = 0; i < idxd->max_groups; i++) { + group = idxd->groups[i]; + memset(&group->grpcfg, 0, sizeof(group->grpcfg)); + group->num_engines = 0; + group->num_wqs = 0; + group->use_rdbuf_limit = false; + /* + * The default value is the same as the value of + * total read buffers in GRPCAP. + */ + group->rdbufs_allowed = idxd->max_rdbufs; + group->rdbufs_reserved = 0; + if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { + group->tc_a = 1; + group->tc_b = 1; + } else { + group->tc_a = -1; + group->tc_b = -1; + } + group->desc_progress_limit = 0; + group->batch_progress_limit = 0; + } } -void idxd_msix_perm_clear(struct idxd_device *idxd) +static void idxd_device_wqs_clear_state(struct idxd_device *idxd) { - union msix_perm mperm; - int i, msixcnt; + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + + mutex_lock(&wq->wq_lock); + idxd_wq_disable_cleanup(wq); + idxd_wq_device_reset_cleanup(wq); + mutex_unlock(&wq->wq_lock); + } +} + +void idxd_device_clear_state(struct idxd_device *idxd) +{ + /* IDXD is always disabled. Other states are cleared only when IDXD is configurable. */ + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + /* + * Clearing wq state is protected by wq lock. + * So no need to be protected by device lock. + */ + idxd_device_wqs_clear_state(idxd); + + spin_lock(&idxd->dev_lock); + idxd_groups_clear_state(idxd); + idxd_engines_clear_state(idxd); + } else { + spin_lock(&idxd->dev_lock); + } + + idxd->state = IDXD_DEV_DISABLED; + spin_unlock(&idxd->dev_lock); +} + +static int idxd_device_evl_setup(struct idxd_device *idxd) +{ + union gencfg_reg gencfg; + union evlcfg_reg evlcfg; + union genctrl_reg genctrl; + struct device *dev = &idxd->pdev->dev; + void *addr; + dma_addr_t dma_addr; + int size; + struct idxd_evl *evl = idxd->evl; + unsigned long *bmap; + int rc; + + if (!evl) + return 0; + + size = evl_size(idxd); - msixcnt = pci_msix_vec_count(idxd->pdev); - if (msixcnt < 0) + bmap = bitmap_zalloc(size, GFP_KERNEL); + if (!bmap) { + rc = -ENOMEM; + goto err_bmap; + } + + /* + * Address needs to be page aligned. However, dma_alloc_coherent() provides + * at minimal page size aligned address. No manual alignment required. + */ + addr = dma_alloc_coherent(dev, size, &dma_addr, GFP_KERNEL); + if (!addr) { + rc = -ENOMEM; + goto err_alloc; + } + + mutex_lock(&evl->lock); + evl->log = addr; + evl->dma = dma_addr; + evl->log_size = size; + evl->bmap = bmap; + + memset(&evlcfg, 0, sizeof(evlcfg)); + evlcfg.bits[0] = dma_addr & GENMASK(63, 12); + evlcfg.size = evl->size; + + iowrite64(evlcfg.bits[0], idxd->reg_base + IDXD_EVLCFG_OFFSET); + iowrite64(evlcfg.bits[1], idxd->reg_base + IDXD_EVLCFG_OFFSET + 8); + + genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); + genctrl.evl_int_en = 1; + iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); + + gencfg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); + gencfg.evl_en = 1; + iowrite32(gencfg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); + + mutex_unlock(&evl->lock); + return 0; + +err_alloc: + bitmap_free(bmap); +err_bmap: + return rc; +} + +static void idxd_device_evl_free(struct idxd_device *idxd) +{ + void *evl_log; + unsigned int evl_log_size; + dma_addr_t evl_dma; + union gencfg_reg gencfg; + union genctrl_reg genctrl; + struct device *dev = &idxd->pdev->dev; + struct idxd_evl *evl = idxd->evl; + + gencfg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); + if (!gencfg.evl_en) return; - mperm.bits = 0; - for (i = 1; i < msixcnt; i++) - iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); + mutex_lock(&evl->lock); + gencfg.evl_en = 0; + iowrite32(gencfg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); + + genctrl.bits = ioread32(idxd->reg_base + IDXD_GENCTRL_OFFSET); + genctrl.evl_int_en = 0; + iowrite32(genctrl.bits, idxd->reg_base + IDXD_GENCTRL_OFFSET); + + iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET); + iowrite64(0, idxd->reg_base + IDXD_EVLCFG_OFFSET + 8); + + bitmap_free(evl->bmap); + evl_log = evl->log; + evl_log_size = evl->log_size; + evl_dma = evl->dma; + evl->log = NULL; + evl->size = IDXD_EVL_SIZE_MIN; + mutex_unlock(&evl->lock); + + dma_free_coherent(dev, evl_log_size, evl_log, evl_dma); } static void idxd_group_config_write(struct idxd_group *group) @@ -741,10 +882,10 @@ static void idxd_group_config_write(struct idxd_group *group) /* setup GRPFLAGS */ grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); - iowrite32(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); - dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", + iowrite64(group->grpcfg.flags.bits, idxd->reg_base + grpcfg_offset); + dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#llx\n", group->id, grpcfg_offset, - ioread32(idxd->reg_base + grpcfg_offset)); + ioread64(idxd->reg_base + grpcfg_offset)); } static int idxd_groups_config_write(struct idxd_device *idxd) @@ -754,10 +895,10 @@ static int idxd_groups_config_write(struct idxd_device *idxd) int i; struct device *dev = &idxd->pdev->dev; - /* Setup bandwidth token limit */ - if (idxd->token_limit) { + /* Setup bandwidth rdbuf limit */ + if (idxd->hw.gen_cap.config_en && idxd->rdbuf_limit) { reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); - reg.token_limit = idxd->token_limit; + reg.rdbuf_limit = idxd->rdbuf_limit; iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); } @@ -773,12 +914,21 @@ static int idxd_groups_config_write(struct idxd_device *idxd) return 0; } +static bool idxd_device_pasid_priv_enabled(struct idxd_device *idxd) +{ + struct pci_dev *pdev = idxd->pdev; + + if (pdev->pasid_enabled && (pdev->pasid_features & PCI_PASID_CAP_PRIV)) + return true; + return false; +} + static int idxd_wq_config_write(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; u32 wq_offset; - int i; + int i, n; if (!wq->group) return 0; @@ -789,43 +939,67 @@ static int idxd_wq_config_write(struct idxd_wq *wq) */ for (i = 0; i < WQCFG_STRIDES(idxd); i++) { wq_offset = WQCFG_OFFSET(idxd, wq->id, i); - wq->wqcfg->bits[i] = ioread32(idxd->reg_base + wq_offset); + wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset); } + if (wq->size == 0 && wq->type != IDXD_WQT_NONE) + wq->size = WQ_DEFAULT_QUEUE_DEPTH; + /* byte 0-3 */ wq->wqcfg->wq_size = wq->size; - if (wq->size == 0) { - dev_warn(dev, "Incorrect work queue size: 0\n"); - return -EINVAL; - } - /* bytes 4-7 */ wq->wqcfg->wq_thresh = wq->threshold; /* byte 8-11 */ - wq->wqcfg->priv = !!(wq->type == IDXD_WQT_KERNEL); if (wq_dedicated(wq)) wq->wqcfg->mode = 1; - if (device_pasid_enabled(idxd)) { - wq->wqcfg->pasid_en = 1; - if (wq->type == IDXD_WQT_KERNEL && wq_dedicated(wq)) - wq->wqcfg->pasid = idxd->pasid; + /* + * The WQ priv bit is set depending on the WQ type. priv = 1 if the + * WQ type is kernel to indicate privileged access. This setting only + * matters for dedicated WQ. According to the DSA spec: + * If the WQ is in dedicated mode, WQ PASID Enable is 1, and the + * Privileged Mode Enable field of the PCI Express PASID capability + * is 0, this field must be 0. + * + * In the case of a dedicated kernel WQ that is not able to support + * the PASID cap, then the configuration will be rejected. + */ + if (wq_dedicated(wq) && wq->wqcfg->pasid_en && + !idxd_device_pasid_priv_enabled(idxd) && + wq->type == IDXD_WQT_KERNEL) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_PRIV; + return -EOPNOTSUPP; } wq->wqcfg->priority = wq->priority; if (idxd->hw.gen_cap.block_on_fault && - test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)) + test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags) && + !test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags)) wq->wqcfg->bof = 1; if (idxd->hw.wq_cap.wq_ats_support) - wq->wqcfg->wq_ats_disable = wq->ats_dis; + wq->wqcfg->wq_ats_disable = test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); + + if (idxd->hw.wq_cap.wq_prs_support) + wq->wqcfg->wq_prs_disable = test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags); /* bytes 12-15 */ wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); - wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); + idxd_wqcfg_set_max_batch_shift(idxd->data->type, wq->wqcfg, ilog2(wq->max_batch_size)); + + /* bytes 32-63 */ + if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) { + memset(wq->wqcfg->op_config, 0, IDXD_MAX_OPCAP_BITS / 8); + for_each_set_bit(n, wq->opcap_bmap, IDXD_MAX_OPCAP_BITS) { + int pos = n % BITS_PER_LONG_LONG; + int idx = n / BITS_PER_LONG_LONG; + + wq->wqcfg->op_config[idx] |= BIT(pos); + } + } dev_dbg(dev, "WQ %d CFGs\n", wq->id); for (i = 0; i < WQCFG_STRIDES(idxd); i++) { @@ -870,13 +1044,11 @@ static void idxd_group_flags_setup(struct idxd_device *idxd) group->tc_b = group->grpcfg.flags.tc_b = 1; else group->grpcfg.flags.tc_b = group->tc_b; - group->grpcfg.flags.use_token_limit = group->use_token_limit; - group->grpcfg.flags.tokens_reserved = group->tokens_reserved; - if (group->tokens_allowed) - group->grpcfg.flags.tokens_allowed = - group->tokens_allowed; - else - group->grpcfg.flags.tokens_allowed = idxd->max_tokens; + group->grpcfg.flags.use_rdbuf_limit = group->use_rdbuf_limit; + group->grpcfg.flags.rdbufs_reserved = group->rdbufs_reserved; + group->grpcfg.flags.rdbufs_allowed = group->rdbufs_allowed; + group->grpcfg.flags.desc_progress_limit = group->desc_progress_limit; + group->grpcfg.flags.batch_progress_limit = group->batch_progress_limit; } } @@ -927,10 +1099,9 @@ static int idxd_wqs_setup(struct idxd_device *idxd) if (!wq->group) continue; - if (!wq->size) - continue; - if (wq_shared(wq) && !device_swq_supported(idxd)) { + if (wq_shared(wq) && !wq_shared_supported(wq)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_SWQ_SUPPORT; dev_warn(dev, "No shared wq support but configured.\n"); return -EINVAL; } @@ -939,8 +1110,10 @@ static int idxd_wqs_setup(struct idxd_device *idxd) configured++; } - if (configured == 0) + if (configured == 0) { + idxd->cmd_status = IDXD_SCMD_WQ_NONE_CONFIGURED; return -EINVAL; + } return 0; } @@ -983,8 +1156,6 @@ static int idxd_wq_load_config(struct idxd_wq *wq) wq->size = wq->wqcfg->wq_size; wq->threshold = wq->wqcfg->wq_thresh; - if (wq->wqcfg->priv) - wq->type = IDXD_WQT_KERNEL; /* The driver does not support shared WQ mode in read-only config yet */ if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en) @@ -994,6 +1165,9 @@ static int idxd_wq_load_config(struct idxd_wq *wq) wq->priority = wq->wqcfg->priority; + wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift; + idxd_wq_set_max_batch_size(idxd->data->type, wq, 1U << wq->wqcfg->max_batch_shift); + for (i = 0; i < WQCFG_STRIDES(idxd); i++) { wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i); dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]); @@ -1057,8 +1231,8 @@ static void idxd_group_load_config(struct idxd_group *group) } grpcfg_offset = GRPFLGCFG_OFFSET(idxd, group->id); - group->grpcfg.flags.bits = ioread32(idxd->reg_base + grpcfg_offset); - dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#x\n", + group->grpcfg.flags.bits = ioread64(idxd->reg_base + grpcfg_offset); + dev_dbg(dev, "GRPFLAGS flags[%d: %#x]: %#llx\n", group->id, grpcfg_offset, group->grpcfg.flags.bits); } @@ -1068,7 +1242,7 @@ int idxd_device_load_config(struct idxd_device *idxd) int i, rc; reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); - idxd->token_limit = reg.token_limit; + idxd->rdbuf_limit = reg.rdbuf_limit; for (i = 0; i < idxd->max_groups; i++) { struct idxd_group *group = idxd->groups[i]; @@ -1086,3 +1260,370 @@ int idxd_device_load_config(struct idxd_device *idxd) return 0; } + +static void idxd_flush_pending_descs(struct idxd_irq_entry *ie) +{ + struct idxd_desc *desc, *itr; + struct llist_node *head; + LIST_HEAD(flist); + enum idxd_complete_type ctype; + + spin_lock(&ie->list_lock); + head = llist_del_all(&ie->pending_llist); + if (head) { + llist_for_each_entry_safe(desc, itr, head, llnode) + list_add_tail(&desc->list, &ie->work_list); + } + + list_for_each_entry_safe(desc, itr, &ie->work_list, list) + list_move_tail(&desc->list, &flist); + spin_unlock(&ie->list_lock); + + list_for_each_entry_safe(desc, itr, &flist, list) { + struct dma_async_tx_descriptor *tx; + + list_del(&desc->list); + ctype = desc->completion->status ? IDXD_COMPLETE_NORMAL : IDXD_COMPLETE_ABORT; + /* + * wq is being disabled. Any remaining descriptors are + * likely to be stuck and can be dropped. callback could + * point to code that is no longer accessible, for example + * if dmatest module has been unloaded. + */ + tx = &desc->txd; + tx->callback = NULL; + tx->callback_result = NULL; + idxd_dma_complete_txd(desc, ctype, true, NULL, NULL); + } +} + +static void idxd_device_set_perm_entry(struct idxd_device *idxd, + struct idxd_irq_entry *ie) +{ + union msix_perm mperm; + + if (ie->pasid == IOMMU_PASID_INVALID) + return; + + mperm.bits = 0; + mperm.pasid = ie->pasid; + mperm.pasid_en = 1; + iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + ie->id * 8); +} + +static void idxd_device_clear_perm_entry(struct idxd_device *idxd, + struct idxd_irq_entry *ie) +{ + iowrite32(0, idxd->reg_base + idxd->msix_perm_offset + ie->id * 8); +} + +void idxd_wq_free_irq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct idxd_irq_entry *ie = &wq->ie; + + if (wq->type != IDXD_WQT_KERNEL) + return; + + free_irq(ie->vector, ie); + idxd_flush_pending_descs(ie); + if (idxd->request_int_handles) + idxd_device_release_int_handle(idxd, ie->int_handle, IDXD_IRQ_MSIX); + idxd_device_clear_perm_entry(idxd, ie); + ie->vector = -1; + ie->int_handle = INVALID_INT_HANDLE; + ie->pasid = IOMMU_PASID_INVALID; +} + +int idxd_wq_request_irq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct idxd_irq_entry *ie; + int rc; + + if (wq->type != IDXD_WQT_KERNEL) + return 0; + + ie = &wq->ie; + ie->vector = pci_irq_vector(pdev, ie->id); + ie->pasid = device_pasid_enabled(idxd) ? idxd->pasid : IOMMU_PASID_INVALID; + idxd_device_set_perm_entry(idxd, ie); + + rc = request_threaded_irq(ie->vector, NULL, idxd_wq_thread, 0, "idxd-portal", ie); + if (rc < 0) { + dev_err(dev, "Failed to request irq %d.\n", ie->vector); + goto err_irq; + } + + if (idxd->request_int_handles) { + rc = idxd_device_request_int_handle(idxd, ie->id, &ie->int_handle, + IDXD_IRQ_MSIX); + if (rc < 0) + goto err_int_handle; + } else { + ie->int_handle = ie->id; + } + + return 0; + +err_int_handle: + ie->int_handle = INVALID_INT_HANDLE; + free_irq(ie->vector, ie); +err_irq: + idxd_device_clear_perm_entry(idxd, ie); + ie->pasid = IOMMU_PASID_INVALID; + return rc; +} + +int idxd_drv_enable_wq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct device *dev = &idxd->pdev->dev; + int rc = -ENXIO; + + lockdep_assert_held(&wq->wq_lock); + + if (idxd->state != IDXD_DEV_ENABLED) { + idxd->cmd_status = IDXD_SCMD_DEV_NOT_ENABLED; + goto err; + } + + if (wq->state != IDXD_WQ_DISABLED) { + dev_dbg(dev, "wq %d already enabled.\n", wq->id); + idxd->cmd_status = IDXD_SCMD_WQ_ENABLED; + rc = -EBUSY; + goto err; + } + + if (!wq->group) { + dev_dbg(dev, "wq %d not attached to group.\n", wq->id); + idxd->cmd_status = IDXD_SCMD_WQ_NO_GRP; + goto err; + } + + if (strlen(wq->name) == 0) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_NAME; + dev_dbg(dev, "wq %d name not set.\n", wq->id); + goto err; + } + + /* Shared WQ checks */ + if (wq_shared(wq)) { + if (!wq_shared_supported(wq)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_SVM; + dev_dbg(dev, "PASID not enabled and shared wq.\n"); + goto err; + } + /* + * Shared wq with the threshold set to 0 means the user + * did not set the threshold or transitioned from a + * dedicated wq but did not set threshold. A value + * of 0 would effectively disable the shared wq. The + * driver does not allow a value of 0 to be set for + * threshold via sysfs. + */ + if (wq->threshold == 0) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_THRESH; + dev_dbg(dev, "Shared wq and threshold 0.\n"); + goto err; + } + } + + /* + * In the event that the WQ is configurable for pasid, the driver + * should setup the pasid, pasid_en bit. This is true for both kernel + * and user shared workqueues. There is no need to setup priv bit in + * that in-kernel DMA will also do user privileged requests. + * A dedicated wq that is not 'kernel' type will configure pasid and + * pasid_en later on so there is no need to setup. + */ + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + if (wq_pasid_enabled(wq)) { + if (is_idxd_wq_kernel(wq) || wq_shared(wq)) { + u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0; + + __idxd_wq_set_pasid_locked(wq, pasid); + } + } + } + + rc = 0; + spin_lock(&idxd->dev_lock); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + rc = idxd_device_config(idxd); + spin_unlock(&idxd->dev_lock); + if (rc < 0) { + dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc); + goto err; + } + + rc = idxd_wq_enable(wq); + if (rc < 0) { + dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc); + goto err; + } + + rc = idxd_wq_map_portal(wq); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_WQ_PORTAL_ERR; + dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc); + goto err_map_portal; + } + + wq->client_count = 0; + + rc = idxd_wq_request_irq(wq); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_WQ_IRQ_ERR; + dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc); + goto err_irq; + } + + rc = idxd_wq_alloc_resources(wq); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_WQ_RES_ALLOC_ERR; + dev_dbg(dev, "WQ resource alloc failed\n"); + goto err_res_alloc; + } + + rc = idxd_wq_init_percpu_ref(wq); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_PERCPU_ERR; + dev_dbg(dev, "percpu_ref setup failed\n"); + goto err_ref; + } + + return 0; + +err_ref: + idxd_wq_free_resources(wq); +err_res_alloc: + idxd_wq_free_irq(wq); +err_irq: + idxd_wq_unmap_portal(wq); +err_map_portal: + if (idxd_wq_disable(wq, false)) + dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq))); +err: + return rc; +} +EXPORT_SYMBOL_NS_GPL(idxd_drv_enable_wq, "IDXD"); + +void idxd_drv_disable_wq(struct idxd_wq *wq) +{ + struct idxd_device *idxd = wq->idxd; + struct device *dev = &idxd->pdev->dev; + + lockdep_assert_held(&wq->wq_lock); + + if (idxd_wq_refcount(wq)) + dev_warn(dev, "Clients has claim on wq %d: %d\n", + wq->id, idxd_wq_refcount(wq)); + + idxd_wq_unmap_portal(wq); + idxd_wq_drain(wq); + idxd_wq_free_irq(wq); + idxd_wq_reset(wq); + idxd_wq_free_resources(wq); + percpu_ref_exit(&wq->wq_active); + wq->type = IDXD_WQT_NONE; + wq->client_count = 0; +} +EXPORT_SYMBOL_NS_GPL(idxd_drv_disable_wq, "IDXD"); + +int idxd_device_drv_probe(struct idxd_dev *idxd_dev) +{ + struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); + int rc = 0; + + /* + * Device should be in disabled state for the idxd_drv to load. If it's in + * enabled state, then the device was altered outside of driver's control. + * If the state is in halted state, then we don't want to proceed. + */ + if (idxd->state != IDXD_DEV_DISABLED) { + idxd->cmd_status = IDXD_SCMD_DEV_ENABLED; + return -ENXIO; + } + + /* Device configuration */ + spin_lock(&idxd->dev_lock); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + rc = idxd_device_config(idxd); + spin_unlock(&idxd->dev_lock); + if (rc < 0) + return -ENXIO; + + /* + * System PASID is preserved across device disable/enable cycle, but + * genconfig register content gets cleared during device reset. We + * need to re-enable user interrupts for kernel work queue completion + * IRQ to function. + */ + if (idxd->pasid != IOMMU_PASID_INVALID) + idxd_set_user_intr(idxd, 1); + + rc = idxd_device_evl_setup(idxd); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_DEV_EVL_ERR; + return rc; + } + + /* Start device */ + rc = idxd_device_enable(idxd); + if (rc < 0) { + idxd_device_evl_free(idxd); + return rc; + } + + /* Setup DMA device without channels */ + rc = idxd_register_dma_device(idxd); + if (rc < 0) { + idxd_device_disable(idxd); + idxd_device_evl_free(idxd); + idxd->cmd_status = IDXD_SCMD_DEV_DMA_ERR; + return rc; + } + + idxd->cmd_status = 0; + return 0; +} + +void idxd_device_drv_remove(struct idxd_dev *idxd_dev) +{ + struct device *dev = &idxd_dev->conf_dev; + struct idxd_device *idxd = idxd_dev_to_idxd(idxd_dev); + int i; + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *wq = idxd->wqs[i]; + struct device *wq_dev = wq_confdev(wq); + + if (wq->state == IDXD_WQ_DISABLED) + continue; + dev_warn(dev, "Active wq %d on disable %s.\n", i, dev_name(wq_dev)); + device_release_driver(wq_dev); + } + + idxd_unregister_dma_device(idxd); + idxd_device_disable(idxd); + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + idxd_device_reset(idxd); + idxd_device_evl_free(idxd); +} + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_DSA, + IDXD_DEV_IAX, + IDXD_DEV_NONE, +}; + +struct idxd_device_driver idxd_drv = { + .type = dev_types, + .probe = idxd_device_drv_probe, + .remove = idxd_device_drv_remove, + .name = "idxd", +}; +EXPORT_SYMBOL_GPL(idxd_drv); diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c index 77439b645044..dbecd699237e 100644 --- a/drivers/dma/idxd/dma.c +++ b/drivers/dma/idxd/dma.c @@ -21,20 +21,27 @@ static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c) } void idxd_dma_complete_txd(struct idxd_desc *desc, - enum idxd_complete_type comp_type) + enum idxd_complete_type comp_type, + bool free_desc, void *ctx, u32 *status) { + struct idxd_device *idxd = desc->wq->idxd; struct dma_async_tx_descriptor *tx; struct dmaengine_result res; int complete = 1; - if (desc->completion->status == DSA_COMP_SUCCESS) + if (desc->completion->status == DSA_COMP_SUCCESS) { res.result = DMA_TRANS_NOERROR; - else if (desc->completion->status) + } else if (desc->completion->status) { + if (idxd->request_int_handles && comp_type != IDXD_COMPLETE_ABORT && + desc->completion->status == DSA_COMP_INT_HANDLE_INVAL && + idxd_queue_int_handle_resubmit(desc)) + return; res.result = DMA_TRANS_WRITE_FAILED; - else if (comp_type == IDXD_COMPLETE_ABORT) + } else if (comp_type == IDXD_COMPLETE_ABORT) { res.result = DMA_TRANS_ABORTED; - else + } else { complete = 0; + } tx = &desc->txd; if (complete && tx->cookie) { @@ -44,6 +51,9 @@ void idxd_dma_complete_txd(struct idxd_desc *desc, tx->callback = NULL; tx->callback_result = NULL; } + + if (free_desc) + idxd_free_desc(desc->wq, desc); } static void op_flag_setup(unsigned long flags, u32 *desc_flags) @@ -53,12 +63,6 @@ static void op_flag_setup(unsigned long flags, u32 *desc_flags) *desc_flags |= IDXD_OP_FLAG_RCI; } -static inline void set_completion_address(struct idxd_desc *desc, - u64 *compl_addr) -{ - *compl_addr = desc->compl_dma; -} - static inline void idxd_prep_desc_common(struct idxd_wq *wq, struct dsa_hw_desc *hw, char opcode, u64 addr_f1, u64 addr_f2, u64 len, @@ -69,11 +73,37 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq, hw->src_addr = addr_f1; hw->dst_addr = addr_f2; hw->xfer_size = len; - hw->priv = !!(wq->type == IDXD_WQT_KERNEL); + /* + * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv + * field instead. This field should be set to 0 for kernel descriptors + * since kernel DMA on VT-d supports "user" privilege only. + */ + hw->priv = 0; hw->completion_addr = compl; } static struct dma_async_tx_descriptor * +idxd_dma_prep_interrupt(struct dma_chan *c, unsigned long flags) +{ + struct idxd_wq *wq = to_idxd_wq(c); + u32 desc_flags; + struct idxd_desc *desc; + + if (wq->state != IDXD_WQ_ENABLED) + return NULL; + + op_flag_setup(flags, &desc_flags); + desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK); + if (IS_ERR(desc)) + return NULL; + + idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_NOOP, + 0, 0, 0, desc->compl_dma, desc_flags); + desc->txd.flags = flags; + return &desc->txd; +} + +static struct dma_async_tx_descriptor * idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest, dma_addr_t dma_src, size_t len, unsigned long flags) { @@ -179,10 +209,12 @@ int idxd_register_dma_device(struct idxd_device *idxd) INIT_LIST_HEAD(&dma->channels); dma->dev = dev; + dma_cap_set(DMA_INTERRUPT, dma->cap_mask); dma_cap_set(DMA_PRIVATE, dma->cap_mask); dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask); dma->device_release = idxd_dma_release; + dma->device_prep_dma_interrupt = idxd_dma_prep_interrupt; if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) { dma_cap_set(DMA_MEMCPY, dma->cap_mask); dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy; @@ -213,7 +245,7 @@ void idxd_unregister_dma_device(struct idxd_device *idxd) dma_async_device_unregister(&idxd->idxd_dma->dma); } -int idxd_register_dma_channel(struct idxd_wq *wq) +static int idxd_register_dma_channel(struct idxd_wq *wq) { struct idxd_device *idxd = wq->idxd; struct dma_device *dma = &idxd->idxd_dma->dma; @@ -237,7 +269,7 @@ int idxd_register_dma_channel(struct idxd_wq *wq) desc->txd.tx_submit = idxd_dma_tx_submit; } - rc = dma_async_device_channel_register(dma, chan); + rc = dma_async_device_channel_register(dma, chan, NULL); if (rc < 0) { kfree(idxd_chan); return rc; @@ -245,12 +277,12 @@ int idxd_register_dma_channel(struct idxd_wq *wq) wq->idxd_chan = idxd_chan; idxd_chan->wq = wq; - get_device(&wq->conf_dev); + get_device(wq_confdev(wq)); return 0; } -void idxd_unregister_dma_channel(struct idxd_wq *wq) +static void idxd_unregister_dma_channel(struct idxd_wq *wq) { struct idxd_dma_chan *idxd_chan = wq->idxd_chan; struct dma_chan *chan = &idxd_chan->chan; @@ -260,5 +292,75 @@ void idxd_unregister_dma_channel(struct idxd_wq *wq) list_del(&chan->device_node); kfree(wq->idxd_chan); wq->idxd_chan = NULL; - put_device(&wq->conf_dev); + put_device(wq_confdev(wq)); +} + +static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev) +{ + struct device *dev = &idxd_dev->conf_dev; + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + struct idxd_device *idxd = wq->idxd; + int rc; + + if (idxd->state != IDXD_DEV_ENABLED) + return -ENXIO; + + mutex_lock(&wq->wq_lock); + if (!idxd_wq_driver_name_match(wq, dev)) { + idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; + rc = -ENODEV; + goto err; + } + + wq->type = IDXD_WQT_KERNEL; + + rc = idxd_drv_enable_wq(wq); + if (rc < 0) { + dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc); + rc = -ENXIO; + goto err; + } + + rc = idxd_register_dma_channel(wq); + if (rc < 0) { + idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR; + dev_dbg(dev, "Failed to register dma channel\n"); + goto err_dma; + } + + idxd->cmd_status = 0; + mutex_unlock(&wq->wq_lock); + return 0; + +err_dma: + idxd_drv_disable_wq(wq); +err: + wq->type = IDXD_WQT_NONE; + mutex_unlock(&wq->wq_lock); + return rc; +} + +static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev) +{ + struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); + + mutex_lock(&wq->wq_lock); + __idxd_wq_quiesce(wq); + idxd_unregister_dma_channel(wq); + idxd_drv_disable_wq(wq); + mutex_unlock(&wq->wq_lock); } + +static enum idxd_dev_type dev_types[] = { + IDXD_DEV_WQ, + IDXD_DEV_NONE, +}; + +struct idxd_device_driver idxd_dmaengine_drv = { + .probe = idxd_dmaengine_drv_probe, + .remove = idxd_dmaengine_drv_remove, + .desc_complete = idxd_dma_complete_txd, + .name = "dmaengine", + .type = dev_types, +}; +EXPORT_SYMBOL_GPL(idxd_dmaengine_drv); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 26482c7d4c3a..74e6695881e6 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -10,15 +10,36 @@ #include <linux/cdev.h> #include <linux/idr.h> #include <linux/pci.h> +#include <linux/bitmap.h> #include <linux/perf_event.h> +#include <linux/iommu.h> +#include <linux/crypto.h> +#include <uapi/linux/idxd.h> #include "registers.h" #define IDXD_DRIVER_VERSION "1.00" -extern struct kmem_cache *idxd_desc_pool; +extern bool tc_override; -struct idxd_device; struct idxd_wq; +struct idxd_dev; + +enum idxd_dev_type { + IDXD_DEV_NONE = -1, + IDXD_DEV_DSA = 0, + IDXD_DEV_IAX, + IDXD_DEV_WQ, + IDXD_DEV_GROUP, + IDXD_DEV_ENGINE, + IDXD_DEV_CDEV, + IDXD_DEV_CDEV_FILE, + IDXD_DEV_MAX_TYPE, +}; + +struct idxd_dev { + struct device conf_dev; + enum idxd_dev_type type; +}; #define IDXD_REG_TIMEOUT 50 #define IDXD_DRAIN_TIMEOUT 5000 @@ -33,12 +54,36 @@ enum idxd_type { #define IDXD_NAME_SIZE 128 #define IDXD_PMU_EVENT_MAX 64 +#define IDXD_ENQCMDS_RETRIES 32 +#define IDXD_ENQCMDS_MAX_RETRIES 64 + +enum idxd_complete_type { + IDXD_COMPLETE_NORMAL = 0, + IDXD_COMPLETE_ABORT, + IDXD_COMPLETE_DEV_FAIL, +}; + +struct idxd_desc; + struct idxd_device_driver { + const char *name; + enum idxd_dev_type *type; + int (*probe)(struct idxd_dev *idxd_dev); + void (*remove)(struct idxd_dev *idxd_dev); + void (*desc_complete)(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc, + void *ctx, u32 *status); struct device_driver drv; }; +extern struct idxd_device_driver dsa_drv; +extern struct idxd_device_driver idxd_drv; +extern struct idxd_device_driver idxd_dmaengine_drv; +extern struct idxd_device_driver idxd_user_drv; + +#define INVALID_INT_HANDLE -1 struct idxd_irq_entry { - struct idxd_device *idxd; int id; int vector; struct llist_head pending_llist; @@ -48,20 +93,24 @@ struct idxd_irq_entry { * and irq thread processing error descriptor. */ spinlock_t list_lock; + int int_handle; + ioasid_t pasid; }; struct idxd_group { - struct device conf_dev; + struct idxd_dev idxd_dev; struct idxd_device *idxd; struct grpcfg grpcfg; int id; int num_engines; int num_wqs; - bool use_token_limit; - u8 tokens_allowed; - u8 tokens_reserved; + bool use_rdbuf_limit; + u8 rdbufs_allowed; + u8 rdbufs_reserved; int tc_a; int tc_b; + int desc_progress_limit; + int batch_progress_limit; }; struct idxd_pmu { @@ -74,7 +123,6 @@ struct idxd_pmu { struct pmu pmu; char name[IDXD_NAME_SIZE]; - int cpu; int n_counters; int counter_width; @@ -85,12 +133,16 @@ struct idxd_pmu { unsigned long supported_filters; int n_filters; - - struct hlist_node cpuhp_node; }; #define IDXD_MAX_PRIORITY 0xf +enum { + COUNTER_FAULTS = 0, + COUNTER_FAULT_FAILS, + COUNTER_MAX +}; + enum idxd_wq_state { IDXD_WQ_DISABLED = 0, IDXD_WQ_ENABLED, @@ -99,6 +151,8 @@ enum idxd_wq_state { enum idxd_wq_flag { WQ_FLAG_DEDICATED = 0, WQ_FLAG_BLOCK_ON_FAULT, + WQ_FLAG_ATS_DISABLE, + WQ_FLAG_PRS_DISABLE, }; enum idxd_wq_type { @@ -110,25 +164,24 @@ enum idxd_wq_type { struct idxd_cdev { struct idxd_wq *wq; struct cdev cdev; - struct device dev; + struct idxd_dev idxd_dev; int minor; }; -#define IDXD_ALLOCATED_BATCH_SIZE 128U +#define DRIVER_NAME_SIZE 128 + #define WQ_NAME_SIZE 1024 #define WQ_TYPE_SIZE 10 +#define WQ_DEFAULT_QUEUE_DEPTH 16 +#define WQ_DEFAULT_MAX_XFER SZ_2M +#define WQ_DEFAULT_MAX_BATCH 32 + enum idxd_op_type { IDXD_OP_BLOCK = 0, IDXD_OP_NONBLOCK = 1, }; -enum idxd_complete_type { - IDXD_COMPLETE_NORMAL = 0, - IDXD_COMPLETE_ABORT, - IDXD_COMPLETE_DEV_FAIL, -}; - struct idxd_dma_chan { struct dma_chan chan; struct idxd_wq *wq; @@ -136,13 +189,18 @@ struct idxd_dma_chan { struct idxd_wq { void __iomem *portal; + u32 portal_offset; + unsigned int enqcmds_retries; struct percpu_ref wq_active; struct completion wq_dead; - struct device conf_dev; + struct completion wq_resurrect; + struct idxd_dev idxd_dev; struct idxd_cdev *idxd_cdev; struct wait_queue_head err_queue; + struct workqueue_struct *wq; struct idxd_device *idxd; int id; + struct idxd_irq_entry ie; enum idxd_wq_type type; struct idxd_group *group; int client_count; @@ -153,16 +211,15 @@ struct idxd_wq { enum idxd_wq_state state; unsigned long flags; union wqcfg *wqcfg; - u32 vec_ptr; /* interrupt steering */ + unsigned long *opcap_bmap; + struct dsa_hw_desc **hw_descs; int num_descs; union { struct dsa_completion_record *compls; struct iax_completion_record *iax_compls; }; - void *compls_raw; dma_addr_t compls_addr; - dma_addr_t compls_addr_raw; int compls_size; struct idxd_desc **descs; struct sbitmap_queue sbq; @@ -170,11 +227,16 @@ struct idxd_wq { char name[WQ_NAME_SIZE + 1]; u64 max_xfer_bytes; u32 max_batch_size; - bool ats_dis; + + /* Lock to protect upasid_xa access. */ + struct mutex uc_lock; + struct xarray upasid_xa; + + char driver_name[DRIVER_NAME_SIZE + 1]; }; struct idxd_engine { - struct device conf_dev; + struct idxd_dev idxd_dev; int id; struct idxd_group *group; struct idxd_device *idxd; @@ -189,12 +251,12 @@ struct idxd_hw { union engine_cap_reg engine_cap; struct opcap opcap; u32 cmd_cap; + union iaa_cap_reg iaa_cap; }; enum idxd_device_state { IDXD_DEV_HALTED = -1, IDXD_DEV_DISABLED = 0, - IDXD_DEV_CONF_READY, IDXD_DEV_ENABLED, }; @@ -202,6 +264,7 @@ enum idxd_device_flag { IDXD_FLAG_CONFIGURABLE = 0, IDXD_FLAG_CMD_RUNNING, IDXD_FLAG_PASID_ENABLED, + IDXD_FLAG_USER_PASID_ENABLED, }; struct idxd_dma_dev { @@ -209,16 +272,45 @@ struct idxd_dma_dev { struct dma_device dma; }; +typedef int (*load_device_defaults_fn_t) (struct idxd_device *idxd); + struct idxd_driver_data { const char *name_prefix; enum idxd_type type; - struct device_type *dev_type; + const struct device_type *dev_type; int compl_size; int align; + int evl_cr_off; + int cr_status_off; + int cr_result_off; + bool user_submission_safe; + load_device_defaults_fn_t load_device_defaults; +}; + +struct idxd_evl { + /* Lock to protect event log access. */ + struct mutex lock; + void *log; + dma_addr_t dma; + /* Total size of event log = number of entries * entry size. */ + unsigned int log_size; + /* The number of entries in the event log. */ + u16 size; + unsigned long *bmap; + bool batch_fail[IDXD_MAX_BATCH_IDENT]; +}; + +struct idxd_evl_fault { + struct work_struct work; + struct idxd_wq *wq; + u8 status; + + /* make this last member always */ + struct __evl_entry entry[]; }; struct idxd_device { - struct device conf_dev; + struct idxd_dev idxd_dev; struct idxd_driver_data *data; struct list_head list; struct idxd_hw hw; @@ -226,7 +318,8 @@ struct idxd_device { unsigned long flags; int id; int major; - u8 cmd_status; + u32 cmd_status; + struct idxd_irq_entry ie; /* misc irq, msix 0 */ struct pci_dev *pdev; void __iomem *reg_base; @@ -242,6 +335,8 @@ struct idxd_device { unsigned int pasid; int num_groups; + int irq_cnt; + bool request_int_handles; u32 msix_perm_offset; u32 wqcfg_offset; @@ -252,25 +347,61 @@ struct idxd_device { u32 max_batch_size; int max_groups; int max_engines; - int max_tokens; + int max_rdbufs; int max_wqs; int max_wq_size; - int token_limit; - int nr_tokens; /* non-reserved tokens */ + int rdbuf_limit; + int nr_rdbufs; /* non-reserved read buffers */ unsigned int wqcfg_size; + unsigned long *wq_enable_map; union sw_err_reg sw_err; wait_queue_head_t cmd_waitq; - int num_wq_irqs; - struct idxd_irq_entry *irq_entries; struct idxd_dma_dev *idxd_dma; struct workqueue_struct *wq; struct work_struct work; - int *int_handles; - struct idxd_pmu *idxd_pmu; + + unsigned long *opcap_bmap; + struct idxd_evl *evl; + struct kmem_cache *evl_cache; + + struct dentry *dbgfs_dir; + struct dentry *dbgfs_evl_file; + + bool user_submission_safe; + + struct idxd_saved_states *idxd_saved; +}; + +struct idxd_saved_states { + struct idxd_device saved_idxd; + struct idxd_evl saved_evl; + struct idxd_engine **saved_engines; + struct idxd_wq **saved_wqs; + struct idxd_group **saved_groups; + unsigned long *saved_wq_enable_map; +}; + +static inline unsigned int evl_ent_size(struct idxd_device *idxd) +{ + return idxd->hw.gen_cap.evl_support ? + (32 * (1 << idxd->hw.gen_cap.evl_support)) : 0; +} + +static inline unsigned int evl_size(struct idxd_device *idxd) +{ + return idxd->evl->size * evl_ent_size(idxd); +} + +struct crypto_ctx { + struct acomp_req *req; + struct crypto_tfm *tfm; + dma_addr_t src_addr; + dma_addr_t dst_addr; + bool compress; }; /* IDXD software descriptor */ @@ -285,47 +416,142 @@ struct idxd_desc { struct iax_completion_record *iax_completion; }; dma_addr_t compl_dma; - struct dma_async_tx_descriptor txd; + union { + struct dma_async_tx_descriptor txd; + struct crypto_ctx crypto; + }; struct llist_node llnode; struct list_head list; int id; int cpu; - unsigned int vector; struct idxd_wq *wq; }; -#define confdev_to_idxd(dev) container_of(dev, struct idxd_device, conf_dev) -#define confdev_to_wq(dev) container_of(dev, struct idxd_wq, conf_dev) +/* + * This is software defined error for the completion status. We overload the error code + * that will never appear in completion status and only SWERR register. + */ +enum idxd_completion_status { + IDXD_COMP_DESC_ABORT = 0xff, +}; + +#define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev +#define wq_confdev(wq) &wq->idxd_dev.conf_dev +#define engine_confdev(engine) &engine->idxd_dev.conf_dev +#define group_confdev(group) &group->idxd_dev.conf_dev +#define cdev_dev(cdev) &cdev->idxd_dev.conf_dev +#define user_ctx_dev(ctx) (&(ctx)->idxd_dev.conf_dev) + +#define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev) +#define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev) +#define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev) + +static inline struct idxd_device_driver *wq_to_idxd_drv(struct idxd_wq *wq) +{ + struct device *dev = wq_confdev(wq); + struct idxd_device_driver *idxd_drv = + container_of(dev->driver, struct idxd_device_driver, drv); + + return idxd_drv; +} + +static inline struct idxd_device *confdev_to_idxd(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return idxd_dev_to_idxd(idxd_dev); +} + +static inline struct idxd_wq *confdev_to_wq(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return idxd_dev_to_wq(idxd_dev); +} + +static inline struct idxd_engine *confdev_to_engine(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_engine, idxd_dev); +} + +static inline struct idxd_group *confdev_to_group(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); -extern struct bus_type dsa_bus_type; -extern struct bus_type iax_bus_type; + return container_of(idxd_dev, struct idxd_group, idxd_dev); +} + +static inline struct idxd_cdev *dev_to_cdev(struct device *dev) +{ + struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); + + return container_of(idxd_dev, struct idxd_cdev, idxd_dev); +} + +static inline void idxd_dev_set_type(struct idxd_dev *idev, int type) +{ + if (type >= IDXD_DEV_MAX_TYPE) { + idev->type = IDXD_DEV_NONE; + return; + } + + idev->type = type; +} + +static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx) +{ + return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie; +} + +static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie) +{ + return container_of(ie, struct idxd_wq, ie); +} + +static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie) +{ + return container_of(ie, struct idxd_device, ie); +} + +static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable) +{ + union gencfg_reg reg; + + reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET); + reg.user_int_en = enable; + iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET); +} + +extern const struct bus_type dsa_bus_type; extern bool support_enqcmd; extern struct ida idxd_ida; -extern struct device_type dsa_device_type; -extern struct device_type iax_device_type; -extern struct device_type idxd_wq_device_type; -extern struct device_type idxd_engine_device_type; -extern struct device_type idxd_group_device_type; +extern const struct device_type dsa_device_type; +extern const struct device_type iax_device_type; +extern const struct device_type idxd_wq_device_type; +extern const struct device_type idxd_engine_device_type; +extern const struct device_type idxd_group_device_type; -static inline bool is_dsa_dev(struct device *dev) +static inline bool is_dsa_dev(struct idxd_dev *idxd_dev) { - return dev->type == &dsa_device_type; + return idxd_dev->type == IDXD_DEV_DSA; } -static inline bool is_iax_dev(struct device *dev) +static inline bool is_iax_dev(struct idxd_dev *idxd_dev) { - return dev->type == &iax_device_type; + return idxd_dev->type == IDXD_DEV_IAX; } -static inline bool is_idxd_dev(struct device *dev) +static inline bool is_idxd_dev(struct idxd_dev *idxd_dev) { - return is_dsa_dev(dev) || is_iax_dev(dev); + return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev); } -static inline bool is_idxd_wq_dev(struct device *dev) +static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev) { - return dev->type == &idxd_wq_device_type; + return idxd_dev->type == IDXD_DEV_WQ; } static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) @@ -335,11 +561,16 @@ static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq) return false; } -static inline bool is_idxd_wq_cdev(struct idxd_wq *wq) +static inline bool is_idxd_wq_user(struct idxd_wq *wq) { return wq->type == IDXD_WQT_USER; } +static inline bool is_idxd_wq_kernel(struct idxd_wq *wq) +{ + return wq->type == IDXD_WQT_KERNEL; +} + static inline bool wq_dedicated(struct idxd_wq *wq) { return test_bit(WQ_FLAG_DEDICATED, &wq->flags); @@ -355,9 +586,20 @@ static inline bool device_pasid_enabled(struct idxd_device *idxd) return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); } -static inline bool device_swq_supported(struct idxd_device *idxd) +static inline bool device_user_pasid_enabled(struct idxd_device *idxd) +{ + return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); +} + +static inline bool wq_pasid_enabled(struct idxd_wq *wq) +{ + return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) || + (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd)); +} + +static inline bool wq_shared_supported(struct idxd_wq *wq) { - return (support_enqcmd && device_pasid_enabled(idxd)); + return (support_enqcmd && wq_pasid_enabled(wq)); } enum idxd_portal_prot { @@ -381,6 +623,24 @@ static inline int idxd_get_wq_portal_full_offset(int wq_id, return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot); } +#define IDXD_PORTAL_MASK (PAGE_SIZE - 1) + +/* + * Even though this function can be accessed by multiple threads, it is safe to use. + * At worst the address gets used more than once before it gets incremented. We don't + * hit a threshold until iops becomes many million times a second. So the occasional + * reuse of the same address is tolerable compare to using an atomic variable. This is + * safe on a system that has atomic load/store for 32bit integers. Given that this is an + * Intel iEP device, that should not be a problem. + */ +static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq) +{ + int ofs = wq->portal_offset; + + wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK; + return wq->portal + ofs; +} + static inline void idxd_wq_get(struct idxd_wq *wq) { wq->client_count++; @@ -396,33 +656,110 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; -int idxd_register_bus_type(void); -void idxd_unregister_bus_type(void); +static inline void idxd_wq_set_private(struct idxd_wq *wq, void *private) +{ + dev_set_drvdata(wq_confdev(wq), private); +} + +static inline void *idxd_wq_get_private(struct idxd_wq *wq) +{ + return dev_get_drvdata(wq_confdev(wq)); +} + +/* + * Intel IAA does not support batch processing. + * The max batch size of device, max batch size of wq and + * max batch shift of wqcfg should be always 0 on IAA. + */ +static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd, + u32 max_batch_size) +{ + if (idxd_type == IDXD_TYPE_IAX) + idxd->max_batch_size = 0; + else + idxd->max_batch_size = max_batch_size; +} + +static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq, + u32 max_batch_size) +{ + if (idxd_type == IDXD_TYPE_IAX) + wq->max_batch_size = 0; + else + wq->max_batch_size = max_batch_size; +} + +static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg, + u32 max_batch_shift) +{ + if (idxd_type == IDXD_TYPE_IAX) + wqcfg->max_batch_shift = 0; + else + wqcfg->max_batch_shift = max_batch_shift; +} + +static inline int idxd_wq_driver_name_match(struct idxd_wq *wq, struct device *dev) +{ + return (strncmp(wq->driver_name, dev->driver->name, strlen(dev->driver->name)) == 0); +} + +#define MODULE_ALIAS_IDXD_DEVICE(type) MODULE_ALIAS("idxd:t" __stringify(type) "*") +#define IDXD_DEVICES_MODALIAS_FMT "idxd:t%d" + +int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, + struct module *module, const char *mod_name); +#define idxd_driver_register(driver) \ + __idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) + +void idxd_driver_unregister(struct idxd_device_driver *idxd_drv); + +#define module_idxd_driver(__idxd_driver) \ + module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister) + +void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); +void idxd_dma_complete_txd(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc, void *ctx, u32 *status); + +static inline void idxd_desc_complete(struct idxd_desc *desc, + enum idxd_complete_type comp_type, + bool free_desc) +{ + struct idxd_device_driver *drv; + u32 status; + + drv = wq_to_idxd_drv(desc->wq); + if (drv->desc_complete) + drv->desc_complete(desc, comp_type, free_desc, + &desc->txd, &status); +} + int idxd_register_devices(struct idxd_device *idxd); void idxd_unregister_devices(struct idxd_device *idxd); -int idxd_register_driver(void); -void idxd_unregister_driver(void); void idxd_wqs_quiesce(struct idxd_device *idxd); +bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc); +void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count); +int idxd_load_iaa_device_defaults(struct idxd_device *idxd); /* device interrupt control */ -void idxd_msix_perm_setup(struct idxd_device *idxd); -void idxd_msix_perm_clear(struct idxd_device *idxd); irqreturn_t idxd_misc_thread(int vec, void *data); irqreturn_t idxd_wq_thread(int irq, void *data); void idxd_mask_error_interrupts(struct idxd_device *idxd); void idxd_unmask_error_interrupts(struct idxd_device *idxd); -void idxd_mask_msix_vectors(struct idxd_device *idxd); -void idxd_mask_msix_vector(struct idxd_device *idxd, int vec_id); -void idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id); /* device control */ +int idxd_device_drv_probe(struct idxd_dev *idxd_dev); +int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, + const struct pci_device_id *id); +void idxd_device_drv_remove(struct idxd_dev *idxd_dev); +int idxd_drv_enable_wq(struct idxd_wq *wq); +void idxd_drv_disable_wq(struct idxd_wq *wq); int idxd_device_init_reset(struct idxd_device *idxd); int idxd_device_enable(struct idxd_device *idxd); int idxd_device_disable(struct idxd_device *idxd); void idxd_device_reset(struct idxd_device *idxd); -void idxd_device_cleanup(struct idxd_device *idxd); +void idxd_device_clear_state(struct idxd_device *idxd); int idxd_device_config(struct idxd_device *idxd); -void idxd_device_wqs_clear_state(struct idxd_device *idxd); void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid); int idxd_device_load_config(struct idxd_device *idxd); int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle, @@ -435,30 +772,27 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd); int idxd_wq_alloc_resources(struct idxd_wq *wq); void idxd_wq_free_resources(struct idxd_wq *wq); int idxd_wq_enable(struct idxd_wq *wq); -int idxd_wq_disable(struct idxd_wq *wq); +int idxd_wq_disable(struct idxd_wq *wq, bool reset_config); void idxd_wq_drain(struct idxd_wq *wq); void idxd_wq_reset(struct idxd_wq *wq); int idxd_wq_map_portal(struct idxd_wq *wq); void idxd_wq_unmap_portal(struct idxd_wq *wq); -void idxd_wq_disable_cleanup(struct idxd_wq *wq); int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid); int idxd_wq_disable_pasid(struct idxd_wq *wq); +void __idxd_wq_quiesce(struct idxd_wq *wq); void idxd_wq_quiesce(struct idxd_wq *wq); int idxd_wq_init_percpu_ref(struct idxd_wq *wq); +void idxd_wq_free_irq(struct idxd_wq *wq); +int idxd_wq_request_irq(struct idxd_wq *wq); /* submission */ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc); struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype); -void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc); +int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc); /* dmaengine */ int idxd_register_dma_device(struct idxd_device *idxd); void idxd_unregister_dma_device(struct idxd_device *idxd); -int idxd_register_dma_channel(struct idxd_wq *wq); -void idxd_unregister_dma_channel(struct idxd_wq *wq); -void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res); -void idxd_dma_complete_txd(struct idxd_desc *desc, - enum idxd_complete_type comp_type); /* cdev */ int idxd_cdev_register(void); @@ -466,20 +800,25 @@ void idxd_cdev_remove(void); int idxd_cdev_get_major(struct idxd_device *idxd); int idxd_wq_add_cdev(struct idxd_wq *wq); void idxd_wq_del_cdev(struct idxd_wq *wq); +int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, + void *buf, int len); +void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index); /* perfmon */ #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON) int perfmon_pmu_init(struct idxd_device *idxd); void perfmon_pmu_remove(struct idxd_device *idxd); void perfmon_counter_overflow(struct idxd_device *idxd); -void perfmon_init(void); -void perfmon_exit(void); #else static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; } static inline void perfmon_pmu_remove(struct idxd_device *idxd) {} static inline void perfmon_counter_overflow(struct idxd_device *idxd) {} -static inline void perfmon_init(void) {} -static inline void perfmon_exit(void) {} #endif +/* debugfs */ +int idxd_device_init_debugfs(struct idxd_device *idxd); +void idxd_device_remove_debugfs(struct idxd_device *idxd); +int idxd_init_debugfs(void); +void idxd_remove_debugfs(void); + #endif diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index c8ae41d36040..2acc34b3daff 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -9,12 +9,10 @@ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/workqueue.h> -#include <linux/aer.h> #include <linux/fs.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/device.h> #include <linux/idr.h> -#include <linux/intel-svm.h> #include <linux/iommu.h> #include <uapi/linux/idxd.h> #include <linux/dmaengine.h> @@ -24,13 +22,19 @@ #include "perfmon.h" MODULE_VERSION(IDXD_DRIVER_VERSION); +MODULE_DESCRIPTION("Intel Data Streaming Accelerator and In-Memory Analytics Accelerator common driver"); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Intel Corporation"); +MODULE_IMPORT_NS("IDXD"); static bool sva = true; module_param(sva, bool, 0644); MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); +bool tc_override; +module_param(tc_override, bool, 0644); +MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); + #define DRV_NAME "idxd" bool support_enqcmd; @@ -43,6 +47,10 @@ static struct idxd_driver_data idxd_driver_data[] = { .compl_size = sizeof(struct dsa_completion_record), .align = 32, .dev_type = &dsa_device_type, + .evl_cr_off = offsetof(struct dsa_evl_entry, cr), + .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ + .cr_status_off = offsetof(struct dsa_completion_record, status), + .cr_result_off = offsetof(struct dsa_completion_record, result), }, [IDXD_TYPE_IAX] = { .name_prefix = "iax", @@ -50,15 +58,30 @@ static struct idxd_driver_data idxd_driver_data[] = { .compl_size = sizeof(struct iax_completion_record), .align = 64, .dev_type = &iax_device_type, + .evl_cr_off = offsetof(struct iax_evl_entry, cr), + .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ + .cr_status_off = offsetof(struct iax_completion_record, status), + .cr_result_off = offsetof(struct iax_completion_record, error_code), + .load_device_defaults = idxd_load_iaa_device_defaults, }, }; static struct pci_device_id idxd_pci_tbl[] = { /* DSA ver 1.0 platforms */ { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, + /* DSA on GNR-D platforms */ + { PCI_DEVICE_DATA(INTEL, DSA_GNRD, &idxd_driver_data[IDXD_TYPE_DSA]) }, + /* DSA on DMR platforms */ + { PCI_DEVICE_DATA(INTEL, DSA_DMR, &idxd_driver_data[IDXD_TYPE_DSA]) }, /* IAX ver 1.0 platforms */ { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, + /* IAA on DMR platforms */ + { PCI_DEVICE_DATA(INTEL, IAA_DMR, &idxd_driver_data[IDXD_TYPE_IAX]) }, + /* IAA PTL platforms */ + { PCI_DEVICE_DATA(INTEL, IAA_PTL, &idxd_driver_data[IDXD_TYPE_IAX]) }, + /* IAA WCL platforms */ + { PCI_DEVICE_DATA(INTEL, IAA_WCL, &idxd_driver_data[IDXD_TYPE_IAX]) }, { 0, } }; MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); @@ -67,7 +90,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) { struct pci_dev *pdev = idxd->pdev; struct device *dev = &pdev->dev; - struct idxd_irq_entry *irq_entry; + struct idxd_irq_entry *ie; int i, msixcnt; int rc = 0; @@ -76,6 +99,7 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) dev_err(dev, "Not MSI-X interrupt capable.\n"); return -ENOSPC; } + idxd->irq_cnt = msixcnt; rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); if (rc != msixcnt) { @@ -84,85 +108,34 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) } dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); - /* - * We implement 1 completion list per MSI-X entry except for - * entry 0, which is for errors and others. - */ - idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry), - GFP_KERNEL, dev_to_node(dev)); - if (!idxd->irq_entries) { - rc = -ENOMEM; - goto err_irq_entries; - } - - for (i = 0; i < msixcnt; i++) { - idxd->irq_entries[i].id = i; - idxd->irq_entries[i].idxd = idxd; - idxd->irq_entries[i].vector = pci_irq_vector(pdev, i); - spin_lock_init(&idxd->irq_entries[i].list_lock); - } - irq_entry = &idxd->irq_entries[0]; - rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread, - 0, "idxd-misc", irq_entry); + ie = idxd_get_ie(idxd, 0); + ie->vector = pci_irq_vector(pdev, 0); + rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); if (rc < 0) { dev_err(dev, "Failed to allocate misc interrupt.\n"); goto err_misc_irq; } + dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); - dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector); - - /* first MSI-X entry is not for wq interrupts */ - idxd->num_wq_irqs = msixcnt - 1; + for (i = 0; i < idxd->max_wqs; i++) { + int msix_idx = i + 1; - for (i = 1; i < msixcnt; i++) { - irq_entry = &idxd->irq_entries[i]; + ie = idxd_get_ie(idxd, msix_idx); + ie->id = msix_idx; + ie->int_handle = INVALID_INT_HANDLE; + ie->pasid = IOMMU_PASID_INVALID; - init_llist_head(&idxd->irq_entries[i].pending_llist); - INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); - rc = request_threaded_irq(irq_entry->vector, NULL, - idxd_wq_thread, 0, "idxd-portal", irq_entry); - if (rc < 0) { - dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); - goto err_wq_irqs; - } - - dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector); - if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { - /* - * The MSIX vector enumeration starts at 1 with vector 0 being the - * misc interrupt that handles non I/O completion events. The - * interrupt handles are for IMS enumeration on guest. The misc - * interrupt vector does not require a handle and therefore we start - * the int_handles at index 0. Since 'i' starts at 1, the first - * int_handles index will be 0. - */ - rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1], - IDXD_IRQ_MSIX); - if (rc < 0) { - free_irq(irq_entry->vector, irq_entry); - goto err_wq_irqs; - } - dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]); - } + spin_lock_init(&ie->list_lock); + init_llist_head(&ie->pending_llist); + INIT_LIST_HEAD(&ie->work_list); } idxd_unmask_error_interrupts(idxd); - idxd_msix_perm_setup(idxd); return 0; - err_wq_irqs: - while (--i >= 0) { - irq_entry = &idxd->irq_entries[i]; - free_irq(irq_entry->vector, irq_entry); - if (i != 0) - idxd_device_release_int_handle(idxd, - idxd->int_handles[i], IDXD_IRQ_MSIX); - } err_misc_irq: - /* Disable error interrupt generation */ idxd_mask_error_interrupts(idxd); - err_irq_entries: pci_free_irq_vectors(pdev); dev_err(dev, "No usable interrupts\n"); return rc; @@ -171,33 +144,43 @@ static int idxd_setup_interrupts(struct idxd_device *idxd) static void idxd_cleanup_interrupts(struct idxd_device *idxd) { struct pci_dev *pdev = idxd->pdev; - struct idxd_irq_entry *irq_entry; - int i, msixcnt; + struct idxd_irq_entry *ie; + int msixcnt; msixcnt = pci_msix_vec_count(pdev); if (msixcnt <= 0) return; - irq_entry = &idxd->irq_entries[0]; - free_irq(irq_entry->vector, irq_entry); + ie = idxd_get_ie(idxd, 0); + idxd_mask_error_interrupts(idxd); + free_irq(ie->vector, ie); + pci_free_irq_vectors(pdev); +} - for (i = 1; i < msixcnt; i++) { +static void idxd_clean_wqs(struct idxd_device *idxd) +{ + struct idxd_wq *wq; + struct device *conf_dev; + int i; - irq_entry = &idxd->irq_entries[i]; - if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) - idxd_device_release_int_handle(idxd, idxd->int_handles[i], - IDXD_IRQ_MSIX); - free_irq(irq_entry->vector, irq_entry); + for (i = 0; i < idxd->max_wqs; i++) { + wq = idxd->wqs[i]; + if (idxd->hw.wq_cap.op_config) + bitmap_free(wq->opcap_bmap); + kfree(wq->wqcfg); + conf_dev = wq_confdev(wq); + put_device(conf_dev); + kfree(wq); } - - idxd_mask_error_interrupts(idxd); - pci_free_irq_vectors(pdev); + bitmap_free(idxd->wq_enable_map); + kfree(idxd->wqs); } static int idxd_setup_wqs(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; struct idxd_wq *wq; + struct device *conf_dev; int i, rc; idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), @@ -205,51 +188,105 @@ static int idxd_setup_wqs(struct idxd_device *idxd) if (!idxd->wqs) return -ENOMEM; + idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); + if (!idxd->wq_enable_map) { + rc = -ENOMEM; + goto err_free_wqs; + } + for (i = 0; i < idxd->max_wqs; i++) { wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); if (!wq) { rc = -ENOMEM; - goto err; + goto err_unwind; } + idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); + conf_dev = wq_confdev(wq); wq->id = i; wq->idxd = idxd; - device_initialize(&wq->conf_dev); - wq->conf_dev.parent = &idxd->conf_dev; - wq->conf_dev.bus = &dsa_bus_type; - wq->conf_dev.type = &idxd_wq_device_type; - rc = dev_set_name(&wq->conf_dev, "wq%d.%d", idxd->id, wq->id); + device_initialize(conf_dev); + conf_dev->parent = idxd_confdev(idxd); + conf_dev->bus = &dsa_bus_type; + conf_dev->type = &idxd_wq_device_type; + rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); if (rc < 0) { - put_device(&wq->conf_dev); - goto err; + put_device(conf_dev); + kfree(wq); + goto err_unwind; } mutex_init(&wq->wq_lock); init_waitqueue_head(&wq->err_queue); init_completion(&wq->wq_dead); - wq->max_xfer_bytes = idxd->max_xfer_bytes; - wq->max_batch_size = idxd->max_batch_size; + init_completion(&wq->wq_resurrect); + wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; + idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); + wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); if (!wq->wqcfg) { - put_device(&wq->conf_dev); + put_device(conf_dev); + kfree(wq); rc = -ENOMEM; - goto err; + goto err_unwind; } + + if (idxd->hw.wq_cap.op_config) { + wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); + if (!wq->opcap_bmap) { + kfree(wq->wqcfg); + put_device(conf_dev); + kfree(wq); + rc = -ENOMEM; + goto err_unwind; + } + bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); + } + mutex_init(&wq->uc_lock); + xa_init(&wq->upasid_xa); idxd->wqs[i] = wq; } return 0; - err: - while (--i >= 0) - put_device(&idxd->wqs[i]->conf_dev); +err_unwind: + while (--i >= 0) { + wq = idxd->wqs[i]; + if (idxd->hw.wq_cap.op_config) + bitmap_free(wq->opcap_bmap); + kfree(wq->wqcfg); + conf_dev = wq_confdev(wq); + put_device(conf_dev); + kfree(wq); + } + bitmap_free(idxd->wq_enable_map); + +err_free_wqs: + kfree(idxd->wqs); + return rc; } +static void idxd_clean_engines(struct idxd_device *idxd) +{ + struct idxd_engine *engine; + struct device *conf_dev; + int i; + + for (i = 0; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + conf_dev = engine_confdev(engine); + put_device(conf_dev); + kfree(engine); + } + kfree(idxd->engines); +} + static int idxd_setup_engines(struct idxd_device *idxd) { struct idxd_engine *engine; struct device *dev = &idxd->pdev->dev; + struct device *conf_dev; int i, rc; idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), @@ -264,15 +301,18 @@ static int idxd_setup_engines(struct idxd_device *idxd) goto err; } + idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); + conf_dev = engine_confdev(engine); engine->id = i; engine->idxd = idxd; - device_initialize(&engine->conf_dev); - engine->conf_dev.parent = &idxd->conf_dev; - engine->conf_dev.bus = &dsa_bus_type; - engine->conf_dev.type = &idxd_engine_device_type; - rc = dev_set_name(&engine->conf_dev, "engine%d.%d", idxd->id, engine->id); + device_initialize(conf_dev); + conf_dev->parent = idxd_confdev(idxd); + conf_dev->bus = &dsa_bus_type; + conf_dev->type = &idxd_engine_device_type; + rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); if (rc < 0) { - put_device(&engine->conf_dev); + put_device(conf_dev); + kfree(engine); goto err; } @@ -282,14 +322,34 @@ static int idxd_setup_engines(struct idxd_device *idxd) return 0; err: - while (--i >= 0) - put_device(&idxd->engines[i]->conf_dev); + while (--i >= 0) { + engine = idxd->engines[i]; + conf_dev = engine_confdev(engine); + put_device(conf_dev); + kfree(engine); + } + kfree(idxd->engines); + return rc; } +static void idxd_clean_groups(struct idxd_device *idxd) +{ + struct idxd_group *group; + int i; + + for (i = 0; i < idxd->max_groups; i++) { + group = idxd->groups[i]; + put_device(group_confdev(group)); + kfree(group); + } + kfree(idxd->groups); +} + static int idxd_setup_groups(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; + struct device *conf_dev; struct idxd_group *group; int i, rc; @@ -305,58 +365,100 @@ static int idxd_setup_groups(struct idxd_device *idxd) goto err; } + idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); + conf_dev = group_confdev(group); group->id = i; group->idxd = idxd; - device_initialize(&group->conf_dev); - group->conf_dev.parent = &idxd->conf_dev; - group->conf_dev.bus = &dsa_bus_type; - group->conf_dev.type = &idxd_group_device_type; - rc = dev_set_name(&group->conf_dev, "group%d.%d", idxd->id, group->id); + device_initialize(conf_dev); + conf_dev->parent = idxd_confdev(idxd); + conf_dev->bus = &dsa_bus_type; + conf_dev->type = &idxd_group_device_type; + rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); if (rc < 0) { - put_device(&group->conf_dev); + put_device(conf_dev); + kfree(group); goto err; } idxd->groups[i] = group; - group->tc_a = -1; - group->tc_b = -1; + if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { + group->tc_a = 1; + group->tc_b = 1; + } else { + group->tc_a = -1; + group->tc_b = -1; + } + /* + * The default value is the same as the value of + * total read buffers in GRPCAP. + */ + group->rdbufs_allowed = idxd->max_rdbufs; } return 0; err: - while (--i >= 0) - put_device(&idxd->groups[i]->conf_dev); + while (--i >= 0) { + group = idxd->groups[i]; + put_device(group_confdev(group)); + kfree(group); + } + kfree(idxd->groups); + return rc; } static void idxd_cleanup_internals(struct idxd_device *idxd) { - int i; - - for (i = 0; i < idxd->max_groups; i++) - put_device(&idxd->groups[i]->conf_dev); - for (i = 0; i < idxd->max_engines; i++) - put_device(&idxd->engines[i]->conf_dev); - for (i = 0; i < idxd->max_wqs; i++) - put_device(&idxd->wqs[i]->conf_dev); + idxd_clean_groups(idxd); + idxd_clean_engines(idxd); + idxd_clean_wqs(idxd); destroy_workqueue(idxd->wq); } -static int idxd_setup_internals(struct idxd_device *idxd) +static int idxd_init_evl(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; - int rc, i; + unsigned int evl_cache_size; + struct idxd_evl *evl; + const char *idxd_name; - init_waitqueue_head(&idxd->cmd_waitq); + if (idxd->hw.gen_cap.evl_support == 0) + return 0; - if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { - idxd->int_handles = kcalloc_node(idxd->max_wqs, sizeof(int), GFP_KERNEL, - dev_to_node(dev)); - if (!idxd->int_handles) - return -ENOMEM; + evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); + if (!evl) + return -ENOMEM; + + mutex_init(&evl->lock); + evl->size = IDXD_EVL_SIZE_MIN; + + idxd_name = dev_name(idxd_confdev(idxd)); + evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); + /* + * Since completion record in evl_cache will be copied to user + * when handling completion record page fault, need to create + * the cache suitable for user copy. + */ + idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, + 0, 0, 0, evl_cache_size, + NULL); + if (!idxd->evl_cache) { + kfree(evl); + return -ENOMEM; } + idxd->evl = evl; + return 0; +} + +static int idxd_setup_internals(struct idxd_device *idxd) +{ + struct device *dev = &idxd->pdev->dev; + int rc; + + init_waitqueue_head(&idxd->cmd_waitq); + rc = idxd_setup_wqs(idxd); if (rc < 0) goto err_wqs; @@ -375,19 +477,21 @@ static int idxd_setup_internals(struct idxd_device *idxd) goto err_wkq_create; } + rc = idxd_init_evl(idxd); + if (rc < 0) + goto err_evl; + return 0; + err_evl: + destroy_workqueue(idxd->wq); err_wkq_create: - for (i = 0; i < idxd->max_groups; i++) - put_device(&idxd->groups[i]->conf_dev); + idxd_clean_groups(idxd); err_group: - for (i = 0; i < idxd->max_engines; i++) - put_device(&idxd->engines[i]->conf_dev); + idxd_clean_engines(idxd); err_engine: - for (i = 0; i < idxd->max_wqs; i++) - put_device(&idxd->wqs[i]->conf_dev); + idxd_clean_wqs(idxd); err_wqs: - kfree(idxd->int_handles); return rc; } @@ -408,6 +512,19 @@ static void idxd_read_table_offsets(struct idxd_device *idxd) dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); } +void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) +{ + int i, j, nr; + + for (i = 0, nr = 0; i < count; i++) { + for (j = 0; j < BITS_PER_LONG_LONG; j++) { + if (val[i] & BIT(j)) + set_bit(nr, bmap); + nr++; + } + } +} + static void idxd_read_caps(struct idxd_device *idxd) { struct device *dev = &idxd->pdev->dev; @@ -422,9 +539,13 @@ static void idxd_read_caps(struct idxd_device *idxd) dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); } + /* reading command capabilities */ + if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) + idxd->request_int_handles = true; + idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); - idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; + idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); if (idxd->hw.gen_cap.config_en) set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); @@ -435,9 +556,9 @@ static void idxd_read_caps(struct idxd_device *idxd) dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); idxd->max_groups = idxd->hw.group_cap.num_groups; dev_dbg(dev, "max groups: %u\n", idxd->max_groups); - idxd->max_tokens = idxd->hw.group_cap.total_tokens; - dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens); - idxd->nr_tokens = idxd->max_tokens; + idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; + dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); + idxd->nr_rdbufs = idxd->max_rdbufs; /* read engine capabilities */ idxd->hw.engine_cap.bits = @@ -462,11 +583,28 @@ static void idxd_read_caps(struct idxd_device *idxd) IDXD_OPCAP_OFFSET + i * sizeof(u64)); dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); } + multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); + + /* read iaa cap */ + if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) + idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); +} + +static void idxd_free(struct idxd_device *idxd) +{ + if (!idxd) + return; + + put_device(idxd_confdev(idxd)); + bitmap_free(idxd->opcap_bmap); + ida_free(&idxd_ida, idxd->id); + kfree(idxd); } static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) { struct device *dev = &pdev->dev; + struct device *conf_dev; struct idxd_device *idxd; int rc; @@ -474,60 +612,97 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d if (!idxd) return NULL; + conf_dev = idxd_confdev(idxd); idxd->pdev = pdev; idxd->data = data; + idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); if (idxd->id < 0) - return NULL; + goto err_ida; - device_initialize(&idxd->conf_dev); - idxd->conf_dev.parent = dev; - idxd->conf_dev.bus = &dsa_bus_type; - idxd->conf_dev.type = idxd->data->dev_type; - rc = dev_set_name(&idxd->conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); - if (rc < 0) { - put_device(&idxd->conf_dev); - return NULL; - } + idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); + if (!idxd->opcap_bmap) + goto err_opcap; + + device_initialize(conf_dev); + conf_dev->parent = dev; + conf_dev->bus = &dsa_bus_type; + conf_dev->type = idxd->data->dev_type; + rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); + if (rc < 0) + goto err_name; spin_lock_init(&idxd->dev_lock); spin_lock_init(&idxd->cmd_lock); return idxd; + +err_name: + put_device(conf_dev); + bitmap_free(idxd->opcap_bmap); +err_opcap: + ida_free(&idxd_ida, idxd->id); +err_ida: + kfree(idxd); + + return NULL; } static int idxd_enable_system_pasid(struct idxd_device *idxd) { - int flags; - unsigned int pasid; - struct iommu_sva *sva; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iommu_domain *domain; + ioasid_t pasid; + int ret; - flags = SVM_FLAG_SUPERVISOR_MODE; + /* + * Attach a global PASID to the DMA domain so that we can use ENQCMDS + * to submit work on buffers mapped by DMA API. + */ + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return -EPERM; - sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags); - if (IS_ERR(sva)) { - dev_warn(&idxd->pdev->dev, - "iommu sva bind failed: %ld\n", PTR_ERR(sva)); - return PTR_ERR(sva); - } + pasid = iommu_alloc_global_pasid(dev); + if (pasid == IOMMU_PASID_INVALID) + return -ENOSPC; - pasid = iommu_sva_get_pasid(sva); - if (pasid == IOMMU_PASID_INVALID) { - iommu_sva_unbind_device(sva); - return -ENODEV; + /* + * DMA domain is owned by the driver, it should support all valid + * types such as DMA-FQ, identity, etc. + */ + ret = iommu_attach_device_pasid(domain, dev, pasid, NULL); + if (ret) { + dev_err(dev, "failed to attach device pasid %d, domain type %d", + pasid, domain->type); + iommu_free_global_pasid(pasid); + return ret; } - idxd->sva = sva; + /* Since we set user privilege for kernel DMA, enable completion IRQ */ + idxd_set_user_intr(idxd, 1); idxd->pasid = pasid; - dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid); - return 0; + + return ret; } static void idxd_disable_system_pasid(struct idxd_device *idxd) { + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + struct iommu_domain *domain; - iommu_sva_unbind_device(idxd->sva); + domain = iommu_get_domain_for_dev(dev); + if (!domain) + return; + + iommu_detach_device_pasid(domain, dev, idxd->pasid); + iommu_free_global_pasid(idxd->pasid); + + idxd_set_user_intr(idxd, 0); idxd->sva = NULL; + idxd->pasid = IOMMU_PASID_INVALID; } static int idxd_probe(struct idxd_device *idxd) @@ -544,18 +719,13 @@ static int idxd_probe(struct idxd_device *idxd) dev_dbg(dev, "IDXD reset complete\n"); if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { - rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA); - if (rc == 0) { - rc = idxd_enable_system_pasid(idxd); - if (rc < 0) { - iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); - dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); - } else { - set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); - } - } else { - dev_warn(dev, "Unable to turn on SVA feature.\n"); - } + set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); + + rc = idxd_enable_system_pasid(idxd); + if (rc) + dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); + else + set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); } else if (!sva) { dev_warn(dev, "User forced SVA off via module param.\n"); } @@ -579,8 +749,6 @@ static int idxd_probe(struct idxd_device *idxd) if (rc) goto err_config; - dev_dbg(dev, "IDXD interrupt setup complete.\n"); - idxd->major = idxd_cdev_get_major(idxd); rc = perfmon_pmu_init(idxd); @@ -595,82 +763,475 @@ static int idxd_probe(struct idxd_device *idxd) err: if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); return rc; } static void idxd_cleanup(struct idxd_device *idxd) { - struct device *dev = &idxd->pdev->dev; - perfmon_pmu_remove(idxd); idxd_cleanup_interrupts(idxd); idxd_cleanup_internals(idxd); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); } -static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +/* + * Attach IDXD device to IDXD driver. + */ +static int idxd_bind(struct device_driver *drv, const char *buf) { - struct device *dev = &pdev->dev; - struct idxd_device *idxd; - struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; + const struct bus_type *bus = drv->bus; + struct device *dev; + int err = -ENODEV; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev) + err = device_driver_attach(drv, dev); + + put_device(dev); + + return err; +} + +/* + * Detach IDXD device from driver. + */ +static void idxd_unbind(struct device_driver *drv, const char *buf) +{ + const struct bus_type *bus = drv->bus; + struct device *dev; + + dev = bus_find_device_by_name(bus, NULL, buf); + if (dev && dev->driver == drv) + device_release_driver(dev); + + put_device(dev); +} + +#define idxd_free_saved_configs(saved_configs, count) \ + do { \ + int i; \ + \ + for (i = 0; i < (count); i++) \ + kfree(saved_configs[i]); \ + } while (0) + +static void idxd_free_saved(struct idxd_group **saved_groups, + struct idxd_engine **saved_engines, + struct idxd_wq **saved_wqs, + struct idxd_device *idxd) +{ + if (saved_groups) + idxd_free_saved_configs(saved_groups, idxd->max_groups); + if (saved_engines) + idxd_free_saved_configs(saved_engines, idxd->max_engines); + if (saved_wqs) + idxd_free_saved_configs(saved_wqs, idxd->max_wqs); +} + +/* + * Save IDXD device configurations including engines, groups, wqs etc. + * The saved configurations can be restored when needed. + */ +static int idxd_device_config_save(struct idxd_device *idxd, + struct idxd_saved_states *idxd_saved) +{ + struct device *dev = &idxd->pdev->dev; + int i; + + memcpy(&idxd_saved->saved_idxd, idxd, sizeof(*idxd)); + + if (idxd->evl) { + memcpy(&idxd_saved->saved_evl, idxd->evl, + sizeof(struct idxd_evl)); + } + + struct idxd_group **saved_groups __free(kfree) = + kcalloc_node(idxd->max_groups, + sizeof(struct idxd_group *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_groups) + return -ENOMEM; + + for (i = 0; i < idxd->max_groups; i++) { + struct idxd_group *saved_group __free(kfree) = + kzalloc_node(sizeof(*saved_group), GFP_KERNEL, + dev_to_node(dev)); + + if (!saved_group) { + /* Free saved groups */ + idxd_free_saved(saved_groups, NULL, NULL, idxd); + + return -ENOMEM; + } + + memcpy(saved_group, idxd->groups[i], sizeof(*saved_group)); + saved_groups[i] = no_free_ptr(saved_group); + } + + struct idxd_engine **saved_engines = + kcalloc_node(idxd->max_engines, + sizeof(struct idxd_engine *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_engines) { + /* Free saved groups */ + idxd_free_saved(saved_groups, NULL, NULL, idxd); + + return -ENOMEM; + } + for (i = 0; i < idxd->max_engines; i++) { + struct idxd_engine *saved_engine __free(kfree) = + kzalloc_node(sizeof(*saved_engine), GFP_KERNEL, + dev_to_node(dev)); + if (!saved_engine) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, + idxd); + + return -ENOMEM; + } + + memcpy(saved_engine, idxd->engines[i], sizeof(*saved_engine)); + saved_engines[i] = no_free_ptr(saved_engine); + } + + unsigned long *saved_wq_enable_map __free(bitmap) = + bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, + dev_to_node(dev)); + if (!saved_wq_enable_map) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, idxd); + + return -ENOMEM; + } + + bitmap_copy(saved_wq_enable_map, idxd->wq_enable_map, idxd->max_wqs); + + struct idxd_wq **saved_wqs __free(kfree) = + kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), + GFP_KERNEL, dev_to_node(dev)); + if (!saved_wqs) { + /* Free saved groups and engines */ + idxd_free_saved(saved_groups, saved_engines, NULL, idxd); + + return -ENOMEM; + } + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *saved_wq __free(kfree) = + kzalloc_node(sizeof(*saved_wq), GFP_KERNEL, + dev_to_node(dev)); + struct idxd_wq *wq; + + if (!saved_wq) { + /* Free saved groups, engines, and wqs */ + idxd_free_saved(saved_groups, saved_engines, saved_wqs, + idxd); + + return -ENOMEM; + } + + if (!test_bit(i, saved_wq_enable_map)) + continue; + + wq = idxd->wqs[i]; + mutex_lock(&wq->wq_lock); + memcpy(saved_wq, wq, sizeof(*saved_wq)); + saved_wqs[i] = no_free_ptr(saved_wq); + mutex_unlock(&wq->wq_lock); + } + + /* Save configurations */ + idxd_saved->saved_groups = no_free_ptr(saved_groups); + idxd_saved->saved_engines = no_free_ptr(saved_engines); + idxd_saved->saved_wq_enable_map = no_free_ptr(saved_wq_enable_map); + idxd_saved->saved_wqs = no_free_ptr(saved_wqs); + + return 0; +} + +/* + * Restore IDXD device configurations including engines, groups, wqs etc + * that were saved before. + */ +static void idxd_device_config_restore(struct idxd_device *idxd, + struct idxd_saved_states *idxd_saved) +{ + struct idxd_evl *saved_evl = &idxd_saved->saved_evl; + int i; + + idxd->rdbuf_limit = idxd_saved->saved_idxd.rdbuf_limit; + + idxd->evl->size = saved_evl->size; + + for (i = 0; i < idxd->max_groups; i++) { + struct idxd_group *saved_group, *group; + + saved_group = idxd_saved->saved_groups[i]; + group = idxd->groups[i]; + + group->rdbufs_allowed = saved_group->rdbufs_allowed; + group->rdbufs_reserved = saved_group->rdbufs_reserved; + group->tc_a = saved_group->tc_a; + group->tc_b = saved_group->tc_b; + group->use_rdbuf_limit = saved_group->use_rdbuf_limit; + + kfree(saved_group); + } + kfree(idxd_saved->saved_groups); + + for (i = 0; i < idxd->max_engines; i++) { + struct idxd_engine *saved_engine, *engine; + + saved_engine = idxd_saved->saved_engines[i]; + engine = idxd->engines[i]; + + engine->group = saved_engine->group; + + kfree(saved_engine); + } + kfree(idxd_saved->saved_engines); + + bitmap_copy(idxd->wq_enable_map, idxd_saved->saved_wq_enable_map, + idxd->max_wqs); + bitmap_free(idxd_saved->saved_wq_enable_map); + + for (i = 0; i < idxd->max_wqs; i++) { + struct idxd_wq *saved_wq, *wq; + size_t len; + + if (!test_bit(i, idxd->wq_enable_map)) + continue; + + saved_wq = idxd_saved->saved_wqs[i]; + wq = idxd->wqs[i]; + + mutex_lock(&wq->wq_lock); + + wq->group = saved_wq->group; + wq->flags = saved_wq->flags; + wq->threshold = saved_wq->threshold; + wq->size = saved_wq->size; + wq->priority = saved_wq->priority; + wq->type = saved_wq->type; + len = strlen(saved_wq->name) + 1; + strscpy(wq->name, saved_wq->name, len); + wq->max_xfer_bytes = saved_wq->max_xfer_bytes; + wq->max_batch_size = saved_wq->max_batch_size; + wq->enqcmds_retries = saved_wq->enqcmds_retries; + wq->descs = saved_wq->descs; + wq->idxd_chan = saved_wq->idxd_chan; + len = strlen(saved_wq->driver_name) + 1; + strscpy(wq->driver_name, saved_wq->driver_name, len); + + mutex_unlock(&wq->wq_lock); + + kfree(saved_wq); + } + + kfree(idxd_saved->saved_wqs); +} + +static void idxd_reset_prepare(struct pci_dev *pdev) +{ + struct idxd_device *idxd = pci_get_drvdata(pdev); + struct device *dev = &idxd->pdev->dev; + const char *idxd_name; int rc; - rc = pci_enable_device(pdev); - if (rc) - return rc; + idxd_name = dev_name(idxd_confdev(idxd)); - dev_dbg(dev, "Alloc IDXD context\n"); - idxd = idxd_alloc(pdev, data); - if (!idxd) { - rc = -ENOMEM; - goto err_idxd_alloc; + struct idxd_saved_states *idxd_saved __free(kfree) = + kzalloc_node(sizeof(*idxd_saved), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!idxd_saved) { + dev_err(dev, "HALT: no memory\n"); + + return; } - dev_dbg(dev, "Mapping BARs\n"); - idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); - if (!idxd->reg_base) { - rc = -ENOMEM; - goto err_iomap; + /* Save IDXD configurations. */ + rc = idxd_device_config_save(idxd, idxd_saved); + if (rc < 0) { + dev_err(dev, "HALT: cannot save %s configs\n", idxd_name); + + return; } - dev_dbg(dev, "Set DMA masks\n"); - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (rc) - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) - goto err; + idxd->idxd_saved = no_free_ptr(idxd_saved); - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (rc) - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + /* Save PCI device state. */ + pci_save_state(idxd->pdev); +} + +static void idxd_reset_done(struct pci_dev *pdev) +{ + struct idxd_device *idxd = pci_get_drvdata(pdev); + const char *idxd_name; + struct device *dev; + int rc, i; + + if (!idxd->idxd_saved) + return; + + dev = &idxd->pdev->dev; + idxd_name = dev_name(idxd_confdev(idxd)); + + /* Restore PCI device state. */ + pci_restore_state(idxd->pdev); + + /* Unbind idxd device from driver. */ + idxd_unbind(&idxd_drv.drv, idxd_name); + + /* + * Probe PCI device without allocating or changing + * idxd software data which keeps the same as before FLR. + */ + idxd_pci_probe_alloc(idxd, NULL, NULL); + + /* Restore IDXD configurations. */ + idxd_device_config_restore(idxd, idxd->idxd_saved); + + /* Re-configure IDXD device if allowed. */ + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + rc = idxd_device_config(idxd); + if (rc < 0) { + dev_err(dev, "HALT: %s config fails\n", idxd_name); + goto out; + } + } + + /* Bind IDXD device to driver. */ + rc = idxd_bind(&idxd_drv.drv, idxd_name); + if (rc < 0) { + dev_err(dev, "HALT: binding %s to driver fails\n", idxd_name); + goto out; + } + + /* Bind enabled wq in the IDXD device to driver. */ + for (i = 0; i < idxd->max_wqs; i++) { + if (test_bit(i, idxd->wq_enable_map)) { + struct idxd_wq *wq = idxd->wqs[i]; + char wq_name[32]; + + wq->state = IDXD_WQ_DISABLED; + sprintf(wq_name, "wq%d.%d", idxd->id, wq->id); + /* + * Bind to user driver depending on wq type. + * + * Currently only support user type WQ. Will support + * kernel type WQ in the future. + */ + if (wq->type == IDXD_WQT_USER) + rc = idxd_bind(&idxd_user_drv.drv, wq_name); + else + rc = -EINVAL; + if (rc < 0) { + clear_bit(i, idxd->wq_enable_map); + dev_err(dev, + "HALT: unable to re-enable wq %s\n", + dev_name(wq_confdev(wq))); + } + } + } +out: + kfree(idxd->idxd_saved); +} + +static const struct pci_error_handlers idxd_error_handler = { + .reset_prepare = idxd_reset_prepare, + .reset_done = idxd_reset_done, +}; + +/* + * Probe idxd PCI device. + * If idxd is not given, need to allocate idxd and set up its data. + * + * If idxd is given, idxd was allocated and setup already. Just need to + * configure device without re-allocating and re-configuring idxd data. + * This is useful for recovering from FLR. + */ +int idxd_pci_probe_alloc(struct idxd_device *idxd, struct pci_dev *pdev, + const struct pci_device_id *id) +{ + bool alloc_idxd = idxd ? false : true; + struct idxd_driver_data *data; + struct device *dev; + int rc; + + pdev = idxd ? idxd->pdev : pdev; + dev = &pdev->dev; + data = id ? (struct idxd_driver_data *)id->driver_data : NULL; + rc = pci_enable_device(pdev); if (rc) - goto err; + return rc; + + if (alloc_idxd) { + dev_dbg(dev, "Alloc IDXD context\n"); + idxd = idxd_alloc(pdev, data); + if (!idxd) { + rc = -ENOMEM; + goto err_idxd_alloc; + } + + dev_dbg(dev, "Mapping BARs\n"); + idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); + if (!idxd->reg_base) { + rc = -ENOMEM; + goto err_iomap; + } + + dev_dbg(dev, "Set DMA masks\n"); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (rc) + goto err; + } dev_dbg(dev, "Set PCI master\n"); pci_set_master(pdev); pci_set_drvdata(pdev, idxd); - idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); - rc = idxd_probe(idxd); - if (rc) { - dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); - goto err; - } + if (alloc_idxd) { + idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); + rc = idxd_probe(idxd); + if (rc) { + dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); + goto err; + } - rc = idxd_register_devices(idxd); - if (rc) { - dev_err(dev, "IDXD sysfs setup failed\n"); - goto err_dev_register; + if (data->load_device_defaults) { + rc = data->load_device_defaults(idxd); + if (rc) + dev_warn(dev, "IDXD loading device defaults failed\n"); + } + + rc = idxd_register_devices(idxd); + if (rc) { + dev_err(dev, "IDXD sysfs setup failed\n"); + goto err_dev_register; + } + + rc = idxd_device_init_debugfs(idxd); + if (rc) + dev_warn(dev, "IDXD debugfs failed to setup\n"); } - idxd->state = IDXD_DEV_CONF_READY; + if (!alloc_idxd) { + /* Release interrupts in the IDXD device. */ + idxd_cleanup_interrupts(idxd); + + /* Re-enable interrupts in the IDXD device. */ + rc = idxd_setup_interrupts(idxd); + if (rc) + dev_warn(dev, "IDXD interrupts failed to setup\n"); + } dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", idxd->hw.version); + if (data) + idxd->user_submission_safe = data->user_submission_safe; + return 0; err_dev_register: @@ -678,36 +1239,15 @@ static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) err: pci_iounmap(pdev, idxd->reg_base); err_iomap: - put_device(&idxd->conf_dev); + idxd_free(idxd); err_idxd_alloc: pci_disable_device(pdev); return rc; } -static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) -{ - struct idxd_desc *desc, *itr; - struct llist_node *head; - - head = llist_del_all(&ie->pending_llist); - if (!head) - return; - - llist_for_each_entry_safe(desc, itr, head, llnode) { - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); - idxd_free_desc(desc->wq, desc); - } -} - -static void idxd_flush_work_list(struct idxd_irq_entry *ie) +static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - struct idxd_desc *desc, *iter; - - list_for_each_entry_safe(desc, iter, &ie->work_list, list) { - list_del(&desc->list); - idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); - idxd_free_desc(desc->wq, desc); - } + return idxd_pci_probe_alloc(NULL, pdev, id); } void idxd_wqs_quiesce(struct idxd_device *idxd) @@ -722,68 +1262,45 @@ void idxd_wqs_quiesce(struct idxd_device *idxd) } } -static void idxd_release_int_handles(struct idxd_device *idxd) -{ - struct device *dev = &idxd->pdev->dev; - int i, rc; - - for (i = 0; i < idxd->num_wq_irqs; i++) { - if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) { - rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i], - IDXD_IRQ_MSIX); - if (rc < 0) - dev_warn(dev, "irq handle %d release failed\n", - idxd->int_handles[i]); - else - dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]); - } - } -} - static void idxd_shutdown(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); - int rc, i; struct idxd_irq_entry *irq_entry; - int msixcnt = pci_msix_vec_count(pdev); + int rc; rc = idxd_device_disable(idxd); if (rc) dev_err(&pdev->dev, "Disabling device failed\n"); - dev_dbg(&pdev->dev, "%s called\n", __func__); - idxd_mask_msix_vectors(idxd); + irq_entry = &idxd->ie; + synchronize_irq(irq_entry->vector); idxd_mask_error_interrupts(idxd); - - for (i = 0; i < msixcnt; i++) { - irq_entry = &idxd->irq_entries[i]; - synchronize_irq(irq_entry->vector); - free_irq(irq_entry->vector, irq_entry); - if (i == 0) - continue; - idxd_flush_pending_llist(irq_entry); - idxd_flush_work_list(irq_entry); - } - - idxd_msix_perm_clear(idxd); - idxd_release_int_handles(idxd); - pci_free_irq_vectors(pdev); - pci_iounmap(pdev, idxd->reg_base); - pci_disable_device(pdev); - destroy_workqueue(idxd->wq); + flush_workqueue(idxd->wq); } static void idxd_remove(struct pci_dev *pdev) { struct idxd_device *idxd = pci_get_drvdata(pdev); - dev_dbg(&pdev->dev, "%s called\n", __func__); + idxd_unregister_devices(idxd); + /* + * When ->release() is called for the idxd->conf_dev, it frees all the memory related + * to the idxd context. The driver still needs those bits in order to do the rest of + * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref + * on the device here to hold off the freeing while allowing the idxd sub-driver + * to unbind. + */ + get_device(idxd_confdev(idxd)); + device_unregister(idxd_confdev(idxd)); idxd_shutdown(pdev); + idxd_device_remove_debugfs(idxd); + perfmon_pmu_remove(idxd); + idxd_cleanup_interrupts(idxd); if (device_pasid_enabled(idxd)) idxd_disable_system_pasid(idxd); - idxd_unregister_devices(idxd); - perfmon_pmu_remove(idxd); - iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); + pci_iounmap(pdev, idxd->reg_base); + put_device(idxd_confdev(idxd)); + pci_disable_device(pdev); } static struct pci_driver idxd_pci_driver = { @@ -792,6 +1309,7 @@ static struct pci_driver idxd_pci_driver = { .probe = idxd_pci_probe, .remove = idxd_remove, .shutdown = idxd_shutdown, + .err_handler = &idxd_error_handler, }; static int __init idxd_init_module(void) @@ -812,20 +1330,26 @@ static int __init idxd_init_module(void) else support_enqcmd = true; - perfmon_init(); + err = idxd_driver_register(&idxd_drv); + if (err < 0) + goto err_idxd_driver_register; - err = idxd_register_bus_type(); + err = idxd_driver_register(&idxd_dmaengine_drv); if (err < 0) - return err; + goto err_idxd_dmaengine_driver_register; - err = idxd_register_driver(); + err = idxd_driver_register(&idxd_user_drv); if (err < 0) - goto err_idxd_driver_register; + goto err_idxd_user_driver_register; err = idxd_cdev_register(); if (err) goto err_cdev_register; + err = idxd_init_debugfs(); + if (err) + goto err_debugfs; + err = pci_register_driver(&idxd_pci_driver); if (err) goto err_pci_register; @@ -833,21 +1357,27 @@ static int __init idxd_init_module(void) return 0; err_pci_register: + idxd_remove_debugfs(); +err_debugfs: idxd_cdev_remove(); err_cdev_register: - idxd_unregister_driver(); + idxd_driver_unregister(&idxd_user_drv); +err_idxd_user_driver_register: + idxd_driver_unregister(&idxd_dmaengine_drv); +err_idxd_dmaengine_driver_register: + idxd_driver_unregister(&idxd_drv); err_idxd_driver_register: - idxd_unregister_bus_type(); return err; } module_init(idxd_init_module); static void __exit idxd_exit_module(void) { - idxd_unregister_driver(); + idxd_driver_unregister(&idxd_user_drv); + idxd_driver_unregister(&idxd_dmaengine_drv); + idxd_driver_unregister(&idxd_drv); pci_unregister_driver(&idxd_pci_driver); idxd_cdev_remove(); - idxd_unregister_bus_type(); - perfmon_exit(); + idxd_remove_debugfs(); } module_exit(idxd_exit_module); diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c index ae68e1e5487a..1107db3ce0a3 100644 --- a/drivers/dma/idxd/irq.c +++ b/drivers/dma/idxd/irq.c @@ -6,6 +6,9 @@ #include <linux/pci.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/dmaengine.h> +#include <linux/delay.h> +#include <linux/iommu.h> +#include <linux/sched/mm.h> #include <uapi/linux/idxd.h> #include "../dmaengine.h" #include "idxd.h" @@ -16,18 +19,15 @@ enum irq_work_type { IRQ_WORK_PROCESS_FAULT, }; -struct idxd_fault { +struct idxd_resubmit { struct work_struct work; - u64 addr; - struct idxd_device *idxd; + struct idxd_desc *desc; }; -static int irq_process_work_list(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, - int *processed, u64 data); -static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, - int *processed, u64 data); +struct idxd_int_handle_revoke { + struct work_struct work; + struct idxd_device *idxd; +}; static void idxd_device_reinit(struct work_struct *work) { @@ -45,13 +45,14 @@ static void idxd_device_reinit(struct work_struct *work) goto out; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; + if (test_bit(i, idxd->wq_enable_map)) { + struct idxd_wq *wq = idxd->wqs[i]; - if (wq->state == IDXD_WQ_ENABLED) { rc = idxd_wq_enable(wq); if (rc < 0) { + clear_bit(i, idxd->wq_enable_map); dev_warn(dev, "Unable to re-enable wq %s\n", - dev_name(&wq->conf_dev)); + dev_name(wq_confdev(wq))); } } } @@ -59,59 +60,401 @@ static void idxd_device_reinit(struct work_struct *work) return; out: - idxd_device_wqs_clear_state(idxd); + idxd_device_clear_state(idxd); } -static void idxd_device_fault_work(struct work_struct *work) +/* + * The function sends a drain descriptor for the interrupt handle. The drain ensures + * all descriptors with this interrupt handle is flushed and the interrupt + * will allow the cleanup of the outstanding descriptors. + */ +static void idxd_int_handle_revoke_drain(struct idxd_irq_entry *ie) { - struct idxd_fault *fault = container_of(work, struct idxd_fault, work); - struct idxd_irq_entry *ie; - int i; - int processed; - int irqcnt = fault->idxd->num_wq_irqs + 1; - - for (i = 1; i < irqcnt; i++) { - ie = &fault->idxd->irq_entries[i]; - irq_process_work_list(ie, IRQ_WORK_PROCESS_FAULT, - &processed, fault->addr); - if (processed) - break; - - irq_process_pending_llist(ie, IRQ_WORK_PROCESS_FAULT, - &processed, fault->addr); - if (processed) - break; + struct idxd_wq *wq = ie_to_wq(ie); + struct idxd_device *idxd = wq->idxd; + struct device *dev = &idxd->pdev->dev; + struct dsa_hw_desc desc = {}; + void __iomem *portal; + int rc; + + /* Issue a simple drain operation with interrupt but no completion record */ + desc.flags = IDXD_OP_FLAG_RCI; + desc.opcode = DSA_OPCODE_DRAIN; + desc.priv = 1; + + if (ie->pasid != IOMMU_PASID_INVALID) + desc.pasid = ie->pasid; + desc.int_handle = ie->int_handle; + portal = idxd_wq_portal_addr(wq); + + /* + * The wmb() makes sure that the descriptor is all there before we + * issue. + */ + wmb(); + if (wq_dedicated(wq)) { + iosubmit_cmds512(portal, &desc, 1); + } else { + rc = idxd_enqcmds(wq, portal, &desc); + /* This should not fail unless hardware failed. */ + if (rc < 0) + dev_warn(dev, "Failed to submit drain desc on wq %d\n", wq->id); } +} + +static void idxd_abort_invalid_int_handle_descs(struct idxd_irq_entry *ie) +{ + LIST_HEAD(flist); + struct idxd_desc *d, *t; + struct llist_node *head; + + spin_lock(&ie->list_lock); + head = llist_del_all(&ie->pending_llist); + if (head) { + llist_for_each_entry_safe(d, t, head, llnode) + list_add_tail(&d->list, &ie->work_list); + } + + list_for_each_entry_safe(d, t, &ie->work_list, list) { + if (d->completion->status == DSA_COMP_INT_HANDLE_INVAL) + list_move_tail(&d->list, &flist); + } + spin_unlock(&ie->list_lock); - kfree(fault); + list_for_each_entry_safe(d, t, &flist, list) { + list_del(&d->list); + idxd_desc_complete(d, IDXD_COMPLETE_ABORT, true); + } } -static int idxd_device_schedule_fault_process(struct idxd_device *idxd, - u64 fault_addr) +static void idxd_int_handle_revoke(struct work_struct *work) { - struct idxd_fault *fault; + struct idxd_int_handle_revoke *revoke = + container_of(work, struct idxd_int_handle_revoke, work); + struct idxd_device *idxd = revoke->idxd; + struct pci_dev *pdev = idxd->pdev; + struct device *dev = &pdev->dev; + int i, new_handle, rc; + + if (!idxd->request_int_handles) { + kfree(revoke); + dev_warn(dev, "Unexpected int handle refresh interrupt.\n"); + return; + } - fault = kmalloc(sizeof(*fault), GFP_ATOMIC); - if (!fault) - return -ENOMEM; + /* + * The loop attempts to acquire new interrupt handle for all interrupt + * vectors that supports a handle. If a new interrupt handle is acquired and the + * wq is kernel type, the driver will kill the percpu_ref to pause all + * ongoing descriptor submissions. The interrupt handle is then changed. + * After change, the percpu_ref is revived and all the pending submissions + * are woken to try again. A drain is sent to for the interrupt handle + * at the end to make sure all invalid int handle descriptors are processed. + */ + for (i = 1; i < idxd->irq_cnt; i++) { + struct idxd_irq_entry *ie = idxd_get_ie(idxd, i); + struct idxd_wq *wq = ie_to_wq(ie); + + if (ie->int_handle == INVALID_INT_HANDLE) + continue; + + rc = idxd_device_request_int_handle(idxd, i, &new_handle, IDXD_IRQ_MSIX); + if (rc < 0) { + dev_warn(dev, "get int handle %d failed: %d\n", i, rc); + /* + * Failed to acquire new interrupt handle. Kill the WQ + * and release all the pending submitters. The submitters will + * get error return code and handle appropriately. + */ + ie->int_handle = INVALID_INT_HANDLE; + idxd_wq_quiesce(wq); + idxd_abort_invalid_int_handle_descs(ie); + continue; + } + + /* No change in interrupt handle, nothing needs to be done */ + if (ie->int_handle == new_handle) + continue; + + if (wq->state != IDXD_WQ_ENABLED || wq->type != IDXD_WQT_KERNEL) { + /* + * All the MSIX interrupts are allocated at once during probe. + * Therefore we need to update all interrupts even if the WQ + * isn't supporting interrupt operations. + */ + ie->int_handle = new_handle; + continue; + } - fault->addr = fault_addr; - fault->idxd = idxd; - INIT_WORK(&fault->work, idxd_device_fault_work); - queue_work(idxd->wq, &fault->work); - return 0; + mutex_lock(&wq->wq_lock); + reinit_completion(&wq->wq_resurrect); + + /* Kill percpu_ref to pause additional descriptor submissions */ + percpu_ref_kill(&wq->wq_active); + + /* Wait for all submitters quiesce before we change interrupt handle */ + wait_for_completion(&wq->wq_dead); + + ie->int_handle = new_handle; + + /* Revive percpu ref and wake up all the waiting submitters */ + percpu_ref_reinit(&wq->wq_active); + complete_all(&wq->wq_resurrect); + mutex_unlock(&wq->wq_lock); + + /* + * The delay here is to wait for all possible MOVDIR64B that + * are issued before percpu_ref_kill() has happened to have + * reached the PCIe domain before the drain is issued. The driver + * needs to ensure that the drain descriptor issued does not pass + * all the other issued descriptors that contain the invalid + * interrupt handle in order to ensure that the drain descriptor + * interrupt will allow the cleanup of all the descriptors with + * invalid interrupt handle. + */ + if (wq_dedicated(wq)) + udelay(100); + idxd_int_handle_revoke_drain(ie); + } + kfree(revoke); } -static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) +static void idxd_evl_fault_work(struct work_struct *work) { + struct idxd_evl_fault *fault = container_of(work, struct idxd_evl_fault, work); + struct idxd_wq *wq = fault->wq; + struct idxd_device *idxd = wq->idxd; struct device *dev = &idxd->pdev->dev; + struct idxd_evl *evl = idxd->evl; + struct __evl_entry *entry_head = fault->entry; + void *cr = (void *)entry_head + idxd->data->evl_cr_off; + int cr_size = idxd->data->compl_size; + u8 *status = (u8 *)cr + idxd->data->cr_status_off; + u8 *result = (u8 *)cr + idxd->data->cr_result_off; + int copied, copy_size; + bool *bf; + + switch (fault->status) { + case DSA_COMP_CRA_XLAT: + if (entry_head->batch && entry_head->first_err_in_batch) + evl->batch_fail[entry_head->batch_id] = false; + + copy_size = cr_size; + idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULTS); + break; + case DSA_COMP_BATCH_EVL_ERR: + bf = &evl->batch_fail[entry_head->batch_id]; + + copy_size = entry_head->rcr || *bf ? cr_size : 0; + if (*bf) { + if (*status == DSA_COMP_SUCCESS) + *status = DSA_COMP_BATCH_FAIL; + *result = 1; + *bf = false; + } + idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULTS); + break; + case DSA_COMP_DRAIN_EVL: + copy_size = cr_size; + break; + default: + copy_size = 0; + dev_dbg_ratelimited(dev, "Unrecognized error code: %#x\n", fault->status); + break; + } + + if (copy_size == 0) + return; + + /* + * Copy completion record to fault_addr in user address space + * that is found by wq and PASID. + */ + copied = idxd_copy_cr(wq, entry_head->pasid, entry_head->fault_addr, + cr, copy_size); + /* + * The task that triggered the page fault is unknown currently + * because multiple threads may share the user address + * space or the task exits already before this fault. + * So if the copy fails, SIGSEGV can not be sent to the task. + * Just print an error for the failure. The user application + * waiting for the completion record will time out on this + * failure. + */ + switch (fault->status) { + case DSA_COMP_CRA_XLAT: + if (copied != copy_size) { + idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULT_FAILS); + dev_dbg_ratelimited(dev, "Failed to write to completion record: (%d:%d)\n", + copy_size, copied); + if (entry_head->batch) + evl->batch_fail[entry_head->batch_id] = true; + } + break; + case DSA_COMP_BATCH_EVL_ERR: + if (copied != copy_size) { + idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULT_FAILS); + dev_dbg_ratelimited(dev, "Failed to write to batch completion record: (%d:%d)\n", + copy_size, copied); + } + break; + case DSA_COMP_DRAIN_EVL: + if (copied != copy_size) + dev_dbg_ratelimited(dev, "Failed to write to drain completion record: (%d:%d)\n", + copy_size, copied); + break; + } + + kmem_cache_free(idxd->evl_cache, fault); +} + +static void process_evl_entry(struct idxd_device *idxd, + struct __evl_entry *entry_head, unsigned int index) +{ + struct device *dev = &idxd->pdev->dev; + struct idxd_evl *evl = idxd->evl; + u8 status; + + if (test_bit(index, evl->bmap)) { + clear_bit(index, evl->bmap); + } else { + status = DSA_COMP_STATUS(entry_head->error); + + if (status == DSA_COMP_CRA_XLAT || status == DSA_COMP_DRAIN_EVL || + status == DSA_COMP_BATCH_EVL_ERR) { + struct idxd_evl_fault *fault; + int ent_size = evl_ent_size(idxd); + + if (entry_head->rci) + dev_dbg(dev, "Completion Int Req set, ignoring!\n"); + + if (!entry_head->rcr && status == DSA_COMP_DRAIN_EVL) + return; + + fault = kmem_cache_alloc(idxd->evl_cache, GFP_ATOMIC); + if (fault) { + struct idxd_wq *wq = idxd->wqs[entry_head->wq_idx]; + + fault->wq = wq; + fault->status = status; + memcpy(&fault->entry, entry_head, ent_size); + INIT_WORK(&fault->work, idxd_evl_fault_work); + queue_work(wq->wq, &fault->work); + } else { + dev_warn(dev, "Failed to service fault work.\n"); + } + } else { + dev_warn_ratelimited(dev, "Device error %#x operation: %#x fault addr: %#llx\n", + status, entry_head->operation, + entry_head->fault_addr); + } + } +} + +static void process_evl_entries(struct idxd_device *idxd) +{ + union evl_status_reg evl_status; + unsigned int h, t; + struct idxd_evl *evl = idxd->evl; + struct __evl_entry *entry_head; + unsigned int ent_size = evl_ent_size(idxd); + u32 size; + + evl_status.bits = 0; + evl_status.int_pending = 1; + + mutex_lock(&evl->lock); + /* Clear interrupt pending bit */ + iowrite32(evl_status.bits_upper32, + idxd->reg_base + IDXD_EVLSTATUS_OFFSET + sizeof(u32)); + evl_status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); + t = evl_status.tail; + h = evl_status.head; + size = idxd->evl->size; + + while (h != t) { + entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); + process_evl_entry(idxd, entry_head, h); + h = (h + 1) % size; + } + + evl_status.head = h; + iowrite32(evl_status.bits_lower32, idxd->reg_base + IDXD_EVLSTATUS_OFFSET); + mutex_unlock(&evl->lock); +} + +static void idxd_device_flr(struct work_struct *work) +{ + struct idxd_device *idxd = container_of(work, struct idxd_device, work); + int rc; + + /* + * IDXD device requires a Function Level Reset (FLR). + * pci_reset_function() will reset the device with FLR. + */ + rc = pci_reset_function(idxd->pdev); + if (rc) + dev_err(&idxd->pdev->dev, "FLR failed\n"); +} + +static irqreturn_t idxd_halt(struct idxd_device *idxd) +{ union gensts_reg gensts; + + gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); + if (gensts.state == IDXD_DEVICE_STATE_HALT) { + idxd->state = IDXD_DEV_HALTED; + if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { + /* + * If we need a software reset, we will throw the work + * on a system workqueue in order to allow interrupts + * for the device command completions. + */ + INIT_WORK(&idxd->work, idxd_device_reinit); + queue_work(idxd->wq, &idxd->work); + } else if (gensts.reset_type == IDXD_DEVICE_RESET_FLR) { + idxd->state = IDXD_DEV_HALTED; + idxd_mask_error_interrupts(idxd); + dev_dbg(&idxd->pdev->dev, + "idxd halted, doing FLR. After FLR, configs are restored\n"); + INIT_WORK(&idxd->work, idxd_device_flr); + queue_work(idxd->wq, &idxd->work); + + } else { + idxd->state = IDXD_DEV_HALTED; + idxd_wqs_quiesce(idxd); + idxd_wqs_unmap_portal(idxd); + idxd_device_clear_state(idxd); + dev_err(&idxd->pdev->dev, + "idxd halted, need system reset"); + + return -ENXIO; + } + } + + return IRQ_HANDLED; +} + +irqreturn_t idxd_misc_thread(int vec, void *data) +{ + struct idxd_irq_entry *irq_entry = data; + struct idxd_device *idxd = ie_to_idxd(irq_entry); + struct device *dev = &idxd->pdev->dev; u32 val = 0; int i; - bool err = false; + u32 cause; + + cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); + if (!cause) + return IRQ_NONE; + + iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); + + if (cause & IDXD_INTC_HALT_STATE) + return idxd_halt(idxd); if (cause & IDXD_INTC_ERR) { - spin_lock_bh(&idxd->dev_lock); + spin_lock(&idxd->dev_lock); for (i = 0; i < 4; i++) idxd->sw_err.bits[i] = ioread64(idxd->reg_base + IDXD_SWERR_OFFSET + i * sizeof(u64)); @@ -136,13 +479,29 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) } } - spin_unlock_bh(&idxd->dev_lock); + spin_unlock(&idxd->dev_lock); val |= IDXD_INTC_ERR; for (i = 0; i < 4; i++) - dev_warn(dev, "err[%d]: %#16.16llx\n", - i, idxd->sw_err.bits[i]); - err = true; + dev_warn_ratelimited(dev, "err[%d]: %#16.16llx\n", + i, idxd->sw_err.bits[i]); + } + + if (cause & IDXD_INTC_INT_HANDLE_REVOKED) { + struct idxd_int_handle_revoke *revoke; + + val |= IDXD_INTC_INT_HANDLE_REVOKED; + + revoke = kzalloc(sizeof(*revoke), GFP_ATOMIC); + if (revoke) { + revoke->idxd = idxd; + INIT_WORK(&revoke->work, idxd_int_handle_revoke); + queue_work(idxd->wq, &revoke->work); + + } else { + dev_err(dev, "Failed to allocate work for int handle revoke\n"); + idxd_wqs_quiesce(idxd); + } } if (cause & IDXD_INTC_CMD) { @@ -160,186 +519,138 @@ static int process_misc_interrupts(struct idxd_device *idxd, u32 cause) perfmon_counter_overflow(idxd); } + if (cause & IDXD_INTC_EVL) { + val |= IDXD_INTC_EVL; + process_evl_entries(idxd); + } + val ^= cause; if (val) dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n", val); - if (!err) - return 0; - - /* - * This case should rarely happen and typically is due to software - * programming error by the driver. - */ - if (idxd->sw_err.valid && - idxd->sw_err.desc_valid && - idxd->sw_err.fault_addr) - idxd_device_schedule_fault_process(idxd, idxd->sw_err.fault_addr); - - gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET); - if (gensts.state == IDXD_DEVICE_STATE_HALT) { - idxd->state = IDXD_DEV_HALTED; - if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) { - /* - * If we need a software reset, we will throw the work - * on a system workqueue in order to allow interrupts - * for the device command completions. - */ - INIT_WORK(&idxd->work, idxd_device_reinit); - queue_work(idxd->wq, &idxd->work); - } else { - spin_lock_bh(&idxd->dev_lock); - idxd_wqs_quiesce(idxd); - idxd_wqs_unmap_portal(idxd); - idxd_device_wqs_clear_state(idxd); - dev_err(&idxd->pdev->dev, - "idxd halted, need %s.\n", - gensts.reset_type == IDXD_DEVICE_RESET_FLR ? - "FLR" : "system reset"); - spin_unlock_bh(&idxd->dev_lock); - return -ENXIO; - } - } - - return 0; -} - -irqreturn_t idxd_misc_thread(int vec, void *data) -{ - struct idxd_irq_entry *irq_entry = data; - struct idxd_device *idxd = irq_entry->idxd; - int rc; - u32 cause; - - cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); - if (cause) - iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); - - while (cause) { - rc = process_misc_interrupts(idxd, cause); - if (rc < 0) - break; - cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET); - if (cause) - iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET); - } - return IRQ_HANDLED; } -static inline bool match_fault(struct idxd_desc *desc, u64 fault_addr) +static void idxd_int_handle_resubmit_work(struct work_struct *work) { - /* - * Completion address can be bad as well. Check fault address match for descriptor - * and completion address. - */ - if ((u64)desc->hw == fault_addr || (u64)desc->completion == fault_addr) { - struct idxd_device *idxd = desc->wq->idxd; - struct device *dev = &idxd->pdev->dev; + struct idxd_resubmit *irw = container_of(work, struct idxd_resubmit, work); + struct idxd_desc *desc = irw->desc; + struct idxd_wq *wq = desc->wq; + int rc; - dev_warn(dev, "desc with fault address: %#llx\n", fault_addr); - return true; + desc->completion->status = 0; + rc = idxd_submit_desc(wq, desc); + if (rc < 0) { + dev_dbg(&wq->idxd->pdev->dev, "Failed to resubmit desc %d to wq %d.\n", + desc->id, wq->id); + /* + * If the error is not -EAGAIN, it means the submission failed due to wq + * has been killed instead of ENQCMDS failure. Here the driver needs to + * notify the submitter of the failure by reporting abort status. + * + * -EAGAIN comes from ENQCMDS failure. idxd_submit_desc() will handle the + * abort. + */ + if (rc != -EAGAIN) { + desc->completion->status = IDXD_COMP_DESC_ABORT; + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, false); + } + idxd_free_desc(wq, desc); } - - return false; + kfree(irw); } -static inline void complete_desc(struct idxd_desc *desc, enum idxd_complete_type reason) +bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc) { - idxd_dma_complete_txd(desc, reason); - idxd_free_desc(desc->wq, desc); + struct idxd_wq *wq = desc->wq; + struct idxd_device *idxd = wq->idxd; + struct idxd_resubmit *irw; + + irw = kzalloc(sizeof(*irw), GFP_KERNEL); + if (!irw) + return false; + + irw->desc = desc; + INIT_WORK(&irw->work, idxd_int_handle_resubmit_work); + queue_work(idxd->wq, &irw->work); + return true; } -static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, - int *processed, u64 data) +static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry) { struct idxd_desc *desc, *t; struct llist_node *head; - int queued = 0; - unsigned long flags; - enum idxd_complete_type reason; - *processed = 0; head = llist_del_all(&irq_entry->pending_llist); if (!head) - goto out; - - if (wtype == IRQ_WORK_NORMAL) - reason = IDXD_COMPLETE_NORMAL; - else - reason = IDXD_COMPLETE_DEV_FAIL; + return; llist_for_each_entry_safe(desc, t, head, llnode) { - if (desc->completion->status) { - if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS) - match_fault(desc, data); - complete_desc(desc, reason); - (*processed)++; + u8 status = desc->completion->status & DSA_COMP_STATUS_MASK; + + if (status) { + /* + * Check against the original status as ABORT is software defined + * and 0xff, which DSA_COMP_STATUS_MASK can mask out. + */ + if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true); + continue; + } + + idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true); } else { - spin_lock_irqsave(&irq_entry->list_lock, flags); + spin_lock(&irq_entry->list_lock); list_add_tail(&desc->list, &irq_entry->work_list); - spin_unlock_irqrestore(&irq_entry->list_lock, flags); - queued++; + spin_unlock(&irq_entry->list_lock); } } - - out: - return queued; } -static int irq_process_work_list(struct idxd_irq_entry *irq_entry, - enum irq_work_type wtype, - int *processed, u64 data) +static void irq_process_work_list(struct idxd_irq_entry *irq_entry) { - int queued = 0; - unsigned long flags; LIST_HEAD(flist); struct idxd_desc *desc, *n; - enum idxd_complete_type reason; - - *processed = 0; - if (wtype == IRQ_WORK_NORMAL) - reason = IDXD_COMPLETE_NORMAL; - else - reason = IDXD_COMPLETE_DEV_FAIL; /* * This lock protects list corruption from access of list outside of the irq handler * thread. */ - spin_lock_irqsave(&irq_entry->list_lock, flags); + spin_lock(&irq_entry->list_lock); if (list_empty(&irq_entry->work_list)) { - spin_unlock_irqrestore(&irq_entry->list_lock, flags); - return 0; + spin_unlock(&irq_entry->list_lock); + return; } list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) { if (desc->completion->status) { - list_del(&desc->list); - (*processed)++; - list_add_tail(&desc->list, &flist); - } else { - queued++; + list_move_tail(&desc->list, &flist); } } - spin_unlock_irqrestore(&irq_entry->list_lock, flags); + spin_unlock(&irq_entry->list_lock); - list_for_each_entry(desc, &flist, list) { - if ((desc->completion->status & DSA_COMP_STATUS_MASK) != DSA_COMP_SUCCESS) - match_fault(desc, data); - complete_desc(desc, reason); - } + list_for_each_entry_safe(desc, n, &flist, list) { + /* + * Check against the original status as ABORT is software defined + * and 0xff, which DSA_COMP_STATUS_MASK can mask out. + */ + list_del(&desc->list); + + if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) { + idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true); + continue; + } - return queued; + idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true); + } } -static int idxd_desc_process(struct idxd_irq_entry *irq_entry) +irqreturn_t idxd_wq_thread(int irq, void *data) { - int rc, processed, total = 0; + struct idxd_irq_entry *irq_entry = data; /* * There are two lists we are processing. The pending_llist is where @@ -358,31 +669,9 @@ static int idxd_desc_process(struct idxd_irq_entry *irq_entry) * and process the completed entries. * 4. If the entry is still waiting on hardware, list_add_tail() to * the work_list. - * 5. Repeat until no more descriptors. */ - do { - rc = irq_process_work_list(irq_entry, IRQ_WORK_NORMAL, - &processed, 0); - total += processed; - if (rc != 0) - continue; - - rc = irq_process_pending_llist(irq_entry, IRQ_WORK_NORMAL, - &processed, 0); - total += processed; - } while (rc != 0); - - return total; -} - -irqreturn_t idxd_wq_thread(int irq, void *data) -{ - struct idxd_irq_entry *irq_entry = data; - int processed; - - processed = idxd_desc_process(irq_entry); - if (processed == 0) - return IRQ_NONE; + irq_process_work_list(irq_entry); + irq_process_pending_llist(irq_entry); return IRQ_HANDLED; } diff --git a/drivers/dma/idxd/perfmon.c b/drivers/dma/idxd/perfmon.c index d73004f47cf4..4b6af2f15d8a 100644 --- a/drivers/dma/idxd/perfmon.c +++ b/drivers/dma/idxd/perfmon.c @@ -6,29 +6,6 @@ #include "idxd.h" #include "perfmon.h" -static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, - char *buf); - -static cpumask_t perfmon_dsa_cpu_mask; -static bool cpuhp_set_up; -static enum cpuhp_state cpuhp_slot; - -/* - * perf userspace reads this attribute to determine which cpus to open - * counters on. It's connected to perfmon_dsa_cpu_mask, which is - * maintained by the cpu hotplug handlers. - */ -static DEVICE_ATTR_RO(cpumask); - -static struct attribute *perfmon_cpumask_attrs[] = { - &dev_attr_cpumask.attr, - NULL, -}; - -static struct attribute_group cpumask_attr_group = { - .attrs = perfmon_cpumask_attrs, -}; - /* * These attributes specify the bits in the config word that the perf * syscall uses to pass the event ids and categories to perfmon. @@ -67,16 +44,9 @@ static struct attribute_group perfmon_format_attr_group = { static const struct attribute_group *perfmon_attr_groups[] = { &perfmon_format_attr_group, - &cpumask_attr_group, NULL, }; -static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - return cpumap_print_to_pagebuf(true, buf, &perfmon_dsa_cpu_mask); -} - static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event) { return &idxd_pmu->pmu == event->pmu; @@ -217,7 +187,6 @@ static int perfmon_pmu_event_init(struct perf_event *event) return -EINVAL; event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd)); - event->cpu = idxd->idxd_pmu->cpu; event->hw.config = event->attr.config; if (event->group_leader != event) @@ -245,12 +214,11 @@ static void perfmon_pmu_event_update(struct perf_event *event) int shift = 64 - idxd->idxd_pmu->counter_width; struct hw_perf_event *hwc = &event->hw; + prev_raw_count = local64_read(&hwc->prev_count); do { - prev_raw_count = local64_read(&hwc->prev_count); new_raw_count = perfmon_pmu_read_counter(event); - } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, - new_raw_count) != prev_raw_count); - + } while (!local64_try_cmpxchg(&hwc->prev_count, + &prev_raw_count, new_raw_count)); n = (new_raw_count << shift); p = (prev_raw_count << shift); @@ -481,14 +449,15 @@ static void idxd_pmu_init(struct idxd_pmu *idxd_pmu) idxd_pmu->pmu.attr_groups = perfmon_attr_groups; idxd_pmu->pmu.task_ctx_nr = perf_invalid_context; idxd_pmu->pmu.event_init = perfmon_pmu_event_init; - idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable, - idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable, + idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable; + idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable; idxd_pmu->pmu.add = perfmon_pmu_event_add; idxd_pmu->pmu.del = perfmon_pmu_event_del; idxd_pmu->pmu.start = perfmon_pmu_event_start; idxd_pmu->pmu.stop = perfmon_pmu_event_stop; idxd_pmu->pmu.read = perfmon_pmu_event_update; idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE; + idxd_pmu->pmu.scope = PERF_PMU_SCOPE_SYS_WIDE; idxd_pmu->pmu.module = THIS_MODULE; } @@ -497,50 +466,11 @@ void perfmon_pmu_remove(struct idxd_device *idxd) if (!idxd->idxd_pmu) return; - cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node); perf_pmu_unregister(&idxd->idxd_pmu->pmu); kfree(idxd->idxd_pmu); idxd->idxd_pmu = NULL; } -static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node) -{ - struct idxd_pmu *idxd_pmu; - - idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); - - /* select the first online CPU as the designated reader */ - if (cpumask_empty(&perfmon_dsa_cpu_mask)) { - cpumask_set_cpu(cpu, &perfmon_dsa_cpu_mask); - idxd_pmu->cpu = cpu; - } - - return 0; -} - -static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node) -{ - struct idxd_pmu *idxd_pmu; - unsigned int target; - - idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node); - - if (!cpumask_test_and_clear_cpu(cpu, &perfmon_dsa_cpu_mask)) - return 0; - - target = cpumask_any_but(cpu_online_mask, cpu); - - /* migrate events if there is a valid target */ - if (target < nr_cpu_ids) - cpumask_set_cpu(target, &perfmon_dsa_cpu_mask); - else - target = -1; - - perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target); - - return 0; -} - int perfmon_pmu_init(struct idxd_device *idxd) { union idxd_perfcap perfcap; @@ -548,12 +478,6 @@ int perfmon_pmu_init(struct idxd_device *idxd) int rc = -ENODEV; /* - * perfmon module initialization failed, nothing to do - */ - if (!cpuhp_set_up) - return -ENODEV; - - /* * If perfmon_offset or num_counters is 0, it means perfmon is * not supported on this hardware. */ @@ -628,11 +552,6 @@ int perfmon_pmu_init(struct idxd_device *idxd) if (rc) goto free; - rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node); - if (rc) { - perf_pmu_unregister(&idxd->idxd_pmu->pmu); - goto free; - } out: return rc; free: @@ -641,22 +560,3 @@ free: goto out; } - -void __init perfmon_init(void) -{ - int rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, - "driver/dma/idxd/perf:online", - perf_event_cpu_online, - perf_event_cpu_offline); - if (WARN_ON(rc < 0)) - return; - - cpuhp_slot = rc; - cpuhp_set_up = true; -} - -void __exit perfmon_exit(void) -{ - if (cpuhp_set_up) - cpuhp_remove_multi_state(cpuhp_slot); -} diff --git a/drivers/dma/idxd/registers.h b/drivers/dma/idxd/registers.h index c970c3f025f0..8dc2e8bca779 100644 --- a/drivers/dma/idxd/registers.h +++ b/drivers/dma/idxd/registers.h @@ -3,9 +3,21 @@ #ifndef _IDXD_REGISTERS_H_ #define _IDXD_REGISTERS_H_ +#ifdef __KERNEL__ +#include <uapi/linux/idxd.h> +#else +#include <linux/idxd.h> +#endif + /* PCI Config */ -#define PCI_DEVICE_ID_INTEL_DSA_SPR0 0x0b25 -#define PCI_DEVICE_ID_INTEL_IAX_SPR0 0x0cfe +#define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb +#define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 +#define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 +#define PCI_DEVICE_ID_INTEL_IAA_PTL 0xb02d +#define PCI_DEVICE_ID_INTEL_IAA_WCL 0xfd2d + +#define DEVICE_VERSION_1 0x100 +#define DEVICE_VERSION_2 0x200 #define IDXD_MMIO_BAR 0 #define IDXD_WQ_BAR 2 @@ -28,16 +40,17 @@ union gen_cap_reg { u64 rsvd:3; u64 dest_readback:1; u64 drain_readback:1; - u64 rsvd2:6; + u64 rsvd2:3; + u64 evl_support:2; + u64 batch_continuation:1; u64 max_xfer_shift:5; u64 max_batch_shift:4; u64 max_ims_mult:6; u64 config_en:1; - u64 max_descs_per_engine:8; - u64 rsvd3:24; + u64 rsvd3:32; }; u64 bits; -} __packed; +}; #define IDXD_GENCAP_OFFSET 0x10 union wq_cap_reg { @@ -52,23 +65,26 @@ union wq_cap_reg { u64 priority:1; u64 occupancy:1; u64 occupancy_int:1; - u64 rsvd3:10; + u64 op_config:1; + u64 wq_prs_support:1; + u64 rsvd4:8; }; u64 bits; -} __packed; +}; #define IDXD_WQCAP_OFFSET 0x20 #define IDXD_WQCFG_MIN 5 union group_cap_reg { struct { u64 num_groups:8; - u64 total_tokens:8; - u64 token_en:1; - u64 token_limit:1; - u64 rsvd:46; + u64 total_rdbufs:8; /* formerly total_tokens */ + u64 rdbuf_ctrl:1; /* formerly token_en */ + u64 rdbuf_limit:1; /* formerly token_limit */ + u64 progress_limit:1; /* descriptor and batch descriptor */ + u64 rsvd:45; }; u64 bits; -} __packed; +}; #define IDXD_GRPCAP_OFFSET 0x30 union engine_cap_reg { @@ -77,7 +93,7 @@ union engine_cap_reg { u64 rsvd:56; }; u64 bits; -} __packed; +}; #define IDXD_ENGCAP_OFFSET 0x38 @@ -88,6 +104,8 @@ struct opcap { u64 bits[4]; }; +#define IDXD_MAX_OPCAP_BITS 256U + #define IDXD_OPCAP_OFFSET 0x40 #define IDXD_TABLE_OFFSET 0x60 @@ -101,30 +119,32 @@ union offsets_reg { u64 rsvd:48; }; u64 bits[2]; -} __packed; +}; #define IDXD_TABLE_MULT 0x100 #define IDXD_GENCFG_OFFSET 0x80 union gencfg_reg { struct { - u32 token_limit:8; + u32 rdbuf_limit:8; u32 rsvd:4; u32 user_int_en:1; - u32 rsvd2:19; + u32 evl_en:1; + u32 rsvd2:18; }; u32 bits; -} __packed; +}; #define IDXD_GENCTRL_OFFSET 0x88 union genctrl_reg { struct { u32 softerr_int_en:1; u32 halt_int_en:1; - u32 rsvd:30; + u32 evl_int_en:1; + u32 rsvd:29; }; u32 bits; -} __packed; +}; #define IDXD_GENSTATS_OFFSET 0x90 union gensts_reg { @@ -134,7 +154,7 @@ union gensts_reg { u32 rsvd:28; }; u32 bits; -} __packed; +}; enum idxd_device_status_state { IDXD_DEVICE_STATE_DISABLED = 0, @@ -155,6 +175,9 @@ enum idxd_device_reset_type { #define IDXD_INTC_CMD 0x02 #define IDXD_INTC_OCCUPY 0x04 #define IDXD_INTC_PERFMON_OVFL 0x08 +#define IDXD_INTC_HALT_STATE 0x10 +#define IDXD_INTC_EVL 0x20 +#define IDXD_INTC_INT_HANDLE_REVOKED 0x80000000 #define IDXD_CMD_OFFSET 0xa0 union idxd_command_reg { @@ -165,7 +188,7 @@ union idxd_command_reg { u32 int_req:1; }; u32 bits; -} __packed; +}; enum idxd_cmd { IDXD_CMD_ENABLE_DEVICE = 1, @@ -195,7 +218,7 @@ union cmdsts_reg { u8 active:1; }; u32 bits; -} __packed; +}; #define IDXD_CMDSTS_ACTIVE 0x80000000 #define IDXD_CMDSTS_ERR_MASK 0xff #define IDXD_CMDSTS_RES_SHIFT 8 @@ -266,7 +289,46 @@ union sw_err_reg { u64 rsvd5; }; u64 bits[4]; -} __packed; +}; + +union iaa_cap_reg { + struct { + u64 dec_aecs_format_ver:1; + u64 drop_init_bits:1; + u64 chaining:1; + u64 force_array_output_mod:1; + u64 load_part_aecs:1; + u64 comp_early_abort:1; + u64 nested_comp:1; + u64 diction_comp:1; + u64 header_gen:1; + u64 crypto_gcm:1; + u64 crypto_cfb:1; + u64 crypto_xts:1; + u64 rsvd:52; + }; + u64 bits; +}; + +#define IDXD_IAACAP_OFFSET 0x180 + +#define IDXD_EVLCFG_OFFSET 0xe0 +union evlcfg_reg { + struct { + u64 pasid_en:1; + u64 priv:1; + u64 rsvd:10; + u64 base_addr:52; + + u64 size:16; + u64 pasid:20; + u64 rsvd2:28; + }; + u64 bits[2]; +}; + +#define IDXD_EVL_SIZE_MIN 0x0040 +#define IDXD_EVL_SIZE_MAX 0xffff union msix_perm { struct { @@ -277,27 +339,31 @@ union msix_perm { u32 pasid:20; }; u32 bits; -} __packed; +}; union group_flags { struct { - u32 tc_a:3; - u32 tc_b:3; - u32 rsvd:1; - u32 use_token_limit:1; - u32 tokens_reserved:8; - u32 rsvd2:4; - u32 tokens_allowed:8; - u32 rsvd3:4; + u64 tc_a:3; + u64 tc_b:3; + u64 rsvd:1; + u64 use_rdbuf_limit:1; + u64 rdbufs_reserved:8; + u64 rsvd2:4; + u64 rdbufs_allowed:8; + u64 rsvd3:4; + u64 desc_progress_limit:2; + u64 rsvd4:2; + u64 batch_progress_limit:2; + u64 rsvd5:26; }; - u32 bits; -} __packed; + u64 bits; +}; struct grpcfg { u64 wqs[4]; u64 engines; union group_flags flags; -} __packed; +}; union wqcfg { struct { @@ -313,7 +379,7 @@ union wqcfg { u32 mode:1; /* shared or dedicated */ u32 bof:1; /* block on fault */ u32 wq_ats_disable:1; - u32 rsvd2:1; + u32 wq_prs_disable:1; u32 priority:4; u32 pasid:20; u32 pasid_en:1; @@ -344,11 +410,18 @@ union wqcfg { /* bytes 28-31 */ u32 rsvd8; + + /* bytes 32-63 */ + u64 op_config[4]; }; - u32 bits[8]; -} __packed; + u32 bits[16]; +}; #define WQCFG_PASID_IDX 2 +#define WQCFG_PRIVL_IDX 2 +#define WQCFG_OCCUP_IDX 6 + +#define WQCFG_OCCUP_MASK 0xffff /* * This macro calculates the offset into the WQCFG register @@ -374,12 +447,14 @@ union wqcfg { /* * This macro calculates the offset into the GRPCFG register * idxd - struct idxd * - * n - wq id - * ofs - the index of the 32b dword for the config register + * n - group id + * ofs - the index of the 64b qword for the config register * - * The WQCFG register block is divided into groups per each wq. The n index - * allows us to move to the register group that's for that particular wq. - * Each register is 32bits. The ofs gives us the number of register to access. + * The GRPCFG register block is divided into three sub-registers, which + * are GRPWQCFG, GRPENGCFG and GRPFLGCFG. The n index allows us to move + * to the register block that contains the three sub-registers. + * Each register block is 64bits. And the ofs gives us the offset + * within the GRPWQCFG register to access. */ #define GRPWQCFG_OFFSET(idxd_dev, n, ofs) ((idxd_dev)->grpcfg_offset +\ (n) * GRPCFG_SIZE + sizeof(u64) * (ofs)) @@ -404,7 +479,7 @@ union idxd_perfcap { u64 rsvd3:8; }; u64 bits; -} __packed; +}; #define IDXD_EVNTCAP_OFFSET 0x80 union idxd_evntcap { @@ -413,7 +488,7 @@ union idxd_evntcap { u64 rsvd:36; }; u64 bits; -} __packed; +}; struct idxd_event { union { @@ -423,7 +498,7 @@ struct idxd_event { }; u32 val; }; -} __packed; +}; #define IDXD_CNTRCAP_OFFSET 0x800 struct idxd_cntrcap { @@ -436,7 +511,7 @@ struct idxd_cntrcap { u32 val; }; struct idxd_event events[]; -} __packed; +}; #define IDXD_PERFRST_OFFSET 0x10 union idxd_perfrst { @@ -446,7 +521,7 @@ union idxd_perfrst { u32 rsvd:30; }; u32 val; -} __packed; +}; #define IDXD_OVFSTATUS_OFFSET 0x30 #define IDXD_PERFFRZ_OFFSET 0x20 @@ -463,7 +538,7 @@ union idxd_cntrcfg { u64 rsvd3:4; }; u64 val; -} __packed; +}; #define IDXD_FLTCFG_OFFSET 0x300 @@ -473,7 +548,7 @@ union idxd_cntrdata { u64 event_count_value; }; u64 val; -} __packed; +}; union event_cfg { struct { @@ -481,7 +556,7 @@ union event_cfg { u64 event_enc:28; }; u64 val; -} __packed; +}; union filter_cfg { struct { @@ -492,6 +567,75 @@ union filter_cfg { u64 eng:8; }; u64 val; -} __packed; +}; + +#define IDXD_EVLSTATUS_OFFSET 0xf0 + +union evl_status_reg { + struct { + u32 head:16; + u32 rsvd:16; + u32 tail:16; + u32 rsvd2:14; + u32 int_pending:1; + u32 rsvd3:1; + }; + struct { + u32 bits_lower32; + u32 bits_upper32; + }; + u64 bits; +}; + +#define IDXD_MAX_BATCH_IDENT 256 + +struct __evl_entry { + u64 rsvd:2; + u64 desc_valid:1; + u64 wq_idx_valid:1; + u64 batch:1; + u64 fault_rw:1; + u64 priv:1; + u64 err_info_valid:1; + u64 error:8; + u64 wq_idx:8; + u64 batch_id:8; + u64 operation:8; + u64 pasid:20; + u64 rsvd2:4; + + u16 batch_idx; + u16 rsvd3; + union { + /* Invalid Flags 0x11 */ + u32 invalid_flags; + /* Invalid Int Handle 0x19 */ + /* Page fault 0x1a */ + /* Page fault 0x06, 0x1f, only operand_id */ + /* Page fault before drain or in batch, 0x26, 0x27 */ + struct { + u16 int_handle; + u16 rci:1; + u16 ims:1; + u16 rcr:1; + u16 first_err_in_batch:1; + u16 rsvd4_2:9; + u16 operand_id:3; + }; + }; + u64 fault_addr; + u64 rsvd5; +}; + +struct dsa_evl_entry { + struct __evl_entry e; + struct dsa_completion_record cr; +}; + +struct iax_evl_entry { + struct __evl_entry e; + u64 rsvd[4]; + struct iax_completion_record cr; +}; #endif diff --git a/drivers/dma/idxd/submit.c b/drivers/dma/idxd/submit.c index 19afb62abaff..6db1c5fcedc5 100644 --- a/drivers/dma/idxd/submit.c +++ b/drivers/dma/idxd/submit.c @@ -21,24 +21,6 @@ static struct idxd_desc *__get_desc(struct idxd_wq *wq, int idx, int cpu) if (device_pasid_enabled(idxd)) desc->hw->pasid = idxd->pasid; - /* - * Descriptor completion vectors are 1...N for MSIX. We will round - * robin through the N vectors. - */ - wq->vec_ptr = (wq->vec_ptr % idxd->num_wq_irqs) + 1; - if (!idxd->int_handles) { - desc->hw->int_handle = wq->vec_ptr; - } else { - desc->vector = wq->vec_ptr; - /* - * int_handles are only for descriptor completion. However for device - * MSIX enumeration, vec 0 is used for misc interrupts. Therefore even - * though we are rotating through 1...N for descriptor interrupts, we - * need to acqurie the int_handles from 0..N-1. - */ - desc->hw->int_handle = idxd->int_handles[desc->vector - 1]; - } - return desc; } @@ -68,7 +50,7 @@ struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype) if (signal_pending_state(TASK_INTERRUPTIBLE, current)) break; idx = sbitmap_queue_get(sbq, &cpu); - if (idx > 0) + if (idx >= 0) break; schedule(); } @@ -79,6 +61,7 @@ struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype) return __get_desc(wq, idx, cpu); } +EXPORT_SYMBOL_NS_GPL(idxd_alloc_desc, "IDXD"); void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) { @@ -87,20 +70,131 @@ void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc) desc->cpu = -1; sbitmap_queue_clear(&wq->sbq, desc->id, cpu); } +EXPORT_SYMBOL_NS_GPL(idxd_free_desc, "IDXD"); + +static struct idxd_desc *list_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, + struct idxd_desc *desc) +{ + struct idxd_desc *d, *n; + + lockdep_assert_held(&ie->list_lock); + list_for_each_entry_safe(d, n, &ie->work_list, list) { + if (d == desc) { + list_del(&d->list); + return d; + } + } + + /* + * At this point, the desc needs to be aborted is held by the completion + * handler where it has taken it off the pending list but has not added to the + * work list. It will be cleaned up by the interrupt handler when it sees the + * IDXD_COMP_DESC_ABORT for completion status. + */ + return NULL; +} + +static void llist_abort_desc(struct idxd_wq *wq, struct idxd_irq_entry *ie, + struct idxd_desc *desc) +{ + struct idxd_desc *d, *t, *found = NULL; + struct llist_node *head; + LIST_HEAD(flist); + + desc->completion->status = IDXD_COMP_DESC_ABORT; + /* + * Grab the list lock so it will block the irq thread handler. This allows the + * abort code to locate the descriptor need to be aborted. + */ + spin_lock(&ie->list_lock); + head = llist_del_all(&ie->pending_llist); + if (head) { + llist_for_each_entry_safe(d, t, head, llnode) { + if (d == desc) { + found = desc; + continue; + } + + if (d->completion->status) + list_add_tail(&d->list, &flist); + else + list_add_tail(&d->list, &ie->work_list); + } + } + + if (!found) + found = list_abort_desc(wq, ie, desc); + spin_unlock(&ie->list_lock); + + if (found) + idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, false, + NULL, NULL); + + /* + * completing the descriptor will return desc to allocator and + * the desc can be acquired by a different process and the + * desc->list can be modified. Delete desc from list so the + * list traversing does not get corrupted by the other process. + */ + list_for_each_entry_safe(d, t, &flist, list) { + list_del_init(&d->list); + idxd_dma_complete_txd(found, IDXD_COMPLETE_ABORT, true, + NULL, NULL); + } +} + +/* + * ENQCMDS typically fail when the WQ is inactive or busy. On host submission, the driver + * has better control of number of descriptors being submitted to a shared wq by limiting + * the number of driver allocated descriptors to the wq size. However, when the swq is + * exported to a guest kernel, it may be shared with multiple guest kernels. This means + * the likelihood of getting busy returned on the swq when submitting goes significantly up. + * Having a tunable retry mechanism allows the driver to keep trying for a bit before giving + * up. The sysfs knob can be tuned by the system administrator. + */ +int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc) +{ + unsigned int retries = wq->enqcmds_retries; + int rc; + + do { + rc = enqcmds(portal, desc); + if (rc == 0) + break; + cpu_relax(); + } while (retries--); + + return rc; +} int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) { struct idxd_device *idxd = wq->idxd; + struct idxd_irq_entry *ie = NULL; + u32 desc_flags = desc->hw->flags; void __iomem *portal; int rc; if (idxd->state != IDXD_DEV_ENABLED) return -EIO; - if (!percpu_ref_tryget_live(&wq->wq_active)) - return -ENXIO; + if (!percpu_ref_tryget_live(&wq->wq_active)) { + wait_for_completion(&wq->wq_resurrect); + if (!percpu_ref_tryget_live(&wq->wq_active)) + return -ENXIO; + } - portal = wq->portal; + portal = idxd_wq_portal_addr(wq); + + /* + * Pending the descriptor to the lockless list for the irq_entry + * that we designated the descriptor to. + */ + if (desc_flags & IDXD_OP_FLAG_RCI) { + ie = &wq->ie; + desc->hw->int_handle = ie->int_handle; + llist_add(&desc->llnode, &ie->pending_llist); + } /* * The wmb() flushes writes to coherent DMA data before @@ -108,39 +202,21 @@ int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc) * even on UP because the recipient is a device. */ wmb(); + if (wq_dedicated(wq)) { iosubmit_cmds512(portal, desc->hw, 1); } else { - /* - * It's not likely that we would receive queue full rejection - * since the descriptor allocation gates at wq size. If we - * receive a -EAGAIN, that means something went wrong such as the - * device is not accepting descriptor at all. - */ - rc = enqcmds(portal, desc->hw); - if (rc < 0) + rc = idxd_enqcmds(wq, portal, desc->hw); + if (rc < 0) { + percpu_ref_put(&wq->wq_active); + /* abort operation frees the descriptor */ + if (ie) + llist_abort_desc(wq, ie, desc); return rc; + } } percpu_ref_put(&wq->wq_active); - - /* - * Pending the descriptor to the lockless list for the irq_entry - * that we designated the descriptor to. - */ - if (desc->hw->flags & IDXD_OP_FLAG_RCI) { - int vec; - - /* - * If the driver is on host kernel, it would be the value - * assigned to interrupt handle, which is index for MSIX - * vector. If it's guest then can't use the int_handle since - * that is the index to IMS for the entire device. The guest - * device local index will be used. - */ - vec = !idxd->int_handles ? desc->hw->int_handle : desc->vector; - llist_add(&desc->llnode, &idxd->irq_entries[vec].pending_llist); - } - return 0; } +EXPORT_SYMBOL_NS_GPL(idxd_submit_desc, "IDXD"); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index 0460d58e3941..9f0701021af0 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -16,338 +16,11 @@ static char *idxd_wq_type_names[] = { [IDXD_WQT_USER] = "user", }; -static int idxd_config_bus_match(struct device *dev, - struct device_driver *drv) -{ - int matched = 0; - - if (is_idxd_dev(dev)) { - struct idxd_device *idxd = confdev_to_idxd(dev); - - if (idxd->state != IDXD_DEV_CONF_READY) - return 0; - matched = 1; - } else if (is_idxd_wq_dev(dev)) { - struct idxd_wq *wq = confdev_to_wq(dev); - struct idxd_device *idxd = wq->idxd; - - if (idxd->state < IDXD_DEV_CONF_READY) - return 0; - - if (wq->state != IDXD_WQ_DISABLED) { - dev_dbg(dev, "%s not disabled\n", dev_name(dev)); - return 0; - } - matched = 1; - } - - if (matched) - dev_dbg(dev, "%s matched\n", dev_name(dev)); - - return matched; -} - -static int enable_wq(struct idxd_wq *wq) -{ - struct idxd_device *idxd = wq->idxd; - struct device *dev = &idxd->pdev->dev; - unsigned long flags; - int rc; - - mutex_lock(&wq->wq_lock); - - if (idxd->state != IDXD_DEV_ENABLED) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "Enabling while device not enabled.\n"); - return -EPERM; - } - - if (wq->state != IDXD_WQ_DISABLED) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ %d already enabled.\n", wq->id); - return -EBUSY; - } - - if (!wq->group) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ not attached to group.\n"); - return -EINVAL; - } - - if (strlen(wq->name) == 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ name not set.\n"); - return -EINVAL; - } - - /* Shared WQ checks */ - if (wq_shared(wq)) { - if (!device_swq_supported(idxd)) { - dev_warn(dev, "PASID not enabled and shared WQ.\n"); - mutex_unlock(&wq->wq_lock); - return -ENXIO; - } - /* - * Shared wq with the threshold set to 0 means the user - * did not set the threshold or transitioned from a - * dedicated wq but did not set threshold. A value - * of 0 would effectively disable the shared wq. The - * driver does not allow a value of 0 to be set for - * threshold via sysfs. - */ - if (wq->threshold == 0) { - dev_warn(dev, "Shared WQ and threshold 0.\n"); - mutex_unlock(&wq->wq_lock); - return -EINVAL; - } - } - - rc = idxd_wq_alloc_resources(wq); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ resource alloc failed\n"); - return rc; - } - - spin_lock_irqsave(&idxd->dev_lock, flags); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock_irqrestore(&idxd->dev_lock, flags); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "Writing WQ %d config failed: %d\n", wq->id, rc); - return rc; - } - - rc = idxd_wq_enable(wq); - if (rc < 0) { - mutex_unlock(&wq->wq_lock); - dev_warn(dev, "WQ %d enabling failed: %d\n", wq->id, rc); - return rc; - } - - rc = idxd_wq_map_portal(wq); - if (rc < 0) { - dev_warn(dev, "wq portal mapping failed: %d\n", rc); - rc = idxd_wq_disable(wq); - if (rc < 0) - dev_warn(dev, "IDXD wq disable failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - - wq->client_count = 0; - - if (wq->type == IDXD_WQT_KERNEL) { - rc = idxd_wq_init_percpu_ref(wq); - if (rc < 0) { - dev_dbg(dev, "percpu_ref setup failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } - - if (is_idxd_wq_dmaengine(wq)) { - rc = idxd_register_dma_channel(wq); - if (rc < 0) { - dev_dbg(dev, "DMA channel register failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } else if (is_idxd_wq_cdev(wq)) { - rc = idxd_wq_add_cdev(wq); - if (rc < 0) { - dev_dbg(dev, "Cdev creation failed\n"); - mutex_unlock(&wq->wq_lock); - return rc; - } - } - - mutex_unlock(&wq->wq_lock); - dev_info(dev, "wq %s enabled\n", dev_name(&wq->conf_dev)); - - return 0; -} - -static int idxd_config_bus_probe(struct device *dev) -{ - int rc = 0; - unsigned long flags; - - dev_dbg(dev, "%s called\n", __func__); - - if (is_idxd_dev(dev)) { - struct idxd_device *idxd = confdev_to_idxd(dev); - - if (idxd->state != IDXD_DEV_CONF_READY) { - dev_warn(dev, "Device not ready for config\n"); - return -EBUSY; - } - - if (!try_module_get(THIS_MODULE)) - return -ENXIO; - - /* Perform IDXD configuration and enabling */ - spin_lock_irqsave(&idxd->dev_lock, flags); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - rc = idxd_device_config(idxd); - spin_unlock_irqrestore(&idxd->dev_lock, flags); - if (rc < 0) { - module_put(THIS_MODULE); - dev_warn(dev, "Device config failed: %d\n", rc); - return rc; - } - - /* start device */ - rc = idxd_device_enable(idxd); - if (rc < 0) { - module_put(THIS_MODULE); - dev_warn(dev, "Device enable failed: %d\n", rc); - return rc; - } - - dev_info(dev, "Device %s enabled\n", dev_name(dev)); - - rc = idxd_register_dma_device(idxd); - if (rc < 0) { - module_put(THIS_MODULE); - dev_dbg(dev, "Failed to register dmaengine device\n"); - return rc; - } - return 0; - } else if (is_idxd_wq_dev(dev)) { - struct idxd_wq *wq = confdev_to_wq(dev); - - return enable_wq(wq); - } - - return -ENODEV; -} - -static void disable_wq(struct idxd_wq *wq) -{ - struct idxd_device *idxd = wq->idxd; - struct device *dev = &idxd->pdev->dev; - - mutex_lock(&wq->wq_lock); - dev_dbg(dev, "%s removing WQ %s\n", __func__, dev_name(&wq->conf_dev)); - if (wq->state == IDXD_WQ_DISABLED) { - mutex_unlock(&wq->wq_lock); - return; - } - - if (wq->type == IDXD_WQT_KERNEL) - idxd_wq_quiesce(wq); - - if (is_idxd_wq_dmaengine(wq)) - idxd_unregister_dma_channel(wq); - else if (is_idxd_wq_cdev(wq)) - idxd_wq_del_cdev(wq); - - if (idxd_wq_refcount(wq)) - dev_warn(dev, "Clients has claim on wq %d: %d\n", - wq->id, idxd_wq_refcount(wq)); - - idxd_wq_unmap_portal(wq); - - idxd_wq_drain(wq); - idxd_wq_reset(wq); - - idxd_wq_free_resources(wq); - wq->client_count = 0; - mutex_unlock(&wq->wq_lock); - - dev_info(dev, "wq %s disabled\n", dev_name(&wq->conf_dev)); -} - -static int idxd_config_bus_remove(struct device *dev) -{ - int rc; - - dev_dbg(dev, "%s called for %s\n", __func__, dev_name(dev)); - - /* disable workqueue here */ - if (is_idxd_wq_dev(dev)) { - struct idxd_wq *wq = confdev_to_wq(dev); - - disable_wq(wq); - } else if (is_idxd_dev(dev)) { - struct idxd_device *idxd = confdev_to_idxd(dev); - int i; - - dev_dbg(dev, "%s removing dev %s\n", __func__, - dev_name(&idxd->conf_dev)); - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - if (wq->state == IDXD_WQ_DISABLED) - continue; - dev_warn(dev, "Active wq %d on disable %s.\n", i, - dev_name(&idxd->conf_dev)); - device_release_driver(&wq->conf_dev); - } - - idxd_unregister_dma_device(idxd); - rc = idxd_device_disable(idxd); - if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { - for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - mutex_lock(&wq->wq_lock); - idxd_wq_disable_cleanup(wq); - mutex_unlock(&wq->wq_lock); - } - } - module_put(THIS_MODULE); - if (rc < 0) - dev_warn(dev, "Device disable failed\n"); - else - dev_info(dev, "Device %s disabled\n", dev_name(dev)); - - } - - return 0; -} - -static void idxd_config_bus_shutdown(struct device *dev) -{ - dev_dbg(dev, "%s called\n", __func__); -} - -struct bus_type dsa_bus_type = { - .name = "dsa", - .match = idxd_config_bus_match, - .probe = idxd_config_bus_probe, - .remove = idxd_config_bus_remove, - .shutdown = idxd_config_bus_shutdown, -}; - -static struct idxd_device_driver dsa_drv = { - .drv = { - .name = "dsa", - .bus = &dsa_bus_type, - .owner = THIS_MODULE, - .mod_name = KBUILD_MODNAME, - }, -}; - -/* IDXD generic driver setup */ -int idxd_register_driver(void) -{ - return driver_register(&dsa_drv.drv); -} - -void idxd_unregister_driver(void) -{ - driver_unregister(&dsa_drv.drv); -} - /* IDXD engine attributes */ static ssize_t engine_group_id_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_engine *engine = - container_of(dev, struct idxd_engine, conf_dev); + struct idxd_engine *engine = confdev_to_engine(dev); if (engine->group) return sysfs_emit(buf, "%d\n", engine->group->id); @@ -359,8 +32,7 @@ static ssize_t engine_group_id_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_engine *engine = - container_of(dev, struct idxd_engine, conf_dev); + struct idxd_engine *engine = confdev_to_engine(dev); struct idxd_device *idxd = engine->idxd; long id; int rc; @@ -414,12 +86,12 @@ static const struct attribute_group *idxd_engine_attribute_groups[] = { static void idxd_conf_engine_release(struct device *dev) { - struct idxd_engine *engine = container_of(dev, struct idxd_engine, conf_dev); + struct idxd_engine *engine = confdev_to_engine(dev); kfree(engine); } -struct device_type idxd_engine_device_type = { +const struct device_type idxd_engine_device_type = { .name = "engine", .release = idxd_conf_engine_release, .groups = idxd_engine_attribute_groups, @@ -427,35 +99,41 @@ struct device_type idxd_engine_device_type = { /* Group attributes */ -static void idxd_set_free_tokens(struct idxd_device *idxd) +static void idxd_set_free_rdbufs(struct idxd_device *idxd) { - int i, tokens; + int i, rdbufs; - for (i = 0, tokens = 0; i < idxd->max_groups; i++) { + for (i = 0, rdbufs = 0; i < idxd->max_groups; i++) { struct idxd_group *g = idxd->groups[i]; - tokens += g->tokens_reserved; + rdbufs += g->rdbufs_reserved; } - idxd->nr_tokens = idxd->max_tokens - tokens; + idxd->nr_rdbufs = idxd->max_rdbufs - rdbufs; +} + +static ssize_t group_read_buffers_reserved_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct idxd_group *group = confdev_to_group(dev); + + return sysfs_emit(buf, "%u\n", group->rdbufs_reserved); } static ssize_t group_tokens_reserved_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); - - return sysfs_emit(buf, "%u\n", group->tokens_reserved); + dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n"); + return group_read_buffers_reserved_show(dev, attr, buf); } -static ssize_t group_tokens_reserved_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t group_read_buffers_reserved_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; unsigned long val; int rc; @@ -473,37 +151,55 @@ static ssize_t group_tokens_reserved_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; - if (val > idxd->max_tokens) + if (val > idxd->max_rdbufs) return -EINVAL; - if (val > idxd->nr_tokens + group->tokens_reserved) + if (val > idxd->nr_rdbufs + group->rdbufs_reserved) return -EINVAL; - group->tokens_reserved = val; - idxd_set_free_tokens(idxd); + group->rdbufs_reserved = val; + idxd_set_free_rdbufs(idxd); return count; } +static ssize_t group_tokens_reserved_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n"); + return group_read_buffers_reserved_store(dev, attr, buf, count); +} + static struct device_attribute dev_attr_group_tokens_reserved = __ATTR(tokens_reserved, 0644, group_tokens_reserved_show, group_tokens_reserved_store); +static struct device_attribute dev_attr_group_read_buffers_reserved = + __ATTR(read_buffers_reserved, 0644, group_read_buffers_reserved_show, + group_read_buffers_reserved_store); + +static ssize_t group_read_buffers_allowed_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct idxd_group *group = confdev_to_group(dev); + + return sysfs_emit(buf, "%u\n", group->rdbufs_allowed); +} + static ssize_t group_tokens_allowed_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); - - return sysfs_emit(buf, "%u\n", group->tokens_allowed); + dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n"); + return group_read_buffers_allowed_show(dev, attr, buf); } -static ssize_t group_tokens_allowed_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t group_read_buffers_allowed_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; unsigned long val; int rc; @@ -522,33 +218,51 @@ static ssize_t group_tokens_allowed_store(struct device *dev, return -EPERM; if (val < 4 * group->num_engines || - val > group->tokens_reserved + idxd->nr_tokens) + val > group->rdbufs_reserved + idxd->nr_rdbufs) return -EINVAL; - group->tokens_allowed = val; + group->rdbufs_allowed = val; return count; } +static ssize_t group_tokens_allowed_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n"); + return group_read_buffers_allowed_store(dev, attr, buf, count); +} + static struct device_attribute dev_attr_group_tokens_allowed = __ATTR(tokens_allowed, 0644, group_tokens_allowed_show, group_tokens_allowed_store); +static struct device_attribute dev_attr_group_read_buffers_allowed = + __ATTR(read_buffers_allowed, 0644, group_read_buffers_allowed_show, + group_read_buffers_allowed_store); + +static ssize_t group_use_read_buffer_limit_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct idxd_group *group = confdev_to_group(dev); + + return sysfs_emit(buf, "%u\n", group->use_rdbuf_limit); +} + static ssize_t group_use_token_limit_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); - - return sysfs_emit(buf, "%u\n", group->use_token_limit); + dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n"); + return group_use_read_buffer_limit_show(dev, attr, buf); } -static ssize_t group_use_token_limit_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t group_use_read_buffer_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; unsigned long val; int rc; @@ -566,22 +280,33 @@ static ssize_t group_use_token_limit_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; - if (idxd->token_limit == 0) + if (idxd->rdbuf_limit == 0) return -EPERM; - group->use_token_limit = !!val; + group->use_rdbuf_limit = !!val; return count; } +static ssize_t group_use_token_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n"); + return group_use_read_buffer_limit_store(dev, attr, buf, count); +} + static struct device_attribute dev_attr_group_use_token_limit = __ATTR(use_token_limit, 0644, group_use_token_limit_show, group_use_token_limit_store); +static struct device_attribute dev_attr_group_use_read_buffer_limit = + __ATTR(use_read_buffer_limit, 0644, group_use_read_buffer_limit_show, + group_use_read_buffer_limit_store); + static ssize_t group_engines_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); int i, rc = 0; struct idxd_device *idxd = group->idxd; @@ -609,8 +334,7 @@ static struct device_attribute dev_attr_group_engines = static ssize_t group_work_queues_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); int i, rc = 0; struct idxd_device *idxd = group->idxd; @@ -639,8 +363,7 @@ static ssize_t group_traffic_class_a_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); return sysfs_emit(buf, "%d\n", group->tc_a); } @@ -649,8 +372,7 @@ static ssize_t group_traffic_class_a_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; long val; int rc; @@ -665,6 +387,9 @@ static ssize_t group_traffic_class_a_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; + if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) + return -EPERM; + if (val < 0 || val > 7) return -EINVAL; @@ -680,8 +405,7 @@ static ssize_t group_traffic_class_b_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); return sysfs_emit(buf, "%d\n", group->tc_b); } @@ -690,8 +414,7 @@ static ssize_t group_traffic_class_b_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_group *group = - container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); struct idxd_device *idxd = group->idxd; long val; int rc; @@ -706,6 +429,9 @@ static ssize_t group_traffic_class_b_store(struct device *dev, if (idxd->state == IDXD_DEV_ENABLED) return -EPERM; + if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) + return -EPERM; + if (val < 0 || val > 7) return -EINVAL; @@ -717,19 +443,126 @@ static struct device_attribute dev_attr_group_traffic_class_b = __ATTR(traffic_class_b, 0644, group_traffic_class_b_show, group_traffic_class_b_store); +static ssize_t group_desc_progress_limit_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct idxd_group *group = confdev_to_group(dev); + + return sysfs_emit(buf, "%d\n", group->desc_progress_limit); +} + +static ssize_t group_desc_progress_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_group *group = confdev_to_group(dev); + int val, rc; + + rc = kstrtoint(buf, 10, &val); + if (rc < 0) + return -EINVAL; + + if (val & ~GENMASK(1, 0)) + return -EINVAL; + + group->desc_progress_limit = val; + return count; +} + +static struct device_attribute dev_attr_group_desc_progress_limit = + __ATTR(desc_progress_limit, 0644, group_desc_progress_limit_show, + group_desc_progress_limit_store); + +static ssize_t group_batch_progress_limit_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct idxd_group *group = confdev_to_group(dev); + + return sysfs_emit(buf, "%d\n", group->batch_progress_limit); +} + +static ssize_t group_batch_progress_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_group *group = confdev_to_group(dev); + int val, rc; + + rc = kstrtoint(buf, 10, &val); + if (rc < 0) + return -EINVAL; + + if (val & ~GENMASK(1, 0)) + return -EINVAL; + + group->batch_progress_limit = val; + return count; +} + +static struct device_attribute dev_attr_group_batch_progress_limit = + __ATTR(batch_progress_limit, 0644, group_batch_progress_limit_show, + group_batch_progress_limit_store); static struct attribute *idxd_group_attributes[] = { &dev_attr_group_work_queues.attr, &dev_attr_group_engines.attr, &dev_attr_group_use_token_limit.attr, + &dev_attr_group_use_read_buffer_limit.attr, &dev_attr_group_tokens_allowed.attr, + &dev_attr_group_read_buffers_allowed.attr, &dev_attr_group_tokens_reserved.attr, + &dev_attr_group_read_buffers_reserved.attr, &dev_attr_group_traffic_class_a.attr, &dev_attr_group_traffic_class_b.attr, + &dev_attr_group_desc_progress_limit.attr, + &dev_attr_group_batch_progress_limit.attr, NULL, }; +static bool idxd_group_attr_progress_limit_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + return (attr == &dev_attr_group_desc_progress_limit.attr || + attr == &dev_attr_group_batch_progress_limit.attr) && + !idxd->hw.group_cap.progress_limit; +} + +static bool idxd_group_attr_read_buffers_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* + * Intel IAA does not support Read Buffer allocation control, + * make these attributes invisible. + */ + return (attr == &dev_attr_group_use_token_limit.attr || + attr == &dev_attr_group_use_read_buffer_limit.attr || + attr == &dev_attr_group_tokens_allowed.attr || + attr == &dev_attr_group_read_buffers_allowed.attr || + attr == &dev_attr_group_tokens_reserved.attr || + attr == &dev_attr_group_read_buffers_reserved.attr) && + idxd->data->type == IDXD_TYPE_IAX; +} + +static umode_t idxd_group_attr_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct idxd_group *group = confdev_to_group(dev); + struct idxd_device *idxd = group->idxd; + + if (idxd_group_attr_progress_limit_invisible(attr, idxd)) + return 0; + + if (idxd_group_attr_read_buffers_invisible(attr, idxd)) + return 0; + + return attr->mode; +} + static const struct attribute_group idxd_group_attribute_group = { .attrs = idxd_group_attributes, + .is_visible = idxd_group_attr_visible, }; static const struct attribute_group *idxd_group_attribute_groups[] = { @@ -739,12 +572,12 @@ static const struct attribute_group *idxd_group_attribute_groups[] = { static void idxd_conf_group_release(struct device *dev) { - struct idxd_group *group = container_of(dev, struct idxd_group, conf_dev); + struct idxd_group *group = confdev_to_group(dev); kfree(group); } -struct device_type idxd_group_device_type = { +const struct device_type idxd_group_device_type = { .name = "group", .release = idxd_conf_group_release, .groups = idxd_group_attribute_groups, @@ -754,7 +587,7 @@ struct device_type idxd_group_device_type = { static ssize_t wq_clients_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%d\n", wq->client_count); } @@ -765,7 +598,7 @@ static struct device_attribute dev_attr_wq_clients = static ssize_t wq_state_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); switch (wq->state) { case IDXD_WQ_DISABLED: @@ -783,7 +616,7 @@ static struct device_attribute dev_attr_wq_state = static ssize_t wq_group_id_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); if (wq->group) return sysfs_emit(buf, "%u\n", wq->group->id); @@ -795,7 +628,7 @@ static ssize_t wq_group_id_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; long id; int rc; @@ -838,7 +671,7 @@ static struct device_attribute dev_attr_wq_group_id = static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%s\n", wq_dedicated(wq) ? "dedicated" : "shared"); } @@ -847,7 +680,7 @@ static ssize_t wq_mode_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) @@ -859,7 +692,7 @@ static ssize_t wq_mode_store(struct device *dev, if (sysfs_streq(buf, "dedicated")) { set_bit(WQ_FLAG_DEDICATED, &wq->flags); wq->threshold = 0; - } else if (sysfs_streq(buf, "shared") && device_swq_supported(idxd)) { + } else if (sysfs_streq(buf, "shared")) { clear_bit(WQ_FLAG_DEDICATED, &wq->flags); } else { return -EINVAL; @@ -874,7 +707,7 @@ static struct device_attribute dev_attr_wq_mode = static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->size); } @@ -897,7 +730,7 @@ static ssize_t wq_size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); unsigned long size; struct idxd_device *idxd = wq->idxd; int rc; @@ -925,7 +758,7 @@ static struct device_attribute dev_attr_wq_size = static ssize_t wq_priority_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->priority); } @@ -934,7 +767,7 @@ static ssize_t wq_priority_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); unsigned long prio; struct idxd_device *idxd = wq->idxd; int rc; @@ -962,7 +795,7 @@ static struct device_attribute dev_attr_wq_priority = static ssize_t wq_block_on_fault_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags)); } @@ -971,11 +804,14 @@ static ssize_t wq_block_on_fault_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; bool bof; int rc; + if (!idxd->hw.gen_cap.block_on_fault) + return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return -EPERM; @@ -986,10 +822,14 @@ static ssize_t wq_block_on_fault_store(struct device *dev, if (rc < 0) return rc; - if (bof) + if (bof) { + if (test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags)) + return -EOPNOTSUPP; + set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); - else + } else { clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); + } return count; } @@ -1001,7 +841,7 @@ static struct device_attribute dev_attr_wq_block_on_fault = static ssize_t wq_threshold_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->threshold); } @@ -1010,7 +850,7 @@ static ssize_t wq_threshold_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; unsigned int val; int rc; @@ -1042,7 +882,7 @@ static struct device_attribute dev_attr_wq_threshold = static ssize_t wq_type_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); switch (wq->type) { case IDXD_WQT_KERNEL: @@ -1061,7 +901,7 @@ static ssize_t wq_type_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); enum idxd_wq_type old_type; if (wq->state != IDXD_WQ_DISABLED) @@ -1090,7 +930,7 @@ static struct device_attribute dev_attr_wq_type = static ssize_t wq_name_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%s\n", wq->name); } @@ -1099,7 +939,8 @@ static ssize_t wq_name_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); + char *input, *pos; if (wq->state != IDXD_WQ_DISABLED) return -EPERM; @@ -1107,16 +948,14 @@ static ssize_t wq_name_store(struct device *dev, if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0) return -EINVAL; - /* - * This is temporarily placed here until we have SVM support for - * dmaengine. - */ - if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd)) - return -EOPNOTSUPP; + input = kstrndup(buf, count, GFP_KERNEL); + if (!input) + return -ENOMEM; + pos = strim(input); memset(wq->name, 0, WQ_NAME_SIZE + 1); - strncpy(wq->name, buf, WQ_NAME_SIZE); - strreplace(wq->name, '\n', '\0'); + sprintf(wq->name, "%s", pos); + kfree(input); return count; } @@ -1126,7 +965,7 @@ static struct device_attribute dev_attr_wq_name = static ssize_t wq_cdev_minor_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); int minor = -1; mutex_lock(&wq->wq_lock); @@ -1160,7 +999,7 @@ static int __get_sysfs_u64(const char *buf, u64 *val) static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%llu\n", wq->max_xfer_bytes); } @@ -1168,11 +1007,14 @@ static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attri static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; u64 xfer_size; int rc; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; + if (wq->state != IDXD_WQ_DISABLED) return -EPERM; @@ -1194,7 +1036,7 @@ static struct device_attribute dev_attr_wq_max_transfer_size = static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); return sysfs_emit(buf, "%u\n", wq->max_batch_size); } @@ -1202,11 +1044,14 @@ static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribut static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; u64 batch_size; int rc; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; + if (wq->state != IDXD_WQ_DISABLED) return -EPERM; @@ -1217,7 +1062,7 @@ static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribu if (batch_size > idxd->max_batch_size) return -EINVAL; - wq->max_batch_size = (u32)batch_size; + idxd_wq_set_max_batch_size(idxd->data->type, wq, (u32)batch_size); return count; } @@ -1227,15 +1072,15 @@ static struct device_attribute dev_attr_wq_max_batch_size = static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); - return sysfs_emit(buf, "%u\n", wq->ats_dis); + return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags)); } static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); struct idxd_device *idxd = wq->idxd; bool ats_dis; int rc; @@ -1243,14 +1088,17 @@ static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute if (wq->state != IDXD_WQ_DISABLED) return -EPERM; - if (!idxd->hw.wq_cap.wq_ats_support) - return -EOPNOTSUPP; + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; rc = kstrtobool(buf, &ats_dis); if (rc < 0) return rc; - wq->ats_dis = ats_dis; + if (ats_dis) + set_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); + else + clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); return count; } @@ -1258,6 +1106,217 @@ static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute static struct device_attribute dev_attr_wq_ats_disable = __ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store); +static ssize_t wq_prs_disable_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + + return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_PRS_DISABLE, &wq->flags)); +} + +static ssize_t wq_prs_disable_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + struct idxd_device *idxd = wq->idxd; + bool prs_dis; + int rc; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; + + rc = kstrtobool(buf, &prs_dis); + if (rc < 0) + return rc; + + if (prs_dis) { + set_bit(WQ_FLAG_PRS_DISABLE, &wq->flags); + /* when PRS is disabled, BOF needs to be off as well */ + clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags); + } else { + clear_bit(WQ_FLAG_PRS_DISABLE, &wq->flags); + } + return count; +} + +static struct device_attribute dev_attr_wq_prs_disable = + __ATTR(prs_disable, 0644, wq_prs_disable_show, wq_prs_disable_store); + +static ssize_t wq_occupancy_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + struct idxd_device *idxd = wq->idxd; + u32 occup, offset; + + if (!idxd->hw.wq_cap.occupancy) + return -EOPNOTSUPP; + + offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_OCCUP_IDX); + occup = ioread32(idxd->reg_base + offset) & WQCFG_OCCUP_MASK; + + return sysfs_emit(buf, "%u\n", occup); +} + +static struct device_attribute dev_attr_wq_occupancy = + __ATTR(occupancy, 0444, wq_occupancy_show, NULL); + +static ssize_t wq_enqcmds_retries_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + + if (wq_dedicated(wq)) + return -EOPNOTSUPP; + + return sysfs_emit(buf, "%u\n", wq->enqcmds_retries); +} + +static ssize_t wq_enqcmds_retries_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + int rc; + unsigned int retries; + + if (wq_dedicated(wq)) + return -EOPNOTSUPP; + + rc = kstrtouint(buf, 10, &retries); + if (rc < 0) + return rc; + + if (retries > IDXD_ENQCMDS_MAX_RETRIES) + retries = IDXD_ENQCMDS_MAX_RETRIES; + + wq->enqcmds_retries = retries; + return count; +} + +static struct device_attribute dev_attr_wq_enqcmds_retries = + __ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store); + +static ssize_t op_cap_show_common(struct device *dev, char *buf, unsigned long *opcap_bmap) +{ + ssize_t pos; + int i; + + pos = 0; + for (i = IDXD_MAX_OPCAP_BITS/64 - 1; i >= 0; i--) { + unsigned long val = opcap_bmap[i]; + + /* On systems where direct user submissions are not safe, we need to clear out + * the BATCH capability from the capability mask in sysfs since we cannot support + * that command on such systems. Narrow the restriction of operations with the + * BATCH opcode to only DSA version 1 devices. + */ + if (i == DSA_OPCODE_BATCH/64 && !confdev_to_idxd(dev)->user_submission_safe && + confdev_to_idxd(dev)->hw.version == DEVICE_VERSION_1) + clear_bit(DSA_OPCODE_BATCH % 64, &val); + + pos += sysfs_emit_at(buf, pos, "%*pb", 64, &val); + pos += sysfs_emit_at(buf, pos, "%c", i == 0 ? '\n' : ','); + } + + return pos; +} + +static ssize_t wq_op_config_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + + return op_cap_show_common(dev, buf, wq->opcap_bmap); +} + +static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask) +{ + int bit; + + /* + * The OPCAP is defined as 256 bits that represents each operation the device + * supports per bit. Iterate through all the bits and check if the input mask + * is set for bits that are not set in the OPCAP for the device. If no OPCAP + * bit is set and input mask has the bit set, then return error. + */ + for_each_set_bit(bit, opmask, IDXD_MAX_OPCAP_BITS) { + if (!test_bit(bit, idxd->opcap_bmap)) + return -EINVAL; + } + + return 0; +} + +static ssize_t wq_op_config_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + struct idxd_device *idxd = wq->idxd; + unsigned long *opmask; + int rc; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + opmask = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); + if (!opmask) + return -ENOMEM; + + rc = bitmap_parse(buf, count, opmask, IDXD_MAX_OPCAP_BITS); + if (rc < 0) + goto err; + + rc = idxd_verify_supported_opcap(idxd, opmask); + if (rc < 0) + goto err; + + bitmap_copy(wq->opcap_bmap, opmask, IDXD_MAX_OPCAP_BITS); + + bitmap_free(opmask); + return count; + +err: + bitmap_free(opmask); + return rc; +} + +static struct device_attribute dev_attr_wq_op_config = + __ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store); + +static ssize_t wq_driver_name_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + + return sysfs_emit(buf, "%s\n", wq->driver_name); +} + +static ssize_t wq_driver_name_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_wq *wq = confdev_to_wq(dev); + char *input, *pos; + + if (wq->state != IDXD_WQ_DISABLED) + return -EPERM; + + if (strlen(buf) > DRIVER_NAME_SIZE || strlen(buf) == 0) + return -EINVAL; + + input = kstrndup(buf, count, GFP_KERNEL); + if (!input) + return -ENOMEM; + + pos = strim(input); + memset(wq->driver_name, 0, DRIVER_NAME_SIZE + 1); + sprintf(wq->driver_name, "%s", pos); + kfree(input); + return count; +} + +static struct device_attribute dev_attr_wq_driver_name = + __ATTR(driver_name, 0644, wq_driver_name_show, wq_driver_name_store); + static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_clients.attr, &dev_attr_wq_state.attr, @@ -1273,11 +1332,51 @@ static struct attribute *idxd_wq_attributes[] = { &dev_attr_wq_max_transfer_size.attr, &dev_attr_wq_max_batch_size.attr, &dev_attr_wq_ats_disable.attr, + &dev_attr_wq_prs_disable.attr, + &dev_attr_wq_occupancy.attr, + &dev_attr_wq_enqcmds_retries.attr, + &dev_attr_wq_op_config.attr, + &dev_attr_wq_driver_name.attr, NULL, }; +/* A WQ attr is invisible if the feature is not supported in WQCAP. */ +#define idxd_wq_attr_invisible(name, cap_field, a, idxd) \ + ((a) == &dev_attr_wq_##name.attr && !(idxd)->hw.wq_cap.cap_field) + +static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* Intel IAA does not support batch processing, make it invisible */ + return attr == &dev_attr_wq_max_batch_size.attr && + idxd->data->type == IDXD_TYPE_IAX; +} + +static umode_t idxd_wq_attr_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct idxd_wq *wq = confdev_to_wq(dev); + struct idxd_device *idxd = wq->idxd; + + if (idxd_wq_attr_invisible(op_config, op_config, attr, idxd)) + return 0; + + if (idxd_wq_attr_max_batch_size_invisible(attr, idxd)) + return 0; + + if (idxd_wq_attr_invisible(prs_disable, wq_prs_support, attr, idxd)) + return 0; + + if (idxd_wq_attr_invisible(ats_disable, wq_ats_support, attr, idxd)) + return 0; + + return attr->mode; +} + static const struct attribute_group idxd_wq_attribute_group = { .attrs = idxd_wq_attributes, + .is_visible = idxd_wq_attr_visible, }; static const struct attribute_group *idxd_wq_attribute_groups[] = { @@ -1287,13 +1386,15 @@ static const struct attribute_group *idxd_wq_attribute_groups[] = { static void idxd_conf_wq_release(struct device *dev) { - struct idxd_wq *wq = container_of(dev, struct idxd_wq, conf_dev); + struct idxd_wq *wq = confdev_to_wq(dev); + bitmap_free(wq->opcap_bmap); kfree(wq->wqcfg); + xa_destroy(&wq->upasid_xa); kfree(wq); } -struct device_type idxd_wq_device_type = { +const struct device_type idxd_wq_device_type = { .name = "wq", .release = idxd_conf_wq_release, .groups = idxd_wq_attribute_groups, @@ -1303,8 +1404,7 @@ struct device_type idxd_wq_device_type = { static ssize_t version_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%#x\n", idxd->hw.version); } @@ -1314,8 +1414,7 @@ static ssize_t max_work_queues_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_wq_size); } @@ -1324,8 +1423,7 @@ static DEVICE_ATTR_RO(max_work_queues_size); static ssize_t max_groups_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_groups); } @@ -1334,8 +1432,7 @@ static DEVICE_ATTR_RO(max_groups); static ssize_t max_work_queues_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_wqs); } @@ -1344,8 +1441,7 @@ static DEVICE_ATTR_RO(max_work_queues); static ssize_t max_engines_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_engines); } @@ -1354,8 +1450,7 @@ static DEVICE_ATTR_RO(max_engines); static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%d\n", dev_to_node(&idxd->pdev->dev)); } @@ -1364,8 +1459,7 @@ static DEVICE_ATTR_RO(numa_node); static ssize_t max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->max_batch_size); } @@ -1375,8 +1469,7 @@ static ssize_t max_transfer_size_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%llu\n", idxd->max_xfer_bytes); } @@ -1385,24 +1478,16 @@ static DEVICE_ATTR_RO(max_transfer_size); static ssize_t op_cap_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); - int i, rc = 0; + struct idxd_device *idxd = confdev_to_idxd(dev); - for (i = 0; i < 4; i++) - rc += sysfs_emit_at(buf, rc, "%#llx ", idxd->hw.opcap.bits[i]); - - rc--; - rc += sysfs_emit_at(buf, rc, "\n"); - return rc; + return op_cap_show_common(dev, buf, idxd->opcap_bmap); } static DEVICE_ATTR_RO(op_cap); static ssize_t gen_cap_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%#llx\n", idxd->hw.gen_cap.bits); } @@ -1411,8 +1496,7 @@ static DEVICE_ATTR_RO(gen_cap); static ssize_t configurable_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)); } @@ -1421,18 +1505,16 @@ static DEVICE_ATTR_RO(configurable); static ssize_t clients_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); - unsigned long flags; + struct idxd_device *idxd = confdev_to_idxd(dev); int count = 0, i; - spin_lock_irqsave(&idxd->dev_lock, flags); + spin_lock(&idxd->dev_lock); for (i = 0; i < idxd->max_wqs; i++) { struct idxd_wq *wq = idxd->wqs[i]; count += wq->client_count; } - spin_unlock_irqrestore(&idxd->dev_lock, flags); + spin_unlock(&idxd->dev_lock); return sysfs_emit(buf, "%d\n", count); } @@ -1441,22 +1523,19 @@ static DEVICE_ATTR_RO(clients); static ssize_t pasid_enabled_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); - return sysfs_emit(buf, "%u\n", device_pasid_enabled(idxd)); + return sysfs_emit(buf, "%u\n", device_user_pasid_enabled(idxd)); } static DEVICE_ATTR_RO(pasid_enabled); static ssize_t state_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); switch (idxd->state) { case IDXD_DEV_DISABLED: - case IDXD_DEV_CONF_READY: return sysfs_emit(buf, "disabled\n"); case IDXD_DEV_ENABLED: return sysfs_emit(buf, "enabled\n"); @@ -1471,46 +1550,55 @@ static DEVICE_ATTR_RO(state); static ssize_t errors_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); - int i, out = 0; - unsigned long flags; + struct idxd_device *idxd = confdev_to_idxd(dev); + DECLARE_BITMAP(swerr_bmap, 256); - spin_lock_irqsave(&idxd->dev_lock, flags); - for (i = 0; i < 4; i++) - out += sysfs_emit_at(buf, out, "%#018llx ", idxd->sw_err.bits[i]); - spin_unlock_irqrestore(&idxd->dev_lock, flags); - out--; - out += sysfs_emit_at(buf, out, "\n"); - return out; + bitmap_zero(swerr_bmap, 256); + spin_lock(&idxd->dev_lock); + multi_u64_to_bmap(swerr_bmap, &idxd->sw_err.bits[0], 4); + spin_unlock(&idxd->dev_lock); + return sysfs_emit(buf, "%*pb\n", 256, swerr_bmap); } static DEVICE_ATTR_RO(errors); +static ssize_t max_read_buffers_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_device *idxd = confdev_to_idxd(dev); + + return sysfs_emit(buf, "%u\n", idxd->max_rdbufs); +} + static ssize_t max_tokens_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + dev_warn_once(dev, "attribute deprecated, see max_read_buffers.\n"); + return max_read_buffers_show(dev, attr, buf); +} + +static DEVICE_ATTR_RO(max_tokens); /* deprecated */ +static DEVICE_ATTR_RO(max_read_buffers); + +static ssize_t read_buffer_limit_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_device *idxd = confdev_to_idxd(dev); - return sysfs_emit(buf, "%u\n", idxd->max_tokens); + return sysfs_emit(buf, "%u\n", idxd->rdbuf_limit); } -static DEVICE_ATTR_RO(max_tokens); static ssize_t token_limit_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); - - return sysfs_emit(buf, "%u\n", idxd->token_limit); + dev_warn_once(dev, "attribute deprecated, see read_buffer_limit.\n"); + return read_buffer_limit_show(dev, attr, buf); } -static ssize_t token_limit_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) +static ssize_t read_buffer_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); unsigned long val; int rc; @@ -1524,22 +1612,31 @@ static ssize_t token_limit_store(struct device *dev, if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return -EPERM; - if (!idxd->hw.group_cap.token_limit) + if (!idxd->hw.group_cap.rdbuf_limit) return -EPERM; - if (val > idxd->hw.group_cap.total_tokens) + if (val > idxd->hw.group_cap.total_rdbufs) return -EINVAL; - idxd->token_limit = val; + idxd->rdbuf_limit = val; return count; } -static DEVICE_ATTR_RW(token_limit); + +static ssize_t token_limit_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + dev_warn_once(dev, "attribute deprecated, see read_buffer_limit\n"); + return read_buffer_limit_store(dev, attr, buf, count); +} + +static DEVICE_ATTR_RW(token_limit); /* deprecated */ +static DEVICE_ATTR_RW(read_buffer_limit); static ssize_t cdev_major_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = - container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%u\n", idxd->major); } @@ -1548,11 +1645,131 @@ static DEVICE_ATTR_RO(cdev_major); static ssize_t cmd_status_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); return sysfs_emit(buf, "%#x\n", idxd->cmd_status); } -static DEVICE_ATTR_RO(cmd_status); + +static ssize_t cmd_status_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_device *idxd = confdev_to_idxd(dev); + + idxd->cmd_status = 0; + return count; +} +static DEVICE_ATTR_RW(cmd_status); + +static ssize_t iaa_cap_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_device *idxd = confdev_to_idxd(dev); + + if (idxd->hw.version < DEVICE_VERSION_2) + return -EOPNOTSUPP; + + return sysfs_emit(buf, "%#llx\n", idxd->hw.iaa_cap.bits); +} +static DEVICE_ATTR_RO(iaa_cap); + +static ssize_t event_log_size_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct idxd_device *idxd = confdev_to_idxd(dev); + + if (!idxd->evl) + return -EOPNOTSUPP; + + return sysfs_emit(buf, "%u\n", idxd->evl->size); +} + +static ssize_t event_log_size_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct idxd_device *idxd = confdev_to_idxd(dev); + unsigned long val; + int rc; + + if (!idxd->evl) + return -EOPNOTSUPP; + + rc = kstrtoul(buf, 10, &val); + if (rc < 0) + return -EINVAL; + + if (idxd->state == IDXD_DEV_ENABLED) + return -EPERM; + + if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) + return -EPERM; + + if (val < IDXD_EVL_SIZE_MIN || val > IDXD_EVL_SIZE_MAX || + (val * evl_ent_size(idxd) > ULONG_MAX - idxd->evl->dma)) + return -EINVAL; + + idxd->evl->size = val; + return count; +} +static DEVICE_ATTR_RW(event_log_size); + +static bool idxd_device_attr_max_batch_size_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* Intel IAA does not support batch processing, make it invisible */ + return attr == &dev_attr_max_batch_size.attr && + idxd->data->type == IDXD_TYPE_IAX; +} + +static bool idxd_device_attr_read_buffers_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + /* + * Intel IAA does not support Read Buffer allocation control, + * make these attributes invisible. + */ + return (attr == &dev_attr_max_tokens.attr || + attr == &dev_attr_max_read_buffers.attr || + attr == &dev_attr_token_limit.attr || + attr == &dev_attr_read_buffer_limit.attr) && + idxd->data->type == IDXD_TYPE_IAX; +} + +static bool idxd_device_attr_iaa_cap_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + return attr == &dev_attr_iaa_cap.attr && + (idxd->data->type != IDXD_TYPE_IAX || + idxd->hw.version < DEVICE_VERSION_2); +} + +static bool idxd_device_attr_event_log_size_invisible(struct attribute *attr, + struct idxd_device *idxd) +{ + return (attr == &dev_attr_event_log_size.attr && + !idxd->hw.gen_cap.evl_support); +} + +static umode_t idxd_device_attr_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = container_of(kobj, struct device, kobj); + struct idxd_device *idxd = confdev_to_idxd(dev); + + if (idxd_device_attr_max_batch_size_invisible(attr, idxd)) + return 0; + + if (idxd_device_attr_read_buffers_invisible(attr, idxd)) + return 0; + + if (idxd_device_attr_iaa_cap_invisible(attr, idxd)) + return 0; + + if (idxd_device_attr_event_log_size_invisible(attr, idxd)) + return 0; + + return attr->mode; +} static struct attribute *idxd_device_attributes[] = { &dev_attr_version.attr, @@ -1571,14 +1788,19 @@ static struct attribute *idxd_device_attributes[] = { &dev_attr_state.attr, &dev_attr_errors.attr, &dev_attr_max_tokens.attr, + &dev_attr_max_read_buffers.attr, &dev_attr_token_limit.attr, + &dev_attr_read_buffer_limit.attr, &dev_attr_cdev_major.attr, &dev_attr_cmd_status.attr, + &dev_attr_iaa_cap.attr, + &dev_attr_event_log_size.attr, NULL, }; static const struct attribute_group idxd_device_attribute_group = { .attrs = idxd_device_attributes, + .is_visible = idxd_device_attr_visible, }; static const struct attribute_group *idxd_attribute_groups[] = { @@ -1588,24 +1810,26 @@ static const struct attribute_group *idxd_attribute_groups[] = { static void idxd_conf_device_release(struct device *dev) { - struct idxd_device *idxd = container_of(dev, struct idxd_device, conf_dev); + struct idxd_device *idxd = confdev_to_idxd(dev); kfree(idxd->groups); + bitmap_free(idxd->wq_enable_map); kfree(idxd->wqs); kfree(idxd->engines); - kfree(idxd->irq_entries); - kfree(idxd->int_handles); + kfree(idxd->evl); + kmem_cache_destroy(idxd->evl_cache); ida_free(&idxd_ida, idxd->id); + bitmap_free(idxd->opcap_bmap); kfree(idxd); } -struct device_type dsa_device_type = { +const struct device_type dsa_device_type = { .name = "dsa", .release = idxd_conf_device_release, .groups = idxd_attribute_groups, }; -struct device_type iax_device_type = { +const struct device_type iax_device_type = { .name = "iax", .release = idxd_conf_device_release, .groups = idxd_attribute_groups, @@ -1613,12 +1837,12 @@ struct device_type iax_device_type = { static int idxd_register_engine_devices(struct idxd_device *idxd) { + struct idxd_engine *engine; int i, j, rc; for (i = 0; i < idxd->max_engines; i++) { - struct idxd_engine *engine = idxd->engines[i]; - - rc = device_add(&engine->conf_dev); + engine = idxd->engines[i]; + rc = device_add(engine_confdev(engine)); if (rc < 0) goto cleanup; } @@ -1627,22 +1851,26 @@ static int idxd_register_engine_devices(struct idxd_device *idxd) cleanup: j = i - 1; - for (; i < idxd->max_engines; i++) - put_device(&idxd->engines[i]->conf_dev); + for (; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + put_device(engine_confdev(engine)); + } - while (j--) - device_unregister(&idxd->engines[j]->conf_dev); + while (j--) { + engine = idxd->engines[j]; + device_unregister(engine_confdev(engine)); + } return rc; } static int idxd_register_group_devices(struct idxd_device *idxd) { + struct idxd_group *group; int i, j, rc; for (i = 0; i < idxd->max_groups; i++) { - struct idxd_group *group = idxd->groups[i]; - - rc = device_add(&group->conf_dev); + group = idxd->groups[i]; + rc = device_add(group_confdev(group)); if (rc < 0) goto cleanup; } @@ -1651,22 +1879,26 @@ static int idxd_register_group_devices(struct idxd_device *idxd) cleanup: j = i - 1; - for (; i < idxd->max_groups; i++) - put_device(&idxd->groups[i]->conf_dev); + for (; i < idxd->max_groups; i++) { + group = idxd->groups[i]; + put_device(group_confdev(group)); + } - while (j--) - device_unregister(&idxd->groups[j]->conf_dev); + while (j--) { + group = idxd->groups[j]; + device_unregister(group_confdev(group)); + } return rc; } static int idxd_register_wq_devices(struct idxd_device *idxd) { + struct idxd_wq *wq; int i, rc, j; for (i = 0; i < idxd->max_wqs; i++) { - struct idxd_wq *wq = idxd->wqs[i]; - - rc = device_add(&wq->conf_dev); + wq = idxd->wqs[i]; + rc = device_add(wq_confdev(wq)); if (rc < 0) goto cleanup; } @@ -1675,11 +1907,15 @@ static int idxd_register_wq_devices(struct idxd_device *idxd) cleanup: j = i - 1; - for (; i < idxd->max_wqs; i++) - put_device(&idxd->wqs[i]->conf_dev); + for (; i < idxd->max_wqs; i++) { + wq = idxd->wqs[i]; + put_device(wq_confdev(wq)); + } - while (j--) - device_unregister(&idxd->wqs[j]->conf_dev); + while (j--) { + wq = idxd->wqs[j]; + device_unregister(wq_confdev(wq)); + } return rc; } @@ -1688,7 +1924,7 @@ int idxd_register_devices(struct idxd_device *idxd) struct device *dev = &idxd->pdev->dev; int rc, i; - rc = device_add(&idxd->conf_dev); + rc = device_add(idxd_confdev(idxd)); if (rc < 0) return rc; @@ -1714,12 +1950,12 @@ int idxd_register_devices(struct idxd_device *idxd) err_group: for (i = 0; i < idxd->max_engines; i++) - device_unregister(&idxd->engines[i]->conf_dev); + device_unregister(engine_confdev(idxd->engines[i])); err_engine: for (i = 0; i < idxd->max_wqs; i++) - device_unregister(&idxd->wqs[i]->conf_dev); + device_unregister(wq_confdev(idxd->wqs[i])); err_wq: - device_del(&idxd->conf_dev); + device_del(idxd_confdev(idxd)); return rc; } @@ -1730,30 +1966,18 @@ void idxd_unregister_devices(struct idxd_device *idxd) for (i = 0; i < idxd->max_wqs; i++) { struct idxd_wq *wq = idxd->wqs[i]; - device_unregister(&wq->conf_dev); + device_unregister(wq_confdev(wq)); } for (i = 0; i < idxd->max_engines; i++) { struct idxd_engine *engine = idxd->engines[i]; - device_unregister(&engine->conf_dev); + device_unregister(engine_confdev(engine)); } for (i = 0; i < idxd->max_groups; i++) { struct idxd_group *group = idxd->groups[i]; - device_unregister(&group->conf_dev); + device_unregister(group_confdev(group)); } - - device_unregister(&idxd->conf_dev); -} - -int idxd_register_bus_type(void) -{ - return bus_register(&dsa_bus_type); -} - -void idxd_unregister_bus_type(void) -{ - bus_unregister(&dsa_bus_type); } diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c index e4ea107ce78c..fd55bcd060ab 100644 --- a/drivers/dma/img-mdc-dma.c +++ b/drivers/dma/img-mdc-dma.c @@ -17,7 +17,6 @@ #include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -886,7 +885,6 @@ static int img_mdc_runtime_resume(struct device *dev) static int mdc_dma_probe(struct platform_device *pdev) { struct mdc_dma *mdma; - struct resource *res; unsigned int i; u32 val; int ret; @@ -898,8 +896,7 @@ static int mdc_dma_probe(struct platform_device *pdev) mdma->soc = of_device_get_match_data(&pdev->dev); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mdma->regs = devm_ioremap_resource(&pdev->dev, res); + mdma->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mdma->regs)) return PTR_ERR(mdma->regs); @@ -1020,7 +1017,7 @@ suspend: return ret; } -static int mdc_dma_remove(struct platform_device *pdev) +static void mdc_dma_remove(struct platform_device *pdev) { struct mdc_dma *mdma = platform_get_drvdata(pdev); struct mdc_chan *mchan, *next; @@ -1040,8 +1037,6 @@ static int mdc_dma_remove(struct platform_device *pdev) pm_runtime_disable(&pdev->dev); if (!pm_runtime_status_suspended(&pdev->dev)) img_mdc_runtime_suspend(&pdev->dev); - - return 0; } #ifdef CONFIG_PM_SLEEP @@ -1078,7 +1073,7 @@ static struct platform_driver mdc_dma_driver = { .driver = { .name = "img-mdc-dma", .pm = &img_mdc_pm_ops, - .of_match_table = of_match_ptr(mdc_dma_of_match), + .of_match_table = mdc_dma_of_match, }, .probe = mdc_dma_probe, .remove = mdc_dma_remove, diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index 7f116bbcfad2..ba434657059a 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -17,15 +17,16 @@ #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/platform_device.h> #include <linux/clk.h> #include <linux/dmaengine.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <asm/irq.h> -#include <linux/platform_data/dma-imx.h> +#include <linux/dma/imx-dma.h> #include "dmaengine.h" #define IMXDMA_MAX_CHAN_DESCRIPTORS 16 @@ -167,7 +168,6 @@ struct imxdma_channel { enum imx_dma_type { IMX1_DMA, - IMX21_DMA, IMX27_DMA, }; @@ -195,8 +195,6 @@ static const struct of_device_id imx_dma_of_dev_id[] = { { .compatible = "fsl,imx1-dma", .data = (const void *)IMX1_DMA, }, { - .compatible = "fsl,imx21-dma", .data = (const void *)IMX21_DMA, - }, { .compatible = "fsl,imx27-dma", .data = (const void *)IMX27_DMA, }, { /* sentinel */ @@ -326,7 +324,7 @@ static void imxdma_disable_hw(struct imxdma_channel *imxdmac) dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel); if (imxdma_hw_chain(imxdmac)) - del_timer(&imxdmac->watchdog); + timer_delete(&imxdmac->watchdog); local_irq_save(flags); imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) | @@ -339,7 +337,8 @@ static void imxdma_disable_hw(struct imxdma_channel *imxdmac) static void imxdma_watchdog(struct timer_list *t) { - struct imxdma_channel *imxdmac = from_timer(imxdmac, t, watchdog); + struct imxdma_channel *imxdmac = timer_container_of(imxdmac, t, + watchdog); struct imxdma_engine *imxdma = imxdmac->imxdma; int channel = imxdmac->channel; @@ -456,7 +455,7 @@ static void dma_irq_handle_channel(struct imxdma_channel *imxdmac) } if (imxdma_hw_chain(imxdmac)) { - del_timer(&imxdmac->watchdog); + timer_delete(&imxdmac->watchdog); return; } } @@ -750,7 +749,6 @@ static int imxdma_alloc_chan_resources(struct dma_chan *chan) desc = kzalloc(sizeof(*desc), GFP_KERNEL); if (!desc) break; - memset(&desc->desc, 0, sizeof(struct dma_async_tx_descriptor)); dma_async_tx_descriptor_init(&desc->desc, chan); desc->desc.tx_submit = imxdma_tx_submit; /* txd.flags will be overwritten in prep funcs */ @@ -812,6 +810,8 @@ static struct dma_async_tx_descriptor *imxdma_prep_slave_sg( dma_length += sg_dma_len(sg); } + imxdma_config_write(chan, &imxdmac->config, direction); + switch (imxdmac->word_size) { case DMA_SLAVE_BUSWIDTH_4_BYTES: if (sg_dma_len(sgl) & 3 || sgl->dma_address & 3) @@ -944,7 +944,7 @@ static struct dma_async_tx_descriptor *imxdma_prep_dma_interleaved( " src_sgl=%s dst_sgl=%s numf=%zu frame_size=%zu\n", __func__, imxdmac->channel, (unsigned long long)xt->src_start, (unsigned long long) xt->dst_start, - xt->src_sgl ? "true" : "false", xt->dst_sgl ? "true" : "false", + str_true_false(xt->src_sgl), str_true_false(xt->dst_sgl), xt->numf, xt->frame_size); if (list_empty(&imxdmac->ld_free) || @@ -1036,7 +1036,6 @@ static struct dma_chan *imxdma_xlate(struct of_phandle_args *dma_spec, static int __init imxdma_probe(struct platform_device *pdev) { struct imxdma_engine *imxdma; - struct resource *res; int ret, i; int irq, irq_err; @@ -1045,10 +1044,9 @@ static int __init imxdma_probe(struct platform_device *pdev) return -ENOMEM; imxdma->dev = &pdev->dev; - imxdma->devtype = (enum imx_dma_type)of_device_get_match_data(&pdev->dev); + imxdma->devtype = (uintptr_t)of_device_get_match_data(&pdev->dev); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - imxdma->base = devm_ioremap_resource(&pdev->dev, res); + imxdma->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(imxdma->base)) return PTR_ERR(imxdma->base); @@ -1217,7 +1215,7 @@ static void imxdma_free_irq(struct platform_device *pdev, struct imxdma_engine * } } -static int imxdma_remove(struct platform_device *pdev) +static void imxdma_remove(struct platform_device *pdev) { struct imxdma_engine *imxdma = platform_get_drvdata(pdev); @@ -1230,8 +1228,6 @@ static int imxdma_remove(struct platform_device *pdev) clk_disable_unprepare(imxdma->dma_ipg); clk_disable_unprepare(imxdma->dma_ahb); - - return 0; } static struct platform_driver imxdma_driver = { diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index 8070fd664bfc..ed9e56de5a9b 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -14,6 +14,7 @@ #include <linux/iopoll.h> #include <linux/module.h> #include <linux/types.h> +#include <linux/bitfield.h> #include <linux/bitops.h> #include <linux/mm.h> #include <linux/interrupt.h> @@ -23,6 +24,7 @@ #include <linux/semaphore.h> #include <linux/spinlock.h> #include <linux/device.h> +#include <linux/genalloc.h> #include <linux/dma-mapping.h> #include <linux/firmware.h> #include <linux/slab.h> @@ -30,12 +32,11 @@ #include <linux/dmaengine.h> #include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/workqueue.h> #include <asm/irq.h> -#include <linux/platform_data/dma-imx.h> +#include <linux/dma/imx-dma.h> #include <linux/regmap.h> #include <linux/mfd/syscon.h> #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h> @@ -73,6 +74,7 @@ #define SDMA_CHNENBL0_IMX35 0x200 #define SDMA_CHNENBL0_IMX31 0x080 #define SDMA_CHNPRI_0 0x100 +#define SDMA_DONE0_CONFIG 0x1000 /* * Buffer descriptor status values. @@ -136,7 +138,11 @@ * 0: Source on AIPS * 12 Destination Bit(DP) 1: Destination on SPBA * 0: Destination on AIPS - * 13-15 --------- MUST BE 0 + * 13 Source FIFO 1: Source is dual FIFO + * 0: Source is single FIFO + * 14 Destination FIFO 1: Destination is dual FIFO + * 0: Destination is single FIFO + * 15 --------- MUST BE 0 * 16-23 Higher WML HWML * 24-27 N Total number of samples after * which Pad adding/Swallowing @@ -167,6 +173,8 @@ #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) #define SDMA_WATERMARK_LEVEL_SP BIT(11) #define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_SD BIT(13) +#define SDMA_WATERMARK_LEVEL_DD BIT(14) #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) #define SDMA_WATERMARK_LEVEL_LWE BIT(28) #define SDMA_WATERMARK_LEVEL_HWE BIT(29) @@ -174,13 +182,22 @@ #define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) #define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ BIT(DMA_MEM_TO_DEV) | \ BIT(DMA_DEV_TO_DEV)) -/** +#define SDMA_WATERMARK_LEVEL_N_FIFOS GENMASK(15, 12) +#define SDMA_WATERMARK_LEVEL_OFF_FIFOS GENMASK(19, 16) +#define SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO GENMASK(31, 28) +#define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23) + +#define SDMA_DONE0_CONFIG_DONE_SEL BIT(7) +#define SDMA_DONE0_CONFIG_DONE_DIS BIT(6) + +/* * struct sdma_script_start_addrs - SDMA script start pointers * * start addresses of the different functions in the physical @@ -223,16 +240,23 @@ struct sdma_script_start_addrs { s32 utra_addr; s32 ram_code_start_addr; /* End of v1 array */ - s32 mcu_2_ssish_addr; + union { s32 v1_end; s32 mcu_2_ssish_addr; }; s32 ssish_2_mcu_addr; s32 hdmi_dma_addr; /* End of v2 array */ - s32 zcanfd_2_mcu_addr; + union { s32 v2_end; s32 zcanfd_2_mcu_addr; }; s32 zqspi_2_mcu_addr; s32 mcu_2_ecspi_addr; + s32 mcu_2_sai_addr; + s32 sai_2_mcu_addr; + s32 uart_2_mcu_rom_addr; + s32 uartsh_2_mcu_rom_addr; + s32 i2c_2_mcu_addr; + s32 mcu_2_i2c_addr; /* End of v3 array */ - s32 mcu_2_zqspi_addr; + union { s32 v3_end; s32 mcu_2_zqspi_addr; }; /* End of v4 array */ + s32 v4_end[]; }; /* @@ -408,10 +432,16 @@ struct sdma_desc { * @shp_addr: value for gReg[6] * @per_addr: value for gReg[2] * @status: status of dma channel - * @context_loaded: ensure context is only loaded once * @data: specific sdma interface structure - * @bd_pool: dma_pool for bd * @terminate_worker: used to call back into terminate work function + * @terminated: terminated list + * @is_ram_script: flag for script in ram + * @n_fifos_src: number of source device fifos + * @n_fifos_dst: number of destination device fifos + * @sw_done: software done flag + * @stride_fifos_src: stride for source device FIFOs + * @stride_fifos_dst: stride for destination device FIFOs + * @words_per_fifo: copy number of words one time for one FIFO */ struct sdma_channel { struct virt_dma_chan vc; @@ -433,9 +463,16 @@ struct sdma_channel { unsigned long watermark_level; u32 shp_addr, per_addr; enum dma_status status; - bool context_loaded; struct imx_dma_data data; struct work_struct terminate_worker; + struct list_head terminated; + bool is_ram_script; + unsigned int n_fifos_src; + unsigned int n_fifos_dst; + unsigned int stride_fifos_src; + unsigned int stride_fifos_dst; + unsigned int words_per_fifo; + bool sw_done; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -458,8 +495,6 @@ struct sdma_channel { * @num_script_addrs: Number of script addresses in this image * @ram_code_start: offset of SDMA ram image in this firmware image * @ram_code_size: size of SDMA ram image - * @script_addrs: Stores the start address of the SDMA scripts - * (in SDMA memory space) */ struct sdma_firmware_header { u32 magic; @@ -476,6 +511,13 @@ struct sdma_driver_data { int num_events; struct sdma_script_start_addrs *script_addrs; bool check_ratio; + /* + * ecspi ERR009165 fixed should be done in sdma script + * and it has been fixed in soc from i.mx6ul. + * please get more information from the below link: + * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf + */ + bool ecspi_fixed; }; struct sdma_engine { @@ -499,6 +541,8 @@ struct sdma_engine { struct sdma_buffer_descriptor *bd0; /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ bool clk_ratio; + bool fw_loaded; + struct gen_pool *iram_pool; }; static int sdma_config_write(struct dma_chan *chan, @@ -595,6 +639,13 @@ static struct sdma_driver_data sdma_imx6q = { .script_addrs = &sdma_script_imx6q, }; +static struct sdma_driver_data sdma_imx6ul = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx6q, + .ecspi_fixed = true, +}; + static struct sdma_script_start_addrs sdma_script_imx7d = { .ap_2_ap_addr = 644, .uart_2_mcu_addr = 819, @@ -628,6 +679,7 @@ static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, }, + { .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, }, { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, }, { /* sentinel */ } }; @@ -680,6 +732,11 @@ static int sdma_config_ownership(struct sdma_channel *sdmac, return 0; } +static int is_sdma_channel_enabled(struct sdma_engine *sdma, int channel) +{ + return !!(readl(sdma->regs + SDMA_H_STATSTOP) & BIT(channel)); +} + static void sdma_enable_channel(struct sdma_engine *sdma, int channel) { writel(BIT(channel), sdma->regs + SDMA_H_START); @@ -720,9 +777,8 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, unsigned long flags; buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL); - if (!buf_virt) { + if (!buf_virt) return -ENOMEM; - } spin_lock_irqsave(&sdma->channel_0_lock, flags); @@ -753,6 +809,14 @@ static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event) val = readl_relaxed(sdma->regs + chnenbl); __set_bit(channel, &val); writel_relaxed(val, sdma->regs + chnenbl); + + /* Set SDMA_DONEx_CONFIG is sw_done enabled */ + if (sdmac->sw_done) { + val = readl_relaxed(sdma->regs + SDMA_DONE0_CONFIG); + val |= SDMA_DONE0_CONFIG_DONE_SEL; + val &= ~SDMA_DONE0_CONFIG_DONE_DIS; + writel_relaxed(val, sdma->regs + SDMA_DONE0_CONFIG); + } } static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) @@ -822,7 +886,6 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) */ desc->chn_real_count = bd->mode.count; - bd->mode.status |= BD_DONE; bd->mode.count = desc->period_len; desc->buf_ptail = desc->buf_tail; desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd; @@ -837,9 +900,21 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL); spin_lock(&sdmac->vc.lock); + /* Assign buffer ownership to SDMA */ + bd->mode.status |= BD_DONE; + if (error) sdmac->status = old_status; } + + /* + * SDMA stops cyclic channel when DMA request triggers a channel and no SDMA + * owned buffer is available (i.e. BD_DONE was set too late). + */ + if (sdmac->desc && !is_sdma_channel_enabled(sdmac->sdma, sdmac->channel)) { + dev_warn(sdmac->sdma->dev, "restart cyclic channel %d\n", sdmac->channel); + sdma_enable_channel(sdmac->sdma, sdmac->channel); + } } static void mxc_sdma_handle_channel_normal(struct sdma_channel *data) @@ -856,9 +931,9 @@ static void mxc_sdma_handle_channel_normal(struct sdma_channel *data) for (i = 0; i < sdmac->desc->num_bd; i++) { bd = &sdmac->desc->bd[i]; - if (bd->mode.status & (BD_DONE | BD_RROR)) + if (bd->mode.status & (BD_DONE | BD_RROR)) error = -EIO; - sdmac->desc->chn_real_count += bd->mode.count; + sdmac->desc->chn_real_count += bd->mode.count; } if (error) @@ -886,7 +961,10 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) desc = sdmac->desc; if (desc) { if (sdmac->flags & IMX_DMA_SG_LOOP) { - sdma_update_channel_loop(sdmac); + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + sdma_update_channel_loop(sdmac); + else + vchan_cyclic_callback(&desc->vd); } else { mxc_sdma_handle_channel_normal(sdmac); vchan_cookie_complete(&desc->vd); @@ -904,7 +982,7 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) /* * sets the pc of SDMA script according to the peripheral type */ -static void sdma_get_pc(struct sdma_channel *sdmac, +static int sdma_get_pc(struct sdma_channel *sdmac, enum sdma_peripheral_type peripheral_type) { struct sdma_engine *sdma = sdmac->sdma; @@ -919,6 +997,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, sdmac->pc_to_device = 0; sdmac->device_to_device = 0; sdmac->pc_to_pc = 0; + sdmac->is_ram_script = false; switch (peripheral_type) { case IMX_DMATYPE_MEMORY: @@ -945,6 +1024,17 @@ static void sdma_get_pc(struct sdma_channel *sdmac, emi_2_per = sdma->script_addrs->mcu_2_ata_addr; break; case IMX_DMATYPE_CSPI: + per_2_emi = sdma->script_addrs->app_2_mcu_addr; + + /* Use rom script mcu_2_app if ERR009165 fixed */ + if (sdmac->sdma->drvdata->ecspi_fixed) { + emi_2_per = sdma->script_addrs->mcu_2_app_addr; + } else { + emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr; + sdmac->is_ram_script = true; + } + + break; case IMX_DMATYPE_EXT: case IMX_DMATYPE_SSI: case IMX_DMATYPE_SAI: @@ -954,6 +1044,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_SSI_DUAL: per_2_emi = sdma->script_addrs->ssish_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_ssish_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_SSI_SP: case IMX_DMATYPE_MMC: @@ -968,6 +1059,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, per_2_emi = sdma->script_addrs->asrc_2_mcu_addr; emi_2_per = sdma->script_addrs->asrc_2_mcu_addr; per_2_per = sdma->script_addrs->per_2_per_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_ASRC_SP: per_2_emi = sdma->script_addrs->shp_2_mcu_addr; @@ -988,14 +1080,31 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_IPU_MEMORY: emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr; break; - default: + case IMX_DMATYPE_MULTI_SAI: + per_2_emi = sdma->script_addrs->sai_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_sai_addr; break; + case IMX_DMATYPE_I2C: + per_2_emi = sdma->script_addrs->i2c_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_i2c_addr; + sdmac->is_ram_script = true; + break; + case IMX_DMATYPE_HDMI: + emi_2_per = sdma->script_addrs->hdmi_dma_addr; + sdmac->is_ram_script = true; + break; + default: + dev_err(sdma->dev, "Unsupported transfer type %d\n", + peripheral_type); + return -EINVAL; } sdmac->pc_from_device = per_2_emi; sdmac->pc_to_device = emi_2_per; sdmac->device_to_device = per_2_per; sdmac->pc_to_pc = emi_2_emi; + + return 0; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -1008,9 +1117,6 @@ static int sdma_load_context(struct sdma_channel *sdmac) int ret; unsigned long flags; - if (sdmac->context_loaded) - return 0; - if (sdmac->direction == DMA_DEV_TO_MEM) load_address = sdmac->pc_from_device; else if (sdmac->direction == DMA_DEV_TO_DEV) @@ -1038,11 +1144,16 @@ static int sdma_load_context(struct sdma_channel *sdmac) /* Send by context the event mask,base address for peripheral * and watermark level */ - context->gReg[0] = sdmac->event_mask[1]; - context->gReg[1] = sdmac->event_mask[0]; - context->gReg[2] = sdmac->per_addr; - context->gReg[6] = sdmac->shp_addr; - context->gReg[7] = sdmac->watermark_level; + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + context->gReg[4] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + } else { + context->gReg[0] = sdmac->event_mask[1]; + context->gReg[1] = sdmac->event_mask[0]; + context->gReg[2] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + context->gReg[7] = sdmac->watermark_level; + } bd0->mode.command = C0_SETDM; bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; @@ -1053,8 +1164,6 @@ static int sdma_load_context(struct sdma_channel *sdmac) spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - sdmac->context_loaded = true; - return ret; } @@ -1078,9 +1187,6 @@ static void sdma_channel_terminate_work(struct work_struct *work) { struct sdma_channel *sdmac = container_of(work, struct sdma_channel, terminate_worker); - unsigned long flags; - LIST_HEAD(head); - /* * According to NXP R&D team a delay of one BD SDMA cost time * (maximum is 1ms) should be added after disable of the channel @@ -1089,11 +1195,7 @@ static void sdma_channel_terminate_work(struct work_struct *work) */ usleep_range(1000, 2000); - spin_lock_irqsave(&sdmac->vc.lock, flags); - vchan_get_all_descriptors(&sdmac->vc, &head); - spin_unlock_irqrestore(&sdmac->vc.lock, flags); - vchan_dma_desc_free_list(&sdmac->vc, &head); - sdmac->context_loaded = false; + vchan_dma_desc_free_list(&sdmac->vc, &sdmac->terminated); } static int sdma_terminate_all(struct dma_chan *chan) @@ -1107,6 +1209,13 @@ static int sdma_terminate_all(struct dma_chan *chan) if (sdmac->desc) { vchan_terminate_vdesc(&sdmac->desc->vd); + /* + * move out current descriptor into terminated list so that + * it could be free in sdma_channel_terminate_work alone + * later without potential involving next descriptor raised + * up before the last descriptor terminated. + */ + vchan_get_all_descriptors(&sdmac->vc, &sdmac->terminated); sdmac->desc = NULL; schedule_work(&sdmac->terminate_worker); } @@ -1163,6 +1272,44 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; + + /* + * Limitation: The p2p script support dual fifos in maximum, + * So when fifo number is larger than 1, force enable dual + * fifos. + */ + if (sdmac->n_fifos_src > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD; + if (sdmac->n_fifos_dst > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; +} + +static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) +{ + unsigned int n_fifos; + unsigned int stride_fifos; + unsigned int words_per_fifo; + + if (sdmac->sw_done) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE; + + if (sdmac->direction == DMA_DEV_TO_MEM) { + n_fifos = sdmac->n_fifos_src; + stride_fifos = sdmac->stride_fifos_src; + } else { + n_fifos = sdmac->n_fifos_dst; + stride_fifos = sdmac->stride_fifos_dst; + } + + words_per_fifo = sdmac->words_per_fifo; + + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_N_FIFOS, n_fifos); + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_OFF_FIFOS, stride_fifos); + if (words_per_fifo) + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO, (words_per_fifo - 1)); } static int sdma_config_channel(struct dma_chan *chan) @@ -1189,7 +1336,9 @@ static int sdma_config_channel(struct dma_chan *chan) break; } - sdma_get_pc(sdmac, sdmac->peripheral_type); + ret = sdma_get_pc(sdmac, sdmac->peripheral_type); + if (ret) + return ret; if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) && (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { @@ -1198,8 +1347,13 @@ static int sdma_config_channel(struct dma_chan *chan) if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || sdmac->peripheral_type == IMX_DMATYPE_ASRC) sdma_set_watermarklevel_for_p2p(sdmac); - } else + } else { + if (sdmac->peripheral_type == + IMX_DMATYPE_MULTI_SAI) + sdma_set_watermarklevel_for_sais(sdmac); + __set_bit(sdmac->event_id0, sdmac->event_mask); + } /* Address */ sdmac->shp_addr = sdmac->per_address; @@ -1208,13 +1362,11 @@ static int sdma_config_channel(struct dma_chan *chan) sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ } - ret = sdma_load_context(sdmac); - - return ret; + return 0; } static int sdma_set_channel_priority(struct sdma_channel *sdmac, - unsigned int priority) + unsigned int priority) { struct sdma_engine *sdma = sdmac->sdma; int channel = sdmac->channel; @@ -1233,8 +1385,14 @@ static int sdma_request_channel0(struct sdma_engine *sdma) { int ret = -EBUSY; - sdma->bd0 = dma_alloc_coherent(sdma->dev, PAGE_SIZE, &sdma->bd0_phys, - GFP_NOWAIT); + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); if (!sdma->bd0) { ret = -ENOMEM; goto out; @@ -1254,10 +1412,14 @@ out: static int sdma_alloc_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; int ret = 0; - desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size, - &desc->bd_phys, GFP_NOWAIT); + if (sdma->iram_pool) + desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size, &desc->bd_phys); + else + desc->bd = dma_alloc_coherent(sdma->dev, bd_size, &desc->bd_phys, GFP_NOWAIT); + if (!desc->bd) { ret = -ENOMEM; goto out; @@ -1269,9 +1431,12 @@ out: static void sdma_free_bd(struct sdma_desc *desc) { u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; - dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, - desc->bd_phys); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, bd_size); + else + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, desc->bd_phys); } static void sdma_desc_free(struct virt_dma_desc *vd) @@ -1294,9 +1459,8 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) * dmatest, thus create 'struct imx_dma_data mem_data' for this case. * Please note in any other slave case, you have to setup chan->private * with 'struct imx_dma_data' in your own filter function if you want to - * request dma channel by dma_request_channel() rather than - * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear - * to warn you to correct your filter function. + * request DMA channel by dma_request_channel(), otherwise, 'MEMCPY in + * case?' will appear to warn you to correct your filter function. */ if (!data) { dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n"); @@ -1306,7 +1470,9 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) mem_data.dma_request2 = 0; data = &mem_data; - sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY); + ret = sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY); + if (ret) + return ret; } switch (data->priority) { @@ -1361,7 +1527,6 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdmac->event_id0 = 0; sdmac->event_id1 = 0; - sdmac->context_loaded = false; sdma_set_channel_priority(sdmac, 0); @@ -1374,6 +1539,11 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, { struct sdma_desc *desc; + if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) { + dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n"); + goto err_out; + } + desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); if (!desc) goto err_out; @@ -1389,7 +1559,7 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, desc->sdmac = sdmac; desc->num_bd = bds; - if (sdma_alloc_bd(desc)) + if (bds && sdma_alloc_bd(desc)) goto err_desc_out; /* No slave_config called in MEMCPY case, so do here */ @@ -1397,10 +1567,12 @@ static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, sdma_config_ownership(sdmac, false, true, false); if (sdma_load_context(sdmac)) - goto err_desc_out; + goto err_bd_out; return desc; +err_bd_out: + sdma_free_bd(desc); err_desc_out: kfree(desc); err_out: @@ -1510,6 +1682,9 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( if (count & 3 || sg->dma_address & 3) goto err_bd_out; break; + case DMA_SLAVE_BUSWIDTH_3_BYTES: + bd->mode.command = 3; + break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) @@ -1554,13 +1729,16 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - int num_periods = buf_len / period_len; + int num_periods = 0; int channel = sdmac->channel; int i = 0, buf = 0; struct sdma_desc *desc; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + num_periods = buf_len / period_len; + sdma_config_write(chan, &sdmac->slave_config, direction); desc = sdma_transfer_init(sdmac, direction, num_periods); @@ -1577,6 +1755,9 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( goto err_bd_out; } + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); + while (buf < buf_len) { struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; @@ -1637,6 +1818,10 @@ static int sdma_config_write(struct dma_chan *chan, sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & SDMA_WATERMARK_LEVEL_HWML; sdmac->word_size = dmaengine_cfg->dst_addr_width; + } else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->per_address2 = dmaengine_cfg->src_addr; + sdmac->watermark_level = 0; } else { sdmac->per_address = dmaengine_cfg->dst_addr; sdmac->watermark_level = dmaengine_cfg->dst_maxburst * @@ -1651,9 +1836,26 @@ static int sdma_config(struct dma_chan *chan, struct dma_slave_config *dmaengine_cfg) { struct sdma_channel *sdmac = to_sdma_chan(chan); + struct sdma_engine *sdma = sdmac->sdma; memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg)); + if (dmaengine_cfg->peripheral_config) { + struct sdma_peripheral_config *sdmacfg = dmaengine_cfg->peripheral_config; + if (dmaengine_cfg->peripheral_size != sizeof(struct sdma_peripheral_config)) { + dev_err(sdma->dev, "Invalid peripheral size %zu, expected %zu\n", + dmaengine_cfg->peripheral_size, + sizeof(struct sdma_peripheral_config)); + return -EINVAL; + } + sdmac->n_fifos_src = sdmacfg->n_fifos_src; + sdmac->n_fifos_dst = sdmacfg->n_fifos_dst; + sdmac->stride_fifos_src = sdmacfg->stride_fifos_src; + sdmac->stride_fifos_dst = sdmacfg->stride_fifos_dst; + sdmac->words_per_fifo = sdmacfg->words_per_fifo; + sdmac->sw_done = sdmacfg->sw_done; + } + /* Set ENBLn earlier to make sure dma request triggered after that */ if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) return -EINVAL; @@ -1720,13 +1922,20 @@ static void sdma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&sdmac->vc.lock, flags); } -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 42 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 \ +(offsetof(struct sdma_script_start_addrs, v1_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 \ +(offsetof(struct sdma_script_start_addrs, v2_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 \ +(offsetof(struct sdma_script_start_addrs, v3_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 \ +(offsetof(struct sdma_script_start_addrs, v4_end) / sizeof(s32)) static void sdma_add_scripts(struct sdma_engine *sdma, - const struct sdma_script_start_addrs *addr) + const struct sdma_script_start_addrs *addr) { s32 *addr_arr = (u32 *)addr; s32 *saddr_arr = (u32 *)sdma->script_addrs; @@ -1747,6 +1956,19 @@ static void sdma_add_scripts(struct sdma_engine *sdma, for (i = 0; i < sdma->script_number; i++) if (addr_arr[i] > 0) saddr_arr[i] = addr_arr[i]; + + /* + * For compatibility with NXP internal legacy kernel before 4.19 which + * is based on uart ram script and mainline kernel based on uart rom + * script, both uart ram/rom scripts are present in newer sdma + * firmware. Use the rom versions if they are present (V3 or newer). + */ + if (sdma->script_number >= SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3) { + if (addr->uart_2_mcu_rom_addr) + sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_rom_addr; + if (addr->uartsh_2_mcu_rom_addr) + sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_rom_addr; + } } static void sdma_load_firmware(const struct firmware *fw, void *context) @@ -1796,16 +2018,18 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) clk_enable(sdma->clk_ahb); /* download the RAM image for SDMA */ sdma_load_script(sdma, ram_code, - header->ram_code_size, - addr->ram_code_start_addr); + header->ram_code_size, + addr->ram_code_start_addr); clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); sdma_add_scripts(sdma, addr); + sdma->fw_loaded = true; + dev_info(sdma->dev, "loaded firmware %d.%d\n", - header->version_major, - header->version_minor); + header->version_major, + header->version_minor); err_firmware: release_firmware(fw); @@ -1823,7 +2047,7 @@ static int sdma_event_remap(struct sdma_engine *sdma) u32 reg, val, shift, num_map, i; int ret = 0; - if (IS_ERR(np) || IS_ERR(gpr_np)) + if (IS_ERR(np) || !gpr_np) goto out; event_remap = of_find_property(np, propname, NULL); @@ -1871,7 +2095,7 @@ static int sdma_event_remap(struct sdma_engine *sdma) } out: - if (!IS_ERR(gpr_np)) + if (gpr_np) of_node_put(gpr_np); return ret; @@ -1882,9 +2106,8 @@ static int sdma_get_firmware(struct sdma_engine *sdma, { int ret; - ret = request_firmware_nowait(THIS_MODULE, - FW_ACTION_UEVENT, fw_name, sdma->dev, - GFP_KERNEL, sdma, sdma_load_firmware); + ret = firmware_request_nowait_nowarn(THIS_MODULE, fw_name, sdma->dev, + GFP_KERNEL, sdma, sdma_load_firmware); return ret; } @@ -1893,6 +2116,7 @@ static int sdma_init(struct sdma_engine *sdma) { int i, ret; dma_addr_t ccb_phys; + int ccbsize; ret = clk_enable(sdma->clk_ipg); if (ret) @@ -1908,10 +2132,14 @@ static int sdma_init(struct sdma_engine *sdma) /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - sdma->channel_control = dma_alloc_coherent(sdma->dev, - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + - sizeof(struct sdma_context_data), - &ccb_phys, GFP_KERNEL); + ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data)); + + if (sdma->iram_pool) + sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, ccbsize, &ccb_phys); + else + sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, &ccb_phys, + GFP_KERNEL); if (!sdma->channel_control) { ret = -ENOMEM; @@ -1919,9 +2147,9 @@ static int sdma_init(struct sdma_engine *sdma) } sdma->context = (void *)sdma->channel_control + - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); + MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control); sdma->context_phys = ccb_phys + - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); + MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control); /* disable all channels */ for (i = 0; i < sdma->drvdata->num_events; i++) @@ -2011,7 +2239,6 @@ static int sdma_probe(struct platform_device *pdev) const char *fw_name; int ret; int irq; - struct resource *iores; struct resource spba_res; int i; struct sdma_engine *sdma; @@ -2034,8 +2261,7 @@ static int sdma_probe(struct platform_device *pdev) if (irq < 0) return irq; - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); - sdma->regs = devm_ioremap_resource(&pdev->dev, iores); + sdma->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sdma->regs)) return PTR_ERR(sdma->regs); @@ -2055,8 +2281,8 @@ static int sdma_probe(struct platform_device *pdev) if (ret) goto err_clk; - ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma", - sdma); + ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, + dev_name(&pdev->dev), sdma); if (ret) goto err_irq; @@ -2076,6 +2302,7 @@ static int sdma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask); dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask); dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask); + dma_cap_set(DMA_PRIVATE, sdma->dma_device.cap_mask); INIT_LIST_HEAD(&sdma->dma_device.channels); /* Initialize channel parameters */ @@ -2086,6 +2313,7 @@ static int sdma_probe(struct platform_device *pdev) sdmac->channel = i; sdmac->vc.desc_free = sdma_desc_free; + INIT_LIST_HEAD(&sdmac->terminated); INIT_WORK(&sdmac->terminate_worker, sdma_channel_terminate_work); /* @@ -2097,6 +2325,12 @@ static int sdma_probe(struct platform_device *pdev) vchan_init(&sdmac->vc, &sdma->dma_device); } + if (np) { + sdma->iram_pool = of_gen_pool_get(np, "iram", 0); + if (sdma->iram_pool) + dev_info(&pdev->dev, "alloc bd from iram.\n"); + } + ret = sdma_init(sdma); if (ret) goto err_init; @@ -2179,7 +2413,7 @@ err_clk: return ret; } -static int sdma_remove(struct platform_device *pdev) +static void sdma_remove(struct platform_device *pdev) { struct sdma_engine *sdma = platform_get_drvdata(pdev); int i; @@ -2198,7 +2432,6 @@ static int sdma_remove(struct platform_device *pdev) } platform_set_drvdata(pdev, NULL); - return 0; } static struct platform_driver sdma_driver = { @@ -2217,7 +2450,7 @@ MODULE_DESCRIPTION("i.MX SDMA driver"); #if IS_ENABLED(CONFIG_SOC_IMX6Q) MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin"); #endif -#if IS_ENABLED(CONFIG_SOC_IMX7D) +#if IS_ENABLED(CONFIG_SOC_IMX7D) || IS_ENABLED(CONFIG_SOC_IMX8M) MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin"); #endif MODULE_LICENSE("GPL"); diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index 289c59ed74b9..5d3c0ae6b342 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -10,6 +10,8 @@ #include <linux/interrupt.h> #include <linux/dca.h> +#include <asm/cpuid/api.h> + /* either a kernel change is needed, or we need something like this in kernel */ #ifndef CONFIG_SMP #include <asm/smp.h> @@ -51,18 +53,18 @@ /* pack PCI B/D/F into a u16 */ static inline u16 dcaid_from_pcidev(struct pci_dev *pci) { - return (pci->bus->number << 8) | pci->devfn; + return pci_dev_id(pci); } static int dca_enabled_in_bios(struct pci_dev *pdev) { /* CPUID level 9 returns DCA configuration */ /* Bit 0 indicates DCA enabled by the BIOS */ - unsigned long cpuid_level_9; + u32 eax; int res; - cpuid_level_9 = cpuid_eax(9); - res = test_bit(0, &cpuid_level_9); + eax = cpuid_eax(CPUID_LEAF_DCA); + res = eax & BIT(0); if (!res) dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n"); diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 37ff4ec7db76..b8fff8333aef 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -33,7 +33,7 @@ MODULE_PARM_DESC(completion_timeout, static int idle_timeout = 2000; module_param(idle_timeout, int, 0644); MODULE_PARM_DESC(idle_timeout, - "set ioat idel timeout [msec] (default 2000 [msec])"); + "set ioat idle timeout [msec] (default 2000 [msec])"); #define IDLE_TIMEOUT msecs_to_jiffies(idle_timeout) #define COMPLETION_TIMEOUT msecs_to_jiffies(completion_timeout) @@ -159,7 +159,7 @@ void ioat_stop(struct ioatdma_chan *ioat_chan) } /* flush inflight timers */ - del_timer_sync(&ioat_chan->timer); + timer_delete_sync(&ioat_chan->timer); /* flush inflight tasklet runs */ tasklet_kill(&ioat_chan->cleanup_task); @@ -584,11 +584,11 @@ desc_get_errstat(struct ioatdma_chan *ioat_chan, struct ioat_ring_ent *desc) } /** - * __cleanup - reclaim used descriptors + * __ioat_cleanup - reclaim used descriptors * @ioat_chan: channel (ring) to clean * @phys_complete: zeroed (or not) completion address (from status) */ -static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) +static void __ioat_cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) { struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma; struct ioat_ring_ent *desc; @@ -656,7 +656,7 @@ static void __cleanup(struct ioatdma_chan *ioat_chan, dma_addr_t phys_complete) if (active - i == 0) { dev_dbg(to_dev(ioat_chan), "%s: cancel completion timeout\n", __func__); - mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); } /* microsecond delay by sysfs variable per pending descriptor */ @@ -675,14 +675,14 @@ static void ioat_cleanup(struct ioatdma_chan *ioat_chan) spin_lock_bh(&ioat_chan->cleanup_lock); if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) - __cleanup(ioat_chan, phys_complete); + __ioat_cleanup(ioat_chan, phys_complete); if (is_ioat_halted(*ioat_chan->completion)) { u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); if (chanerr & (IOAT_CHANERR_HANDLE_MASK | IOAT_CHANERR_RECOVER_MASK)) { - mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); ioat_eh(ioat_chan); } } @@ -712,7 +712,7 @@ static void ioat_restart_channel(struct ioatdma_chan *ioat_chan) ioat_quiesce(ioat_chan, 0); if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) - __cleanup(ioat_chan, phys_complete); + __ioat_cleanup(ioat_chan, phys_complete); __ioat_restart_chan(ioat_chan); } @@ -786,7 +786,7 @@ static void ioat_eh(struct ioatdma_chan *ioat_chan) /* cleanup so tail points to descriptor that caused the error */ if (ioat_cleanup_preamble(ioat_chan, &phys_complete)) - __cleanup(ioat_chan, phys_complete); + __ioat_cleanup(ioat_chan, phys_complete); chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr_int); @@ -879,7 +879,7 @@ static void check_active(struct ioatdma_chan *ioat_chan) } if (test_and_clear_bit(IOAT_CHAN_ACTIVE, &ioat_chan->state)) - mod_timer(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); + mod_timer_pending(&ioat_chan->timer, jiffies + IDLE_TIMEOUT); } static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan) @@ -901,7 +901,8 @@ static void ioat_reboot_chan(struct ioatdma_chan *ioat_chan) void ioat_timer_event(struct timer_list *t) { - struct ioatdma_chan *ioat_chan = from_timer(ioat_chan, t, timer); + struct ioatdma_chan *ioat_chan = timer_container_of(ioat_chan, t, + timer); dma_addr_t phys_complete; u64 status; @@ -943,7 +944,7 @@ void ioat_timer_event(struct timer_list *t) /* timer restarted in ioat_cleanup_preamble * and IOAT_COMPLETION_ACK cleared */ - __cleanup(ioat_chan, phys_complete); + __ioat_cleanup(ioat_chan, phys_complete); goto unlock_out; } diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 140cfe3782fb..12a4a4860a74 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -19,6 +19,8 @@ #define IOAT_DMA_DCA_ANY_CPU ~0 +int system_has_dca_enabled(struct pci_dev *pdev); + #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, dma_dev) #define to_dev(ioat_chan) (&(ioat_chan)->ioat_dma->pdev->dev) #define to_pdev(ioat_chan) ((ioat_chan)->ioat_dma->pdev) @@ -74,6 +76,7 @@ struct ioatdma_device { struct dca_provider *dca; enum ioat_irq_mode irq_mode; u32 cap; + int chancnt; /* shadow version for CB3.3 chan reset errata workaround */ u64 msixtba0; @@ -196,10 +199,8 @@ extern const struct sysfs_ops ioat_sysfs_ops; extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_cap_attr; extern int ioat_pending_level; -extern int ioat_ring_alloc_order; extern struct kobj_type ioat_ktype; extern struct kmem_cache *ioat_cache; -extern int ioat_ring_max_alloc_order; extern struct kmem_cache *ioat_sed_cache; static inline struct ioatdma_chan *to_ioat_chan(struct dma_chan *c) diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index 79e4e4c09c18..0373c48520c9 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -63,9 +63,6 @@ #define IOAT_VER_3_3 0x33 /* Version 3.3 */ #define IOAT_VER_3_4 0x34 /* Version 3.4 */ - -int system_has_dca_enabled(struct pci_dev *pdev); - #define IOAT_DESC_SZ 64 struct ioat_dma_descriptor { diff --git a/drivers/dma/ioat/init.c b/drivers/dma/ioat/init.c index 191b59279007..227398673b73 100644 --- a/drivers/dma/ioat/init.c +++ b/drivers/dma/ioat/init.c @@ -15,7 +15,6 @@ #include <linux/workqueue.h> #include <linux/prefetch.h> #include <linux/dca.h> -#include <linux/aer.h> #include <linux/sizes.h> #include "dma.h" #include "registers.h" @@ -24,6 +23,7 @@ #include "../dmaengine.h" MODULE_VERSION(IOAT_DMA_VERSION); +MODULE_DESCRIPTION("Intel I/OAT DMA Linux driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Intel Corporation"); @@ -421,7 +421,7 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *ioat_dma) msix: /* The number of MSI-X vectors should equal the number of channels */ - msixcnt = ioat_dma->dma_dev.chancnt; + msixcnt = ioat_dma->chancnt; for (i = 0; i < msixcnt; i++) ioat_dma->msix_entries[i].entry = i; @@ -512,7 +512,7 @@ static int ioat_probe(struct ioatdma_device *ioat_dma) dma_cap_set(DMA_MEMCPY, dma->cap_mask); dma->dev = &pdev->dev; - if (!dma->chancnt) { + if (!ioat_dma->chancnt) { dev_err(dev, "channel enumeration error\n"); goto err_setup_interrupts; } @@ -535,18 +535,6 @@ err_out: return err; } -static int ioat_register(struct ioatdma_device *ioat_dma) -{ - int err = dma_async_device_register(&ioat_dma->dma_dev); - - if (err) { - ioat_disable_interrupts(ioat_dma); - dma_pool_destroy(ioat_dma->completion_pool); - } - - return err; -} - static void ioat_dma_remove(struct ioatdma_device *ioat_dma) { struct dma_device *dma = &ioat_dma->dma_dev; @@ -568,15 +556,16 @@ static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) struct device *dev = &ioat_dma->pdev->dev; struct dma_device *dma = &ioat_dma->dma_dev; u8 xfercap_log; + int chancnt; int i; INIT_LIST_HEAD(&dma->channels); - dma->chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); - dma->chancnt &= 0x1f; /* bits [4:0] valid */ - if (dma->chancnt > ARRAY_SIZE(ioat_dma->idx)) { + chancnt = readb(ioat_dma->reg_base + IOAT_CHANCNT_OFFSET); + chancnt &= 0x1f; /* bits [4:0] valid */ + if (chancnt > ARRAY_SIZE(ioat_dma->idx)) { dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", - dma->chancnt, ARRAY_SIZE(ioat_dma->idx)); - dma->chancnt = ARRAY_SIZE(ioat_dma->idx); + chancnt, ARRAY_SIZE(ioat_dma->idx)); + chancnt = ARRAY_SIZE(ioat_dma->idx); } xfercap_log = readb(ioat_dma->reg_base + IOAT_XFERCAP_OFFSET); xfercap_log &= 0x1f; /* bits [4:0] valid */ @@ -584,7 +573,7 @@ static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) return; dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); - for (i = 0; i < dma->chancnt; i++) { + for (i = 0; i < chancnt; i++) { ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); if (!ioat_chan) break; @@ -597,7 +586,7 @@ static void ioat_enumerate_channels(struct ioatdma_device *ioat_dma) break; } } - dma->chancnt = i; + ioat_dma->chancnt = i; } /** @@ -916,7 +905,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *ioat_dma) op = IOAT_OP_XOR_VAL; - /* validate the sources with the destintation page */ + /* validate the sources with the destination page */ for (i = 0; i < IOAT_NUM_SRC_TEST; i++) xor_val_srcs[i] = xor_srcs[i]; xor_val_srcs[i] = dest; @@ -1181,9 +1170,9 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) ioat_chan->reg_base + IOAT_DCACTRL_OFFSET); } - err = ioat_register(ioat_dma); + err = dma_async_device_register(&ioat_dma->dma_dev); if (err) - return err; + goto err_disable_interrupts; ioat_kobject_add(ioat_dma, &ioat_ktype); @@ -1191,21 +1180,30 @@ static int ioat3_dma_probe(struct ioatdma_device *ioat_dma, int dca) ioat_dma->dca = ioat_dca_init(pdev, ioat_dma->reg_base); /* disable relaxed ordering */ - err = pcie_capability_read_word(pdev, IOAT_DEVCTRL_OFFSET, &val16); - if (err) - return pcibios_err_to_errno(err); + err = pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &val16); + if (err) { + err = pcibios_err_to_errno(err); + goto err_disable_interrupts; + } /* clear relaxed ordering enable */ - val16 &= ~IOAT_DEVCTRL_ROE; - err = pcie_capability_write_word(pdev, IOAT_DEVCTRL_OFFSET, val16); - if (err) - return pcibios_err_to_errno(err); + val16 &= ~PCI_EXP_DEVCTL_RELAX_EN; + err = pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, val16); + if (err) { + err = pcibios_err_to_errno(err); + goto err_disable_interrupts; + } if (ioat_dma->cap & IOAT_CAP_DPS) writeb(ioat_pending_level + 1, ioat_dma->reg_base + IOAT_PREFETCH_LIMIT_OFFSET); return 0; + +err_disable_interrupts: + ioat_disable_interrupts(ioat_dma); + dma_pool_destroy(ioat_dma->completion_pool); + return err; } static void ioat_shutdown(struct pci_dev *pdev) @@ -1226,12 +1224,12 @@ static void ioat_shutdown(struct pci_dev *pdev) set_bit(IOAT_CHAN_DOWN, &ioat_chan->state); spin_unlock_bh(&ioat_chan->prep_lock); /* - * Synchronization rule for del_timer_sync(): + * Synchronization rule for timer_delete_sync(): * - The caller must not hold locks which would prevent * completion of the timer's handler. * So prep_lock cannot be held before calling it. */ - del_timer_sync(&ioat_chan->timer); + timer_delete_sync(&ioat_chan->timer); /* this should quiesce then reset */ ioat_reset_hw(ioat_chan); @@ -1288,7 +1286,6 @@ static pci_ers_result_t ioat_pcie_error_slot_reset(struct pci_dev *pdev) } else { pci_set_master(pdev); pci_restore_state(pdev); - pci_save_state(pdev); pci_wake_from_d3(pdev, false); } @@ -1350,6 +1347,8 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) void __iomem * const *iomap; struct device *dev = &pdev->dev; struct ioatdma_device *device; + unsigned int i; + u8 version; int err; err = pcim_enable_device(pdev); @@ -1363,15 +1362,11 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!iomap) return -ENOMEM; - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) - return err; + version = readb(iomap[IOAT_MMIO_BAR] + IOAT_VER_OFFSET); + if (version < IOAT_VER_3_0) + return -ENODEV; - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (err) return err; @@ -1381,22 +1376,19 @@ static int ioat_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) pci_set_master(pdev); pci_set_drvdata(pdev, device); - device->version = readb(device->reg_base + IOAT_VER_OFFSET); + device->version = version; if (device->version >= IOAT_VER_3_4) ioat_dca_enabled = 0; - if (device->version >= IOAT_VER_3_0) { - if (is_skx_ioat(pdev)) - device->version = IOAT_VER_3_2; - err = ioat3_dma_probe(device, ioat_dca_enabled); - - if (device->version >= IOAT_VER_3_3) - pci_enable_pcie_error_reporting(pdev); - } else - return -ENODEV; + if (is_skx_ioat(pdev)) + device->version = IOAT_VER_3_2; + + err = ioat3_dma_probe(device, ioat_dca_enabled); if (err) { + for (i = 0; i < IOAT_MAX_CHANS; i++) + kfree(device->idx[i]); + kfree(device); dev_err(dev, "Intel(R) I/OAT DMA Engine init failed\n"); - pci_disable_pcie_error_reporting(pdev); return -ENODEV; } @@ -1419,7 +1411,6 @@ static void ioat_remove(struct pci_dev *pdev) device->dca = NULL; } - pci_disable_pcie_error_reporting(pdev); ioat_dma_remove(device); } @@ -1458,6 +1449,7 @@ module_init(ioat_init_module); static void __exit ioat_exit_module(void) { pci_unregister_driver(&ioat_pci_driver); + kmem_cache_destroy(ioat_sed_cache); kmem_cache_destroy(ioat_cache); } module_exit(ioat_exit_module); diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index f55a5f92f185..54cf0ad39887 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -14,13 +14,6 @@ #define IOAT_PCI_CHANERR_INT_OFFSET 0x180 #define IOAT_PCI_CHANERRMASK_INT_OFFSET 0x184 -/* PCIe config registers */ - -/* EXPCAPID + N */ -#define IOAT_DEVCTRL_OFFSET 0x8 -/* relaxed ordering enable */ -#define IOAT_DEVCTRL_ROE 0x10 - /* MMIO Device Registers */ #define IOAT_CHANCNT_OFFSET 0x00 /* 8-bit */ diff --git a/drivers/dma/ioat/sysfs.c b/drivers/dma/ioat/sysfs.c index aa44bcd6a356..168adf28c5b1 100644 --- a/drivers/dma/ioat/sysfs.c +++ b/drivers/dma/ioat/sysfs.c @@ -158,8 +158,9 @@ static struct attribute *ioat_attrs[] = { &intr_coalesce_attr.attr, NULL, }; +ATTRIBUTE_GROUPS(ioat); struct kobj_type ioat_ktype = { .sysfs_ops = &ioat_sysfs_ops, - .default_attrs = ioat_attrs, + .default_groups = ioat_groups, }; diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c deleted file mode 100644 index 310b899d581f..000000000000 --- a/drivers/dma/iop-adma.c +++ /dev/null @@ -1,1554 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * offload engine driver for the Intel Xscale series of i/o processors - * Copyright © 2006, Intel Corporation. - */ - -/* - * This driver supports the asynchrounous DMA copy and RAID engines available - * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) - */ - -#include <linux/init.h> -#include <linux/module.h> -#include <linux/delay.h> -#include <linux/dma-mapping.h> -#include <linux/spinlock.h> -#include <linux/interrupt.h> -#include <linux/platform_device.h> -#include <linux/prefetch.h> -#include <linux/memory.h> -#include <linux/ioport.h> -#include <linux/raid/pq.h> -#include <linux/slab.h> - -#include "iop-adma.h" -#include "dmaengine.h" - -#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) -#define to_iop_adma_device(dev) \ - container_of(dev, struct iop_adma_device, common) -#define tx_to_iop_adma_slot(tx) \ - container_of(tx, struct iop_adma_desc_slot, async_tx) - -/** - * iop_adma_free_slots - flags descriptor slots for reuse - * @slot: Slot to free - * Caller must hold &iop_chan->lock while calling this function - */ -static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) -{ - int stride = slot->slots_per_op; - - while (stride--) { - slot->slots_per_op = 0; - slot = list_entry(slot->slot_node.next, - struct iop_adma_desc_slot, - slot_node); - } -} - -static dma_cookie_t -iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *iop_chan, dma_cookie_t cookie) -{ - struct dma_async_tx_descriptor *tx = &desc->async_tx; - - BUG_ON(tx->cookie < 0); - if (tx->cookie > 0) { - cookie = tx->cookie; - tx->cookie = 0; - - /* call the callback (must not sleep or submit new - * operations to this channel) - */ - dmaengine_desc_get_callback_invoke(tx, NULL); - - dma_descriptor_unmap(tx); - if (desc->group_head) - desc->group_head = NULL; - } - - /* run dependent operations */ - dma_run_dependencies(tx); - - return cookie; -} - -static int -iop_adma_clean_slot(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *iop_chan) -{ - /* the client is allowed to attach dependent operations - * until 'ack' is set - */ - if (!async_tx_test_ack(&desc->async_tx)) - return 0; - - /* leave the last descriptor in the chain - * so we can append to it - */ - if (desc->chain_node.next == &iop_chan->chain) - return 1; - - dev_dbg(iop_chan->device->common.dev, - "\tfree slot: %d slots_per_op: %d\n", - desc->idx, desc->slots_per_op); - - list_del(&desc->chain_node); - iop_adma_free_slots(desc); - - return 0; -} - -static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) -{ - struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; - dma_cookie_t cookie = 0; - u32 current_desc = iop_chan_get_current_descriptor(iop_chan); - int busy = iop_chan_is_busy(iop_chan); - int seen_current = 0, slot_cnt = 0, slots_per_op = 0; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - /* free completed slots from the chain starting with - * the oldest descriptor - */ - list_for_each_entry_safe(iter, _iter, &iop_chan->chain, - chain_node) { - pr_debug("\tcookie: %d slot: %d busy: %d " - "this_desc: %pad next_desc: %#llx ack: %d\n", - iter->async_tx.cookie, iter->idx, busy, - &iter->async_tx.phys, (u64)iop_desc_get_next_desc(iter), - async_tx_test_ack(&iter->async_tx)); - prefetch(_iter); - prefetch(&_iter->async_tx); - - /* do not advance past the current descriptor loaded into the - * hardware channel, subsequent descriptors are either in - * process or have not been submitted - */ - if (seen_current) - break; - - /* stop the search if we reach the current descriptor and the - * channel is busy, or if it appears that the current descriptor - * needs to be re-read (i.e. has been appended to) - */ - if (iter->async_tx.phys == current_desc) { - BUG_ON(seen_current++); - if (busy || iop_desc_get_next_desc(iter)) - break; - } - - /* detect the start of a group transaction */ - if (!slot_cnt && !slots_per_op) { - slot_cnt = iter->slot_cnt; - slots_per_op = iter->slots_per_op; - if (slot_cnt <= slots_per_op) { - slot_cnt = 0; - slots_per_op = 0; - } - } - - if (slot_cnt) { - pr_debug("\tgroup++\n"); - if (!grp_start) - grp_start = iter; - slot_cnt -= slots_per_op; - } - - /* all the members of a group are complete */ - if (slots_per_op != 0 && slot_cnt == 0) { - struct iop_adma_desc_slot *grp_iter, *_grp_iter; - int end_of_chain = 0; - pr_debug("\tgroup end\n"); - - /* collect the total results */ - if (grp_start->xor_check_result) { - u32 zero_sum_result = 0; - slot_cnt = grp_start->slot_cnt; - grp_iter = grp_start; - - list_for_each_entry_from(grp_iter, - &iop_chan->chain, chain_node) { - zero_sum_result |= - iop_desc_get_zero_result(grp_iter); - pr_debug("\titer%d result: %d\n", - grp_iter->idx, zero_sum_result); - slot_cnt -= slots_per_op; - if (slot_cnt == 0) - break; - } - pr_debug("\tgrp_start->xor_check_result: %p\n", - grp_start->xor_check_result); - *grp_start->xor_check_result = zero_sum_result; - } - - /* clean up the group */ - slot_cnt = grp_start->slot_cnt; - grp_iter = grp_start; - list_for_each_entry_safe_from(grp_iter, _grp_iter, - &iop_chan->chain, chain_node) { - cookie = iop_adma_run_tx_complete_actions( - grp_iter, iop_chan, cookie); - - slot_cnt -= slots_per_op; - end_of_chain = iop_adma_clean_slot(grp_iter, - iop_chan); - - if (slot_cnt == 0 || end_of_chain) - break; - } - - /* the group should be complete at this point */ - BUG_ON(slot_cnt); - - slots_per_op = 0; - grp_start = NULL; - if (end_of_chain) - break; - else - continue; - } else if (slots_per_op) /* wait for group completion */ - continue; - - /* write back zero sum results (single descriptor case) */ - if (iter->xor_check_result && iter->async_tx.cookie) - *iter->xor_check_result = - iop_desc_get_zero_result(iter); - - cookie = iop_adma_run_tx_complete_actions( - iter, iop_chan, cookie); - - if (iop_adma_clean_slot(iter, iop_chan)) - break; - } - - if (cookie > 0) { - iop_chan->common.completed_cookie = cookie; - pr_debug("\tcompleted cookie %d\n", cookie); - } -} - -static void -iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) -{ - spin_lock_bh(&iop_chan->lock); - __iop_adma_slot_cleanup(iop_chan); - spin_unlock_bh(&iop_chan->lock); -} - -static void iop_adma_tasklet(struct tasklet_struct *t) -{ - struct iop_adma_chan *iop_chan = from_tasklet(iop_chan, t, - irq_tasklet); - - /* lockdep will flag depedency submissions as potentially - * recursive locking, this is not the case as a dependency - * submission will never recurse a channels submit routine. - * There are checks in async_tx.c to prevent this. - */ - spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); - __iop_adma_slot_cleanup(iop_chan); - spin_unlock(&iop_chan->lock); -} - -static struct iop_adma_desc_slot * -iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, - int slots_per_op) -{ - struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; - LIST_HEAD(chain); - int slots_found, retry = 0; - - /* start search from the last allocated descrtiptor - * if a contiguous allocation can not be found start searching - * from the beginning of the list - */ -retry: - slots_found = 0; - if (retry == 0) - iter = iop_chan->last_used; - else - iter = list_entry(&iop_chan->all_slots, - struct iop_adma_desc_slot, - slot_node); - - list_for_each_entry_safe_continue( - iter, _iter, &iop_chan->all_slots, slot_node) { - prefetch(_iter); - prefetch(&_iter->async_tx); - if (iter->slots_per_op) { - /* give up after finding the first busy slot - * on the second pass through the list - */ - if (retry) - break; - - slots_found = 0; - continue; - } - - /* start the allocation if the slot is correctly aligned */ - if (!slots_found++) { - if (iop_desc_is_aligned(iter, slots_per_op)) - alloc_start = iter; - else { - slots_found = 0; - continue; - } - } - - if (slots_found == num_slots) { - struct iop_adma_desc_slot *alloc_tail = NULL; - struct iop_adma_desc_slot *last_used = NULL; - iter = alloc_start; - while (num_slots) { - int i; - dev_dbg(iop_chan->device->common.dev, - "allocated slot: %d " - "(desc %p phys: %#llx) slots_per_op %d\n", - iter->idx, iter->hw_desc, - (u64)iter->async_tx.phys, slots_per_op); - - /* pre-ack all but the last descriptor */ - if (num_slots != slots_per_op) - async_tx_ack(&iter->async_tx); - - list_add_tail(&iter->chain_node, &chain); - alloc_tail = iter; - iter->async_tx.cookie = 0; - iter->slot_cnt = num_slots; - iter->xor_check_result = NULL; - for (i = 0; i < slots_per_op; i++) { - iter->slots_per_op = slots_per_op - i; - last_used = iter; - iter = list_entry(iter->slot_node.next, - struct iop_adma_desc_slot, - slot_node); - } - num_slots -= slots_per_op; - } - alloc_tail->group_head = alloc_start; - alloc_tail->async_tx.cookie = -EBUSY; - list_splice(&chain, &alloc_tail->tx_list); - iop_chan->last_used = last_used; - iop_desc_clear_next_desc(alloc_start); - iop_desc_clear_next_desc(alloc_tail); - return alloc_tail; - } - } - if (!retry++) - goto retry; - - /* perform direct reclaim if the allocation fails */ - __iop_adma_slot_cleanup(iop_chan); - - return NULL; -} - -static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) -{ - dev_dbg(iop_chan->device->common.dev, "pending: %d\n", - iop_chan->pending); - - if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { - iop_chan->pending = 0; - iop_chan_append(iop_chan); - } -} - -static dma_cookie_t -iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); - struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); - struct iop_adma_desc_slot *grp_start, *old_chain_tail; - int slot_cnt; - dma_cookie_t cookie; - dma_addr_t next_dma; - - grp_start = sw_desc->group_head; - slot_cnt = grp_start->slot_cnt; - - spin_lock_bh(&iop_chan->lock); - cookie = dma_cookie_assign(tx); - - old_chain_tail = list_entry(iop_chan->chain.prev, - struct iop_adma_desc_slot, chain_node); - list_splice_init(&sw_desc->tx_list, - &old_chain_tail->chain_node); - - /* fix up the hardware chain */ - next_dma = grp_start->async_tx.phys; - iop_desc_set_next_desc(old_chain_tail, next_dma); - BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ - - /* check for pre-chained descriptors */ - iop_paranoia(iop_desc_get_next_desc(sw_desc)); - - /* increment the pending count by the number of slots - * memcpy operations have a 1:1 (slot:operation) relation - * other operations are heavier and will pop the threshold - * more often. - */ - iop_chan->pending += slot_cnt; - iop_adma_check_threshold(iop_chan); - spin_unlock_bh(&iop_chan->lock); - - dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", - __func__, sw_desc->async_tx.cookie, sw_desc->idx); - - return cookie; -} - -static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); -static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); - -/** - * iop_adma_alloc_chan_resources - returns the number of allocated descriptors - * @chan: allocate descriptor resources for this channel - * - * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To - * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be - * greater than 2x the number slots needed to satisfy a device->max_xor - * request. - * */ -static int iop_adma_alloc_chan_resources(struct dma_chan *chan) -{ - char *hw_desc; - dma_addr_t dma_desc; - int idx; - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *slot = NULL; - int init = iop_chan->slots_allocated ? 0 : 1; - struct iop_adma_platform_data *plat_data = - dev_get_platdata(&iop_chan->device->pdev->dev); - int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; - - /* Allocate descriptor slots */ - do { - idx = iop_chan->slots_allocated; - if (idx == num_descs_in_pool) - break; - - slot = kzalloc(sizeof(*slot), GFP_KERNEL); - if (!slot) { - printk(KERN_INFO "IOP ADMA Channel only initialized" - " %d descriptor slots", idx); - break; - } - hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; - slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; - - dma_async_tx_descriptor_init(&slot->async_tx, chan); - slot->async_tx.tx_submit = iop_adma_tx_submit; - INIT_LIST_HEAD(&slot->tx_list); - INIT_LIST_HEAD(&slot->chain_node); - INIT_LIST_HEAD(&slot->slot_node); - dma_desc = iop_chan->device->dma_desc_pool; - slot->async_tx.phys = dma_desc + idx * IOP_ADMA_SLOT_SIZE; - slot->idx = idx; - - spin_lock_bh(&iop_chan->lock); - iop_chan->slots_allocated++; - list_add_tail(&slot->slot_node, &iop_chan->all_slots); - spin_unlock_bh(&iop_chan->lock); - } while (iop_chan->slots_allocated < num_descs_in_pool); - - if (idx && !iop_chan->last_used) - iop_chan->last_used = list_entry(iop_chan->all_slots.next, - struct iop_adma_desc_slot, - slot_node); - - dev_dbg(iop_chan->device->common.dev, - "allocated %d descriptor slots last_used: %p\n", - iop_chan->slots_allocated, iop_chan->last_used); - - /* initialize the channel and the chain with a null operation */ - if (init) { - if (dma_has_cap(DMA_MEMCPY, - iop_chan->device->common.cap_mask)) - iop_chan_start_null_memcpy(iop_chan); - else if (dma_has_cap(DMA_XOR, - iop_chan->device->common.cap_mask)) - iop_chan_start_null_xor(iop_chan); - else - BUG(); - } - - return (idx > 0) ? idx : -ENOMEM; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_interrupt(grp_start, iop_chan); - sw_desc->async_tx.flags = flags; - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, - dma_addr_t dma_src, size_t len, unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, "%s len: %zu\n", - __func__, len); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_memcpy(grp_start, flags); - iop_desc_set_byte_count(grp_start, iop_chan, len); - iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); - iop_desc_set_memcpy_src_addr(grp_start, dma_src); - sw_desc->async_tx.flags = flags; - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, - dma_addr_t *dma_src, unsigned int src_cnt, size_t len, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, - "%s src_cnt: %d len: %zu flags: %lx\n", - __func__, src_cnt, len, flags); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_xor(grp_start, src_cnt, flags); - iop_desc_set_byte_count(grp_start, iop_chan, len); - iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); - sw_desc->async_tx.flags = flags; - while (src_cnt--) - iop_desc_set_xor_src_addr(grp_start, src_cnt, - dma_src[src_cnt]); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, - unsigned int src_cnt, size_t len, u32 *result, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *grp_start; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - - dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", - __func__, src_cnt, len); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - iop_desc_init_zero_sum(grp_start, src_cnt, flags); - iop_desc_set_zero_sum_byte_count(grp_start, len); - grp_start->xor_check_result = result; - pr_debug("\t%s: grp_start->xor_check_result: %p\n", - __func__, grp_start->xor_check_result); - sw_desc->async_tx.flags = flags; - while (src_cnt--) - iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, - dma_src[src_cnt]); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, size_t len, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *g; - int slot_cnt, slots_per_op; - int continue_srcs; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, - "%s src_cnt: %d len: %zu flags: %lx\n", - __func__, src_cnt, len, flags); - - if (dmaf_p_disabled_continue(flags)) - continue_srcs = 1+src_cnt; - else if (dmaf_continue(flags)) - continue_srcs = 3+src_cnt; - else - continue_srcs = 0+src_cnt; - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - int i; - - g = sw_desc->group_head; - iop_desc_set_byte_count(g, iop_chan, len); - - /* even if P is disabled its destination address (bits - * [3:0]) must match Q. It is ok if P points to an - * invalid address, it won't be written. - */ - if (flags & DMA_PREP_PQ_DISABLE_P) - dst[0] = dst[1] & 0x7; - - iop_desc_set_pq_addr(g, dst); - sw_desc->async_tx.flags = flags; - for (i = 0; i < src_cnt; i++) - iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); - - /* if we are continuing a previous operation factor in - * the old p and q values, see the comment for dma_maxpq - * in include/linux/dmaengine.h - */ - if (dmaf_p_disabled_continue(flags)) - iop_desc_set_pq_src_addr(g, i++, dst[1], 1); - else if (dmaf_continue(flags)) { - iop_desc_set_pq_src_addr(g, i++, dst[0], 0); - iop_desc_set_pq_src_addr(g, i++, dst[1], 1); - iop_desc_set_pq_src_addr(g, i++, dst[1], 0); - } - iop_desc_init_pq(g, i, flags); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static struct dma_async_tx_descriptor * -iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, - unsigned int src_cnt, const unsigned char *scf, - size_t len, enum sum_check_flags *pqres, - unsigned long flags) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *sw_desc, *g; - int slot_cnt, slots_per_op; - - if (unlikely(!len)) - return NULL; - BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); - - dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %zu\n", - __func__, src_cnt, len); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - /* for validate operations p and q are tagged onto the - * end of the source list - */ - int pq_idx = src_cnt; - - g = sw_desc->group_head; - iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); - iop_desc_set_pq_zero_sum_byte_count(g, len); - g->pq_check_result = pqres; - pr_debug("\t%s: g->pq_check_result: %p\n", - __func__, g->pq_check_result); - sw_desc->async_tx.flags = flags; - while (src_cnt--) - iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, - src[src_cnt], - scf[src_cnt]); - iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); - } - spin_unlock_bh(&iop_chan->lock); - - return sw_desc ? &sw_desc->async_tx : NULL; -} - -static void iop_adma_free_chan_resources(struct dma_chan *chan) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - struct iop_adma_desc_slot *iter, *_iter; - int in_use_descs = 0; - - iop_adma_slot_cleanup(iop_chan); - - spin_lock_bh(&iop_chan->lock); - list_for_each_entry_safe(iter, _iter, &iop_chan->chain, - chain_node) { - in_use_descs++; - list_del(&iter->chain_node); - } - list_for_each_entry_safe_reverse( - iter, _iter, &iop_chan->all_slots, slot_node) { - list_del(&iter->slot_node); - kfree(iter); - iop_chan->slots_allocated--; - } - iop_chan->last_used = NULL; - - dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", - __func__, iop_chan->slots_allocated); - spin_unlock_bh(&iop_chan->lock); - - /* one is ok since we left it on there on purpose */ - if (in_use_descs > 1) - printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", - in_use_descs - 1); -} - -/** - * iop_adma_status - poll the status of an ADMA transaction - * @chan: ADMA channel handle - * @cookie: ADMA transaction identifier - * @txstate: a holder for the current state of the channel or NULL - */ -static enum dma_status iop_adma_status(struct dma_chan *chan, - dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - int ret; - - ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_COMPLETE) - return ret; - - iop_adma_slot_cleanup(iop_chan); - - return dma_cookie_status(chan, cookie, txstate); -} - -static irqreturn_t iop_adma_eot_handler(int irq, void *data) -{ - struct iop_adma_chan *chan = data; - - dev_dbg(chan->device->common.dev, "%s\n", __func__); - - tasklet_schedule(&chan->irq_tasklet); - - iop_adma_device_clear_eot_status(chan); - - return IRQ_HANDLED; -} - -static irqreturn_t iop_adma_eoc_handler(int irq, void *data) -{ - struct iop_adma_chan *chan = data; - - dev_dbg(chan->device->common.dev, "%s\n", __func__); - - tasklet_schedule(&chan->irq_tasklet); - - iop_adma_device_clear_eoc_status(chan); - - return IRQ_HANDLED; -} - -static irqreturn_t iop_adma_err_handler(int irq, void *data) -{ - struct iop_adma_chan *chan = data; - unsigned long status = iop_chan_get_status(chan); - - dev_err(chan->device->common.dev, - "error ( %s%s%s%s%s%s%s)\n", - iop_is_err_int_parity(status, chan) ? "int_parity " : "", - iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", - iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", - iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", - iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", - iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", - iop_is_err_split_tx(status, chan) ? "split_tx " : ""); - - iop_adma_device_clear_err_status(chan); - - BUG(); - - return IRQ_HANDLED; -} - -static void iop_adma_issue_pending(struct dma_chan *chan) -{ - struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); - - if (iop_chan->pending) { - iop_chan->pending = 0; - iop_chan_append(iop_chan); - } -} - -/* - * Perform a transaction to verify the HW works. - */ -#define IOP_ADMA_TEST_SIZE 2000 - -static int iop_adma_memcpy_self_test(struct iop_adma_device *device) -{ - int i; - void *src, *dest; - dma_addr_t src_dma, dest_dma; - struct dma_chan *dma_chan; - dma_cookie_t cookie; - struct dma_async_tx_descriptor *tx; - int err = 0; - struct iop_adma_chan *iop_chan; - - dev_dbg(device->common.dev, "%s\n", __func__); - - src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); - if (!src) - return -ENOMEM; - dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); - if (!dest) { - kfree(src); - return -ENOMEM; - } - - /* Fill in src buffer */ - for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) - ((u8 *) src)[i] = (u8)i; - - /* Start copy, using first DMA channel */ - dma_chan = container_of(device->common.channels.next, - struct dma_chan, - device_node); - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { - err = -ENODEV; - goto out; - } - - dest_dma = dma_map_single(dma_chan->device->dev, dest, - IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); - src_dma = dma_map_single(dma_chan->device->dev, src, - IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); - tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, - IOP_ADMA_TEST_SIZE, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(1); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test copy timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - iop_chan = to_iop_adma_chan(dma_chan); - dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, - IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); - if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { - dev_err(dma_chan->device->dev, - "Self-test copy failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - -free_resources: - iop_adma_free_chan_resources(dma_chan); -out: - kfree(src); - kfree(dest); - return err; -} - -#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ -static int -iop_adma_xor_val_self_test(struct iop_adma_device *device) -{ - int i, src_idx; - struct page *dest; - struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; - struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; - dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; - dma_addr_t dest_dma; - struct dma_async_tx_descriptor *tx; - struct dma_chan *dma_chan; - dma_cookie_t cookie; - u8 cmp_byte = 0; - u32 cmp_word; - u32 zero_sum_result; - int err = 0; - struct iop_adma_chan *iop_chan; - - dev_dbg(device->common.dev, "%s\n", __func__); - - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { - xor_srcs[src_idx] = alloc_page(GFP_KERNEL); - if (!xor_srcs[src_idx]) { - while (src_idx--) - __free_page(xor_srcs[src_idx]); - return -ENOMEM; - } - } - - dest = alloc_page(GFP_KERNEL); - if (!dest) { - while (src_idx--) - __free_page(xor_srcs[src_idx]); - return -ENOMEM; - } - - /* Fill in src buffers */ - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { - u8 *ptr = page_address(xor_srcs[src_idx]); - for (i = 0; i < PAGE_SIZE; i++) - ptr[i] = (1 << src_idx); - } - - for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) - cmp_byte ^= (u8) (1 << src_idx); - - cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | - (cmp_byte << 8) | cmp_byte; - - memset(page_address(dest), 0, PAGE_SIZE); - - dma_chan = container_of(device->common.channels.next, - struct dma_chan, - device_node); - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { - err = -ENODEV; - goto out; - } - - /* test xor */ - dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, - PAGE_SIZE, DMA_FROM_DEVICE); - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) - dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], - 0, PAGE_SIZE, DMA_TO_DEVICE); - tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, - IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test xor timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - iop_chan = to_iop_adma_chan(dma_chan); - dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, - PAGE_SIZE, DMA_FROM_DEVICE); - for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { - u32 *ptr = page_address(dest); - if (ptr[i] != cmp_word) { - dev_err(dma_chan->device->dev, - "Self-test xor failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - } - dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, - PAGE_SIZE, DMA_TO_DEVICE); - - /* skip zero sum if the capability is not present */ - if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) - goto free_resources; - - /* zero sum the sources with the destintation page */ - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) - zero_sum_srcs[i] = xor_srcs[i]; - zero_sum_srcs[i] = dest; - - zero_sum_result = 1; - - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = dma_map_page(dma_chan->device->dev, - zero_sum_srcs[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, - IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, - &zero_sum_result, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test zero sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != 0) { - dev_err(dma_chan->device->dev, - "Self-test zero sum failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - /* test for non-zero parity sum */ - zero_sum_result = 0; - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) - dma_srcs[i] = dma_map_page(dma_chan->device->dev, - zero_sum_srcs[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, - IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, - &zero_sum_result, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { - dev_err(dma_chan->device->dev, - "Self-test non-zero sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != 1) { - dev_err(dma_chan->device->dev, - "Self-test non-zero sum failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - -free_resources: - iop_adma_free_chan_resources(dma_chan); -out: - src_idx = IOP_ADMA_NUM_SRC_TEST; - while (src_idx--) - __free_page(xor_srcs[src_idx]); - __free_page(dest); - return err; -} - -#ifdef CONFIG_RAID6_PQ -static int -iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) -{ - /* combined sources, software pq results, and extra hw pq results */ - struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; - /* ptr to the extra hw pq buffers defined above */ - struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; - /* address conversion buffers (dma_map / page_address) */ - void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; - dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2]; - dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST]; - - int i; - struct dma_async_tx_descriptor *tx; - struct dma_chan *dma_chan; - dma_cookie_t cookie; - u32 zero_sum_result; - int err = 0; - struct device *dev; - - dev_dbg(device->common.dev, "%s\n", __func__); - - for (i = 0; i < ARRAY_SIZE(pq); i++) { - pq[i] = alloc_page(GFP_KERNEL); - if (!pq[i]) { - while (i--) - __free_page(pq[i]); - return -ENOMEM; - } - } - - /* Fill in src buffers */ - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { - pq_sw[i] = page_address(pq[i]); - memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); - } - pq_sw[i] = page_address(pq[i]); - pq_sw[i+1] = page_address(pq[i+1]); - - dma_chan = container_of(device->common.channels.next, - struct dma_chan, - device_node); - if (iop_adma_alloc_chan_resources(dma_chan) < 1) { - err = -ENODEV; - goto out; - } - - dev = dma_chan->device->dev; - - /* initialize the dests */ - memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); - memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); - - /* test pq */ - pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); - pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - - tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, - IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, - PAGE_SIZE, - DMA_PREP_INTERRUPT | - DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dev, "Self-test pq timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); - - if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], - page_address(pq_hw[0]), PAGE_SIZE) != 0) { - dev_err(dev, "Self-test p failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], - page_address(pq_hw[1]), PAGE_SIZE) != 0) { - dev_err(dev, "Self-test q failed compare, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - /* test correct zero sum using the software generated pq values */ - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - - zero_sum_result = ~0; - tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], - pq_src, IOP_ADMA_NUM_SRC_TEST, - raid6_gfexp, PAGE_SIZE, &zero_sum_result, - DMA_PREP_INTERRUPT|DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != 0) { - dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", - zero_sum_result); - err = -ENODEV; - goto free_resources; - } - - /* test incorrect zero sum */ - i = IOP_ADMA_NUM_SRC_TEST; - memset(pq_sw[i] + 100, 0, 100); - memset(pq_sw[i+1] + 200, 0, 200); - for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) - pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, - DMA_TO_DEVICE); - - zero_sum_result = 0; - tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], - pq_src, IOP_ADMA_NUM_SRC_TEST, - raid6_gfexp, PAGE_SIZE, &zero_sum_result, - DMA_PREP_INTERRUPT|DMA_CTRL_ACK); - - cookie = iop_adma_tx_submit(tx); - iop_adma_issue_pending(dma_chan); - msleep(8); - - if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_COMPLETE) { - dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); - err = -ENODEV; - goto free_resources; - } - - if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { - dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", - zero_sum_result); - err = -ENODEV; - goto free_resources; - } - -free_resources: - iop_adma_free_chan_resources(dma_chan); -out: - i = ARRAY_SIZE(pq); - while (i--) - __free_page(pq[i]); - return err; -} -#endif - -static int iop_adma_remove(struct platform_device *dev) -{ - struct iop_adma_device *device = platform_get_drvdata(dev); - struct dma_chan *chan, *_chan; - struct iop_adma_chan *iop_chan; - struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev); - - dma_async_device_unregister(&device->common); - - dma_free_coherent(&dev->dev, plat_data->pool_size, - device->dma_desc_pool_virt, device->dma_desc_pool); - - list_for_each_entry_safe(chan, _chan, &device->common.channels, - device_node) { - iop_chan = to_iop_adma_chan(chan); - list_del(&chan->device_node); - kfree(iop_chan); - } - kfree(device); - - return 0; -} - -static int iop_adma_probe(struct platform_device *pdev) -{ - struct resource *res; - int ret = 0, i; - struct iop_adma_device *adev; - struct iop_adma_chan *iop_chan; - struct dma_device *dma_dev; - struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev); - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - - if (!devm_request_mem_region(&pdev->dev, res->start, - resource_size(res), pdev->name)) - return -EBUSY; - - adev = kzalloc(sizeof(*adev), GFP_KERNEL); - if (!adev) - return -ENOMEM; - dma_dev = &adev->common; - - /* allocate coherent memory for hardware descriptors - * note: writecombine gives slightly better performance, but - * requires that we explicitly flush the writes - */ - adev->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev, - plat_data->pool_size, - &adev->dma_desc_pool, - GFP_KERNEL); - if (!adev->dma_desc_pool_virt) { - ret = -ENOMEM; - goto err_free_adev; - } - - dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %pad\n", - __func__, adev->dma_desc_pool_virt, &adev->dma_desc_pool); - - adev->id = plat_data->hw_id; - - /* discover transaction capabilites from the platform data */ - dma_dev->cap_mask = plat_data->cap_mask; - - adev->pdev = pdev; - platform_set_drvdata(pdev, adev); - - INIT_LIST_HEAD(&dma_dev->channels); - - /* set base routines */ - dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; - dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; - dma_dev->device_tx_status = iop_adma_status; - dma_dev->device_issue_pending = iop_adma_issue_pending; - dma_dev->dev = &pdev->dev; - - /* set prep routines based on capability */ - if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) - dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; - if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { - dma_dev->max_xor = iop_adma_get_max_xor(); - dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; - } - if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) - dma_dev->device_prep_dma_xor_val = - iop_adma_prep_dma_xor_val; - if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { - dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); - dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; - } - if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) - dma_dev->device_prep_dma_pq_val = - iop_adma_prep_dma_pq_val; - if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) - dma_dev->device_prep_dma_interrupt = - iop_adma_prep_dma_interrupt; - - iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); - if (!iop_chan) { - ret = -ENOMEM; - goto err_free_dma; - } - iop_chan->device = adev; - - iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, - resource_size(res)); - if (!iop_chan->mmr_base) { - ret = -ENOMEM; - goto err_free_iop_chan; - } - tasklet_setup(&iop_chan->irq_tasklet, iop_adma_tasklet); - - /* clear errors before enabling interrupts */ - iop_adma_device_clear_err_status(iop_chan); - - for (i = 0; i < 3; i++) { - static const irq_handler_t handler[] = { - iop_adma_eot_handler, - iop_adma_eoc_handler, - iop_adma_err_handler - }; - int irq = platform_get_irq(pdev, i); - if (irq < 0) { - ret = -ENXIO; - goto err_free_iop_chan; - } else { - ret = devm_request_irq(&pdev->dev, irq, - handler[i], 0, pdev->name, iop_chan); - if (ret) - goto err_free_iop_chan; - } - } - - spin_lock_init(&iop_chan->lock); - INIT_LIST_HEAD(&iop_chan->chain); - INIT_LIST_HEAD(&iop_chan->all_slots); - iop_chan->common.device = dma_dev; - dma_cookie_init(&iop_chan->common); - list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); - - if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { - ret = iop_adma_memcpy_self_test(adev); - dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); - if (ret) - goto err_free_iop_chan; - } - - if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { - ret = iop_adma_xor_val_self_test(adev); - dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); - if (ret) - goto err_free_iop_chan; - } - - if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && - dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { - #ifdef CONFIG_RAID6_PQ - ret = iop_adma_pq_zero_sum_self_test(adev); - dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); - #else - /* can not test raid6, so do not publish capability */ - dma_cap_clear(DMA_PQ, dma_dev->cap_mask); - dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); - ret = 0; - #endif - if (ret) - goto err_free_iop_chan; - } - - dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n", - dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", - dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", - dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", - dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", - dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", - dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); - - dma_async_device_register(dma_dev); - goto out; - - err_free_iop_chan: - kfree(iop_chan); - err_free_dma: - dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, - adev->dma_desc_pool_virt, adev->dma_desc_pool); - err_free_adev: - kfree(adev); - out: - return ret; -} - -static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) -{ - struct iop_adma_desc_slot *sw_desc, *grp_start; - dma_cookie_t cookie; - int slot_cnt, slots_per_op; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - - list_splice_init(&sw_desc->tx_list, &iop_chan->chain); - async_tx_ack(&sw_desc->async_tx); - iop_desc_init_memcpy(grp_start, 0); - iop_desc_set_byte_count(grp_start, iop_chan, 0); - iop_desc_set_dest_addr(grp_start, iop_chan, 0); - iop_desc_set_memcpy_src_addr(grp_start, 0); - - cookie = dma_cookie_assign(&sw_desc->async_tx); - - /* initialize the completed cookie to be less than - * the most recently used cookie - */ - iop_chan->common.completed_cookie = cookie - 1; - - /* channel should not be busy */ - BUG_ON(iop_chan_is_busy(iop_chan)); - - /* clear any prior error-status bits */ - iop_adma_device_clear_err_status(iop_chan); - - /* disable operation */ - iop_chan_disable(iop_chan); - - /* set the descriptor address */ - iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); - - /* 1/ don't add pre-chained descriptors - * 2/ dummy read to flush next_desc write - */ - BUG_ON(iop_desc_get_next_desc(sw_desc)); - - /* run the descriptor */ - iop_chan_enable(iop_chan); - } else - dev_err(iop_chan->device->common.dev, - "failed to allocate null descriptor\n"); - spin_unlock_bh(&iop_chan->lock); -} - -static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) -{ - struct iop_adma_desc_slot *sw_desc, *grp_start; - dma_cookie_t cookie; - int slot_cnt, slots_per_op; - - dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); - - spin_lock_bh(&iop_chan->lock); - slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); - sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); - if (sw_desc) { - grp_start = sw_desc->group_head; - list_splice_init(&sw_desc->tx_list, &iop_chan->chain); - async_tx_ack(&sw_desc->async_tx); - iop_desc_init_null_xor(grp_start, 2, 0); - iop_desc_set_byte_count(grp_start, iop_chan, 0); - iop_desc_set_dest_addr(grp_start, iop_chan, 0); - iop_desc_set_xor_src_addr(grp_start, 0, 0); - iop_desc_set_xor_src_addr(grp_start, 1, 0); - - cookie = dma_cookie_assign(&sw_desc->async_tx); - - /* initialize the completed cookie to be less than - * the most recently used cookie - */ - iop_chan->common.completed_cookie = cookie - 1; - - /* channel should not be busy */ - BUG_ON(iop_chan_is_busy(iop_chan)); - - /* clear any prior error-status bits */ - iop_adma_device_clear_err_status(iop_chan); - - /* disable operation */ - iop_chan_disable(iop_chan); - - /* set the descriptor address */ - iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); - - /* 1/ don't add pre-chained descriptors - * 2/ dummy read to flush next_desc write - */ - BUG_ON(iop_desc_get_next_desc(sw_desc)); - - /* run the descriptor */ - iop_chan_enable(iop_chan); - } else - dev_err(iop_chan->device->common.dev, - "failed to allocate null descriptor\n"); - spin_unlock_bh(&iop_chan->lock); -} - -static struct platform_driver iop_adma_driver = { - .probe = iop_adma_probe, - .remove = iop_adma_remove, - .driver = { - .name = "iop-adma", - }, -}; - -module_platform_driver(iop_adma_driver); - -MODULE_AUTHOR("Intel Corporation"); -MODULE_DESCRIPTION("IOP ADMA Engine Driver"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS("platform:iop-adma"); diff --git a/drivers/dma/iop-adma.h b/drivers/dma/iop-adma.h deleted file mode 100644 index d44eabb6f5eb..000000000000 --- a/drivers/dma/iop-adma.h +++ /dev/null @@ -1,914 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright © 2006, Intel Corporation. - */ -#ifndef _ADMA_H -#define _ADMA_H -#include <linux/types.h> -#include <linux/io.h> -#include <linux/platform_data/dma-iop32x.h> - -/* Memory copy units */ -#define DMA_CCR(chan) (chan->mmr_base + 0x0) -#define DMA_CSR(chan) (chan->mmr_base + 0x4) -#define DMA_DAR(chan) (chan->mmr_base + 0xc) -#define DMA_NDAR(chan) (chan->mmr_base + 0x10) -#define DMA_PADR(chan) (chan->mmr_base + 0x14) -#define DMA_PUADR(chan) (chan->mmr_base + 0x18) -#define DMA_LADR(chan) (chan->mmr_base + 0x1c) -#define DMA_BCR(chan) (chan->mmr_base + 0x20) -#define DMA_DCR(chan) (chan->mmr_base + 0x24) - -/* Application accelerator unit */ -#define AAU_ACR(chan) (chan->mmr_base + 0x0) -#define AAU_ASR(chan) (chan->mmr_base + 0x4) -#define AAU_ADAR(chan) (chan->mmr_base + 0x8) -#define AAU_ANDAR(chan) (chan->mmr_base + 0xc) -#define AAU_SAR(src, chan) (chan->mmr_base + (0x10 + ((src) << 2))) -#define AAU_DAR(chan) (chan->mmr_base + 0x20) -#define AAU_ABCR(chan) (chan->mmr_base + 0x24) -#define AAU_ADCR(chan) (chan->mmr_base + 0x28) -#define AAU_SAR_EDCR(src_edc) (chan->mmr_base + (0x02c + ((src_edc-4) << 2))) -#define AAU_EDCR0_IDX 8 -#define AAU_EDCR1_IDX 17 -#define AAU_EDCR2_IDX 26 - -struct iop3xx_aau_desc_ctrl { - unsigned int int_en:1; - unsigned int blk1_cmd_ctrl:3; - unsigned int blk2_cmd_ctrl:3; - unsigned int blk3_cmd_ctrl:3; - unsigned int blk4_cmd_ctrl:3; - unsigned int blk5_cmd_ctrl:3; - unsigned int blk6_cmd_ctrl:3; - unsigned int blk7_cmd_ctrl:3; - unsigned int blk8_cmd_ctrl:3; - unsigned int blk_ctrl:2; - unsigned int dual_xor_en:1; - unsigned int tx_complete:1; - unsigned int zero_result_err:1; - unsigned int zero_result_en:1; - unsigned int dest_write_en:1; -}; - -struct iop3xx_aau_e_desc_ctrl { - unsigned int reserved:1; - unsigned int blk1_cmd_ctrl:3; - unsigned int blk2_cmd_ctrl:3; - unsigned int blk3_cmd_ctrl:3; - unsigned int blk4_cmd_ctrl:3; - unsigned int blk5_cmd_ctrl:3; - unsigned int blk6_cmd_ctrl:3; - unsigned int blk7_cmd_ctrl:3; - unsigned int blk8_cmd_ctrl:3; - unsigned int reserved2:7; -}; - -struct iop3xx_dma_desc_ctrl { - unsigned int pci_transaction:4; - unsigned int int_en:1; - unsigned int dac_cycle_en:1; - unsigned int mem_to_mem_en:1; - unsigned int crc_data_tx_en:1; - unsigned int crc_gen_en:1; - unsigned int crc_seed_dis:1; - unsigned int reserved:21; - unsigned int crc_tx_complete:1; -}; - -struct iop3xx_desc_dma { - u32 next_desc; - union { - u32 pci_src_addr; - u32 pci_dest_addr; - u32 src_addr; - }; - union { - u32 upper_pci_src_addr; - u32 upper_pci_dest_addr; - }; - union { - u32 local_pci_src_addr; - u32 local_pci_dest_addr; - u32 dest_addr; - }; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_dma_desc_ctrl desc_ctrl_field; - }; - u32 crc_addr; -}; - -struct iop3xx_desc_aau { - u32 next_desc; - u32 src[4]; - u32 dest_addr; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_aau_desc_ctrl desc_ctrl_field; - }; - union { - u32 src_addr; - u32 e_desc_ctrl; - struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; - } src_edc[31]; -}; - -struct iop3xx_aau_gfmr { - unsigned int gfmr1:8; - unsigned int gfmr2:8; - unsigned int gfmr3:8; - unsigned int gfmr4:8; -}; - -struct iop3xx_desc_pq_xor { - u32 next_desc; - u32 src[3]; - union { - u32 data_mult1; - struct iop3xx_aau_gfmr data_mult1_field; - }; - u32 dest_addr; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_aau_desc_ctrl desc_ctrl_field; - }; - union { - u32 src_addr; - u32 e_desc_ctrl; - struct iop3xx_aau_e_desc_ctrl e_desc_ctrl_field; - u32 data_multiplier; - struct iop3xx_aau_gfmr data_mult_field; - u32 reserved; - } src_edc_gfmr[19]; -}; - -struct iop3xx_desc_dual_xor { - u32 next_desc; - u32 src0_addr; - u32 src1_addr; - u32 h_src_addr; - u32 d_src_addr; - u32 h_dest_addr; - u32 byte_count; - union { - u32 desc_ctrl; - struct iop3xx_aau_desc_ctrl desc_ctrl_field; - }; - u32 d_dest_addr; -}; - -union iop3xx_desc { - struct iop3xx_desc_aau *aau; - struct iop3xx_desc_dma *dma; - struct iop3xx_desc_pq_xor *pq_xor; - struct iop3xx_desc_dual_xor *dual_xor; - void *ptr; -}; - -/* No support for p+q operations */ -static inline int -iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op) -{ - BUG(); - return 0; -} - -static inline void -iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - BUG(); -} - -static inline void -iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr) -{ - BUG(); -} - -static inline void -iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx, - dma_addr_t addr, unsigned char coef) -{ - BUG(); -} - -static inline int -iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op) -{ - BUG(); - return 0; -} - -static inline void -iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - BUG(); -} - -static inline void -iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) -{ - BUG(); -} - -#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr - -static inline void -iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx, - dma_addr_t *src) -{ - BUG(); -} - -static inline int iop_adma_get_max_xor(void) -{ - return 32; -} - -static inline int iop_adma_get_max_pq(void) -{ - BUG(); - return 0; -} - -static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan) -{ - int id = chan->device->id; - - switch (id) { - case DMA0_ID: - case DMA1_ID: - return __raw_readl(DMA_DAR(chan)); - case AAU_ID: - return __raw_readl(AAU_ADAR(chan)); - default: - BUG(); - } - return 0; -} - -static inline void iop_chan_set_next_descriptor(struct iop_adma_chan *chan, - u32 next_desc_addr) -{ - int id = chan->device->id; - - switch (id) { - case DMA0_ID: - case DMA1_ID: - __raw_writel(next_desc_addr, DMA_NDAR(chan)); - break; - case AAU_ID: - __raw_writel(next_desc_addr, AAU_ANDAR(chan)); - break; - } - -} - -#define IOP_ADMA_STATUS_BUSY (1 << 10) -#define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT (1024) -#define IOP_ADMA_XOR_MAX_BYTE_COUNT (16 * 1024 * 1024) -#define IOP_ADMA_MAX_BYTE_COUNT (16 * 1024 * 1024) - -static inline int iop_chan_is_busy(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - return (status & IOP_ADMA_STATUS_BUSY) ? 1 : 0; -} - -static inline int iop_desc_is_aligned(struct iop_adma_desc_slot *desc, - int num_slots) -{ - /* num_slots will only ever be 1, 2, 4, or 8 */ - return (desc->idx & (num_slots - 1)) ? 0 : 1; -} - -/* to do: support large (i.e. > hw max) buffer sizes */ -static inline int iop_chan_memcpy_slot_count(size_t len, int *slots_per_op) -{ - *slots_per_op = 1; - return 1; -} - -/* to do: support large (i.e. > hw max) buffer sizes */ -static inline int iop_chan_memset_slot_count(size_t len, int *slots_per_op) -{ - *slots_per_op = 1; - return 1; -} - -static inline int iop3xx_aau_xor_slot_count(size_t len, int src_cnt, - int *slots_per_op) -{ - static const char slot_count_table[] = { - 1, 1, 1, 1, /* 01 - 04 */ - 2, 2, 2, 2, /* 05 - 08 */ - 4, 4, 4, 4, /* 09 - 12 */ - 4, 4, 4, 4, /* 13 - 16 */ - 8, 8, 8, 8, /* 17 - 20 */ - 8, 8, 8, 8, /* 21 - 24 */ - 8, 8, 8, 8, /* 25 - 28 */ - 8, 8, 8, 8, /* 29 - 32 */ - }; - *slots_per_op = slot_count_table[src_cnt - 1]; - return *slots_per_op; -} - -static inline int -iop_chan_interrupt_slot_count(int *slots_per_op, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return iop_chan_memcpy_slot_count(0, slots_per_op); - case AAU_ID: - return iop3xx_aau_xor_slot_count(0, 2, slots_per_op); - default: - BUG(); - } - return 0; -} - -static inline int iop_chan_xor_slot_count(size_t len, int src_cnt, - int *slots_per_op) -{ - int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); - - if (len <= IOP_ADMA_XOR_MAX_BYTE_COUNT) - return slot_cnt; - - len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; - while (len > IOP_ADMA_XOR_MAX_BYTE_COUNT) { - len -= IOP_ADMA_XOR_MAX_BYTE_COUNT; - slot_cnt += *slots_per_op; - } - - slot_cnt += *slots_per_op; - - return slot_cnt; -} - -/* zero sum on iop3xx is limited to 1k at a time so it requires multiple - * descriptors - */ -static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt, - int *slots_per_op) -{ - int slot_cnt = iop3xx_aau_xor_slot_count(len, src_cnt, slots_per_op); - - if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) - return slot_cnt; - - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - slot_cnt += *slots_per_op; - } - - slot_cnt += *slots_per_op; - - return slot_cnt; -} - -static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return hw_desc.dma->byte_count; - case AAU_ID: - return hw_desc.aau->byte_count; - default: - BUG(); - } - return 0; -} - -/* translate the src_idx to a descriptor word index */ -static inline int __desc_idx(int src_idx) -{ - static const int desc_idx_table[] = { 0, 0, 0, 0, - 0, 1, 2, 3, - 5, 6, 7, 8, - 9, 10, 11, 12, - 14, 15, 16, 17, - 18, 19, 20, 21, - 23, 24, 25, 26, - 27, 28, 29, 30, - }; - - return desc_idx_table[src_idx]; -} - -static inline u32 iop_desc_get_src_addr(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan, - int src_idx) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return hw_desc.dma->src_addr; - case AAU_ID: - break; - default: - BUG(); - } - - if (src_idx < 4) - return hw_desc.aau->src[src_idx]; - else - return hw_desc.aau->src_edc[__desc_idx(src_idx)].src_addr; -} - -static inline void iop3xx_aau_desc_set_src_addr(struct iop3xx_desc_aau *hw_desc, - int src_idx, dma_addr_t addr) -{ - if (src_idx < 4) - hw_desc->src[src_idx] = addr; - else - hw_desc->src_edc[__desc_idx(src_idx)].src_addr = addr; -} - -static inline void -iop_desc_init_memcpy(struct iop_adma_desc_slot *desc, unsigned long flags) -{ - struct iop3xx_desc_dma *hw_desc = desc->hw_desc; - union { - u32 value; - struct iop3xx_dma_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - u_desc_ctrl.field.mem_to_mem_en = 1; - u_desc_ctrl.field.pci_transaction = 0xe; /* memory read block */ - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; - hw_desc->upper_pci_src_addr = 0; - hw_desc->crc_addr = 0; -} - -static inline void -iop_desc_init_memset(struct iop_adma_desc_slot *desc, unsigned long flags) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - u_desc_ctrl.field.blk1_cmd_ctrl = 0x2; /* memory block fill */ - u_desc_ctrl.field.dest_write_en = 1; - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; -} - -static inline u32 -iop3xx_desc_init_xor(struct iop3xx_desc_aau *hw_desc, int src_cnt, - unsigned long flags) -{ - int i, shift; - u32 edcr; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - switch (src_cnt) { - case 25 ... 32: - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - edcr = 0; - shift = 1; - for (i = 24; i < src_cnt; i++) { - edcr |= (1 << shift); - shift += 3; - } - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = edcr; - src_cnt = 24; - fallthrough; - case 17 ... 24: - if (!u_desc_ctrl.field.blk_ctrl) { - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - } - edcr = 0; - shift = 1; - for (i = 16; i < src_cnt; i++) { - edcr |= (1 << shift); - shift += 3; - } - hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = edcr; - src_cnt = 16; - fallthrough; - case 9 ... 16: - if (!u_desc_ctrl.field.blk_ctrl) - u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ - edcr = 0; - shift = 1; - for (i = 8; i < src_cnt; i++) { - edcr |= (1 << shift); - shift += 3; - } - hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = edcr; - src_cnt = 8; - fallthrough; - case 2 ... 8: - shift = 1; - for (i = 0; i < src_cnt; i++) { - u_desc_ctrl.value |= (1 << shift); - shift += 3; - } - - if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) - u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ - } - - u_desc_ctrl.field.dest_write_en = 1; - u_desc_ctrl.field.blk1_cmd_ctrl = 0x7; /* direct fill */ - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; - - return u_desc_ctrl.value; -} - -static inline void -iop_desc_init_xor(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - iop3xx_desc_init_xor(desc->hw_desc, src_cnt, flags); -} - -/* return the number of operations */ -static inline int -iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; - struct iop3xx_desc_aau *hw_desc, *prev_hw_desc, *iter; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - int i, j; - - hw_desc = desc->hw_desc; - - for (i = 0, j = 0; (slot_cnt -= slots_per_op) >= 0; - i += slots_per_op, j++) { - iter = iop_hw_desc_slot_idx(hw_desc, i); - u_desc_ctrl.value = iop3xx_desc_init_xor(iter, src_cnt, flags); - u_desc_ctrl.field.dest_write_en = 0; - u_desc_ctrl.field.zero_result_en = 1; - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - iter->desc_ctrl = u_desc_ctrl.value; - - /* for the subsequent descriptors preserve the store queue - * and chain them together - */ - if (i) { - prev_hw_desc = - iop_hw_desc_slot_idx(hw_desc, i - slots_per_op); - prev_hw_desc->next_desc = - (u32) (desc->async_tx.phys + (i << 5)); - } - } - - return j; -} - -static inline void -iop_desc_init_null_xor(struct iop_adma_desc_slot *desc, int src_cnt, - unsigned long flags) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - union { - u32 value; - struct iop3xx_aau_desc_ctrl field; - } u_desc_ctrl; - - u_desc_ctrl.value = 0; - switch (src_cnt) { - case 25 ... 32: - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; - fallthrough; - case 17 ... 24: - if (!u_desc_ctrl.field.blk_ctrl) { - hw_desc->src_edc[AAU_EDCR2_IDX].e_desc_ctrl = 0; - u_desc_ctrl.field.blk_ctrl = 0x3; /* use EDCR[2:0] */ - } - hw_desc->src_edc[AAU_EDCR1_IDX].e_desc_ctrl = 0; - fallthrough; - case 9 ... 16: - if (!u_desc_ctrl.field.blk_ctrl) - u_desc_ctrl.field.blk_ctrl = 0x2; /* use EDCR0 */ - hw_desc->src_edc[AAU_EDCR0_IDX].e_desc_ctrl = 0; - fallthrough; - case 1 ... 8: - if (!u_desc_ctrl.field.blk_ctrl && src_cnt > 4) - u_desc_ctrl.field.blk_ctrl = 0x1; /* use mini-desc */ - } - - u_desc_ctrl.field.dest_write_en = 0; - u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT; - hw_desc->desc_ctrl = u_desc_ctrl.value; -} - -static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan, - u32 byte_count) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - hw_desc.dma->byte_count = byte_count; - break; - case AAU_ID: - hw_desc.aau->byte_count = byte_count; - break; - default: - BUG(); - } -} - -static inline void -iop_desc_init_interrupt(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - iop_desc_init_memcpy(desc, 1); - hw_desc.dma->byte_count = 0; - hw_desc.dma->dest_addr = 0; - hw_desc.dma->src_addr = 0; - break; - case AAU_ID: - iop_desc_init_null_xor(desc, 2, 1); - hw_desc.aau->byte_count = 0; - hw_desc.aau->dest_addr = 0; - hw_desc.aau->src[0] = 0; - hw_desc.aau->src[1] = 0; - break; - default: - BUG(); - } -} - -static inline void -iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len) -{ - int slots_per_op = desc->slots_per_op; - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; - int i = 0; - - if (len <= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { - hw_desc->byte_count = len; - } else { - do { - iter = iop_hw_desc_slot_idx(hw_desc, i); - iter->byte_count = IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - len -= IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT; - i += slots_per_op; - } while (len > IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT); - - iter = iop_hw_desc_slot_idx(hw_desc, i); - iter->byte_count = len; - } -} - -static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc, - struct iop_adma_chan *chan, - dma_addr_t addr) -{ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - hw_desc.dma->dest_addr = addr; - break; - case AAU_ID: - hw_desc.aau->dest_addr = addr; - break; - default: - BUG(); - } -} - -static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc, - dma_addr_t addr) -{ - struct iop3xx_desc_dma *hw_desc = desc->hw_desc; - hw_desc->src_addr = addr; -} - -static inline void -iop_desc_set_zero_sum_src_addr(struct iop_adma_desc_slot *desc, int src_idx, - dma_addr_t addr) -{ - - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; - int i; - - for (i = 0; (slot_cnt -= slots_per_op) >= 0; - i += slots_per_op, addr += IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT) { - iter = iop_hw_desc_slot_idx(hw_desc, i); - iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); - } -} - -static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc, - int src_idx, dma_addr_t addr) -{ - - struct iop3xx_desc_aau *hw_desc = desc->hw_desc, *iter; - int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op; - int i; - - for (i = 0; (slot_cnt -= slots_per_op) >= 0; - i += slots_per_op, addr += IOP_ADMA_XOR_MAX_BYTE_COUNT) { - iter = iop_hw_desc_slot_idx(hw_desc, i); - iop3xx_aau_desc_set_src_addr(iter, src_idx, addr); - } -} - -static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc, - u32 next_desc_addr) -{ - /* hw_desc->next_desc is the same location for all channels */ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - - iop_paranoia(hw_desc.dma->next_desc); - hw_desc.dma->next_desc = next_desc_addr; -} - -static inline u32 iop_desc_get_next_desc(struct iop_adma_desc_slot *desc) -{ - /* hw_desc->next_desc is the same location for all channels */ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - return hw_desc.dma->next_desc; -} - -static inline void iop_desc_clear_next_desc(struct iop_adma_desc_slot *desc) -{ - /* hw_desc->next_desc is the same location for all channels */ - union iop3xx_desc hw_desc = { .ptr = desc->hw_desc, }; - hw_desc.dma->next_desc = 0; -} - -static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc, - u32 val) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - hw_desc->src[0] = val; -} - -static inline enum sum_check_flags -iop_desc_get_zero_result(struct iop_adma_desc_slot *desc) -{ - struct iop3xx_desc_aau *hw_desc = desc->hw_desc; - struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field; - - iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en)); - return desc_ctrl.zero_result_err << SUM_CHECK_P; -} - -static inline void iop_chan_append(struct iop_adma_chan *chan) -{ - u32 dma_chan_ctrl; - - dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); - dma_chan_ctrl |= 0x2; - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); -} - -static inline u32 iop_chan_get_status(struct iop_adma_chan *chan) -{ - return __raw_readl(DMA_CSR(chan)); -} - -static inline void iop_chan_disable(struct iop_adma_chan *chan) -{ - u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); - dma_chan_ctrl &= ~1; - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); -} - -static inline void iop_chan_enable(struct iop_adma_chan *chan) -{ - u32 dma_chan_ctrl = __raw_readl(DMA_CCR(chan)); - - dma_chan_ctrl |= 1; - __raw_writel(dma_chan_ctrl, DMA_CCR(chan)); -} - -static inline void iop_adma_device_clear_eot_status(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - status &= (1 << 9); - __raw_writel(status, DMA_CSR(chan)); -} - -static inline void iop_adma_device_clear_eoc_status(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - status &= (1 << 8); - __raw_writel(status, DMA_CSR(chan)); -} - -static inline void iop_adma_device_clear_err_status(struct iop_adma_chan *chan) -{ - u32 status = __raw_readl(DMA_CSR(chan)); - - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - status &= (1 << 5) | (1 << 3) | (1 << 2) | (1 << 1); - break; - case AAU_ID: - status &= (1 << 5); - break; - default: - BUG(); - } - - __raw_writel(status, DMA_CSR(chan)); -} - -static inline int -iop_is_err_int_parity(unsigned long status, struct iop_adma_chan *chan) -{ - return 0; -} - -static inline int -iop_is_err_mcu_abort(unsigned long status, struct iop_adma_chan *chan) -{ - return 0; -} - -static inline int -iop_is_err_int_tabort(unsigned long status, struct iop_adma_chan *chan) -{ - return 0; -} - -static inline int -iop_is_err_int_mabort(unsigned long status, struct iop_adma_chan *chan) -{ - return test_bit(5, &status); -} - -static inline int -iop_is_err_pci_tabort(unsigned long status, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return test_bit(2, &status); - default: - return 0; - } -} - -static inline int -iop_is_err_pci_mabort(unsigned long status, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return test_bit(3, &status); - default: - return 0; - } -} - -static inline int -iop_is_err_split_tx(unsigned long status, struct iop_adma_chan *chan) -{ - switch (chan->device->id) { - case DMA0_ID: - case DMA1_ID: - return test_bit(1, &status); - default: - return 0; - } -} -#endif /* _ADMA_H */ diff --git a/drivers/dma/ipu/Makefile b/drivers/dma/ipu/Makefile deleted file mode 100644 index c79ff116daf6..000000000000 --- a/drivers/dma/ipu/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-y += ipu_irq.o ipu_idmac.o diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c deleted file mode 100644 index 104ad420abbe..000000000000 --- a/drivers/dma/ipu/ipu_idmac.c +++ /dev/null @@ -1,1800 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2008 - * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de> - * - * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved. - */ - -#include <linux/dma-mapping.h> -#include <linux/init.h> -#include <linux/platform_device.h> -#include <linux/err.h> -#include <linux/spinlock.h> -#include <linux/delay.h> -#include <linux/list.h> -#include <linux/clk.h> -#include <linux/vmalloc.h> -#include <linux/string.h> -#include <linux/interrupt.h> -#include <linux/io.h> -#include <linux/module.h> -#include <linux/dma/ipu-dma.h> - -#include "../dmaengine.h" -#include "ipu_intern.h" - -#define FS_VF_IN_VALID 0x00000002 -#define FS_ENC_IN_VALID 0x00000001 - -static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan, - bool wait_for_stop); - -/* - * There can be only one, we could allocate it dynamically, but then we'd have - * to add an extra parameter to some functions, and use something as ugly as - * struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device)); - * in the ISR - */ -static struct ipu ipu_data; - -#define to_ipu(id) container_of(id, struct ipu, idmac) - -static u32 __idmac_read_icreg(struct ipu *ipu, unsigned long reg) -{ - return __raw_readl(ipu->reg_ic + reg); -} - -#define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF) - -static void __idmac_write_icreg(struct ipu *ipu, u32 value, unsigned long reg) -{ - __raw_writel(value, ipu->reg_ic + reg); -} - -#define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF) - -static u32 idmac_read_ipureg(struct ipu *ipu, unsigned long reg) -{ - return __raw_readl(ipu->reg_ipu + reg); -} - -static void idmac_write_ipureg(struct ipu *ipu, u32 value, unsigned long reg) -{ - __raw_writel(value, ipu->reg_ipu + reg); -} - -/***************************************************************************** - * IPU / IC common functions - */ -static void dump_idmac_reg(struct ipu *ipu) -{ - dev_dbg(ipu->dev, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, " - "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n", - idmac_read_icreg(ipu, IDMAC_CONF), - idmac_read_icreg(ipu, IC_CONF), - idmac_read_icreg(ipu, IDMAC_CHA_EN), - idmac_read_icreg(ipu, IDMAC_CHA_PRI), - idmac_read_icreg(ipu, IDMAC_CHA_BUSY)); - dev_dbg(ipu->dev, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, " - "DB_MODE 0x%x, TASKS_STAT 0x%x\n", - idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY), - idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY), - idmac_read_ipureg(ipu, IPU_CHA_CUR_BUF), - idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL), - idmac_read_ipureg(ipu, IPU_TASKS_STAT)); -} - -static uint32_t bytes_per_pixel(enum pixel_fmt fmt) -{ - switch (fmt) { - case IPU_PIX_FMT_GENERIC: /* generic data */ - case IPU_PIX_FMT_RGB332: - case IPU_PIX_FMT_YUV420P: - case IPU_PIX_FMT_YUV422P: - default: - return 1; - case IPU_PIX_FMT_RGB565: - case IPU_PIX_FMT_YUYV: - case IPU_PIX_FMT_UYVY: - return 2; - case IPU_PIX_FMT_BGR24: - case IPU_PIX_FMT_RGB24: - return 3; - case IPU_PIX_FMT_GENERIC_32: /* generic data */ - case IPU_PIX_FMT_BGR32: - case IPU_PIX_FMT_RGB32: - case IPU_PIX_FMT_ABGR32: - return 4; - } -} - -/* Enable direct write to memory by the Camera Sensor Interface */ -static void ipu_ic_enable_task(struct ipu *ipu, enum ipu_channel channel) -{ - uint32_t ic_conf, mask; - - switch (channel) { - case IDMAC_IC_0: - mask = IC_CONF_PRPENC_EN; - break; - case IDMAC_IC_7: - mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN; - break; - default: - return; - } - ic_conf = idmac_read_icreg(ipu, IC_CONF) | mask; - idmac_write_icreg(ipu, ic_conf, IC_CONF); -} - -/* Called under spin_lock_irqsave(&ipu_data.lock) */ -static void ipu_ic_disable_task(struct ipu *ipu, enum ipu_channel channel) -{ - uint32_t ic_conf, mask; - - switch (channel) { - case IDMAC_IC_0: - mask = IC_CONF_PRPENC_EN; - break; - case IDMAC_IC_7: - mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN; - break; - default: - return; - } - ic_conf = idmac_read_icreg(ipu, IC_CONF) & ~mask; - idmac_write_icreg(ipu, ic_conf, IC_CONF); -} - -static uint32_t ipu_channel_status(struct ipu *ipu, enum ipu_channel channel) -{ - uint32_t stat = TASK_STAT_IDLE; - uint32_t task_stat_reg = idmac_read_ipureg(ipu, IPU_TASKS_STAT); - - switch (channel) { - case IDMAC_IC_7: - stat = (task_stat_reg & TSTAT_CSI2MEM_MASK) >> - TSTAT_CSI2MEM_OFFSET; - break; - case IDMAC_IC_0: - case IDMAC_SDC_0: - case IDMAC_SDC_1: - default: - break; - } - return stat; -} - -struct chan_param_mem_planar { - /* Word 0 */ - u32 xv:10; - u32 yv:10; - u32 xb:12; - - u32 yb:12; - u32 res1:2; - u32 nsb:1; - u32 lnpb:6; - u32 ubo_l:11; - - u32 ubo_h:15; - u32 vbo_l:17; - - u32 vbo_h:9; - u32 res2:3; - u32 fw:12; - u32 fh_l:8; - - u32 fh_h:4; - u32 res3:28; - - /* Word 1 */ - u32 eba0; - - u32 eba1; - - u32 bpp:3; - u32 sl:14; - u32 pfs:3; - u32 bam:3; - u32 res4:2; - u32 npb:6; - u32 res5:1; - - u32 sat:2; - u32 res6:30; -} __attribute__ ((packed)); - -struct chan_param_mem_interleaved { - /* Word 0 */ - u32 xv:10; - u32 yv:10; - u32 xb:12; - - u32 yb:12; - u32 sce:1; - u32 res1:1; - u32 nsb:1; - u32 lnpb:6; - u32 sx:10; - u32 sy_l:1; - - u32 sy_h:9; - u32 ns:10; - u32 sm:10; - u32 sdx_l:3; - - u32 sdx_h:2; - u32 sdy:5; - u32 sdrx:1; - u32 sdry:1; - u32 sdr1:1; - u32 res2:2; - u32 fw:12; - u32 fh_l:8; - - u32 fh_h:4; - u32 res3:28; - - /* Word 1 */ - u32 eba0; - - u32 eba1; - - u32 bpp:3; - u32 sl:14; - u32 pfs:3; - u32 bam:3; - u32 res4:2; - u32 npb:6; - u32 res5:1; - - u32 sat:2; - u32 scc:1; - u32 ofs0:5; - u32 ofs1:5; - u32 ofs2:5; - u32 ofs3:5; - u32 wid0:3; - u32 wid1:3; - u32 wid2:3; - - u32 wid3:3; - u32 dec_sel:1; - u32 res6:28; -} __attribute__ ((packed)); - -union chan_param_mem { - struct chan_param_mem_planar pp; - struct chan_param_mem_interleaved ip; -}; - -static void ipu_ch_param_set_plane_offset(union chan_param_mem *params, - u32 u_offset, u32 v_offset) -{ - params->pp.ubo_l = u_offset & 0x7ff; - params->pp.ubo_h = u_offset >> 11; - params->pp.vbo_l = v_offset & 0x1ffff; - params->pp.vbo_h = v_offset >> 17; -} - -static void ipu_ch_param_set_size(union chan_param_mem *params, - uint32_t pixel_fmt, uint16_t width, - uint16_t height, uint16_t stride) -{ - u32 u_offset; - u32 v_offset; - - params->pp.fw = width - 1; - params->pp.fh_l = height - 1; - params->pp.fh_h = (height - 1) >> 8; - params->pp.sl = stride - 1; - - switch (pixel_fmt) { - case IPU_PIX_FMT_GENERIC: - /*Represents 8-bit Generic data */ - params->pp.bpp = 3; - params->pp.pfs = 7; - params->pp.npb = 31; - params->pp.sat = 2; /* SAT = use 32-bit access */ - break; - case IPU_PIX_FMT_GENERIC_32: - /*Represents 32-bit Generic data */ - params->pp.bpp = 0; - params->pp.pfs = 7; - params->pp.npb = 7; - params->pp.sat = 2; /* SAT = use 32-bit access */ - break; - case IPU_PIX_FMT_RGB565: - params->ip.bpp = 2; - params->ip.pfs = 4; - params->ip.npb = 15; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 0; /* Red bit offset */ - params->ip.ofs1 = 5; /* Green bit offset */ - params->ip.ofs2 = 11; /* Blue bit offset */ - params->ip.ofs3 = 16; /* Alpha bit offset */ - params->ip.wid0 = 4; /* Red bit width - 1 */ - params->ip.wid1 = 5; /* Green bit width - 1 */ - params->ip.wid2 = 4; /* Blue bit width - 1 */ - break; - case IPU_PIX_FMT_BGR24: - params->ip.bpp = 1; /* 24 BPP & RGB PFS */ - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 0; /* Red bit offset */ - params->ip.ofs1 = 8; /* Green bit offset */ - params->ip.ofs2 = 16; /* Blue bit offset */ - params->ip.ofs3 = 24; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - break; - case IPU_PIX_FMT_RGB24: - params->ip.bpp = 1; /* 24 BPP & RGB PFS */ - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 16; /* Red bit offset */ - params->ip.ofs1 = 8; /* Green bit offset */ - params->ip.ofs2 = 0; /* Blue bit offset */ - params->ip.ofs3 = 24; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - break; - case IPU_PIX_FMT_BGRA32: - case IPU_PIX_FMT_BGR32: - case IPU_PIX_FMT_ABGR32: - params->ip.bpp = 0; - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 8; /* Red bit offset */ - params->ip.ofs1 = 16; /* Green bit offset */ - params->ip.ofs2 = 24; /* Blue bit offset */ - params->ip.ofs3 = 0; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - params->ip.wid3 = 7; /* Alpha bit width - 1 */ - break; - case IPU_PIX_FMT_RGBA32: - case IPU_PIX_FMT_RGB32: - params->ip.bpp = 0; - params->ip.pfs = 4; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - params->ip.ofs0 = 24; /* Red bit offset */ - params->ip.ofs1 = 16; /* Green bit offset */ - params->ip.ofs2 = 8; /* Blue bit offset */ - params->ip.ofs3 = 0; /* Alpha bit offset */ - params->ip.wid0 = 7; /* Red bit width - 1 */ - params->ip.wid1 = 7; /* Green bit width - 1 */ - params->ip.wid2 = 7; /* Blue bit width - 1 */ - params->ip.wid3 = 7; /* Alpha bit width - 1 */ - break; - case IPU_PIX_FMT_UYVY: - params->ip.bpp = 2; - params->ip.pfs = 6; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - break; - case IPU_PIX_FMT_YUV420P2: - case IPU_PIX_FMT_YUV420P: - params->ip.bpp = 3; - params->ip.pfs = 3; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - u_offset = stride * height; - v_offset = u_offset + u_offset / 4; - ipu_ch_param_set_plane_offset(params, u_offset, v_offset); - break; - case IPU_PIX_FMT_YVU422P: - params->ip.bpp = 3; - params->ip.pfs = 2; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - v_offset = stride * height; - u_offset = v_offset + v_offset / 2; - ipu_ch_param_set_plane_offset(params, u_offset, v_offset); - break; - case IPU_PIX_FMT_YUV422P: - params->ip.bpp = 3; - params->ip.pfs = 2; - params->ip.npb = 7; - params->ip.sat = 2; /* SAT = 32-bit access */ - u_offset = stride * height; - v_offset = u_offset + u_offset / 2; - ipu_ch_param_set_plane_offset(params, u_offset, v_offset); - break; - default: - dev_err(ipu_data.dev, - "mx3 ipu: unimplemented pixel format %d\n", pixel_fmt); - break; - } - - params->pp.nsb = 1; -} - -static void ipu_ch_param_set_buffer(union chan_param_mem *params, - dma_addr_t buf0, dma_addr_t buf1) -{ - params->pp.eba0 = buf0; - params->pp.eba1 = buf1; -} - -static void ipu_ch_param_set_rotation(union chan_param_mem *params, - enum ipu_rotate_mode rotate) -{ - params->pp.bam = rotate; -} - -static void ipu_write_param_mem(uint32_t addr, uint32_t *data, - uint32_t num_words) -{ - for (; num_words > 0; num_words--) { - dev_dbg(ipu_data.dev, - "write param mem - addr = 0x%08X, data = 0x%08X\n", - addr, *data); - idmac_write_ipureg(&ipu_data, addr, IPU_IMA_ADDR); - idmac_write_ipureg(&ipu_data, *data++, IPU_IMA_DATA); - addr++; - if ((addr & 0x7) == 5) { - addr &= ~0x7; /* set to word 0 */ - addr += 8; /* increment to next row */ - } - } -} - -static int calc_resize_coeffs(uint32_t in_size, uint32_t out_size, - uint32_t *resize_coeff, - uint32_t *downsize_coeff) -{ - uint32_t temp_size; - uint32_t temp_downsize; - - *resize_coeff = 1 << 13; - *downsize_coeff = 1 << 13; - - /* Cannot downsize more than 8:1 */ - if (out_size << 3 < in_size) - return -EINVAL; - - /* compute downsizing coefficient */ - temp_downsize = 0; - temp_size = in_size; - while (temp_size >= out_size * 2 && temp_downsize < 2) { - temp_size >>= 1; - temp_downsize++; - } - *downsize_coeff = temp_downsize; - - /* - * compute resizing coefficient using the following formula: - * resize_coeff = M*(SI -1)/(SO - 1) - * where M = 2^13, SI - input size, SO - output size - */ - *resize_coeff = (8192L * (temp_size - 1)) / (out_size - 1); - if (*resize_coeff >= 16384L) { - dev_err(ipu_data.dev, "Warning! Overflow on resize coeff.\n"); - *resize_coeff = 0x3FFF; - } - - dev_dbg(ipu_data.dev, "resizing from %u -> %u pixels, " - "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size, out_size, - *downsize_coeff, *resize_coeff >= 8192L ? 1 : 0, - ((*resize_coeff & 0x1FFF) * 10000L) / 8192L, *resize_coeff); - - return 0; -} - -static enum ipu_color_space format_to_colorspace(enum pixel_fmt fmt) -{ - switch (fmt) { - case IPU_PIX_FMT_RGB565: - case IPU_PIX_FMT_BGR24: - case IPU_PIX_FMT_RGB24: - case IPU_PIX_FMT_BGR32: - case IPU_PIX_FMT_RGB32: - return IPU_COLORSPACE_RGB; - default: - return IPU_COLORSPACE_YCBCR; - } -} - -static int ipu_ic_init_prpenc(struct ipu *ipu, - union ipu_channel_param *params, bool src_is_csi) -{ - uint32_t reg, ic_conf; - uint32_t downsize_coeff, resize_coeff; - enum ipu_color_space in_fmt, out_fmt; - - /* Setup vertical resizing */ - calc_resize_coeffs(params->video.in_height, - params->video.out_height, - &resize_coeff, &downsize_coeff); - reg = (downsize_coeff << 30) | (resize_coeff << 16); - - /* Setup horizontal resizing */ - calc_resize_coeffs(params->video.in_width, - params->video.out_width, - &resize_coeff, &downsize_coeff); - reg |= (downsize_coeff << 14) | resize_coeff; - - /* Setup color space conversion */ - in_fmt = format_to_colorspace(params->video.in_pixel_fmt); - out_fmt = format_to_colorspace(params->video.out_pixel_fmt); - - /* - * Colourspace conversion unsupported yet - see _init_csc() in - * Freescale sources - */ - if (in_fmt != out_fmt) { - dev_err(ipu->dev, "Colourspace conversion unsupported!\n"); - return -EOPNOTSUPP; - } - - idmac_write_icreg(ipu, reg, IC_PRP_ENC_RSC); - - ic_conf = idmac_read_icreg(ipu, IC_CONF); - - if (src_is_csi) - ic_conf &= ~IC_CONF_RWS_EN; - else - ic_conf |= IC_CONF_RWS_EN; - - idmac_write_icreg(ipu, ic_conf, IC_CONF); - - return 0; -} - -static uint32_t dma_param_addr(uint32_t dma_ch) -{ - /* Channel Parameter Memory */ - return 0x10000 | (dma_ch << 4); -} - -static void ipu_channel_set_priority(struct ipu *ipu, enum ipu_channel channel, - bool prio) -{ - u32 reg = idmac_read_icreg(ipu, IDMAC_CHA_PRI); - - if (prio) - reg |= 1UL << channel; - else - reg &= ~(1UL << channel); - - idmac_write_icreg(ipu, reg, IDMAC_CHA_PRI); - - dump_idmac_reg(ipu); -} - -static uint32_t ipu_channel_conf_mask(enum ipu_channel channel) -{ - uint32_t mask; - - switch (channel) { - case IDMAC_IC_0: - case IDMAC_IC_7: - mask = IPU_CONF_CSI_EN | IPU_CONF_IC_EN; - break; - case IDMAC_SDC_0: - case IDMAC_SDC_1: - mask = IPU_CONF_SDC_EN | IPU_CONF_DI_EN; - break; - default: - mask = 0; - break; - } - - return mask; -} - -/** - * ipu_enable_channel() - enable an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: IDMAC channel. - * @return: 0 on success or negative error code on failure. - */ -static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan) -{ - struct ipu *ipu = to_ipu(idmac); - enum ipu_channel channel = ichan->dma_chan.chan_id; - uint32_t reg; - unsigned long flags; - - spin_lock_irqsave(&ipu->lock, flags); - - /* Reset to buffer 0 */ - idmac_write_ipureg(ipu, 1UL << channel, IPU_CHA_CUR_BUF); - ichan->active_buffer = 0; - ichan->status = IPU_CHANNEL_ENABLED; - - switch (channel) { - case IDMAC_SDC_0: - case IDMAC_SDC_1: - case IDMAC_IC_7: - ipu_channel_set_priority(ipu, channel, true); - default: - break; - } - - reg = idmac_read_icreg(ipu, IDMAC_CHA_EN); - - idmac_write_icreg(ipu, reg | (1UL << channel), IDMAC_CHA_EN); - - ipu_ic_enable_task(ipu, channel); - - spin_unlock_irqrestore(&ipu->lock, flags); - return 0; -} - -/** - * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel. - * @ichan: IDMAC channel. - * @pixel_fmt: pixel format of buffer. Pixel format is a FOURCC ASCII code. - * @width: width of buffer in pixels. - * @height: height of buffer in pixels. - * @stride: stride length of buffer in pixels. - * @rot_mode: rotation mode of buffer. A rotation setting other than - * IPU_ROTATE_VERT_FLIP should only be used for input buffers of - * rotation channels. - * @phyaddr_0: buffer 0 physical address. - * @phyaddr_1: buffer 1 physical address. Setting this to a value other than - * NULL enables double buffering mode. - * @return: 0 on success or negative error code on failure. - */ -static int ipu_init_channel_buffer(struct idmac_channel *ichan, - enum pixel_fmt pixel_fmt, - uint16_t width, uint16_t height, - uint32_t stride, - enum ipu_rotate_mode rot_mode, - dma_addr_t phyaddr_0, dma_addr_t phyaddr_1) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - struct idmac *idmac = to_idmac(ichan->dma_chan.device); - struct ipu *ipu = to_ipu(idmac); - union chan_param_mem params = {}; - unsigned long flags; - uint32_t reg; - uint32_t stride_bytes; - - stride_bytes = stride * bytes_per_pixel(pixel_fmt); - - if (stride_bytes % 4) { - dev_err(ipu->dev, - "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n", - stride, stride_bytes); - return -EINVAL; - } - - /* IC channel's stride must be a multiple of 8 pixels */ - if ((channel <= IDMAC_IC_13) && (stride % 8)) { - dev_err(ipu->dev, "Stride must be 8 pixel multiple\n"); - return -EINVAL; - } - - /* Build parameter memory data for DMA channel */ - ipu_ch_param_set_size(¶ms, pixel_fmt, width, height, stride_bytes); - ipu_ch_param_set_buffer(¶ms, phyaddr_0, phyaddr_1); - ipu_ch_param_set_rotation(¶ms, rot_mode); - - spin_lock_irqsave(&ipu->lock, flags); - - ipu_write_param_mem(dma_param_addr(channel), (uint32_t *)¶ms, 10); - - reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL); - - if (phyaddr_1) - reg |= 1UL << channel; - else - reg &= ~(1UL << channel); - - idmac_write_ipureg(ipu, reg, IPU_CHA_DB_MODE_SEL); - - ichan->status = IPU_CHANNEL_READY; - - spin_unlock_irqrestore(&ipu->lock, flags); - - return 0; -} - -/** - * ipu_select_buffer() - mark a channel's buffer as ready. - * @channel: channel ID. - * @buffer_n: buffer number to mark ready. - */ -static void ipu_select_buffer(enum ipu_channel channel, int buffer_n) -{ - /* No locking - this is a write-one-to-set register, cleared by IPU */ - if (buffer_n == 0) - /* Mark buffer 0 as ready. */ - idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF0_RDY); - else - /* Mark buffer 1 as ready. */ - idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF1_RDY); -} - -/** - * ipu_update_channel_buffer() - update physical address of a channel buffer. - * @ichan: IDMAC channel. - * @buffer_n: buffer number to update. - * 0 or 1 are the only valid values. - * @phyaddr: buffer physical address. - */ -/* Called under spin_lock(_irqsave)(&ichan->lock) */ -static void ipu_update_channel_buffer(struct idmac_channel *ichan, - int buffer_n, dma_addr_t phyaddr) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - uint32_t reg; - unsigned long flags; - - spin_lock_irqsave(&ipu_data.lock, flags); - - if (buffer_n == 0) { - reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY); - if (reg & (1UL << channel)) { - ipu_ic_disable_task(&ipu_data, channel); - ichan->status = IPU_CHANNEL_READY; - } - - /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */ - idmac_write_ipureg(&ipu_data, dma_param_addr(channel) + - 0x0008UL, IPU_IMA_ADDR); - idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA); - } else { - reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY); - if (reg & (1UL << channel)) { - ipu_ic_disable_task(&ipu_data, channel); - ichan->status = IPU_CHANNEL_READY; - } - - /* Check if double-buffering is already enabled */ - reg = idmac_read_ipureg(&ipu_data, IPU_CHA_DB_MODE_SEL); - - if (!(reg & (1UL << channel))) - idmac_write_ipureg(&ipu_data, reg | (1UL << channel), - IPU_CHA_DB_MODE_SEL); - - /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */ - idmac_write_ipureg(&ipu_data, dma_param_addr(channel) + - 0x0009UL, IPU_IMA_ADDR); - idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA); - } - - spin_unlock_irqrestore(&ipu_data.lock, flags); -} - -/* Called under spin_lock_irqsave(&ichan->lock) */ -static int ipu_submit_buffer(struct idmac_channel *ichan, - struct idmac_tx_desc *desc, struct scatterlist *sg, int buf_idx) -{ - unsigned int chan_id = ichan->dma_chan.chan_id; - struct device *dev = &ichan->dma_chan.dev->device; - - if (async_tx_test_ack(&desc->txd)) - return -EINTR; - - /* - * On first invocation this shouldn't be necessary, the call to - * ipu_init_channel_buffer() above will set addresses for us, so we - * could make it conditional on status >= IPU_CHANNEL_ENABLED, but - * doing it again shouldn't hurt either. - */ - ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg)); - - ipu_select_buffer(chan_id, buf_idx); - dev_dbg(dev, "Updated sg %p on channel 0x%x buffer %d\n", - sg, chan_id, buf_idx); - - return 0; -} - -/* Called under spin_lock_irqsave(&ichan->lock) */ -static int ipu_submit_channel_buffers(struct idmac_channel *ichan, - struct idmac_tx_desc *desc) -{ - struct scatterlist *sg; - int i, ret = 0; - - for (i = 0, sg = desc->sg; i < 2 && sg; i++) { - if (!ichan->sg[i]) { - ichan->sg[i] = sg; - - ret = ipu_submit_buffer(ichan, desc, sg, i); - if (ret < 0) - return ret; - - sg = sg_next(sg); - } - } - - return ret; -} - -static dma_cookie_t idmac_tx_submit(struct dma_async_tx_descriptor *tx) -{ - struct idmac_tx_desc *desc = to_tx_desc(tx); - struct idmac_channel *ichan = to_idmac_chan(tx->chan); - struct idmac *idmac = to_idmac(tx->chan->device); - struct ipu *ipu = to_ipu(idmac); - struct device *dev = &ichan->dma_chan.dev->device; - dma_cookie_t cookie; - unsigned long flags; - int ret; - - /* Sanity check */ - if (!list_empty(&desc->list)) { - /* The descriptor doesn't belong to client */ - dev_err(dev, "Descriptor %p not prepared!\n", tx); - return -EBUSY; - } - - mutex_lock(&ichan->chan_mutex); - - async_tx_clear_ack(tx); - - if (ichan->status < IPU_CHANNEL_READY) { - struct idmac_video_param *video = &ichan->params.video; - /* - * Initial buffer assignment - the first two sg-entries from - * the descriptor will end up in the IDMAC buffers - */ - dma_addr_t dma_1 = sg_is_last(desc->sg) ? 0 : - sg_dma_address(&desc->sg[1]); - - WARN_ON(ichan->sg[0] || ichan->sg[1]); - - cookie = ipu_init_channel_buffer(ichan, - video->out_pixel_fmt, - video->out_width, - video->out_height, - video->out_stride, - IPU_ROTATE_NONE, - sg_dma_address(&desc->sg[0]), - dma_1); - if (cookie < 0) - goto out; - } - - dev_dbg(dev, "Submitting sg %p\n", &desc->sg[0]); - - cookie = dma_cookie_assign(tx); - - /* ipu->lock can be taken under ichan->lock, but not v.v. */ - spin_lock_irqsave(&ichan->lock, flags); - - list_add_tail(&desc->list, &ichan->queue); - /* submit_buffers() atomically verifies and fills empty sg slots */ - ret = ipu_submit_channel_buffers(ichan, desc); - - spin_unlock_irqrestore(&ichan->lock, flags); - - if (ret < 0) { - cookie = ret; - goto dequeue; - } - - if (ichan->status < IPU_CHANNEL_ENABLED) { - ret = ipu_enable_channel(idmac, ichan); - if (ret < 0) { - cookie = ret; - goto dequeue; - } - } - - dump_idmac_reg(ipu); - -dequeue: - if (cookie < 0) { - spin_lock_irqsave(&ichan->lock, flags); - list_del_init(&desc->list); - spin_unlock_irqrestore(&ichan->lock, flags); - tx->cookie = cookie; - ichan->dma_chan.cookie = cookie; - } - -out: - mutex_unlock(&ichan->chan_mutex); - - return cookie; -} - -/* Called with ichan->chan_mutex held */ -static int idmac_desc_alloc(struct idmac_channel *ichan, int n) -{ - struct idmac_tx_desc *desc = - vmalloc(array_size(n, sizeof(struct idmac_tx_desc))); - struct idmac *idmac = to_idmac(ichan->dma_chan.device); - - if (!desc) - return -ENOMEM; - - /* No interrupts, just disable the tasklet for a moment */ - tasklet_disable(&to_ipu(idmac)->tasklet); - - ichan->n_tx_desc = n; - ichan->desc = desc; - INIT_LIST_HEAD(&ichan->queue); - INIT_LIST_HEAD(&ichan->free_list); - - while (n--) { - struct dma_async_tx_descriptor *txd = &desc->txd; - - memset(txd, 0, sizeof(*txd)); - dma_async_tx_descriptor_init(txd, &ichan->dma_chan); - txd->tx_submit = idmac_tx_submit; - - list_add(&desc->list, &ichan->free_list); - - desc++; - } - - tasklet_enable(&to_ipu(idmac)->tasklet); - - return 0; -} - -/** - * ipu_init_channel() - initialize an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: pointer to the channel object. - * @return 0 on success or negative error code on failure. - */ -static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan) -{ - union ipu_channel_param *params = &ichan->params; - uint32_t ipu_conf; - enum ipu_channel channel = ichan->dma_chan.chan_id; - unsigned long flags; - uint32_t reg; - struct ipu *ipu = to_ipu(idmac); - int ret = 0, n_desc = 0; - - dev_dbg(ipu->dev, "init channel = %d\n", channel); - - if (channel != IDMAC_SDC_0 && channel != IDMAC_SDC_1 && - channel != IDMAC_IC_7) - return -EINVAL; - - spin_lock_irqsave(&ipu->lock, flags); - - switch (channel) { - case IDMAC_IC_7: - n_desc = 16; - reg = idmac_read_icreg(ipu, IC_CONF); - idmac_write_icreg(ipu, reg & ~IC_CONF_CSI_MEM_WR_EN, IC_CONF); - break; - case IDMAC_IC_0: - n_desc = 16; - reg = idmac_read_ipureg(ipu, IPU_FS_PROC_FLOW); - idmac_write_ipureg(ipu, reg & ~FS_ENC_IN_VALID, IPU_FS_PROC_FLOW); - ret = ipu_ic_init_prpenc(ipu, params, true); - break; - case IDMAC_SDC_0: - case IDMAC_SDC_1: - n_desc = 4; - default: - break; - } - - ipu->channel_init_mask |= 1L << channel; - - /* Enable IPU sub module */ - ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) | - ipu_channel_conf_mask(channel); - idmac_write_ipureg(ipu, ipu_conf, IPU_CONF); - - spin_unlock_irqrestore(&ipu->lock, flags); - - if (n_desc && !ichan->desc) - ret = idmac_desc_alloc(ichan, n_desc); - - dump_idmac_reg(ipu); - - return ret; -} - -/** - * ipu_uninit_channel() - uninitialize an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: pointer to the channel object. - */ -static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - unsigned long flags; - uint32_t reg; - unsigned long chan_mask = 1UL << channel; - uint32_t ipu_conf; - struct ipu *ipu = to_ipu(idmac); - - spin_lock_irqsave(&ipu->lock, flags); - - if (!(ipu->channel_init_mask & chan_mask)) { - dev_err(ipu->dev, "Channel already uninitialized %d\n", - channel); - spin_unlock_irqrestore(&ipu->lock, flags); - return; - } - - /* Reset the double buffer */ - reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL); - idmac_write_ipureg(ipu, reg & ~chan_mask, IPU_CHA_DB_MODE_SEL); - - ichan->sec_chan_en = false; - - switch (channel) { - case IDMAC_IC_7: - reg = idmac_read_icreg(ipu, IC_CONF); - idmac_write_icreg(ipu, reg & ~(IC_CONF_RWS_EN | IC_CONF_PRPENC_EN), - IC_CONF); - break; - case IDMAC_IC_0: - reg = idmac_read_icreg(ipu, IC_CONF); - idmac_write_icreg(ipu, reg & ~(IC_CONF_PRPENC_EN | IC_CONF_PRPENC_CSC1), - IC_CONF); - break; - case IDMAC_SDC_0: - case IDMAC_SDC_1: - default: - break; - } - - ipu->channel_init_mask &= ~(1L << channel); - - ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) & - ~ipu_channel_conf_mask(channel); - idmac_write_ipureg(ipu, ipu_conf, IPU_CONF); - - spin_unlock_irqrestore(&ipu->lock, flags); - - ichan->n_tx_desc = 0; - vfree(ichan->desc); - ichan->desc = NULL; -} - -/** - * ipu_disable_channel() - disable an IPU channel. - * @idmac: IPU DMAC context. - * @ichan: channel object pointer. - * @wait_for_stop: flag to set whether to wait for channel end of frame or - * return immediately. - * @return: 0 on success or negative error code on failure. - */ -static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan, - bool wait_for_stop) -{ - enum ipu_channel channel = ichan->dma_chan.chan_id; - struct ipu *ipu = to_ipu(idmac); - uint32_t reg; - unsigned long flags; - unsigned long chan_mask = 1UL << channel; - unsigned int timeout; - - if (wait_for_stop && channel != IDMAC_SDC_1 && channel != IDMAC_SDC_0) { - timeout = 40; - /* This waiting always fails. Related to spurious irq problem */ - while ((idmac_read_icreg(ipu, IDMAC_CHA_BUSY) & chan_mask) || - (ipu_channel_status(ipu, channel) == TASK_STAT_ACTIVE)) { - timeout--; - msleep(10); - - if (!timeout) { - dev_dbg(ipu->dev, - "Warning: timeout waiting for channel %u to " - "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, " - "busy = 0x%08X, tstat = 0x%08X\n", channel, - idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY), - idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY), - idmac_read_icreg(ipu, IDMAC_CHA_BUSY), - idmac_read_ipureg(ipu, IPU_TASKS_STAT)); - break; - } - } - dev_dbg(ipu->dev, "timeout = %d * 10ms\n", 40 - timeout); - } - /* SDC BG and FG must be disabled before DMA is disabled */ - if (wait_for_stop && (channel == IDMAC_SDC_0 || - channel == IDMAC_SDC_1)) { - for (timeout = 5; - timeout && !ipu_irq_status(ichan->eof_irq); timeout--) - msleep(5); - } - - spin_lock_irqsave(&ipu->lock, flags); - - /* Disable IC task */ - ipu_ic_disable_task(ipu, channel); - - /* Disable DMA channel(s) */ - reg = idmac_read_icreg(ipu, IDMAC_CHA_EN); - idmac_write_icreg(ipu, reg & ~chan_mask, IDMAC_CHA_EN); - - spin_unlock_irqrestore(&ipu->lock, flags); - - return 0; -} - -static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan, - struct idmac_tx_desc **desc, struct scatterlist *sg) -{ - struct scatterlist *sgnew = sg ? sg_next(sg) : NULL; - - if (sgnew) - /* next sg-element in this list */ - return sgnew; - - if ((*desc)->list.next == &ichan->queue) - /* No more descriptors on the queue */ - return NULL; - - /* Fetch next descriptor */ - *desc = list_entry((*desc)->list.next, struct idmac_tx_desc, list); - return (*desc)->sg; -} - -/* - * We have several possibilities here: - * current BUF next BUF - * - * not last sg next not last sg - * not last sg next last sg - * last sg first sg from next descriptor - * last sg NULL - * - * Besides, the descriptor queue might be empty or not. We process all these - * cases carefully. - */ -static irqreturn_t idmac_interrupt(int irq, void *dev_id) -{ - struct idmac_channel *ichan = dev_id; - struct device *dev = &ichan->dma_chan.dev->device; - unsigned int chan_id = ichan->dma_chan.chan_id; - struct scatterlist **sg, *sgnext, *sgnew = NULL; - /* Next transfer descriptor */ - struct idmac_tx_desc *desc, *descnew; - bool done = false; - u32 ready0, ready1, curbuf, err; - struct dmaengine_desc_callback cb; - - /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */ - - dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer); - - spin_lock(&ipu_data.lock); - - ready0 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY); - ready1 = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY); - curbuf = idmac_read_ipureg(&ipu_data, IPU_CHA_CUR_BUF); - err = idmac_read_ipureg(&ipu_data, IPU_INT_STAT_4); - - if (err & (1 << chan_id)) { - idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4); - spin_unlock(&ipu_data.lock); - /* - * Doing this - * ichan->sg[0] = ichan->sg[1] = NULL; - * you can force channel re-enable on the next tx_submit(), but - * this is dirty - think about descriptors with multiple - * sg elements. - */ - dev_warn(dev, "NFB4EOF on channel %d, ready %x, %x, cur %x\n", - chan_id, ready0, ready1, curbuf); - return IRQ_HANDLED; - } - spin_unlock(&ipu_data.lock); - - /* Other interrupts do not interfere with this channel */ - spin_lock(&ichan->lock); - if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) || - (!ichan->active_buffer && (ready0 >> chan_id) & 1) - )) { - spin_unlock(&ichan->lock); - dev_dbg(dev, - "IRQ with active buffer still ready on channel %x, " - "active %d, ready %x, %x!\n", chan_id, - ichan->active_buffer, ready0, ready1); - return IRQ_NONE; - } - - if (unlikely(list_empty(&ichan->queue))) { - ichan->sg[ichan->active_buffer] = NULL; - spin_unlock(&ichan->lock); - dev_err(dev, - "IRQ without queued buffers on channel %x, active %d, " - "ready %x, %x!\n", chan_id, - ichan->active_buffer, ready0, ready1); - return IRQ_NONE; - } - - /* - * active_buffer is a software flag, it shows which buffer we are - * currently expecting back from the hardware, IDMAC should be - * processing the other buffer already - */ - sg = &ichan->sg[ichan->active_buffer]; - sgnext = ichan->sg[!ichan->active_buffer]; - - if (!*sg) { - spin_unlock(&ichan->lock); - return IRQ_HANDLED; - } - - desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list); - descnew = desc; - - dev_dbg(dev, "IDMAC irq %d, dma %#llx, next dma %#llx, current %d, curbuf %#x\n", - irq, (u64)sg_dma_address(*sg), - sgnext ? (u64)sg_dma_address(sgnext) : 0, - ichan->active_buffer, curbuf); - - /* Find the descriptor of sgnext */ - sgnew = idmac_sg_next(ichan, &descnew, *sg); - if (sgnext != sgnew) - dev_err(dev, "Submitted buffer %p, next buffer %p\n", sgnext, sgnew); - - /* - * if sgnext == NULL sg must be the last element in a scatterlist and - * queue must be empty - */ - if (unlikely(!sgnext)) { - if (!WARN_ON(sg_next(*sg))) - dev_dbg(dev, "Underrun on channel %x\n", chan_id); - ichan->sg[!ichan->active_buffer] = sgnew; - - if (unlikely(sgnew)) { - ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer); - } else { - spin_lock(&ipu_data.lock); - ipu_ic_disable_task(&ipu_data, chan_id); - spin_unlock(&ipu_data.lock); - ichan->status = IPU_CHANNEL_READY; - /* Continue to check for complete descriptor */ - } - } - - /* Calculate and submit the next sg element */ - sgnew = idmac_sg_next(ichan, &descnew, sgnew); - - if (unlikely(!sg_next(*sg)) || !sgnext) { - /* - * Last element in scatterlist done, remove from the queue, - * _init for debugging - */ - list_del_init(&desc->list); - done = true; - } - - *sg = sgnew; - - if (likely(sgnew) && - ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) { - dmaengine_desc_get_callback(&descnew->txd, &cb); - - list_del_init(&descnew->list); - spin_unlock(&ichan->lock); - - dmaengine_desc_callback_invoke(&cb, NULL); - spin_lock(&ichan->lock); - } - - /* Flip the active buffer - even if update above failed */ - ichan->active_buffer = !ichan->active_buffer; - if (done) - dma_cookie_complete(&desc->txd); - - dmaengine_desc_get_callback(&desc->txd, &cb); - - spin_unlock(&ichan->lock); - - if (done && (desc->txd.flags & DMA_PREP_INTERRUPT)) - dmaengine_desc_callback_invoke(&cb, NULL); - - return IRQ_HANDLED; -} - -static void ipu_gc_tasklet(struct tasklet_struct *t) -{ - struct ipu *ipu = from_tasklet(ipu, t, tasklet); - int i; - - for (i = 0; i < IPU_CHANNELS_NUM; i++) { - struct idmac_channel *ichan = ipu->channel + i; - struct idmac_tx_desc *desc; - unsigned long flags; - struct scatterlist *sg; - int j, k; - - for (j = 0; j < ichan->n_tx_desc; j++) { - desc = ichan->desc + j; - spin_lock_irqsave(&ichan->lock, flags); - if (async_tx_test_ack(&desc->txd)) { - list_move(&desc->list, &ichan->free_list); - for_each_sg(desc->sg, sg, desc->sg_len, k) { - if (ichan->sg[0] == sg) - ichan->sg[0] = NULL; - else if (ichan->sg[1] == sg) - ichan->sg[1] = NULL; - } - async_tx_clear_ack(&desc->txd); - } - spin_unlock_irqrestore(&ichan->lock, flags); - } - } -} - -/* Allocate and initialise a transfer descriptor. */ -static struct dma_async_tx_descriptor *idmac_prep_slave_sg(struct dma_chan *chan, - struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction direction, unsigned long tx_flags, - void *context) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac_tx_desc *desc = NULL; - struct dma_async_tx_descriptor *txd = NULL; - unsigned long flags; - - /* We only can handle these three channels so far */ - if (chan->chan_id != IDMAC_SDC_0 && chan->chan_id != IDMAC_SDC_1 && - chan->chan_id != IDMAC_IC_7) - return NULL; - - if (!is_slave_direction(direction)) { - dev_err(chan->device->dev, "Invalid DMA direction %d!\n", direction); - return NULL; - } - - mutex_lock(&ichan->chan_mutex); - - spin_lock_irqsave(&ichan->lock, flags); - if (!list_empty(&ichan->free_list)) { - desc = list_entry(ichan->free_list.next, - struct idmac_tx_desc, list); - - list_del_init(&desc->list); - - desc->sg_len = sg_len; - desc->sg = sgl; - txd = &desc->txd; - txd->flags = tx_flags; - } - spin_unlock_irqrestore(&ichan->lock, flags); - - mutex_unlock(&ichan->chan_mutex); - - tasklet_schedule(&to_ipu(to_idmac(chan->device))->tasklet); - - return txd; -} - -/* Re-select the current buffer and re-activate the channel */ -static void idmac_issue_pending(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - struct ipu *ipu = to_ipu(idmac); - unsigned long flags; - - /* This is not always needed, but doesn't hurt either */ - spin_lock_irqsave(&ipu->lock, flags); - ipu_select_buffer(chan->chan_id, ichan->active_buffer); - spin_unlock_irqrestore(&ipu->lock, flags); - - /* - * Might need to perform some parts of initialisation from - * ipu_enable_channel(), but not all, we do not want to reset to buffer - * 0, don't need to set priority again either, but re-enabling the task - * and the channel might be a good idea. - */ -} - -static int idmac_pause(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - struct ipu *ipu = to_ipu(idmac); - struct list_head *list, *tmp; - unsigned long flags; - - mutex_lock(&ichan->chan_mutex); - - spin_lock_irqsave(&ipu->lock, flags); - ipu_ic_disable_task(ipu, chan->chan_id); - - /* Return all descriptors into "prepared" state */ - list_for_each_safe(list, tmp, &ichan->queue) - list_del_init(list); - - ichan->sg[0] = NULL; - ichan->sg[1] = NULL; - - spin_unlock_irqrestore(&ipu->lock, flags); - - ichan->status = IPU_CHANNEL_INITIALIZED; - - mutex_unlock(&ichan->chan_mutex); - - return 0; -} - -static int __idmac_terminate_all(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - struct ipu *ipu = to_ipu(idmac); - unsigned long flags; - int i; - - ipu_disable_channel(idmac, ichan, - ichan->status >= IPU_CHANNEL_ENABLED); - - tasklet_disable(&ipu->tasklet); - - /* ichan->queue is modified in ISR, have to spinlock */ - spin_lock_irqsave(&ichan->lock, flags); - list_splice_init(&ichan->queue, &ichan->free_list); - - if (ichan->desc) - for (i = 0; i < ichan->n_tx_desc; i++) { - struct idmac_tx_desc *desc = ichan->desc + i; - if (list_empty(&desc->list)) - /* Descriptor was prepared, but not submitted */ - list_add(&desc->list, &ichan->free_list); - - async_tx_clear_ack(&desc->txd); - } - - ichan->sg[0] = NULL; - ichan->sg[1] = NULL; - spin_unlock_irqrestore(&ichan->lock, flags); - - tasklet_enable(&ipu->tasklet); - - ichan->status = IPU_CHANNEL_INITIALIZED; - - return 0; -} - -static int idmac_terminate_all(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - int ret; - - mutex_lock(&ichan->chan_mutex); - - ret = __idmac_terminate_all(chan); - - mutex_unlock(&ichan->chan_mutex); - - return ret; -} - -#ifdef DEBUG -static irqreturn_t ic_sof_irq(int irq, void *dev_id) -{ - struct idmac_channel *ichan = dev_id; - printk(KERN_DEBUG "Got SOF IRQ %d on Channel %d\n", - irq, ichan->dma_chan.chan_id); - disable_irq_nosync(irq); - return IRQ_HANDLED; -} - -static irqreturn_t ic_eof_irq(int irq, void *dev_id) -{ - struct idmac_channel *ichan = dev_id; - printk(KERN_DEBUG "Got EOF IRQ %d on Channel %d\n", - irq, ichan->dma_chan.chan_id); - disable_irq_nosync(irq); - return IRQ_HANDLED; -} - -static int ic_sof = -EINVAL, ic_eof = -EINVAL; -#endif - -static int idmac_alloc_chan_resources(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - int ret; - - /* dmaengine.c now guarantees to only offer free channels */ - BUG_ON(chan->client_count > 1); - WARN_ON(ichan->status != IPU_CHANNEL_FREE); - - dma_cookie_init(chan); - - ret = ipu_irq_map(chan->chan_id); - if (ret < 0) - goto eimap; - - ichan->eof_irq = ret; - - /* - * Important to first disable the channel, because maybe someone - * used it before us, e.g., the bootloader - */ - ipu_disable_channel(idmac, ichan, true); - - ret = ipu_init_channel(idmac, ichan); - if (ret < 0) - goto eichan; - - ret = request_irq(ichan->eof_irq, idmac_interrupt, 0, - ichan->eof_name, ichan); - if (ret < 0) - goto erirq; - -#ifdef DEBUG - if (chan->chan_id == IDMAC_IC_7) { - ic_sof = ipu_irq_map(69); - if (ic_sof > 0) { - ret = request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan); - if (ret) - dev_err(&chan->dev->device, "request irq failed for IC SOF"); - } - ic_eof = ipu_irq_map(70); - if (ic_eof > 0) { - ret = request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan); - if (ret) - dev_err(&chan->dev->device, "request irq failed for IC EOF"); - } - } -#endif - - ichan->status = IPU_CHANNEL_INITIALIZED; - - dev_dbg(&chan->dev->device, "Found channel 0x%x, irq %d\n", - chan->chan_id, ichan->eof_irq); - - return ret; - -erirq: - ipu_uninit_channel(idmac, ichan); -eichan: - ipu_irq_unmap(chan->chan_id); -eimap: - return ret; -} - -static void idmac_free_chan_resources(struct dma_chan *chan) -{ - struct idmac_channel *ichan = to_idmac_chan(chan); - struct idmac *idmac = to_idmac(chan->device); - - mutex_lock(&ichan->chan_mutex); - - __idmac_terminate_all(chan); - - if (ichan->status > IPU_CHANNEL_FREE) { -#ifdef DEBUG - if (chan->chan_id == IDMAC_IC_7) { - if (ic_sof > 0) { - free_irq(ic_sof, ichan); - ipu_irq_unmap(69); - ic_sof = -EINVAL; - } - if (ic_eof > 0) { - free_irq(ic_eof, ichan); - ipu_irq_unmap(70); - ic_eof = -EINVAL; - } - } -#endif - free_irq(ichan->eof_irq, ichan); - ipu_irq_unmap(chan->chan_id); - } - - ichan->status = IPU_CHANNEL_FREE; - - ipu_uninit_channel(idmac, ichan); - - mutex_unlock(&ichan->chan_mutex); - - tasklet_schedule(&to_ipu(idmac)->tasklet); -} - -static enum dma_status idmac_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, struct dma_tx_state *txstate) -{ - return dma_cookie_status(chan, cookie, txstate); -} - -static int __init ipu_idmac_init(struct ipu *ipu) -{ - struct idmac *idmac = &ipu->idmac; - struct dma_device *dma = &idmac->dma; - int i; - - dma_cap_set(DMA_SLAVE, dma->cap_mask); - dma_cap_set(DMA_PRIVATE, dma->cap_mask); - - /* Compulsory common fields */ - dma->dev = ipu->dev; - dma->device_alloc_chan_resources = idmac_alloc_chan_resources; - dma->device_free_chan_resources = idmac_free_chan_resources; - dma->device_tx_status = idmac_tx_status; - dma->device_issue_pending = idmac_issue_pending; - - /* Compulsory for DMA_SLAVE fields */ - dma->device_prep_slave_sg = idmac_prep_slave_sg; - dma->device_pause = idmac_pause; - dma->device_terminate_all = idmac_terminate_all; - - INIT_LIST_HEAD(&dma->channels); - for (i = 0; i < IPU_CHANNELS_NUM; i++) { - struct idmac_channel *ichan = ipu->channel + i; - struct dma_chan *dma_chan = &ichan->dma_chan; - - spin_lock_init(&ichan->lock); - mutex_init(&ichan->chan_mutex); - - ichan->status = IPU_CHANNEL_FREE; - ichan->sec_chan_en = false; - snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i); - - dma_chan->device = &idmac->dma; - dma_cookie_init(dma_chan); - dma_chan->chan_id = i; - list_add_tail(&dma_chan->device_node, &dma->channels); - } - - idmac_write_icreg(ipu, 0x00000070, IDMAC_CONF); - - return dma_async_device_register(&idmac->dma); -} - -static void ipu_idmac_exit(struct ipu *ipu) -{ - int i; - struct idmac *idmac = &ipu->idmac; - - for (i = 0; i < IPU_CHANNELS_NUM; i++) { - struct idmac_channel *ichan = ipu->channel + i; - - idmac_terminate_all(&ichan->dma_chan); - } - - dma_async_device_unregister(&idmac->dma); -} - -/***************************************************************************** - * IPU common probe / remove - */ - -static int __init ipu_probe(struct platform_device *pdev) -{ - struct resource *mem_ipu, *mem_ic; - int ret; - - spin_lock_init(&ipu_data.lock); - - mem_ipu = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mem_ic = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!mem_ipu || !mem_ic) - return -EINVAL; - - ipu_data.dev = &pdev->dev; - - platform_set_drvdata(pdev, &ipu_data); - - ret = platform_get_irq(pdev, 0); - if (ret < 0) - goto err_noirq; - - ipu_data.irq_fn = ret; - ret = platform_get_irq(pdev, 1); - if (ret < 0) - goto err_noirq; - - ipu_data.irq_err = ret; - - dev_dbg(&pdev->dev, "fn irq %u, err irq %u\n", - ipu_data.irq_fn, ipu_data.irq_err); - - /* Remap IPU common registers */ - ipu_data.reg_ipu = ioremap(mem_ipu->start, resource_size(mem_ipu)); - if (!ipu_data.reg_ipu) { - ret = -ENOMEM; - goto err_ioremap_ipu; - } - - /* Remap Image Converter and Image DMA Controller registers */ - ipu_data.reg_ic = ioremap(mem_ic->start, resource_size(mem_ic)); - if (!ipu_data.reg_ic) { - ret = -ENOMEM; - goto err_ioremap_ic; - } - - /* Get IPU clock */ - ipu_data.ipu_clk = clk_get(&pdev->dev, NULL); - if (IS_ERR(ipu_data.ipu_clk)) { - ret = PTR_ERR(ipu_data.ipu_clk); - goto err_clk_get; - } - - /* Make sure IPU HSP clock is running */ - clk_prepare_enable(ipu_data.ipu_clk); - - /* Disable all interrupts */ - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_1); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_2); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_3); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_4); - idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_5); - - dev_dbg(&pdev->dev, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev->name, - (unsigned long)mem_ipu->start, ipu_data.irq_fn, ipu_data.irq_err); - - ret = ipu_irq_attach_irq(&ipu_data, pdev); - if (ret < 0) - goto err_attach_irq; - - /* Initialize DMA engine */ - ret = ipu_idmac_init(&ipu_data); - if (ret < 0) - goto err_idmac_init; - - tasklet_setup(&ipu_data.tasklet, ipu_gc_tasklet); - - ipu_data.dev = &pdev->dev; - - dev_dbg(ipu_data.dev, "IPU initialized\n"); - - return 0; - -err_idmac_init: -err_attach_irq: - ipu_irq_detach_irq(&ipu_data, pdev); - clk_disable_unprepare(ipu_data.ipu_clk); - clk_put(ipu_data.ipu_clk); -err_clk_get: - iounmap(ipu_data.reg_ic); -err_ioremap_ic: - iounmap(ipu_data.reg_ipu); -err_ioremap_ipu: -err_noirq: - dev_err(&pdev->dev, "Failed to probe IPU: %d\n", ret); - return ret; -} - -static int ipu_remove(struct platform_device *pdev) -{ - struct ipu *ipu = platform_get_drvdata(pdev); - - ipu_idmac_exit(ipu); - ipu_irq_detach_irq(ipu, pdev); - clk_disable_unprepare(ipu->ipu_clk); - clk_put(ipu->ipu_clk); - iounmap(ipu->reg_ic); - iounmap(ipu->reg_ipu); - tasklet_kill(&ipu->tasklet); - - return 0; -} - -/* - * We need two MEM resources - with IPU-common and Image Converter registers, - * including PF_CONF and IDMAC_* registers, and two IRQs - function and error - */ -static struct platform_driver ipu_platform_driver = { - .driver = { - .name = "ipu-core", - }, - .remove = ipu_remove, -}; - -static int __init ipu_init(void) -{ - return platform_driver_probe(&ipu_platform_driver, ipu_probe); -} -subsys_initcall(ipu_init); - -MODULE_DESCRIPTION("IPU core driver"); -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>"); -MODULE_ALIAS("platform:ipu-core"); diff --git a/drivers/dma/ipu/ipu_intern.h b/drivers/dma/ipu/ipu_intern.h deleted file mode 100644 index e7ec1dec3edf..000000000000 --- a/drivers/dma/ipu/ipu_intern.h +++ /dev/null @@ -1,173 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2008 - * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de> - * - * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved. - */ - -#ifndef _IPU_INTERN_H_ -#define _IPU_INTERN_H_ - -#include <linux/dmaengine.h> -#include <linux/platform_device.h> -#include <linux/interrupt.h> - -/* IPU Common registers */ -#define IPU_CONF 0x00 -#define IPU_CHA_BUF0_RDY 0x04 -#define IPU_CHA_BUF1_RDY 0x08 -#define IPU_CHA_DB_MODE_SEL 0x0C -#define IPU_CHA_CUR_BUF 0x10 -#define IPU_FS_PROC_FLOW 0x14 -#define IPU_FS_DISP_FLOW 0x18 -#define IPU_TASKS_STAT 0x1C -#define IPU_IMA_ADDR 0x20 -#define IPU_IMA_DATA 0x24 -#define IPU_INT_CTRL_1 0x28 -#define IPU_INT_CTRL_2 0x2C -#define IPU_INT_CTRL_3 0x30 -#define IPU_INT_CTRL_4 0x34 -#define IPU_INT_CTRL_5 0x38 -#define IPU_INT_STAT_1 0x3C -#define IPU_INT_STAT_2 0x40 -#define IPU_INT_STAT_3 0x44 -#define IPU_INT_STAT_4 0x48 -#define IPU_INT_STAT_5 0x4C -#define IPU_BRK_CTRL_1 0x50 -#define IPU_BRK_CTRL_2 0x54 -#define IPU_BRK_STAT 0x58 -#define IPU_DIAGB_CTRL 0x5C - -/* IPU_CONF Register bits */ -#define IPU_CONF_CSI_EN 0x00000001 -#define IPU_CONF_IC_EN 0x00000002 -#define IPU_CONF_ROT_EN 0x00000004 -#define IPU_CONF_PF_EN 0x00000008 -#define IPU_CONF_SDC_EN 0x00000010 -#define IPU_CONF_ADC_EN 0x00000020 -#define IPU_CONF_DI_EN 0x00000040 -#define IPU_CONF_DU_EN 0x00000080 -#define IPU_CONF_PXL_ENDIAN 0x00000100 - -/* Image Converter Registers */ -#define IC_CONF 0x88 -#define IC_PRP_ENC_RSC 0x8C -#define IC_PRP_VF_RSC 0x90 -#define IC_PP_RSC 0x94 -#define IC_CMBP_1 0x98 -#define IC_CMBP_2 0x9C -#define PF_CONF 0xA0 -#define IDMAC_CONF 0xA4 -#define IDMAC_CHA_EN 0xA8 -#define IDMAC_CHA_PRI 0xAC -#define IDMAC_CHA_BUSY 0xB0 - -/* Image Converter Register bits */ -#define IC_CONF_PRPENC_EN 0x00000001 -#define IC_CONF_PRPENC_CSC1 0x00000002 -#define IC_CONF_PRPENC_ROT_EN 0x00000004 -#define IC_CONF_PRPVF_EN 0x00000100 -#define IC_CONF_PRPVF_CSC1 0x00000200 -#define IC_CONF_PRPVF_CSC2 0x00000400 -#define IC_CONF_PRPVF_CMB 0x00000800 -#define IC_CONF_PRPVF_ROT_EN 0x00001000 -#define IC_CONF_PP_EN 0x00010000 -#define IC_CONF_PP_CSC1 0x00020000 -#define IC_CONF_PP_CSC2 0x00040000 -#define IC_CONF_PP_CMB 0x00080000 -#define IC_CONF_PP_ROT_EN 0x00100000 -#define IC_CONF_IC_GLB_LOC_A 0x10000000 -#define IC_CONF_KEY_COLOR_EN 0x20000000 -#define IC_CONF_RWS_EN 0x40000000 -#define IC_CONF_CSI_MEM_WR_EN 0x80000000 - -#define IDMA_CHAN_INVALID 0x000000FF -#define IDMA_IC_0 0x00000001 -#define IDMA_IC_1 0x00000002 -#define IDMA_IC_2 0x00000004 -#define IDMA_IC_3 0x00000008 -#define IDMA_IC_4 0x00000010 -#define IDMA_IC_5 0x00000020 -#define IDMA_IC_6 0x00000040 -#define IDMA_IC_7 0x00000080 -#define IDMA_IC_8 0x00000100 -#define IDMA_IC_9 0x00000200 -#define IDMA_IC_10 0x00000400 -#define IDMA_IC_11 0x00000800 -#define IDMA_IC_12 0x00001000 -#define IDMA_IC_13 0x00002000 -#define IDMA_SDC_BG 0x00004000 -#define IDMA_SDC_FG 0x00008000 -#define IDMA_SDC_MASK 0x00010000 -#define IDMA_SDC_PARTIAL 0x00020000 -#define IDMA_ADC_SYS1_WR 0x00040000 -#define IDMA_ADC_SYS2_WR 0x00080000 -#define IDMA_ADC_SYS1_CMD 0x00100000 -#define IDMA_ADC_SYS2_CMD 0x00200000 -#define IDMA_ADC_SYS1_RD 0x00400000 -#define IDMA_ADC_SYS2_RD 0x00800000 -#define IDMA_PF_QP 0x01000000 -#define IDMA_PF_BSP 0x02000000 -#define IDMA_PF_Y_IN 0x04000000 -#define IDMA_PF_U_IN 0x08000000 -#define IDMA_PF_V_IN 0x10000000 -#define IDMA_PF_Y_OUT 0x20000000 -#define IDMA_PF_U_OUT 0x40000000 -#define IDMA_PF_V_OUT 0x80000000 - -#define TSTAT_PF_H264_PAUSE 0x00000001 -#define TSTAT_CSI2MEM_MASK 0x0000000C -#define TSTAT_CSI2MEM_OFFSET 2 -#define TSTAT_VF_MASK 0x00000600 -#define TSTAT_VF_OFFSET 9 -#define TSTAT_VF_ROT_MASK 0x000C0000 -#define TSTAT_VF_ROT_OFFSET 18 -#define TSTAT_ENC_MASK 0x00000180 -#define TSTAT_ENC_OFFSET 7 -#define TSTAT_ENC_ROT_MASK 0x00030000 -#define TSTAT_ENC_ROT_OFFSET 16 -#define TSTAT_PP_MASK 0x00001800 -#define TSTAT_PP_OFFSET 11 -#define TSTAT_PP_ROT_MASK 0x00300000 -#define TSTAT_PP_ROT_OFFSET 20 -#define TSTAT_PF_MASK 0x00C00000 -#define TSTAT_PF_OFFSET 22 -#define TSTAT_ADCSYS1_MASK 0x03000000 -#define TSTAT_ADCSYS1_OFFSET 24 -#define TSTAT_ADCSYS2_MASK 0x0C000000 -#define TSTAT_ADCSYS2_OFFSET 26 - -#define TASK_STAT_IDLE 0 -#define TASK_STAT_ACTIVE 1 -#define TASK_STAT_WAIT4READY 2 - -struct idmac { - struct dma_device dma; -}; - -struct ipu { - void __iomem *reg_ipu; - void __iomem *reg_ic; - unsigned int irq_fn; /* IPU Function IRQ to the CPU */ - unsigned int irq_err; /* IPU Error IRQ to the CPU */ - unsigned int irq_base; /* Beginning of the IPU IRQ range */ - unsigned long channel_init_mask; - spinlock_t lock; - struct clk *ipu_clk; - struct device *dev; - struct idmac idmac; - struct idmac_channel channel[IPU_CHANNELS_NUM]; - struct tasklet_struct tasklet; -}; - -#define to_idmac(d) container_of(d, struct idmac, dma) - -extern int ipu_irq_attach_irq(struct ipu *ipu, struct platform_device *dev); -extern void ipu_irq_detach_irq(struct ipu *ipu, struct platform_device *dev); - -extern bool ipu_irq_status(uint32_t irq); -extern int ipu_irq_map(unsigned int source); -extern int ipu_irq_unmap(unsigned int source); - -#endif diff --git a/drivers/dma/ipu/ipu_irq.c b/drivers/dma/ipu/ipu_irq.c deleted file mode 100644 index 97d9a6f04f2a..000000000000 --- a/drivers/dma/ipu/ipu_irq.c +++ /dev/null @@ -1,367 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2008 - * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de> - */ - -#include <linux/init.h> -#include <linux/err.h> -#include <linux/spinlock.h> -#include <linux/delay.h> -#include <linux/clk.h> -#include <linux/irq.h> -#include <linux/io.h> -#include <linux/module.h> -#include <linux/dma/ipu-dma.h> - -#include "ipu_intern.h" - -/* - * Register read / write - shall be inlined by the compiler - */ -static u32 ipu_read_reg(struct ipu *ipu, unsigned long reg) -{ - return __raw_readl(ipu->reg_ipu + reg); -} - -static void ipu_write_reg(struct ipu *ipu, u32 value, unsigned long reg) -{ - __raw_writel(value, ipu->reg_ipu + reg); -} - - -/* - * IPU IRQ chip driver - */ - -#define IPU_IRQ_NR_FN_BANKS 3 -#define IPU_IRQ_NR_ERR_BANKS 2 -#define IPU_IRQ_NR_BANKS (IPU_IRQ_NR_FN_BANKS + IPU_IRQ_NR_ERR_BANKS) - -struct ipu_irq_bank { - unsigned int control; - unsigned int status; - struct ipu *ipu; -}; - -static struct ipu_irq_bank irq_bank[IPU_IRQ_NR_BANKS] = { - /* 3 groups of functional interrupts */ - { - .control = IPU_INT_CTRL_1, - .status = IPU_INT_STAT_1, - }, { - .control = IPU_INT_CTRL_2, - .status = IPU_INT_STAT_2, - }, { - .control = IPU_INT_CTRL_3, - .status = IPU_INT_STAT_3, - }, - /* 2 groups of error interrupts */ - { - .control = IPU_INT_CTRL_4, - .status = IPU_INT_STAT_4, - }, { - .control = IPU_INT_CTRL_5, - .status = IPU_INT_STAT_5, - }, -}; - -struct ipu_irq_map { - unsigned int irq; - int source; - struct ipu_irq_bank *bank; - struct ipu *ipu; -}; - -static struct ipu_irq_map irq_map[CONFIG_MX3_IPU_IRQS]; -/* Protects allocations from the above array of maps */ -static DEFINE_MUTEX(map_lock); -/* Protects register accesses and individual mappings */ -static DEFINE_RAW_SPINLOCK(bank_lock); - -static struct ipu_irq_map *src2map(unsigned int src) -{ - int i; - - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) - if (irq_map[i].source == src) - return irq_map + i; - - return NULL; -} - -static void ipu_irq_unmask(struct irq_data *d) -{ - struct ipu_irq_map *map = irq_data_get_irq_chip_data(d); - struct ipu_irq_bank *bank; - uint32_t reg; - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - - bank = map->bank; - if (!bank) { - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - pr_err("IPU: %s(%u) - unmapped!\n", __func__, d->irq); - return; - } - - reg = ipu_read_reg(bank->ipu, bank->control); - reg |= (1UL << (map->source & 31)); - ipu_write_reg(bank->ipu, reg, bank->control); - - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); -} - -static void ipu_irq_mask(struct irq_data *d) -{ - struct ipu_irq_map *map = irq_data_get_irq_chip_data(d); - struct ipu_irq_bank *bank; - uint32_t reg; - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - - bank = map->bank; - if (!bank) { - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - pr_err("IPU: %s(%u) - unmapped!\n", __func__, d->irq); - return; - } - - reg = ipu_read_reg(bank->ipu, bank->control); - reg &= ~(1UL << (map->source & 31)); - ipu_write_reg(bank->ipu, reg, bank->control); - - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); -} - -static void ipu_irq_ack(struct irq_data *d) -{ - struct ipu_irq_map *map = irq_data_get_irq_chip_data(d); - struct ipu_irq_bank *bank; - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - - bank = map->bank; - if (!bank) { - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - pr_err("IPU: %s(%u) - unmapped!\n", __func__, d->irq); - return; - } - - ipu_write_reg(bank->ipu, 1UL << (map->source & 31), bank->status); - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); -} - -/** - * ipu_irq_status() - returns the current interrupt status of the specified IRQ. - * @irq: interrupt line to get status for. - * @return: true if the interrupt is pending/asserted or false if the - * interrupt is not pending. - */ -bool ipu_irq_status(unsigned int irq) -{ - struct ipu_irq_map *map = irq_get_chip_data(irq); - struct ipu_irq_bank *bank; - unsigned long lock_flags; - bool ret; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - bank = map->bank; - ret = bank && ipu_read_reg(bank->ipu, bank->status) & - (1UL << (map->source & 31)); - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - - return ret; -} - -/** - * ipu_irq_map() - map an IPU interrupt source to an IRQ number - * @source: interrupt source bit position (see below) - * @return: mapped IRQ number or negative error code - * - * The source parameter has to be explained further. On i.MX31 IPU has 137 IRQ - * sources, they are broken down in 5 32-bit registers, like 32, 32, 24, 32, 17. - * However, the source argument of this function is not the sequence number of - * the possible IRQ, but rather its bit position. So, first interrupt in fourth - * register has source number 96, and not 88. This makes calculations easier, - * and also provides forward compatibility with any future IPU implementations - * with any interrupt bit assignments. - */ -int ipu_irq_map(unsigned int source) -{ - int i, ret = -ENOMEM; - struct ipu_irq_map *map; - - might_sleep(); - - mutex_lock(&map_lock); - map = src2map(source); - if (map) { - pr_err("IPU: Source %u already mapped to IRQ %u\n", source, map->irq); - ret = -EBUSY; - goto out; - } - - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) { - if (irq_map[i].source < 0) { - unsigned long lock_flags; - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - irq_map[i].source = source; - irq_map[i].bank = irq_bank + source / 32; - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - - ret = irq_map[i].irq; - pr_debug("IPU: mapped source %u to IRQ %u\n", - source, ret); - break; - } - } -out: - mutex_unlock(&map_lock); - - if (ret < 0) - pr_err("IPU: couldn't map source %u: %d\n", source, ret); - - return ret; -} - -/** - * ipu_irq_unmap() - unmap an IPU interrupt source - * @source: interrupt source bit position (see ipu_irq_map()) - * @return: 0 or negative error code - */ -int ipu_irq_unmap(unsigned int source) -{ - int i, ret = -EINVAL; - - might_sleep(); - - mutex_lock(&map_lock); - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) { - if (irq_map[i].source == source) { - unsigned long lock_flags; - - pr_debug("IPU: unmapped source %u from IRQ %u\n", - source, irq_map[i].irq); - - raw_spin_lock_irqsave(&bank_lock, lock_flags); - irq_map[i].source = -EINVAL; - irq_map[i].bank = NULL; - raw_spin_unlock_irqrestore(&bank_lock, lock_flags); - - ret = 0; - break; - } - } - mutex_unlock(&map_lock); - - return ret; -} - -/* Chained IRQ handler for IPU function and error interrupt */ -static void ipu_irq_handler(struct irq_desc *desc) -{ - struct ipu *ipu = irq_desc_get_handler_data(desc); - u32 status; - int i, line; - - for (i = 0; i < IPU_IRQ_NR_BANKS; i++) { - struct ipu_irq_bank *bank = irq_bank + i; - - raw_spin_lock(&bank_lock); - status = ipu_read_reg(ipu, bank->status); - /* - * Don't think we have to clear all interrupts here, they will - * be acked by ->handle_irq() (handle_level_irq). However, we - * might want to clear unhandled interrupts after the loop... - */ - status &= ipu_read_reg(ipu, bank->control); - raw_spin_unlock(&bank_lock); - while ((line = ffs(status))) { - struct ipu_irq_map *map; - unsigned int irq; - - line--; - status &= ~(1UL << line); - - raw_spin_lock(&bank_lock); - map = src2map(32 * i + line); - if (!map) { - raw_spin_unlock(&bank_lock); - pr_err("IPU: Interrupt on unmapped source %u bank %d\n", - line, i); - continue; - } - irq = map->irq; - raw_spin_unlock(&bank_lock); - generic_handle_irq(irq); - } - } -} - -static struct irq_chip ipu_irq_chip = { - .name = "ipu_irq", - .irq_ack = ipu_irq_ack, - .irq_mask = ipu_irq_mask, - .irq_unmask = ipu_irq_unmask, -}; - -/* Install the IRQ handler */ -int __init ipu_irq_attach_irq(struct ipu *ipu, struct platform_device *dev) -{ - unsigned int irq, i; - int irq_base = irq_alloc_descs(-1, 0, CONFIG_MX3_IPU_IRQS, - numa_node_id()); - - if (irq_base < 0) - return irq_base; - - for (i = 0; i < IPU_IRQ_NR_BANKS; i++) - irq_bank[i].ipu = ipu; - - for (i = 0; i < CONFIG_MX3_IPU_IRQS; i++) { - int ret; - - irq = irq_base + i; - ret = irq_set_chip(irq, &ipu_irq_chip); - if (ret < 0) - return ret; - ret = irq_set_chip_data(irq, irq_map + i); - if (ret < 0) - return ret; - irq_map[i].ipu = ipu; - irq_map[i].irq = irq; - irq_map[i].source = -EINVAL; - irq_set_handler(irq, handle_level_irq); - irq_clear_status_flags(irq, IRQ_NOREQUEST | IRQ_NOPROBE); - } - - irq_set_chained_handler_and_data(ipu->irq_fn, ipu_irq_handler, ipu); - - irq_set_chained_handler_and_data(ipu->irq_err, ipu_irq_handler, ipu); - - ipu->irq_base = irq_base; - - return 0; -} - -void ipu_irq_detach_irq(struct ipu *ipu, struct platform_device *dev) -{ - unsigned int irq, irq_base; - - irq_base = ipu->irq_base; - - irq_set_chained_handler_and_data(ipu->irq_fn, NULL, NULL); - - irq_set_chained_handler_and_data(ipu->irq_err, NULL, NULL); - - for (irq = irq_base; irq < irq_base + CONFIG_MX3_IPU_IRQS; irq++) { - irq_set_status_flags(irq, IRQ_NOREQUEST); - irq_set_chip(irq, NULL); - irq_set_chip_data(irq, NULL); - } -} diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index ecdaada95120..0f9cd7815f88 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -15,7 +15,6 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/of_device.h> #include <linux/of.h> #include <linux/clk.h> #include <linux/of_dma.h> @@ -839,7 +838,6 @@ static int k3_dma_probe(struct platform_device *op) { const struct k3dma_soc_data *soc_data; struct k3_dma_dev *d; - const struct of_device_id *of_id; int i, ret, irq = 0; d = devm_kzalloc(&op->dev, sizeof(*d), GFP_KERNEL); @@ -854,19 +852,16 @@ static int k3_dma_probe(struct platform_device *op) if (IS_ERR(d->base)) return PTR_ERR(d->base); - of_id = of_match_device(k3_pdma_dt_ids, &op->dev); - if (of_id) { - of_property_read_u32((&op->dev)->of_node, - "dma-channels", &d->dma_channels); - of_property_read_u32((&op->dev)->of_node, - "dma-requests", &d->dma_requests); - ret = of_property_read_u32((&op->dev)->of_node, - "dma-channel-mask", &d->dma_channel_mask); - if (ret) { - dev_warn(&op->dev, - "dma-channel-mask doesn't exist, considering all as available.\n"); - d->dma_channel_mask = (u32)~0UL; - } + of_property_read_u32((&op->dev)->of_node, + "dma-channels", &d->dma_channels); + of_property_read_u32((&op->dev)->of_node, + "dma-requests", &d->dma_requests); + ret = of_property_read_u32((&op->dev)->of_node, + "dma-channel-mask", &d->dma_channel_mask); + if (ret) { + dev_warn(&op->dev, + "dma-channel-mask doesn't exist, considering all as available.\n"); + d->dma_channel_mask = (u32)~0UL; } if (!(soc_data->flags & K3_FLAG_NOCLK)) { @@ -974,7 +969,7 @@ dma_async_register_fail: return ret; } -static int k3_dma_remove(struct platform_device *op) +static void k3_dma_remove(struct platform_device *op) { struct k3_dma_chan *c, *cn; struct k3_dma_dev *d = platform_get_drvdata(op); @@ -990,7 +985,6 @@ static int k3_dma_remove(struct platform_device *op) } tasklet_kill(&d->task); clk_disable_unprepare(d->clk); - return 0; } #ifdef CONFIG_PM_SLEEP @@ -1040,5 +1034,4 @@ static struct platform_driver k3_pdma_driver = { module_platform_driver(k3_pdma_driver); MODULE_DESCRIPTION("HiSilicon k3 DMA Driver"); -MODULE_ALIAS("platform:k3dma"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/lgm/lgm-dma.c b/drivers/dma/lgm/lgm-dma.c index efe8bd3a0e2a..8173c3f1075a 100644 --- a/drivers/dma/lgm/lgm-dma.c +++ b/drivers/dma/lgm/lgm-dma.c @@ -107,7 +107,7 @@ * If header mode is set in DMA descriptor, * If bit 30 is disabled, HDR_LEN must be configured according to channel * requirement. - * If bit 30 is enabled(checksum with heade mode), HDR_LEN has no need to + * If bit 30 is enabled(checksum with header mode), HDR_LEN has no need to * be configured. It will enable check sum for switch * If header mode is not set in DMA descriptor, * This register setting doesn't matter @@ -914,7 +914,7 @@ static void ldma_dev_init(struct ldma_dev *d) } } -static int ldma_cfg_init(struct ldma_dev *d) +static int ldma_parse_dt(struct ldma_dev *d) { struct fwnode_handle *fwnode = dev_fwnode(d->dev); struct ldma_port *p; @@ -1593,11 +1593,12 @@ static int intel_ldma_probe(struct platform_device *pdev) d->core_clk = devm_clk_get_optional(dev, NULL); if (IS_ERR(d->core_clk)) return PTR_ERR(d->core_clk); - clk_prepare_enable(d->core_clk); d->rst = devm_reset_control_get_optional(dev, NULL); if (IS_ERR(d->rst)) return PTR_ERR(d->rst); + + clk_prepare_enable(d->core_clk); reset_control_deassert(d->rst); ret = devm_add_action_or_reset(dev, ldma_clk_disable, d); @@ -1660,10 +1661,6 @@ static int intel_ldma_probe(struct platform_device *pdev) p->ldev = d; } - ret = ldma_cfg_init(d); - if (ret) - return ret; - dma_dev->dev = &pdev->dev; ch_mask = (unsigned long)d->channels_mask; @@ -1674,6 +1671,10 @@ static int intel_ldma_probe(struct platform_device *pdev) ldma_dma_init_v3X(j, d); } + ret = ldma_parse_dt(d); + if (ret) + return ret; + dma_dev->device_alloc_chan_resources = ldma_alloc_chan_resources; dma_dev->device_free_chan_resources = ldma_free_chan_resources; dma_dev->device_terminate_all = ldma_terminate_all; @@ -1731,9 +1732,4 @@ static struct platform_driver intel_ldma_driver = { * registered DMA channels and DMA capabilities to clients before their * initialization. */ -static int __init intel_ldma_init(void) -{ - return platform_driver_register(&intel_ldma_driver); -} - -device_initcall(intel_ldma_init); +builtin_platform_driver(intel_ldma_driver); diff --git a/drivers/dma/loongson1-apb-dma.c b/drivers/dma/loongson1-apb-dma.c new file mode 100644 index 000000000000..255fe7eca212 --- /dev/null +++ b/drivers/dma/loongson1-apb-dma.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for Loongson-1 APB DMA Controller + * + * Copyright (C) 2015-2024 Keguang Zhang <keguang.zhang@gmail.com> + */ + +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +/* Loongson-1 DMA Control Register */ +#define LS1X_DMA_CTRL 0x0 + +/* DMA Control Register Bits */ +#define LS1X_DMA_STOP BIT(4) +#define LS1X_DMA_START BIT(3) +#define LS1X_DMA_ASK_VALID BIT(2) + +/* DMA Next Field Bits */ +#define LS1X_DMA_NEXT_VALID BIT(0) + +/* DMA Command Field Bits */ +#define LS1X_DMA_RAM2DEV BIT(12) +#define LS1X_DMA_INT BIT(1) +#define LS1X_DMA_INT_MASK BIT(0) + +#define LS1X_DMA_LLI_ALIGNMENT 64 +#define LS1X_DMA_LLI_ADDR_MASK GENMASK(31, __ffs(LS1X_DMA_LLI_ALIGNMENT)) +#define LS1X_DMA_MAX_CHANNELS 3 + +enum ls1x_dmadesc_offsets { + LS1X_DMADESC_NEXT = 0, + LS1X_DMADESC_SADDR, + LS1X_DMADESC_DADDR, + LS1X_DMADESC_LENGTH, + LS1X_DMADESC_STRIDE, + LS1X_DMADESC_CYCLES, + LS1X_DMADESC_CMD, + LS1X_DMADESC_SIZE +}; + +struct ls1x_dma_lli { + unsigned int hw[LS1X_DMADESC_SIZE]; + dma_addr_t phys; + struct list_head node; +} __aligned(LS1X_DMA_LLI_ALIGNMENT); + +struct ls1x_dma_desc { + struct virt_dma_desc vd; + struct list_head lli_list; +}; + +struct ls1x_dma_chan { + struct virt_dma_chan vc; + struct dma_pool *lli_pool; + phys_addr_t src_addr; + phys_addr_t dst_addr; + enum dma_slave_buswidth src_addr_width; + enum dma_slave_buswidth dst_addr_width; + unsigned int bus_width; + void __iomem *reg_base; + int irq; + bool is_cyclic; + struct ls1x_dma_lli *curr_lli; +}; + +struct ls1x_dma { + struct dma_device ddev; + unsigned int nr_chans; + struct ls1x_dma_chan chan[]; +}; + +static irqreturn_t ls1x_dma_irq_handler(int irq, void *data); + +#define to_ls1x_dma_chan(dchan) \ + container_of(dchan, struct ls1x_dma_chan, vc.chan) + +#define to_ls1x_dma_desc(d) \ + container_of(d, struct ls1x_dma_desc, vd) + +static inline struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static inline int ls1x_dma_query(struct ls1x_dma_chan *chan, + dma_addr_t *lli_phys) +{ + struct dma_chan *dchan = &chan->vc.chan; + int val, ret; + + val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK; + val |= LS1X_DMA_ASK_VALID; + val |= dchan->chan_id; + writel(val, chan->reg_base + LS1X_DMA_CTRL); + ret = readl_poll_timeout_atomic(chan->reg_base + LS1X_DMA_CTRL, val, + !(val & LS1X_DMA_ASK_VALID), 0, 3000); + if (ret) + dev_err(chan2dev(dchan), "failed to query DMA\n"); + + return ret; +} + +static inline int ls1x_dma_start(struct ls1x_dma_chan *chan, + dma_addr_t *lli_phys) +{ + struct dma_chan *dchan = &chan->vc.chan; + struct device *dev = chan2dev(dchan); + int val, ret; + + val = *lli_phys & LS1X_DMA_LLI_ADDR_MASK; + val |= LS1X_DMA_START; + val |= dchan->chan_id; + writel(val, chan->reg_base + LS1X_DMA_CTRL); + ret = readl_poll_timeout(chan->reg_base + LS1X_DMA_CTRL, val, + !(val & LS1X_DMA_START), 0, 1000); + if (!ret) + dev_dbg(dev, "start DMA with lli_phys=%pad\n", lli_phys); + else + dev_err(dev, "failed to start DMA\n"); + + return ret; +} + +static inline void ls1x_dma_stop(struct ls1x_dma_chan *chan) +{ + int val = readl(chan->reg_base + LS1X_DMA_CTRL); + + writel(val | LS1X_DMA_STOP, chan->reg_base + LS1X_DMA_CTRL); +} + +static void ls1x_dma_free_chan_resources(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct device *dev = chan2dev(dchan); + + dma_free_coherent(dev, sizeof(struct ls1x_dma_lli), + chan->curr_lli, chan->curr_lli->phys); + dma_pool_destroy(chan->lli_pool); + chan->lli_pool = NULL; + devm_free_irq(dev, chan->irq, chan); + vchan_free_chan_resources(&chan->vc); +} + +static int ls1x_dma_alloc_chan_resources(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct device *dev = chan2dev(dchan); + dma_addr_t phys; + int ret; + + ret = devm_request_irq(dev, chan->irq, ls1x_dma_irq_handler, + IRQF_SHARED, dma_chan_name(dchan), chan); + if (ret) { + dev_err(dev, "failed to request IRQ %d\n", chan->irq); + return ret; + } + + chan->lli_pool = dma_pool_create(dma_chan_name(dchan), dev, + sizeof(struct ls1x_dma_lli), + __alignof__(struct ls1x_dma_lli), 0); + if (!chan->lli_pool) + return -ENOMEM; + + /* allocate memory for querying the current lli */ + dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + chan->curr_lli = dma_alloc_coherent(dev, sizeof(struct ls1x_dma_lli), + &phys, GFP_KERNEL); + if (!chan->curr_lli) { + dma_pool_destroy(chan->lli_pool); + return -ENOMEM; + } + chan->curr_lli->phys = phys; + + return 0; +} + +static void ls1x_dma_free_desc(struct virt_dma_desc *vd) +{ + struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd); + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(vd->tx.chan); + struct ls1x_dma_lli *lli, *_lli; + + list_for_each_entry_safe(lli, _lli, &desc->lli_list, node) { + list_del(&lli->node); + dma_pool_free(chan->lli_pool, lli, lli->phys); + } + + kfree(desc); +} + +static struct ls1x_dma_desc *ls1x_dma_alloc_desc(void) +{ + struct ls1x_dma_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_NOWAIT); + if (!desc) + return NULL; + + INIT_LIST_HEAD(&desc->lli_list); + + return desc; +} + +static int ls1x_dma_prep_lli(struct dma_chan *dchan, struct ls1x_dma_desc *desc, + struct scatterlist *sgl, unsigned int sg_len, + enum dma_transfer_direction dir, bool is_cyclic) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct ls1x_dma_lli *lli, *prev = NULL, *first = NULL; + struct device *dev = chan2dev(dchan); + struct list_head *pos = NULL; + struct scatterlist *sg; + unsigned int dev_addr, cmd, i; + + switch (dir) { + case DMA_MEM_TO_DEV: + dev_addr = chan->dst_addr; + chan->bus_width = chan->dst_addr_width; + cmd = LS1X_DMA_RAM2DEV | LS1X_DMA_INT; + break; + case DMA_DEV_TO_MEM: + dev_addr = chan->src_addr; + chan->bus_width = chan->src_addr_width; + cmd = LS1X_DMA_INT; + break; + default: + dev_err(dev, "unsupported DMA direction: %s\n", + dmaengine_get_direction_text(dir)); + return -EINVAL; + } + + for_each_sg(sgl, sg, sg_len, i) { + dma_addr_t buf_addr = sg_dma_address(sg); + size_t buf_len = sg_dma_len(sg); + dma_addr_t phys; + + if (!is_dma_copy_aligned(dchan->device, buf_addr, 0, buf_len)) { + dev_err(dev, "buffer is not aligned\n"); + return -EINVAL; + } + + /* allocate HW descriptors */ + lli = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, &phys); + if (!lli) { + dev_err(dev, "failed to alloc lli %u\n", i); + return -ENOMEM; + } + + /* setup HW descriptors */ + lli->phys = phys; + lli->hw[LS1X_DMADESC_SADDR] = buf_addr; + lli->hw[LS1X_DMADESC_DADDR] = dev_addr; + lli->hw[LS1X_DMADESC_LENGTH] = buf_len / chan->bus_width; + lli->hw[LS1X_DMADESC_STRIDE] = 0; + lli->hw[LS1X_DMADESC_CYCLES] = 1; + lli->hw[LS1X_DMADESC_CMD] = cmd; + + if (prev) + prev->hw[LS1X_DMADESC_NEXT] = + lli->phys | LS1X_DMA_NEXT_VALID; + prev = lli; + + if (!first) + first = lli; + + list_add_tail(&lli->node, &desc->lli_list); + } + + if (is_cyclic) { + lli->hw[LS1X_DMADESC_NEXT] = first->phys | LS1X_DMA_NEXT_VALID; + chan->is_cyclic = is_cyclic; + } + + list_for_each(pos, &desc->lli_list) { + lli = list_entry(pos, struct ls1x_dma_lli, node); + print_hex_dump_debug("LLI: ", DUMP_PREFIX_OFFSET, 16, 4, + lli, sizeof(*lli), false); + } + + return 0; +} + +static struct dma_async_tx_descriptor * +ls1x_dma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct ls1x_dma_desc *desc; + + dev_dbg(chan2dev(dchan), "sg_len=%u flags=0x%lx dir=%s\n", + sg_len, flags, dmaengine_get_direction_text(dir)); + + desc = ls1x_dma_alloc_desc(); + if (!desc) + return NULL; + + if (ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, false)) { + ls1x_dma_free_desc(&desc->vd); + return NULL; + } + + return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags); +} + +static struct dma_async_tx_descriptor * +ls1x_dma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction dir, unsigned long flags) +{ + struct ls1x_dma_desc *desc; + struct scatterlist *sgl; + unsigned int sg_len; + unsigned int i; + int ret; + + dev_dbg(chan2dev(dchan), + "buf_len=%zu period_len=%zu flags=0x%lx dir=%s\n", + buf_len, period_len, flags, dmaengine_get_direction_text(dir)); + + desc = ls1x_dma_alloc_desc(); + if (!desc) + return NULL; + + /* allocate the scatterlist */ + sg_len = buf_len / period_len; + sgl = kmalloc_array(sg_len, sizeof(*sgl), GFP_NOWAIT); + if (!sgl) + return NULL; + + sg_init_table(sgl, sg_len); + for (i = 0; i < sg_len; ++i) { + sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(buf_addr)), + period_len, offset_in_page(buf_addr)); + sg_dma_address(&sgl[i]) = buf_addr; + sg_dma_len(&sgl[i]) = period_len; + buf_addr += period_len; + } + + ret = ls1x_dma_prep_lli(dchan, desc, sgl, sg_len, dir, true); + kfree(sgl); + if (ret) { + ls1x_dma_free_desc(&desc->vd); + return NULL; + } + + return vchan_tx_prep(to_virt_chan(dchan), &desc->vd, flags); +} + +static int ls1x_dma_slave_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + + chan->src_addr = config->src_addr; + chan->src_addr_width = config->src_addr_width; + chan->dst_addr = config->dst_addr; + chan->dst_addr_width = config->dst_addr_width; + + return 0; +} + +static int ls1x_dma_pause(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + int ret; + + guard(spinlock_irqsave)(&chan->vc.lock); + /* save the current lli */ + ret = ls1x_dma_query(chan, &chan->curr_lli->phys); + if (!ret) + ls1x_dma_stop(chan); + + return ret; +} + +static int ls1x_dma_resume(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + + guard(spinlock_irqsave)(&chan->vc.lock); + + return ls1x_dma_start(chan, &chan->curr_lli->phys); +} + +static int ls1x_dma_terminate_all(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct virt_dma_desc *vd; + LIST_HEAD(head); + + ls1x_dma_stop(chan); + + scoped_guard(spinlock_irqsave, &chan->vc.lock) { + vd = vchan_next_desc(&chan->vc); + if (vd) + vchan_terminate_vdesc(vd); + + vchan_get_all_descriptors(&chan->vc, &head); + } + + vchan_dma_desc_free_list(&chan->vc, &head); + + return 0; +} + +static void ls1x_dma_synchronize(struct dma_chan *dchan) +{ + vchan_synchronize(to_virt_chan(dchan)); +} + +static enum dma_status ls1x_dma_tx_status(struct dma_chan *dchan, + dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + struct virt_dma_desc *vd; + enum dma_status status; + size_t bytes = 0; + + status = dma_cookie_status(dchan, cookie, state); + if (status == DMA_COMPLETE) + return status; + + scoped_guard(spinlock_irqsave, &chan->vc.lock) { + vd = vchan_find_desc(&chan->vc, cookie); + if (vd) { + struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd); + struct ls1x_dma_lli *lli; + dma_addr_t next_phys; + + /* get the current lli */ + if (ls1x_dma_query(chan, &chan->curr_lli->phys)) + return status; + + /* locate the current lli */ + next_phys = chan->curr_lli->hw[LS1X_DMADESC_NEXT]; + list_for_each_entry(lli, &desc->lli_list, node) + if (lli->hw[LS1X_DMADESC_NEXT] == next_phys) + break; + + dev_dbg(chan2dev(dchan), "current lli_phys=%pad", + &lli->phys); + + /* count the residues */ + list_for_each_entry_from(lli, &desc->lli_list, node) + bytes += lli->hw[LS1X_DMADESC_LENGTH] * + chan->bus_width; + } + } + + dma_set_residue(state, bytes); + + return status; +} + +static void ls1x_dma_issue_pending(struct dma_chan *dchan) +{ + struct ls1x_dma_chan *chan = to_ls1x_dma_chan(dchan); + + guard(spinlock_irqsave)(&chan->vc.lock); + + if (vchan_issue_pending(&chan->vc)) { + struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); + + if (vd) { + struct ls1x_dma_desc *desc = to_ls1x_dma_desc(vd); + struct ls1x_dma_lli *lli; + + lli = list_first_entry(&desc->lli_list, + struct ls1x_dma_lli, node); + ls1x_dma_start(chan, &lli->phys); + } + } +} + +static irqreturn_t ls1x_dma_irq_handler(int irq, void *data) +{ + struct ls1x_dma_chan *chan = data; + struct dma_chan *dchan = &chan->vc.chan; + struct device *dev = chan2dev(dchan); + struct virt_dma_desc *vd; + + scoped_guard(spinlock, &chan->vc.lock) { + vd = vchan_next_desc(&chan->vc); + if (!vd) { + dev_warn(dev, + "IRQ %d with no active desc on channel %d\n", + irq, dchan->chan_id); + return IRQ_NONE; + } + + if (chan->is_cyclic) { + vchan_cyclic_callback(vd); + } else { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + } + + dev_dbg(dev, "DMA IRQ %d on channel %d\n", irq, dchan->chan_id); + + return IRQ_HANDLED; +} + +static int ls1x_dma_chan_probe(struct platform_device *pdev, + struct ls1x_dma *dma) +{ + void __iomem *reg_base; + int id; + + reg_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(reg_base)) + return PTR_ERR(reg_base); + + for (id = 0; id < dma->nr_chans; id++) { + struct ls1x_dma_chan *chan = &dma->chan[id]; + char pdev_irqname[16]; + + snprintf(pdev_irqname, sizeof(pdev_irqname), "ch%d", id); + chan->irq = platform_get_irq_byname(pdev, pdev_irqname); + if (chan->irq < 0) + return dev_err_probe(&pdev->dev, chan->irq, + "failed to get IRQ for ch%d\n", + id); + + chan->reg_base = reg_base; + chan->vc.desc_free = ls1x_dma_free_desc; + vchan_init(&chan->vc, &dma->ddev); + } + + return 0; +} + +static void ls1x_dma_chan_remove(struct ls1x_dma *dma) +{ + int id; + + for (id = 0; id < dma->nr_chans; id++) { + struct ls1x_dma_chan *chan = &dma->chan[id]; + + if (chan->vc.chan.device == &dma->ddev) { + list_del(&chan->vc.chan.device_node); + tasklet_kill(&chan->vc.task); + } + } +} + +static int ls1x_dma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dma_device *ddev; + struct ls1x_dma *dma; + int ret; + + ret = platform_irq_count(pdev); + if (ret <= 0 || ret > LS1X_DMA_MAX_CHANNELS) + return dev_err_probe(dev, -EINVAL, + "Invalid number of IRQ channels: %d\n", + ret); + + dma = devm_kzalloc(dev, struct_size(dma, chan, ret), GFP_KERNEL); + if (!dma) + return -ENOMEM; + dma->nr_chans = ret; + + /* initialize DMA device */ + ddev = &dma->ddev; + ddev->dev = dev; + ddev->copy_align = DMAENGINE_ALIGN_4_BYTES; + ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + ddev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + ddev->device_alloc_chan_resources = ls1x_dma_alloc_chan_resources; + ddev->device_free_chan_resources = ls1x_dma_free_chan_resources; + ddev->device_prep_slave_sg = ls1x_dma_prep_slave_sg; + ddev->device_prep_dma_cyclic = ls1x_dma_prep_dma_cyclic; + ddev->device_config = ls1x_dma_slave_config; + ddev->device_pause = ls1x_dma_pause; + ddev->device_resume = ls1x_dma_resume; + ddev->device_terminate_all = ls1x_dma_terminate_all; + ddev->device_synchronize = ls1x_dma_synchronize; + ddev->device_tx_status = ls1x_dma_tx_status; + ddev->device_issue_pending = ls1x_dma_issue_pending; + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + INIT_LIST_HEAD(&ddev->channels); + + /* initialize DMA channels */ + ret = ls1x_dma_chan_probe(pdev, dma); + if (ret) + goto err; + + ret = dmaenginem_async_device_register(ddev); + if (ret) { + dev_err(dev, "failed to register DMA device\n"); + goto err; + } + + ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, + ddev); + if (ret) { + dev_err(dev, "failed to register DMA controller\n"); + goto err; + } + + platform_set_drvdata(pdev, dma); + dev_info(dev, "Loongson1 DMA driver registered\n"); + + return 0; + +err: + ls1x_dma_chan_remove(dma); + + return ret; +} + +static void ls1x_dma_remove(struct platform_device *pdev) +{ + struct ls1x_dma *dma = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + ls1x_dma_chan_remove(dma); +} + +static const struct of_device_id ls1x_dma_match[] = { + { .compatible = "loongson,ls1b-apbdma" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ls1x_dma_match); + +static struct platform_driver ls1x_dma_driver = { + .probe = ls1x_dma_probe, + .remove = ls1x_dma_remove, + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = ls1x_dma_match, + }, +}; + +module_platform_driver(ls1x_dma_driver); + +MODULE_AUTHOR("Keguang Zhang <keguang.zhang@gmail.com>"); +MODULE_DESCRIPTION("Loongson-1 APB DMA Controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/loongson2-apb-dma.c b/drivers/dma/loongson2-apb-dma.c new file mode 100644 index 000000000000..c528f02b9f84 --- /dev/null +++ b/drivers/dma/loongson2-apb-dma.c @@ -0,0 +1,705 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Driver for the Loongson-2 APB DMA Controller + * + * Copyright (C) 2017-2023 Loongson Corporation + */ + +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +/* Global Configuration Register */ +#define LDMA_ORDER_ERG 0x0 + +/* Bitfield definitions */ + +/* Bitfields in Global Configuration Register */ +#define LDMA_64BIT_EN BIT(0) /* 1: 64 bit support */ +#define LDMA_UNCOHERENT_EN BIT(1) /* 0: cache, 1: uncache */ +#define LDMA_ASK_VALID BIT(2) +#define LDMA_START BIT(3) /* DMA start operation */ +#define LDMA_STOP BIT(4) /* DMA stop operation */ +#define LDMA_CONFIG_MASK GENMASK_ULL(4, 0) /* DMA controller config bits mask */ + +/* Bitfields in ndesc_addr field of HW descriptor */ +#define LDMA_DESC_EN BIT(0) /*1: The next descriptor is valid */ +#define LDMA_DESC_ADDR_LOW GENMASK(31, 1) + +/* Bitfields in cmd field of HW descriptor */ +#define LDMA_INT BIT(1) /* Enable DMA interrupts */ +#define LDMA_DATA_DIRECTION BIT(12) /* 1: write to device, 0: read from device */ + +#define LDMA_SLAVE_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +#define LDMA_MAX_TRANS_LEN U32_MAX + +/*-- descriptors -----------------------------------------------------*/ + +/* + * struct ls2x_dma_hw_desc - DMA HW descriptor + * @ndesc_addr: the next descriptor low address. + * @mem_addr: memory low address. + * @apb_addr: device buffer address. + * @len: length of a piece of carried content, in words. + * @step_len: length between two moved memory data blocks. + * @step_times: number of blocks to be carried in a single DMA operation. + * @cmd: descriptor command or state. + * @stats: DMA status. + * @high_ndesc_addr: the next descriptor high address. + * @high_mem_addr: memory high address. + * @reserved: reserved + */ +struct ls2x_dma_hw_desc { + u32 ndesc_addr; + u32 mem_addr; + u32 apb_addr; + u32 len; + u32 step_len; + u32 step_times; + u32 cmd; + u32 stats; + u32 high_ndesc_addr; + u32 high_mem_addr; + u32 reserved[2]; +} __packed; + +/* + * struct ls2x_dma_sg - ls2x dma scatter gather entry + * @hw: the pointer to DMA HW descriptor. + * @llp: physical address of the DMA HW descriptor. + * @phys: destination or source address(mem). + * @len: number of Bytes to read. + */ +struct ls2x_dma_sg { + struct ls2x_dma_hw_desc *hw; + dma_addr_t llp; + dma_addr_t phys; + u32 len; +}; + +/* + * struct ls2x_dma_desc - software descriptor + * @vdesc: pointer to the virtual dma descriptor. + * @cyclic: flag to dma cyclic + * @burst_size: burst size of transaction, in words. + * @desc_num: number of sg entries. + * @direction: transfer direction, to or from device. + * @status: dma controller status. + * @sg: array of sgs. + */ +struct ls2x_dma_desc { + struct virt_dma_desc vdesc; + bool cyclic; + size_t burst_size; + u32 desc_num; + enum dma_transfer_direction direction; + enum dma_status status; + struct ls2x_dma_sg sg[] __counted_by(desc_num); +}; + +/*-- Channels --------------------------------------------------------*/ + +/* + * struct ls2x_dma_chan - internal representation of an LS2X APB DMA channel + * @vchan: virtual dma channel entry. + * @desc: pointer to the ls2x sw dma descriptor. + * @pool: hw desc table + * @irq: irq line + * @sconfig: configuration for slave transfers, passed via .device_config + */ +struct ls2x_dma_chan { + struct virt_dma_chan vchan; + struct ls2x_dma_desc *desc; + void *pool; + int irq; + struct dma_slave_config sconfig; +}; + +/*-- Controller ------------------------------------------------------*/ + +/* + * struct ls2x_dma_priv - LS2X APB DMAC specific information + * @ddev: dmaengine dma_device object members + * @dma_clk: DMAC clock source + * @regs: memory mapped register base + * @lchan: channel to store ls2x_dma_chan structures + */ +struct ls2x_dma_priv { + struct dma_device ddev; + struct clk *dma_clk; + void __iomem *regs; + struct ls2x_dma_chan lchan; +}; + +/*-- Helper functions ------------------------------------------------*/ + +static inline struct ls2x_dma_desc *to_ldma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct ls2x_dma_desc, vdesc); +} + +static inline struct ls2x_dma_chan *to_ldma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct ls2x_dma_chan, vchan.chan); +} + +static inline struct ls2x_dma_priv *to_ldma_priv(struct dma_device *ddev) +{ + return container_of(ddev, struct ls2x_dma_priv, ddev); +} + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static void ls2x_dma_desc_free(struct virt_dma_desc *vdesc) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(vdesc->tx.chan); + struct ls2x_dma_desc *desc = to_ldma_desc(vdesc); + int i; + + for (i = 0; i < desc->desc_num; i++) { + if (desc->sg[i].hw) + dma_pool_free(lchan->pool, desc->sg[i].hw, + desc->sg[i].llp); + } + + kfree(desc); +} + +static void ls2x_dma_write_cmd(struct ls2x_dma_chan *lchan, bool cmd) +{ + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); + u64 val; + + val = lo_hi_readq(priv->regs + LDMA_ORDER_ERG) & ~LDMA_CONFIG_MASK; + val |= LDMA_64BIT_EN | cmd; + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); +} + +static void ls2x_dma_start_transfer(struct ls2x_dma_chan *lchan) +{ + struct ls2x_dma_priv *priv = to_ldma_priv(lchan->vchan.chan.device); + struct ls2x_dma_sg *ldma_sg; + struct virt_dma_desc *vdesc; + u64 val; + + /* Get the next descriptor */ + vdesc = vchan_next_desc(&lchan->vchan); + if (!vdesc) { + lchan->desc = NULL; + return; + } + + list_del(&vdesc->node); + lchan->desc = to_ldma_desc(vdesc); + ldma_sg = &lchan->desc->sg[0]; + + /* Start DMA */ + lo_hi_writeq(0, priv->regs + LDMA_ORDER_ERG); + val = (ldma_sg->llp & ~LDMA_CONFIG_MASK) | LDMA_64BIT_EN | LDMA_START; + lo_hi_writeq(val, priv->regs + LDMA_ORDER_ERG); +} + +static size_t ls2x_dmac_detect_burst(struct ls2x_dma_chan *lchan) +{ + u32 maxburst, buswidth; + + /* Reject definitely invalid configurations */ + if ((lchan->sconfig.src_addr_width & LDMA_SLAVE_BUSWIDTHS) && + (lchan->sconfig.dst_addr_width & LDMA_SLAVE_BUSWIDTHS)) + return 0; + + if (lchan->sconfig.direction == DMA_MEM_TO_DEV) { + maxburst = lchan->sconfig.dst_maxburst; + buswidth = lchan->sconfig.dst_addr_width; + } else { + maxburst = lchan->sconfig.src_maxburst; + buswidth = lchan->sconfig.src_addr_width; + } + + /* If maxburst is zero, fallback to LDMA_MAX_TRANS_LEN */ + return maxburst ? (maxburst * buswidth) >> 2 : LDMA_MAX_TRANS_LEN; +} + +static void ls2x_dma_fill_desc(struct ls2x_dma_chan *lchan, u32 sg_index, + struct ls2x_dma_desc *desc) +{ + struct ls2x_dma_sg *ldma_sg = &desc->sg[sg_index]; + u32 num_segments, segment_size; + + if (desc->direction == DMA_MEM_TO_DEV) { + ldma_sg->hw->cmd = LDMA_INT | LDMA_DATA_DIRECTION; + ldma_sg->hw->apb_addr = lchan->sconfig.dst_addr; + } else { + ldma_sg->hw->cmd = LDMA_INT; + ldma_sg->hw->apb_addr = lchan->sconfig.src_addr; + } + + ldma_sg->hw->mem_addr = lower_32_bits(ldma_sg->phys); + ldma_sg->hw->high_mem_addr = upper_32_bits(ldma_sg->phys); + + /* Split into multiple equally sized segments if necessary */ + num_segments = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, desc->burst_size); + segment_size = DIV_ROUND_UP((ldma_sg->len + 3) >> 2, num_segments); + + /* Word count register takes input in words */ + ldma_sg->hw->len = segment_size; + ldma_sg->hw->step_times = num_segments; + ldma_sg->hw->step_len = 0; + + /* lets make a link list */ + if (sg_index) { + desc->sg[sg_index - 1].hw->ndesc_addr = ldma_sg->llp | LDMA_DESC_EN; + desc->sg[sg_index - 1].hw->high_ndesc_addr = upper_32_bits(ldma_sg->llp); + } +} + +/*-- DMA Engine API --------------------------------------------------*/ + +/* + * ls2x_dma_alloc_chan_resources - allocate resources for DMA channel + * @chan: allocate descriptor resources for this channel + * + * return - the number of allocated descriptors + */ +static int ls2x_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + /* Create a pool of consistent memory blocks for hardware descriptors */ + lchan->pool = dma_pool_create(dev_name(chan2dev(chan)), + chan->device->dev, PAGE_SIZE, + __alignof__(struct ls2x_dma_hw_desc), 0); + if (!lchan->pool) { + dev_err(chan2dev(chan), "No memory for descriptors\n"); + return -ENOMEM; + } + + return 1; +} + +/* + * ls2x_dma_free_chan_resources - free all channel resources + * @chan: DMA channel + */ +static void ls2x_dma_free_chan_resources(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + vchan_free_chan_resources(to_virt_chan(chan)); + dma_pool_destroy(lchan->pool); + lchan->pool = NULL; +} + +/* + * ls2x_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction + * @chan: DMA channel + * @sgl: scatterlist to transfer to/from + * @sg_len: number of entries in @scatterlist + * @direction: DMA direction + * @flags: tx descriptor status flags + * @context: transaction context (ignored) + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +ls2x_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + u32 sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + struct ls2x_dma_desc *desc; + struct scatterlist *sg; + size_t burst_size; + int i; + + if (unlikely(!sg_len || !is_slave_direction(direction))) + return NULL; + + burst_size = ls2x_dmac_detect_burst(lchan); + if (!burst_size) + return NULL; + + desc = kzalloc(struct_size(desc, sg, sg_len), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->desc_num = sg_len; + desc->direction = direction; + desc->burst_size = burst_size; + + for_each_sg(sgl, sg, sg_len, i) { + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; + + /* Allocate DMA capable memory for hardware descriptor */ + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); + if (!ldma_sg->hw) { + desc->desc_num = i; + ls2x_dma_desc_free(&desc->vdesc); + return NULL; + } + + ldma_sg->phys = sg_dma_address(sg); + ldma_sg->len = sg_dma_len(sg); + + ls2x_dma_fill_desc(lchan, i, desc); + } + + /* Setting the last descriptor enable bit */ + desc->sg[sg_len - 1].hw->ndesc_addr &= ~LDMA_DESC_EN; + desc->status = DMA_IN_PROGRESS; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +/* + * ls2x_dma_prep_dma_cyclic - prepare the cyclic DMA transfer + * @chan: the DMA channel to prepare + * @buf_addr: physical DMA address where the buffer starts + * @buf_len: total number of bytes for the entire buffer + * @period_len: number of bytes for each period + * @direction: transfer direction, to or from device + * @flags: tx descriptor status flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +ls2x_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + struct ls2x_dma_desc *desc; + size_t burst_size; + u32 num_periods; + int i; + + if (unlikely(!buf_len || !period_len)) + return NULL; + + if (unlikely(!is_slave_direction(direction))) + return NULL; + + burst_size = ls2x_dmac_detect_burst(lchan); + if (!burst_size) + return NULL; + + num_periods = buf_len / period_len; + desc = kzalloc(struct_size(desc, sg, num_periods), GFP_NOWAIT); + if (!desc) + return NULL; + + desc->desc_num = num_periods; + desc->direction = direction; + desc->burst_size = burst_size; + + /* Build cyclic linked list */ + for (i = 0; i < num_periods; i++) { + struct ls2x_dma_sg *ldma_sg = &desc->sg[i]; + + /* Allocate DMA capable memory for hardware descriptor */ + ldma_sg->hw = dma_pool_alloc(lchan->pool, GFP_NOWAIT, &ldma_sg->llp); + if (!ldma_sg->hw) { + desc->desc_num = i; + ls2x_dma_desc_free(&desc->vdesc); + return NULL; + } + + ldma_sg->phys = buf_addr + period_len * i; + ldma_sg->len = period_len; + + ls2x_dma_fill_desc(lchan, i, desc); + } + + /* Lets make a cyclic list */ + desc->sg[num_periods - 1].hw->ndesc_addr = desc->sg[0].llp | LDMA_DESC_EN; + desc->sg[num_periods - 1].hw->high_ndesc_addr = upper_32_bits(desc->sg[0].llp); + desc->cyclic = true; + desc->status = DMA_IN_PROGRESS; + + return vchan_tx_prep(&lchan->vchan, &desc->vdesc, flags); +} + +/* + * ls2x_slave_config - set slave configuration for channel + * @chan: dma channel + * @cfg: slave configuration + * + * Sets slave configuration for channel + */ +static int ls2x_dma_slave_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + memcpy(&lchan->sconfig, config, sizeof(*config)); + return 0; +} + +/* + * ls2x_dma_issue_pending - push pending transactions to the hardware + * @chan: channel + * + * When this function is called, all pending transactions are pushed to the + * hardware and executed. + */ +static void ls2x_dma_issue_pending(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (vchan_issue_pending(&lchan->vchan) && !lchan->desc) + ls2x_dma_start_transfer(lchan); + spin_unlock_irqrestore(&lchan->vchan.lock, flags); +} + +/* + * ls2x_dma_terminate_all - terminate all transactions + * @chan: channel + * + * Stops all DMA transactions. + */ +static int ls2x_dma_terminate_all(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&lchan->vchan.lock, flags); + /* Setting stop cmd */ + ls2x_dma_write_cmd(lchan, LDMA_STOP); + if (lchan->desc) { + vchan_terminate_vdesc(&lchan->desc->vdesc); + lchan->desc = NULL; + } + + vchan_get_all_descriptors(&lchan->vchan, &head); + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + vchan_dma_desc_free_list(&lchan->vchan, &head); + return 0; +} + +/* + * ls2x_dma_synchronize - Synchronizes the termination of transfers to the + * current context. + * @chan: channel + */ +static void ls2x_dma_synchronize(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + + vchan_synchronize(&lchan->vchan); +} + +static int ls2x_dma_pause(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (lchan->desc && lchan->desc->status == DMA_IN_PROGRESS) { + ls2x_dma_write_cmd(lchan, LDMA_STOP); + lchan->desc->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + return 0; +} + +static int ls2x_dma_resume(struct dma_chan *chan) +{ + struct ls2x_dma_chan *lchan = to_ldma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&lchan->vchan.lock, flags); + if (lchan->desc && lchan->desc->status == DMA_PAUSED) { + lchan->desc->status = DMA_IN_PROGRESS; + ls2x_dma_write_cmd(lchan, LDMA_START); + } + spin_unlock_irqrestore(&lchan->vchan.lock, flags); + + return 0; +} + +/* + * ls2x_dma_isr - LS2X DMA Interrupt handler + * @irq: IRQ number + * @dev_id: Pointer to ls2x_dma_chan + * + * Return: IRQ_HANDLED/IRQ_NONE + */ +static irqreturn_t ls2x_dma_isr(int irq, void *dev_id) +{ + struct ls2x_dma_chan *lchan = dev_id; + struct ls2x_dma_desc *desc; + + spin_lock(&lchan->vchan.lock); + desc = lchan->desc; + if (desc) { + if (desc->cyclic) { + vchan_cyclic_callback(&desc->vdesc); + } else { + desc->status = DMA_COMPLETE; + vchan_cookie_complete(&desc->vdesc); + ls2x_dma_start_transfer(lchan); + } + + /* ls2x_dma_start_transfer() updates lchan->desc */ + if (!lchan->desc) + ls2x_dma_write_cmd(lchan, LDMA_STOP); + } + spin_unlock(&lchan->vchan.lock); + + return IRQ_HANDLED; +} + +static int ls2x_dma_chan_init(struct platform_device *pdev, + struct ls2x_dma_priv *priv) +{ + struct ls2x_dma_chan *lchan = &priv->lchan; + struct device *dev = &pdev->dev; + int ret; + + lchan->irq = platform_get_irq(pdev, 0); + if (lchan->irq < 0) + return lchan->irq; + + ret = devm_request_irq(dev, lchan->irq, ls2x_dma_isr, IRQF_TRIGGER_RISING, + dev_name(&pdev->dev), lchan); + if (ret) + return ret; + + /* Initialize channels related values */ + INIT_LIST_HEAD(&priv->ddev.channels); + lchan->vchan.desc_free = ls2x_dma_desc_free; + vchan_init(&lchan->vchan, &priv->ddev); + + return 0; +} + +/* + * ls2x_dma_probe - Driver probe function + * @pdev: Pointer to the platform_device structure + * + * Return: '0' on success and failure value on error + */ +static int ls2x_dma_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ls2x_dma_priv *priv; + struct dma_device *ddev; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->regs)) + return dev_err_probe(dev, PTR_ERR(priv->regs), + "devm_platform_ioremap_resource failed.\n"); + + priv->dma_clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(priv->dma_clk)) + return dev_err_probe(dev, PTR_ERR(priv->dma_clk), "devm_clk_get failed.\n"); + + ret = clk_prepare_enable(priv->dma_clk); + if (ret) + return dev_err_probe(dev, ret, "clk_prepare_enable failed.\n"); + + ret = ls2x_dma_chan_init(pdev, priv); + if (ret) + goto disable_clk; + + ddev = &priv->ddev; + ddev->dev = dev; + dma_cap_zero(ddev->cap_mask); + dma_cap_set(DMA_SLAVE, ddev->cap_mask); + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); + + ddev->device_alloc_chan_resources = ls2x_dma_alloc_chan_resources; + ddev->device_free_chan_resources = ls2x_dma_free_chan_resources; + ddev->device_tx_status = dma_cookie_status; + ddev->device_issue_pending = ls2x_dma_issue_pending; + ddev->device_prep_slave_sg = ls2x_dma_prep_slave_sg; + ddev->device_prep_dma_cyclic = ls2x_dma_prep_dma_cyclic; + ddev->device_config = ls2x_dma_slave_config; + ddev->device_terminate_all = ls2x_dma_terminate_all; + ddev->device_synchronize = ls2x_dma_synchronize; + ddev->device_pause = ls2x_dma_pause; + ddev->device_resume = ls2x_dma_resume; + + ddev->src_addr_widths = LDMA_SLAVE_BUSWIDTHS; + ddev->dst_addr_widths = LDMA_SLAVE_BUSWIDTHS; + ddev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + + ret = dma_async_device_register(&priv->ddev); + if (ret < 0) + goto disable_clk; + + ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, priv); + if (ret < 0) + goto unregister_dmac; + + platform_set_drvdata(pdev, priv); + + dev_info(dev, "Loongson LS2X APB DMA driver registered successfully.\n"); + return 0; + +unregister_dmac: + dma_async_device_unregister(&priv->ddev); +disable_clk: + clk_disable_unprepare(priv->dma_clk); + + return ret; +} + +/* + * ls2x_dma_remove - Driver remove function + * @pdev: Pointer to the platform_device structure + */ +static void ls2x_dma_remove(struct platform_device *pdev) +{ + struct ls2x_dma_priv *priv = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&priv->ddev); + clk_disable_unprepare(priv->dma_clk); +} + +static const struct of_device_id ls2x_dma_of_match_table[] = { + { .compatible = "loongson,ls2k1000-apbdma" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ls2x_dma_of_match_table); + +static struct platform_driver ls2x_dmac_driver = { + .probe = ls2x_dma_probe, + .remove = ls2x_dma_remove, + .driver = { + .name = "ls2x-apbdma", + .of_match_table = ls2x_dma_of_match_table, + }, +}; +module_platform_driver(ls2x_dmac_driver); + +MODULE_DESCRIPTION("Loongson-2 APB DMA Controller driver"); +MODULE_AUTHOR("Loongson Technology Corporation Limited"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/lpc18xx-dmamux.c b/drivers/dma/lpc18xx-dmamux.c index df98cae8792b..2b6436f4b193 100644 --- a/drivers/dma/lpc18xx-dmamux.c +++ b/drivers/dma/lpc18xx-dmamux.c @@ -12,8 +12,10 @@ #include <linux/err.h> #include <linux/init.h> #include <linux/mfd/syscon.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/regmap.h> #include <linux/spinlock.h> diff --git a/drivers/dma/lpc32xx-dmamux.c b/drivers/dma/lpc32xx-dmamux.c new file mode 100644 index 000000000000..351d7e23e615 --- /dev/null +++ b/drivers/dma/lpc32xx-dmamux.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Copyright 2024 Timesys Corporation <piotr.wojtaszczyk@timesys.com> +// +// Based on TI DMA Crossbar driver by: +// Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com +// Author: Peter Ujfalusi <peter.ujfalusi@ti.com> + +#include <linux/err.h> +#include <linux/init.h> +#include <linux/mfd/syscon.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/spinlock.h> + +#define LPC32XX_SSP_CLK_CTRL 0x78 +#define LPC32XX_I2S_CLK_CTRL 0x7c + +struct lpc32xx_dmamux { + int signal; + char *name_sel0; + char *name_sel1; + int muxval; + int muxreg; + int bit; + bool busy; +}; + +struct lpc32xx_dmamux_data { + struct dma_router dmarouter; + struct regmap *reg; + spinlock_t lock; /* protects busy status flag */ +}; + +/* From LPC32x0 User manual "3.2.1 DMA request signals" */ +static struct lpc32xx_dmamux lpc32xx_muxes[] = { + { + .signal = 3, + .name_sel0 = "spi2-rx-tx", + .name_sel1 = "ssp1-rx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 5, + }, + { + .signal = 10, + .name_sel0 = "uart7-rx", + .name_sel1 = "i2s1-dma1", + .muxreg = LPC32XX_I2S_CLK_CTRL, + .bit = 4, + }, + { + .signal = 11, + .name_sel0 = "spi1-rx-tx", + .name_sel1 = "ssp1-tx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 4, + }, + { + .signal = 14, + .name_sel0 = "none", + .name_sel1 = "ssp0-rx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 3, + }, + { + .signal = 15, + .name_sel0 = "none", + .name_sel1 = "ssp0-tx", + .muxreg = LPC32XX_SSP_CLK_CTRL, + .bit = 2, + }, +}; + +static void lpc32xx_dmamux_release(struct device *dev, void *route_data) +{ + struct lpc32xx_dmamux_data *dmamux = dev_get_drvdata(dev); + struct lpc32xx_dmamux *mux = route_data; + + dev_dbg(dev, "releasing dma request signal %d routed to %s\n", + mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); + + guard(spinlock)(&dmamux->lock); + + mux->busy = false; +} + +static void *lpc32xx_dmamux_reserve(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct platform_device *pdev = of_find_device_by_node(ofdma->of_node); + struct device *dev = &pdev->dev; + struct lpc32xx_dmamux_data *dmamux = platform_get_drvdata(pdev); + unsigned long flags; + struct lpc32xx_dmamux *mux = NULL; + int i; + + if (dma_spec->args_count != 3) { + dev_err(&pdev->dev, "invalid number of dma mux args\n"); + return ERR_PTR(-EINVAL); + } + + for (i = 0; i < ARRAY_SIZE(lpc32xx_muxes); i++) { + if (lpc32xx_muxes[i].signal == dma_spec->args[0]) { + mux = &lpc32xx_muxes[i]; + break; + } + } + if (!mux) { + dev_err(&pdev->dev, "invalid mux request number: %d\n", + dma_spec->args[0]); + return ERR_PTR(-EINVAL); + } + + if (dma_spec->args[2] > 1) { + dev_err(&pdev->dev, "invalid dma mux value: %d\n", + dma_spec->args[1]); + return ERR_PTR(-EINVAL); + } + + /* The of_node_put() will be done in the core for the node */ + dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); + if (!dma_spec->np) { + dev_err(&pdev->dev, "can't get dma master\n"); + return ERR_PTR(-EINVAL); + } + + spin_lock_irqsave(&dmamux->lock, flags); + if (mux->busy) { + spin_unlock_irqrestore(&dmamux->lock, flags); + dev_err(dev, "dma request signal %d busy, routed to %s\n", + mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); + of_node_put(dma_spec->np); + return ERR_PTR(-EBUSY); + } + + mux->busy = true; + mux->muxval = dma_spec->args[2] ? BIT(mux->bit) : 0; + + regmap_update_bits(dmamux->reg, mux->muxreg, BIT(mux->bit), mux->muxval); + spin_unlock_irqrestore(&dmamux->lock, flags); + + dma_spec->args[2] = 0; + dma_spec->args_count = 2; + + dev_dbg(dev, "dma request signal %d routed to %s\n", + mux->signal, mux->muxval ? mux->name_sel1 : mux->name_sel1); + + return mux; +} + +static int lpc32xx_dmamux_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct lpc32xx_dmamux_data *dmamux; + + dmamux = devm_kzalloc(&pdev->dev, sizeof(*dmamux), GFP_KERNEL); + if (!dmamux) + return -ENOMEM; + + dmamux->reg = syscon_node_to_regmap(np->parent); + if (IS_ERR(dmamux->reg)) { + dev_err(&pdev->dev, "syscon lookup failed\n"); + return PTR_ERR(dmamux->reg); + } + + spin_lock_init(&dmamux->lock); + platform_set_drvdata(pdev, dmamux); + dmamux->dmarouter.dev = &pdev->dev; + dmamux->dmarouter.route_free = lpc32xx_dmamux_release; + + return of_dma_router_register(np, lpc32xx_dmamux_reserve, + &dmamux->dmarouter); +} + +static const struct of_device_id lpc32xx_dmamux_match[] = { + { .compatible = "nxp,lpc3220-dmamux" }, + {}, +}; + +static struct platform_driver lpc32xx_dmamux_driver = { + .probe = lpc32xx_dmamux_probe, + .driver = { + .name = "lpc32xx-dmamux", + .of_match_table = lpc32xx_dmamux_match, + }, +}; + +static int __init lpc32xx_dmamux_init(void) +{ + return platform_driver_register(&lpc32xx_dmamux_driver); +} +arch_initcall(lpc32xx_dmamux_init); diff --git a/drivers/dma/mcf-edma.c b/drivers/dma/mcf-edma-main.c index e12b754e6398..9e1c6400c77b 100644 --- a/drivers/dma/mcf-edma.c +++ b/drivers/dma/mcf-edma-main.c @@ -19,7 +19,6 @@ static irqreturn_t mcf_edma_tx_handler(int irq, void *dev_id) struct fsl_edma_engine *mcf_edma = dev_id; struct edma_regs *regs = &mcf_edma->regs; unsigned int ch; - struct fsl_edma_chan *mcf_chan; u64 intmap; intmap = ioread32(regs->inth); @@ -31,31 +30,7 @@ static irqreturn_t mcf_edma_tx_handler(int irq, void *dev_id) for (ch = 0; ch < mcf_edma->n_chans; ch++) { if (intmap & BIT(ch)) { iowrite8(EDMA_MASK_CH(ch), regs->cint); - - mcf_chan = &mcf_edma->chans[ch]; - - spin_lock(&mcf_chan->vchan.lock); - - if (!mcf_chan->edesc) { - /* terminate_all called before */ - spin_unlock(&mcf_chan->vchan.lock); - continue; - } - - if (!mcf_chan->edesc->iscyclic) { - list_del(&mcf_chan->edesc->vdesc.node); - vchan_cookie_complete(&mcf_chan->edesc->vdesc); - mcf_chan->edesc = NULL; - mcf_chan->status = DMA_COMPLETE; - mcf_chan->idle = true; - } else { - vchan_cyclic_callback(&mcf_chan->edesc->vdesc); - } - - if (!mcf_chan->edesc) - fsl_edma_xfer_desc(mcf_chan); - - spin_unlock(&mcf_chan->vchan.lock); + fsl_edma_tx_chan_handler(&mcf_edma->chans[ch]); } } @@ -76,8 +51,7 @@ static irqreturn_t mcf_edma_err_handler(int irq, void *dev_id) if (err & BIT(ch)) { fsl_edma_disable_request(&mcf_edma->chans[ch]); iowrite8(EDMA_CERR_CERR(ch), regs->cerr); - mcf_edma->chans[ch].status = DMA_ERROR; - mcf_edma->chans[ch].idle = true; + fsl_edma_err_chan_handler(&mcf_edma->chans[ch]); } } @@ -90,7 +64,6 @@ static irqreturn_t mcf_edma_err_handler(int irq, void *dev_id) fsl_edma_disable_request(&mcf_edma->chans[ch]); iowrite8(EDMA_CERR_CERR(ch), regs->cerr); mcf_edma->chans[ch].status = DMA_ERROR; - mcf_edma->chans[ch].idle = true; } } @@ -172,7 +145,7 @@ static void mcf_edma_irq_free(struct platform_device *pdev, } static struct fsl_edma_drvdata mcf_data = { - .version = v2, + .flags = FSL_EDMA_DRV_EDMA64, .setup_irq = mcf_edma_irq_init, }; @@ -180,10 +153,8 @@ static int mcf_edma_probe(struct platform_device *pdev) { struct mcf_edma_platform_data *pdata; struct fsl_edma_engine *mcf_edma; - struct fsl_edma_chan *mcf_chan; struct edma_regs *regs; - struct resource *res; - int ret, i, len, chans; + int ret, i, chans; pdata = dev_get_platdata(&pdev->dev); if (!pdata) { @@ -191,9 +162,15 @@ static int mcf_edma_probe(struct platform_device *pdev) return -EINVAL; } - chans = pdata->dma_channels; - len = sizeof(*mcf_edma) + sizeof(*mcf_chan) * chans; - mcf_edma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!pdata->dma_channels) { + dev_info(&pdev->dev, "setting default channel number to 64"); + chans = 64; + } else { + chans = pdata->dma_channels; + } + + mcf_edma = devm_kzalloc(&pdev->dev, struct_size(mcf_edma, chans, chans), + GFP_KERNEL); if (!mcf_edma) return -ENOMEM; @@ -203,16 +180,9 @@ static int mcf_edma_probe(struct platform_device *pdev) mcf_edma->drvdata = &mcf_data; mcf_edma->big_endian = 1; - if (!mcf_edma->n_chans) { - dev_info(&pdev->dev, "setting default channel number to 64"); - mcf_edma->n_chans = 64; - } - mutex_init(&mcf_edma->fsl_edma_mutex); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - - mcf_edma->membase = devm_ioremap_resource(&pdev->dev, res); + mcf_edma->membase = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mcf_edma->membase)) return PTR_ERR(mcf_edma->membase); @@ -224,12 +194,13 @@ static int mcf_edma_probe(struct platform_device *pdev) struct fsl_edma_chan *mcf_chan = &mcf_edma->chans[i]; mcf_chan->edma = mcf_edma; - mcf_chan->slave_id = i; - mcf_chan->idle = true; + mcf_chan->srcid = i; mcf_chan->dma_dir = DMA_NONE; mcf_chan->vchan.desc_free = fsl_edma_free_desc; vchan_init(&mcf_chan->vchan, &mcf_edma->dma_dev); - iowrite32(0x0, ®s->tcd[i].csr); + mcf_chan->tcd = mcf_edma->membase + EDMA_TCD + + i * sizeof(struct fsl_edma_hw_tcd); + edma_write_tcdreg(mcf_chan, cpu_to_le32(0), csr); } iowrite32(~0, regs->inth); @@ -282,15 +253,13 @@ static int mcf_edma_probe(struct platform_device *pdev) return 0; } -static int mcf_edma_remove(struct platform_device *pdev) +static void mcf_edma_remove(struct platform_device *pdev) { struct fsl_edma_engine *mcf_edma = platform_get_drvdata(pdev); mcf_edma_irq_free(pdev, mcf_edma); fsl_edma_cleanup_vchan(&mcf_edma->dma_dev); dma_async_device_unregister(&mcf_edma->dma_dev); - - return 0; } static struct platform_driver mcf_edma_driver = { @@ -306,7 +275,7 @@ bool mcf_edma_filter_fn(struct dma_chan *chan, void *param) if (chan->device->dev->driver == &mcf_edma_driver.driver) { struct fsl_edma_chan *mcf_chan = to_fsl_edma_chan(chan); - return (mcf_chan->slave_id == (uintptr_t)param); + return (mcf_chan->srcid == (uintptr_t)param); } return false; diff --git a/drivers/dma/mediatek/mtk-cqdma.c b/drivers/dma/mediatek/mtk-cqdma.c index 41ef9f15d3d5..9f0c41ca7770 100644 --- a/drivers/dma/mediatek/mtk-cqdma.c +++ b/drivers/dma/mediatek/mtk-cqdma.c @@ -18,7 +18,6 @@ #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -373,7 +372,7 @@ static void mtk_cqdma_tasklet_cb(struct tasklet_struct *t) /* * free child CVD after completion. - * the parent CVD would be freeed with desc_free by user. + * the parent CVD would be freed with desc_free by user. */ if (cvd->parent != cvd) kfree(cvd); @@ -421,15 +420,11 @@ static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c, { struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c); struct virt_dma_desc *vd; - unsigned long flags; - spin_lock_irqsave(&cvc->pc->lock, flags); list_for_each_entry(vd, &cvc->pc->queue, node) if (vd->tx.cookie == cookie) { - spin_unlock_irqrestore(&cvc->pc->lock, flags); return vd; } - spin_unlock_irqrestore(&cvc->pc->lock, flags); list_for_each_entry(vd, &cvc->vc.desc_issued, node) if (vd->tx.cookie == cookie) @@ -453,9 +448,11 @@ static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c, if (ret == DMA_COMPLETE || !txstate) return ret; - spin_lock_irqsave(&cvc->vc.lock, flags); + spin_lock_irqsave(&cvc->pc->lock, flags); + spin_lock(&cvc->vc.lock); vd = mtk_cqdma_find_active_desc(c, cookie); - spin_unlock_irqrestore(&cvc->vc.lock, flags); + spin_unlock(&cvc->vc.lock); + spin_unlock_irqrestore(&cvc->pc->lock, flags); if (vd) { cvd = to_cqdma_vdesc(vd); @@ -519,7 +516,7 @@ mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, /* setup dma channel */ cvd[i]->ch = c; - /* setup sourece, destination, and length */ + /* setup source, destination, and length */ tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len; cvd[i]->len = tlen; cvd[i]->src = src; @@ -618,7 +615,7 @@ static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c) u32 i, min_refcnt = U32_MAX, refcnt; unsigned long flags; - /* allocate PC with the minimun refcount */ + /* allocate PC with the minimum refcount */ for (i = 0; i < cqdma->dma_channels; ++i) { refcnt = refcount_read(&cqdma->pc[i]->refcnt); if (refcnt < min_refcnt) { @@ -751,7 +748,6 @@ static int mtk_cqdma_probe(struct platform_device *pdev) struct mtk_cqdma_device *cqdma; struct mtk_cqdma_vchan *vc; struct dma_device *dd; - struct resource *res; int err; u32 i; @@ -824,13 +820,10 @@ static int mtk_cqdma_probe(struct platform_device *pdev) return PTR_ERR(cqdma->pc[i]->base); /* allocate IRQ resource */ - res = platform_get_resource(pdev, IORESOURCE_IRQ, i); - if (!res) { - dev_err(&pdev->dev, "No irq resource for %s\n", - dev_name(&pdev->dev)); - return -EINVAL; - } - cqdma->pc[i]->irq = res->start; + err = platform_get_irq(pdev, i); + if (err < 0) + return err; + cqdma->pc[i]->irq = err; err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq, mtk_cqdma_irq, 0, dev_name(&pdev->dev), @@ -890,7 +883,7 @@ err_unregister: return err; } -static int mtk_cqdma_remove(struct platform_device *pdev) +static void mtk_cqdma_remove(struct platform_device *pdev) { struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev); struct mtk_cqdma_vchan *vc; @@ -923,8 +916,6 @@ static int mtk_cqdma_remove(struct platform_device *pdev) dma_async_device_unregister(&cqdma->ddev); of_dma_controller_free(pdev->dev.of_node); - - return 0; } static struct platform_driver mtk_cqdma_driver = { diff --git a/drivers/dma/mediatek/mtk-hsdma.c b/drivers/dma/mediatek/mtk-hsdma.c index 6ad8afbb95f2..fa77bb24a430 100644 --- a/drivers/dma/mediatek/mtk-hsdma.c +++ b/drivers/dma/mediatek/mtk-hsdma.c @@ -17,7 +17,6 @@ #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -138,7 +137,7 @@ struct mtk_hsdma_vdesc { /** * struct mtk_hsdma_cb - This is the struct holding extra info required for RX - * ring to know what relevant VD the the PD is being + * ring to know what relevant VD the PD is being * mapped to. * @vd: Pointer to the relevant VD. * @flag: Flag indicating what action should be taken when VD @@ -227,7 +226,7 @@ struct mtk_hsdma_soc { * @pc_refcnt: Track how many VCs are using the PC * @lock: Lock protect agaisting multiple VCs access PC * @soc: The pointer to area holding differences among - * vaious platform + * various platform */ struct mtk_hsdma_device { struct dma_device ddev; @@ -601,7 +600,7 @@ static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma) cb->flag = 0; } - cb->vd = 0; + cb->vd = NULL; /* * Recycle the RXD with the helper WRITE_ONCE that can ensure @@ -761,7 +760,7 @@ static void mtk_hsdma_free_active_desc(struct dma_chan *c) /* * Once issue_synchronize is being set, which means once the hardware * consumes all descriptors for the channel in the ring, the - * synchronization must be be notified immediately it is completed. + * synchronization must be notified immediately it is completed. */ spin_lock(&hvc->vc.lock); if (!list_empty(&hvc->desc_hw_processing)) { @@ -896,7 +895,6 @@ static int mtk_hsdma_probe(struct platform_device *pdev) struct mtk_hsdma_device *hsdma; struct mtk_hsdma_vchan *vc; struct dma_device *dd; - struct resource *res; int i, err; hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL); @@ -905,8 +903,7 @@ static int mtk_hsdma_probe(struct platform_device *pdev) dd = &hsdma->ddev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - hsdma->base = devm_ioremap_resource(&pdev->dev, res); + hsdma->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(hsdma->base)) return PTR_ERR(hsdma->base); @@ -923,13 +920,10 @@ static int mtk_hsdma_probe(struct platform_device *pdev) return PTR_ERR(hsdma->clk); } - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res) { - dev_err(&pdev->dev, "No irq resource for %s\n", - dev_name(&pdev->dev)); - return -EINVAL; - } - hsdma->irq = res->start; + err = platform_get_irq(pdev, 0); + if (err < 0) + return err; + hsdma->irq = err; refcount_set(&hsdma->pc_refcnt, 0); spin_lock_init(&hsdma->lock); @@ -1015,7 +1009,7 @@ err_unregister: return err; } -static int mtk_hsdma_remove(struct platform_device *pdev) +static void mtk_hsdma_remove(struct platform_device *pdev) { struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev); struct mtk_hsdma_vchan *vc; @@ -1040,8 +1034,6 @@ static int mtk_hsdma_remove(struct platform_device *pdev) dma_async_device_unregister(&hsdma->ddev); of_dma_controller_free(pdev->dev.of_node); - - return 0; } static struct platform_driver mtk_hsdma_driver = { diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index 375e7e647df6..08e15177427b 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -16,7 +16,6 @@ #include <linux/kernel.h> #include <linux/list.h> #include <linux/module.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -274,7 +273,7 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) unsigned int status; int ret; - ret = pm_runtime_get_sync(mtkd->ddev.dev); + ret = pm_runtime_resume_and_get(mtkd->ddev.dev); if (ret < 0) { pm_runtime_put_noidle(chan->device->dev); return ret; @@ -288,18 +287,21 @@ static int mtk_uart_apdma_alloc_chan_resources(struct dma_chan *chan) ret = readx_poll_timeout(readl, c->base + VFF_EN, status, !status, 10, 100); if (ret) - return ret; + goto err_pm; ret = request_irq(c->irq, mtk_uart_apdma_irq_handler, IRQF_TRIGGER_NONE, KBUILD_MODNAME, chan); if (ret < 0) { dev_err(chan->device->dev, "Can't request dma IRQ\n"); - return -EINVAL; + ret = -EINVAL; + goto err_pm; } if (mtkd->support_33bits) mtk_uart_apdma_write(c, VFF_4G_SUPPORT, VFF_4G_SUPPORT_CLR_B); +err_pm: + pm_runtime_put_noidle(mtkd->ddev.dev); return ret; } @@ -448,9 +450,8 @@ static int mtk_uart_apdma_device_pause(struct dma_chan *chan) mtk_uart_apdma_write(c, VFF_EN, VFF_EN_CLR_B); mtk_uart_apdma_write(c, VFF_INT_EN, VFF_INT_EN_CLR_B); - synchronize_irq(c->irq); - spin_unlock_irqrestore(&c->vc.lock, flags); + synchronize_irq(c->irq); return 0; } @@ -548,7 +549,6 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev) } pm_runtime_enable(&pdev->dev); - pm_runtime_set_active(&pdev->dev); rc = dma_async_device_register(&mtkd->ddev); if (rc) @@ -572,7 +572,7 @@ err_no_dma: return rc; } -static int mtk_uart_apdma_remove(struct platform_device *pdev) +static void mtk_uart_apdma_remove(struct platform_device *pdev) { struct mtk_uart_apdmadev *mtkd = platform_get_drvdata(pdev); @@ -583,8 +583,6 @@ static int mtk_uart_apdma_remove(struct platform_device *pdev) dma_async_device_unregister(&mtkd->ddev); pm_runtime_disable(&pdev->dev); - - return 0; } #ifdef CONFIG_PM_SLEEP @@ -639,7 +637,7 @@ static const struct dev_pm_ops mtk_uart_apdma_pm_ops = { static struct platform_driver mtk_uart_apdma_driver = { .probe = mtk_uart_apdma_probe, - .remove = mtk_uart_apdma_remove, + .remove = mtk_uart_apdma_remove, .driver = { .name = KBUILD_MODNAME, .pm = &mtk_uart_apdma_pm_ops, diff --git a/drivers/dma/milbeaut-hdmac.c b/drivers/dma/milbeaut-hdmac.c index a8cfb59f6efe..9a5ec247ed6d 100644 --- a/drivers/dma/milbeaut-hdmac.c +++ b/drivers/dma/milbeaut-hdmac.c @@ -269,7 +269,7 @@ milbeaut_hdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!md) return NULL; - md->sgl = kzalloc(sizeof(*sgl) * sg_len, GFP_NOWAIT); + md->sgl = kcalloc(sg_len, sizeof(*sgl), GFP_NOWAIT); if (!md->sgl) { kfree(md); return NULL; @@ -531,7 +531,7 @@ disable_clk: return ret; } -static int milbeaut_hdmac_remove(struct platform_device *pdev) +static void milbeaut_hdmac_remove(struct platform_device *pdev) { struct milbeaut_hdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -546,16 +546,21 @@ static int milbeaut_hdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } milbeaut_hdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&mdev->ddev); clk_disable_unprepare(mdev->clk); - - return 0; } static const struct of_device_id milbeaut_hdmac_match[] = { diff --git a/drivers/dma/milbeaut-xdmac.c b/drivers/dma/milbeaut-xdmac.c index d29d01e730aa..58d4fd6df0bf 100644 --- a/drivers/dma/milbeaut-xdmac.c +++ b/drivers/dma/milbeaut-xdmac.c @@ -368,7 +368,7 @@ disable_xdmac: return ret; } -static int milbeaut_xdmac_remove(struct platform_device *pdev) +static void milbeaut_xdmac_remove(struct platform_device *pdev) { struct milbeaut_xdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -383,8 +383,15 @@ static int milbeaut_xdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } milbeaut_xdmac_free_chan_resources(chan); } @@ -392,8 +399,6 @@ static int milbeaut_xdmac_remove(struct platform_device *pdev) dma_async_device_unregister(&mdev->ddev); disable_xdmac(mdev); - - return 0; } static const struct of_device_id milbeaut_xdmac_match[] = { diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index 89f1814ff27a..d07229a74886 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -15,7 +15,8 @@ #include <linux/device.h> #include <linux/platform_data/mmp_dma.h> #include <linux/dmapool.h> -#include <linux/of_device.h> +#include <linux/clk.h> +#include <linux/reset.h> #include <linux/of_dma.h> #include <linux/of.h> @@ -24,9 +25,12 @@ #define DCSR 0x0000 #define DALGN 0x00a0 #define DINT 0x00f0 -#define DDADR 0x0200 +#define DDADR(n) (0x0200 + ((n) << 4)) #define DSADR(n) (0x0204 + ((n) << 4)) #define DTADR(n) (0x0208 + ((n) << 4)) +#define DDADRH(n) (0x0300 + ((n) << 4)) +#define DSADRH(n) (0x0304 + ((n) << 4)) +#define DTADRH(n) (0x0308 + ((n) << 4)) #define DCMD 0x020c #define DCSR_RUN BIT(31) /* Run Bit (read / write) */ @@ -43,6 +47,7 @@ #define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ #define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ #define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ +#define DCSR_LPAEEN BIT(21) /* Long Physical Address Extension Enable */ #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ #define DCSR_EORINTR BIT(9) /* The end of Receive */ @@ -75,6 +80,16 @@ struct mmp_pdma_desc_hw { u32 dsadr; /* DSADR value for the current transfer */ u32 dtadr; /* DTADR value for the current transfer */ u32 dcmd; /* DCMD value for the current transfer */ + /* + * The following 32-bit words are only used in the 64-bit, ie. + * LPAE (Long Physical Address Extension) mode. + * They are used to specify the high 32 bits of the descriptor's + * addresses. + */ + u32 ddadrh; /* High 32-bit of DDADR */ + u32 dsadrh; /* High 32-bit of DSADR */ + u32 dtadrh; /* High 32-bit of DTADR */ + u32 rsvd; /* reserved */ } __aligned(32); struct mmp_pdma_desc_sw { @@ -119,12 +134,55 @@ struct mmp_pdma_phy { struct mmp_pdma_chan *vchan; }; +/** + * struct mmp_pdma_ops - Operations for the MMP PDMA controller + * + * Hardware Register Operations (read/write hardware registers): + * @write_next_addr: Function to program address of next descriptor into + * DDADR/DDADRH + * @read_src_addr: Function to read the source address from DSADR/DSADRH + * @read_dst_addr: Function to read the destination address from DTADR/DTADRH + * + * Descriptor Memory Operations (manipulate descriptor structs in memory): + * @set_desc_next_addr: Function to set next descriptor address in descriptor + * @set_desc_src_addr: Function to set the source address in descriptor + * @set_desc_dst_addr: Function to set the destination address in descriptor + * @get_desc_src_addr: Function to get the source address from descriptor + * @get_desc_dst_addr: Function to get the destination address from descriptor + * + * Controller Configuration: + * @run_bits: Control bits in DCSR register for channel start/stop + * @dma_mask: DMA addressing capability of controller. 0 to use OF/platform + * settings, or explicit mask like DMA_BIT_MASK(32/64) + */ +struct mmp_pdma_ops { + /* Hardware Register Operations */ + void (*write_next_addr)(struct mmp_pdma_phy *phy, dma_addr_t addr); + u64 (*read_src_addr)(struct mmp_pdma_phy *phy); + u64 (*read_dst_addr)(struct mmp_pdma_phy *phy); + + /* Descriptor Memory Operations */ + void (*set_desc_next_addr)(struct mmp_pdma_desc_hw *desc, + dma_addr_t addr); + void (*set_desc_src_addr)(struct mmp_pdma_desc_hw *desc, + dma_addr_t addr); + void (*set_desc_dst_addr)(struct mmp_pdma_desc_hw *desc, + dma_addr_t addr); + u64 (*get_desc_src_addr)(const struct mmp_pdma_desc_hw *desc); + u64 (*get_desc_dst_addr)(const struct mmp_pdma_desc_hw *desc); + + /* Controller Configuration */ + u32 run_bits; + u64 dma_mask; +}; + struct mmp_pdma_device { int dma_channels; void __iomem *base; struct device *dev; struct dma_device device; struct mmp_pdma_phy *phy; + const struct mmp_pdma_ops *ops; spinlock_t phy_lock; /* protect alloc/free phy channels */ }; @@ -137,24 +195,112 @@ struct mmp_pdma_device { #define to_mmp_pdma_dev(dmadev) \ container_of(dmadev, struct mmp_pdma_device, device) -static int mmp_pdma_config_write(struct dma_chan *dchan, - struct dma_slave_config *cfg, - enum dma_transfer_direction direction); +/* For 32-bit PDMA */ +static void write_next_addr_32(struct mmp_pdma_phy *phy, dma_addr_t addr) +{ + writel(addr, phy->base + DDADR(phy->idx)); +} + +static u64 read_src_addr_32(struct mmp_pdma_phy *phy) +{ + return readl(phy->base + DSADR(phy->idx)); +} + +static u64 read_dst_addr_32(struct mmp_pdma_phy *phy) +{ + return readl(phy->base + DTADR(phy->idx)); +} -static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr) +static void set_desc_next_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) { - u32 reg = (phy->idx << 4) + DDADR; + desc->ddadr = addr; +} - writel(addr, phy->base + reg); +static void set_desc_src_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dsadr = addr; } +static void set_desc_dst_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dtadr = addr; +} + +static u64 get_desc_src_addr_32(const struct mmp_pdma_desc_hw *desc) +{ + return desc->dsadr; +} + +static u64 get_desc_dst_addr_32(const struct mmp_pdma_desc_hw *desc) +{ + return desc->dtadr; +} + +/* For 64-bit PDMA */ +static void write_next_addr_64(struct mmp_pdma_phy *phy, dma_addr_t addr) +{ + writel(lower_32_bits(addr), phy->base + DDADR(phy->idx)); + writel(upper_32_bits(addr), phy->base + DDADRH(phy->idx)); +} + +static u64 read_src_addr_64(struct mmp_pdma_phy *phy) +{ + u32 low = readl(phy->base + DSADR(phy->idx)); + u32 high = readl(phy->base + DSADRH(phy->idx)); + + return ((u64)high << 32) | low; +} + +static u64 read_dst_addr_64(struct mmp_pdma_phy *phy) +{ + u32 low = readl(phy->base + DTADR(phy->idx)); + u32 high = readl(phy->base + DTADRH(phy->idx)); + + return ((u64)high << 32) | low; +} + +static void set_desc_next_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->ddadr = lower_32_bits(addr); + desc->ddadrh = upper_32_bits(addr); +} + +static void set_desc_src_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dsadr = lower_32_bits(addr); + desc->dsadrh = upper_32_bits(addr); +} + +static void set_desc_dst_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) +{ + desc->dtadr = lower_32_bits(addr); + desc->dtadrh = upper_32_bits(addr); +} + +static u64 get_desc_src_addr_64(const struct mmp_pdma_desc_hw *desc) +{ + return ((u64)desc->dsadrh << 32) | desc->dsadr; +} + +static u64 get_desc_dst_addr_64(const struct mmp_pdma_desc_hw *desc) +{ + return ((u64)desc->dtadrh << 32) | desc->dtadr; +} + +static int mmp_pdma_config_write(struct dma_chan *dchan, + struct dma_slave_config *cfg, + enum dma_transfer_direction direction); + static void enable_chan(struct mmp_pdma_phy *phy) { u32 reg, dalgn; + struct mmp_pdma_device *pdev; if (!phy->vchan) return; + pdev = to_mmp_pdma_dev(phy->vchan->chan.device); + reg = DRCMR(phy->vchan->drcmr); writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); @@ -166,18 +312,29 @@ static void enable_chan(struct mmp_pdma_phy *phy) writel(dalgn, phy->base + DALGN); reg = (phy->idx << 2) + DCSR; - writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg); + writel(readl(phy->base + reg) | pdev->ops->run_bits, + phy->base + reg); } static void disable_chan(struct mmp_pdma_phy *phy) { - u32 reg; + u32 reg, dcsr; if (!phy) return; reg = (phy->idx << 2) + DCSR; - writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg); + dcsr = readl(phy->base + reg); + + if (phy->vchan) { + struct mmp_pdma_device *pdev; + + pdev = to_mmp_pdma_dev(phy->vchan->chan.device); + writel(dcsr & ~pdev->ops->run_bits, phy->base + reg); + } else { + /* If no vchan, just clear the RUN bit */ + writel(dcsr & ~DCSR_RUN, phy->base + reg); + } } static int clear_chan_irq(struct mmp_pdma_phy *phy) @@ -296,6 +453,7 @@ static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) static void start_pending_queue(struct mmp_pdma_chan *chan) { struct mmp_pdma_desc_sw *desc; + struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device); /* still in running, irq will start the pending list */ if (!chan->idle) { @@ -330,7 +488,7 @@ static void start_pending_queue(struct mmp_pdma_chan *chan) * Program the descriptor's address into the DMA controller, * then start the DMA transaction */ - set_desc(chan->phy, desc->async_tx.phys); + pdev->ops->write_next_addr(chan->phy, desc->async_tx.phys); enable_chan(chan->phy); chan->idle = false; } @@ -446,15 +604,14 @@ mmp_pdma_prep_memcpy(struct dma_chan *dchan, size_t len, unsigned long flags) { struct mmp_pdma_chan *chan; + struct mmp_pdma_device *pdev; struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; size_t copy = 0; - if (!dchan) - return NULL; - - if (!len) + if (!dchan || !len) return NULL; + pdev = to_mmp_pdma_dev(dchan->device); chan = to_mmp_pdma_chan(dchan); chan->byte_align = false; @@ -477,13 +634,14 @@ mmp_pdma_prep_memcpy(struct dma_chan *dchan, chan->byte_align = true; new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy); - new->desc.dsadr = dma_src; - new->desc.dtadr = dma_dst; + pdev->ops->set_desc_src_addr(&new->desc, dma_src); + pdev->ops->set_desc_dst_addr(&new->desc, dma_dst); if (!first) first = new; else - prev->desc.ddadr = new->async_tx.phys; + pdev->ops->set_desc_next_addr(&prev->desc, + new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -527,6 +685,7 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, unsigned long flags, void *context) { struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); + struct mmp_pdma_device *pdev = to_mmp_pdma_dev(dchan->device); struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; size_t len, avail; struct scatterlist *sg; @@ -558,17 +717,18 @@ mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len); if (dir == DMA_MEM_TO_DEV) { - new->desc.dsadr = addr; + pdev->ops->set_desc_src_addr(&new->desc, addr); new->desc.dtadr = chan->dev_addr; } else { new->desc.dsadr = chan->dev_addr; - new->desc.dtadr = addr; + pdev->ops->set_desc_dst_addr(&new->desc, addr); } if (!first) first = new; else - prev->desc.ddadr = new->async_tx.phys; + pdev->ops->set_desc_next_addr(&prev->desc, + new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -608,12 +768,15 @@ mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, unsigned long flags) { struct mmp_pdma_chan *chan; + struct mmp_pdma_device *pdev; struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; dma_addr_t dma_src, dma_dst; if (!dchan || !len || !period_len) return NULL; + pdev = to_mmp_pdma_dev(dchan->device); + /* the buffer length must be a multiple of period_len */ if (len % period_len != 0) return NULL; @@ -650,13 +813,14 @@ mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | (DCMD_LENGTH & period_len)); - new->desc.dsadr = dma_src; - new->desc.dtadr = dma_dst; + pdev->ops->set_desc_src_addr(&new->desc, dma_src); + pdev->ops->set_desc_dst_addr(&new->desc, dma_dst); if (!first) first = new; else - prev->desc.ddadr = new->async_tx.phys; + pdev->ops->set_desc_next_addr(&prev->desc, + new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -677,7 +841,7 @@ mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, first->async_tx.cookie = -EBUSY; /* make the cyclic link */ - new->desc.ddadr = first->async_tx.phys; + pdev->ops->set_desc_next_addr(&new->desc, first->async_tx.phys); chan->cyclic_first = first; return &first->async_tx; @@ -727,12 +891,6 @@ static int mmp_pdma_config_write(struct dma_chan *dchan, chan->dir = direction; chan->dev_addr = addr; - /* FIXME: drivers should be ported over to use the filter - * function. Once that's done, the following two lines can - * be removed. - */ - if (cfg->slave_id) - chan->drcmr = cfg->slave_id; return 0; } @@ -769,7 +927,9 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, dma_cookie_t cookie) { struct mmp_pdma_desc_sw *sw; - u32 curr, residue = 0; + struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device); + u64 curr; + u32 residue = 0; bool passed = false; bool cyclic = chan->cyclic_first != NULL; @@ -781,17 +941,18 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, return 0; if (chan->dir == DMA_DEV_TO_MEM) - curr = readl(chan->phy->base + DTADR(chan->phy->idx)); + curr = pdev->ops->read_dst_addr(chan->phy); else - curr = readl(chan->phy->base + DSADR(chan->phy->idx)); + curr = pdev->ops->read_src_addr(chan->phy); list_for_each_entry(sw, &chan->chain_running, node) { - u32 start, end, len; + u64 start, end; + u32 len; if (chan->dir == DMA_DEV_TO_MEM) - start = sw->desc.dtadr; + start = pdev->ops->get_desc_dst_addr(&sw->desc); else - start = sw->desc.dsadr; + start = pdev->ops->get_desc_src_addr(&sw->desc); len = sw->desc.dcmd & DCMD_LENGTH; end = start + len; @@ -807,7 +968,7 @@ static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, if (passed) { residue += len; } else if (curr >= start && curr <= end) { - residue += end - curr; + residue += (u32)(end - curr); passed = true; } @@ -938,7 +1099,7 @@ static void dma_do_tasklet(struct tasklet_struct *t) } } -static int mmp_pdma_remove(struct platform_device *op) +static void mmp_pdma_remove(struct platform_device *op) { struct mmp_pdma_device *pdev = platform_get_drvdata(op); struct mmp_pdma_phy *phy; @@ -964,7 +1125,6 @@ static int mmp_pdma_remove(struct platform_device *op) } dma_async_device_unregister(&pdev->device); - return 0; } static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) @@ -1002,9 +1162,42 @@ static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) return 0; } +static const struct mmp_pdma_ops marvell_pdma_v1_ops = { + .write_next_addr = write_next_addr_32, + .read_src_addr = read_src_addr_32, + .read_dst_addr = read_dst_addr_32, + .set_desc_next_addr = set_desc_next_addr_32, + .set_desc_src_addr = set_desc_src_addr_32, + .set_desc_dst_addr = set_desc_dst_addr_32, + .get_desc_src_addr = get_desc_src_addr_32, + .get_desc_dst_addr = get_desc_dst_addr_32, + .run_bits = (DCSR_RUN), + .dma_mask = 0, /* let OF/platform set DMA mask */ +}; + +static const struct mmp_pdma_ops spacemit_k1_pdma_ops = { + .write_next_addr = write_next_addr_64, + .read_src_addr = read_src_addr_64, + .read_dst_addr = read_dst_addr_64, + .set_desc_next_addr = set_desc_next_addr_64, + .set_desc_src_addr = set_desc_src_addr_64, + .set_desc_dst_addr = set_desc_dst_addr_64, + .get_desc_src_addr = get_desc_src_addr_64, + .get_desc_dst_addr = get_desc_dst_addr_64, + .run_bits = (DCSR_RUN | DCSR_LPAEEN), + .dma_mask = DMA_BIT_MASK(64), /* force 64-bit DMA addr capability */ +}; + static const struct of_device_id mmp_pdma_dt_ids[] = { - { .compatible = "marvell,pdma-1.0", }, - {} + { + .compatible = "marvell,pdma-1.0", + .data = &marvell_pdma_v1_ops + }, { + .compatible = "spacemit,k1-pdma", + .data = &spacemit_k1_pdma_ops + }, { + /* sentinel */ + } }; MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids); @@ -1026,9 +1219,9 @@ static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, static int mmp_pdma_probe(struct platform_device *op) { struct mmp_pdma_device *pdev; - const struct of_device_id *of_id; struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); - struct resource *iores; + struct clk *clk; + struct reset_control *rst; int i, ret, irq = 0; int dma_channels = 0, irq_num = 0; const enum dma_slave_buswidth widths = @@ -1043,19 +1236,34 @@ static int mmp_pdma_probe(struct platform_device *op) spin_lock_init(&pdev->phy_lock); - iores = platform_get_resource(op, IORESOURCE_MEM, 0); - pdev->base = devm_ioremap_resource(pdev->dev, iores); + pdev->base = devm_platform_ioremap_resource(op, 0); if (IS_ERR(pdev->base)) return PTR_ERR(pdev->base); - of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev); - if (of_id) - of_property_read_u32(pdev->dev->of_node, "#dma-channels", - &dma_channels); - else if (pdata && pdata->dma_channels) + clk = devm_clk_get_optional_enabled(pdev->dev, NULL); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + rst = devm_reset_control_get_optional_exclusive_deasserted(pdev->dev, + NULL); + if (IS_ERR(rst)) + return PTR_ERR(rst); + + pdev->ops = of_device_get_match_data(&op->dev); + if (!pdev->ops) + return -ENODEV; + + if (pdev->dev->of_node) { + /* Parse new and deprecated dma-channels properties */ + if (of_property_read_u32(pdev->dev->of_node, "dma-channels", + &dma_channels)) + of_property_read_u32(pdev->dev->of_node, "#dma-channels", + &dma_channels); + } else if (pdata && pdata->dma_channels) { dma_channels = pdata->dma_channels; - else + } else { dma_channels = 32; /* default 32 channel */ + } pdev->dma_channels = dma_channels; for (i = 0; i < dma_channels; i++) { @@ -1106,7 +1314,10 @@ static int mmp_pdma_probe(struct platform_device *op) pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - if (pdev->dev->coherent_dma_mask) + /* Set DMA mask based on ops->dma_mask, or OF/platform */ + if (pdev->ops->dma_mask) + dma_set_mask(pdev->dev, pdev->ops->dma_mask); + else if (pdev->dev->coherent_dma_mask) dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask); else dma_set_mask(pdev->dev, DMA_BIT_MASK(64)); @@ -1123,6 +1334,7 @@ static int mmp_pdma_probe(struct platform_device *op) mmp_pdma_dma_xlate, pdev); if (ret < 0) { dev_err(&op->dev, "of_dma_controller_register failed\n"); + dma_async_device_unregister(&pdev->device); return ret; } } diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index a262e0eb4cc9..ba03321eeff7 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -14,9 +14,9 @@ #include <linux/slab.h> #include <linux/dmaengine.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <linux/device.h> -#include <linux/platform_data/dma-mmp_tdma.h> -#include <linux/of_device.h> +#include <linux/genalloc.h> #include <linux/of_dma.h> #include "dmaengine.h" @@ -552,12 +552,9 @@ static void mmp_tdma_issue_pending(struct dma_chan *chan) mmp_tdma_enable_chan(tdmac); } -static int mmp_tdma_remove(struct platform_device *pdev) +static void mmp_tdma_remove(struct platform_device *pdev) { - if (pdev->dev.of_node) - of_dma_controller_free(pdev->dev.of_node); - - return 0; + of_dma_controller_free(pdev->dev.of_node); } static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev, @@ -637,19 +634,13 @@ MODULE_DEVICE_TABLE(of, mmp_tdma_dt_ids); static int mmp_tdma_probe(struct platform_device *pdev) { enum mmp_tdma_type type; - const struct of_device_id *of_id; struct mmp_tdma_device *tdev; - struct resource *iores; int i, ret; int irq = 0, irq_num = 0; int chan_num = TDMA_CHANNEL_NUM; struct gen_pool *pool = NULL; - of_id = of_match_device(mmp_tdma_dt_ids, &pdev->dev); - if (of_id) - type = (enum mmp_tdma_type) of_id->data; - else - type = platform_get_device_id(pdev)->driver_data; + type = (kernel_ulong_t)device_get_match_data(&pdev->dev); /* always have couple channels */ tdev = devm_kzalloc(&pdev->dev, sizeof(*tdev), GFP_KERNEL); @@ -663,17 +654,13 @@ static int mmp_tdma_probe(struct platform_device *pdev) irq_num++; } - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); - tdev->base = devm_ioremap_resource(&pdev->dev, iores); + tdev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(tdev->base)) return PTR_ERR(tdev->base); INIT_LIST_HEAD(&tdev->device.channels); - if (pdev->dev.of_node) - pool = of_gen_pool_get(pdev->dev.of_node, "asram", 0); - else - pool = sram_get_gpool("asram"); + pool = of_gen_pool_get(pdev->dev.of_node, "asram", 0); if (!pool) { dev_err(&pdev->dev, "asram pool not available\n"); return -ENOMEM; @@ -731,32 +718,22 @@ static int mmp_tdma_probe(struct platform_device *pdev) return ret; } - if (pdev->dev.of_node) { - ret = of_dma_controller_register(pdev->dev.of_node, - mmp_tdma_xlate, tdev); - if (ret) { - dev_err(tdev->device.dev, - "failed to register controller\n"); - return ret; - } + ret = of_dma_controller_register(pdev->dev.of_node, + mmp_tdma_xlate, tdev); + if (ret) { + dev_err(tdev->device.dev, "failed to register controller\n"); + return ret; } dev_info(tdev->device.dev, "initialized\n"); return 0; } -static const struct platform_device_id mmp_tdma_id_table[] = { - { "mmp-adma", MMP_AUD_TDMA }, - { "pxa910-squ", PXA910_SQU }, - { }, -}; - static struct platform_driver mmp_tdma_driver = { .driver = { .name = "mmp-tdma", .of_match_table = mmp_tdma_dt_ids, }, - .id_table = mmp_tdma_id_table, .probe = mmp_tdma_probe, .remove = mmp_tdma_remove, }; @@ -765,6 +742,5 @@ module_platform_driver(mmp_tdma_driver); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("MMP Two-Channel DMA Driver"); -MODULE_ALIAS("platform:mmp-tdma"); MODULE_AUTHOR("Leo Yan <leoy@marvell.com>"); MODULE_AUTHOR("Zhangfei Gao <zhangfei.gao@marvell.com>"); diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c index 74755093e14b..de09e1ab7767 100644 --- a/drivers/dma/moxart-dma.c +++ b/drivers/dma/moxart-dma.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * MOXA ART SoCs DMA Engine support. * * Copyright (C) 2013 Jonas Jensen * * Jonas Jensen <jonas.jensen@gmail.com> - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #include <linux/dmaengine.h> @@ -127,7 +124,7 @@ struct moxart_desc { unsigned int dma_cycles; struct virt_dma_desc vd; uint8_t es; - struct moxart_sg sg[]; + struct moxart_sg sg[] __counted_by(sglen); }; struct moxart_chan { @@ -151,11 +148,6 @@ struct moxart_dmadev { unsigned int irq; }; -struct moxart_filter_data { - struct moxart_dmadev *mdc; - struct of_phandle_args *dma_spec; -}; - static const unsigned int es_bytes[] = { [MOXART_DMA_DATA_TYPE_S8] = 1, [MOXART_DMA_DATA_TYPE_S16] = 2, @@ -312,6 +304,7 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg( d = kzalloc(struct_size(d, sg, sg_len), GFP_ATOMIC); if (!d) return NULL; + d->sglen = sg_len; d->dma_dir = dir; d->dev_addr = dev_addr; @@ -322,8 +315,6 @@ static struct dma_async_tx_descriptor *moxart_prep_slave_sg( d->sg[i].len = sg_dma_len(sgent); } - d->sglen = sg_len; - ch->error = 0; return vchan_tx_prep(&ch->vc, &d->vd, tx_flags); @@ -566,7 +557,6 @@ static int moxart_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct device_node *node = dev->of_node; - struct resource *res; void __iomem *dma_base_addr; int ret, i; unsigned int irq; @@ -583,8 +573,7 @@ static int moxart_probe(struct platform_device *pdev) return -EINVAL; } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dma_base_addr = devm_ioremap_resource(dev, res); + dma_base_addr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dma_base_addr)) return PTR_ERR(dma_base_addr); @@ -635,7 +624,7 @@ static int moxart_probe(struct platform_device *pdev) return 0; } -static int moxart_remove(struct platform_device *pdev) +static void moxart_remove(struct platform_device *pdev) { struct moxart_dmadev *m = platform_get_drvdata(pdev); @@ -645,8 +634,6 @@ static int moxart_remove(struct platform_device *pdev) if (pdev->dev.of_node) of_dma_controller_free(pdev->dev.of_node); - - return 0; } static const struct of_device_id moxart_dma_match[] = { @@ -657,7 +644,7 @@ MODULE_DEVICE_TABLE(of, moxart_dma_match); static struct platform_driver moxart_driver = { .probe = moxart_probe, - .remove = moxart_remove, + .remove = moxart_remove, .driver = { .name = "moxart-dma-engine", .of_match_table = moxart_dma_match, diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index c1a69149c8bf..bf131cb5db66 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -36,11 +36,11 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/slab.h> +#include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/of_dma.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/random.h> @@ -813,6 +813,7 @@ inline bool is_buswidth_valid(u8 buswidth, bool is_mpc8308) case 16: if (is_mpc8308) return false; + break; case 1: case 2: case 4: @@ -1083,7 +1084,7 @@ err: return retval; } -static int mpc_dma_remove(struct platform_device *op) +static void mpc_dma_remove(struct platform_device *op) { struct device *dev = &op->dev; struct mpc_dma *mdma = dev_get_drvdata(dev); @@ -1098,8 +1099,6 @@ static int mpc_dma_remove(struct platform_device *op) free_irq(mdma->irq, mdma); irq_dispose_mapping(mdma->irq); tasklet_kill(&mdma->tasklet); - - return 0; } static const struct of_device_id mpc_dma_match[] = { diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 23b232b57518..5e8386296046 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -10,8 +10,8 @@ #include <linux/dma-mapping.h> #include <linux/spinlock.h> #include <linux/interrupt.h> -#include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <linux/memory.h> #include <linux/clk.h> #include <linux/of.h> @@ -414,7 +414,7 @@ mv_xor_tx_submit(struct dma_async_tx_descriptor *tx) if (!mv_chan_is_busy(mv_chan)) { u32 current_desc = mv_chan_get_current_desc(mv_chan); /* - * and the curren desc is the end of the chain before + * and the current desc is the end of the chain before * the append, then we need to start the channel */ if (current_desc == old_chain_tail->async_tx.phys) @@ -1013,7 +1013,7 @@ static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan) dma_async_device_unregister(&mv_chan->dmadev); - dma_free_coherent(dev, MV_XOR_POOL_SIZE, + dma_free_wc(dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); dma_unmap_single(dev, mv_chan->dummy_src_addr, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); @@ -1061,8 +1061,16 @@ mv_xor_channel_add(struct mv_xor_device *xordev, */ mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev, mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); + if (dma_mapping_error(dma_dev->dev, mv_chan->dummy_src_addr)) + return ERR_PTR(-ENOMEM); + mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev, mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); + if (dma_mapping_error(dma_dev->dev, mv_chan->dummy_dst_addr)) { + ret = -ENOMEM; + goto err_unmap_src; + } + /* allocate coherent memory for hardware descriptors * note: writecombine gives slightly better performance, but @@ -1071,10 +1079,12 @@ mv_xor_channel_add(struct mv_xor_device *xordev, mv_chan->dma_desc_pool_virt = dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool, GFP_KERNEL); - if (!mv_chan->dma_desc_pool_virt) - return ERR_PTR(-ENOMEM); + if (!mv_chan->dma_desc_pool_virt) { + ret = -ENOMEM; + goto err_unmap_dst; + } - /* discover transaction capabilites from the platform data */ + /* discover transaction capabilities from the platform data */ dma_dev->cap_mask = cap_mask; INIT_LIST_HEAD(&dma_dev->channels); @@ -1153,8 +1163,15 @@ mv_xor_channel_add(struct mv_xor_device *xordev, err_free_irq: free_irq(mv_chan->irq, mv_chan); err_free_dma: - dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE, + dma_free_wc(&pdev->dev, MV_XOR_POOL_SIZE, mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); +err_unmap_dst: + dma_unmap_single(dma_dev->dev, mv_chan->dummy_dst_addr, + MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); +err_unmap_src: + dma_unmap_single(dma_dev->dev, mv_chan->dummy_src_addr, + MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); + return ERR_PTR(ret); } @@ -1328,13 +1345,8 @@ static int mv_xor_probe(struct platform_device *pdev) * setting up. In non-dt case it can only be the legacy one. */ xordev->xor_type = XOR_ORION; - if (pdev->dev.of_node) { - const struct of_device_id *of_id = - of_match_device(mv_xor_dt_ids, - &pdev->dev); - - xordev->xor_type = (uintptr_t)of_id->data; - } + if (pdev->dev.of_node) + xordev->xor_type = (uintptr_t)device_get_match_data(&pdev->dev); /* * (Re-)program MBUS remapping windows if we are asked to. @@ -1374,10 +1386,9 @@ static int mv_xor_probe(struct platform_device *pdev) return 0; if (pdev->dev.of_node) { - struct device_node *np; int i = 0; - for_each_child_of_node(pdev->dev.of_node, np) { + for_each_child_of_node_scoped(pdev->dev.of_node, np) { struct mv_xor_chan *chan; dma_cap_mask_t cap_mask; int irq; diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h index d86086b05b0e..c87cefd38a07 100644 --- a/drivers/dma/mv_xor.h +++ b/drivers/dma/mv_xor.h @@ -99,7 +99,7 @@ struct mv_xor_device { * @common: common dmaengine channel object members * @slots_allocated: records the actual size of the descriptor slot pool * @irq_tasklet: bottom half where mv_xor_slot_cleanup runs - * @op_in_desc: new mode of driver, each op is writen to descriptor. + * @op_in_desc: new mode of driver, each op is written to descriptor. */ struct mv_xor_chan { int pending; diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c index 9b0d463f89bb..cad4d4fb51ac 100644 --- a/drivers/dma/mv_xor_v2.c +++ b/drivers/dma/mv_xor_v2.c @@ -149,7 +149,7 @@ struct mv_xor_v2_descriptor { * @desc_size: HW descriptor size * @npendings: number of pending descriptors (for which tx_submit has * @hw_queue_idx: HW queue index - * @msi_desc: local interrupt descriptor information + * @irq: The Linux interrupt number * been called, but not yet issue_pending) */ struct mv_xor_v2_device { @@ -168,14 +168,14 @@ struct mv_xor_v2_device { int desc_size; unsigned int npendings; unsigned int hw_queue_idx; - struct msi_desc *msi_desc; + unsigned int irq; }; /** * struct mv_xor_v2_sw_desc - implements a xor SW descriptor * @idx: descriptor index * @async_tx: support for the async_tx api - * @hw_desc: assosiated HW descriptor + * @hw_desc: associated HW descriptor * @free_list: node of the free SW descriprots list */ struct mv_xor_v2_sw_desc { @@ -313,7 +313,7 @@ mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx) "%s sw_desc %p: async_tx %p\n", __func__, sw_desc, &sw_desc->async_tx); - /* assign coookie */ + /* assign cookie */ spin_lock_bh(&xor_dev->lock); cookie = dma_cookie_assign(tx); @@ -591,14 +591,14 @@ static void mv_xor_v2_tasklet(struct tasklet_struct *t) dma_run_dependencies(&next_pending_sw_desc->async_tx); /* Lock the channel */ - spin_lock_bh(&xor_dev->lock); + spin_lock(&xor_dev->lock); /* add the SW descriptor to the free descriptors list */ list_add(&next_pending_sw_desc->free_list, &xor_dev->free_sw_desc); /* Release the channel */ - spin_unlock_bh(&xor_dev->lock); + spin_unlock(&xor_dev->lock); /* increment the next descriptor */ pending_ptr++; @@ -635,7 +635,7 @@ static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev) writel(MV_XOR_V2_DESC_NUM, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF); - /* write the DESQ address to the DMA enngine*/ + /* write the DESQ address to the DMA engine*/ writel(lower_32_bits(xor_dev->hw_desq), xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF); writel(upper_32_bits(xor_dev->hw_desq), @@ -714,11 +714,9 @@ static int mv_xor_v2_resume(struct platform_device *dev) static int mv_xor_v2_probe(struct platform_device *pdev) { struct mv_xor_v2_device *xor_dev; - struct resource *res; int i, ret = 0; struct dma_device *dma_dev; struct mv_xor_v2_sw_desc *sw_desc; - struct msi_desc *msi_desc; BUILD_BUG_ON(sizeof(struct mv_xor_v2_descriptor) != MV_XOR_V2_EXT_DESC_SIZE); @@ -727,13 +725,11 @@ static int mv_xor_v2_probe(struct platform_device *pdev) if (!xor_dev) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - xor_dev->dma_base = devm_ioremap_resource(&pdev->dev, res); + xor_dev->dma_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(xor_dev->dma_base)) return PTR_ERR(xor_dev->dma_base); - res = platform_get_resource(pdev, IORESOURCE_MEM, 1); - xor_dev->glob_base = devm_ioremap_resource(&pdev->dev, res); + xor_dev->glob_base = devm_platform_ioremap_resource(pdev, 1); if (IS_ERR(xor_dev->glob_base)) return PTR_ERR(xor_dev->glob_base); @@ -743,41 +739,22 @@ static int mv_xor_v2_probe(struct platform_device *pdev) if (ret) return ret; - xor_dev->reg_clk = devm_clk_get(&pdev->dev, "reg"); - if (PTR_ERR(xor_dev->reg_clk) != -ENOENT) { - if (!IS_ERR(xor_dev->reg_clk)) { - ret = clk_prepare_enable(xor_dev->reg_clk); - if (ret) - return ret; - } else { - return PTR_ERR(xor_dev->reg_clk); - } - } + xor_dev->reg_clk = devm_clk_get_optional_enabled(&pdev->dev, "reg"); + if (IS_ERR(xor_dev->reg_clk)) + return PTR_ERR(xor_dev->reg_clk); - xor_dev->clk = devm_clk_get(&pdev->dev, NULL); - if (PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) { - ret = EPROBE_DEFER; - goto disable_reg_clk; - } - if (!IS_ERR(xor_dev->clk)) { - ret = clk_prepare_enable(xor_dev->clk); - if (ret) - goto disable_reg_clk; - } + xor_dev->clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(xor_dev->clk)) + return PTR_ERR(xor_dev->clk); - ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1, - mv_xor_v2_set_msi_msg); + ret = platform_device_msi_init_and_alloc_irqs(&pdev->dev, 1, + mv_xor_v2_set_msi_msg); if (ret) - goto disable_clk; + return ret; - msi_desc = first_msi_entry(&pdev->dev); - if (!msi_desc) { - ret = -ENODEV; - goto free_msi_irqs; - } - xor_dev->msi_desc = msi_desc; + xor_dev->irq = msi_get_virq(&pdev->dev, 0); - ret = devm_request_irq(&pdev->dev, msi_desc->irq, + ret = devm_request_irq(&pdev->dev, xor_dev->irq, mv_xor_v2_interrupt_handler, 0, dev_name(&pdev->dev), xor_dev); if (ret) @@ -874,15 +851,11 @@ free_hw_desq: xor_dev->desc_size * MV_XOR_V2_DESC_NUM, xor_dev->hw_desq_virt, xor_dev->hw_desq); free_msi_irqs: - platform_msi_domain_free_irqs(&pdev->dev); -disable_clk: - clk_disable_unprepare(xor_dev->clk); -disable_reg_clk: - clk_disable_unprepare(xor_dev->reg_clk); + platform_device_msi_free_irqs_all(&pdev->dev); return ret; } -static int mv_xor_v2_remove(struct platform_device *pdev) +static void mv_xor_v2_remove(struct platform_device *pdev) { struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev); @@ -892,15 +865,11 @@ static int mv_xor_v2_remove(struct platform_device *pdev) xor_dev->desc_size * MV_XOR_V2_DESC_NUM, xor_dev->hw_desq_virt, xor_dev->hw_desq); - devm_free_irq(&pdev->dev, xor_dev->msi_desc->irq, xor_dev); + devm_free_irq(&pdev->dev, xor_dev->irq, xor_dev); - platform_msi_domain_free_irqs(&pdev->dev); + platform_device_msi_free_irqs_all(&pdev->dev); tasklet_kill(&xor_dev->irq_tasklet); - - clk_disable_unprepare(xor_dev->clk); - - return 0; } #ifdef CONFIG_OF @@ -925,4 +894,3 @@ static struct platform_driver mv_xor_v2_driver = { module_platform_driver(mv_xor_v2_driver); MODULE_DESCRIPTION("DMA engine driver for Marvell's Version 2 of XOR engine"); -MODULE_LICENSE("GPL"); diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index 994fc4d2aca4..cfb9962417ef 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -21,7 +21,6 @@ #include <linux/module.h> #include <linux/stmp_device.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/list.h> #include <linux/dma/mxs-dma.h> @@ -670,7 +669,7 @@ static enum dma_status mxs_dma_tx_status(struct dma_chan *chan, return mxs_chan->status; } -static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma) +static int mxs_dma_init(struct mxs_dma_engine *mxs_dma) { int ret; @@ -741,12 +740,11 @@ static struct dma_chan *mxs_dma_xlate(struct of_phandle_args *dma_spec, ofdma->of_node); } -static int __init mxs_dma_probe(struct platform_device *pdev) +static int mxs_dma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; const struct mxs_dma_type *dma_type; struct mxs_dma_engine *mxs_dma; - struct resource *iores; int ret, i; mxs_dma = devm_kzalloc(&pdev->dev, sizeof(*mxs_dma), GFP_KERNEL); @@ -763,8 +761,7 @@ static int __init mxs_dma_probe(struct platform_device *pdev) mxs_dma->type = dma_type->type; mxs_dma->dev_id = dma_type->id; - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); - mxs_dma->base = devm_ioremap_resource(&pdev->dev, iores); + mxs_dma->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mxs_dma->base)) return PTR_ERR(mxs_dma->base); @@ -839,10 +836,7 @@ static struct platform_driver mxs_dma_driver = { .name = "mxs-dma", .of_match_table = mxs_dma_dt_ids, }, + .probe = mxs_dma_probe, }; -static int __init mxs_dma_module_init(void) -{ - return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe); -} -subsys_initcall(mxs_dma_module_init); +builtin_platform_driver(mxs_dma_driver); diff --git a/drivers/dma/nbpfaxi.c b/drivers/dma/nbpfaxi.c index 9c52c57919c6..334425faac00 100644 --- a/drivers/dma/nbpfaxi.c +++ b/drivers/dma/nbpfaxi.c @@ -15,7 +15,6 @@ #include <linux/log2.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/slab.h> @@ -712,6 +711,9 @@ static int nbpf_desc_page_alloc(struct nbpf_channel *chan) list_add_tail(&ldesc->node, &lhead); ldesc->hwdesc_dma_addr = dma_map_single(dchan->device->dev, hwdesc, sizeof(*hwdesc), DMA_TO_DEVICE); + if (dma_mapping_error(dchan->device->dev, + ldesc->hwdesc_dma_addr)) + goto unmap_error; dev_dbg(dev, "%s(): mapped 0x%p to %pad\n", __func__, hwdesc, &ldesc->hwdesc_dma_addr); @@ -738,6 +740,16 @@ static int nbpf_desc_page_alloc(struct nbpf_channel *chan) spin_unlock_irq(&chan->lock); return ARRAY_SIZE(dpage->desc); + +unmap_error: + while (i--) { + ldesc--; hwdesc--; + + dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr, + sizeof(hwdesc), DMA_TO_DEVICE); + } + + return -ENOMEM; } static void nbpf_desc_put(struct nbpf_desc *desc) @@ -898,7 +910,7 @@ static int nbpf_config(struct dma_chan *dchan, /* * We could check config->slave_id to match chan->terminal here, * but with DT they would be coming from the same source, so - * such a check would be superflous + * such a check would be superfluous */ chan->slave_dst_addr = config->dst_addr; @@ -1294,7 +1306,6 @@ static int nbpf_probe(struct platform_device *pdev) struct device_node *np = dev->of_node; struct nbpf_device *nbpf; struct dma_device *dma_dev; - struct resource *iomem, *irq_res; const struct nbpf_config *cfg; int num_channels; int ret, irq, eirq, i; @@ -1318,8 +1329,7 @@ static int nbpf_probe(struct platform_device *pdev) dma_dev = &nbpf->dma_dev; dma_dev->dev = dev; - iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - nbpf->base = devm_ioremap_resource(dev, iomem); + nbpf->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(nbpf->base)) return PTR_ERR(nbpf->base); @@ -1335,13 +1345,11 @@ static int nbpf_probe(struct platform_device *pdev) nbpf->config = cfg; for (i = 0; irqs < ARRAY_SIZE(irqbuf); i++) { - irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, i); - if (!irq_res) - break; - - for (irq = irq_res->start; irq <= irq_res->end; - irq++, irqs++) - irqbuf[irqs] = irq; + irq = platform_get_irq_optional(pdev, i); + if (irq < 0 && irq != -ENXIO) + return irq; + if (irq > 0) + irqbuf[irqs++] = irq; } /* @@ -1356,7 +1364,7 @@ static int nbpf_probe(struct platform_device *pdev) if (irqs == 1) { eirq = irqbuf[0]; - for (i = 0; i <= num_channels; i++) + for (i = 0; i < num_channels; i++) nbpf->chan[i].irq = irqbuf[0]; } else { eirq = platform_get_irq_byname(pdev, "error"); @@ -1366,16 +1374,15 @@ static int nbpf_probe(struct platform_device *pdev) if (irqs == num_channels + 1) { struct nbpf_channel *chan; - for (i = 0, chan = nbpf->chan; i <= num_channels; + for (i = 0, chan = nbpf->chan; i < num_channels; i++, chan++) { /* Skip the error IRQ */ if (irqbuf[i] == eirq) i++; + if (i >= ARRAY_SIZE(irqbuf)) + return -EINVAL; chan->irq = irqbuf[i]; } - - if (chan != nbpf->chan + num_channels) - return -EINVAL; } else { /* 2 IRQs and more than one channel */ if (irqbuf[0] == eirq) @@ -1383,7 +1390,7 @@ static int nbpf_probe(struct platform_device *pdev) else irq = irqbuf[0]; - for (i = 0; i <= num_channels; i++) + for (i = 0; i < num_channels; i++) nbpf->chan[i].irq = irq; } } @@ -1459,7 +1466,7 @@ e_clk_off: return ret; } -static int nbpf_remove(struct platform_device *pdev) +static void nbpf_remove(struct platform_device *pdev) { struct nbpf_device *nbpf = platform_get_drvdata(pdev); int i; @@ -1477,8 +1484,6 @@ static int nbpf_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&nbpf->dma_dev); clk_disable_unprepare(nbpf->clk); - - return 0; } static const struct platform_device_id nbpf_ids[] = { @@ -1495,7 +1500,6 @@ static const struct platform_device_id nbpf_ids[] = { }; MODULE_DEVICE_TABLE(platform, nbpf_ids); -#ifdef CONFIG_PM static int nbpf_runtime_suspend(struct device *dev) { struct nbpf_device *nbpf = dev_get_drvdata(dev); @@ -1508,17 +1512,16 @@ static int nbpf_runtime_resume(struct device *dev) struct nbpf_device *nbpf = dev_get_drvdata(dev); return clk_prepare_enable(nbpf->clk); } -#endif static const struct dev_pm_ops nbpf_pm_ops = { - SET_RUNTIME_PM_OPS(nbpf_runtime_suspend, nbpf_runtime_resume, NULL) + RUNTIME_PM_OPS(nbpf_runtime_suspend, nbpf_runtime_resume, NULL) }; static struct platform_driver nbpf_driver = { .driver = { .name = "dma-nbpf", .of_match_table = nbpf_match, - .pm = &nbpf_pm_ops, + .pm = pm_ptr(&nbpf_pm_ops), }, .id_table = nbpf_ids, .probe = nbpf_probe, diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c index ec00b20ae8e4..423442e55d36 100644 --- a/drivers/dma/of-dma.c +++ b/drivers/dma/of-dma.c @@ -26,10 +26,10 @@ static DEFINE_MUTEX(of_dma_lock); * * Finds a DMA controller with matching device node and number for dma cells * in a list of registered DMA controllers. If a match is found a valid pointer - * to the DMA data stored is retuned. A NULL pointer is returned if no match is + * to the DMA data stored is returned. A NULL pointer is returned if no match is * found. */ -static struct of_dma *of_dma_find_controller(struct of_phandle_args *dma_spec) +static struct of_dma *of_dma_find_controller(const struct of_phandle_args *dma_spec) { struct of_dma *ofdma; @@ -67,8 +67,12 @@ static struct dma_chan *of_dma_router_xlate(struct of_phandle_args *dma_spec, return NULL; ofdma_target = of_dma_find_controller(&dma_spec_target); - if (!ofdma_target) - return NULL; + if (!ofdma_target) { + ofdma->dma_router->route_free(ofdma->dma_router->dev, + route_data); + chan = ERR_PTR(-EPROBE_DEFER); + goto err; + } chan = ofdma_target->of_dma_xlate(&dma_spec_target, ofdma_target); if (IS_ERR_OR_NULL(chan)) { @@ -89,6 +93,7 @@ static struct dma_chan *of_dma_router_xlate(struct of_phandle_args *dma_spec, } } +err: /* * Need to put the node back since the ofdma->of_dma_route_allocate * has taken it for generating the new, translated dma_spec @@ -259,7 +264,7 @@ struct dma_chan *of_dma_request_slave_channel(struct device_node *np, } /* Silently fail if there is not even the "dmas" property */ - if (!of_find_property(np, "dmas", NULL)) + if (!of_property_present(np, "dmas")) return ERR_PTR(-ENODEV); count = of_property_count_strings(np, "dma-names"); @@ -337,7 +342,7 @@ EXPORT_SYMBOL_GPL(of_dma_simple_xlate); * * This function can be used as the of xlate callback for DMA driver which wants * to match the channel based on the channel id. When using this xlate function - * the #dma-cells propety of the DMA controller dt node needs to be set to 1. + * the #dma-cells property of the DMA controller dt node needs to be set to 1. * The data parameter of of_dma_controller_register must be a pointer to the * dma_device struct the function should match upon. * diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c index 1f0bbaed4643..57cec757d8f5 100644 --- a/drivers/dma/owl-dma.c +++ b/drivers/dma/owl-dma.c @@ -20,8 +20,9 @@ #include <linux/io.h> #include <linux/mm.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include "virt-dma.h" @@ -192,8 +193,8 @@ struct owl_dma_pchan { }; /** - * struct owl_dma_pchan - Wrapper for DMA ENGINE channel - * @vc: wrappped virtual channel + * struct owl_dma_vchan - Wrapper for DMA ENGINE channel + * @vc: wrapped virtual channel * @pchan: the physical channel utilized by this channel * @txd: active transaction on this channel * @cfg: slave configuration for this channel @@ -249,7 +250,7 @@ static void pchan_update(struct owl_dma_pchan *pchan, u32 reg, else regval &= ~val; - writel(val, pchan->base + reg); + writel(regval, pchan->base + reg); } static void pchan_writel(struct owl_dma_pchan *pchan, u32 reg, u32 data) @@ -273,7 +274,7 @@ static void dma_update(struct owl_dma *od, u32 reg, u32 val, bool state) else regval &= ~val; - writel(val, od->base + reg); + writel(regval, od->base + reg); } static void dma_writel(struct owl_dma *od, u32 reg, u32 data) @@ -1116,7 +1117,7 @@ static int owl_dma_probe(struct platform_device *pdev) dev_info(&pdev->dev, "dma-channels %d, dma-requests %d\n", nr_channels, nr_requests); - od->devid = (enum owl_dma_id)of_device_get_match_data(&pdev->dev); + od->devid = (uintptr_t)of_device_get_match_data(&pdev->dev); od->nr_pchans = nr_channels; od->nr_vchans = nr_requests; @@ -1155,7 +1156,7 @@ static int owl_dma_probe(struct platform_device *pdev) } /* - * Eventhough the DMA controller is capable of generating 4 + * Even though the DMA controller is capable of generating 4 * IRQ's for DMA priority feature, we only use 1 IRQ for * simplification. */ @@ -1230,7 +1231,7 @@ err_pool_free: return ret; } -static int owl_dma_remove(struct platform_device *pdev) +static void owl_dma_remove(struct platform_device *pdev) { struct owl_dma *od = platform_get_drvdata(pdev); @@ -1247,13 +1248,11 @@ static int owl_dma_remove(struct platform_device *pdev) clk_disable_unprepare(od->clk); dma_pool_destroy(od->lli_pool); - - return 0; } static struct platform_driver owl_dma_driver = { .probe = owl_dma_probe, - .remove = owl_dma_remove, + .remove = owl_dma_remove, .driver = { .name = "dma-owl", .of_match_table = of_match_ptr(owl_dma_match), diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index 1da04112fcdb..6b2793b07694 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -155,11 +155,6 @@ static inline struct device *chan2dev(struct dma_chan *chan) return &chan->dev->device; } -static inline struct device *chan2parent(struct dma_chan *chan) -{ - return chan->dev->device.parent; -} - static inline struct pch_dma_desc *pdc_first_active(struct pch_dma_chan *pd_chan) { @@ -835,7 +830,7 @@ static int pch_dma_probe(struct pci_dev *pdev, goto err_disable_pdev; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); + err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Cannot set proper DMA config\n"); goto err_free_res; diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 110de8a60058..82a9fe88ad54 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -404,6 +404,12 @@ enum desc_status { */ BUSY, /* + * Pause was called while descriptor was BUSY. Due to hardware + * limitations, only termination is possible for descriptors + * that have been paused. + */ + PAUSED, + /* * Sitting on the channel work_list but xfer done * by PL330 core */ @@ -1050,7 +1056,7 @@ static bool _trigger(struct pl330_thread *thrd) return true; } -static bool _start(struct pl330_thread *thrd) +static bool pl330_start_thread(struct pl330_thread *thrd) { switch (_state(thrd)) { case PL330_STATE_FAULT_COMPLETING: @@ -1702,7 +1708,7 @@ static int pl330_update(struct pl330_dmac *pl330) thrd->req_running = -1; /* Get going again ASAP */ - _start(thrd); + pl330_start_thread(thrd); /* For now, just make a list of callbacks to be done */ list_add_tail(&descdone->rqd, &pl330->req_done); @@ -2041,7 +2047,7 @@ static inline void fill_queue(struct dma_pl330_chan *pch) list_for_each_entry(desc, &pch->work_list, node) { /* If already submitted */ - if (desc->status == BUSY) + if (desc->status == BUSY || desc->status == PAUSED) continue; ret = pl330_submit_req(pch->thread, desc); @@ -2089,7 +2095,7 @@ static void pl330_tasklet(struct tasklet_struct *t) } else { /* Make sure the PL330 Channel thread is active */ spin_lock(&pch->thread->dmac->lock); - _start(pch->thread); + pl330_start_thread(pch->thread); spin_unlock(&pch->thread->dmac->lock); } @@ -2107,7 +2113,7 @@ static void pl330_tasklet(struct tasklet_struct *t) if (power_down) { pch->active = true; spin_lock(&pch->thread->dmac->lock); - _start(pch->thread); + pl330_start_thread(pch->thread); spin_unlock(&pch->thread->dmac->lock); power_down = false; } @@ -2326,6 +2332,7 @@ static int pl330_pause(struct dma_chan *chan) { struct dma_pl330_chan *pch = to_pchan(chan); struct pl330_dmac *pl330 = pch->dmac; + struct dma_pl330_desc *desc; unsigned long flags; pm_runtime_get_sync(pl330->ddma.dev); @@ -2335,6 +2342,10 @@ static int pl330_pause(struct dma_chan *chan) _stop(pch->thread); spin_unlock(&pl330->lock); + list_for_each_entry(desc, &pch->work_list, node) { + if (desc->status == BUSY) + desc->status = PAUSED; + } spin_unlock_irqrestore(&pch->lock, flags); pm_runtime_mark_last_busy(pl330->ddma.dev); pm_runtime_put_autosuspend(pl330->ddma.dev); @@ -2425,7 +2436,7 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, else if (running && desc == running) transferred = pl330_get_current_xferred_count(pch, desc); - else if (desc->status == BUSY) + else if (desc->status == BUSY || desc->status == PAUSED) /* * Busy but not running means either just enqueued, * or finished and not yet marked done @@ -2442,6 +2453,9 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, case DONE: ret = DMA_COMPLETE; break; + case PAUSED: + ret = DMA_PAUSED; + break; case PREP: case BUSY: ret = DMA_IN_PROGRESS; @@ -2571,6 +2585,7 @@ static struct dma_pl330_desc *pluck_desc(struct list_head *pool, desc->status = PREP; desc->txd.callback = NULL; + desc->txd.callback_result = NULL; } spin_unlock_irqrestore(lock, flags); @@ -2589,7 +2604,7 @@ static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch) /* If the DMAC pool is empty, alloc new */ if (!desc) { - DEFINE_SPINLOCK(lock); + static DEFINE_SPINLOCK(lock); LIST_HEAD(pool); if (!add_desc(&pool, &lock, GFP_ATOMIC, 1)) @@ -2752,7 +2767,6 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( return NULL; pch->cyclic = true; - desc->txd.flags = flags; return &desc->txd; } @@ -2804,8 +2818,6 @@ pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, desc->bytes_requested = len; - desc->txd.flags = flags; - return &desc->txd; } @@ -2889,7 +2901,6 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, } /* Return the last desc in the chain */ - desc->txd.flags = flg; return &desc->txd; } @@ -2968,7 +2979,7 @@ static int __maybe_unused pl330_suspend(struct device *dev) struct amba_device *pcdev = to_amba_device(dev); pm_runtime_force_suspend(dev); - amba_pclk_unprepare(pcdev); + clk_unprepare(pcdev->pclk); return 0; } @@ -2978,7 +2989,7 @@ static int __maybe_unused pl330_resume(struct device *dev) struct amba_device *pcdev = to_amba_device(dev); int ret; - ret = amba_pclk_prepare(pcdev); + ret = clk_prepare(pcdev->pclk); if (ret) return ret; @@ -3152,10 +3163,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) * This is the limit for transfers with a buswidth of 1, larger * buswidths will have larger limits. */ - ret = dma_set_max_seg_size(&adev->dev, 1900800); - if (ret) - dev_err(&adev->dev, "unable to set the seg size\n"); - + dma_set_max_seg_size(&adev->dev, 1900800); init_pl330_debugfs(pl330); dev_info(&adev->dev, @@ -3251,7 +3259,6 @@ MODULE_DEVICE_TABLE(amba, pl330_ids); static struct amba_driver pl330_driver = { .drv = { - .owner = THIS_MODULE, .name = "dma-pl330", .pm = &pl330_pm, }, diff --git a/drivers/dma/plx_dma.c b/drivers/dma/plx_dma.c index 166934544161..34b6416c3287 100644 --- a/drivers/dma/plx_dma.c +++ b/drivers/dma/plx_dma.c @@ -137,7 +137,7 @@ static void plx_dma_process_desc(struct plx_dma_dev *plxdev) struct plx_dma_desc *desc; u32 flags; - spin_lock_bh(&plxdev->ring_lock); + spin_lock(&plxdev->ring_lock); while (plxdev->tail != plxdev->head) { desc = plx_dma_get_desc(plxdev, plxdev->tail); @@ -165,7 +165,7 @@ static void plx_dma_process_desc(struct plx_dma_dev *plxdev) plxdev->tail++; } - spin_unlock_bh(&plxdev->ring_lock); + spin_unlock(&plxdev->ring_lock); } static void plx_dma_abort_desc(struct plx_dma_dev *plxdev) @@ -517,7 +517,6 @@ static int plx_dma_create(struct pci_dev *pdev) plxdev->bar = pcim_iomap_table(pdev)[0]; dma = &plxdev->dma_dev; - dma->chancnt = 1; INIT_LIST_HEAD(&dma->channels); dma_cap_set(DMA_MEMCPY, dma->cap_mask); dma->copy_align = DMAENGINE_ALIGN_1_BYTE; @@ -563,15 +562,9 @@ static int plx_dma_probe(struct pci_dev *pdev, if (rc) return rc; - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); - if (rc) - rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (rc) - return rc; - - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (rc) - rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (rc) return rc; diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index df7704053d91..61500ad7c850 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -9,7 +9,7 @@ */ /* - * This driver supports the asynchrounous DMA copy and RAID engines available + * This driver supports the asynchronous DMA copy and RAID engines available * on the AMCC PPC440SPe Processors. * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) * ADMA driver written by D.Williams. @@ -28,7 +28,7 @@ #include <linux/of.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <asm/dcr.h> #include <asm/dcr-regs.h> #include "adma.h" @@ -874,7 +874,7 @@ static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", __func__, src_cnt, state, addr_count, order); for (i = 0; i < src_cnt; i++) - pr_err("\t[%d] 0x%llx \n", i, srcs[i]); + pr_err("\t[%d] 0x%llx\n", i, srcs[i]); BUG(); } @@ -1686,8 +1686,8 @@ static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( { struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; struct ppc440spe_adma_desc_slot *alloc_start = NULL; - struct list_head chain = LIST_HEAD_INIT(chain); int slots_found, retry = 0; + LIST_HEAD(chain); BUG_ON(!num_slots || !slots_per_op); @@ -3240,7 +3240,6 @@ static int ppc440spe_adma_dma2rxor_prep_src( struct ppc440spe_rxor *cursor, int index, int src_cnt, u32 addr) { - int rval = 0; u32 sign; struct ppc440spe_adma_desc_slot *desc = hdesc; int i; @@ -3348,7 +3347,7 @@ static int ppc440spe_adma_dma2rxor_prep_src( break; } - return rval; + return 0; } /** @@ -3637,7 +3636,7 @@ static void ppc440spe_adma_issue_pending(struct dma_chan *chan) ppc440spe_chan = to_ppc440spe_adma_chan(chan); dev_dbg(ppc440spe_chan->device->common.dev, - "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id, + "ppc440spe adma%d: %s %d\n", ppc440spe_chan->device->id, __func__, ppc440spe_chan->pending); if (ppc440spe_chan->pending) { @@ -4231,7 +4230,7 @@ out: /** * ppc440spe_adma_remove - remove the asynch device */ -static int ppc440spe_adma_remove(struct platform_device *ofdev) +static void ppc440spe_adma_remove(struct platform_device *ofdev) { struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); struct device_node *np = ofdev->dev.of_node; @@ -4279,7 +4278,6 @@ static int ppc440spe_adma_remove(struct platform_device *ofdev) of_address_to_resource(np, 0, &res); release_mem_region(res.start, resource_size(&res)); kfree(adev); - return 0; } /* @@ -4300,9 +4298,8 @@ static ssize_t devices_show(struct device_driver *dev, char *buf) for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { if (ppc440spe_adma_devices[i] == -1) continue; - size += scnprintf(buf + size, PAGE_SIZE - size, - "PPC440SP(E)-ADMA.%d: %s\n", i, - ppc_adma_errors[ppc440spe_adma_devices[i]]); + size += sysfs_emit_at(buf, size, "PPC440SP(E)-ADMA.%d: %s\n", + i, ppc_adma_errors[ppc440spe_adma_devices[i]]); } return size; } @@ -4310,15 +4307,15 @@ static DRIVER_ATTR_RO(devices); static ssize_t enable_show(struct device_driver *dev, char *buf) { - return snprintf(buf, PAGE_SIZE, - "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", - ppc440spe_r6_enabled ? "EN" : "DIS"); + return sysfs_emit(buf, "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", + ppc440spe_r6_enabled ? "EN" : "DIS"); } static ssize_t enable_store(struct device_driver *dev, const char *buf, size_t count) { unsigned long val; + int err; if (!count || count > 11) return -EINVAL; @@ -4327,7 +4324,10 @@ static ssize_t enable_store(struct device_driver *dev, const char *buf, return -EFAULT; /* Write a key */ - sscanf(buf, "%lx", &val); + err = kstrtoul(buf, 16, &val); + if (err) + return err; + dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); isync(); @@ -4359,7 +4359,7 @@ static ssize_t poly_show(struct device_driver *dev, char *buf) reg &= 0xFF; #endif - size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver " + size = sysfs_emit(buf, "PPC440SP(e) RAID-6 driver " "uses 0x1%02x polynomial.\n", reg); return size; } @@ -4368,7 +4368,7 @@ static ssize_t poly_store(struct device_driver *dev, const char *buf, size_t count) { unsigned long reg, val; - + int err; #ifdef CONFIG_440SP /* 440SP uses default 0x14D polynomial only */ return -EINVAL; @@ -4378,7 +4378,9 @@ static ssize_t poly_store(struct device_driver *dev, const char *buf, return -EINVAL; /* e.g., 0x14D or 0x11D */ - sscanf(buf, "%lx", &val); + err = kstrtoul(buf, 16, &val); + if (err) + return err; if (val & ~0x1FF) return -EINVAL; diff --git a/drivers/dma/ppc4xx/adma.h b/drivers/dma/ppc4xx/adma.h index 26b7a5ed9ac7..f8a5d7c1fb40 100644 --- a/drivers/dma/ppc4xx/adma.h +++ b/drivers/dma/ppc4xx/adma.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * 2006-2009 (C) DENX Software Engineering. * * Author: Yuri Tikhonov <yur@emcraft.com> - * - * This file is licensed under the terms of the GNU General Public License - * version 2. This program is licensed "as is" without any warranty of - * any kind, whether express or implied. */ #ifndef _PPC440SPE_ADMA_H diff --git a/drivers/dma/ppc4xx/dma.h b/drivers/dma/ppc4xx/dma.h index bcde2df2f373..b5725481bfa6 100644 --- a/drivers/dma/ppc4xx/dma.h +++ b/drivers/dma/ppc4xx/dma.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * 440SPe's DMA engines support header file * * 2006-2009 (C) DENX Software Engineering. * * Author: Yuri Tikhonov <yur@emcraft.com> - * - * This file is licensed under the term of the GNU General Public License - * version 2. The program licensed "as is" without any warranty of any - * kind, whether express or implied. */ #ifndef _PPC440SPE_DMA_H @@ -17,7 +14,7 @@ /* Number of elements in the array with statical CDBs */ #define MAX_STAT_DMA_CDBS 16 -/* Number of DMA engines available on the contoller */ +/* Number of DMA engines available on the controller */ #define DMA_ENGINES_NUM 2 /* Maximum h/w supported number of destinations */ diff --git a/drivers/dma/ppc4xx/xor.h b/drivers/dma/ppc4xx/xor.h index daed7384daac..da1230df2817 100644 --- a/drivers/dma/ppc4xx/xor.h +++ b/drivers/dma/ppc4xx/xor.h @@ -1,13 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * 440SPe's XOR engines support header file * * 2006-2009 (C) DENX Software Engineering. * * Author: Yuri Tikhonov <yur@emcraft.com> - * - * This file is licensed under the term of the GNU General Public License - * version 2. The program licensed "as is" without any warranty of any - * kind, whether express or implied. */ #ifndef _PPC440SPE_XOR_H diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index 4a2a796e348c..249296389771 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -10,14 +10,14 @@ #include <linux/interrupt.h> #include <linux/dma-mapping.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/dmaengine.h> #include <linux/platform_device.h> #include <linux/device.h> #include <linux/platform_data/mmp_dma.h> #include <linux/dmapool.h> -#include <linux/of_device.h> -#include <linux/of_dma.h> #include <linux/of.h> +#include <linux/of_dma.h> #include <linux/wait.h> #include <linux/dma/pxa-dma.h> @@ -91,7 +91,8 @@ struct pxad_desc_sw { bool cyclic; struct dma_pool *desc_pool; /* Channel's used allocator */ - struct pxad_desc_hw *hw_desc[]; /* DMA coherent descriptors */ + struct pxad_desc_hw *hw_desc[] __counted_by(nb_desc); + /* DMA coherent descriptors */ }; struct pxad_phy { @@ -277,8 +278,7 @@ static int chan_state_show(struct seq_file *s, void *p) seq_printf(s, "\tPriority : %s\n", str_prio[(phy->idx & 0xf) / 4]); seq_printf(s, "\tUnaligned transfer bit: %s\n", - _phy_readl_relaxed(phy, DALGN) & BIT(phy->idx) ? - "yes" : "no"); + str_yes_no(_phy_readl_relaxed(phy, DALGN) & BIT(phy->idx))); seq_printf(s, "\tDCSR = %08x (%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", dcsr, PXA_DCSR_STR(RUN), PXA_DCSR_STR(NODESC), PXA_DCSR_STR(STOPIRQEN), PXA_DCSR_STR(EORIRQEN), @@ -722,7 +722,6 @@ static void pxad_free_desc(struct virt_dma_desc *vd) dma_addr_t dma; struct pxad_desc_sw *sw_desc = to_pxad_sw_desc(vd); - BUG_ON(sw_desc->nb_desc == 0); for (i = sw_desc->nb_desc - 1; i >= 0; i--) { if (i > 0) dma = sw_desc->hw_desc[i - 1]->ddadr; @@ -740,30 +739,31 @@ pxad_alloc_desc(struct pxad_chan *chan, unsigned int nb_hw_desc) { struct pxad_desc_sw *sw_desc; dma_addr_t dma; + void *desc; int i; - sw_desc = kzalloc(sizeof(*sw_desc) + - nb_hw_desc * sizeof(struct pxad_desc_hw *), + sw_desc = kzalloc(struct_size(sw_desc, hw_desc, nb_hw_desc), GFP_NOWAIT); if (!sw_desc) return NULL; sw_desc->desc_pool = chan->desc_pool; for (i = 0; i < nb_hw_desc; i++) { - sw_desc->hw_desc[i] = dma_pool_alloc(sw_desc->desc_pool, - GFP_NOWAIT, &dma); - if (!sw_desc->hw_desc[i]) { + desc = dma_pool_alloc(sw_desc->desc_pool, GFP_NOWAIT, &dma); + if (!desc) { dev_err(&chan->vc.chan.dev->device, "%s(): Couldn't allocate the %dth hw_desc from dma_pool %p\n", __func__, i, sw_desc->desc_pool); goto err; } + sw_desc->nb_desc++; + sw_desc->hw_desc[i] = desc; + if (i == 0) sw_desc->first = dma; else sw_desc->hw_desc[i - 1]->ddadr = dma; - sw_desc->nb_desc++; } return sw_desc; @@ -910,13 +910,6 @@ static void pxad_get_config(struct pxad_chan *chan, *dcmd |= PXA_DCMD_BURST16; else if (maxburst == 32) *dcmd |= PXA_DCMD_BURST32; - - /* FIXME: drivers should be ported over to use the filter - * function. Once that's done, the following two lines can - * be removed. - */ - if (chan->cfg.slave_id) - chan->drcmr = chan->cfg.slave_id; } static struct dma_async_tx_descriptor * @@ -1229,13 +1222,12 @@ static void pxad_free_channels(struct dma_device *dmadev) } } -static int pxad_remove(struct platform_device *op) +static void pxad_remove(struct platform_device *op) { struct pxad_device *pdev = platform_get_drvdata(op); pxad_cleanup_debugfs(pdev); pxad_free_channels(&pdev->slave); - return 0; } static int pxad_init_phys(struct platform_device *op, @@ -1255,14 +1247,14 @@ static int pxad_init_phys(struct platform_device *op, return -ENOMEM; for (i = 0; i < nb_phy_chans; i++) - if (platform_get_irq(op, i) > 0) + if (platform_get_irq_optional(op, i) > 0) nr_irq++; for (i = 0; i < nb_phy_chans; i++) { phy = &pdev->phys[i]; phy->base = pdev->base; phy->idx = i; - irq = platform_get_irq(op, i); + irq = platform_get_irq_optional(op, i); if ((nr_irq > 1) && (irq > 0)) ret = devm_request_irq(&op->dev, irq, pxad_chan_handler, @@ -1351,10 +1343,8 @@ static int pxad_init_dmadev(struct platform_device *op, static int pxad_probe(struct platform_device *op) { struct pxad_device *pdev; - const struct of_device_id *of_id; const struct dma_slave_map *slave_map = NULL; struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); - struct resource *iores; int ret, dma_channels = 0, nb_requestors = 0, slave_map_cnt = 0; const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | @@ -1366,17 +1356,22 @@ static int pxad_probe(struct platform_device *op) spin_lock_init(&pdev->phy_lock); - iores = platform_get_resource(op, IORESOURCE_MEM, 0); - pdev->base = devm_ioremap_resource(&op->dev, iores); + pdev->base = devm_platform_ioremap_resource(op, 0); if (IS_ERR(pdev->base)) return PTR_ERR(pdev->base); - of_id = of_match_device(pxad_dt_ids, &op->dev); - if (of_id) { - of_property_read_u32(op->dev.of_node, "#dma-channels", - &dma_channels); - ret = of_property_read_u32(op->dev.of_node, "#dma-requests", + if (op->dev.of_node) { + /* Parse new and deprecated dma-channels properties */ + if (of_property_read_u32(op->dev.of_node, "dma-channels", + &dma_channels)) + of_property_read_u32(op->dev.of_node, "#dma-channels", + &dma_channels); + /* Parse new and deprecated dma-requests properties */ + ret = of_property_read_u32(op->dev.of_node, "dma-requests", &nb_requestors); + if (ret) + ret = of_property_read_u32(op->dev.of_node, "#dma-requests", + &nb_requestors); if (ret) { dev_warn(pdev->slave.dev, "#dma-requests set to default 32 as missing in OF: %d", diff --git a/drivers/dma/qcom/Kconfig b/drivers/dma/qcom/Kconfig index 3f926a653bd8..ace75d7b835a 100644 --- a/drivers/dma/qcom/Kconfig +++ b/drivers/dma/qcom/Kconfig @@ -45,6 +45,7 @@ config QCOM_HIDMA_MGMT config QCOM_HIDMA tristate "Qualcomm Technologies HIDMA Channel support" + depends on HAS_IOMEM select DMA_ENGINE help Enable support for the Qualcomm Technologies HIDMA controller. diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index c8a77b428b52..2cf060174795 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -74,7 +74,7 @@ struct bam_async_desc { struct list_head desc_node; enum dma_transfer_direction dir; size_t length; - struct bam_desc_hw desc[]; + struct bam_desc_hw desc[] __counted_by(num_desc); }; enum bam_reg { @@ -388,6 +388,8 @@ struct bam_device { /* execution environment ID, from DT */ u32 ee; bool controlled_remotely; + bool powered_remotely; + u32 active_channels; const struct reg_offset_data *layout; @@ -416,6 +418,44 @@ static inline void __iomem *bam_addr(struct bam_device *bdev, u32 pipe, } /** + * bam_reset() - reset and initialize BAM registers + * @bdev: bam device + */ +static void bam_reset(struct bam_device *bdev) +{ + u32 val; + + /* s/w reset bam */ + /* after reset all pipes are disabled and idle */ + val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL)); + val |= BAM_SW_RST; + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); + val &= ~BAM_SW_RST; + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); + + /* make sure previous stores are visible before enabling BAM */ + wmb(); + + /* enable bam */ + val |= BAM_EN; + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); + + /* set descriptor threshold, start with 4 bytes */ + writel_relaxed(DEFAULT_CNT_THRSHLD, + bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); + + /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */ + writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS)); + + /* enable irqs for errors */ + writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN, + bam_addr(bdev, 0, BAM_IRQ_EN)); + + /* unmask global bam interrupt */ + writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); +} + +/** * bam_reset_channel - Reset individual BAM DMA channel * @bchan: bam channel * @@ -512,13 +552,8 @@ static int bam_alloc_chan(struct dma_chan *chan) return -ENOMEM; } - return 0; -} - -static int bam_pm_runtime_get_sync(struct device *dev) -{ - if (pm_runtime_enabled(dev)) - return pm_runtime_get_sync(dev); + if (bdev->active_channels++ == 0 && bdev->powered_remotely) + bam_reset(bdev); return 0; } @@ -538,7 +573,7 @@ static void bam_free_chan(struct dma_chan *chan) unsigned long flags; int ret; - ret = bam_pm_runtime_get_sync(bdev->dev); + ret = pm_runtime_get_sync(bdev->dev); if (ret < 0) return; @@ -565,6 +600,13 @@ static void bam_free_chan(struct dma_chan *chan) /* disable irq */ writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN)); + if (--bdev->active_channels == 0 && bdev->powered_remotely) { + /* s/w reset bam */ + val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL)); + val |= BAM_SW_RST; + writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); + } + err: pm_runtime_mark_last_busy(bdev->dev); pm_runtime_put_autosuspend(bdev->dev); @@ -625,7 +667,7 @@ static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, for_each_sg(sgl, sg, sg_len, i) num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_FIFO_SIZE); - /* allocate enough room to accomodate the number of entries */ + /* allocate enough room to accommodate the number of entries */ async_desc = kzalloc(struct_size(async_desc, desc, num_alloc), GFP_NOWAIT); @@ -734,7 +776,7 @@ static int bam_pause(struct dma_chan *chan) unsigned long flag; int ret; - ret = bam_pm_runtime_get_sync(bdev->dev); + ret = pm_runtime_get_sync(bdev->dev); if (ret < 0) return ret; @@ -760,7 +802,7 @@ static int bam_resume(struct dma_chan *chan) unsigned long flag; int ret; - ret = bam_pm_runtime_get_sync(bdev->dev); + ret = pm_runtime_get_sync(bdev->dev); if (ret < 0) return ret; @@ -869,7 +911,7 @@ static irqreturn_t bam_dma_irq(int irq, void *data) if (srcs & P_IRQ) tasklet_schedule(&bdev->task); - ret = bam_pm_runtime_get_sync(bdev->dev); + ret = pm_runtime_get_sync(bdev->dev); if (ret < 0) return IRQ_NONE; @@ -987,7 +1029,7 @@ static void bam_start_dma(struct bam_chan *bchan) if (!vd) return; - ret = bam_pm_runtime_get_sync(bdev->dev); + ret = pm_runtime_get_sync(bdev->dev); if (ret < 0) return; @@ -1164,37 +1206,9 @@ static int bam_init(struct bam_device *bdev) bdev->num_channels = val & BAM_NUM_PIPES_MASK; } - if (bdev->controlled_remotely) - return 0; - - /* s/w reset bam */ - /* after reset all pipes are disabled and idle */ - val = readl_relaxed(bam_addr(bdev, 0, BAM_CTRL)); - val |= BAM_SW_RST; - writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); - val &= ~BAM_SW_RST; - writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); - - /* make sure previous stores are visible before enabling BAM */ - wmb(); - - /* enable bam */ - val |= BAM_EN; - writel_relaxed(val, bam_addr(bdev, 0, BAM_CTRL)); - - /* set descriptor threshhold, start with 4 bytes */ - writel_relaxed(DEFAULT_CNT_THRSHLD, - bam_addr(bdev, 0, BAM_DESC_CNT_TRSHLD)); - - /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */ - writel_relaxed(BAM_CNFG_BITS_DEFAULT, bam_addr(bdev, 0, BAM_CNFG_BITS)); - - /* enable irqs for errors */ - writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN, - bam_addr(bdev, 0, BAM_IRQ_EN)); - - /* unmask global bam interrupt */ - writel_relaxed(BAM_IRQ_MSK, bam_addr(bdev, 0, BAM_IRQ_SRCS_MSK_EE)); + /* Reset BAM now if fully controlled locally */ + if (!bdev->controlled_remotely && !bdev->powered_remotely) + bam_reset(bdev); return 0; } @@ -1223,7 +1237,6 @@ static int bam_dma_probe(struct platform_device *pdev) { struct bam_device *bdev; const struct of_device_id *match; - struct resource *iores; int ret, i; bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL); @@ -1240,8 +1253,7 @@ static int bam_dma_probe(struct platform_device *pdev) bdev->layout = match->data; - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); - bdev->regs = devm_ioremap_resource(&pdev->dev, iores); + bdev->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(bdev->regs)) return PTR_ERR(bdev->regs); @@ -1257,27 +1269,33 @@ static int bam_dma_probe(struct platform_device *pdev) bdev->controlled_remotely = of_property_read_bool(pdev->dev.of_node, "qcom,controlled-remotely"); + bdev->powered_remotely = of_property_read_bool(pdev->dev.of_node, + "qcom,powered-remotely"); - if (bdev->controlled_remotely) { + if (bdev->controlled_remotely || bdev->powered_remotely) + bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk"); + else + bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); + + if (IS_ERR(bdev->bamclk)) + return PTR_ERR(bdev->bamclk); + + if (!bdev->bamclk) { ret = of_property_read_u32(pdev->dev.of_node, "num-channels", &bdev->num_channels); - if (ret) + if (ret) { dev_err(bdev->dev, "num-channels unspecified in dt\n"); + return ret; + } ret = of_property_read_u32(pdev->dev.of_node, "qcom,num-ees", &bdev->num_ees); - if (ret) + if (ret) { dev_err(bdev->dev, "num-ees unspecified in dt\n"); + return ret; + } } - if (bdev->controlled_remotely) - bdev->bamclk = devm_clk_get_optional(bdev->dev, "bam_clk"); - else - bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); - - if (IS_ERR(bdev->bamclk)) - return PTR_ERR(bdev->bamclk); - ret = clk_prepare_enable(bdev->bamclk); if (ret) { dev_err(bdev->dev, "failed to prepare/enable clock\n"); @@ -1311,11 +1329,7 @@ static int bam_dma_probe(struct platform_device *pdev) /* set max dma segment size */ bdev->common.dev = bdev->dev; - ret = dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE); - if (ret) { - dev_err(bdev->dev, "cannot set maximum segment size\n"); - goto err_bam_channel_exit; - } + dma_set_max_seg_size(bdev->common.dev, BAM_FIFO_SIZE); platform_set_drvdata(pdev, bdev); @@ -1350,11 +1364,6 @@ static int bam_dma_probe(struct platform_device *pdev) if (ret) goto err_unregister_dma; - if (!bdev->bamclk) { - pm_runtime_disable(&pdev->dev); - return 0; - } - pm_runtime_irq_safe(&pdev->dev); pm_runtime_set_autosuspend_delay(&pdev->dev, BAM_DMA_AUTOSUSPEND_DELAY); pm_runtime_use_autosuspend(&pdev->dev); @@ -1377,7 +1386,7 @@ err_disable_clk: return ret; } -static int bam_dma_remove(struct platform_device *pdev) +static void bam_dma_remove(struct platform_device *pdev) { struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; @@ -1407,8 +1416,6 @@ static int bam_dma_remove(struct platform_device *pdev) tasklet_kill(&bdev->task); clk_disable_unprepare(bdev->bamclk); - - return 0; } static int __maybe_unused bam_dma_runtime_suspend(struct device *dev) @@ -1438,10 +1445,8 @@ static int __maybe_unused bam_dma_suspend(struct device *dev) { struct bam_device *bdev = dev_get_drvdata(dev); - if (bdev->bamclk) { - pm_runtime_force_suspend(dev); - clk_unprepare(bdev->bamclk); - } + pm_runtime_force_suspend(dev); + clk_unprepare(bdev->bamclk); return 0; } @@ -1451,13 +1456,11 @@ static int __maybe_unused bam_dma_resume(struct device *dev) struct bam_device *bdev = dev_get_drvdata(dev); int ret; - if (bdev->bamclk) { - ret = clk_prepare(bdev->bamclk); - if (ret) - return ret; + ret = clk_prepare(bdev->bamclk); + if (ret) + return ret; - pm_runtime_force_resume(dev); - } + pm_runtime_force_resume(dev); return 0; } diff --git a/drivers/dma/qcom/gpi.c b/drivers/dma/qcom/gpi.c index 1a1b7d8458c9..66bfea1f156d 100644 --- a/drivers/dma/qcom/gpi.c +++ b/drivers/dma/qcom/gpi.c @@ -18,6 +18,7 @@ #include "../virt-dma.h" #define TRE_TYPE_DMA 0x10 +#define TRE_TYPE_IMMEDIATE_DMA 0x11 #define TRE_TYPE_GO 0x20 #define TRE_TYPE_CONFIG0 0x22 @@ -64,6 +65,7 @@ /* DMA TRE */ #define TRE_DMA_LEN GENMASK(23, 0) +#define TRE_DMA_IMMEDIATE_LEN GENMASK(3, 0) /* Register offsets from gpi-top */ #define GPII_n_CH_k_CNTXT_0_OFFS(n, k) (0x20000 + (0x4000 * (n)) + (0x80 * (k))) @@ -476,12 +478,6 @@ struct gpi_dev { struct gpii *gpiis; }; -struct reg_info { - char *name; - u32 offset; - u32 val; -}; - struct gchan { struct virt_dma_chan vc; u32 chid; @@ -573,17 +569,6 @@ static inline void gpi_write_reg(struct gpii *gpii, void __iomem *addr, u32 val) writel_relaxed(val, addr); } -/* gpi_write_reg_field - write to specific bit field */ -static inline void gpi_write_reg_field(struct gpii *gpii, void __iomem *addr, - u32 mask, u32 shift, u32 val) -{ - u32 tmp = gpi_read_reg(gpii, addr); - - tmp &= ~mask; - val = tmp | ((val << shift) & mask); - gpi_write_reg(gpii, addr, val); -} - static __always_inline void gpi_update_reg(struct gpii *gpii, u32 offset, u32 mask, u32 val) { @@ -1150,9 +1135,9 @@ static void gpi_ev_tasklet(unsigned long data) { struct gpii *gpii = (struct gpii *)data; - read_lock_bh(&gpii->pm_lock); + read_lock(&gpii->pm_lock); if (!REG_ACCESS_VALID(gpii->pm_state)) { - read_unlock_bh(&gpii->pm_lock); + read_unlock(&gpii->pm_lock); dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n", TO_GPI_PM_STR(gpii->pm_state)); return; @@ -1163,7 +1148,7 @@ static void gpi_ev_tasklet(unsigned long data) /* enable IEOB, switching back to interrupts */ gpi_config_interrupts(gpii, MASK_IEOB_SETTINGS, 1); - read_unlock_bh(&gpii->pm_lock); + read_unlock(&gpii->pm_lock); } /* marks all pending events for the channel as stale */ @@ -1197,7 +1182,6 @@ static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd) { struct gpii *gpii = gchan->gpii; struct gpi_ring *ch_ring = &gchan->ch_ring; - unsigned long flags; LIST_HEAD(list); int ret; @@ -1220,9 +1204,9 @@ static int gpi_reset_chan(struct gchan *gchan, enum gpi_cmd gpi_cmd) gpi_mark_stale_events(gchan); /* remove all async descriptors */ - spin_lock_irqsave(&gchan->vc.lock, flags); + spin_lock(&gchan->vc.lock); vchan_get_all_descriptors(&gchan->vc, &list); - spin_unlock_irqrestore(&gchan->vc.lock, flags); + spin_unlock(&gchan->vc.lock); write_unlock_irq(&gpii->pm_lock); vchan_dma_desc_free_list(&gchan->vc, &list); @@ -1635,7 +1619,8 @@ gpi_peripheral_config(struct dma_chan *chan, struct dma_slave_config *config) } static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, - struct scatterlist *sgl, enum dma_transfer_direction direction) + struct scatterlist *sgl, enum dma_transfer_direction direction, + unsigned long flags) { struct gpi_i2c_config *i2c = chan->config; struct device *dev = chan->gpii->gpi_dev->dev; @@ -1700,6 +1685,9 @@ static int gpi_create_i2c_tre(struct gchan *chan, struct gpi_desc *desc, tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + + if (!(flags & DMA_PREP_INTERRUPT)) + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_BEI); } for (i = 0; i < tre_idx; i++) @@ -1718,6 +1706,7 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, dma_addr_t address; struct gpi_tre *tre; unsigned int i; + int len; /* first create config tre if applicable */ if (direction == DMA_MEM_TO_DEV && spi->set_config) { @@ -1754,10 +1743,15 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN); tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE); - if (spi->cmd == SPI_RX) + if (spi->cmd == SPI_RX) { tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB); - else + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK); + } else if (spi->cmd == SPI_TX) { tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); + } else { /* SPI_DUPLEX */ + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); + tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK); + } } /* create the dma tre */ @@ -1765,14 +1759,30 @@ static int gpi_create_spi_tre(struct gchan *chan, struct gpi_desc *desc, tre_idx++; address = sg_dma_address(sgl); - tre->dword[0] = lower_32_bits(address); - tre->dword[1] = upper_32_bits(address); + len = sg_dma_len(sgl); - tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN); + /* Support Immediate dma for write transfers for data length up to 8 bytes */ + if (direction == DMA_MEM_TO_DEV && len <= 2 * sizeof(tre->dword[0])) { + /* + * For Immediate dma, data length may not always be length of 8 bytes, + * it can be length less than 8, hence initialize both dword's with 0 + */ + tre->dword[0] = 0; + tre->dword[1] = 0; + memcpy(&tre->dword[0], sg_virt(sgl), len); - tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); - if (direction == DMA_MEM_TO_DEV) - tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); + tre->dword[2] = u32_encode_bits(len, TRE_DMA_IMMEDIATE_LEN); + tre->dword[3] = u32_encode_bits(TRE_TYPE_IMMEDIATE_DMA, TRE_FLAGS_TYPE); + } else { + tre->dword[0] = lower_32_bits(address); + tre->dword[1] = upper_32_bits(address); + + tre->dword[2] = u32_encode_bits(len, TRE_DMA_LEN); + tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); + } + + tre->dword[3] |= u32_encode_bits(direction == DMA_MEM_TO_DEV, + TRE_FLAGS_IEOT); for (i = 0; i < tre_idx; i++) dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], @@ -1821,6 +1831,9 @@ gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return NULL; } + if (!(flags & DMA_PREP_INTERRUPT) && (nr - nr_tre < 2)) + return NULL; + gpi_desc = kzalloc(sizeof(*gpi_desc), GFP_NOWAIT); if (!gpi_desc) return NULL; @@ -1829,7 +1842,7 @@ gpi_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (gchan->protocol == QCOM_GPI_SPI) { i = gpi_create_spi_tre(gchan, gpi_desc, sgl, direction); } else if (gchan->protocol == QCOM_GPI_I2C) { - i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction); + i = gpi_create_i2c_tre(gchan, gpi_desc, sgl, direction, flags); } else { dev_err(dev, "invalid peripheral: %d\n", gchan->protocol); kfree(gpi_desc); @@ -1858,7 +1871,7 @@ static void gpi_issue_pending(struct dma_chan *chan) read_lock_irqsave(&gpii->pm_lock, pm_lock_flags); - /* move all submitted discriptors to issued list */ + /* move all submitted descriptors to issued list */ spin_lock_irqsave(&gchan->vc.lock, flags); if (vchan_issue_pending(&gchan->vc)) vd = list_last_entry(&gchan->vc.desc_issued, @@ -1961,7 +1974,6 @@ error_alloc_ev_ring: error_config_int: gpi_free_ring(&gpii->ev_ring, gpii); exit_gpi_init: - mutex_unlock(&gpii->ctrl_lock); return ret; } @@ -2148,6 +2160,7 @@ static int gpi_probe(struct platform_device *pdev) { struct gpi_dev *gpi_dev; unsigned int i; + u32 ee_offset; int ret; gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL); @@ -2155,8 +2168,7 @@ static int gpi_probe(struct platform_device *pdev) return -ENOMEM; gpi_dev->dev = &pdev->dev; - gpi_dev->res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - gpi_dev->regs = devm_ioremap_resource(gpi_dev->dev, gpi_dev->res); + gpi_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &gpi_dev->res); if (IS_ERR(gpi_dev->regs)) return PTR_ERR(gpi_dev->regs); gpi_dev->ee_base = gpi_dev->regs; @@ -2175,6 +2187,9 @@ static int gpi_probe(struct platform_device *pdev) return ret; } + ee_offset = (uintptr_t)device_get_match_data(gpi_dev->dev); + gpi_dev->ee_base = gpi_dev->ee_base - ee_offset; + gpi_dev->ev_factor = EV_FACTOR; ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64)); @@ -2206,10 +2221,8 @@ static int gpi_probe(struct platform_device *pdev) /* set up irq */ ret = platform_get_irq(pdev, i); - if (ret < 0) { - dev_err(gpi_dev->dev, "platform_get_irq failed for %d:%d\n", i, ret); + if (ret < 0) return ret; - } gpii->irq = ret; /* set up channel specific register info */ @@ -2280,9 +2293,18 @@ static int gpi_probe(struct platform_device *pdev) } static const struct of_device_id gpi_of_match[] = { - { .compatible = "qcom,sdm845-gpi-dma" }, - { .compatible = "qcom,sm8150-gpi-dma" }, - { .compatible = "qcom,sm8250-gpi-dma" }, + { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 }, + { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 }, + /* + * Do not grow the list for compatible devices. Instead use + * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma + * (for ee_offset = 0x10000). + */ + { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 }, + { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 }, + { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 }, + { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 }, + { .compatible = "qcom,sm8450-gpi-dma", .data = (void *)0x10000 }, { }, }; MODULE_DEVICE_TABLE(of, gpi_of_match); diff --git a/drivers/dma/qcom/hidma.c b/drivers/dma/qcom/hidma.c index 23d64489d25f..c2b3e4452e71 100644 --- a/drivers/dma/qcom/hidma.c +++ b/drivers/dma/qcom/hidma.c @@ -45,12 +45,11 @@ #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/list.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/of_dma.h> -#include <linux/of_device.h> #include <linux/property.h> #include <linux/delay.h> #include <linux/acpi.h> @@ -214,7 +213,6 @@ static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig) spin_lock_init(&mchan->lock); list_add_tail(&mchan->chan.device_node, &ddev->channels); - dmadev->ddev.chancnt++; return 0; } @@ -431,6 +429,7 @@ hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value, struct hidma_desc *mdesc = NULL; struct hidma_dev *mdma = mchan->dmadev; unsigned long irqflags; + u64 byte_pattern, fill_pattern; /* Get free descriptor */ spin_lock_irqsave(&mchan->lock, irqflags); @@ -443,9 +442,19 @@ hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value, if (!mdesc) return NULL; + byte_pattern = (char)value; + fill_pattern = (byte_pattern << 56) | + (byte_pattern << 48) | + (byte_pattern << 40) | + (byte_pattern << 32) | + (byte_pattern << 24) | + (byte_pattern << 16) | + (byte_pattern << 8) | + byte_pattern; + mdesc->desc.flags = flags; hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, - value, dest, len, flags, + fill_pattern, dest, len, flags, HIDMA_TRE_MEMSET); /* Place descriptor in prepared list */ @@ -599,7 +608,7 @@ static irqreturn_t hidma_chirq_handler(int chirq, void *arg) return hidma_ll_inthandler(chirq, lldev); } -#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN +#ifdef CONFIG_GENERIC_MSI_IRQ static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg) { struct hidma_lldev **lldevp = arg; @@ -660,13 +669,13 @@ static int hidma_sysfs_init(struct hidma_dev *dev) return device_create_file(dev->ddev.dev, dev->chid_attrs); } -#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN +#ifdef CONFIG_GENERIC_MSI_IRQ static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) { struct device *dev = msi_desc_to_dev(desc); struct hidma_dev *dmadev = dev_get_drvdata(dev); - if (!desc->platform.msi_index) { + if (!desc->msi_index) { writel(msg->address_lo, dmadev->dev_evca + 0x118); writel(msg->address_hi, dmadev->dev_evca + 0x11C); writel(msg->data, dmadev->dev_evca + 0x120); @@ -676,61 +685,55 @@ static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) static void hidma_free_msis(struct hidma_dev *dmadev) { -#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN +#ifdef CONFIG_GENERIC_MSI_IRQ struct device *dev = dmadev->ddev.dev; - struct msi_desc *desc; + int i, virq; - /* free allocated MSI interrupts above */ - for_each_msi_entry(desc, dev) - devm_free_irq(dev, desc->irq, &dmadev->lldev); + for (i = 0; i < HIDMA_MSI_INTS; i++) { + virq = msi_get_virq(dev, i); + if (virq) + devm_free_irq(dev, virq, &dmadev->lldev); + } - platform_msi_domain_free_irqs(dev); + platform_device_msi_free_irqs_all(dev); #endif } static int hidma_request_msi(struct hidma_dev *dmadev, struct platform_device *pdev) { -#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN - int rc; - struct msi_desc *desc; - struct msi_desc *failed_desc = NULL; +#ifdef CONFIG_GENERIC_MSI_IRQ + int rc, i, virq; - rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, - hidma_write_msi_msg); + rc = platform_device_msi_init_and_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, + hidma_write_msi_msg); if (rc) return rc; - for_each_msi_entry(desc, &pdev->dev) { - if (!desc->platform.msi_index) - dmadev->msi_virqbase = desc->irq; - - rc = devm_request_irq(&pdev->dev, desc->irq, + for (i = 0; i < HIDMA_MSI_INTS; i++) { + virq = msi_get_virq(&pdev->dev, i); + rc = devm_request_irq(&pdev->dev, virq, hidma_chirq_handler_msi, 0, "qcom-hidma-msi", &dmadev->lldev); - if (rc) { - failed_desc = desc; + if (rc) break; - } + if (!i) + dmadev->msi_virqbase = virq; } if (rc) { /* free allocated MSI interrupts above */ - for_each_msi_entry(desc, &pdev->dev) { - if (desc == failed_desc) - break; - devm_free_irq(&pdev->dev, desc->irq, - &dmadev->lldev); + for (--i; i >= 0; i--) { + virq = msi_get_virq(&pdev->dev, i); + devm_free_irq(&pdev->dev, virq, &dmadev->lldev); } + dev_warn(&pdev->dev, + "failed to request MSI irq, falling back to wired IRQ\n"); } else { /* Add callback to free MSIs on teardown */ hidma_ll_setup_irq(dmadev->lldev, true); - } - if (rc) - dev_warn(&pdev->dev, - "failed to request MSI irq, falling back to wired IRQ\n"); return rc; #else return -EINVAL; @@ -741,7 +744,7 @@ static bool hidma_test_capability(struct device *dev, enum hidma_cap test_cap) { enum hidma_cap cap; - cap = (enum hidma_cap) device_get_match_data(dev); + cap = (uintptr_t) device_get_match_data(dev); return cap ? ((cap & test_cap) > 0) : 0; } @@ -761,17 +764,15 @@ static int hidma_probe(struct platform_device *pdev) pm_runtime_set_active(&pdev->dev); pm_runtime_enable(&pdev->dev); - trca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); - trca = devm_ioremap_resource(&pdev->dev, trca_resource); + trca = devm_platform_get_and_ioremap_resource(pdev, 0, &trca_resource); if (IS_ERR(trca)) { - rc = -ENOMEM; + rc = PTR_ERR(trca); goto bailout; } - evca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 1); - evca = devm_ioremap_resource(&pdev->dev, evca_resource); + evca = devm_platform_get_and_ioremap_resource(pdev, 1, &evca_resource); if (IS_ERR(evca)) { - rc = -ENOMEM; + rc = PTR_ERR(evca); goto bailout; } @@ -781,7 +782,7 @@ static int hidma_probe(struct platform_device *pdev) */ chirq = platform_get_irq(pdev, 0); if (chirq < 0) { - rc = -ENODEV; + rc = chirq; goto bailout; } @@ -844,9 +845,7 @@ static int hidma_probe(struct platform_device *pdev) rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (rc) { dev_warn(&pdev->dev, "unable to set coherent mask to 64"); - rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (rc) - goto dmafree; + goto dmafree; } dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, @@ -915,7 +914,7 @@ static void hidma_shutdown(struct platform_device *pdev) } -static int hidma_remove(struct platform_device *pdev) +static void hidma_remove(struct platform_device *pdev) { struct hidma_dev *dmadev = platform_get_drvdata(pdev); @@ -935,8 +934,6 @@ static int hidma_remove(struct platform_device *pdev) dev_info(&pdev->dev, "HI-DMA engine removed\n"); pm_runtime_put_sync_suspend(&pdev->dev); pm_runtime_disable(&pdev->dev); - - return 0; } #if IS_ENABLED(CONFIG_ACPI) @@ -949,25 +946,16 @@ static const struct acpi_device_id hidma_acpi_ids[] = { MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids); #endif -static const struct of_device_id hidma_match[] = { - {.compatible = "qcom,hidma-1.0",}, - {.compatible = "qcom,hidma-1.1", .data = (void *)(HIDMA_MSI_CAP),}, - {.compatible = "qcom,hidma-1.2", - .data = (void *)(HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP),}, - {}, -}; -MODULE_DEVICE_TABLE(of, hidma_match); - static struct platform_driver hidma_driver = { .probe = hidma_probe, .remove = hidma_remove, .shutdown = hidma_shutdown, .driver = { .name = "hidma", - .of_match_table = hidma_match, .acpi_match_table = ACPI_PTR(hidma_acpi_ids), }, }; module_platform_driver(hidma_driver); +MODULE_DESCRIPTION("Qualcomm Technologies HIDMA Channel support"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/qcom/hidma_mgmt.c b/drivers/dma/qcom/hidma_mgmt.c index 62026607f3f8..4805ce390ffa 100644 --- a/drivers/dma/qcom/hidma_mgmt.c +++ b/drivers/dma/qcom/hidma_mgmt.c @@ -7,11 +7,8 @@ #include <linux/dmaengine.h> #include <linux/acpi.h> -#include <linux/of.h> #include <linux/property.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/module.h> #include <linux/uaccess.h> #include <linux/slab.h> @@ -174,10 +171,9 @@ static int hidma_mgmt_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); pm_runtime_get_sync(&pdev->dev); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - virtaddr = devm_ioremap_resource(&pdev->dev, res); + virtaddr = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(virtaddr)) { - rc = -ENOMEM; + rc = PTR_ERR(virtaddr); goto out; } @@ -326,115 +322,14 @@ static const struct acpi_device_id hidma_mgmt_acpi_ids[] = { MODULE_DEVICE_TABLE(acpi, hidma_mgmt_acpi_ids); #endif -static const struct of_device_id hidma_mgmt_match[] = { - {.compatible = "qcom,hidma-mgmt-1.0",}, - {}, -}; -MODULE_DEVICE_TABLE(of, hidma_mgmt_match); - static struct platform_driver hidma_mgmt_driver = { .probe = hidma_mgmt_probe, .driver = { .name = "hidma-mgmt", - .of_match_table = hidma_mgmt_match, .acpi_match_table = ACPI_PTR(hidma_mgmt_acpi_ids), }, }; -#if defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) -static int object_counter; - -static int __init hidma_mgmt_of_populate_channels(struct device_node *np) -{ - struct platform_device *pdev_parent = of_find_device_by_node(np); - struct platform_device_info pdevinfo; - struct device_node *child; - struct resource *res; - int ret = 0; - - /* allocate a resource array */ - res = kcalloc(3, sizeof(*res), GFP_KERNEL); - if (!res) - return -ENOMEM; - - for_each_available_child_of_node(np, child) { - struct platform_device *new_pdev; - - ret = of_address_to_resource(child, 0, &res[0]); - if (!ret) - goto out; - - ret = of_address_to_resource(child, 1, &res[1]); - if (!ret) - goto out; - - ret = of_irq_to_resource(child, 0, &res[2]); - if (ret <= 0) - goto out; - - memset(&pdevinfo, 0, sizeof(pdevinfo)); - pdevinfo.fwnode = &child->fwnode; - pdevinfo.parent = pdev_parent ? &pdev_parent->dev : NULL; - pdevinfo.name = child->name; - pdevinfo.id = object_counter++; - pdevinfo.res = res; - pdevinfo.num_res = 3; - pdevinfo.data = NULL; - pdevinfo.size_data = 0; - pdevinfo.dma_mask = DMA_BIT_MASK(64); - new_pdev = platform_device_register_full(&pdevinfo); - if (IS_ERR(new_pdev)) { - ret = PTR_ERR(new_pdev); - goto out; - } - new_pdev->dev.of_node = child; - of_dma_configure(&new_pdev->dev, child, true); - /* - * It is assumed that calling of_msi_configure is safe on - * platforms with or without MSI support. - */ - of_msi_configure(&new_pdev->dev, child); - } - - kfree(res); - - return ret; - -out: - of_node_put(child); - kfree(res); - - return ret; -} -#endif - -static int __init hidma_mgmt_init(void) -{ -#if defined(CONFIG_OF) && defined(CONFIG_OF_IRQ) - struct device_node *child; - - for_each_matching_node(child, hidma_mgmt_match) { - /* device tree based firmware here */ - hidma_mgmt_of_populate_channels(child); - } -#endif - /* - * We do not check for return value here, as it is assumed that - * platform_driver_register must not fail. The reason for this is that - * the (potential) hidma_mgmt_of_populate_channels calls above are not - * cleaned up if it does fail, and to do this work is quite - * complicated. In particular, various calls of of_address_to_resource, - * of_irq_to_resource, platform_device_register_full, of_dma_configure, - * and of_msi_configure which then call other functions and so on, must - * be cleaned up - this is not a trivial exercise. - * - * Currently, this module is not intended to be unloaded, and there is - * no module_exit function defined which does the needed cleanup. For - * this reason, we have to assume success here. - */ - platform_driver_register(&hidma_mgmt_driver); - - return 0; -} -module_init(hidma_mgmt_init); +module_platform_driver(hidma_mgmt_driver); +MODULE_DESCRIPTION("Qualcomm Technologies HIDMA DMA engine interface"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/qcom/qcom_adm.c b/drivers/dma/qcom/qcom_adm.c index ee78bed8d60d..6be54fddcee1 100644 --- a/drivers/dma/qcom/qcom_adm.c +++ b/drivers/dma/qcom/qcom_adm.c @@ -8,6 +8,7 @@ #include <linux/device.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> +#include <linux/dma/qcom_adm.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -140,6 +141,8 @@ struct adm_chan { struct adm_async_desc *curr_txd; struct dma_slave_config slave; + u32 crci; + u32 mux; struct list_head node; int error; @@ -376,13 +379,13 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, if (blk_size < 0) { dev_err(adev->dev, "invalid burst value: %d\n", burst); - return ERR_PTR(-EINVAL); + return NULL; } - crci = achan->slave.slave_id & 0xf; - if (!crci || achan->slave.slave_id > 0x1f) { + crci = achan->crci & 0xf; + if (!crci || achan->crci > 0x1f) { dev_err(adev->dev, "invalid crci value\n"); - return ERR_PTR(-EINVAL); + return NULL; } } @@ -400,12 +403,12 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, } async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT); - if (!async_desc) - return ERR_PTR(-ENOMEM); + if (!async_desc) { + dev_err(adev->dev, "not enough memory for async_desc struct\n"); + return NULL; + } - if (crci) - async_desc->mux = achan->slave.slave_id & ADM_CRCI_MUX_SEL ? - ADM_CRCI_CTL_MUX_SEL : 0; + async_desc->mux = achan->mux ? ADM_CRCI_CTL_MUX_SEL : 0; async_desc->crci = crci; async_desc->blk_size = blk_size; async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) + @@ -413,8 +416,10 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, sizeof(*cple) + 2 * ADM_DESC_ALIGN; async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT); - if (!async_desc->cpl) + if (!async_desc->cpl) { + dev_err(adev->dev, "not enough memory for cpl struct\n"); goto free; + } async_desc->adev = adev; @@ -436,8 +441,10 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl, async_desc->dma_len, DMA_TO_DEVICE); - if (dma_mapping_error(adev->dev, async_desc->dma_addr)) + if (dma_mapping_error(adev->dev, async_desc->dma_addr)) { + dev_err(adev->dev, "dma mapping error for cpl\n"); goto free; + } cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl); @@ -453,7 +460,7 @@ static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, free: kfree(async_desc); - return ERR_PTR(-ENOMEM); + return NULL; } /** @@ -488,10 +495,13 @@ static int adm_terminate_all(struct dma_chan *chan) static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) { struct adm_chan *achan = to_adm_chan(chan); + struct qcom_adm_peripheral_config *config = cfg->peripheral_config; unsigned long flag; spin_lock_irqsave(&achan->vc.lock, flag); memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config)); + if (cfg->peripheral_size == sizeof(*config)) + achan->crci = config->crci; spin_unlock_irqrestore(&achan->vc.lock, flag); return 0; @@ -640,7 +650,7 @@ static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie, /* * residue is either the full length if it is in the issued list, or 0 * if it is in progress. We have no reliable way of determining - * anything inbetween + * anything in between */ dma_set_residue(txstate, residue); @@ -694,6 +704,45 @@ static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan, achan->vc.desc_free = adm_dma_free_desc; } +/** + * adm_dma_xlate + * @dma_spec: pointer to DMA specifier as found in the device tree + * @ofdma: pointer to DMA controller data + * + * This can use either 1-cell or 2-cell formats, the first cell + * identifies the slave device, while the optional second cell + * contains the crci value. + * + * Returns pointer to appropriate dma channel on success or NULL on error. + */ +static struct dma_chan *adm_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct dma_device *dev = ofdma->of_dma_data; + struct dma_chan *chan, *candidate = NULL; + struct adm_chan *achan; + + if (!dev || dma_spec->args_count > 2) + return NULL; + + list_for_each_entry(chan, &dev->channels, device_node) + if (chan->chan_id == dma_spec->args[0]) { + candidate = chan; + break; + } + + if (!candidate) + return NULL; + + achan = to_adm_chan(candidate); + if (dma_spec->args_count == 2) + achan->crci = dma_spec->args[1]; + else + achan->crci = 0; + + return dma_get_slave_channel(candidate); +} + static int adm_dma_probe(struct platform_device *pdev) { struct adm_device *adev; @@ -838,8 +887,7 @@ static int adm_dma_probe(struct platform_device *pdev) goto err_disable_clks; } - ret = of_dma_controller_register(pdev->dev.of_node, - of_dma_xlate_by_chan_id, + ret = of_dma_controller_register(pdev->dev.of_node, adm_dma_xlate, &adev->common); if (ret) goto err_unregister_dma; @@ -856,7 +904,7 @@ err_disable_core_clk: return ret; } -static int adm_dma_remove(struct platform_device *pdev) +static void adm_dma_remove(struct platform_device *pdev) { struct adm_device *adev = platform_get_drvdata(pdev); struct adm_chan *achan; @@ -879,8 +927,6 @@ static int adm_dma_remove(struct platform_device *pdev) clk_disable_unprepare(adev->core_clk); clk_disable_unprepare(adev->iface_clk); - - return 0; } static const struct of_device_id adm_of_match[] = { diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c deleted file mode 100644 index 8e14c72d03f0..000000000000 --- a/drivers/dma/s3c24xx-dma.c +++ /dev/null @@ -1,1428 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * S3C24XX DMA handling - * - * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de> - * - * based on amba-pl08x.c - * - * Copyright (c) 2006 ARM Ltd. - * Copyright (c) 2010 ST-Ericsson SA - * - * Author: Peter Pearse <peter.pearse@arm.com> - * Author: Linus Walleij <linus.walleij@stericsson.com> - * - * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals - * that can be routed to any of the 4 to 8 hardware-channels. - * - * Therefore on these DMA controllers the number of channels - * and the number of incoming DMA signals are two totally different things. - * It is usually not possible to theoretically handle all physical signals, - * so a multiplexing scheme with possible denial of use is necessary. - * - * Open items: - * - bursts - */ - -#include <linux/platform_device.h> -#include <linux/types.h> -#include <linux/dmaengine.h> -#include <linux/dma-mapping.h> -#include <linux/interrupt.h> -#include <linux/clk.h> -#include <linux/module.h> -#include <linux/mod_devicetable.h> -#include <linux/slab.h> -#include <linux/platform_data/dma-s3c24xx.h> - -#include "dmaengine.h" -#include "virt-dma.h" - -#define MAX_DMA_CHANNELS 8 - -#define S3C24XX_DISRC 0x00 -#define S3C24XX_DISRCC 0x04 -#define S3C24XX_DISRCC_INC_INCREMENT 0 -#define S3C24XX_DISRCC_INC_FIXED BIT(0) -#define S3C24XX_DISRCC_LOC_AHB 0 -#define S3C24XX_DISRCC_LOC_APB BIT(1) - -#define S3C24XX_DIDST 0x08 -#define S3C24XX_DIDSTC 0x0c -#define S3C24XX_DIDSTC_INC_INCREMENT 0 -#define S3C24XX_DIDSTC_INC_FIXED BIT(0) -#define S3C24XX_DIDSTC_LOC_AHB 0 -#define S3C24XX_DIDSTC_LOC_APB BIT(1) -#define S3C24XX_DIDSTC_INT_TC0 0 -#define S3C24XX_DIDSTC_INT_RELOAD BIT(2) - -#define S3C24XX_DCON 0x10 - -#define S3C24XX_DCON_TC_MASK 0xfffff -#define S3C24XX_DCON_DSZ_BYTE (0 << 20) -#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20) -#define S3C24XX_DCON_DSZ_WORD (2 << 20) -#define S3C24XX_DCON_DSZ_MASK (3 << 20) -#define S3C24XX_DCON_DSZ_SHIFT 20 -#define S3C24XX_DCON_AUTORELOAD 0 -#define S3C24XX_DCON_NORELOAD BIT(22) -#define S3C24XX_DCON_HWTRIG BIT(23) -#define S3C24XX_DCON_HWSRC_SHIFT 24 -#define S3C24XX_DCON_SERV_SINGLE 0 -#define S3C24XX_DCON_SERV_WHOLE BIT(27) -#define S3C24XX_DCON_TSZ_UNIT 0 -#define S3C24XX_DCON_TSZ_BURST4 BIT(28) -#define S3C24XX_DCON_INT BIT(29) -#define S3C24XX_DCON_SYNC_PCLK 0 -#define S3C24XX_DCON_SYNC_HCLK BIT(30) -#define S3C24XX_DCON_DEMAND 0 -#define S3C24XX_DCON_HANDSHAKE BIT(31) - -#define S3C24XX_DSTAT 0x14 -#define S3C24XX_DSTAT_STAT_BUSY BIT(20) -#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff - -#define S3C24XX_DMASKTRIG 0x20 -#define S3C24XX_DMASKTRIG_SWTRIG BIT(0) -#define S3C24XX_DMASKTRIG_ON BIT(1) -#define S3C24XX_DMASKTRIG_STOP BIT(2) - -#define S3C24XX_DMAREQSEL 0x24 -#define S3C24XX_DMAREQSEL_HW BIT(0) - -/* - * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel - * for a DMA source. Instead only specific channels are valid. - * All of these SoCs have 4 physical channels and the number of request - * source bits is 3. Additionally we also need 1 bit to mark the channel - * as valid. - * Therefore we separate the chansel element of the channel data into 4 - * parts of 4 bits each, to hold the information if the channel is valid - * and the hw request source to use. - * - * Example: - * SDI is valid on channels 0, 2 and 3 - with varying hw request sources. - * For it the chansel field would look like - * - * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1 - * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2 - * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2 - */ -#define S3C24XX_CHANSEL_WIDTH 4 -#define S3C24XX_CHANSEL_VALID BIT(3) -#define S3C24XX_CHANSEL_REQ_MASK 7 - -/* - * struct soc_data - vendor-specific config parameters for individual SoCs - * @stride: spacing between the registers of each channel - * @has_reqsel: does the controller use the newer requestselection mechanism - * @has_clocks: are controllable dma-clocks present - */ -struct soc_data { - int stride; - bool has_reqsel; - bool has_clocks; -}; - -/* - * enum s3c24xx_dma_chan_state - holds the virtual channel states - * @S3C24XX_DMA_CHAN_IDLE: the channel is idle - * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport - * channel and is running a transfer on it - * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport - * channel to become available (only pertains to memcpy channels) - */ -enum s3c24xx_dma_chan_state { - S3C24XX_DMA_CHAN_IDLE, - S3C24XX_DMA_CHAN_RUNNING, - S3C24XX_DMA_CHAN_WAITING, -}; - -/* - * struct s3c24xx_sg - structure containing data per sg - * @src_addr: src address of sg - * @dst_addr: dst address of sg - * @len: transfer len in bytes - * @node: node for txd's dsg_list - */ -struct s3c24xx_sg { - dma_addr_t src_addr; - dma_addr_t dst_addr; - size_t len; - struct list_head node; -}; - -/* - * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor - * @vd: virtual DMA descriptor - * @dsg_list: list of children sg's - * @at: sg currently being transfered - * @width: transfer width - * @disrcc: value for source control register - * @didstc: value for destination control register - * @dcon: base value for dcon register - * @cyclic: indicate cyclic transfer - */ -struct s3c24xx_txd { - struct virt_dma_desc vd; - struct list_head dsg_list; - struct list_head *at; - u8 width; - u32 disrcc; - u32 didstc; - u32 dcon; - bool cyclic; -}; - -struct s3c24xx_dma_chan; - -/* - * struct s3c24xx_dma_phy - holder for the physical channels - * @id: physical index to this channel - * @valid: does the channel have all required elements - * @base: virtual memory base (remapped) for the this channel - * @irq: interrupt for this channel - * @clk: clock for this channel - * @lock: a lock to use when altering an instance of this struct - * @serving: virtual channel currently being served by this physicalchannel - * @host: a pointer to the host (internal use) - */ -struct s3c24xx_dma_phy { - unsigned int id; - bool valid; - void __iomem *base; - int irq; - struct clk *clk; - spinlock_t lock; - struct s3c24xx_dma_chan *serving; - struct s3c24xx_dma_engine *host; -}; - -/* - * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel - * @id: the id of the channel - * @name: name of the channel - * @vc: wrappped virtual channel - * @phy: the physical channel utilized by this channel, if there is one - * @runtime_addr: address for RX/TX according to the runtime config - * @at: active transaction on this channel - * @lock: a lock for this channel data - * @host: a pointer to the host (internal use) - * @state: whether the channel is idle, running etc - * @slave: whether this channel is a device (slave) or for memcpy - */ -struct s3c24xx_dma_chan { - int id; - const char *name; - struct virt_dma_chan vc; - struct s3c24xx_dma_phy *phy; - struct dma_slave_config cfg; - struct s3c24xx_txd *at; - struct s3c24xx_dma_engine *host; - enum s3c24xx_dma_chan_state state; - bool slave; -}; - -/* - * struct s3c24xx_dma_engine - the local state holder for the S3C24XX - * @pdev: the corresponding platform device - * @pdata: platform data passed in from the platform/machine - * @base: virtual memory base (remapped) - * @slave: slave engine for this instance - * @memcpy: memcpy engine for this instance - * @phy_chans: array of data for the physical channels - */ -struct s3c24xx_dma_engine { - struct platform_device *pdev; - const struct s3c24xx_dma_platdata *pdata; - struct soc_data *sdata; - void __iomem *base; - struct dma_device slave; - struct dma_device memcpy; - struct s3c24xx_dma_phy *phy_chans; -}; - -/* - * Physical channel handling - */ - -/* - * Check whether a certain channel is busy or not. - */ -static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy) -{ - unsigned int val = readl(phy->base + S3C24XX_DSTAT); - return val & S3C24XX_DSTAT_STAT_BUSY; -} - -static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan, - struct s3c24xx_dma_phy *phy) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; - int phyvalid; - - /* every phy is valid for memcopy channels */ - if (!s3cchan->slave) - return true; - - /* On newer variants all phys can be used for all virtual channels */ - if (s3cdma->sdata->has_reqsel) - return true; - - phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH)); - return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false; -} - -/* - * Allocate a physical channel for a virtual channel - * - * Try to locate a physical channel to be used for this transfer. If all - * are taken return NULL and the requester will have to cope by using - * some fallback PIO mode or retrying later. - */ -static -struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_phy *phy = NULL; - unsigned long flags; - int i; - int ret; - - for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { - phy = &s3cdma->phy_chans[i]; - - if (!phy->valid) - continue; - - if (!s3c24xx_dma_phy_valid(s3cchan, phy)) - continue; - - spin_lock_irqsave(&phy->lock, flags); - - if (!phy->serving) { - phy->serving = s3cchan; - spin_unlock_irqrestore(&phy->lock, flags); - break; - } - - spin_unlock_irqrestore(&phy->lock, flags); - } - - /* No physical channel available, cope with it */ - if (i == s3cdma->pdata->num_phy_channels) { - dev_warn(&s3cdma->pdev->dev, "no phy channel available\n"); - return NULL; - } - - /* start the phy clock */ - if (s3cdma->sdata->has_clocks) { - ret = clk_enable(phy->clk); - if (ret) { - dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n", - phy->id, ret); - phy->serving = NULL; - return NULL; - } - } - - return phy; -} - -/* - * Mark the physical channel as free. - * - * This drops the link between the physical and virtual channel. - */ -static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy) -{ - struct s3c24xx_dma_engine *s3cdma = phy->host; - - if (s3cdma->sdata->has_clocks) - clk_disable(phy->clk); - - phy->serving = NULL; -} - -/* - * Stops the channel by writing the stop bit. - * This should not be used for an on-going transfer, but as a method of - * shutting down a channel (eg, when it's no longer used) or terminating a - * transfer. - */ -static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy) -{ - writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG); -} - -/* - * Virtual channel handling - */ - -static inline -struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan) -{ - return container_of(chan, struct s3c24xx_dma_chan, vc.chan); -} - -static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_phy *phy = s3cchan->phy; - struct s3c24xx_txd *txd = s3cchan->at; - u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK; - - return tc * txd->width; -} - -static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan, - struct dma_slave_config *config) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - unsigned long flags; - int ret = 0; - - /* Reject definitely invalid configurations */ - if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || - config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) - return -EINVAL; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - - if (!s3cchan->slave) { - ret = -EINVAL; - goto out; - } - - s3cchan->cfg = *config; - -out: - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - return ret; -} - -/* - * Transfer handling - */ - -static inline -struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx) -{ - return container_of(tx, struct s3c24xx_txd, vd.tx); -} - -static struct s3c24xx_txd *s3c24xx_dma_get_txd(void) -{ - struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); - - if (txd) { - INIT_LIST_HEAD(&txd->dsg_list); - txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD; - } - - return txd; -} - -static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd) -{ - struct s3c24xx_sg *dsg, *_dsg; - - list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { - list_del(&dsg->node); - kfree(dsg); - } - - kfree(txd); -} - -static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan, - struct s3c24xx_txd *txd) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_phy *phy = s3cchan->phy; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node); - u32 dcon = txd->dcon; - u32 val; - - /* transfer-size and -count from len and width */ - switch (txd->width) { - case 1: - dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len; - break; - case 2: - dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2); - break; - case 4: - dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4); - break; - } - - if (s3cchan->slave) { - struct s3c24xx_dma_channel *cdata = - &pdata->channels[s3cchan->id]; - - if (s3cdma->sdata->has_reqsel) { - writel_relaxed((cdata->chansel << 1) | - S3C24XX_DMAREQSEL_HW, - phy->base + S3C24XX_DMAREQSEL); - } else { - int csel = cdata->chansel >> (phy->id * - S3C24XX_CHANSEL_WIDTH); - - csel &= S3C24XX_CHANSEL_REQ_MASK; - dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT; - dcon |= S3C24XX_DCON_HWTRIG; - } - } else { - if (s3cdma->sdata->has_reqsel) - writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL); - } - - writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC); - writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC); - writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST); - writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC); - writel_relaxed(dcon, phy->base + S3C24XX_DCON); - - val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG); - val &= ~S3C24XX_DMASKTRIG_STOP; - val |= S3C24XX_DMASKTRIG_ON; - - /* trigger the dma operation for memcpy transfers */ - if (!s3cchan->slave) - val |= S3C24XX_DMASKTRIG_SWTRIG; - - writel(val, phy->base + S3C24XX_DMASKTRIG); -} - -/* - * Set the initial DMA register values and start first sg. - */ -static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_phy *phy = s3cchan->phy; - struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc); - struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); - - list_del(&txd->vd.node); - - s3cchan->at = txd; - - /* Wait for channel inactive */ - while (s3c24xx_dma_phy_busy(phy)) - cpu_relax(); - - /* point to the first element of the sg list */ - txd->at = txd->dsg_list.next; - s3c24xx_dma_start_next_sg(s3cchan, txd); -} - -/* - * Try to allocate a physical channel. When successful, assign it to - * this virtual channel, and initiate the next descriptor. The - * virtual channel lock must be held at this point. - */ -static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_phy *phy; - - phy = s3c24xx_dma_get_phy(s3cchan); - if (!phy) { - dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n", - s3cchan->name); - s3cchan->state = S3C24XX_DMA_CHAN_WAITING; - return; - } - - dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n", - phy->id, s3cchan->name); - - s3cchan->phy = phy; - s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; - - s3c24xx_dma_start_next_txd(s3cchan); -} - -static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy, - struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - - dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n", - phy->id, s3cchan->name); - - /* - * We do this without taking the lock; we're really only concerned - * about whether this pointer is NULL or not, and we're guaranteed - * that this will only be called when it _already_ is non-NULL. - */ - phy->serving = s3cchan; - s3cchan->phy = phy; - s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; - s3c24xx_dma_start_next_txd(s3cchan); -} - -/* - * Free a physical DMA channel, potentially reallocating it to another - * virtual channel if we have any pending. - */ -static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan) -{ - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_dma_chan *p, *next; - -retry: - next = NULL; - - /* Find a waiting virtual channel for the next transfer. */ - list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node) - if (p->state == S3C24XX_DMA_CHAN_WAITING) { - next = p; - break; - } - - if (!next) { - list_for_each_entry(p, &s3cdma->slave.channels, - vc.chan.device_node) - if (p->state == S3C24XX_DMA_CHAN_WAITING && - s3c24xx_dma_phy_valid(p, s3cchan->phy)) { - next = p; - break; - } - } - - /* Ensure that the physical channel is stopped */ - s3c24xx_dma_terminate_phy(s3cchan->phy); - - if (next) { - bool success; - - /* - * Eww. We know this isn't going to deadlock - * but lockdep probably doesn't. - */ - spin_lock(&next->vc.lock); - /* Re-check the state now that we have the lock */ - success = next->state == S3C24XX_DMA_CHAN_WAITING; - if (success) - s3c24xx_dma_phy_reassign_start(s3cchan->phy, next); - spin_unlock(&next->vc.lock); - - /* If the state changed, try to find another channel */ - if (!success) - goto retry; - } else { - /* No more jobs, so free up the physical channel */ - s3c24xx_dma_put_phy(s3cchan->phy); - } - - s3cchan->phy = NULL; - s3cchan->state = S3C24XX_DMA_CHAN_IDLE; -} - -static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd) -{ - struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan); - - if (!s3cchan->slave) - dma_descriptor_unmap(&vd->tx); - - s3c24xx_dma_free_txd(txd); -} - -static irqreturn_t s3c24xx_dma_irq(int irq, void *data) -{ - struct s3c24xx_dma_phy *phy = data; - struct s3c24xx_dma_chan *s3cchan = phy->serving; - struct s3c24xx_txd *txd; - - dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id); - - /* - * Interrupts happen to notify the completion of a transfer and the - * channel should have moved into its stop state already on its own. - * Therefore interrupts on channels not bound to a virtual channel - * should never happen. Nevertheless send a terminate command to the - * channel if the unlikely case happens. - */ - if (unlikely(!s3cchan)) { - dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n", - phy->id); - - s3c24xx_dma_terminate_phy(phy); - - return IRQ_HANDLED; - } - - spin_lock(&s3cchan->vc.lock); - txd = s3cchan->at; - if (txd) { - /* when more sg's are in this txd, start the next one */ - if (!list_is_last(txd->at, &txd->dsg_list)) { - txd->at = txd->at->next; - if (txd->cyclic) - vchan_cyclic_callback(&txd->vd); - s3c24xx_dma_start_next_sg(s3cchan, txd); - } else if (!txd->cyclic) { - s3cchan->at = NULL; - vchan_cookie_complete(&txd->vd); - - /* - * And start the next descriptor (if any), - * otherwise free this channel. - */ - if (vchan_next_desc(&s3cchan->vc)) - s3c24xx_dma_start_next_txd(s3cchan); - else - s3c24xx_dma_phy_free(s3cchan); - } else { - vchan_cyclic_callback(&txd->vd); - - /* Cyclic: reset at beginning */ - txd->at = txd->dsg_list.next; - s3c24xx_dma_start_next_sg(s3cchan, txd); - } - } - spin_unlock(&s3cchan->vc.lock); - - return IRQ_HANDLED; -} - -/* - * The DMA ENGINE API - */ - -static int s3c24xx_dma_terminate_all(struct dma_chan *chan) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - LIST_HEAD(head); - unsigned long flags; - int ret; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - - if (!s3cchan->phy && !s3cchan->at) { - dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n", - s3cchan->id); - ret = -EINVAL; - goto unlock; - } - - s3cchan->state = S3C24XX_DMA_CHAN_IDLE; - - /* Mark physical channel as free */ - if (s3cchan->phy) - s3c24xx_dma_phy_free(s3cchan); - - /* Dequeue current job */ - if (s3cchan->at) { - vchan_terminate_vdesc(&s3cchan->at->vd); - s3cchan->at = NULL; - } - - /* Dequeue jobs not yet fired as well */ - - vchan_get_all_descriptors(&s3cchan->vc, &head); - - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - - vchan_dma_desc_free_list(&s3cchan->vc, &head); - - return 0; - -unlock: - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - - return ret; -} - -static void s3c24xx_dma_synchronize(struct dma_chan *chan) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - - vchan_synchronize(&s3cchan->vc); -} - -static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan) -{ - /* Ensure all queued descriptors are freed */ - vchan_free_chan_resources(to_virt_chan(chan)); -} - -static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan, - dma_cookie_t cookie, struct dma_tx_state *txstate) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - struct virt_dma_desc *vd; - unsigned long flags; - enum dma_status ret; - size_t bytes = 0; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - ret = dma_cookie_status(chan, cookie, txstate); - - /* - * There's no point calculating the residue if there's - * no txstate to store the value. - */ - if (ret == DMA_COMPLETE || !txstate) { - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - return ret; - } - - vd = vchan_find_desc(&s3cchan->vc, cookie); - if (vd) { - /* On the issued list, so hasn't been processed yet */ - txd = to_s3c24xx_txd(&vd->tx); - - list_for_each_entry(dsg, &txd->dsg_list, node) - bytes += dsg->len; - } else { - /* - * Currently running, so sum over the pending sg's and - * the currently active one. - */ - txd = s3cchan->at; - - dsg = list_entry(txd->at, struct s3c24xx_sg, node); - list_for_each_entry_from(dsg, &txd->dsg_list, node) - bytes += dsg->len; - - bytes += s3c24xx_dma_getbytes_chan(s3cchan); - } - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); - - /* - * This cookie not complete yet - * Get number of bytes left in the active transactions and queue - */ - dma_set_residue(txstate, bytes); - - /* Whether waiting or running, we're in progress */ - return ret; -} - -/* - * Initialize a descriptor to be used by memcpy submit - */ -static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy( - struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, - size_t len, unsigned long flags) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - int src_mod, dest_mod; - - dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n", - len, s3cchan->name); - - if ((len & S3C24XX_DCON_TC_MASK) != len) { - dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len); - return NULL; - } - - txd = s3c24xx_dma_get_txd(); - if (!txd) - return NULL; - - dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); - if (!dsg) { - s3c24xx_dma_free_txd(txd); - return NULL; - } - list_add_tail(&dsg->node, &txd->dsg_list); - - dsg->src_addr = src; - dsg->dst_addr = dest; - dsg->len = len; - - /* - * Determine a suitable transfer width. - * The DMA controller cannot fetch/store information which is not - * naturally aligned on the bus, i.e., a 4 byte fetch must start at - * an address divisible by 4 - more generally addr % width must be 0. - */ - src_mod = src % 4; - dest_mod = dest % 4; - switch (len % 4) { - case 0: - txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1; - break; - case 2: - txd->width = ((src_mod == 2 || src_mod == 0) && - (dest_mod == 2 || dest_mod == 0)) ? 2 : 1; - break; - default: - txd->width = 1; - break; - } - - txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT; - txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT; - txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK | - S3C24XX_DCON_SERV_WHOLE; - - return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); -} - -static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic( - struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period, - enum dma_transfer_direction direction, unsigned long flags) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - unsigned sg_len; - dma_addr_t slave_addr; - u32 hwcfg = 0; - int i; - - dev_dbg(&s3cdma->pdev->dev, - "prepare cyclic transaction of %zu bytes with period %zu from %s\n", - size, period, s3cchan->name); - - if (!is_slave_direction(direction)) { - dev_err(&s3cdma->pdev->dev, - "direction %d unsupported\n", direction); - return NULL; - } - - txd = s3c24xx_dma_get_txd(); - if (!txd) - return NULL; - - txd->cyclic = 1; - - if (cdata->handshake) - txd->dcon |= S3C24XX_DCON_HANDSHAKE; - - switch (cdata->bus) { - case S3C24XX_DMA_APB: - txd->dcon |= S3C24XX_DCON_SYNC_PCLK; - hwcfg |= S3C24XX_DISRCC_LOC_APB; - break; - case S3C24XX_DMA_AHB: - txd->dcon |= S3C24XX_DCON_SYNC_HCLK; - hwcfg |= S3C24XX_DISRCC_LOC_AHB; - break; - } - - /* - * Always assume our peripheral desintation is a fixed - * address in memory. - */ - hwcfg |= S3C24XX_DISRCC_INC_FIXED; - - /* - * Individual dma operations are requested by the slave, - * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE). - */ - txd->dcon |= S3C24XX_DCON_SERV_SINGLE; - - if (direction == DMA_MEM_TO_DEV) { - txd->disrcc = S3C24XX_DISRCC_LOC_AHB | - S3C24XX_DISRCC_INC_INCREMENT; - txd->didstc = hwcfg; - slave_addr = s3cchan->cfg.dst_addr; - txd->width = s3cchan->cfg.dst_addr_width; - } else { - txd->disrcc = hwcfg; - txd->didstc = S3C24XX_DIDSTC_LOC_AHB | - S3C24XX_DIDSTC_INC_INCREMENT; - slave_addr = s3cchan->cfg.src_addr; - txd->width = s3cchan->cfg.src_addr_width; - } - - sg_len = size / period; - - for (i = 0; i < sg_len; i++) { - dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); - if (!dsg) { - s3c24xx_dma_free_txd(txd); - return NULL; - } - list_add_tail(&dsg->node, &txd->dsg_list); - - dsg->len = period; - /* Check last period length */ - if (i == sg_len - 1) - dsg->len = size - period * i; - if (direction == DMA_MEM_TO_DEV) { - dsg->src_addr = addr + period * i; - dsg->dst_addr = slave_addr; - } else { /* DMA_DEV_TO_MEM */ - dsg->src_addr = slave_addr; - dsg->dst_addr = addr + period * i; - } - } - - return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); -} - -static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - struct s3c24xx_dma_engine *s3cdma = s3cchan->host; - const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; - struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; - struct s3c24xx_txd *txd; - struct s3c24xx_sg *dsg; - struct scatterlist *sg; - dma_addr_t slave_addr; - u32 hwcfg = 0; - int tmp; - - dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n", - sg_dma_len(sgl), s3cchan->name); - - txd = s3c24xx_dma_get_txd(); - if (!txd) - return NULL; - - if (cdata->handshake) - txd->dcon |= S3C24XX_DCON_HANDSHAKE; - - switch (cdata->bus) { - case S3C24XX_DMA_APB: - txd->dcon |= S3C24XX_DCON_SYNC_PCLK; - hwcfg |= S3C24XX_DISRCC_LOC_APB; - break; - case S3C24XX_DMA_AHB: - txd->dcon |= S3C24XX_DCON_SYNC_HCLK; - hwcfg |= S3C24XX_DISRCC_LOC_AHB; - break; - } - - /* - * Always assume our peripheral desintation is a fixed - * address in memory. - */ - hwcfg |= S3C24XX_DISRCC_INC_FIXED; - - /* - * Individual dma operations are requested by the slave, - * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE). - */ - txd->dcon |= S3C24XX_DCON_SERV_SINGLE; - - if (direction == DMA_MEM_TO_DEV) { - txd->disrcc = S3C24XX_DISRCC_LOC_AHB | - S3C24XX_DISRCC_INC_INCREMENT; - txd->didstc = hwcfg; - slave_addr = s3cchan->cfg.dst_addr; - txd->width = s3cchan->cfg.dst_addr_width; - } else if (direction == DMA_DEV_TO_MEM) { - txd->disrcc = hwcfg; - txd->didstc = S3C24XX_DIDSTC_LOC_AHB | - S3C24XX_DIDSTC_INC_INCREMENT; - slave_addr = s3cchan->cfg.src_addr; - txd->width = s3cchan->cfg.src_addr_width; - } else { - s3c24xx_dma_free_txd(txd); - dev_err(&s3cdma->pdev->dev, - "direction %d unsupported\n", direction); - return NULL; - } - - for_each_sg(sgl, sg, sg_len, tmp) { - dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); - if (!dsg) { - s3c24xx_dma_free_txd(txd); - return NULL; - } - list_add_tail(&dsg->node, &txd->dsg_list); - - dsg->len = sg_dma_len(sg); - if (direction == DMA_MEM_TO_DEV) { - dsg->src_addr = sg_dma_address(sg); - dsg->dst_addr = slave_addr; - } else { /* DMA_DEV_TO_MEM */ - dsg->src_addr = slave_addr; - dsg->dst_addr = sg_dma_address(sg); - } - } - - return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); -} - -/* - * Slave transactions callback to the slave device to allow - * synchronization of slave DMA signals with the DMAC enable - */ -static void s3c24xx_dma_issue_pending(struct dma_chan *chan) -{ - struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); - unsigned long flags; - - spin_lock_irqsave(&s3cchan->vc.lock, flags); - if (vchan_issue_pending(&s3cchan->vc)) { - if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING) - s3c24xx_dma_phy_alloc_and_start(s3cchan); - } - spin_unlock_irqrestore(&s3cchan->vc.lock, flags); -} - -/* - * Bringup and teardown - */ - -/* - * Initialise the DMAC memcpy/slave channels. - * Make a local wrapper to hold required data - */ -static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma, - struct dma_device *dmadev, unsigned int channels, bool slave) -{ - struct s3c24xx_dma_chan *chan; - int i; - - INIT_LIST_HEAD(&dmadev->channels); - - /* - * Register as many many memcpy as we have physical channels, - * we won't always be able to use all but the code will have - * to cope with that situation. - */ - for (i = 0; i < channels; i++) { - chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL); - if (!chan) - return -ENOMEM; - - chan->id = i; - chan->host = s3cdma; - chan->state = S3C24XX_DMA_CHAN_IDLE; - - if (slave) { - chan->slave = true; - chan->name = kasprintf(GFP_KERNEL, "slave%d", i); - if (!chan->name) - return -ENOMEM; - } else { - chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); - if (!chan->name) - return -ENOMEM; - } - dev_dbg(dmadev->dev, - "initialize virtual channel \"%s\"\n", - chan->name); - - chan->vc.desc_free = s3c24xx_dma_desc_free; - vchan_init(&chan->vc, dmadev); - } - dev_info(dmadev->dev, "initialized %d virtual %s channels\n", - i, slave ? "slave" : "memcpy"); - return i; -} - -static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev) -{ - struct s3c24xx_dma_chan *chan = NULL; - struct s3c24xx_dma_chan *next; - - list_for_each_entry_safe(chan, - next, &dmadev->channels, vc.chan.device_node) { - list_del(&chan->vc.chan.device_node); - tasklet_kill(&chan->vc.task); - } -} - -/* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */ -static struct soc_data soc_s3c2410 = { - .stride = 0x40, - .has_reqsel = false, - .has_clocks = false, -}; - -/* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */ -static struct soc_data soc_s3c2412 = { - .stride = 0x40, - .has_reqsel = true, - .has_clocks = true, -}; - -/* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */ -static struct soc_data soc_s3c2443 = { - .stride = 0x100, - .has_reqsel = true, - .has_clocks = true, -}; - -static const struct platform_device_id s3c24xx_dma_driver_ids[] = { - { - .name = "s3c2410-dma", - .driver_data = (kernel_ulong_t)&soc_s3c2410, - }, { - .name = "s3c2412-dma", - .driver_data = (kernel_ulong_t)&soc_s3c2412, - }, { - .name = "s3c2443-dma", - .driver_data = (kernel_ulong_t)&soc_s3c2443, - }, - { }, -}; - -static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev) -{ - return (struct soc_data *) - platform_get_device_id(pdev)->driver_data; -} - -static int s3c24xx_dma_probe(struct platform_device *pdev) -{ - const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev); - struct s3c24xx_dma_engine *s3cdma; - struct soc_data *sdata; - struct resource *res; - int ret; - int i; - - if (!pdata) { - dev_err(&pdev->dev, "platform data missing\n"); - return -ENODEV; - } - - /* Basic sanity check */ - if (pdata->num_phy_channels > MAX_DMA_CHANNELS) { - dev_err(&pdev->dev, "too many dma channels %d, max %d\n", - pdata->num_phy_channels, MAX_DMA_CHANNELS); - return -EINVAL; - } - - sdata = s3c24xx_dma_get_soc_data(pdev); - if (!sdata) - return -EINVAL; - - s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL); - if (!s3cdma) - return -ENOMEM; - - s3cdma->pdev = pdev; - s3cdma->pdata = pdata; - s3cdma->sdata = sdata; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - s3cdma->base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(s3cdma->base)) - return PTR_ERR(s3cdma->base); - - s3cdma->phy_chans = devm_kcalloc(&pdev->dev, - pdata->num_phy_channels, - sizeof(struct s3c24xx_dma_phy), - GFP_KERNEL); - if (!s3cdma->phy_chans) - return -ENOMEM; - - /* acquire irqs and clocks for all physical channels */ - for (i = 0; i < pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - char clk_name[6]; - - phy->id = i; - phy->base = s3cdma->base + (i * sdata->stride); - phy->host = s3cdma; - - phy->irq = platform_get_irq(pdev, i); - if (phy->irq < 0) - continue; - - ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq, - 0, pdev->name, phy); - if (ret) { - dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n", - i, ret); - continue; - } - - if (sdata->has_clocks) { - sprintf(clk_name, "dma.%d", i); - phy->clk = devm_clk_get(&pdev->dev, clk_name); - if (IS_ERR(phy->clk) && sdata->has_clocks) { - dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n", - i, PTR_ERR(phy->clk)); - continue; - } - - ret = clk_prepare(phy->clk); - if (ret) { - dev_err(&pdev->dev, "clock for phy %d failed, error %d\n", - i, ret); - continue; - } - } - - spin_lock_init(&phy->lock); - phy->valid = true; - - dev_dbg(&pdev->dev, "physical channel %d is %s\n", - i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE"); - } - - /* Initialize memcpy engine */ - dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask); - dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask); - s3cdma->memcpy.dev = &pdev->dev; - s3cdma->memcpy.device_free_chan_resources = - s3c24xx_dma_free_chan_resources; - s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy; - s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status; - s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending; - s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config; - s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all; - s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize; - - /* Initialize slave engine for SoC internal dedicated peripherals */ - dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask); - dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask); - dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask); - s3cdma->slave.dev = &pdev->dev; - s3cdma->slave.device_free_chan_resources = - s3c24xx_dma_free_chan_resources; - s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status; - s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending; - s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg; - s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic; - s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config; - s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all; - s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize; - s3cdma->slave.filter.map = pdata->slave_map; - s3cdma->slave.filter.mapcnt = pdata->slavecnt; - s3cdma->slave.filter.fn = s3c24xx_dma_filter; - - /* Register as many memcpy channels as there are physical channels */ - ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy, - pdata->num_phy_channels, false); - if (ret <= 0) { - dev_warn(&pdev->dev, - "%s failed to enumerate memcpy channels - %d\n", - __func__, ret); - goto err_memcpy; - } - - /* Register slave channels */ - ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave, - pdata->num_channels, true); - if (ret <= 0) { - dev_warn(&pdev->dev, - "%s failed to enumerate slave channels - %d\n", - __func__, ret); - goto err_slave; - } - - ret = dma_async_device_register(&s3cdma->memcpy); - if (ret) { - dev_warn(&pdev->dev, - "%s failed to register memcpy as an async device - %d\n", - __func__, ret); - goto err_memcpy_reg; - } - - ret = dma_async_device_register(&s3cdma->slave); - if (ret) { - dev_warn(&pdev->dev, - "%s failed to register slave as an async device - %d\n", - __func__, ret); - goto err_slave_reg; - } - - platform_set_drvdata(pdev, s3cdma); - dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n", - pdata->num_phy_channels); - - return 0; - -err_slave_reg: - dma_async_device_unregister(&s3cdma->memcpy); -err_memcpy_reg: - s3c24xx_dma_free_virtual_channels(&s3cdma->slave); -err_slave: - s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy); -err_memcpy: - if (sdata->has_clocks) - for (i = 0; i < pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - if (phy->valid) - clk_unprepare(phy->clk); - } - - return ret; -} - -static void s3c24xx_dma_free_irq(struct platform_device *pdev, - struct s3c24xx_dma_engine *s3cdma) -{ - int i; - - for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - - devm_free_irq(&pdev->dev, phy->irq, phy); - } -} - -static int s3c24xx_dma_remove(struct platform_device *pdev) -{ - const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev); - struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev); - struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev); - int i; - - dma_async_device_unregister(&s3cdma->slave); - dma_async_device_unregister(&s3cdma->memcpy); - - s3c24xx_dma_free_irq(pdev, s3cdma); - - s3c24xx_dma_free_virtual_channels(&s3cdma->slave); - s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy); - - if (sdata->has_clocks) - for (i = 0; i < pdata->num_phy_channels; i++) { - struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; - if (phy->valid) - clk_unprepare(phy->clk); - } - - return 0; -} - -static struct platform_driver s3c24xx_dma_driver = { - .driver = { - .name = "s3c24xx-dma", - }, - .id_table = s3c24xx_dma_driver_ids, - .probe = s3c24xx_dma_probe, - .remove = s3c24xx_dma_remove, -}; - -module_platform_driver(s3c24xx_dma_driver); - -bool s3c24xx_dma_filter(struct dma_chan *chan, void *param) -{ - struct s3c24xx_dma_chan *s3cchan; - - if (chan->device->dev->driver != &s3c24xx_dma_driver.driver) - return false; - - s3cchan = to_s3c24xx_dma_chan(chan); - - return s3cchan->id == (uintptr_t)param; -} -EXPORT_SYMBOL(s3c24xx_dma_filter); - -MODULE_DESCRIPTION("S3C24XX DMA Driver"); -MODULE_AUTHOR("Heiko Stuebner"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index 1e918e284fc0..dc1a9a05252e 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -78,7 +78,7 @@ struct sa11x0_dma_desc { bool cyclic; unsigned sglen; - struct sa11x0_dma_sg sg[]; + struct sa11x0_dma_sg sg[] __counted_by(sglen); }; struct sa11x0_dma_phy; @@ -558,6 +558,7 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg( dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc); return NULL; } + txd->sglen = j; j = 0; for_each_sg(sg, sgent, sglen, i) { @@ -593,7 +594,6 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg( txd->ddar = c->ddar; txd->size = size; - txd->sglen = j; dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n", &c->vc, &txd->vd, txd->size, txd->sglen); @@ -628,6 +628,7 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic( dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc); return NULL; } + txd->sglen = sglen; for (i = k = 0; i < size / period; i++) { size_t tlen, len = period; @@ -653,7 +654,6 @@ static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic( txd->ddar = c->ddar; txd->size = size; - txd->sglen = sglen; txd->cyclic = 1; txd->period = sgperiod; @@ -984,7 +984,7 @@ static int sa11x0_dma_probe(struct platform_device *pdev) return ret; } -static int sa11x0_dma_remove(struct platform_device *pdev) +static void sa11x0_dma_remove(struct platform_device *pdev) { struct sa11x0_dma_dev *d = platform_get_drvdata(pdev); unsigned pch; @@ -997,11 +997,9 @@ static int sa11x0_dma_remove(struct platform_device *pdev) tasklet_kill(&d->task); iounmap(d->base); kfree(d); - - return 0; } -static int sa11x0_dma_suspend(struct device *dev) +static __maybe_unused int sa11x0_dma_suspend(struct device *dev) { struct sa11x0_dma_dev *d = dev_get_drvdata(dev); unsigned pch; @@ -1039,7 +1037,7 @@ static int sa11x0_dma_suspend(struct device *dev) return 0; } -static int sa11x0_dma_resume(struct device *dev) +static __maybe_unused int sa11x0_dma_resume(struct device *dev) { struct sa11x0_dma_dev *d = dev_get_drvdata(dev); unsigned pch; @@ -1072,12 +1070,7 @@ static int sa11x0_dma_resume(struct device *dev) } static const struct dev_pm_ops sa11x0_dma_pm_ops = { - .suspend_noirq = sa11x0_dma_suspend, - .resume_noirq = sa11x0_dma_resume, - .freeze_noirq = sa11x0_dma_suspend, - .thaw_noirq = sa11x0_dma_resume, - .poweroff_noirq = sa11x0_dma_suspend, - .restore_noirq = sa11x0_dma_resume, + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sa11x0_dma_suspend, sa11x0_dma_resume) }; static struct platform_driver sa11x0_dma_driver = { diff --git a/drivers/dma/sf-pdma/sf-pdma.c b/drivers/dma/sf-pdma/sf-pdma.c index f12606aeff87..7ad3c29be146 100644 --- a/drivers/dma/sf-pdma/sf-pdma.c +++ b/drivers/dma/sf-pdma/sf-pdma.c @@ -20,10 +20,13 @@ #include <linux/mod_devicetable.h> #include <linux/dma-mapping.h> #include <linux/of.h> +#include <linux/of_dma.h> #include <linux/slab.h> #include "sf-pdma.h" +#define PDMA_QUIRK_NO_STRICT_ORDERING BIT(0) + #ifndef readq static inline unsigned long long readq(void __iomem *addr) { @@ -52,16 +55,6 @@ static inline struct sf_pdma_desc *to_sf_pdma_desc(struct virt_dma_desc *vd) static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan) { struct sf_pdma_desc *desc; - unsigned long flags; - - spin_lock_irqsave(&chan->lock, flags); - - if (chan->desc && !chan->desc->in_use) { - spin_unlock_irqrestore(&chan->lock, flags); - return chan->desc; - } - - spin_unlock_irqrestore(&chan->lock, flags); desc = kzalloc(sizeof(*desc), GFP_NOWAIT); if (!desc) @@ -75,7 +68,7 @@ static struct sf_pdma_desc *sf_pdma_alloc_desc(struct sf_pdma_chan *chan) static void sf_pdma_fill_desc(struct sf_pdma_desc *desc, u64 dst, u64 src, u64 size) { - desc->xfer_type = PDMA_FULL_SPEED; + desc->xfer_type = desc->chan->pdma->transfer_type; desc->xfer_size = size; desc->dst_addr = dst; desc->src_addr = src; @@ -106,12 +99,10 @@ sf_pdma_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dest, dma_addr_t src, if (!desc) return NULL; - desc->in_use = true; desc->dirn = DMA_MEM_TO_MEM; desc->async_tx = vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); spin_lock_irqsave(&chan->vchan.lock, iflags); - chan->desc = desc; sf_pdma_fill_desc(desc, dest, src, len); spin_unlock_irqrestore(&chan->vchan.lock, iflags); @@ -170,11 +161,17 @@ static size_t sf_pdma_desc_residue(struct sf_pdma_chan *chan, unsigned long flags; u64 residue = 0; struct sf_pdma_desc *desc; - struct dma_async_tx_descriptor *tx; + struct dma_async_tx_descriptor *tx = NULL; spin_lock_irqsave(&chan->vchan.lock, flags); - tx = &chan->desc->vdesc.tx; + list_for_each_entry(vd, &chan->vchan.desc_submitted, node) + if (vd->tx.cookie == cookie) + tx = &vd->tx; + + if (!tx) + goto out; + if (cookie == tx->chan->completed_cookie) goto out; @@ -241,6 +238,19 @@ static void sf_pdma_enable_request(struct sf_pdma_chan *chan) writel(v, regs->ctrl); } +static struct sf_pdma_desc *sf_pdma_get_first_pending_desc(struct sf_pdma_chan *chan) +{ + struct virt_dma_chan *vchan = &chan->vchan; + struct virt_dma_desc *vdesc; + + if (list_empty(&vchan->desc_issued)) + return NULL; + + vdesc = list_first_entry(&vchan->desc_issued, struct virt_dma_desc, node); + + return container_of(vdesc, struct sf_pdma_desc, vdesc); +} + static void sf_pdma_xfer_desc(struct sf_pdma_chan *chan) { struct sf_pdma_desc *desc = chan->desc; @@ -268,8 +278,11 @@ static void sf_pdma_issue_pending(struct dma_chan *dchan) spin_lock_irqsave(&chan->vchan.lock, flags); - if (vchan_issue_pending(&chan->vchan) && chan->desc) + if (!chan->desc && vchan_issue_pending(&chan->vchan)) { + /* vchan_issue_pending has made a check that desc in not NULL */ + chan->desc = sf_pdma_get_first_pending_desc(chan); sf_pdma_xfer_desc(chan); + } spin_unlock_irqrestore(&chan->vchan.lock, flags); } @@ -279,7 +292,7 @@ static void sf_pdma_free_desc(struct virt_dma_desc *vdesc) struct sf_pdma_desc *desc; desc = to_sf_pdma_desc(vdesc); - desc->in_use = false; + kfree(desc); } static void sf_pdma_donebh_tasklet(struct tasklet_struct *t) @@ -298,6 +311,11 @@ static void sf_pdma_donebh_tasklet(struct tasklet_struct *t) spin_lock_irqsave(&chan->vchan.lock, flags); list_del(&chan->desc->vdesc.node); vchan_cookie_complete(&chan->desc->vdesc); + + chan->desc = sf_pdma_get_first_pending_desc(chan); + if (chan->desc) + sf_pdma_xfer_desc(chan); + spin_unlock_irqrestore(&chan->vchan.lock, flags); } @@ -336,7 +354,7 @@ static irqreturn_t sf_pdma_done_isr(int irq, void *dev_id) if (!residue) { tasklet_hi_schedule(&chan->done_tasklet); } else { - /* submit next trascatioin if possible */ + /* submit next transaction if possible */ struct sf_pdma_desc *desc = chan->desc; desc->src_addr += desc->xfer_size - residue; @@ -389,10 +407,8 @@ static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) chan = &pdma->chans[i]; irq = platform_get_irq(pdev, i * 2); - if (irq < 0) { - dev_err(&pdev->dev, "ch(%d) Can't get done irq.\n", i); + if (irq < 0) return -EINVAL; - } r = devm_request_irq(&pdev->dev, irq, sf_pdma_done_isr, 0, dev_name(&pdev->dev), (void *)chan); @@ -404,10 +420,8 @@ static int sf_pdma_irq_init(struct platform_device *pdev, struct sf_pdma *pdma) chan->txirq = irq; irq = platform_get_irq(pdev, (i * 2) + 1); - if (irq < 0) { - dev_err(&pdev->dev, "ch(%d) Can't get err irq.\n", i); + if (irq < 0) return -EINVAL; - } r = devm_request_irq(&pdev->dev, irq, sf_pdma_err_isr, 0, dev_name(&pdev->dev), (void *)chan); @@ -481,27 +495,41 @@ static void sf_pdma_setup_chans(struct sf_pdma *pdma) static int sf_pdma_probe(struct platform_device *pdev) { + const struct sf_pdma_driver_platdata *ddata; struct sf_pdma *pdma; - struct sf_pdma_chan *chan; - struct resource *res; - int len, chans; - int ret; + int ret, n_chans; const enum dma_slave_buswidth widths = DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | DMA_SLAVE_BUSWIDTH_4_BYTES | DMA_SLAVE_BUSWIDTH_8_BYTES | DMA_SLAVE_BUSWIDTH_16_BYTES | DMA_SLAVE_BUSWIDTH_32_BYTES | DMA_SLAVE_BUSWIDTH_64_BYTES; - chans = PDMA_NR_CH; - len = sizeof(*pdma) + sizeof(*chan) * chans; - pdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + ret = of_property_read_u32(pdev->dev.of_node, "dma-channels", &n_chans); + if (ret) { + /* backwards-compatibility for no dma-channels property */ + dev_dbg(&pdev->dev, "set number of channels to default value: 4\n"); + n_chans = PDMA_MAX_NR_CH; + } else if (n_chans > PDMA_MAX_NR_CH) { + dev_err(&pdev->dev, "the number of channels exceeds the maximum\n"); + return -EINVAL; + } + + pdma = devm_kzalloc(&pdev->dev, struct_size(pdma, chans, n_chans), + GFP_KERNEL); if (!pdma) return -ENOMEM; - pdma->n_chans = chans; + pdma->n_chans = n_chans; + + pdma->transfer_type = PDMA_FULL_SPEED | PDMA_STRICT_ORDERING; + + ddata = device_get_match_data(&pdev->dev); + if (ddata) { + if (ddata->quirks & PDMA_QUIRK_NO_STRICT_ORDERING) + pdma->transfer_type &= ~PDMA_STRICT_ORDERING; + } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - pdma->membase = devm_ioremap_resource(&pdev->dev, res); + pdma->membase = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pdma->membase)) return PTR_ERR(pdma->membase); @@ -547,16 +575,29 @@ static int sf_pdma_probe(struct platform_device *pdev) return ret; } + ret = of_dma_controller_register(pdev->dev.of_node, + of_dma_xlate_by_chan_id, pdma); + if (ret < 0) { + dev_err(&pdev->dev, + "Can't register SiFive Platform OF_DMA. (%d)\n", ret); + goto err_unregister; + } + return 0; + +err_unregister: + dma_async_device_unregister(&pdma->dma_dev); + + return ret; } -static int sf_pdma_remove(struct platform_device *pdev) +static void sf_pdma_remove(struct platform_device *pdev) { struct sf_pdma *pdma = platform_get_drvdata(pdev); struct sf_pdma_chan *ch; int i; - for (i = 0; i < PDMA_NR_CH; i++) { + for (i = 0; i < pdma->n_chans; i++) { ch = &pdma->chans[i]; devm_free_irq(&pdev->dev, ch->txirq, ch); @@ -567,13 +608,25 @@ static int sf_pdma_remove(struct platform_device *pdev) tasklet_kill(&ch->err_tasklet); } - dma_async_device_unregister(&pdma->dma_dev); + if (pdev->dev.of_node) + of_dma_controller_free(pdev->dev.of_node); - return 0; + dma_async_device_unregister(&pdma->dma_dev); } +static const struct sf_pdma_driver_platdata mpfs_pdma = { + .quirks = PDMA_QUIRK_NO_STRICT_ORDERING, +}; + static const struct of_device_id sf_pdma_dt_ids[] = { - { .compatible = "sifive,fu540-c000-pdma" }, + { + .compatible = "sifive,fu540-c000-pdma", + }, { + .compatible = "sifive,pdma0", + }, { + .compatible = "microchip,mpfs-pdma", + .data = &mpfs_pdma, + }, {}, }; MODULE_DEVICE_TABLE(of, sf_pdma_dt_ids); diff --git a/drivers/dma/sf-pdma/sf-pdma.h b/drivers/dma/sf-pdma/sf-pdma.h index 0c20167b097d..215e07183d7e 100644 --- a/drivers/dma/sf-pdma/sf-pdma.h +++ b/drivers/dma/sf-pdma/sf-pdma.h @@ -22,11 +22,7 @@ #include "../dmaengine.h" #include "../virt-dma.h" -#define PDMA_NR_CH 4 - -#if (PDMA_NR_CH != 4) -#error "Please define PDMA_NR_CH to 4" -#endif +#define PDMA_MAX_NR_CH 4 #define PDMA_BASE_ADDR 0x3000000 #define PDMA_CHAN_OFFSET 0x1000 @@ -52,7 +48,8 @@ #define PDMA_ERR_STATUS_MASK GENMASK(31, 31) /* Transfer Type */ -#define PDMA_FULL_SPEED 0xFF000008 +#define PDMA_FULL_SPEED 0xFF000000 +#define PDMA_STRICT_ORDERING BIT(3) /* Error Recovery */ #define MAX_RETRY 1 @@ -82,7 +79,6 @@ struct sf_pdma_desc { u64 src_addr; struct virt_dma_desc vdesc; struct sf_pdma_chan *chan; - bool in_use; enum dma_transfer_direction dirn; struct dma_async_tx_descriptor *async_tx; }; @@ -117,8 +113,13 @@ struct sf_pdma { struct dma_device dma_dev; void __iomem *membase; void __iomem *mappedbase; + u32 transfer_type; u32 n_chans; - struct sf_pdma_chan chans[PDMA_NR_CH]; + struct sf_pdma_chan chans[] __counted_by(n_chans); +}; + +struct sf_pdma_driver_platdata { + u32 quirks; }; #endif /* _SF_PDMA_H */ diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig index 13437323a85b..a16c7e83bd14 100644 --- a/drivers/dma/sh/Kconfig +++ b/drivers/dma/sh/Kconfig @@ -16,7 +16,7 @@ config SH_DMAE_BASE depends on SUPERH || COMPILE_TEST depends on !SUPERH || SH_DMA depends on !SH_DMA_API - default y + default SUPERH || SH_DMA select RENESAS_DMA help Enable support for the Renesas SuperH DMA controllers. @@ -47,3 +47,12 @@ config RENESAS_USB_DMAC help This driver supports the USB-DMA controller found in the Renesas SoCs. + +config RZ_DMAC + tristate "Renesas RZ DMA Controller" + depends on ARCH_RENESAS || COMPILE_TEST + select RENESAS_DMA + select DMA_VIRTUAL_CHANNELS + help + This driver supports the general purpose DMA controller typically + found in the Renesas RZ SoC variants. diff --git a/drivers/dma/sh/Makefile b/drivers/dma/sh/Makefile index abdf10341725..360ab6d25e76 100644 --- a/drivers/dma/sh/Makefile +++ b/drivers/dma/sh/Makefile @@ -15,3 +15,4 @@ obj-$(CONFIG_SH_DMAE) += shdma.o obj-$(CONFIG_RCAR_DMAC) += rcar-dmac.o obj-$(CONFIG_RENESAS_USB_DMAC) += usb-dmac.o +obj-$(CONFIG_RZ_DMAC) += rz-dmac.o diff --git a/drivers/dma/sh/rcar-dmac.c b/drivers/dma/sh/rcar-dmac.c index 6885b3dcd7a9..475a347cae1b 100644 --- a/drivers/dma/sh/rcar-dmac.c +++ b/drivers/dma/sh/rcar-dmac.c @@ -103,8 +103,8 @@ struct rcar_dmac_desc_page { struct list_head node; union { - struct rcar_dmac_desc descs[0]; - struct rcar_dmac_xfer_chunk chunks[0]; + DECLARE_FLEX_ARRAY(struct rcar_dmac_desc, descs); + DECLARE_FLEX_ARRAY(struct rcar_dmac_xfer_chunk, chunks); }; }; @@ -236,7 +236,7 @@ struct rcar_dmac_of_data { #define RCAR_DMAOR_PRI_ROUND_ROBIN (3 << 8) #define RCAR_DMAOR_AE (1 << 2) #define RCAR_DMAOR_DME (1 << 0) -#define RCAR_DMACHCLR 0x0080 /* Not on R-Car V3U */ +#define RCAR_DMACHCLR 0x0080 /* Not on R-Car Gen4 */ #define RCAR_DMADPSEC 0x00a0 #define RCAR_DMASAR 0x0000 @@ -299,8 +299,8 @@ struct rcar_dmac_of_data { #define RCAR_DMAFIXDAR 0x0014 #define RCAR_DMAFIXDPBASE 0x0060 -/* For R-Car V3U */ -#define RCAR_V3U_DMACHCLR 0x0100 +/* For R-Car Gen4 */ +#define RCAR_GEN4_DMACHCLR 0x0100 /* Hardcode the MEMCPY transfer size to 4 bytes. */ #define RCAR_DMAC_MEMCPY_XFER_SIZE 4 @@ -345,7 +345,7 @@ static void rcar_dmac_chan_clear(struct rcar_dmac *dmac, struct rcar_dmac_chan *chan) { if (dmac->chan_base) - rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + rcar_dmac_chan_write(chan, RCAR_GEN4_DMACHCLR, 1); else rcar_dmac_write(dmac, RCAR_DMACHCLR, BIT(chan->index)); } @@ -357,7 +357,7 @@ static void rcar_dmac_chan_clear_all(struct rcar_dmac *dmac) if (dmac->chan_base) { for_each_rcar_dmac_chan(i, dmac, chan) - rcar_dmac_chan_write(chan, RCAR_V3U_DMACHCLR, 1); + rcar_dmac_chan_write(chan, RCAR_GEN4_DMACHCLR, 1); } else { rcar_dmac_write(dmac, RCAR_DMACHCLR, dmac->channels_mask); } @@ -1728,19 +1728,12 @@ static struct dma_chan *rcar_dmac_of_xlate(struct of_phandle_args *dma_spec, * Power management */ -#ifdef CONFIG_PM -static int rcar_dmac_runtime_suspend(struct device *dev) -{ - return 0; -} - static int rcar_dmac_runtime_resume(struct device *dev) { struct rcar_dmac *dmac = dev_get_drvdata(dev); return rcar_dmac_init(dmac); } -#endif static const struct dev_pm_ops rcar_dmac_pm = { /* @@ -1748,10 +1741,9 @@ static const struct dev_pm_ops rcar_dmac_pm = { * - Wait for the current transfer to complete and stop the device, * - Resume transfers, if any. */ - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(rcar_dmac_runtime_suspend, rcar_dmac_runtime_resume, - NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + RUNTIME_PM_OPS(NULL, rcar_dmac_runtime_resume, NULL) }; /* ----------------------------------------------------------------------------- @@ -1869,7 +1861,10 @@ static int rcar_dmac_probe(struct platform_device *pdev) dmac->dev = &pdev->dev; platform_set_drvdata(pdev, dmac); dma_set_max_seg_size(dmac->dev, RCAR_DMATCR_MASK); - dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); + + ret = dma_set_mask_and_coherent(dmac->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; ret = rcar_dmac_parse_of(&pdev->dev, dmac); if (ret < 0) @@ -1916,7 +1911,7 @@ static int rcar_dmac_probe(struct platform_device *pdev) ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "runtime PM get sync failed (%d)\n", ret); - return ret; + goto err_pm_disable; } ret = rcar_dmac_init(dmac); @@ -1924,7 +1919,7 @@ static int rcar_dmac_probe(struct platform_device *pdev) if (ret) { dev_err(&pdev->dev, "failed to reset device\n"); - goto error; + goto err_pm_disable; } /* Initialize engine */ @@ -1958,14 +1953,14 @@ static int rcar_dmac_probe(struct platform_device *pdev) for_each_rcar_dmac_chan(i, dmac, chan) { ret = rcar_dmac_chan_probe(dmac, chan); if (ret < 0) - goto error; + goto err_pm_disable; } /* Register the DMAC as a DMA provider for DT. */ ret = of_dma_controller_register(pdev->dev.of_node, rcar_dmac_of_xlate, NULL); if (ret < 0) - goto error; + goto err_pm_disable; /* * Register the DMA engine device. @@ -1974,17 +1969,18 @@ static int rcar_dmac_probe(struct platform_device *pdev) */ ret = dma_async_device_register(engine); if (ret < 0) - goto error; + goto err_dma_free; return 0; -error: +err_dma_free: of_dma_controller_free(pdev->dev.of_node); +err_pm_disable: pm_runtime_disable(&pdev->dev); return ret; } -static int rcar_dmac_remove(struct platform_device *pdev) +static void rcar_dmac_remove(struct platform_device *pdev) { struct rcar_dmac *dmac = platform_get_drvdata(pdev); @@ -1992,8 +1988,6 @@ static int rcar_dmac_remove(struct platform_device *pdev) dma_async_device_unregister(&dmac->engine); pm_runtime_disable(&pdev->dev); - - return 0; } static void rcar_dmac_shutdown(struct platform_device *pdev) @@ -2008,7 +2002,7 @@ static const struct rcar_dmac_of_data rcar_dmac_data = { .chan_offset_stride = 0x80, }; -static const struct rcar_dmac_of_data rcar_v3u_dmac_data = { +static const struct rcar_dmac_of_data rcar_gen4_dmac_data = { .chan_offset_base = 0x0, .chan_offset_stride = 0x1000, }; @@ -2018,8 +2012,15 @@ static const struct of_device_id rcar_dmac_of_ids[] = { .compatible = "renesas,rcar-dmac", .data = &rcar_dmac_data, }, { + .compatible = "renesas,rcar-gen4-dmac", + .data = &rcar_gen4_dmac_data, + }, { + /* + * Backward compatibility for between v5.12 - v5.19 + * which didn't combined with "renesas,rcar-gen4-dmac" + */ .compatible = "renesas,dmac-r8a779a0", - .data = &rcar_v3u_dmac_data, + .data = &rcar_gen4_dmac_data, }, { /* Sentinel */ } }; @@ -2027,7 +2028,7 @@ MODULE_DEVICE_TABLE(of, rcar_dmac_of_ids); static struct platform_driver rcar_dmac_driver = { .driver = { - .pm = &rcar_dmac_pm, + .pm = pm_ptr(&rcar_dmac_pm), .name = "rcar-dmac", .of_match_table = rcar_dmac_of_ids, }, diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c new file mode 100644 index 000000000000..1f687b08d6b8 --- /dev/null +++ b/drivers/dma/sh/rz-dmac.c @@ -0,0 +1,1082 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas RZ/G2L DMA Controller Driver + * + * Based on imx-dma.c + * + * Copyright (C) 2021 Renesas Electronics Corp. + * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> + * Copyright 2012 Javier Martin, Vista Silicon <javier.martin@vista-silicon.com> + */ + +#include <linux/bitfield.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/irqchip/irq-renesas-rzv2h.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#include "../dmaengine.h" +#include "../virt-dma.h" + +enum rz_dmac_prep_type { + RZ_DMAC_DESC_MEMCPY, + RZ_DMAC_DESC_SLAVE_SG, +}; + +struct rz_lmdesc { + u32 header; + u32 sa; + u32 da; + u32 tb; + u32 chcfg; + u32 chitvl; + u32 chext; + u32 nxla; +}; + +struct rz_dmac_desc { + struct virt_dma_desc vd; + dma_addr_t src; + dma_addr_t dest; + size_t len; + struct list_head node; + enum dma_transfer_direction direction; + enum rz_dmac_prep_type type; + /* For slave sg */ + struct scatterlist *sg; + unsigned int sgcount; +}; + +#define to_rz_dmac_desc(d) container_of(d, struct rz_dmac_desc, vd) + +struct rz_dmac_chan { + struct virt_dma_chan vc; + void __iomem *ch_base; + void __iomem *ch_cmn_base; + unsigned int index; + int irq; + struct rz_dmac_desc *desc; + int descs_allocated; + + dma_addr_t src_per_address; + dma_addr_t dst_per_address; + + u32 chcfg; + u32 chctrl; + int mid_rid; + + struct list_head ld_free; + struct list_head ld_queue; + struct list_head ld_active; + + struct { + struct rz_lmdesc *base; + struct rz_lmdesc *head; + struct rz_lmdesc *tail; + dma_addr_t base_dma; + } lmdesc; +}; + +#define to_rz_dmac_chan(c) container_of(c, struct rz_dmac_chan, vc.chan) + +struct rz_dmac_icu { + struct platform_device *pdev; + u8 dmac_index; +}; + +struct rz_dmac { + struct dma_device engine; + struct rz_dmac_icu icu; + struct device *dev; + struct reset_control *rstc; + void __iomem *base; + void __iomem *ext_base; + + unsigned int n_channels; + struct rz_dmac_chan *channels; + + bool has_icu; + + DECLARE_BITMAP(modules, 1024); +}; + +#define to_rz_dmac(d) container_of(d, struct rz_dmac, engine) + +/* + * ----------------------------------------------------------------------------- + * Registers + */ + +#define CHSTAT 0x0024 +#define CHCTRL 0x0028 +#define CHCFG 0x002c +#define NXLA 0x0038 + +#define DCTRL 0x0000 + +#define EACH_CHANNEL_OFFSET 0x0040 +#define CHANNEL_0_7_OFFSET 0x0000 +#define CHANNEL_0_7_COMMON_BASE 0x0300 +#define CHANNEL_8_15_OFFSET 0x0400 +#define CHANNEL_8_15_COMMON_BASE 0x0700 + +#define CHSTAT_ER BIT(4) +#define CHSTAT_EN BIT(0) + +#define CHCTRL_CLRINTMSK BIT(17) +#define CHCTRL_CLRSUS BIT(9) +#define CHCTRL_CLRTC BIT(6) +#define CHCTRL_CLREND BIT(5) +#define CHCTRL_CLRRQ BIT(4) +#define CHCTRL_SWRST BIT(3) +#define CHCTRL_STG BIT(2) +#define CHCTRL_CLREN BIT(1) +#define CHCTRL_SETEN BIT(0) +#define CHCTRL_DEFAULT (CHCTRL_CLRINTMSK | CHCTRL_CLRSUS | \ + CHCTRL_CLRTC | CHCTRL_CLREND | \ + CHCTRL_CLRRQ | CHCTRL_SWRST | \ + CHCTRL_CLREN) + +#define CHCFG_DMS BIT(31) +#define CHCFG_DEM BIT(24) +#define CHCFG_DAD BIT(21) +#define CHCFG_SAD BIT(20) +#define CHCFG_REQD BIT(3) +#define CHCFG_SEL(bits) ((bits) & 0x07) +#define CHCFG_MEM_COPY (0x80400008) +#define CHCFG_FILL_DDS_MASK GENMASK(19, 16) +#define CHCFG_FILL_SDS_MASK GENMASK(15, 12) +#define CHCFG_FILL_TM(a) (((a) & BIT(5)) << 22) +#define CHCFG_FILL_AM(a) (((a) & GENMASK(4, 2)) << 6) +#define CHCFG_FILL_LVL(a) (((a) & BIT(1)) << 5) +#define CHCFG_FILL_HIEN(a) (((a) & BIT(0)) << 5) + +#define MID_RID_MASK GENMASK(9, 0) +#define CHCFG_MASK GENMASK(15, 10) +#define CHCFG_DS_INVALID 0xFF +#define DCTRL_LVINT BIT(1) +#define DCTRL_PR BIT(0) +#define DCTRL_DEFAULT (DCTRL_LVINT | DCTRL_PR) + +/* LINK MODE DESCRIPTOR */ +#define HEADER_LV BIT(0) + +#define RZ_DMAC_MAX_CHAN_DESCRIPTORS 16 +#define RZ_DMAC_MAX_CHANNELS 16 +#define DMAC_NR_LMDESC 64 + +/* RZ/V2H ICU related */ +#define RZV2H_MAX_DMAC_INDEX 4 + +/* + * ----------------------------------------------------------------------------- + * Device access + */ + +static void rz_dmac_writel(struct rz_dmac *dmac, unsigned int val, + unsigned int offset) +{ + writel(val, dmac->base + offset); +} + +static void rz_dmac_ext_writel(struct rz_dmac *dmac, unsigned int val, + unsigned int offset) +{ + writel(val, dmac->ext_base + offset); +} + +static u32 rz_dmac_ext_readl(struct rz_dmac *dmac, unsigned int offset) +{ + return readl(dmac->ext_base + offset); +} + +static void rz_dmac_ch_writel(struct rz_dmac_chan *channel, unsigned int val, + unsigned int offset, int which) +{ + if (which) + writel(val, channel->ch_base + offset); + else + writel(val, channel->ch_cmn_base + offset); +} + +static u32 rz_dmac_ch_readl(struct rz_dmac_chan *channel, + unsigned int offset, int which) +{ + if (which) + return readl(channel->ch_base + offset); + else + return readl(channel->ch_cmn_base + offset); +} + +/* + * ----------------------------------------------------------------------------- + * Initialization + */ + +static void rz_lmdesc_setup(struct rz_dmac_chan *channel, + struct rz_lmdesc *lmdesc) +{ + u32 nxla; + + channel->lmdesc.base = lmdesc; + channel->lmdesc.head = lmdesc; + channel->lmdesc.tail = lmdesc; + nxla = channel->lmdesc.base_dma; + while (lmdesc < (channel->lmdesc.base + (DMAC_NR_LMDESC - 1))) { + lmdesc->header = 0; + nxla += sizeof(*lmdesc); + lmdesc->nxla = nxla; + lmdesc++; + } + + lmdesc->header = 0; + lmdesc->nxla = channel->lmdesc.base_dma; +} + +/* + * ----------------------------------------------------------------------------- + * Descriptors preparation + */ + +static void rz_dmac_lmdesc_recycle(struct rz_dmac_chan *channel) +{ + struct rz_lmdesc *lmdesc = channel->lmdesc.head; + + while (!(lmdesc->header & HEADER_LV)) { + lmdesc->header = 0; + lmdesc++; + if (lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) + lmdesc = channel->lmdesc.base; + } + channel->lmdesc.head = lmdesc; +} + +static void rz_dmac_enable_hw(struct rz_dmac_chan *channel) +{ + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + unsigned long flags; + u32 nxla; + u32 chctrl; + u32 chstat; + + dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); + + local_irq_save(flags); + + rz_dmac_lmdesc_recycle(channel); + + nxla = channel->lmdesc.base_dma + + (sizeof(struct rz_lmdesc) * (channel->lmdesc.head - + channel->lmdesc.base)); + + chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); + if (!(chstat & CHSTAT_EN)) { + chctrl = (channel->chctrl | CHCTRL_SETEN); + rz_dmac_ch_writel(channel, nxla, NXLA, 1); + rz_dmac_ch_writel(channel, channel->chcfg, CHCFG, 1); + rz_dmac_ch_writel(channel, CHCTRL_SWRST, CHCTRL, 1); + rz_dmac_ch_writel(channel, chctrl, CHCTRL, 1); + } + + local_irq_restore(flags); +} + +static void rz_dmac_disable_hw(struct rz_dmac_chan *channel) +{ + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + unsigned long flags; + + dev_dbg(dmac->dev, "%s channel %d\n", __func__, channel->index); + + local_irq_save(flags); + rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); + local_irq_restore(flags); +} + +static void rz_dmac_set_dmars_register(struct rz_dmac *dmac, int nr, u32 dmars) +{ + u32 dmars_offset = (nr / 2) * 4; + u32 shift = (nr % 2) * 16; + u32 dmars32; + + dmars32 = rz_dmac_ext_readl(dmac, dmars_offset); + dmars32 &= ~(0xffff << shift); + dmars32 |= dmars << shift; + + rz_dmac_ext_writel(dmac, dmars32, dmars_offset); +} + +static void rz_dmac_prepare_desc_for_memcpy(struct rz_dmac_chan *channel) +{ + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + struct rz_lmdesc *lmdesc = channel->lmdesc.tail; + struct rz_dmac_desc *d = channel->desc; + u32 chcfg = CHCFG_MEM_COPY; + + /* prepare descriptor */ + lmdesc->sa = d->src; + lmdesc->da = d->dest; + lmdesc->tb = d->len; + lmdesc->chcfg = chcfg; + lmdesc->chitvl = 0; + lmdesc->chext = 0; + lmdesc->header = HEADER_LV; + + if (dmac->has_icu) { + rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + channel->index, + RZV2H_ICU_DMAC_REQ_NO_DEFAULT); + } else { + rz_dmac_set_dmars_register(dmac, channel->index, 0); + } + + channel->chcfg = chcfg; + channel->chctrl = CHCTRL_STG | CHCTRL_SETEN; +} + +static void rz_dmac_prepare_descs_for_slave_sg(struct rz_dmac_chan *channel) +{ + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + struct rz_dmac_desc *d = channel->desc; + struct scatterlist *sg, *sgl = d->sg; + struct rz_lmdesc *lmdesc; + unsigned int i, sg_len = d->sgcount; + + channel->chcfg |= CHCFG_SEL(channel->index) | CHCFG_DEM | CHCFG_DMS; + + if (d->direction == DMA_DEV_TO_MEM) { + channel->chcfg |= CHCFG_SAD; + channel->chcfg &= ~CHCFG_REQD; + } else { + channel->chcfg |= CHCFG_DAD | CHCFG_REQD; + } + + lmdesc = channel->lmdesc.tail; + + for (i = 0, sg = sgl; i < sg_len; i++, sg = sg_next(sg)) { + if (d->direction == DMA_DEV_TO_MEM) { + lmdesc->sa = channel->src_per_address; + lmdesc->da = sg_dma_address(sg); + } else { + lmdesc->sa = sg_dma_address(sg); + lmdesc->da = channel->dst_per_address; + } + + lmdesc->tb = sg_dma_len(sg); + lmdesc->chitvl = 0; + lmdesc->chext = 0; + if (i == (sg_len - 1)) { + lmdesc->chcfg = (channel->chcfg & ~CHCFG_DEM); + lmdesc->header = HEADER_LV; + } else { + lmdesc->chcfg = channel->chcfg; + lmdesc->header = HEADER_LV; + } + if (++lmdesc >= (channel->lmdesc.base + DMAC_NR_LMDESC)) + lmdesc = channel->lmdesc.base; + } + + channel->lmdesc.tail = lmdesc; + + if (dmac->has_icu) { + rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + channel->index, channel->mid_rid); + } else { + rz_dmac_set_dmars_register(dmac, channel->index, channel->mid_rid); + } + + channel->chctrl = CHCTRL_SETEN; +} + +static int rz_dmac_xfer_desc(struct rz_dmac_chan *chan) +{ + struct rz_dmac_desc *d = chan->desc; + struct virt_dma_desc *vd; + + vd = vchan_next_desc(&chan->vc); + if (!vd) + return 0; + + list_del(&vd->node); + + switch (d->type) { + case RZ_DMAC_DESC_MEMCPY: + rz_dmac_prepare_desc_for_memcpy(chan); + break; + + case RZ_DMAC_DESC_SLAVE_SG: + rz_dmac_prepare_descs_for_slave_sg(chan); + break; + + default: + return -EINVAL; + } + + rz_dmac_enable_hw(chan); + + return 0; +} + +/* + * ----------------------------------------------------------------------------- + * DMA engine operations + */ + +static int rz_dmac_alloc_chan_resources(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + + while (channel->descs_allocated < RZ_DMAC_MAX_CHAN_DESCRIPTORS) { + struct rz_dmac_desc *desc; + + desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) + break; + + list_add_tail(&desc->node, &channel->ld_free); + channel->descs_allocated++; + } + + if (!channel->descs_allocated) + return -ENOMEM; + + return channel->descs_allocated; +} + +static void rz_dmac_free_chan_resources(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + struct rz_lmdesc *lmdesc = channel->lmdesc.base; + struct rz_dmac_desc *desc, *_desc; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&channel->vc.lock, flags); + + for (i = 0; i < DMAC_NR_LMDESC; i++) + lmdesc[i].header = 0; + + rz_dmac_disable_hw(channel); + list_splice_tail_init(&channel->ld_active, &channel->ld_free); + list_splice_tail_init(&channel->ld_queue, &channel->ld_free); + + if (channel->mid_rid >= 0) { + clear_bit(channel->mid_rid, dmac->modules); + channel->mid_rid = -EINVAL; + } + + spin_unlock_irqrestore(&channel->vc.lock, flags); + + list_for_each_entry_safe(desc, _desc, &channel->ld_free, node) { + kfree(desc); + channel->descs_allocated--; + } + + INIT_LIST_HEAD(&channel->ld_free); + vchan_free_chan_resources(&channel->vc); +} + +static struct dma_async_tx_descriptor * +rz_dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + struct rz_dmac_desc *desc; + + dev_dbg(dmac->dev, "%s channel: %d src=0x%pad dst=0x%pad len=%zu\n", + __func__, channel->index, &src, &dest, len); + + if (list_empty(&channel->ld_free)) + return NULL; + + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + + desc->type = RZ_DMAC_DESC_MEMCPY; + desc->src = src; + desc->dest = dest; + desc->len = len; + desc->direction = DMA_MEM_TO_MEM; + + list_move_tail(channel->ld_free.next, &channel->ld_queue); + return vchan_tx_prep(&channel->vc, &desc->vd, flags); +} + +static struct dma_async_tx_descriptor * +rz_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac_desc *desc; + struct scatterlist *sg; + int dma_length = 0; + int i = 0; + + if (list_empty(&channel->ld_free)) + return NULL; + + desc = list_first_entry(&channel->ld_free, struct rz_dmac_desc, node); + + for_each_sg(sgl, sg, sg_len, i) { + dma_length += sg_dma_len(sg); + } + + desc->type = RZ_DMAC_DESC_SLAVE_SG; + desc->sg = sgl; + desc->sgcount = sg_len; + desc->len = dma_length; + desc->direction = direction; + + if (direction == DMA_DEV_TO_MEM) + desc->src = channel->src_per_address; + else + desc->dest = channel->dst_per_address; + + list_move_tail(channel->ld_free.next, &channel->ld_queue); + return vchan_tx_prep(&channel->vc, &desc->vd, flags); +} + +static int rz_dmac_terminate_all(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + rz_dmac_disable_hw(channel); + spin_lock_irqsave(&channel->vc.lock, flags); + list_splice_tail_init(&channel->ld_active, &channel->ld_free); + list_splice_tail_init(&channel->ld_queue, &channel->ld_free); + vchan_get_all_descriptors(&channel->vc, &head); + spin_unlock_irqrestore(&channel->vc.lock, flags); + vchan_dma_desc_free_list(&channel->vc, &head); + + return 0; +} + +static void rz_dmac_issue_pending(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + struct rz_dmac_desc *desc; + unsigned long flags; + + spin_lock_irqsave(&channel->vc.lock, flags); + + if (!list_empty(&channel->ld_queue)) { + desc = list_first_entry(&channel->ld_queue, + struct rz_dmac_desc, node); + channel->desc = desc; + if (vchan_issue_pending(&channel->vc)) { + if (rz_dmac_xfer_desc(channel) < 0) + dev_warn(dmac->dev, "ch: %d couldn't issue DMA xfer\n", + channel->index); + else + list_move_tail(channel->ld_queue.next, + &channel->ld_active); + } + } + + spin_unlock_irqrestore(&channel->vc.lock, flags); +} + +static u8 rz_dmac_ds_to_val_mapping(enum dma_slave_buswidth ds) +{ + u8 i; + static const enum dma_slave_buswidth ds_lut[] = { + DMA_SLAVE_BUSWIDTH_1_BYTE, + DMA_SLAVE_BUSWIDTH_2_BYTES, + DMA_SLAVE_BUSWIDTH_4_BYTES, + DMA_SLAVE_BUSWIDTH_8_BYTES, + DMA_SLAVE_BUSWIDTH_16_BYTES, + DMA_SLAVE_BUSWIDTH_32_BYTES, + DMA_SLAVE_BUSWIDTH_64_BYTES, + DMA_SLAVE_BUSWIDTH_128_BYTES, + }; + + for (i = 0; i < ARRAY_SIZE(ds_lut); i++) { + if (ds_lut[i] == ds) + return i; + } + + return CHCFG_DS_INVALID; +} + +static int rz_dmac_config(struct dma_chan *chan, + struct dma_slave_config *config) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + u32 val; + + channel->dst_per_address = config->dst_addr; + channel->chcfg &= ~CHCFG_FILL_DDS_MASK; + if (channel->dst_per_address) { + val = rz_dmac_ds_to_val_mapping(config->dst_addr_width); + if (val == CHCFG_DS_INVALID) + return -EINVAL; + + channel->chcfg |= FIELD_PREP(CHCFG_FILL_DDS_MASK, val); + } + + channel->src_per_address = config->src_addr; + channel->chcfg &= ~CHCFG_FILL_SDS_MASK; + if (channel->src_per_address) { + val = rz_dmac_ds_to_val_mapping(config->src_addr_width); + if (val == CHCFG_DS_INVALID) + return -EINVAL; + + channel->chcfg |= FIELD_PREP(CHCFG_FILL_SDS_MASK, val); + } + + return 0; +} + +static void rz_dmac_virt_desc_free(struct virt_dma_desc *vd) +{ + /* + * Place holder + * Descriptor allocation is done during alloc_chan_resources and + * get freed during free_chan_resources. + * list is used to manage the descriptors and avoid any memory + * allocation/free during DMA read/write. + */ +} + +static void rz_dmac_device_synchronize(struct dma_chan *chan) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + u32 chstat; + int ret; + + ret = read_poll_timeout(rz_dmac_ch_readl, chstat, !(chstat & CHSTAT_EN), + 100, 100000, false, channel, CHSTAT, 1); + if (ret < 0) + dev_warn(dmac->dev, "DMA Timeout"); + + if (dmac->has_icu) { + rzv2h_icu_register_dma_req(dmac->icu.pdev, dmac->icu.dmac_index, + channel->index, + RZV2H_ICU_DMAC_REQ_NO_DEFAULT); + } else { + rz_dmac_set_dmars_register(dmac, channel->index, 0); + } +} + +/* + * ----------------------------------------------------------------------------- + * IRQ handling + */ + +static void rz_dmac_irq_handle_channel(struct rz_dmac_chan *channel) +{ + struct dma_chan *chan = &channel->vc.chan; + struct rz_dmac *dmac = to_rz_dmac(chan->device); + u32 chstat, chctrl; + + chstat = rz_dmac_ch_readl(channel, CHSTAT, 1); + if (chstat & CHSTAT_ER) { + dev_err(dmac->dev, "DMAC err CHSTAT_%d = %08X\n", + channel->index, chstat); + rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); + goto done; + } + + chctrl = rz_dmac_ch_readl(channel, CHCTRL, 1); + rz_dmac_ch_writel(channel, chctrl | CHCTRL_CLREND, CHCTRL, 1); +done: + return; +} + +static irqreturn_t rz_dmac_irq_handler(int irq, void *dev_id) +{ + struct rz_dmac_chan *channel = dev_id; + + if (channel) { + rz_dmac_irq_handle_channel(channel); + return IRQ_WAKE_THREAD; + } + /* handle DMAERR irq */ + return IRQ_HANDLED; +} + +static irqreturn_t rz_dmac_irq_handler_thread(int irq, void *dev_id) +{ + struct rz_dmac_chan *channel = dev_id; + struct rz_dmac_desc *desc = NULL; + unsigned long flags; + + spin_lock_irqsave(&channel->vc.lock, flags); + + if (list_empty(&channel->ld_active)) { + /* Someone might have called terminate all */ + goto out; + } + + desc = list_first_entry(&channel->ld_active, struct rz_dmac_desc, node); + vchan_cookie_complete(&desc->vd); + list_move_tail(channel->ld_active.next, &channel->ld_free); + if (!list_empty(&channel->ld_queue)) { + desc = list_first_entry(&channel->ld_queue, struct rz_dmac_desc, + node); + channel->desc = desc; + if (rz_dmac_xfer_desc(channel) == 0) + list_move_tail(channel->ld_queue.next, &channel->ld_active); + } +out: + spin_unlock_irqrestore(&channel->vc.lock, flags); + + return IRQ_HANDLED; +} + +/* + * ----------------------------------------------------------------------------- + * OF xlate and channel filter + */ + +static bool rz_dmac_chan_filter(struct dma_chan *chan, void *arg) +{ + struct rz_dmac_chan *channel = to_rz_dmac_chan(chan); + struct rz_dmac *dmac = to_rz_dmac(chan->device); + struct of_phandle_args *dma_spec = arg; + u32 ch_cfg; + + channel->mid_rid = dma_spec->args[0] & MID_RID_MASK; + ch_cfg = (dma_spec->args[0] & CHCFG_MASK) >> 10; + channel->chcfg = CHCFG_FILL_TM(ch_cfg) | CHCFG_FILL_AM(ch_cfg) | + CHCFG_FILL_LVL(ch_cfg) | CHCFG_FILL_HIEN(ch_cfg); + + return !test_and_set_bit(channel->mid_rid, dmac->modules); +} + +static struct dma_chan *rz_dmac_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + dma_cap_mask_t mask; + + if (dma_spec->args_count != 1) + return NULL; + + /* Only slave DMA channels can be allocated via DT */ + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + return __dma_request_channel(&mask, rz_dmac_chan_filter, dma_spec, + ofdma->of_node); +} + +/* + * ----------------------------------------------------------------------------- + * Probe and remove + */ + +static int rz_dmac_chan_probe(struct rz_dmac *dmac, + struct rz_dmac_chan *channel, + u8 index) +{ + struct platform_device *pdev = to_platform_device(dmac->dev); + struct rz_lmdesc *lmdesc; + char pdev_irqname[6]; + char *irqname; + int ret; + + channel->index = index; + channel->mid_rid = -EINVAL; + + /* Request the channel interrupt. */ + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); + channel->irq = platform_get_irq_byname(pdev, pdev_irqname); + if (channel->irq < 0) + return channel->irq; + + irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", + dev_name(dmac->dev), index); + if (!irqname) + return -ENOMEM; + + ret = devm_request_threaded_irq(dmac->dev, channel->irq, + rz_dmac_irq_handler, + rz_dmac_irq_handler_thread, 0, + irqname, channel); + if (ret) { + dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", + channel->irq, ret); + return ret; + } + + /* Set io base address for each channel */ + if (index < 8) { + channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + + EACH_CHANNEL_OFFSET * index; + channel->ch_cmn_base = dmac->base + CHANNEL_0_7_COMMON_BASE; + } else { + channel->ch_base = dmac->base + CHANNEL_8_15_OFFSET + + EACH_CHANNEL_OFFSET * (index - 8); + channel->ch_cmn_base = dmac->base + CHANNEL_8_15_COMMON_BASE; + } + + /* Allocate descriptors */ + lmdesc = dma_alloc_coherent(&pdev->dev, + sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, + &channel->lmdesc.base_dma, GFP_KERNEL); + if (!lmdesc) { + dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); + return -ENOMEM; + } + rz_lmdesc_setup(channel, lmdesc); + + /* Initialize register for each channel */ + rz_dmac_ch_writel(channel, CHCTRL_DEFAULT, CHCTRL, 1); + + channel->vc.desc_free = rz_dmac_virt_desc_free; + vchan_init(&channel->vc, &dmac->engine); + INIT_LIST_HEAD(&channel->ld_queue); + INIT_LIST_HEAD(&channel->ld_free); + INIT_LIST_HEAD(&channel->ld_active); + + return 0; +} + +static int rz_dmac_parse_of_icu(struct device *dev, struct rz_dmac *dmac) +{ + struct device_node *np = dev->of_node; + struct of_phandle_args args; + uint32_t dmac_index; + int ret; + + ret = of_parse_phandle_with_fixed_args(np, "renesas,icu", 1, 0, &args); + if (ret == -ENOENT) + return 0; + if (ret) + return ret; + + dmac->has_icu = true; + + dmac->icu.pdev = of_find_device_by_node(args.np); + of_node_put(args.np); + if (!dmac->icu.pdev) { + dev_err(dev, "ICU device not found.\n"); + return -ENODEV; + } + + dmac_index = args.args[0]; + if (dmac_index > RZV2H_MAX_DMAC_INDEX) { + dev_err(dev, "DMAC index %u invalid.\n", dmac_index); + return -EINVAL; + } + dmac->icu.dmac_index = dmac_index; + + return 0; +} + +static int rz_dmac_parse_of(struct device *dev, struct rz_dmac *dmac) +{ + struct device_node *np = dev->of_node; + int ret; + + ret = of_property_read_u32(np, "dma-channels", &dmac->n_channels); + if (ret < 0) { + dev_err(dev, "unable to read dma-channels property\n"); + return ret; + } + + if (!dmac->n_channels || dmac->n_channels > RZ_DMAC_MAX_CHANNELS) { + dev_err(dev, "invalid number of channels %u\n", dmac->n_channels); + return -EINVAL; + } + + return rz_dmac_parse_of_icu(dev, dmac); +} + +static int rz_dmac_probe(struct platform_device *pdev) +{ + const char *irqname = "error"; + struct dma_device *engine; + struct rz_dmac *dmac; + int channel_num; + int ret; + int irq; + u8 i; + + dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); + if (!dmac) + return -ENOMEM; + + dmac->dev = &pdev->dev; + platform_set_drvdata(pdev, dmac); + + ret = rz_dmac_parse_of(&pdev->dev, dmac); + if (ret < 0) + return ret; + + dmac->channels = devm_kcalloc(&pdev->dev, dmac->n_channels, + sizeof(*dmac->channels), GFP_KERNEL); + if (!dmac->channels) + return -ENOMEM; + + /* Request resources */ + dmac->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dmac->base)) + return PTR_ERR(dmac->base); + + if (!dmac->has_icu) { + dmac->ext_base = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(dmac->ext_base)) + return PTR_ERR(dmac->ext_base); + } + + /* Register interrupt handler for error */ + irq = platform_get_irq_byname(pdev, irqname); + if (irq < 0) + return irq; + + ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, + irqname, NULL); + if (ret) { + dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", + irq, ret); + return ret; + } + + /* Initialize the channels. */ + INIT_LIST_HEAD(&dmac->engine.channels); + + dmac->rstc = devm_reset_control_array_get_optional_exclusive(&pdev->dev); + if (IS_ERR(dmac->rstc)) + return dev_err_probe(&pdev->dev, PTR_ERR(dmac->rstc), + "failed to get resets\n"); + + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); + goto err_pm_disable; + } + + ret = reset_control_deassert(dmac->rstc); + if (ret) + goto err_pm_runtime_put; + + for (i = 0; i < dmac->n_channels; i++) { + ret = rz_dmac_chan_probe(dmac, &dmac->channels[i], i); + if (ret < 0) + goto err; + } + + /* Register the DMAC as a DMA provider for DT. */ + ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, + NULL); + if (ret < 0) + goto err; + + /* Register the DMA engine device. */ + engine = &dmac->engine; + dma_cap_set(DMA_SLAVE, engine->cap_mask); + dma_cap_set(DMA_MEMCPY, engine->cap_mask); + rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_0_7_COMMON_BASE + DCTRL); + rz_dmac_writel(dmac, DCTRL_DEFAULT, CHANNEL_8_15_COMMON_BASE + DCTRL); + + engine->dev = &pdev->dev; + + engine->device_alloc_chan_resources = rz_dmac_alloc_chan_resources; + engine->device_free_chan_resources = rz_dmac_free_chan_resources; + engine->device_tx_status = dma_cookie_status; + engine->device_prep_slave_sg = rz_dmac_prep_slave_sg; + engine->device_prep_dma_memcpy = rz_dmac_prep_dma_memcpy; + engine->device_config = rz_dmac_config; + engine->device_terminate_all = rz_dmac_terminate_all; + engine->device_issue_pending = rz_dmac_issue_pending; + engine->device_synchronize = rz_dmac_device_synchronize; + + engine->copy_align = DMAENGINE_ALIGN_1_BYTE; + dma_set_max_seg_size(engine->dev, U32_MAX); + + ret = dma_async_device_register(engine); + if (ret < 0) { + dev_err(&pdev->dev, "unable to register\n"); + goto dma_register_err; + } + return 0; + +dma_register_err: + of_dma_controller_free(pdev->dev.of_node); +err: + channel_num = i ? i - 1 : 0; + for (i = 0; i < channel_num; i++) { + struct rz_dmac_chan *channel = &dmac->channels[i]; + + dma_free_coherent(&pdev->dev, + sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, + channel->lmdesc.base, + channel->lmdesc.base_dma); + } + + reset_control_assert(dmac->rstc); +err_pm_runtime_put: + pm_runtime_put(&pdev->dev); +err_pm_disable: + pm_runtime_disable(&pdev->dev); + + return ret; +} + +static void rz_dmac_remove(struct platform_device *pdev) +{ + struct rz_dmac *dmac = platform_get_drvdata(pdev); + unsigned int i; + + dma_async_device_unregister(&dmac->engine); + of_dma_controller_free(pdev->dev.of_node); + for (i = 0; i < dmac->n_channels; i++) { + struct rz_dmac_chan *channel = &dmac->channels[i]; + + dma_free_coherent(&pdev->dev, + sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, + channel->lmdesc.base, + channel->lmdesc.base_dma); + } + reset_control_assert(dmac->rstc); + pm_runtime_put(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + platform_device_put(dmac->icu.pdev); +} + +static const struct of_device_id of_rz_dmac_match[] = { + { .compatible = "renesas,r9a09g057-dmac", }, + { .compatible = "renesas,rz-dmac", }, + { /* Sentinel */ } +}; +MODULE_DEVICE_TABLE(of, of_rz_dmac_match); + +static struct platform_driver rz_dmac_driver = { + .driver = { + .name = "rz-dmac", + .of_match_table = of_rz_dmac_match, + }, + .probe = rz_dmac_probe, + .remove = rz_dmac_remove, +}; + +module_platform_driver(rz_dmac_driver); + +MODULE_DESCRIPTION("Renesas RZ/G2L DMA Controller Driver"); +MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sh/shdma-arm.h b/drivers/dma/sh/shdma-arm.h deleted file mode 100644 index 7459f9a13b5b..000000000000 --- a/drivers/dma/sh/shdma-arm.h +++ /dev/null @@ -1,48 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Renesas SuperH DMA Engine support - * - * Copyright (C) 2013 Renesas Electronics, Inc. - */ - -#ifndef SHDMA_ARM_H -#define SHDMA_ARM_H - -#include "shdma.h" - -/* Transmit sizes and respective CHCR register values */ -enum { - XMIT_SZ_8BIT = 0, - XMIT_SZ_16BIT = 1, - XMIT_SZ_32BIT = 2, - XMIT_SZ_64BIT = 7, - XMIT_SZ_128BIT = 3, - XMIT_SZ_256BIT = 4, - XMIT_SZ_512BIT = 5, -}; - -/* log2(size / 8) - used to calculate number of transfers */ -#define SH_DMAE_TS_SHIFT { \ - [XMIT_SZ_8BIT] = 0, \ - [XMIT_SZ_16BIT] = 1, \ - [XMIT_SZ_32BIT] = 2, \ - [XMIT_SZ_64BIT] = 3, \ - [XMIT_SZ_128BIT] = 4, \ - [XMIT_SZ_256BIT] = 5, \ - [XMIT_SZ_512BIT] = 6, \ -} - -#define TS_LOW_BIT 0x3 /* --xx */ -#define TS_HI_BIT 0xc /* xx-- */ - -#define TS_LOW_SHIFT (3) -#define TS_HI_SHIFT (20 - 2) /* 2 bits for shifted low TS */ - -#define TS_INDEX2VAL(i) \ - ((((i) & TS_LOW_BIT) << TS_LOW_SHIFT) |\ - (((i) & TS_HI_BIT) << TS_HI_SHIFT)) - -#define CHCR_TX(xmit_sz) (DM_FIX | SM_INC | RS_ERS | TS_INDEX2VAL((xmit_sz))) -#define CHCR_RX(xmit_sz) (DM_INC | SM_FIX | RS_ERS | TS_INDEX2VAL((xmit_sz))) - -#endif diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index 7f72b3f4cd1a..834741adadaa 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -129,12 +129,25 @@ static dma_cookie_t shdma_tx_submit(struct dma_async_tx_descriptor *tx) const struct shdma_ops *ops = sdev->ops; dev_dbg(schan->dev, "Bring up channel %d\n", schan->id); - /* - * TODO: .xfer_setup() might fail on some platforms. - * Make it int then, on error remove chunks from the - * queue again - */ - ops->setup_xfer(schan, schan->slave_id); + + ret = ops->setup_xfer(schan, schan->slave_id); + if (ret < 0) { + dev_err(schan->dev, "setup_xfer failed: %d\n", ret); + + /* Remove chunks from the queue and mark them as idle */ + list_for_each_entry_safe(chunk, c, &schan->ld_queue, node) { + if (chunk->cookie == cookie) { + chunk->mark = DESC_IDLE; + list_move(&chunk->node, &schan->ld_free); + } + } + + schan->pm_state = SHDMA_PM_ESTABLISHED; + ret = pm_runtime_put(schan->dev); + + spin_unlock_irq(&schan->chan_lock); + return ret; + } if (schan->pm_state == SHDMA_PM_PENDING) shdma_chan_xfer_ld_queue(schan); @@ -725,7 +738,7 @@ static struct dma_async_tx_descriptor *shdma_prep_dma_cyclic( slave_addr = ops->slave_addr(schan); /* - * Allocate the sg list dynamically as it would consumer too much stack + * Allocate the sg list dynamically as it would consume too much stack * space. */ sgl = kmalloc_array(sg_len, sizeof(*sgl), GFP_KERNEL); @@ -787,14 +800,6 @@ static int shdma_config(struct dma_chan *chan, return -EINVAL; /* - * overriding the slave_id through dma_slave_config is deprecated, - * but possibly some out-of-tree drivers still do it. - */ - if (WARN_ON_ONCE(config->slave_id && - config->slave_id != schan->real_slave_id)) - schan->real_slave_id = config->slave_id; - - /* * We could lock this, but you shouldn't be configuring the * channel, while using it... */ @@ -969,7 +974,7 @@ void shdma_chan_probe(struct shdma_dev *sdev, spin_lock_init(&schan->chan_lock); - /* Init descripter manage list */ + /* Init descriptor manage list */ INIT_LIST_HEAD(&schan->ld_queue); INIT_LIST_HEAD(&schan->ld_free); @@ -1042,9 +1047,7 @@ EXPORT_SYMBOL(shdma_cleanup); static int __init shdma_enter(void) { - shdma_slave_used = kcalloc(DIV_ROUND_UP(slave_num, BITS_PER_LONG), - sizeof(long), - GFP_KERNEL); + shdma_slave_used = bitmap_zalloc(slave_num, GFP_KERNEL); if (!shdma_slave_used) return -ENOMEM; return 0; @@ -1053,10 +1056,9 @@ module_init(shdma_enter); static void __exit shdma_exit(void) { - kfree(shdma_slave_used); + bitmap_free(shdma_slave_used); } module_exit(shdma_exit); -MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("SH-DMA driver base library"); MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>"); diff --git a/drivers/dma/sh/shdma.h b/drivers/dma/sh/shdma.h index 9c121a4b33ad..f97d80343aea 100644 --- a/drivers/dma/sh/shdma.h +++ b/drivers/dma/sh/shdma.h @@ -25,7 +25,7 @@ struct sh_dmae_chan { const struct sh_dmae_slave_config *config; /* Slave DMA configuration */ int xmit_shift; /* log_2(bytes_per_xfer) */ void __iomem *base; - char dev_id[16]; /* unique name per DMAC of channel */ + char dev_id[32]; /* unique name per DMAC of channel */ int pm_error; dma_addr_t slave_addr; }; diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 5aafe548ca5f..603e15102e45 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -23,7 +23,6 @@ #include <linux/module.h> #include <linux/notifier.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/rculist.h> @@ -301,25 +300,34 @@ static bool sh_dmae_channel_busy(struct shdma_chan *schan) return dmae_is_busy(sh_chan); } -static void sh_dmae_setup_xfer(struct shdma_chan *schan, - int slave_id) +static int sh_dmae_setup_xfer(struct shdma_chan *schan, int slave_id) { struct sh_dmae_chan *sh_chan = container_of(schan, struct sh_dmae_chan, shdma_chan); + int ret = 0; if (slave_id >= 0) { const struct sh_dmae_slave_config *cfg = sh_chan->config; - dmae_set_dmars(sh_chan, cfg->mid_rid); - dmae_set_chcr(sh_chan, cfg->chcr); + ret = dmae_set_dmars(sh_chan, cfg->mid_rid); + if (ret < 0) + goto END; + + ret = dmae_set_chcr(sh_chan, cfg->chcr); + if (ret < 0) + goto END; + } else { dmae_init(sh_chan); } + +END: + return ret; } /* - * Find a slave channel configuration from the contoller list by either a slave + * Find a slave channel configuration from the controller list by either a slave * ID in the non-DT case, or by a MID/RID value in the DT case */ static const struct sh_dmae_slave_config *dmae_find_slave( @@ -678,7 +686,7 @@ static int sh_dmae_probe(struct platform_device *pdev) int err, errirq, i, irq_cnt = 0, irqres = 0, irq_cap = 0; struct sh_dmae_device *shdev; struct dma_device *dma_dev; - struct resource *chan, *dmars, *errirq_res, *chanirq_res; + struct resource *dmars, *errirq_res, *chanirq_res; if (pdev->dev.of_node) pdata = of_device_get_match_data(&pdev->dev); @@ -689,7 +697,6 @@ static int sh_dmae_probe(struct platform_device *pdev) if (!pdata || !pdata->channel_num) return -ENODEV; - chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); /* DMARS area is optional */ dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); /* @@ -709,7 +716,7 @@ static int sh_dmae_probe(struct platform_device *pdev) * requested with the IRQF_SHARED flag */ errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!chan || !errirq_res) + if (!errirq_res) return -ENODEV; shdev = devm_kzalloc(&pdev->dev, sizeof(struct sh_dmae_device), @@ -719,7 +726,7 @@ static int sh_dmae_probe(struct platform_device *pdev) dma_dev = &shdev->shdma_dev.dma_dev; - shdev->chan_reg = devm_ioremap_resource(&pdev->dev, chan); + shdev->chan_reg = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(shdev->chan_reg)) return PTR_ERR(shdev->chan_reg); if (dmars) { @@ -884,7 +891,7 @@ eshdma: return err; } -static int sh_dmae_remove(struct platform_device *pdev) +static void sh_dmae_remove(struct platform_device *pdev) { struct sh_dmae_device *shdev = platform_get_drvdata(pdev); struct dma_device *dma_dev = &shdev->shdma_dev.dma_dev; @@ -901,8 +908,6 @@ static int sh_dmae_remove(struct platform_device *pdev) shdma_cleanup(&shdev->shdma_dev); synchronize_rcu(); - - return 0; } static struct platform_driver sh_dmae_driver = { diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c index 8f7ceb698226..b42e5a66fd95 100644 --- a/drivers/dma/sh/usb-dmac.c +++ b/drivers/dma/sh/usb-dmac.c @@ -57,7 +57,7 @@ struct usb_dmac_desc { u32 residue; struct list_head node; dma_cookie_t done_cookie; - struct usb_dmac_sg sg[]; + struct usb_dmac_sg sg[] __counted_by(sg_allocated_len); }; #define to_usb_dmac_desc(vd) container_of(vd, struct usb_dmac_desc, vd) @@ -301,7 +301,7 @@ static struct usb_dmac_desc *usb_dmac_desc_get(struct usb_dmac_chan *chan, struct usb_dmac_desc *desc = NULL; unsigned long flags; - /* Get a freed descritpor */ + /* Get a freed descriptor */ spin_lock_irqsave(&chan->vc.lock, flags); list_for_each_entry(desc, &chan->desc_freed, node) { if (sg_len <= desc->sg_allocated_len) { @@ -466,7 +466,7 @@ static int usb_dmac_chan_terminate_all(struct dma_chan *chan) static unsigned int usb_dmac_get_current_residue(struct usb_dmac_chan *chan, struct usb_dmac_desc *desc, - int sg_index) + unsigned int sg_index) { struct usb_dmac_sg *sg = desc->sg + sg_index; u32 mem_addr = sg->mem_addr & 0xffffffff; @@ -670,7 +670,6 @@ static struct dma_chan *usb_dmac_of_xlate(struct of_phandle_args *dma_spec, * Power management */ -#ifdef CONFIG_PM static int usb_dmac_runtime_suspend(struct device *dev) { struct usb_dmac *dmac = dev_get_drvdata(dev); @@ -691,13 +690,11 @@ static int usb_dmac_runtime_resume(struct device *dev) return usb_dmac_init(dmac); } -#endif /* CONFIG_PM */ static const struct dev_pm_ops usb_dmac_pm = { - SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(usb_dmac_runtime_suspend, usb_dmac_runtime_resume, - NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + RUNTIME_PM_OPS(usb_dmac_runtime_suspend, usb_dmac_runtime_resume, NULL) }; /* ----------------------------------------------------------------------------- @@ -706,10 +703,10 @@ static const struct dev_pm_ops usb_dmac_pm = { static int usb_dmac_chan_probe(struct usb_dmac *dmac, struct usb_dmac_chan *uchan, - unsigned int index) + u8 index) { struct platform_device *pdev = to_platform_device(dmac->dev); - char pdev_irqname[5]; + char pdev_irqname[6]; char *irqname; int ret; @@ -717,7 +714,7 @@ static int usb_dmac_chan_probe(struct usb_dmac *dmac, uchan->iomem = dmac->iomem + USB_DMAC_CHAN_OFFSET(index); /* Request the channel interrupt. */ - sprintf(pdev_irqname, "ch%u", index); + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); uchan->irq = platform_get_irq_byname(pdev, pdev_irqname); if (uchan->irq < 0) return -ENODEV; @@ -768,9 +765,8 @@ static int usb_dmac_probe(struct platform_device *pdev) const enum dma_slave_buswidth widths = USB_DMAC_SLAVE_BUSWIDTH; struct dma_device *engine; struct usb_dmac *dmac; - struct resource *mem; - unsigned int i; int ret; + u8 i; dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); if (!dmac) @@ -789,8 +785,7 @@ static int usb_dmac_probe(struct platform_device *pdev) return -ENOMEM; /* Request resources. */ - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dmac->iomem = devm_ioremap_resource(&pdev->dev, mem); + dmac->iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dmac->iomem)) return PTR_ERR(dmac->iomem); @@ -855,8 +850,8 @@ static int usb_dmac_probe(struct platform_device *pdev) error: of_dma_controller_free(pdev->dev.of_node); - pm_runtime_put(&pdev->dev); error_pm: + pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); return ret; } @@ -868,10 +863,10 @@ static void usb_dmac_chan_remove(struct usb_dmac *dmac, devm_free_irq(dmac->dev, uchan->irq, uchan); } -static int usb_dmac_remove(struct platform_device *pdev) +static void usb_dmac_remove(struct platform_device *pdev) { struct usb_dmac *dmac = platform_get_drvdata(pdev); - int i; + u8 i; for (i = 0; i < dmac->n_channels; ++i) usb_dmac_chan_remove(dmac, &dmac->channels[i]); @@ -879,8 +874,6 @@ static int usb_dmac_remove(struct platform_device *pdev) dma_async_device_unregister(&dmac->engine); pm_runtime_disable(&pdev->dev); - - return 0; } static void usb_dmac_shutdown(struct platform_device *pdev) @@ -898,7 +891,7 @@ MODULE_DEVICE_TABLE(of, usb_dmac_of_ids); static struct platform_driver usb_dmac_driver = { .driver = { - .pm = &usb_dmac_pm, + .pm = pm_ptr(&usb_dmac_pm), .name = "usb-dmac", .of_match_table = usb_dmac_of_ids, }, diff --git a/drivers/dma/sprd-dma.c b/drivers/dma/sprd-dma.c index 0ef5ca81ba4d..6207e0b185e1 100644 --- a/drivers/dma/sprd-dma.c +++ b/drivers/dma/sprd-dma.c @@ -15,7 +15,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> @@ -212,7 +212,7 @@ struct sprd_dma_dev { struct clk *ashb_clk; int irq; u32 total_chns; - struct sprd_dma_chn channels[]; + struct sprd_dma_chn channels[] __counted_by(total_chns); }; static void sprd_dma_free_desc(struct virt_dma_desc *vd); @@ -572,8 +572,7 @@ static void sprd_dma_stop(struct sprd_dma_chn *schan) schan->cur_desc = NULL; } -static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc, - enum sprd_dma_int_type int_type, +static bool sprd_dma_check_trans_done(enum sprd_dma_int_type int_type, enum sprd_dma_req_mode req_mode) { if (int_type == SPRD_DMA_NO_INT) @@ -619,8 +618,7 @@ static irqreturn_t dma_irq_handle(int irq, void *dev_id) vchan_cyclic_callback(&sdesc->vd); } else { /* Check if the dma request descriptor is done. */ - trans_done = sprd_dma_check_trans_done(sdesc, int_type, - req_type); + trans_done = sprd_dma_check_trans_done(int_type, req_type); if (trans_done == true) { vchan_cookie_complete(&sdesc->vd); schan->cur_desc = NULL; @@ -795,9 +793,6 @@ static int sprd_dma_fill_desc(struct dma_chan *chan, return dst_datawidth; } - if (slave_cfg->slave_id) - schan->dev_id = slave_cfg->slave_id; - hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET; /* @@ -1120,7 +1115,20 @@ static int sprd_dma_probe(struct platform_device *pdev) u32 chn_count; int ret, i; - ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); + if (ret) { + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(&pdev->dev, "unable to set coherent mask to 32\n"); + return ret; + } + } + + /* Parse new and deprecated dma-channels properties */ + ret = device_property_read_u32(&pdev->dev, "dma-channels", &chn_count); + if (ret) + ret = device_property_read_u32(&pdev->dev, "#dma-channels", + &chn_count); if (ret) { dev_err(&pdev->dev, "get dma channels count failed\n"); return ret; @@ -1168,7 +1176,6 @@ static int sprd_dma_probe(struct platform_device *pdev) dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask); sdev->total_chns = chn_count; - sdev->dma_dev.chancnt = chn_count; INIT_LIST_HEAD(&sdev->dma_dev.channels); INIT_LIST_HEAD(&sdev->dma_dev.global_node); sdev->dma_dev.dev = &pdev->dev; @@ -1232,15 +1239,12 @@ err_rpm: return ret; } -static int sprd_dma_remove(struct platform_device *pdev) +static void sprd_dma_remove(struct platform_device *pdev) { struct sprd_dma_dev *sdev = platform_get_drvdata(pdev); struct sprd_dma_chn *c, *cn; - int ret; - ret = pm_runtime_get_sync(&pdev->dev); - if (ret < 0) - return ret; + pm_runtime_get_sync(&pdev->dev); /* explicitly free the irq */ if (sdev->irq > 0) @@ -1258,13 +1262,13 @@ static int sprd_dma_remove(struct platform_device *pdev) pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); - return 0; } static const struct of_device_id sprd_dma_match[] = { { .compatible = "sprd,sc9860-dma", }, {}, }; +MODULE_DEVICE_TABLE(of, sprd_dma_match); static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev) { @@ -1307,4 +1311,3 @@ MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("DMA driver for Spreadtrum"); MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>"); MODULE_AUTHOR("Eric Long <eric.long@spreadtrum.com>"); -MODULE_ALIAS("platform:sprd-dma"); diff --git a/drivers/dma/st_fdma.c b/drivers/dma/st_fdma.c index 962b6e05287b..dc2ab7d16cf2 100644 --- a/drivers/dma/st_fdma.c +++ b/drivers/dma/st_fdma.c @@ -10,9 +10,10 @@ #include <linux/init.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <linux/interrupt.h> #include <linux/remoteproc.h> #include <linux/slab.h> @@ -739,18 +740,11 @@ static void st_fdma_free(struct st_fdma_dev *fdev) static int st_fdma_probe(struct platform_device *pdev) { struct st_fdma_dev *fdev; - const struct of_device_id *match; struct device_node *np = pdev->dev.of_node; const struct st_fdma_driverdata *drvdata; int ret, i; - match = of_match_device((st_fdma_match), &pdev->dev); - if (!match || !match->data) { - dev_err(&pdev->dev, "No device match found\n"); - return -ENODEV; - } - - drvdata = match->data; + drvdata = device_get_match_data(&pdev->dev); fdev = devm_kzalloc(&pdev->dev, sizeof(*fdev), GFP_KERNEL); if (!fdev) @@ -849,15 +843,13 @@ err: return ret; } -static int st_fdma_remove(struct platform_device *pdev) +static void st_fdma_remove(struct platform_device *pdev) { struct st_fdma_dev *fdev = platform_get_drvdata(pdev); devm_free_irq(&pdev->dev, fdev->irq, fdev); st_slim_rproc_put(fdev->slim_rproc); of_dma_controller_free(pdev->dev.of_node); - - return 0; } static struct platform_driver st_fdma_platform_driver = { @@ -874,4 +866,3 @@ MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("STMicroelectronics FDMA engine driver"); MODULE_AUTHOR("Ludovic.barre <Ludovic.barre@st.com>"); MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>"); -MODULE_ALIAS("platform: " DRIVER_NAME); diff --git a/drivers/dma/st_fdma.h b/drivers/dma/st_fdma.h index fa15b97a3bab..f296412e96b6 100644 --- a/drivers/dma/st_fdma.h +++ b/drivers/dma/st_fdma.h @@ -97,7 +97,7 @@ struct st_fdma_desc { struct st_fdma_chan *fchan; bool iscyclic; unsigned int n_nodes; - struct st_fdma_sw_node node[]; + struct st_fdma_sw_node node[] __counted_by(n_nodes); }; enum st_fdma_type { diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index e1827393143f..d52e1685aed5 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -19,14 +19,41 @@ #include <linux/pm_runtime.h> #include <linux/err.h> #include <linux/of.h> +#include <linux/of_address.h> #include <linux/of_dma.h> #include <linux/amba/bus.h> #include <linux/regulator/consumer.h> -#include <linux/platform_data/dma-ste-dma40.h> #include "dmaengine.h" +#include "ste_dma40.h" #include "ste_dma40_ll.h" +/** + * struct stedma40_platform_data - Configuration struct for the dma device. + * + * @disabled_channels: A vector, ending with -1, that marks physical channels + * that are for different reasons not available for the driver. + * @soft_lli_chans: A vector, that marks physical channels will use LLI by SW + * which avoids HW bug that exists in some versions of the controller. + * SoftLLI introduces relink overhead that could impact performance for + * certain use cases. + * @num_of_soft_lli_chans: The number of channels that needs to be configured + * to use SoftLLI. + * @use_esram_lcla: flag for mapping the lcla into esram region + * @num_of_memcpy_chans: The number of channels reserved for memcpy. + * @num_of_phy_chans: The number of physical channels implemented in HW. + * 0 means reading the number of channels from DMA HW but this is only valid + * for 'multiple of 4' channels, like 8. + */ +struct stedma40_platform_data { + int disabled_channels[STEDMA40_MAX_PHYS]; + int *soft_lli_chans; + int num_of_soft_lli_chans; + bool use_esram_lcla; + int num_of_memcpy_chans; + int num_of_phy_chans; +}; + #define D40_NAME "dma40" #define D40_PHY_CHAN -1 @@ -107,7 +134,7 @@ static const struct stedma40_chan_cfg dma40_memcpy_conf_log = { }; /** - * enum 40_command - The different commands and/or statuses. + * enum d40_command - The different commands and/or statuses. * * @D40_DMA_STOP: DMA channel command STOP or status STOPPED, * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN. @@ -155,7 +182,7 @@ static __maybe_unused u32 d40_backup_regs[] = { /* * since 9540 and 8540 has the same HW revision - * use v4a for 9540 or ealier + * use v4a for 9540 or earlier * use v4b for 8540 or later * HW revision: * DB8500ed has revision 0 @@ -382,7 +409,7 @@ struct d40_desc { * * @base: The virtual address of LCLA. 18 bit aligned. * @dma_addr: DMA address, if mapped - * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used. + * @base_unaligned: The original kmalloc pointer, if kmalloc is used. * This pointer is only there for clean-up on error. * @pages: The number of pages needed for all physical channels. * Only used later for clean-up on error @@ -525,8 +552,6 @@ struct d40_gen_dmac { * @virtbase: The virtual base address of the DMA's register. * @rev: silicon revision detected. * @clk: Pointer to the DMA clock structure. - * @phy_start: Physical memory start of the DMA registers. - * @phy_size: Size of the DMA register map. * @irq: The IRQ number. * @num_memcpy_chans: The number of channels used for memcpy (mem-to-mem * transfers). @@ -570,8 +595,6 @@ struct d40_base { void __iomem *virtbase; u8 rev:4; struct clk *clk; - phys_addr_t phy_start; - resource_size_t phy_size; int irq; int num_memcpy_chans; int num_phy_chans; @@ -1630,7 +1653,7 @@ static void dma_tasklet(struct tasklet_struct *t) return; check_pending_tx: - /* Rescue manouver if receiving double interrupts */ + /* Rescue maneuver if receiving double interrupts */ if (d40c->pending_tx > 0) d40c->pending_tx--; spin_unlock_irqrestore(&d40c->lock, flags); @@ -1970,7 +1993,7 @@ static int d40_config_memcpy(struct d40_chan *d40c) dma_has_cap(DMA_SLAVE, cap)) { d40c->dma_cfg = dma40_memcpy_conf_phy; - /* Generate interrrupt at end of transfer or relink. */ + /* Generate interrupt at end of transfer or relink. */ d40c->dst_def_cfg |= BIT(D40_SREG_CFG_TIM_POS); /* Generate interrupt on error. */ @@ -2268,7 +2291,7 @@ d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src, return NULL; } -bool stedma40_filter(struct dma_chan *chan, void *data) +static bool stedma40_filter(struct dma_chan *chan, void *data) { struct stedma40_chan_cfg *info = data; struct d40_chan *d40c = @@ -2287,7 +2310,6 @@ bool stedma40_filter(struct dma_chan *chan, void *data) return err == 0; } -EXPORT_SYMBOL(stedma40_filter); static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src) { @@ -3100,64 +3122,57 @@ static int __init d40_phy_res_init(struct d40_base *base) return num_phy_chans_avail; } -static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) +/* Called from the registered devm action */ +static void d40_drop_kmem_cache_action(void *d) +{ + struct kmem_cache *desc_slab = d; + + kmem_cache_destroy(desc_slab); +} + +static int __init d40_hw_detect_init(struct platform_device *pdev, + struct d40_base **retbase) { struct stedma40_platform_data *plat_data = dev_get_platdata(&pdev->dev); + struct device *dev = &pdev->dev; struct clk *clk; void __iomem *virtbase; - struct resource *res; struct d40_base *base; int num_log_chans; int num_phy_chans; int num_memcpy_chans; - int clk_ret = -EINVAL; int i; u32 pid; u32 cid; u8 rev; + int ret; - clk = clk_get(&pdev->dev, NULL); - if (IS_ERR(clk)) { - d40_err(&pdev->dev, "No matching clock found\n"); - goto check_prepare_enabled; - } - - clk_ret = clk_prepare_enable(clk); - if (clk_ret) { - d40_err(&pdev->dev, "Failed to prepare/enable clock\n"); - goto disable_unprepare; - } + clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(clk)) + return PTR_ERR(clk); /* Get IO for DMAC base address */ - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base"); - if (!res) - goto disable_unprepare; - - if (request_mem_region(res->start, resource_size(res), - D40_NAME " I/O base") == NULL) - goto release_region; - - virtbase = ioremap(res->start, resource_size(res)); - if (!virtbase) - goto release_region; + virtbase = devm_platform_ioremap_resource_byname(pdev, "base"); + if (IS_ERR(virtbase)) + return PTR_ERR(virtbase); /* This is just a regular AMBA PrimeCell ID actually */ for (pid = 0, i = 0; i < 4; i++) - pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i) + pid |= (readl(virtbase + SZ_4K - 0x20 + 4 * i) & 255) << (i * 8); for (cid = 0, i = 0; i < 4; i++) - cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i) + cid |= (readl(virtbase + SZ_4K - 0x10 + 4 * i) & 255) << (i * 8); if (cid != AMBA_CID) { - d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID\n"); - goto unmap_io; + d40_err(dev, "Unknown hardware! No PrimeCell ID\n"); + return -EINVAL; } if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) { - d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n", + d40_err(dev, "Unknown designer! Got %x wanted %x\n", AMBA_MANF_BITS(pid), AMBA_VENDOR_ST); - goto unmap_io; + return -EINVAL; } /* * HW revision: @@ -3170,8 +3185,8 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) */ rev = AMBA_REV_BITS(pid); if (rev < 2) { - d40_err(&pdev->dev, "hardware revision: %d is not supported", rev); - goto unmap_io; + d40_err(dev, "hardware revision: %d is not supported", rev); + return -EINVAL; } /* The number of physical channels on this HW */ @@ -3188,27 +3203,26 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) num_log_chans = num_phy_chans * D40_MAX_LOG_CHAN_PER_PHY; - dev_info(&pdev->dev, - "hardware rev: %d @ %pa with %d physical and %d logical channels\n", - rev, &res->start, num_phy_chans, num_log_chans); + dev_info(dev, + "hardware rev: %d with %d physical and %d logical channels\n", + rev, num_phy_chans, num_log_chans); - base = kzalloc(ALIGN(sizeof(struct d40_base), 4) + - (num_phy_chans + num_log_chans + num_memcpy_chans) * - sizeof(struct d40_chan), GFP_KERNEL); + base = devm_kzalloc(dev, + ALIGN(sizeof(struct d40_base), 4) + + (num_phy_chans + num_log_chans + num_memcpy_chans) * + sizeof(struct d40_chan), GFP_KERNEL); - if (base == NULL) - goto unmap_io; + if (!base) + return -ENOMEM; base->rev = rev; base->clk = clk; base->num_memcpy_chans = num_memcpy_chans; base->num_phy_chans = num_phy_chans; base->num_log_chans = num_log_chans; - base->phy_start = res->start; - base->phy_size = resource_size(res); base->virtbase = virtbase; base->plat_data = plat_data; - base->dev = &pdev->dev; + base->dev = dev; base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4); base->log_chans = &base->phy_chans[num_phy_chans]; @@ -3242,76 +3256,57 @@ static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) base->gen_dmac.init_reg_size = ARRAY_SIZE(dma_init_reg_v4a); } - base->phy_res = kcalloc(num_phy_chans, - sizeof(*base->phy_res), - GFP_KERNEL); + base->phy_res = devm_kcalloc(dev, num_phy_chans, + sizeof(*base->phy_res), + GFP_KERNEL); if (!base->phy_res) - goto free_base; + return -ENOMEM; - base->lookup_phy_chans = kcalloc(num_phy_chans, - sizeof(*base->lookup_phy_chans), - GFP_KERNEL); + base->lookup_phy_chans = devm_kcalloc(dev, num_phy_chans, + sizeof(*base->lookup_phy_chans), + GFP_KERNEL); if (!base->lookup_phy_chans) - goto free_phy_res; + return -ENOMEM; - base->lookup_log_chans = kcalloc(num_log_chans, - sizeof(*base->lookup_log_chans), - GFP_KERNEL); + base->lookup_log_chans = devm_kcalloc(dev, num_log_chans, + sizeof(*base->lookup_log_chans), + GFP_KERNEL); if (!base->lookup_log_chans) - goto free_phy_chans; + return -ENOMEM; - base->reg_val_backup_chan = kmalloc_array(base->num_phy_chans, + base->reg_val_backup_chan = devm_kmalloc_array(dev, base->num_phy_chans, sizeof(d40_backup_regs_chan), GFP_KERNEL); if (!base->reg_val_backup_chan) - goto free_log_chans; + return -ENOMEM; - base->lcla_pool.alloc_map = kcalloc(num_phy_chans + base->lcla_pool.alloc_map = devm_kcalloc(dev, num_phy_chans * D40_LCLA_LINK_PER_EVENT_GRP, sizeof(*base->lcla_pool.alloc_map), GFP_KERNEL); if (!base->lcla_pool.alloc_map) - goto free_backup_chan; + return -ENOMEM; - base->regs_interrupt = kmalloc_array(base->gen_dmac.il_size, + base->regs_interrupt = devm_kmalloc_array(dev, base->gen_dmac.il_size, sizeof(*base->regs_interrupt), GFP_KERNEL); if (!base->regs_interrupt) - goto free_map; + return -ENOMEM; base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc), 0, SLAB_HWCACHE_ALIGN, NULL); - if (base->desc_slab == NULL) - goto free_regs; - - - return base; - free_regs: - kfree(base->regs_interrupt); - free_map: - kfree(base->lcla_pool.alloc_map); - free_backup_chan: - kfree(base->reg_val_backup_chan); - free_log_chans: - kfree(base->lookup_log_chans); - free_phy_chans: - kfree(base->lookup_phy_chans); - free_phy_res: - kfree(base->phy_res); - free_base: - kfree(base); - unmap_io: - iounmap(virtbase); - release_region: - release_mem_region(res->start, resource_size(res)); - check_prepare_enabled: - if (!clk_ret) - disable_unprepare: - clk_disable_unprepare(clk); - if (!IS_ERR(clk)) - clk_put(clk); - return NULL; + if (!base->desc_slab) + return -ENOMEM; + + ret = devm_add_action_or_reset(dev, d40_drop_kmem_cache_action, + base->desc_slab); + if (ret) + return ret; + + *retbase = base; + + return 0; } static void __init d40_hw_init(struct d40_base *base) @@ -3415,7 +3410,7 @@ static int __init d40_lcla_allocate(struct d40_base *base) base->lcla_pool.base = (void *)page_list[i]; } else { /* - * After many attempts and no succees with finding the correct + * After many attempts and no success with finding the correct * alignment, try with allocating a big buffer. */ dev_warn(base->dev, @@ -3451,14 +3446,14 @@ static int __init d40_lcla_allocate(struct d40_base *base) return ret; } -static int __init d40_of_probe(struct platform_device *pdev, +static int __init d40_of_probe(struct device *dev, struct device_node *np) { struct stedma40_platform_data *pdata; int num_phy = 0, num_memcpy = 0, num_disabled = 0; const __be32 *list; - pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return -ENOMEM; @@ -3471,7 +3466,7 @@ static int __init d40_of_probe(struct platform_device *pdev, num_memcpy /= sizeof(*list); if (num_memcpy > D40_MEMCPY_MAX_CHANS || num_memcpy <= 0) { - d40_err(&pdev->dev, + d40_err(dev, "Invalid number of memcpy channels specified (%d)\n", num_memcpy); return -EINVAL; @@ -3486,7 +3481,7 @@ static int __init d40_of_probe(struct platform_device *pdev, num_disabled /= sizeof(*list); if (num_disabled >= STEDMA40_MAX_PHYS || num_disabled < 0) { - d40_err(&pdev->dev, + d40_err(dev, "Invalid number of disabled channels specified (%d)\n", num_disabled); return -EINVAL; @@ -3497,35 +3492,30 @@ static int __init d40_of_probe(struct platform_device *pdev, num_disabled); pdata->disabled_channels[num_disabled] = -1; - pdev->dev.platform_data = pdata; + dev->platform_data = pdata; return 0; } static int __init d40_probe(struct platform_device *pdev) { - struct stedma40_platform_data *plat_data = dev_get_platdata(&pdev->dev); + struct device *dev = &pdev->dev; struct device_node *np = pdev->dev.of_node; - int ret = -ENOENT; + struct device_node *np_lcpa; struct d40_base *base; struct resource *res; + struct resource res_lcpa; int num_reserved_chans; u32 val; + int ret; - if (!plat_data) { - if (np) { - if (d40_of_probe(pdev, np)) { - ret = -ENOMEM; - goto report_failure; - } - } else { - d40_err(&pdev->dev, "No pdata or Device Tree provided\n"); - goto report_failure; - } + if (d40_of_probe(dev, np)) { + ret = -ENOMEM; + goto report_failure; } - base = d40_hw_detect_init(pdev); - if (!base) + ret = d40_hw_detect_init(pdev, &base); + if (ret) goto report_failure; num_reserved_chans = d40_phy_res_init(base); @@ -3535,37 +3525,38 @@ static int __init d40_probe(struct platform_device *pdev) spin_lock_init(&base->interrupt_lock); spin_lock_init(&base->execmd_lock); - /* Get IO for logical channel parameter address */ - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa"); - if (!res) { - ret = -ENOENT; - d40_err(&pdev->dev, "No \"lcpa\" memory resource\n"); - goto destroy_cache; + /* Get IO for logical channel parameter address (LCPA) */ + np_lcpa = of_parse_phandle(np, "sram", 0); + if (!np_lcpa) { + dev_err(dev, "no LCPA SRAM node\n"); + ret = -EINVAL; + goto report_failure; } - base->lcpa_size = resource_size(res); - base->phy_lcpa = res->start; - - if (request_mem_region(res->start, resource_size(res), - D40_NAME " I/O lcpa") == NULL) { - ret = -EBUSY; - d40_err(&pdev->dev, "Failed to request LCPA region %pR\n", res); - goto destroy_cache; + /* This is no device so read the address directly from the node */ + ret = of_address_to_resource(np_lcpa, 0, &res_lcpa); + if (ret) { + dev_err(dev, "no LCPA SRAM resource\n"); + goto report_failure; } + base->lcpa_size = resource_size(&res_lcpa); + base->phy_lcpa = res_lcpa.start; + dev_info(dev, "found LCPA SRAM at %pad, size %pa\n", + &base->phy_lcpa, &base->lcpa_size); /* We make use of ESRAM memory for this. */ val = readl(base->virtbase + D40_DREG_LCPA); - if (res->start != val && val != 0) { - dev_warn(&pdev->dev, - "[%s] Mismatch LCPA dma 0x%x, def %pa\n", - __func__, val, &res->start); + if (base->phy_lcpa != val && val != 0) { + dev_warn(dev, + "[%s] Mismatch LCPA dma 0x%x, def %08x\n", + __func__, val, (u32)base->phy_lcpa); } else - writel(res->start, base->virtbase + D40_DREG_LCPA); + writel(base->phy_lcpa, base->virtbase + D40_DREG_LCPA); - base->lcpa_base = ioremap(res->start, resource_size(res)); + base->lcpa_base = devm_ioremap(dev, base->phy_lcpa, base->lcpa_size); if (!base->lcpa_base) { ret = -ENOMEM; - d40_err(&pdev->dev, "Failed to ioremap LCPA region\n"); - goto destroy_cache; + d40_err(dev, "Failed to ioremap LCPA region\n"); + goto report_failure; } /* If lcla has to be located in ESRAM we don't need to allocate */ if (base->plat_data->use_esram_lcla) { @@ -3573,23 +3564,23 @@ static int __init d40_probe(struct platform_device *pdev) "lcla_esram"); if (!res) { ret = -ENOENT; - d40_err(&pdev->dev, + d40_err(dev, "No \"lcla_esram\" memory resource\n"); - goto destroy_cache; + goto report_failure; } - base->lcla_pool.base = ioremap(res->start, - resource_size(res)); + base->lcla_pool.base = devm_ioremap(dev, res->start, + resource_size(res)); if (!base->lcla_pool.base) { ret = -ENOMEM; - d40_err(&pdev->dev, "Failed to ioremap LCLA region\n"); - goto destroy_cache; + d40_err(dev, "Failed to ioremap LCLA region\n"); + goto report_failure; } writel(res->start, base->virtbase + D40_DREG_LCLA); } else { ret = d40_lcla_allocate(base); if (ret) { - d40_err(&pdev->dev, "Failed to allocate LCLA area\n"); + d40_err(dev, "Failed to allocate LCLA area\n"); goto destroy_cache; } } @@ -3597,10 +3588,14 @@ static int __init d40_probe(struct platform_device *pdev) spin_lock_init(&base->lcla_pool.lock); base->irq = platform_get_irq(pdev, 0); + if (base->irq < 0) { + ret = base->irq; + goto destroy_cache; + } ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); if (ret) { - d40_err(&pdev->dev, "No IRQ defined\n"); + d40_err(dev, "No IRQ defined\n"); goto destroy_cache; } @@ -3608,7 +3603,7 @@ static int __init d40_probe(struct platform_device *pdev) base->lcpa_regulator = regulator_get(base->dev, "lcla_esram"); if (IS_ERR(base->lcpa_regulator)) { - d40_err(&pdev->dev, "Failed to get lcpa_regulator\n"); + d40_err(dev, "Failed to get lcpa_regulator\n"); ret = PTR_ERR(base->lcpa_regulator); base->lcpa_regulator = NULL; goto destroy_cache; @@ -3616,7 +3611,7 @@ static int __init d40_probe(struct platform_device *pdev) ret = regulator_enable(base->lcpa_regulator); if (ret) { - d40_err(&pdev->dev, + d40_err(dev, "Failed to enable lcpa_regulator\n"); regulator_put(base->lcpa_regulator); base->lcpa_regulator = NULL; @@ -3637,33 +3632,21 @@ static int __init d40_probe(struct platform_device *pdev) if (ret) goto destroy_cache; - ret = dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE); - if (ret) { - d40_err(&pdev->dev, "Failed to set dma max seg size\n"); - goto destroy_cache; - } + dma_set_max_seg_size(base->dev, STEDMA40_MAX_SEG_SIZE); d40_hw_init(base); - if (np) { - ret = of_dma_controller_register(np, d40_xlate, NULL); - if (ret) - dev_err(&pdev->dev, - "could not register of_dma_controller\n"); + ret = of_dma_controller_register(np, d40_xlate, NULL); + if (ret) { + dev_err(dev, + "could not register of_dma_controller\n"); + goto destroy_cache; } dev_info(base->dev, "initialized\n"); return 0; - destroy_cache: - kmem_cache_destroy(base->desc_slab); - if (base->virtbase) - iounmap(base->virtbase); - - if (base->lcla_pool.base && base->plat_data->use_esram_lcla) { - iounmap(base->lcla_pool.base); - base->lcla_pool.base = NULL; - } + destroy_cache: if (base->lcla_pool.dma_addr) dma_unmap_single(base->dev, base->lcla_pool.dma_addr, SZ_1K * base->num_phy_chans, @@ -3675,32 +3658,14 @@ static int __init d40_probe(struct platform_device *pdev) kfree(base->lcla_pool.base_unaligned); - if (base->lcpa_base) - iounmap(base->lcpa_base); - - if (base->phy_lcpa) - release_mem_region(base->phy_lcpa, - base->lcpa_size); - if (base->phy_start) - release_mem_region(base->phy_start, - base->phy_size); - if (base->clk) { - clk_disable_unprepare(base->clk); - clk_put(base->clk); - } - if (base->lcpa_regulator) { regulator_disable(base->lcpa_regulator); regulator_put(base->lcpa_regulator); } + pm_runtime_disable(base->dev); - kfree(base->lcla_pool.alloc_map); - kfree(base->lookup_log_chans); - kfree(base->lookup_phy_chans); - kfree(base->phy_res); - kfree(base); report_failure: - d40_err(&pdev->dev, "probe failed\n"); + d40_err(dev, "probe failed\n"); return ret; } diff --git a/drivers/dma/ste_dma40.h b/drivers/dma/ste_dma40.h new file mode 100644 index 000000000000..a90c786acc1f --- /dev/null +++ b/drivers/dma/ste_dma40.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef STE_DMA40_H +#define STE_DMA40_H + +/* + * Maximum size for a single dma descriptor + * Size is limited to 16 bits. + * Size is in the units of addr-widths (1,2,4,8 bytes) + * Larger transfers will be split up to multiple linked desc + */ +#define STEDMA40_MAX_SEG_SIZE 0xFFFF + +/* dev types for memcpy */ +#define STEDMA40_DEV_DST_MEMORY (-1) +#define STEDMA40_DEV_SRC_MEMORY (-1) + +enum stedma40_mode { + STEDMA40_MODE_LOGICAL = 0, + STEDMA40_MODE_PHYSICAL, + STEDMA40_MODE_OPERATION, +}; + +enum stedma40_mode_opt { + STEDMA40_PCHAN_BASIC_MODE = 0, + STEDMA40_LCHAN_SRC_LOG_DST_LOG = 0, + STEDMA40_PCHAN_MODULO_MODE, + STEDMA40_PCHAN_DOUBLE_DST_MODE, + STEDMA40_LCHAN_SRC_PHY_DST_LOG, + STEDMA40_LCHAN_SRC_LOG_DST_PHY, +}; + +#define STEDMA40_ESIZE_8_BIT 0x0 +#define STEDMA40_ESIZE_16_BIT 0x1 +#define STEDMA40_ESIZE_32_BIT 0x2 +#define STEDMA40_ESIZE_64_BIT 0x3 + +/* The value 4 indicates that PEN-reg shall be set to 0 */ +#define STEDMA40_PSIZE_PHY_1 0x4 +#define STEDMA40_PSIZE_PHY_2 0x0 +#define STEDMA40_PSIZE_PHY_4 0x1 +#define STEDMA40_PSIZE_PHY_8 0x2 +#define STEDMA40_PSIZE_PHY_16 0x3 + +/* + * The number of elements differ in logical and + * physical mode + */ +#define STEDMA40_PSIZE_LOG_1 STEDMA40_PSIZE_PHY_2 +#define STEDMA40_PSIZE_LOG_4 STEDMA40_PSIZE_PHY_4 +#define STEDMA40_PSIZE_LOG_8 STEDMA40_PSIZE_PHY_8 +#define STEDMA40_PSIZE_LOG_16 STEDMA40_PSIZE_PHY_16 + +/* Maximum number of possible physical channels */ +#define STEDMA40_MAX_PHYS 32 + +enum stedma40_flow_ctrl { + STEDMA40_NO_FLOW_CTRL, + STEDMA40_FLOW_CTRL, +}; + +/** + * struct stedma40_half_channel_info - dst/src channel configuration + * + * @big_endian: true if the src/dst should be read as big endian + * @data_width: Data width of the src/dst hardware + * @p_size: Burst size + * @flow_ctrl: Flow control on/off. + */ +struct stedma40_half_channel_info { + bool big_endian; + enum dma_slave_buswidth data_width; + int psize; + enum stedma40_flow_ctrl flow_ctrl; +}; + +/** + * struct stedma40_chan_cfg - Structure to be filled by client drivers. + * + * @dir: MEM 2 MEM, PERIPH 2 MEM , MEM 2 PERIPH, PERIPH 2 PERIPH + * @high_priority: true if high-priority + * @realtime: true if realtime mode is to be enabled. Only available on DMA40 + * version 3+, i.e DB8500v2+ + * @mode: channel mode: physical, logical, or operation + * @mode_opt: options for the chosen channel mode + * @dev_type: src/dst device type (driver uses dir to figure out which) + * @src_info: Parameters for dst half channel + * @dst_info: Parameters for dst half channel + * @use_fixed_channel: if true, use physical channel specified by phy_channel + * @phy_channel: physical channel to use, only if use_fixed_channel is true + * + * This structure has to be filled by the client drivers. + * It is recommended to do all dma configurations for clients in the machine. + * + */ +struct stedma40_chan_cfg { + enum dma_transfer_direction dir; + bool high_priority; + bool realtime; + enum stedma40_mode mode; + enum stedma40_mode_opt mode_opt; + int dev_type; + struct stedma40_half_channel_info src_info; + struct stedma40_half_channel_info dst_info; + + bool use_fixed_channel; + int phy_channel; +}; + +#endif /* STE_DMA40_H */ diff --git a/drivers/dma/ste_dma40_ll.c b/drivers/dma/ste_dma40_ll.c index b5287c661eb7..4c489b126cb2 100644 --- a/drivers/dma/ste_dma40_ll.c +++ b/drivers/dma/ste_dma40_ll.c @@ -6,8 +6,9 @@ */ #include <linux/kernel.h> -#include <linux/platform_data/dma-ste-dma40.h> +#include <linux/dmaengine.h> +#include "ste_dma40.h" #include "ste_dma40_ll.h" static u8 d40_width_to_bits(enum dma_slave_buswidth width) diff --git a/drivers/dma/ste_dma40_ll.h b/drivers/dma/ste_dma40_ll.h index c504e855eb02..2e30e9a94a1e 100644 --- a/drivers/dma/ste_dma40_ll.h +++ b/drivers/dma/ste_dma40_ll.h @@ -369,7 +369,7 @@ struct d40_phy_lli_bidir { * @lcsp02: Either maps to register lcsp0 if src or lcsp2 if dst. * @lcsp13: Either maps to register lcsp1 if src or lcsp3 if dst. * - * This struct must be 8 bytes aligned since it will be accessed directy by + * This struct must be 8 bytes aligned since it will be accessed directly by * the DMA. Never add any none hw mapped registers to this struct. */ diff --git a/drivers/dma/stm32/Kconfig b/drivers/dma/stm32/Kconfig new file mode 100644 index 000000000000..4d8d8063133b --- /dev/null +++ b/drivers/dma/stm32/Kconfig @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# STM32 DMA controllers drivers +# +if ARCH_STM32 || COMPILE_TEST + +config STM32_DMA + bool "STMicroelectronics STM32 DMA support" + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the on-chip DMA controller on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA controller + and want to use DMA say Y here. + +config STM32_DMAMUX + bool "STMicroelectronics STM32 DMA multiplexer support" + depends on STM32_DMA + help + Enable support for the on-chip DMA multiplexer on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA multiplexer + and want to use DMAMUX say Y here. + +config STM32_MDMA + bool "STMicroelectronics STM32 master DMA support" + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the on-chip MDMA controller on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA controller + and want to use MDMA say Y here. + +config STM32_DMA3 + tristate "STMicroelectronics STM32 DMA3 support" + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Enable support for the on-chip DMA3 controller on STMicroelectronics + STM32 platforms. + If you have a board based on STM32 SoC with such DMA3 controller + and want to use DMA3, say Y here. + +endif diff --git a/drivers/dma/stm32/Makefile b/drivers/dma/stm32/Makefile new file mode 100644 index 000000000000..5082db4b4c1c --- /dev/null +++ b/drivers/dma/stm32/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_STM32_DMA) += stm32-dma.o +obj-$(CONFIG_STM32_DMAMUX) += stm32-dmamux.o +obj-$(CONFIG_STM32_MDMA) += stm32-mdma.o +obj-$(CONFIG_STM32_DMA3) += stm32-dma3.o diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32/stm32-dma.c index f54ecb123a52..04389936c8a6 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32/stm32-dma.c @@ -9,6 +9,7 @@ * Pierre-Yves Mordret <pierre-yves.mordret@st.com> */ +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/delay.h> #include <linux/dmaengine.h> @@ -20,7 +21,6 @@ #include <linux/list.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> @@ -28,12 +28,14 @@ #include <linux/sched.h> #include <linux/slab.h> -#include "virt-dma.h" +#include "../virt-dma.h" #define STM32_DMA_LISR 0x0000 /* DMA Low Int Status Reg */ #define STM32_DMA_HISR 0x0004 /* DMA High Int Status Reg */ +#define STM32_DMA_ISR(n) (((n) & 4) ? STM32_DMA_HISR : STM32_DMA_LISR) #define STM32_DMA_LIFCR 0x0008 /* DMA Low Int Flag Clear Reg */ #define STM32_DMA_HIFCR 0x000c /* DMA High Int Flag Clear Reg */ +#define STM32_DMA_IFCR(n) (((n) & 4) ? STM32_DMA_HIFCR : STM32_DMA_LIFCR) #define STM32_DMA_TCI BIT(5) /* Transfer Complete Interrupt */ #define STM32_DMA_HTI BIT(4) /* Half Transfer Interrupt */ #define STM32_DMA_TEI BIT(3) /* Transfer Error Interrupt */ @@ -43,23 +45,23 @@ | STM32_DMA_TEI \ | STM32_DMA_DMEI \ | STM32_DMA_FEI) +/* + * If (chan->id % 4) is 2 or 3, left shift the mask by 16 bits; + * if (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. + */ +#define STM32_DMA_FLAGS_SHIFT(n) ({ typeof(n) (_n) = (n); \ + (((_n) & 2) << 3) | (((_n) & 1) * 6); }) /* DMA Stream x Configuration Register */ #define STM32_DMA_SCR(x) (0x0010 + 0x18 * (x)) /* x = 0..7 */ -#define STM32_DMA_SCR_REQ(n) ((n & 0x7) << 25) +#define STM32_DMA_SCR_REQ_MASK GENMASK(27, 25) #define STM32_DMA_SCR_MBURST_MASK GENMASK(24, 23) -#define STM32_DMA_SCR_MBURST(n) ((n & 0x3) << 23) #define STM32_DMA_SCR_PBURST_MASK GENMASK(22, 21) -#define STM32_DMA_SCR_PBURST(n) ((n & 0x3) << 21) #define STM32_DMA_SCR_PL_MASK GENMASK(17, 16) -#define STM32_DMA_SCR_PL(n) ((n & 0x3) << 16) #define STM32_DMA_SCR_MSIZE_MASK GENMASK(14, 13) -#define STM32_DMA_SCR_MSIZE(n) ((n & 0x3) << 13) #define STM32_DMA_SCR_PSIZE_MASK GENMASK(12, 11) -#define STM32_DMA_SCR_PSIZE(n) ((n & 0x3) << 11) -#define STM32_DMA_SCR_PSIZE_GET(n) ((n & STM32_DMA_SCR_PSIZE_MASK) >> 11) #define STM32_DMA_SCR_DIR_MASK GENMASK(7, 6) -#define STM32_DMA_SCR_DIR(n) ((n & 0x3) << 6) +#define STM32_DMA_SCR_TRBUFF BIT(20) /* Bufferable transfer for USART/UART */ #define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */ #define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */ #define STM32_DMA_SCR_PINCOS BIT(15) /* Peripheral inc offset size */ @@ -95,7 +97,6 @@ /* DMA stream x FIFO control register */ #define STM32_DMA_SFCR(x) (0x0024 + 0x18 * (x)) #define STM32_DMA_SFCR_FTH_MASK GENMASK(1, 0) -#define STM32_DMA_SFCR_FTH(n) (n & STM32_DMA_SFCR_FTH_MASK) #define STM32_DMA_SFCR_FEIE BIT(7) /* FIFO error interrupt enable */ #define STM32_DMA_SFCR_DMDIS BIT(2) /* Direct mode disable */ #define STM32_DMA_SFCR_MASK (STM32_DMA_SFCR_FEIE \ @@ -136,10 +137,9 @@ /* DMA Features */ #define STM32_DMA_THRESHOLD_FTR_MASK GENMASK(1, 0) -#define STM32_DMA_THRESHOLD_FTR_GET(n) ((n) & STM32_DMA_THRESHOLD_FTR_MASK) #define STM32_DMA_DIRECT_MODE_MASK BIT(2) -#define STM32_DMA_DIRECT_MODE_GET(n) (((n) & STM32_DMA_DIRECT_MODE_MASK) \ - >> 2) +#define STM32_DMA_ALT_ACK_MODE_MASK BIT(4) +#define STM32_DMA_MDMA_STREAM_ID_MASK GENMASK(19, 16) enum stm32_dma_width { STM32_DMA_BYTE, @@ -190,7 +190,20 @@ struct stm32_dma_desc { struct virt_dma_desc vdesc; bool cyclic; u32 num_sgs; - struct stm32_dma_sg_req sg_req[]; + struct stm32_dma_sg_req sg_req[] __counted_by(num_sgs); +}; + +/** + * struct stm32_dma_mdma_config - STM32 DMA MDMA configuration + * @stream_id: DMA request to trigger STM32 MDMA transfer + * @ifcr: DMA interrupt flag clear register address, + * used by STM32 MDMA to clear DMA Transfer Complete flag + * @tcf: DMA Transfer Complete flag + */ +struct stm32_dma_mdma_config { + u32 stream_id; + u32 ifcr; + u32 tcf; }; struct stm32_dma_chan { @@ -206,6 +219,9 @@ struct stm32_dma_chan { u32 threshold; u32 mem_burst; u32 mem_width; + enum dma_status status; + bool trig_mdma; + struct stm32_dma_mdma_config mdma_config; }; struct stm32_dma_device { @@ -268,7 +284,6 @@ static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len, u32 threshold) { enum dma_slave_buswidth max_width; - u64 addr = buf_addr; if (threshold == STM32_DMA_FIFO_THRESHOLD_FULL) max_width = DMA_SLAVE_BUSWIDTH_4_BYTES; @@ -279,7 +294,7 @@ static enum dma_slave_buswidth stm32_dma_get_max_width(u32 buf_len, max_width > DMA_SLAVE_BUSWIDTH_1_BYTE) max_width = max_width >> 1; - if (do_div(addr, max_width)) + if (buf_addr & (max_width - 1)) max_width = DMA_SLAVE_BUSWIDTH_1_BYTE; return max_width; @@ -386,6 +401,13 @@ static int stm32_dma_slave_config(struct dma_chan *c, memcpy(&chan->dma_sconfig, config, sizeof(*config)); + /* Check if user is requesting DMA to trigger STM32 MDMA */ + if (config->peripheral_size) { + config->peripheral_config = &chan->mdma_config; + config->peripheral_size = sizeof(chan->mdma_config); + chan->trig_mdma = true; + } + chan->config_init = true; return 0; @@ -399,17 +421,10 @@ static u32 stm32_dma_irq_status(struct stm32_dma_chan *chan) /* * Read "flags" from DMA_xISR register corresponding to the selected * DMA channel at the correct bit offset inside that register. - * - * If (ch % 4) is 2 or 3, left shift the mask by 16 bits. - * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. */ - if (chan->id & 4) - dma_isr = stm32_dma_read(dmadev, STM32_DMA_HISR); - else - dma_isr = stm32_dma_read(dmadev, STM32_DMA_LISR); - - flags = dma_isr >> (((chan->id & 2) << 3) | ((chan->id & 1) * 6)); + dma_isr = stm32_dma_read(dmadev, STM32_DMA_ISR(chan->id)); + flags = dma_isr >> STM32_DMA_FLAGS_SHIFT(chan->id); return flags & STM32_DMA_MASKI; } @@ -422,17 +437,11 @@ static void stm32_dma_irq_clear(struct stm32_dma_chan *chan, u32 flags) /* * Write "flags" to the DMA_xIFCR register corresponding to the selected * DMA channel at the correct bit offset inside that register. - * - * If (ch % 4) is 2 or 3, left shift the mask by 16 bits. - * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits. */ flags &= STM32_DMA_MASKI; - dma_ifcr = flags << (((chan->id & 2) << 3) | ((chan->id & 1) * 6)); + dma_ifcr = flags << STM32_DMA_FLAGS_SHIFT(chan->id); - if (chan->id & 4) - stm32_dma_write(dmadev, STM32_DMA_HIFCR, dma_ifcr); - else - stm32_dma_write(dmadev, STM32_DMA_LIFCR, dma_ifcr); + stm32_dma_write(dmadev, STM32_DMA_IFCR(chan->id), dma_ifcr); } static int stm32_dma_disable_chan(struct stm32_dma_chan *chan) @@ -484,6 +493,7 @@ static void stm32_dma_stop(struct stm32_dma_chan *chan) } chan->busy = false; + chan->status = DMA_COMPLETE; } static int stm32_dma_terminate_all(struct dma_chan *c) @@ -495,6 +505,7 @@ static int stm32_dma_terminate_all(struct dma_chan *c) spin_lock_irqsave(&chan->vchan.lock, flags); if (chan->desc) { + dma_cookie_complete(&chan->desc->vdesc.tx); vchan_terminate_vdesc(&chan->desc->vdesc); if (chan->busy) stm32_dma_stop(chan); @@ -533,6 +544,13 @@ static void stm32_dma_dump_reg(struct stm32_dma_chan *chan) dev_dbg(chan2dev(chan), "SFCR: 0x%08x\n", sfcr); } +static void stm32_dma_sg_inc(struct stm32_dma_chan *chan) +{ + chan->next_sg++; + if (chan->desc->cyclic && (chan->next_sg == chan->desc->num_sgs)) + chan->next_sg = 0; +} + static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan); static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) @@ -565,6 +583,10 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) sg_req = &chan->desc->sg_req[chan->next_sg]; reg = &sg_req->chan_reg; + /* When DMA triggers STM32 MDMA, DMA Transfer Complete is managed by STM32 MDMA */ + if (chan->trig_mdma && chan->dma_sconfig.direction != DMA_MEM_TO_DEV) + reg->dma_scr &= ~STM32_DMA_SCR_TCIE; + reg->dma_scr &= ~STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); stm32_dma_write(dmadev, STM32_DMA_SPAR(chan->id), reg->dma_spar); @@ -573,7 +595,7 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) stm32_dma_write(dmadev, STM32_DMA_SM1AR(chan->id), reg->dma_sm1ar); stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), reg->dma_sndtr); - chan->next_sg++; + stm32_dma_sg_inc(chan); /* Clear interrupt status if it is there */ status = stm32_dma_irq_status(chan); @@ -586,12 +608,12 @@ static void stm32_dma_start_transfer(struct stm32_dma_chan *chan) stm32_dma_dump_reg(chan); /* Start DMA */ + chan->busy = true; + chan->status = DMA_IN_PROGRESS; reg->dma_scr |= STM32_DMA_SCR_EN; stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), reg->dma_scr); - chan->busy = true; - - dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); } static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) @@ -603,41 +625,135 @@ static void stm32_dma_configure_next_sg(struct stm32_dma_chan *chan) id = chan->id; dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); - if (dma_scr & STM32_DMA_SCR_DBM) { - if (chan->next_sg == chan->desc->num_sgs) - chan->next_sg = 0; + sg_req = &chan->desc->sg_req[chan->next_sg]; - sg_req = &chan->desc->sg_req[chan->next_sg]; + if (dma_scr & STM32_DMA_SCR_CT) { + dma_sm0ar = sg_req->chan_reg.dma_sm0ar; + stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar); + dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n", + stm32_dma_read(dmadev, STM32_DMA_SM0AR(id))); + } else { + dma_sm1ar = sg_req->chan_reg.dma_sm1ar; + stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar); + dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n", + stm32_dma_read(dmadev, STM32_DMA_SM1AR(id))); + } +} - if (dma_scr & STM32_DMA_SCR_CT) { - dma_sm0ar = sg_req->chan_reg.dma_sm0ar; - stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), dma_sm0ar); - dev_dbg(chan2dev(chan), "CT=1 <=> SM0AR: 0x%08x\n", - stm32_dma_read(dmadev, STM32_DMA_SM0AR(id))); - } else { - dma_sm1ar = sg_req->chan_reg.dma_sm1ar; - stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), dma_sm1ar); - dev_dbg(chan2dev(chan), "CT=0 <=> SM1AR: 0x%08x\n", - stm32_dma_read(dmadev, STM32_DMA_SM1AR(id))); - } +static void stm32_dma_handle_chan_paused(struct stm32_dma_chan *chan) +{ + struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); + u32 dma_scr; + + /* + * Read and store current remaining data items and peripheral/memory addresses to be + * updated on resume + */ + dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); + /* + * Transfer can be paused while between a previous resume and reconfiguration on transfer + * complete. If transfer is cyclic and CIRC and DBM have been deactivated for resume, need + * to set it here in SCR backup to ensure a good reconfiguration on transfer complete. + */ + if (chan->desc && chan->desc->cyclic) { + if (chan->desc->num_sgs == 1) + dma_scr |= STM32_DMA_SCR_CIRC; + else + dma_scr |= STM32_DMA_SCR_DBM; } + chan->chan_reg.dma_scr = dma_scr; + + /* + * Need to temporarily deactivate CIRC/DBM until next Transfer Complete interrupt, otherwise + * on resume NDTR autoreload value will be wrong (lower than the initial period length) + */ + if (chan->desc && chan->desc->cyclic) { + dma_scr &= ~(STM32_DMA_SCR_DBM | STM32_DMA_SCR_CIRC); + stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); + } + + chan->chan_reg.dma_sndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); + + chan->status = DMA_PAUSED; + + dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan); } -static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan) +static void stm32_dma_post_resume_reconfigure(struct stm32_dma_chan *chan) { - if (chan->desc) { - if (chan->desc->cyclic) { - vchan_cyclic_callback(&chan->desc->vdesc); - chan->next_sg++; + struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); + struct stm32_dma_sg_req *sg_req; + u32 dma_scr, status, id; + + id = chan->id; + dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); + + /* Clear interrupt status if it is there */ + status = stm32_dma_irq_status(chan); + if (status) + stm32_dma_irq_clear(chan, status); + + if (!chan->next_sg) + sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1]; + else + sg_req = &chan->desc->sg_req[chan->next_sg - 1]; + + /* Reconfigure NDTR with the initial value */ + stm32_dma_write(dmadev, STM32_DMA_SNDTR(chan->id), sg_req->chan_reg.dma_sndtr); + + /* Restore SPAR */ + stm32_dma_write(dmadev, STM32_DMA_SPAR(id), sg_req->chan_reg.dma_spar); + + /* Restore SM0AR/SM1AR whatever DBM/CT as they may have been modified */ + stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), sg_req->chan_reg.dma_sm0ar); + stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), sg_req->chan_reg.dma_sm1ar); + + /* Reactivate CIRC/DBM if needed */ + if (chan->chan_reg.dma_scr & STM32_DMA_SCR_DBM) { + dma_scr |= STM32_DMA_SCR_DBM; + /* Restore CT */ + if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CT) + dma_scr &= ~STM32_DMA_SCR_CT; + else + dma_scr |= STM32_DMA_SCR_CT; + } else if (chan->chan_reg.dma_scr & STM32_DMA_SCR_CIRC) { + dma_scr |= STM32_DMA_SCR_CIRC; + } + stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); + + stm32_dma_configure_next_sg(chan); + + stm32_dma_dump_reg(chan); + + dma_scr |= STM32_DMA_SCR_EN; + stm32_dma_write(dmadev, STM32_DMA_SCR(chan->id), dma_scr); + + dev_dbg(chan2dev(chan), "vchan %p: reconfigured after pause/resume\n", &chan->vchan); +} + +static void stm32_dma_handle_chan_done(struct stm32_dma_chan *chan, u32 scr) +{ + if (!chan->desc) + return; + + if (chan->desc->cyclic) { + vchan_cyclic_callback(&chan->desc->vdesc); + if (chan->trig_mdma) + return; + stm32_dma_sg_inc(chan); + /* cyclic while CIRC/DBM disable => post resume reconfiguration needed */ + if (!(scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM))) + stm32_dma_post_resume_reconfigure(chan); + else if (scr & STM32_DMA_SCR_DBM && chan->desc->num_sgs > 2) stm32_dma_configure_next_sg(chan); - } else { - chan->busy = false; - if (chan->next_sg == chan->desc->num_sgs) { - vchan_cookie_complete(&chan->desc->vdesc); - chan->desc = NULL; - } - stm32_dma_start_transfer(chan); + } else { + chan->busy = false; + chan->status = DMA_COMPLETE; + if (chan->next_sg == chan->desc->num_sgs) { + vchan_cookie_complete(&chan->desc->vdesc); + chan->desc = NULL; } + stm32_dma_start_transfer(chan); } } @@ -673,8 +789,10 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) if (status & STM32_DMA_TCI) { stm32_dma_irq_clear(chan, STM32_DMA_TCI); - if (scr & STM32_DMA_SCR_TCIE) - stm32_dma_handle_chan_done(chan); + if (scr & STM32_DMA_SCR_TCIE) { + if (chan->status != DMA_PAUSED) + stm32_dma_handle_chan_done(chan, scr); + } status &= ~STM32_DMA_TCI; } @@ -702,13 +820,113 @@ static void stm32_dma_issue_pending(struct dma_chan *c) spin_lock_irqsave(&chan->vchan.lock, flags); if (vchan_issue_pending(&chan->vchan) && !chan->desc && !chan->busy) { - dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); stm32_dma_start_transfer(chan); } spin_unlock_irqrestore(&chan->vchan.lock, flags); } +static int stm32_dma_pause(struct dma_chan *c) +{ + struct stm32_dma_chan *chan = to_stm32_dma_chan(c); + unsigned long flags; + int ret; + + if (chan->status != DMA_IN_PROGRESS) + return -EPERM; + + spin_lock_irqsave(&chan->vchan.lock, flags); + + ret = stm32_dma_disable_chan(chan); + if (!ret) + stm32_dma_handle_chan_paused(chan); + + spin_unlock_irqrestore(&chan->vchan.lock, flags); + + return ret; +} + +static int stm32_dma_resume(struct dma_chan *c) +{ + struct stm32_dma_chan *chan = to_stm32_dma_chan(c); + struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); + struct stm32_dma_chan_reg chan_reg = chan->chan_reg; + u32 id = chan->id, scr, ndtr, offset, spar, sm0ar, sm1ar; + struct stm32_dma_sg_req *sg_req; + unsigned long flags; + + if (chan->status != DMA_PAUSED) + return -EPERM; + + scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); + if (WARN_ON(scr & STM32_DMA_SCR_EN)) + return -EPERM; + + spin_lock_irqsave(&chan->vchan.lock, flags); + + /* sg_reg[prev_sg] contains original ndtr, sm0ar and sm1ar before pausing the transfer */ + if (!chan->next_sg) + sg_req = &chan->desc->sg_req[chan->desc->num_sgs - 1]; + else + sg_req = &chan->desc->sg_req[chan->next_sg - 1]; + + ndtr = sg_req->chan_reg.dma_sndtr; + offset = (ndtr - chan_reg.dma_sndtr); + offset <<= FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, chan_reg.dma_scr); + spar = sg_req->chan_reg.dma_spar; + sm0ar = sg_req->chan_reg.dma_sm0ar; + sm1ar = sg_req->chan_reg.dma_sm1ar; + + /* + * The peripheral and/or memory addresses have to be updated in order to adjust the + * address pointers. Need to check increment. + */ + if (chan_reg.dma_scr & STM32_DMA_SCR_PINC) + stm32_dma_write(dmadev, STM32_DMA_SPAR(id), spar + offset); + else + stm32_dma_write(dmadev, STM32_DMA_SPAR(id), spar); + + if (!(chan_reg.dma_scr & STM32_DMA_SCR_MINC)) + offset = 0; + + /* + * In case of DBM, the current target could be SM1AR. + * Need to temporarily deactivate CIRC/DBM to finish the current transfer, so + * SM0AR becomes the current target and must be updated with SM1AR + offset if CT=1. + */ + if ((chan_reg.dma_scr & STM32_DMA_SCR_DBM) && (chan_reg.dma_scr & STM32_DMA_SCR_CT)) + stm32_dma_write(dmadev, STM32_DMA_SM1AR(id), sm1ar + offset); + else + stm32_dma_write(dmadev, STM32_DMA_SM0AR(id), sm0ar + offset); + + /* NDTR must be restored otherwise internal HW counter won't be correctly reset */ + stm32_dma_write(dmadev, STM32_DMA_SNDTR(id), chan_reg.dma_sndtr); + + /* + * Need to temporarily deactivate CIRC/DBM until next Transfer Complete interrupt, + * otherwise NDTR autoreload value will be wrong (lower than the initial period length) + */ + if (chan_reg.dma_scr & (STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM)) + chan_reg.dma_scr &= ~(STM32_DMA_SCR_CIRC | STM32_DMA_SCR_DBM); + + if (chan_reg.dma_scr & STM32_DMA_SCR_DBM) + stm32_dma_configure_next_sg(chan); + + stm32_dma_dump_reg(chan); + + /* The stream may then be re-enabled to restart transfer from the point it was stopped */ + chan->status = DMA_IN_PROGRESS; + chan_reg.dma_scr |= STM32_DMA_SCR_EN; + stm32_dma_write(dmadev, STM32_DMA_SCR(id), chan_reg.dma_scr); + + spin_unlock_irqrestore(&chan->vchan.lock, flags); + + dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan); + + return 0; +} + static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, enum dma_transfer_direction direction, enum dma_slave_buswidth *buswidth, @@ -751,8 +969,14 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, if (src_bus_width < 0) return src_bus_width; - /* Set memory burst size */ - src_maxburst = STM32_DMA_MAX_BURST; + /* + * Set memory burst size - burst not possible if address is not aligned on + * the address boundary equal to the size of the transfer + */ + if (buf_addr & (buf_len - 1)) + src_maxburst = 1; + else + src_maxburst = STM32_DMA_MAX_BURST; src_best_burst = stm32_dma_get_best_burst(buf_len, src_maxburst, fifoth, @@ -761,16 +985,16 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, if (src_burst_size < 0) return src_burst_size; - dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_DEV) | - STM32_DMA_SCR_PSIZE(dst_bus_width) | - STM32_DMA_SCR_MSIZE(src_bus_width) | - STM32_DMA_SCR_PBURST(dst_burst_size) | - STM32_DMA_SCR_MBURST(src_burst_size); + dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_DEV) | + FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, dst_bus_width) | + FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, src_bus_width) | + FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dst_burst_size) | + FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, src_burst_size); /* Set FIFO threshold */ chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK; if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE) - chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth); + chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth); /* Set peripheral address */ chan->chan_reg.dma_spar = chan->dma_sconfig.dst_addr; @@ -801,8 +1025,14 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, if (dst_bus_width < 0) return dst_bus_width; - /* Set memory burst size */ - dst_maxburst = STM32_DMA_MAX_BURST; + /* + * Set memory burst size - burst not possible if address is not aligned on + * the address boundary equal to the size of the transfer + */ + if (buf_addr & (buf_len - 1)) + dst_maxburst = 1; + else + dst_maxburst = STM32_DMA_MAX_BURST; dst_best_burst = stm32_dma_get_best_burst(buf_len, dst_maxburst, fifoth, @@ -812,16 +1042,16 @@ static int stm32_dma_set_xfer_param(struct stm32_dma_chan *chan, if (dst_burst_size < 0) return dst_burst_size; - dma_scr = STM32_DMA_SCR_DIR(STM32_DMA_DEV_TO_MEM) | - STM32_DMA_SCR_PSIZE(src_bus_width) | - STM32_DMA_SCR_MSIZE(dst_bus_width) | - STM32_DMA_SCR_PBURST(src_burst_size) | - STM32_DMA_SCR_MBURST(dst_burst_size); + dma_scr = FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_DEV_TO_MEM) | + FIELD_PREP(STM32_DMA_SCR_PSIZE_MASK, src_bus_width) | + FIELD_PREP(STM32_DMA_SCR_MSIZE_MASK, dst_bus_width) | + FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, src_burst_size) | + FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dst_burst_size); /* Set FIFO threshold */ chan->chan_reg.dma_sfcr &= ~STM32_DMA_SFCR_FTH_MASK; if (fifoth != STM32_DMA_FIFO_THRESHOLD_NONE) - chan->chan_reg.dma_sfcr |= STM32_DMA_SFCR_FTH(fifoth); + chan->chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, fifoth); /* Set peripheral address */ chan->chan_reg.dma_spar = chan->dma_sconfig.src_addr; @@ -874,6 +1104,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( desc = kzalloc(struct_size(desc, sg_req, sg_len), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = sg_len; /* Set peripheral flow controller */ if (chan->dma_sconfig.device_fc) @@ -881,6 +1112,12 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( else chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; + /* Activate Double Buffer Mode if DMA triggers STM32 MDMA and more than 1 sg */ + if (chan->trig_mdma && sg_len > 1) { + chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM; + chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT; + } + for_each_sg(sgl, sg, sg_len, i) { ret = stm32_dma_set_xfer_param(chan, direction, &buswidth, sg_dma_len(sg), @@ -902,10 +1139,10 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_slave_sg( desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; desc->sg_req[i].chan_reg.dma_sm0ar = sg_dma_address(sg); desc->sg_req[i].chan_reg.dma_sm1ar = sg_dma_address(sg); + if (chan->trig_mdma) + desc->sg_req[i].chan_reg.dma_sm1ar += sg_dma_len(sg); desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; } - - desc->num_sgs = sg_len; desc->cyclic = false; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -964,10 +1201,12 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( } /* Enable Circular mode or double buffer mode */ - if (buf_len == period_len) + if (buf_len == period_len) { chan->chan_reg.dma_scr |= STM32_DMA_SCR_CIRC; - else + } else { chan->chan_reg.dma_scr |= STM32_DMA_SCR_DBM; + chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_CT; + } /* Clear periph ctrl if client set it */ chan->chan_reg.dma_scr &= ~STM32_DMA_SCR_PFCTRL; @@ -977,6 +1216,7 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( desc = kzalloc(struct_size(desc, sg_req, num_periods), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = num_periods; for (i = 0; i < num_periods; i++) { desc->sg_req[i].len = period_len; @@ -987,11 +1227,12 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_cyclic( desc->sg_req[i].chan_reg.dma_spar = chan->chan_reg.dma_spar; desc->sg_req[i].chan_reg.dma_sm0ar = buf_addr; desc->sg_req[i].chan_reg.dma_sm1ar = buf_addr; + if (chan->trig_mdma) + desc->sg_req[i].chan_reg.dma_sm1ar += period_len; desc->sg_req[i].chan_reg.dma_sndtr = nb_data_items; - buf_addr += period_len; + if (!chan->trig_mdma) + buf_addr += period_len; } - - desc->num_sgs = num_periods; desc->cyclic = true; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -1005,13 +1246,14 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( enum dma_slave_buswidth max_width; struct stm32_dma_desc *desc; size_t xfer_count, offset; - u32 num_sgs, best_burst, dma_burst, threshold; - int i; + u32 num_sgs, best_burst, threshold; + int dma_burst, i; num_sgs = DIV_ROUND_UP(len, STM32_DMA_ALIGNED_MAX_DATA_ITEMS); desc = kzalloc(struct_size(desc, sg_req, num_sgs), GFP_NOWAIT); if (!desc) return NULL; + desc->num_sgs = num_sgs; threshold = chan->threshold; @@ -1024,26 +1266,27 @@ static struct dma_async_tx_descriptor *stm32_dma_prep_dma_memcpy( best_burst = stm32_dma_get_best_burst(len, STM32_DMA_MAX_BURST, threshold, max_width); dma_burst = stm32_dma_get_burst(chan, best_burst); + if (dma_burst < 0) { + kfree(desc); + return NULL; + } stm32_dma_clear_reg(&desc->sg_req[i].chan_reg); desc->sg_req[i].chan_reg.dma_scr = - STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_MEM) | - STM32_DMA_SCR_PBURST(dma_burst) | - STM32_DMA_SCR_MBURST(dma_burst) | + FIELD_PREP(STM32_DMA_SCR_DIR_MASK, STM32_DMA_MEM_TO_MEM) | + FIELD_PREP(STM32_DMA_SCR_PBURST_MASK, dma_burst) | + FIELD_PREP(STM32_DMA_SCR_MBURST_MASK, dma_burst) | STM32_DMA_SCR_MINC | STM32_DMA_SCR_PINC | STM32_DMA_SCR_TCIE | STM32_DMA_SCR_TEIE; desc->sg_req[i].chan_reg.dma_sfcr |= STM32_DMA_SFCR_MASK; - desc->sg_req[i].chan_reg.dma_sfcr |= - STM32_DMA_SFCR_FTH(threshold); + desc->sg_req[i].chan_reg.dma_sfcr |= FIELD_PREP(STM32_DMA_SFCR_FTH_MASK, threshold); desc->sg_req[i].chan_reg.dma_spar = src + offset; desc->sg_req[i].chan_reg.dma_sm0ar = dest + offset; desc->sg_req[i].chan_reg.dma_sndtr = xfer_count; desc->sg_req[i].len = xfer_count; } - - desc->num_sgs = num_sgs; desc->cyclic = false; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -1055,7 +1298,7 @@ static u32 stm32_dma_get_remaining_bytes(struct stm32_dma_chan *chan) struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(chan->id)); - width = STM32_DMA_SCR_PSIZE_GET(dma_scr); + width = FIELD_GET(STM32_DMA_SCR_PSIZE_MASK, dma_scr); ndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); return ndtr << width; @@ -1077,24 +1320,36 @@ static bool stm32_dma_is_current_sg(struct stm32_dma_chan *chan) { struct stm32_dma_device *dmadev = stm32_dma_get_dev(chan); struct stm32_dma_sg_req *sg_req; - u32 dma_scr, dma_smar, id; + u32 dma_scr, dma_smar, id, period_len; id = chan->id; dma_scr = stm32_dma_read(dmadev, STM32_DMA_SCR(id)); + /* In cyclic CIRC but not DBM, CT is not used */ if (!(dma_scr & STM32_DMA_SCR_DBM)) return true; sg_req = &chan->desc->sg_req[chan->next_sg]; + period_len = sg_req->len; + /* DBM - take care of a previous pause/resume not yet post reconfigured */ if (dma_scr & STM32_DMA_SCR_CT) { dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM0AR(id)); - return (dma_smar == sg_req->chan_reg.dma_sm0ar); + /* + * If transfer has been pause/resumed, + * SM0AR is in the range of [SM0AR:SM0AR+period_len] + */ + return (dma_smar >= sg_req->chan_reg.dma_sm0ar && + dma_smar < sg_req->chan_reg.dma_sm0ar + period_len); } dma_smar = stm32_dma_read(dmadev, STM32_DMA_SM1AR(id)); - - return (dma_smar == sg_req->chan_reg.dma_sm1ar); + /* + * If transfer has been pause/resumed, + * SM1AR is in the range of [SM1AR:SM1AR+period_len] + */ + return (dma_smar >= sg_req->chan_reg.dma_sm1ar && + dma_smar < sg_req->chan_reg.dma_sm1ar + period_len); } static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, @@ -1134,11 +1389,12 @@ static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, residue = stm32_dma_get_remaining_bytes(chan); - if (!stm32_dma_is_current_sg(chan)) { + if ((chan->desc->cyclic || chan->trig_mdma) && !stm32_dma_is_current_sg(chan)) { n_sg++; if (n_sg == chan->desc->num_sgs) n_sg = 0; - residue = sg_req->len; + if (!chan->trig_mdma) + residue = sg_req->len; } /* @@ -1148,7 +1404,7 @@ static size_t stm32_dma_desc_residue(struct stm32_dma_chan *chan, * residue = remaining bytes from NDTR + remaining * periods/sg to be transferred */ - if (!chan->desc->cyclic || n_sg != 0) + if ((!chan->desc->cyclic && !chan->trig_mdma) || n_sg != 0) for (i = n_sg; i < desc->num_sgs; i++) residue += desc->sg_req[i].len; @@ -1174,7 +1430,12 @@ static enum dma_status stm32_dma_tx_status(struct dma_chan *c, u32 residue = 0; status = dma_cookie_status(c, cookie, state); - if (status == DMA_COMPLETE || !state) + if (status == DMA_COMPLETE) + return status; + + status = chan->status; + + if (!state) return status; spin_lock_irqsave(&chan->vchan.lock, flags); @@ -1200,7 +1461,7 @@ static int stm32_dma_alloc_chan_resources(struct dma_chan *c) chan->config_init = false; - ret = pm_runtime_get_sync(dmadev->ddev.dev); + ret = pm_runtime_resume_and_get(dmadev->ddev.dev); if (ret < 0) return ret; @@ -1244,14 +1505,17 @@ static void stm32_dma_set_config(struct stm32_dma_chan *chan, stm32_dma_clear_reg(&chan->chan_reg); chan->chan_reg.dma_scr = cfg->stream_config & STM32_DMA_SCR_CFG_MASK; - chan->chan_reg.dma_scr |= STM32_DMA_SCR_REQ(cfg->request_line); + chan->chan_reg.dma_scr |= FIELD_PREP(STM32_DMA_SCR_REQ_MASK, cfg->request_line); /* Enable Interrupts */ chan->chan_reg.dma_scr |= STM32_DMA_SCR_TEIE | STM32_DMA_SCR_TCIE; - chan->threshold = STM32_DMA_THRESHOLD_FTR_GET(cfg->features); - if (STM32_DMA_DIRECT_MODE_GET(cfg->features)) + chan->threshold = FIELD_GET(STM32_DMA_THRESHOLD_FTR_MASK, cfg->features); + if (FIELD_GET(STM32_DMA_DIRECT_MODE_MASK, cfg->features)) chan->threshold = STM32_DMA_FIFO_THRESHOLD_NONE; + if (FIELD_GET(STM32_DMA_ALT_ACK_MODE_MASK, cfg->features)) + chan->chan_reg.dma_scr |= STM32_DMA_SCR_TRBUFF; + chan->mdma_config.stream_id = FIELD_GET(STM32_DMA_MDMA_STREAM_ID_MASK, cfg->features); } static struct dma_chan *stm32_dma_of_xlate(struct of_phandle_args *dma_spec, @@ -1303,25 +1567,17 @@ static int stm32_dma_probe(struct platform_device *pdev) struct stm32_dma_chan *chan; struct stm32_dma_device *dmadev; struct dma_device *dd; - const struct of_device_id *match; struct resource *res; struct reset_control *rst; int i, ret; - match = of_match_device(stm32_dma_of_match, &pdev->dev); - if (!match) { - dev_err(&pdev->dev, "Error: No device match found\n"); - return -ENODEV; - } - dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); if (!dmadev) return -ENOMEM; dd = &dmadev->ddev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dmadev->base = devm_ioremap_resource(&pdev->dev, res); + dmadev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(dmadev->base)) return PTR_ERR(dmadev->base); @@ -1361,6 +1617,8 @@ static int stm32_dma_probe(struct platform_device *pdev) dd->device_prep_slave_sg = stm32_dma_prep_slave_sg; dd->device_prep_dma_cyclic = stm32_dma_prep_dma_cyclic; dd->device_config = stm32_dma_slave_config; + dd->device_pause = stm32_dma_pause; + dd->device_resume = stm32_dma_resume; dd->device_terminate_all = stm32_dma_terminate_all; dd->device_synchronize = stm32_dma_synchronize; dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | @@ -1373,6 +1631,7 @@ static int stm32_dma_probe(struct platform_device *pdev) dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; dd->copy_align = DMAENGINE_ALIGN_32_BYTES; dd->max_burst = STM32_DMA_MAX_BURST; + dd->max_sg_burst = STM32_DMA_ALIGNED_MAX_DATA_ITEMS; dd->descriptor_reuse = true; dd->dev = &pdev->dev; INIT_LIST_HEAD(&dd->channels); @@ -1388,6 +1647,12 @@ static int stm32_dma_probe(struct platform_device *pdev) chan->id = i; chan->vchan.desc_free = stm32_dma_desc_free; vchan_init(&chan->vchan, dd); + + chan->mdma_config.ifcr = res->start; + chan->mdma_config.ifcr += STM32_DMA_IFCR(chan->id); + + chan->mdma_config.tcf = STM32_DMA_TCI; + chan->mdma_config.tcf <<= STM32_DMA_FLAGS_SHIFT(chan->id); } ret = dma_async_device_register(dd); @@ -1465,12 +1730,12 @@ static int stm32_dma_runtime_resume(struct device *dev) #endif #ifdef CONFIG_PM_SLEEP -static int stm32_dma_suspend(struct device *dev) +static int stm32_dma_pm_suspend(struct device *dev) { struct stm32_dma_device *dmadev = dev_get_drvdata(dev); int id, ret, scr; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) return ret; @@ -1489,14 +1754,14 @@ static int stm32_dma_suspend(struct device *dev) return 0; } -static int stm32_dma_resume(struct device *dev) +static int stm32_dma_pm_resume(struct device *dev) { return pm_runtime_force_resume(dev); } #endif static const struct dev_pm_ops stm32_dma_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_suspend, stm32_dma_resume) + SET_SYSTEM_SLEEP_PM_OPS(stm32_dma_pm_suspend, stm32_dma_pm_resume) SET_RUNTIME_PM_OPS(stm32_dma_runtime_suspend, stm32_dma_runtime_resume, NULL) }; diff --git a/drivers/dma/stm32/stm32-dma3.c b/drivers/dma/stm32/stm32-dma3.c new file mode 100644 index 000000000000..50e7106c5cb7 --- /dev/null +++ b/drivers/dma/stm32/stm32-dma3.c @@ -0,0 +1,1926 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * STM32 DMA3 controller driver + * + * Copyright (C) STMicroelectronics 2024 + * Author(s): Amelie Delaunay <amelie.delaunay@foss.st.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/dmapool.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> +#include <linux/slab.h> + +#include "../virt-dma.h" + +#define STM32_DMA3_SECCFGR 0x00 +#define STM32_DMA3_PRIVCFGR 0x04 +#define STM32_DMA3_RCFGLOCKR 0x08 +#define STM32_DMA3_MISR 0x0c +#define STM32_DMA3_SMISR 0x10 + +#define STM32_DMA3_CLBAR(x) (0x50 + 0x80 * (x)) +#define STM32_DMA3_CCIDCFGR(x) (0x54 + 0x80 * (x)) +#define STM32_DMA3_CSEMCR(x) (0x58 + 0x80 * (x)) +#define STM32_DMA3_CFCR(x) (0x5c + 0x80 * (x)) +#define STM32_DMA3_CSR(x) (0x60 + 0x80 * (x)) +#define STM32_DMA3_CCR(x) (0x64 + 0x80 * (x)) +#define STM32_DMA3_CTR1(x) (0x90 + 0x80 * (x)) +#define STM32_DMA3_CTR2(x) (0x94 + 0x80 * (x)) +#define STM32_DMA3_CBR1(x) (0x98 + 0x80 * (x)) +#define STM32_DMA3_CSAR(x) (0x9c + 0x80 * (x)) +#define STM32_DMA3_CDAR(x) (0xa0 + 0x80 * (x)) +#define STM32_DMA3_CLLR(x) (0xcc + 0x80 * (x)) + +#define STM32_DMA3_HWCFGR13 0xfc0 /* G_PER_CTRL(X) x=8..15 */ +#define STM32_DMA3_HWCFGR12 0xfc4 /* G_PER_CTRL(X) x=0..7 */ +#define STM32_DMA3_HWCFGR4 0xfe4 /* G_FIFO_SIZE(X) x=8..15 */ +#define STM32_DMA3_HWCFGR3 0xfe8 /* G_FIFO_SIZE(X) x=0..7 */ +#define STM32_DMA3_HWCFGR2 0xfec /* G_MAX_REQ_ID */ +#define STM32_DMA3_HWCFGR1 0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */ +#define STM32_DMA3_VERR 0xff4 + +/* SECCFGR DMA secure configuration register */ +#define SECCFGR_SEC(x) BIT(x) + +/* MISR DMA non-secure/secure masked interrupt status register */ +#define MISR_MIS(x) BIT(x) + +/* CxLBAR DMA channel x linked_list base address register */ +#define CLBAR_LBA GENMASK(31, 16) + +/* CxCIDCFGR DMA channel x CID register */ +#define CCIDCFGR_CFEN BIT(0) +#define CCIDCFGR_SEM_EN BIT(1) +#define CCIDCFGR_SCID GENMASK(5, 4) +#define CCIDCFGR_SEM_WLIST_CID0 BIT(16) +#define CCIDCFGR_SEM_WLIST_CID1 BIT(17) +#define CCIDCFGR_SEM_WLIST_CID2 BIT(18) + +enum ccidcfgr_cid { + CCIDCFGR_CID0, + CCIDCFGR_CID1, + CCIDCFGR_CID2, +}; + +/* CxSEMCR DMA channel x semaphore control register */ +#define CSEMCR_SEM_MUTEX BIT(0) +#define CSEMCR_SEM_CCID GENMASK(5, 4) + +/* CxFCR DMA channel x flag clear register */ +#define CFCR_TCF BIT(8) +#define CFCR_HTF BIT(9) +#define CFCR_DTEF BIT(10) +#define CFCR_ULEF BIT(11) +#define CFCR_USEF BIT(12) +#define CFCR_SUSPF BIT(13) + +/* CxSR DMA channel x status register */ +#define CSR_IDLEF BIT(0) +#define CSR_TCF BIT(8) +#define CSR_HTF BIT(9) +#define CSR_DTEF BIT(10) +#define CSR_ULEF BIT(11) +#define CSR_USEF BIT(12) +#define CSR_SUSPF BIT(13) +#define CSR_ALL_F GENMASK(13, 8) +#define CSR_FIFOL GENMASK(24, 16) + +/* CxCR DMA channel x control register */ +#define CCR_EN BIT(0) +#define CCR_RESET BIT(1) +#define CCR_SUSP BIT(2) +#define CCR_TCIE BIT(8) +#define CCR_HTIE BIT(9) +#define CCR_DTEIE BIT(10) +#define CCR_ULEIE BIT(11) +#define CCR_USEIE BIT(12) +#define CCR_SUSPIE BIT(13) +#define CCR_ALLIE GENMASK(13, 8) +#define CCR_LSM BIT(16) +#define CCR_LAP BIT(17) +#define CCR_PRIO GENMASK(23, 22) + +enum ccr_prio { + CCR_PRIO_LOW, + CCR_PRIO_MID, + CCR_PRIO_HIGH, + CCR_PRIO_VERY_HIGH, +}; + +/* CxTR1 DMA channel x transfer register 1 */ +#define CTR1_SINC BIT(3) +#define CTR1_SBL_1 GENMASK(9, 4) +#define CTR1_DINC BIT(19) +#define CTR1_DBL_1 GENMASK(25, 20) +#define CTR1_SDW_LOG2 GENMASK(1, 0) +#define CTR1_PAM GENMASK(12, 11) +#define CTR1_SAP BIT(14) +#define CTR1_DDW_LOG2 GENMASK(17, 16) +#define CTR1_DAP BIT(30) + +enum ctr1_dw { + CTR1_DW_BYTE, + CTR1_DW_HWORD, + CTR1_DW_WORD, + CTR1_DW_DWORD, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */ +}; + +enum ctr1_pam { + CTR1_PAM_0S_LT, /* if DDW > SDW, padded with 0s else left-truncated */ + CTR1_PAM_SE_RT, /* if DDW > SDW, sign extended else right-truncated */ + CTR1_PAM_PACK_UNPACK, /* FIFO queued */ +}; + +/* CxTR2 DMA channel x transfer register 2 */ +#define CTR2_REQSEL GENMASK(7, 0) +#define CTR2_SWREQ BIT(9) +#define CTR2_DREQ BIT(10) +#define CTR2_BREQ BIT(11) +#define CTR2_PFREQ BIT(12) +#define CTR2_TCEM GENMASK(31, 30) + +enum ctr2_tcem { + CTR2_TCEM_BLOCK, + CTR2_TCEM_REPEAT_BLOCK, + CTR2_TCEM_LLI, + CTR2_TCEM_CHANNEL, +}; + +/* CxBR1 DMA channel x block register 1 */ +#define CBR1_BNDT GENMASK(15, 0) + +/* CxLLR DMA channel x linked-list address register */ +#define CLLR_LA GENMASK(15, 2) +#define CLLR_ULL BIT(16) +#define CLLR_UDA BIT(27) +#define CLLR_USA BIT(28) +#define CLLR_UB1 BIT(29) +#define CLLR_UT2 BIT(30) +#define CLLR_UT1 BIT(31) + +/* HWCFGR13 DMA hardware configuration register 13 x=8..15 */ +/* HWCFGR12 DMA hardware configuration register 12 x=0..7 */ +#define G_PER_CTRL(x) (ULL(0x1) << (4 * (x))) + +/* HWCFGR4 DMA hardware configuration register 4 x=8..15 */ +/* HWCFGR3 DMA hardware configuration register 3 x=0..7 */ +#define G_FIFO_SIZE(x) (ULL(0x7) << (4 * (x))) + +#define get_chan_hwcfg(x, mask, reg) (((reg) & (mask)) >> (4 * (x))) + +/* HWCFGR2 DMA hardware configuration register 2 */ +#define G_MAX_REQ_ID GENMASK(7, 0) + +/* HWCFGR1 DMA hardware configuration register 1 */ +#define G_MASTER_PORTS GENMASK(2, 0) +#define G_NUM_CHANNELS GENMASK(12, 8) +#define G_M0_DATA_WIDTH_ENC GENMASK(25, 24) +#define G_M1_DATA_WIDTH_ENC GENMASK(29, 28) + +enum stm32_dma3_master_ports { + AXI64, /* 1x AXI: 64-bit port 0 */ + AHB32, /* 1x AHB: 32-bit port 0 */ + AHB32_AHB32, /* 2x AHB: 32-bit port 0 and 32-bit port 1 */ + AXI64_AHB32, /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */ + AXI64_AXI64, /* 2x AXI: 64-bit port 0 and 64-bit port 1 */ + AXI128_AHB32, /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */ +}; + +enum stm32_dma3_port_data_width { + DW_32, /* 32-bit, for AHB */ + DW_64, /* 64-bit, for AXI */ + DW_128, /* 128-bit, for AXI */ + DW_INVALID, +}; + +/* VERR DMA version register */ +#define VERR_MINREV GENMASK(3, 0) +#define VERR_MAJREV GENMASK(7, 4) + +/* Device tree */ +/* struct stm32_dma3_dt_conf */ +/* .ch_conf */ +#define STM32_DMA3_DT_PRIO GENMASK(1, 0) /* CCR_PRIO */ +#define STM32_DMA3_DT_FIFO GENMASK(7, 4) +/* .tr_conf */ +#define STM32_DMA3_DT_SINC BIT(0) /* CTR1_SINC */ +#define STM32_DMA3_DT_SAP BIT(1) /* CTR1_SAP */ +#define STM32_DMA3_DT_DINC BIT(4) /* CTR1_DINC */ +#define STM32_DMA3_DT_DAP BIT(5) /* CTR1_DAP */ +#define STM32_DMA3_DT_BREQ BIT(8) /* CTR2_BREQ */ +#define STM32_DMA3_DT_PFREQ BIT(9) /* CTR2_PFREQ */ +#define STM32_DMA3_DT_TCEM GENMASK(13, 12) /* CTR2_TCEM */ +#define STM32_DMA3_DT_NOPACK BIT(16) /* CTR1_PAM */ +#define STM32_DMA3_DT_NOREFACT BIT(17) + +/* struct stm32_dma3_chan .config_set bitfield */ +#define STM32_DMA3_CFG_SET_DT BIT(0) +#define STM32_DMA3_CFG_SET_DMA BIT(1) +#define STM32_DMA3_CFG_SET_BOTH (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA) + +#define STM32_DMA3_MAX_BLOCK_SIZE ALIGN_DOWN(CBR1_BNDT, 64) +#define STM32_DMA3_MAX_BURST_LEN (1 + min_t(u32, FIELD_MAX(CTR1_SBL_1), \ + FIELD_MAX(CTR1_DBL_1))) +#define port_is_ahb(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \ + ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); }) +#define port_is_axi(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \ + ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); }) +#define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) || \ + (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \ + DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES) + +/* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */ +struct stm32_dma3_hwdesc { + u32 ctr1; + u32 ctr2; + u32 cbr1; + u32 csar; + u32 cdar; + u32 cllr; +} __packed __aligned(32); + +/* + * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed + * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte + * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents + * the sg size limitation. + */ +#define STM32_DMA3_MAX_SEG_SIZE ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE) + +/* + * Linked-list items + */ +struct stm32_dma3_lli { + struct stm32_dma3_hwdesc *hwdesc; + dma_addr_t hwdesc_addr; +}; + +struct stm32_dma3_swdesc { + struct virt_dma_desc vdesc; + u32 ccr; + bool cyclic; + u32 lli_size; + struct stm32_dma3_lli lli[] __counted_by(lli_size); +}; + +struct stm32_dma3_dt_conf { + u32 ch_id; + u32 req_line; + u32 ch_conf; + u32 tr_conf; +}; + +struct stm32_dma3_chan { + struct virt_dma_chan vchan; + u32 id; + int irq; + u32 fifo_size; + u32 max_burst; + bool semaphore_mode; + struct stm32_dma3_dt_conf dt_config; + struct dma_slave_config dma_config; + u8 config_set; + struct dma_pool *lli_pool; + struct stm32_dma3_swdesc *swdesc; + enum ctr2_tcem tcem; + u32 dma_status; +}; + +struct stm32_dma3_pdata { + u32 axi_max_burst_len; +}; + +struct stm32_dma3_ddata { + struct dma_device dma_dev; + void __iomem *base; + struct clk *clk; + struct stm32_dma3_chan *chans; + u32 dma_channels; + u32 dma_requests; + enum stm32_dma3_port_data_width ports_max_dw[2]; + u32 axi_max_burst_len; +}; + +static inline struct stm32_dma3_ddata *to_stm32_dma3_ddata(struct stm32_dma3_chan *chan) +{ + return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev); +} + +static inline struct stm32_dma3_chan *to_stm32_dma3_chan(struct dma_chan *c) +{ + return container_of(c, struct stm32_dma3_chan, vchan.chan); +} + +static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct stm32_dma3_swdesc, vdesc); +} + +static struct device *chan2dev(struct stm32_dma3_chan *chan) +{ + return &chan->vchan.chan.dev->device; +} + +static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct device *dev = chan2dev(chan); + u32 id = chan->id, offset; + + offset = STM32_DMA3_SECCFGR; + dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_PRIVCFGR; + dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CCIDCFGR(id); + dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CSEMCR(id); + dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CSR(id); + dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CCR(id); + dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CTR1(id); + dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CTR2(id); + dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CBR1(id); + dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CSAR(id); + dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CDAR(id); + dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CLLR(id); + dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); + offset = STM32_DMA3_CLBAR(id); + dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset)); +} + +static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc) +{ + struct stm32_dma3_hwdesc *hwdesc; + int i; + + for (i = 0; i < swdesc->lli_size; i++) { + hwdesc = swdesc->lli[i].hwdesc; + if (i) + dev_dbg(chan2dev(chan), "V\n"); + dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr); + dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1); + dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2); + dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1); + dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar); + dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar); + dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr); + } + + if (swdesc->cyclic) { + dev_dbg(chan2dev(chan), "|\n"); + dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr); + } else { + dev_dbg(chan2dev(chan), "X\n"); + } +} + +static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan *chan, u32 count) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct stm32_dma3_swdesc *swdesc; + int i; + + /* + * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes + * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be + * addressed, so abort the allocation. + */ + if ((count * 32) > CLLR_LA) { + dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE); + return NULL; + } + + swdesc = kzalloc(struct_size(swdesc, lli, count), GFP_NOWAIT); + if (!swdesc) + return NULL; + swdesc->lli_size = count; + + for (i = 0; i < count; i++) { + swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT, + &swdesc->lli[i].hwdesc_addr); + if (!swdesc->lli[i].hwdesc) + goto err_pool_free; + } + swdesc->ccr = 0; + + /* Set LL base address */ + writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA, + ddata->base + STM32_DMA3_CLBAR(chan->id)); + + /* Set LL allocated port */ + swdesc->ccr &= ~CCR_LAP; + + return swdesc; + +err_pool_free: + dev_err(chan2dev(chan), "Failed to alloc descriptors\n"); + while (--i >= 0) + dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); + kfree(swdesc); + + return NULL; +} + +static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc) +{ + int i; + + for (i = 0; i < swdesc->lli_size; i++) + dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr); + + kfree(swdesc); +} + +static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc) +{ + struct stm32_dma3_swdesc *swdesc = to_stm32_dma3_swdesc(vdesc); + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan); + + stm32_dma3_chan_desc_free(chan, swdesc); +} + +static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct device *dev = chan2dev(chan); + u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id)); + u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); + u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id)); + u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); + u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); + u32 bndt = FIELD_GET(CBR1_BNDT, cbr1); + u32 sdw = 1 << FIELD_GET(CTR1_SDW_LOG2, ctr1); + u32 ddw = 1 << FIELD_GET(CTR1_DDW_LOG2, ctr1); + u32 sap = FIELD_GET(CTR1_SAP, ctr1); + u32 dap = FIELD_GET(CTR1_DAP, ctr1); + + if (!bndt && !FIELD_GET(CLLR_UB1, cllr)) + dev_err(dev, "null source block size and no update of this value\n"); + if (bndt % sdw) + dev_err(dev, "source block size not multiple of src data width\n"); + if (FIELD_GET(CTR1_PAM, ctr1) == CTR1_PAM_PACK_UNPACK && bndt % ddw) + dev_err(dev, "(un)packing mode w/ src block size not multiple of dst data width\n"); + if (csar % sdw) + dev_err(dev, "unaligned source address not multiple of src data width\n"); + if (cdar % ddw) + dev_err(dev, "unaligned destination address not multiple of dst data width\n"); + if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap])) + dev_err(dev, "double-word source data width not supported on port %u\n", sap); + if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap])) + dev_err(dev, "double-word destination data width not supported on port %u\n", dap); +} + +static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc, + u32 curr, dma_addr_t src, dma_addr_t dst, u32 len, + u32 ctr1, u32 ctr2, bool is_last, bool is_cyclic) +{ + struct stm32_dma3_hwdesc *hwdesc; + dma_addr_t next_lli; + u32 next = curr + 1; + + hwdesc = swdesc->lli[curr].hwdesc; + hwdesc->ctr1 = ctr1; + hwdesc->ctr2 = ctr2; + hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len); + hwdesc->csar = src; + hwdesc->cdar = dst; + + if (is_last) { + if (is_cyclic) + next_lli = swdesc->lli[0].hwdesc_addr; + else + next_lli = 0; + } else { + next_lli = swdesc->lli[next].hwdesc_addr; + } + + hwdesc->cllr = 0; + if (next_lli) { + hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1; + hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL; + hwdesc->cllr |= (next_lli & CLLR_LA); + } + + /* + * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read + * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel + * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors + * are reused (DMA_CTRL_REUSE). + */ + if (is_last) + dma_wmb(); +} + +static enum dma_slave_buswidth stm32_dma3_get_max_dw(u32 chan_max_burst, + enum stm32_dma3_port_data_width port_max_dw, + u32 len, dma_addr_t addr) +{ + enum dma_slave_buswidth max_dw = get_chan_max_dw(port_max_dw, chan_max_burst); + + /* len and addr must be a multiple of dw */ + return 1 << __ffs(len | addr | max_dw); +} + +static u32 stm32_dma3_get_max_burst(u32 len, enum dma_slave_buswidth dw, + u32 chan_max_burst, u32 bus_max_burst) +{ + u32 max_burst = chan_max_burst ? chan_max_burst / dw : 1; + + /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */ + if (len < chan_max_burst) + max_burst = len / dw; + + /* + * HW doesn't modify the burst if burst size <= half of the fifo size. + * If len is not a multiple of burst size, last burst is shortened by HW. + * Take care of maximum burst supported on interconnect bus. + */ + return min_t(u32, max_burst, bus_max_burst); +} + +static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transfer_direction dir, + u32 *ccr, u32 *ctr1, u32 *ctr2, + dma_addr_t src_addr, dma_addr_t dst_addr, u32 len) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct dma_device dma_device = ddata->dma_dev; + u32 src_max_burst = STM32_DMA3_MAX_BURST_LEN, dst_max_burst = STM32_DMA3_MAX_BURST_LEN; + u32 sdw, ddw, sbl_max, dbl_max, tcem, init_dw, init_bl_max; + u32 _ctr1 = 0, _ctr2 = 0; + u32 ch_conf = chan->dt_config.ch_conf; + u32 tr_conf = chan->dt_config.tr_conf; + u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw; + u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw; + + dev_dbg(chan2dev(chan), "%s from %pad to %pad\n", + dmaengine_get_direction_text(dir), &src_addr, &dst_addr); + + sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst); + ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst); + sbl_max = chan->dma_config.src_maxburst ? : 1; + dbl_max = chan->dma_config.dst_maxburst ? : 1; + + /* Following conditions would raise User Setting Error interrupt */ + if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) { + dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw); + return -EINVAL; + } + + if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) { + dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n"); + return -EINVAL; + } + + sap_max_dw = ddata->ports_max_dw[sap]; + dap_max_dw = ddata->ports_max_dw[dap]; + if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) || + (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) { + dev_err(chan2dev(chan), + "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n", + sdw, ddw, sap, dap); + return -EINVAL; + } + + if (FIELD_GET(STM32_DMA3_DT_SINC, tr_conf)) + _ctr1 |= CTR1_SINC; + if (sap) + _ctr1 |= CTR1_SAP; + if (port_is_axi(sap_max_dw)) /* AXI - apply axi maximum burst limitation */ + src_max_burst = ddata->axi_max_burst_len; + if (FIELD_GET(STM32_DMA3_DT_DINC, tr_conf)) + _ctr1 |= CTR1_DINC; + if (dap) + _ctr1 |= CTR1_DAP; + if (port_is_axi(dap_max_dw)) /* AXI - apply axi maximum burst limitation */ + dst_max_burst = ddata->axi_max_burst_len; + + _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ; + if (FIELD_GET(STM32_DMA3_DT_BREQ, tr_conf)) + _ctr2 |= CTR2_BREQ; + if (dir == DMA_DEV_TO_MEM && FIELD_GET(STM32_DMA3_DT_PFREQ, tr_conf)) + _ctr2 |= CTR2_PFREQ; + tcem = FIELD_GET(STM32_DMA3_DT_TCEM, tr_conf); + _ctr2 |= FIELD_PREP(CTR2_TCEM, tcem); + + /* Store TCEM to know on which event TC flag occurred */ + chan->tcem = tcem; + /* Store direction for residue computation */ + chan->dma_config.direction = dir; + + switch (dir) { + case DMA_MEM_TO_DEV: + /* Set destination (device) data width and burst */ + ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, + len, dst_addr)); + dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst, + dst_max_burst)); + + /* Set source (memory) data width and burst */ + sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); + sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst); + if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf)) { + sdw = ddw; + sbl_max = dbl_max; + } + + _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); + _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); + _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); + _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); + + if (ddw != sdw) { + _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); + /* Should never reach this case as ddw is clamped down */ + if (len & (ddw - 1)) { + dev_err(chan2dev(chan), + "Packing mode is enabled and len is not multiple of ddw"); + return -EINVAL; + } + } + + /* dst = dev */ + _ctr2 |= CTR2_DREQ; + + break; + + case DMA_DEV_TO_MEM: + /* Set source (device) data width and burst */ + sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, + len, src_addr)); + sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst, + src_max_burst)); + + /* Set destination (memory) data width and burst */ + ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); + dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst); + if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf) || + ((_ctr2 & CTR2_PFREQ) && ddw > sdw)) { /* Packing to wider ddw not supported */ + ddw = sdw; + dbl_max = sbl_max; + } + + _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); + _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); + _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); + _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); + + if (ddw != sdw) { + _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); + /* Should never reach this case as ddw is clamped down */ + if (len & (ddw - 1)) { + dev_err(chan2dev(chan), + "Packing mode is enabled and len is not multiple of ddw\n"); + return -EINVAL; + } + } + + /* dst = mem */ + _ctr2 &= ~CTR2_DREQ; + + break; + + case DMA_MEM_TO_MEM: + /* Set source (memory) data width and burst */ + init_dw = sdw; + init_bl_max = sbl_max; + sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr); + sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst); + if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { + sdw = min_t(u32, init_dw, sdw); + sbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, sdw, + chan->max_burst, + src_max_burst)); + } + + /* Set destination (memory) data width and burst */ + init_dw = ddw; + init_bl_max = dbl_max; + ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr); + dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst); + if (chan->config_set & STM32_DMA3_CFG_SET_DMA) { + ddw = min_t(u32, init_dw, ddw); + dbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, ddw, + chan->max_burst, + dst_max_burst)); + } + + _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw)); + _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1); + _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw)); + _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1); + + if (ddw != sdw) { + _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK); + /* Should never reach this case as ddw is clamped down */ + if (len & (ddw - 1)) { + dev_err(chan2dev(chan), + "Packing mode is enabled and len is not multiple of ddw"); + return -EINVAL; + } + } + + /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */ + _ctr2 |= CTR2_SWREQ; + + break; + + default: + dev_err(chan2dev(chan), "Direction %s not supported\n", + dmaengine_get_direction_text(dir)); + return -EINVAL; + } + + *ccr |= FIELD_PREP(CCR_PRIO, FIELD_GET(STM32_DMA3_DT_PRIO, ch_conf)); + *ctr1 = _ctr1; + *ctr2 = _ctr2; + + dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n", + __func__, sdw, sbl_max, ddw, dbl_max); + + return 0; +} + +static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct virt_dma_desc *vdesc; + struct stm32_dma3_hwdesc *hwdesc; + u32 id = chan->id; + u32 csr, ccr; + + vdesc = vchan_next_desc(&chan->vchan); + if (!vdesc) { + chan->swdesc = NULL; + return; + } + list_del(&vdesc->node); + + chan->swdesc = to_stm32_dma3_swdesc(vdesc); + hwdesc = chan->swdesc->lli[0].hwdesc; + + stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc); + + writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id)); + writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id)); + writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id)); + writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id)); + writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id)); + writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id)); + writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id)); + + /* Clear any pending interrupts */ + csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id)); + if (csr & CSR_ALL_F) + writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id)); + + stm32_dma3_chan_dump_reg(chan); + + ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id)); + writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id)); + + chan->dma_status = DMA_IN_PROGRESS; + + dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); +} + +static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; + int ret = 0; + + if (susp) + ccr |= CCR_SUSP; + else + ccr &= ~CCR_SUSP; + + writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id)); + + if (susp) { + ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, + csr & CSR_SUSPF, 1, 10); + if (!ret) + writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); + + stm32_dma3_chan_dump_reg(chan); + } + + return ret; +} + +static void stm32_dma3_chan_reset(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN; + + writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id)); +} + +static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc *swdesc, u32 cllr, u32 *residue) +{ + u32 i, lli_offset, next_lli_offset = cllr & CLLR_LA; + + /* If cllr is null, it means it is either the last or single item */ + if (!cllr) + return swdesc->lli_size - 1; + + /* In cyclic mode, go fast and first check we are not on the last item */ + if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA)) + return swdesc->lli_size - 1; + + /* As transfer is in progress, look backward from the last item */ + for (i = swdesc->lli_size - 1; i > 0; i--) { + *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1); + lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA; + if (lli_offset == next_lli_offset) + return i - 1; + } + + return -EINVAL; +} + +static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan, + struct stm32_dma3_swdesc *swdesc, + struct dma_tx_state *txstate) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct device *dev = chan2dev(chan); + struct stm32_dma3_hwdesc *hwdesc; + u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol; + bool pack_unpack; + int ret; + + csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); + if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) { + /* Suspend current transfer to read registers for a snapshot */ + writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id)); + ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr, + csr & (CSR_SUSPF | CSR_IDLEF), 1, 10); + + if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) { + writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); + writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); + if (ret) + dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr); + } + } + + /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */ + if (!(csr & CSR_IDLEF)) + dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr); + + /* + * If channel is not suspended, but Idle and Transfer Complete are set, + * linked-list is over, no residue + */ + if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF)) + return; + + /* Read registers to have a snapshot */ + cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id)); + cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id)); + cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id)); + + /* Resume current transfer */ + if (csr & CSR_SUSPF) { + writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id)); + writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id)); + } + + /* Add current BNDT */ + bndt = FIELD_GET(CBR1_BNDT, cbr1); + residue = bndt; + + /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */ + ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue); + if (ret < 0) { + dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n"); + return; + } + curr_lli = ret; + + /* Read current FIFO level - in units of programmed destination data width */ + hwdesc = swdesc->lli[curr_lli].hwdesc; + fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1)); + /* If the FIFO contains as many bytes as its size, it can't contain more */ + if (fifol == (1 << (chan->fifo_size + 1))) + goto skip_fifol_update; + + /* + * In case of PACKING (Destination burst length > Source burst length) or UNPACKING + * (Source burst length > Destination burst length), bytes could be pending in the FIFO + * (to be packed up to Destination burst length or unpacked into Destination burst length + * chunks). + * BNDT is not reliable, as it reflects the number of bytes read from the source but not the + * number of bytes written to the destination. + * FIFOL is also not sufficient, because it reflects the number of available write beats in + * units of Destination data width but not the bytes not yet packed or unpacked. + * In case of Destination increment DINC, it is possible to compute the number of bytes in + * the FIFO: + * fifol_in_bytes = bytes_read - bytes_written. + */ + pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK); + if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) { + int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt; + int bytes_written = cdar - hwdesc->cdar; + + if (bytes_read > 0) + fifol = bytes_read - bytes_written; + } + +skip_fifol_update: + if (fifol) { + dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol); + dma_set_in_flight_bytes(txstate, fifol); + /* + * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from + * the source memory buffer, so just need to add fifol to residue in case of + * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination + * memory buffer. + */ + if (chan->dma_config.direction == DMA_DEV_TO_MEM) + residue += fifol; + } + dma_set_residue(txstate, residue); +} + +static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 ccr; + int ret = 0; + + chan->dma_status = DMA_COMPLETE; + + /* Disable interrupts */ + ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)); + writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id)); + + if (!(ccr & CCR_SUSP) && (ccr & CCR_EN)) { + /* Suspend the channel */ + ret = stm32_dma3_chan_suspend(chan, true); + if (ret) + dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__); + } + + /* + * Reset the channel: this causes the reset of the FIFO and the reset of the channel + * internal state, the reset of CCR_EN and CCR_SUSP bits. + */ + stm32_dma3_chan_reset(chan); + + return ret; +} + +static void stm32_dma3_chan_complete(struct stm32_dma3_chan *chan) +{ + if (!chan->swdesc) + return; + + vchan_cookie_complete(&chan->swdesc->vdesc); + chan->swdesc = NULL; + stm32_dma3_chan_start(chan); +} + +static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid) +{ + struct stm32_dma3_chan *chan = devid; + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 misr, csr, ccr; + + spin_lock(&chan->vchan.lock); + + misr = readl_relaxed(ddata->base + STM32_DMA3_MISR); + if (!(misr & MISR_MIS(chan->id))) { + spin_unlock(&chan->vchan.lock); + return IRQ_NONE; + } + + csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id)); + ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE; + + if (csr & CSR_TCF && ccr & CCR_TCIE) { + if (chan->swdesc->cyclic) + vchan_cyclic_callback(&chan->swdesc->vdesc); + else + stm32_dma3_chan_complete(chan); + } + + if (csr & CSR_USEF && ccr & CCR_USEIE) { + dev_err(chan2dev(chan), "User setting error\n"); + chan->dma_status = DMA_ERROR; + /* CCR.EN automatically cleared by HW */ + stm32_dma3_check_user_setting(chan); + stm32_dma3_chan_reset(chan); + } + + if (csr & CSR_ULEF && ccr & CCR_ULEIE) { + dev_err(chan2dev(chan), "Update link transfer error\n"); + chan->dma_status = DMA_ERROR; + /* CCR.EN automatically cleared by HW */ + stm32_dma3_chan_reset(chan); + } + + if (csr & CSR_DTEF && ccr & CCR_DTEIE) { + dev_err(chan2dev(chan), "Data transfer error\n"); + chan->dma_status = DMA_ERROR; + /* CCR.EN automatically cleared by HW */ + stm32_dma3_chan_reset(chan); + } + + /* + * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set, + * ensure HTF flag to be cleared, with other flags. + */ + csr &= (ccr | CCR_HTIE); + + if (csr) + writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id)); + + spin_unlock(&chan->vchan.lock); + + return IRQ_HANDLED; +} + +static int stm32_dma3_alloc_chan_resources(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 id = chan->id, csemcr, ccid; + int ret; + + ret = pm_runtime_resume_and_get(ddata->dma_dev.dev); + if (ret < 0) + return ret; + + /* Ensure the channel is free */ + if (chan->semaphore_mode && + readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) { + ret = -EBUSY; + goto err_put_sync; + } + + chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev, + sizeof(struct stm32_dma3_hwdesc), + __alignof__(struct stm32_dma3_hwdesc), SZ_64K); + if (!chan->lli_pool) { + dev_err(chan2dev(chan), "Failed to create LLI pool\n"); + ret = -ENOMEM; + goto err_put_sync; + } + + /* Take the channel semaphore */ + if (chan->semaphore_mode) { + writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id)); + csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id)); + ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr); + /* Check that the channel is well taken */ + if (ccid != CCIDCFGR_CID1) { + dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid); + ret = -EPERM; + goto err_pool_destroy; + } + dev_dbg(chan2dev(chan), "Under CID1 control (semcr=0x%08x)\n", csemcr); + } + + return 0; + +err_pool_destroy: + dmam_pool_destroy(chan->lli_pool); + chan->lli_pool = NULL; + +err_put_sync: + pm_runtime_put_sync(ddata->dma_dev.dev); + + return ret; +} + +static void stm32_dma3_free_chan_resources(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + unsigned long flags; + + /* Ensure channel is in idle state */ + spin_lock_irqsave(&chan->vchan.lock, flags); + stm32_dma3_chan_stop(chan); + chan->swdesc = NULL; + spin_unlock_irqrestore(&chan->vchan.lock, flags); + + vchan_free_chan_resources(to_virt_chan(c)); + + dmam_pool_destroy(chan->lli_pool); + chan->lli_pool = NULL; + + /* Release the channel semaphore */ + if (chan->semaphore_mode) + writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id)); + + pm_runtime_put_sync(ddata->dma_dev.dev); + + /* Reset configuration */ + memset(&chan->dt_config, 0, sizeof(chan->dt_config)); + memset(&chan->dma_config, 0, sizeof(chan->dma_config)); + chan->config_set = 0; +} + +static u32 stm32_dma3_get_ll_count(struct stm32_dma3_chan *chan, size_t len, bool prevent_refactor) +{ + u32 count; + + if (prevent_refactor) + return DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE); + + count = len / STM32_DMA3_MAX_BLOCK_SIZE; + len -= (len / STM32_DMA3_MAX_BLOCK_SIZE) * STM32_DMA3_MAX_BLOCK_SIZE; + + if (len >= chan->max_burst) { + count += 1; /* len < STM32_DMA3_MAX_BLOCK_SIZE here, so it fits in one item */ + len -= (len / chan->max_burst) * chan->max_burst; + } + + /* Unaligned remainder fits in one extra item */ + if (len > 0) + count += 1; + + return count; +} + +static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan *chan, + dma_addr_t dst, dma_addr_t src) +{ + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */ + u32 burst = chan->max_burst / dw; + + /* Initialize dt_config if channel not pre-configured through DT */ + if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) { + chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH); + chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size); + chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC; + chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL); + } + + /* Initialize dma_config if dmaengine_slave_config() not used */ + if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) { + chan->dma_config.src_addr_width = dw; + chan->dma_config.dst_addr_width = dw; + chan->dma_config.src_maxburst = burst; + chan->dma_config.dst_maxburst = burst; + chan->dma_config.src_addr = src; + chan->dma_config.dst_addr = dst; + } +} + +static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_chan *c, + dma_addr_t dst, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc; + size_t next_size, offset; + u32 count, i, ctr1, ctr2; + bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) || + !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf); + + count = stm32_dma3_get_ll_count(chan, len, prevent_refactor); + + swdesc = stm32_dma3_chan_desc_alloc(chan, count); + if (!swdesc) + return NULL; + + if (chan->config_set != STM32_DMA3_CFG_SET_BOTH) + stm32_dma3_init_chan_config_for_memcpy(chan, dst, src); + + for (i = 0, offset = 0; offset < len; i++, offset += next_size) { + size_t remaining; + int ret; + + remaining = len - offset; + next_size = min_t(size_t, remaining, STM32_DMA3_MAX_BLOCK_SIZE); + + if (!prevent_refactor && + (next_size < STM32_DMA3_MAX_BLOCK_SIZE && next_size >= chan->max_burst)) + next_size = chan->max_burst * (remaining / chan->max_burst); + + ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, + src + offset, dst + offset, next_size); + if (ret) + goto err_desc_free; + + stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src + offset, dst + offset, next_size, + ctr1, ctr2, next_size == remaining, false); + } + + /* Enable Errors interrupts */ + swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; + /* Enable Transfer state interrupts */ + swdesc->ccr |= CCR_TCIE; + + swdesc->cyclic = false; + + return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); + +err_desc_free: + stm32_dma3_chan_desc_free(chan, swdesc); + + return NULL; +} + +static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan *c, + struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc; + struct scatterlist *sg; + size_t len; + dma_addr_t sg_addr, dev_addr, src, dst; + u32 i, j, count, ctr1, ctr2; + bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) || + !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf); + int ret; + + count = 0; + for_each_sg(sgl, sg, sg_len, i) + count += stm32_dma3_get_ll_count(chan, sg_dma_len(sg), prevent_refactor); + + swdesc = stm32_dma3_chan_desc_alloc(chan, count); + if (!swdesc) + return NULL; + + /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */ + j = 0; + for_each_sg(sgl, sg, sg_len, i) { + sg_addr = sg_dma_address(sg); + dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr : + chan->dma_config.src_addr; + len = sg_dma_len(sg); + + do { + size_t chunk = min_t(size_t, len, STM32_DMA3_MAX_BLOCK_SIZE); + + if (!prevent_refactor && + (chunk < STM32_DMA3_MAX_BLOCK_SIZE && chunk >= chan->max_burst)) + chunk = chan->max_burst * (len / chan->max_burst); + + if (dir == DMA_MEM_TO_DEV) { + src = sg_addr; + dst = dev_addr; + + ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, + src, dst, chunk); + + if (FIELD_GET(CTR1_DINC, ctr1)) + dev_addr += chunk; + } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */ + src = dev_addr; + dst = sg_addr; + + ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2, + src, dst, chunk); + + if (FIELD_GET(CTR1_SINC, ctr1)) + dev_addr += chunk; + } + + if (ret) + goto err_desc_free; + + stm32_dma3_chan_prep_hwdesc(chan, swdesc, j, src, dst, chunk, + ctr1, ctr2, j == (count - 1), false); + + sg_addr += chunk; + len -= chunk; + j++; + } while (len); + } + + if (count != sg_len && chan->tcem != CTR2_TCEM_CHANNEL) + dev_warn(chan2dev(chan), "Linked-list refactored, %d items instead of %d\n", + count, sg_len); + + /* Enable Error interrupts */ + swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; + /* Enable Transfer state interrupts */ + swdesc->ccr |= CCR_TCIE; + + swdesc->cyclic = false; + + return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); + +err_desc_free: + stm32_dma3_chan_desc_free(chan, swdesc); + + return NULL; +} + +static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_chan *c, + dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction dir, + unsigned long flags) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc; + dma_addr_t src, dst; + u32 count, i, ctr1, ctr2; + int ret; + + if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) { + dev_err(chan2dev(chan), "Invalid buffer/period length\n"); + return NULL; + } + + if (buf_len % period_len) { + dev_err(chan2dev(chan), "Buffer length not multiple of period length\n"); + return NULL; + } + + count = buf_len / period_len; + swdesc = stm32_dma3_chan_desc_alloc(chan, count); + if (!swdesc) + return NULL; + + if (dir == DMA_MEM_TO_DEV) { + src = buf_addr; + dst = chan->dma_config.dst_addr; + + ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2, + src, dst, period_len); + } else if (dir == DMA_DEV_TO_MEM) { + src = chan->dma_config.src_addr; + dst = buf_addr; + + ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2, + src, dst, period_len); + } else { + dev_err(chan2dev(chan), "Invalid direction\n"); + ret = -EINVAL; + } + + if (ret) + goto err_desc_free; + + for (i = 0; i < count; i++) { + if (dir == DMA_MEM_TO_DEV) { + src = buf_addr + i * period_len; + dst = chan->dma_config.dst_addr; + } else { /* (dir == DMA_DEV_TO_MEM) */ + src = chan->dma_config.src_addr; + dst = buf_addr + i * period_len; + } + + stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src, dst, period_len, + ctr1, ctr2, i == (count - 1), true); + } + + /* Enable Error interrupts */ + swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE; + /* Enable Transfer state interrupts */ + swdesc->ccr |= CCR_TCIE; + + swdesc->cyclic = true; + + return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags); + +err_desc_free: + stm32_dma3_chan_desc_free(chan, swdesc); + + return NULL; +} + +static void stm32_dma3_caps(struct dma_chan *c, struct dma_slave_caps *caps) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + if (!chan->fifo_size) { + caps->max_burst = 0; + caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + } else { + /* Burst transfer should not exceed half of the fifo size */ + caps->max_burst = chan->max_burst; + if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) { + caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + } + } +} + +static int stm32_dma3_config(struct dma_chan *c, struct dma_slave_config *config) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + memcpy(&chan->dma_config, config, sizeof(*config)); + chan->config_set |= STM32_DMA3_CFG_SET_DMA; + + return 0; +} + +static int stm32_dma3_pause(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + int ret; + + ret = stm32_dma3_chan_suspend(chan, true); + if (ret) + return ret; + + chan->dma_status = DMA_PAUSED; + + dev_dbg(chan2dev(chan), "vchan %p: paused\n", &chan->vchan); + + return 0; +} + +static int stm32_dma3_resume(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + stm32_dma3_chan_suspend(chan, false); + + chan->dma_status = DMA_IN_PROGRESS; + + dev_dbg(chan2dev(chan), "vchan %p: resumed\n", &chan->vchan); + + return 0; +} + +static int stm32_dma3_terminate_all(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&chan->vchan.lock, flags); + + if (chan->swdesc) { + vchan_terminate_vdesc(&chan->swdesc->vdesc); + chan->swdesc = NULL; + } + + stm32_dma3_chan_stop(chan); + + vchan_get_all_descriptors(&chan->vchan, &head); + + spin_unlock_irqrestore(&chan->vchan.lock, flags); + vchan_dma_desc_free_list(&chan->vchan, &head); + + dev_dbg(chan2dev(chan), "vchan %p: terminated\n", &chan->vchan); + + return 0; +} + +static void stm32_dma3_synchronize(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + + vchan_synchronize(&chan->vchan); +} + +static enum dma_status stm32_dma3_tx_status(struct dma_chan *c, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_swdesc *swdesc = NULL; + enum dma_status status; + unsigned long flags; + struct virt_dma_desc *vd; + + status = dma_cookie_status(c, cookie, txstate); + if (status == DMA_COMPLETE) + return status; + + if (!txstate) + return chan->dma_status; + + spin_lock_irqsave(&chan->vchan.lock, flags); + + vd = vchan_find_desc(&chan->vchan, cookie); + if (vd) + swdesc = to_stm32_dma3_swdesc(vd); + else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie) + swdesc = chan->swdesc; + + /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */ + if (swdesc) + stm32_dma3_chan_set_residue(chan, swdesc, txstate); + + spin_unlock_irqrestore(&chan->vchan.lock, flags); + + return chan->dma_status; +} + +static void stm32_dma3_issue_pending(struct dma_chan *c) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + unsigned long flags; + + spin_lock_irqsave(&chan->vchan.lock, flags); + + if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) { + dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); + stm32_dma3_chan_start(chan); + } + + spin_unlock_irqrestore(&chan->vchan.lock, flags); +} + +static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param) +{ + struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c); + struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan); + struct stm32_dma3_dt_conf *conf = fn_param; + u32 mask, semcr; + int ret; + + dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n", + __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf); + + if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask)) + if (!(mask & BIT(chan->id))) + return false; + + ret = pm_runtime_resume_and_get(ddata->dma_dev.dev); + if (ret < 0) + return false; + semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)); + pm_runtime_put_sync(ddata->dma_dev.dev); + + /* Check if chan is free */ + if (semcr & CSEMCR_SEM_MUTEX) + return false; + + /* Check if chan fifo fits well */ + if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size) + return false; + + return true; +} + +static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) +{ + struct stm32_dma3_ddata *ddata = ofdma->of_dma_data; + dma_cap_mask_t mask = ddata->dma_dev.cap_mask; + struct stm32_dma3_dt_conf conf; + struct stm32_dma3_chan *chan; + struct dma_chan *c; + + if (dma_spec->args_count < 3) { + dev_err(ddata->dma_dev.dev, "Invalid args count\n"); + return NULL; + } + + conf.req_line = dma_spec->args[0]; + conf.ch_conf = dma_spec->args[1]; + conf.tr_conf = dma_spec->args[2]; + + if (conf.req_line >= ddata->dma_requests) { + dev_err(ddata->dma_dev.dev, "Invalid request line\n"); + return NULL; + } + + /* Request dma channel among the generic dma controller list */ + c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf); + if (!c) { + dev_err(ddata->dma_dev.dev, "No suitable channel found\n"); + return NULL; + } + + chan = to_stm32_dma3_chan(c); + chan->dt_config = conf; + chan->config_set |= STM32_DMA3_CFG_SET_DT; + + return c; +} + +static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata) +{ + u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0; + + /* Reserve Secure channels */ + chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR); + + /* + * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of + * the processor which is configuring and using the given channel. + * In case CID filtering is not configured, dma-channel-mask property can be used to + * specify available DMA channels to the kernel. + */ + of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask); + + /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */ + for (i = 0; i < ddata->dma_channels; i++) { + ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i)); + + if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */ + invalid_cid |= BIT(i); + if (!(mask & BIT(i))) /* Not in dma-channel-mask */ + chan_reserved |= BIT(i); + } else { /* CID-filtered */ + if (!(ccidcfgr & CCIDCFGR_SEM_EN)) { /* Static CID mode */ + if (FIELD_GET(CCIDCFGR_SCID, ccidcfgr) != CCIDCFGR_CID1) + chan_reserved |= BIT(i); + } else { /* Semaphore mode */ + if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1, ccidcfgr)) + chan_reserved |= BIT(i); + ddata->chans[i].semaphore_mode = true; + } + } + dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i, + !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" : + ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID", + (chan_reserved & BIT(i)) ? "denied" : + mask & BIT(i) ? "force allowed" : "allowed"); + } + + if (invalid_cid) + dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n", + ddata->dma_channels, &invalid_cid); + + return chan_reserved; +} + +static struct stm32_dma3_pdata stm32mp25_pdata = { + .axi_max_burst_len = 16, +}; + +static const struct of_device_id stm32_dma3_of_match[] = { + { .compatible = "st,stm32mp25-dma3", .data = &stm32mp25_pdata, }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, stm32_dma3_of_match); + +static int stm32_dma3_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + const struct stm32_dma3_pdata *pdata; + struct stm32_dma3_ddata *ddata; + struct reset_control *reset; + struct stm32_dma3_chan *chan; + struct dma_device *dma_dev; + u32 master_ports, chan_reserved, i, verr; + u64 hwcfgr; + int ret; + + ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL); + if (!ddata) + return -ENOMEM; + platform_set_drvdata(pdev, ddata); + + dma_dev = &ddata->dma_dev; + + ddata->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ddata->base)) + return PTR_ERR(ddata->base); + + ddata->clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(ddata->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n"); + + reset = devm_reset_control_get_optional(&pdev->dev, NULL); + if (IS_ERR(reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n"); + + ret = clk_prepare_enable(ddata->clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n"); + + reset_control_reset(reset); + + INIT_LIST_HEAD(&dma_dev->channels); + + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); + dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); + dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); + dma_dev->dev = &pdev->dev; + /* + * This controller supports up to 8-byte buswidth depending on the port used and the + * channel, and can only access address at even boundaries, multiple of the buswidth. + */ + dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES; + dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES); + dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM); + + dma_dev->descriptor_reuse = true; + dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE; + dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources; + dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources; + dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy; + dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg; + dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic; + dma_dev->device_caps = stm32_dma3_caps; + dma_dev->device_config = stm32_dma3_config; + dma_dev->device_pause = stm32_dma3_pause; + dma_dev->device_resume = stm32_dma3_resume; + dma_dev->device_terminate_all = stm32_dma3_terminate_all; + dma_dev->device_synchronize = stm32_dma3_synchronize; + dma_dev->device_tx_status = stm32_dma3_tx_status; + dma_dev->device_issue_pending = stm32_dma3_issue_pending; + + /* if dma_channels is not modified, get it from hwcfgr1 */ + if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) { + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); + ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr); + } + + /* if dma_requests is not modified, get it from hwcfgr2 */ + if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) { + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2); + ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1; + } + + /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */ + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1); + master_ports = FIELD_GET(G_MASTER_PORTS, hwcfgr); + + ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr); + if (master_ports == AXI64 || master_ports == AHB32) /* Single master port */ + ddata->ports_max_dw[1] = DW_INVALID; + else /* Dual master ports */ + ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr); + + /* axi_max_burst_len is optional, if not defined, use STM32_DMA3_MAX_BURST_LEN */ + ddata->axi_max_burst_len = STM32_DMA3_MAX_BURST_LEN; + pdata = device_get_match_data(&pdev->dev); + if (pdata && pdata->axi_max_burst_len) { + ddata->axi_max_burst_len = min_t(u32, pdata->axi_max_burst_len, + STM32_DMA3_MAX_BURST_LEN); + dev_dbg(&pdev->dev, "Burst is limited to %u beats through AXI port\n", + ddata->axi_max_burst_len); + } + + ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans), + GFP_KERNEL); + if (!ddata->chans) { + ret = -ENOMEM; + goto err_clk_disable; + } + + chan_reserved = stm32_dma3_check_rif(ddata); + + if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) { + ret = -ENODEV; + dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n"); + goto err_clk_disable; + } + + /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */ + hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3); + hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32; + + for (i = 0; i < ddata->dma_channels; i++) { + if (chan_reserved & BIT(i)) + continue; + + chan = &ddata->chans[i]; + chan->id = i; + chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr); + /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */ + chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0; + } + + ret = dmaenginem_async_device_register(dma_dev); + if (ret) + goto err_clk_disable; + + for (i = 0; i < ddata->dma_channels; i++) { + char name[12]; + + if (chan_reserved & BIT(i)) + continue; + + chan = &ddata->chans[i]; + snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id); + + chan->vchan.desc_free = stm32_dma3_chan_vdesc_free; + vchan_init(&chan->vchan, dma_dev); + + ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name); + if (ret) { + dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name); + goto err_clk_disable; + } + + ret = platform_get_irq(pdev, i); + if (ret < 0) + goto err_clk_disable; + chan->irq = ret; + + ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0, + dev_name(chan2dev(chan)), chan); + if (ret) { + dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n", + dev_name(chan2dev(chan))); + goto err_clk_disable; + } + } + + ret = of_dma_controller_register(np, stm32_dma3_of_xlate, ddata); + if (ret) { + dev_err_probe(&pdev->dev, ret, "Failed to register controller\n"); + goto err_clk_disable; + } + + verr = readl_relaxed(ddata->base + STM32_DMA3_VERR); + + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_put(&pdev->dev); + + dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n", + FIELD_GET(VERR_MAJREV, verr), FIELD_GET(VERR_MINREV, verr)); + + return 0; + +err_clk_disable: + clk_disable_unprepare(ddata->clk); + + return ret; +} + +static void stm32_dma3_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); +} + +static int stm32_dma3_runtime_suspend(struct device *dev) +{ + struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); + + clk_disable_unprepare(ddata->clk); + + return 0; +} + +static int stm32_dma3_runtime_resume(struct device *dev) +{ + struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(ddata->clk); + if (ret) + dev_err(dev, "Failed to enable clk: %d\n", ret); + + return ret; +} + +static const struct dev_pm_ops stm32_dma3_pm_ops = { + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL) +}; + +static struct platform_driver stm32_dma3_driver = { + .probe = stm32_dma3_probe, + .remove = stm32_dma3_remove, + .driver = { + .name = "stm32-dma3", + .of_match_table = stm32_dma3_of_match, + .pm = pm_ptr(&stm32_dma3_pm_ops), + }, +}; + +static int __init stm32_dma3_init(void) +{ + return platform_driver_register(&stm32_dma3_driver); +} + +subsys_initcall(stm32_dma3_init); + +MODULE_DESCRIPTION("STM32 DMA3 controller driver"); +MODULE_AUTHOR("Amelie Delaunay <amelie.delaunay@foss.st.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/stm32-dmamux.c b/drivers/dma/stm32/stm32-dmamux.c index ef0d0555103d..8d77e2a7939a 100644 --- a/drivers/dma/stm32-dmamux.c +++ b/drivers/dma/stm32/stm32-dmamux.c @@ -15,8 +15,10 @@ #include <linux/err.h> #include <linux/init.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> @@ -39,13 +41,13 @@ struct stm32_dmamux_data { u32 dma_requests; /* Number of DMA requests connected to DMAMUX */ u32 dmamux_requests; /* Number of DMA requests routed toward DMAs */ spinlock_t lock; /* Protects register access */ - unsigned long *dma_inuse; /* Used DMA channel */ + DECLARE_BITMAP(dma_inuse, STM32_DMAMUX_MAX_DMA_REQUESTS); /* Used DMA channel */ u32 ccr[STM32_DMAMUX_MAX_DMA_REQUESTS]; /* Used to backup CCR register * in suspend */ u32 dma_reqs[]; /* Number of DMA Request per DMA masters. * [0] holds number of DMA Masters. - * To be kept at very end end of this structure + * To be kept at very end of this structure */ }; @@ -137,7 +139,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, /* Set dma request */ spin_lock_irqsave(&dmamux->lock, flags); - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { spin_unlock_irqrestore(&dmamux->lock, flags); goto error; @@ -147,7 +149,7 @@ static void *stm32_dmamux_route_allocate(struct of_phandle_args *dma_spec, mux->request = dma_spec->args[0]; /* craft DMA spec */ - dma_spec->args[3] = dma_spec->args[2]; + dma_spec->args[3] = dma_spec->args[2] | mux->chan_id << 16; dma_spec->args[2] = dma_spec->args[1]; dma_spec->args[1] = 0; dma_spec->args[0] = mux->chan_id - min; @@ -179,7 +181,6 @@ static int stm32_dmamux_probe(struct platform_device *pdev) const struct of_device_id *match; struct device_node *dma_node; struct stm32_dmamux_data *stm32_dmamux; - struct resource *res; void __iomem *iomem; struct reset_control *rst; int i, count, ret; @@ -229,12 +230,6 @@ static int stm32_dmamux_probe(struct platform_device *pdev) stm32_dmamux->dma_requests = dma_req; stm32_dmamux->dma_reqs[0] = count; - stm32_dmamux->dma_inuse = devm_kcalloc(&pdev->dev, - BITS_TO_LONGS(dma_req), - sizeof(unsigned long), - GFP_KERNEL); - if (!stm32_dmamux->dma_inuse) - return -ENOMEM; if (device_property_read_u32(&pdev->dev, "dma-requests", &stm32_dmamux->dmamux_requests)) { @@ -244,8 +239,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev) } pm_runtime_get_noresume(&pdev->dev); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - iomem = devm_ioremap_resource(&pdev->dev, res); + iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(iomem)) return PTR_ERR(iomem); @@ -267,7 +261,7 @@ static int stm32_dmamux_probe(struct platform_device *pdev) ret = PTR_ERR(rst); if (ret == -EPROBE_DEFER) goto err_clk; - } else { + } else if (count > 1) { /* Don't reset if there is only one dma-master */ reset_control_assert(rst); udelay(2); reset_control_deassert(rst); @@ -292,10 +286,12 @@ static int stm32_dmamux_probe(struct platform_device *pdev) ret = of_dma_router_register(node, stm32_dmamux_route_allocate, &stm32_dmamux->dmarouter); if (ret) - goto err_clk; + goto pm_disable; return 0; +pm_disable: + pm_runtime_disable(&pdev->dev); err_clk: clk_disable_unprepare(stm32_dmamux->clk); @@ -336,7 +332,7 @@ static int stm32_dmamux_suspend(struct device *dev) struct stm32_dmamux_data *stm32_dmamux = platform_get_drvdata(pdev); int i, ret; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) return ret; @@ -361,7 +357,7 @@ static int stm32_dmamux_resume(struct device *dev) if (ret < 0) return ret; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) return ret; @@ -404,4 +400,3 @@ arch_initcall(stm32_dmamux_init); MODULE_DESCRIPTION("DMA Router driver for STM32 DMA MUX"); MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>"); MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32/stm32-mdma.c index 18cbd1e43c2e..080c1c725216 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32/stm32-mdma.c @@ -10,6 +10,7 @@ * Inspired by stm32-dma.c and dma-jz4780.c */ +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/delay.h> #include <linux/dmaengine.h> @@ -23,24 +24,15 @@ #include <linux/log2.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/slab.h> -#include "virt-dma.h" - -/* MDMA Generic getter/setter */ -#define STM32_MDMA_SHIFT(n) (ffs(n) - 1) -#define STM32_MDMA_SET(n, mask) (((n) << STM32_MDMA_SHIFT(mask)) & \ - (mask)) -#define STM32_MDMA_GET(n, mask) (((n) & (mask)) >> \ - STM32_MDMA_SHIFT(mask)) +#include "../virt-dma.h" #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */ -#define STM32_MDMA_GISR1 0x0004 /* MDMA Int Status Reg 2 */ /* MDMA Channel x interrupt/status register */ #define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */ @@ -79,9 +71,9 @@ #define STM32_MDMA_CCR_WEX BIT(14) #define STM32_MDMA_CCR_HEX BIT(13) #define STM32_MDMA_CCR_BEX BIT(12) +#define STM32_MDMA_CCR_SM BIT(8) #define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6) -#define STM32_MDMA_CCR_PL(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CCR_PL_MASK) +#define STM32_MDMA_CCR_PL(n) FIELD_PREP(STM32_MDMA_CCR_PL_MASK, (n)) #define STM32_MDMA_CCR_TCIE BIT(5) #define STM32_MDMA_CCR_BTIE BIT(4) #define STM32_MDMA_CCR_BRTIE BIT(3) @@ -99,48 +91,33 @@ #define STM32_MDMA_CTCR_BWM BIT(31) #define STM32_MDMA_CTCR_SWRM BIT(30) #define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28) -#define STM32_MDMA_CTCR_TRGM(n) STM32_MDMA_SET((n), \ - STM32_MDMA_CTCR_TRGM_MSK) -#define STM32_MDMA_CTCR_TRGM_GET(n) STM32_MDMA_GET((n), \ - STM32_MDMA_CTCR_TRGM_MSK) +#define STM32_MDMA_CTCR_TRGM(n) FIELD_PREP(STM32_MDMA_CTCR_TRGM_MSK, (n)) +#define STM32_MDMA_CTCR_TRGM_GET(n) FIELD_GET(STM32_MDMA_CTCR_TRGM_MSK, (n)) #define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26) -#define STM32_MDMA_CTCR_PAM(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CTCR_PAM_MASK) +#define STM32_MDMA_CTCR_PAM(n) FIELD_PREP(STM32_MDMA_CTCR_PAM_MASK, (n)) #define STM32_MDMA_CTCR_PKE BIT(25) #define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18) -#define STM32_MDMA_CTCR_TLEN(n) STM32_MDMA_SET((n), \ - STM32_MDMA_CTCR_TLEN_MSK) -#define STM32_MDMA_CTCR_TLEN_GET(n) STM32_MDMA_GET((n), \ - STM32_MDMA_CTCR_TLEN_MSK) +#define STM32_MDMA_CTCR_TLEN(n) FIELD_PREP(STM32_MDMA_CTCR_TLEN_MSK, (n)) +#define STM32_MDMA_CTCR_TLEN_GET(n) FIELD_GET(STM32_MDMA_CTCR_TLEN_MSK, (n)) #define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18) -#define STM32_MDMA_CTCR_LEN2(n) STM32_MDMA_SET((n), \ - STM32_MDMA_CTCR_LEN2_MSK) -#define STM32_MDMA_CTCR_LEN2_GET(n) STM32_MDMA_GET((n), \ - STM32_MDMA_CTCR_LEN2_MSK) +#define STM32_MDMA_CTCR_LEN2(n) FIELD_PREP(STM32_MDMA_CTCR_LEN2_MSK, (n)) +#define STM32_MDMA_CTCR_LEN2_GET(n) FIELD_GET(STM32_MDMA_CTCR_LEN2_MSK, (n)) #define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15) -#define STM32_MDMA_CTCR_DBURST(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CTCR_DBURST_MASK) +#define STM32_MDMA_CTCR_DBURST(n) FIELD_PREP(STM32_MDMA_CTCR_DBURST_MASK, (n)) #define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12) -#define STM32_MDMA_CTCR_SBURST(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CTCR_SBURST_MASK) +#define STM32_MDMA_CTCR_SBURST(n) FIELD_PREP(STM32_MDMA_CTCR_SBURST_MASK, (n)) #define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10) -#define STM32_MDMA_CTCR_DINCOS(n) STM32_MDMA_SET((n), \ - STM32_MDMA_CTCR_DINCOS_MASK) +#define STM32_MDMA_CTCR_DINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_DINCOS_MASK, (n)) #define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8) -#define STM32_MDMA_CTCR_SINCOS(n) STM32_MDMA_SET((n), \ - STM32_MDMA_CTCR_SINCOS_MASK) +#define STM32_MDMA_CTCR_SINCOS(n) FIELD_PREP(STM32_MDMA_CTCR_SINCOS_MASK, (n)) #define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6) -#define STM32_MDMA_CTCR_DSIZE(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CTCR_DSIZE_MASK) +#define STM32_MDMA_CTCR_DSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_DSIZE_MASK, (n)) #define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4) -#define STM32_MDMA_CTCR_SSIZE(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CTCR_SSIZE_MASK) +#define STM32_MDMA_CTCR_SSIZE(n) FIELD_PREP(STM32_MDMA_CTCR_SSIZE_MASK, (n)) #define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2) -#define STM32_MDMA_CTCR_DINC(n) STM32_MDMA_SET((n), \ - STM32_MDMA_CTCR_DINC_MASK) +#define STM32_MDMA_CTCR_DINC(n) FIELD_PREP(STM32_MDMA_CTCR_DINC_MASK, (n)) #define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0) -#define STM32_MDMA_CTCR_SINC(n) STM32_MDMA_SET((n), \ - STM32_MDMA_CTCR_SINC_MASK) +#define STM32_MDMA_CTCR_SINC(n) FIELD_PREP(STM32_MDMA_CTCR_SINC_MASK, (n)) #define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \ | STM32_MDMA_CTCR_DINC_MASK \ | STM32_MDMA_CTCR_SINCOS_MASK \ @@ -151,16 +128,13 @@ /* MDMA Channel x block number of data register */ #define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x)) #define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20) -#define STM32_MDMA_CBNDTR_BRC(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CBNDTR_BRC_MK) -#define STM32_MDMA_CBNDTR_BRC_GET(n) STM32_MDMA_GET((n), \ - STM32_MDMA_CBNDTR_BRC_MK) +#define STM32_MDMA_CBNDTR_BRC(n) FIELD_PREP(STM32_MDMA_CBNDTR_BRC_MK, (n)) +#define STM32_MDMA_CBNDTR_BRC_GET(n) FIELD_GET(STM32_MDMA_CBNDTR_BRC_MK, (n)) #define STM32_MDMA_CBNDTR_BRDUM BIT(19) #define STM32_MDMA_CBNDTR_BRSUM BIT(18) #define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0) -#define STM32_MDMA_CBNDTR_BNDT(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CBNDTR_BNDT_MASK) +#define STM32_MDMA_CBNDTR_BNDT(n) FIELD_PREP(STM32_MDMA_CBNDTR_BNDT_MASK, (n)) /* MDMA Channel x source address register */ #define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x)) @@ -171,11 +145,9 @@ /* MDMA Channel x block repeat address update register */ #define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x)) #define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16) -#define STM32_MDMA_CBRUR_DUV(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CBRUR_DUV_MASK) +#define STM32_MDMA_CBRUR_DUV(n) FIELD_PREP(STM32_MDMA_CBRUR_DUV_MASK, (n)) #define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0) -#define STM32_MDMA_CBRUR_SUV(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CBRUR_SUV_MASK) +#define STM32_MDMA_CBRUR_SUV(n) FIELD_PREP(STM32_MDMA_CBRUR_SUV_MASK, (n)) /* MDMA Channel x link address register */ #define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x)) @@ -184,9 +156,8 @@ #define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x)) #define STM32_MDMA_CTBR_DBUS BIT(17) #define STM32_MDMA_CTBR_SBUS BIT(16) -#define STM32_MDMA_CTBR_TSEL_MASK GENMASK(7, 0) -#define STM32_MDMA_CTBR_TSEL(n) STM32_MDMA_SET(n, \ - STM32_MDMA_CTBR_TSEL_MASK) +#define STM32_MDMA_CTBR_TSEL_MASK GENMASK(5, 0) +#define STM32_MDMA_CTBR_TSEL(n) FIELD_PREP(STM32_MDMA_CTBR_TSEL_MASK, (n)) /* MDMA Channel x mask address register */ #define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x)) @@ -196,7 +167,7 @@ #define STM32_MDMA_MAX_BUF_LEN 128 #define STM32_MDMA_MAX_BLOCK_LEN 65536 -#define STM32_MDMA_MAX_CHANNELS 63 +#define STM32_MDMA_MAX_CHANNELS 32 #define STM32_MDMA_MAX_REQUESTS 256 #define STM32_MDMA_MAX_BURST 128 #define STM32_MDMA_VERY_HIGH_PRIORITY 0x3 @@ -227,6 +198,7 @@ struct stm32_mdma_chan_config { u32 transfer_config; u32 mask_addr; u32 mask_data; + bool m2m_hw; /* True when MDMA is triggered by STM32 DMA */ }; struct stm32_mdma_hwdesc { @@ -252,7 +224,13 @@ struct stm32_mdma_desc { u32 ccr; bool cyclic; u32 count; - struct stm32_mdma_desc_node node[]; + struct stm32_mdma_desc_node node[] __counted_by(count); +}; + +struct stm32_mdma_dma_config { + u32 request; /* STM32 DMA channel stream id, triggering MDMA */ + u32 cmar; /* STM32 DMA interrupt flag clear register address */ + u32 cmdr; /* STM32 DMA Transfer Complete flag */ }; struct stm32_mdma_chan { @@ -276,8 +254,9 @@ struct stm32_mdma_device { u32 nr_channels; u32 nr_requests; u32 nr_ahb_addr_masks; + u32 chan_reserved; struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS]; - u32 ahb_addr_masks[]; + u32 ahb_addr_masks[] __counted_by(nr_ahb_addr_masks); }; static struct stm32_mdma_device *stm32_mdma_get_dev( @@ -342,6 +321,7 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc( desc = kzalloc(struct_size(desc, node, count), GFP_NOWAIT); if (!desc) return NULL; + desc->count = count; for (i = 0; i < count; i++) { desc->node[i].hwdesc = @@ -351,8 +331,6 @@ static struct stm32_mdma_desc *stm32_mdma_alloc_desc( goto err; } - desc->count = count; - return desc; err: @@ -510,7 +488,7 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, src_maxburst = chan->dma_config.src_maxburst; dst_maxburst = chan->dma_config.dst_maxburst; - ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN; ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)); ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)); @@ -566,13 +544,23 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, dst_addr = chan->dma_config.dst_addr; /* Set device data size */ + if (chan_config->m2m_hw) + dst_addr_width = stm32_mdma_get_max_width(dst_addr, buf_len, + STM32_MDMA_MAX_BUF_LEN); dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width); if (dst_bus_width < 0) return dst_bus_width; ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK; ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width); + if (chan_config->m2m_hw) { + ctcr &= ~STM32_MDMA_CTCR_DINCOS_MASK; + ctcr |= STM32_MDMA_CTCR_DINCOS(dst_bus_width); + } /* Set device burst value */ + if (chan_config->m2m_hw) + dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width; + dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, dst_maxburst, dst_addr_width); @@ -615,13 +603,24 @@ static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan, src_addr = chan->dma_config.src_addr; /* Set device data size */ + if (chan_config->m2m_hw) + src_addr_width = stm32_mdma_get_max_width(src_addr, buf_len, + STM32_MDMA_MAX_BUF_LEN); + src_bus_width = stm32_mdma_get_width(chan, src_addr_width); if (src_bus_width < 0) return src_bus_width; ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK; ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width); + if (chan_config->m2m_hw) { + ctcr &= ~STM32_MDMA_CTCR_SINCOS_MASK; + ctcr |= STM32_MDMA_CTCR_SINCOS(src_bus_width); + } /* Set device burst value */ + if (chan_config->m2m_hw) + src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width; + src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen, src_maxburst, src_addr_width); @@ -729,11 +728,15 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, { struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); struct dma_slave_config *dma_config = &chan->dma_config; + struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct scatterlist *sg; dma_addr_t src_addr, dst_addr; - u32 ccr, ctcr, ctbr; + u32 m2m_hw_period, ccr, ctcr, ctbr; int i, ret = 0; + if (chan_config->m2m_hw) + m2m_hw_period = sg_dma_len(sgl); + for_each_sg(sgl, sg, sg_len, i) { if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) { dev_err(chan2dev(chan), "Invalid block len\n"); @@ -743,6 +746,8 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, if (direction == DMA_MEM_TO_DEV) { src_addr = sg_dma_address(sg); dst_addr = dma_config->dst_addr; + if (chan_config->m2m_hw && (i & 1)) + dst_addr += m2m_hw_period; ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr, &ctbr, src_addr, sg_dma_len(sg)); @@ -750,6 +755,8 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, src_addr); } else { src_addr = dma_config->src_addr; + if (chan_config->m2m_hw && (i & 1)) + src_addr += m2m_hw_period; dst_addr = sg_dma_address(sg); ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr, &ctbr, dst_addr, @@ -769,8 +776,6 @@ static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan, /* Enable interrupts */ ccr &= ~STM32_MDMA_CCR_IRQ_MASK; ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE; - if (sg_len > 1) - ccr |= STM32_MDMA_CCR_BTIE; desc->ccr = ccr; return 0; @@ -782,6 +787,7 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl, unsigned long flags, void *context) { struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); + struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct stm32_mdma_desc *desc; int i, ret; @@ -804,6 +810,21 @@ stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl, if (ret < 0) goto xfer_setup_err; + /* + * In case of M2M HW transfer triggered by STM32 DMA, we do not have to clear the + * transfer complete flag by hardware in order to let the CPU rearm the STM32 DMA + * with the next sg element and update some data in dmaengine framework. + */ + if (chan_config->m2m_hw && direction == DMA_MEM_TO_DEV) { + struct stm32_mdma_hwdesc *hwdesc; + + for (i = 0; i < sg_len; i++) { + hwdesc = desc->node[i].hwdesc; + hwdesc->cmar = 0; + hwdesc->cmdr = 0; + } + } + desc->cyclic = false; return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); @@ -825,6 +846,7 @@ stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr, struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); struct dma_slave_config *dma_config = &chan->dma_config; + struct stm32_mdma_chan_config *chan_config = &chan->chan_config; struct stm32_mdma_desc *desc; dma_addr_t src_addr, dst_addr; u32 ccr, ctcr, ctbr, count; @@ -885,8 +907,12 @@ stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr, if (direction == DMA_MEM_TO_DEV) { src_addr = buf_addr + i * period_len; dst_addr = dma_config->dst_addr; + if (chan_config->m2m_hw && (i & 1)) + dst_addr += period_len; } else { src_addr = dma_config->src_addr; + if (chan_config->m2m_hw && (i & 1)) + src_addr += period_len; dst_addr = buf_addr + i * period_len; } @@ -938,7 +964,7 @@ stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src, if (!desc) return NULL; - ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)); + ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & ~STM32_MDMA_CCR_EN; ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)); ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)); cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)); @@ -1161,7 +1187,7 @@ static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan) chan->busy = true; - dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: started\n", &chan->vchan); } static void stm32_mdma_issue_pending(struct dma_chan *c) @@ -1174,7 +1200,7 @@ static void stm32_mdma_issue_pending(struct dma_chan *c) if (!vchan_issue_pending(&chan->vchan)) goto end; - dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: issued\n", &chan->vchan); if (!chan->desc && !chan->busy) stm32_mdma_start_transfer(chan); @@ -1194,7 +1220,7 @@ static int stm32_mdma_pause(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); if (!ret) - dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: pause\n", &chan->vchan); return ret; } @@ -1207,6 +1233,10 @@ static int stm32_mdma_resume(struct dma_chan *c) unsigned long flags; u32 status, reg; + /* Transfer can be terminated */ + if (!chan->desc || (stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)) & STM32_MDMA_CCR_EN)) + return -EPERM; + hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc; spin_lock_irqsave(&chan->vchan.lock, flags); @@ -1231,7 +1261,7 @@ static int stm32_mdma_resume(struct dma_chan *c) spin_unlock_irqrestore(&chan->vchan.lock, flags); - dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan); + dev_dbg(chan2dev(chan), "vchan %p: resume\n", &chan->vchan); return 0; } @@ -1271,26 +1301,51 @@ static int stm32_mdma_slave_config(struct dma_chan *c, memcpy(&chan->dma_config, config, sizeof(*config)); + /* Check if user is requesting STM32 DMA to trigger MDMA */ + if (config->peripheral_size) { + struct stm32_mdma_dma_config *mdma_config; + + mdma_config = (struct stm32_mdma_dma_config *)chan->dma_config.peripheral_config; + chan->chan_config.request = mdma_config->request; + chan->chan_config.mask_addr = mdma_config->cmar; + chan->chan_config.mask_data = mdma_config->cmdr; + chan->chan_config.m2m_hw = true; + } + return 0; } static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan, struct stm32_mdma_desc *desc, - u32 curr_hwdesc) + u32 curr_hwdesc, + struct dma_tx_state *state) { struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); - struct stm32_mdma_hwdesc *hwdesc = desc->node[0].hwdesc; - u32 cbndtr, residue, modulo, burst_size; + struct stm32_mdma_hwdesc *hwdesc; + u32 cisr, clar, cbndtr, residue, modulo, burst_size; int i; + cisr = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id)); + residue = 0; - for (i = curr_hwdesc + 1; i < desc->count; i++) { + /* Get the next hw descriptor to process from current transfer */ + clar = stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)); + for (i = desc->count - 1; i >= 0; i--) { hwdesc = desc->node[i].hwdesc; + + if (hwdesc->clar == clar) + break;/* Current transfer found, stop cumulating */ + + /* Cumulate residue of unprocessed hw descriptors */ residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr); } cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)); residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK; + state->in_flight_bytes = 0; + if (chan->chan_config.m2m_hw && (cisr & STM32_MDMA_CISR_CRQA)) + state->in_flight_bytes = cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK; + if (!chan->mem_burst) return residue; @@ -1320,11 +1375,10 @@ static enum dma_status stm32_mdma_tx_status(struct dma_chan *c, vdesc = vchan_find_desc(&chan->vchan, cookie); if (chan->desc && cookie == chan->desc->vdesc.tx.cookie) - residue = stm32_mdma_desc_residue(chan, chan->desc, - chan->curr_hwdesc); + residue = stm32_mdma_desc_residue(chan, chan->desc, chan->curr_hwdesc, state); else if (vdesc) - residue = stm32_mdma_desc_residue(chan, - to_stm32_mdma_desc(vdesc), 0); + residue = stm32_mdma_desc_residue(chan, to_stm32_mdma_desc(vdesc), 0, state); + dma_set_residue(state, residue); spin_unlock_irqrestore(&chan->vchan.lock, flags); @@ -1345,32 +1399,17 @@ static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan) static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) { struct stm32_mdma_device *dmadev = devid; - struct stm32_mdma_chan *chan = devid; + struct stm32_mdma_chan *chan; u32 reg, id, ccr, ien, status; /* Find out which channel generates the interrupt */ status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0); - if (status) { - id = __ffs(status); - } else { - status = readl_relaxed(dmadev->base + STM32_MDMA_GISR1); - if (!status) { - dev_dbg(mdma2dev(dmadev), "spurious it\n"); - return IRQ_NONE; - } - id = __ffs(status); - /* - * As GISR0 provides status for channel id from 0 to 31, - * so GISR1 provides status for channel id from 32 to 62 - */ - id += 32; - } - - chan = &dmadev->chan[id]; - if (!chan) { - dev_warn(mdma2dev(dmadev), "MDMA channel not initialized\n"); + if (!status) { + dev_dbg(mdma2dev(dmadev), "spurious it\n"); return IRQ_NONE; } + id = __ffs(status); + chan = &dmadev->chan[id]; /* Handle interrupt for the channel */ spin_lock(&chan->vchan.lock); @@ -1382,9 +1421,12 @@ static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid) if (!(status & ien)) { spin_unlock(&chan->vchan.lock); - dev_warn(chan2dev(chan), - "spurious it (status=0x%04x, ien=0x%04x)\n", - status, ien); + if (chan->busy) + dev_warn(chan2dev(chan), + "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien); + else + dev_dbg(chan2dev(chan), + "spurious it (status=0x%04x, ien=0x%04x)\n", status, ien); return IRQ_NONE; } @@ -1484,10 +1526,23 @@ static void stm32_mdma_free_chan_resources(struct dma_chan *c) chan->desc_pool = NULL; } +static bool stm32_mdma_filter_fn(struct dma_chan *c, void *fn_param) +{ + struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c); + struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan); + + /* Check if chan is marked Secure */ + if (dmadev->chan_reserved & BIT(chan->id)) + return false; + + return true; +} + static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct stm32_mdma_device *dmadev = ofdma->of_dma_data; + dma_cap_mask_t mask = dmadev->ddev.cap_mask; struct stm32_mdma_chan *chan; struct dma_chan *c; struct stm32_mdma_chan_config config; @@ -1497,6 +1552,7 @@ static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec, return NULL; } + memset(&config, 0, sizeof(config)); config.request = dma_spec->args[0]; config.priority_level = dma_spec->args[1]; config.transfer_config = dma_spec->args[2]; @@ -1513,7 +1569,7 @@ static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec, return NULL; } - c = dma_get_any_slave_channel(&dmadev->ddev); + c = __dma_request_channel(&mask, stm32_mdma_filter_fn, &config, ofdma->of_node); if (!c) { dev_err(mdma2dev(dmadev), "No more channels available\n"); return NULL; @@ -1537,7 +1593,6 @@ static int stm32_mdma_probe(struct platform_device *pdev) struct stm32_mdma_device *dmadev; struct dma_device *dd; struct device_node *of_node; - struct resource *res; struct reset_control *rst; u32 nr_channels, nr_requests; int i, count, ret; @@ -1566,20 +1621,20 @@ static int stm32_mdma_probe(struct platform_device *pdev) if (count < 0) count = 0; - dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev) + sizeof(u32) * count, + dmadev = devm_kzalloc(&pdev->dev, + struct_size(dmadev, ahb_addr_masks, count), GFP_KERNEL); if (!dmadev) return -ENOMEM; + dmadev->nr_ahb_addr_masks = count; dmadev->nr_channels = nr_channels; dmadev->nr_requests = nr_requests; device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks", dmadev->ahb_addr_masks, count); - dmadev->nr_ahb_addr_masks = count; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dmadev->base = devm_ioremap_resource(&pdev->dev, res); + dmadev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dmadev->base)) return PTR_ERR(dmadev->base); @@ -1642,6 +1697,10 @@ static int stm32_mdma_probe(struct platform_device *pdev) for (i = 0; i < dmadev->nr_channels; i++) { chan = &dmadev->chan[i]; chan->id = i; + + if (stm32_mdma_read(dmadev, STM32_MDMA_CCR(i)) & STM32_MDMA_CCR_SM) + dmadev->chan_reserved |= BIT(i); + chan->vchan.desc_free = stm32_mdma_desc_free; vchan_init(&chan->vchan, dd); } @@ -1768,4 +1827,3 @@ subsys_initcall(stm32_mdma_init); MODULE_DESCRIPTION("Driver for STM32 MDMA controller"); MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>"); MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sun4i-dma.c b/drivers/dma/sun4i-dma.c index 93f1645ae928..00d2fd38d17f 100644 --- a/drivers/dma/sun4i-dma.c +++ b/drivers/dma/sun4i-dma.c @@ -7,12 +7,15 @@ #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/clk.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/dmapool.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/of_dma.h> +#include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/reset.h> #include <linux/slab.h> #include <linux/spinlock.h> @@ -30,12 +33,21 @@ #define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5) #define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type) +#define SUNIV_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 24) +#define SUNIV_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 8) + +#define SUN4I_MAX_BURST 8 +#define SUNIV_MAX_BURST 4 + /** Normal DMA register values **/ /* Normal DMA source/destination data request type values */ #define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16 #define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1) +#define SUNIV_NDMA_DRQ_TYPE_SDRAM 0x11 +#define SUNIV_NDMA_DRQ_TYPE_LIMIT (0x17 + 1) + /** Normal DMA register layout **/ /* Dedicated DMA source/destination address mode values */ @@ -49,6 +61,9 @@ #define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15) #define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6) +#define SUNIV_NDMA_CFG_CONT_MODE BIT(29) +#define SUNIV_NDMA_CFG_WAIT_STATE(n) ((n) << 26) + /** Dedicated DMA register values **/ /* Dedicated DMA source/destination address mode values */ @@ -61,6 +76,9 @@ #define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1 #define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1) +#define SUNIV_DDMA_DRQ_TYPE_SDRAM 0x1 +#define SUNIV_DDMA_DRQ_TYPE_LIMIT (0x9 + 1) + /** Dedicated DMA register layout **/ /* Dedicated DMA configuration register layout */ @@ -114,6 +132,11 @@ #define SUN4I_DMA_NR_MAX_VCHANS \ (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS) +#define SUNIV_NDMA_NR_MAX_CHANNELS 4 +#define SUNIV_DDMA_NR_MAX_CHANNELS 4 +#define SUNIV_NDMA_NR_MAX_VCHANS (24 * 2 - 1) +#define SUNIV_DDMA_NR_MAX_VCHANS 10 + /* This set of SUN4I_DDMA timing parameters were found experimentally while * working with the SPI driver and seem to make it behave correctly */ #define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \ @@ -122,6 +145,42 @@ SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \ SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2)) +/* + * Normal DMA supports individual transfers (segments) up to 128k. + * Dedicated DMA supports transfers up to 16M. We can only report + * one size limit, so we have to use the smaller value. + */ +#define SUN4I_NDMA_MAX_SEG_SIZE SZ_128K +#define SUN4I_DDMA_MAX_SEG_SIZE SZ_16M +#define SUN4I_DMA_MAX_SEG_SIZE SUN4I_NDMA_MAX_SEG_SIZE + +/* + * Hardware channels / ports representation + * + * The hardware is used in several SoCs, with differing numbers + * of channels and endpoints. This structure ties those numbers + * to a certain compatible string. + */ +struct sun4i_dma_config { + u32 ndma_nr_max_channels; + u32 ndma_nr_max_vchans; + + u32 ddma_nr_max_channels; + u32 ddma_nr_max_vchans; + + u32 dma_nr_max_channels; + + void (*set_dst_data_width)(u32 *p_cfg, s8 data_width); + void (*set_src_data_width)(u32 *p_cfg, s8 data_width); + int (*convert_burst)(u32 maxburst); + + u8 ndma_drq_sdram; + u8 ddma_drq_sdram; + + u8 max_burst; + bool has_reset; +}; + struct sun4i_dma_pchan { /* Register base of channel */ void __iomem *base; @@ -155,11 +214,12 @@ struct sun4i_dma_contract { struct virt_dma_desc vd; struct list_head demands; struct list_head completed_demands; - int is_cyclic; + bool is_cyclic : 1; + bool use_half_int : 1; }; struct sun4i_dma_dev { - DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS); + unsigned long *pchans_used; struct dma_device slave; struct sun4i_dma_pchan *pchans; struct sun4i_dma_vchan *vchans; @@ -167,6 +227,8 @@ struct sun4i_dma_dev { struct clk *clk; int irq; spinlock_t lock; + const struct sun4i_dma_config *cfg; + struct reset_control *rst; }; static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev) @@ -189,7 +251,27 @@ static struct device *chan2dev(struct dma_chan *chan) return &chan->dev->device; } -static int convert_burst(u32 maxburst) +static void set_dst_data_width_a10(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(data_width); +} + +static void set_src_data_width_a10(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(data_width); +} + +static void set_dst_data_width_f1c100s(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUNIV_DMA_CFG_DST_DATA_WIDTH(data_width); +} + +static void set_src_data_width_f1c100s(u32 *p_cfg, s8 data_width) +{ + *p_cfg |= SUNIV_DMA_CFG_SRC_DATA_WIDTH(data_width); +} + +static int convert_burst_a10(u32 maxburst) { if (maxburst > 8) return -EINVAL; @@ -198,6 +280,15 @@ static int convert_burst(u32 maxburst) return (maxburst >> 2); } +static int convert_burst_f1c100s(u32 maxburst) +{ + if (maxburst > 4) + return -EINVAL; + + /* 1 -> 0, 4 -> 1 */ + return (maxburst >> 2); +} + static int convert_buswidth(enum dma_slave_buswidth addr_width) { if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES) @@ -222,15 +313,15 @@ static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv, int i, max; /* - * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and - * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones + * pchans 0-priv->cfg->ndma_nr_max_channels are normal, and + * priv->cfg->ndma_nr_max_channels+ are dedicated ones */ if (vchan->is_dedicated) { - i = SUN4I_NDMA_NR_MAX_CHANNELS; - max = SUN4I_DMA_NR_MAX_CHANNELS; + i = priv->cfg->ndma_nr_max_channels; + max = priv->cfg->dma_nr_max_channels; } else { i = 0; - max = SUN4I_NDMA_NR_MAX_CHANNELS; + max = priv->cfg->ndma_nr_max_channels; } spin_lock_irqsave(&priv->lock, flags); @@ -372,7 +463,7 @@ static int __execute_vchan_pending(struct sun4i_dma_dev *priv, if (promise) { vchan->contract = contract; vchan->pchan = pchan; - set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1); + set_pchan_interrupt(priv, pchan, contract->use_half_int, 1); configure_pchan(pchan, promise); } @@ -433,6 +524,7 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, size_t len, struct dma_slave_config *sconfig, enum dma_transfer_direction direction) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_promise *promise; int ret; @@ -456,13 +548,13 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, sconfig->src_addr_width, sconfig->dst_addr_width); /* Source burst */ - ret = convert_burst(sconfig->src_maxburst); + ret = priv->cfg->convert_burst(sconfig->src_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ - ret = convert_burst(sconfig->dst_maxburst); + ret = priv->cfg->convert_burst(sconfig->dst_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); @@ -471,13 +563,13 @@ generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, ret = convert_buswidth(sconfig->src_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + priv->cfg->set_src_data_width(&promise->cfg, ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + priv->cfg->set_dst_data_width(&promise->cfg, ret); return promise; @@ -499,6 +591,7 @@ static struct sun4i_dma_promise * generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, size_t len, struct dma_slave_config *sconfig) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_promise *promise; int ret; @@ -513,13 +606,13 @@ generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN; /* Source burst */ - ret = convert_burst(sconfig->src_maxburst); + ret = priv->cfg->convert_burst(sconfig->src_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret); /* Destination burst */ - ret = convert_burst(sconfig->dst_maxburst); + ret = priv->cfg->convert_burst(sconfig->dst_maxburst); if (ret < 0) goto fail; promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret); @@ -528,13 +621,13 @@ generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest, ret = convert_buswidth(sconfig->src_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret); + priv->cfg->set_src_data_width(&promise->cfg, ret); /* Destination bus width */ ret = convert_buswidth(sconfig->dst_addr_width); if (ret < 0) goto fail; - promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret); + priv->cfg->set_dst_data_width(&promise->cfg, ret); return promise; @@ -611,6 +704,7 @@ static struct dma_async_tx_descriptor * sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -627,8 +721,8 @@ sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, */ sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; - sconfig->src_maxburst = 8; - sconfig->dst_maxburst = 8; + sconfig->src_maxburst = priv->cfg->max_burst; + sconfig->dst_maxburst = priv->cfg->max_burst; if (vchan->is_dedicated) promise = generate_ddma_promise(chan, src, dest, len, sconfig); @@ -643,11 +737,13 @@ sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, /* Configure memcpy mode */ if (vchan->is_dedicated) { - promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) | - SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM); + promise->cfg |= + SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ddma_drq_sdram) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ddma_drq_sdram); } else { - promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) | - SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM); + promise->cfg |= + SUN4I_DMA_CFG_SRC_DRQ_TYPE(priv->cfg->ndma_drq_sdram) | + SUN4I_DMA_CFG_DST_DRQ_TYPE(priv->cfg->ndma_drq_sdram); } /* Fill the contract with our only promise */ @@ -662,6 +758,7 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, size_t period_len, enum dma_transfer_direction dir, unsigned long flags) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -685,11 +782,11 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, if (vchan->is_dedicated) { io_mode = SUN4I_DDMA_ADDR_MODE_IO; linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ddma_drq_sdram; } else { io_mode = SUN4I_NDMA_ADDR_MODE_IO; linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ndma_drq_sdram; } if (dir == DMA_MEM_TO_DEV) { @@ -735,12 +832,21 @@ sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len, * * Which requires half the engine programming for the same * functionality. + * + * This only works if two periods fit in a single promise. That will + * always be the case for dedicated DMA, where the hardware has a much + * larger maximum transfer size than advertised to clients. */ - nr_periods = DIV_ROUND_UP(len / period_len, 2); + if (vchan->is_dedicated || period_len <= SUN4I_NDMA_MAX_SEG_SIZE / 2) { + period_len *= 2; + contract->use_half_int = 1; + } + + nr_periods = DIV_ROUND_UP(len, period_len); for (i = 0; i < nr_periods; i++) { /* Calculate the offset in the buffer and the length needed */ - offset = i * period_len * 2; - plength = min((len - offset), (period_len * 2)); + offset = i * period_len; + plength = min((len - offset), period_len); if (dir == DMA_MEM_TO_DEV) src = buf + offset; else @@ -773,6 +879,7 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { + struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device); struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan); struct dma_slave_config *sconfig = &vchan->cfg; struct sun4i_dma_promise *promise; @@ -798,11 +905,11 @@ sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (vchan->is_dedicated) { io_mode = SUN4I_DDMA_ADDR_MODE_IO; linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ddma_drq_sdram; } else { io_mode = SUN4I_NDMA_ADDR_MODE_IO; linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR; - ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM; + ram_type = priv->cfg->ndma_drq_sdram; } if (dir == DMA_MEM_TO_DEV) @@ -1124,15 +1231,17 @@ handle_pending: static int sun4i_dma_probe(struct platform_device *pdev) { struct sun4i_dma_dev *priv; - struct resource *res; int i, j, ret; priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - priv->base = devm_ioremap_resource(&pdev->dev, res); + priv->cfg = of_device_get_match_data(&pdev->dev); + if (!priv->cfg) + return -ENODEV; + + priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); @@ -1140,15 +1249,23 @@ static int sun4i_dma_probe(struct platform_device *pdev) if (priv->irq < 0) return priv->irq; - priv->clk = devm_clk_get(&pdev->dev, NULL); - if (IS_ERR(priv->clk)) { - dev_err(&pdev->dev, "No clock specified\n"); - return PTR_ERR(priv->clk); + priv->clk = devm_clk_get_enabled(&pdev->dev, NULL); + if (IS_ERR(priv->clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->clk), + "Couldn't start the clock\n"); + + if (priv->cfg->has_reset) { + priv->rst = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL); + if (IS_ERR(priv->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(priv->rst), + "Failed to get reset control\n"); } platform_set_drvdata(pdev, priv); spin_lock_init(&priv->lock); + dma_set_max_seg_size(&pdev->dev, SUN4I_DMA_MAX_SEG_SIZE); + dma_cap_zero(priv->slave.cap_mask); dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask); dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask); @@ -1177,23 +1294,26 @@ static int sun4i_dma_probe(struct platform_device *pdev) priv->slave.dev = &pdev->dev; - priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS, + priv->pchans = devm_kcalloc(&pdev->dev, priv->cfg->dma_nr_max_channels, sizeof(struct sun4i_dma_pchan), GFP_KERNEL); priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS, sizeof(struct sun4i_dma_vchan), GFP_KERNEL); - if (!priv->vchans || !priv->pchans) + priv->pchans_used = devm_kcalloc(&pdev->dev, + BITS_TO_LONGS(priv->cfg->dma_nr_max_channels), + sizeof(unsigned long), GFP_KERNEL); + if (!priv->vchans || !priv->pchans || !priv->pchans_used) return -ENOMEM; /* - * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and - * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are + * [0..priv->cfg->ndma_nr_max_channels) are normal pchans, and + * [priv->cfg->ndma_nr_max_channels..priv->cfg->dma_nr_max_channels) are * dedicated ones */ - for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++) + for (i = 0; i < priv->cfg->ndma_nr_max_channels; i++) priv->pchans[i].base = priv->base + SUN4I_NDMA_CHANNEL_REG_BASE(i); - for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) { + for (j = 0; i < priv->cfg->dma_nr_max_channels; i++, j++) { priv->pchans[i].base = priv->base + SUN4I_DDMA_CHANNEL_REG_BASE(j); priv->pchans[i].is_dedicated = 1; @@ -1207,12 +1327,6 @@ static int sun4i_dma_probe(struct platform_device *pdev) vchan_init(&vchan->vc, &priv->slave); } - ret = clk_prepare_enable(priv->clk); - if (ret) { - dev_err(&pdev->dev, "Couldn't enable the clock\n"); - return ret; - } - /* * Make sure the IRQs are all disabled and accounted for. The bootloader * likes to leave these dirty @@ -1222,36 +1336,26 @@ static int sun4i_dma_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt, 0, dev_name(&pdev->dev), priv); - if (ret) { - dev_err(&pdev->dev, "Cannot request IRQ\n"); - goto err_clk_disable; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, "Cannot request IRQ\n"); - ret = dma_async_device_register(&priv->slave); - if (ret) { - dev_warn(&pdev->dev, "Failed to register DMA engine device\n"); - goto err_clk_disable; - } + ret = dmaenginem_async_device_register(&priv->slave); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to register DMA engine device\n"); ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate, priv); - if (ret) { - dev_err(&pdev->dev, "of_dma_controller_register failed\n"); - goto err_dma_unregister; - } + if (ret) + return dev_err_probe(&pdev->dev, ret, + "Failed to register translation function\n"); dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n"); return 0; - -err_dma_unregister: - dma_async_device_unregister(&priv->slave); -err_clk_disable: - clk_disable_unprepare(priv->clk); - return ret; } -static int sun4i_dma_remove(struct platform_device *pdev) +static void sun4i_dma_remove(struct platform_device *pdev) { struct sun4i_dma_dev *priv = platform_get_drvdata(pdev); @@ -1259,22 +1363,60 @@ static int sun4i_dma_remove(struct platform_device *pdev) disable_irq(priv->irq); of_dma_controller_free(pdev->dev.of_node); - dma_async_device_unregister(&priv->slave); +} - clk_disable_unprepare(priv->clk); +static struct sun4i_dma_config sun4i_a10_dma_cfg = { + .ndma_nr_max_channels = SUN4I_NDMA_NR_MAX_CHANNELS, + .ndma_nr_max_vchans = SUN4I_NDMA_NR_MAX_VCHANS, - return 0; -} + .ddma_nr_max_channels = SUN4I_DDMA_NR_MAX_CHANNELS, + .ddma_nr_max_vchans = SUN4I_DDMA_NR_MAX_VCHANS, + + .dma_nr_max_channels = SUN4I_DMA_NR_MAX_CHANNELS, + + .set_dst_data_width = set_dst_data_width_a10, + .set_src_data_width = set_src_data_width_a10, + .convert_burst = convert_burst_a10, + + .ndma_drq_sdram = SUN4I_NDMA_DRQ_TYPE_SDRAM, + .ddma_drq_sdram = SUN4I_DDMA_DRQ_TYPE_SDRAM, + + .max_burst = SUN4I_MAX_BURST, + .has_reset = false, +}; + +static struct sun4i_dma_config suniv_f1c100s_dma_cfg = { + .ndma_nr_max_channels = SUNIV_NDMA_NR_MAX_CHANNELS, + .ndma_nr_max_vchans = SUNIV_NDMA_NR_MAX_VCHANS, + + .ddma_nr_max_channels = SUNIV_DDMA_NR_MAX_CHANNELS, + .ddma_nr_max_vchans = SUNIV_DDMA_NR_MAX_VCHANS, + + .dma_nr_max_channels = SUNIV_NDMA_NR_MAX_CHANNELS + + SUNIV_DDMA_NR_MAX_CHANNELS, + + .set_dst_data_width = set_dst_data_width_f1c100s, + .set_src_data_width = set_src_data_width_f1c100s, + .convert_burst = convert_burst_f1c100s, + + .ndma_drq_sdram = SUNIV_NDMA_DRQ_TYPE_SDRAM, + .ddma_drq_sdram = SUNIV_DDMA_DRQ_TYPE_SDRAM, + + .max_burst = SUNIV_MAX_BURST, + .has_reset = true, +}; static const struct of_device_id sun4i_dma_match[] = { - { .compatible = "allwinner,sun4i-a10-dma" }, + { .compatible = "allwinner,sun4i-a10-dma", .data = &sun4i_a10_dma_cfg }, + { .compatible = "allwinner,suniv-f1c100s-dma", + .data = &suniv_f1c100s_dma_cfg }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, sun4i_dma_match); static struct platform_driver sun4i_dma_driver = { .probe = sun4i_dma_probe, - .remove = sun4i_dma_remove, + .remove = sun4i_dma_remove, .driver = { .name = "sun4i-dma", .of_match_table = sun4i_dma_match, diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index 5cadd4d2b824..2215ff877bf7 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -9,15 +9,17 @@ #include <linux/clk.h> #include <linux/delay.h> +#include <linux/dma-mapping.h> #include <linux/dmaengine.h> #include <linux/dmapool.h> #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> #include <linux/platform_device.h> #include <linux/reset.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/types.h> #include "virt-dma.h" @@ -90,6 +92,14 @@ #define DMA_CHAN_CUR_PARA 0x1c +/* + * LLI address mangling + * + * The LLI link physical address is also mangled, but we avoid dealing + * with that by allocating LLIs from the DMA32 zone. + */ +#define SRC_HIGH_ADDR(x) (((x) & 0x3U) << 16) +#define DST_HIGH_ADDR(x) (((x) & 0x3U) << 18) /* * Various hardware related defines @@ -132,6 +142,7 @@ struct sun6i_dma_config { u32 dst_burst_lengths; u32 src_addr_widths; u32 dst_addr_widths; + bool has_high_addr; bool has_mbus_clk; }; @@ -241,9 +252,7 @@ static inline void sun6i_dma_dump_com_regs(struct sun6i_dma_dev *sdev) static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, struct sun6i_pchan *pchan) { - phys_addr_t reg = virt_to_phys(pchan->base); - - dev_dbg(sdev->slave.dev, "Chan %d reg: %pa\n" + dev_dbg(sdev->slave.dev, "Chan %d reg:\n" "\t___en(%04x): \t0x%08x\n" "\tpause(%04x): \t0x%08x\n" "\tstart(%04x): \t0x%08x\n" @@ -252,7 +261,7 @@ static inline void sun6i_dma_dump_chan_regs(struct sun6i_dma_dev *sdev, "\t__dst(%04x): \t0x%08x\n" "\tcount(%04x): \t0x%08x\n" "\t_para(%04x): \t0x%08x\n\n", - pchan->idx, ®, + pchan->idx, DMA_CHAN_ENABLE, readl(pchan->base + DMA_CHAN_ENABLE), DMA_CHAN_PAUSE, @@ -385,17 +394,16 @@ static void *sun6i_dma_lli_add(struct sun6i_dma_lli *prev, } static inline void sun6i_dma_dump_lli(struct sun6i_vchan *vchan, - struct sun6i_dma_lli *lli) + struct sun6i_dma_lli *v_lli, + dma_addr_t p_lli) { - phys_addr_t p_lli = virt_to_phys(lli); - dev_dbg(chan2dev(&vchan->vc.chan), - "\n\tdesc: p - %pa v - 0x%p\n" + "\n\tdesc:\tp - %pad v - 0x%p\n" "\t\tc - 0x%08x s - 0x%08x d - 0x%08x\n" "\t\tl - 0x%08x p - 0x%08x n - 0x%08x\n", - &p_lli, lli, - lli->cfg, lli->src, lli->dst, - lli->len, lli->para, lli->p_lli_next); + &p_lli, v_lli, + v_lli->cfg, v_lli->src, v_lli->dst, + v_lli->len, v_lli->para, v_lli->p_lli_next); } static void sun6i_dma_free_desc(struct virt_dma_desc *vd) @@ -445,7 +453,7 @@ static int sun6i_dma_start_desc(struct sun6i_vchan *vchan) pchan->desc = to_sun6i_desc(&desc->tx); pchan->done = NULL; - sun6i_dma_dump_lli(vchan, pchan->desc->v_lli); + sun6i_dma_dump_lli(vchan, pchan->desc->v_lli, pchan->desc->p_lli); irq_reg = pchan->idx / DMA_IRQ_CHAN_NR; irq_offset = pchan->idx % DMA_IRQ_CHAN_NR; @@ -546,7 +554,7 @@ static irqreturn_t sun6i_dma_interrupt(int irq, void *dev_id) continue; dev_dbg(sdev->slave.dev, "DMA irq status %s: 0x%x\n", - i ? "high" : "low", status); + str_high_low(i), status); writel(status, sdev->base + DMA_IRQ_STAT(i)); @@ -626,6 +634,18 @@ static int set_config(struct sun6i_dma_dev *sdev, return 0; } +static inline void sun6i_dma_set_addr(struct sun6i_dma_dev *sdev, + struct sun6i_dma_lli *v_lli, + dma_addr_t src, dma_addr_t dst) +{ + v_lli->src = lower_32_bits(src); + v_lli->dst = lower_32_bits(dst); + + if (sdev->cfg->has_high_addr) + v_lli->para |= SRC_HIGH_ADDR(upper_32_bits(src)) | + DST_HIGH_ADDR(upper_32_bits(dst)); +} + static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) @@ -648,16 +668,15 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( if (!txd) return NULL; - v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); + v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli); if (!v_lli) { dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); goto err_txd_free; } - v_lli->src = src; - v_lli->dst = dest; v_lli->len = len; v_lli->para = NORMAL_WAIT; + sun6i_dma_set_addr(sdev, v_lli, src, dest); burst = convert_burst(8); width = convert_buswidth(DMA_SLAVE_BUSWIDTH_4_BYTES); @@ -670,7 +689,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_memcpy( sun6i_dma_lli_add(NULL, v_lli, p_lli, txd); - sun6i_dma_dump_lli(vchan, v_lli); + sun6i_dma_dump_lli(vchan, v_lli, p_lli); return vchan_tx_prep(&vchan->vc, &txd->vd, flags); @@ -708,7 +727,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( return NULL; for_each_sg(sgl, sg, sg_len, i) { - v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); + v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli); if (!v_lli) goto err_lli_free; @@ -716,8 +735,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( v_lli->para = NORMAL_WAIT; if (dir == DMA_MEM_TO_DEV) { - v_lli->src = sg_dma_address(sg); - v_lli->dst = sconfig->dst_addr; + sun6i_dma_set_addr(sdev, v_lli, + sg_dma_address(sg), + sconfig->dst_addr); v_lli->cfg = lli_cfg; sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); @@ -729,8 +749,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( sg_dma_len(sg), flags); } else { - v_lli->src = sconfig->src_addr; - v_lli->dst = sg_dma_address(sg); + sun6i_dma_set_addr(sdev, v_lli, + sconfig->src_addr, + sg_dma_address(sg)); v_lli->cfg = lli_cfg; sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); @@ -746,14 +767,16 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_slave_sg( } dev_dbg(chan2dev(chan), "First: %pad\n", &txd->p_lli); - for (prev = txd->v_lli; prev; prev = prev->v_lli_next) - sun6i_dma_dump_lli(vchan, prev); + for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; + p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) + sun6i_dma_dump_lli(vchan, v_lli, p_lli); return vchan_tx_prep(&vchan->vc, &txd->vd, flags); err_lli_free: - for (prev = txd->v_lli; prev; prev = prev->v_lli_next) - dma_pool_free(sdev->pool, prev, virt_to_phys(prev)); + for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; + p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) + dma_pool_free(sdev->pool, v_lli, p_lli); kfree(txd); return NULL; } @@ -787,7 +810,7 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( return NULL; for (i = 0; i < periods; i++) { - v_lli = dma_pool_alloc(sdev->pool, GFP_NOWAIT, &p_lli); + v_lli = dma_pool_alloc(sdev->pool, GFP_DMA32 | GFP_NOWAIT, &p_lli); if (!v_lli) { dev_err(sdev->slave.dev, "Failed to alloc lli memory\n"); goto err_lli_free; @@ -797,14 +820,16 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( v_lli->para = NORMAL_WAIT; if (dir == DMA_MEM_TO_DEV) { - v_lli->src = buf_addr + period_len * i; - v_lli->dst = sconfig->dst_addr; + sun6i_dma_set_addr(sdev, v_lli, + buf_addr + period_len * i, + sconfig->dst_addr); v_lli->cfg = lli_cfg; sdev->cfg->set_drq(&v_lli->cfg, DRQ_SDRAM, vchan->port); sdev->cfg->set_mode(&v_lli->cfg, LINEAR_MODE, IO_MODE); } else { - v_lli->src = sconfig->src_addr; - v_lli->dst = buf_addr + period_len * i; + sun6i_dma_set_addr(sdev, v_lli, + sconfig->src_addr, + buf_addr + period_len * i); v_lli->cfg = lli_cfg; sdev->cfg->set_drq(&v_lli->cfg, vchan->port, DRQ_SDRAM); sdev->cfg->set_mode(&v_lli->cfg, IO_MODE, LINEAR_MODE); @@ -820,8 +845,9 @@ static struct dma_async_tx_descriptor *sun6i_dma_prep_dma_cyclic( return vchan_tx_prep(&vchan->vc, &txd->vd, flags); err_lli_free: - for (prev = txd->v_lli; prev; prev = prev->v_lli_next) - dma_pool_free(sdev->pool, prev, virt_to_phys(prev)); + for (p_lli = txd->p_lli, v_lli = txd->v_lli; v_lli; + p_lli = v_lli->p_lli_next, v_lli = v_lli->v_lli_next) + dma_pool_free(sdev->pool, v_lli, p_lli); kfree(txd); return NULL; } @@ -1174,8 +1200,6 @@ static struct sun6i_dma_config sun50i_a64_dma_cfg = { }; /* - * TODO: Add support for more than 4g physical addressing. - * * The A100 binding uses the number of dma channels from the * device tree node. */ @@ -1194,6 +1218,7 @@ static struct sun6i_dma_config sun50i_a100_dma_cfg = { BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | BIT(DMA_SLAVE_BUSWIDTH_8_BYTES), + .has_high_addr = true, .has_mbus_clk = true, }; @@ -1248,6 +1273,7 @@ static const struct of_device_id sun6i_dma_match[] = { { .compatible = "allwinner,sun8i-a83t-dma", .data = &sun8i_a83t_dma_cfg }, { .compatible = "allwinner,sun8i-h3-dma", .data = &sun8i_h3_dma_cfg }, { .compatible = "allwinner,sun8i-v3s-dma", .data = &sun8i_v3s_dma_cfg }, + { .compatible = "allwinner,sun20i-d1-dma", .data = &sun50i_a100_dma_cfg }, { .compatible = "allwinner,sun50i-a64-dma", .data = &sun50i_a64_dma_cfg }, { .compatible = "allwinner,sun50i-a100-dma", .data = &sun50i_a100_dma_cfg }, { .compatible = "allwinner,sun50i-h6-dma", .data = &sun50i_h6_dma_cfg }, @@ -1259,7 +1285,6 @@ static int sun6i_dma_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct sun6i_dma_dev *sdc; - struct resource *res; int ret, i; sdc = devm_kzalloc(&pdev->dev, sizeof(*sdc), GFP_KERNEL); @@ -1270,8 +1295,7 @@ static int sun6i_dma_probe(struct platform_device *pdev) if (!sdc->cfg) return -ENODEV; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - sdc->base = devm_ioremap_resource(&pdev->dev, res); + sdc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sdc->base)) return PTR_ERR(sdc->base); @@ -1310,6 +1334,8 @@ static int sun6i_dma_probe(struct platform_device *pdev) INIT_LIST_HEAD(&sdc->pending); spin_lock_init(&sdc->lock); + dma_set_max_seg_size(&pdev->dev, SZ_32M - 1); + dma_cap_set(DMA_PRIVATE, sdc->slave.cap_mask); dma_cap_set(DMA_MEMCPY, sdc->slave.cap_mask); dma_cap_set(DMA_SLAVE, sdc->slave.cap_mask); @@ -1445,7 +1471,7 @@ err_chan_free: return ret; } -static int sun6i_dma_remove(struct platform_device *pdev) +static void sun6i_dma_remove(struct platform_device *pdev) { struct sun6i_dma_dev *sdc = platform_get_drvdata(pdev); @@ -1459,8 +1485,6 @@ static int sun6i_dma_remove(struct platform_device *pdev) reset_control_assert(sdc->rstc); sun6i_dma_free(sdc); - - return 0; } static struct platform_driver sun6i_dma_driver = { diff --git a/drivers/dma/tegra186-gpc-dma.c b/drivers/dma/tegra186-gpc-dma.c new file mode 100644 index 000000000000..4d6fe0efa76e --- /dev/null +++ b/drivers/dma/tegra186-gpc-dma.c @@ -0,0 +1,1553 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * DMA driver for NVIDIA Tegra GPC DMA controller. + * + * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved. + */ + +#include <linux/bitfield.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/iommu.h> +#include <linux/iopoll.h> +#include <linux/minmax.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_dma.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/slab.h> +#include <dt-bindings/memory/tegra186-mc.h> +#include "virt-dma.h" + +/* CSR register */ +#define TEGRA_GPCDMA_CHAN_CSR 0x00 +#define TEGRA_GPCDMA_CSR_ENB BIT(31) +#define TEGRA_GPCDMA_CSR_IE_EOC BIT(30) +#define TEGRA_GPCDMA_CSR_ONCE BIT(27) + +#define TEGRA_GPCDMA_CSR_FC_MODE GENMASK(25, 24) +#define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO \ + FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0) +#define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO \ + FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1) +#define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO \ + FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2) +#define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO \ + FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3) + +#define TEGRA_GPCDMA_CSR_DMA GENMASK(23, 21) +#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC \ + FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0) +#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC \ + FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1) +#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC \ + FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2) +#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC \ + FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3) +#define TEGRA_GPCDMA_CSR_DMA_MEM2MEM \ + FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4) +#define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT \ + FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6) + +#define TEGRA_GPCDMA_CSR_REQ_SEL_MASK GENMASK(20, 16) +#define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED \ + FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4) +#define TEGRA_GPCDMA_CSR_IRQ_MASK BIT(15) +#define TEGRA_GPCDMA_CSR_WEIGHT GENMASK(13, 10) + +/* STATUS register */ +#define TEGRA_GPCDMA_CHAN_STATUS 0x004 +#define TEGRA_GPCDMA_STATUS_BUSY BIT(31) +#define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30) +#define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28) +#define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY BIT(27) +#define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE BIT(26) +#define TEGRA_GPCDMA_STATUS_CHANNEL_RX BIT(25) +#define TEGRA_GPCDMA_STATUS_CHANNEL_TX BIT(24) +#define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA BIT(23) +#define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21) +#define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20) + +#define TEGRA_GPCDMA_CHAN_CSRE 0x008 +#define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31) + +/* Source address */ +#define TEGRA_GPCDMA_CHAN_SRC_PTR 0x00C + +/* Destination address */ +#define TEGRA_GPCDMA_CHAN_DST_PTR 0x010 + +/* High address pointer */ +#define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR 0x014 +#define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR GENMASK(7, 0) +#define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR GENMASK(23, 16) + +/* MC sequence register */ +#define TEGRA_GPCDMA_CHAN_MCSEQ 0x18 +#define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31) +#define TEGRA_GPCDMA_MCSEQ_REQ_COUNT GENMASK(30, 25) +#define TEGRA_GPCDMA_MCSEQ_BURST GENMASK(24, 23) +#define TEGRA_GPCDMA_MCSEQ_BURST_2 \ + FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0) +#define TEGRA_GPCDMA_MCSEQ_BURST_16 \ + FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3) +#define TEGRA_GPCDMA_MCSEQ_WRAP1 GENMASK(22, 20) +#define TEGRA_GPCDMA_MCSEQ_WRAP0 GENMASK(19, 17) +#define TEGRA_GPCDMA_MCSEQ_WRAP_NONE 0 + +#define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK GENMASK(13, 7) +#define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK GENMASK(6, 0) + +/* MMIO sequence register */ +#define TEGRA_GPCDMA_CHAN_MMIOSEQ 0x01c +#define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31) +#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH GENMASK(30, 28) +#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 \ + FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0) +#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16 \ + FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1) +#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32 \ + FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2) +#define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP BIT(27) +#define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT 23 +#define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN 2U +#define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX 32U +#define TEGRA_GPCDMA_MMIOSEQ_BURST(bs) \ + (GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT) +#define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID GENMASK(22, 19) +#define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD GENMASK(18, 16) +#define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT GENMASK(8, 7) + +/* Channel WCOUNT */ +#define TEGRA_GPCDMA_CHAN_WCOUNT 0x20 + +/* Transfer count */ +#define TEGRA_GPCDMA_CHAN_XFER_COUNT 0x24 + +/* DMA byte count status */ +#define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS 0x28 + +/* Error Status Register */ +#define TEGRA_GPCDMA_CHAN_ERR_STATUS 0x30 +#define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT 8 +#define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK 0xF +#define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ( \ + ((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) & \ + TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK) +#define TEGRA_DMA_BM_FIFO_FULL_ERR 0xF +#define TEGRA_DMA_PERIPH_FIFO_FULL_ERR 0xE +#define TEGRA_DMA_PERIPH_ID_ERR 0xD +#define TEGRA_DMA_STREAM_ID_ERR 0xC +#define TEGRA_DMA_MC_SLAVE_ERR 0xB +#define TEGRA_DMA_MMIO_SLAVE_ERR 0xA + +/* Fixed Pattern */ +#define TEGRA_GPCDMA_CHAN_FIXED_PATTERN 0x34 + +#define TEGRA_GPCDMA_CHAN_TZ 0x38 +#define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1 BIT(0) +#define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1 BIT(1) + +#define TEGRA_GPCDMA_CHAN_SPARE 0x3c +#define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC BIT(16) + +/* + * If any burst is in flight and DMA paused then this is the time to complete + * on-flight burst and update DMA status register. + */ +#define TEGRA_GPCDMA_BURST_COMPLETE_TIME 10 +#define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT 5000 /* 5 msec */ + +/* Channel base address offset from GPCDMA base address */ +#define TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET 0x10000 + +/* Default channel mask reserving channel0 */ +#define TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK 0xfffffffe + +struct tegra_dma; +struct tegra_dma_channel; + +/* + * tegra_dma_chip_data Tegra chip specific DMA data + * @nr_channels: Number of channels available in the controller. + * @channel_reg_size: Channel register size. + * @max_dma_count: Maximum DMA transfer count supported by DMA controller. + * @hw_support_pause: DMA HW engine support pause of the channel. + */ +struct tegra_dma_chip_data { + bool hw_support_pause; + unsigned int nr_channels; + unsigned int channel_reg_size; + unsigned int max_dma_count; + int (*terminate)(struct tegra_dma_channel *tdc); +}; + +/* DMA channel registers */ +struct tegra_dma_channel_regs { + u32 csr; + u32 src_ptr; + u32 dst_ptr; + u32 high_addr_ptr; + u32 mc_seq; + u32 mmio_seq; + u32 wcount; + u32 fixed_pattern; +}; + +/* + * tegra_dma_sg_req: DMA request details to configure hardware. This + * contains the details for one transfer to configure DMA hw. + * The client's request for data transfer can be broken into multiple + * sub-transfer as per requester details and hw support. This sub transfer + * get added as an array in Tegra DMA desc which manages the transfer details. + */ +struct tegra_dma_sg_req { + unsigned int len; + struct tegra_dma_channel_regs ch_regs; +}; + +/* + * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to + * manage client request and keep track of transfer status, callbacks + * and request counts etc. + */ +struct tegra_dma_desc { + bool cyclic; + unsigned int bytes_req; + unsigned int bytes_xfer; + unsigned int sg_idx; + unsigned int sg_count; + struct virt_dma_desc vd; + struct tegra_dma_channel *tdc; + struct tegra_dma_sg_req sg_req[] __counted_by(sg_count); +}; + +/* + * tegra_dma_channel: Channel specific information + */ +struct tegra_dma_channel { + bool config_init; + char name[30]; + enum dma_transfer_direction sid_dir; + enum dma_status status; + int id; + int irq; + int slave_id; + struct tegra_dma *tdma; + struct virt_dma_chan vc; + struct tegra_dma_desc *dma_desc; + struct dma_slave_config dma_sconfig; + unsigned int stream_id; + unsigned long chan_base_offset; +}; + +/* + * tegra_dma: Tegra DMA specific information + */ +struct tegra_dma { + const struct tegra_dma_chip_data *chip_data; + unsigned long sid_m2d_reserved; + unsigned long sid_d2m_reserved; + u32 chan_mask; + void __iomem *base_addr; + struct device *dev; + struct dma_device dma_dev; + struct reset_control *rst; + struct tegra_dma_channel channels[]; +}; + +static inline void tdc_write(struct tegra_dma_channel *tdc, + u32 reg, u32 val) +{ + writel_relaxed(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg); +} + +static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) +{ + return readl_relaxed(tdc->tdma->base_addr + tdc->chan_base_offset + reg); +} + +static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) +{ + return container_of(dc, struct tegra_dma_channel, vc.chan); +} + +static inline struct tegra_dma_desc *vd_to_tegra_dma_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct tegra_dma_desc, vd); +} + +static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) +{ + return tdc->vc.chan.device->dev; +} + +static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc) +{ + dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n", + tdc->id, tdc->name); + dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n", + tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR) + ); + dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n", + tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT), + tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS) + ); + dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n", + tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS)); +} + +static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc, + enum dma_transfer_direction direction) +{ + struct tegra_dma *tdma = tdc->tdma; + int sid = tdc->slave_id; + + if (!is_slave_direction(direction)) + return 0; + + switch (direction) { + case DMA_MEM_TO_DEV: + if (test_and_set_bit(sid, &tdma->sid_m2d_reserved)) { + dev_err(tdma->dev, "slave id already in use\n"); + return -EINVAL; + } + break; + case DMA_DEV_TO_MEM: + if (test_and_set_bit(sid, &tdma->sid_d2m_reserved)) { + dev_err(tdma->dev, "slave id already in use\n"); + return -EINVAL; + } + break; + default: + break; + } + + tdc->sid_dir = direction; + + return 0; +} + +static void tegra_dma_sid_free(struct tegra_dma_channel *tdc) +{ + struct tegra_dma *tdma = tdc->tdma; + int sid = tdc->slave_id; + + switch (tdc->sid_dir) { + case DMA_MEM_TO_DEV: + clear_bit(sid, &tdma->sid_m2d_reserved); + break; + case DMA_DEV_TO_MEM: + clear_bit(sid, &tdma->sid_d2m_reserved); + break; + default: + break; + } + + tdc->sid_dir = DMA_TRANS_NONE; +} + +static void tegra_dma_desc_free(struct virt_dma_desc *vd) +{ + kfree(container_of(vd, struct tegra_dma_desc, vd)); +} + +static int tegra_dma_slave_config(struct dma_chan *dc, + struct dma_slave_config *sconfig) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + + memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); + tdc->config_init = true; + + return 0; +} + +static int tegra_dma_pause(struct tegra_dma_channel *tdc) +{ + int ret; + u32 val; + + val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE); + val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE; + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val); + + /* Wait until busy bit is de-asserted */ + ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + + tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS, + val, + !(val & TEGRA_GPCDMA_STATUS_BUSY), + TEGRA_GPCDMA_BURST_COMPLETE_TIME, + TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT); + + if (ret) { + dev_err(tdc2dev(tdc), "DMA pause timed out\n"); + tegra_dma_dump_chan_regs(tdc); + } + + tdc->status = DMA_PAUSED; + + return ret; +} + +static int tegra_dma_device_pause(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + unsigned long flags; + int ret; + + if (!tdc->tdma->chip_data->hw_support_pause) + return -ENOSYS; + + spin_lock_irqsave(&tdc->vc.lock, flags); + ret = tegra_dma_pause(tdc); + spin_unlock_irqrestore(&tdc->vc.lock, flags); + + return ret; +} + +static void tegra_dma_resume(struct tegra_dma_channel *tdc) +{ + u32 val; + + val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE); + val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE; + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val); + + tdc->status = DMA_IN_PROGRESS; +} + +static int tegra_dma_device_resume(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + unsigned long flags; + + if (!tdc->tdma->chip_data->hw_support_pause) + return -ENOSYS; + + spin_lock_irqsave(&tdc->vc.lock, flags); + tegra_dma_resume(tdc); + spin_unlock_irqrestore(&tdc->vc.lock, flags); + + return 0; +} + +static inline int tegra_dma_pause_noerr(struct tegra_dma_channel *tdc) +{ + /* Return 0 irrespective of PAUSE status. + * This is useful to recover channels that can exit out of flush + * state when the channel is disabled. + */ + + tegra_dma_pause(tdc); + return 0; +} + +static void tegra_dma_disable(struct tegra_dma_channel *tdc) +{ + u32 csr, status; + + csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR); + + /* Disable interrupts */ + csr &= ~TEGRA_GPCDMA_CSR_IE_EOC; + + /* Disable DMA */ + csr &= ~TEGRA_GPCDMA_CSR_ENB; + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr); + + /* Clear interrupt status if it is there */ + status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); + if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) { + dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status); + } +} + +static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc) +{ + struct tegra_dma_desc *dma_desc = tdc->dma_desc; + struct tegra_dma_channel_regs *ch_regs; + int ret; + u32 val; + + dma_desc->sg_idx++; + + /* Reset the sg index for cyclic transfers */ + if (dma_desc->sg_idx == dma_desc->sg_count) + dma_desc->sg_idx = 0; + + /* Configure next transfer immediately after DMA is busy */ + ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + + tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS, + val, + (val & TEGRA_GPCDMA_STATUS_BUSY), 0, + TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT); + if (ret) + return; + + ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs; + + tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr); + + /* Start DMA */ + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, + ch_regs->csr | TEGRA_GPCDMA_CSR_ENB); +} + +static void tegra_dma_start(struct tegra_dma_channel *tdc) +{ + struct tegra_dma_desc *dma_desc = tdc->dma_desc; + struct tegra_dma_channel_regs *ch_regs; + struct virt_dma_desc *vdesc; + + if (!dma_desc) { + vdesc = vchan_next_desc(&tdc->vc); + if (!vdesc) + return; + + dma_desc = vd_to_tegra_dma_desc(vdesc); + list_del(&vdesc->node); + dma_desc->tdc = tdc; + tdc->dma_desc = dma_desc; + + tegra_dma_resume(tdc); + } + + ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs; + + tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr); + + /* Start DMA */ + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, + ch_regs->csr | TEGRA_GPCDMA_CSR_ENB); +} + +static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc) +{ + vchan_cookie_complete(&tdc->dma_desc->vd); + + tegra_dma_sid_free(tdc); + tdc->dma_desc = NULL; + tdc->status = DMA_COMPLETE; +} + +static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc, + unsigned int err_status) +{ + switch (TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) { + case TEGRA_DMA_BM_FIFO_FULL_ERR: + dev_err(tdc->tdma->dev, + "GPCDMA CH%d bm fifo full\n", tdc->id); + break; + + case TEGRA_DMA_PERIPH_FIFO_FULL_ERR: + dev_err(tdc->tdma->dev, + "GPCDMA CH%d peripheral fifo full\n", tdc->id); + break; + + case TEGRA_DMA_PERIPH_ID_ERR: + dev_err(tdc->tdma->dev, + "GPCDMA CH%d illegal peripheral id\n", tdc->id); + break; + + case TEGRA_DMA_STREAM_ID_ERR: + dev_err(tdc->tdma->dev, + "GPCDMA CH%d illegal stream id\n", tdc->id); + break; + + case TEGRA_DMA_MC_SLAVE_ERR: + dev_err(tdc->tdma->dev, + "GPCDMA CH%d mc slave error\n", tdc->id); + break; + + case TEGRA_DMA_MMIO_SLAVE_ERR: + dev_err(tdc->tdma->dev, + "GPCDMA CH%d mmio slave error\n", tdc->id); + break; + + default: + dev_err(tdc->tdma->dev, + "GPCDMA CH%d security violation %x\n", tdc->id, + err_status); + } +} + +static irqreturn_t tegra_dma_isr(int irq, void *dev_id) +{ + struct tegra_dma_channel *tdc = dev_id; + struct tegra_dma_desc *dma_desc = tdc->dma_desc; + struct tegra_dma_sg_req *sg_req; + u32 status; + + /* Check channel error status register */ + status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS); + if (status) { + tegra_dma_chan_decode_error(tdc, status); + tegra_dma_dump_chan_regs(tdc); + tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF); + } + + spin_lock(&tdc->vc.lock); + status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); + if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC)) + goto irq_done; + + tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, + TEGRA_GPCDMA_STATUS_ISE_EOC); + + if (!dma_desc) + goto irq_done; + + sg_req = dma_desc->sg_req; + dma_desc->bytes_xfer += sg_req[dma_desc->sg_idx].len; + + if (dma_desc->cyclic) { + vchan_cyclic_callback(&dma_desc->vd); + tegra_dma_configure_next_sg(tdc); + } else { + dma_desc->sg_idx++; + if (dma_desc->sg_idx == dma_desc->sg_count) + tegra_dma_xfer_complete(tdc); + else + tegra_dma_start(tdc); + } + +irq_done: + spin_unlock(&tdc->vc.lock); + return IRQ_HANDLED; +} + +static void tegra_dma_issue_pending(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + unsigned long flags; + + if (tdc->dma_desc) + return; + + spin_lock_irqsave(&tdc->vc.lock, flags); + if (vchan_issue_pending(&tdc->vc)) + tegra_dma_start(tdc); + + /* + * For cyclic DMA transfers, program the second + * transfer parameters as soon as the first DMA + * transfer is started inorder for the DMA + * controller to trigger the second transfer + * with the correct parameters. + */ + if (tdc->dma_desc && tdc->dma_desc->cyclic) + tegra_dma_configure_next_sg(tdc); + + spin_unlock_irqrestore(&tdc->vc.lock, flags); +} + +static int tegra_dma_stop_client(struct tegra_dma_channel *tdc) +{ + int ret; + u32 status, csr; + + /* + * Change the client associated with the DMA channel + * to stop DMA engine from starting any more bursts for + * the given client and wait for in flight bursts to complete + */ + csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR); + csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK); + csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED; + tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr); + + /* Wait for in flight data transfer to finish */ + udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME); + + /* If TX/RX path is still active wait till it becomes + * inactive + */ + + ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + + tdc->chan_base_offset + + TEGRA_GPCDMA_CHAN_STATUS, + status, + !(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX | + TEGRA_GPCDMA_STATUS_CHANNEL_RX)), + 5, + TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT); + if (ret) { + dev_err(tdc2dev(tdc), "Timeout waiting for DMA burst completion!\n"); + tegra_dma_dump_chan_regs(tdc); + } + + return ret; +} + +static int tegra_dma_terminate_all(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + unsigned long flags; + LIST_HEAD(head); + int err; + + spin_lock_irqsave(&tdc->vc.lock, flags); + + if (tdc->dma_desc) { + err = tdc->tdma->chip_data->terminate(tdc); + if (err) { + spin_unlock_irqrestore(&tdc->vc.lock, flags); + return err; + } + + vchan_terminate_vdesc(&tdc->dma_desc->vd); + tegra_dma_disable(tdc); + tdc->dma_desc = NULL; + } + + tdc->status = DMA_COMPLETE; + tegra_dma_sid_free(tdc); + vchan_get_all_descriptors(&tdc->vc, &head); + spin_unlock_irqrestore(&tdc->vc.lock, flags); + + vchan_dma_desc_free_list(&tdc->vc, &head); + + return 0; +} + +static int tegra_dma_get_residual(struct tegra_dma_channel *tdc) +{ + struct tegra_dma_desc *dma_desc = tdc->dma_desc; + struct tegra_dma_sg_req *sg_req = dma_desc->sg_req; + unsigned int bytes_xfer, residual; + u32 wcount = 0, status; + + wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT); + + /* + * Set wcount = 0 if EOC bit is set. The transfer would have + * already completed and the CHAN_XFER_COUNT could have updated + * for the next transfer, specifically in case of cyclic transfers. + */ + status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); + if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) + wcount = 0; + + bytes_xfer = dma_desc->bytes_xfer + + sg_req[dma_desc->sg_idx].len - (wcount * 4); + + if (dma_desc->bytes_req == bytes_xfer) + return 0; + + residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req); + + return residual; +} + +static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + struct tegra_dma_desc *dma_desc; + struct virt_dma_desc *vd; + unsigned int residual; + unsigned long flags; + enum dma_status ret; + + ret = dma_cookie_status(dc, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + if (tdc->status == DMA_PAUSED) + ret = DMA_PAUSED; + + spin_lock_irqsave(&tdc->vc.lock, flags); + vd = vchan_find_desc(&tdc->vc, cookie); + if (vd) { + dma_desc = vd_to_tegra_dma_desc(vd); + residual = dma_desc->bytes_req; + dma_set_residue(txstate, residual); + } else if (tdc->dma_desc && tdc->dma_desc->vd.tx.cookie == cookie) { + residual = tegra_dma_get_residual(tdc); + dma_set_residue(txstate, residual); + } else { + dev_err(tdc2dev(tdc), "cookie %d is not found\n", cookie); + } + spin_unlock_irqrestore(&tdc->vc.lock, flags); + + return ret; +} + +static inline int get_bus_width(struct tegra_dma_channel *tdc, + enum dma_slave_buswidth slave_bw) +{ + switch (slave_bw) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32; + default: + dev_err(tdc2dev(tdc), "given slave bus width is not supported\n"); + return -EINVAL; + } +} + +static unsigned int get_burst_size(struct tegra_dma_channel *tdc, + u32 burst_size, enum dma_slave_buswidth slave_bw, + int len) +{ + unsigned int burst_mmio_width, burst_byte; + + /* + * burst_size from client is in terms of the bus_width. + * convert that into words. + * If burst_size is not specified from client, then use + * len to calculate the optimum burst size + */ + burst_byte = burst_size ? burst_size * slave_bw : len; + burst_mmio_width = burst_byte / 4; + + if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN) + return 0; + + burst_mmio_width = min(burst_mmio_width, TEGRA_GPCDMA_MMIOSEQ_BURST_MAX); + + return TEGRA_GPCDMA_MMIOSEQ_BURST(burst_mmio_width); +} + +static int get_transfer_param(struct tegra_dma_channel *tdc, + enum dma_transfer_direction direction, + u32 *apb_addr, + u32 *mmio_seq, + u32 *csr, + unsigned int *burst_size, + enum dma_slave_buswidth *slave_bw) +{ + switch (direction) { + case DMA_MEM_TO_DEV: + *apb_addr = tdc->dma_sconfig.dst_addr; + *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width); + *burst_size = tdc->dma_sconfig.dst_maxburst; + *slave_bw = tdc->dma_sconfig.dst_addr_width; + *csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC; + return 0; + case DMA_DEV_TO_MEM: + *apb_addr = tdc->dma_sconfig.src_addr; + *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width); + *burst_size = tdc->dma_sconfig.src_maxburst; + *slave_bw = tdc->dma_sconfig.src_addr_width; + *csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC; + return 0; + default: + dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); + } + + return -EINVAL; +} + +static struct dma_async_tx_descriptor * +tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value, + size_t len, unsigned long flags) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count; + struct tegra_dma_sg_req *sg_req; + struct tegra_dma_desc *dma_desc; + u32 csr, mc_seq; + + if ((len & 3) || (dest & 3) || len > max_dma_count) { + dev_err(tdc2dev(tdc), + "DMA length/memory address is not supported\n"); + return NULL; + } + + /* Set DMA mode to fixed pattern */ + csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT; + /* Enable once or continuous mode */ + csr |= TEGRA_GPCDMA_CSR_ONCE; + /* Enable IRQ mask */ + csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; + /* Enable the DMA interrupt */ + if (flags & DMA_PREP_INTERRUPT) + csr |= TEGRA_GPCDMA_CSR_IE_EOC; + /* Configure default priority weight for the channel */ + csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); + + mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + /* retain stream-id and clean rest */ + mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; + + /* Set the address wrapping */ + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + + /* Program outstanding MC requests */ + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); + /* Set burst size */ + mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; + + dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT); + if (!dma_desc) + return NULL; + + dma_desc->bytes_req = len; + dma_desc->sg_count = 1; + sg_req = dma_desc->sg_req; + + sg_req[0].ch_regs.src_ptr = 0; + sg_req[0].ch_regs.dst_ptr = dest; + sg_req[0].ch_regs.high_addr_ptr = + FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32)); + sg_req[0].ch_regs.fixed_pattern = value; + /* Word count reg takes value as (N +1) words */ + sg_req[0].ch_regs.wcount = ((len - 4) >> 2); + sg_req[0].ch_regs.csr = csr; + sg_req[0].ch_regs.mmio_seq = 0; + sg_req[0].ch_regs.mc_seq = mc_seq; + sg_req[0].len = len; + + dma_desc->cyclic = false; + return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); +} + +static struct dma_async_tx_descriptor * +tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest, + dma_addr_t src, size_t len, unsigned long flags) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + struct tegra_dma_sg_req *sg_req; + struct tegra_dma_desc *dma_desc; + unsigned int max_dma_count; + u32 csr, mc_seq; + + max_dma_count = tdc->tdma->chip_data->max_dma_count; + if ((len & 3) || (src & 3) || (dest & 3) || len > max_dma_count) { + dev_err(tdc2dev(tdc), + "DMA length/memory address is not supported\n"); + return NULL; + } + + /* Set DMA mode to memory to memory transfer */ + csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM; + /* Enable once or continuous mode */ + csr |= TEGRA_GPCDMA_CSR_ONCE; + /* Enable IRQ mask */ + csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; + /* Enable the DMA interrupt */ + if (flags & DMA_PREP_INTERRUPT) + csr |= TEGRA_GPCDMA_CSR_IE_EOC; + /* Configure default priority weight for the channel */ + csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); + + mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + /* retain stream-id and clean rest */ + mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) | + (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK); + + /* Set the address wrapping */ + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + + /* Program outstanding MC requests */ + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); + /* Set burst size */ + mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; + + dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT); + if (!dma_desc) + return NULL; + + dma_desc->bytes_req = len; + dma_desc->sg_count = 1; + sg_req = dma_desc->sg_req; + + sg_req[0].ch_regs.src_ptr = src; + sg_req[0].ch_regs.dst_ptr = dest; + sg_req[0].ch_regs.high_addr_ptr = + FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32)); + sg_req[0].ch_regs.high_addr_ptr |= + FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32)); + /* Word count reg takes value as (N +1) words */ + sg_req[0].ch_regs.wcount = ((len - 4) >> 2); + sg_req[0].ch_regs.csr = csr; + sg_req[0].ch_regs.mmio_seq = 0; + sg_req[0].ch_regs.mc_seq = mc_seq; + sg_req[0].len = len; + + dma_desc->cyclic = false; + return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); +} + +static struct dma_async_tx_descriptor * +tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count; + enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED; + u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0; + struct tegra_dma_sg_req *sg_req; + struct tegra_dma_desc *dma_desc; + struct scatterlist *sg; + u32 burst_size; + unsigned int i; + int ret; + + if (!tdc->config_init) { + dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); + return NULL; + } + if (sg_len < 1) { + dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len); + return NULL; + } + + ret = tegra_dma_sid_reserve(tdc, direction); + if (ret) + return NULL; + + ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr, + &burst_size, &slave_bw); + if (ret < 0) + return NULL; + + /* Enable once or continuous mode */ + csr |= TEGRA_GPCDMA_CSR_ONCE; + /* Program the slave id in requestor select */ + csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id); + /* Enable IRQ mask */ + csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; + /* Configure default priority weight for the channel*/ + csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); + + /* Enable the DMA interrupt */ + if (flags & DMA_PREP_INTERRUPT) + csr |= TEGRA_GPCDMA_CSR_IE_EOC; + + mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + /* retain stream-id and clean rest */ + mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; + + /* Set the address wrapping on both MC and MMIO side */ + + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1); + + /* Program 2 MC outstanding requests by default. */ + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); + + /* Setting MC burst size depending on MMIO burst size */ + if (burst_size == 64) + mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; + else + mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2; + + dma_desc = kzalloc(struct_size(dma_desc, sg_req, sg_len), GFP_NOWAIT); + if (!dma_desc) + return NULL; + + dma_desc->sg_count = sg_len; + sg_req = dma_desc->sg_req; + + /* Make transfer requests */ + for_each_sg(sgl, sg, sg_len, i) { + u32 len; + dma_addr_t mem; + + mem = sg_dma_address(sg); + len = sg_dma_len(sg); + + if ((len & 3) || (mem & 3) || len > max_dma_count) { + dev_err(tdc2dev(tdc), + "DMA length/memory address is not supported\n"); + kfree(dma_desc); + return NULL; + } + + mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len); + dma_desc->bytes_req += len; + + if (direction == DMA_MEM_TO_DEV) { + sg_req[i].ch_regs.src_ptr = mem; + sg_req[i].ch_regs.dst_ptr = apb_ptr; + sg_req[i].ch_regs.high_addr_ptr = + FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32)); + } else if (direction == DMA_DEV_TO_MEM) { + sg_req[i].ch_regs.src_ptr = apb_ptr; + sg_req[i].ch_regs.dst_ptr = mem; + sg_req[i].ch_regs.high_addr_ptr = + FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32)); + } + + /* + * Word count register takes input in words. Writing a value + * of N into word count register means a req of (N+1) words. + */ + sg_req[i].ch_regs.wcount = ((len - 4) >> 2); + sg_req[i].ch_regs.csr = csr; + sg_req[i].ch_regs.mmio_seq = mmio_seq; + sg_req[i].ch_regs.mc_seq = mc_seq; + sg_req[i].len = len; + } + + dma_desc->cyclic = false; + return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); +} + +static struct dma_async_tx_descriptor * +tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags) +{ + enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED; + u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size; + unsigned int max_dma_count, len, period_count, i; + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + struct tegra_dma_desc *dma_desc; + struct tegra_dma_sg_req *sg_req; + dma_addr_t mem = buf_addr; + int ret; + + if (!buf_len || !period_len) { + dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); + return NULL; + } + + if (!tdc->config_init) { + dev_err(tdc2dev(tdc), "DMA slave is not configured\n"); + return NULL; + } + + ret = tegra_dma_sid_reserve(tdc, direction); + if (ret) + return NULL; + + /* + * We only support cycle transfer when buf_len is multiple of + * period_len. + */ + if (buf_len % period_len) { + dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n"); + return NULL; + } + + len = period_len; + max_dma_count = tdc->tdma->chip_data->max_dma_count; + if ((len & 3) || (buf_addr & 3) || len > max_dma_count) { + dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); + return NULL; + } + + ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr, + &burst_size, &slave_bw); + if (ret < 0) + return NULL; + + /* Enable once or continuous mode */ + csr &= ~TEGRA_GPCDMA_CSR_ONCE; + /* Program the slave id in requestor select */ + csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id); + /* Enable IRQ mask */ + csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; + /* Configure default priority weight for the channel*/ + csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); + + /* Enable the DMA interrupt */ + if (flags & DMA_PREP_INTERRUPT) + csr |= TEGRA_GPCDMA_CSR_IE_EOC; + + mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1); + + mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + /* retain stream-id and clean rest */ + mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; + + /* Set the address wrapping on both MC and MMIO side */ + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, + TEGRA_GPCDMA_MCSEQ_WRAP_NONE); + + /* Program 2 MC outstanding requests by default. */ + mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); + /* Setting MC burst size depending on MMIO burst size */ + if (burst_size == 64) + mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; + else + mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2; + + period_count = buf_len / period_len; + dma_desc = kzalloc(struct_size(dma_desc, sg_req, period_count), + GFP_NOWAIT); + if (!dma_desc) + return NULL; + + dma_desc->bytes_req = buf_len; + dma_desc->sg_count = period_count; + sg_req = dma_desc->sg_req; + + /* Split transfer equal to period size */ + for (i = 0; i < period_count; i++) { + mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len); + if (direction == DMA_MEM_TO_DEV) { + sg_req[i].ch_regs.src_ptr = mem; + sg_req[i].ch_regs.dst_ptr = apb_ptr; + sg_req[i].ch_regs.high_addr_ptr = + FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32)); + } else if (direction == DMA_DEV_TO_MEM) { + sg_req[i].ch_regs.src_ptr = apb_ptr; + sg_req[i].ch_regs.dst_ptr = mem; + sg_req[i].ch_regs.high_addr_ptr = + FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32)); + } + /* + * Word count register takes input in words. Writing a value + * of N into word count register means a req of (N+1) words. + */ + sg_req[i].ch_regs.wcount = ((len - 4) >> 2); + sg_req[i].ch_regs.csr = csr; + sg_req[i].ch_regs.mmio_seq = mmio_seq; + sg_req[i].ch_regs.mc_seq = mc_seq; + sg_req[i].len = len; + + mem += len; + } + + dma_desc->cyclic = true; + + return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); +} + +static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + int ret; + + ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc); + if (ret) { + dev_err(tdc2dev(tdc), "request_irq failed for %s\n", tdc->name); + return ret; + } + + dma_cookie_init(&tdc->vc.chan); + tdc->config_init = false; + return 0; +} + +static void tegra_dma_chan_synchronize(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + + synchronize_irq(tdc->irq); + vchan_synchronize(&tdc->vc); +} + +static void tegra_dma_free_chan_resources(struct dma_chan *dc) +{ + struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); + + dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); + + tegra_dma_terminate_all(dc); + synchronize_irq(tdc->irq); + + tasklet_kill(&tdc->vc.task); + tdc->config_init = false; + tdc->slave_id = -1; + tdc->sid_dir = DMA_TRANS_NONE; + free_irq(tdc->irq, tdc); + + vchan_free_chan_resources(&tdc->vc); +} + +static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct tegra_dma *tdma = ofdma->of_dma_data; + struct tegra_dma_channel *tdc; + struct dma_chan *chan; + + chan = dma_get_any_slave_channel(&tdma->dma_dev); + if (!chan) + return NULL; + + tdc = to_tegra_dma_chan(chan); + tdc->slave_id = dma_spec->args[0]; + + return chan; +} + +static const struct tegra_dma_chip_data tegra186_dma_chip_data = { + .nr_channels = 32, + .channel_reg_size = SZ_64K, + .max_dma_count = SZ_1G, + .hw_support_pause = false, + .terminate = tegra_dma_stop_client, +}; + +static const struct tegra_dma_chip_data tegra194_dma_chip_data = { + .nr_channels = 32, + .channel_reg_size = SZ_64K, + .max_dma_count = SZ_1G, + .hw_support_pause = true, + .terminate = tegra_dma_pause, +}; + +static const struct tegra_dma_chip_data tegra234_dma_chip_data = { + .nr_channels = 32, + .channel_reg_size = SZ_64K, + .max_dma_count = SZ_1G, + .hw_support_pause = true, + .terminate = tegra_dma_pause_noerr, +}; + +static const struct of_device_id tegra_dma_of_match[] = { + { + .compatible = "nvidia,tegra186-gpcdma", + .data = &tegra186_dma_chip_data, + }, { + .compatible = "nvidia,tegra194-gpcdma", + .data = &tegra194_dma_chip_data, + }, { + .compatible = "nvidia,tegra234-gpcdma", + .data = &tegra234_dma_chip_data, + }, { + }, +}; +MODULE_DEVICE_TABLE(of, tegra_dma_of_match); + +static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id) +{ + unsigned int reg_val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); + + reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK); + reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK); + + reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id); + reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id); + + tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val); + return 0; +} + +static int tegra_dma_probe(struct platform_device *pdev) +{ + const struct tegra_dma_chip_data *cdata = NULL; + unsigned int i; + u32 stream_id; + struct tegra_dma *tdma; + int ret; + + cdata = of_device_get_match_data(&pdev->dev); + + tdma = devm_kzalloc(&pdev->dev, + struct_size(tdma, channels, cdata->nr_channels), + GFP_KERNEL); + if (!tdma) + return -ENOMEM; + + tdma->dev = &pdev->dev; + tdma->chip_data = cdata; + platform_set_drvdata(pdev, tdma); + + tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); + + tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma"); + if (IS_ERR(tdma->rst)) { + return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst), + "Missing controller reset\n"); + } + reset_control_reset(tdma->rst); + + tdma->dma_dev.dev = &pdev->dev; + + if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) { + dev_err(&pdev->dev, "Missing iommu stream-id\n"); + return -EINVAL; + } + + ret = device_property_read_u32(&pdev->dev, "dma-channel-mask", + &tdma->chan_mask); + if (ret) { + dev_warn(&pdev->dev, + "Missing dma-channel-mask property, using default channel mask %#x\n", + TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK); + tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK; + } + + INIT_LIST_HEAD(&tdma->dma_dev.channels); + for (i = 0; i < cdata->nr_channels; i++) { + struct tegra_dma_channel *tdc = &tdma->channels[i]; + + /* Check for channel mask */ + if (!(tdma->chan_mask & BIT(i))) + continue; + + tdc->irq = platform_get_irq(pdev, i); + if (tdc->irq < 0) + return tdc->irq; + + tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET + + i * cdata->channel_reg_size; + snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i); + tdc->tdma = tdma; + tdc->id = i; + tdc->slave_id = -1; + + vchan_init(&tdc->vc, &tdma->dma_dev); + tdc->vc.desc_free = tegra_dma_desc_free; + + /* program stream-id for this channel */ + tegra_dma_program_sid(tdc, stream_id); + tdc->stream_id = stream_id; + } + + dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); + dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); + dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask); + dma_cap_set(DMA_MEMSET, tdma->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); + + /* + * Only word aligned transfers are supported. Set the copy + * alignment shift. + */ + tdma->dma_dev.copy_align = 2; + tdma->dma_dev.fill_align = 2; + tdma->dma_dev.device_alloc_chan_resources = + tegra_dma_alloc_chan_resources; + tdma->dma_dev.device_free_chan_resources = + tegra_dma_free_chan_resources; + tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg; + tdma->dma_dev.device_prep_dma_memcpy = tegra_dma_prep_dma_memcpy; + tdma->dma_dev.device_prep_dma_memset = tegra_dma_prep_dma_memset; + tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic; + tdma->dma_dev.device_config = tegra_dma_slave_config; + tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; + tdma->dma_dev.device_tx_status = tegra_dma_tx_status; + tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; + tdma->dma_dev.device_pause = tegra_dma_device_pause; + tdma->dma_dev.device_resume = tegra_dma_device_resume; + tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize; + tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + ret = dma_async_device_register(&tdma->dma_dev); + if (ret < 0) { + dev_err_probe(&pdev->dev, ret, + "GPC DMA driver registration failed\n"); + return ret; + } + + ret = of_dma_controller_register(pdev->dev.of_node, + tegra_dma_of_xlate, tdma); + if (ret < 0) { + dev_err_probe(&pdev->dev, ret, + "GPC DMA OF registration failed\n"); + + dma_async_device_unregister(&tdma->dma_dev); + return ret; + } + + dev_info(&pdev->dev, "GPC DMA driver register %lu channels\n", + hweight_long(tdma->chan_mask)); + + return 0; +} + +static void tegra_dma_remove(struct platform_device *pdev) +{ + struct tegra_dma *tdma = platform_get_drvdata(pdev); + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&tdma->dma_dev); +} + +static int __maybe_unused tegra_dma_pm_suspend(struct device *dev) +{ + struct tegra_dma *tdma = dev_get_drvdata(dev); + unsigned int i; + + for (i = 0; i < tdma->chip_data->nr_channels; i++) { + struct tegra_dma_channel *tdc = &tdma->channels[i]; + + if (!(tdma->chan_mask & BIT(i))) + continue; + + if (tdc->dma_desc) { + dev_err(tdma->dev, "channel %u busy\n", i); + return -EBUSY; + } + } + + return 0; +} + +static int __maybe_unused tegra_dma_pm_resume(struct device *dev) +{ + struct tegra_dma *tdma = dev_get_drvdata(dev); + unsigned int i; + + reset_control_reset(tdma->rst); + + for (i = 0; i < tdma->chip_data->nr_channels; i++) { + struct tegra_dma_channel *tdc = &tdma->channels[i]; + + if (!(tdma->chan_mask & BIT(i))) + continue; + + tegra_dma_program_sid(tdc, tdc->stream_id); + } + + return 0; +} + +static const struct dev_pm_ops tegra_dma_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume) +}; + +static struct platform_driver tegra_dma_driver = { + .driver = { + .name = "tegra-gpcdma", + .pm = &tegra_dma_dev_pm_ops, + .of_match_table = tegra_dma_of_match, + }, + .probe = tegra_dma_probe, + .remove = tegra_dma_remove, +}; + +module_platform_driver(tegra_dma_driver); + +MODULE_DESCRIPTION("NVIDIA Tegra GPC DMA Controller driver"); +MODULE_AUTHOR("Pavan Kunapuli <pkunapuli@nvidia.com>"); +MODULE_AUTHOR("Rajesh Gumasta <rgumasta@nvidia.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index b7260749e8ee..14a61e53a41b 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -17,7 +17,6 @@ #include <linux/mm.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm.h> @@ -233,11 +232,6 @@ static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val) writel(val, tdma->base_addr + reg); } -static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg) -{ - return readl(tdma->base_addr + reg); -} - static inline void tdc_write(struct tegra_dma_channel *tdc, u32 reg, u32 val) { @@ -343,12 +337,6 @@ static int tegra_dma_slave_config(struct dma_chan *dc, } memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); - if (tdc->slave_id == TEGRA_APBDMA_SLAVE_ID_INVALID && - sconfig->device_fc) { - if (sconfig->slave_id > TEGRA_APBDMA_CSR_REQ_SEL_MASK) - return -EINVAL; - tdc->slave_id = sconfig->slave_id; - } tdc->config_init = true; return 0; @@ -475,7 +463,7 @@ static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, /* * If interrupt is pending then do nothing as the ISR will handle - * the programing for new request. + * the programming for new request. */ if (status & TEGRA_APBDMA_STATUS_ISE_EOC) { dev_err(tdc2dev(tdc), @@ -1593,7 +1581,7 @@ err_clk_unprepare: return ret; } -static int tegra_dma_remove(struct platform_device *pdev) +static void tegra_dma_remove(struct platform_device *pdev) { struct tegra_dma *tdma = platform_get_drvdata(pdev); @@ -1601,8 +1589,6 @@ static int tegra_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&tdma->dma_dev); pm_runtime_disable(&pdev->dev); clk_unprepare(tdma->dma_clk); - - return 0; } static int __maybe_unused tegra_dma_runtime_suspend(struct device *dev) diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index 4735742e826d..d0e8bb27a03b 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -8,9 +8,10 @@ #include <linux/clk.h> #include <linux/iopoll.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> @@ -26,10 +27,10 @@ #define ADMA_CH_INT_CLEAR 0x1c #define ADMA_CH_CTRL 0x24 -#define ADMA_CH_CTRL_DIR(val) (((val) & 0xf) << 12) +#define ADMA_CH_CTRL_DIR(val, mask, shift) (((val) & (mask)) << (shift)) #define ADMA_CH_CTRL_DIR_AHUB2MEM 2 #define ADMA_CH_CTRL_DIR_MEM2AHUB 4 -#define ADMA_CH_CTRL_MODE_CONTINUOUS (2 << 8) +#define ADMA_CH_CTRL_MODE_CONTINUOUS(shift) (2 << (shift)) #define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1) #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 0 @@ -40,13 +41,27 @@ #define ADMA_CH_CONFIG_MAX_BURST_SIZE 16 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf) #define ADMA_CH_CONFIG_MAX_BUFS 8 -#define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) (reqs << 4) +#define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) ((reqs) << 4) + +#define ADMA_GLOBAL_CH_CONFIG 0x400 +#define ADMA_GLOBAL_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0x7) +#define ADMA_GLOBAL_CH_CONFIG_OUTSTANDING_REQS(reqs) ((reqs) << 8) + +#define TEGRA186_ADMA_GLOBAL_PAGE_CHGRP 0x30 +#define TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ 0x70 +#define TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ 0x84 +#define TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0 0x44 +#define TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1 0x48 +#define TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0 0x100 +#define TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1 0x104 +#define TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0 0x180 +#define TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1 0x184 +#define TEGRA264_ADMA_GLOBAL_PAGE_OFFSET 0x8 #define ADMA_CH_FIFO_CTRL 0x2c -#define TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0xf) << 8) -#define TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0xf) -#define TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(val) (((val) & 0x1f) << 8) -#define TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(val) ((val) & 0x1f) +#define ADMA_CH_TX_FIFO_SIZE_SHIFT 8 +#define ADMA_CH_RX_FIFO_SIZE_SHIFT 0 +#define ADMA_GLOBAL_CH_FIFO_CTRL 0x300 #define ADMA_CH_LOWER_SRC_ADDR 0x34 #define ADMA_CH_LOWER_TRG_ADDR 0x3c @@ -61,43 +76,58 @@ #define TEGRA_ADMA_BURST_COMPLETE_TIME 20 -#define TEGRA210_FIFO_CTRL_DEFAULT (TEGRA210_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \ - TEGRA210_ADMA_CH_FIFO_CTRL_RXSIZE(3)) - -#define TEGRA186_FIFO_CTRL_DEFAULT (TEGRA186_ADMA_CH_FIFO_CTRL_TXSIZE(3) | \ - TEGRA186_ADMA_CH_FIFO_CTRL_RXSIZE(3)) - #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift) struct tegra_adma; /* * struct tegra_adma_chip_data - Tegra chip specific data + * @adma_get_burst_config: Function callback used to set DMA burst size. * @global_reg_offset: Register offset of DMA global register. * @global_int_clear: Register offset of DMA global interrupt clear. + * @global_ch_fifo_base: Global channel fifo ctrl base offset + * @global_ch_config_base: Global channel config base offset * @ch_req_tx_shift: Register offset for AHUB transmit channel select. * @ch_req_rx_shift: Register offset for AHUB receive channel select. + * @ch_dir_shift: Channel direction bit position. + * @ch_mode_shift: Channel mode bit position. * @ch_base_offset: Register offset of DMA channel registers. - * @has_outstanding_reqs: If DMA channel can have outstanding requests. + * @ch_tc_offset_diff: From TC register onwards offset differs for Tegra264 * @ch_fifo_ctrl: Default value for channel FIFO CTRL register. + * @ch_config: Outstanding and WRR config values * @ch_req_mask: Mask for Tx or Rx channel select. + * @ch_dir_mask: Mask for channel direction. * @ch_req_max: Maximum number of Tx or Rx channels available. * @ch_reg_size: Size of DMA channel register space. * @nr_channels: Number of DMA channels available. + * @ch_fifo_size_mask: Mask for FIFO size field. + * @sreq_index_offset: Slave channel index offset. + * @max_page: Maximum ADMA Channel Page. + * @set_global_pg_config: Global page programming. */ struct tegra_adma_chip_data { unsigned int (*adma_get_burst_config)(unsigned int burst_size); unsigned int global_reg_offset; unsigned int global_int_clear; + unsigned int global_ch_fifo_base; + unsigned int global_ch_config_base; unsigned int ch_req_tx_shift; unsigned int ch_req_rx_shift; + unsigned int ch_dir_shift; + unsigned int ch_mode_shift; unsigned int ch_base_offset; + unsigned int ch_tc_offset_diff; unsigned int ch_fifo_ctrl; + unsigned int ch_config; unsigned int ch_req_mask; + unsigned int ch_dir_mask; unsigned int ch_req_max; unsigned int ch_reg_size; unsigned int nr_channels; - bool has_outstanding_reqs; + unsigned int ch_fifo_size_mask; + unsigned int sreq_index_offset; + unsigned int max_page; + void (*set_global_pg_config)(struct tegra_adma *tdma); }; /* @@ -106,6 +136,7 @@ struct tegra_adma_chip_data { struct tegra_adma_chan_regs { unsigned int ctrl; unsigned int config; + unsigned int global_config; unsigned int src_addr; unsigned int trg_addr; unsigned int fifo_ctrl; @@ -144,6 +175,9 @@ struct tegra_adma_chan { /* Transfer count and position info */ unsigned int tx_buf_count; unsigned int tx_buf_pos; + + unsigned int global_ch_fifo_offset; + unsigned int global_ch_config_offset; }; /* @@ -153,18 +187,21 @@ struct tegra_adma { struct dma_device dma_dev; struct device *dev; void __iomem *base_addr; + void __iomem *ch_base_addr; struct clk *ahub_clk; unsigned int nr_channels; + unsigned long *dma_chan_mask; unsigned long rx_requests_reserved; unsigned long tx_requests_reserved; /* Used to store global command register state when suspending */ unsigned int global_cmd; + unsigned int ch_page_no; const struct tegra_adma_chip_data *cdata; /* Last member of the structure */ - struct tegra_adma_chan channels[]; + struct tegra_adma_chan channels[] __counted_by(nr_channels); }; static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) @@ -177,6 +214,11 @@ static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg) return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg); } +static inline void tdma_ch_global_write(struct tegra_adma *tdma, u32 reg, u32 val) +{ + writel(val, tdma->ch_base_addr + tdma->cdata->global_reg_offset + reg); +} + static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val) { writel(val, tdc->chan_addr + reg); @@ -218,13 +260,53 @@ static int tegra_adma_slave_config(struct dma_chan *dc, return 0; } +static void tegra186_adma_global_page_config(struct tegra_adma *tdma) +{ + /* + * Clear the default page1 channel group configs and program + * the global registers based on the actual page usage + */ + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ, 0); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP + (tdma->ch_page_no * 0x4), 0xff); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ + (tdma->ch_page_no * 0x4), 0x1ffffff); + tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ + (tdma->ch_page_no * 0x4), 0xffffff); +} + +static void tegra264_adma_global_page_config(struct tegra_adma *tdma) +{ + u32 global_page_offset = tdma->ch_page_no * TEGRA264_ADMA_GLOBAL_PAGE_OFFSET; + + /* If the default page (page1) is not used, then clear page1 registers */ + if (tdma->ch_page_no) { + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0, 0); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1, 0); + } + + /* Program global registers for selected page */ + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1 + global_page_offset, 0x1); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0 + global_page_offset, 0xffffffff); + tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1 + global_page_offset, 0x1); +} + static int tegra_adma_init(struct tegra_adma *tdma) { u32 status; int ret; - /* Clear any interrupts */ - tdma_write(tdma, tdma->cdata->global_int_clear, 0x1); + /* Clear any channels group global interrupts */ + tdma_ch_global_write(tdma, tdma->cdata->global_int_clear, 0x1); + + if (!tdma->base_addr) + return 0; /* Assert soft reset */ tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1); @@ -238,6 +320,9 @@ static int tegra_adma_init(struct tegra_adma *tdma) if (ret) return ret; + if (tdma->cdata->set_global_pg_config) + tdma->cdata->set_global_pg_config(tdma); + /* Enable global ADMA registers */ tdma_write(tdma, ADMA_GLOBAL_CMD, 1); @@ -370,11 +455,21 @@ static void tegra_adma_start(struct tegra_adma_chan *tdc) tdc->tx_buf_pos = 0; tdc->tx_buf_count = 0; - tdma_ch_write(tdc, ADMA_CH_TC, ch_regs->tc); + tdma_ch_write(tdc, ADMA_CH_TC - tdc->tdma->cdata->ch_tc_offset_diff, ch_regs->tc); tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); - tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_regs->src_addr); - tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_regs->trg_addr); - tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl); + tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR - tdc->tdma->cdata->ch_tc_offset_diff, + ch_regs->src_addr); + tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR - tdc->tdma->cdata->ch_tc_offset_diff, + ch_regs->trg_addr); + + if (!tdc->tdma->cdata->global_ch_fifo_base) + tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl); + else if (tdc->global_ch_fifo_offset) + tdma_write(tdc->tdma, tdc->global_ch_fifo_offset, ch_regs->fifo_ctrl); + + if (tdc->global_ch_config_offset) + tdma_write(tdc->tdma, tdc->global_ch_config_offset, ch_regs->global_config); + tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config); /* Start ADMA */ @@ -387,7 +482,8 @@ static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc) { struct tegra_adma_desc *desc = tdc->desc; unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1; - unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS); + unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS - + tdc->tdma->cdata->ch_tc_offset_diff); unsigned int periods_remaining; /* @@ -560,13 +656,14 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, { struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata; - unsigned int burst_size, adma_dir; + unsigned int burst_size, adma_dir, fifo_size_shift; if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) return -EINVAL; switch (direction) { case DMA_MEM_TO_DEV: + fifo_size_shift = ADMA_CH_TX_FIFO_SIZE_SHIFT; adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB; burst_size = tdc->sconfig.dst_maxburst; ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1); @@ -577,6 +674,7 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, break; case DMA_DEV_TO_MEM: + fifo_size_shift = ADMA_CH_RX_FIFO_SIZE_SHIFT; adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM; burst_size = tdc->sconfig.src_maxburst; ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1); @@ -591,14 +689,37 @@ static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, return -EINVAL; } - ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir) | - ADMA_CH_CTRL_MODE_CONTINUOUS | + ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir, cdata->ch_dir_mask, + cdata->ch_dir_shift) | + ADMA_CH_CTRL_MODE_CONTINUOUS(cdata->ch_mode_shift) | ADMA_CH_CTRL_FLOWCTRL_EN; ch_regs->config |= cdata->adma_get_burst_config(burst_size); - ch_regs->config |= ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1); - if (cdata->has_outstanding_reqs) - ch_regs->config |= TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8); - ch_regs->fifo_ctrl = cdata->ch_fifo_ctrl; + + if (cdata->global_ch_config_base) + ch_regs->global_config |= cdata->ch_config; + else + ch_regs->config |= cdata->ch_config; + + /* + * 'sreq_index' represents the current ADMAIF channel number and as per + * HW recommendation its FIFO size should match with the corresponding + * ADMA channel. + * + * ADMA FIFO size is set as per below (based on default ADMAIF channel + * FIFO sizes): + * fifo_size = 0x2 (sreq_index > sreq_index_offset) + * fifo_size = 0x3 (sreq_index <= sreq_index_offset) + * + */ + if (tdc->sreq_index > cdata->sreq_index_offset) + ch_regs->fifo_ctrl = + ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask, + fifo_size_shift); + else + ch_regs->fifo_ctrl = + ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask, + fifo_size_shift); + ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK; return tegra_adma_request_alloc(tdc, direction); @@ -655,9 +776,8 @@ static int tegra_adma_alloc_chan_resources(struct dma_chan *dc) return ret; } - ret = pm_runtime_get_sync(tdc2dev(tdc)); + ret = pm_runtime_resume_and_get(tdc2dev(tdc)); if (ret < 0) { - pm_runtime_put_noidle(tdc2dev(tdc)); free_irq(tdc->irq, tdc); return ret; } @@ -716,23 +836,40 @@ static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev) struct tegra_adma_chan *tdc; int i; - tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); + if (tdma->base_addr) + tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); + if (!tdma->global_cmd) goto clk_disable; for (i = 0; i < tdma->nr_channels; i++) { tdc = &tdma->channels[i]; + /* skip for reserved channels */ + if (!tdc->tdma) + continue; + ch_reg = &tdc->ch_regs; ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); /* skip if channel is not active */ if (!ch_reg->cmd) continue; - ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC); - ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR); - ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR); + ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC - tdma->cdata->ch_tc_offset_diff); + ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR - + tdma->cdata->ch_tc_offset_diff); + ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR - + tdma->cdata->ch_tc_offset_diff); ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); - ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); + + if (tdc->global_ch_config_offset) + ch_reg->global_config = tdma_read(tdc->tdma, tdc->global_ch_config_offset); + + if (!tdc->tdma->cdata->global_ch_fifo_base) + ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); + else if (tdc->global_ch_fifo_offset) + ch_reg->fifo_ctrl = tdma_read(tdc->tdma, tdc->global_ch_fifo_offset); + ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG); + } clk_disable: @@ -753,23 +890,41 @@ static int __maybe_unused tegra_adma_runtime_resume(struct device *dev) dev_err(dev, "ahub clk_enable failed: %d\n", ret); return ret; } - tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); + if (tdma->base_addr) { + tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); + if (tdma->cdata->set_global_pg_config) + tdma->cdata->set_global_pg_config(tdma); + } if (!tdma->global_cmd) return 0; for (i = 0; i < tdma->nr_channels; i++) { tdc = &tdma->channels[i]; + /* skip for reserved channels */ + if (!tdc->tdma) + continue; ch_reg = &tdc->ch_regs; /* skip if channel was not active earlier */ if (!ch_reg->cmd) continue; - tdma_ch_write(tdc, ADMA_CH_TC, ch_reg->tc); - tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR, ch_reg->src_addr); - tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR, ch_reg->trg_addr); + tdma_ch_write(tdc, ADMA_CH_TC - tdma->cdata->ch_tc_offset_diff, ch_reg->tc); + tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR - tdma->cdata->ch_tc_offset_diff, + ch_reg->src_addr); + tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR - tdma->cdata->ch_tc_offset_diff, + ch_reg->trg_addr); tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl); - tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl); + + if (!tdc->tdma->cdata->global_ch_fifo_base) + tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl); + else if (tdc->global_ch_fifo_offset) + tdma_write(tdc->tdma, tdc->global_ch_fifo_offset, ch_reg->fifo_ctrl); + + if (tdc->global_ch_config_offset) + tdma_write(tdc->tdma, tdc->global_ch_config_offset, ch_reg->global_config); + tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config); + tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd); } @@ -780,35 +935,80 @@ static const struct tegra_adma_chip_data tegra210_chip_data = { .adma_get_burst_config = tegra210_adma_get_burst_config, .global_reg_offset = 0xc00, .global_int_clear = 0x20, + .global_ch_fifo_base = 0, + .global_ch_config_base = 0, .ch_req_tx_shift = 28, .ch_req_rx_shift = 24, + .ch_dir_shift = 12, + .ch_mode_shift = 8, .ch_base_offset = 0, - .has_outstanding_reqs = false, - .ch_fifo_ctrl = TEGRA210_FIFO_CTRL_DEFAULT, + .ch_tc_offset_diff = 0, + .ch_config = ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1), .ch_req_mask = 0xf, + .ch_dir_mask = 0xf, .ch_req_max = 10, .ch_reg_size = 0x80, .nr_channels = 22, + .ch_fifo_size_mask = 0xf, + .sreq_index_offset = 2, + .max_page = 0, + .set_global_pg_config = NULL, }; static const struct tegra_adma_chip_data tegra186_chip_data = { .adma_get_burst_config = tegra186_adma_get_burst_config, .global_reg_offset = 0, .global_int_clear = 0x402c, + .global_ch_fifo_base = 0, + .global_ch_config_base = 0, .ch_req_tx_shift = 27, .ch_req_rx_shift = 22, + .ch_dir_shift = 12, + .ch_mode_shift = 8, .ch_base_offset = 0x10000, - .has_outstanding_reqs = true, - .ch_fifo_ctrl = TEGRA186_FIFO_CTRL_DEFAULT, + .ch_tc_offset_diff = 0, + .ch_config = ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1) | + TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8), .ch_req_mask = 0x1f, + .ch_dir_mask = 0xf, .ch_req_max = 20, .ch_reg_size = 0x100, .nr_channels = 32, + .ch_fifo_size_mask = 0x1f, + .sreq_index_offset = 4, + .max_page = 4, + .set_global_pg_config = tegra186_adma_global_page_config, +}; + +static const struct tegra_adma_chip_data tegra264_chip_data = { + .adma_get_burst_config = tegra186_adma_get_burst_config, + .global_reg_offset = 0, + .global_int_clear = 0x800c, + .global_ch_fifo_base = ADMA_GLOBAL_CH_FIFO_CTRL, + .global_ch_config_base = ADMA_GLOBAL_CH_CONFIG, + .ch_req_tx_shift = 26, + .ch_req_rx_shift = 20, + .ch_dir_shift = 10, + .ch_mode_shift = 7, + .ch_base_offset = 0x10000, + .ch_tc_offset_diff = 4, + .ch_config = ADMA_GLOBAL_CH_CONFIG_WEIGHT_FOR_WRR(1) | + ADMA_GLOBAL_CH_CONFIG_OUTSTANDING_REQS(8), + .ch_req_mask = 0x3f, + .ch_dir_mask = 7, + .ch_req_max = 32, + .ch_reg_size = 0x100, + .nr_channels = 64, + .ch_fifo_size_mask = 0x7f, + .sreq_index_offset = 0, + .max_page = 10, + .set_global_pg_config = tegra264_adma_global_page_config, }; static const struct of_device_id tegra_adma_of_match[] = { { .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data }, { .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data }, + { .compatible = "nvidia,tegra264-adma", .data = &tegra264_chip_data }, { }, }; MODULE_DEVICE_TABLE(of, tegra_adma_of_match); @@ -817,7 +1017,7 @@ static int tegra_adma_probe(struct platform_device *pdev) { const struct tegra_adma_chip_data *cdata; struct tegra_adma *tdma; - struct resource *res; + struct resource *res_page, *res_base; int ret, i; cdata = of_device_get_match_data(&pdev->dev); @@ -837,10 +1037,46 @@ static int tegra_adma_probe(struct platform_device *pdev) tdma->nr_channels = cdata->nr_channels; platform_set_drvdata(pdev, tdma); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - tdma->base_addr = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(tdma->base_addr)) - return PTR_ERR(tdma->base_addr); + res_page = platform_get_resource_byname(pdev, IORESOURCE_MEM, "page"); + if (res_page) { + tdma->ch_base_addr = devm_ioremap_resource(&pdev->dev, res_page); + if (IS_ERR(tdma->ch_base_addr)) + return PTR_ERR(tdma->ch_base_addr); + + res_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "global"); + if (res_base) { + resource_size_t page_offset, page_no; + unsigned int ch_base_offset; + + if (res_page->start < res_base->start) + return -EINVAL; + page_offset = res_page->start - res_base->start; + ch_base_offset = cdata->ch_base_offset; + if (!ch_base_offset) + return -EINVAL; + + page_no = div_u64(page_offset, ch_base_offset); + if (!page_no || page_no > INT_MAX) + return -EINVAL; + + tdma->ch_page_no = page_no - 1; + tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); + } + } else { + /* If no 'page' property found, then reg DT binding would be legacy */ + res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res_base) { + tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); + } else { + return -ENODEV; + } + + tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset; + } tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio"); if (IS_ERR(tdma->ahub_clk)) { @@ -848,12 +1084,41 @@ static int tegra_adma_probe(struct platform_device *pdev) return PTR_ERR(tdma->ahub_clk); } + tdma->dma_chan_mask = devm_kzalloc(&pdev->dev, + BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long), + GFP_KERNEL); + if (!tdma->dma_chan_mask) + return -ENOMEM; + + /* Enable all channels by default */ + bitmap_fill(tdma->dma_chan_mask, tdma->nr_channels); + + ret = of_property_read_u32_array(pdev->dev.of_node, "dma-channel-mask", + (u32 *)tdma->dma_chan_mask, + BITS_TO_U32(tdma->nr_channels)); + if (ret < 0 && (ret != -EINVAL)) { + dev_err(&pdev->dev, "dma-channel-mask is not complete.\n"); + return ret; + } + INIT_LIST_HEAD(&tdma->dma_dev.channels); for (i = 0; i < tdma->nr_channels; i++) { struct tegra_adma_chan *tdc = &tdma->channels[i]; - tdc->chan_addr = tdma->base_addr + cdata->ch_base_offset - + (cdata->ch_reg_size * i); + /* skip for reserved channels */ + if (!test_bit(i, tdma->dma_chan_mask)) + continue; + + tdc->chan_addr = tdma->ch_base_addr + (cdata->ch_reg_size * i); + + if (tdma->base_addr) { + if (cdata->global_ch_fifo_base) + tdc->global_ch_fifo_offset = cdata->global_ch_fifo_base + (4 * i); + + if (cdata->global_ch_config_base) + tdc->global_ch_config_offset = + cdata->global_ch_config_base + (4 * i); + } tdc->irq = of_irq_get(pdev->dev.of_node, i); if (tdc->irq <= 0) { @@ -868,11 +1133,9 @@ static int tegra_adma_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); - ret = pm_runtime_get_sync(&pdev->dev); - if (ret < 0) { - pm_runtime_put_noidle(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) goto rpm_disable; - } ret = tegra_adma_init(tdma); if (ret) @@ -932,7 +1195,7 @@ irq_dispose: return ret; } -static int tegra_adma_remove(struct platform_device *pdev) +static void tegra_adma_remove(struct platform_device *pdev) { struct tegra_adma *tdma = platform_get_drvdata(pdev); int i; @@ -940,13 +1203,12 @@ static int tegra_adma_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&tdma->dma_dev); - for (i = 0; i < tdma->nr_channels; ++i) - irq_dispose_mapping(tdma->channels[i].irq); + for (i = 0; i < tdma->nr_channels; ++i) { + if (tdma->channels[i].irq) + irq_dispose_mapping(tdma->channels[i].irq); + } - pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); - - return 0; } static const struct dev_pm_ops tegra_adma_dev_pm_ops = { @@ -968,7 +1230,6 @@ static struct platform_driver tegra_admac_driver = { module_platform_driver(tegra_admac_driver); -MODULE_ALIAS("platform:tegra210-adma"); MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver"); MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>"); MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>"); diff --git a/drivers/dma/ti/Kconfig b/drivers/dma/ti/Kconfig index 79618fac119a..dbf168146d35 100644 --- a/drivers/dma/ti/Kconfig +++ b/drivers/dma/ti/Kconfig @@ -17,7 +17,7 @@ config TI_EDMA select DMA_ENGINE select DMA_VIRTUAL_CHANNELS select TI_DMA_CROSSBAR if (ARCH_OMAP || COMPILE_TEST) - default y + default ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE help Enable support for the TI EDMA (Enhanced DMA) controller. This DMA engine is found on TI DaVinci, AM33xx, AM43xx, DRA7xx and Keystone 2 @@ -29,13 +29,13 @@ config DMA_OMAP select DMA_ENGINE select DMA_VIRTUAL_CHANNELS select TI_DMA_CROSSBAR if (SOC_DRA7XX || COMPILE_TEST) - default y + default ARCH_OMAP help Enable support for the TI sDMA (System DMA or DMA4) controller. This DMA engine is found on OMAP and DRA7xx parts. config TI_K3_UDMA - bool "Texas Instruments UDMA support" + tristate "Texas Instruments UDMA support" depends on ARCH_K3 depends on TI_SCI_PROTOCOL depends on TI_SCI_INTA_IRQCHIP @@ -48,7 +48,7 @@ config TI_K3_UDMA DMA engine is used in AM65x and j721e. config TI_K3_UDMA_GLUE_LAYER - bool "Texas Instruments UDMA Glue layer for non DMAengine users" + tristate "Texas Instruments UDMA Glue layer for non DMAengine users" depends on ARCH_K3 depends on TI_K3_UDMA help @@ -56,7 +56,8 @@ config TI_K3_UDMA_GLUE_LAYER If unsure, say N. config TI_K3_PSIL - bool + tristate + default TI_K3_UDMA config TI_DMA_CROSSBAR bool diff --git a/drivers/dma/ti/Makefile b/drivers/dma/ti/Makefile index bd496efadff7..d376c117cecf 100644 --- a/drivers/dma/ti/Makefile +++ b/drivers/dma/ti/Makefile @@ -4,9 +4,15 @@ obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_DMA_OMAP) += omap-dma.o obj-$(CONFIG_TI_K3_UDMA) += k3-udma.o obj-$(CONFIG_TI_K3_UDMA_GLUE_LAYER) += k3-udma-glue.o -obj-$(CONFIG_TI_K3_PSIL) += k3-psil.o \ - k3-psil-am654.o \ - k3-psil-j721e.o \ - k3-psil-j7200.o \ - k3-psil-am64.o +k3-psil-lib-objs := k3-psil.o \ + k3-psil-am654.o \ + k3-psil-j721e.o \ + k3-psil-j7200.o \ + k3-psil-am64.o \ + k3-psil-j721s2.o \ + k3-psil-am62.o \ + k3-psil-am62a.o \ + k3-psil-j784s4.o \ + k3-psil-am62p.o +obj-$(CONFIG_TI_K3_PSIL) += k3-psil-lib.o obj-$(CONFIG_TI_DMA_CROSSBAR) += dma-crossbar.o diff --git a/drivers/dma/ti/cppi41.c b/drivers/dma/ti/cppi41.c index 8c2f7ebe998c..8d8c3d6038fc 100644 --- a/drivers/dma/ti/cppi41.c +++ b/drivers/dma/ti/cppi41.c @@ -315,7 +315,7 @@ static irqreturn_t cppi41_irq(int irq, void *data) val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i)); if (i == QMGR_PENDING_SLOT_Q(first_completion_queue) && val) { u32 mask; - /* set corresponding bit for completetion Q 93 */ + /* set corresponding bit for completion Q 93 */ mask = 1 << QMGR_PENDING_BIT_Q(first_completion_queue); /* not set all bits for queues less than Q 93 */ mask--; @@ -703,7 +703,7 @@ static int cppi41_tear_down_chan(struct cppi41_channel *c) * transfer descriptor followed by TD descriptor. Waiting seems not to * cause any difference. * RX seems to be thrown out right away. However once the TearDown - * descriptor gets through we are done. If we have seens the transfer + * descriptor gets through we are done. If we have seen the transfer * descriptor before the TD we fetch it from enqueue, it has to be * there waiting for us. */ @@ -747,7 +747,7 @@ static int cppi41_stop_chan(struct dma_chan *chan) struct cppi41_channel *cc, *_ct; /* - * channels might still be in the pendling list if + * channels might still be in the pending list if * cppi41_dma_issue_pending() is called after * cppi41_runtime_suspend() is called */ @@ -1039,7 +1039,6 @@ static int cppi41_dma_probe(struct platform_device *pdev) struct cppi41_dd *cdd; struct device *dev = &pdev->dev; const struct cppi_glue_infos *glue_info; - struct resource *mem; int index; int irq; int ret; @@ -1072,18 +1071,15 @@ static int cppi41_dma_probe(struct platform_device *pdev) if (index < 0) return index; - mem = platform_get_resource(pdev, IORESOURCE_MEM, index); - cdd->ctrl_mem = devm_ioremap_resource(dev, mem); + cdd->ctrl_mem = devm_platform_ioremap_resource(pdev, index); if (IS_ERR(cdd->ctrl_mem)) return PTR_ERR(cdd->ctrl_mem); - mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 1); - cdd->sched_mem = devm_ioremap_resource(dev, mem); + cdd->sched_mem = devm_platform_ioremap_resource(pdev, index + 1); if (IS_ERR(cdd->sched_mem)) return PTR_ERR(cdd->sched_mem); - mem = platform_get_resource(pdev, IORESOURCE_MEM, index + 2); - cdd->qmgr_mem = devm_ioremap_resource(dev, mem); + cdd->qmgr_mem = devm_platform_ioremap_resource(pdev, index + 2); if (IS_ERR(cdd->qmgr_mem)) return PTR_ERR(cdd->qmgr_mem); @@ -1105,8 +1101,12 @@ static int cppi41_dma_probe(struct platform_device *pdev) cdd->qmgr_num_pend = glue_info->qmgr_num_pend; cdd->first_completion_queue = glue_info->first_completion_queue; + /* Parse new and deprecated dma-channels properties */ ret = of_property_read_u32(dev->of_node, - "#dma-channels", &cdd->n_chans); + "dma-channels", &cdd->n_chans); + if (ret) + ret = of_property_read_u32(dev->of_node, + "#dma-channels", &cdd->n_chans); if (ret) goto err_get_n_chans; @@ -1156,7 +1156,7 @@ err_get_sync: return ret; } -static int cppi41_dma_remove(struct platform_device *pdev) +static void cppi41_dma_remove(struct platform_device *pdev) { struct cppi41_dd *cdd = platform_get_drvdata(pdev); int error; @@ -1173,7 +1173,6 @@ static int cppi41_dma_remove(struct platform_device *pdev) pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); - return 0; } static int __maybe_unused cppi41_suspend(struct device *dev) @@ -1253,5 +1252,6 @@ static struct platform_driver cpp41_dma_driver = { }; module_platform_driver(cpp41_dma_driver); +MODULE_DESCRIPTION("Texas Instruments CPPI 4.1 DMA support"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Sebastian Andrzej Siewior <bigeasy@linutronix.de>"); diff --git a/drivers/dma/ti/dma-crossbar.c b/drivers/dma/ti/dma-crossbar.c index 71d24fc07c00..7f17ee87a6dc 100644 --- a/drivers/dma/ti/dma-crossbar.c +++ b/drivers/dma/ti/dma-crossbar.c @@ -3,14 +3,15 @@ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> */ +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/err.h> #include <linux/init.h> #include <linux/list.h> #include <linux/io.h> -#include <linux/of_address.h> -#include <linux/of_device.h> +#include <linux/of.h> #include <linux/of_dma.h> +#include <linux/of_platform.h> #define TI_XBAR_DRA7 0 #define TI_XBAR_AM335X 1 @@ -245,6 +246,7 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec, if (dma_spec->args[0] >= xbar->xbar_requests) { dev_err(&pdev->dev, "Invalid XBAR request number: %d\n", dma_spec->args[0]); + put_device(&pdev->dev); return ERR_PTR(-EINVAL); } @@ -252,12 +254,14 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec, dma_spec->np = of_parse_phandle(ofdma->of_node, "dma-masters", 0); if (!dma_spec->np) { dev_err(&pdev->dev, "Can't get DMA master\n"); + put_device(&pdev->dev); return ERR_PTR(-EINVAL); } map = kzalloc(sizeof(*map), GFP_KERNEL); if (!map) { of_node_put(dma_spec->np); + put_device(&pdev->dev); return ERR_PTR(-ENOMEM); } @@ -268,6 +272,8 @@ static void *ti_dra7_xbar_route_allocate(struct of_phandle_args *dma_spec, mutex_unlock(&xbar->mutex); dev_err(&pdev->dev, "Run out of free DMA requests\n"); kfree(map); + of_node_put(dma_spec->np); + put_device(&pdev->dev); return ERR_PTR(-ENOMEM); } set_bit(map->xbar_out, xbar->dma_inuse); diff --git a/drivers/dma/ti/edma.c b/drivers/dma/ti/edma.c index 35d81bd857f1..552be71db6c4 100644 --- a/drivers/dma/ti/edma.c +++ b/drivers/dma/ti/edma.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * TI EDMA DMA engine driver * * Copyright 2012 Texas Instruments - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation version 2. - * - * This program is distributed "as is" WITHOUT ANY WARRANTY of any - * kind, whether express or implied; without even the implied warranty - * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. */ #include <linux/dmaengine.h> @@ -24,11 +16,11 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/string_choices.h> #include <linux/of.h> #include <linux/of_dma.h> #include <linux/of_irq.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/pm_runtime.h> #include <linux/platform_data/edma.h> @@ -118,10 +110,10 @@ /* * Max of 20 segments per channel to conserve PaRAM slots - * Also note that MAX_NR_SG should be atleast the no.of periods + * Also note that MAX_NR_SG should be at least the no.of periods * that are required for ASoC, otherwise DMA prep calls will * fail. Today davinci-pcm is the only user of this driver and - * requires atleast 17 slots, so we setup the default to 20. + * requires at least 17 slots, so we setup the default to 20. */ #define MAX_NR_SG 20 #define EDMA_MAX_SLOTS MAX_NR_SG @@ -211,13 +203,12 @@ struct edma_desc { u32 residue; u32 residue_stat; - struct edma_pset pset[]; + struct edma_pset pset[] __counted_by(pset_nr); }; struct edma_cc; struct edma_tc { - struct device_node *node; u16 id; }; @@ -326,14 +317,6 @@ static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and, edma_write(ecc, offset, val); } -static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and) -{ - unsigned val = edma_read(ecc, offset); - - val &= and; - edma_write(ecc, offset, val); -} - static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or) { unsigned val = edma_read(ecc, offset); @@ -360,12 +343,6 @@ static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i, edma_modify(ecc, offset + (i << 2), and, or); } -static inline void edma_or_array(struct edma_cc *ecc, int offset, int i, - unsigned or) -{ - edma_or(ecc, offset + (i << 2), or); -} - static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j, unsigned or) { @@ -378,11 +355,6 @@ static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i, edma_write(ecc, offset + ((i * 2 + j) << 2), val); } -static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset) -{ - return edma_read(ecc, EDMA_SHADOW0 + offset); -} - static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc, int offset, int i) { @@ -401,36 +373,12 @@ static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset, edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val); } -static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset, - int param_no) -{ - return edma_read(ecc, EDMA_PARM + offset + (param_no << 5)); -} - -static inline void edma_param_write(struct edma_cc *ecc, int offset, - int param_no, unsigned val) -{ - edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val); -} - static inline void edma_param_modify(struct edma_cc *ecc, int offset, int param_no, unsigned and, unsigned or) { edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or); } -static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no, - unsigned and) -{ - edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and); -} - -static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no, - unsigned or) -{ - edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or); -} - static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no, int priority) { @@ -751,11 +699,6 @@ static void edma_free_channel(struct edma_chan *echan) edma_setup_interrupt(echan, false); } -static inline struct edma_cc *to_edma_cc(struct dma_device *d) -{ - return container_of(d, struct edma_cc, dma_slave); -} - static inline struct edma_chan *to_edma_chan(struct dma_chan *c) { return container_of(c, struct edma_chan, vchan.chan); @@ -976,7 +919,7 @@ static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset, * and quotient respectively of the division of: * (dma_length / acnt) by (SZ_64K -1). This is so * that in case bcnt over flows, we have ccnt to use. - * Note: In A-sync tranfer only, bcntrld is used, but it + * Note: In A-sync transfer only, bcntrld is used, but it * only applies for sg_dma_len(sg) >= SZ_64K. * In this case, the best way adopted is- bccnt for the * first frame will be the remainder below. Then for @@ -1203,7 +1146,7 @@ static struct dma_async_tx_descriptor *edma_prep_dma_memcpy( * slot2: the remaining amount of data after slot1. * ACNT = full_length - length1, length2 = ACNT * - * When the full_length is multibple of 32767 one slot can be + * When the full_length is a multiple of 32767 one slot can be * used to complete the transfer. */ width = array_size; @@ -1681,8 +1624,7 @@ static irqreturn_t dma_ccerr_handler(int irq, void *data) dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val); emr = val; - for (i = find_next_bit(&emr, 32, 0); i < 32; - i = find_next_bit(&emr, 32, i + 1)) { + for_each_set_bit(i, &emr, 32) { int k = (j << 5) + i; /* Clear the corresponding EMR bits */ @@ -1815,7 +1757,7 @@ static void edma_issue_pending(struct dma_chan *chan) * This limit exists to avoid a possible infinite loop when waiting for proof * that a particular transfer is completed. This limit can be hit if there * are large bursts to/from slow devices or the CPU is never able to catch - * the DMA hardware idle. On an AM335x transfering 48 bytes from the UART + * the DMA hardware idle. On an AM335x transferring 48 bytes from the UART * RX-FIFO, as many as 55 loops have been seen. */ #define EDMA_MAX_TR_WAIT_LOOPS 1000 @@ -2106,7 +2048,7 @@ static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata, dev_dbg(dev, "num_qchannels: %u\n", ecc->num_qchannels); dev_dbg(dev, "num_slots: %u\n", ecc->num_slots); dev_dbg(dev, "num_tc: %u\n", ecc->num_tc); - dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no"); + dev_dbg(dev, "chmap_exist: %s\n", str_yes_no(ecc->chmap_exist)); /* Nothing need to be done if queue priority is provided */ if (pdata->queue_priority_mapping) @@ -2122,8 +2064,8 @@ static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata, * priority. So Q0 is the highest priority queue and the last queue has * the lowest priority. */ - queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8), - GFP_KERNEL); + queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, + sizeof(*queue_priority_map), GFP_KERNEL); if (!queue_priority_map) return -ENOMEM; @@ -2317,8 +2259,12 @@ static struct dma_chan *of_edma_xlate(struct of_phandle_args *dma_spec, return NULL; out: - /* The channel is going to be used as HW synchronized */ - echan->hw_triggered = true; + /* + * The channel is going to be HW synchronized, unless it was + * reserved as a memcpy channel + */ + echan->hw_triggered = + !edma_is_memcpy_channel(i, ecc->info->memcpy_channels); return dma_get_slave_channel(chan); } #else @@ -2459,9 +2405,14 @@ static int edma_probe(struct platform_device *pdev) if (irq < 0 && node) irq = irq_of_parse_and_map(node, 0); - if (irq >= 0) { + if (irq > 0) { irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint", dev_name(dev)); + if (!irq_name) { + ret = -ENOMEM; + goto err_disable_pm; + } + ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name, ecc); if (ret) { @@ -2475,9 +2426,14 @@ static int edma_probe(struct platform_device *pdev) if (irq < 0 && node) irq = irq_of_parse_and_map(node, 2); - if (irq >= 0) { + if (irq > 0) { irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint", dev_name(dev)); + if (!irq_name) { + ret = -ENOMEM; + goto err_disable_pm; + } + ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name, ecc); if (ret) { @@ -2508,19 +2464,19 @@ static int edma_probe(struct platform_device *pdev) goto err_reg1; } - for (i = 0;; i++) { + for (i = 0; i < ecc->num_tc; i++) { ret = of_parse_phandle_with_fixed_args(node, "ti,tptcs", 1, i, &tc_args); - if (ret || i == ecc->num_tc) + if (ret) break; - ecc->tc_list[i].node = tc_args.np; ecc->tc_list[i].id = i; queue_priority_mapping[i][1] = tc_args.args[0]; if (queue_priority_mapping[i][1] > lowest_priority) { lowest_priority = queue_priority_mapping[i][1]; info->default_queue = i; } + of_node_put(tc_args.np); } /* See if we have optional dma-channel-mask array */ @@ -2608,7 +2564,7 @@ static void edma_cleanupp_vchan(struct dma_device *dmadev) } } -static int edma_remove(struct platform_device *pdev) +static void edma_remove(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct edma_cc *ecc = dev_get_drvdata(dev); @@ -2626,8 +2582,6 @@ static int edma_remove(struct platform_device *pdev) edma_free_slot(ecc, ecc->dummy_slot); pm_runtime_put_sync(dev); pm_runtime_disable(dev); - - return 0; } #ifdef CONFIG_PM_SLEEP diff --git a/drivers/dma/ti/k3-psil-am62.c b/drivers/dma/ti/k3-psil-am62.c new file mode 100644 index 000000000000..1272b1541f61 --- /dev/null +++ b/drivers/dma/ti/k3-psil-am62.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/kernel.h> + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = flow_base, \ + }, \ + } + +#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = default_flow, \ + .notdpkt = tx, \ + }, \ + } + +#define PSIL_PDMA_MCASP(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pdma_acc32 = 1, \ + .pdma_burst = 1, \ + }, \ + } + +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep am62_src_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0x7504, 20, 35, 8, 35, 0), + PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), + PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), + PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), + PSIL_PDMA_XY_PKT(0x4302), + PSIL_PDMA_XY_PKT(0x4303), + PSIL_PDMA_XY_PKT(0x4304), + PSIL_PDMA_XY_PKT(0x4305), + PSIL_PDMA_XY_PKT(0x4306), + PSIL_PDMA_XY_PKT(0x4307), + PSIL_PDMA_XY_PKT(0x4308), + PSIL_PDMA_XY_PKT(0x4309), + PSIL_PDMA_XY_PKT(0x430a), + PSIL_PDMA_XY_PKT(0x430b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0x4400), + PSIL_PDMA_XY_PKT(0x4401), + PSIL_PDMA_XY_PKT(0x4402), + PSIL_PDMA_XY_PKT(0x4403), + PSIL_PDMA_XY_PKT(0x4404), + PSIL_PDMA_XY_PKT(0x4405), + PSIL_PDMA_XY_PKT(0x4406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0x4500), + PSIL_PDMA_MCASP(0x4501), + PSIL_PDMA_MCASP(0x4502), + /* CPSW3G */ + PSIL_ETHERNET(0x4600, 19, 19, 16), + /* CSI2RX */ + PSIL_CSI2RX(0x4700), + PSIL_CSI2RX(0x4701), + PSIL_CSI2RX(0x4702), + PSIL_CSI2RX(0x4703), + PSIL_CSI2RX(0x4704), + PSIL_CSI2RX(0x4705), + PSIL_CSI2RX(0x4706), + PSIL_CSI2RX(0x4707), + PSIL_CSI2RX(0x4708), + PSIL_CSI2RX(0x4709), + PSIL_CSI2RX(0x470a), + PSIL_CSI2RX(0x470b), + PSIL_CSI2RX(0x470c), + PSIL_CSI2RX(0x470d), + PSIL_CSI2RX(0x470e), + PSIL_CSI2RX(0x470f), + PSIL_CSI2RX(0x4710), + PSIL_CSI2RX(0x4711), + PSIL_CSI2RX(0x4712), + PSIL_CSI2RX(0x4713), + PSIL_CSI2RX(0x4714), + PSIL_CSI2RX(0x4715), + PSIL_CSI2RX(0x4716), + PSIL_CSI2RX(0x4717), + PSIL_CSI2RX(0x4718), + PSIL_CSI2RX(0x4719), + PSIL_CSI2RX(0x471a), + PSIL_CSI2RX(0x471b), + PSIL_CSI2RX(0x471c), + PSIL_CSI2RX(0x471d), + PSIL_CSI2RX(0x471e), + PSIL_CSI2RX(0x471f), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep am62_dst_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), + PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0xc300), + PSIL_PDMA_XY_PKT(0xc301), + PSIL_PDMA_XY_PKT(0xc302), + PSIL_PDMA_XY_PKT(0xc303), + PSIL_PDMA_XY_PKT(0xc304), + PSIL_PDMA_XY_PKT(0xc305), + PSIL_PDMA_XY_PKT(0xc306), + PSIL_PDMA_XY_PKT(0xc307), + PSIL_PDMA_XY_PKT(0xc308), + PSIL_PDMA_XY_PKT(0xc309), + PSIL_PDMA_XY_PKT(0xc30a), + PSIL_PDMA_XY_PKT(0xc30b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0xc400), + PSIL_PDMA_XY_PKT(0xc401), + PSIL_PDMA_XY_PKT(0xc402), + PSIL_PDMA_XY_PKT(0xc403), + PSIL_PDMA_XY_PKT(0xc404), + PSIL_PDMA_XY_PKT(0xc405), + PSIL_PDMA_XY_PKT(0xc406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0xc500), + PSIL_PDMA_MCASP(0xc501), + PSIL_PDMA_MCASP(0xc502), + /* CPSW3G */ + PSIL_ETHERNET(0xc600, 19, 19, 8), + PSIL_ETHERNET(0xc601, 20, 27, 8), + PSIL_ETHERNET(0xc602, 21, 35, 8), + PSIL_ETHERNET(0xc603, 22, 43, 8), + PSIL_ETHERNET(0xc604, 23, 51, 8), + PSIL_ETHERNET(0xc605, 24, 59, 8), + PSIL_ETHERNET(0xc606, 25, 67, 8), + PSIL_ETHERNET(0xc607, 26, 75, 8), +}; + +struct psil_ep_map am62_ep_map = { + .name = "am62", + .src = am62_src_ep_map, + .src_count = ARRAY_SIZE(am62_src_ep_map), + .dst = am62_dst_ep_map, + .dst_count = ARRAY_SIZE(am62_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-am62a.c b/drivers/dma/ti/k3-psil-am62a.c new file mode 100644 index 000000000000..4cf9123b0e93 --- /dev/null +++ b/drivers/dma/ti/k3-psil-am62a.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/kernel.h> + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_TR(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + }, \ + } + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = flow_base, \ + }, \ + } + +#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = default_flow, \ + .notdpkt = tx, \ + }, \ + } + +#define PSIL_PDMA_MCASP(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pdma_acc32 = 1, \ + .pdma_burst = 1, \ + }, \ + } + +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep am62a_src_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0x7504, 20, 35, 8, 35, 0), + PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), + PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), + PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), + PSIL_PDMA_XY_PKT(0x4302), + PSIL_PDMA_XY_PKT(0x4303), + PSIL_PDMA_XY_PKT(0x4304), + PSIL_PDMA_XY_PKT(0x4305), + PSIL_PDMA_XY_PKT(0x4306), + PSIL_PDMA_XY_PKT(0x4307), + PSIL_PDMA_XY_PKT(0x4308), + PSIL_PDMA_XY_PKT(0x4309), + PSIL_PDMA_XY_PKT(0x430a), + PSIL_PDMA_XY_PKT(0x430b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0x4400), + PSIL_PDMA_XY_PKT(0x4401), + PSIL_PDMA_XY_PKT(0x4402), + PSIL_PDMA_XY_PKT(0x4403), + PSIL_PDMA_XY_PKT(0x4404), + PSIL_PDMA_XY_PKT(0x4405), + PSIL_PDMA_XY_PKT(0x4406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0x4500), + PSIL_PDMA_MCASP(0x4501), + PSIL_PDMA_MCASP(0x4502), + /* CPSW3G */ + PSIL_ETHERNET(0x4600, 19, 19, 16), + /* CSI2RX */ + PSIL_CSI2RX(0x5000), + PSIL_CSI2RX(0x5001), + PSIL_CSI2RX(0x5002), + PSIL_CSI2RX(0x5003), + PSIL_CSI2RX(0x5004), + PSIL_CSI2RX(0x5005), + PSIL_CSI2RX(0x5006), + PSIL_CSI2RX(0x5007), + PSIL_CSI2RX(0x5008), + PSIL_CSI2RX(0x5009), + PSIL_CSI2RX(0x500a), + PSIL_CSI2RX(0x500b), + PSIL_CSI2RX(0x500c), + PSIL_CSI2RX(0x500d), + PSIL_CSI2RX(0x500e), + PSIL_CSI2RX(0x500f), + PSIL_CSI2RX(0x5010), + PSIL_CSI2RX(0x5011), + PSIL_CSI2RX(0x5012), + PSIL_CSI2RX(0x5013), + PSIL_CSI2RX(0x5014), + PSIL_CSI2RX(0x5015), + PSIL_CSI2RX(0x5016), + PSIL_CSI2RX(0x5017), + PSIL_CSI2RX(0x5018), + PSIL_CSI2RX(0x5019), + PSIL_CSI2RX(0x501a), + PSIL_CSI2RX(0x501b), + PSIL_CSI2RX(0x501c), + PSIL_CSI2RX(0x501d), + PSIL_CSI2RX(0x501e), + PSIL_CSI2RX(0x501f), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep am62a_dst_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), + PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0xc300), + PSIL_PDMA_XY_PKT(0xc301), + PSIL_PDMA_XY_PKT(0xc302), + PSIL_PDMA_XY_PKT(0xc303), + PSIL_PDMA_XY_PKT(0xc304), + PSIL_PDMA_XY_PKT(0xc305), + PSIL_PDMA_XY_PKT(0xc306), + PSIL_PDMA_XY_PKT(0xc307), + PSIL_PDMA_XY_PKT(0xc308), + PSIL_PDMA_XY_PKT(0xc309), + PSIL_PDMA_XY_PKT(0xc30a), + PSIL_PDMA_XY_PKT(0xc30b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0xc400), + PSIL_PDMA_XY_PKT(0xc401), + PSIL_PDMA_XY_PKT(0xc402), + PSIL_PDMA_XY_PKT(0xc403), + PSIL_PDMA_XY_PKT(0xc404), + PSIL_PDMA_XY_PKT(0xc405), + PSIL_PDMA_XY_PKT(0xc406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0xc500), + PSIL_PDMA_MCASP(0xc501), + PSIL_PDMA_MCASP(0xc502), + /* CPSW3G */ + PSIL_ETHERNET(0xc600, 19, 19, 8), + PSIL_ETHERNET(0xc601, 20, 27, 8), + PSIL_ETHERNET(0xc602, 21, 35, 8), + PSIL_ETHERNET(0xc603, 22, 43, 8), + PSIL_ETHERNET(0xc604, 23, 51, 8), + PSIL_ETHERNET(0xc605, 24, 59, 8), + PSIL_ETHERNET(0xc606, 25, 67, 8), + PSIL_ETHERNET(0xc607, 26, 75, 8), +}; + +struct psil_ep_map am62a_ep_map = { + .name = "am62a", + .src = am62a_src_ep_map, + .src_count = ARRAY_SIZE(am62a_src_ep_map), + .dst = am62a_dst_ep_map, + .dst_count = ARRAY_SIZE(am62a_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-am62p.c b/drivers/dma/ti/k3-psil-am62p.c new file mode 100644 index 000000000000..0f338e16d971 --- /dev/null +++ b/drivers/dma/ti/k3-psil-am62p.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/kernel.h> + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_TR(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + }, \ + } + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .mapped_channel_id = -1, \ + .default_flow_id = -1, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x, ch, flow_base, flow_cnt) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = flow_base, \ + }, \ + } + +#define PSIL_SAUL(x, ch, flow_base, flow_cnt, default_flow, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .mapped_channel_id = ch, \ + .flow_start = flow_base, \ + .flow_num = flow_cnt, \ + .default_flow_id = default_flow, \ + .notdpkt = tx, \ + }, \ + } + +#define PSIL_PDMA_MCASP(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pdma_acc32 = 1, \ + .pdma_burst = 1, \ + }, \ + } + +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep am62p_src_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0x7504, 20, 35, 8, 35, 0), + PSIL_SAUL(0x7505, 21, 35, 8, 36, 0), + PSIL_SAUL(0x7506, 22, 43, 8, 43, 0), + PSIL_SAUL(0x7507, 23, 43, 8, 44, 0), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0x4300), + PSIL_PDMA_XY_PKT(0x4301), + PSIL_PDMA_XY_PKT(0x4302), + PSIL_PDMA_XY_PKT(0x4303), + PSIL_PDMA_XY_PKT(0x4304), + PSIL_PDMA_XY_PKT(0x4305), + PSIL_PDMA_XY_PKT(0x4306), + PSIL_PDMA_XY_PKT(0x4307), + PSIL_PDMA_XY_PKT(0x4308), + PSIL_PDMA_XY_PKT(0x4309), + PSIL_PDMA_XY_PKT(0x430a), + PSIL_PDMA_XY_PKT(0x430b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0x4400), + PSIL_PDMA_XY_PKT(0x4401), + PSIL_PDMA_XY_PKT(0x4402), + PSIL_PDMA_XY_PKT(0x4403), + PSIL_PDMA_XY_PKT(0x4404), + PSIL_PDMA_XY_PKT(0x4405), + PSIL_PDMA_XY_PKT(0x4406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0x4500), + PSIL_PDMA_MCASP(0x4501), + PSIL_PDMA_MCASP(0x4502), + /* CPSW3G */ + PSIL_ETHERNET(0x4600, 19, 19, 16), + /* CSI2RX */ + PSIL_CSI2RX(0x5000), + PSIL_CSI2RX(0x5001), + PSIL_CSI2RX(0x5002), + PSIL_CSI2RX(0x5003), + PSIL_CSI2RX(0x5004), + PSIL_CSI2RX(0x5005), + PSIL_CSI2RX(0x5006), + PSIL_CSI2RX(0x5007), + PSIL_CSI2RX(0x5008), + PSIL_CSI2RX(0x5009), + PSIL_CSI2RX(0x500a), + PSIL_CSI2RX(0x500b), + PSIL_CSI2RX(0x500c), + PSIL_CSI2RX(0x500d), + PSIL_CSI2RX(0x500e), + PSIL_CSI2RX(0x500f), + PSIL_CSI2RX(0x5010), + PSIL_CSI2RX(0x5011), + PSIL_CSI2RX(0x5012), + PSIL_CSI2RX(0x5013), + PSIL_CSI2RX(0x5014), + PSIL_CSI2RX(0x5015), + PSIL_CSI2RX(0x5016), + PSIL_CSI2RX(0x5017), + PSIL_CSI2RX(0x5018), + PSIL_CSI2RX(0x5019), + PSIL_CSI2RX(0x501a), + PSIL_CSI2RX(0x501b), + PSIL_CSI2RX(0x501c), + PSIL_CSI2RX(0x501d), + PSIL_CSI2RX(0x501e), + PSIL_CSI2RX(0x501f), + PSIL_CSI2RX(0x5000), + PSIL_CSI2RX(0x5001), + PSIL_CSI2RX(0x5002), + PSIL_CSI2RX(0x5003), + PSIL_CSI2RX(0x5004), + PSIL_CSI2RX(0x5005), + PSIL_CSI2RX(0x5006), + PSIL_CSI2RX(0x5007), + PSIL_CSI2RX(0x5008), + PSIL_CSI2RX(0x5009), + PSIL_CSI2RX(0x500a), + PSIL_CSI2RX(0x500b), + PSIL_CSI2RX(0x500c), + PSIL_CSI2RX(0x500d), + PSIL_CSI2RX(0x500e), + PSIL_CSI2RX(0x500f), + PSIL_CSI2RX(0x5010), + PSIL_CSI2RX(0x5011), + PSIL_CSI2RX(0x5012), + PSIL_CSI2RX(0x5013), + PSIL_CSI2RX(0x5014), + PSIL_CSI2RX(0x5015), + PSIL_CSI2RX(0x5016), + PSIL_CSI2RX(0x5017), + PSIL_CSI2RX(0x5018), + PSIL_CSI2RX(0x5019), + PSIL_CSI2RX(0x501a), + PSIL_CSI2RX(0x501b), + PSIL_CSI2RX(0x501c), + PSIL_CSI2RX(0x501d), + PSIL_CSI2RX(0x501e), + PSIL_CSI2RX(0x501f), + /* CSIRX 1-3 (only for J722S) */ + PSIL_CSI2RX(0x5100), + PSIL_CSI2RX(0x5101), + PSIL_CSI2RX(0x5102), + PSIL_CSI2RX(0x5103), + PSIL_CSI2RX(0x5104), + PSIL_CSI2RX(0x5105), + PSIL_CSI2RX(0x5106), + PSIL_CSI2RX(0x5107), + PSIL_CSI2RX(0x5108), + PSIL_CSI2RX(0x5109), + PSIL_CSI2RX(0x510a), + PSIL_CSI2RX(0x510b), + PSIL_CSI2RX(0x510c), + PSIL_CSI2RX(0x510d), + PSIL_CSI2RX(0x510e), + PSIL_CSI2RX(0x510f), + PSIL_CSI2RX(0x5110), + PSIL_CSI2RX(0x5111), + PSIL_CSI2RX(0x5112), + PSIL_CSI2RX(0x5113), + PSIL_CSI2RX(0x5114), + PSIL_CSI2RX(0x5115), + PSIL_CSI2RX(0x5116), + PSIL_CSI2RX(0x5117), + PSIL_CSI2RX(0x5118), + PSIL_CSI2RX(0x5119), + PSIL_CSI2RX(0x511a), + PSIL_CSI2RX(0x511b), + PSIL_CSI2RX(0x511c), + PSIL_CSI2RX(0x511d), + PSIL_CSI2RX(0x511e), + PSIL_CSI2RX(0x511f), + PSIL_CSI2RX(0x5200), + PSIL_CSI2RX(0x5201), + PSIL_CSI2RX(0x5202), + PSIL_CSI2RX(0x5203), + PSIL_CSI2RX(0x5204), + PSIL_CSI2RX(0x5205), + PSIL_CSI2RX(0x5206), + PSIL_CSI2RX(0x5207), + PSIL_CSI2RX(0x5208), + PSIL_CSI2RX(0x5209), + PSIL_CSI2RX(0x520a), + PSIL_CSI2RX(0x520b), + PSIL_CSI2RX(0x520c), + PSIL_CSI2RX(0x520d), + PSIL_CSI2RX(0x520e), + PSIL_CSI2RX(0x520f), + PSIL_CSI2RX(0x5210), + PSIL_CSI2RX(0x5211), + PSIL_CSI2RX(0x5212), + PSIL_CSI2RX(0x5213), + PSIL_CSI2RX(0x5214), + PSIL_CSI2RX(0x5215), + PSIL_CSI2RX(0x5216), + PSIL_CSI2RX(0x5217), + PSIL_CSI2RX(0x5218), + PSIL_CSI2RX(0x5219), + PSIL_CSI2RX(0x521a), + PSIL_CSI2RX(0x521b), + PSIL_CSI2RX(0x521c), + PSIL_CSI2RX(0x521d), + PSIL_CSI2RX(0x521e), + PSIL_CSI2RX(0x521f), + PSIL_CSI2RX(0x5300), + PSIL_CSI2RX(0x5301), + PSIL_CSI2RX(0x5302), + PSIL_CSI2RX(0x5303), + PSIL_CSI2RX(0x5304), + PSIL_CSI2RX(0x5305), + PSIL_CSI2RX(0x5306), + PSIL_CSI2RX(0x5307), + PSIL_CSI2RX(0x5308), + PSIL_CSI2RX(0x5309), + PSIL_CSI2RX(0x530a), + PSIL_CSI2RX(0x530b), + PSIL_CSI2RX(0x530c), + PSIL_CSI2RX(0x530d), + PSIL_CSI2RX(0x530e), + PSIL_CSI2RX(0x530f), + PSIL_CSI2RX(0x5310), + PSIL_CSI2RX(0x5311), + PSIL_CSI2RX(0x5312), + PSIL_CSI2RX(0x5313), + PSIL_CSI2RX(0x5314), + PSIL_CSI2RX(0x5315), + PSIL_CSI2RX(0x5316), + PSIL_CSI2RX(0x5317), + PSIL_CSI2RX(0x5318), + PSIL_CSI2RX(0x5319), + PSIL_CSI2RX(0x531a), + PSIL_CSI2RX(0x531b), + PSIL_CSI2RX(0x531c), + PSIL_CSI2RX(0x531d), + PSIL_CSI2RX(0x531e), + PSIL_CSI2RX(0x531f), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep am62p_dst_ep_map[] = { + /* SAUL */ + PSIL_SAUL(0xf500, 27, 83, 8, 83, 1), + PSIL_SAUL(0xf501, 28, 91, 8, 91, 1), + /* PDMA_MAIN0 - SPI0-2 */ + PSIL_PDMA_XY_PKT(0xc300), + PSIL_PDMA_XY_PKT(0xc301), + PSIL_PDMA_XY_PKT(0xc302), + PSIL_PDMA_XY_PKT(0xc303), + PSIL_PDMA_XY_PKT(0xc304), + PSIL_PDMA_XY_PKT(0xc305), + PSIL_PDMA_XY_PKT(0xc306), + PSIL_PDMA_XY_PKT(0xc307), + PSIL_PDMA_XY_PKT(0xc308), + PSIL_PDMA_XY_PKT(0xc309), + PSIL_PDMA_XY_PKT(0xc30a), + PSIL_PDMA_XY_PKT(0xc30b), + /* PDMA_MAIN1 - UART0-6 */ + PSIL_PDMA_XY_PKT(0xc400), + PSIL_PDMA_XY_PKT(0xc401), + PSIL_PDMA_XY_PKT(0xc402), + PSIL_PDMA_XY_PKT(0xc403), + PSIL_PDMA_XY_PKT(0xc404), + PSIL_PDMA_XY_PKT(0xc405), + PSIL_PDMA_XY_PKT(0xc406), + /* PDMA_MAIN2 - MCASP0-2 */ + PSIL_PDMA_MCASP(0xc500), + PSIL_PDMA_MCASP(0xc501), + PSIL_PDMA_MCASP(0xc502), + /* CPSW3G */ + PSIL_ETHERNET(0xc600, 19, 19, 8), + PSIL_ETHERNET(0xc601, 20, 27, 8), + PSIL_ETHERNET(0xc602, 21, 35, 8), + PSIL_ETHERNET(0xc603, 22, 43, 8), + PSIL_ETHERNET(0xc604, 23, 51, 8), + PSIL_ETHERNET(0xc605, 24, 59, 8), + PSIL_ETHERNET(0xc606, 25, 67, 8), + PSIL_ETHERNET(0xc607, 26, 75, 8), +}; + +struct psil_ep_map am62p_ep_map = { + .name = "am62p", + .src = am62p_src_ep_map, + .src_count = ARRAY_SIZE(am62p_src_ep_map), + .dst = am62p_dst_ep_map, + .dst_count = ARRAY_SIZE(am62p_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-j7200.c b/drivers/dma/ti/k3-psil-j7200.c index 5ea63ea74822..e3feff869991 100644 --- a/drivers/dma/ti/k3-psil-j7200.c +++ b/drivers/dma/ti/k3-psil-j7200.c @@ -143,6 +143,57 @@ static struct psil_ep j7200_src_ep_map[] = { /* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ static struct psil_ep j7200_dst_ep_map[] = { + /* PDMA_MCASP - McASP0-2 */ + PSIL_PDMA_MCASP(0xc400), + PSIL_PDMA_MCASP(0xc401), + PSIL_PDMA_MCASP(0xc402), + /* PDMA_SPI_G0 - SPI0-3 */ + PSIL_PDMA_XY_PKT(0xc600), + PSIL_PDMA_XY_PKT(0xc601), + PSIL_PDMA_XY_PKT(0xc602), + PSIL_PDMA_XY_PKT(0xc603), + PSIL_PDMA_XY_PKT(0xc604), + PSIL_PDMA_XY_PKT(0xc605), + PSIL_PDMA_XY_PKT(0xc606), + PSIL_PDMA_XY_PKT(0xc607), + PSIL_PDMA_XY_PKT(0xc608), + PSIL_PDMA_XY_PKT(0xc609), + PSIL_PDMA_XY_PKT(0xc60a), + PSIL_PDMA_XY_PKT(0xc60b), + PSIL_PDMA_XY_PKT(0xc60c), + PSIL_PDMA_XY_PKT(0xc60d), + PSIL_PDMA_XY_PKT(0xc60e), + PSIL_PDMA_XY_PKT(0xc60f), + /* PDMA_SPI_G1 - SPI4-7 */ + PSIL_PDMA_XY_PKT(0xc610), + PSIL_PDMA_XY_PKT(0xc611), + PSIL_PDMA_XY_PKT(0xc612), + PSIL_PDMA_XY_PKT(0xc613), + PSIL_PDMA_XY_PKT(0xc614), + PSIL_PDMA_XY_PKT(0xc615), + PSIL_PDMA_XY_PKT(0xc616), + PSIL_PDMA_XY_PKT(0xc617), + PSIL_PDMA_XY_PKT(0xc618), + PSIL_PDMA_XY_PKT(0xc619), + PSIL_PDMA_XY_PKT(0xc61a), + PSIL_PDMA_XY_PKT(0xc61b), + PSIL_PDMA_XY_PKT(0xc61c), + PSIL_PDMA_XY_PKT(0xc61d), + PSIL_PDMA_XY_PKT(0xc61e), + PSIL_PDMA_XY_PKT(0xc61f), + /* PDMA_USART_G0 - UART0-1 */ + PSIL_PDMA_XY_PKT(0xc700), + PSIL_PDMA_XY_PKT(0xc701), + /* PDMA_USART_G1 - UART2-3 */ + PSIL_PDMA_XY_PKT(0xc702), + PSIL_PDMA_XY_PKT(0xc703), + /* PDMA_USART_G2 - UART4-9 */ + PSIL_PDMA_XY_PKT(0xc704), + PSIL_PDMA_XY_PKT(0xc705), + PSIL_PDMA_XY_PKT(0xc706), + PSIL_PDMA_XY_PKT(0xc707), + PSIL_PDMA_XY_PKT(0xc708), + PSIL_PDMA_XY_PKT(0xc709), /* CPSW5 */ PSIL_ETHERNET(0xca00), PSIL_ETHERNET(0xca01), @@ -161,6 +212,22 @@ static struct psil_ep j7200_dst_ep_map[] = { PSIL_ETHERNET(0xf005), PSIL_ETHERNET(0xf006), PSIL_ETHERNET(0xf007), + /* MCU_PDMA_MISC_G0 - SPI0 */ + PSIL_PDMA_XY_PKT(0xf100), + PSIL_PDMA_XY_PKT(0xf101), + PSIL_PDMA_XY_PKT(0xf102), + PSIL_PDMA_XY_PKT(0xf103), + /* MCU_PDMA_MISC_G1 - SPI1-2 */ + PSIL_PDMA_XY_PKT(0xf200), + PSIL_PDMA_XY_PKT(0xf201), + PSIL_PDMA_XY_PKT(0xf202), + PSIL_PDMA_XY_PKT(0xf203), + PSIL_PDMA_XY_PKT(0xf204), + PSIL_PDMA_XY_PKT(0xf205), + PSIL_PDMA_XY_PKT(0xf206), + PSIL_PDMA_XY_PKT(0xf207), + /* MCU_PDMA_MISC_G2 - UART0 */ + PSIL_PDMA_XY_PKT(0xf300), /* SA2UL */ PSIL_SA2UL(0xf500, 1), PSIL_SA2UL(0xf501, 1), diff --git a/drivers/dma/ti/k3-psil-j721e.c b/drivers/dma/ti/k3-psil-j721e.c index 7580870ed746..e7c83d668bb6 100644 --- a/drivers/dma/ti/k3-psil-j721e.c +++ b/drivers/dma/ti/k3-psil-j721e.c @@ -58,6 +58,14 @@ }, \ } +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + /* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ static struct psil_ep j721e_src_ep_map[] = { /* SA2UL */ @@ -138,6 +146,71 @@ static struct psil_ep j721e_src_ep_map[] = { PSIL_PDMA_XY_PKT(0x4707), PSIL_PDMA_XY_PKT(0x4708), PSIL_PDMA_XY_PKT(0x4709), + /* CSI2RX */ + PSIL_CSI2RX(0x4940), + PSIL_CSI2RX(0x4941), + PSIL_CSI2RX(0x4942), + PSIL_CSI2RX(0x4943), + PSIL_CSI2RX(0x4944), + PSIL_CSI2RX(0x4945), + PSIL_CSI2RX(0x4946), + PSIL_CSI2RX(0x4947), + PSIL_CSI2RX(0x4948), + PSIL_CSI2RX(0x4949), + PSIL_CSI2RX(0x494a), + PSIL_CSI2RX(0x494b), + PSIL_CSI2RX(0x494c), + PSIL_CSI2RX(0x494d), + PSIL_CSI2RX(0x494e), + PSIL_CSI2RX(0x494f), + PSIL_CSI2RX(0x4950), + PSIL_CSI2RX(0x4951), + PSIL_CSI2RX(0x4952), + PSIL_CSI2RX(0x4953), + PSIL_CSI2RX(0x4954), + PSIL_CSI2RX(0x4955), + PSIL_CSI2RX(0x4956), + PSIL_CSI2RX(0x4957), + PSIL_CSI2RX(0x4958), + PSIL_CSI2RX(0x4959), + PSIL_CSI2RX(0x495a), + PSIL_CSI2RX(0x495b), + PSIL_CSI2RX(0x495c), + PSIL_CSI2RX(0x495d), + PSIL_CSI2RX(0x495e), + PSIL_CSI2RX(0x495f), + PSIL_CSI2RX(0x4960), + PSIL_CSI2RX(0x4961), + PSIL_CSI2RX(0x4962), + PSIL_CSI2RX(0x4963), + PSIL_CSI2RX(0x4964), + PSIL_CSI2RX(0x4965), + PSIL_CSI2RX(0x4966), + PSIL_CSI2RX(0x4967), + PSIL_CSI2RX(0x4968), + PSIL_CSI2RX(0x4969), + PSIL_CSI2RX(0x496a), + PSIL_CSI2RX(0x496b), + PSIL_CSI2RX(0x496c), + PSIL_CSI2RX(0x496d), + PSIL_CSI2RX(0x496e), + PSIL_CSI2RX(0x496f), + PSIL_CSI2RX(0x4970), + PSIL_CSI2RX(0x4971), + PSIL_CSI2RX(0x4972), + PSIL_CSI2RX(0x4973), + PSIL_CSI2RX(0x4974), + PSIL_CSI2RX(0x4975), + PSIL_CSI2RX(0x4976), + PSIL_CSI2RX(0x4977), + PSIL_CSI2RX(0x4978), + PSIL_CSI2RX(0x4979), + PSIL_CSI2RX(0x497a), + PSIL_CSI2RX(0x497b), + PSIL_CSI2RX(0x497c), + PSIL_CSI2RX(0x497d), + PSIL_CSI2RX(0x497e), + PSIL_CSI2RX(0x497f), /* CPSW9 */ PSIL_ETHERNET(0x4a00), /* CPSW0 */ @@ -193,6 +266,69 @@ static struct psil_ep j721e_dst_ep_map[] = { PSIL_ETHERNET(0xc205), PSIL_ETHERNET(0xc206), PSIL_ETHERNET(0xc207), + /* PDMA6 (PSIL_PDMA_MCASP_G0) - McASP0-2 */ + PSIL_PDMA_MCASP(0xc400), + PSIL_PDMA_MCASP(0xc401), + PSIL_PDMA_MCASP(0xc402), + /* PDMA7 (PSIL_PDMA_MCASP_G1) - McASP3-11 */ + PSIL_PDMA_MCASP(0xc500), + PSIL_PDMA_MCASP(0xc501), + PSIL_PDMA_MCASP(0xc502), + PSIL_PDMA_MCASP(0xc503), + PSIL_PDMA_MCASP(0xc504), + PSIL_PDMA_MCASP(0xc505), + PSIL_PDMA_MCASP(0xc506), + PSIL_PDMA_MCASP(0xc507), + PSIL_PDMA_MCASP(0xc508), + /* PDMA8 (PDMA_MISC_G0) - SPI0-1 */ + PSIL_PDMA_XY_PKT(0xc600), + PSIL_PDMA_XY_PKT(0xc601), + PSIL_PDMA_XY_PKT(0xc602), + PSIL_PDMA_XY_PKT(0xc603), + PSIL_PDMA_XY_PKT(0xc604), + PSIL_PDMA_XY_PKT(0xc605), + PSIL_PDMA_XY_PKT(0xc606), + PSIL_PDMA_XY_PKT(0xc607), + /* PDMA9 (PDMA_MISC_G1) - SPI2-3 */ + PSIL_PDMA_XY_PKT(0xc60c), + PSIL_PDMA_XY_PKT(0xc60d), + PSIL_PDMA_XY_PKT(0xc60e), + PSIL_PDMA_XY_PKT(0xc60f), + PSIL_PDMA_XY_PKT(0xc610), + PSIL_PDMA_XY_PKT(0xc611), + PSIL_PDMA_XY_PKT(0xc612), + PSIL_PDMA_XY_PKT(0xc613), + /* PDMA10 (PDMA_MISC_G2) - SPI4-5 */ + PSIL_PDMA_XY_PKT(0xc618), + PSIL_PDMA_XY_PKT(0xc619), + PSIL_PDMA_XY_PKT(0xc61a), + PSIL_PDMA_XY_PKT(0xc61b), + PSIL_PDMA_XY_PKT(0xc61c), + PSIL_PDMA_XY_PKT(0xc61d), + PSIL_PDMA_XY_PKT(0xc61e), + PSIL_PDMA_XY_PKT(0xc61f), + /* PDMA11 (PDMA_MISC_G3) */ + PSIL_PDMA_XY_PKT(0xc624), + PSIL_PDMA_XY_PKT(0xc625), + PSIL_PDMA_XY_PKT(0xc626), + PSIL_PDMA_XY_PKT(0xc627), + PSIL_PDMA_XY_PKT(0xc628), + PSIL_PDMA_XY_PKT(0xc629), + PSIL_PDMA_XY_PKT(0xc630), + PSIL_PDMA_XY_PKT(0xc63a), + /* PDMA13 (PDMA_USART_G0) - UART0-1 */ + PSIL_PDMA_XY_PKT(0xc700), + PSIL_PDMA_XY_PKT(0xc701), + /* PDMA14 (PDMA_USART_G1) - UART2-3 */ + PSIL_PDMA_XY_PKT(0xc702), + PSIL_PDMA_XY_PKT(0xc703), + /* PDMA15 (PDMA_USART_G2) - UART4-9 */ + PSIL_PDMA_XY_PKT(0xc704), + PSIL_PDMA_XY_PKT(0xc705), + PSIL_PDMA_XY_PKT(0xc706), + PSIL_PDMA_XY_PKT(0xc707), + PSIL_PDMA_XY_PKT(0xc708), + PSIL_PDMA_XY_PKT(0xc709), /* CPSW9 */ PSIL_ETHERNET(0xca00), PSIL_ETHERNET(0xca01), @@ -211,6 +347,22 @@ static struct psil_ep j721e_dst_ep_map[] = { PSIL_ETHERNET(0xf005), PSIL_ETHERNET(0xf006), PSIL_ETHERNET(0xf007), + /* MCU_PDMA0 (MCU_PDMA_MISC_G0) - SPI0 */ + PSIL_PDMA_XY_PKT(0xf100), + PSIL_PDMA_XY_PKT(0xf101), + PSIL_PDMA_XY_PKT(0xf102), + PSIL_PDMA_XY_PKT(0xf103), + /* MCU_PDMA1 (MCU_PDMA_MISC_G1) - SPI1-2 */ + PSIL_PDMA_XY_PKT(0xf200), + PSIL_PDMA_XY_PKT(0xf201), + PSIL_PDMA_XY_PKT(0xf202), + PSIL_PDMA_XY_PKT(0xf203), + PSIL_PDMA_XY_PKT(0xf204), + PSIL_PDMA_XY_PKT(0xf205), + PSIL_PDMA_XY_PKT(0xf206), + PSIL_PDMA_XY_PKT(0xf207), + /* MCU_PDMA2 (MCU_PDMA_MISC_G2) - UART0 */ + PSIL_PDMA_XY_PKT(0xf300), /* SA2UL */ PSIL_SA2UL(0xf500, 1), PSIL_SA2UL(0xf501, 1), diff --git a/drivers/dma/ti/k3-psil-j721s2.c b/drivers/dma/ti/k3-psil-j721s2.c new file mode 100644 index 000000000000..ba08bdcdcd2b --- /dev/null +++ b/drivers/dma/ti/k3-psil-j721s2.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/kernel.h> + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_TR(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + }, \ + } + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_PDMA_MCASP(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pdma_acc32 = 1, \ + .pdma_burst = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + }, \ + } + +#define PSIL_SA2UL(x, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .notdpkt = tx, \ + }, \ + } + +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep j721s2_src_ep_map[] = { + /* PDMA_MCASP - McASP0-4 */ + PSIL_PDMA_MCASP(0x4400), + PSIL_PDMA_MCASP(0x4401), + PSIL_PDMA_MCASP(0x4402), + PSIL_PDMA_MCASP(0x4403), + PSIL_PDMA_MCASP(0x4404), + /* PDMA_SPI_G0 - SPI0-3 */ + PSIL_PDMA_XY_PKT(0x4600), + PSIL_PDMA_XY_PKT(0x4601), + PSIL_PDMA_XY_PKT(0x4602), + PSIL_PDMA_XY_PKT(0x4603), + PSIL_PDMA_XY_PKT(0x4604), + PSIL_PDMA_XY_PKT(0x4605), + PSIL_PDMA_XY_PKT(0x4606), + PSIL_PDMA_XY_PKT(0x4607), + PSIL_PDMA_XY_PKT(0x4608), + PSIL_PDMA_XY_PKT(0x4609), + PSIL_PDMA_XY_PKT(0x460a), + PSIL_PDMA_XY_PKT(0x460b), + PSIL_PDMA_XY_PKT(0x460c), + PSIL_PDMA_XY_PKT(0x460d), + PSIL_PDMA_XY_PKT(0x460e), + PSIL_PDMA_XY_PKT(0x460f), + /* PDMA_SPI_G1 - SPI4-7 */ + PSIL_PDMA_XY_PKT(0x4610), + PSIL_PDMA_XY_PKT(0x4611), + PSIL_PDMA_XY_PKT(0x4612), + PSIL_PDMA_XY_PKT(0x4613), + PSIL_PDMA_XY_PKT(0x4614), + PSIL_PDMA_XY_PKT(0x4615), + PSIL_PDMA_XY_PKT(0x4616), + PSIL_PDMA_XY_PKT(0x4617), + PSIL_PDMA_XY_PKT(0x4618), + PSIL_PDMA_XY_PKT(0x4619), + PSIL_PDMA_XY_PKT(0x461a), + PSIL_PDMA_XY_PKT(0x461b), + PSIL_PDMA_XY_PKT(0x461c), + PSIL_PDMA_XY_PKT(0x461d), + PSIL_PDMA_XY_PKT(0x461e), + PSIL_PDMA_XY_PKT(0x461f), + /* MAIN_CPSW2G */ + PSIL_ETHERNET(0x4640), + /* PDMA_USART_G0 - UART0-1 */ + PSIL_PDMA_XY_PKT(0x4700), + PSIL_PDMA_XY_PKT(0x4701), + /* PDMA_USART_G1 - UART2-3 */ + PSIL_PDMA_XY_PKT(0x4702), + PSIL_PDMA_XY_PKT(0x4703), + /* PDMA_USART_G2 - UART4-9 */ + PSIL_PDMA_XY_PKT(0x4704), + PSIL_PDMA_XY_PKT(0x4705), + PSIL_PDMA_XY_PKT(0x4706), + PSIL_PDMA_XY_PKT(0x4707), + PSIL_PDMA_XY_PKT(0x4708), + PSIL_PDMA_XY_PKT(0x4709), + /* CSI2RX */ + PSIL_CSI2RX(0x4940), + PSIL_CSI2RX(0x4941), + PSIL_CSI2RX(0x4942), + PSIL_CSI2RX(0x4943), + PSIL_CSI2RX(0x4944), + PSIL_CSI2RX(0x4945), + PSIL_CSI2RX(0x4946), + PSIL_CSI2RX(0x4947), + PSIL_CSI2RX(0x4948), + PSIL_CSI2RX(0x4949), + PSIL_CSI2RX(0x494a), + PSIL_CSI2RX(0x494b), + PSIL_CSI2RX(0x494c), + PSIL_CSI2RX(0x494d), + PSIL_CSI2RX(0x494e), + PSIL_CSI2RX(0x494f), + PSIL_CSI2RX(0x4950), + PSIL_CSI2RX(0x4951), + PSIL_CSI2RX(0x4952), + PSIL_CSI2RX(0x4953), + PSIL_CSI2RX(0x4954), + PSIL_CSI2RX(0x4955), + PSIL_CSI2RX(0x4956), + PSIL_CSI2RX(0x4957), + PSIL_CSI2RX(0x4958), + PSIL_CSI2RX(0x4959), + PSIL_CSI2RX(0x495a), + PSIL_CSI2RX(0x495b), + PSIL_CSI2RX(0x495c), + PSIL_CSI2RX(0x495d), + PSIL_CSI2RX(0x495e), + PSIL_CSI2RX(0x495f), + PSIL_CSI2RX(0x4960), + PSIL_CSI2RX(0x4961), + PSIL_CSI2RX(0x4962), + PSIL_CSI2RX(0x4963), + PSIL_CSI2RX(0x4964), + PSIL_CSI2RX(0x4965), + PSIL_CSI2RX(0x4966), + PSIL_CSI2RX(0x4967), + PSIL_CSI2RX(0x4968), + PSIL_CSI2RX(0x4969), + PSIL_CSI2RX(0x496a), + PSIL_CSI2RX(0x496b), + PSIL_CSI2RX(0x496c), + PSIL_CSI2RX(0x496d), + PSIL_CSI2RX(0x496e), + PSIL_CSI2RX(0x496f), + PSIL_CSI2RX(0x4970), + PSIL_CSI2RX(0x4971), + PSIL_CSI2RX(0x4972), + PSIL_CSI2RX(0x4973), + PSIL_CSI2RX(0x4974), + PSIL_CSI2RX(0x4975), + PSIL_CSI2RX(0x4976), + PSIL_CSI2RX(0x4977), + PSIL_CSI2RX(0x4978), + PSIL_CSI2RX(0x4979), + PSIL_CSI2RX(0x497a), + PSIL_CSI2RX(0x497b), + PSIL_CSI2RX(0x497c), + PSIL_CSI2RX(0x497d), + PSIL_CSI2RX(0x497e), + PSIL_CSI2RX(0x497f), + /* MAIN SA2UL */ + PSIL_SA2UL(0x4a40, 0), + PSIL_SA2UL(0x4a41, 0), + PSIL_SA2UL(0x4a42, 0), + PSIL_SA2UL(0x4a43, 0), + /* CPSW0 */ + PSIL_ETHERNET(0x7000), + /* MCU_PDMA0 (MCU_PDMA_MISC_G0) - SPI0 */ + PSIL_PDMA_XY_PKT(0x7100), + PSIL_PDMA_XY_PKT(0x7101), + PSIL_PDMA_XY_PKT(0x7102), + PSIL_PDMA_XY_PKT(0x7103), + /* MCU_PDMA1 (MCU_PDMA_MISC_G1) - SPI1-2 */ + PSIL_PDMA_XY_PKT(0x7200), + PSIL_PDMA_XY_PKT(0x7201), + PSIL_PDMA_XY_PKT(0x7202), + PSIL_PDMA_XY_PKT(0x7203), + PSIL_PDMA_XY_PKT(0x7204), + PSIL_PDMA_XY_PKT(0x7205), + PSIL_PDMA_XY_PKT(0x7206), + PSIL_PDMA_XY_PKT(0x7207), + /* MCU_PDMA2 (MCU_PDMA_MISC_G2) - UART0 */ + PSIL_PDMA_XY_PKT(0x7300), + /* MCU_PDMA_ADC - ADC0-1 */ + PSIL_PDMA_XY_TR(0x7400), + PSIL_PDMA_XY_TR(0x7401), + PSIL_PDMA_XY_TR(0x7402), + PSIL_PDMA_XY_TR(0x7403), + /* SA2UL */ + PSIL_SA2UL(0x7500, 0), + PSIL_SA2UL(0x7501, 0), + PSIL_SA2UL(0x7502, 0), + PSIL_SA2UL(0x7503, 0), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep j721s2_dst_ep_map[] = { + /* MAIN SA2UL */ + PSIL_SA2UL(0xca40, 1), + PSIL_SA2UL(0xca41, 1), + /* CPSW0 */ + PSIL_ETHERNET(0xf000), + PSIL_ETHERNET(0xf001), + PSIL_ETHERNET(0xf002), + PSIL_ETHERNET(0xf003), + PSIL_ETHERNET(0xf004), + PSIL_ETHERNET(0xf005), + PSIL_ETHERNET(0xf006), + PSIL_ETHERNET(0xf007), + /* MAIN_CPSW2G */ + PSIL_ETHERNET(0xc640), + PSIL_ETHERNET(0xc641), + PSIL_ETHERNET(0xc642), + PSIL_ETHERNET(0xc643), + PSIL_ETHERNET(0xc644), + PSIL_ETHERNET(0xc645), + PSIL_ETHERNET(0xc646), + PSIL_ETHERNET(0xc647), + /* SA2UL */ + PSIL_SA2UL(0xf500, 1), + PSIL_SA2UL(0xf501, 1), +}; + +struct psil_ep_map j721s2_ep_map = { + .name = "j721s2", + .src = j721s2_src_ep_map, + .src_count = ARRAY_SIZE(j721s2_src_ep_map), + .dst = j721s2_dst_ep_map, + .dst_count = ARRAY_SIZE(j721s2_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-j784s4.c b/drivers/dma/ti/k3-psil-j784s4.c new file mode 100644 index 000000000000..12bfa2478f92 --- /dev/null +++ b/drivers/dma/ti/k3-psil-j784s4.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Texas Instruments Incorporated - https://www.ti.com + */ + +#include <linux/kernel.h> + +#include "k3-psil-priv.h" + +#define PSIL_PDMA_XY_TR(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + }, \ + } + +#define PSIL_PDMA_XY_PKT(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pkt_mode = 1, \ + }, \ + } + +#define PSIL_PDMA_MCASP(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_PDMA_XY, \ + .pdma_acc32 = 1, \ + .pdma_burst = 1, \ + }, \ + } + +#define PSIL_ETHERNET(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 16, \ + }, \ + } + +#define PSIL_SA2UL(x, tx) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + .pkt_mode = 1, \ + .needs_epib = 1, \ + .psd_size = 64, \ + .notdpkt = tx, \ + }, \ + } + +#define PSIL_CSI2RX(x) \ + { \ + .thread_id = x, \ + .ep_config = { \ + .ep_type = PSIL_EP_NATIVE, \ + }, \ + } + +/* PSI-L source thread IDs, used for RX (DMA_DEV_TO_MEM) */ +static struct psil_ep j784s4_src_ep_map[] = { + /* PDMA_MCASP - McASP0-4 */ + PSIL_PDMA_MCASP(0x4400), + PSIL_PDMA_MCASP(0x4401), + PSIL_PDMA_MCASP(0x4402), + PSIL_PDMA_MCASP(0x4403), + PSIL_PDMA_MCASP(0x4404), + /* PDMA_SPI_G0 - SPI0-3 */ + PSIL_PDMA_XY_PKT(0x4600), + PSIL_PDMA_XY_PKT(0x4601), + PSIL_PDMA_XY_PKT(0x4602), + PSIL_PDMA_XY_PKT(0x4603), + PSIL_PDMA_XY_PKT(0x4604), + PSIL_PDMA_XY_PKT(0x4605), + PSIL_PDMA_XY_PKT(0x4606), + PSIL_PDMA_XY_PKT(0x4607), + PSIL_PDMA_XY_PKT(0x4608), + PSIL_PDMA_XY_PKT(0x4609), + PSIL_PDMA_XY_PKT(0x460a), + PSIL_PDMA_XY_PKT(0x460b), + PSIL_PDMA_XY_PKT(0x460c), + PSIL_PDMA_XY_PKT(0x460d), + PSIL_PDMA_XY_PKT(0x460e), + PSIL_PDMA_XY_PKT(0x460f), + /* PDMA_SPI_G1 - SPI4-7 */ + PSIL_PDMA_XY_PKT(0x4620), + PSIL_PDMA_XY_PKT(0x4621), + PSIL_PDMA_XY_PKT(0x4622), + PSIL_PDMA_XY_PKT(0x4623), + PSIL_PDMA_XY_PKT(0x4624), + PSIL_PDMA_XY_PKT(0x4625), + PSIL_PDMA_XY_PKT(0x4626), + PSIL_PDMA_XY_PKT(0x4627), + PSIL_PDMA_XY_PKT(0x4628), + PSIL_PDMA_XY_PKT(0x4629), + PSIL_PDMA_XY_PKT(0x462a), + PSIL_PDMA_XY_PKT(0x462b), + PSIL_PDMA_XY_PKT(0x462c), + PSIL_PDMA_XY_PKT(0x462d), + PSIL_PDMA_XY_PKT(0x462e), + PSIL_PDMA_XY_PKT(0x462f), + /* MAIN_CPSW2G */ + PSIL_ETHERNET(0x4640), + /* PDMA_USART_G0 - UART0-1 */ + PSIL_PDMA_XY_PKT(0x4700), + PSIL_PDMA_XY_PKT(0x4701), + /* PDMA_USART_G1 - UART2-3 */ + PSIL_PDMA_XY_PKT(0x4702), + PSIL_PDMA_XY_PKT(0x4703), + /* PDMA_USART_G2 - UART4-9 */ + PSIL_PDMA_XY_PKT(0x4704), + PSIL_PDMA_XY_PKT(0x4705), + PSIL_PDMA_XY_PKT(0x4706), + PSIL_PDMA_XY_PKT(0x4707), + PSIL_PDMA_XY_PKT(0x4708), + PSIL_PDMA_XY_PKT(0x4709), + /* CSI2RX */ + PSIL_CSI2RX(0x4900), + PSIL_CSI2RX(0x4901), + PSIL_CSI2RX(0x4902), + PSIL_CSI2RX(0x4903), + PSIL_CSI2RX(0x4940), + PSIL_CSI2RX(0x4941), + PSIL_CSI2RX(0x4942), + PSIL_CSI2RX(0x4943), + PSIL_CSI2RX(0x4944), + PSIL_CSI2RX(0x4945), + PSIL_CSI2RX(0x4946), + PSIL_CSI2RX(0x4947), + PSIL_CSI2RX(0x4948), + PSIL_CSI2RX(0x4949), + PSIL_CSI2RX(0x494a), + PSIL_CSI2RX(0x494b), + PSIL_CSI2RX(0x494c), + PSIL_CSI2RX(0x494d), + PSIL_CSI2RX(0x494e), + PSIL_CSI2RX(0x494f), + PSIL_CSI2RX(0x4950), + PSIL_CSI2RX(0x4951), + PSIL_CSI2RX(0x4952), + PSIL_CSI2RX(0x4953), + PSIL_CSI2RX(0x4954), + PSIL_CSI2RX(0x4955), + PSIL_CSI2RX(0x4956), + PSIL_CSI2RX(0x4957), + PSIL_CSI2RX(0x4958), + PSIL_CSI2RX(0x4959), + PSIL_CSI2RX(0x495a), + PSIL_CSI2RX(0x495b), + PSIL_CSI2RX(0x495c), + PSIL_CSI2RX(0x495d), + PSIL_CSI2RX(0x495e), + PSIL_CSI2RX(0x495f), + PSIL_CSI2RX(0x4960), + PSIL_CSI2RX(0x4961), + PSIL_CSI2RX(0x4962), + PSIL_CSI2RX(0x4963), + PSIL_CSI2RX(0x4964), + PSIL_CSI2RX(0x4965), + PSIL_CSI2RX(0x4966), + PSIL_CSI2RX(0x4967), + PSIL_CSI2RX(0x4968), + PSIL_CSI2RX(0x4969), + PSIL_CSI2RX(0x496a), + PSIL_CSI2RX(0x496b), + PSIL_CSI2RX(0x496c), + PSIL_CSI2RX(0x496d), + PSIL_CSI2RX(0x496e), + PSIL_CSI2RX(0x496f), + PSIL_CSI2RX(0x4970), + PSIL_CSI2RX(0x4971), + PSIL_CSI2RX(0x4972), + PSIL_CSI2RX(0x4973), + PSIL_CSI2RX(0x4974), + PSIL_CSI2RX(0x4975), + PSIL_CSI2RX(0x4976), + PSIL_CSI2RX(0x4977), + PSIL_CSI2RX(0x4978), + PSIL_CSI2RX(0x4979), + PSIL_CSI2RX(0x497a), + PSIL_CSI2RX(0x497b), + PSIL_CSI2RX(0x497c), + PSIL_CSI2RX(0x497d), + PSIL_CSI2RX(0x497e), + PSIL_CSI2RX(0x497f), + PSIL_CSI2RX(0x4980), + PSIL_CSI2RX(0x4981), + PSIL_CSI2RX(0x4982), + PSIL_CSI2RX(0x4983), + PSIL_CSI2RX(0x4984), + PSIL_CSI2RX(0x4985), + PSIL_CSI2RX(0x4986), + PSIL_CSI2RX(0x4987), + PSIL_CSI2RX(0x4988), + PSIL_CSI2RX(0x4989), + PSIL_CSI2RX(0x498a), + PSIL_CSI2RX(0x498b), + PSIL_CSI2RX(0x498c), + PSIL_CSI2RX(0x498d), + PSIL_CSI2RX(0x498e), + PSIL_CSI2RX(0x498f), + PSIL_CSI2RX(0x4990), + PSIL_CSI2RX(0x4991), + PSIL_CSI2RX(0x4992), + PSIL_CSI2RX(0x4993), + PSIL_CSI2RX(0x4994), + PSIL_CSI2RX(0x4995), + PSIL_CSI2RX(0x4996), + PSIL_CSI2RX(0x4997), + PSIL_CSI2RX(0x4998), + PSIL_CSI2RX(0x4999), + PSIL_CSI2RX(0x499a), + PSIL_CSI2RX(0x499b), + PSIL_CSI2RX(0x499c), + PSIL_CSI2RX(0x499d), + PSIL_CSI2RX(0x499e), + PSIL_CSI2RX(0x499f), + /* MAIN_CPSW9G */ + PSIL_ETHERNET(0x4a00), + /* MAIN-SA2UL */ + PSIL_SA2UL(0x4a40, 0), + PSIL_SA2UL(0x4a41, 0), + PSIL_SA2UL(0x4a42, 0), + PSIL_SA2UL(0x4a43, 0), + /* MCU_CPSW0 */ + PSIL_ETHERNET(0x7000), + /* MCU_PDMA0 (MCU_PDMA_MISC_G0) - SPI0 */ + PSIL_PDMA_XY_PKT(0x7100), + PSIL_PDMA_XY_PKT(0x7101), + PSIL_PDMA_XY_PKT(0x7102), + PSIL_PDMA_XY_PKT(0x7103), + /* MCU_PDMA1 (MCU_PDMA_MISC_G1) - SPI1-2 */ + PSIL_PDMA_XY_PKT(0x7200), + PSIL_PDMA_XY_PKT(0x7201), + PSIL_PDMA_XY_PKT(0x7202), + PSIL_PDMA_XY_PKT(0x7203), + PSIL_PDMA_XY_PKT(0x7204), + PSIL_PDMA_XY_PKT(0x7205), + PSIL_PDMA_XY_PKT(0x7206), + PSIL_PDMA_XY_PKT(0x7207), + /* MCU_PDMA2 (MCU_PDMA_MISC_G2) - UART0 */ + PSIL_PDMA_XY_PKT(0x7300), + /* MCU_PDMA_ADC - ADC0-1 */ + PSIL_PDMA_XY_TR(0x7400), + PSIL_PDMA_XY_TR(0x7401), + PSIL_PDMA_XY_TR(0x7402), + PSIL_PDMA_XY_TR(0x7403), + /* MCU_SA2UL */ + PSIL_SA2UL(0x7500, 0), + PSIL_SA2UL(0x7501, 0), + PSIL_SA2UL(0x7502, 0), + PSIL_SA2UL(0x7503, 0), +}; + +/* PSI-L destination thread IDs, used for TX (DMA_MEM_TO_DEV) */ +static struct psil_ep j784s4_dst_ep_map[] = { + /* MAIN_CPSW2G */ + PSIL_ETHERNET(0xc640), + PSIL_ETHERNET(0xc641), + PSIL_ETHERNET(0xc642), + PSIL_ETHERNET(0xc643), + PSIL_ETHERNET(0xc644), + PSIL_ETHERNET(0xc645), + PSIL_ETHERNET(0xc646), + PSIL_ETHERNET(0xc647), + /* MAIN_CPSW9G */ + PSIL_ETHERNET(0xca00), + PSIL_ETHERNET(0xca01), + PSIL_ETHERNET(0xca02), + PSIL_ETHERNET(0xca03), + PSIL_ETHERNET(0xca04), + PSIL_ETHERNET(0xca05), + PSIL_ETHERNET(0xca06), + PSIL_ETHERNET(0xca07), + /* MAIN-SA2UL */ + PSIL_SA2UL(0xca40, 1), + PSIL_SA2UL(0xca41, 1), + /* PDMA_SPI_G0 - SPI0-3 */ + PSIL_PDMA_XY_PKT(0xc600), + PSIL_PDMA_XY_PKT(0xc601), + PSIL_PDMA_XY_PKT(0xc602), + PSIL_PDMA_XY_PKT(0xc603), + PSIL_PDMA_XY_PKT(0xc604), + PSIL_PDMA_XY_PKT(0xc605), + PSIL_PDMA_XY_PKT(0xc606), + PSIL_PDMA_XY_PKT(0xc607), + PSIL_PDMA_XY_PKT(0xc608), + PSIL_PDMA_XY_PKT(0xc609), + PSIL_PDMA_XY_PKT(0xc60a), + PSIL_PDMA_XY_PKT(0xc60b), + PSIL_PDMA_XY_PKT(0xc60c), + PSIL_PDMA_XY_PKT(0xc60d), + PSIL_PDMA_XY_PKT(0xc60e), + PSIL_PDMA_XY_PKT(0xc60f), + /* PDMA_SPI_G1 - SPI4-7 */ + PSIL_PDMA_XY_PKT(0xc620), + PSIL_PDMA_XY_PKT(0xc621), + PSIL_PDMA_XY_PKT(0xc622), + PSIL_PDMA_XY_PKT(0xc623), + PSIL_PDMA_XY_PKT(0xc624), + PSIL_PDMA_XY_PKT(0xc625), + PSIL_PDMA_XY_PKT(0xc626), + PSIL_PDMA_XY_PKT(0xc627), + PSIL_PDMA_XY_PKT(0xc628), + PSIL_PDMA_XY_PKT(0xc629), + PSIL_PDMA_XY_PKT(0xc62a), + PSIL_PDMA_XY_PKT(0xc62b), + PSIL_PDMA_XY_PKT(0xc62c), + PSIL_PDMA_XY_PKT(0xc62d), + PSIL_PDMA_XY_PKT(0xc62e), + PSIL_PDMA_XY_PKT(0xc62f), + /* MCU_CPSW0 */ + PSIL_ETHERNET(0xf000), + PSIL_ETHERNET(0xf001), + PSIL_ETHERNET(0xf002), + PSIL_ETHERNET(0xf003), + PSIL_ETHERNET(0xf004), + PSIL_ETHERNET(0xf005), + PSIL_ETHERNET(0xf006), + PSIL_ETHERNET(0xf007), + /* MCU_PDMA_MISC_G0 - SPI0 */ + PSIL_PDMA_XY_PKT(0xf100), + PSIL_PDMA_XY_PKT(0xf101), + PSIL_PDMA_XY_PKT(0xf102), + PSIL_PDMA_XY_PKT(0xf103), + /* MCU_PDMA_MISC_G1 - SPI1-2 */ + PSIL_PDMA_XY_PKT(0xf200), + PSIL_PDMA_XY_PKT(0xf201), + PSIL_PDMA_XY_PKT(0xf202), + PSIL_PDMA_XY_PKT(0xf203), + PSIL_PDMA_XY_PKT(0xf204), + PSIL_PDMA_XY_PKT(0xf205), + PSIL_PDMA_XY_PKT(0xf206), + PSIL_PDMA_XY_PKT(0xf207), + /* MCU_SA2UL */ + PSIL_SA2UL(0xf500, 1), + PSIL_SA2UL(0xf501, 1), +}; + +struct psil_ep_map j784s4_ep_map = { + .name = "j784s4", + .src = j784s4_src_ep_map, + .src_count = ARRAY_SIZE(j784s4_src_ep_map), + .dst = j784s4_dst_ep_map, + .dst_count = ARRAY_SIZE(j784s4_dst_ep_map), +}; diff --git a/drivers/dma/ti/k3-psil-priv.h b/drivers/dma/ti/k3-psil-priv.h index b74e192e3c2d..a577be97e344 100644 --- a/drivers/dma/ti/k3-psil-priv.h +++ b/drivers/dma/ti/k3-psil-priv.h @@ -41,5 +41,10 @@ extern struct psil_ep_map am654_ep_map; extern struct psil_ep_map j721e_ep_map; extern struct psil_ep_map j7200_ep_map; extern struct psil_ep_map am64_ep_map; +extern struct psil_ep_map j721s2_ep_map; +extern struct psil_ep_map am62_ep_map; +extern struct psil_ep_map am62a_ep_map; +extern struct psil_ep_map j784s4_ep_map; +extern struct psil_ep_map am62p_ep_map; #endif /* K3_PSIL_PRIV_H_ */ diff --git a/drivers/dma/ti/k3-psil.c b/drivers/dma/ti/k3-psil.c index 13ce7367d870..c4b6f0df4686 100644 --- a/drivers/dma/ti/k3-psil.c +++ b/drivers/dma/ti/k3-psil.c @@ -5,6 +5,7 @@ */ #include <linux/kernel.h> +#include <linux/module.h> #include <linux/device.h> #include <linux/init.h> #include <linux/mutex.h> @@ -21,6 +22,12 @@ static const struct soc_device_attribute k3_soc_devices[] = { { .family = "J721E", .data = &j721e_ep_map }, { .family = "J7200", .data = &j7200_ep_map }, { .family = "AM64X", .data = &am64_ep_map }, + { .family = "J721S2", .data = &j721s2_ep_map }, + { .family = "AM62X", .data = &am62_ep_map }, + { .family = "AM62AX", .data = &am62a_ep_map }, + { .family = "J784S4", .data = &j784s4_ep_map }, + { .family = "AM62PX", .data = &am62p_ep_map }, + { .family = "J722S", .data = &am62p_ep_map }, { /* sentinel */ } }; @@ -99,3 +106,5 @@ int psil_set_new_ep_config(struct device *dev, const char *name, return 0; } EXPORT_SYMBOL_GPL(psil_set_new_ep_config); +MODULE_DESCRIPTION("K3 PSI-L endpoint configuration"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 4fdd9f06b723..f87d244cc2d6 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -6,6 +6,7 @@ * */ +#include <linux/module.h> #include <linux/atomic.h> #include <linux/delay.h> #include <linux/dma-mapping.h> @@ -83,6 +84,7 @@ struct k3_udma_glue_rx_channel { struct k3_udma_glue_rx_flow *flows; u32 flow_num; u32 flows_ready; + bool single_fdq; /* one FDQ for all flows */ }; static void k3_udma_chan_dev_release(struct device *dev) @@ -110,6 +112,35 @@ static int of_k3_udma_glue_parse(struct device_node *udmax_np, return 0; } +static int of_k3_udma_glue_parse_chn_common(struct k3_udma_glue_common *common, u32 thread_id, + bool tx_chn) +{ + if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) + return -EINVAL; + + if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) + return -EINVAL; + + /* get psil endpoint config */ + common->ep_config = psil_get_ep_config(thread_id); + if (IS_ERR(common->ep_config)) { + dev_err(common->dev, + "No configuration for psi-l thread 0x%04x\n", + thread_id); + return PTR_ERR(common->ep_config); + } + + common->epib = common->ep_config->needs_epib; + common->psdata_size = common->ep_config->psd_size; + + if (tx_chn) + common->dst_thread = thread_id; + else + common->src_thread = thread_id; + + return 0; +} + static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, const char *name, struct k3_udma_glue_common *common, bool tx_chn) @@ -152,38 +183,29 @@ static int of_k3_udma_glue_parse_chn(struct device_node *chn_np, common->atype_asel = dma_spec.args[1]; } - if (tx_chn && !(thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { - ret = -EINVAL; - goto out_put_spec; - } + ret = of_k3_udma_glue_parse_chn_common(common, thread_id, tx_chn); - if (!tx_chn && (thread_id & K3_PSIL_DST_THREAD_ID_OFFSET)) { - ret = -EINVAL; - goto out_put_spec; - } +out_put_spec: + of_node_put(dma_spec.np); + return ret; +} - /* get psil endpoint config */ - common->ep_config = psil_get_ep_config(thread_id); - if (IS_ERR(common->ep_config)) { - dev_err(common->dev, - "No configuration for psi-l thread 0x%04x\n", - thread_id); - ret = PTR_ERR(common->ep_config); - goto out_put_spec; - } +static int +of_k3_udma_glue_parse_chn_by_id(struct device_node *udmax_np, struct k3_udma_glue_common *common, + bool tx_chn, u32 thread_id) +{ + int ret = 0; - common->epib = common->ep_config->needs_epib; - common->psdata_size = common->ep_config->psd_size; + if (unlikely(!udmax_np)) + return -EINVAL; - if (tx_chn) - common->dst_thread = thread_id; - else - common->src_thread = thread_id; + ret = of_k3_udma_glue_parse(udmax_np, common); + if (ret) + return ret; -out_put_spec: - of_node_put(dma_spec.np); + ret = of_k3_udma_glue_parse_chn_common(common, thread_id, tx_chn); return ret; -}; +} static void k3_udma_glue_dump_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) { @@ -250,29 +272,13 @@ static int k3_udma_glue_cfg_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) return tisci_rm->tisci_udmap_ops->tx_ch_cfg(tisci_rm->tisci, &req); } -struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, - const char *name, struct k3_udma_glue_tx_channel_cfg *cfg) +static int +k3_udma_glue_request_tx_chn_common(struct device *dev, + struct k3_udma_glue_tx_channel *tx_chn, + struct k3_udma_glue_tx_channel_cfg *cfg) { - struct k3_udma_glue_tx_channel *tx_chn; int ret; - tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); - if (!tx_chn) - return ERR_PTR(-ENOMEM); - - tx_chn->common.dev = dev; - tx_chn->common.swdata_size = cfg->swdata_size; - tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; - tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; - tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; - tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; - - /* parse of udmap channel */ - ret = of_k3_udma_glue_parse_chn(dev->of_node, name, - &tx_chn->common, true); - if (ret) - goto err; - tx_chn->common.hdesc_size = cppi5_hdesc_calc_size(tx_chn->common.epib, tx_chn->common.psdata_size, tx_chn->common.swdata_size); @@ -288,7 +294,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, if (IS_ERR(tx_chn->udma_tchanx)) { ret = PTR_ERR(tx_chn->udma_tchanx); dev_err(dev, "UDMAX tchanx get err %d\n", ret); - goto err; + return ret; } tx_chn->udma_tchan_id = xudma_tchan_get_id(tx_chn->udma_tchanx); @@ -299,8 +305,9 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = device_register(&tx_chn->common.chan_dev); if (ret) { dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&tx_chn->common.chan_dev); tx_chn->common.chan_dev.parent = NULL; - goto err; + return ret; } if (xudma_is_pktdma(tx_chn->common.udmax)) { @@ -324,7 +331,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, &tx_chn->ringtxcq); if (ret) { dev_err(dev, "Failed to get TX/TXCQ rings %d\n", ret); - goto err; + return ret; } /* Set the dma_dev for the rings to be configured */ @@ -340,13 +347,13 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = k3_ringacc_ring_cfg(tx_chn->ringtx, &cfg->tx_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); - goto err; + return ret; } ret = k3_ringacc_ring_cfg(tx_chn->ringtxcq, &cfg->txcq_cfg); if (ret) { dev_err(dev, "Failed to cfg ringtx %d\n", ret); - goto err; + return ret; } /* request and cfg psi-l */ @@ -357,11 +364,42 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = k3_udma_glue_cfg_tx_chn(tx_chn); if (ret) { dev_err(dev, "Failed to cfg tchan %d\n", ret); - goto err; + return ret; } k3_udma_glue_dump_tx_chn(tx_chn); + return 0; +} + +struct k3_udma_glue_tx_channel * +k3_udma_glue_request_tx_chn(struct device *dev, const char *name, + struct k3_udma_glue_tx_channel_cfg *cfg) +{ + struct k3_udma_glue_tx_channel *tx_chn; + int ret; + + tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); + if (!tx_chn) + return ERR_PTR(-ENOMEM); + + tx_chn->common.dev = dev; + tx_chn->common.swdata_size = cfg->swdata_size; + tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; + tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; + tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; + tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; + + /* parse of udmap channel */ + ret = of_k3_udma_glue_parse_chn(dev->of_node, name, + &tx_chn->common, true); + if (ret) + goto err; + + ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); + if (ret) + goto err; + return tx_chn; err: @@ -370,6 +408,41 @@ err: } EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn); +struct k3_udma_glue_tx_channel * +k3_udma_glue_request_tx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_tx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id) +{ + struct k3_udma_glue_tx_channel *tx_chn; + int ret; + + tx_chn = devm_kzalloc(dev, sizeof(*tx_chn), GFP_KERNEL); + if (!tx_chn) + return ERR_PTR(-ENOMEM); + + tx_chn->common.dev = dev; + tx_chn->common.swdata_size = cfg->swdata_size; + tx_chn->tx_pause_on_err = cfg->tx_pause_on_err; + tx_chn->tx_filt_einfo = cfg->tx_filt_einfo; + tx_chn->tx_filt_pswords = cfg->tx_filt_pswords; + tx_chn->tx_supr_tdpkt = cfg->tx_supr_tdpkt; + + ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, &tx_chn->common, true, thread_id); + if (ret) + goto err; + + ret = k3_udma_glue_request_tx_chn_common(dev, tx_chn, cfg); + if (ret) + goto err; + + return tx_chn; + +err: + k3_udma_glue_release_tx_chn(tx_chn); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(k3_udma_glue_request_tx_chn_for_thread_id); + void k3_udma_glue_release_tx_chn(struct k3_udma_glue_tx_channel *tx_chn) { if (tx_chn->psil_paired) { @@ -556,6 +629,9 @@ int k3_udma_glue_tx_get_irq(struct k3_udma_glue_tx_channel *tx_chn) tx_chn->virq = k3_ringacc_get_ring_irq_num(tx_chn->ringtxcq); } + if (!tx_chn->virq) + return -ENXIO; + return tx_chn->virq; } EXPORT_SYMBOL_GPL(k3_udma_glue_tx_get_irq); @@ -895,10 +971,13 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, ep_cfg = rx_chn->common.ep_config; - if (xudma_is_pktdma(rx_chn->common.udmax)) + if (xudma_is_pktdma(rx_chn->common.udmax)) { rx_chn->udma_rchan_id = ep_cfg->mapped_channel_id; - else + rx_chn->single_fdq = false; + } else { rx_chn->udma_rchan_id = -1; + rx_chn->single_fdq = true; + } /* request and cfg UDMAP RX channel */ rx_chn->udma_rchanx = xudma_rchan_get(rx_chn->common.udmax, @@ -917,6 +996,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, ret = device_register(&rx_chn->common.chan_dev); if (ret) { dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&rx_chn->common.chan_dev); rx_chn->common.chan_dev.parent = NULL; goto err; } @@ -994,12 +1074,62 @@ err: return ERR_PTR(ret); } +static int +k3_udma_glue_request_remote_rx_chn_common(struct k3_udma_glue_rx_channel *rx_chn, + struct k3_udma_glue_rx_channel_cfg *cfg, + struct device *dev) +{ + int ret, i; + + rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, + rx_chn->common.psdata_size, + rx_chn->common.swdata_size); + + rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, + sizeof(*rx_chn->flows), GFP_KERNEL); + if (!rx_chn->flows) + return -ENOMEM; + + rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; + rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); + dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x-0x%02x", + rx_chn->common.src_thread, rx_chn->flow_id_base); + ret = device_register(&rx_chn->common.chan_dev); + if (ret) { + dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&rx_chn->common.chan_dev); + rx_chn->common.chan_dev.parent = NULL; + return ret; + } + + if (xudma_is_pktdma(rx_chn->common.udmax)) { + /* prepare the channel device as coherent */ + rx_chn->common.chan_dev.dma_coherent = true; + dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, + DMA_BIT_MASK(48)); + rx_chn->single_fdq = false; + } else { + rx_chn->single_fdq = true; + } + + ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); + if (ret) + return ret; + + for (i = 0; i < rx_chn->flow_num; i++) + rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; + + k3_udma_glue_dump_rx_chn(rx_chn); + + return 0; +} + static struct k3_udma_glue_rx_channel * k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, struct k3_udma_glue_rx_channel_cfg *cfg) { struct k3_udma_glue_rx_channel *rx_chn; - int ret, i; + int ret; if (cfg->flow_id_num <= 0 || cfg->flow_id_use_rxchan_id || @@ -1030,43 +1160,55 @@ k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, if (ret) goto err; - rx_chn->common.hdesc_size = cppi5_hdesc_calc_size(rx_chn->common.epib, - rx_chn->common.psdata_size, - rx_chn->common.swdata_size); - - rx_chn->flows = devm_kcalloc(dev, rx_chn->flow_num, - sizeof(*rx_chn->flows), GFP_KERNEL); - if (!rx_chn->flows) { - ret = -ENOMEM; + ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); + if (ret) goto err; - } - rx_chn->common.chan_dev.class = &k3_udma_glue_devclass; - rx_chn->common.chan_dev.parent = xudma_get_device(rx_chn->common.udmax); - dev_set_name(&rx_chn->common.chan_dev, "rchan_remote-0x%04x", - rx_chn->common.src_thread); - ret = device_register(&rx_chn->common.chan_dev); - if (ret) { - dev_err(dev, "Channel Device registration failed %d\n", ret); - rx_chn->common.chan_dev.parent = NULL; - goto err; - } + return rx_chn; - if (xudma_is_pktdma(rx_chn->common.udmax)) { - /* prepare the channel device as coherent */ - rx_chn->common.chan_dev.dma_coherent = true; - dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev, - DMA_BIT_MASK(48)); - } +err: + k3_udma_glue_release_rx_chn(rx_chn); + return ERR_PTR(ret); +} - ret = k3_udma_glue_allocate_rx_flows(rx_chn, cfg); +struct k3_udma_glue_rx_channel * +k3_udma_glue_request_remote_rx_chn_for_thread_id(struct device *dev, + struct k3_udma_glue_rx_channel_cfg *cfg, + struct device_node *udmax_np, u32 thread_id) +{ + struct k3_udma_glue_rx_channel *rx_chn; + int ret; + + if (cfg->flow_id_num <= 0 || + cfg->flow_id_use_rxchan_id || + cfg->def_flow_cfg || + cfg->flow_id_base < 0) + return ERR_PTR(-EINVAL); + + /* + * Remote RX channel is under control of Remote CPU core, so + * Linux can only request and manipulate by dedicated RX flows + */ + + rx_chn = devm_kzalloc(dev, sizeof(*rx_chn), GFP_KERNEL); + if (!rx_chn) + return ERR_PTR(-ENOMEM); + + rx_chn->common.dev = dev; + rx_chn->common.swdata_size = cfg->swdata_size; + rx_chn->remote = true; + rx_chn->udma_rchan_id = -1; + rx_chn->flow_num = cfg->flow_id_num; + rx_chn->flow_id_base = cfg->flow_id_base; + rx_chn->psil_paired = false; + + ret = of_k3_udma_glue_parse_chn_by_id(udmax_np, &rx_chn->common, false, thread_id); if (ret) goto err; - for (i = 0; i < rx_chn->flow_num; i++) - rx_chn->flows[i].udma_rflow_id = rx_chn->flow_id_base + i; - - k3_udma_glue_dump_rx_chn(rx_chn); + ret = k3_udma_glue_request_remote_rx_chn_common(rx_chn, cfg, dev); + if (ret) + goto err; return rx_chn; @@ -1074,6 +1216,7 @@ err: k3_udma_glue_release_rx_chn(rx_chn); return ERR_PTR(ret); } +EXPORT_SYMBOL_GPL(k3_udma_glue_request_remote_rx_chn_for_thread_id); struct k3_udma_glue_rx_channel * k3_udma_glue_request_rx_chn(struct device *dev, const char *name, @@ -1317,7 +1460,7 @@ EXPORT_SYMBOL_GPL(k3_udma_glue_tdown_rx_chn); void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, u32 flow_num, void *data, - void (*cleanup)(void *data, dma_addr_t desc_dma), bool skip_fdq) + void (*cleanup)(void *data, dma_addr_t desc_dma)) { struct k3_udma_glue_rx_flow *flow = &rx_chn->flows[flow_num]; struct device *dev = rx_chn->common.dev; @@ -1329,7 +1472,7 @@ void k3_udma_glue_reset_rx_chn(struct k3_udma_glue_rx_channel *rx_chn, dev_dbg(dev, "RX reset flow %u occ_rx %u\n", flow_num, occ_rx); /* Skip RX FDQ in case one FDQ is used for the set of flows */ - if (skip_fdq) + if (rx_chn->single_fdq && flow_num) goto do_reset; /* @@ -1392,6 +1535,9 @@ int k3_udma_glue_rx_get_irq(struct k3_udma_glue_rx_channel *rx_chn, flow->virq = k3_ringacc_get_ring_irq_num(flow->ringrx); } + if (!flow->virq) + return -ENXIO; + return flow->virq; } EXPORT_SYMBOL_GPL(k3_udma_glue_rx_get_irq); @@ -1433,4 +1579,7 @@ static int __init k3_udma_glue_class_init(void) { return class_register(&k3_udma_glue_devclass); } -arch_initcall(k3_udma_glue_class_init); + +module_init(k3_udma_glue_class_init); +MODULE_DESCRIPTION("TI K3 NAVSS DMA glue interface"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ti/k3-udma-private.c b/drivers/dma/ti/k3-udma-private.c index aada84f40723..05228bf00033 100644 --- a/drivers/dma/ti/k3-udma-private.c +++ b/drivers/dma/ti/k3-udma-private.c @@ -3,6 +3,8 @@ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> */ +#include <linux/of.h> +#include <linux/of_platform.h> int xudma_navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) { @@ -31,14 +33,14 @@ struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property) } pdev = of_find_device_by_node(udma_node); + if (np != udma_node) + of_node_put(udma_node); + if (!pdev) { pr_debug("UDMA device not found\n"); return ERR_PTR(-EPROBE_DEFER); } - if (np != udma_node) - of_node_put(udma_node); - ud = platform_get_drvdata(pdev); if (!ud) { pr_debug("UDMA has not been probed\n"); @@ -168,8 +170,7 @@ int xudma_pktdma_tflow_get_irq(struct udma_dev *ud, int udma_tflow_id) { const struct udma_oes_offsets *oes = &ud->soc_data->oes; - return ti_sci_inta_msi_get_virq(ud->dev, udma_tflow_id + - oes->pktdma_tchan_flow); + return msi_get_virq(ud->dev, udma_tflow_id + oes->pktdma_tchan_flow); } EXPORT_SYMBOL(xudma_pktdma_tflow_get_irq); @@ -177,7 +178,6 @@ int xudma_pktdma_rflow_get_irq(struct udma_dev *ud, int udma_rflow_id) { const struct udma_oes_offsets *oes = &ud->soc_data->oes; - return ti_sci_inta_msi_get_virq(ud->dev, udma_rflow_id + - oes->pktdma_rchan_flow); + return msi_get_virq(ud->dev, udma_rflow_id + oes->pktdma_rchan_flow); } EXPORT_SYMBOL(xudma_pktdma_rflow_get_irq); diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 96ad21869ba7..aa2dc762140f 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -5,6 +5,7 @@ */ #include <linux/kernel.h> +#include <linux/module.h> #include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> @@ -19,7 +20,6 @@ #include <linux/sys_soc.h> #include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> #include <linux/of_irq.h> #include <linux/workqueue.h> #include <linux/completion.h> @@ -134,6 +134,7 @@ struct udma_match_data { u32 flags; u32 statictr_z_mask; u8 burst_size[3]; + struct udma_soc_data *soc_data; }; struct udma_soc_data { @@ -263,6 +264,7 @@ struct udma_chan_config { enum udma_tp_level channel_tpl; /* Channel Throughput Level */ u32 tr_trigger_type; + unsigned long tx_flags; /* PKDMA mapped channel */ int mapped_channel_id; @@ -300,10 +302,10 @@ struct udma_chan { struct udma_tx_drain tx_drain; - u32 bcnt; /* number of bytes completed since the start of the channel */ - /* Channel configuration parameters */ struct udma_chan_config config; + /* Channel configuration parameters (backup) */ + struct udma_chan_config backup_config; /* dmapool for packet mode descriptors */ bool use_dma_pool; @@ -757,6 +759,21 @@ static void udma_reset_rings(struct udma_chan *uc) } } +static void udma_decrement_byte_counters(struct udma_chan *uc, u32 val) +{ + if (uc->desc->dir == DMA_DEV_TO_MEM) { + udma_rchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); + udma_rchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); + if (uc->config.ep_type != PSIL_EP_NATIVE) + udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); + } else { + udma_tchanrt_write(uc, UDMA_CHAN_RT_BCNT_REG, val); + udma_tchanrt_write(uc, UDMA_CHAN_RT_SBCNT_REG, val); + if (!uc->bchan && uc->config.ep_type != PSIL_EP_NATIVE) + udma_tchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); + } +} + static void udma_reset_counters(struct udma_chan *uc) { u32 val; @@ -790,8 +807,6 @@ static void udma_reset_counters(struct udma_chan *uc) val = udma_rchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); udma_rchanrt_write(uc, UDMA_CHAN_RT_PEER_BCNT_REG, val); } - - uc->bcnt = 0; } static int udma_reset_chan(struct udma_chan *uc, bool hard) @@ -1045,9 +1060,14 @@ static bool udma_is_desc_really_done(struct udma_chan *uc, struct udma_desc *d) { u32 peer_bcnt, bcnt; - /* Only TX towards PDMA is affected */ + /* + * Only TX towards PDMA is affected. + * If DMA_PREP_INTERRUPT is not set by consumer then skip the transfer + * completion calculation, consumer must ensure that there is no stale + * data in DMA fabric in this case. + */ if (uc->config.ep_type == PSIL_EP_NATIVE || - uc->config.dir != DMA_MEM_TO_DEV) + uc->config.dir != DMA_MEM_TO_DEV || !(uc->config.tx_flags & DMA_PREP_INTERRUPT)) return true; peer_bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_PEER_BCNT_REG); @@ -1071,8 +1091,11 @@ static void udma_check_tx_completion(struct work_struct *work) u32 residue_diff; ktime_t time_diff; unsigned long delay; + unsigned long flags; while (1) { + spin_lock_irqsave(&uc->vc.lock, flags); + if (uc->desc) { /* Get previous residue and time stamp */ residue_diff = uc->tx_drain.residue; @@ -1107,6 +1130,8 @@ static void udma_check_tx_completion(struct work_struct *work) break; } + spin_unlock_irqrestore(&uc->vc.lock, flags); + usleep_range(ktime_to_us(delay), ktime_to_us(delay) + 10); continue; @@ -1115,7 +1140,7 @@ static void udma_check_tx_completion(struct work_struct *work) if (uc->desc) { struct udma_desc *d = uc->desc; - uc->bcnt += d->residue; + udma_decrement_byte_counters(uc, d->residue); udma_start(uc); vchan_cookie_complete(&d->vd); break; @@ -1123,6 +1148,8 @@ static void udma_check_tx_completion(struct work_struct *work) break; } + + spin_unlock_irqrestore(&uc->vc.lock, flags); } static irqreturn_t udma_ring_irq_handler(int irq, void *data) @@ -1168,7 +1195,7 @@ static irqreturn_t udma_ring_irq_handler(int irq, void *data) vchan_cyclic_callback(&d->vd); } else { if (udma_is_desc_really_done(uc, d)) { - uc->bcnt += d->residue; + udma_decrement_byte_counters(uc, d->residue); udma_start(uc); vchan_cookie_complete(&d->vd); } else { @@ -1204,7 +1231,7 @@ static irqreturn_t udma_udma_irq_handler(int irq, void *data) vchan_cyclic_callback(&d->vd); } else { /* TODO: figure out the real amount of data */ - uc->bcnt += d->residue; + udma_decrement_byte_counters(uc, d->residue); udma_start(uc); vchan_cookie_complete(&d->vd); } @@ -1348,6 +1375,7 @@ static int bcdma_get_bchan(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; enum udma_tp_level tpl; + int ret; if (uc->bchan) { dev_dbg(ud->dev, "chan%d: already have bchan%d allocated\n", @@ -1365,8 +1393,11 @@ static int bcdma_get_bchan(struct udma_chan *uc) tpl = ud->bchan_tpl.levels - 1; uc->bchan = __udma_reserve_bchan(ud, tpl, -1); - if (IS_ERR(uc->bchan)) - return PTR_ERR(uc->bchan); + if (IS_ERR(uc->bchan)) { + ret = PTR_ERR(uc->bchan); + uc->bchan = NULL; + return ret; + } uc->tchan = uc->bchan; @@ -1376,6 +1407,7 @@ static int bcdma_get_bchan(struct udma_chan *uc) static int udma_get_tchan(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; + int ret; if (uc->tchan) { dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", @@ -1390,8 +1422,11 @@ static int udma_get_tchan(struct udma_chan *uc) */ uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, uc->config.mapped_channel_id); - if (IS_ERR(uc->tchan)) - return PTR_ERR(uc->tchan); + if (IS_ERR(uc->tchan)) { + ret = PTR_ERR(uc->tchan); + uc->tchan = NULL; + return ret; + } if (ud->tflow_cnt) { int tflow_id; @@ -1421,6 +1456,7 @@ static int udma_get_tchan(struct udma_chan *uc) static int udma_get_rchan(struct udma_chan *uc) { struct udma_dev *ud = uc->ud; + int ret; if (uc->rchan) { dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", @@ -1435,8 +1471,13 @@ static int udma_get_rchan(struct udma_chan *uc) */ uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, uc->config.mapped_channel_id); + if (IS_ERR(uc->rchan)) { + ret = PTR_ERR(uc->rchan); + uc->rchan = NULL; + return ret; + } - return PTR_ERR_OR_ZERO(uc->rchan); + return 0; } static int udma_get_chan_pair(struct udma_chan *uc) @@ -1490,6 +1531,7 @@ static int udma_get_chan_pair(struct udma_chan *uc) static int udma_get_rflow(struct udma_chan *uc, int flow_id) { struct udma_dev *ud = uc->ud; + int ret; if (!uc->rchan) { dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); @@ -1503,8 +1545,13 @@ static int udma_get_rflow(struct udma_chan *uc, int flow_id) } uc->rflow = __udma_get_rflow(ud, flow_id); + if (IS_ERR(uc->rflow)) { + ret = PTR_ERR(uc->rflow); + uc->rflow = NULL; + return ret; + } - return PTR_ERR_OR_ZERO(uc->rflow); + return 0; } static void bcdma_put_bchan(struct udma_chan *uc) @@ -2293,8 +2340,7 @@ static int udma_alloc_chan_resources(struct dma_chan *chan) /* Event from UDMA (TR events) only needed for slave TR mode channels */ if (is_slave_direction(uc->config.dir) && !uc->config.pkt_mode) { - uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, - irq_udma_idx); + uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); if (uc->irq_num_udma <= 0) { dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", irq_udma_idx); @@ -2466,7 +2512,7 @@ static int bcdma_alloc_chan_resources(struct dma_chan *chan) uc->psil_paired = true; } - uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); if (uc->irq_num_ring <= 0) { dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", irq_ring_idx); @@ -2483,8 +2529,7 @@ static int bcdma_alloc_chan_resources(struct dma_chan *chan) /* Event from BCDMA (TR events) only needed for slave channels */ if (is_slave_direction(uc->config.dir)) { - uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, - irq_udma_idx); + uc->irq_num_udma = msi_get_virq(ud->dev, irq_udma_idx); if (uc->irq_num_udma <= 0) { dev_err(ud->dev, "Failed to get bcdma irq (index: %u)\n", irq_udma_idx); @@ -2652,7 +2697,7 @@ static int pktdma_alloc_chan_resources(struct dma_chan *chan) uc->psil_paired = true; - uc->irq_num_ring = ti_sci_inta_msi_get_virq(ud->dev, irq_ring_idx); + uc->irq_num_ring = msi_get_virq(ud->dev, irq_ring_idx); if (uc->irq_num_ring <= 0) { dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", irq_ring_idx); @@ -2927,6 +2972,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, struct scatterlist *sgent; struct cppi5_tr_type15_t *tr_req = NULL; enum dma_slave_buswidth dev_width; + u32 csf = CPPI5_TR_CSF_SUPR_EVT; u16 tr_cnt0, tr_cnt1; dma_addr_t dev_addr; struct udma_desc *d; @@ -2997,6 +3043,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, if (uc->ud->match_data->type == DMA_TYPE_UDMA) { asel = 0; + csf |= CPPI5_TR_CSF_EOL_ICNT0; } else { asel = (u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT; dev_addr |= asel; @@ -3020,7 +3067,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[tr_idx].flags, CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); cppi5_tr_set_trigger(&tr_req[tr_idx].flags, uc->config.tr_trigger_type, CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, 0, 0); @@ -3066,8 +3113,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, cppi5_tr_init(&tr_req[tr_idx].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[tr_idx].flags, - CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[tr_idx].flags, csf); cppi5_tr_set_trigger(&tr_req[tr_idx].flags, uc->config.tr_trigger_type, CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, @@ -3111,8 +3157,7 @@ udma_prep_slave_sg_triggered_tr(struct udma_chan *uc, struct scatterlist *sgl, d->residue += sg_len; } - cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, - CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); + cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, csf | CPPI5_TR_CSF_EOP); return d; } @@ -3147,27 +3192,40 @@ static int udma_configure_statictr(struct udma_chan *uc, struct udma_desc *d, d->static_tr.elcnt = elcnt; - /* - * PDMA must to close the packet when the channel is in packet mode. - * For TR mode when the channel is not cyclic we also need PDMA to close - * the packet otherwise the transfer will stall because PDMA holds on - * the data it has received from the peripheral. - */ if (uc->config.pkt_mode || !uc->cyclic) { + /* + * PDMA must close the packet when the channel is in packet mode. + * For TR mode when the channel is not cyclic we also need PDMA + * to close the packet otherwise the transfer will stall because + * PDMA holds on the data it has received from the peripheral. + */ unsigned int div = dev_width * elcnt; if (uc->cyclic) d->static_tr.bstcnt = d->residue / d->sglen / div; else d->static_tr.bstcnt = d->residue / div; + } else if (uc->ud->match_data->type == DMA_TYPE_BCDMA && + uc->config.dir == DMA_DEV_TO_MEM && + uc->cyclic) { + /* + * For cyclic mode with BCDMA we have to set EOP in each TR to + * prevent short packet errors seen on channel teardown. So the + * PDMA must close the packet after every TR transfer by setting + * burst count equal to the number of bytes transferred. + */ + struct cppi5_tr_type1_t *tr_req = d->hwdesc[0].tr_req_base; - if (uc->config.dir == DMA_DEV_TO_MEM && - d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) - return -EINVAL; + d->static_tr.bstcnt = + (tr_req->icnt0 * tr_req->icnt1) / dev_width; } else { d->static_tr.bstcnt = 0; } + if (uc->config.dir == DMA_DEV_TO_MEM && + d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) + return -EINVAL; + return 0; } @@ -3390,6 +3448,8 @@ udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!burst) burst = 1; + uc->config.tx_flags = tx_flags; + if (uc->config.pkt_mode) d = udma_prep_slave_sg_pkt(uc, sgl, sglen, dir, tx_flags, context); @@ -3410,8 +3470,9 @@ udma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, /* static TR for remote PDMA */ if (udma_configure_statictr(uc, d, dev_width, burst)) { dev_err(uc->ud->dev, - "%s: StaticTR Z is limited to maximum 4095 (%u)\n", - __func__, d->static_tr.bstcnt); + "%s: StaticTR Z is limited to maximum %u (%u)\n", + __func__, uc->ud->match_data->statictr_z_mask, + d->static_tr.bstcnt); udma_free_hwdesc(uc, d); kfree(d); @@ -3436,6 +3497,7 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; unsigned int i; int num_tr; + u32 period_csf = 0; num_tr = udma_get_tr_counters(period_len, __ffs(buf_addr), &tr0_cnt0, &tr0_cnt1, &tr1_cnt0); @@ -3458,6 +3520,20 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, period_addr = buf_addr | ((u64)uc->config.asel << K3_ADDRESS_ASEL_SHIFT); + /* + * For BCDMA <-> PDMA transfers, the EOP flag needs to be set on the + * last TR of a descriptor, to mark the packet as complete. + * This is required for getting the teardown completion message in case + * of TX, and to avoid short-packet error in case of RX. + * + * As we are in cyclic mode, we do not know which period might be the + * last one, so set the flag for each period. + */ + if (uc->config.ep_type == PSIL_EP_PDMA_XY && + uc->ud->match_data->type == DMA_TYPE_BCDMA) { + period_csf = CPPI5_TR_CSF_EOP; + } + for (i = 0; i < periods; i++) { int tr_idx = i * num_tr; @@ -3485,8 +3561,10 @@ udma_prep_dma_cyclic_tr(struct udma_chan *uc, dma_addr_t buf_addr, } if (!(flags & DMA_PREP_INTERRUPT)) - cppi5_tr_csf_set(&tr_req[tr_idx].flags, - CPPI5_TR_CSF_SUPR_EVT); + period_csf |= CPPI5_TR_CSF_SUPR_EVT; + + if (period_csf) + cppi5_tr_csf_set(&tr_req[tr_idx].flags, period_csf); period_addr += period_len; } @@ -3615,8 +3693,9 @@ udma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, /* static TR for remote PDMA */ if (udma_configure_statictr(uc, d, dev_width, burst)) { dev_err(uc->ud->dev, - "%s: StaticTR Z is limited to maximum 4095 (%u)\n", - __func__, d->static_tr.bstcnt); + "%s: StaticTR Z is limited to maximum %u (%u)\n", + __func__, uc->ud->match_data->statictr_z_mask, + d->static_tr.bstcnt); udma_free_hwdesc(uc, d); kfree(d); @@ -3639,6 +3718,7 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, int num_tr; size_t tr_size = sizeof(struct cppi5_tr_type15_t); u16 tr0_cnt0, tr0_cnt1, tr1_cnt0; + u32 csf = CPPI5_TR_CSF_SUPR_EVT; if (uc->config.dir != DMA_MEM_TO_MEM) { dev_err(chan->device->dev, @@ -3669,13 +3749,15 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if (uc->ud->match_data->type != DMA_TYPE_UDMA) { src |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; dest |= (u64)uc->ud->asel << K3_ADDRESS_ASEL_SHIFT; + } else { + csf |= CPPI5_TR_CSF_EOL_ICNT0; } tr_req = d->hwdesc[0].tr_req_base; cppi5_tr_init(&tr_req[0].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[0].flags, CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[0].flags, csf); tr_req[0].addr = src; tr_req[0].icnt0 = tr0_cnt0; @@ -3694,7 +3776,7 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, if (num_tr == 2) { cppi5_tr_init(&tr_req[1].flags, CPPI5_TR_TYPE15, false, true, CPPI5_TR_EVENT_SIZE_COMPLETION, 0); - cppi5_tr_csf_set(&tr_req[1].flags, CPPI5_TR_CSF_SUPR_EVT); + cppi5_tr_csf_set(&tr_req[1].flags, csf); tr_req[1].addr = src + tr0_cnt1 * tr0_cnt0; tr_req[1].icnt0 = tr1_cnt0; @@ -3709,8 +3791,7 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, tr_req[1].dicnt3 = 1; } - cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, - CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP); + cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, csf | CPPI5_TR_CSF_EOP); if (uc->config.metadata_size) d->vd.tx.metadata_ops = &metadata_ops; @@ -3791,7 +3872,6 @@ static enum dma_status udma_tx_status(struct dma_chan *chan, bcnt = udma_tchanrt_read(uc, UDMA_CHAN_RT_BCNT_REG); } - bcnt -= uc->bcnt; if (bcnt && !(bcnt % uc->desc->residue)) residue = 0; else @@ -3927,6 +4007,7 @@ static void udma_desc_pre_callback(struct virt_dma_chan *vc, { struct udma_chan *uc = to_udma_chan(&vc->chan); struct udma_desc *d; + u8 status; if (!vd) return; @@ -3936,12 +4017,12 @@ static void udma_desc_pre_callback(struct virt_dma_chan *vc, if (d->metadata_size) udma_fetch_epib(uc, d); - /* Provide residue information for the client */ if (result) { void *desc_vaddr = udma_curr_cppi5_desc_vaddr(d, d->desc_idx); if (cppi5_desc_get_type(desc_vaddr) == CPPI5_INFO0_DESC_TYPE_VAL_HOST) { + /* Provide residue information for the client */ result->residue = d->residue - cppi5_hdesc_get_pktlen(desc_vaddr); if (result->residue) @@ -3950,7 +4031,12 @@ static void udma_desc_pre_callback(struct virt_dma_chan *vc, result->result = DMA_TRANS_NOERROR; } else { result->residue = 0; - result->result = DMA_TRANS_NOERROR; + /* Propagate TR Response errors to the client */ + status = d->hwdesc[0].tr_resp_base->status; + if (status) + result->result = DMA_TRANS_ABORTED; + else + result->result = DMA_TRANS_NOERROR; } } } @@ -4167,7 +4253,6 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct udma_dev *ud = ofdma->of_dma_data; - dma_cap_mask_t mask = ud->ddev.cap_mask; struct udma_filter_param filter_param; struct dma_chan *chan; @@ -4199,7 +4284,7 @@ static struct dma_chan *udma_of_xlate(struct of_phandle_args *dma_spec, } } - chan = __dma_request_channel(&mask, udma_dma_filter_fn, &filter_param, + chan = __dma_request_channel(&ud->ddev.cap_mask, udma_dma_filter_fn, &filter_param, ofdma->of_node); if (!chan) { dev_err(ud->dev, "get channel fail in %s.\n", __func__); @@ -4259,6 +4344,34 @@ static struct udma_match_data j721e_mcu_data = { }, }; +static struct udma_soc_data am62a_dmss_csi_soc_data = { + .oes = { + .bcdma_rchan_data = 0xe00, + .bcdma_rchan_ring = 0x1000, + }, +}; + +static struct udma_soc_data j721s2_bcdma_csi_soc_data = { + .oes = { + .bcdma_tchan_data = 0x800, + .bcdma_tchan_ring = 0xa00, + .bcdma_rchan_data = 0xe00, + .bcdma_rchan_ring = 0x1000, + }, +}; + +static struct udma_match_data am62a_bcdma_csirx_data = { + .type = DMA_TYPE_BCDMA, + .psil_base = 0x3100, + .enable_memcpy_support = false, + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, + .soc_data = &am62a_dmss_csi_soc_data, +}; + static struct udma_match_data am64_bcdma_data = { .type = DMA_TYPE_BCDMA, .psil_base = 0x2000, /* for tchan and rchan, not applicable to bchan */ @@ -4285,6 +4398,30 @@ static struct udma_match_data am64_pktdma_data = { }, }; +static struct udma_match_data j721s2_bcdma_csi_data = { + .type = DMA_TYPE_BCDMA, + .psil_base = 0x2000, + .enable_memcpy_support = false, + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, + .soc_data = &j721s2_bcdma_csi_soc_data, +}; + +static struct udma_match_data j722s_bcdma_csi_data = { + .type = DMA_TYPE_BCDMA, + .psil_base = 0x3100, + .enable_memcpy_support = false, + .burst_size = { + TI_SCI_RM_UDMAP_CHAN_BURST_SIZE_64_BYTES, /* Normal Channels */ + 0, /* No H Channels */ + 0, /* No UH Channels */ + }, + .soc_data = &j721s2_bcdma_csi_soc_data, +}; + static const struct of_device_id udma_of_match[] = { { .compatible = "ti,am654-navss-main-udmap", @@ -4300,24 +4437,29 @@ static const struct of_device_id udma_of_match[] = { .compatible = "ti,j721e-navss-mcu-udmap", .data = &j721e_mcu_data, }, - { /* Sentinel */ }, -}; - -static const struct of_device_id bcdma_of_match[] = { { .compatible = "ti,am64-dmss-bcdma", .data = &am64_bcdma_data, }, - { /* Sentinel */ }, -}; - -static const struct of_device_id pktdma_of_match[] = { { .compatible = "ti,am64-dmss-pktdma", .data = &am64_pktdma_data, }, + { + .compatible = "ti,am62a-dmss-bcdma-csirx", + .data = &am62a_bcdma_csirx_data, + }, + { + .compatible = "ti,j721s2-dmss-bcdma-csi", + .data = &j721s2_bcdma_csi_data, + }, + { + .compatible = "ti,j722s-dmss-bcdma-csi", + .data = &j722s_bcdma_csi_data, + }, { /* Sentinel */ }, }; +MODULE_DEVICE_TABLE(of, udma_of_match); static struct udma_soc_data am654_soc_data = { .oes = { @@ -4356,6 +4498,12 @@ static const struct soc_device_attribute k3_soc_devices[] = { { .family = "J721E", .data = &j721e_soc_data }, { .family = "J7200", .data = &j7200_soc_data }, { .family = "AM64X", .data = &am64_soc_data }, + { .family = "J721S2", .data = &j721e_soc_data}, + { .family = "AM62X", .data = &am64_soc_data }, + { .family = "AM62AX", .data = &am64_soc_data }, + { .family = "J784S4", .data = &j721e_soc_data }, + { .family = "AM62PX", .data = &am64_soc_data }, + { .family = "J722S", .data = &am64_soc_data }, { /* sentinel */ } }; @@ -4379,7 +4527,9 @@ static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); break; case DMA_TYPE_BCDMA: - ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2); + ud->bchan_cnt = BCDMA_CAP2_BCHAN_CNT(cap2) + + BCDMA_CAP3_HBCHAN_CNT(cap3) + + BCDMA_CAP3_UBCHAN_CNT(cap3); ud->tchan_cnt = BCDMA_CAP2_TCHAN_CNT(cap2); ud->rchan_cnt = BCDMA_CAP2_RCHAN_CNT(cap2); ud->rflow_cnt = ud->rchan_cnt; @@ -4514,45 +4664,60 @@ static int udma_setup_resources(struct udma_dev *ud) rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; if (IS_ERR(rm_res)) { bitmap_zero(ud->tchan_map, ud->tchan_cnt); + irq_res.sets = 1; } else { bitmap_fill(ud->tchan_map, ud->tchan_cnt); for (i = 0; i < rm_res->sets; i++) udma_mark_resource_ranges(ud, ud->tchan_map, &rm_res->desc[i], "tchan"); + irq_res.sets = rm_res->sets; } - irq_res.sets = rm_res->sets; /* rchan and matching default flow ranges */ rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; if (IS_ERR(rm_res)) { bitmap_zero(ud->rchan_map, ud->rchan_cnt); + irq_res.sets++; } else { bitmap_fill(ud->rchan_map, ud->rchan_cnt); for (i = 0; i < rm_res->sets; i++) udma_mark_resource_ranges(ud, ud->rchan_map, &rm_res->desc[i], "rchan"); + irq_res.sets += rm_res->sets; } - irq_res.sets += rm_res->sets; irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); + if (!irq_res.desc) + return -ENOMEM; rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; - for (i = 0; i < rm_res->sets; i++) { - irq_res.desc[i].start = rm_res->desc[i].start; - irq_res.desc[i].num = rm_res->desc[i].num; - irq_res.desc[i].start_sec = rm_res->desc[i].start_sec; - irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; + if (IS_ERR(rm_res)) { + irq_res.desc[0].start = 0; + irq_res.desc[0].num = ud->tchan_cnt; + i = 1; + } else { + for (i = 0; i < rm_res->sets; i++) { + irq_res.desc[i].start = rm_res->desc[i].start; + irq_res.desc[i].num = rm_res->desc[i].num; + irq_res.desc[i].start_sec = rm_res->desc[i].start_sec; + irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; + } } rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; - for (j = 0; j < rm_res->sets; j++, i++) { - if (rm_res->desc[j].num) { - irq_res.desc[i].start = rm_res->desc[j].start + - ud->soc_data->oes.udma_rchan; - irq_res.desc[i].num = rm_res->desc[j].num; - } - if (rm_res->desc[j].num_sec) { - irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + - ud->soc_data->oes.udma_rchan; - irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + if (IS_ERR(rm_res)) { + irq_res.desc[i].start = 0; + irq_res.desc[i].num = ud->rchan_cnt; + } else { + for (j = 0; j < rm_res->sets; j++, i++) { + if (rm_res->desc[j].num) { + irq_res.desc[i].start = rm_res->desc[j].start + + ud->soc_data->oes.udma_rchan; + irq_res.desc[i].num = rm_res->desc[j].num; + } + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + ud->soc_data->oes.udma_rchan; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + } } } ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); @@ -4670,14 +4835,15 @@ static int bcdma_setup_resources(struct udma_dev *ud) rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; if (IS_ERR(rm_res)) { bitmap_zero(ud->bchan_map, ud->bchan_cnt); + irq_res.sets++; } else { bitmap_fill(ud->bchan_map, ud->bchan_cnt); for (i = 0; i < rm_res->sets; i++) udma_mark_resource_ranges(ud, ud->bchan_map, &rm_res->desc[i], "bchan"); + irq_res.sets += rm_res->sets; } - irq_res.sets += rm_res->sets; } /* tchan ranges */ @@ -4685,14 +4851,15 @@ static int bcdma_setup_resources(struct udma_dev *ud) rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; if (IS_ERR(rm_res)) { bitmap_zero(ud->tchan_map, ud->tchan_cnt); + irq_res.sets += 2; } else { bitmap_fill(ud->tchan_map, ud->tchan_cnt); for (i = 0; i < rm_res->sets; i++) udma_mark_resource_ranges(ud, ud->tchan_map, &rm_res->desc[i], "tchan"); + irq_res.sets += rm_res->sets * 2; } - irq_res.sets += rm_res->sets * 2; } /* rchan ranges */ @@ -4700,47 +4867,99 @@ static int bcdma_setup_resources(struct udma_dev *ud) rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; if (IS_ERR(rm_res)) { bitmap_zero(ud->rchan_map, ud->rchan_cnt); + irq_res.sets += 2; } else { bitmap_fill(ud->rchan_map, ud->rchan_cnt); for (i = 0; i < rm_res->sets; i++) udma_mark_resource_ranges(ud, ud->rchan_map, &rm_res->desc[i], "rchan"); + irq_res.sets += rm_res->sets * 2; } - irq_res.sets += rm_res->sets * 2; } irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); + if (!irq_res.desc) + return -ENOMEM; if (ud->bchan_cnt) { rm_res = tisci_rm->rm_ranges[RM_RANGE_BCHAN]; - for (i = 0; i < rm_res->sets; i++) { - irq_res.desc[i].start = rm_res->desc[i].start + - oes->bcdma_bchan_ring; - irq_res.desc[i].num = rm_res->desc[i].num; + if (IS_ERR(rm_res)) { + irq_res.desc[0].start = oes->bcdma_bchan_ring; + irq_res.desc[0].num = ud->bchan_cnt; + i = 1; + } else { + for (i = 0; i < rm_res->sets; i++) { + irq_res.desc[i].start = rm_res->desc[i].start + + oes->bcdma_bchan_ring; + irq_res.desc[i].num = rm_res->desc[i].num; + + if (rm_res->desc[i].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[i].start_sec + + oes->bcdma_bchan_ring; + irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; + } + } } + } else { + i = 0; } + if (ud->tchan_cnt) { rm_res = tisci_rm->rm_ranges[RM_RANGE_TCHAN]; - for (j = 0; j < rm_res->sets; j++, i += 2) { - irq_res.desc[i].start = rm_res->desc[j].start + - oes->bcdma_tchan_data; - irq_res.desc[i].num = rm_res->desc[j].num; - - irq_res.desc[i + 1].start = rm_res->desc[j].start + - oes->bcdma_tchan_ring; - irq_res.desc[i + 1].num = rm_res->desc[j].num; + if (IS_ERR(rm_res)) { + irq_res.desc[i].start = oes->bcdma_tchan_data; + irq_res.desc[i].num = ud->tchan_cnt; + irq_res.desc[i + 1].start = oes->bcdma_tchan_ring; + irq_res.desc[i + 1].num = ud->tchan_cnt; + i += 2; + } else { + for (j = 0; j < rm_res->sets; j++, i += 2) { + irq_res.desc[i].start = rm_res->desc[j].start + + oes->bcdma_tchan_data; + irq_res.desc[i].num = rm_res->desc[j].num; + + irq_res.desc[i + 1].start = rm_res->desc[j].start + + oes->bcdma_tchan_ring; + irq_res.desc[i + 1].num = rm_res->desc[j].num; + + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_tchan_data; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + irq_res.desc[i + 1].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_tchan_ring; + irq_res.desc[i + 1].num_sec = rm_res->desc[j].num_sec; + } + } } } if (ud->rchan_cnt) { rm_res = tisci_rm->rm_ranges[RM_RANGE_RCHAN]; - for (j = 0; j < rm_res->sets; j++, i += 2) { - irq_res.desc[i].start = rm_res->desc[j].start + - oes->bcdma_rchan_data; - irq_res.desc[i].num = rm_res->desc[j].num; - - irq_res.desc[i + 1].start = rm_res->desc[j].start + - oes->bcdma_rchan_ring; - irq_res.desc[i + 1].num = rm_res->desc[j].num; + if (IS_ERR(rm_res)) { + irq_res.desc[i].start = oes->bcdma_rchan_data; + irq_res.desc[i].num = ud->rchan_cnt; + irq_res.desc[i + 1].start = oes->bcdma_rchan_ring; + irq_res.desc[i + 1].num = ud->rchan_cnt; + i += 2; + } else { + for (j = 0; j < rm_res->sets; j++, i += 2) { + irq_res.desc[i].start = rm_res->desc[j].start + + oes->bcdma_rchan_data; + irq_res.desc[i].num = rm_res->desc[j].num; + + irq_res.desc[i + 1].start = rm_res->desc[j].start + + oes->bcdma_rchan_ring; + irq_res.desc[i + 1].num = rm_res->desc[j].num; + + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_rchan_data; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + irq_res.desc[i + 1].start_sec = rm_res->desc[j].start_sec + + oes->bcdma_rchan_ring; + irq_res.desc[i + 1].num_sec = rm_res->desc[j].num_sec; + } + } } } @@ -4838,39 +5057,66 @@ static int pktdma_setup_resources(struct udma_dev *ud) if (IS_ERR(rm_res)) { /* all rflows are assigned exclusively to Linux */ bitmap_zero(ud->rflow_in_use, ud->rflow_cnt); + irq_res.sets = 1; } else { bitmap_fill(ud->rflow_in_use, ud->rflow_cnt); for (i = 0; i < rm_res->sets; i++) udma_mark_resource_ranges(ud, ud->rflow_in_use, &rm_res->desc[i], "rflow"); + irq_res.sets = rm_res->sets; } - irq_res.sets = rm_res->sets; /* tflow ranges */ rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; if (IS_ERR(rm_res)) { /* all tflows are assigned exclusively to Linux */ bitmap_zero(ud->tflow_map, ud->tflow_cnt); + irq_res.sets++; } else { bitmap_fill(ud->tflow_map, ud->tflow_cnt); for (i = 0; i < rm_res->sets; i++) udma_mark_resource_ranges(ud, ud->tflow_map, &rm_res->desc[i], "tflow"); + irq_res.sets += rm_res->sets; } - irq_res.sets += rm_res->sets; irq_res.desc = kcalloc(irq_res.sets, sizeof(*irq_res.desc), GFP_KERNEL); + if (!irq_res.desc) + return -ENOMEM; rm_res = tisci_rm->rm_ranges[RM_RANGE_TFLOW]; - for (i = 0; i < rm_res->sets; i++) { - irq_res.desc[i].start = rm_res->desc[i].start + - oes->pktdma_tchan_flow; - irq_res.desc[i].num = rm_res->desc[i].num; + if (IS_ERR(rm_res)) { + irq_res.desc[0].start = oes->pktdma_tchan_flow; + irq_res.desc[0].num = ud->tflow_cnt; + i = 1; + } else { + for (i = 0; i < rm_res->sets; i++) { + irq_res.desc[i].start = rm_res->desc[i].start + + oes->pktdma_tchan_flow; + irq_res.desc[i].num = rm_res->desc[i].num; + + if (rm_res->desc[i].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[i].start_sec + + oes->pktdma_tchan_flow; + irq_res.desc[i].num_sec = rm_res->desc[i].num_sec; + } + } } rm_res = tisci_rm->rm_ranges[RM_RANGE_RFLOW]; - for (j = 0; j < rm_res->sets; j++, i++) { - irq_res.desc[i].start = rm_res->desc[j].start + - oes->pktdma_rchan_flow; - irq_res.desc[i].num = rm_res->desc[j].num; + if (IS_ERR(rm_res)) { + irq_res.desc[i].start = oes->pktdma_rchan_flow; + irq_res.desc[i].num = ud->rflow_cnt; + } else { + for (j = 0; j < rm_res->sets; j++, i++) { + irq_res.desc[i].start = rm_res->desc[j].start + + oes->pktdma_rchan_flow; + irq_res.desc[i].num = rm_res->desc[j].num; + + if (rm_res->desc[j].num_sec) { + irq_res.desc[i].start_sec = rm_res->desc[j].start_sec + + oes->pktdma_rchan_flow; + irq_res.desc[i].num_sec = rm_res->desc[j].num_sec; + } + } } ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); kfree(irq_res.desc); @@ -4948,6 +5194,7 @@ static int setup_resources(struct udma_dev *ud) ud->tchan_cnt), ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt)); + break; default: break; } @@ -5176,23 +5423,21 @@ static int udma_probe(struct platform_device *pdev) return -ENOMEM; match = of_match_node(udma_of_match, dev->of_node); - if (!match) - match = of_match_node(bcdma_of_match, dev->of_node); if (!match) { - match = of_match_node(pktdma_of_match, dev->of_node); - if (!match) { - dev_err(dev, "No compatible match found\n"); - return -ENODEV; - } + dev_err(dev, "No compatible match found\n"); + return -ENODEV; } ud->match_data = match->data; - soc = soc_device_match(k3_soc_devices); - if (!soc) { - dev_err(dev, "No compatible SoC found\n"); - return -ENODEV; + ud->soc_data = ud->match_data->soc_data; + if (!ud->soc_data) { + soc = soc_device_match(k3_soc_devices); + if (!soc) { + dev_err(dev, "No compatible SoC found\n"); + return -ENODEV; + } + ud->soc_data = soc->data; } - ud->soc_data = soc->data; ret = udma_get_mmrs(pdev, ud); if (ret) @@ -5258,10 +5503,9 @@ static int udma_probe(struct platform_device *pdev) if (IS_ERR(ud->ringacc)) return PTR_ERR(ud->ringacc); - dev->msi_domain = of_msi_get_domain(dev, dev->of_node, + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_TI_SCI_INTA_MSI); - if (!dev->msi_domain) { - dev_err(dev, "Failed to get MSI domain\n"); + if (!dev->msi.domain) { return -EPROBE_DEFER; } @@ -5380,7 +5624,8 @@ static int udma_probe(struct platform_device *pdev) uc->config.dir = DMA_MEM_TO_MEM; uc->name = devm_kasprintf(dev, GFP_KERNEL, "%s chan%d", dev_name(dev), i); - + if (!uc->name) + return -ENOMEM; vchan_init(&uc->vc, &ud->ddev); /* Use custom vchan completion handling */ tasklet_setup(&uc->vc.task, udma_vchan_complete); @@ -5408,35 +5653,70 @@ static int udma_probe(struct platform_device *pdev) return ret; } +static int __maybe_unused udma_pm_suspend(struct device *dev) +{ + struct udma_dev *ud = dev_get_drvdata(dev); + struct dma_device *dma_dev = &ud->ddev; + struct dma_chan *chan; + struct udma_chan *uc; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + if (chan->client_count) { + uc = to_udma_chan(chan); + /* backup the channel configuration */ + memcpy(&uc->backup_config, &uc->config, + sizeof(struct udma_chan_config)); + dev_dbg(dev, "Suspending channel %s\n", + dma_chan_name(chan)); + ud->ddev.device_free_chan_resources(chan); + } + } + + return 0; +} + +static int __maybe_unused udma_pm_resume(struct device *dev) +{ + struct udma_dev *ud = dev_get_drvdata(dev); + struct dma_device *dma_dev = &ud->ddev; + struct dma_chan *chan; + struct udma_chan *uc; + int ret; + + list_for_each_entry(chan, &dma_dev->channels, device_node) { + if (chan->client_count) { + uc = to_udma_chan(chan); + /* restore the channel configuration */ + memcpy(&uc->config, &uc->backup_config, + sizeof(struct udma_chan_config)); + dev_dbg(dev, "Resuming channel %s\n", + dma_chan_name(chan)); + ret = ud->ddev.device_alloc_chan_resources(chan); + if (ret) + return ret; + } + } + + return 0; +} + +static const struct dev_pm_ops udma_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(udma_pm_suspend, udma_pm_resume) +}; + static struct platform_driver udma_driver = { .driver = { .name = "ti-udma", .of_match_table = udma_of_match, .suppress_bind_attrs = true, + .pm = &udma_pm_ops, }, .probe = udma_probe, }; -builtin_platform_driver(udma_driver); -static struct platform_driver bcdma_driver = { - .driver = { - .name = "ti-bcdma", - .of_match_table = bcdma_of_match, - .suppress_bind_attrs = true, - }, - .probe = udma_probe, -}; -builtin_platform_driver(bcdma_driver); - -static struct platform_driver pktdma_driver = { - .driver = { - .name = "ti-pktdma", - .of_match_table = pktdma_of_match, - .suppress_bind_attrs = true, - }, - .probe = udma_probe, -}; -builtin_platform_driver(pktdma_driver); +module_platform_driver(udma_driver); +MODULE_DESCRIPTION("Texas Instruments UDMA support"); +MODULE_LICENSE("GPL v2"); /* Private interfaces to UDMA */ #include "k3-udma-private.c" diff --git a/drivers/dma/ti/k3-udma.h b/drivers/dma/ti/k3-udma.h index d349c6d482ae..9062a237cd16 100644 --- a/drivers/dma/ti/k3-udma.h +++ b/drivers/dma/ti/k3-udma.h @@ -131,7 +131,6 @@ int xudma_navss_psil_unpair(struct udma_dev *ud, u32 src_thread, struct udma_dev *of_xudma_dev_get(struct device_node *np, const char *property); struct device *xudma_get_device(struct udma_dev *ud); struct k3_ringacc *xudma_get_ringacc(struct udma_dev *ud); -void xudma_dev_put(struct udma_dev *ud); u32 xudma_dev_get_psil_base(struct udma_dev *ud); struct udma_tisci_rm *xudma_dev_get_tisci_rm(struct udma_dev *ud); diff --git a/drivers/dma/ti/omap-dma.c b/drivers/dma/ti/omap-dma.c index 7cb577e6587b..8c023c6e623a 100644 --- a/drivers/dma/ti/omap-dma.c +++ b/drivers/dma/ti/omap-dma.c @@ -16,8 +16,8 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_device.h> #include "../virt-dma.h" @@ -124,7 +124,7 @@ struct omap_desc { uint32_t csdp; /* CSDP value */ unsigned sglen; - struct omap_sg sg[]; + struct omap_sg sg[] __counted_by(sglen); }; enum { @@ -699,6 +699,11 @@ static void omap_dma_put_lch(struct omap_dmadev *od, int lch) mutex_unlock(&od->lch_lock); } +static inline bool omap_dma_legacy(struct omap_dmadev *od) +{ + return IS_ENABLED(CONFIG_ARCH_OMAP1) && od->legacy; +} + static int omap_dma_alloc_chan_resources(struct dma_chan *chan) { struct omap_dmadev *od = to_omap_dma_dev(chan->device); @@ -706,7 +711,7 @@ static int omap_dma_alloc_chan_resources(struct dma_chan *chan) struct device *dev = od->ddev.dev; int ret; - if (od->legacy) { + if (omap_dma_legacy(od)) { ret = omap_request_dma(c->dma_sig, "DMA engine", omap_dma_callback, c, &c->dma_ch); } else { @@ -718,7 +723,7 @@ static int omap_dma_alloc_chan_resources(struct dma_chan *chan) if (ret >= 0) { omap_dma_assign(od, c, c->dma_ch); - if (!od->legacy) { + if (!omap_dma_legacy(od)) { unsigned val; spin_lock_irq(&od->irq_lock); @@ -757,7 +762,7 @@ static void omap_dma_free_chan_resources(struct dma_chan *chan) struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); - if (!od->legacy) { + if (!omap_dma_legacy(od)) { spin_lock_irq(&od->irq_lock); od->irq_enable_mask &= ~BIT(c->dma_ch); omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); @@ -768,7 +773,7 @@ static void omap_dma_free_chan_resources(struct dma_chan *chan) od->lch_map[c->dma_ch] = NULL; vchan_free_chan_resources(&c->vc); - if (od->legacy) + if (omap_dma_legacy(od)) omap_free_dma(c->dma_ch); else omap_dma_put_lch(od, c->dma_ch); @@ -1000,6 +1005,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC); if (!d) return NULL; + d->sglen = sglen; d->dir = dir; d->dev_addr = dev_addr; @@ -1115,8 +1121,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( } } - d->sglen = sglen; - /* Release the dma_pool entries if one allocation failed */ if (ll_failed) { for (i = 0; i < d->sglen; i++) { @@ -1182,10 +1186,10 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->dev_addr = dev_addr; d->fi = burst; d->es = es; + d->sglen = 1; d->sg[0].addr = buf_addr; d->sg[0].en = period_len / es_bytes[es]; d->sg[0].fn = buf_len / period_len; - d->sglen = 1; d->ccr = c->ccr; if (dir == DMA_DEV_TO_MEM) @@ -1254,10 +1258,10 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy( d->dev_addr = src; d->fi = 0; d->es = data_type; + d->sglen = 1; d->sg[0].en = len / BIT(data_type); d->sg[0].fn = 1; d->sg[0].addr = dest; - d->sglen = 1; d->ccr = c->ccr; d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC; @@ -1305,6 +1309,7 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved( if (data_type > CSDP_DATA_TYPE_32) data_type = CSDP_DATA_TYPE_32; + d->sglen = 1; sg = &d->sg[0]; d->dir = DMA_MEM_TO_MEM; d->dev_addr = xt->src_start; @@ -1312,7 +1317,6 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved( sg->en = xt->sgl[0].size / BIT(data_type); sg->fn = xt->numf; sg->addr = xt->dst_start; - d->sglen = 1; d->ccr = c->ccr; src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); @@ -1442,7 +1446,7 @@ static int omap_dma_pause(struct dma_chan *chan) * A source-synchronised channel is one where the fetching of data is * under control of the device. In other words, a device-to-memory * transfer. So, a destination-synchronised channel (which would be a - * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE + * memory-to-device transfer) undergoes an abort if the CCR_ENABLE * bit is cleared. * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel * aborts immediately after completion of current read/write @@ -1653,7 +1657,6 @@ static int omap_dma_probe(struct platform_device *pdev) { const struct omap_dma_config *conf; struct omap_dmadev *od; - struct resource *res; int rc, i, irq; u32 val; @@ -1661,8 +1664,7 @@ static int omap_dma_probe(struct platform_device *pdev) if (!od) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - od->base = devm_ioremap_resource(&pdev->dev, res); + od->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(od->base)) return PTR_ERR(od->base); @@ -1674,12 +1676,14 @@ static int omap_dma_probe(struct platform_device *pdev) dev_err(&pdev->dev, "omap_system_dma_plat_info is missing"); return -ENODEV; } - } else { + } else if (IS_ENABLED(CONFIG_ARCH_OMAP1)) { od->cfg = &default_cfg; od->plat = omap_get_plat_info(); if (!od->plat) return -EPROBE_DEFER; + } else { + return -ENODEV; } od->reg_map = od->plat->reg_map; @@ -1839,7 +1843,7 @@ static int omap_dma_probe(struct platform_device *pdev) return rc; } -static int omap_dma_remove(struct platform_device *pdev) +static void omap_dma_remove(struct platform_device *pdev) { struct omap_dmadev *od = platform_get_drvdata(pdev); int irq; @@ -1855,7 +1859,7 @@ static int omap_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&od->ddev); - if (!od->legacy) { + if (!omap_dma_legacy(od)) { /* Disable all interrupts */ omap_dma_glbl_write(od, IRQENABLE_L0, 0); } @@ -1864,8 +1868,6 @@ static int omap_dma_remove(struct platform_device *pdev) dma_pool_destroy(od->desc_pool); omap_dma_free(od); - - return 0; } static const struct omap_dma_config omap2420_data = { @@ -1913,7 +1915,7 @@ MODULE_DEVICE_TABLE(of, omap_dma_match); static struct platform_driver omap_dma_driver = { .probe = omap_dma_probe, - .remove = omap_dma_remove, + .remove = omap_dma_remove, .driver = { .name = "omap-dma-engine", .of_match_table = omap_dma_match, @@ -1948,4 +1950,5 @@ static void __exit omap_dma_exit(void) module_exit(omap_dma_exit); MODULE_AUTHOR("Russell King"); +MODULE_DESCRIPTION("Texas Instruments sDMA DMAengine support"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index 3f524be69efb..ecaf002558af 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -740,7 +740,7 @@ err_release_region: } -static int td_remove(struct platform_device *pdev) +static void td_remove(struct platform_device *pdev) { struct timb_dma *td = platform_get_drvdata(pdev); struct resource *iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); @@ -754,7 +754,6 @@ static int td_remove(struct platform_device *pdev) release_mem_region(iomem->start, resource_size(iomem)); dev_dbg(&pdev->dev, "Removed...\n"); - return 0; } static struct platform_driver td_driver = { @@ -762,7 +761,7 @@ static struct platform_driver td_driver = { .name = DRIVER_NAME, }, .probe = td_probe, - .remove = td_remove, + .remove = td_remove, }; module_platform_driver(td_driver); diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 5b6b375a257e..35d5221683b2 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -1151,7 +1151,7 @@ static int __init txx9dmac_chan_probe(struct platform_device *pdev) return 0; } -static int txx9dmac_chan_remove(struct platform_device *pdev) +static void txx9dmac_chan_remove(struct platform_device *pdev) { struct txx9dmac_chan *dc = platform_get_drvdata(pdev); @@ -1162,7 +1162,6 @@ static int txx9dmac_chan_remove(struct platform_device *pdev) tasklet_kill(&dc->tasklet); } dc->ddev->chan[pdev->id % TXX9_DMA_MAX_NR_CHANNELS] = NULL; - return 0; } static int __init txx9dmac_probe(struct platform_device *pdev) @@ -1215,7 +1214,7 @@ static int __init txx9dmac_probe(struct platform_device *pdev) return 0; } -static int txx9dmac_remove(struct platform_device *pdev) +static void txx9dmac_remove(struct platform_device *pdev) { struct txx9dmac_dev *ddev = platform_get_drvdata(pdev); @@ -1224,7 +1223,6 @@ static int txx9dmac_remove(struct platform_device *pdev) devm_free_irq(&pdev->dev, ddev->irq, ddev); tasklet_kill(&ddev->tasklet); } - return 0; } static void txx9dmac_shutdown(struct platform_device *pdev) diff --git a/drivers/dma/uniphier-mdmac.c b/drivers/dma/uniphier-mdmac.c index 618839df0748..7a99f86ecb5a 100644 --- a/drivers/dma/uniphier-mdmac.c +++ b/drivers/dma/uniphier-mdmac.c @@ -453,7 +453,7 @@ disable_clk: return ret; } -static int uniphier_mdmac_remove(struct platform_device *pdev) +static void uniphier_mdmac_remove(struct platform_device *pdev) { struct uniphier_mdmac_device *mdev = platform_get_drvdata(pdev); struct dma_chan *chan; @@ -468,16 +468,21 @@ static int uniphier_mdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &mdev->ddev.channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *mdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } uniphier_mdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&mdev->ddev); clk_disable_unprepare(mdev->clk); - - return 0; } static const struct of_device_id uniphier_mdmac_match[] = { diff --git a/drivers/dma/uniphier-xdmac.c b/drivers/dma/uniphier-xdmac.c index 16b19654873d..ceeb6171c9d1 100644 --- a/drivers/dma/uniphier-xdmac.c +++ b/drivers/dma/uniphier-xdmac.c @@ -80,7 +80,7 @@ struct uniphier_xdmac_desc { unsigned int nr_node; unsigned int cur_node; enum dma_transfer_direction dir; - struct uniphier_xdmac_desc_node nodes[]; + struct uniphier_xdmac_desc_node nodes[] __counted_by(nr_node); }; struct uniphier_xdmac_chan { @@ -97,7 +97,7 @@ struct uniphier_xdmac_device { struct dma_device ddev; void __iomem *reg_base; int nr_chans; - struct uniphier_xdmac_chan channels[]; + struct uniphier_xdmac_chan channels[] __counted_by(nr_chans); }; static struct uniphier_xdmac_chan * @@ -131,8 +131,9 @@ uniphier_xdmac_next_desc(struct uniphier_xdmac_chan *xc) static void uniphier_xdmac_chan_start(struct uniphier_xdmac_chan *xc, struct uniphier_xdmac_desc *xd) { - u32 src_mode, src_addr, src_width; - u32 dst_mode, dst_addr, dst_width; + u32 src_mode, src_width; + u32 dst_mode, dst_width; + dma_addr_t src_addr, dst_addr; u32 val, its, tnum; enum dma_slave_buswidth buswidth; @@ -209,8 +210,8 @@ static int uniphier_xdmac_chan_stop(struct uniphier_xdmac_chan *xc) writel(0, xc->reg_ch_base + XDMAC_TSS); /* wait until transfer is stopped */ - return readl_poll_timeout(xc->reg_ch_base + XDMAC_STAT, val, - !(val & XDMAC_STAT_TENF), 100, 1000); + return readl_poll_timeout_atomic(xc->reg_ch_base + XDMAC_STAT, val, + !(val & XDMAC_STAT_TENF), 100, 1000); } /* xc->vc.lock must be held by caller */ @@ -294,6 +295,7 @@ uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, xd = kzalloc(struct_size(xd, nodes, nr), GFP_NOWAIT); if (!xd) return NULL; + xd->nr_node = nr; for (i = 0; i < nr; i++) { burst_size = min_t(size_t, len, XDMAC_MAX_WORD_SIZE); @@ -308,7 +310,6 @@ uniphier_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, } xd->dir = DMA_MEM_TO_MEM; - xd->nr_node = nr; xd->cur_node = 0; return vchan_tx_prep(vc, &xd->vd, flags); @@ -350,6 +351,7 @@ uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, xd = kzalloc(struct_size(xd, nodes, sg_len), GFP_NOWAIT); if (!xd) return NULL; + xd->nr_node = sg_len; for_each_sg(sgl, sg, sg_len, i) { xd->nodes[i].src = (direction == DMA_DEV_TO_MEM) @@ -384,7 +386,6 @@ uniphier_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, } xd->dir = direction; - xd->nr_node = sg_len; xd->cur_node = 0; return vchan_tx_prep(vc, &xd->vd, flags); @@ -562,7 +563,7 @@ out_unregister_dmac: return ret; } -static int uniphier_xdmac_remove(struct platform_device *pdev) +static void uniphier_xdmac_remove(struct platform_device *pdev) { struct uniphier_xdmac_device *xdev = platform_get_drvdata(pdev); struct dma_device *ddev = &xdev->ddev; @@ -578,15 +579,20 @@ static int uniphier_xdmac_remove(struct platform_device *pdev) */ list_for_each_entry(chan, &ddev->channels, device_node) { ret = dmaengine_terminate_sync(chan); - if (ret) - return ret; + if (ret) { + /* + * This results in resource leakage and maybe also + * use-after-free errors as e.g. *xdev is kfreed. + */ + dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", + chan->chan_id, ERR_PTR(ret)); + return; + } uniphier_xdmac_free_chan_resources(chan); } of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(ddev); - - return 0; } static const struct of_device_id uniphier_xdmac_match[] = { diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c index a6f4265be0c9..7961172a780d 100644 --- a/drivers/dma/virt-dma.c +++ b/drivers/dma/virt-dma.c @@ -139,4 +139,5 @@ void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev) EXPORT_SYMBOL_GPL(vchan_init); MODULE_AUTHOR("Russell King"); +MODULE_DESCRIPTION("Virtual DMA channel support for DMAengine"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index e9f5250fbe4d..59d9eabc8b67 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -81,6 +81,8 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan */ static inline bool vchan_issue_pending(struct virt_dma_chan *vc) { + lockdep_assert_held(&vc->lock); + list_splice_tail_init(&vc->desc_submitted, &vc->desc_issued); return !list_empty(&vc->desc_issued); } @@ -96,6 +98,8 @@ static inline void vchan_cookie_complete(struct virt_dma_desc *vd) struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); dma_cookie_t cookie; + lockdep_assert_held(&vc->lock); + cookie = vd->tx.cookie; dma_cookie_complete(&vd->tx); dev_vdbg(vc->chan.device->dev, "txd %p[%x]: marked complete\n", @@ -146,6 +150,8 @@ static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd) { struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); + lockdep_assert_held(&vc->lock); + list_add_tail(&vd->node, &vc->desc_terminated); if (vc->cyclic == vd) @@ -160,6 +166,8 @@ static inline void vchan_terminate_vdesc(struct virt_dma_desc *vd) */ static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) { + lockdep_assert_held(&vc->lock); + return list_first_entry_or_null(&vc->desc_issued, struct virt_dma_desc, node); } @@ -177,6 +185,8 @@ static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) static inline void vchan_get_all_descriptors(struct virt_dma_chan *vc, struct list_head *head) { + lockdep_assert_held(&vc->lock); + list_splice_tail_init(&vc->desc_allocated, head); list_splice_tail_init(&vc->desc_submitted, head); list_splice_tail_init(&vc->desc_issued, head); diff --git a/drivers/dma/xgene-dma.c b/drivers/dma/xgene-dma.c index 3589b4ef50b8..f64624ea44ad 100644 --- a/drivers/dma/xgene-dma.c +++ b/drivers/dma/xgene-dma.c @@ -18,8 +18,9 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> -#include <linux/of_device.h> +#include <linux/platform_device.h> #include "dmaengine.h" @@ -1741,7 +1742,7 @@ static int xgene_dma_probe(struct platform_device *pdev) /* Initialize DMA channels software state */ xgene_dma_init_channels(pdma); - /* Configue DMA rings */ + /* Configure DMA rings */ ret = xgene_dma_init_rings(pdma); if (ret) goto err_clk_enable; @@ -1775,7 +1776,7 @@ err_clk_enable: return ret; } -static int xgene_dma_remove(struct platform_device *pdev) +static void xgene_dma_remove(struct platform_device *pdev) { struct xgene_dma *pdma = platform_get_drvdata(pdev); struct xgene_dma_chan *chan; @@ -1796,8 +1797,6 @@ static int xgene_dma_remove(struct platform_device *pdev) if (!IS_ERR(pdma->clk)) clk_disable_unprepare(pdma->clk); - - return 0; } #ifdef CONFIG_ACPI diff --git a/drivers/dma/xilinx/Makefile b/drivers/dma/xilinx/Makefile index 767bb45f641f..ebaa93644c94 100644 --- a/drivers/dma/xilinx/Makefile +++ b/drivers/dma/xilinx/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_XILINX_DMA) += xilinx_dma.o +obj-$(CONFIG_XILINX_XDMA) += xdma.o obj-$(CONFIG_XILINX_ZYNQMP_DMA) += zynqmp_dma.o obj-$(CONFIG_XILINX_ZYNQMP_DPDMA) += xilinx_dpdma.o diff --git a/drivers/dma/xilinx/xdma-regs.h b/drivers/dma/xilinx/xdma-regs.h new file mode 100644 index 000000000000..6ad08878e938 --- /dev/null +++ b/drivers/dma/xilinx/xdma-regs.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved. + * Copyright (C) 2022, Advanced Micro Devices, Inc. + */ + +#ifndef __DMA_XDMA_REGS_H +#define __DMA_XDMA_REGS_H + +/* The length of register space exposed to host */ +#define XDMA_REG_SPACE_LEN 65536 + +/* + * maximum number of DMA channels for each direction: + * Host to Card (H2C) or Card to Host (C2H) + */ +#define XDMA_MAX_CHANNELS 4 + +/* + * macros to define the number of descriptor blocks can be used in one + * DMA transfer request. + * the DMA engine uses a linked list of descriptor blocks that specify the + * source, destination, and length of the DMA transfers. + */ +#define XDMA_DESC_BLOCK_NUM BIT(7) +#define XDMA_DESC_BLOCK_MASK (XDMA_DESC_BLOCK_NUM - 1) + +/* descriptor definitions */ +#define XDMA_DESC_ADJACENT 32 +#define XDMA_DESC_ADJACENT_MASK (XDMA_DESC_ADJACENT - 1) +#define XDMA_DESC_ADJACENT_BITS GENMASK(13, 8) +#define XDMA_DESC_MAGIC 0xad4bUL +#define XDMA_DESC_MAGIC_BITS GENMASK(31, 16) +#define XDMA_DESC_FLAGS_BITS GENMASK(7, 0) +#define XDMA_DESC_STOPPED BIT(0) +#define XDMA_DESC_COMPLETED BIT(1) +#define XDMA_DESC_BLEN_BITS 28 +#define XDMA_DESC_BLEN_MAX (BIT(XDMA_DESC_BLEN_BITS) - PAGE_SIZE) + +/* macros to construct the descriptor control word */ +#define XDMA_DESC_CONTROL(adjacent, flag) \ + (FIELD_PREP(XDMA_DESC_MAGIC_BITS, XDMA_DESC_MAGIC) | \ + FIELD_PREP(XDMA_DESC_ADJACENT_BITS, (adjacent) - 1) | \ + FIELD_PREP(XDMA_DESC_FLAGS_BITS, (flag))) +#define XDMA_DESC_CONTROL_LAST \ + XDMA_DESC_CONTROL(1, XDMA_DESC_STOPPED | XDMA_DESC_COMPLETED) +#define XDMA_DESC_CONTROL_CYCLIC \ + XDMA_DESC_CONTROL(1, XDMA_DESC_COMPLETED) + +/* + * Descriptor for a single contiguous memory block transfer. + * + * Multiple descriptors are linked by means of the next pointer. An additional + * extra adjacent number gives the amount of extra contiguous descriptors. + * + * The descriptors are in root complex memory, and the bytes in the 32-bit + * words must be in little-endian byte ordering. + */ +struct xdma_hw_desc { + __le32 control; + __le32 bytes; + __le64 src_addr; + __le64 dst_addr; + __le64 next_desc; +}; + +#define XDMA_DESC_SIZE sizeof(struct xdma_hw_desc) +#define XDMA_DESC_BLOCK_SIZE (XDMA_DESC_SIZE * XDMA_DESC_ADJACENT) +#define XDMA_DESC_BLOCK_ALIGN 32 +#define XDMA_DESC_BLOCK_BOUNDARY 4096 + +/* + * Channel registers + */ +#define XDMA_CHAN_IDENTIFIER 0x0 +#define XDMA_CHAN_CONTROL 0x4 +#define XDMA_CHAN_CONTROL_W1S 0x8 +#define XDMA_CHAN_CONTROL_W1C 0xc +#define XDMA_CHAN_STATUS 0x40 +#define XDMA_CHAN_STATUS_RC 0x44 +#define XDMA_CHAN_COMPLETED_DESC 0x48 +#define XDMA_CHAN_ALIGNMENTS 0x4c +#define XDMA_CHAN_INTR_ENABLE 0x90 +#define XDMA_CHAN_INTR_ENABLE_W1S 0x94 +#define XDMA_CHAN_INTR_ENABLE_W1C 0x9c + +#define XDMA_CHAN_STRIDE 0x100 +#define XDMA_CHAN_H2C_OFFSET 0x0 +#define XDMA_CHAN_C2H_OFFSET 0x1000 +#define XDMA_CHAN_H2C_TARGET 0x0 +#define XDMA_CHAN_C2H_TARGET 0x1 + +/* macro to check if channel is available */ +#define XDMA_CHAN_MAGIC 0x1fc0 +#define XDMA_CHAN_CHECK_TARGET(id, target) \ + (((u32)(id) >> 16) == XDMA_CHAN_MAGIC + (target)) + +/* bits of the channel control register */ +#define CHAN_CTRL_RUN_STOP BIT(0) +#define CHAN_CTRL_IE_DESC_STOPPED BIT(1) +#define CHAN_CTRL_IE_DESC_COMPLETED BIT(2) +#define CHAN_CTRL_IE_DESC_ALIGN_MISMATCH BIT(3) +#define CHAN_CTRL_IE_MAGIC_STOPPED BIT(4) +#define CHAN_CTRL_IE_IDLE_STOPPED BIT(6) +#define CHAN_CTRL_IE_READ_ERROR GENMASK(13, 9) +#define CHAN_CTRL_IE_WRITE_ERROR GENMASK(18, 14) +#define CHAN_CTRL_IE_DESC_ERROR GENMASK(23, 19) +#define CHAN_CTRL_NON_INCR_ADDR BIT(25) +#define CHAN_CTRL_POLL_MODE_WB BIT(26) + +#define CHAN_CTRL_START (CHAN_CTRL_RUN_STOP | \ + CHAN_CTRL_IE_DESC_STOPPED | \ + CHAN_CTRL_IE_DESC_COMPLETED | \ + CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ + CHAN_CTRL_IE_MAGIC_STOPPED | \ + CHAN_CTRL_IE_READ_ERROR | \ + CHAN_CTRL_IE_WRITE_ERROR | \ + CHAN_CTRL_IE_DESC_ERROR) + +/* bits of the channel status register */ +#define XDMA_CHAN_STATUS_BUSY BIT(0) + +#define XDMA_CHAN_STATUS_MASK CHAN_CTRL_START + +#define XDMA_CHAN_ERROR_MASK (CHAN_CTRL_IE_DESC_ALIGN_MISMATCH | \ + CHAN_CTRL_IE_MAGIC_STOPPED | \ + CHAN_CTRL_IE_READ_ERROR | \ + CHAN_CTRL_IE_WRITE_ERROR | \ + CHAN_CTRL_IE_DESC_ERROR) + +/* bits of the channel interrupt enable mask */ +#define CHAN_IM_DESC_ERROR BIT(19) +#define CHAN_IM_READ_ERROR BIT(9) +#define CHAN_IM_IDLE_STOPPED BIT(6) +#define CHAN_IM_MAGIC_STOPPED BIT(4) +#define CHAN_IM_DESC_COMPLETED BIT(2) +#define CHAN_IM_DESC_STOPPED BIT(1) + +#define CHAN_IM_ALL (CHAN_IM_DESC_ERROR | CHAN_IM_READ_ERROR | \ + CHAN_IM_IDLE_STOPPED | CHAN_IM_MAGIC_STOPPED | \ + CHAN_IM_DESC_COMPLETED | CHAN_IM_DESC_STOPPED) + +/* + * Channel SGDMA registers + */ +#define XDMA_SGDMA_IDENTIFIER 0x4000 +#define XDMA_SGDMA_DESC_LO 0x4080 +#define XDMA_SGDMA_DESC_HI 0x4084 +#define XDMA_SGDMA_DESC_ADJ 0x4088 +#define XDMA_SGDMA_DESC_CREDIT 0x408c + +/* + * interrupt registers + */ +#define XDMA_IRQ_IDENTIFIER 0x2000 +#define XDMA_IRQ_USER_INT_EN 0x2004 +#define XDMA_IRQ_USER_INT_EN_W1S 0x2008 +#define XDMA_IRQ_USER_INT_EN_W1C 0x200c +#define XDMA_IRQ_CHAN_INT_EN 0x2010 +#define XDMA_IRQ_CHAN_INT_EN_W1S 0x2014 +#define XDMA_IRQ_CHAN_INT_EN_W1C 0x2018 +#define XDMA_IRQ_USER_INT_REQ 0x2040 +#define XDMA_IRQ_CHAN_INT_REQ 0x2044 +#define XDMA_IRQ_USER_INT_PEND 0x2048 +#define XDMA_IRQ_CHAN_INT_PEND 0x204c +#define XDMA_IRQ_USER_VEC_NUM 0x2080 +#define XDMA_IRQ_CHAN_VEC_NUM 0x20a0 + +#define XDMA_IRQ_VEC_SHIFT 8 + +#endif /* __DMA_XDMA_REGS_H */ diff --git a/drivers/dma/xilinx/xdma.c b/drivers/dma/xilinx/xdma.c new file mode 100644 index 000000000000..0d88b1a670e1 --- /dev/null +++ b/drivers/dma/xilinx/xdma.c @@ -0,0 +1,1321 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DMA driver for Xilinx DMA/Bridge Subsystem + * + * Copyright (C) 2017-2020 Xilinx, Inc. All rights reserved. + * Copyright (C) 2022, Advanced Micro Devices, Inc. + */ + +/* + * The DMA/Bridge Subsystem for PCI Express allows for the movement of data + * between Host memory and the DMA subsystem. It does this by operating on + * 'descriptors' that contain information about the source, destination and + * amount of data to transfer. These direct memory transfers can be both in + * the Host to Card (H2C) and Card to Host (C2H) transfers. The DMA can be + * configured to have a single AXI4 Master interface shared by all channels + * or one AXI4-Stream interface for each channel enabled. Memory transfers are + * specified on a per-channel basis in descriptor linked lists, which the DMA + * fetches from host memory and processes. Events such as descriptor completion + * and errors are signaled using interrupts. The core also provides up to 16 + * user interrupt wires that generate interrupts to the host. + */ + +#include <linux/mod_devicetable.h> +#include <linux/bitfield.h> +#include <linux/dmapool.h> +#include <linux/regmap.h> +#include <linux/dmaengine.h> +#include <linux/dma/amd_xdma.h> +#include <linux/platform_device.h> +#include <linux/platform_data/amd_xdma.h> +#include <linux/dma-mapping.h> +#include <linux/pci.h> +#include "../virt-dma.h" +#include "xdma-regs.h" + +/* mmio regmap config for all XDMA registers */ +static const struct regmap_config xdma_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .max_register = XDMA_REG_SPACE_LEN, +}; + +/** + * struct xdma_desc_block - Descriptor block + * @virt_addr: Virtual address of block start + * @dma_addr: DMA address of block start + */ +struct xdma_desc_block { + void *virt_addr; + dma_addr_t dma_addr; +}; + +/** + * struct xdma_chan - Driver specific DMA channel structure + * @vchan: Virtual channel + * @xdev_hdl: Pointer to DMA device structure + * @base: Offset of channel registers + * @desc_pool: Descriptor pool + * @busy: Busy flag of the channel + * @dir: Transferring direction of the channel + * @cfg: Transferring config of the channel + * @irq: IRQ assigned to the channel + */ +struct xdma_chan { + struct virt_dma_chan vchan; + void *xdev_hdl; + u32 base; + struct dma_pool *desc_pool; + bool busy; + enum dma_transfer_direction dir; + struct dma_slave_config cfg; + u32 irq; + struct completion last_interrupt; + bool stop_requested; +}; + +/** + * struct xdma_desc - DMA desc structure + * @vdesc: Virtual DMA descriptor + * @chan: DMA channel pointer + * @dir: Transferring direction of the request + * @desc_blocks: Hardware descriptor blocks + * @dblk_num: Number of hardware descriptor blocks + * @desc_num: Number of hardware descriptors + * @completed_desc_num: Completed hardware descriptors + * @cyclic: Cyclic transfer vs. scatter-gather + * @interleaved_dma: Interleaved DMA transfer + * @periods: Number of periods in the cyclic transfer + * @period_size: Size of a period in bytes in cyclic transfers + * @frames_left: Number of frames left in interleaved DMA transfer + * @error: tx error flag + */ +struct xdma_desc { + struct virt_dma_desc vdesc; + struct xdma_chan *chan; + enum dma_transfer_direction dir; + struct xdma_desc_block *desc_blocks; + u32 dblk_num; + u32 desc_num; + u32 completed_desc_num; + bool cyclic; + bool interleaved_dma; + u32 periods; + u32 period_size; + u32 frames_left; + bool error; +}; + +#define XDMA_DEV_STATUS_REG_DMA BIT(0) +#define XDMA_DEV_STATUS_INIT_MSIX BIT(1) + +/** + * struct xdma_device - DMA device structure + * @pdev: Platform device pointer + * @dma_dev: DMA device structure + * @rmap: MMIO regmap for DMA registers + * @h2c_chans: Host to Card channels + * @c2h_chans: Card to Host channels + * @h2c_chan_num: Number of H2C channels + * @c2h_chan_num: Number of C2H channels + * @irq_start: Start IRQ assigned to device + * @irq_num: Number of IRQ assigned to device + * @status: Initialization status + */ +struct xdma_device { + struct platform_device *pdev; + struct dma_device dma_dev; + struct regmap *rmap; + struct xdma_chan *h2c_chans; + struct xdma_chan *c2h_chans; + u32 h2c_chan_num; + u32 c2h_chan_num; + u32 irq_start; + u32 irq_num; + u32 status; +}; + +#define xdma_err(xdev, fmt, args...) \ + dev_err(&(xdev)->pdev->dev, fmt, ##args) +#define XDMA_CHAN_NUM(_xd) ({ \ + typeof(_xd) (xd) = (_xd); \ + ((xd)->h2c_chan_num + (xd)->c2h_chan_num); }) + +/* Get the last desc in a desc block */ +static inline void *xdma_blk_last_desc(struct xdma_desc_block *block) +{ + return block->virt_addr + (XDMA_DESC_ADJACENT - 1) * XDMA_DESC_SIZE; +} + +/** + * xdma_link_sg_desc_blocks - Link SG descriptor blocks for DMA transfer + * @sw_desc: Tx descriptor pointer + */ +static void xdma_link_sg_desc_blocks(struct xdma_desc *sw_desc) +{ + struct xdma_desc_block *block; + u32 last_blk_desc, desc_control; + struct xdma_hw_desc *desc; + int i; + + desc_control = XDMA_DESC_CONTROL(XDMA_DESC_ADJACENT, 0); + for (i = 1; i < sw_desc->dblk_num; i++) { + block = &sw_desc->desc_blocks[i - 1]; + desc = xdma_blk_last_desc(block); + + if (!(i & XDMA_DESC_BLOCK_MASK)) { + desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST); + continue; + } + desc->control = cpu_to_le32(desc_control); + desc->next_desc = cpu_to_le64(block[1].dma_addr); + } + + /* update the last block */ + last_blk_desc = (sw_desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK; + if (((sw_desc->dblk_num - 1) & XDMA_DESC_BLOCK_MASK) > 0) { + block = &sw_desc->desc_blocks[sw_desc->dblk_num - 2]; + desc = xdma_blk_last_desc(block); + desc_control = XDMA_DESC_CONTROL(last_blk_desc + 1, 0); + desc->control = cpu_to_le32(desc_control); + } + + block = &sw_desc->desc_blocks[sw_desc->dblk_num - 1]; + desc = block->virt_addr + last_blk_desc * XDMA_DESC_SIZE; + desc->control = cpu_to_le32(XDMA_DESC_CONTROL_LAST); +} + +/** + * xdma_link_cyclic_desc_blocks - Link cyclic descriptor blocks for DMA transfer + * @sw_desc: Tx descriptor pointer + */ +static void xdma_link_cyclic_desc_blocks(struct xdma_desc *sw_desc) +{ + struct xdma_desc_block *block; + struct xdma_hw_desc *desc; + int i; + + block = sw_desc->desc_blocks; + for (i = 0; i < sw_desc->desc_num - 1; i++) { + desc = block->virt_addr + i * XDMA_DESC_SIZE; + desc->next_desc = cpu_to_le64(block->dma_addr + ((i + 1) * XDMA_DESC_SIZE)); + } + desc = block->virt_addr + i * XDMA_DESC_SIZE; + desc->next_desc = cpu_to_le64(block->dma_addr); +} + +static inline struct xdma_chan *to_xdma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct xdma_chan, vchan.chan); +} + +static inline struct xdma_desc *to_xdma_desc(struct virt_dma_desc *vdesc) +{ + return container_of(vdesc, struct xdma_desc, vdesc); +} + +/** + * xdma_channel_init - Initialize DMA channel registers + * @chan: DMA channel pointer + */ +static int xdma_channel_init(struct xdma_chan *chan) +{ + struct xdma_device *xdev = chan->xdev_hdl; + int ret; + + ret = regmap_write(xdev->rmap, chan->base + XDMA_CHAN_CONTROL_W1C, + CHAN_CTRL_NON_INCR_ADDR); + if (ret) + return ret; + + ret = regmap_write(xdev->rmap, chan->base + XDMA_CHAN_INTR_ENABLE, + CHAN_IM_ALL); + if (ret) + return ret; + + return 0; +} + +/** + * xdma_free_desc - Free descriptor + * @vdesc: Virtual DMA descriptor + */ +static void xdma_free_desc(struct virt_dma_desc *vdesc) +{ + struct xdma_desc *sw_desc; + int i; + + sw_desc = to_xdma_desc(vdesc); + for (i = 0; i < sw_desc->dblk_num; i++) { + if (!sw_desc->desc_blocks[i].virt_addr) + break; + dma_pool_free(sw_desc->chan->desc_pool, + sw_desc->desc_blocks[i].virt_addr, + sw_desc->desc_blocks[i].dma_addr); + } + kfree(sw_desc->desc_blocks); + kfree(sw_desc); +} + +/** + * xdma_alloc_desc - Allocate descriptor + * @chan: DMA channel pointer + * @desc_num: Number of hardware descriptors + * @cyclic: Whether this is a cyclic transfer + */ +static struct xdma_desc * +xdma_alloc_desc(struct xdma_chan *chan, u32 desc_num, bool cyclic) +{ + struct xdma_desc *sw_desc; + struct xdma_hw_desc *desc; + dma_addr_t dma_addr; + u32 dblk_num; + u32 control; + void *addr; + int i, j; + + sw_desc = kzalloc(sizeof(*sw_desc), GFP_NOWAIT); + if (!sw_desc) + return NULL; + + sw_desc->chan = chan; + sw_desc->desc_num = desc_num; + sw_desc->cyclic = cyclic; + sw_desc->error = false; + dblk_num = DIV_ROUND_UP(desc_num, XDMA_DESC_ADJACENT); + sw_desc->desc_blocks = kcalloc(dblk_num, sizeof(*sw_desc->desc_blocks), + GFP_NOWAIT); + if (!sw_desc->desc_blocks) + goto failed; + + if (cyclic) + control = XDMA_DESC_CONTROL_CYCLIC; + else + control = XDMA_DESC_CONTROL(1, 0); + + sw_desc->dblk_num = dblk_num; + for (i = 0; i < sw_desc->dblk_num; i++) { + addr = dma_pool_alloc(chan->desc_pool, GFP_NOWAIT, &dma_addr); + if (!addr) + goto failed; + + sw_desc->desc_blocks[i].virt_addr = addr; + sw_desc->desc_blocks[i].dma_addr = dma_addr; + for (j = 0, desc = addr; j < XDMA_DESC_ADJACENT; j++) + desc[j].control = cpu_to_le32(control); + } + + if (cyclic) + xdma_link_cyclic_desc_blocks(sw_desc); + else + xdma_link_sg_desc_blocks(sw_desc); + + return sw_desc; + +failed: + xdma_free_desc(&sw_desc->vdesc); + return NULL; +} + +/** + * xdma_xfer_start - Start DMA transfer + * @xchan: DMA channel pointer + */ +static int xdma_xfer_start(struct xdma_chan *xchan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&xchan->vchan); + struct xdma_device *xdev = xchan->xdev_hdl; + struct xdma_desc_block *block; + u32 val, completed_blocks; + struct xdma_desc *desc; + int ret; + + /* + * check if there is not any submitted descriptor or channel is busy. + * vchan lock should be held where this function is called. + */ + if (!vd || xchan->busy) + return -EINVAL; + + /* clear run stop bit to get ready for transfer */ + ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, + CHAN_CTRL_RUN_STOP); + if (ret) + return ret; + + desc = to_xdma_desc(vd); + if (desc->dir != xchan->dir) { + xdma_err(xdev, "incorrect request direction"); + return -EINVAL; + } + + /* set DMA engine to the first descriptor block */ + completed_blocks = desc->completed_desc_num / XDMA_DESC_ADJACENT; + block = &desc->desc_blocks[completed_blocks]; + val = lower_32_bits(block->dma_addr); + ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_LO, val); + if (ret) + return ret; + + val = upper_32_bits(block->dma_addr); + ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_HI, val); + if (ret) + return ret; + + if (completed_blocks + 1 == desc->dblk_num) + val = (desc->desc_num - 1) & XDMA_DESC_ADJACENT_MASK; + else + val = XDMA_DESC_ADJACENT - 1; + ret = regmap_write(xdev->rmap, xchan->base + XDMA_SGDMA_DESC_ADJ, val); + if (ret) + return ret; + + /* kick off DMA transfer */ + ret = regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL, + CHAN_CTRL_START); + if (ret) + return ret; + + xchan->busy = true; + xchan->stop_requested = false; + reinit_completion(&xchan->last_interrupt); + + return 0; +} + +/** + * xdma_xfer_stop - Stop DMA transfer + * @xchan: DMA channel pointer + */ +static int xdma_xfer_stop(struct xdma_chan *xchan) +{ + struct xdma_device *xdev = xchan->xdev_hdl; + + /* clear run stop bit to prevent any further auto-triggering */ + return regmap_write(xdev->rmap, xchan->base + XDMA_CHAN_CONTROL_W1C, + CHAN_CTRL_RUN_STOP); +} + +/** + * xdma_alloc_channels - Detect and allocate DMA channels + * @xdev: DMA device pointer + * @dir: Channel direction + */ +static int xdma_alloc_channels(struct xdma_device *xdev, + enum dma_transfer_direction dir) +{ + struct xdma_platdata *pdata = dev_get_platdata(&xdev->pdev->dev); + struct xdma_chan **chans, *xchan; + u32 base, identifier, target; + u32 *chan_num; + int i, j, ret; + + if (dir == DMA_MEM_TO_DEV) { + base = XDMA_CHAN_H2C_OFFSET; + target = XDMA_CHAN_H2C_TARGET; + chans = &xdev->h2c_chans; + chan_num = &xdev->h2c_chan_num; + } else if (dir == DMA_DEV_TO_MEM) { + base = XDMA_CHAN_C2H_OFFSET; + target = XDMA_CHAN_C2H_TARGET; + chans = &xdev->c2h_chans; + chan_num = &xdev->c2h_chan_num; + } else { + xdma_err(xdev, "invalid direction specified"); + return -EINVAL; + } + + /* detect number of available DMA channels */ + for (i = 0, *chan_num = 0; i < pdata->max_dma_channels; i++) { + ret = regmap_read(xdev->rmap, base + i * XDMA_CHAN_STRIDE, + &identifier); + if (ret) + return ret; + + /* check if it is available DMA channel */ + if (XDMA_CHAN_CHECK_TARGET(identifier, target)) + (*chan_num)++; + } + + if (!*chan_num) { + xdma_err(xdev, "does not probe any channel"); + return -EINVAL; + } + + *chans = devm_kcalloc(&xdev->pdev->dev, *chan_num, sizeof(**chans), + GFP_KERNEL); + if (!*chans) + return -ENOMEM; + + for (i = 0, j = 0; i < pdata->max_dma_channels; i++) { + ret = regmap_read(xdev->rmap, base + i * XDMA_CHAN_STRIDE, + &identifier); + if (ret) + return ret; + + if (!XDMA_CHAN_CHECK_TARGET(identifier, target)) + continue; + + if (j == *chan_num) { + xdma_err(xdev, "invalid channel number"); + return -EIO; + } + + /* init channel structure and hardware */ + xchan = &(*chans)[j]; + xchan->xdev_hdl = xdev; + xchan->base = base + i * XDMA_CHAN_STRIDE; + xchan->dir = dir; + xchan->stop_requested = false; + init_completion(&xchan->last_interrupt); + + ret = xdma_channel_init(xchan); + if (ret) + return ret; + xchan->vchan.desc_free = xdma_free_desc; + vchan_init(&xchan->vchan, &xdev->dma_dev); + + j++; + } + + dev_info(&xdev->pdev->dev, "configured %d %s channels", j, + (dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H"); + + return 0; +} + +/** + * xdma_issue_pending - Issue pending transactions + * @chan: DMA channel pointer + */ +static void xdma_issue_pending(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); + if (vchan_issue_pending(&xdma_chan->vchan)) + xdma_xfer_start(xdma_chan); + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); +} + +/** + * xdma_terminate_all - Terminate all transactions + * @chan: DMA channel pointer + */ +static int xdma_terminate_all(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct virt_dma_desc *vd; + unsigned long flags; + LIST_HEAD(head); + + xdma_xfer_stop(xdma_chan); + + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); + + xdma_chan->busy = false; + xdma_chan->stop_requested = true; + vd = vchan_next_desc(&xdma_chan->vchan); + if (vd) { + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); + } + vchan_get_all_descriptors(&xdma_chan->vchan, &head); + list_splice_tail(&head, &xdma_chan->vchan.desc_terminated); + + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); + + return 0; +} + +/** + * xdma_synchronize - Synchronize terminated transactions + * @chan: DMA channel pointer + */ +static void xdma_synchronize(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_device *xdev = xdma_chan->xdev_hdl; + int st = 0; + + /* If the engine continues running, wait for the last interrupt */ + regmap_read(xdev->rmap, xdma_chan->base + XDMA_CHAN_STATUS, &st); + if (st & XDMA_CHAN_STATUS_BUSY) + wait_for_completion_timeout(&xdma_chan->last_interrupt, msecs_to_jiffies(1000)); + + vchan_synchronize(&xdma_chan->vchan); +} + +/** + * xdma_fill_descs() - Fill hardware descriptors for one contiguous memory chunk. + * More than one descriptor will be used if the size is bigger + * than XDMA_DESC_BLEN_MAX. + * @sw_desc: Descriptor container + * @src_addr: First value for the ->src_addr field + * @dst_addr: First value for the ->dst_addr field + * @size: Size of the contiguous memory block + * @filled_descs_num: Index of the first descriptor to take care of in @sw_desc + */ +static inline u32 xdma_fill_descs(struct xdma_desc *sw_desc, u64 src_addr, + u64 dst_addr, u32 size, u32 filled_descs_num) +{ + u32 left = size, len, desc_num = filled_descs_num; + struct xdma_desc_block *dblk; + struct xdma_hw_desc *desc; + + dblk = sw_desc->desc_blocks + (desc_num / XDMA_DESC_ADJACENT); + desc = dblk->virt_addr; + desc += desc_num & XDMA_DESC_ADJACENT_MASK; + do { + len = min_t(u32, left, XDMA_DESC_BLEN_MAX); + /* set hardware descriptor */ + desc->bytes = cpu_to_le32(len); + desc->src_addr = cpu_to_le64(src_addr); + desc->dst_addr = cpu_to_le64(dst_addr); + if (!(++desc_num & XDMA_DESC_ADJACENT_MASK)) + desc = (++dblk)->virt_addr; + else + desc++; + + src_addr += len; + dst_addr += len; + left -= len; + } while (left); + + return desc_num - filled_descs_num; +} + +/** + * xdma_prep_device_sg - prepare a descriptor for a DMA transaction + * @chan: DMA channel pointer + * @sgl: Transfer scatter gather list + * @sg_len: Length of scatter gather list + * @dir: Transfer direction + * @flags: transfer ack flags + * @context: APP words of the descriptor + */ +static struct dma_async_tx_descriptor * +xdma_prep_device_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct dma_async_tx_descriptor *tx_desc; + struct xdma_desc *sw_desc; + u32 desc_num = 0, i; + u64 addr, dev_addr, *src, *dst; + struct scatterlist *sg; + + for_each_sg(sgl, sg, sg_len, i) + desc_num += DIV_ROUND_UP(sg_dma_len(sg), XDMA_DESC_BLEN_MAX); + + sw_desc = xdma_alloc_desc(xdma_chan, desc_num, false); + if (!sw_desc) + return NULL; + sw_desc->dir = dir; + sw_desc->cyclic = false; + sw_desc->interleaved_dma = false; + + if (dir == DMA_MEM_TO_DEV) { + dev_addr = xdma_chan->cfg.dst_addr; + src = &addr; + dst = &dev_addr; + } else { + dev_addr = xdma_chan->cfg.src_addr; + src = &dev_addr; + dst = &addr; + } + + desc_num = 0; + for_each_sg(sgl, sg, sg_len, i) { + addr = sg_dma_address(sg); + desc_num += xdma_fill_descs(sw_desc, *src, *dst, sg_dma_len(sg), desc_num); + dev_addr += sg_dma_len(sg); + } + + tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); + if (!tx_desc) + goto failed; + + return tx_desc; + +failed: + xdma_free_desc(&sw_desc->vdesc); + + return NULL; +} + +/** + * xdma_prep_dma_cyclic - prepare for cyclic DMA transactions + * @chan: DMA channel pointer + * @address: Device DMA address to access + * @size: Total length to transfer + * @period_size: Period size to use for each transfer + * @dir: Transfer direction + * @flags: Transfer ack flags + */ +static struct dma_async_tx_descriptor * +xdma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t address, + size_t size, size_t period_size, + enum dma_transfer_direction dir, + unsigned long flags) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_device *xdev = xdma_chan->xdev_hdl; + unsigned int periods = size / period_size; + struct dma_async_tx_descriptor *tx_desc; + struct xdma_desc *sw_desc; + u64 addr, dev_addr, *src, *dst; + u32 desc_num; + unsigned int i; + + /* + * Simplify the whole logic by preventing an abnormally high number of + * periods and periods size. + */ + if (period_size > XDMA_DESC_BLEN_MAX) { + xdma_err(xdev, "period size limited to %lu bytes\n", XDMA_DESC_BLEN_MAX); + return NULL; + } + + if (periods > XDMA_DESC_ADJACENT) { + xdma_err(xdev, "number of periods limited to %u\n", XDMA_DESC_ADJACENT); + return NULL; + } + + sw_desc = xdma_alloc_desc(xdma_chan, periods, true); + if (!sw_desc) + return NULL; + + sw_desc->periods = periods; + sw_desc->period_size = period_size; + sw_desc->dir = dir; + sw_desc->interleaved_dma = false; + + addr = address; + if (dir == DMA_MEM_TO_DEV) { + dev_addr = xdma_chan->cfg.dst_addr; + src = &addr; + dst = &dev_addr; + } else { + dev_addr = xdma_chan->cfg.src_addr; + src = &dev_addr; + dst = &addr; + } + + desc_num = 0; + for (i = 0; i < periods; i++) { + desc_num += xdma_fill_descs(sw_desc, *src, *dst, period_size, desc_num); + addr += period_size; + } + + tx_desc = vchan_tx_prep(&xdma_chan->vchan, &sw_desc->vdesc, flags); + if (!tx_desc) + goto failed; + + return tx_desc; + +failed: + xdma_free_desc(&sw_desc->vdesc); + + return NULL; +} + +/** + * xdma_prep_interleaved_dma - Prepare virtual descriptor for interleaved DMA transfers + * @chan: DMA channel + * @xt: DMA transfer template + * @flags: tx flags + */ +static struct dma_async_tx_descriptor * +xdma_prep_interleaved_dma(struct dma_chan *chan, + struct dma_interleaved_template *xt, + unsigned long flags) +{ + int i; + u32 desc_num = 0, period_size = 0; + struct dma_async_tx_descriptor *tx_desc; + struct xdma_chan *xchan = to_xdma_chan(chan); + struct xdma_desc *sw_desc; + u64 src_addr, dst_addr; + + for (i = 0; i < xt->frame_size; ++i) + desc_num += DIV_ROUND_UP(xt->sgl[i].size, XDMA_DESC_BLEN_MAX); + + sw_desc = xdma_alloc_desc(xchan, desc_num, false); + if (!sw_desc) + return NULL; + sw_desc->dir = xt->dir; + sw_desc->interleaved_dma = true; + sw_desc->cyclic = flags & DMA_PREP_REPEAT; + sw_desc->frames_left = xt->numf; + sw_desc->periods = xt->numf; + + desc_num = 0; + src_addr = xt->src_start; + dst_addr = xt->dst_start; + for (i = 0; i < xt->frame_size; ++i) { + desc_num += xdma_fill_descs(sw_desc, src_addr, dst_addr, xt->sgl[i].size, desc_num); + src_addr += dmaengine_get_src_icg(xt, &xt->sgl[i]) + (xt->src_inc ? + xt->sgl[i].size : 0); + dst_addr += dmaengine_get_dst_icg(xt, &xt->sgl[i]) + (xt->dst_inc ? + xt->sgl[i].size : 0); + period_size += xt->sgl[i].size; + } + sw_desc->period_size = period_size; + + tx_desc = vchan_tx_prep(&xchan->vchan, &sw_desc->vdesc, flags); + if (tx_desc) + return tx_desc; + + xdma_free_desc(&sw_desc->vdesc); + return NULL; +} + +/** + * xdma_device_config - Configure the DMA channel + * @chan: DMA channel + * @cfg: channel configuration + */ +static int xdma_device_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + + memcpy(&xdma_chan->cfg, cfg, sizeof(*cfg)); + + return 0; +} + +/** + * xdma_free_chan_resources - Free channel resources + * @chan: DMA channel + */ +static void xdma_free_chan_resources(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + + vchan_free_chan_resources(&xdma_chan->vchan); + dma_pool_destroy(xdma_chan->desc_pool); + xdma_chan->desc_pool = NULL; +} + +/** + * xdma_alloc_chan_resources - Allocate channel resources + * @chan: DMA channel + */ +static int xdma_alloc_chan_resources(struct dma_chan *chan) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_device *xdev = xdma_chan->xdev_hdl; + struct device *dev = xdev->dma_dev.dev; + + while (dev && !dev_is_pci(dev)) + dev = dev->parent; + if (!dev) { + xdma_err(xdev, "unable to find pci device"); + return -EINVAL; + } + + xdma_chan->desc_pool = dma_pool_create(dma_chan_name(chan), dev, XDMA_DESC_BLOCK_SIZE, + XDMA_DESC_BLOCK_ALIGN, XDMA_DESC_BLOCK_BOUNDARY); + if (!xdma_chan->desc_pool) { + xdma_err(xdev, "unable to allocate descriptor pool"); + return -ENOMEM; + } + + return 0; +} + +static enum dma_status xdma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *state) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_desc *desc = NULL; + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; + unsigned int period_idx; + u32 residue = 0; + + ret = dma_cookie_status(chan, cookie, state); + if (ret == DMA_COMPLETE) + return ret; + + spin_lock_irqsave(&xdma_chan->vchan.lock, flags); + + vd = vchan_find_desc(&xdma_chan->vchan, cookie); + if (!vd) + goto out; + + desc = to_xdma_desc(vd); + if (desc->error) { + ret = DMA_ERROR; + } else if (desc->cyclic) { + period_idx = desc->completed_desc_num % desc->periods; + residue = (desc->periods - period_idx) * desc->period_size; + dma_set_residue(state, residue); + } +out: + spin_unlock_irqrestore(&xdma_chan->vchan.lock, flags); + + return ret; +} + +/** + * xdma_channel_isr - XDMA channel interrupt handler + * @irq: IRQ number + * @dev_id: Pointer to the DMA channel structure + */ +static irqreturn_t xdma_channel_isr(int irq, void *dev_id) +{ + struct xdma_chan *xchan = dev_id; + u32 complete_desc_num = 0; + struct xdma_device *xdev = xchan->xdev_hdl; + struct virt_dma_desc *vd, *next_vd; + struct xdma_desc *desc; + int ret; + u32 st; + bool repeat_tx; + + spin_lock(&xchan->vchan.lock); + + if (xchan->stop_requested) + complete(&xchan->last_interrupt); + + /* get submitted request */ + vd = vchan_next_desc(&xchan->vchan); + if (!vd) + goto out; + + /* Clear-on-read the status register */ + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_STATUS_RC, &st); + if (ret) + goto out; + + desc = to_xdma_desc(vd); + + st &= XDMA_CHAN_STATUS_MASK; + if ((st & XDMA_CHAN_ERROR_MASK) || + !(st & (CHAN_CTRL_IE_DESC_COMPLETED | CHAN_CTRL_IE_DESC_STOPPED))) { + desc->error = true; + xdma_err(xdev, "channel error, status register value: 0x%x", st); + goto out; + } + + ret = regmap_read(xdev->rmap, xchan->base + XDMA_CHAN_COMPLETED_DESC, + &complete_desc_num); + if (ret) + goto out; + + if (desc->interleaved_dma) { + xchan->busy = false; + desc->completed_desc_num += complete_desc_num; + if (complete_desc_num == XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) { + xdma_xfer_start(xchan); + goto out; + } + + /* last desc of any frame */ + desc->frames_left--; + if (desc->frames_left) + goto out; + + /* last desc of the last frame */ + repeat_tx = vd->tx.flags & DMA_PREP_REPEAT; + next_vd = list_first_entry_or_null(&vd->node, struct virt_dma_desc, node); + if (next_vd) + repeat_tx = repeat_tx && !(next_vd->tx.flags & DMA_PREP_LOAD_EOT); + if (repeat_tx) { + desc->frames_left = desc->periods; + desc->completed_desc_num = 0; + vchan_cyclic_callback(vd); + } else { + list_del(&vd->node); + vchan_cookie_complete(vd); + } + /* start (or continue) the tx of a first desc on the vc.desc_issued list, if any */ + xdma_xfer_start(xchan); + } else if (!desc->cyclic) { + xchan->busy = false; + desc->completed_desc_num += complete_desc_num; + + /* if all data blocks are transferred, remove and complete the request */ + if (desc->completed_desc_num == desc->desc_num) { + list_del(&vd->node); + vchan_cookie_complete(vd); + goto out; + } + + if (desc->completed_desc_num > desc->desc_num || + complete_desc_num != XDMA_DESC_BLOCK_NUM * XDMA_DESC_ADJACENT) + goto out; + + /* transfer the rest of data */ + xdma_xfer_start(xchan); + } else { + desc->completed_desc_num = complete_desc_num; + vchan_cyclic_callback(vd); + } + +out: + spin_unlock(&xchan->vchan.lock); + return IRQ_HANDLED; +} + +/** + * xdma_irq_fini - Uninitialize IRQ + * @xdev: DMA device pointer + */ +static void xdma_irq_fini(struct xdma_device *xdev) +{ + int i; + + /* disable interrupt */ + regmap_write(xdev->rmap, XDMA_IRQ_CHAN_INT_EN_W1C, ~0); + + /* free irq handler */ + for (i = 0; i < xdev->h2c_chan_num; i++) + free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]); + + for (i = 0; i < xdev->c2h_chan_num; i++) + free_irq(xdev->c2h_chans[i].irq, &xdev->c2h_chans[i]); +} + +/** + * xdma_set_vector_reg - configure hardware IRQ registers + * @xdev: DMA device pointer + * @vec_tbl_start: Start of IRQ registers + * @irq_start: Start of IRQ + * @irq_num: Number of IRQ + */ +static int xdma_set_vector_reg(struct xdma_device *xdev, u32 vec_tbl_start, + u32 irq_start, u32 irq_num) +{ + u32 shift, i, val = 0; + int ret; + + /* Each IRQ register is 32 bit and contains 4 IRQs */ + while (irq_num > 0) { + for (i = 0; i < 4; i++) { + shift = XDMA_IRQ_VEC_SHIFT * i; + val |= irq_start << shift; + irq_start++; + irq_num--; + if (!irq_num) + break; + } + + /* write IRQ register */ + ret = regmap_write(xdev->rmap, vec_tbl_start, val); + if (ret) + return ret; + vec_tbl_start += sizeof(u32); + val = 0; + } + + return 0; +} + +/** + * xdma_irq_init - initialize IRQs + * @xdev: DMA device pointer + */ +static int xdma_irq_init(struct xdma_device *xdev) +{ + u32 irq = xdev->irq_start; + u32 user_irq_start; + int i, j, ret; + + /* return failure if there are not enough IRQs */ + if (xdev->irq_num < XDMA_CHAN_NUM(xdev)) { + xdma_err(xdev, "not enough irq"); + return -EINVAL; + } + + /* setup H2C interrupt handler */ + for (i = 0; i < xdev->h2c_chan_num; i++) { + ret = request_irq(irq, xdma_channel_isr, 0, + "xdma-h2c-channel", &xdev->h2c_chans[i]); + if (ret) { + xdma_err(xdev, "H2C channel%d request irq%d failed: %d", + i, irq, ret); + goto failed_init_h2c; + } + xdev->h2c_chans[i].irq = irq; + irq++; + } + + /* setup C2H interrupt handler */ + for (j = 0; j < xdev->c2h_chan_num; j++) { + ret = request_irq(irq, xdma_channel_isr, 0, + "xdma-c2h-channel", &xdev->c2h_chans[j]); + if (ret) { + xdma_err(xdev, "C2H channel%d request irq%d failed: %d", + j, irq, ret); + goto failed_init_c2h; + } + xdev->c2h_chans[j].irq = irq; + irq++; + } + + /* config hardware IRQ registers */ + ret = xdma_set_vector_reg(xdev, XDMA_IRQ_CHAN_VEC_NUM, 0, + XDMA_CHAN_NUM(xdev)); + if (ret) { + xdma_err(xdev, "failed to set channel vectors: %d", ret); + goto failed_init_c2h; + } + + /* config user IRQ registers if needed */ + user_irq_start = XDMA_CHAN_NUM(xdev); + if (xdev->irq_num > user_irq_start) { + ret = xdma_set_vector_reg(xdev, XDMA_IRQ_USER_VEC_NUM, + user_irq_start, + xdev->irq_num - user_irq_start); + if (ret) { + xdma_err(xdev, "failed to set user vectors: %d", ret); + goto failed_init_c2h; + } + } + + /* enable interrupt */ + ret = regmap_write(xdev->rmap, XDMA_IRQ_CHAN_INT_EN_W1S, ~0); + if (ret) + goto failed_init_c2h; + + return 0; + +failed_init_c2h: + while (j--) + free_irq(xdev->c2h_chans[j].irq, &xdev->c2h_chans[j]); +failed_init_h2c: + while (i--) + free_irq(xdev->h2c_chans[i].irq, &xdev->h2c_chans[i]); + + return ret; +} + +static bool xdma_filter_fn(struct dma_chan *chan, void *param) +{ + struct xdma_chan *xdma_chan = to_xdma_chan(chan); + struct xdma_chan_info *chan_info = param; + + return chan_info->dir == xdma_chan->dir; +} + +/** + * xdma_disable_user_irq - Disable user interrupt + * @pdev: Pointer to the platform_device structure + * @irq_num: System IRQ number + */ +void xdma_disable_user_irq(struct platform_device *pdev, u32 irq_num) +{ + struct xdma_device *xdev = platform_get_drvdata(pdev); + u32 index; + + index = irq_num - xdev->irq_start; + if (index < XDMA_CHAN_NUM(xdev) || index >= xdev->irq_num) { + xdma_err(xdev, "invalid user irq number"); + return; + } + index -= XDMA_CHAN_NUM(xdev); + + regmap_write(xdev->rmap, XDMA_IRQ_USER_INT_EN_W1C, 1 << index); +} +EXPORT_SYMBOL(xdma_disable_user_irq); + +/** + * xdma_enable_user_irq - Enable user logic interrupt + * @pdev: Pointer to the platform_device structure + * @irq_num: System IRQ number + */ +int xdma_enable_user_irq(struct platform_device *pdev, u32 irq_num) +{ + struct xdma_device *xdev = platform_get_drvdata(pdev); + u32 index; + int ret; + + index = irq_num - xdev->irq_start; + if (index < XDMA_CHAN_NUM(xdev) || index >= xdev->irq_num) { + xdma_err(xdev, "invalid user irq number"); + return -EINVAL; + } + index -= XDMA_CHAN_NUM(xdev); + + ret = regmap_write(xdev->rmap, XDMA_IRQ_USER_INT_EN_W1S, 1 << index); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL(xdma_enable_user_irq); + +/** + * xdma_get_user_irq - Get system IRQ number + * @pdev: Pointer to the platform_device structure + * @user_irq_index: User logic IRQ wire index + * + * Return: The system IRQ number allocated for the given wire index. + */ +int xdma_get_user_irq(struct platform_device *pdev, u32 user_irq_index) +{ + struct xdma_device *xdev = platform_get_drvdata(pdev); + + if (XDMA_CHAN_NUM(xdev) + user_irq_index >= xdev->irq_num) { + xdma_err(xdev, "invalid user irq index"); + return -EINVAL; + } + + return xdev->irq_start + XDMA_CHAN_NUM(xdev) + user_irq_index; +} +EXPORT_SYMBOL(xdma_get_user_irq); + +/** + * xdma_remove - Driver remove function + * @pdev: Pointer to the platform_device structure + */ +static void xdma_remove(struct platform_device *pdev) +{ + struct xdma_device *xdev = platform_get_drvdata(pdev); + + if (xdev->status & XDMA_DEV_STATUS_INIT_MSIX) + xdma_irq_fini(xdev); + + if (xdev->status & XDMA_DEV_STATUS_REG_DMA) + dma_async_device_unregister(&xdev->dma_dev); +} + +/** + * xdma_probe - Driver probe function + * @pdev: Pointer to the platform_device structure + */ +static int xdma_probe(struct platform_device *pdev) +{ + struct xdma_platdata *pdata = dev_get_platdata(&pdev->dev); + struct xdma_device *xdev; + void __iomem *reg_base; + struct resource *res; + int ret = -ENODEV; + + if (pdata->max_dma_channels > XDMA_MAX_CHANNELS) { + dev_err(&pdev->dev, "invalid max dma channels %d", + pdata->max_dma_channels); + return -EINVAL; + } + + xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); + if (!xdev) + return -ENOMEM; + + platform_set_drvdata(pdev, xdev); + xdev->pdev = pdev; + + res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!res) { + xdma_err(xdev, "failed to get irq resource"); + goto failed; + } + xdev->irq_start = res->start; + xdev->irq_num = resource_size(res); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + xdma_err(xdev, "failed to get io resource"); + goto failed; + } + + reg_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(reg_base)) { + xdma_err(xdev, "ioremap failed"); + goto failed; + } + + xdev->rmap = devm_regmap_init_mmio(&pdev->dev, reg_base, + &xdma_regmap_config); + if (!xdev->rmap) { + xdma_err(xdev, "config regmap failed: %d", ret); + goto failed; + } + INIT_LIST_HEAD(&xdev->dma_dev.channels); + + ret = xdma_alloc_channels(xdev, DMA_MEM_TO_DEV); + if (ret) { + xdma_err(xdev, "config H2C channels failed: %d", ret); + goto failed; + } + + ret = xdma_alloc_channels(xdev, DMA_DEV_TO_MEM); + if (ret) { + xdma_err(xdev, "config C2H channels failed: %d", ret); + goto failed; + } + + dma_cap_set(DMA_SLAVE, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_PRIVATE, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_INTERLEAVE, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_REPEAT, xdev->dma_dev.cap_mask); + dma_cap_set(DMA_LOAD_EOT, xdev->dma_dev.cap_mask); + + xdev->dma_dev.dev = &pdev->dev; + xdev->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + xdev->dma_dev.device_free_chan_resources = xdma_free_chan_resources; + xdev->dma_dev.device_alloc_chan_resources = xdma_alloc_chan_resources; + xdev->dma_dev.device_tx_status = xdma_tx_status; + xdev->dma_dev.device_prep_slave_sg = xdma_prep_device_sg; + xdev->dma_dev.device_config = xdma_device_config; + xdev->dma_dev.device_issue_pending = xdma_issue_pending; + xdev->dma_dev.device_terminate_all = xdma_terminate_all; + xdev->dma_dev.device_synchronize = xdma_synchronize; + xdev->dma_dev.filter.map = pdata->device_map; + xdev->dma_dev.filter.mapcnt = pdata->device_map_cnt; + xdev->dma_dev.filter.fn = xdma_filter_fn; + xdev->dma_dev.device_prep_dma_cyclic = xdma_prep_dma_cyclic; + xdev->dma_dev.device_prep_interleaved_dma = xdma_prep_interleaved_dma; + + ret = dma_async_device_register(&xdev->dma_dev); + if (ret) { + xdma_err(xdev, "failed to register Xilinx XDMA: %d", ret); + goto failed; + } + xdev->status |= XDMA_DEV_STATUS_REG_DMA; + + ret = xdma_irq_init(xdev); + if (ret) { + xdma_err(xdev, "failed to init msix: %d", ret); + goto failed; + } + xdev->status |= XDMA_DEV_STATUS_INIT_MSIX; + + return 0; + +failed: + xdma_remove(pdev); + + return ret; +} + +static const struct platform_device_id xdma_id_table[] = { + { "xdma", 0}, + { }, +}; +MODULE_DEVICE_TABLE(platform, xdma_id_table); + +static struct platform_driver xdma_driver = { + .driver = { + .name = "xdma", + }, + .id_table = xdma_id_table, + .probe = xdma_probe, + .remove = xdma_remove, +}; + +module_platform_driver(xdma_driver); + +MODULE_DESCRIPTION("AMD XDMA driver"); +MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 75c0b8e904e5..fabff602065f 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -41,11 +41,12 @@ #include <linux/io.h> #include <linux/iopoll.h> #include <linux/module.h> -#include <linux/of_address.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_platform.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/clk.h> #include <linux/io-64-nonatomic-lo-hi.h> @@ -112,7 +113,9 @@ /* Register Direct Mode Registers */ #define XILINX_DMA_REG_VSIZE 0x0000 +#define XILINX_DMA_VSIZE_MASK GENMASK(12, 0) #define XILINX_DMA_REG_HSIZE 0x0004 +#define XILINX_DMA_HSIZE_MASK GENMASK(15, 0) #define XILINX_DMA_REG_FRMDLY_STRIDE 0x0008 #define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT 24 @@ -173,12 +176,15 @@ #define XILINX_DMA_MAX_TRANS_LEN_MAX 23 #define XILINX_DMA_V2_MAX_TRANS_LEN_MAX 26 #define XILINX_DMA_CR_COALESCE_MAX GENMASK(23, 16) +#define XILINX_DMA_CR_DELAY_MAX GENMASK(31, 24) #define XILINX_DMA_CR_CYCLIC_BD_EN_MASK BIT(4) #define XILINX_DMA_CR_COALESCE_SHIFT 16 +#define XILINX_DMA_CR_DELAY_SHIFT 24 #define XILINX_DMA_BD_SOP BIT(27) #define XILINX_DMA_BD_EOP BIT(26) +#define XILINX_DMA_BD_COMP_MASK BIT(31) #define XILINX_DMA_COALESCE_MAX 255 -#define XILINX_DMA_NUM_DESCS 255 +#define XILINX_DMA_NUM_DESCS 512 #define XILINX_DMA_NUM_APP_WORDS 5 /* AXI CDMA Specific Registers/Offsets */ @@ -394,6 +400,7 @@ struct xilinx_dma_tx_descriptor { * @genlock: Support genlock mode * @err: Channel has errors * @idle: Check for channel idle + * @terminating: Check for channel being synchronized by user * @tasklet: Cleanup work after irq * @config: Device configuration info * @flush_on_fsync: Flush on Frame sync @@ -409,6 +416,7 @@ struct xilinx_dma_tx_descriptor { * @stop_transfer: Differentiate b/w DMA IP's quiesce * @tdest: TDEST value for mcdma * @has_vflip: S2MM vertical flip + * @irq_delay: Interrupt delay timeout */ struct xilinx_dma_chan { struct xilinx_dma_device *xdev; @@ -431,6 +439,7 @@ struct xilinx_dma_chan { bool genlock; bool err; bool idle; + bool terminating; struct tasklet_struct tasklet; struct xilinx_vdma_config config; bool flush_on_fsync; @@ -446,6 +455,7 @@ struct xilinx_dma_chan { int (*stop_transfer)(struct xilinx_dma_chan *chan); u16 tdest; bool has_vflip; + u8 irq_delay; }; /** @@ -491,6 +501,7 @@ struct xilinx_dma_config { * @s2mm_chan_id: DMA s2mm channel identifier * @mm2s_chan_id: DMA mm2s channel identifier * @max_buffer_len: Max buffer length + * @has_axistream_connected: AXI DMA connected to AXI Stream IP */ struct xilinx_dma_device { void __iomem *regs; @@ -509,6 +520,7 @@ struct xilinx_dma_device { u32 s2mm_chan_id; u32 mm2s_chan_id; u32 max_buffer_len; + bool has_axistream_connected; }; /* Macros */ @@ -621,6 +633,29 @@ static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan, } } +/** + * xilinx_dma_get_metadata_ptr- Populate metadata pointer and payload length + * @tx: async transaction descriptor + * @payload_len: metadata payload length + * @max_len: metadata max length + * Return: The app field pointer. + */ +static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx, + size_t *payload_len, size_t *max_len) +{ + struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); + struct xilinx_axidma_tx_segment *seg; + + *max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS; + seg = list_first_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + return seg->hw.app; +} + +static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { + .get_ptr = xilinx_dma_get_metadata_ptr, +}; + /* ----------------------------------------------------------------------------- * Descriptors and segments alloc and free */ @@ -790,7 +825,7 @@ static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan, } /** - * xilinx_dma_tx_descriptor - Allocate transaction descriptor + * xilinx_dma_alloc_tx_descriptor - Allocate transaction descriptor * @chan: Driver specific DMA channel * * Return: The allocated descriptor on success and NULL on failure. @@ -996,14 +1031,12 @@ static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan, struct xilinx_dma_tx_descriptor *desc, unsigned long *flags) { - dma_async_tx_callback callback; - void *callback_param; + struct dmaengine_desc_callback cb; - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; - if (callback) { + dmaengine_desc_get_callback(&desc->async_tx, &cb); + if (dmaengine_desc_callback_valid(&cb)) { spin_unlock_irqrestore(&chan->lock, *flags); - callback(callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irqsave(&chan->lock, *flags); } } @@ -1049,6 +1082,13 @@ static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan) /* Run any dependencies, then free the descriptor */ dma_run_dependencies(&desc->async_tx); xilinx_dma_free_tx_descriptor(chan, desc); + + /* + * While we ran a callback the user called a terminate function, + * which takes care of cleaning up any remaining descriptors + */ + if (chan->terminating) + break; } spin_unlock_irqrestore(&chan->lock, flags); @@ -1365,16 +1405,18 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); - j = chan->desc_submitcount; - reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); - if (chan->direction == DMA_MEM_TO_DEV) { - reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; - } else { - reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + if (config->park) { + j = chan->desc_submitcount; + reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); + if (chan->direction == DMA_MEM_TO_DEV) { + reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; + } else { + reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + } + dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); } - dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); /* Start the hardware */ xilinx_dma_start(chan); @@ -1411,8 +1453,7 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) chan->desc_submitcount++; chan->desc_pendingcount--; - list_del(&desc->node); - list_add_tail(&desc->node, &chan->active_list); + list_move_tail(&desc->node, &chan->active_list); if (chan->desc_submitcount == chan->num_frms) chan->desc_submitcount = 0; @@ -1529,6 +1570,9 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) if (chan->has_sg) xilinx_write(chan, XILINX_DMA_REG_CURDESC, head_desc->async_tx.phys); + reg &= ~XILINX_DMA_CR_DELAY_MAX; + reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); xilinx_dma_start(chan); @@ -1650,6 +1694,19 @@ static void xilinx_dma_issue_pending(struct dma_chan *dchan) } /** + * xilinx_dma_device_config - Configure the DMA channel + * @dchan: DMA channel + * @config: channel configuration + * + * Return: 0 always. + */ +static int xilinx_dma_device_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + return 0; +} + +/** * xilinx_dma_complete_descriptor - Mark the active descriptor as complete * @chan : xilinx DMA channel * @@ -1664,6 +1721,14 @@ static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan) return; list_for_each_entry_safe(desc, next, &chan->active_list, node) { + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + struct xilinx_axidma_tx_segment *seg; + + seg = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg) + break; + } if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA) desc->residue = xilinx_dma_get_residue(chan, desc); @@ -1797,7 +1862,7 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) spin_unlock(&chan->lock); } - tasklet_schedule(&chan->tasklet); + tasklet_hi_schedule(&chan->tasklet); return IRQ_HANDLED; } @@ -1845,15 +1910,8 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) } } - if (status & XILINX_DMA_DMASR_DLY_CNT_IRQ) { - /* - * Device takes too long to do the transfer when user requires - * responsiveness. - */ - dev_dbg(chan->dev, "Inter-packet latency too long\n"); - } - - if (status & XILINX_DMA_DMASR_FRM_CNT_IRQ) { + if (status & (XILINX_DMA_DMASR_FRM_CNT_IRQ | + XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); chan->idle = true; @@ -1965,6 +2023,8 @@ static dma_cookie_t xilinx_dma_tx_submit(struct dma_async_tx_descriptor *tx) if (desc->cyclic) chan->cyclic = true; + chan->terminating = false; + spin_unlock_irqrestore(&chan->lock, flags); return cookie; @@ -1995,6 +2055,10 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, if (!xt->numf || !xt->sgl[0].size) return NULL; + if (xt->numf & ~XILINX_DMA_VSIZE_MASK || + xt->sgl[0].size & ~XILINX_DMA_HSIZE_MASK) + return NULL; + if (xt->frame_size != 1) return NULL; @@ -2109,6 +2173,99 @@ error: } /** + * xilinx_dma_prep_peripheral_dma_vec - prepare descriptors for a DMA_SLAVE + * transaction from DMA vectors + * @dchan: DMA channel + * @vecs: Array of DMA vectors that should be transferred + * @nb: number of entries in @vecs + * @direction: DMA direction + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor *xilinx_dma_prep_peripheral_dma_vec( + struct dma_chan *dchan, const struct dma_vec *vecs, size_t nb, + enum dma_transfer_direction direction, unsigned long flags) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_axidma_tx_segment *segment, *head, *prev = NULL; + size_t copy; + size_t sg_used; + unsigned int i; + + if (!is_slave_direction(direction) || direction != chan->direction) + return NULL; + + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + + /* Build transactions using information from DMA vectors */ + for (i = 0; i < nb; i++) { + sg_used = 0; + + /* Loop until the entire dma_vec entry is used */ + while (sg_used < vecs[i].len) { + struct xilinx_axidma_desc_hw *hw; + + /* Get a free segment */ + segment = xilinx_axidma_alloc_tx_segment(chan); + if (!segment) + goto error; + + /* + * Calculate the maximum number of bytes to transfer, + * making sure it is less than the hw limit + */ + copy = xilinx_dma_calc_copysize(chan, vecs[i].len, + sg_used); + hw = &segment->hw; + + /* Fill in the descriptor */ + xilinx_axidma_buf(chan, hw, vecs[i].addr, sg_used, 0); + hw->control = copy; + + if (prev) + prev->hw.next_desc = segment->phys; + + prev = segment; + sg_used += copy; + + /* + * Insert the segment into the descriptor segments + * list. + */ + list_add_tail(&segment->node, &desc->segments); + } + } + + head = list_first_entry(&desc->segments, struct xilinx_axidma_tx_segment, node); + desc->async_tx.phys = head->phys; + + /* For the last DMA_MEM_TO_DEV transfer, set EOP */ + if (chan->direction == DMA_MEM_TO_DEV) { + segment->hw.control |= XILINX_DMA_BD_SOP; + segment = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, + node); + segment->hw.control |= XILINX_DMA_BD_EOP; + } + + if (chan->xdev->has_axistream_connected) + desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction * @dchan: DMA channel * @sgl: scatterlist to transfer to/from @@ -2200,6 +2357,9 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg( segment->hw.control |= XILINX_DMA_BD_EOP; } + if (chan->xdev->has_axistream_connected) + desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; + return &desc->async_tx; error: @@ -2436,6 +2596,7 @@ static int xilinx_dma_terminate_all(struct dma_chan *dchan) xilinx_dma_chan_reset(chan); /* Remove and free all of the descriptors in the lists */ + chan->terminating = true; xilinx_dma_free_descriptors(chan); chan->idle = true; @@ -2461,7 +2622,7 @@ static void xilinx_dma_synchronize(struct dma_chan *dchan) } /** - * xilinx_dma_channel_set_config - Configure VDMA channel + * xilinx_vdma_channel_set_config - Configure VDMA channel * Run-time configuration for Axi VDMA, supports: * . halt the channel * . configure interrupt coalescing and inter-packet delay threshold @@ -2774,6 +2935,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, /* Retrieve the channel properties from the device tree */ has_dre = of_property_read_bool(node, "xlnx,include-dre"); + of_property_read_u8(node, "xlnx,irq-delay", &chan->irq_delay); + chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); err = of_property_read_u32(node, "xlnx,datawidth", &value); @@ -2839,8 +3002,12 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, return -EINVAL; } + xdev->common.directions |= chan->direction; + /* Request the interrupt */ - chan->irq = irq_of_parse_and_map(node, chan->tdest); + chan->irq = of_irq_get(node, chan->tdest); + if (chan->irq < 0) + return dev_err_probe(xdev->dev, chan->irq, "failed to get irq\n"); err = request_irq(chan->irq, xdev->dma_config->irq_handler, IRQF_SHARED, "xilinx-dma-controller", chan); if (err) { @@ -2869,7 +3036,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, XILINX_DMA_DMASR_SG_MASK) chan->has_sg = true; dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id, - chan->has_sg ? "enabled" : "disabled"); + str_enabled_disabled(chan->has_sg)); } /* Initialize the tasklet */ @@ -2902,7 +3069,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, * @xdev: Driver specific device structure * @node: Device node * - * Return: 0 always. + * Return: '0' on success and failure value on error. */ static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, struct device_node *node) @@ -2914,8 +3081,11 @@ static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0) dev_warn(xdev->dev, "missing dma-channels property\n"); - for (i = 0; i < nr_channels; i++) - xilinx_dma_chan_probe(xdev, node); + for (i = 0; i < nr_channels; i++) { + ret = xilinx_dma_chan_probe(xdev, node); + if (ret) + return ret; + } return 0; } @@ -3015,9 +3185,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) /* Request and map I/O memory */ xdev->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(xdev->regs)) - return PTR_ERR(xdev->regs); - + if (IS_ERR(xdev->regs)) { + err = PTR_ERR(xdev->regs); + goto disable_clks; + } /* Retrieve the DMA engine properties from the device tree */ xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; @@ -3039,13 +3210,20 @@ static int xilinx_dma_probe(struct platform_device *pdev) } } + dma_set_max_seg_size(xdev->dev, xdev->max_buffer_len); + + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + xdev->has_axistream_connected = + of_property_read_bool(node, "xlnx,axistream-connected"); + } + if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { err = of_property_read_u32(node, "xlnx,num-fstores", &num_frames); if (err < 0) { dev_err(xdev->dev, "missing xlnx,num-fstores property\n"); - return err; + goto disable_clks; } err = of_property_read_u32(node, "xlnx,flush-fsync", @@ -3064,8 +3242,16 @@ static int xilinx_dma_probe(struct platform_device *pdev) else xdev->ext_addr = false; + /* Set metadata mode */ + if (xdev->has_axistream_connected) + xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE; + /* Set the dma mask bits */ - dma_set_mask(xdev->dev, DMA_BIT_MASK(addr_width)); + err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width)); + if (err < 0) { + dev_err(xdev->dev, "DMA mask error %d\n", err); + goto disable_clks; + } /* Initialize the DMA engine */ xdev->common.dev = &pdev->dev; @@ -3084,8 +3270,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) xdev->common.device_synchronize = xilinx_dma_synchronize; xdev->common.device_tx_status = xilinx_dma_tx_status; xdev->common.device_issue_pending = xilinx_dma_issue_pending; + xdev->common.device_config = xilinx_dma_device_config; if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask); + xdev->common.device_prep_peripheral_dma_vec = xilinx_dma_prep_peripheral_dma_vec; xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg; xdev->common.device_prep_dma_cyclic = xilinx_dma_prep_dma_cyclic; @@ -3110,8 +3298,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) /* Initialize the channels */ for_each_child_of_node(node, child) { err = xilinx_dma_child_probe(xdev, child); - if (err < 0) - goto disable_clks; + if (err < 0) { + of_node_put(child); + goto error; + } } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { @@ -3146,12 +3336,12 @@ static int xilinx_dma_probe(struct platform_device *pdev) return 0; -disable_clks: - xdma_disable_allclks(xdev); error: for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xilinx_dma_chan_remove(xdev->chan[i]); +disable_clks: + xdma_disable_allclks(xdev); return err; } @@ -3159,10 +3349,8 @@ error: /** * xilinx_dma_remove - Driver remove function * @pdev: Pointer to the platform_device structure - * - * Return: Always '0' */ -static int xilinx_dma_remove(struct platform_device *pdev) +static void xilinx_dma_remove(struct platform_device *pdev) { struct xilinx_dma_device *xdev = platform_get_drvdata(pdev); int i; @@ -3176,8 +3364,6 @@ static int xilinx_dma_remove(struct platform_device *pdev) xilinx_dma_chan_remove(xdev->chan[i]); xdma_disable_allclks(xdev); - - return 0; } static struct platform_driver xilinx_vdma_driver = { diff --git a/drivers/dma/xilinx/xilinx_dpdma.c b/drivers/dma/xilinx/xilinx_dpdma.c index b280a53e8570..ee5d9fdbfd7f 100644 --- a/drivers/dma/xilinx/xilinx_dpdma.c +++ b/drivers/dma/xilinx/xilinx_dpdma.c @@ -12,6 +12,7 @@ #include <linux/clk.h> #include <linux/debugfs.h> #include <linux/delay.h> +#include <linux/dma/xilinx_dpdma.h> #include <linux/dmaengine.h> #include <linux/dmapool.h> #include <linux/interrupt.h> @@ -148,7 +149,7 @@ struct xilinx_dpdma_chan; * @addr_ext: upper 16 bit of 48 bit address (next_desc and src_addr) * @next_desc: next descriptor 32 bit address * @src_addr: payload source address (1st page, 32 LSB) - * @addr_ext_23: payload source address (3nd and 3rd pages, 16 LSBs) + * @addr_ext_23: payload source address (2nd and 3rd pages, 16 LSBs) * @addr_ext_45: payload source address (4th and 5th pages, 16 LSBs) * @src_addr2: payload source address (2nd page, 32 LSB) * @src_addr3: payload source address (3rd page, 32 LSB) @@ -209,11 +210,12 @@ struct xilinx_dpdma_tx_desc { * @vchan: virtual DMA channel * @reg: register base address * @id: channel ID - * @wait_to_stop: queue to wait for outstanding transacitons before stopping + * @wait_to_stop: queue to wait for outstanding transactions before stopping * @running: true if the channel is running * @first_frame: flag for the first frame of stream * @video_group: flag if multi-channel operation is needed for video channels - * @lock: lock to access struct xilinx_dpdma_chan + * @lock: lock to access struct xilinx_dpdma_chan. Must be taken before + * @vchan.lock, if both are to be held. * @desc_pool: descriptor allocation pool * @err_task: error IRQ bottom half handler * @desc: References to descriptors being processed @@ -271,9 +273,6 @@ struct xilinx_dpdma_device { /* ----------------------------------------------------------------------------- * DebugFS */ - -#ifdef CONFIG_DEBUG_FS - #define XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE 32 #define XILINX_DPDMA_DEBUGFS_UINT16_MAX_STR "65535" @@ -299,7 +298,7 @@ struct xilinx_dpdma_debugfs_request { static void xilinx_dpdma_debugfs_desc_done_irq(struct xilinx_dpdma_chan *chan) { - if (chan->id == dpdma_debugfs.chan_id) + if (IS_ENABLED(CONFIG_DEBUG_FS) && chan->id == dpdma_debugfs.chan_id) dpdma_debugfs.xilinx_dpdma_irq_done_count++; } @@ -311,7 +310,7 @@ static ssize_t xilinx_dpdma_debugfs_desc_done_irq_read(char *buf) out_str_len = strlen(XILINX_DPDMA_DEBUGFS_UINT16_MAX_STR); out_str_len = min_t(size_t, XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE, - out_str_len); + out_str_len + 1); snprintf(buf, out_str_len, "%d", dpdma_debugfs.xilinx_dpdma_irq_done_count); @@ -378,7 +377,7 @@ static ssize_t xilinx_dpdma_debugfs_read(struct file *f, char __user *buf, if (ret < 0) goto done; } else { - strlcpy(kern_buff, "No testcase executed", + strscpy(kern_buff, "No testcase executed", XILINX_DPDMA_DEBUGFS_READ_MAX_SIZE); } @@ -462,16 +461,6 @@ static void xilinx_dpdma_debugfs_init(struct xilinx_dpdma_device *xdev) dev_err(xdev->dev, "Failed to create debugfs testcase file\n"); } -#else -static void xilinx_dpdma_debugfs_init(struct xilinx_dpdma_device *xdev) -{ -} - -static void xilinx_dpdma_debugfs_desc_done_irq(struct xilinx_dpdma_chan *chan) -{ -} -#endif /* CONFIG_DEBUG_FS */ - /* ----------------------------------------------------------------------------- * I/O Accessors */ @@ -682,6 +671,84 @@ static void xilinx_dpdma_chan_free_tx_desc(struct virt_dma_desc *vdesc) } /** + * xilinx_dpdma_chan_prep_cyclic - Prepare a cyclic dma descriptor + * @chan: DPDMA channel + * @buf_addr: buffer address + * @buf_len: buffer length + * @period_len: number of periods + * @flags: tx flags argument passed in to prepare function + * + * Prepare a tx descriptor incudling internal software/hardware descriptors + * for the given cyclic transaction. + * + * Return: A dma async tx descriptor on success, or NULL. + */ +static struct dma_async_tx_descriptor * +xilinx_dpdma_chan_prep_cyclic(struct xilinx_dpdma_chan *chan, + dma_addr_t buf_addr, size_t buf_len, + size_t period_len, unsigned long flags) +{ + struct xilinx_dpdma_tx_desc *tx_desc; + struct xilinx_dpdma_sw_desc *sw_desc, *last = NULL; + unsigned int periods = buf_len / period_len; + unsigned int i; + + tx_desc = xilinx_dpdma_chan_alloc_tx_desc(chan); + if (!tx_desc) + return NULL; + + for (i = 0; i < periods; i++) { + struct xilinx_dpdma_hw_desc *hw_desc; + + if (!IS_ALIGNED(buf_addr, XILINX_DPDMA_ALIGN_BYTES)) { + dev_err(chan->xdev->dev, + "buffer should be aligned at %d B\n", + XILINX_DPDMA_ALIGN_BYTES); + goto error; + } + + sw_desc = xilinx_dpdma_chan_alloc_sw_desc(chan); + if (!sw_desc) + goto error; + + xilinx_dpdma_sw_desc_set_dma_addrs(chan->xdev, sw_desc, last, + &buf_addr, 1); + hw_desc = &sw_desc->hw; + hw_desc->xfer_size = period_len; + hw_desc->hsize_stride = + FIELD_PREP(XILINX_DPDMA_DESC_HSIZE_STRIDE_HSIZE_MASK, + period_len) | + FIELD_PREP(XILINX_DPDMA_DESC_HSIZE_STRIDE_STRIDE_MASK, + period_len); + hw_desc->control = XILINX_DPDMA_DESC_CONTROL_PREEMBLE | + XILINX_DPDMA_DESC_CONTROL_IGNORE_DONE | + XILINX_DPDMA_DESC_CONTROL_COMPLETE_INTR; + + list_add_tail(&sw_desc->node, &tx_desc->descriptors); + + buf_addr += period_len; + last = sw_desc; + } + + sw_desc = list_first_entry(&tx_desc->descriptors, + struct xilinx_dpdma_sw_desc, node); + last->hw.next_desc = lower_32_bits(sw_desc->dma_addr); + if (chan->xdev->ext_addr) + last->hw.addr_ext |= + FIELD_PREP(XILINX_DPDMA_DESC_ADDR_EXT_NEXT_ADDR_MASK, + upper_32_bits(sw_desc->dma_addr)); + + last->hw.control |= XILINX_DPDMA_DESC_CONTROL_LAST_OF_FRAME; + + return vchan_tx_prep(&chan->vchan, &tx_desc->vdesc, flags); + +error: + xilinx_dpdma_chan_free_tx_desc(&tx_desc->vdesc); + + return NULL; +} + +/** * xilinx_dpdma_chan_prep_interleaved_dma - Prepare an interleaved dma * descriptor * @chan: DPDMA channel @@ -1054,9 +1121,8 @@ static int xilinx_dpdma_chan_stop(struct xilinx_dpdma_chan *chan) static void xilinx_dpdma_chan_done_irq(struct xilinx_dpdma_chan *chan) { struct xilinx_dpdma_tx_desc *active; - unsigned long flags; - spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->lock); xilinx_dpdma_debugfs_desc_done_irq(chan); @@ -1068,7 +1134,7 @@ static void xilinx_dpdma_chan_done_irq(struct xilinx_dpdma_chan *chan) "chan%u: DONE IRQ with no active descriptor!\n", chan->id); - spin_unlock_irqrestore(&chan->lock, flags); + spin_unlock(&chan->lock); } /** @@ -1083,10 +1149,9 @@ static void xilinx_dpdma_chan_vsync_irq(struct xilinx_dpdma_chan *chan) { struct xilinx_dpdma_tx_desc *pending; struct xilinx_dpdma_sw_desc *sw_desc; - unsigned long flags; u32 desc_id; - spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->lock); pending = chan->desc.pending; if (!chan->running || !pending) @@ -1109,15 +1174,17 @@ static void xilinx_dpdma_chan_vsync_irq(struct xilinx_dpdma_chan *chan) * Complete the active descriptor, if any, promote the pending * descriptor to active, and queue the next transfer, if any. */ + spin_lock(&chan->vchan.lock); if (chan->desc.active) vchan_cookie_complete(&chan->desc.active->vdesc); chan->desc.active = pending; chan->desc.pending = NULL; xilinx_dpdma_chan_queue_transfer(chan); + spin_unlock(&chan->vchan.lock); out: - spin_unlock_irqrestore(&chan->lock, flags); + spin_unlock(&chan->lock); } /** @@ -1200,6 +1267,23 @@ out_unlock: /* ----------------------------------------------------------------------------- * DMA Engine Operations */ +static struct dma_async_tx_descriptor * +xilinx_dpdma_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t buf_addr, + size_t buf_len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags) +{ + struct xilinx_dpdma_chan *chan = to_xilinx_chan(dchan); + + if (direction != DMA_MEM_TO_DEV) + return NULL; + + if (buf_len % period_len) + return NULL; + + return xilinx_dpdma_chan_prep_cyclic(chan, buf_addr, buf_len, + period_len, flags); +} static struct dma_async_tx_descriptor * xilinx_dpdma_prep_interleaved_dma(struct dma_chan *dchan, @@ -1276,16 +1360,19 @@ static void xilinx_dpdma_issue_pending(struct dma_chan *dchan) struct xilinx_dpdma_chan *chan = to_xilinx_chan(dchan); unsigned long flags; - spin_lock_irqsave(&chan->vchan.lock, flags); + spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->vchan.lock); if (vchan_issue_pending(&chan->vchan)) xilinx_dpdma_chan_queue_transfer(chan); - spin_unlock_irqrestore(&chan->vchan.lock, flags); + spin_unlock(&chan->vchan.lock); + spin_unlock_irqrestore(&chan->lock, flags); } static int xilinx_dpdma_config(struct dma_chan *dchan, struct dma_slave_config *config) { struct xilinx_dpdma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dpdma_peripheral_config *pconfig; unsigned long flags; /* @@ -1295,15 +1382,18 @@ static int xilinx_dpdma_config(struct dma_chan *dchan, * fixed both on the DPDMA side and on the DP controller side. */ - spin_lock_irqsave(&chan->lock, flags); - /* - * Abuse the slave_id to indicate that the channel is part of a video - * group. + * Use the peripheral_config to indicate that the channel is part + * of a video group. This requires matching use of the custom + * structure in each driver. */ - if (chan->id <= ZYNQMP_DPDMA_VIDEO2) - chan->video_group = config->slave_id != 0; + pconfig = config->peripheral_config; + if (WARN_ON(pconfig && config->peripheral_size != sizeof(*pconfig))) + return -EINVAL; + spin_lock_irqsave(&chan->lock, flags); + if (chan->id <= ZYNQMP_DPDMA_VIDEO2 && pconfig) + chan->video_group = pconfig->video_group; spin_unlock_irqrestore(&chan->lock, flags); return 0; @@ -1503,7 +1593,9 @@ static void xilinx_dpdma_chan_err_task(struct tasklet_struct *t) XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id); spin_lock_irqsave(&chan->lock, flags); + spin_lock(&chan->vchan.lock); xilinx_dpdma_chan_queue_transfer(chan); + spin_unlock(&chan->vchan.lock); spin_unlock_irqrestore(&chan->lock, flags); } @@ -1660,10 +1752,8 @@ static int xilinx_dpdma_probe(struct platform_device *pdev) dpdma_hw_init(xdev); xdev->irq = platform_get_irq(pdev, 0); - if (xdev->irq < 0) { - dev_err(xdev->dev, "failed to get platform irq\n"); + if (xdev->irq < 0) return xdev->irq; - } ret = request_irq(xdev->irq, xilinx_dpdma_irq_handler, IRQF_SHARED, dev_name(xdev->dev), xdev); @@ -1677,6 +1767,7 @@ static int xilinx_dpdma_probe(struct platform_device *pdev) dma_cap_set(DMA_SLAVE, ddev->cap_mask); dma_cap_set(DMA_PRIVATE, ddev->cap_mask); + dma_cap_set(DMA_CYCLIC, ddev->cap_mask); dma_cap_set(DMA_INTERLEAVE, ddev->cap_mask); dma_cap_set(DMA_REPEAT, ddev->cap_mask); dma_cap_set(DMA_LOAD_EOT, ddev->cap_mask); @@ -1684,6 +1775,7 @@ static int xilinx_dpdma_probe(struct platform_device *pdev) ddev->device_alloc_chan_resources = xilinx_dpdma_alloc_chan_resources; ddev->device_free_chan_resources = xilinx_dpdma_free_chan_resources; + ddev->device_prep_dma_cyclic = xilinx_dpdma_prep_dma_cyclic; ddev->device_prep_interleaved_dma = xilinx_dpdma_prep_interleaved_dma; /* TODO: Can we achieve better granularity ? */ ddev->device_tx_status = dma_cookie_status; @@ -1746,7 +1838,7 @@ error: return ret; } -static int xilinx_dpdma_remove(struct platform_device *pdev) +static void xilinx_dpdma_remove(struct platform_device *pdev) { struct xilinx_dpdma_device *xdev = platform_get_drvdata(pdev); unsigned int i; @@ -1761,8 +1853,6 @@ static int xilinx_dpdma_remove(struct platform_device *pdev) for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) xilinx_dpdma_chan_remove(xdev->chan[i]); - - return 0; } static const struct of_device_id xilinx_dpdma_of_match[] = { diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index 5fecf5aa6e85..f7e584de4335 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -6,16 +6,14 @@ */ #include <linux/bitops.h> -#include <linux/dmapool.h> -#include <linux/dma/xilinx_dma.h> +#include <linux/dma-mapping.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/module.h> -#include <linux/of_address.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/clk.h> #include <linux/io-64-nonatomic-lo-hi.h> @@ -24,10 +22,10 @@ #include "../dmaengine.h" /* Register Offsets */ -#define ZYNQMP_DMA_ISR 0x100 -#define ZYNQMP_DMA_IMR 0x104 -#define ZYNQMP_DMA_IER 0x108 -#define ZYNQMP_DMA_IDS 0x10C +#define ZYNQMP_DMA_ISR (chan->irq_offset + 0x100) +#define ZYNQMP_DMA_IMR (chan->irq_offset + 0x104) +#define ZYNQMP_DMA_IER (chan->irq_offset + 0x108) +#define ZYNQMP_DMA_IDS (chan->irq_offset + 0x10c) #define ZYNQMP_DMA_CTRL0 0x110 #define ZYNQMP_DMA_CTRL1 0x114 #define ZYNQMP_DMA_DATA_ATTR 0x120 @@ -147,6 +145,9 @@ #define tx_to_desc(tx) container_of(tx, struct zynqmp_dma_desc_sw, \ async_tx) +/* IRQ Register offset for Versal Gen 2 */ +#define IRQ_REG_OFFSET 0x308 + /** * struct zynqmp_dma_desc_ll - Hw linked list descriptor * @addr: Buffer address @@ -213,6 +214,7 @@ struct zynqmp_dma_desc_sw { * @bus_width: Bus width * @src_burst_len: Source burst length * @dst_burst_len: Dest burst length + * @irq_offset: Irq register offset */ struct zynqmp_dma_chan { struct zynqmp_dma_device *zdev; @@ -232,11 +234,12 @@ struct zynqmp_dma_chan { bool is_dmacoherent; struct tasklet_struct tasklet; bool idle; - u32 desc_size; + size_t desc_size; bool err; u32 bus_width; u32 src_burst_len; u32 dst_burst_len; + u32 irq_offset; }; /** @@ -255,6 +258,14 @@ struct zynqmp_dma_device { struct clk *clk_apb; }; +struct zynqmp_dma_config { + u32 offset; +}; + +static const struct zynqmp_dma_config versal2_dma_config = { + .offset = IRQ_REG_OFFSET, +}; + static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg, u64 value) { @@ -355,7 +366,7 @@ static void zynqmp_dma_init(struct zynqmp_dma_chan *chan) } writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR); - /* Clearing the interrupt account rgisters */ + /* Clearing the interrupt account registers */ val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); @@ -434,8 +445,7 @@ static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan, struct zynqmp_dma_desc_sw *child, *next; chan->desc_free_cnt++; - list_del(&sdesc->node); - list_add_tail(&sdesc->node, &chan->free_list); + list_move_tail(&sdesc->node, &chan->free_list); list_for_each_entry_safe(child, next, &sdesc->tx_list, node) { chan->desc_free_cnt++; list_move_tail(&child->node, &chan->free_list); @@ -490,7 +500,8 @@ static int zynqmp_dma_alloc_chan_resources(struct dma_chan *dchan) } chan->desc_pool_v = dma_alloc_coherent(chan->dev, - (2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS), + (2 * ZYNQMP_DMA_DESC_SIZE(chan) * + ZYNQMP_DMA_NUM_DESCS), &chan->desc_pool_p, GFP_KERNEL); if (!chan->desc_pool_v) return -ENOMEM; @@ -604,22 +615,25 @@ static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan) static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan) { struct zynqmp_dma_desc_sw *desc, *next; + unsigned long irqflags; + + spin_lock_irqsave(&chan->lock, irqflags); list_for_each_entry_safe(desc, next, &chan->done_list, node) { - dma_async_tx_callback callback; - void *callback_param; - - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; - if (callback) { - spin_unlock(&chan->lock); - callback(callback_param); - spin_lock(&chan->lock); + struct dmaengine_desc_callback cb; + + dmaengine_desc_get_callback(&desc->async_tx, &cb); + if (dmaengine_desc_callback_valid(&cb)) { + spin_unlock_irqrestore(&chan->lock, irqflags); + dmaengine_desc_callback_invoke(&cb, NULL); + spin_lock_irqsave(&chan->lock, irqflags); } /* Run any dependencies, then free the descriptor */ zynqmp_dma_free_descriptor(chan, desc); } + + spin_unlock_irqrestore(&chan->lock, irqflags); } /** @@ -659,9 +673,13 @@ static void zynqmp_dma_issue_pending(struct dma_chan *dchan) */ static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan) { + unsigned long irqflags; + + spin_lock_irqsave(&chan->lock, irqflags); zynqmp_dma_free_desc_list(chan, &chan->active_list); zynqmp_dma_free_desc_list(chan, &chan->pending_list); zynqmp_dma_free_desc_list(chan, &chan->done_list); + spin_unlock_irqrestore(&chan->lock, irqflags); } /** @@ -671,11 +689,8 @@ static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan) static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan) { struct zynqmp_dma_chan *chan = to_chan(dchan); - unsigned long irqflags; - spin_lock_irqsave(&chan->lock, irqflags); zynqmp_dma_free_descriptors(chan); - spin_unlock_irqrestore(&chan->lock, irqflags); dma_free_coherent(chan->dev, (2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS), chan->desc_pool_v, chan->desc_pool_p); @@ -690,11 +705,16 @@ static void zynqmp_dma_free_chan_resources(struct dma_chan *dchan) */ static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan) { + unsigned long irqflags; + writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); + spin_lock_irqsave(&chan->lock, irqflags); zynqmp_dma_complete_descriptor(chan); + spin_unlock_irqrestore(&chan->lock, irqflags); zynqmp_dma_chan_desc_cleanup(chan); zynqmp_dma_free_descriptors(chan); + zynqmp_dma_init(chan); } @@ -750,27 +770,27 @@ static void zynqmp_dma_do_tasklet(struct tasklet_struct *t) u32 count; unsigned long irqflags; - spin_lock_irqsave(&chan->lock, irqflags); - if (chan->err) { zynqmp_dma_reset(chan); chan->err = false; - goto unlock; + return; } + spin_lock_irqsave(&chan->lock, irqflags); count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); - while (count) { zynqmp_dma_complete_descriptor(chan); - zynqmp_dma_chan_desc_cleanup(chan); count--; } + spin_unlock_irqrestore(&chan->lock, irqflags); - if (chan->idle) - zynqmp_dma_start_transfer(chan); + zynqmp_dma_chan_desc_cleanup(chan); -unlock: - spin_unlock_irqrestore(&chan->lock, irqflags); + if (chan->idle) { + spin_lock_irqsave(&chan->lock, irqflags); + zynqmp_dma_start_transfer(chan); + spin_unlock_irqrestore(&chan->lock, irqflags); + } } /** @@ -782,17 +802,25 @@ unlock: static int zynqmp_dma_device_terminate_all(struct dma_chan *dchan) { struct zynqmp_dma_chan *chan = to_chan(dchan); - unsigned long irqflags; - spin_lock_irqsave(&chan->lock, irqflags); writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); zynqmp_dma_free_descriptors(chan); - spin_unlock_irqrestore(&chan->lock, irqflags); return 0; } /** + * zynqmp_dma_synchronize - Synchronizes the termination of a transfers to the current context. + * @dchan: DMA channel pointer + */ +static void zynqmp_dma_synchronize(struct dma_chan *dchan) +{ + struct zynqmp_dma_chan *chan = to_chan(dchan); + + tasklet_kill(&chan->tasklet); +} + +/** * zynqmp_dma_prep_memcpy - prepare descriptors for memcpy transaction * @dchan: DMA channel * @dma_dst: Destination buffer address @@ -846,7 +874,7 @@ static struct dma_async_tx_descriptor *zynqmp_dma_prep_memcpy( zynqmp_dma_desc_config_eod(chan, desc); async_tx_ack(&first->async_tx); - first->async_tx.flags = flags; + first->async_tx.flags = (enum dma_ctrl_flags)flags; return &first->async_tx; } @@ -876,8 +904,8 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, struct platform_device *pdev) { struct zynqmp_dma_chan *chan; - struct resource *res; struct device_node *node = pdev->dev.of_node; + const struct zynqmp_dma_config *match_data; int err; chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL); @@ -886,8 +914,7 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, chan->dev = zdev->dev; chan->zdev = zdev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - chan->regs = devm_ioremap_resource(&pdev->dev, res); + chan->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(chan->regs)) return PTR_ERR(chan->regs); @@ -906,6 +933,10 @@ static int zynqmp_dma_chan_probe(struct zynqmp_dma_device *zdev, return -EINVAL; } + match_data = of_device_get_match_data(&pdev->dev); + if (match_data) + chan->irq_offset = match_data->offset; + chan->is_dmacoherent = of_property_read_bool(node, "dma-coherent"); zdev->chan = chan; tasklet_setup(&chan->tasklet, zynqmp_dma_do_tasklet); @@ -1048,12 +1079,17 @@ static int zynqmp_dma_probe(struct platform_device *pdev) zdev->dev = &pdev->dev; INIT_LIST_HEAD(&zdev->common.channels); - dma_set_mask(&pdev->dev, DMA_BIT_MASK(44)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + if (ret) { + dev_err(&pdev->dev, "DMA not available for address range\n"); + return ret; + } dma_cap_set(DMA_MEMCPY, zdev->common.cap_mask); p = &zdev->common; p->device_prep_dma_memcpy = zynqmp_dma_prep_memcpy; p->device_terminate_all = zynqmp_dma_device_terminate_all; + p->device_synchronize = zynqmp_dma_synchronize; p->device_issue_pending = zynqmp_dma_issue_pending; p->device_alloc_chan_resources = zynqmp_dma_alloc_chan_resources; p->device_free_chan_resources = zynqmp_dma_free_chan_resources; @@ -1062,22 +1098,24 @@ static int zynqmp_dma_probe(struct platform_device *pdev) p->dev = &pdev->dev; zdev->clk_main = devm_clk_get(&pdev->dev, "clk_main"); - if (IS_ERR(zdev->clk_main)) { - dev_err(&pdev->dev, "main clock not found.\n"); - return PTR_ERR(zdev->clk_main); - } + if (IS_ERR(zdev->clk_main)) + return dev_err_probe(&pdev->dev, PTR_ERR(zdev->clk_main), + "main clock not found.\n"); zdev->clk_apb = devm_clk_get(&pdev->dev, "clk_apb"); - if (IS_ERR(zdev->clk_apb)) { - dev_err(&pdev->dev, "apb clock not found.\n"); - return PTR_ERR(zdev->clk_apb); - } + if (IS_ERR(zdev->clk_apb)) + return dev_err_probe(&pdev->dev, PTR_ERR(zdev->clk_apb), + "apb clock not found.\n"); platform_set_drvdata(pdev, zdev); pm_runtime_set_autosuspend_delay(zdev->dev, ZDMA_PM_TIMEOUT); pm_runtime_use_autosuspend(zdev->dev); pm_runtime_enable(zdev->dev); - pm_runtime_get_sync(zdev->dev); + ret = pm_runtime_resume_and_get(zdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "device wakeup failed.\n"); + pm_runtime_disable(zdev->dev); + } if (!pm_runtime_enabled(zdev->dev)) { ret = zynqmp_dma_runtime_resume(zdev->dev); if (ret) @@ -1086,19 +1124,23 @@ static int zynqmp_dma_probe(struct platform_device *pdev) ret = zynqmp_dma_chan_probe(zdev, pdev); if (ret) { - dev_err(&pdev->dev, "Probing channel failed\n"); + dev_err_probe(&pdev->dev, ret, "Probing channel failed\n"); goto err_disable_pm; } p->dst_addr_widths = BIT(zdev->chan->bus_width / 8); p->src_addr_widths = BIT(zdev->chan->bus_width / 8); - dma_async_device_register(&zdev->common); + ret = dma_async_device_register(&zdev->common); + if (ret) { + dev_err(zdev->dev, "failed to register the dma device\n"); + goto free_chan_resources; + } ret = of_dma_controller_register(pdev->dev.of_node, of_zynqmp_dma_xlate, zdev); if (ret) { - dev_err(&pdev->dev, "Unable to register DMA to DT\n"); + dev_err_probe(&pdev->dev, ret, "Unable to register DMA to DT\n"); dma_async_device_unregister(&zdev->common); goto free_chan_resources; } @@ -1106,8 +1148,6 @@ static int zynqmp_dma_probe(struct platform_device *pdev) pm_runtime_mark_last_busy(zdev->dev); pm_runtime_put_sync_autosuspend(zdev->dev); - dev_info(&pdev->dev, "ZynqMP DMA driver Probe success\n"); - return 0; free_chan_resources: @@ -1125,7 +1165,7 @@ err_disable_pm: * * Return: Always '0' */ -static int zynqmp_dma_remove(struct platform_device *pdev) +static void zynqmp_dma_remove(struct platform_device *pdev) { struct zynqmp_dma_device *zdev = platform_get_drvdata(pdev); @@ -1133,14 +1173,13 @@ static int zynqmp_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&zdev->common); zynqmp_dma_chan_remove(zdev->chan); - pm_runtime_disable(zdev->dev); - if (!pm_runtime_enabled(zdev->dev)) + if (pm_runtime_active(zdev->dev)) zynqmp_dma_runtime_suspend(zdev->dev); - - return 0; + pm_runtime_disable(zdev->dev); } static const struct of_device_id zynqmp_dma_of_match[] = { + { .compatible = "amd,versal2-dma-1.0", .data = &versal2_dma_config }, { .compatible = "xlnx,zynqmp-dma-1.0", }, {} }; @@ -1154,6 +1193,7 @@ static struct platform_driver zynqmp_dma_driver = { }, .probe = zynqmp_dma_probe, .remove = zynqmp_dma_remove, + .shutdown = zynqmp_dma_remove, }; module_platform_driver(zynqmp_dma_driver); |