diff options
Diffstat (limited to 'drivers/dma/imx-sdma.c')
| -rw-r--r-- | drivers/dma/imx-sdma.c | 1232 |
1 files changed, 877 insertions, 355 deletions
diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index a67ec1bdc4e0..ed9e56de5a9b 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -1,26 +1,20 @@ -/* - * drivers/dma/imx-sdma.c - * - * This file contains a driver for the Freescale Smart DMA engine - * - * Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> - * - * Based on code from Freescale: - * - * Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved. - * - * The code contained herein is licensed under the GNU General Public - * License. You may obtain a copy of the GNU General Public License - * Version 2 or later at the following locations: - * - * http://www.opensource.org/licenses/gpl-license.html - * http://www.gnu.org/copyleft/gpl.html - */ +// SPDX-License-Identifier: GPL-2.0+ +// +// drivers/dma/imx-sdma.c +// +// This file contains a driver for the Freescale Smart DMA engine +// +// Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de> +// +// Based on code from Freescale: +// +// Copyright 2004-2009 Freescale Semiconductor, Inc. All Rights Reserved. #include <linux/init.h> #include <linux/iopoll.h> #include <linux/module.h> #include <linux/types.h> +#include <linux/bitfield.h> #include <linux/bitops.h> #include <linux/mm.h> #include <linux/interrupt.h> @@ -30,6 +24,7 @@ #include <linux/semaphore.h> #include <linux/spinlock.h> #include <linux/device.h> +#include <linux/genalloc.h> #include <linux/dma-mapping.h> #include <linux/firmware.h> #include <linux/slab.h> @@ -37,17 +32,17 @@ #include <linux/dmaengine.h> #include <linux/of.h> #include <linux/of_address.h> -#include <linux/of_device.h> #include <linux/of_dma.h> +#include <linux/workqueue.h> #include <asm/irq.h> -#include <linux/platform_data/dma-imx-sdma.h> -#include <linux/platform_data/dma-imx.h> +#include <linux/dma/imx-dma.h> #include <linux/regmap.h> #include <linux/mfd/syscon.h> #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h> #include "dmaengine.h" +#include "virt-dma.h" /* SDMA registers */ #define SDMA_H_C0PTR 0x000 @@ -79,6 +74,7 @@ #define SDMA_CHNENBL0_IMX35 0x200 #define SDMA_CHNENBL0_IMX31 0x080 #define SDMA_CHNPRI_0 0x100 +#define SDMA_DONE0_CONFIG 0x1000 /* * Buffer descriptor status values. @@ -142,7 +138,11 @@ * 0: Source on AIPS * 12 Destination Bit(DP) 1: Destination on SPBA * 0: Destination on AIPS - * 13-15 --------- MUST BE 0 + * 13 Source FIFO 1: Source is dual FIFO + * 0: Source is single FIFO + * 14 Destination FIFO 1: Destination is dual FIFO + * 0: Destination is single FIFO + * 15 --------- MUST BE 0 * 16-23 Higher WML HWML * 24-27 N Total number of samples after * which Pad adding/Swallowing @@ -173,15 +173,97 @@ #define SDMA_WATERMARK_LEVEL_SPDIF BIT(10) #define SDMA_WATERMARK_LEVEL_SP BIT(11) #define SDMA_WATERMARK_LEVEL_DP BIT(12) +#define SDMA_WATERMARK_LEVEL_SD BIT(13) +#define SDMA_WATERMARK_LEVEL_DD BIT(14) #define SDMA_WATERMARK_LEVEL_HWML (0xFF << 16) #define SDMA_WATERMARK_LEVEL_LWE BIT(28) #define SDMA_WATERMARK_LEVEL_HWE BIT(29) #define SDMA_WATERMARK_LEVEL_CONT BIT(31) +#define SDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +#define SDMA_DMA_DIRECTIONS (BIT(DMA_DEV_TO_MEM) | \ + BIT(DMA_MEM_TO_DEV) | \ + BIT(DMA_DEV_TO_DEV)) + +#define SDMA_WATERMARK_LEVEL_N_FIFOS GENMASK(15, 12) +#define SDMA_WATERMARK_LEVEL_OFF_FIFOS GENMASK(19, 16) +#define SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO GENMASK(31, 28) +#define SDMA_WATERMARK_LEVEL_SW_DONE BIT(23) + +#define SDMA_DONE0_CONFIG_DONE_SEL BIT(7) +#define SDMA_DONE0_CONFIG_DONE_DIS BIT(6) + +/* + * struct sdma_script_start_addrs - SDMA script start pointers + * + * start addresses of the different functions in the physical + * address space of the SDMA engine. + */ +struct sdma_script_start_addrs { + s32 ap_2_ap_addr; + s32 ap_2_bp_addr; + s32 ap_2_ap_fixed_addr; + s32 bp_2_ap_addr; + s32 loopback_on_dsp_side_addr; + s32 mcu_interrupt_only_addr; + s32 firi_2_per_addr; + s32 firi_2_mcu_addr; + s32 per_2_firi_addr; + s32 mcu_2_firi_addr; + s32 uart_2_per_addr; + s32 uart_2_mcu_addr; + s32 per_2_app_addr; + s32 mcu_2_app_addr; + s32 per_2_per_addr; + s32 uartsh_2_per_addr; + s32 uartsh_2_mcu_addr; + s32 per_2_shp_addr; + s32 mcu_2_shp_addr; + s32 ata_2_mcu_addr; + s32 mcu_2_ata_addr; + s32 app_2_per_addr; + s32 app_2_mcu_addr; + s32 shp_2_per_addr; + s32 shp_2_mcu_addr; + s32 mshc_2_mcu_addr; + s32 mcu_2_mshc_addr; + s32 spdif_2_mcu_addr; + s32 mcu_2_spdif_addr; + s32 asrc_2_mcu_addr; + s32 ext_mem_2_ipu_addr; + s32 descrambler_addr; + s32 dptc_dvfs_addr; + s32 utra_addr; + s32 ram_code_start_addr; + /* End of v1 array */ + union { s32 v1_end; s32 mcu_2_ssish_addr; }; + s32 ssish_2_mcu_addr; + s32 hdmi_dma_addr; + /* End of v2 array */ + union { s32 v2_end; s32 zcanfd_2_mcu_addr; }; + s32 zqspi_2_mcu_addr; + s32 mcu_2_ecspi_addr; + s32 mcu_2_sai_addr; + s32 sai_2_mcu_addr; + s32 uart_2_mcu_rom_addr; + s32 uartsh_2_mcu_rom_addr; + s32 i2c_2_mcu_addr; + s32 mcu_2_i2c_addr; + /* End of v3 array */ + union { s32 v3_end; s32 mcu_2_zqspi_addr; }; + /* End of v4 array */ + s32 v4_end[]; +}; + /* * Mode/Count of data node descriptors - IPCv2 */ struct sdma_mode_count { +#define SDMA_BD_MAX_CNT 0xffff u32 count : 16; /* size of the buffer pointed by this BD */ u32 status : 8; /* E,R,I,C,W,D status bits stored here */ u32 command : 8; /* command mostly used for channel 0 */ @@ -199,9 +281,9 @@ struct sdma_buffer_descriptor { /** * struct sdma_channel_control - Channel control Block * - * @current_bd_ptr current buffer descriptor processed - * @base_bd_ptr first element of buffer descriptor array - * @unused padding. The SDMA engine expects an array of 128 byte + * @current_bd_ptr: current buffer descriptor processed + * @base_bd_ptr: first element of buffer descriptor array + * @unused: padding. The SDMA engine expects an array of 128 byte * control blocks */ struct sdma_channel_control { @@ -214,10 +296,13 @@ struct sdma_channel_control { * struct sdma_state_registers - SDMA context for a channel * * @pc: program counter + * @unused1: unused * @t: test bit: status of arithmetic & test instruction * @rpc: return program counter + * @unused0: unused * @sf: source fault while loading data * @spc: loop start program counter + * @unused2: unused * @df: destination fault while storing data * @epc: loop end program counter * @lm: loop mode @@ -255,6 +340,14 @@ struct sdma_state_registers { * @dsa: dedicated core source address register * @ds: dedicated core status register * @dd: dedicated core data register + * @scratch0: 1st word of dedicated ram for context switch + * @scratch1: 2nd word of dedicated ram for context switch + * @scratch2: 3rd word of dedicated ram for context switch + * @scratch3: 4th word of dedicated ram for context switch + * @scratch4: 5th word of dedicated ram for context switch + * @scratch5: 6th word of dedicated ram for context switch + * @scratch6: 7th word of dedicated ram for context switch + * @scratch7: 8th word of dedicated ram for context switch */ struct sdma_context_data { struct sdma_state_registers channel_state; @@ -283,53 +376,103 @@ struct sdma_context_data { u32 scratch7; } __attribute__ ((packed)); -#define NUM_BD (int)(PAGE_SIZE / sizeof(struct sdma_buffer_descriptor)) struct sdma_engine; /** + * struct sdma_desc - descriptor structor for one transfer + * @vd: descriptor for virt dma + * @num_bd: number of descriptors currently handling + * @bd_phys: physical address of bd + * @buf_tail: ID of the buffer that was processed + * @buf_ptail: ID of the previous buffer that was processed + * @period_len: period length, used in cyclic. + * @chn_real_count: the real count updated from bd->mode.count + * @chn_count: the transfer count set + * @sdmac: sdma_channel pointer + * @bd: pointer of allocate bd + */ +struct sdma_desc { + struct virt_dma_desc vd; + unsigned int num_bd; + dma_addr_t bd_phys; + unsigned int buf_tail; + unsigned int buf_ptail; + unsigned int period_len; + unsigned int chn_real_count; + unsigned int chn_count; + struct sdma_channel *sdmac; + struct sdma_buffer_descriptor *bd; +}; + +/** * struct sdma_channel - housekeeping for a SDMA channel * - * @sdma pointer to the SDMA engine for this channel - * @channel the channel number, matches dmaengine chan_id + 1 - * @direction transfer type. Needed for setting SDMA script - * @peripheral_type Peripheral type. Needed for setting SDMA script - * @event_id0 aka dma request line - * @event_id1 for channels that use 2 events - * @word_size peripheral access size - * @buf_tail ID of the buffer that was processed - * @buf_ptail ID of the previous buffer that was processed - * @num_bd max NUM_BD. number of descriptors currently handling + * @vc: virt_dma base structure + * @desc: sdma description including vd and other special member + * @sdma: pointer to the SDMA engine for this channel + * @channel: the channel number, matches dmaengine chan_id + 1 + * @direction: transfer type. Needed for setting SDMA script + * @slave_config: Slave configuration + * @peripheral_type: Peripheral type. Needed for setting SDMA script + * @event_id0: aka dma request line + * @event_id1: for channels that use 2 events + * @word_size: peripheral access size + * @pc_from_device: script address for those device_2_memory + * @pc_to_device: script address for those memory_2_device + * @device_to_device: script address for those device_2_device + * @pc_to_pc: script address for those memory_2_memory + * @flags: loop mode or not + * @per_address: peripheral source or destination address in common case + * destination address in p_2_p case + * @per_address2: peripheral source address in p_2_p case + * @event_mask: event mask used in p_2_p script + * @watermark_level: value for gReg[7], some script will extend it from + * basic watermark such as p_2_p + * @shp_addr: value for gReg[6] + * @per_addr: value for gReg[2] + * @status: status of dma channel + * @data: specific sdma interface structure + * @terminate_worker: used to call back into terminate work function + * @terminated: terminated list + * @is_ram_script: flag for script in ram + * @n_fifos_src: number of source device fifos + * @n_fifos_dst: number of destination device fifos + * @sw_done: software done flag + * @stride_fifos_src: stride for source device FIFOs + * @stride_fifos_dst: stride for destination device FIFOs + * @words_per_fifo: copy number of words one time for one FIFO */ struct sdma_channel { + struct virt_dma_chan vc; + struct sdma_desc *desc; struct sdma_engine *sdma; unsigned int channel; enum dma_transfer_direction direction; + struct dma_slave_config slave_config; enum sdma_peripheral_type peripheral_type; unsigned int event_id0; unsigned int event_id1; enum dma_slave_buswidth word_size; - unsigned int buf_tail; - unsigned int buf_ptail; - unsigned int num_bd; - unsigned int period_len; - struct sdma_buffer_descriptor *bd; - dma_addr_t bd_phys; unsigned int pc_from_device, pc_to_device; unsigned int device_to_device; + unsigned int pc_to_pc; unsigned long flags; dma_addr_t per_address, per_address2; unsigned long event_mask[2]; unsigned long watermark_level; u32 shp_addr, per_addr; - struct dma_chan chan; - spinlock_t lock; - struct dma_async_tx_descriptor desc; enum dma_status status; - unsigned int chn_count; - unsigned int chn_real_count; - struct tasklet_struct tasklet; struct imx_dma_data data; + struct work_struct terminate_worker; + struct list_head terminated; + bool is_ram_script; + unsigned int n_fifos_src; + unsigned int n_fifos_dst; + unsigned int stride_fifos_src; + unsigned int stride_fifos_dst; + unsigned int words_per_fifo; + bool sw_done; }; #define IMX_DMA_SG_LOOP BIT(0) @@ -344,16 +487,14 @@ struct sdma_channel { /** * struct sdma_firmware_header - Layout of the firmware image * - * @magic "SDMA" - * @version_major increased whenever layout of struct sdma_script_start_addrs - * changes. - * @version_minor firmware minor version (for binary compatible changes) - * @script_addrs_start offset of struct sdma_script_start_addrs in this image - * @num_script_addrs Number of script addresses in this image - * @ram_code_start offset of SDMA ram image in this firmware image - * @ram_code_size size of SDMA ram image - * @script_addrs Stores the start address of the SDMA scripts - * (in SDMA memory space) + * @magic: "SDMA" + * @version_major: increased whenever layout of struct + * sdma_script_start_addrs changes. + * @version_minor: firmware minor version (for binary compatible changes) + * @script_addrs_start: offset of struct sdma_script_start_addrs in this image + * @num_script_addrs: Number of script addresses in this image + * @ram_code_start: offset of SDMA ram image in this firmware image + * @ram_code_size: size of SDMA ram image */ struct sdma_firmware_header { u32 magic; @@ -369,11 +510,18 @@ struct sdma_driver_data { int chnenbl0; int num_events; struct sdma_script_start_addrs *script_addrs; + bool check_ratio; + /* + * ecspi ERR009165 fixed should be done in sdma script + * and it has been fixed in soc from i.mx6ul. + * please get more information from the below link: + * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf + */ + bool ecspi_fixed; }; struct sdma_engine { struct device *dev; - struct device_dma_parameters dma_parms; struct sdma_channel channel[MAX_DMA_CHANNELS]; struct sdma_channel_control *channel_control; void __iomem *regs; @@ -389,8 +537,18 @@ struct sdma_engine { u32 spba_start_addr; u32 spba_end_addr; unsigned int irq; + dma_addr_t bd0_phys; + struct sdma_buffer_descriptor *bd0; + /* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/ + bool clk_ratio; + bool fw_loaded; + struct gen_pool *iram_pool; }; +static int sdma_config_write(struct dma_chan *chan, + struct dma_slave_config *dmaengine_cfg, + enum dma_transfer_direction direction); + static struct sdma_driver_data sdma_imx31 = { .chnenbl0 = SDMA_CHNENBL0_IMX31, .num_events = 32, @@ -481,6 +639,13 @@ static struct sdma_driver_data sdma_imx6q = { .script_addrs = &sdma_script_imx6q, }; +static struct sdma_driver_data sdma_imx6ul = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx6q, + .ecspi_fixed = true, +}; + static struct sdma_script_start_addrs sdma_script_imx7d = { .ap_2_ap_addr = 644, .uart_2_mcu_addr = 819, @@ -499,33 +664,12 @@ static struct sdma_driver_data sdma_imx7d = { .script_addrs = &sdma_script_imx7d, }; -static const struct platform_device_id sdma_devtypes[] = { - { - .name = "imx25-sdma", - .driver_data = (unsigned long)&sdma_imx25, - }, { - .name = "imx31-sdma", - .driver_data = (unsigned long)&sdma_imx31, - }, { - .name = "imx35-sdma", - .driver_data = (unsigned long)&sdma_imx35, - }, { - .name = "imx51-sdma", - .driver_data = (unsigned long)&sdma_imx51, - }, { - .name = "imx53-sdma", - .driver_data = (unsigned long)&sdma_imx53, - }, { - .name = "imx6q-sdma", - .driver_data = (unsigned long)&sdma_imx6q, - }, { - .name = "imx7d-sdma", - .driver_data = (unsigned long)&sdma_imx7d, - }, { - /* sentinel */ - } +static struct sdma_driver_data sdma_imx8mq = { + .chnenbl0 = SDMA_CHNENBL0_IMX35, + .num_events = 48, + .script_addrs = &sdma_script_imx7d, + .check_ratio = 1, }; -MODULE_DEVICE_TABLE(platform, sdma_devtypes); static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, }, @@ -535,6 +679,8 @@ static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, { .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, }, + { .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, }, + { .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, sdma_dt_ids); @@ -586,6 +732,11 @@ static int sdma_config_ownership(struct sdma_channel *sdmac, return 0; } +static int is_sdma_channel_enabled(struct sdma_engine *sdma, int channel) +{ + return !!(readl(sdma->regs + SDMA_H_STATSTOP) & BIT(channel)); +} + static void sdma_enable_channel(struct sdma_engine *sdma, int channel) { writel(BIT(channel), sdma->regs + SDMA_H_START); @@ -607,8 +758,11 @@ static int sdma_run_channel0(struct sdma_engine *sdma) dev_err(sdma->dev, "Timeout waiting for CH0 ready\n"); /* Set bits of CONFIG register with dynamic context switching */ - if (readl(sdma->regs + SDMA_H_CONFIG) == 0) - writel_relaxed(SDMA_H_CONFIG_CSM, sdma->regs + SDMA_H_CONFIG); + reg = readl(sdma->regs + SDMA_H_CONFIG); + if ((reg & SDMA_H_CONFIG_CSM) == 0) { + reg |= SDMA_H_CONFIG_CSM; + writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG); + } return ret; } @@ -616,23 +770,20 @@ static int sdma_run_channel0(struct sdma_engine *sdma) static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, u32 address) { - struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd; + struct sdma_buffer_descriptor *bd0 = sdma->bd0; void *buf_virt; dma_addr_t buf_phys; int ret; unsigned long flags; - buf_virt = dma_alloc_coherent(NULL, - size, - &buf_phys, GFP_KERNEL); - if (!buf_virt) { + buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys, GFP_KERNEL); + if (!buf_virt) return -ENOMEM; - } spin_lock_irqsave(&sdma->channel_0_lock, flags); bd0->mode.command = C0_SETPM; - bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD; + bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; bd0->mode.count = size / 2; bd0->buffer_addr = buf_phys; bd0->ext_buffer_addr = address; @@ -643,7 +794,7 @@ static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size, spin_unlock_irqrestore(&sdma->channel_0_lock, flags); - dma_free_coherent(NULL, size, buf_virt, buf_phys); + dma_free_coherent(sdma->dev, size, buf_virt, buf_phys); return ret; } @@ -658,6 +809,14 @@ static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event) val = readl_relaxed(sdma->regs + chnenbl); __set_bit(channel, &val); writel_relaxed(val, sdma->regs + chnenbl); + + /* Set SDMA_DONEx_CONFIG is sw_done enabled */ + if (sdmac->sw_done) { + val = readl_relaxed(sdma->regs + SDMA_DONE0_CONFIG); + val |= SDMA_DONE0_CONFIG_DONE_SEL; + val &= ~SDMA_DONE0_CONFIG_DONE_DIS; + writel_relaxed(val, sdma->regs + SDMA_DONE0_CONFIG); + } } static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) @@ -672,6 +831,31 @@ static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) writel_relaxed(val, sdma->regs + chnenbl); } +static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct sdma_desc, vd.tx); +} + +static void sdma_start_desc(struct sdma_channel *sdmac) +{ + struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc); + struct sdma_desc *desc; + struct sdma_engine *sdma = sdmac->sdma; + int channel = sdmac->channel; + + if (!vd) { + sdmac->desc = NULL; + return; + } + sdmac->desc = desc = to_sdma_desc(&vd->tx); + + list_del(&vd->node); + + sdma->channel_control[channel].base_bd_ptr = desc->bd_phys; + sdma->channel_control[channel].current_bd_ptr = desc->bd_phys; + sdma_enable_channel(sdma, sdmac->channel); +} + static void sdma_update_channel_loop(struct sdma_channel *sdmac) { struct sdma_buffer_descriptor *bd; @@ -682,8 +866,10 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * loop mode. Iterate over descriptors, re-setup them and * call callback function. */ - while (1) { - bd = &sdmac->bd[sdmac->buf_tail]; + while (sdmac->desc) { + struct sdma_desc *desc = sdmac->desc; + + bd = &desc->bd[desc->buf_tail]; if (bd->mode.status & BD_DONE) break; @@ -699,11 +885,10 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * the number of bytes present in the current buffer descriptor. */ - sdmac->chn_real_count = bd->mode.count; - bd->mode.status |= BD_DONE; - bd->mode.count = sdmac->period_len; - sdmac->buf_ptail = sdmac->buf_tail; - sdmac->buf_tail = (sdmac->buf_tail + 1) % sdmac->num_bd; + desc->chn_real_count = bd->mode.count; + bd->mode.count = desc->period_len; + desc->buf_ptail = desc->buf_tail; + desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd; /* * The callback is called from the interrupt context in order @@ -711,41 +896,50 @@ static void sdma_update_channel_loop(struct sdma_channel *sdmac) * SDMA transaction status by the time the client tasklet is * executed. */ + spin_unlock(&sdmac->vc.lock); + dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL); + spin_lock(&sdmac->vc.lock); - dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL); + /* Assign buffer ownership to SDMA */ + bd->mode.status |= BD_DONE; if (error) sdmac->status = old_status; } + + /* + * SDMA stops cyclic channel when DMA request triggers a channel and no SDMA + * owned buffer is available (i.e. BD_DONE was set too late). + */ + if (sdmac->desc && !is_sdma_channel_enabled(sdmac->sdma, sdmac->channel)) { + dev_warn(sdmac->sdma->dev, "restart cyclic channel %d\n", sdmac->channel); + sdma_enable_channel(sdmac->sdma, sdmac->channel); + } } -static void mxc_sdma_handle_channel_normal(unsigned long data) +static void mxc_sdma_handle_channel_normal(struct sdma_channel *data) { struct sdma_channel *sdmac = (struct sdma_channel *) data; struct sdma_buffer_descriptor *bd; int i, error = 0; - sdmac->chn_real_count = 0; + sdmac->desc->chn_real_count = 0; /* * non loop mode. Iterate over all descriptors, collect * errors and call callback function */ - for (i = 0; i < sdmac->num_bd; i++) { - bd = &sdmac->bd[i]; + for (i = 0; i < sdmac->desc->num_bd; i++) { + bd = &sdmac->desc->bd[i]; - if (bd->mode.status & (BD_DONE | BD_RROR)) + if (bd->mode.status & (BD_DONE | BD_RROR)) error = -EIO; - sdmac->chn_real_count += bd->mode.count; + sdmac->desc->chn_real_count += bd->mode.count; } if (error) sdmac->status = DMA_ERROR; else sdmac->status = DMA_COMPLETE; - - dma_cookie_complete(&sdmac->desc); - - dmaengine_desc_get_callback_invoke(&sdmac->desc, NULL); } static irqreturn_t sdma_int_handler(int irq, void *dev_id) @@ -761,12 +955,24 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) while (stat) { int channel = fls(stat) - 1; struct sdma_channel *sdmac = &sdma->channel[channel]; + struct sdma_desc *desc; + + spin_lock(&sdmac->vc.lock); + desc = sdmac->desc; + if (desc) { + if (sdmac->flags & IMX_DMA_SG_LOOP) { + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + sdma_update_channel_loop(sdmac); + else + vchan_cyclic_callback(&desc->vd); + } else { + mxc_sdma_handle_channel_normal(sdmac); + vchan_cookie_complete(&desc->vd); + sdma_start_desc(sdmac); + } + } - if (sdmac->flags & IMX_DMA_SG_LOOP) - sdma_update_channel_loop(sdmac); - else - tasklet_schedule(&sdmac->tasklet); - + spin_unlock(&sdmac->vc.lock); __clear_bit(channel, &stat); } @@ -776,7 +982,7 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) /* * sets the pc of SDMA script according to the peripheral type */ -static void sdma_get_pc(struct sdma_channel *sdmac, +static int sdma_get_pc(struct sdma_channel *sdmac, enum sdma_peripheral_type peripheral_type) { struct sdma_engine *sdma = sdmac->sdma; @@ -785,14 +991,17 @@ static void sdma_get_pc(struct sdma_channel *sdmac, * These are needed once we start to support transfers between * two peripherals or memory-to-memory transfers */ - int per_2_per = 0; + int per_2_per = 0, emi_2_emi = 0; sdmac->pc_from_device = 0; sdmac->pc_to_device = 0; sdmac->device_to_device = 0; + sdmac->pc_to_pc = 0; + sdmac->is_ram_script = false; switch (peripheral_type) { case IMX_DMATYPE_MEMORY: + emi_2_emi = sdma->script_addrs->ap_2_ap_addr; break; case IMX_DMATYPE_DSP: emi_2_per = sdma->script_addrs->bp_2_ap_addr; @@ -815,6 +1024,17 @@ static void sdma_get_pc(struct sdma_channel *sdmac, emi_2_per = sdma->script_addrs->mcu_2_ata_addr; break; case IMX_DMATYPE_CSPI: + per_2_emi = sdma->script_addrs->app_2_mcu_addr; + + /* Use rom script mcu_2_app if ERR009165 fixed */ + if (sdmac->sdma->drvdata->ecspi_fixed) { + emi_2_per = sdma->script_addrs->mcu_2_app_addr; + } else { + emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr; + sdmac->is_ram_script = true; + } + + break; case IMX_DMATYPE_EXT: case IMX_DMATYPE_SSI: case IMX_DMATYPE_SAI: @@ -824,6 +1044,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_SSI_DUAL: per_2_emi = sdma->script_addrs->ssish_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_ssish_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_SSI_SP: case IMX_DMATYPE_MMC: @@ -838,6 +1059,7 @@ static void sdma_get_pc(struct sdma_channel *sdmac, per_2_emi = sdma->script_addrs->asrc_2_mcu_addr; emi_2_per = sdma->script_addrs->asrc_2_mcu_addr; per_2_per = sdma->script_addrs->per_2_per_addr; + sdmac->is_ram_script = true; break; case IMX_DMATYPE_ASRC_SP: per_2_emi = sdma->script_addrs->shp_2_mcu_addr; @@ -858,13 +1080,31 @@ static void sdma_get_pc(struct sdma_channel *sdmac, case IMX_DMATYPE_IPU_MEMORY: emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr; break; - default: + case IMX_DMATYPE_MULTI_SAI: + per_2_emi = sdma->script_addrs->sai_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_sai_addr; + break; + case IMX_DMATYPE_I2C: + per_2_emi = sdma->script_addrs->i2c_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_i2c_addr; + sdmac->is_ram_script = true; break; + case IMX_DMATYPE_HDMI: + emi_2_per = sdma->script_addrs->hdmi_dma_addr; + sdmac->is_ram_script = true; + break; + default: + dev_err(sdma->dev, "Unsupported transfer type %d\n", + peripheral_type); + return -EINVAL; } sdmac->pc_from_device = per_2_emi; sdmac->pc_to_device = emi_2_per; sdmac->device_to_device = per_2_per; + sdmac->pc_to_pc = emi_2_emi; + + return 0; } static int sdma_load_context(struct sdma_channel *sdmac) @@ -873,7 +1113,7 @@ static int sdma_load_context(struct sdma_channel *sdmac) int channel = sdmac->channel; int load_address; struct sdma_context_data *context = sdma->context; - struct sdma_buffer_descriptor *bd0 = sdma->channel[0].bd; + struct sdma_buffer_descriptor *bd0 = sdma->bd0; int ret; unsigned long flags; @@ -881,6 +1121,8 @@ static int sdma_load_context(struct sdma_channel *sdmac) load_address = sdmac->pc_from_device; else if (sdmac->direction == DMA_DEV_TO_DEV) load_address = sdmac->device_to_device; + else if (sdmac->direction == DMA_MEM_TO_MEM) + load_address = sdmac->pc_to_pc; else load_address = sdmac->pc_to_device; @@ -902,14 +1144,19 @@ static int sdma_load_context(struct sdma_channel *sdmac) /* Send by context the event mask,base address for peripheral * and watermark level */ - context->gReg[0] = sdmac->event_mask[1]; - context->gReg[1] = sdmac->event_mask[0]; - context->gReg[2] = sdmac->per_addr; - context->gReg[6] = sdmac->shp_addr; - context->gReg[7] = sdmac->watermark_level; + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + context->gReg[4] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + } else { + context->gReg[0] = sdmac->event_mask[1]; + context->gReg[1] = sdmac->event_mask[0]; + context->gReg[2] = sdmac->per_addr; + context->gReg[6] = sdmac->shp_addr; + context->gReg[7] = sdmac->watermark_level; + } bd0->mode.command = C0_SETDM; - bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD; + bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD; bd0->mode.count = sizeof(*context) / 4; bd0->buffer_addr = sdma->context_phys; bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel; @@ -922,7 +1169,7 @@ static int sdma_load_context(struct sdma_channel *sdmac) static struct sdma_channel *to_sdma_chan(struct dma_chan *chan) { - return container_of(chan, struct sdma_channel, chan); + return container_of(chan, struct sdma_channel, vc.chan); } static int sdma_disable_channel(struct dma_chan *chan) @@ -936,22 +1183,57 @@ static int sdma_disable_channel(struct dma_chan *chan) return 0; } - -static int sdma_disable_channel_with_delay(struct dma_chan *chan) +static void sdma_channel_terminate_work(struct work_struct *work) { - sdma_disable_channel(chan); - + struct sdma_channel *sdmac = container_of(work, struct sdma_channel, + terminate_worker); /* * According to NXP R&D team a delay of one BD SDMA cost time * (maximum is 1ms) should be added after disable of the channel * bit, to ensure SDMA core has really been stopped after SDMA * clients call .device_terminate_all. */ - mdelay(1); + usleep_range(1000, 2000); + + vchan_dma_desc_free_list(&sdmac->vc, &sdmac->terminated); +} + +static int sdma_terminate_all(struct dma_chan *chan) +{ + struct sdma_channel *sdmac = to_sdma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&sdmac->vc.lock, flags); + + sdma_disable_channel(chan); + + if (sdmac->desc) { + vchan_terminate_vdesc(&sdmac->desc->vd); + /* + * move out current descriptor into terminated list so that + * it could be free in sdma_channel_terminate_work alone + * later without potential involving next descriptor raised + * up before the last descriptor terminated. + */ + vchan_get_all_descriptors(&sdmac->vc, &sdmac->terminated); + sdmac->desc = NULL; + schedule_work(&sdmac->terminate_worker); + } + + spin_unlock_irqrestore(&sdmac->vc.lock, flags); return 0; } +static void sdma_channel_synchronize(struct dma_chan *chan) +{ + struct sdma_channel *sdmac = to_sdma_chan(chan); + + vchan_synchronize(&sdmac->vc); + + flush_work(&sdmac->terminate_worker); +} + static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) { struct sdma_engine *sdma = sdmac->sdma; @@ -990,6 +1272,44 @@ static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac) sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP; sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT; + + /* + * Limitation: The p2p script support dual fifos in maximum, + * So when fifo number is larger than 1, force enable dual + * fifos. + */ + if (sdmac->n_fifos_src > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD; + if (sdmac->n_fifos_dst > 1) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD; +} + +static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac) +{ + unsigned int n_fifos; + unsigned int stride_fifos; + unsigned int words_per_fifo; + + if (sdmac->sw_done) + sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE; + + if (sdmac->direction == DMA_DEV_TO_MEM) { + n_fifos = sdmac->n_fifos_src; + stride_fifos = sdmac->stride_fifos_src; + } else { + n_fifos = sdmac->n_fifos_dst; + stride_fifos = sdmac->stride_fifos_dst; + } + + words_per_fifo = sdmac->words_per_fifo; + + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_N_FIFOS, n_fifos); + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_OFF_FIFOS, stride_fifos); + if (words_per_fifo) + sdmac->watermark_level |= + FIELD_PREP(SDMA_WATERMARK_LEVEL_WORDS_PER_FIFO, (words_per_fifo - 1)); } static int sdma_config_channel(struct dma_chan *chan) @@ -1004,18 +1324,6 @@ static int sdma_config_channel(struct dma_chan *chan) sdmac->shp_addr = 0; sdmac->per_addr = 0; - if (sdmac->event_id0) { - if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) - return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id0); - } - - if (sdmac->event_id1) { - if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) - return -EINVAL; - sdma_event_enable(sdmac, sdmac->event_id1); - } - switch (sdmac->peripheral_type) { case IMX_DMATYPE_DSP: sdma_config_ownership(sdmac, false, true, true); @@ -1028,7 +1336,9 @@ static int sdma_config_channel(struct dma_chan *chan) break; } - sdma_get_pc(sdmac, sdmac->peripheral_type); + ret = sdma_get_pc(sdmac, sdmac->peripheral_type); + if (ret) + return ret; if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) && (sdmac->peripheral_type != IMX_DMATYPE_DSP)) { @@ -1037,8 +1347,13 @@ static int sdma_config_channel(struct dma_chan *chan) if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP || sdmac->peripheral_type == IMX_DMATYPE_ASRC) sdma_set_watermarklevel_for_p2p(sdmac); - } else + } else { + if (sdmac->peripheral_type == + IMX_DMATYPE_MULTI_SAI) + sdma_set_watermarklevel_for_sais(sdmac); + __set_bit(sdmac->event_id0, sdmac->event_mask); + } /* Address */ sdmac->shp_addr = sdmac->per_address; @@ -1047,13 +1362,11 @@ static int sdma_config_channel(struct dma_chan *chan) sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */ } - ret = sdma_load_context(sdmac); - - return ret; + return 0; } static int sdma_set_channel_priority(struct sdma_channel *sdmac, - unsigned int priority) + unsigned int priority) { struct sdma_engine *sdma = sdmac->sdma; int channel = sdmac->channel; @@ -1068,52 +1381,99 @@ static int sdma_set_channel_priority(struct sdma_channel *sdmac, return 0; } -static int sdma_request_channel(struct sdma_channel *sdmac) +static int sdma_request_channel0(struct sdma_engine *sdma) { - struct sdma_engine *sdma = sdmac->sdma; - int channel = sdmac->channel; int ret = -EBUSY; - sdmac->bd = dma_zalloc_coherent(NULL, PAGE_SIZE, &sdmac->bd_phys, - GFP_KERNEL); - if (!sdmac->bd) { + if (sdma->iram_pool) + sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys); + else + sdma->bd0 = dma_alloc_coherent(sdma->dev, + sizeof(struct sdma_buffer_descriptor), + &sdma->bd0_phys, GFP_NOWAIT); + if (!sdma->bd0) { ret = -ENOMEM; goto out; } - sdma->channel_control[channel].base_bd_ptr = sdmac->bd_phys; - sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; + sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys; + sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys; - sdma_set_channel_priority(sdmac, MXC_SDMA_DEFAULT_PRIORITY); + sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY); return 0; out: return ret; } -static dma_cookie_t sdma_tx_submit(struct dma_async_tx_descriptor *tx) + +static int sdma_alloc_bd(struct sdma_desc *desc) { - unsigned long flags; - struct sdma_channel *sdmac = to_sdma_chan(tx->chan); - dma_cookie_t cookie; + u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; + int ret = 0; + + if (sdma->iram_pool) + desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size, &desc->bd_phys); + else + desc->bd = dma_alloc_coherent(sdma->dev, bd_size, &desc->bd_phys, GFP_NOWAIT); + + if (!desc->bd) { + ret = -ENOMEM; + goto out; + } +out: + return ret; +} - spin_lock_irqsave(&sdmac->lock, flags); +static void sdma_free_bd(struct sdma_desc *desc) +{ + u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor); + struct sdma_engine *sdma = desc->sdmac->sdma; - cookie = dma_cookie_assign(tx); + if (sdma->iram_pool) + gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd, bd_size); + else + dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd, desc->bd_phys); +} - spin_unlock_irqrestore(&sdmac->lock, flags); +static void sdma_desc_free(struct virt_dma_desc *vd) +{ + struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd); - return cookie; + sdma_free_bd(desc); + kfree(desc); } static int sdma_alloc_chan_resources(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); struct imx_dma_data *data = chan->private; + struct imx_dma_data mem_data; int prio, ret; - if (!data) - return -EINVAL; + /* + * MEMCPY may never setup chan->private by filter function such as + * dmatest, thus create 'struct imx_dma_data mem_data' for this case. + * Please note in any other slave case, you have to setup chan->private + * with 'struct imx_dma_data' in your own filter function if you want to + * request DMA channel by dma_request_channel(), otherwise, 'MEMCPY in + * case?' will appear to warn you to correct your filter function. + */ + if (!data) { + dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n"); + mem_data.priority = 2; + mem_data.peripheral_type = IMX_DMATYPE_MEMORY; + mem_data.dma_request = 0; + mem_data.dma_request2 = 0; + data = &mem_data; + + ret = sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY); + if (ret) + return ret; + } switch (data->priority) { case DMA_PRIO_HIGH: @@ -1139,19 +1499,10 @@ static int sdma_alloc_chan_resources(struct dma_chan *chan) if (ret) goto disable_clk_ipg; - ret = sdma_request_channel(sdmac); - if (ret) - goto disable_clk_ahb; - ret = sdma_set_channel_priority(sdmac, prio); if (ret) goto disable_clk_ahb; - dma_async_tx_descriptor_init(&sdmac->desc, chan); - sdmac->desc.tx_submit = sdma_tx_submit; - /* txd.flags will be overwritten in prep funcs */ - sdmac->desc.flags = DMA_CTRL_ACK; - return 0; disable_clk_ahb: @@ -1166,10 +1517,11 @@ static void sdma_free_chan_resources(struct dma_chan *chan) struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - sdma_disable_channel(chan); + sdma_terminate_all(chan); - if (sdmac->event_id0) - sdma_event_disable(sdmac, sdmac->event_id0); + sdma_channel_synchronize(chan); + + sdma_event_disable(sdmac, sdmac->event_id0); if (sdmac->event_id1) sdma_event_disable(sdmac, sdmac->event_id1); @@ -1178,12 +1530,111 @@ static void sdma_free_chan_resources(struct dma_chan *chan) sdma_set_channel_priority(sdmac, 0); - dma_free_coherent(NULL, PAGE_SIZE, sdmac->bd, sdmac->bd_phys); - clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); } +static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac, + enum dma_transfer_direction direction, u32 bds) +{ + struct sdma_desc *desc; + + if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) { + dev_warn_once(sdmac->sdma->dev, "sdma firmware not ready!\n"); + goto err_out; + } + + desc = kzalloc((sizeof(*desc)), GFP_NOWAIT); + if (!desc) + goto err_out; + + sdmac->status = DMA_IN_PROGRESS; + sdmac->direction = direction; + sdmac->flags = 0; + + desc->chn_count = 0; + desc->chn_real_count = 0; + desc->buf_tail = 0; + desc->buf_ptail = 0; + desc->sdmac = sdmac; + desc->num_bd = bds; + + if (bds && sdma_alloc_bd(desc)) + goto err_desc_out; + + /* No slave_config called in MEMCPY case, so do here */ + if (direction == DMA_MEM_TO_MEM) + sdma_config_ownership(sdmac, false, true, false); + + if (sdma_load_context(sdmac)) + goto err_bd_out; + + return desc; + +err_bd_out: + sdma_free_bd(desc); +err_desc_out: + kfree(desc); +err_out: + return NULL; +} + +static struct dma_async_tx_descriptor *sdma_prep_memcpy( + struct dma_chan *chan, dma_addr_t dma_dst, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + struct sdma_channel *sdmac = to_sdma_chan(chan); + struct sdma_engine *sdma = sdmac->sdma; + int channel = sdmac->channel; + size_t count; + int i = 0, param; + struct sdma_buffer_descriptor *bd; + struct sdma_desc *desc; + + if (!chan || !len) + return NULL; + + dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.\n", + &dma_src, &dma_dst, len, channel); + + desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM, + len / SDMA_BD_MAX_CNT + 1); + if (!desc) + return NULL; + + do { + count = min_t(size_t, len, SDMA_BD_MAX_CNT); + bd = &desc->bd[i]; + bd->buffer_addr = dma_src; + bd->ext_buffer_addr = dma_dst; + bd->mode.count = count; + desc->chn_count += count; + bd->mode.command = 0; + + dma_src += count; + dma_dst += count; + len -= count; + i++; + + param = BD_DONE | BD_EXTD | BD_CONT; + /* last bd */ + if (!len) { + param |= BD_INTR; + param |= BD_LAST; + param &= ~BD_CONT; + } + + dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s\n", + i, count, bd->buffer_addr, + param & BD_WRAP ? "wrap" : "", + param & BD_INTR ? " intr" : ""); + + bd->mode.status = param; + } while (len); + + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); +} + static struct dma_async_tx_descriptor *sdma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, @@ -1191,75 +1642,59 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - int ret, i, count; + int i, count; int channel = sdmac->channel; struct scatterlist *sg; + struct sdma_desc *desc; - if (sdmac->status == DMA_IN_PROGRESS) - return NULL; - sdmac->status = DMA_IN_PROGRESS; + sdma_config_write(chan, &sdmac->slave_config, direction); - sdmac->flags = 0; - - sdmac->buf_tail = 0; - sdmac->buf_ptail = 0; - sdmac->chn_real_count = 0; + desc = sdma_transfer_init(sdmac, direction, sg_len); + if (!desc) + goto err_out; dev_dbg(sdma->dev, "setting up %d entries for channel %d.\n", sg_len, channel); - sdmac->direction = direction; - ret = sdma_load_context(sdmac); - if (ret) - goto err_out; - - if (sg_len > NUM_BD) { - dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n", - channel, sg_len, NUM_BD); - ret = -EINVAL; - goto err_out; - } - - sdmac->chn_count = 0; for_each_sg(sgl, sg, sg_len, i) { - struct sdma_buffer_descriptor *bd = &sdmac->bd[i]; + struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; bd->buffer_addr = sg->dma_address; count = sg_dma_len(sg); - if (count > 0xffff) { + if (count > SDMA_BD_MAX_CNT) { dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d\n", - channel, count, 0xffff); - ret = -EINVAL; - goto err_out; + channel, count, SDMA_BD_MAX_CNT); + goto err_bd_out; } bd->mode.count = count; - sdmac->chn_count += count; + desc->chn_count += count; - if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) { - ret = -EINVAL; - goto err_out; - } + if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) + goto err_bd_out; switch (sdmac->word_size) { case DMA_SLAVE_BUSWIDTH_4_BYTES: bd->mode.command = 0; if (count & 3 || sg->dma_address & 3) - return NULL; + goto err_bd_out; + break; + case DMA_SLAVE_BUSWIDTH_3_BYTES: + bd->mode.command = 3; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: bd->mode.command = 2; if (count & 1 || sg->dma_address & 1) - return NULL; + goto err_bd_out; break; case DMA_SLAVE_BUSWIDTH_1_BYTE: bd->mode.command = 1; break; default: - return NULL; + goto err_bd_out; } param = BD_DONE | BD_EXTD | BD_CONT; @@ -1278,10 +1713,10 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( bd->mode.status = param; } - sdmac->num_bd = sg_len; - sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; - - return &sdmac->desc; + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); +err_bd_out: + sdma_free_bd(desc); + kfree(desc); err_out: sdmac->status = DMA_ERROR; return NULL; @@ -1294,42 +1729,37 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( { struct sdma_channel *sdmac = to_sdma_chan(chan); struct sdma_engine *sdma = sdmac->sdma; - int num_periods = buf_len / period_len; + int num_periods = 0; int channel = sdmac->channel; - int ret, i = 0, buf = 0; + int i = 0, buf = 0; + struct sdma_desc *desc; dev_dbg(sdma->dev, "%s channel: %d\n", __func__, channel); - if (sdmac->status == DMA_IN_PROGRESS) - return NULL; - - sdmac->status = DMA_IN_PROGRESS; + if (sdmac->peripheral_type != IMX_DMATYPE_HDMI) + num_periods = buf_len / period_len; - sdmac->buf_tail = 0; - sdmac->buf_ptail = 0; - sdmac->chn_real_count = 0; - sdmac->period_len = period_len; + sdma_config_write(chan, &sdmac->slave_config, direction); - sdmac->flags |= IMX_DMA_SG_LOOP; - sdmac->direction = direction; - ret = sdma_load_context(sdmac); - if (ret) + desc = sdma_transfer_init(sdmac, direction, num_periods); + if (!desc) goto err_out; - if (num_periods > NUM_BD) { - dev_err(sdma->dev, "SDMA channel %d: maximum number of sg exceeded: %d > %d\n", - channel, num_periods, NUM_BD); - goto err_out; - } + desc->period_len = period_len; - if (period_len > 0xffff) { + sdmac->flags |= IMX_DMA_SG_LOOP; + + if (period_len > SDMA_BD_MAX_CNT) { dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d\n", - channel, period_len, 0xffff); - goto err_out; + channel, period_len, SDMA_BD_MAX_CNT); + goto err_bd_out; } + if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); + while (buf < buf_len) { - struct sdma_buffer_descriptor *bd = &sdmac->bd[i]; + struct sdma_buffer_descriptor *bd = &desc->bd[i]; int param; bd->buffer_addr = dma_addr; @@ -1337,7 +1767,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( bd->mode.count = period_len; if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES) - goto err_out; + goto err_bd_out; if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES) bd->mode.command = 0; else @@ -1360,26 +1790,27 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( i++; } - sdmac->num_bd = num_periods; - sdma->channel_control[channel].current_bd_ptr = sdmac->bd_phys; - - return &sdmac->desc; + return vchan_tx_prep(&sdmac->vc, &desc->vd, flags); +err_bd_out: + sdma_free_bd(desc); + kfree(desc); err_out: sdmac->status = DMA_ERROR; return NULL; } -static int sdma_config(struct dma_chan *chan, - struct dma_slave_config *dmaengine_cfg) +static int sdma_config_write(struct dma_chan *chan, + struct dma_slave_config *dmaengine_cfg, + enum dma_transfer_direction direction) { struct sdma_channel *sdmac = to_sdma_chan(chan); - if (dmaengine_cfg->direction == DMA_DEV_TO_MEM) { + if (direction == DMA_DEV_TO_MEM) { sdmac->per_address = dmaengine_cfg->src_addr; sdmac->watermark_level = dmaengine_cfg->src_maxburst * dmaengine_cfg->src_addr_width; sdmac->word_size = dmaengine_cfg->src_addr_width; - } else if (dmaengine_cfg->direction == DMA_DEV_TO_DEV) { + } else if (direction == DMA_DEV_TO_DEV) { sdmac->per_address2 = dmaengine_cfg->src_addr; sdmac->per_address = dmaengine_cfg->dst_addr; sdmac->watermark_level = dmaengine_cfg->src_maxburst & @@ -1387,28 +1818,92 @@ static int sdma_config(struct dma_chan *chan, sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) & SDMA_WATERMARK_LEVEL_HWML; sdmac->word_size = dmaengine_cfg->dst_addr_width; + } else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) { + sdmac->per_address = dmaengine_cfg->dst_addr; + sdmac->per_address2 = dmaengine_cfg->src_addr; + sdmac->watermark_level = 0; } else { sdmac->per_address = dmaengine_cfg->dst_addr; sdmac->watermark_level = dmaengine_cfg->dst_maxburst * dmaengine_cfg->dst_addr_width; sdmac->word_size = dmaengine_cfg->dst_addr_width; } - sdmac->direction = dmaengine_cfg->direction; + sdmac->direction = direction; return sdma_config_channel(chan); } +static int sdma_config(struct dma_chan *chan, + struct dma_slave_config *dmaengine_cfg) +{ + struct sdma_channel *sdmac = to_sdma_chan(chan); + struct sdma_engine *sdma = sdmac->sdma; + + memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg)); + + if (dmaengine_cfg->peripheral_config) { + struct sdma_peripheral_config *sdmacfg = dmaengine_cfg->peripheral_config; + if (dmaengine_cfg->peripheral_size != sizeof(struct sdma_peripheral_config)) { + dev_err(sdma->dev, "Invalid peripheral size %zu, expected %zu\n", + dmaengine_cfg->peripheral_size, + sizeof(struct sdma_peripheral_config)); + return -EINVAL; + } + sdmac->n_fifos_src = sdmacfg->n_fifos_src; + sdmac->n_fifos_dst = sdmacfg->n_fifos_dst; + sdmac->stride_fifos_src = sdmacfg->stride_fifos_src; + sdmac->stride_fifos_dst = sdmacfg->stride_fifos_dst; + sdmac->words_per_fifo = sdmacfg->words_per_fifo; + sdmac->sw_done = sdmacfg->sw_done; + } + + /* Set ENBLn earlier to make sure dma request triggered after that */ + if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events) + return -EINVAL; + sdma_event_enable(sdmac, sdmac->event_id0); + + if (sdmac->event_id1) { + if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events) + return -EINVAL; + sdma_event_enable(sdmac, sdmac->event_id1); + } + + return 0; +} + static enum dma_status sdma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct sdma_channel *sdmac = to_sdma_chan(chan); + struct sdma_desc *desc = NULL; u32 residue; + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; - if (sdmac->flags & IMX_DMA_SG_LOOP) - residue = (sdmac->num_bd - sdmac->buf_ptail) * - sdmac->period_len - sdmac->chn_real_count; - else - residue = sdmac->chn_count - sdmac->chn_real_count; + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&sdmac->vc.lock, flags); + + vd = vchan_find_desc(&sdmac->vc, cookie); + if (vd) + desc = to_sdma_desc(&vd->tx); + else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie) + desc = sdmac->desc; + + if (desc) { + if (sdmac->flags & IMX_DMA_SG_LOOP) + residue = (desc->num_bd - desc->buf_ptail) * + desc->period_len - desc->chn_real_count; + else + residue = desc->chn_count - desc->chn_real_count; + } else { + residue = 0; + } + + spin_unlock_irqrestore(&sdmac->vc.lock, flags); dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, residue); @@ -1419,19 +1914,28 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan, static void sdma_issue_pending(struct dma_chan *chan) { struct sdma_channel *sdmac = to_sdma_chan(chan); - struct sdma_engine *sdma = sdmac->sdma; + unsigned long flags; - if (sdmac->status == DMA_IN_PROGRESS) - sdma_enable_channel(sdma, sdmac->channel); + spin_lock_irqsave(&sdmac->vc.lock, flags); + if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc) + sdma_start_desc(sdmac); + spin_unlock_irqrestore(&sdmac->vc.lock, flags); } -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 41 -#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 42 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 \ +(offsetof(struct sdma_script_start_addrs, v1_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 \ +(offsetof(struct sdma_script_start_addrs, v2_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3 \ +(offsetof(struct sdma_script_start_addrs, v3_end) / sizeof(s32)) + +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4 \ +(offsetof(struct sdma_script_start_addrs, v4_end) / sizeof(s32)) static void sdma_add_scripts(struct sdma_engine *sdma, - const struct sdma_script_start_addrs *addr) + const struct sdma_script_start_addrs *addr) { s32 *addr_arr = (u32 *)addr; s32 *saddr_arr = (u32 *)sdma->script_addrs; @@ -1441,9 +1945,30 @@ static void sdma_add_scripts(struct sdma_engine *sdma, if (!sdma->script_number) sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; + if (sdma->script_number > sizeof(struct sdma_script_start_addrs) + / sizeof(s32)) { + dev_err(sdma->dev, + "SDMA script number %d not match with firmware.\n", + sdma->script_number); + return; + } + for (i = 0; i < sdma->script_number; i++) if (addr_arr[i] > 0) saddr_arr[i] = addr_arr[i]; + + /* + * For compatibility with NXP internal legacy kernel before 4.19 which + * is based on uart ram script and mainline kernel based on uart rom + * script, both uart ram/rom scripts are present in newer sdma + * firmware. Use the rom versions if they are present (V3 or newer). + */ + if (sdma->script_number >= SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3) { + if (addr->uart_2_mcu_rom_addr) + sdma->script_addrs->uart_2_mcu_addr = addr->uart_2_mcu_rom_addr; + if (addr->uartsh_2_mcu_rom_addr) + sdma->script_addrs->uartsh_2_mcu_addr = addr->uartsh_2_mcu_rom_addr; + } } static void sdma_load_firmware(const struct firmware *fw, void *context) @@ -1493,16 +2018,18 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) clk_enable(sdma->clk_ahb); /* download the RAM image for SDMA */ sdma_load_script(sdma, ram_code, - header->ram_code_size, - addr->ram_code_start_addr); + header->ram_code_size, + addr->ram_code_start_addr); clk_disable(sdma->clk_ipg); clk_disable(sdma->clk_ahb); sdma_add_scripts(sdma, addr); + sdma->fw_loaded = true; + dev_info(sdma->dev, "loaded firmware %d.%d\n", - header->version_major, - header->version_minor); + header->version_major, + header->version_minor); err_firmware: release_firmware(fw); @@ -1520,7 +2047,7 @@ static int sdma_event_remap(struct sdma_engine *sdma) u32 reg, val, shift, num_map, i; int ret = 0; - if (IS_ERR(np) || IS_ERR(gpr_np)) + if (IS_ERR(np) || !gpr_np) goto out; event_remap = of_find_property(np, propname, NULL); @@ -1568,7 +2095,7 @@ static int sdma_event_remap(struct sdma_engine *sdma) } out: - if (!IS_ERR(gpr_np)) + if (gpr_np) of_node_put(gpr_np); return ret; @@ -1579,9 +2106,8 @@ static int sdma_get_firmware(struct sdma_engine *sdma, { int ret; - ret = request_firmware_nowait(THIS_MODULE, - FW_ACTION_HOTPLUG, fw_name, sdma->dev, - GFP_KERNEL, sdma, sdma_load_firmware); + ret = firmware_request_nowait_nowarn(THIS_MODULE, fw_name, sdma->dev, + GFP_KERNEL, sdma, sdma_load_firmware); return ret; } @@ -1590,6 +2116,7 @@ static int sdma_init(struct sdma_engine *sdma) { int i, ret; dma_addr_t ccb_phys; + int ccbsize; ret = clk_enable(sdma->clk_ipg); if (ret) @@ -1598,13 +2125,21 @@ static int sdma_init(struct sdma_engine *sdma) if (ret) goto disable_clk_ipg; + if (sdma->drvdata->check_ratio && + (clk_get_rate(sdma->clk_ahb) == clk_get_rate(sdma->clk_ipg))) + sdma->clk_ratio = 1; + /* Be sure SDMA has not started yet */ writel_relaxed(0, sdma->regs + SDMA_H_C0PTR); - sdma->channel_control = dma_alloc_coherent(NULL, - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control) + - sizeof(struct sdma_context_data), - &ccb_phys, GFP_KERNEL); + ccbsize = MAX_DMA_CHANNELS * (sizeof(struct sdma_channel_control) + + sizeof(struct sdma_context_data)); + + if (sdma->iram_pool) + sdma->channel_control = gen_pool_dma_alloc(sdma->iram_pool, ccbsize, &ccb_phys); + else + sdma->channel_control = dma_alloc_coherent(sdma->dev, ccbsize, &ccb_phys, + GFP_KERNEL); if (!sdma->channel_control) { ret = -ENOMEM; @@ -1612,13 +2147,9 @@ static int sdma_init(struct sdma_engine *sdma) } sdma->context = (void *)sdma->channel_control + - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); + MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control); sdma->context_phys = ccb_phys + - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control); - - /* Zero-out the CCB structures array just allocated */ - memset(sdma->channel_control, 0, - MAX_DMA_CHANNELS * sizeof (struct sdma_channel_control)); + MAX_DMA_CHANNELS * sizeof(struct sdma_channel_control); /* disable all channels */ for (i = 0; i < sdma->drvdata->num_events; i++) @@ -1628,7 +2159,7 @@ static int sdma_init(struct sdma_engine *sdma) for (i = 0; i < MAX_DMA_CHANNELS; i++) writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4); - ret = sdma_request_channel(&sdma->channel[0]); + ret = sdma_request_channel0(sdma); if (ret) goto err_dma_alloc; @@ -1638,8 +2169,10 @@ static int sdma_init(struct sdma_engine *sdma) writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR); /* Set bits of CONFIG register but with static context switching */ - /* FIXME: Check whether to set ACR bit depending on clock ratios */ - writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); + if (sdma->clk_ratio) + writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG); + else + writel_relaxed(0, sdma->regs + SDMA_H_CONFIG); writel_relaxed(ccb_phys, sdma->regs + SDMA_H_C0PTR); @@ -1695,35 +2228,21 @@ static struct dma_chan *sdma_xlate(struct of_phandle_args *dma_spec, */ data.dma_request2 = 0; - return dma_request_channel(mask, sdma_filter_fn, &data); + return __dma_request_channel(&mask, sdma_filter_fn, &data, + ofdma->of_node); } static int sdma_probe(struct platform_device *pdev) { - const struct of_device_id *of_id = - of_match_device(sdma_dt_ids, &pdev->dev); struct device_node *np = pdev->dev.of_node; struct device_node *spba_bus; const char *fw_name; int ret; int irq; - struct resource *iores; struct resource spba_res; - struct sdma_platform_data *pdata = dev_get_platdata(&pdev->dev); int i; struct sdma_engine *sdma; s32 *saddr_arr; - const struct sdma_driver_data *drvdata = NULL; - - if (of_id) - drvdata = of_id->data; - else if (pdev->id_entry) - drvdata = (void *)pdev->id_entry->driver_data; - - if (!drvdata) { - dev_err(&pdev->dev, "unable to find driver data\n"); - return -EINVAL; - } ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (ret) @@ -1736,14 +2255,13 @@ static int sdma_probe(struct platform_device *pdev) spin_lock_init(&sdma->channel_0_lock); sdma->dev = &pdev->dev; - sdma->drvdata = drvdata; + sdma->drvdata = of_device_get_match_data(sdma->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); - sdma->regs = devm_ioremap_resource(&pdev->dev, iores); + sdma->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sdma->regs)) return PTR_ERR(sdma->regs); @@ -1763,8 +2281,8 @@ static int sdma_probe(struct platform_device *pdev) if (ret) goto err_clk; - ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, "sdma", - sdma); + ret = devm_request_irq(&pdev->dev, irq, sdma_int_handler, 0, + dev_name(&pdev->dev), sdma); if (ret) goto err_irq; @@ -1778,11 +2296,13 @@ static int sdma_probe(struct platform_device *pdev) /* initially no scripts available */ saddr_arr = (s32 *)sdma->script_addrs; - for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++) + for (i = 0; i < sizeof(*sdma->script_addrs) / sizeof(s32); i++) saddr_arr[i] = -EINVAL; dma_cap_set(DMA_SLAVE, sdma->dma_device.cap_mask); dma_cap_set(DMA_CYCLIC, sdma->dma_device.cap_mask); + dma_cap_set(DMA_MEMCPY, sdma->dma_device.cap_mask); + dma_cap_set(DMA_PRIVATE, sdma->dma_device.cap_mask); INIT_LIST_HEAD(&sdma->dma_device.channels); /* Initialize channel parameters */ @@ -1790,22 +2310,25 @@ static int sdma_probe(struct platform_device *pdev) struct sdma_channel *sdmac = &sdma->channel[i]; sdmac->sdma = sdma; - spin_lock_init(&sdmac->lock); - sdmac->chan.device = &sdma->dma_device; - dma_cookie_init(&sdmac->chan); sdmac->channel = i; - - tasklet_init(&sdmac->tasklet, mxc_sdma_handle_channel_normal, - (unsigned long) sdmac); + sdmac->vc.desc_free = sdma_desc_free; + INIT_LIST_HEAD(&sdmac->terminated); + INIT_WORK(&sdmac->terminate_worker, + sdma_channel_terminate_work); /* * Add the channel to the DMAC list. Do not add channel 0 though * because we need it internally in the SDMA driver. This also means * that channel 0 in dmaengine counting matches sdma channel 1. */ if (i) - list_add_tail(&sdmac->chan.device_node, - &sdma->dma_device.channels); + vchan_init(&sdmac->vc, &sdma->dma_device); + } + + if (np) { + sdma->iram_pool = of_gen_pool_get(np, "iram", 0); + if (sdma->iram_pool) + dev_info(&pdev->dev, "alloc bd from iram.\n"); } ret = sdma_init(sdma); @@ -1818,29 +2341,6 @@ static int sdma_probe(struct platform_device *pdev) if (sdma->drvdata->script_addrs) sdma_add_scripts(sdma, sdma->drvdata->script_addrs); - if (pdata && pdata->script_addrs) - sdma_add_scripts(sdma, pdata->script_addrs); - - if (pdata) { - ret = sdma_get_firmware(sdma, pdata->fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from platform data\n"); - } else { - /* - * Because that device tree does not encode ROM script address, - * the RAM script in firmware is mandatory for device tree - * probe, otherwise it fails. - */ - ret = of_property_read_string(np, "fsl,sdma-ram-script-name", - &fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware name\n"); - else { - ret = sdma_get_firmware(sdma, fw_name); - if (ret) - dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); - } - } sdma->dma_device.dev = &pdev->dev; @@ -1850,14 +2350,16 @@ static int sdma_probe(struct platform_device *pdev) sdma->dma_device.device_prep_slave_sg = sdma_prep_slave_sg; sdma->dma_device.device_prep_dma_cyclic = sdma_prep_dma_cyclic; sdma->dma_device.device_config = sdma_config; - sdma->dma_device.device_terminate_all = sdma_disable_channel_with_delay; - sdma->dma_device.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); - sdma->dma_device.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); - sdma->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + sdma->dma_device.device_terminate_all = sdma_terminate_all; + sdma->dma_device.device_synchronize = sdma_channel_synchronize; + sdma->dma_device.src_addr_widths = SDMA_DMA_BUSWIDTHS; + sdma->dma_device.dst_addr_widths = SDMA_DMA_BUSWIDTHS; + sdma->dma_device.directions = SDMA_DMA_DIRECTIONS; sdma->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; + sdma->dma_device.device_prep_dma_memcpy = sdma_prep_memcpy; sdma->dma_device.device_issue_pending = sdma_issue_pending; - sdma->dma_device.dev->dma_parms = &sdma->dma_parms; - dma_set_max_seg_size(sdma->dma_device.dev, 65535); + sdma->dma_device.copy_align = 2; + dma_set_max_seg_size(sdma->dma_device.dev, SDMA_BD_MAX_CNT); platform_set_drvdata(pdev, sdma); @@ -1883,6 +2385,21 @@ static int sdma_probe(struct platform_device *pdev) of_node_put(spba_bus); } + /* + * Because that device tree does not encode ROM script address, + * the RAM script in firmware is mandatory for device tree + * probe, otherwise it fails. + */ + ret = of_property_read_string(np, "fsl,sdma-ram-script-name", + &fw_name); + if (ret) { + dev_warn(&pdev->dev, "failed to get firmware name\n"); + } else { + ret = sdma_get_firmware(sdma, fw_name); + if (ret) + dev_warn(&pdev->dev, "failed to get firmware from device tree\n"); + } + return 0; err_register: @@ -1896,7 +2413,7 @@ err_clk: return ret; } -static int sdma_remove(struct platform_device *pdev) +static void sdma_remove(struct platform_device *pdev) { struct sdma_engine *sdma = platform_get_drvdata(pdev); int i; @@ -1910,11 +2427,11 @@ static int sdma_remove(struct platform_device *pdev) for (i = 0; i < MAX_DMA_CHANNELS; i++) { struct sdma_channel *sdmac = &sdma->channel[i]; - tasklet_kill(&sdmac->tasklet); + tasklet_kill(&sdmac->vc.task); + sdma_free_chan_resources(&sdmac->vc.chan); } platform_set_drvdata(pdev, NULL); - return 0; } static struct platform_driver sdma_driver = { @@ -1922,7 +2439,6 @@ static struct platform_driver sdma_driver = { .name = "imx-sdma", .of_match_table = sdma_dt_ids, }, - .id_table = sdma_devtypes, .remove = sdma_remove, .probe = sdma_probe, }; @@ -1931,4 +2447,10 @@ module_platform_driver(sdma_driver); MODULE_AUTHOR("Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>"); MODULE_DESCRIPTION("i.MX SDMA driver"); +#if IS_ENABLED(CONFIG_SOC_IMX6Q) +MODULE_FIRMWARE("imx/sdma/sdma-imx6q.bin"); +#endif +#if IS_ENABLED(CONFIG_SOC_IMX7D) || IS_ENABLED(CONFIG_SOC_IMX8M) +MODULE_FIRMWARE("imx/sdma/sdma-imx7d.bin"); +#endif MODULE_LICENSE("GPL"); |