diff options
Diffstat (limited to 'drivers/dma/xilinx/xilinx_dma.c')
| -rw-r--r-- | drivers/dma/xilinx/xilinx_dma.c | 288 |
1 files changed, 237 insertions, 51 deletions
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 75c0b8e904e5..fabff602065f 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -41,11 +41,12 @@ #include <linux/io.h> #include <linux/iopoll.h> #include <linux/module.h> -#include <linux/of_address.h> +#include <linux/of.h> #include <linux/of_dma.h> -#include <linux/of_platform.h> #include <linux/of_irq.h> +#include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/clk.h> #include <linux/io-64-nonatomic-lo-hi.h> @@ -112,7 +113,9 @@ /* Register Direct Mode Registers */ #define XILINX_DMA_REG_VSIZE 0x0000 +#define XILINX_DMA_VSIZE_MASK GENMASK(12, 0) #define XILINX_DMA_REG_HSIZE 0x0004 +#define XILINX_DMA_HSIZE_MASK GENMASK(15, 0) #define XILINX_DMA_REG_FRMDLY_STRIDE 0x0008 #define XILINX_DMA_FRMDLY_STRIDE_FRMDLY_SHIFT 24 @@ -173,12 +176,15 @@ #define XILINX_DMA_MAX_TRANS_LEN_MAX 23 #define XILINX_DMA_V2_MAX_TRANS_LEN_MAX 26 #define XILINX_DMA_CR_COALESCE_MAX GENMASK(23, 16) +#define XILINX_DMA_CR_DELAY_MAX GENMASK(31, 24) #define XILINX_DMA_CR_CYCLIC_BD_EN_MASK BIT(4) #define XILINX_DMA_CR_COALESCE_SHIFT 16 +#define XILINX_DMA_CR_DELAY_SHIFT 24 #define XILINX_DMA_BD_SOP BIT(27) #define XILINX_DMA_BD_EOP BIT(26) +#define XILINX_DMA_BD_COMP_MASK BIT(31) #define XILINX_DMA_COALESCE_MAX 255 -#define XILINX_DMA_NUM_DESCS 255 +#define XILINX_DMA_NUM_DESCS 512 #define XILINX_DMA_NUM_APP_WORDS 5 /* AXI CDMA Specific Registers/Offsets */ @@ -394,6 +400,7 @@ struct xilinx_dma_tx_descriptor { * @genlock: Support genlock mode * @err: Channel has errors * @idle: Check for channel idle + * @terminating: Check for channel being synchronized by user * @tasklet: Cleanup work after irq * @config: Device configuration info * @flush_on_fsync: Flush on Frame sync @@ -409,6 +416,7 @@ struct xilinx_dma_tx_descriptor { * @stop_transfer: Differentiate b/w DMA IP's quiesce * @tdest: TDEST value for mcdma * @has_vflip: S2MM vertical flip + * @irq_delay: Interrupt delay timeout */ struct xilinx_dma_chan { struct xilinx_dma_device *xdev; @@ -431,6 +439,7 @@ struct xilinx_dma_chan { bool genlock; bool err; bool idle; + bool terminating; struct tasklet_struct tasklet; struct xilinx_vdma_config config; bool flush_on_fsync; @@ -446,6 +455,7 @@ struct xilinx_dma_chan { int (*stop_transfer)(struct xilinx_dma_chan *chan); u16 tdest; bool has_vflip; + u8 irq_delay; }; /** @@ -491,6 +501,7 @@ struct xilinx_dma_config { * @s2mm_chan_id: DMA s2mm channel identifier * @mm2s_chan_id: DMA mm2s channel identifier * @max_buffer_len: Max buffer length + * @has_axistream_connected: AXI DMA connected to AXI Stream IP */ struct xilinx_dma_device { void __iomem *regs; @@ -509,6 +520,7 @@ struct xilinx_dma_device { u32 s2mm_chan_id; u32 mm2s_chan_id; u32 max_buffer_len; + bool has_axistream_connected; }; /* Macros */ @@ -621,6 +633,29 @@ static inline void xilinx_aximcdma_buf(struct xilinx_dma_chan *chan, } } +/** + * xilinx_dma_get_metadata_ptr- Populate metadata pointer and payload length + * @tx: async transaction descriptor + * @payload_len: metadata payload length + * @max_len: metadata max length + * Return: The app field pointer. + */ +static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx, + size_t *payload_len, size_t *max_len) +{ + struct xilinx_dma_tx_descriptor *desc = to_dma_tx_descriptor(tx); + struct xilinx_axidma_tx_segment *seg; + + *max_len = *payload_len = sizeof(u32) * XILINX_DMA_NUM_APP_WORDS; + seg = list_first_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + return seg->hw.app; +} + +static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { + .get_ptr = xilinx_dma_get_metadata_ptr, +}; + /* ----------------------------------------------------------------------------- * Descriptors and segments alloc and free */ @@ -790,7 +825,7 @@ static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan, } /** - * xilinx_dma_tx_descriptor - Allocate transaction descriptor + * xilinx_dma_alloc_tx_descriptor - Allocate transaction descriptor * @chan: Driver specific DMA channel * * Return: The allocated descriptor on success and NULL on failure. @@ -996,14 +1031,12 @@ static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan, struct xilinx_dma_tx_descriptor *desc, unsigned long *flags) { - dma_async_tx_callback callback; - void *callback_param; + struct dmaengine_desc_callback cb; - callback = desc->async_tx.callback; - callback_param = desc->async_tx.callback_param; - if (callback) { + dmaengine_desc_get_callback(&desc->async_tx, &cb); + if (dmaengine_desc_callback_valid(&cb)) { spin_unlock_irqrestore(&chan->lock, *flags); - callback(callback_param); + dmaengine_desc_callback_invoke(&cb, NULL); spin_lock_irqsave(&chan->lock, *flags); } } @@ -1049,6 +1082,13 @@ static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan) /* Run any dependencies, then free the descriptor */ dma_run_dependencies(&desc->async_tx); xilinx_dma_free_tx_descriptor(chan, desc); + + /* + * While we ran a callback the user called a terminate function, + * which takes care of cleaning up any remaining descriptors + */ + if (chan->terminating) + break; } spin_unlock_irqrestore(&chan->lock, flags); @@ -1365,16 +1405,18 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); - j = chan->desc_submitcount; - reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); - if (chan->direction == DMA_MEM_TO_DEV) { - reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; - } else { - reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; - reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + if (config->park) { + j = chan->desc_submitcount; + reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR); + if (chan->direction == DMA_MEM_TO_DEV) { + reg &= ~XILINX_DMA_PARK_PTR_RD_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_RD_REF_SHIFT; + } else { + reg &= ~XILINX_DMA_PARK_PTR_WR_REF_MASK; + reg |= j << XILINX_DMA_PARK_PTR_WR_REF_SHIFT; + } + dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); } - dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg); /* Start the hardware */ xilinx_dma_start(chan); @@ -1411,8 +1453,7 @@ static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan) chan->desc_submitcount++; chan->desc_pendingcount--; - list_del(&desc->node); - list_add_tail(&desc->node, &chan->active_list); + list_move_tail(&desc->node, &chan->active_list); if (chan->desc_submitcount == chan->num_frms) chan->desc_submitcount = 0; @@ -1529,6 +1570,9 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) if (chan->has_sg) xilinx_write(chan, XILINX_DMA_REG_CURDESC, head_desc->async_tx.phys); + reg &= ~XILINX_DMA_CR_DELAY_MAX; + reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; + dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); xilinx_dma_start(chan); @@ -1650,6 +1694,19 @@ static void xilinx_dma_issue_pending(struct dma_chan *dchan) } /** + * xilinx_dma_device_config - Configure the DMA channel + * @dchan: DMA channel + * @config: channel configuration + * + * Return: 0 always. + */ +static int xilinx_dma_device_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + return 0; +} + +/** * xilinx_dma_complete_descriptor - Mark the active descriptor as complete * @chan : xilinx DMA channel * @@ -1664,6 +1721,14 @@ static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan) return; list_for_each_entry_safe(desc, next, &chan->active_list, node) { + if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + struct xilinx_axidma_tx_segment *seg; + + seg = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, node); + if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg) + break; + } if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA) desc->residue = xilinx_dma_get_residue(chan, desc); @@ -1797,7 +1862,7 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) spin_unlock(&chan->lock); } - tasklet_schedule(&chan->tasklet); + tasklet_hi_schedule(&chan->tasklet); return IRQ_HANDLED; } @@ -1845,15 +1910,8 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) } } - if (status & XILINX_DMA_DMASR_DLY_CNT_IRQ) { - /* - * Device takes too long to do the transfer when user requires - * responsiveness. - */ - dev_dbg(chan->dev, "Inter-packet latency too long\n"); - } - - if (status & XILINX_DMA_DMASR_FRM_CNT_IRQ) { + if (status & (XILINX_DMA_DMASR_FRM_CNT_IRQ | + XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); chan->idle = true; @@ -1965,6 +2023,8 @@ static dma_cookie_t xilinx_dma_tx_submit(struct dma_async_tx_descriptor *tx) if (desc->cyclic) chan->cyclic = true; + chan->terminating = false; + spin_unlock_irqrestore(&chan->lock, flags); return cookie; @@ -1995,6 +2055,10 @@ xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, if (!xt->numf || !xt->sgl[0].size) return NULL; + if (xt->numf & ~XILINX_DMA_VSIZE_MASK || + xt->sgl[0].size & ~XILINX_DMA_HSIZE_MASK) + return NULL; + if (xt->frame_size != 1) return NULL; @@ -2109,6 +2173,99 @@ error: } /** + * xilinx_dma_prep_peripheral_dma_vec - prepare descriptors for a DMA_SLAVE + * transaction from DMA vectors + * @dchan: DMA channel + * @vecs: Array of DMA vectors that should be transferred + * @nb: number of entries in @vecs + * @direction: DMA direction + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor *xilinx_dma_prep_peripheral_dma_vec( + struct dma_chan *dchan, const struct dma_vec *vecs, size_t nb, + enum dma_transfer_direction direction, unsigned long flags) +{ + struct xilinx_dma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_dma_tx_descriptor *desc; + struct xilinx_axidma_tx_segment *segment, *head, *prev = NULL; + size_t copy; + size_t sg_used; + unsigned int i; + + if (!is_slave_direction(direction) || direction != chan->direction) + return NULL; + + desc = xilinx_dma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_dma_tx_submit; + + /* Build transactions using information from DMA vectors */ + for (i = 0; i < nb; i++) { + sg_used = 0; + + /* Loop until the entire dma_vec entry is used */ + while (sg_used < vecs[i].len) { + struct xilinx_axidma_desc_hw *hw; + + /* Get a free segment */ + segment = xilinx_axidma_alloc_tx_segment(chan); + if (!segment) + goto error; + + /* + * Calculate the maximum number of bytes to transfer, + * making sure it is less than the hw limit + */ + copy = xilinx_dma_calc_copysize(chan, vecs[i].len, + sg_used); + hw = &segment->hw; + + /* Fill in the descriptor */ + xilinx_axidma_buf(chan, hw, vecs[i].addr, sg_used, 0); + hw->control = copy; + + if (prev) + prev->hw.next_desc = segment->phys; + + prev = segment; + sg_used += copy; + + /* + * Insert the segment into the descriptor segments + * list. + */ + list_add_tail(&segment->node, &desc->segments); + } + } + + head = list_first_entry(&desc->segments, struct xilinx_axidma_tx_segment, node); + desc->async_tx.phys = head->phys; + + /* For the last DMA_MEM_TO_DEV transfer, set EOP */ + if (chan->direction == DMA_MEM_TO_DEV) { + segment->hw.control |= XILINX_DMA_BD_SOP; + segment = list_last_entry(&desc->segments, + struct xilinx_axidma_tx_segment, + node); + segment->hw.control |= XILINX_DMA_BD_EOP; + } + + if (chan->xdev->has_axistream_connected) + desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; + + return &desc->async_tx; + +error: + xilinx_dma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction * @dchan: DMA channel * @sgl: scatterlist to transfer to/from @@ -2200,6 +2357,9 @@ static struct dma_async_tx_descriptor *xilinx_dma_prep_slave_sg( segment->hw.control |= XILINX_DMA_BD_EOP; } + if (chan->xdev->has_axistream_connected) + desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; + return &desc->async_tx; error: @@ -2436,6 +2596,7 @@ static int xilinx_dma_terminate_all(struct dma_chan *dchan) xilinx_dma_chan_reset(chan); /* Remove and free all of the descriptors in the lists */ + chan->terminating = true; xilinx_dma_free_descriptors(chan); chan->idle = true; @@ -2461,7 +2622,7 @@ static void xilinx_dma_synchronize(struct dma_chan *dchan) } /** - * xilinx_dma_channel_set_config - Configure VDMA channel + * xilinx_vdma_channel_set_config - Configure VDMA channel * Run-time configuration for Axi VDMA, supports: * . halt the channel * . configure interrupt coalescing and inter-packet delay threshold @@ -2774,6 +2935,8 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, /* Retrieve the channel properties from the device tree */ has_dre = of_property_read_bool(node, "xlnx,include-dre"); + of_property_read_u8(node, "xlnx,irq-delay", &chan->irq_delay); + chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); err = of_property_read_u32(node, "xlnx,datawidth", &value); @@ -2839,8 +3002,12 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, return -EINVAL; } + xdev->common.directions |= chan->direction; + /* Request the interrupt */ - chan->irq = irq_of_parse_and_map(node, chan->tdest); + chan->irq = of_irq_get(node, chan->tdest); + if (chan->irq < 0) + return dev_err_probe(xdev->dev, chan->irq, "failed to get irq\n"); err = request_irq(chan->irq, xdev->dma_config->irq_handler, IRQF_SHARED, "xilinx-dma-controller", chan); if (err) { @@ -2869,7 +3036,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, XILINX_DMA_DMASR_SG_MASK) chan->has_sg = true; dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id, - chan->has_sg ? "enabled" : "disabled"); + str_enabled_disabled(chan->has_sg)); } /* Initialize the tasklet */ @@ -2902,7 +3069,7 @@ static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev, * @xdev: Driver specific device structure * @node: Device node * - * Return: 0 always. + * Return: '0' on success and failure value on error. */ static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, struct device_node *node) @@ -2914,8 +3081,11 @@ static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev, if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0) dev_warn(xdev->dev, "missing dma-channels property\n"); - for (i = 0; i < nr_channels; i++) - xilinx_dma_chan_probe(xdev, node); + for (i = 0; i < nr_channels; i++) { + ret = xilinx_dma_chan_probe(xdev, node); + if (ret) + return ret; + } return 0; } @@ -3015,9 +3185,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) /* Request and map I/O memory */ xdev->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(xdev->regs)) - return PTR_ERR(xdev->regs); - + if (IS_ERR(xdev->regs)) { + err = PTR_ERR(xdev->regs); + goto disable_clks; + } /* Retrieve the DMA engine properties from the device tree */ xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; @@ -3039,13 +3210,20 @@ static int xilinx_dma_probe(struct platform_device *pdev) } } + dma_set_max_seg_size(xdev->dev, xdev->max_buffer_len); + + if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { + xdev->has_axistream_connected = + of_property_read_bool(node, "xlnx,axistream-connected"); + } + if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { err = of_property_read_u32(node, "xlnx,num-fstores", &num_frames); if (err < 0) { dev_err(xdev->dev, "missing xlnx,num-fstores property\n"); - return err; + goto disable_clks; } err = of_property_read_u32(node, "xlnx,flush-fsync", @@ -3064,8 +3242,16 @@ static int xilinx_dma_probe(struct platform_device *pdev) else xdev->ext_addr = false; + /* Set metadata mode */ + if (xdev->has_axistream_connected) + xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE; + /* Set the dma mask bits */ - dma_set_mask(xdev->dev, DMA_BIT_MASK(addr_width)); + err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width)); + if (err < 0) { + dev_err(xdev->dev, "DMA mask error %d\n", err); + goto disable_clks; + } /* Initialize the DMA engine */ xdev->common.dev = &pdev->dev; @@ -3084,8 +3270,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) xdev->common.device_synchronize = xilinx_dma_synchronize; xdev->common.device_tx_status = xilinx_dma_tx_status; xdev->common.device_issue_pending = xilinx_dma_issue_pending; + xdev->common.device_config = xilinx_dma_device_config; if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask); + xdev->common.device_prep_peripheral_dma_vec = xilinx_dma_prep_peripheral_dma_vec; xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg; xdev->common.device_prep_dma_cyclic = xilinx_dma_prep_dma_cyclic; @@ -3110,8 +3298,10 @@ static int xilinx_dma_probe(struct platform_device *pdev) /* Initialize the channels */ for_each_child_of_node(node, child) { err = xilinx_dma_child_probe(xdev, child); - if (err < 0) - goto disable_clks; + if (err < 0) { + of_node_put(child); + goto error; + } } if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { @@ -3146,12 +3336,12 @@ static int xilinx_dma_probe(struct platform_device *pdev) return 0; -disable_clks: - xdma_disable_allclks(xdev); error: for (i = 0; i < xdev->dma_config->max_channels; i++) if (xdev->chan[i]) xilinx_dma_chan_remove(xdev->chan[i]); +disable_clks: + xdma_disable_allclks(xdev); return err; } @@ -3159,10 +3349,8 @@ error: /** * xilinx_dma_remove - Driver remove function * @pdev: Pointer to the platform_device structure - * - * Return: Always '0' */ -static int xilinx_dma_remove(struct platform_device *pdev) +static void xilinx_dma_remove(struct platform_device *pdev) { struct xilinx_dma_device *xdev = platform_get_drvdata(pdev); int i; @@ -3176,8 +3364,6 @@ static int xilinx_dma_remove(struct platform_device *pdev) xilinx_dma_chan_remove(xdev->chan[i]); xdma_disable_allclks(xdev); - - return 0; } static struct platform_driver xilinx_vdma_driver = { |
