summaryrefslogtreecommitdiff
path: root/drivers/dma/pl330.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma/pl330.c')
-rw-r--r--drivers/dma/pl330.c543
1 files changed, 383 insertions, 160 deletions
diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c
index b19ee04567b5..82a9fe88ad54 100644
--- a/drivers/dma/pl330.c
+++ b/drivers/dma/pl330.c
@@ -1,16 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
* Copyright (c) 2012 Samsung Electronics Co., Ltd.
* http://www.samsung.com
*
* Copyright (C) 2010 Samsung Electronics Co. Ltd.
* Jaswinder Singh <jassi.brar@samsung.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
*/
+#include <linux/debugfs.h>
#include <linux/kernel.h>
#include <linux/io.h>
#include <linux/init.h>
@@ -27,6 +24,8 @@
#include <linux/of_dma.h>
#include <linux/err.h>
#include <linux/pm_runtime.h>
+#include <linux/bug.h>
+#include <linux/reset.h>
#include "dmaengine.h"
#define PL330_MAX_CHAN 8
@@ -34,7 +33,8 @@
#define PL330_MAX_PERI 32
#define PL330_MAX_BURST 16
-#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)
+#define PL330_QUIRK_BROKEN_NO_FLUSHP BIT(0)
+#define PL330_QUIRK_PERIPH_BURST BIT(1)
enum pl330_cachectrl {
CCTRL0, /* Noncacheable and nonbufferable */
@@ -255,7 +255,7 @@ enum pl330_byteswap {
static unsigned cmd_line;
#define PL330_DBGCMD_DUMP(off, x...) do { \
printk("%x:", cmd_line); \
- printk(x); \
+ printk(KERN_CONT x); \
cmd_line += off; \
} while (0)
#define PL330_DBGMC_START(addr) (cmd_line = addr)
@@ -285,7 +285,7 @@ struct pl330_config {
u32 irq_ns;
};
-/**
+/*
* Request Configuration.
* The PL330 core does not modify this and uses the last
* working configuration if the request doesn't provide any.
@@ -404,6 +404,12 @@ enum desc_status {
*/
BUSY,
/*
+ * Pause was called while descriptor was BUSY. Due to hardware
+ * limitations, only termination is possible for descriptors
+ * that have been paused.
+ */
+ PAUSED,
+ /*
* Sitting on the channel work_list but xfer done
* by PL330 core
*/
@@ -447,6 +453,7 @@ struct dma_pl330_chan {
/* DMA-mapped view of the FIFO; may differ if an IOMMU is present */
dma_addr_t fifo_dma;
enum dma_data_direction dir;
+ struct dma_slave_config slave_config;
/* for cyclic capability */
bool cyclic;
@@ -459,9 +466,6 @@ struct pl330_dmac {
/* DMA-Engine Device */
struct dma_device ddma;
- /* Holds info about sg limitations */
- struct device_dma_parameters dma_parms;
-
/* Pool of descriptors available for the DMAC's channels */
struct list_head desc_pool;
/* To protect desc_pool manipulation */
@@ -497,6 +501,9 @@ struct pl330_dmac {
unsigned int num_peripherals;
struct dma_pl330_chan *peripherals; /* keep at end */
int quirks;
+
+ struct reset_control *rstc;
+ struct reset_control *rstc_ocp;
};
static struct pl330_of_quirks {
@@ -506,6 +513,10 @@ static struct pl330_of_quirks {
{
.quirk = "arm,pl330-broken-no-flushp",
.id = PL330_QUIRK_BROKEN_NO_FLUSHP,
+ },
+ {
+ .quirk = "arm,pl330-periph-burst",
+ .id = PL330_QUIRK_PERIPH_BURST,
}
};
@@ -541,6 +552,10 @@ struct _xfer_spec {
struct dma_pl330_desc *desc;
};
+static int pl330_config_write(struct dma_chan *chan,
+ struct dma_slave_config *slave_config,
+ enum dma_transfer_direction direction);
+
static inline bool _queue_full(struct pl330_thread *thrd)
{
return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL;
@@ -878,6 +893,12 @@ static inline void _execute_DBGINSN(struct pl330_thread *thrd,
void __iomem *regs = thrd->dmac->base;
u32 val;
+ /* If timed out due to halted state-machine */
+ if (_until_dmac_idle(thrd)) {
+ dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");
+ return;
+ }
+
val = (insn[0] << 16) | (insn[1] << 24);
if (!as_manager) {
val |= (1 << 0);
@@ -888,12 +909,6 @@ static inline void _execute_DBGINSN(struct pl330_thread *thrd,
val = le32_to_cpu(*((__le32 *)&insn[2]));
writel(val, regs + DBGINST1);
- /* If timed out due to halted state-machine */
- if (_until_dmac_idle(thrd)) {
- dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");
- return;
- }
-
/* Get going */
writel(0, regs + DBGCMD);
}
@@ -960,6 +975,7 @@ static void _stop(struct pl330_thread *thrd)
{
void __iomem *regs = thrd->dmac->base;
u8 insn[6] = {0, 0, 0, 0, 0, 0};
+ u32 inten = readl(regs + INTEN);
if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
@@ -972,10 +988,13 @@ static void _stop(struct pl330_thread *thrd)
_emit_KILL(0, insn);
- /* Stop generating interrupts for SEV */
- writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
-
_execute_DBGINSN(thrd, insn, is_manager(thrd));
+
+ /* clear the event */
+ if (inten & (1 << thrd->ev))
+ writel(1 << thrd->ev, regs + INTCLR);
+ /* Stop generating interrupts for SEV */
+ writel(inten & ~(1 << thrd->ev), regs + INTEN);
}
/* Start doing req 'idx' of thread 'thrd' */
@@ -1037,7 +1056,7 @@ static bool _trigger(struct pl330_thread *thrd)
return true;
}
-static bool _start(struct pl330_thread *thrd)
+static bool pl330_start_thread(struct pl330_thread *thrd)
{
switch (_state(thrd)) {
case PL330_STATE_FAULT_COMPLETING:
@@ -1045,13 +1064,16 @@ static bool _start(struct pl330_thread *thrd)
if (_state(thrd) == PL330_STATE_KILLING)
UNTIL(thrd, PL330_STATE_STOPPED)
+ fallthrough;
case PL330_STATE_FAULTING:
_stop(thrd);
+ fallthrough;
case PL330_STATE_KILLING:
case PL330_STATE_COMPLETING:
UNTIL(thrd, PL330_STATE_STOPPED)
+ fallthrough;
case PL330_STATE_STOPPED:
return _trigger(thrd);
@@ -1094,51 +1116,91 @@ static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
return off;
}
-static inline int _ldst_devtomem(struct pl330_dmac *pl330, unsigned dry_run,
- u8 buf[], const struct _xfer_spec *pxs,
- int cyc)
+static u32 _emit_load(unsigned int dry_run, u8 buf[],
+ enum pl330_cond cond, enum dma_transfer_direction direction,
+ u8 peri)
{
int off = 0;
- enum pl330_cond cond;
- if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
- cond = BURST;
- else
- cond = SINGLE;
+ switch (direction) {
+ case DMA_MEM_TO_MEM:
+ case DMA_MEM_TO_DEV:
+ off += _emit_LD(dry_run, &buf[off], cond);
+ break;
- while (cyc--) {
- off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
- off += _emit_LDP(dry_run, &buf[off], cond, pxs->desc->peri);
- off += _emit_ST(dry_run, &buf[off], ALWAYS);
+ case DMA_DEV_TO_MEM:
+ if (cond == ALWAYS) {
+ off += _emit_LDP(dry_run, &buf[off], SINGLE,
+ peri);
+ off += _emit_LDP(dry_run, &buf[off], BURST,
+ peri);
+ } else {
+ off += _emit_LDP(dry_run, &buf[off], cond,
+ peri);
+ }
+ break;
- if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
- off += _emit_FLUSHP(dry_run, &buf[off],
- pxs->desc->peri);
+ default:
+ /* this code should be unreachable */
+ WARN_ON(1);
+ break;
}
return off;
}
-static inline int _ldst_memtodev(struct pl330_dmac *pl330,
- unsigned dry_run, u8 buf[],
- const struct _xfer_spec *pxs, int cyc)
+static inline u32 _emit_store(unsigned int dry_run, u8 buf[],
+ enum pl330_cond cond, enum dma_transfer_direction direction,
+ u8 peri)
{
int off = 0;
- enum pl330_cond cond;
- if (pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP)
- cond = BURST;
- else
- cond = SINGLE;
+ switch (direction) {
+ case DMA_MEM_TO_MEM:
+ case DMA_DEV_TO_MEM:
+ off += _emit_ST(dry_run, &buf[off], cond);
+ break;
+
+ case DMA_MEM_TO_DEV:
+ if (cond == ALWAYS) {
+ off += _emit_STP(dry_run, &buf[off], SINGLE,
+ peri);
+ off += _emit_STP(dry_run, &buf[off], BURST,
+ peri);
+ } else {
+ off += _emit_STP(dry_run, &buf[off], cond,
+ peri);
+ }
+ break;
+ default:
+ /* this code should be unreachable */
+ WARN_ON(1);
+ break;
+ }
+
+ return off;
+}
+
+static inline int _ldst_peripheral(struct pl330_dmac *pl330,
+ unsigned dry_run, u8 buf[],
+ const struct _xfer_spec *pxs, int cyc,
+ enum pl330_cond cond)
+{
+ int off = 0;
+
+ /*
+ * do FLUSHP at beginning to clear any stale dma requests before the
+ * first WFP.
+ */
+ if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
+ off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
while (cyc--) {
off += _emit_WFP(dry_run, &buf[off], cond, pxs->desc->peri);
- off += _emit_LD(dry_run, &buf[off], ALWAYS);
- off += _emit_STP(dry_run, &buf[off], cond, pxs->desc->peri);
-
- if (!(pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP))
- off += _emit_FLUSHP(dry_run, &buf[off],
- pxs->desc->peri);
+ off += _emit_load(dry_run, &buf[off], cond, pxs->desc->rqtype,
+ pxs->desc->peri);
+ off += _emit_store(dry_run, &buf[off], cond, pxs->desc->rqtype,
+ pxs->desc->peri);
}
return off;
@@ -1148,19 +1210,76 @@ static int _bursts(struct pl330_dmac *pl330, unsigned dry_run, u8 buf[],
const struct _xfer_spec *pxs, int cyc)
{
int off = 0;
+ enum pl330_cond cond = BRST_LEN(pxs->ccr) > 1 ? BURST : SINGLE;
+
+ if (pl330->quirks & PL330_QUIRK_PERIPH_BURST)
+ cond = BURST;
switch (pxs->desc->rqtype) {
case DMA_MEM_TO_DEV:
- off += _ldst_memtodev(pl330, dry_run, &buf[off], pxs, cyc);
- break;
case DMA_DEV_TO_MEM:
- off += _ldst_devtomem(pl330, dry_run, &buf[off], pxs, cyc);
+ off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, cyc,
+ cond);
break;
+
case DMA_MEM_TO_MEM:
off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
break;
+
default:
- off += 0x40000000; /* Scare off the Client */
+ /* this code should be unreachable */
+ WARN_ON(1);
+ break;
+ }
+
+ return off;
+}
+
+/*
+ * only the unaligned burst transfers have the dregs.
+ * so, still transfer dregs with a reduced size burst
+ * for mem-to-mem, mem-to-dev or dev-to-mem.
+ */
+static int _dregs(struct pl330_dmac *pl330, unsigned int dry_run, u8 buf[],
+ const struct _xfer_spec *pxs, int transfer_length)
+{
+ int off = 0;
+ int dregs_ccr;
+
+ if (transfer_length == 0)
+ return off;
+
+ /*
+ * dregs_len = (total bytes - BURST_TO_BYTE(bursts, ccr)) /
+ * BRST_SIZE(ccr)
+ * the dregs len must be smaller than burst len,
+ * so, for higher efficiency, we can modify CCR
+ * to use a reduced size burst len for the dregs.
+ */
+ dregs_ccr = pxs->ccr;
+ dregs_ccr &= ~((0xf << CC_SRCBRSTLEN_SHFT) |
+ (0xf << CC_DSTBRSTLEN_SHFT));
+ dregs_ccr |= (((transfer_length - 1) & 0xf) <<
+ CC_SRCBRSTLEN_SHFT);
+ dregs_ccr |= (((transfer_length - 1) & 0xf) <<
+ CC_DSTBRSTLEN_SHFT);
+
+ switch (pxs->desc->rqtype) {
+ case DMA_MEM_TO_DEV:
+ case DMA_DEV_TO_MEM:
+ off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);
+ off += _ldst_peripheral(pl330, dry_run, &buf[off], pxs, 1,
+ BURST);
+ break;
+
+ case DMA_MEM_TO_MEM:
+ off += _emit_MOV(dry_run, &buf[off], CCR, dregs_ccr);
+ off += _ldst_memtomem(dry_run, &buf[off], pxs, 1);
+ break;
+
+ default:
+ /* this code should be unreachable */
+ WARN_ON(1);
break;
}
@@ -1256,6 +1375,8 @@ static inline int _setup_loops(struct pl330_dmac *pl330,
struct pl330_xfer *x = &pxs->desc->px;
u32 ccr = pxs->ccr;
unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
+ int num_dregs = (x->bytes - BURST_TO_BYTE(bursts, ccr)) /
+ BRST_SIZE(ccr);
int off = 0;
while (bursts) {
@@ -1263,6 +1384,7 @@ static inline int _setup_loops(struct pl330_dmac *pl330,
off += _loop(pl330, dry_run, &buf[off], &c, pxs);
bursts -= c;
}
+ off += _dregs(pl330, dry_run, &buf[off], pxs, num_dregs);
return off;
}
@@ -1294,7 +1416,6 @@ static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
struct _xfer_spec *pxs)
{
struct _pl330_req *req = &thrd->req[index];
- struct pl330_xfer *x;
u8 *buf = req->mc_cpu;
int off = 0;
@@ -1303,11 +1424,6 @@ static int _setup_req(struct pl330_dmac *pl330, unsigned dry_run,
/* DMAMOV CCR, ccr */
off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
- x = &pxs->desc->px;
- /* Error if xfer length is not aligned at burst size */
- if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
- return -EINVAL;
-
off += _setup_xfer(pl330, dry_run, &buf[off], pxs);
/* DMASEV peripheral/event */
@@ -1365,6 +1481,20 @@ static int pl330_submit_req(struct pl330_thread *thrd,
u32 ccr;
int ret = 0;
+ switch (desc->rqtype) {
+ case DMA_MEM_TO_DEV:
+ break;
+
+ case DMA_DEV_TO_MEM:
+ break;
+
+ case DMA_MEM_TO_MEM:
+ break;
+
+ default:
+ return -ENOTSUPP;
+ }
+
if (pl330->state == DYING
|| pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
@@ -1403,8 +1533,6 @@ static int pl330_submit_req(struct pl330_thread *thrd,
/* First dry run to check if req is acceptable */
ret = _setup_req(pl330, 1, thrd, idx, &xs);
- if (ret < 0)
- goto xfer_exit;
if (ret > pl330->mcbufsz / 2) {
dev_info(pl330->ddma.dev, "%s:%d Try increasing mcbufsz (%i/%i)\n",
@@ -1449,9 +1577,9 @@ static void dma_pl330_rqcb(struct dma_pl330_desc *desc, enum pl330_op_err err)
tasklet_schedule(&pch->task);
}
-static void pl330_dotask(unsigned long data)
+static void pl330_dotask(struct tasklet_struct *t)
{
- struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
+ struct pl330_dmac *pl330 = from_tasklet(pl330, t, tasks);
unsigned long flags;
int i;
@@ -1510,7 +1638,7 @@ static void pl330_dotask(unsigned long data)
/* Returns 1 if state was updated, 0 otherwise */
static int pl330_update(struct pl330_dmac *pl330)
{
- struct dma_pl330_desc *descdone, *tmp;
+ struct dma_pl330_desc *descdone;
unsigned long flags;
void __iomem *regs;
u32 val;
@@ -1580,7 +1708,7 @@ static int pl330_update(struct pl330_dmac *pl330)
thrd->req_running = -1;
/* Get going again ASAP */
- _start(thrd);
+ pl330_start_thread(thrd);
/* For now, just make a list of callbacks to be done */
list_add_tail(&descdone->rqd, &pl330->req_done);
@@ -1588,7 +1716,9 @@ static int pl330_update(struct pl330_dmac *pl330)
}
/* Now that we are in no hurry, do the callbacks */
- list_for_each_entry_safe(descdone, tmp, &pl330->req_done, rqd) {
+ while (!list_empty(&pl330->req_done)) {
+ descdone = list_first_entry(&pl330->req_done,
+ struct dma_pl330_desc, rqd);
list_del(&descdone->rqd);
spin_unlock_irqrestore(&pl330->lock, flags);
dma_pl330_rqcb(descdone, PL330_ERR_NONE);
@@ -1674,8 +1804,6 @@ static inline void _free_event(struct pl330_thread *thrd, int ev)
static void pl330_release_channel(struct pl330_thread *thrd)
{
- struct pl330_dmac *pl330;
-
if (!thrd || thrd->free)
return;
@@ -1684,8 +1812,6 @@ static void pl330_release_channel(struct pl330_thread *thrd)
dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);
dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);
- pl330 = thrd->dmac;
-
_free_event(thrd, thrd->ev);
thrd->free = true;
}
@@ -1761,7 +1887,7 @@ static int dmac_alloc_threads(struct pl330_dmac *pl330)
int i;
/* Allocate 1 Manager and 'chans' Channel threads */
- pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
+ pl330->channels = kcalloc(1 + chans, sizeof(*thrd),
GFP_KERNEL);
if (!pl330->channels)
return -ENOMEM;
@@ -1808,9 +1934,10 @@ static int dmac_alloc_resources(struct pl330_dmac *pl330)
if (ret) {
dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
__func__, __LINE__);
- dma_free_coherent(pl330->ddma.dev,
+ dma_free_attrs(pl330->ddma.dev,
chans * pl330->mcbufsz,
- pl330->mcode_cpu, pl330->mcode_bus);
+ pl330->mcode_cpu, pl330->mcode_bus,
+ DMA_ATTR_PRIVILEGED);
return ret;
}
@@ -1856,7 +1983,7 @@ static int pl330_add(struct pl330_dmac *pl330)
return ret;
}
- tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
+ tasklet_setup(&pl330->tasks, pl330_dotask);
pl330->state = INIT;
@@ -1889,9 +2016,9 @@ static void pl330_del(struct pl330_dmac *pl330)
/* Free DMAC resources */
dmac_free_threads(pl330);
- dma_free_coherent(pl330->ddma.dev,
+ dma_free_attrs(pl330->ddma.dev,
pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,
- pl330->mcode_bus);
+ pl330->mcode_bus, DMA_ATTR_PRIVILEGED);
}
/* forward declaration */
@@ -1920,7 +2047,7 @@ static inline void fill_queue(struct dma_pl330_chan *pch)
list_for_each_entry(desc, &pch->work_list, node) {
/* If already submitted */
- if (desc->status == BUSY)
+ if (desc->status == BUSY || desc->status == PAUSED)
continue;
ret = pl330_submit_req(pch->thread, desc);
@@ -1939,9 +2066,9 @@ static inline void fill_queue(struct dma_pl330_chan *pch)
}
}
-static void pl330_tasklet(unsigned long data)
+static void pl330_tasklet(struct tasklet_struct *t)
{
- struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
+ struct dma_pl330_chan *pch = from_tasklet(pch, t, task);
struct dma_pl330_desc *desc, *_dt;
unsigned long flags;
bool power_down = false;
@@ -1968,7 +2095,7 @@ static void pl330_tasklet(unsigned long data)
} else {
/* Make sure the PL330 Channel thread is active */
spin_lock(&pch->thread->dmac->lock);
- _start(pch->thread);
+ pl330_start_thread(pch->thread);
spin_unlock(&pch->thread->dmac->lock);
}
@@ -1986,7 +2113,7 @@ static void pl330_tasklet(unsigned long data)
if (power_down) {
pch->active = true;
spin_lock(&pch->thread->dmac->lock);
- _start(pch->thread);
+ pl330_start_thread(pch->thread);
spin_unlock(&pch->thread->dmac->lock);
power_down = false;
}
@@ -2049,7 +2176,7 @@ static int pl330_alloc_chan_resources(struct dma_chan *chan)
return -ENOMEM;
}
- tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
+ tasklet_setup(&pch->task, pl330_tasklet);
spin_unlock_irqrestore(&pl330->lock, flags);
@@ -2104,49 +2231,70 @@ static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,
return true;
}
-static int pl330_config(struct dma_chan *chan,
- struct dma_slave_config *slave_config)
+static int fixup_burst_len(int max_burst_len, int quirks)
+{
+ if (max_burst_len > PL330_MAX_BURST)
+ return PL330_MAX_BURST;
+ else if (max_burst_len < 1)
+ return 1;
+ else
+ return max_burst_len;
+}
+
+static int pl330_config_write(struct dma_chan *chan,
+ struct dma_slave_config *slave_config,
+ enum dma_transfer_direction direction)
{
struct dma_pl330_chan *pch = to_pchan(chan);
pl330_unprep_slave_fifo(pch);
- if (slave_config->direction == DMA_MEM_TO_DEV) {
+ if (direction == DMA_MEM_TO_DEV) {
if (slave_config->dst_addr)
pch->fifo_addr = slave_config->dst_addr;
if (slave_config->dst_addr_width)
pch->burst_sz = __ffs(slave_config->dst_addr_width);
- if (slave_config->dst_maxburst)
- pch->burst_len = slave_config->dst_maxburst;
- } else if (slave_config->direction == DMA_DEV_TO_MEM) {
+ pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,
+ pch->dmac->quirks);
+ } else if (direction == DMA_DEV_TO_MEM) {
if (slave_config->src_addr)
pch->fifo_addr = slave_config->src_addr;
if (slave_config->src_addr_width)
pch->burst_sz = __ffs(slave_config->src_addr_width);
- if (slave_config->src_maxburst)
- pch->burst_len = slave_config->src_maxburst;
+ pch->burst_len = fixup_burst_len(slave_config->src_maxburst,
+ pch->dmac->quirks);
}
return 0;
}
+static int pl330_config(struct dma_chan *chan,
+ struct dma_slave_config *slave_config)
+{
+ struct dma_pl330_chan *pch = to_pchan(chan);
+
+ memcpy(&pch->slave_config, slave_config, sizeof(*slave_config));
+
+ return 0;
+}
+
static int pl330_terminate_all(struct dma_chan *chan)
{
struct dma_pl330_chan *pch = to_pchan(chan);
struct dma_pl330_desc *desc;
unsigned long flags;
struct pl330_dmac *pl330 = pch->dmac;
- LIST_HEAD(list);
bool power_down = false;
pm_runtime_get_sync(pl330->ddma.dev);
spin_lock_irqsave(&pch->lock, flags);
+
spin_lock(&pl330->lock);
_stop(pch->thread);
- spin_unlock(&pl330->lock);
-
pch->thread->req[0].desc = NULL;
pch->thread->req[1].desc = NULL;
pch->thread->req_running = -1;
+ spin_unlock(&pl330->lock);
+
power_down = pch->active;
pch->active = false;
@@ -2184,6 +2332,7 @@ static int pl330_pause(struct dma_chan *chan)
{
struct dma_pl330_chan *pch = to_pchan(chan);
struct pl330_dmac *pl330 = pch->dmac;
+ struct dma_pl330_desc *desc;
unsigned long flags;
pm_runtime_get_sync(pl330->ddma.dev);
@@ -2193,6 +2342,10 @@ static int pl330_pause(struct dma_chan *chan)
_stop(pch->thread);
spin_unlock(&pl330->lock);
+ list_for_each_entry(desc, &pch->work_list, node) {
+ if (desc->status == BUSY)
+ desc->status = PAUSED;
+ }
spin_unlock_irqrestore(&pch->lock, flags);
pm_runtime_mark_last_busy(pl330->ddma.dev);
pm_runtime_put_autosuspend(pl330->ddma.dev);
@@ -2283,7 +2436,7 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
else if (running && desc == running)
transferred =
pl330_get_current_xferred_count(pch, desc);
- else if (desc->status == BUSY)
+ else if (desc->status == BUSY || desc->status == PAUSED)
/*
* Busy but not running means either just enqueued,
* or finished and not yet marked done
@@ -2300,6 +2453,9 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
case DONE:
ret = DMA_COMPLETE;
break;
+ case PAUSED:
+ ret = DMA_PAUSED;
+ break;
case PREP:
case BUSY:
ret = DMA_IN_PROGRESS;
@@ -2340,7 +2496,7 @@ static void pl330_issue_pending(struct dma_chan *chan)
list_splice_tail_init(&pch->submitted_list, &pch->work_list);
spin_unlock_irqrestore(&pch->lock, flags);
- pl330_tasklet((unsigned long)pch);
+ pl330_tasklet(&pch->task);
}
/*
@@ -2390,7 +2546,8 @@ static inline void _init_desc(struct dma_pl330_desc *desc)
}
/* Returns the number of descriptors added to the DMAC pool */
-static int add_desc(struct pl330_dmac *pl330, gfp_t flg, int count)
+static int add_desc(struct list_head *pool, spinlock_t *lock,
+ gfp_t flg, int count)
{
struct dma_pl330_desc *desc;
unsigned long flags;
@@ -2400,36 +2557,38 @@ static int add_desc(struct pl330_dmac *pl330, gfp_t flg, int count)
if (!desc)
return 0;
- spin_lock_irqsave(&pl330->pool_lock, flags);
+ spin_lock_irqsave(lock, flags);
for (i = 0; i < count; i++) {
_init_desc(&desc[i]);
- list_add_tail(&desc[i].node, &pl330->desc_pool);
+ list_add_tail(&desc[i].node, pool);
}
- spin_unlock_irqrestore(&pl330->pool_lock, flags);
+ spin_unlock_irqrestore(lock, flags);
return count;
}
-static struct dma_pl330_desc *pluck_desc(struct pl330_dmac *pl330)
+static struct dma_pl330_desc *pluck_desc(struct list_head *pool,
+ spinlock_t *lock)
{
struct dma_pl330_desc *desc = NULL;
unsigned long flags;
- spin_lock_irqsave(&pl330->pool_lock, flags);
+ spin_lock_irqsave(lock, flags);
- if (!list_empty(&pl330->desc_pool)) {
- desc = list_entry(pl330->desc_pool.next,
+ if (!list_empty(pool)) {
+ desc = list_entry(pool->next,
struct dma_pl330_desc, node);
list_del_init(&desc->node);
desc->status = PREP;
desc->txd.callback = NULL;
+ desc->txd.callback_result = NULL;
}
- spin_unlock_irqrestore(&pl330->pool_lock, flags);
+ spin_unlock_irqrestore(lock, flags);
return desc;
}
@@ -2441,20 +2600,18 @@ static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
struct dma_pl330_desc *desc;
/* Pluck one desc from the pool of DMAC */
- desc = pluck_desc(pl330);
+ desc = pluck_desc(&pl330->desc_pool, &pl330->pool_lock);
/* If the DMAC pool is empty, alloc new */
if (!desc) {
- if (!add_desc(pl330, GFP_ATOMIC, 1))
- return NULL;
+ static DEFINE_SPINLOCK(lock);
+ LIST_HEAD(pool);
- /* Try again */
- desc = pluck_desc(pl330);
- if (!desc) {
- dev_err(pch->dmac->ddma.dev,
- "%s:%d ALERT!\n", __func__, __LINE__);
+ if (!add_desc(&pool, &lock, GFP_ATOMIC, 1))
return NULL;
- }
+
+ desc = pluck_desc(&pool, &lock);
+ WARN_ON(!desc || !list_empty(&pool));
}
/* Initialize the descriptor */
@@ -2517,14 +2674,8 @@ static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
burst_len >>= desc->rqcfg.brst_size;
/* src/dst_burst_len can't be more than 16 */
- if (burst_len > 16)
- burst_len = 16;
-
- while (burst_len > 1) {
- if (!(len % (burst_len << desc->rqcfg.brst_size)))
- break;
- burst_len--;
- }
+ if (burst_len > PL330_MAX_BURST)
+ burst_len = PL330_MAX_BURST;
return burst_len;
}
@@ -2550,19 +2701,23 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
return NULL;
}
+ pl330_config_write(chan, &pch->slave_config, direction);
+
if (!pl330_prep_slave_fifo(pch, direction))
return NULL;
for (i = 0; i < len / period_len; i++) {
desc = pl330_get_desc(pch);
if (!desc) {
+ unsigned long iflags;
+
dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
__func__, __LINE__);
if (!first)
return NULL;
- spin_lock_irqsave(&pl330->pool_lock, flags);
+ spin_lock_irqsave(&pl330->pool_lock, iflags);
while (!list_empty(&first->node)) {
desc = list_entry(first->node.next,
@@ -2572,7 +2727,7 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
list_move_tail(&first->node, &pl330->desc_pool);
- spin_unlock_irqrestore(&pl330->pool_lock, flags);
+ spin_unlock_irqrestore(&pl330->pool_lock, iflags);
return NULL;
}
@@ -2596,7 +2751,7 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
desc->rqtype = direction;
desc->rqcfg.brst_size = pch->burst_sz;
- desc->rqcfg.brst_len = 1;
+ desc->rqcfg.brst_len = pch->burst_len;
desc->bytes_requested = period_len;
fill_px(&desc->px, dst, src, period_len);
@@ -2612,7 +2767,6 @@ static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
return NULL;
pch->cyclic = true;
- desc->txd.flags = flags;
return &desc->txd;
}
@@ -2654,18 +2808,16 @@ pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
while (burst != (1 << desc->rqcfg.brst_size))
desc->rqcfg.brst_size++;
+ desc->rqcfg.brst_len = get_burst_len(desc, len);
/*
* If burst size is smaller than bus width then make sure we only
* transfer one at a time to avoid a burst stradling an MFIFO entry.
*/
- if (desc->rqcfg.brst_size * 8 < pl330->pcfg.data_bus_width)
+ if (burst * 8 < pl330->pcfg.data_bus_width)
desc->rqcfg.brst_len = 1;
- desc->rqcfg.brst_len = get_burst_len(desc, len);
desc->bytes_requested = len;
- desc->txd.flags = flags;
-
return &desc->txd;
}
@@ -2704,6 +2856,8 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
if (unlikely(!pch || !sgl || !sg_len))
return NULL;
+ pl330_config_write(chan, &pch->slave_config, direction);
+
if (!pl330_prep_slave_fifo(pch, direction))
return NULL;
@@ -2741,13 +2895,12 @@ pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
}
desc->rqcfg.brst_size = pch->burst_sz;
- desc->rqcfg.brst_len = 1;
+ desc->rqcfg.brst_len = pch->burst_len;
desc->rqtype = direction;
desc->bytes_requested = sg_dma_len(sg);
}
/* Return the last desc in the chain */
- desc->txd.flags = flg;
return &desc->txd;
}
@@ -2766,6 +2919,55 @@ static irqreturn_t pl330_irq_handler(int irq, void *data)
BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
+#ifdef CONFIG_DEBUG_FS
+static int pl330_debugfs_show(struct seq_file *s, void *data)
+{
+ struct pl330_dmac *pl330 = s->private;
+ int chans, pchs, ch, pr;
+
+ chans = pl330->pcfg.num_chan;
+ pchs = pl330->num_peripherals;
+
+ seq_puts(s, "PL330 physical channels:\n");
+ seq_puts(s, "THREAD:\t\tCHANNEL:\n");
+ seq_puts(s, "--------\t-----\n");
+ for (ch = 0; ch < chans; ch++) {
+ struct pl330_thread *thrd = &pl330->channels[ch];
+ int found = -1;
+
+ for (pr = 0; pr < pchs; pr++) {
+ struct dma_pl330_chan *pch = &pl330->peripherals[pr];
+
+ if (!pch->thread || thrd->id != pch->thread->id)
+ continue;
+
+ found = pr;
+ }
+
+ seq_printf(s, "%d\t\t", thrd->id);
+ if (found == -1)
+ seq_puts(s, "--\n");
+ else
+ seq_printf(s, "%d\n", found);
+ }
+
+ return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
+
+static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
+{
+ debugfs_create_file(dev_name(pl330->ddma.dev),
+ S_IFREG | 0444, NULL, pl330,
+ &pl330_debugfs_fops);
+}
+#else
+static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
+{
+}
+#endif
+
/*
* Runtime PM callbacks are provided by amba/bus.c driver.
*
@@ -2776,13 +2978,8 @@ static int __maybe_unused pl330_suspend(struct device *dev)
{
struct amba_device *pcdev = to_amba_device(dev);
- pm_runtime_disable(dev);
-
- if (!pm_runtime_status_suspended(dev)) {
- /* amba did not disable the clock */
- amba_pclk_disable(pcdev);
- }
- amba_pclk_unprepare(pcdev);
+ pm_runtime_force_suspend(dev);
+ clk_unprepare(pcdev->pclk);
return 0;
}
@@ -2792,19 +2989,18 @@ static int __maybe_unused pl330_resume(struct device *dev)
struct amba_device *pcdev = to_amba_device(dev);
int ret;
- ret = amba_pclk_prepare(pcdev);
+ ret = clk_prepare(pcdev->pclk);
if (ret)
return ret;
- if (!pm_runtime_status_suspended(dev))
- ret = amba_pclk_enable(pcdev);
-
- pm_runtime_enable(dev);
+ pm_runtime_force_resume(dev);
return ret;
}
-static SIMPLE_DEV_PM_OPS(pl330_pm, pl330_suspend, pl330_resume);
+static const struct dev_pm_ops pl330_pm = {
+ SET_LATE_SYSTEM_SLEEP_PM_OPS(pl330_suspend, pl330_resume)
+};
static int
pl330_probe(struct amba_device *adev, const struct amba_id *id)
@@ -2844,6 +3040,29 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
amba_set_drvdata(adev, pl330);
+ pl330->rstc = devm_reset_control_get_optional(&adev->dev, "dma");
+ if (IS_ERR(pl330->rstc)) {
+ return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc), "Failed to get reset!\n");
+ } else {
+ ret = reset_control_deassert(pl330->rstc);
+ if (ret) {
+ dev_err(&adev->dev, "Couldn't deassert the device from reset!\n");
+ return ret;
+ }
+ }
+
+ pl330->rstc_ocp = devm_reset_control_get_optional(&adev->dev, "dma-ocp");
+ if (IS_ERR(pl330->rstc_ocp)) {
+ return dev_err_probe(&adev->dev, PTR_ERR(pl330->rstc_ocp),
+ "Failed to get OCP reset!\n");
+ } else {
+ ret = reset_control_deassert(pl330->rstc_ocp);
+ if (ret) {
+ dev_err(&adev->dev, "Couldn't deassert the device from OCP reset!\n");
+ return ret;
+ }
+ }
+
for (i = 0; i < AMBA_NR_IRQS; i++) {
irq = adev->irq[i];
if (irq) {
@@ -2868,7 +3087,8 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
spin_lock_init(&pl330->pool_lock);
/* Create a descriptor pool of default size */
- if (!add_desc(pl330, GFP_KERNEL, NR_DEFAULT_DESC))
+ if (!add_desc(&pl330->desc_pool, &pl330->pool_lock,
+ GFP_KERNEL, NR_DEFAULT_DESC))
dev_warn(&adev->dev, "unable to allocate desc\n");
INIT_LIST_HEAD(&pd->channels);
@@ -2878,7 +3098,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
pl330->num_peripherals = num_chan;
- pl330->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
+ pl330->peripherals = kcalloc(num_chan, sizeof(*pch), GFP_KERNEL);
if (!pl330->peripherals) {
ret = -ENOMEM;
goto probe_err2;
@@ -2921,9 +3141,8 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
pd->src_addr_widths = PL330_DMA_BUSWIDTHS;
pd->dst_addr_widths = PL330_DMA_BUSWIDTHS;
pd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
- pd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
- pd->max_burst = ((pl330->quirks & PL330_QUIRK_BROKEN_NO_FLUSHP) ?
- 1 : PL330_MAX_BURST);
+ pd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
+ pd->max_burst = PL330_MAX_BURST;
ret = dma_async_device_register(pd);
if (ret) {
@@ -2940,17 +3159,13 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id)
}
}
- adev->dev.dma_parms = &pl330->dma_parms;
-
/*
* This is the limit for transfers with a buswidth of 1, larger
* buswidths will have larger limits.
*/
- ret = dma_set_max_seg_size(&adev->dev, 1900800);
- if (ret)
- dev_err(&adev->dev, "unable to set the seg size\n");
-
+ dma_set_max_seg_size(&adev->dev, 1900800);
+ init_pl330_debugfs(pl330);
dev_info(&adev->dev,
"Loaded driver for PL330 DMAC-%x\n", adev->periphid);
dev_info(&adev->dev,
@@ -2982,10 +3197,15 @@ probe_err3:
probe_err2:
pl330_del(pl330);
+ if (pl330->rstc_ocp)
+ reset_control_assert(pl330->rstc_ocp);
+
+ if (pl330->rstc)
+ reset_control_assert(pl330->rstc);
return ret;
}
-static int pl330_remove(struct amba_device *adev)
+static void pl330_remove(struct amba_device *adev)
{
struct pl330_dmac *pl330 = amba_get_drvdata(adev);
struct dma_pl330_chan *pch, *_p;
@@ -3020,10 +3240,14 @@ static int pl330_remove(struct amba_device *adev)
pl330_del(pl330);
- return 0;
+ if (pl330->rstc_ocp)
+ reset_control_assert(pl330->rstc_ocp);
+
+ if (pl330->rstc)
+ reset_control_assert(pl330->rstc);
}
-static struct amba_id pl330_ids[] = {
+static const struct amba_id pl330_ids[] = {
{
.id = 0x00041330,
.mask = 0x000fffff,
@@ -3035,7 +3259,6 @@ MODULE_DEVICE_TABLE(amba, pl330_ids);
static struct amba_driver pl330_driver = {
.drv = {
- .owner = THIS_MODULE,
.name = "dma-pl330",
.pm = &pl330_pm,
},
generated by cgit (git 2.34.1) at 2025-12-25 13:03:10 +0000