summaryrefslogtreecommitdiff
path: root/drivers/dma/sa11x0-dma.c
diff options
context:
space:
mode:
authorRussell King <rmk+kernel@arm.linux.org.uk>2012-01-09 21:44:07 +0000
committerRussell King <rmk+kernel@arm.linux.org.uk>2012-03-07 11:32:21 +0000
commit6365bead25efc84a4cf4aa9b0a7638f8a970cdff (patch)
tree3dc5a72a6f25a48dfea6842b9e11af1b209ccec2 /drivers/dma/sa11x0-dma.c
parentd65b4e98d7ea3038b767b70fe8be959b2913f16d (diff)
DMA: sa11x0: add SA-11x0 DMA driver
Add support for the SA-11x0 DMA driver, which replaces the private API version in arch/arm/mach-sa1100/dma.c. We model this as a set of virtual DMA channels, one for each request signal, and assign the virtual DMA channel to a physical DMA channel when there is work to be done. This allows DMA users to claim their channels, and hold them while not in use, without affecting the availability of the physical channels. Another advantage over this approach, compared to the private version, is that a channel can be reconfigured on the fly without having to release and re-request it - which for the IrDA driver, allows us to use DMA for SIR mode transmit without eating up three physical channels. As IrDA is half-duplex, we actually only need one physical channel, and this architecture allows us to achieve that. Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
Diffstat (limited to 'drivers/dma/sa11x0-dma.c')
-rw-r--r--drivers/dma/sa11x0-dma.c1109
1 files changed, 1109 insertions, 0 deletions
diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c
new file mode 100644
index 000000000000..16a6b48883cf
--- /dev/null
+++ b/drivers/dma/sa11x0-dma.c
@@ -0,0 +1,1109 @@
+/*
+ * SA11x0 DMAengine support
+ *
+ * Copyright (C) 2012 Russell King
+ * Derived in part from arch/arm/mach-sa1100/dma.c,
+ * Copyright (C) 2000, 2001 by Nicolas Pitre
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/sched.h>
+#include <linux/device.h>
+#include <linux/dmaengine.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/sa11x0-dma.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#define NR_PHY_CHAN 6
+#define DMA_ALIGN 3
+#define DMA_MAX_SIZE 0x1fff
+#define DMA_CHUNK_SIZE 0x1000
+
+#define DMA_DDAR 0x00
+#define DMA_DCSR_S 0x04
+#define DMA_DCSR_C 0x08
+#define DMA_DCSR_R 0x0c
+#define DMA_DBSA 0x10
+#define DMA_DBTA 0x14
+#define DMA_DBSB 0x18
+#define DMA_DBTB 0x1c
+#define DMA_SIZE 0x20
+
+#define DCSR_RUN (1 << 0)
+#define DCSR_IE (1 << 1)
+#define DCSR_ERROR (1 << 2)
+#define DCSR_DONEA (1 << 3)
+#define DCSR_STRTA (1 << 4)
+#define DCSR_DONEB (1 << 5)
+#define DCSR_STRTB (1 << 6)
+#define DCSR_BIU (1 << 7)
+
+#define DDAR_RW (1 << 0) /* 0 = W, 1 = R */
+#define DDAR_E (1 << 1) /* 0 = LE, 1 = BE */
+#define DDAR_BS (1 << 2) /* 0 = BS4, 1 = BS8 */
+#define DDAR_DW (1 << 3) /* 0 = 8b, 1 = 16b */
+#define DDAR_Ser0UDCTr (0x0 << 4)
+#define DDAR_Ser0UDCRc (0x1 << 4)
+#define DDAR_Ser1SDLCTr (0x2 << 4)
+#define DDAR_Ser1SDLCRc (0x3 << 4)
+#define DDAR_Ser1UARTTr (0x4 << 4)
+#define DDAR_Ser1UARTRc (0x5 << 4)
+#define DDAR_Ser2ICPTr (0x6 << 4)
+#define DDAR_Ser2ICPRc (0x7 << 4)
+#define DDAR_Ser3UARTTr (0x8 << 4)
+#define DDAR_Ser3UARTRc (0x9 << 4)
+#define DDAR_Ser4MCP0Tr (0xa << 4)
+#define DDAR_Ser4MCP0Rc (0xb << 4)
+#define DDAR_Ser4MCP1Tr (0xc << 4)
+#define DDAR_Ser4MCP1Rc (0xd << 4)
+#define DDAR_Ser4SSPTr (0xe << 4)
+#define DDAR_Ser4SSPRc (0xf << 4)
+
+struct sa11x0_dma_sg {
+ u32 addr;
+ u32 len;
+};
+
+struct sa11x0_dma_desc {
+ struct dma_async_tx_descriptor tx;
+ u32 ddar;
+ size_t size;
+
+ /* maybe protected by c->lock */
+ struct list_head node;
+ unsigned sglen;
+ struct sa11x0_dma_sg sg[0];
+};
+
+struct sa11x0_dma_phy;
+
+struct sa11x0_dma_chan {
+ struct dma_chan chan;
+ spinlock_t lock;
+ dma_cookie_t lc;
+
+ /* protected by c->lock */
+ struct sa11x0_dma_phy *phy;
+ enum dma_status status;
+ struct list_head desc_submitted;
+ struct list_head desc_issued;
+
+ /* protected by d->lock */
+ struct list_head node;
+
+ u32 ddar;
+ const char *name;
+};
+
+struct sa11x0_dma_phy {
+ void __iomem *base;
+ struct sa11x0_dma_dev *dev;
+ unsigned num;
+
+ struct sa11x0_dma_chan *vchan;
+
+ /* Protected by c->lock */
+ unsigned sg_load;
+ struct sa11x0_dma_desc *txd_load;
+ unsigned sg_done;
+ struct sa11x0_dma_desc *txd_done;
+#ifdef CONFIG_PM_SLEEP
+ u32 dbs[2];
+ u32 dbt[2];
+ u32 dcsr;
+#endif
+};
+
+struct sa11x0_dma_dev {
+ struct dma_device slave;
+ void __iomem *base;
+ spinlock_t lock;
+ struct tasklet_struct task;
+ struct list_head chan_pending;
+ struct list_head desc_complete;
+ struct sa11x0_dma_phy phy[NR_PHY_CHAN];
+};
+
+static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
+{
+ return container_of(chan, struct sa11x0_dma_chan, chan);
+}
+
+static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
+{
+ return container_of(dmadev, struct sa11x0_dma_dev, slave);
+}
+
+static struct sa11x0_dma_desc *to_sa11x0_dma_tx(struct dma_async_tx_descriptor *tx)
+{
+ return container_of(tx, struct sa11x0_dma_desc, tx);
+}
+
+static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
+{
+ if (list_empty(&c->desc_issued))
+ return NULL;
+
+ return list_first_entry(&c->desc_issued, struct sa11x0_dma_desc, node);
+}
+
+static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
+{
+ list_del(&txd->node);
+ p->txd_load = txd;
+ p->sg_load = 0;
+
+ dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
+ p->num, txd, txd->tx.cookie, txd->ddar);
+}
+
+static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
+ struct sa11x0_dma_chan *c)
+{
+ struct sa11x0_dma_desc *txd = p->txd_load;
+ struct sa11x0_dma_sg *sg;
+ void __iomem *base = p->base;
+ unsigned dbsx, dbtx;
+ u32 dcsr;
+
+ if (!txd)
+ return;
+
+ dcsr = readl_relaxed(base + DMA_DCSR_R);
+
+ /* Don't try to load the next transfer if both buffers are started */
+ if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
+ return;
+
+ if (p->sg_load == txd->sglen) {
+ struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
+
+ /*
+ * We have reached the end of the current descriptor.
+ * Peek at the next descriptor, and if compatible with
+ * the current, start processing it.
+ */
+ if (txn && txn->ddar == txd->ddar) {
+ txd = txn;
+ sa11x0_dma_start_desc(p, txn);
+ } else {
+ p->txd_load = NULL;
+ return;
+ }
+ }
+
+ sg = &txd->sg[p->sg_load++];
+
+ /* Select buffer to load according to channel status */
+ if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
+ ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
+ dbsx = DMA_DBSA;
+ dbtx = DMA_DBTA;
+ dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
+ } else {
+ dbsx = DMA_DBSB;
+ dbtx = DMA_DBTB;
+ dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
+ }
+
+ writel_relaxed(sg->addr, base + dbsx);
+ writel_relaxed(sg->len, base + dbtx);
+ writel(dcsr, base + DMA_DCSR_S);
+
+ dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
+ p->num, dcsr,
+ 'A' + (dbsx == DMA_DBSB), sg->addr,
+ 'A' + (dbtx == DMA_DBTB), sg->len);
+}
+
+static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
+ struct sa11x0_dma_chan *c)
+{
+ struct sa11x0_dma_desc *txd = p->txd_done;
+
+ if (++p->sg_done == txd->sglen) {
+ struct sa11x0_dma_dev *d = p->dev;
+
+ dev_vdbg(d->slave.dev, "pchan %u: txd %p[%x]: completed\n",
+ p->num, p->txd_done, p->txd_done->tx.cookie);
+
+ c->lc = txd->tx.cookie;
+
+ spin_lock(&d->lock);
+ list_add_tail(&txd->node, &d->desc_complete);
+ spin_unlock(&d->lock);
+
+ p->sg_done = 0;
+ p->txd_done = p->txd_load;
+
+ tasklet_schedule(&d->task);
+ }
+
+ sa11x0_dma_start_sg(p, c);
+}
+
+static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
+{
+ struct sa11x0_dma_phy *p = dev_id;
+ struct sa11x0_dma_dev *d = p->dev;
+ struct sa11x0_dma_chan *c;
+ u32 dcsr;
+
+ dcsr = readl_relaxed(p->base + DMA_DCSR_R);
+ if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
+ return IRQ_NONE;
+
+ /* Clear reported status bits */
+ writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
+ p->base + DMA_DCSR_C);
+
+ dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
+
+ if (dcsr & DCSR_ERROR) {
+ dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
+ p->num, dcsr,
+ readl_relaxed(p->base + DMA_DDAR),
+ readl_relaxed(p->base + DMA_DBSA),
+ readl_relaxed(p->base + DMA_DBTA),
+ readl_relaxed(p->base + DMA_DBSB),
+ readl_relaxed(p->base + DMA_DBTB));
+ }
+
+ c = p->vchan;
+ if (c) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->lock, flags);
+ /*
+ * Now that we're holding the lock, check that the vchan
+ * really is associated with this pchan before touching the
+ * hardware. This should always succeed, because we won't
+ * change p->vchan or c->phy while the channel is actively
+ * transferring.
+ */
+ if (c->phy == p) {
+ if (dcsr & DCSR_DONEA)
+ sa11x0_dma_complete(p, c);
+ if (dcsr & DCSR_DONEB)
+ sa11x0_dma_complete(p, c);
+ }
+ spin_unlock_irqrestore(&c->lock, flags);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
+{
+ struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
+
+ /* If the issued list is empty, we have no further txds to process */
+ if (txd) {
+ struct sa11x0_dma_phy *p = c->phy;
+
+ sa11x0_dma_start_desc(p, txd);
+ p->txd_done = txd;
+ p->sg_done = 0;
+
+ /* The channel should not have any transfers started */
+ WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
+ (DCSR_STRTA | DCSR_STRTB));
+
+ /* Clear the run and start bits before changing DDAR */
+ writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
+ p->base + DMA_DCSR_C);
+ writel_relaxed(txd->ddar, p->base + DMA_DDAR);
+
+ /* Try to start both buffers */
+ sa11x0_dma_start_sg(p, c);
+ sa11x0_dma_start_sg(p, c);
+ }
+}
+
+static void sa11x0_dma_tasklet(unsigned long arg)
+{
+ struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
+ struct sa11x0_dma_phy *p;
+ struct sa11x0_dma_chan *c;
+ struct sa11x0_dma_desc *txd, *txn;
+ LIST_HEAD(head);
+ unsigned pch, pch_alloc = 0;
+
+ dev_dbg(d->slave.dev, "tasklet enter\n");
+
+ /* Get the completed tx descriptors */
+ spin_lock_irq(&d->lock);
+ list_splice_init(&d->desc_complete, &head);
+ spin_unlock_irq(&d->lock);
+
+ list_for_each_entry(txd, &head, node) {
+ c = to_sa11x0_dma_chan(txd->tx.chan);
+
+ dev_dbg(d->slave.dev, "vchan %p: txd %p[%x] completed\n",
+ c, txd, txd->tx.cookie);
+
+ spin_lock_irq(&c->lock);
+ p = c->phy;
+ if (p) {
+ if (!p->txd_done)
+ sa11x0_dma_start_txd(c);
+ if (!p->txd_done) {
+ /* No current txd associated with this channel */
+ dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
+
+ /* Mark this channel free */
+ c->phy = NULL;
+ p->vchan = NULL;
+ }
+ }
+ spin_unlock_irq(&c->lock);
+ }
+
+ spin_lock_irq(&d->lock);
+ for (pch = 0; pch < NR_PHY_CHAN; pch++) {
+ p = &d->phy[pch];
+
+ if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
+ c = list_first_entry(&d->chan_pending,
+ struct sa11x0_dma_chan, node);
+ list_del_init(&c->node);
+
+ pch_alloc |= 1 << pch;
+
+ /* Mark this channel allocated */
+ p->vchan = c;
+
+ dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, c);
+ }
+ }
+ spin_unlock_irq(&d->lock);
+
+ for (pch = 0; pch < NR_PHY_CHAN; pch++) {
+ if (pch_alloc & (1 << pch)) {
+ p = &d->phy[pch];
+ c = p->vchan;
+
+ spin_lock_irq(&c->lock);
+ c->phy = p;
+
+ sa11x0_dma_start_txd(c);
+ spin_unlock_irq(&c->lock);
+ }
+ }
+
+ /* Now free the completed tx descriptor, and call their callbacks */
+ list_for_each_entry_safe(txd, txn, &head, node) {
+ dma_async_tx_callback callback = txd->tx.callback;
+ void *callback_param = txd->tx.callback_param;
+
+ dev_dbg(d->slave.dev, "txd %p[%x]: callback and free\n",
+ txd, txd->tx.cookie);
+
+ kfree(txd);
+
+ if (callback)
+ callback(callback_param);
+ }
+
+ dev_dbg(d->slave.dev, "tasklet exit\n");
+}
+
+
+static void sa11x0_dma_desc_free(struct sa11x0_dma_dev *d, struct list_head *head)
+{
+ struct sa11x0_dma_desc *txd, *txn;
+
+ list_for_each_entry_safe(txd, txn, head, node) {
+ dev_dbg(d->slave.dev, "txd %p: freeing\n", txd);
+ kfree(txd);
+ }
+}
+
+static int sa11x0_dma_alloc_chan_resources(struct dma_chan *chan)
+{
+ return 0;
+}
+
+static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
+{
+ struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
+ struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
+ unsigned long flags;
+ LIST_HEAD(head);
+
+ spin_lock_irqsave(&c->lock, flags);
+ spin_lock(&d->lock);
+ list_del_init(&c->node);
+ spin_unlock(&d->lock);
+
+ list_splice_tail_init(&c->desc_submitted, &head);
+ list_splice_tail_init(&c->desc_issued, &head);
+ spin_unlock_irqrestore(&c->lock, flags);
+
+ sa11x0_dma_desc_free(d, &head);
+}
+
+static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
+{
+ unsigned reg;
+ u32 dcsr;
+
+ dcsr = readl_relaxed(p->base + DMA_DCSR_R);
+
+ if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
+ (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
+ reg = DMA_DBSA;
+ else
+ reg = DMA_DBSB;
+
+ return readl_relaxed(p->base + reg);
+}
+
+static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
+ dma_cookie_t cookie, struct dma_tx_state *state)
+{
+ struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
+ struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
+ struct sa11x0_dma_phy *p;
+ struct sa11x0_dma_desc *txd;
+ dma_cookie_t last_used, last_complete;
+ unsigned long flags;
+ enum dma_status ret;
+ size_t bytes = 0;
+
+ last_used = c->chan.cookie;
+ last_complete = c->lc;
+
+ ret = dma_async_is_complete(cookie, last_complete, last_used);
+ if (ret == DMA_SUCCESS) {
+ dma_set_tx_state(state, last_complete, last_used, 0);
+ return ret;
+ }
+
+ spin_lock_irqsave(&c->lock, flags);
+ p = c->phy;
+ ret = c->status;
+ if (p) {
+ dma_addr_t addr = sa11x0_dma_pos(p);
+
+ dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
+
+ txd = p->txd_done;
+ if (txd) {
+ unsigned i;
+
+ for (i = 0; i < txd->sglen; i++) {
+ dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
+ i, txd->sg[i].addr, txd->sg[i].len);
+ if (addr >= txd->sg[i].addr &&
+ addr < txd->sg[i].addr + txd->sg[i].len) {
+ unsigned len;
+
+ len = txd->sg[i].len -
+ (addr - txd->sg[i].addr);
+ dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
+ i, len);
+ bytes += len;
+ i++;
+ break;
+ }
+ }
+ for (; i < txd->sglen; i++) {
+ dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
+ i, txd->sg[i].addr, txd->sg[i].len);
+ bytes += txd->sg[i].len;
+ }
+ }
+ if (txd != p->txd_load && p->txd_load)
+ bytes += p->txd_load->size;
+ }
+ list_for_each_entry(txd, &c->desc_issued, node) {
+ bytes += txd->size;
+ }
+ spin_unlock_irqrestore(&c->lock, flags);
+
+ dma_set_tx_state(state, last_complete, last_used, bytes);
+
+ dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", bytes);
+
+ return ret;
+}
+
+/*
+ * Move pending txds to the issued list, and re-init pending list.
+ * If not already pending, add this channel to the list of pending
+ * channels and trigger the tasklet to run.
+ */
+static void sa11x0_dma_issue_pending(struct dma_chan *chan)
+{
+ struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
+ struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->lock, flags);
+ list_splice_tail_init(&c->desc_submitted, &c->desc_issued);
+ if (!list_empty(&c->desc_issued)) {
+ spin_lock(&d->lock);
+ if (!c->phy && list_empty(&c->node)) {
+ list_add_tail(&c->node, &d->chan_pending);
+ tasklet_schedule(&d->task);
+ dev_dbg(d->slave.dev, "vchan %p: issued\n", c);
+ }
+ spin_unlock(&d->lock);
+ } else
+ dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", c);
+ spin_unlock_irqrestore(&c->lock, flags);
+}
+
+static dma_cookie_t sa11x0_dma_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+ struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(tx->chan);
+ struct sa11x0_dma_desc *txd = to_sa11x0_dma_tx(tx);
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->lock, flags);
+ c->chan.cookie += 1;
+ if (c->chan.cookie < 0)
+ c->chan.cookie = 1;
+ txd->tx.cookie = c->chan.cookie;
+
+ list_add_tail(&txd->node, &c->desc_submitted);
+ spin_unlock_irqrestore(&c->lock, flags);
+
+ dev_dbg(tx->chan->device->dev, "vchan %p: txd %p[%x]: submitted\n",
+ c, txd, txd->tx.cookie);
+
+ return txd->tx.cookie;
+}
+
+static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
+ struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
+ enum dma_transfer_direction dir, unsigned long flags)
+{
+ struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
+ struct sa11x0_dma_desc *txd;
+ struct scatterlist *sgent;
+ unsigned i, j = sglen;
+ size_t size = 0;
+
+ /* SA11x0 channels can only operate in their native direction */
+ if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
+ dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
+ c, c->ddar, dir);
+ return NULL;
+ }
+
+ /* Do not allow zero-sized txds */
+ if (sglen == 0)
+ return NULL;
+
+ for_each_sg(sg, sgent, sglen, i) {
+ dma_addr_t addr = sg_dma_address(sgent);
+ unsigned int len = sg_dma_len(sgent);
+
+ if (len > DMA_MAX_SIZE)
+ j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
+ if (addr & DMA_ALIGN) {
+ dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
+ c, addr);
+ return NULL;
+ }
+ }
+
+ txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC);
+ if (!txd) {
+ dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", c);
+ return NULL;
+ }
+
+ j = 0;
+ for_each_sg(sg, sgent, sglen, i) {
+ dma_addr_t addr = sg_dma_address(sgent);
+ unsigned len = sg_dma_len(sgent);
+
+ size += len;
+
+ do {
+ unsigned tlen = len;
+
+ /*
+ * Check whether the transfer will fit. If not, try
+ * to split the transfer up such that we end up with
+ * equal chunks - but make sure that we preserve the
+ * alignment. This avoids small segments.
+ */
+ if (tlen > DMA_MAX_SIZE) {
+ unsigned mult = DIV_ROUND_UP(tlen,
+ DMA_MAX_SIZE & ~DMA_ALIGN);
+
+ tlen = (tlen / mult) & ~DMA_ALIGN;
+ }
+
+ txd->sg[j].addr = addr;
+ txd->sg[j].len = tlen;
+
+ addr += tlen;
+ len -= tlen;
+ j++;
+ } while (len);
+ }
+
+ dma_async_tx_descriptor_init(&txd->tx, &c->chan);
+ txd->tx.flags = flags;
+ txd->tx.tx_submit = sa11x0_dma_tx_submit;
+ txd->ddar = c->ddar;
+ txd->size = size;
+ txd->sglen = j;
+
+ dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
+ c, txd, txd->size, txd->sglen);
+
+ return &txd->tx;
+}
+
+static int sa11x0_dma_slave_config(struct sa11x0_dma_chan *c, struct dma_slave_config *cfg)
+{
+ u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
+ dma_addr_t addr;
+ enum dma_slave_buswidth width;
+ u32 maxburst;
+
+ if (ddar & DDAR_RW) {
+ addr = cfg->src_addr;
+ width = cfg->src_addr_width;
+ maxburst = cfg->src_maxburst;
+ } else {
+ addr = cfg->dst_addr;
+ width = cfg->dst_addr_width;
+ maxburst = cfg->dst_maxburst;
+ }
+
+ if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
+ width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
+ (maxburst != 4 && maxburst != 8))
+ return -EINVAL;
+
+ if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
+ ddar |= DDAR_DW;
+ if (maxburst == 8)
+ ddar |= DDAR_BS;
+
+ dev_dbg(c->chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
+ c, addr, width, maxburst);
+
+ c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
+
+ return 0;
+}
+
+static int sa11x0_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+ unsigned long arg)
+{
+ struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
+ struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
+ struct sa11x0_dma_phy *p;
+ LIST_HEAD(head);
+ unsigned long flags;
+ int ret;
+
+ switch (cmd) {
+ case DMA_SLAVE_CONFIG:
+ return sa11x0_dma_slave_config(c, (struct dma_slave_config *)arg);
+
+ case DMA_TERMINATE_ALL:
+ dev_dbg(d->slave.dev, "vchan %p: terminate all\n", c);
+ /* Clear the tx descriptor lists */
+ spin_lock_irqsave(&c->lock, flags);
+ list_splice_tail_init(&c->desc_submitted, &head);
+ list_splice_tail_init(&c->desc_issued, &head);
+
+ p = c->phy;
+ if (p) {
+ struct sa11x0_dma_desc *txd, *txn;
+
+ dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
+ /* vchan is assigned to a pchan - stop the channel */
+ writel(DCSR_RUN | DCSR_IE |
+ DCSR_STRTA | DCSR_DONEA |
+ DCSR_STRTB | DCSR_DONEB,
+ p->base + DMA_DCSR_C);
+
+ list_for_each_entry_safe(txd, txn, &d->desc_complete, node)
+ if (txd->tx.chan == &c->chan)
+ list_move(&txd->node, &head);
+
+ if (p->txd_load) {
+ if (p->txd_load != p->txd_done)
+ list_add_tail(&p->txd_load->node, &head);
+ p->txd_load = NULL;
+ }
+ if (p->txd_done) {
+ list_add_tail(&p->txd_done->node, &head);
+ p->txd_done = NULL;
+ }
+ c->phy = NULL;
+ spin_lock(&d->lock);
+ p->vchan = NULL;
+ spin_unlock(&d->lock);
+ tasklet_schedule(&d->task);
+ }
+ spin_unlock_irqrestore(&c->lock, flags);
+ sa11x0_dma_desc_free(d, &head);
+ ret = 0;
+ break;
+
+ case DMA_PAUSE:
+ dev_dbg(d->slave.dev, "vchan %p: pause\n", c);
+ spin_lock_irqsave(&c->lock, flags);
+ if (c->status == DMA_IN_PROGRESS) {
+ c->status = DMA_PAUSED;
+
+ p = c->phy;
+ if (p) {
+ writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
+ } else {
+ spin_lock(&d->lock);
+ list_del_init(&c->node);
+ spin_unlock(&d->lock);
+ }
+ }
+ spin_unlock_irqrestore(&c->lock, flags);
+ ret = 0;
+ break;
+
+ case DMA_RESUME:
+ dev_dbg(d->slave.dev, "vchan %p: resume\n", c);
+ spin_lock_irqsave(&c->lock, flags);
+ if (c->status == DMA_PAUSED) {
+ c->status = DMA_IN_PROGRESS;
+
+ p = c->phy;
+ if (p) {
+ writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
+ } else if (!list_empty(&c->desc_issued)) {
+ spin_lock(&d->lock);
+ list_add_tail(&c->node, &d->chan_pending);
+ spin_unlock(&d->lock);
+ }
+ }
+ spin_unlock_irqrestore(&c->lock, flags);
+ ret = 0;
+ break;
+
+ default:
+ ret = -ENXIO;
+ break;
+ }
+
+ return ret;
+}
+
+struct sa11x0_dma_channel_desc {
+ u32 ddar;
+ const char *name;
+};
+
+#define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
+static const struct sa11x0_dma_channel_desc chan_desc[] = {
+ CD(Ser0UDCTr, 0),
+ CD(Ser0UDCRc, DDAR_RW),
+ CD(Ser1SDLCTr, 0),
+ CD(Ser1SDLCRc, DDAR_RW),
+ CD(Ser1UARTTr, 0),
+ CD(Ser1UARTRc, DDAR_RW),
+ CD(Ser2ICPTr, 0),
+ CD(Ser2ICPRc, DDAR_RW),
+ CD(Ser3UARTTr, 0),
+ CD(Ser3UARTRc, DDAR_RW),
+ CD(Ser4MCP0Tr, 0),
+ CD(Ser4MCP0Rc, DDAR_RW),
+ CD(Ser4MCP1Tr, 0),
+ CD(Ser4MCP1Rc, DDAR_RW),
+ CD(Ser4SSPTr, 0),
+ CD(Ser4SSPRc, DDAR_RW),
+};
+
+static int __devinit sa11x0_dma_init_dmadev(struct dma_device *dmadev,
+ struct device *dev)
+{
+ unsigned i;
+
+ dmadev->chancnt = ARRAY_SIZE(chan_desc);
+ INIT_LIST_HEAD(&dmadev->channels);
+ dmadev->dev = dev;
+ dmadev->device_alloc_chan_resources = sa11x0_dma_alloc_chan_resources;
+ dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
+ dmadev->device_control = sa11x0_dma_control;
+ dmadev->device_tx_status = sa11x0_dma_tx_status;
+ dmadev->device_issue_pending = sa11x0_dma_issue_pending;
+
+ for (i = 0; i < dmadev->chancnt; i++) {
+ struct sa11x0_dma_chan *c;
+
+ c = kzalloc(sizeof(*c), GFP_KERNEL);
+ if (!c) {
+ dev_err(dev, "no memory for channel %u\n", i);
+ return -ENOMEM;
+ }
+
+ c->chan.device = dmadev;
+ c->status = DMA_IN_PROGRESS;
+ c->ddar = chan_desc[i].ddar;
+ c->name = chan_desc[i].name;
+ spin_lock_init(&c->lock);
+ INIT_LIST_HEAD(&c->desc_submitted);
+ INIT_LIST_HEAD(&c->desc_issued);
+ INIT_LIST_HEAD(&c->node);
+ list_add_tail(&c->chan.device_node, &dmadev->channels);
+ }
+
+ return dma_async_device_register(dmadev);
+}
+
+static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
+ void *data)
+{
+ int irq = platform_get_irq(pdev, nr);
+
+ if (irq <= 0)
+ return -ENXIO;
+
+ return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
+}
+
+static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
+ void *data)
+{
+ int irq = platform_get_irq(pdev, nr);
+ if (irq > 0)
+ free_irq(irq, data);
+}
+
+static void sa11x0_dma_free_channels(struct dma_device *dmadev)
+{
+ struct sa11x0_dma_chan *c, *cn;
+
+ list_for_each_entry_safe(c, cn, &dmadev->channels, chan.device_node) {
+ list_del(&c->chan.device_node);
+ kfree(c);
+ }
+}
+
+static int __devinit sa11x0_dma_probe(struct platform_device *pdev)
+{
+ struct sa11x0_dma_dev *d;
+ struct resource *res;
+ unsigned i;
+ int ret;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res)
+ return -ENXIO;
+
+ d = kzalloc(sizeof(*d), GFP_KERNEL);
+ if (!d) {
+ ret = -ENOMEM;
+ goto err_alloc;
+ }
+
+ spin_lock_init(&d->lock);
+ INIT_LIST_HEAD(&d->chan_pending);
+ INIT_LIST_HEAD(&d->desc_complete);
+
+ d->base = ioremap(res->start, resource_size(res));
+ if (!d->base) {
+ ret = -ENOMEM;
+ goto err_ioremap;
+ }
+
+ tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
+
+ for (i = 0; i < NR_PHY_CHAN; i++) {
+ struct sa11x0_dma_phy *p = &d->phy[i];
+
+ p->dev = d;
+ p->num = i;
+ p->base = d->base + i * DMA_SIZE;
+ writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
+ DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
+ p->base + DMA_DCSR_C);
+ writel_relaxed(0, p->base + DMA_DDAR);
+
+ ret = sa11x0_dma_request_irq(pdev, i, p);
+ if (ret) {
+ while (i) {
+ i--;
+ sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
+ }
+ goto err_irq;
+ }
+ }
+
+ dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
+ d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
+ ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
+ if (ret) {
+ dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
+ ret);
+ goto err_slave_reg;
+ }
+
+ platform_set_drvdata(pdev, d);
+ return 0;
+
+ err_slave_reg:
+ sa11x0_dma_free_channels(&d->slave);
+ for (i = 0; i < NR_PHY_CHAN; i++)
+ sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
+ err_irq:
+ tasklet_kill(&d->task);
+ iounmap(d->base);
+ err_ioremap:
+ kfree(d);
+ err_alloc:
+ return ret;
+}
+
+static int __devexit sa11x0_dma_remove(struct platform_device *pdev)
+{
+ struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
+ unsigned pch;
+
+ dma_async_device_unregister(&d->slave);
+
+ sa11x0_dma_free_channels(&d->slave);
+ for (pch = 0; pch < NR_PHY_CHAN; pch++)
+ sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
+ tasklet_kill(&d->task);
+ iounmap(d->base);
+ kfree(d);
+
+ return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int sa11x0_dma_suspend(struct device *dev)
+{
+ struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
+ unsigned pch;
+
+ for (pch = 0; pch < NR_PHY_CHAN; pch++) {
+ struct sa11x0_dma_phy *p = &d->phy[pch];
+ u32 dcsr, saved_dcsr;
+
+ dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
+ if (dcsr & DCSR_RUN) {
+ writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
+ dcsr = readl_relaxed(p->base + DMA_DCSR_R);
+ }
+
+ saved_dcsr &= DCSR_RUN | DCSR_IE;
+ if (dcsr & DCSR_BIU) {
+ p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
+ p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
+ p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
+ p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
+ saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
+ (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
+ } else {
+ p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
+ p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
+ p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
+ p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
+ saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
+ }
+ p->dcsr = saved_dcsr;
+
+ writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
+ }
+
+ return 0;
+}
+
+static int sa11x0_dma_resume(struct device *dev)
+{
+ struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
+ unsigned pch;
+
+ for (pch = 0; pch < NR_PHY_CHAN; pch++) {
+ struct sa11x0_dma_phy *p = &d->phy[pch];
+ struct sa11x0_dma_desc *txd = NULL;
+ u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
+
+ WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
+
+ if (p->txd_done)
+ txd = p->txd_done;
+ else if (p->txd_load)
+ txd = p->txd_load;
+
+ if (!txd)
+ continue;
+
+ writel_relaxed(txd->ddar, p->base + DMA_DDAR);
+
+ writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
+ writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
+ writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
+ writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
+ writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
+ }
+
+ return 0;
+}
+#endif
+
+static const struct dev_pm_ops sa11x0_dma_pm_ops = {
+ .suspend_noirq = sa11x0_dma_suspend,
+ .resume_noirq = sa11x0_dma_resume,
+ .freeze_noirq = sa11x0_dma_suspend,
+ .thaw_noirq = sa11x0_dma_resume,
+ .poweroff_noirq = sa11x0_dma_suspend,
+ .restore_noirq = sa11x0_dma_resume,
+};
+
+static struct platform_driver sa11x0_dma_driver = {
+ .driver = {
+ .name = "sa11x0-dma",
+ .owner = THIS_MODULE,
+ .pm = &sa11x0_dma_pm_ops,
+ },
+ .probe = sa11x0_dma_probe,
+ .remove = __devexit_p(sa11x0_dma_remove),
+};
+
+bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
+{
+ if (chan->device->dev->driver == &sa11x0_dma_driver.driver) {
+ struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
+ const char *p = param;
+
+ return !strcmp(c->name, p);
+ }
+ return false;
+}
+EXPORT_SYMBOL(sa11x0_dma_filter_fn);
+
+static int __init sa11x0_dma_init(void)
+{
+ return platform_driver_register(&sa11x0_dma_driver);
+}
+subsys_initcall(sa11x0_dma_init);
+
+static void __exit sa11x0_dma_exit(void)
+{
+ platform_driver_unregister(&sa11x0_dma_driver);
+}
+module_exit(sa11x0_dma_exit);
+
+MODULE_AUTHOR("Russell King");
+MODULE_DESCRIPTION("SA-11x0 DMA driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:sa11x0-dma");