diff options
Diffstat (limited to 'drivers/usb/musb/cppi_dma.c')
| -rw-r--r-- | drivers/usb/musb/cppi_dma.c | 1547 |
1 files changed, 0 insertions, 1547 deletions
diff --git a/drivers/usb/musb/cppi_dma.c b/drivers/usb/musb/cppi_dma.c deleted file mode 100644 index edb5b63d7063..000000000000 --- a/drivers/usb/musb/cppi_dma.c +++ /dev/null @@ -1,1547 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (C) 2005-2006 by Texas Instruments - * - * This file implements a DMA interface using TI's CPPI DMA. - * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB. - * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci. - */ - -#include <linux/module.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/usb.h> - -#include "musb_core.h" -#include "musb_debug.h" -#include "cppi_dma.h" -#include "davinci.h" - - -/* CPPI DMA status 7-mar-2006: - * - * - See musb_{host,gadget}.c for more info - * - * - Correct RX DMA generally forces the engine into irq-per-packet mode, - * which can easily saturate the CPU under non-mass-storage loads. - * - * NOTES 24-aug-2006 (2.6.18-rc4): - * - * - peripheral RXDMA wedged in a test with packets of length 512/512/1. - * evidently after the 1 byte packet was received and acked, the queue - * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003, - * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401 - * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx - * of its next (512 byte) packet. IRQ issues? - * - * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will - * evidently also directly update the RX and TX CSRs ... so audit all - * host and peripheral side DMA code to avoid CSR access after DMA has - * been started. - */ - -/* REVISIT now we can avoid preallocating these descriptors; or - * more simply, switch to a global freelist not per-channel ones. - * Note: at full speed, 64 descriptors == 4K bulk data. - */ -#define NUM_TXCHAN_BD 64 -#define NUM_RXCHAN_BD 64 - -static inline void cpu_drain_writebuffer(void) -{ - wmb(); -#ifdef CONFIG_CPU_ARM926T - /* REVISIT this "should not be needed", - * but lack of it sure seemed to hurt ... - */ - asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n"); -#endif -} - -static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c) -{ - struct cppi_descriptor *bd = c->freelist; - - if (bd) - c->freelist = bd->next; - return bd; -} - -static inline void -cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd) -{ - if (!bd) - return; - bd->next = c->freelist; - c->freelist = bd; -} - -/* - * Start DMA controller - * - * Initialize the DMA controller as necessary. - */ - -/* zero out entire rx state RAM entry for the channel */ -static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx) -{ - musb_writel(&rx->rx_skipbytes, 0, 0); - musb_writel(&rx->rx_head, 0, 0); - musb_writel(&rx->rx_sop, 0, 0); - musb_writel(&rx->rx_current, 0, 0); - musb_writel(&rx->rx_buf_current, 0, 0); - musb_writel(&rx->rx_len_len, 0, 0); - musb_writel(&rx->rx_cnt_cnt, 0, 0); -} - -/* zero out entire tx state RAM entry for the channel */ -static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr) -{ - musb_writel(&tx->tx_head, 0, 0); - musb_writel(&tx->tx_buf, 0, 0); - musb_writel(&tx->tx_current, 0, 0); - musb_writel(&tx->tx_buf_current, 0, 0); - musb_writel(&tx->tx_info, 0, 0); - musb_writel(&tx->tx_rem_len, 0, 0); - /* musb_writel(&tx->tx_dummy, 0, 0); */ - musb_writel(&tx->tx_complete, 0, ptr); -} - -static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c) -{ - int j; - - /* initialize channel fields */ - c->head = NULL; - c->tail = NULL; - c->last_processed = NULL; - c->channel.status = MUSB_DMA_STATUS_UNKNOWN; - c->controller = cppi; - c->is_rndis = 0; - c->freelist = NULL; - - /* build the BD Free list for the channel */ - for (j = 0; j < NUM_TXCHAN_BD + 1; j++) { - struct cppi_descriptor *bd; - dma_addr_t dma; - - bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma); - bd->dma = dma; - cppi_bd_free(c, bd); - } -} - -static int cppi_channel_abort(struct dma_channel *); - -static void cppi_pool_free(struct cppi_channel *c) -{ - struct cppi *cppi = c->controller; - struct cppi_descriptor *bd; - - (void) cppi_channel_abort(&c->channel); - c->channel.status = MUSB_DMA_STATUS_UNKNOWN; - c->controller = NULL; - - /* free all its bds */ - bd = c->last_processed; - do { - if (bd) - dma_pool_free(cppi->pool, bd, bd->dma); - bd = cppi_bd_alloc(c); - } while (bd); - c->last_processed = NULL; -} - -static void cppi_controller_start(struct cppi *controller) -{ - void __iomem *tibase; - int i; - - /* do whatever is necessary to start controller */ - for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { - controller->tx[i].transmit = true; - controller->tx[i].index = i; - } - for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { - controller->rx[i].transmit = false; - controller->rx[i].index = i; - } - - /* setup BD list on a per channel basis */ - for (i = 0; i < ARRAY_SIZE(controller->tx); i++) - cppi_pool_init(controller, controller->tx + i); - for (i = 0; i < ARRAY_SIZE(controller->rx); i++) - cppi_pool_init(controller, controller->rx + i); - - tibase = controller->tibase; - INIT_LIST_HEAD(&controller->tx_complete); - - /* initialise tx/rx channel head pointers to zero */ - for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { - struct cppi_channel *tx_ch = controller->tx + i; - struct cppi_tx_stateram __iomem *tx; - - INIT_LIST_HEAD(&tx_ch->tx_complete); - - tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i); - tx_ch->state_ram = tx; - cppi_reset_tx(tx, 0); - } - for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { - struct cppi_channel *rx_ch = controller->rx + i; - struct cppi_rx_stateram __iomem *rx; - - INIT_LIST_HEAD(&rx_ch->tx_complete); - - rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i); - rx_ch->state_ram = rx; - cppi_reset_rx(rx); - } - - /* enable individual cppi channels */ - musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG, - DAVINCI_DMA_ALL_CHANNELS_ENABLE); - musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG, - DAVINCI_DMA_ALL_CHANNELS_ENABLE); - - /* enable tx/rx CPPI control */ - musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); - musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE); - - /* disable RNDIS mode, also host rx RNDIS autorequest */ - musb_writel(tibase, DAVINCI_RNDIS_REG, 0); - musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0); -} - -/* - * Stop DMA controller - * - * De-Init the DMA controller as necessary. - */ - -static void cppi_controller_stop(struct cppi *controller) -{ - void __iomem *tibase; - int i; - struct musb *musb; - - musb = controller->controller.musb; - - tibase = controller->tibase; - /* DISABLE INDIVIDUAL CHANNEL Interrupts */ - musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG, - DAVINCI_DMA_ALL_CHANNELS_ENABLE); - musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG, - DAVINCI_DMA_ALL_CHANNELS_ENABLE); - - musb_dbg(musb, "Tearing down RX and TX Channels"); - for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { - /* FIXME restructure of txdma to use bds like rxdma */ - controller->tx[i].last_processed = NULL; - cppi_pool_free(controller->tx + i); - } - for (i = 0; i < ARRAY_SIZE(controller->rx); i++) - cppi_pool_free(controller->rx + i); - - /* in Tx Case proper teardown is supported. We resort to disabling - * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is - * complete TX CPPI cannot be disabled. - */ - /*disable tx/rx cppi */ - musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); - musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE); -} - -/* While dma channel is allocated, we only want the core irqs active - * for fault reports, otherwise we'd get irqs that we don't care about. - * Except for TX irqs, where dma done != fifo empty and reusable ... - * - * NOTE: docs don't say either way, but irq masking **enables** irqs. - * - * REVISIT same issue applies to pure PIO usage too, and non-cppi dma... - */ -static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum) -{ - musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8)); -} - -static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum) -{ - musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8)); -} - - -/* - * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to - * each transfer direction of a non-control endpoint, so allocating - * (and deallocating) is mostly a way to notice bad housekeeping on - * the software side. We assume the irqs are always active. - */ -static struct dma_channel * -cppi_channel_allocate(struct dma_controller *c, - struct musb_hw_ep *ep, u8 transmit) -{ - struct cppi *controller; - u8 index; - struct cppi_channel *cppi_ch; - void __iomem *tibase; - struct musb *musb; - - controller = container_of(c, struct cppi, controller); - tibase = controller->tibase; - musb = c->musb; - - /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */ - index = ep->epnum - 1; - - /* return the corresponding CPPI Channel Handle, and - * probably disable the non-CPPI irq until we need it. - */ - if (transmit) { - if (index >= ARRAY_SIZE(controller->tx)) { - musb_dbg(musb, "no %cX%d CPPI channel", 'T', index); - return NULL; - } - cppi_ch = controller->tx + index; - } else { - if (index >= ARRAY_SIZE(controller->rx)) { - musb_dbg(musb, "no %cX%d CPPI channel", 'R', index); - return NULL; - } - cppi_ch = controller->rx + index; - core_rxirq_disable(tibase, ep->epnum); - } - - /* REVISIT make this an error later once the same driver code works - * with the other DMA engine too - */ - if (cppi_ch->hw_ep) - musb_dbg(musb, "re-allocating DMA%d %cX channel %p", - index, transmit ? 'T' : 'R', cppi_ch); - cppi_ch->hw_ep = ep; - cppi_ch->channel.status = MUSB_DMA_STATUS_FREE; - cppi_ch->channel.max_len = 0x7fffffff; - - musb_dbg(musb, "Allocate CPPI%d %cX", index, transmit ? 'T' : 'R'); - return &cppi_ch->channel; -} - -/* Release a CPPI Channel. */ -static void cppi_channel_release(struct dma_channel *channel) -{ - struct cppi_channel *c; - void __iomem *tibase; - - /* REVISIT: for paranoia, check state and abort if needed... */ - - c = container_of(channel, struct cppi_channel, channel); - tibase = c->controller->tibase; - if (!c->hw_ep) - musb_dbg(c->controller->controller.musb, - "releasing idle DMA channel %p", c); - else if (!c->transmit) - core_rxirq_enable(tibase, c->index + 1); - - /* for now, leave its cppi IRQ enabled (we won't trigger it) */ - c->hw_ep = NULL; - channel->status = MUSB_DMA_STATUS_UNKNOWN; -} - -/* Context: controller irqlocked */ -static void -cppi_dump_rx(int level, struct cppi_channel *c, const char *tag) -{ - void __iomem *base = c->controller->mregs; - struct cppi_rx_stateram __iomem *rx = c->state_ram; - - musb_ep_select(base, c->index + 1); - - musb_dbg(c->controller->controller.musb, - "RX DMA%d%s: %d left, csr %04x, " - "%08x H%08x S%08x C%08x, " - "B%08x L%08x %08x .. %08x", - c->index, tag, - musb_readl(c->controller->tibase, - DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index), - musb_readw(c->hw_ep->regs, MUSB_RXCSR), - - musb_readl(&rx->rx_skipbytes, 0), - musb_readl(&rx->rx_head, 0), - musb_readl(&rx->rx_sop, 0), - musb_readl(&rx->rx_current, 0), - - musb_readl(&rx->rx_buf_current, 0), - musb_readl(&rx->rx_len_len, 0), - musb_readl(&rx->rx_cnt_cnt, 0), - musb_readl(&rx->rx_complete, 0) - ); -} - -/* Context: controller irqlocked */ -static void -cppi_dump_tx(int level, struct cppi_channel *c, const char *tag) -{ - void __iomem *base = c->controller->mregs; - struct cppi_tx_stateram __iomem *tx = c->state_ram; - - musb_ep_select(base, c->index + 1); - - musb_dbg(c->controller->controller.musb, - "TX DMA%d%s: csr %04x, " - "H%08x S%08x C%08x %08x, " - "F%08x L%08x .. %08x", - c->index, tag, - musb_readw(c->hw_ep->regs, MUSB_TXCSR), - - musb_readl(&tx->tx_head, 0), - musb_readl(&tx->tx_buf, 0), - musb_readl(&tx->tx_current, 0), - musb_readl(&tx->tx_buf_current, 0), - - musb_readl(&tx->tx_info, 0), - musb_readl(&tx->tx_rem_len, 0), - /* dummy/unused word 6 */ - musb_readl(&tx->tx_complete, 0) - ); -} - -/* Context: controller irqlocked */ -static inline void -cppi_rndis_update(struct cppi_channel *c, int is_rx, - void __iomem *tibase, int is_rndis) -{ - /* we may need to change the rndis flag for this cppi channel */ - if (c->is_rndis != is_rndis) { - u32 value = musb_readl(tibase, DAVINCI_RNDIS_REG); - u32 temp = 1 << (c->index); - - if (is_rx) - temp <<= 16; - if (is_rndis) - value |= temp; - else - value &= ~temp; - musb_writel(tibase, DAVINCI_RNDIS_REG, value); - c->is_rndis = is_rndis; - } -} - -static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd) -{ - pr_debug("RXBD/%s %08x: " - "nxt %08x buf %08x off.blen %08x opt.plen %08x\n", - tag, bd->dma, - bd->hw_next, bd->hw_bufp, bd->hw_off_len, - bd->hw_options); -} - -static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) -{ - struct cppi_descriptor *bd; - - cppi_dump_rx(level, rx, tag); - if (rx->last_processed) - cppi_dump_rxbd("last", rx->last_processed); - for (bd = rx->head; bd; bd = bd->next) - cppi_dump_rxbd("active", bd); -} - - -/* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX; - * so we won't ever use it (see "CPPI RX Woes" below). - */ -static inline int cppi_autoreq_update(struct cppi_channel *rx, - void __iomem *tibase, int onepacket, unsigned n_bds) -{ - u32 val; - -#ifdef RNDIS_RX_IS_USABLE - u32 tmp; - /* assert(is_host_active(musb)) */ - - /* start from "AutoReq never" */ - tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); - val = tmp & ~((0x3) << (rx->index * 2)); - - /* HCD arranged reqpkt for packet #1. we arrange int - * for all but the last one, maybe in two segments. - */ - if (!onepacket) { -#if 0 - /* use two segments, autoreq "all" then the last "never" */ - val |= ((0x3) << (rx->index * 2)); - n_bds--; -#else - /* one segment, autoreq "all-but-last" */ - val |= ((0x1) << (rx->index * 2)); -#endif - } - - if (val != tmp) { - int n = 100; - - /* make sure that autoreq is updated before continuing */ - musb_writel(tibase, DAVINCI_AUTOREQ_REG, val); - do { - tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG); - if (tmp == val) - break; - cpu_relax(); - } while (n-- > 0); - } -#endif - - /* REQPKT is turned off after each segment */ - if (n_bds && rx->channel.actual_len) { - void __iomem *regs = rx->hw_ep->regs; - - val = musb_readw(regs, MUSB_RXCSR); - if (!(val & MUSB_RXCSR_H_REQPKT)) { - val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS; - musb_writew(regs, MUSB_RXCSR, val); - /* flush writebuffer */ - val = musb_readw(regs, MUSB_RXCSR); - } - } - return n_bds; -} - - -/* Buffer enqueuing Logic: - * - * - RX builds new queues each time, to help handle routine "early - * termination" cases (faults, including errors and short reads) - * more correctly. - * - * - for now, TX reuses the same queue of BDs every time - * - * REVISIT long term, we want a normal dynamic model. - * ... the goal will be to append to the - * existing queue, processing completed "dma buffers" (segments) on the fly. - * - * Otherwise we force an IRQ latency between requests, which slows us a lot - * (especially in "transparent" dma). Unfortunately that model seems to be - * inherent in the DMA model from the Mentor code, except in the rare case - * of transfers big enough (~128+ KB) that we could append "middle" segments - * in the TX paths. (RX can't do this, see below.) - * - * That's true even in the CPPI- friendly iso case, where most urbs have - * several small segments provided in a group and where the "packet at a time" - * "transparent" DMA model is always correct, even on the RX side. - */ - -/* - * CPPI TX: - * ======== - * TX is a lot more reasonable than RX; it doesn't need to run in - * irq-per-packet mode very often. RNDIS mode seems to behave too - * (except how it handles the exactly-N-packets case). Building a - * txdma queue with multiple requests (urb or usb_request) looks - * like it would work ... but fault handling would need much testing. - * - * The main issue with TX mode RNDIS relates to transfer lengths that - * are an exact multiple of the packet length. It appears that there's - * a hiccup in that case (maybe the DMA completes before the ZLP gets - * written?) boiling down to not being able to rely on CPPI writing any - * terminating zero length packet before the next transfer is written. - * So that's punted to PIO; better yet, gadget drivers can avoid it. - * - * Plus, there's allegedly an undocumented constraint that rndis transfer - * length be a multiple of 64 bytes ... but the chip doesn't act that - * way, and we really don't _want_ that behavior anyway. - * - * On TX, "transparent" mode works ... although experiments have shown - * problems trying to use the SOP/EOP bits in different USB packets. - * - * REVISIT try to handle terminating zero length packets using CPPI - * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet - * links avoid that issue by forcing them to avoid zlps.) - */ -static void -cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx) -{ - unsigned maxpacket = tx->maxpacket; - dma_addr_t addr = tx->buf_dma + tx->offset; - size_t length = tx->buf_len - tx->offset; - struct cppi_descriptor *bd; - unsigned n_bds; - unsigned i; - struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram; - int rndis; - - /* TX can use the CPPI "rndis" mode, where we can probably fit this - * transfer in one BD and one IRQ. The only time we would NOT want - * to use it is when hardware constraints prevent it, or if we'd - * trigger the "send a ZLP?" confusion. - */ - rndis = (maxpacket & 0x3f) == 0 - && length > maxpacket - && length < 0xffff - && (length % maxpacket) != 0; - - if (rndis) { - maxpacket = length; - n_bds = 1; - } else { - if (length) - n_bds = DIV_ROUND_UP(length, maxpacket); - else - n_bds = 1; - n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD); - length = min(n_bds * maxpacket, length); - } - - musb_dbg(musb, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u", - tx->index, - maxpacket, - rndis ? "rndis" : "transparent", - n_bds, - (unsigned long long)addr, length); - - cppi_rndis_update(tx, 0, musb->ctrl_base, rndis); - - /* assuming here that channel_program is called during - * transfer initiation ... current code maintains state - * for one outstanding request only (no queues, not even - * the implicit ones of an iso urb). - */ - - bd = tx->freelist; - tx->head = bd; - tx->last_processed = NULL; - - /* FIXME use BD pool like RX side does, and just queue - * the minimum number for this request. - */ - - /* Prepare queue of BDs first, then hand it to hardware. - * All BDs except maybe the last should be of full packet - * size; for RNDIS there _is_ only that last packet. - */ - for (i = 0; i < n_bds; ) { - if (++i < n_bds && bd->next) - bd->hw_next = bd->next->dma; - else - bd->hw_next = 0; - - bd->hw_bufp = tx->buf_dma + tx->offset; - - /* FIXME set EOP only on the last packet, - * SOP only on the first ... avoid IRQs - */ - if ((tx->offset + maxpacket) <= tx->buf_len) { - tx->offset += maxpacket; - bd->hw_off_len = maxpacket; - bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET - | CPPI_OWN_SET | maxpacket; - } else { - /* only this one may be a partial USB Packet */ - u32 partial_len; - - partial_len = tx->buf_len - tx->offset; - tx->offset = tx->buf_len; - bd->hw_off_len = partial_len; - - bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET - | CPPI_OWN_SET | partial_len; - if (partial_len == 0) - bd->hw_options |= CPPI_ZERO_SET; - } - - musb_dbg(musb, "TXBD %p: nxt %08x buf %08x len %04x opt %08x", - bd, bd->hw_next, bd->hw_bufp, - bd->hw_off_len, bd->hw_options); - - /* update the last BD enqueued to the list */ - tx->tail = bd; - bd = bd->next; - } - - /* BDs live in DMA-coherent memory, but writes might be pending */ - cpu_drain_writebuffer(); - - /* Write to the HeadPtr in state RAM to trigger */ - musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma); - - cppi_dump_tx(5, tx, "/S"); -} - -/* - * CPPI RX Woes: - * ============= - * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte - * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back. - * (Full speed transfers have similar scenarios.) - * - * The correct behavior for Linux is that (a) fills the buffer with 300 bytes, - * and the next packet goes into a buffer that's queued later; while (b) fills - * the buffer with 1024 bytes. How to do that with CPPI? - * - * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but - * (b) loses **BADLY** because nothing (!) happens when that second packet - * fills the buffer, much less when a third one arrives. (Which makes this - * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination - * is optional, and it's fine if peripherals -- not hosts! -- pad messages - * out to end-of-buffer. Standard PCI host controller DMA descriptors - * implement that mode by default ... which is no accident.) - * - * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have - * converse problems: (b) is handled right, but (a) loses badly. CPPI RX - * ignores SOP/EOP markings and processes both of those BDs; so both packets - * are loaded into the buffer (with a 212 byte gap between them), and the next - * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP - * are intended as outputs for RX queues, not inputs...) - * - * - A variant of "transparent" mode -- one BD at a time -- is the only way to - * reliably make both cases work, with software handling both cases correctly - * and at the significant penalty of needing an IRQ per packet. (The lack of - * I/O overlap can be slightly ameliorated by enabling double buffering.) - * - * So how to get rid of IRQ-per-packet? The transparent multi-BD case could - * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK - * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors - * with guaranteed driver level fault recovery and scrubbing out what's left - * of that garbaged datastream. - * - * But there seems to be no way to identify the cases where CPPI RNDIS mode - * is appropriate -- which do NOT include RNDIS host drivers, but do include - * the CDC Ethernet driver! -- and the documentation is incomplete/wrong. - * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic - * that applies best on the peripheral side (and which could fail rudely). - * - * Leaving only "transparent" mode; we avoid multi-bd modes in almost all - * cases other than mass storage class. Otherwise we're correct but slow, - * since CPPI penalizes our need for a "true RNDIS" default mode. - */ - - -/* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY - * - * IFF - * (a) peripheral mode ... since rndis peripherals could pad their - * writes to hosts, causing i/o failure; or we'd have to cope with - * a largely unknowable variety of host side protocol variants - * (b) and short reads are NOT errors ... since full reads would - * cause those same i/o failures - * (c) and read length is - * - less than 64KB (max per cppi descriptor) - * - not a multiple of 4096 (g_zero default, full reads typical) - * - N (>1) packets long, ditto (full reads not EXPECTED) - * THEN - * try rx rndis mode - * - * Cost of heuristic failing: RXDMA wedges at the end of transfers that - * fill out the whole buffer. Buggy host side usb network drivers could - * trigger that, but "in the field" such bugs seem to be all but unknown. - * - * So this module parameter lets the heuristic be disabled. When using - * gadgetfs, the heuristic will probably need to be disabled. - */ -static bool cppi_rx_rndis = 1; - -module_param(cppi_rx_rndis, bool, 0); -MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic"); - - -/** - * cppi_next_rx_segment - dma read for the next chunk of a buffer - * @musb: the controller - * @rx: dma channel - * @onepacket: true unless caller treats short reads as errors, and - * performs fault recovery above usbcore. - * Context: controller irqlocked - * - * See above notes about why we can't use multi-BD RX queues except in - * rare cases (mass storage class), and can never use the hardware "rndis" - * mode (since it's not a "true" RNDIS mode) with complete safety.. - * - * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in - * code to recover from corrupted datastreams after each short transfer. - */ -static void -cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) -{ - unsigned maxpacket = rx->maxpacket; - dma_addr_t addr = rx->buf_dma + rx->offset; - size_t length = rx->buf_len - rx->offset; - struct cppi_descriptor *bd, *tail; - unsigned n_bds; - unsigned i; - void __iomem *tibase = musb->ctrl_base; - int is_rndis = 0; - struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; - struct cppi_descriptor *d; - - if (onepacket) { - /* almost every USB driver, host or peripheral side */ - n_bds = 1; - - /* maybe apply the heuristic above */ - if (cppi_rx_rndis - && is_peripheral_active(musb) - && length > maxpacket - && (length & ~0xffff) == 0 - && (length & 0x0fff) != 0 - && (length & (maxpacket - 1)) == 0) { - maxpacket = length; - is_rndis = 1; - } - } else { - /* virtually nothing except mass storage class */ - if (length > 0xffff) { - n_bds = 0xffff / maxpacket; - length = n_bds * maxpacket; - } else { - n_bds = DIV_ROUND_UP(length, maxpacket); - } - if (n_bds == 1) - onepacket = 1; - else - n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD); - } - - /* In host mode, autorequest logic can generate some IN tokens; it's - * tricky since we can't leave REQPKT set in RXCSR after the transfer - * finishes. So: multipacket transfers involve two or more segments. - * And always at least two IRQs ... RNDIS mode is not an option. - */ - if (is_host_active(musb)) - n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); - - cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); - - length = min(n_bds * maxpacket, length); - - musb_dbg(musb, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " - "dma 0x%llx len %u %u/%u", - rx->index, maxpacket, - onepacket - ? (is_rndis ? "rndis" : "onepacket") - : "multipacket", - n_bds, - musb_readl(tibase, - DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) - & 0xffff, - (unsigned long long)addr, length, - rx->channel.actual_len, rx->buf_len); - - /* only queue one segment at a time, since the hardware prevents - * correct queue shutdown after unexpected short packets - */ - bd = cppi_bd_alloc(rx); - rx->head = bd; - - /* Build BDs for all packets in this segment */ - for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) { - u32 bd_len; - - if (i) { - bd = cppi_bd_alloc(rx); - if (!bd) - break; - tail->next = bd; - tail->hw_next = bd->dma; - } - bd->hw_next = 0; - - /* all but the last packet will be maxpacket size */ - if (maxpacket < length) - bd_len = maxpacket; - else - bd_len = length; - - bd->hw_bufp = addr; - addr += bd_len; - rx->offset += bd_len; - - bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; - bd->buflen = bd_len; - - bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); - length -= bd_len; - } - - /* we always expect at least one reusable BD! */ - if (!tail) { - WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); - return; - } else if (i < n_bds) - WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); - - tail->next = NULL; - tail->hw_next = 0; - - bd = rx->head; - rx->tail = tail; - - /* short reads and other faults should terminate this entire - * dma segment. we want one "dma packet" per dma segment, not - * one per USB packet, terminating the whole queue at once... - * NOTE that current hardware seems to ignore SOP and EOP. - */ - bd->hw_options |= CPPI_SOP_SET; - tail->hw_options |= CPPI_EOP_SET; - - for (d = rx->head; d; d = d->next) - cppi_dump_rxbd("S", d); - - /* in case the preceding transfer left some state... */ - tail = rx->last_processed; - if (tail) { - tail->next = bd; - tail->hw_next = bd->dma; - } - - core_rxirq_enable(tibase, rx->index + 1); - - /* BDs live in DMA-coherent memory, but writes might be pending */ - cpu_drain_writebuffer(); - - /* REVISIT specs say to write this AFTER the BUFCNT register - * below ... but that loses badly. - */ - musb_writel(&rx_ram->rx_head, 0, bd->dma); - - /* bufferCount must be at least 3, and zeroes on completion - * unless it underflows below zero, or stops at two, or keeps - * growing ... grr. - */ - i = musb_readl(tibase, - DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) - & 0xffff; - - if (!i) - musb_writel(tibase, - DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), - n_bds + 2); - else if (n_bds > (i - 3)) - musb_writel(tibase, - DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), - n_bds - (i - 3)); - - i = musb_readl(tibase, - DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) - & 0xffff; - if (i < (2 + n_bds)) { - musb_dbg(musb, "bufcnt%d underrun - %d (for %d)", - rx->index, i, n_bds); - musb_writel(tibase, - DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), - n_bds + 2); - } - - cppi_dump_rx(4, rx, "/S"); -} - -/** - * cppi_channel_program - program channel for data transfer - * @ch: the channel - * @maxpacket: max packet size - * @mode: For RX, 1 unless the usb protocol driver promised to treat - * all short reads as errors and kick in high level fault recovery. - * For TX, ignored because of RNDIS mode races/glitches. - * @dma_addr: dma address of buffer - * @len: length of buffer - * Context: controller irqlocked - */ -static int cppi_channel_program(struct dma_channel *ch, - u16 maxpacket, u8 mode, - dma_addr_t dma_addr, u32 len) -{ - struct cppi_channel *cppi_ch; - struct cppi *controller; - struct musb *musb; - - cppi_ch = container_of(ch, struct cppi_channel, channel); - controller = cppi_ch->controller; - musb = controller->controller.musb; - - switch (ch->status) { - case MUSB_DMA_STATUS_BUS_ABORT: - case MUSB_DMA_STATUS_CORE_ABORT: - /* fault irq handler should have handled cleanup */ - WARNING("%cX DMA%d not cleaned up after abort!\n", - cppi_ch->transmit ? 'T' : 'R', - cppi_ch->index); - /* WARN_ON(1); */ - break; - case MUSB_DMA_STATUS_BUSY: - WARNING("program active channel? %cX DMA%d\n", - cppi_ch->transmit ? 'T' : 'R', - cppi_ch->index); - /* WARN_ON(1); */ - break; - case MUSB_DMA_STATUS_UNKNOWN: - musb_dbg(musb, "%cX DMA%d not allocated!", - cppi_ch->transmit ? 'T' : 'R', - cppi_ch->index); - fallthrough; - case MUSB_DMA_STATUS_FREE: - break; - } - - ch->status = MUSB_DMA_STATUS_BUSY; - - /* set transfer parameters, then queue up its first segment */ - cppi_ch->buf_dma = dma_addr; - cppi_ch->offset = 0; - cppi_ch->maxpacket = maxpacket; - cppi_ch->buf_len = len; - cppi_ch->channel.actual_len = 0; - - /* TX channel? or RX? */ - if (cppi_ch->transmit) - cppi_next_tx_segment(musb, cppi_ch); - else - cppi_next_rx_segment(musb, cppi_ch, mode); - - return true; -} - -static bool cppi_rx_scan(struct cppi *cppi, unsigned ch) -{ - struct cppi_channel *rx = &cppi->rx[ch]; - struct cppi_rx_stateram __iomem *state = rx->state_ram; - struct cppi_descriptor *bd; - struct cppi_descriptor *last = rx->last_processed; - bool completed = false; - bool acked = false; - int i; - dma_addr_t safe2ack; - void __iomem *regs = rx->hw_ep->regs; - struct musb *musb = cppi->controller.musb; - - cppi_dump_rx(6, rx, "/K"); - - bd = last ? last->next : rx->head; - if (!bd) - return false; - - /* run through all completed BDs */ - for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); - (safe2ack || completed) && bd && i < NUM_RXCHAN_BD; - i++, bd = bd->next) { - u16 len; - - /* catch latest BD writes from CPPI */ - rmb(); - if (!completed && (bd->hw_options & CPPI_OWN_SET)) - break; - - musb_dbg(musb, "C/RXBD %llx: nxt %08x buf %08x " - "off.len %08x opt.len %08x (%d)", - (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp, - bd->hw_off_len, bd->hw_options, - rx->channel.actual_len); - - /* actual packet received length */ - if ((bd->hw_options & CPPI_SOP_SET) && !completed) - len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; - else - len = 0; - - if (bd->hw_options & CPPI_EOQ_MASK) - completed = true; - - if (!completed && len < bd->buflen) { - /* NOTE: when we get a short packet, RXCSR_H_REQPKT - * must have been cleared, and no more DMA packets may - * active be in the queue... TI docs didn't say, but - * CPPI ignores those BDs even though OWN is still set. - */ - completed = true; - musb_dbg(musb, "rx short %d/%d (%d)", - len, bd->buflen, - rx->channel.actual_len); - } - - /* If we got here, we expect to ack at least one BD; meanwhile - * CPPI may completing other BDs while we scan this list... - * - * RACE: we can notice OWN cleared before CPPI raises the - * matching irq by writing that BD as the completion pointer. - * In such cases, stop scanning and wait for the irq, avoiding - * lost acks and states where BD ownership is unclear. - */ - if (bd->dma == safe2ack) { - musb_writel(&state->rx_complete, 0, safe2ack); - safe2ack = musb_readl(&state->rx_complete, 0); - acked = true; - if (bd->dma == safe2ack) - safe2ack = 0; - } - - rx->channel.actual_len += len; - - cppi_bd_free(rx, last); - last = bd; - - /* stop scanning on end-of-segment */ - if (bd->hw_next == 0) - completed = true; - } - rx->last_processed = last; - - /* dma abort, lost ack, or ... */ - if (!acked && last) { - int csr; - - if (safe2ack == 0 || safe2ack == rx->last_processed->dma) - musb_writel(&state->rx_complete, 0, safe2ack); - if (safe2ack == 0) { - cppi_bd_free(rx, last); - rx->last_processed = NULL; - - /* if we land here on the host side, H_REQPKT will - * be clear and we need to restart the queue... - */ - WARN_ON(rx->head); - } - musb_ep_select(cppi->mregs, rx->index + 1); - csr = musb_readw(regs, MUSB_RXCSR); - if (csr & MUSB_RXCSR_DMAENAB) { - musb_dbg(musb, "list%d %p/%p, last %llx%s, csr %04x", - rx->index, - rx->head, rx->tail, - rx->last_processed - ? (unsigned long long) - rx->last_processed->dma - : 0, - completed ? ", completed" : "", - csr); - cppi_dump_rxq(4, "/what?", rx); - } - } - if (!completed) { - int csr; - - rx->head = bd; - - /* REVISIT seems like "autoreq all but EOP" doesn't... - * setting it here "should" be racey, but seems to work - */ - csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); - if (is_host_active(cppi->controller.musb) - && bd - && !(csr & MUSB_RXCSR_H_REQPKT)) { - csr |= MUSB_RXCSR_H_REQPKT; - musb_writew(regs, MUSB_RXCSR, - MUSB_RXCSR_H_WZC_BITS | csr); - csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); - } - } else { - rx->head = NULL; - rx->tail = NULL; - } - - cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); - return completed; -} - -irqreturn_t cppi_interrupt(int irq, void *dev_id) -{ - struct musb *musb = dev_id; - struct cppi *cppi; - void __iomem *tibase; - struct musb_hw_ep *hw_ep = NULL; - u32 rx, tx; - int i, index; - unsigned long flags; - - cppi = container_of(musb->dma_controller, struct cppi, controller); - if (cppi->irq) - spin_lock_irqsave(&musb->lock, flags); - - tibase = musb->ctrl_base; - - tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG); - rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG); - - if (!tx && !rx) { - if (cppi->irq) - spin_unlock_irqrestore(&musb->lock, flags); - return IRQ_NONE; - } - - musb_dbg(musb, "CPPI IRQ Tx%x Rx%x", tx, rx); - - /* process TX channels */ - for (index = 0; tx; tx = tx >> 1, index++) { - struct cppi_channel *tx_ch; - struct cppi_tx_stateram __iomem *tx_ram; - bool completed = false; - struct cppi_descriptor *bd; - - if (!(tx & 1)) - continue; - - tx_ch = cppi->tx + index; - tx_ram = tx_ch->state_ram; - - /* FIXME need a cppi_tx_scan() routine, which - * can also be called from abort code - */ - - cppi_dump_tx(5, tx_ch, "/E"); - - bd = tx_ch->head; - - /* - * If Head is null then this could mean that a abort interrupt - * that needs to be acknowledged. - */ - if (NULL == bd) { - musb_dbg(musb, "null BD"); - musb_writel(&tx_ram->tx_complete, 0, 0); - continue; - } - - /* run through all completed BDs */ - for (i = 0; !completed && bd && i < NUM_TXCHAN_BD; - i++, bd = bd->next) { - u16 len; - - /* catch latest BD writes from CPPI */ - rmb(); - if (bd->hw_options & CPPI_OWN_SET) - break; - - musb_dbg(musb, "C/TXBD %p n %x b %x off %x opt %x", - bd, bd->hw_next, bd->hw_bufp, - bd->hw_off_len, bd->hw_options); - - len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK; - tx_ch->channel.actual_len += len; - - tx_ch->last_processed = bd; - - /* write completion register to acknowledge - * processing of completed BDs, and possibly - * release the IRQ; EOQ might not be set ... - * - * REVISIT use the same ack strategy as rx - * - * REVISIT have observed bit 18 set; huh?? - */ - /* if ((bd->hw_options & CPPI_EOQ_MASK)) */ - musb_writel(&tx_ram->tx_complete, 0, bd->dma); - - /* stop scanning on end-of-segment */ - if (bd->hw_next == 0) - completed = true; - } - - /* on end of segment, maybe go to next one */ - if (completed) { - /* cppi_dump_tx(4, tx_ch, "/complete"); */ - - /* transfer more, or report completion */ - if (tx_ch->offset >= tx_ch->buf_len) { - tx_ch->head = NULL; - tx_ch->tail = NULL; - tx_ch->channel.status = MUSB_DMA_STATUS_FREE; - - hw_ep = tx_ch->hw_ep; - - musb_dma_completion(musb, index + 1, 1); - - } else { - /* Bigger transfer than we could fit in - * that first batch of descriptors... - */ - cppi_next_tx_segment(musb, tx_ch); - } - } else - tx_ch->head = bd; - } - - /* Start processing the RX block */ - for (index = 0; rx; rx = rx >> 1, index++) { - - if (rx & 1) { - struct cppi_channel *rx_ch; - - rx_ch = cppi->rx + index; - - /* let incomplete dma segments finish */ - if (!cppi_rx_scan(cppi, index)) - continue; - - /* start another dma segment if needed */ - if (rx_ch->channel.actual_len != rx_ch->buf_len - && rx_ch->channel.actual_len - == rx_ch->offset) { - cppi_next_rx_segment(musb, rx_ch, 1); - continue; - } - - /* all segments completed! */ - rx_ch->channel.status = MUSB_DMA_STATUS_FREE; - - hw_ep = rx_ch->hw_ep; - - core_rxirq_disable(tibase, index + 1); - musb_dma_completion(musb, index + 1, 0); - } - } - - /* write to CPPI EOI register to re-enable interrupts */ - musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0); - - if (cppi->irq) - spin_unlock_irqrestore(&musb->lock, flags); - - return IRQ_HANDLED; -} -EXPORT_SYMBOL_GPL(cppi_interrupt); - -/* Instantiate a software object representing a DMA controller. */ -struct dma_controller * -cppi_dma_controller_create(struct musb *musb, void __iomem *mregs) -{ - struct cppi *controller; - struct device *dev = musb->controller; - struct platform_device *pdev = to_platform_device(dev); - int irq = platform_get_irq_byname(pdev, "dma"); - - controller = kzalloc(sizeof *controller, GFP_KERNEL); - if (!controller) - return NULL; - - controller->mregs = mregs; - controller->tibase = mregs - DAVINCI_BASE_OFFSET; - - controller->controller.musb = musb; - controller->controller.channel_alloc = cppi_channel_allocate; - controller->controller.channel_release = cppi_channel_release; - controller->controller.channel_program = cppi_channel_program; - controller->controller.channel_abort = cppi_channel_abort; - - /* NOTE: allocating from on-chip SRAM would give the least - * contention for memory access, if that ever matters here. - */ - - /* setup BufferPool */ - controller->pool = dma_pool_create("cppi", - controller->controller.musb->controller, - sizeof(struct cppi_descriptor), - CPPI_DESCRIPTOR_ALIGN, 0); - if (!controller->pool) { - kfree(controller); - return NULL; - } - - if (irq > 0) { - if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) { - dev_err(dev, "request_irq %d failed!\n", irq); - musb_dma_controller_destroy(&controller->controller); - return NULL; - } - controller->irq = irq; - } - - cppi_controller_start(controller); - return &controller->controller; -} -EXPORT_SYMBOL_GPL(cppi_dma_controller_create); - -/* - * Destroy a previously-instantiated DMA controller. - */ -void cppi_dma_controller_destroy(struct dma_controller *c) -{ - struct cppi *cppi; - - cppi = container_of(c, struct cppi, controller); - - cppi_controller_stop(cppi); - - if (cppi->irq) - free_irq(cppi->irq, cppi->controller.musb); - - /* assert: caller stopped the controller first */ - dma_pool_destroy(cppi->pool); - - kfree(cppi); -} -EXPORT_SYMBOL_GPL(cppi_dma_controller_destroy); - -/* - * Context: controller irqlocked, endpoint selected - */ -static int cppi_channel_abort(struct dma_channel *channel) -{ - struct cppi_channel *cppi_ch; - struct cppi *controller; - void __iomem *mbase; - void __iomem *tibase; - void __iomem *regs; - u32 value; - struct cppi_descriptor *queue; - - cppi_ch = container_of(channel, struct cppi_channel, channel); - - controller = cppi_ch->controller; - - switch (channel->status) { - case MUSB_DMA_STATUS_BUS_ABORT: - case MUSB_DMA_STATUS_CORE_ABORT: - /* from RX or TX fault irq handler */ - case MUSB_DMA_STATUS_BUSY: - /* the hardware needs shutting down */ - regs = cppi_ch->hw_ep->regs; - break; - case MUSB_DMA_STATUS_UNKNOWN: - case MUSB_DMA_STATUS_FREE: - return 0; - default: - return -EINVAL; - } - - if (!cppi_ch->transmit && cppi_ch->head) - cppi_dump_rxq(3, "/abort", cppi_ch); - - mbase = controller->mregs; - tibase = controller->tibase; - - queue = cppi_ch->head; - cppi_ch->head = NULL; - cppi_ch->tail = NULL; - - /* REVISIT should rely on caller having done this, - * and caller should rely on us not changing it. - * peripheral code is safe ... check host too. - */ - musb_ep_select(mbase, cppi_ch->index + 1); - - if (cppi_ch->transmit) { - struct cppi_tx_stateram __iomem *tx_ram; - /* REVISIT put timeouts on these controller handshakes */ - - cppi_dump_tx(6, cppi_ch, " (teardown)"); - - /* teardown DMA engine then usb core */ - do { - value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG); - } while (!(value & CPPI_TEAR_READY)); - musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); - - tx_ram = cppi_ch->state_ram; - do { - value = musb_readl(&tx_ram->tx_complete, 0); - } while (0xFFFFFFFC != value); - - /* FIXME clean up the transfer state ... here? - * the completion routine should get called with - * an appropriate status code. - */ - - value = musb_readw(regs, MUSB_TXCSR); - value &= ~MUSB_TXCSR_DMAENAB; - value |= MUSB_TXCSR_FLUSHFIFO; - musb_writew(regs, MUSB_TXCSR, value); - musb_writew(regs, MUSB_TXCSR, value); - - /* - * 1. Write to completion Ptr value 0x1(bit 0 set) - * (write back mode) - * 2. Wait for abort interrupt and then put the channel in - * compare mode by writing 1 to the tx_complete register. - */ - cppi_reset_tx(tx_ram, 1); - cppi_ch->head = NULL; - musb_writel(&tx_ram->tx_complete, 0, 1); - cppi_dump_tx(5, cppi_ch, " (done teardown)"); - - /* REVISIT tx side _should_ clean up the same way - * as the RX side ... this does no cleanup at all! - */ - - } else /* RX */ { - u16 csr; - - /* NOTE: docs don't guarantee any of this works ... we - * expect that if the usb core stops telling the cppi core - * to pull more data from it, then it'll be safe to flush - * current RX DMA state iff any pending fifo transfer is done. - */ - - core_rxirq_disable(tibase, cppi_ch->index + 1); - - /* for host, ensure ReqPkt is never set again */ - if (is_host_active(cppi_ch->controller->controller.musb)) { - value = musb_readl(tibase, DAVINCI_AUTOREQ_REG); - value &= ~((0x3) << (cppi_ch->index * 2)); - musb_writel(tibase, DAVINCI_AUTOREQ_REG, value); - } - - csr = musb_readw(regs, MUSB_RXCSR); - - /* for host, clear (just) ReqPkt at end of current packet(s) */ - if (is_host_active(cppi_ch->controller->controller.musb)) { - csr |= MUSB_RXCSR_H_WZC_BITS; - csr &= ~MUSB_RXCSR_H_REQPKT; - } else - csr |= MUSB_RXCSR_P_WZC_BITS; - - /* clear dma enable */ - csr &= ~(MUSB_RXCSR_DMAENAB); - musb_writew(regs, MUSB_RXCSR, csr); - csr = musb_readw(regs, MUSB_RXCSR); - - /* Quiesce: wait for current dma to finish (if not cleanup). - * We can't use bit zero of stateram->rx_sop, since that - * refers to an entire "DMA packet" not just emptying the - * current fifo. Most segments need multiple usb packets. - */ - if (channel->status == MUSB_DMA_STATUS_BUSY) - udelay(50); - - /* scan the current list, reporting any data that was - * transferred and acking any IRQ - */ - cppi_rx_scan(controller, cppi_ch->index); - - /* clobber the existing state once it's idle - * - * NOTE: arguably, we should also wait for all the other - * RX channels to quiesce (how??) and then temporarily - * disable RXCPPI_CTRL_REG ... but it seems that we can - * rely on the controller restarting from state ram, with - * only RXCPPI_BUFCNT state being bogus. BUFCNT will - * correct itself after the next DMA transfer though. - * - * REVISIT does using rndis mode change that? - */ - cppi_reset_rx(cppi_ch->state_ram); - - /* next DMA request _should_ load cppi head ptr */ - - /* ... we don't "free" that list, only mutate it in place. */ - cppi_dump_rx(5, cppi_ch, " (done abort)"); - - /* clean up previously pending bds */ - cppi_bd_free(cppi_ch, cppi_ch->last_processed); - cppi_ch->last_processed = NULL; - - while (queue) { - struct cppi_descriptor *tmp = queue->next; - - cppi_bd_free(cppi_ch, queue); - queue = tmp; - } - } - - channel->status = MUSB_DMA_STATUS_FREE; - cppi_ch->buf_dma = 0; - cppi_ch->offset = 0; - cppi_ch->buf_len = 0; - cppi_ch->maxpacket = 0; - return 0; -} - -/* TBD Queries: - * - * Power Management ... probably turn off cppi during suspend, restart; - * check state ram? Clocking is presumably shared with usb core. - */ |