diff options
Diffstat (limited to 'drivers/usb/mtu3/mtu3_qmu.c')
| -rw-r--r-- | drivers/usb/mtu3/mtu3_qmu.c | 293 |
1 files changed, 203 insertions, 90 deletions
diff --git a/drivers/usb/mtu3/mtu3_qmu.c b/drivers/usb/mtu3/mtu3_qmu.c index 7d9ba8a52368..03f26589b056 100644 --- a/drivers/usb/mtu3/mtu3_qmu.c +++ b/drivers/usb/mtu3/mtu3_qmu.c @@ -1,19 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 /* * mtu3_qmu.c - Queue Management Unit driver for device controller * * Copyright (C) 2016 MediaTek Inc. * * Author: Chunfeng Yun <chunfeng.yun@mediatek.com> - * - * This software is licensed under the terms of the GNU General Public - * License version 2, as published by the Free Software Foundation, and - * may be copied, distributed, and modified under those terms. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * */ /* @@ -31,16 +22,99 @@ #include <linux/iopoll.h> #include "mtu3.h" +#include "mtu3_trace.h" #define QMU_CHECKSUM_LEN 16 #define GPD_FLAGS_HWO BIT(0) #define GPD_FLAGS_BDP BIT(1) #define GPD_FLAGS_BPS BIT(2) +#define GPD_FLAGS_ZLP BIT(6) #define GPD_FLAGS_IOC BIT(7) +#define GET_GPD_HWO(gpd) (le32_to_cpu((gpd)->dw0_info) & GPD_FLAGS_HWO) + +#define GPD_RX_BUF_LEN_OG(x) (((x) & 0xffff) << 16) +#define GPD_RX_BUF_LEN_EL(x) (((x) & 0xfffff) << 12) +#define GPD_RX_BUF_LEN(mtu, x) \ +({ \ + typeof(x) x_ = (x); \ + ((mtu)->gen2cp) ? GPD_RX_BUF_LEN_EL(x_) : GPD_RX_BUF_LEN_OG(x_); \ +}) + +#define GPD_DATA_LEN_OG(x) ((x) & 0xffff) +#define GPD_DATA_LEN_EL(x) ((x) & 0xfffff) +#define GPD_DATA_LEN(mtu, x) \ +({ \ + typeof(x) x_ = (x); \ + ((mtu)->gen2cp) ? GPD_DATA_LEN_EL(x_) : GPD_DATA_LEN_OG(x_); \ +}) + +#define GPD_EXT_FLAG_ZLP BIT(29) +#define GPD_EXT_NGP_OG(x) (((x) & 0xf) << 20) +#define GPD_EXT_BUF_OG(x) (((x) & 0xf) << 16) +#define GPD_EXT_NGP_EL(x) (((x) & 0xf) << 28) +#define GPD_EXT_BUF_EL(x) (((x) & 0xf) << 24) +#define GPD_EXT_NGP(mtu, x) \ +({ \ + typeof(x) x_ = (x); \ + ((mtu)->gen2cp) ? GPD_EXT_NGP_EL(x_) : GPD_EXT_NGP_OG(x_); \ +}) + +#define GPD_EXT_BUF(mtu, x) \ +({ \ + typeof(x) x_ = (x); \ + ((mtu)->gen2cp) ? GPD_EXT_BUF_EL(x_) : GPD_EXT_BUF_OG(x_); \ +}) + +#define HILO_GEN64(hi, lo) (((u64)(hi) << 32) + (lo)) +#define HILO_DMA(hi, lo) \ + ((dma_addr_t)HILO_GEN64((le32_to_cpu(hi)), (le32_to_cpu(lo)))) + +static dma_addr_t read_txq_cur_addr(void __iomem *mbase, u8 epnum) +{ + u32 txcpr; + u32 txhiar; -#define GPD_EXT_FLAG_ZLP BIT(5) + txcpr = mtu3_readl(mbase, USB_QMU_TQCPR(epnum)); + txhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum)); + return HILO_DMA(QMU_CUR_GPD_ADDR_HI(txhiar), txcpr); +} + +static dma_addr_t read_rxq_cur_addr(void __iomem *mbase, u8 epnum) +{ + u32 rxcpr; + u32 rxhiar; + + rxcpr = mtu3_readl(mbase, USB_QMU_RQCPR(epnum)); + rxhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum)); + + return HILO_DMA(QMU_CUR_GPD_ADDR_HI(rxhiar), rxcpr); +} + +static void write_txq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma) +{ + u32 tqhiar; + + mtu3_writel(mbase, USB_QMU_TQSAR(epnum), + cpu_to_le32(lower_32_bits(dma))); + tqhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum)); + tqhiar &= ~QMU_START_ADDR_HI_MSK; + tqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma)); + mtu3_writel(mbase, USB_QMU_TQHIAR(epnum), tqhiar); +} + +static void write_rxq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma) +{ + u32 rqhiar; + + mtu3_writel(mbase, USB_QMU_RQSAR(epnum), + cpu_to_le32(lower_32_bits(dma))); + rqhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum)); + rqhiar &= ~QMU_START_ADDR_HI_MSK; + rqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma)); + mtu3_writel(mbase, USB_QMU_RQHIAR(epnum), rqhiar); +} static struct qmu_gpd *gpd_dma_to_virt(struct mtu3_gpd_ring *ring, dma_addr_t dma_addr) @@ -83,7 +157,7 @@ static void reset_gpd_list(struct mtu3_ep *mep) struct qmu_gpd *gpd = ring->start; if (gpd) { - gpd->flag &= ~GPD_FLAGS_HWO; + gpd->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO); gpd_ring_init(ring, gpd); } } @@ -112,27 +186,6 @@ void mtu3_gpd_ring_free(struct mtu3_ep *mep) memset(ring, 0, sizeof(*ring)); } -/* - * calculate check sum of a gpd or bd - * add "noinline" and "mb" to prevent wrong calculation - */ -static noinline u8 qmu_calc_checksum(u8 *data) -{ - u8 chksum = 0; - int i; - - data[1] = 0x0; /* set checksum to 0 */ - - mb(); /* ensure the gpd/bd is really up-to-date */ - for (i = 0; i < QMU_CHECKSUM_LEN; i++) - chksum += data[i]; - - /* Default: HWO=1, @flag[bit0] */ - chksum += 1; - - return 0xFF - chksum; -} - void mtu3_qmu_resume(struct mtu3_ep *mep) { struct mtu3 *mtu = mep->mtu; @@ -157,6 +210,7 @@ static struct qmu_gpd *advance_enq_gpd(struct mtu3_gpd_ring *ring) return ring->enqueue; } +/* @dequeue may be NULL if ring is unallocated or freed */ static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring) { if (ring->dequeue < ring->end) @@ -167,8 +221,8 @@ static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring) return ring->dequeue; } -/* check if a ring is emtpy */ -static int gpd_ring_empty(struct mtu3_gpd_ring *ring) +/* check if a ring is empty */ +static bool gpd_ring_empty(struct mtu3_gpd_ring *ring) { struct qmu_gpd *enq = ring->enqueue; struct qmu_gpd *next; @@ -193,29 +247,39 @@ static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq) struct mtu3_gpd_ring *ring = &mep->gpd_ring; struct qmu_gpd *gpd = ring->enqueue; struct usb_request *req = &mreq->request; + struct mtu3 *mtu = mep->mtu; + dma_addr_t enq_dma; + u32 ext_addr; - /* set all fields to zero as default value */ - memset(gpd, 0, sizeof(*gpd)); - - gpd->buffer = cpu_to_le32((u32)req->dma); - gpd->buf_len = cpu_to_le16(req->length); - gpd->flag |= GPD_FLAGS_IOC; + gpd->dw0_info = 0; /* SW own it */ + gpd->buffer = cpu_to_le32(lower_32_bits(req->dma)); + ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma)); + gpd->dw3_info = cpu_to_le32(GPD_DATA_LEN(mtu, req->length)); /* get the next GPD */ enq = advance_enq_gpd(ring); - dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p\n", - mep->epnum, gpd, enq); - - enq->flag &= ~GPD_FLAGS_HWO; - gpd->next_gpd = cpu_to_le32((u32)gpd_virt_to_dma(ring, enq)); - - if (req->zero) - gpd->ext_flag |= GPD_EXT_FLAG_ZLP; + enq_dma = gpd_virt_to_dma(ring, enq); + dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n", + mep->epnum, gpd, enq, &enq_dma); + + enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO); + gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma)); + ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma)); + gpd->dw0_info = cpu_to_le32(ext_addr); + + if (req->zero) { + if (mtu->gen2cp) + gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_ZLP); + else + gpd->dw3_info |= cpu_to_le32(GPD_EXT_FLAG_ZLP); + } - gpd->chksum = qmu_calc_checksum((u8 *)gpd); - gpd->flag |= GPD_FLAGS_HWO; + /* prevent reorder, make sure GPD's HWO is set last */ + mb(); + gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO); mreq->gpd = gpd; + trace_mtu3_prepare_gpd(mep, gpd); return 0; } @@ -226,25 +290,31 @@ static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq) struct mtu3_gpd_ring *ring = &mep->gpd_ring; struct qmu_gpd *gpd = ring->enqueue; struct usb_request *req = &mreq->request; + struct mtu3 *mtu = mep->mtu; + dma_addr_t enq_dma; + u32 ext_addr; - /* set all fields to zero as default value */ - memset(gpd, 0, sizeof(*gpd)); - - gpd->buffer = cpu_to_le32((u32)req->dma); - gpd->data_buf_len = cpu_to_le16(req->length); - gpd->flag |= GPD_FLAGS_IOC; + gpd->dw0_info = 0; /* SW own it */ + gpd->buffer = cpu_to_le32(lower_32_bits(req->dma)); + ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma)); + gpd->dw0_info = cpu_to_le32(GPD_RX_BUF_LEN(mtu, req->length)); /* get the next GPD */ enq = advance_enq_gpd(ring); - dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p\n", - mep->epnum, gpd, enq); - - enq->flag &= ~GPD_FLAGS_HWO; - gpd->next_gpd = cpu_to_le32((u32)gpd_virt_to_dma(ring, enq)); - gpd->chksum = qmu_calc_checksum((u8 *)gpd); - gpd->flag |= GPD_FLAGS_HWO; + enq_dma = gpd_virt_to_dma(ring, enq); + dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n", + mep->epnum, gpd, enq, &enq_dma); + + enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO); + gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma)); + ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma)); + gpd->dw3_info = cpu_to_le32(ext_addr); + /* prevent reorder, make sure GPD's HWO is set last */ + mb(); + gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO); mreq->gpd = gpd; + trace_mtu3_prepare_gpd(mep, gpd); return 0; } @@ -267,9 +337,8 @@ int mtu3_qmu_start(struct mtu3_ep *mep) if (mep->is_in) { /* set QMU start address */ - mtu3_writel(mbase, USB_QMU_TQSAR(mep->epnum), ring->dma); - mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN); - mtu3_setbits(mbase, U3D_QCR0, QMU_TX_CS_EN(epnum)); + write_txq_start_addr(mbase, epnum, ring->dma); + mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_DMAREQEN); /* send zero length packet according to ZLP flag in GPD */ mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum)); mtu3_writel(mbase, U3D_TQERRIESR0, @@ -282,9 +351,8 @@ int mtu3_qmu_start(struct mtu3_ep *mep) mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START); } else { - mtu3_writel(mbase, USB_QMU_RQSAR(mep->epnum), ring->dma); - mtu3_setbits(mbase, MU3D_EP_RXCR0(mep->epnum), RX_DMAREQEN); - mtu3_setbits(mbase, U3D_QCR0, QMU_RX_CS_EN(epnum)); + write_rxq_start_addr(mbase, epnum, ring->dma); + mtu3_setbits(mbase, MU3D_EP_RXCR0(epnum), RX_DMAREQEN); /* don't expect ZLP */ mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum)); /* move to next GPD when receive ZLP */ @@ -321,6 +389,9 @@ void mtu3_qmu_stop(struct mtu3_ep *mep) } mtu3_writel(mbase, qcsr, QMU_Q_STOP); + if (mep->is_in) + mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_FLUSHFIFO); + ret = readl_poll_timeout_atomic(mbase + qcsr, value, !(value & QMU_Q_ACTIVE), 1, 1000); if (ret) { @@ -328,6 +399,10 @@ void mtu3_qmu_stop(struct mtu3_ep *mep) return; } + /* flush fifo again to make sure the fifo is empty */ + if (mep->is_in) + mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_FLUSHFIFO); + dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name); } @@ -353,26 +428,25 @@ static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum) struct mtu3_gpd_ring *ring = &mep->gpd_ring; void __iomem *mbase = mtu->mac_base; struct qmu_gpd *gpd_current = NULL; - dma_addr_t gpd_dma = mtu3_readl(mbase, USB_QMU_TQCPR(epnum)); - struct usb_request *req = NULL; struct mtu3_request *mreq; + dma_addr_t cur_gpd_dma; u32 txcsr = 0; int ret; mreq = next_request(mep); - if (mreq && mreq->request.length == 0) - req = &mreq->request; - else + if (mreq && mreq->request.length != 0) return; - gpd_current = gpd_dma_to_virt(ring, gpd_dma); + cur_gpd_dma = read_txq_cur_addr(mbase, epnum); + gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma); - if (le16_to_cpu(gpd_current->buf_len) != 0) { + if (GPD_DATA_LEN(mtu, le32_to_cpu(gpd_current->dw3_info)) != 0) { dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum); return; } dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq); + trace_mtu3_zlp_exp_gpd(mep, gpd_current); mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN); @@ -383,11 +457,10 @@ static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum) return; } mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY); - + /* prevent reorder, make sure GPD's HWO is set last */ + mb(); /* by pass the current GDP */ - gpd_current->flag |= GPD_FLAGS_BPS; - gpd_current->chksum = qmu_calc_checksum((u8 *)gpd_current); - gpd_current->flag |= GPD_FLAGS_HWO; + gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO); /*enable DMAREQEN, switch back to QMU mode */ mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN); @@ -395,6 +468,37 @@ static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum) } /* + * when rx error happens (except zlperr), QMU will stop, and RQCPR saves + * the GPD encountered error, Done irq will arise after resuming QMU again. + */ +static void qmu_error_rx(struct mtu3 *mtu, u8 epnum) +{ + struct mtu3_ep *mep = mtu->out_eps + epnum; + struct mtu3_gpd_ring *ring = &mep->gpd_ring; + struct qmu_gpd *gpd_current = NULL; + struct mtu3_request *mreq; + dma_addr_t cur_gpd_dma; + + cur_gpd_dma = read_rxq_cur_addr(mtu->mac_base, epnum); + gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma); + + mreq = next_request(mep); + if (!mreq || mreq->gpd != gpd_current) { + dev_err(mtu->dev, "no correct RX req is found\n"); + return; + } + + mreq->request.status = -EAGAIN; + + /* by pass the current GDP */ + gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO); + mtu3_qmu_resume(mep); + + dev_dbg(mtu->dev, "%s EP%d, current=%p, req=%p\n", + __func__, epnum, gpd_current, mreq); +} + +/* * NOTE: request list maybe is already empty as following case: * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)--> * queue_tx --> process_tasklet(meanwhile, the second one is transferred, @@ -408,17 +512,18 @@ static void qmu_done_tx(struct mtu3 *mtu, u8 epnum) void __iomem *mbase = mtu->mac_base; struct qmu_gpd *gpd = ring->dequeue; struct qmu_gpd *gpd_current = NULL; - dma_addr_t gpd_dma = mtu3_readl(mbase, USB_QMU_TQCPR(epnum)); struct usb_request *request = NULL; struct mtu3_request *mreq; + dma_addr_t cur_gpd_dma; /*transfer phy address got from QMU register to virtual address */ - gpd_current = gpd_dma_to_virt(ring, gpd_dma); + cur_gpd_dma = read_txq_cur_addr(mbase, epnum); + gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma); dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n", __func__, epnum, gpd, gpd_current, ring->enqueue); - while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) { + while (gpd && gpd != gpd_current && !GET_GPD_HWO(gpd)) { mreq = next_request(mep); @@ -428,7 +533,8 @@ static void qmu_done_tx(struct mtu3 *mtu, u8 epnum) } request = &mreq->request; - request->actual = le16_to_cpu(gpd->buf_len); + request->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info)); + trace_mtu3_complete_gpd(mep, gpd); mtu3_req_complete(mep, request, 0); gpd = advance_deq_gpd(ring); @@ -446,16 +552,17 @@ static void qmu_done_rx(struct mtu3 *mtu, u8 epnum) void __iomem *mbase = mtu->mac_base; struct qmu_gpd *gpd = ring->dequeue; struct qmu_gpd *gpd_current = NULL; - dma_addr_t gpd_dma = mtu3_readl(mbase, USB_QMU_RQCPR(epnum)); struct usb_request *req = NULL; struct mtu3_request *mreq; + dma_addr_t cur_gpd_dma; - gpd_current = gpd_dma_to_virt(ring, gpd_dma); + cur_gpd_dma = read_rxq_cur_addr(mbase, epnum); + gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma); dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n", __func__, epnum, gpd, gpd_current, ring->enqueue); - while (gpd != gpd_current && !(gpd->flag & GPD_FLAGS_HWO)) { + while (gpd && gpd != gpd_current && !GET_GPD_HWO(gpd)) { mreq = next_request(mep); @@ -465,7 +572,8 @@ static void qmu_done_rx(struct mtu3 *mtu, u8 epnum) } req = &mreq->request; - req->actual = le16_to_cpu(gpd->buf_len); + req->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info)); + trace_mtu3_complete_gpd(mep, gpd); mtu3_req_complete(mep, req, 0); gpd = advance_deq_gpd(ring); @@ -495,14 +603,18 @@ static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status) if ((qmu_status & RXQ_CSERR_INT) || (qmu_status & RXQ_LENERR_INT)) { errval = mtu3_readl(mbase, U3D_RQERRIR0); + mtu3_writel(mbase, U3D_RQERRIR0, errval); + for (i = 1; i < mtu->num_eps; i++) { if (errval & QMU_RX_CS_ERR(i)) dev_err(mtu->dev, "Rx %d CS error!\n", i); if (errval & QMU_RX_LEN_ERR(i)) dev_err(mtu->dev, "RX %d Length error\n", i); + + if (errval & (QMU_RX_CS_ERR(i) | QMU_RX_LEN_ERR(i))) + qmu_error_rx(mtu, i); } - mtu3_writel(mbase, U3D_RQERRIR0, errval); } if (qmu_status & RXQ_ZLPERR_INT) { @@ -543,6 +655,7 @@ irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu) dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n", (qmu_done_status & 0xFFFF), qmu_done_status >> 16, qmu_status); + trace_mtu3_qmu_isr(qmu_done_status, qmu_status); if (qmu_done_status) qmu_done_isr(mtu, qmu_done_status); |
