diff options
Diffstat (limited to 'drivers/spi/spi-qcom-qspi.c')
| -rw-r--r-- | drivers/spi/spi-qcom-qspi.c | 349 |
1 files changed, 301 insertions, 48 deletions
diff --git a/drivers/spi/spi-qcom-qspi.c b/drivers/spi/spi-qcom-qspi.c index c334dfec4117..3aeddada58e1 100644 --- a/drivers/spi/spi-qcom-qspi.c +++ b/drivers/spi/spi-qcom-qspi.c @@ -2,12 +2,15 @@ // Copyright (c) 2017-2018, The Linux foundation. All rights reserved. #include <linux/clk.h> +#include <linux/dmapool.h> +#include <linux/dma-mapping.h> #include <linux/interconnect.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/module.h> #include <linux/of.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/pinctrl/consumer.h> #include <linux/pm_runtime.h> #include <linux/pm_opp.h> #include <linux/spi/spi.h> @@ -61,11 +64,12 @@ #define WR_FIFO_FULL BIT(10) #define WR_FIFO_OVERRUN BIT(11) #define TRANSACTION_DONE BIT(16) +#define DMA_CHAIN_DONE BIT(31) #define QSPI_ERR_IRQS (RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \ WR_FIFO_OVERRUN) #define QSPI_ALL_IRQS (QSPI_ERR_IRQS | RESP_FIFO_RDY | \ WR_FIFO_EMPTY | WR_FIFO_FULL | \ - TRANSACTION_DONE) + TRANSACTION_DONE | DMA_CHAIN_DONE) #define PIO_XFER_CTRL 0x0014 #define REQUEST_COUNT_MSK 0xffff @@ -107,18 +111,34 @@ #define RD_FIFO_RESET 0x0030 #define RESET_FIFO BIT(0) +#define NEXT_DMA_DESC_ADDR 0x0040 +#define CURRENT_DMA_DESC_ADDR 0x0044 +#define CURRENT_MEM_ADDR 0x0048 + #define CUR_MEM_ADDR 0x0048 #define HW_VERSION 0x004c #define RD_FIFO 0x0050 #define SAMPLING_CLK_CFG 0x0090 #define SAMPLING_CLK_STATUS 0x0094 +#define QSPI_ALIGN_REQ 32 enum qspi_dir { QSPI_READ, QSPI_WRITE, }; +struct qspi_cmd_desc { + u32 data_address; + u32 next_descriptor; + u32 direction:1; + u32 multi_io_mode:3; + u32 reserved1:4; + u32 fragment:1; + u32 reserved2:7; + u32 length:16; +}; + struct qspi_xfer { union { const void *tx_buf; @@ -136,11 +156,23 @@ enum qspi_clocks { QSPI_NUM_CLKS }; +/* + * Number of entries in sgt returned from spi framework that- + * will be supported. Can be modified as required. + * In practice, given max_dma_len is 64KB, the number of + * entries is not expected to exceed 1. + */ +#define QSPI_MAX_SG 5 + struct qcom_qspi { void __iomem *base; struct device *dev; struct clk_bulk_data *clks; struct qspi_xfer xfer; + struct dma_pool *dma_cmd_pool; + dma_addr_t dma_cmd_desc[QSPI_MAX_SG]; + void *virt_cmd_desc[QSPI_MAX_SG]; + unsigned int n_cmd_desc; struct icc_path *icc_path_cpu_to_qspi; unsigned long last_speed; /* Lock to protect data accessed by IRQs */ @@ -152,21 +184,22 @@ static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl, { switch (buswidth) { case 1: - return SDR_1BIT << MULTI_IO_MODE_SHFT; + return SDR_1BIT; case 2: - return SDR_2BIT << MULTI_IO_MODE_SHFT; + return SDR_2BIT; case 4: - return SDR_4BIT << MULTI_IO_MODE_SHFT; + return SDR_4BIT; default: dev_warn_once(ctrl->dev, "Unexpected bus width: %u\n", buswidth); - return SDR_1BIT << MULTI_IO_MODE_SHFT; + return SDR_1BIT; } } static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl) { u32 pio_xfer_cfg; + u32 iomode; const struct qspi_xfer *xfer; xfer = &ctrl->xfer; @@ -178,7 +211,8 @@ static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl) else pio_xfer_cfg |= TRANSFER_FRAGMENT; pio_xfer_cfg &= ~MULTI_IO_MODE_MSK; - pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth); + iomode = qspi_buswidth_to_iomode(ctrl, xfer->buswidth); + pio_xfer_cfg |= iomode << MULTI_IO_MODE_SHFT; writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG); } @@ -213,15 +247,25 @@ static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl) qcom_qspi_pio_xfer_ctrl(ctrl); } -static void qcom_qspi_handle_err(struct spi_master *master, +static void qcom_qspi_handle_err(struct spi_controller *host, struct spi_message *msg) { - struct qcom_qspi *ctrl = spi_master_get_devdata(master); + u32 int_status; + struct qcom_qspi *ctrl = spi_controller_get_devdata(host); unsigned long flags; + int i; spin_lock_irqsave(&ctrl->lock, flags); writel(0, ctrl->base + MSTR_INT_EN); + int_status = readl(ctrl->base + MSTR_INT_STATUS); + writel(int_status, ctrl->base + MSTR_INT_STATUS); ctrl->xfer.rem_bytes = 0; + + /* free cmd descriptors if they are around (DMA mode) */ + for (i = 0; i < ctrl->n_cmd_desc; i++) + dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i], + ctrl->dma_cmd_desc[i]); + ctrl->n_cmd_desc = 0; spin_unlock_irqrestore(&ctrl->lock, flags); } @@ -241,7 +285,7 @@ static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz) } /* - * Set BW quota for CPU as driver supports FIFO mode only. + * Set BW quota for CPU. * We don't have explicit peak requirement so keep it equal to avg_bw. */ avg_bw_cpu = Bps_to_icc(speed_hz); @@ -257,14 +301,125 @@ static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz) return 0; } -static int qcom_qspi_transfer_one(struct spi_master *master, +static int qcom_qspi_alloc_desc(struct qcom_qspi *ctrl, dma_addr_t dma_ptr, + uint32_t n_bytes) +{ + struct qspi_cmd_desc *virt_cmd_desc, *prev; + dma_addr_t dma_cmd_desc; + + /* allocate for dma cmd descriptor */ + virt_cmd_desc = dma_pool_alloc(ctrl->dma_cmd_pool, GFP_ATOMIC | __GFP_ZERO, &dma_cmd_desc); + if (!virt_cmd_desc) { + dev_warn_once(ctrl->dev, "Couldn't find memory for descriptor\n"); + return -EAGAIN; + } + + ctrl->virt_cmd_desc[ctrl->n_cmd_desc] = virt_cmd_desc; + ctrl->dma_cmd_desc[ctrl->n_cmd_desc] = dma_cmd_desc; + ctrl->n_cmd_desc++; + + /* setup cmd descriptor */ + virt_cmd_desc->data_address = dma_ptr; + virt_cmd_desc->direction = ctrl->xfer.dir; + virt_cmd_desc->multi_io_mode = qspi_buswidth_to_iomode(ctrl, ctrl->xfer.buswidth); + virt_cmd_desc->fragment = !ctrl->xfer.is_last; + virt_cmd_desc->length = n_bytes; + + /* update previous descriptor */ + if (ctrl->n_cmd_desc >= 2) { + prev = (ctrl->virt_cmd_desc)[ctrl->n_cmd_desc - 2]; + prev->next_descriptor = dma_cmd_desc; + prev->fragment = 1; + } + + return 0; +} + +static int qcom_qspi_setup_dma_desc(struct qcom_qspi *ctrl, + struct spi_transfer *xfer) +{ + int ret; + struct sg_table *sgt; + dma_addr_t dma_ptr_sg; + unsigned int dma_len_sg; + int i; + + if (ctrl->n_cmd_desc) { + dev_err(ctrl->dev, "Remnant dma buffers n_cmd_desc-%d\n", ctrl->n_cmd_desc); + return -EIO; + } + + sgt = (ctrl->xfer.dir == QSPI_READ) ? &xfer->rx_sg : &xfer->tx_sg; + if (!sgt->nents || sgt->nents > QSPI_MAX_SG) { + dev_warn_once(ctrl->dev, "Cannot handle %d entries in scatter list\n", sgt->nents); + return -EAGAIN; + } + + for (i = 0; i < sgt->nents; i++) { + dma_ptr_sg = sg_dma_address(sgt->sgl + i); + dma_len_sg = sg_dma_len(sgt->sgl + i); + if (!IS_ALIGNED(dma_ptr_sg, QSPI_ALIGN_REQ)) { + dev_warn_once(ctrl->dev, "dma_address not aligned to %d\n", QSPI_ALIGN_REQ); + return -EAGAIN; + } + /* + * When reading with DMA the controller writes to memory 1 word + * at a time. If the length isn't a multiple of 4 bytes then + * the controller can clobber the things later in memory. + * Fallback to PIO to be safe. + */ + if (ctrl->xfer.dir == QSPI_READ && (dma_len_sg & 0x03)) { + dev_warn_once(ctrl->dev, "fallback to PIO for read of size %#010x\n", + dma_len_sg); + return -EAGAIN; + } + } + + for (i = 0; i < sgt->nents; i++) { + dma_ptr_sg = sg_dma_address(sgt->sgl + i); + dma_len_sg = sg_dma_len(sgt->sgl + i); + + ret = qcom_qspi_alloc_desc(ctrl, dma_ptr_sg, dma_len_sg); + if (ret) + goto cleanup; + } + return 0; + +cleanup: + for (i = 0; i < ctrl->n_cmd_desc; i++) + dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i], + ctrl->dma_cmd_desc[i]); + ctrl->n_cmd_desc = 0; + return ret; +} + +static void qcom_qspi_dma_xfer(struct qcom_qspi *ctrl) +{ + /* Setup new interrupts */ + writel(DMA_CHAIN_DONE, ctrl->base + MSTR_INT_EN); + + /* kick off transfer */ + writel((u32)((ctrl->dma_cmd_desc)[0]), ctrl->base + NEXT_DMA_DESC_ADDR); +} + +/* Switch to DMA if transfer length exceeds this */ +#define QSPI_MAX_BYTES_FIFO 64 + +static bool qcom_qspi_can_dma(struct spi_controller *ctlr, + struct spi_device *slv, struct spi_transfer *xfer) +{ + return xfer->len > QSPI_MAX_BYTES_FIFO; +} + +static int qcom_qspi_transfer_one(struct spi_controller *host, struct spi_device *slv, struct spi_transfer *xfer) { - struct qcom_qspi *ctrl = spi_master_get_devdata(master); + struct qcom_qspi *ctrl = spi_controller_get_devdata(host); int ret; unsigned long speed_hz; unsigned long flags; + u32 mstr_cfg; speed_hz = slv->max_speed_hz; if (xfer->speed_hz) @@ -275,6 +430,7 @@ static int qcom_qspi_transfer_one(struct spi_master *master, return ret; spin_lock_irqsave(&ctrl->lock, flags); + mstr_cfg = readl(ctrl->base + MSTR_CONFIG); /* We are half duplex, so either rx or tx will be set */ if (xfer->rx_buf) { @@ -287,17 +443,45 @@ static int qcom_qspi_transfer_one(struct spi_master *master, ctrl->xfer.tx_buf = xfer->tx_buf; } ctrl->xfer.is_last = list_is_last(&xfer->transfer_list, - &master->cur_msg->transfers); + &host->cur_msg->transfers); ctrl->xfer.rem_bytes = xfer->len; + + if (xfer->rx_sg.nents || xfer->tx_sg.nents) { + /* do DMA transfer */ + if (!(mstr_cfg & DMA_ENABLE)) { + mstr_cfg |= DMA_ENABLE; + writel(mstr_cfg, ctrl->base + MSTR_CONFIG); + } + + ret = qcom_qspi_setup_dma_desc(ctrl, xfer); + if (ret != -EAGAIN) { + if (!ret) { + dma_wmb(); + qcom_qspi_dma_xfer(ctrl); + } + goto exit; + } + dev_warn_once(ctrl->dev, "DMA failure, falling back to PIO\n"); + ret = 0; /* We'll retry w/ PIO */ + } + + if (mstr_cfg & DMA_ENABLE) { + mstr_cfg &= ~DMA_ENABLE; + writel(mstr_cfg, ctrl->base + MSTR_CONFIG); + } qcom_qspi_pio_xfer(ctrl); +exit: spin_unlock_irqrestore(&ctrl->lock, flags); + if (ret) + return ret; + /* We'll call spi_finalize_current_transfer() when done */ return 1; } -static int qcom_qspi_prepare_message(struct spi_master *master, +static int qcom_qspi_prepare_message(struct spi_controller *host, struct spi_message *message) { u32 mstr_cfg; @@ -306,12 +490,12 @@ static int qcom_qspi_prepare_message(struct spi_master *master, int tx_data_delay = 1; unsigned long flags; - ctrl = spi_master_get_devdata(master); + ctrl = spi_controller_get_devdata(host); spin_lock_irqsave(&ctrl->lock, flags); mstr_cfg = readl(ctrl->base + MSTR_CONFIG); mstr_cfg &= ~CHIP_SELECT_NUM; - if (message->spi->chip_select) + if (spi_get_chipselect(message->spi, 0)) mstr_cfg |= CHIP_SELECT_NUM; mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE; @@ -327,6 +511,16 @@ static int qcom_qspi_prepare_message(struct spi_master *master, return 0; } +static int qcom_qspi_alloc_dma(struct qcom_qspi *ctrl) +{ + ctrl->dma_cmd_pool = dmam_pool_create("qspi cmd desc pool", + ctrl->dev, sizeof(struct qspi_cmd_desc), 0, 0); + if (!ctrl->dma_cmd_pool) + return -ENOMEM; + + return 0; +} + static irqreturn_t pio_read(struct qcom_qspi *ctrl) { u32 rd_fifo_status; @@ -425,6 +619,10 @@ static irqreturn_t qcom_qspi_irq(int irq, void *dev_id) int_status = readl(ctrl->base + MSTR_INT_STATUS); writel(int_status, ctrl->base + MSTR_INT_STATUS); + /* Ignore disabled interrupts */ + int_status &= readl(ctrl->base + MSTR_INT_EN); + + /* PIO mode handling */ if (ctrl->xfer.dir == QSPI_WRITE) { if (int_status & WR_FIFO_EMPTY) ret = pio_write(ctrl); @@ -448,26 +646,66 @@ static irqreturn_t qcom_qspi_irq(int irq, void *dev_id) spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev)); } + /* DMA mode handling */ + if (int_status & DMA_CHAIN_DONE) { + int i; + + writel(0, ctrl->base + MSTR_INT_EN); + ctrl->xfer.rem_bytes = 0; + + for (i = 0; i < ctrl->n_cmd_desc; i++) + dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i], + ctrl->dma_cmd_desc[i]); + ctrl->n_cmd_desc = 0; + + ret = IRQ_HANDLED; + spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev)); + } + spin_unlock(&ctrl->lock); return ret; } +static int qcom_qspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op) +{ + /* + * If qcom_qspi_can_dma() is going to return false we don't need to + * adjust anything. + */ + if (op->data.nbytes <= QSPI_MAX_BYTES_FIFO) + return 0; + + /* + * When reading, the transfer needs to be a multiple of 4 bytes so + * shrink the transfer if that's not true. The caller will then do a + * second transfer to finish things up. + */ + if (op->data.dir == SPI_MEM_DATA_IN && (op->data.nbytes & 0x3)) + op->data.nbytes &= ~0x3; + + return 0; +} + +static const struct spi_controller_mem_ops qcom_qspi_mem_ops = { + .adjust_op_size = qcom_qspi_adjust_op_size, +}; + static int qcom_qspi_probe(struct platform_device *pdev) { int ret; struct device *dev; - struct spi_master *master; + struct spi_controller *host; struct qcom_qspi *ctrl; dev = &pdev->dev; - master = devm_spi_alloc_master(dev, sizeof(*ctrl)); - if (!master) + host = devm_spi_alloc_host(dev, sizeof(*ctrl)); + if (!host) return -ENOMEM; - platform_set_drvdata(pdev, master); + platform_set_drvdata(pdev, host); - ctrl = spi_master_get_devdata(master); + ctrl = spi_controller_get_devdata(host); spin_lock_init(&ctrl->lock); ctrl->dev = dev; @@ -516,18 +754,27 @@ static int qcom_qspi_probe(struct platform_device *pdev) return ret; } - master->max_speed_hz = 300000000; - master->num_chipselect = QSPI_NUM_CS; - master->bus_num = -1; - master->dev.of_node = pdev->dev.of_node; - master->mode_bits = SPI_MODE_0 | - SPI_TX_DUAL | SPI_RX_DUAL | - SPI_TX_QUAD | SPI_RX_QUAD; - master->flags = SPI_MASTER_HALF_DUPLEX; - master->prepare_message = qcom_qspi_prepare_message; - master->transfer_one = qcom_qspi_transfer_one; - master->handle_err = qcom_qspi_handle_err; - master->auto_runtime_pm = true; + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (ret) + return dev_err_probe(dev, ret, "could not set DMA mask\n"); + + host->max_speed_hz = 300000000; + host->max_dma_len = 65536; /* as per HPG */ + host->dma_alignment = QSPI_ALIGN_REQ; + host->num_chipselect = QSPI_NUM_CS; + host->bus_num = -1; + host->dev.of_node = pdev->dev.of_node; + host->mode_bits = SPI_MODE_0 | + SPI_TX_DUAL | SPI_RX_DUAL | + SPI_TX_QUAD | SPI_RX_QUAD; + host->flags = SPI_CONTROLLER_HALF_DUPLEX; + host->prepare_message = qcom_qspi_prepare_message; + host->transfer_one = qcom_qspi_transfer_one; + host->handle_err = qcom_qspi_handle_err; + if (of_property_present(pdev->dev.of_node, "iommus")) + host->can_dma = qcom_qspi_can_dma; + host->auto_runtime_pm = true; + host->mem_ops = &qcom_qspi_mem_ops; ret = devm_pm_opp_set_clkname(&pdev->dev, "core"); if (ret) @@ -539,11 +786,15 @@ static int qcom_qspi_probe(struct platform_device *pdev) return ret; } + ret = qcom_qspi_alloc_dma(ctrl); + if (ret) + return ret; + pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, 250); pm_runtime_enable(dev); - ret = spi_register_master(master); + ret = spi_register_controller(host); if (!ret) return 0; @@ -552,22 +803,20 @@ static int qcom_qspi_probe(struct platform_device *pdev) return ret; } -static int qcom_qspi_remove(struct platform_device *pdev) +static void qcom_qspi_remove(struct platform_device *pdev) { - struct spi_master *master = platform_get_drvdata(pdev); + struct spi_controller *host = platform_get_drvdata(pdev); /* Unregister _before_ disabling pm_runtime() so we stop transfers */ - spi_unregister_master(master); + spi_unregister_controller(host); pm_runtime_disable(&pdev->dev); - - return 0; } static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev) { - struct spi_master *master = dev_get_drvdata(dev); - struct qcom_qspi *ctrl = spi_master_get_devdata(master); + struct spi_controller *host = dev_get_drvdata(dev); + struct qcom_qspi *ctrl = spi_controller_get_devdata(host); int ret; /* Drop the performance state vote */ @@ -581,15 +830,19 @@ static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev) return ret; } + pinctrl_pm_select_sleep_state(dev); + return 0; } static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev) { - struct spi_master *master = dev_get_drvdata(dev); - struct qcom_qspi *ctrl = spi_master_get_devdata(master); + struct spi_controller *host = dev_get_drvdata(dev); + struct qcom_qspi *ctrl = spi_controller_get_devdata(host); int ret; + pinctrl_pm_select_default_state(dev); + ret = icc_enable(ctrl->icc_path_cpu_to_qspi); if (ret) { dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n", @@ -606,30 +859,30 @@ static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev) static int __maybe_unused qcom_qspi_suspend(struct device *dev) { - struct spi_master *master = dev_get_drvdata(dev); + struct spi_controller *host = dev_get_drvdata(dev); int ret; - ret = spi_master_suspend(master); + ret = spi_controller_suspend(host); if (ret) return ret; ret = pm_runtime_force_suspend(dev); if (ret) - spi_master_resume(master); + spi_controller_resume(host); return ret; } static int __maybe_unused qcom_qspi_resume(struct device *dev) { - struct spi_master *master = dev_get_drvdata(dev); + struct spi_controller *host = dev_get_drvdata(dev); int ret; ret = pm_runtime_force_resume(dev); if (ret) return ret; - ret = spi_master_resume(master); + ret = spi_controller_resume(host); if (ret) pm_runtime_force_suspend(dev); |