diff options
Diffstat (limited to 'drivers/mmc/host/renesas_sdhi_internal_dmac.c')
| -rw-r--r-- | drivers/mmc/host/renesas_sdhi_internal_dmac.c | 496 |
1 files changed, 378 insertions, 118 deletions
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c index 92c9b15252da..f6ebb7bc7ede 100644 --- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c +++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c @@ -2,19 +2,22 @@ /* * DMA support for Internal DMAC with SDHI SD/SDIO controller * - * Copyright (C) 2016-17 Renesas Electronics Corporation + * Copyright (C) 2016-19 Renesas Electronics Corporation * Copyright (C) 2016-17 Horms Solutions, Simon Horman + * Copyright (C) 2018-19 Sang Engineering, Wolfram Sang */ #include <linux/bitops.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/io-64-nonatomic-hi-lo.h> -#include <linux/mfd/tmio.h> #include <linux/mmc/host.h> #include <linux/mod_devicetable.h> #include <linux/module.h> +#include <linux/of.h> #include <linux/pagemap.h> +#include <linux/platform_data/tmio.h> +#include <linux/platform_device.h> #include <linux/scatterlist.h> #include <linux/sys_soc.h> @@ -45,9 +48,9 @@ #define RST_RESERVED_BITS GENMASK_ULL(31, 0) /* DM_CM_INFO1 and DM_CM_INFO1_MASK */ -#define INFO1_CLEAR 0 #define INFO1_MASK_CLEAR GENMASK_ULL(31, 0) -#define INFO1_DTRANEND1 BIT(17) +#define INFO1_DTRANEND1 BIT(20) +#define INFO1_DTRANEND1_OLD BIT(17) #define INFO1_DTRANEND0 BIT(16) /* DM_CM_INFO2 and DM_CM_INFO2_MASK */ @@ -55,6 +58,12 @@ #define INFO2_DTRANERR1 BIT(17) #define INFO2_DTRANERR0 BIT(16) +enum renesas_sdhi_dma_cookie { + COOKIE_UNMAPPED, + COOKIE_PRE_MAPPED, + COOKIE_MAPPED, +}; + /* * Specification of this driver: * - host->chan_{rx,tx} will be used as a flag of enabling/disabling the dma @@ -64,178 +73,403 @@ static unsigned long global_flags; /* - * Workaround for avoiding to use RX DMAC by multiple channels. - * On R-Car H3 ES1.* and M3-W ES1.0, when multiple SDHI channels use - * RX DMAC simultaneously, sometimes hundreds of bytes data are not - * stored into the system memory even if the DMAC interrupt happened. - * So, this driver then uses one RX DMAC channel only. + * Workaround for avoiding to use RX DMAC by multiple channels. On R-Car M3-W + * ES1.0, when multiple SDHI channels use RX DMAC simultaneously, sometimes + * hundreds of data bytes are not stored into the system memory even if the + * DMAC interrupt happened. So, this driver then uses one RX DMAC channel only. */ -#define SDHI_INTERNAL_DMAC_ONE_RX_ONLY 0 -#define SDHI_INTERNAL_DMAC_RX_IN_USE 1 - -/* RZ/A2 does not have the ADRR_MODE bit */ -#define SDHI_INTERNAL_DMAC_ADDR_MODE_FIXED_ONLY 2 +#define SDHI_INTERNAL_DMAC_RX_IN_USE 0 /* Definitions for sampling clocks */ static struct renesas_sdhi_scc rcar_gen3_scc_taps[] = { { .clk_rate = 0, .tap = 0x00000300, + .tap_hs400_4tap = 0x00000100, }, }; -static const struct renesas_sdhi_of_data of_rza2_compatible = { +static const struct renesas_sdhi_of_data of_data_rza2 = { .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | TMIO_MMC_HAVE_CBSY, .tmio_ocr_mask = MMC_VDD_32_33, .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | - MMC_CAP_CMD23, + MMC_CAP_CMD23 | MMC_CAP_WAIT_WHILE_BUSY, .bus_shift = 2, .scc_offset = 0 - 0x1000, .taps = rcar_gen3_scc_taps, .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), - /* DMAC can handle 0xffffffff blk count but only 1 segment */ - .max_blk_count = 0xffffffff, + /* DMAC can handle 32bit blk count but only 1 segment */ + .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, + .max_segs = 1, +}; + +static const struct renesas_sdhi_of_data of_data_rcar_gen3 = { + .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | + TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2 | + TMIO_MMC_64BIT_DATA_PORT, + .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | + MMC_CAP_CMD23 | MMC_CAP_WAIT_WHILE_BUSY, + .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT | MMC_CAP2_MERGE_CAPABLE, + .bus_shift = 2, + .scc_offset = 0x1000, + .taps = rcar_gen3_scc_taps, + .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), + /* DMAC can handle 32bit blk count but only 1 segment */ + .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, .max_segs = 1, + .sdhi_flags = SDHI_FLAG_NEED_CLKH_FALLBACK, }; -static const struct renesas_sdhi_of_data of_rcar_gen3_compatible = { +static const struct renesas_sdhi_of_data of_data_rcar_gen3_no_sdh_fallback = { .tmio_flags = TMIO_MMC_HAS_IDLE_WAIT | TMIO_MMC_CLK_ACTUAL | - TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2, + TMIO_MMC_HAVE_CBSY | TMIO_MMC_MIN_RCAR2 | + TMIO_MMC_64BIT_DATA_PORT, .capabilities = MMC_CAP_SD_HIGHSPEED | MMC_CAP_SDIO_IRQ | - MMC_CAP_CMD23, - .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT, + MMC_CAP_CMD23 | MMC_CAP_WAIT_WHILE_BUSY, + .capabilities2 = MMC_CAP2_NO_WRITE_PROTECT | MMC_CAP2_MERGE_CAPABLE, .bus_shift = 2, .scc_offset = 0x1000, .taps = rcar_gen3_scc_taps, .taps_num = ARRAY_SIZE(rcar_gen3_scc_taps), - /* DMAC can handle 0xffffffff blk count but only 1 segment */ - .max_blk_count = 0xffffffff, + /* DMAC can handle 32bit blk count but only 1 segment */ + .max_blk_count = UINT_MAX / TMIO_MAX_BLK_SIZE, .max_segs = 1, }; +static const u8 r8a7796_es13_calib_table[2][SDHI_CALIB_TABLE_MAX] = { + { 3, 3, 3, 3, 3, 3, 3, 4, 4, 5, 6, 7, 8, 9, 10, 15, + 16, 16, 16, 16, 16, 16, 17, 18, 18, 19, 20, 21, 22, 23, 24, 25 }, + { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 7, 8, 11, + 12, 17, 18, 18, 18, 18, 18, 18, 18, 19, 20, 21, 22, 23, 25, 25 } +}; + +static const u8 r8a77965_calib_table[2][SDHI_CALIB_TABLE_MAX] = { + { 1, 2, 6, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25, 25, 26, 27, 28, 29, 30, 31 }, + { 2, 3, 4, 4, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 17, 17, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 31, 31, 31 } +}; + +static const u8 r8a77990_calib_table[2][SDHI_CALIB_TABLE_MAX] = { + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0, 0, 1, 2, 3, 3, 4, 4, 4, 5, 5, 6, 8, 9, 10, + 11, 12, 13, 15, 16, 17, 17, 18, 18, 19, 20, 22, 24, 25, 26, 26 } +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_4tap_nohs400 = { + .hs400_disabled = true, + .hs400_4taps = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_4tap_nohs400_one_rx = { + .hs400_disabled = true, + .hs400_4taps = true, + .dma_one_rx_only = true, + .old_info1_layout = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_4tap = { + .hs400_4taps = true, + .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), + .manual_tap_correction = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_nohs400 = { + .hs400_disabled = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_fixed_addr = { + .fixed_addr_mode = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_bad_taps1357 = { + .hs400_bad_taps = BIT(1) | BIT(3) | BIT(5) | BIT(7), + .manual_tap_correction = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_bad_taps2367 = { + .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), + .manual_tap_correction = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_r8a7796_es13 = { + .hs400_4taps = true, + .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), + .hs400_calib_table = r8a7796_es13_calib_table, + .manual_tap_correction = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_r8a77965 = { + .hs400_bad_taps = BIT(2) | BIT(3) | BIT(6) | BIT(7), + .hs400_calib_table = r8a77965_calib_table, + .manual_tap_correction = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_r8a77990 = { + .hs400_calib_table = r8a77990_calib_table, + .manual_tap_correction = true, +}; + +static const struct renesas_sdhi_quirks sdhi_quirks_rzg2l = { + .fixed_addr_mode = true, + .hs400_disabled = true, +}; + +/* + * Note for r8a7796 / r8a774a1: we can't distinguish ES1.1 and 1.2 as of now. + * So, we want to treat them equally and only have a match for ES1.2 to enforce + * this if there ever will be a way to distinguish ES1.2. + */ +static const struct soc_device_attribute sdhi_quirks_match[] = { + { .soc_id = "r8a774a1", .revision = "ES1.[012]", .data = &sdhi_quirks_4tap_nohs400 }, + { .soc_id = "r8a7795", .revision = "ES2.0", .data = &sdhi_quirks_4tap }, + { .soc_id = "r8a7796", .revision = "ES1.0", .data = &sdhi_quirks_4tap_nohs400_one_rx }, + { .soc_id = "r8a7796", .revision = "ES1.[12]", .data = &sdhi_quirks_4tap_nohs400 }, + { .soc_id = "r8a7796", .revision = "ES1.*", .data = &sdhi_quirks_r8a7796_es13 }, + { .soc_id = "r8a77980", .revision = "ES1.*", .data = &sdhi_quirks_nohs400 }, + { /* Sentinel. */ } +}; + +static const struct renesas_sdhi_of_data_with_quirks of_r8a7795_compatible = { + .of_data = &of_data_rcar_gen3, + .quirks = &sdhi_quirks_bad_taps2367, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_r8a77961_compatible = { + .of_data = &of_data_rcar_gen3, + .quirks = &sdhi_quirks_bad_taps1357, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_r8a77965_compatible = { + .of_data = &of_data_rcar_gen3, + .quirks = &sdhi_quirks_r8a77965, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_r8a77970_compatible = { + .of_data = &of_data_rcar_gen3_no_sdh_fallback, + .quirks = &sdhi_quirks_nohs400, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_r8a77990_compatible = { + .of_data = &of_data_rcar_gen3, + .quirks = &sdhi_quirks_r8a77990, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_rzg2l_compatible = { + .of_data = &of_data_rcar_gen3, + .quirks = &sdhi_quirks_rzg2l, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_rcar_gen3_compatible = { + .of_data = &of_data_rcar_gen3, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_rcar_gen3_nohs400_compatible = { + .of_data = &of_data_rcar_gen3, + .quirks = &sdhi_quirks_nohs400, +}; + +static const struct renesas_sdhi_of_data_with_quirks of_rza2_compatible = { + .of_data = &of_data_rza2, + .quirks = &sdhi_quirks_fixed_addr, +}; + static const struct of_device_id renesas_sdhi_internal_dmac_of_match[] = { { .compatible = "renesas,sdhi-r7s9210", .data = &of_rza2_compatible, }, { .compatible = "renesas,sdhi-mmc-r8a77470", .data = &of_rcar_gen3_compatible, }, - { .compatible = "renesas,sdhi-r8a7795", .data = &of_rcar_gen3_compatible, }, - { .compatible = "renesas,sdhi-r8a7796", .data = &of_rcar_gen3_compatible, }, + { .compatible = "renesas,sdhi-r8a7795", .data = &of_r8a7795_compatible, }, + { .compatible = "renesas,sdhi-r8a77961", .data = &of_r8a77961_compatible, }, + { .compatible = "renesas,sdhi-r8a77965", .data = &of_r8a77965_compatible, }, + { .compatible = "renesas,sdhi-r8a77970", .data = &of_r8a77970_compatible, }, + { .compatible = "renesas,sdhi-r8a77990", .data = &of_r8a77990_compatible, }, + { .compatible = "renesas,sdhi-r8a77995", .data = &of_rcar_gen3_nohs400_compatible, }, + { .compatible = "renesas,sdhi-r9a09g011", .data = &of_rzg2l_compatible, }, + { .compatible = "renesas,sdhi-r9a09g057", .data = &of_rzg2l_compatible, }, + { .compatible = "renesas,rzg2l-sdhi", .data = &of_rzg2l_compatible, }, { .compatible = "renesas,rcar-gen3-sdhi", .data = &of_rcar_gen3_compatible, }, + { .compatible = "renesas,rcar-gen4-sdhi", .data = &of_rcar_gen3_compatible, }, {}, }; MODULE_DEVICE_TABLE(of, renesas_sdhi_internal_dmac_of_match); static void -renesas_sdhi_internal_dmac_dm_write(struct tmio_mmc_host *host, - int addr, u64 val) -{ - writeq(val, host->ctl + addr); -} - -static void renesas_sdhi_internal_dmac_enable_dma(struct tmio_mmc_host *host, bool enable) { struct renesas_sdhi *priv = host_to_priv(host); + u32 dma_irqs = INFO1_DTRANEND0 | + (sdhi_has_quirk(priv, old_info1_layout) ? + INFO1_DTRANEND1_OLD : INFO1_DTRANEND1); if (!host->chan_tx || !host->chan_rx) return; - if (!enable) - renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO1, - INFO1_CLEAR); + writel(enable ? ~dma_irqs : INFO1_MASK_CLEAR, host->ctl + DM_CM_INFO1_MASK); if (priv->dma_priv.enable) priv->dma_priv.enable(host, enable); } static void -renesas_sdhi_internal_dmac_abort_dma(struct tmio_mmc_host *host) { +renesas_sdhi_internal_dmac_abort_dma(struct tmio_mmc_host *host) +{ u64 val = RST_DTRANRST1 | RST_DTRANRST0; renesas_sdhi_internal_dmac_enable_dma(host, false); - renesas_sdhi_internal_dmac_dm_write(host, DM_CM_RST, - RST_RESERVED_BITS & ~val); - renesas_sdhi_internal_dmac_dm_write(host, DM_CM_RST, - RST_RESERVED_BITS | val); + writel(RST_RESERVED_BITS & ~val, host->ctl + DM_CM_RST); + writel(RST_RESERVED_BITS | val, host->ctl + DM_CM_RST); clear_bit(SDHI_INTERNAL_DMAC_RX_IN_USE, &global_flags); renesas_sdhi_internal_dmac_enable_dma(host, true); } +static bool renesas_sdhi_internal_dmac_dma_irq(struct tmio_mmc_host *host) +{ + struct renesas_sdhi *priv = host_to_priv(host); + struct renesas_sdhi_dma *dma_priv = &priv->dma_priv; + + u32 dma_irqs = INFO1_DTRANEND0 | + (sdhi_has_quirk(priv, old_info1_layout) ? + INFO1_DTRANEND1_OLD : INFO1_DTRANEND1); + u32 status = readl(host->ctl + DM_CM_INFO1); + + if (status & dma_irqs) { + writel(status ^ dma_irqs, host->ctl + DM_CM_INFO1); + set_bit(SDHI_DMA_END_FLAG_DMA, &dma_priv->end_flags); + if (test_bit(SDHI_DMA_END_FLAG_ACCESS, &dma_priv->end_flags)) + queue_work(system_bh_wq, &dma_priv->dma_complete); + } + + return status & dma_irqs; +} + static void -renesas_sdhi_internal_dmac_dataend_dma(struct tmio_mmc_host *host) { +renesas_sdhi_internal_dmac_dataend_dma(struct tmio_mmc_host *host) +{ struct renesas_sdhi *priv = host_to_priv(host); + struct renesas_sdhi_dma *dma_priv = &priv->dma_priv; + + set_bit(SDHI_DMA_END_FLAG_ACCESS, &dma_priv->end_flags); + if (test_bit(SDHI_DMA_END_FLAG_DMA, &dma_priv->end_flags) || + host->data->error) + queue_work(system_bh_wq, &dma_priv->dma_complete); +} + +/* + * renesas_sdhi_internal_dmac_map() will be called with two different + * sg pointers in two mmc_data by .pre_req(), but tmio host can have a single + * sg_ptr only. So, renesas_sdhi_internal_dmac_{un}map() should use a sg + * pointer in a mmc_data instead of host->sg_ptr. + */ +static void +renesas_sdhi_internal_dmac_unmap(struct tmio_mmc_host *host, + struct mmc_data *data, + enum renesas_sdhi_dma_cookie cookie) +{ + bool unmap = cookie == COOKIE_UNMAPPED ? (data->host_cookie != cookie) : + (data->host_cookie == cookie); + + if (unmap) { + dma_unmap_sg(&host->pdev->dev, data->sg, data->sg_len, + mmc_get_dma_dir(data)); + data->host_cookie = COOKIE_UNMAPPED; + } +} + +static bool +renesas_sdhi_internal_dmac_map(struct tmio_mmc_host *host, + struct mmc_data *data, + enum renesas_sdhi_dma_cookie cookie) +{ + if (data->host_cookie == COOKIE_PRE_MAPPED) + return true; + + if (!dma_map_sg(&host->pdev->dev, data->sg, data->sg_len, + mmc_get_dma_dir(data))) + return false; - tasklet_schedule(&priv->dma_priv.dma_complete); + data->host_cookie = cookie; + + /* This DMAC needs buffers to be 128-byte aligned */ + if (!IS_ALIGNED(sg_dma_address(data->sg), 128)) { + renesas_sdhi_internal_dmac_unmap(host, data, cookie); + return false; + } + + return true; } static void renesas_sdhi_internal_dmac_start_dma(struct tmio_mmc_host *host, struct mmc_data *data) { + struct renesas_sdhi *priv = host_to_priv(host); struct scatterlist *sg = host->sg_ptr; u32 dtran_mode = DTRAN_MODE_BUS_WIDTH; - if (!test_bit(SDHI_INTERNAL_DMAC_ADDR_MODE_FIXED_ONLY, &global_flags)) + if (!sdhi_has_quirk(priv, fixed_addr_mode)) dtran_mode |= DTRAN_MODE_ADDR_MODE; - if (!dma_map_sg(&host->pdev->dev, sg, host->sg_len, - mmc_get_dma_dir(data))) + if (!renesas_sdhi_internal_dmac_map(host, data, COOKIE_MAPPED)) goto force_pio; - /* This DMAC cannot handle if buffer is not 8-bytes alignment */ - if (!IS_ALIGNED(sg_dma_address(sg), 8)) - goto force_pio_with_unmap; - if (data->flags & MMC_DATA_READ) { dtran_mode |= DTRAN_MODE_CH_NUM_CH1; - if (test_bit(SDHI_INTERNAL_DMAC_ONE_RX_ONLY, &global_flags) && + if (sdhi_has_quirk(priv, dma_one_rx_only) && test_and_set_bit(SDHI_INTERNAL_DMAC_RX_IN_USE, &global_flags)) goto force_pio_with_unmap; } else { dtran_mode |= DTRAN_MODE_CH_NUM_CH0; } + priv->dma_priv.end_flags = 0; renesas_sdhi_internal_dmac_enable_dma(host, true); /* set dma parameters */ - renesas_sdhi_internal_dmac_dm_write(host, DM_CM_DTRAN_MODE, - dtran_mode); - renesas_sdhi_internal_dmac_dm_write(host, DM_DTRAN_ADDR, - sg_dma_address(sg)); + writel(dtran_mode, host->ctl + DM_CM_DTRAN_MODE); + writel(sg_dma_address(sg), host->ctl + DM_DTRAN_ADDR); host->dma_on = true; return; force_pio_with_unmap: - dma_unmap_sg(&host->pdev->dev, sg, host->sg_len, mmc_get_dma_dir(data)); + renesas_sdhi_internal_dmac_unmap(host, data, COOKIE_UNMAPPED); force_pio: renesas_sdhi_internal_dmac_enable_dma(host, false); } -static void renesas_sdhi_internal_dmac_issue_tasklet_fn(unsigned long arg) +static void renesas_sdhi_internal_dmac_issue_work_fn(struct work_struct *work) { - struct tmio_mmc_host *host = (struct tmio_mmc_host *)arg; + struct tmio_mmc_host *host = from_work(host, work, dma_issue); + struct renesas_sdhi *priv = host_to_priv(host); tmio_mmc_enable_mmc_irqs(host, TMIO_STAT_DATAEND); - /* start the DMAC */ - renesas_sdhi_internal_dmac_dm_write(host, DM_CM_DTRAN_CTRL, - DTRAN_CTRL_DM_START); + if (!host->cmd->error) { + /* start the DMAC */ + writel(DTRAN_CTRL_DM_START, host->ctl + DM_CM_DTRAN_CTRL); + } else { + /* on CMD errors, simulate DMA end immediately */ + set_bit(SDHI_DMA_END_FLAG_DMA, &priv->dma_priv.end_flags); + if (test_bit(SDHI_DMA_END_FLAG_ACCESS, &priv->dma_priv.end_flags)) + queue_work(system_bh_wq, &priv->dma_priv.dma_complete); + } } -static void renesas_sdhi_internal_dmac_complete_tasklet_fn(unsigned long arg) +static bool renesas_sdhi_internal_dmac_complete(struct tmio_mmc_host *host) { - struct tmio_mmc_host *host = (struct tmio_mmc_host *)arg; enum dma_data_direction dir; - spin_lock_irq(&host->lock); + if (!host->dma_on) + return false; if (!host->data) - goto out; + return false; if (host->data->flags & MMC_DATA_READ) dir = DMA_FROM_DEVICE; @@ -243,37 +477,86 @@ static void renesas_sdhi_internal_dmac_complete_tasklet_fn(unsigned long arg) dir = DMA_TO_DEVICE; renesas_sdhi_internal_dmac_enable_dma(host, false); - dma_unmap_sg(&host->pdev->dev, host->sg_ptr, host->sg_len, dir); + renesas_sdhi_internal_dmac_unmap(host, host->data, COOKIE_MAPPED); if (dir == DMA_FROM_DEVICE) clear_bit(SDHI_INTERNAL_DMAC_RX_IN_USE, &global_flags); + host->dma_on = false; + + return true; +} + +static void renesas_sdhi_internal_dmac_complete_work_fn(struct work_struct *work) +{ + struct renesas_sdhi_dma *dma_priv = from_work(dma_priv, work, dma_complete); + struct renesas_sdhi *priv = container_of(dma_priv, typeof(*priv), dma_priv); + struct tmio_mmc_host *host = priv->host; + + spin_lock_irq(&host->lock); + if (!renesas_sdhi_internal_dmac_complete(host)) + goto out; + tmio_mmc_do_data_irq(host); out: spin_unlock_irq(&host->lock); } +static void renesas_sdhi_internal_dmac_end_dma(struct tmio_mmc_host *host) +{ + if (host->data) + renesas_sdhi_internal_dmac_complete(host); +} + +static void renesas_sdhi_internal_dmac_post_req(struct mmc_host *mmc, + struct mmc_request *mrq, + int err) +{ + struct tmio_mmc_host *host = mmc_priv(mmc); + struct mmc_data *data = mrq->data; + + if (!data) + return; + + renesas_sdhi_internal_dmac_unmap(host, data, COOKIE_UNMAPPED); +} + +static void renesas_sdhi_internal_dmac_pre_req(struct mmc_host *mmc, + struct mmc_request *mrq) +{ + struct tmio_mmc_host *host = mmc_priv(mmc); + struct mmc_data *data = mrq->data; + + if (!data) + return; + + data->host_cookie = COOKIE_UNMAPPED; + renesas_sdhi_internal_dmac_map(host, data, COOKIE_PRE_MAPPED); +} + static void renesas_sdhi_internal_dmac_request_dma(struct tmio_mmc_host *host, struct tmio_mmc_data *pdata) { struct renesas_sdhi *priv = host_to_priv(host); - /* Disable DMAC interrupts, we don't use them */ - renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO1_MASK, - INFO1_MASK_CLEAR); - renesas_sdhi_internal_dmac_dm_write(host, DM_CM_INFO2_MASK, - INFO2_MASK_CLEAR); + /* Disable DMAC interrupts initially */ + writel(INFO1_MASK_CLEAR, host->ctl + DM_CM_INFO1_MASK); + writel(INFO2_MASK_CLEAR, host->ctl + DM_CM_INFO2_MASK); + writel(0, host->ctl + DM_CM_INFO1); + writel(0, host->ctl + DM_CM_INFO2); /* Each value is set to non-zero to assume "enabling" each DMA */ host->chan_rx = host->chan_tx = (void *)0xdeadbeaf; - tasklet_init(&priv->dma_priv.dma_complete, - renesas_sdhi_internal_dmac_complete_tasklet_fn, - (unsigned long)host); - tasklet_init(&host->dma_issue, - renesas_sdhi_internal_dmac_issue_tasklet_fn, - (unsigned long)host); + INIT_WORK(&priv->dma_priv.dma_complete, + renesas_sdhi_internal_dmac_complete_work_fn); + INIT_WORK(&host->dma_issue, + renesas_sdhi_internal_dmac_issue_work_fn); + + /* Add pre_req and post_req */ + host->ops.pre_req = renesas_sdhi_internal_dmac_pre_req; + host->ops.post_req = renesas_sdhi_internal_dmac_post_req; } static void @@ -290,66 +573,43 @@ static const struct tmio_mmc_dma_ops renesas_sdhi_internal_dmac_dma_ops = { .release = renesas_sdhi_internal_dmac_release_dma, .abort = renesas_sdhi_internal_dmac_abort_dma, .dataend = renesas_sdhi_internal_dmac_dataend_dma, -}; - -/* - * Whitelist of specific R-Car Gen3 SoC ES versions to use this DMAC - * implementation as others may use a different implementation. - */ -static const struct soc_device_attribute soc_whitelist[] = { - /* specific ones */ - { .soc_id = "r7s9210", - .data = (void *)BIT(SDHI_INTERNAL_DMAC_ADDR_MODE_FIXED_ONLY) }, - { .soc_id = "r8a7795", .revision = "ES1.*", - .data = (void *)BIT(SDHI_INTERNAL_DMAC_ONE_RX_ONLY) }, - { .soc_id = "r8a7796", .revision = "ES1.0", - .data = (void *)BIT(SDHI_INTERNAL_DMAC_ONE_RX_ONLY) }, - /* generic ones */ - { .soc_id = "r8a774a1" }, - { .soc_id = "r8a774c0" }, - { .soc_id = "r8a77470" }, - { .soc_id = "r8a7795" }, - { .soc_id = "r8a7796" }, - { .soc_id = "r8a77965" }, - { .soc_id = "r8a77970" }, - { .soc_id = "r8a77980" }, - { .soc_id = "r8a77990" }, - { .soc_id = "r8a77995" }, - { /* sentinel */ } + .end = renesas_sdhi_internal_dmac_end_dma, + .dma_irq = renesas_sdhi_internal_dmac_dma_irq, }; static int renesas_sdhi_internal_dmac_probe(struct platform_device *pdev) { - const struct soc_device_attribute *soc = soc_device_match(soc_whitelist); + const struct soc_device_attribute *attr; + const struct renesas_sdhi_of_data_with_quirks *of_data_quirks; + const struct renesas_sdhi_quirks *quirks; struct device *dev = &pdev->dev; - if (!soc) - return -ENODEV; - - global_flags |= (unsigned long)soc->data; + of_data_quirks = of_device_get_match_data(&pdev->dev); + quirks = of_data_quirks->quirks; - dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL); - if (!dev->dma_parms) - return -ENOMEM; + attr = soc_device_match(sdhi_quirks_match); + if (attr) + quirks = attr->data; /* value is max of SD_SECCNT. Confirmed by HW engineers */ dma_set_max_seg_size(dev, 0xffffffff); - return renesas_sdhi_probe(pdev, &renesas_sdhi_internal_dmac_dma_ops); + return renesas_sdhi_probe(pdev, &renesas_sdhi_internal_dmac_dma_ops, + of_data_quirks->of_data, quirks); } static const struct dev_pm_ops renesas_sdhi_internal_dmac_dev_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) - SET_RUNTIME_PM_OPS(tmio_mmc_host_runtime_suspend, - tmio_mmc_host_runtime_resume, - NULL) + SYSTEM_SLEEP_PM_OPS(renesas_sdhi_suspend, renesas_sdhi_resume) + RUNTIME_PM_OPS(tmio_mmc_host_runtime_suspend, + tmio_mmc_host_runtime_resume, + NULL) }; static struct platform_driver renesas_internal_dmac_sdhi_driver = { .driver = { .name = "renesas_sdhi_internal_dmac", - .pm = &renesas_sdhi_internal_dmac_dev_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + .pm = pm_ptr(&renesas_sdhi_internal_dmac_dev_pm_ops), .of_match_table = renesas_sdhi_internal_dmac_of_match, }, .probe = renesas_sdhi_internal_dmac_probe, |