diff options
Diffstat (limited to 'drivers/mmc/host/sdhci.c')
| -rw-r--r-- | drivers/mmc/host/sdhci.c | 2514 |
1 files changed, 1819 insertions, 695 deletions
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index ecd0d4350e8a..ac7e11f37af7 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -1,19 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * linux/drivers/mmc/host/sdhci.c - Secure Digital Host Controller Interface driver * * Copyright (C) 2005-2008 Pierre Ossman, All Rights Reserved. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or (at - * your option) any later version. - * * Thanks to the following companies for their support: * * - JMicron (hardware and technical support) */ +#include <linux/bitfield.h> #include <linux/delay.h> +#include <linux/dmaengine.h> #include <linux/ktime.h> #include <linux/highmem.h> #include <linux/io.h> @@ -21,10 +19,11 @@ #include <linux/dma-mapping.h> #include <linux/slab.h> #include <linux/scatterlist.h> +#include <linux/sizes.h> #include <linux/regulator/consumer.h> #include <linux/pm_runtime.h> #include <linux/of.h> - +#include <linux/bug.h> #include <linux/leds.h> #include <linux/mmc/mmc.h> @@ -48,9 +47,7 @@ static unsigned int debug_quirks = 0; static unsigned int debug_quirks2; -static void sdhci_finish_data(struct sdhci_host *); - -static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable); +static bool sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd); void sdhci_dumpregs(struct sdhci_host *host) { @@ -80,8 +77,8 @@ void sdhci_dumpregs(struct sdhci_host *host) SDHCI_DUMP("Int enab: 0x%08x | Sig enab: 0x%08x\n", sdhci_readl(host, SDHCI_INT_ENABLE), sdhci_readl(host, SDHCI_SIGNAL_ENABLE)); - SDHCI_DUMP("AC12 err: 0x%08x | Slot int: 0x%08x\n", - sdhci_readw(host, SDHCI_ACMD12_ERR), + SDHCI_DUMP("ACmd stat: 0x%08x | Slot int: 0x%08x\n", + sdhci_readw(host, SDHCI_AUTO_CMD_STATUS), sdhci_readw(host, SDHCI_SLOT_INT_STATUS)); SDHCI_DUMP("Caps: 0x%08x | Caps_1: 0x%08x\n", sdhci_readl(host, SDHCI_CAPABILITIES), @@ -111,6 +108,12 @@ void sdhci_dumpregs(struct sdhci_host *host) } } + if (host->ops->dump_uhs2_regs) + host->ops->dump_uhs2_regs(host); + + if (host->ops->dump_vendor_regs) + host->ops->dump_vendor_regs(host); + SDHCI_DUMP("============================================\n"); } EXPORT_SYMBOL_GPL(sdhci_dumpregs); @@ -121,17 +124,41 @@ EXPORT_SYMBOL_GPL(sdhci_dumpregs); * * \*****************************************************************************/ -static inline bool sdhci_data_line_cmd(struct mmc_command *cmd) +static void sdhci_do_enable_v4_mode(struct sdhci_host *host) +{ + u16 ctrl2; + + ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2); + if (ctrl2 & SDHCI_CTRL_V4_MODE) + return; + + ctrl2 |= SDHCI_CTRL_V4_MODE; + sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2); +} + +/* + * This can be called before sdhci_add_host() by Vendor's host controller + * driver to enable v4 mode if supported. + */ +void sdhci_enable_v4_mode(struct sdhci_host *host) +{ + host->v4_mode = true; + sdhci_do_enable_v4_mode(host); +} +EXPORT_SYMBOL_GPL(sdhci_enable_v4_mode); + +bool sdhci_data_line_cmd(struct mmc_command *cmd) { return cmd->data || cmd->flags & MMC_RSP_BUSY; } +EXPORT_SYMBOL_GPL(sdhci_data_line_cmd); static void sdhci_set_card_detection(struct sdhci_host *host, bool enable) { u32 present; if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) || - !mmc_card_is_removable(host->mmc)) + !mmc_card_is_removable(host->mmc) || mmc_host_can_gpio_cd(host->mmc)) return; if (enable) { @@ -163,7 +190,7 @@ static void sdhci_runtime_pm_bus_on(struct sdhci_host *host) if (host->bus_on) return; host->bus_on = true; - pm_runtime_get_noresume(host->mmc->parent); + pm_runtime_get_noresume(mmc_dev(host->mmc)); } static void sdhci_runtime_pm_bus_off(struct sdhci_host *host) @@ -171,7 +198,7 @@ static void sdhci_runtime_pm_bus_off(struct sdhci_host *host) if (!host->bus_on) return; host->bus_on = false; - pm_runtime_put_noidle(host->mmc->parent); + pm_runtime_put_noidle(mmc_dev(host->mmc)); } void sdhci_reset(struct sdhci_host *host, u8 mask) @@ -191,10 +218,15 @@ void sdhci_reset(struct sdhci_host *host, u8 mask) timeout = ktime_add_ms(ktime_get(), 100); /* hw clears the bit when it's done */ - while (sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask) { - if (ktime_after(ktime_get(), timeout)) { + while (1) { + bool timedout = ktime_after(ktime_get(), timeout); + + if (!(sdhci_readb(host, SDHCI_SOFTWARE_RESET) & mask)) + break; + if (timedout) { pr_err("%s: Reset 0x%x never completed.\n", mmc_hostname(host->mmc), (int)mask); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); sdhci_dumpregs(host); return; } @@ -203,28 +235,68 @@ void sdhci_reset(struct sdhci_host *host, u8 mask) } EXPORT_SYMBOL_GPL(sdhci_reset); -static void sdhci_do_reset(struct sdhci_host *host, u8 mask) +bool sdhci_do_reset(struct sdhci_host *host, u8 mask) { if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) { struct mmc_host *mmc = host->mmc; if (!mmc->ops->get_cd(mmc)) - return; + return false; } host->ops->reset(host, mask); - if (mask & SDHCI_RESET_ALL) { + return true; +} +EXPORT_SYMBOL_GPL(sdhci_do_reset); + +static void sdhci_reset_for_all(struct sdhci_host *host) +{ + if (sdhci_do_reset(host, SDHCI_RESET_ALL)) { if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) { if (host->ops->enable_dma) host->ops->enable_dma(host); } - /* Resetting the controller clears many */ host->preset_enabled = false; } } +enum sdhci_reset_reason { + SDHCI_RESET_FOR_INIT, + SDHCI_RESET_FOR_REQUEST_ERROR, + SDHCI_RESET_FOR_REQUEST_ERROR_DATA_ONLY, + SDHCI_RESET_FOR_TUNING_ABORT, + SDHCI_RESET_FOR_CARD_REMOVED, + SDHCI_RESET_FOR_CQE_RECOVERY, +}; + +static void sdhci_reset_for_reason(struct sdhci_host *host, enum sdhci_reset_reason reason) +{ + if (host->quirks2 & SDHCI_QUIRK2_ISSUE_CMD_DAT_RESET_TOGETHER) { + sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); + return; + } + + switch (reason) { + case SDHCI_RESET_FOR_INIT: + sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); + break; + case SDHCI_RESET_FOR_REQUEST_ERROR: + case SDHCI_RESET_FOR_TUNING_ABORT: + case SDHCI_RESET_FOR_CARD_REMOVED: + case SDHCI_RESET_FOR_CQE_RECOVERY: + sdhci_do_reset(host, SDHCI_RESET_CMD); + sdhci_do_reset(host, SDHCI_RESET_DATA); + break; + case SDHCI_RESET_FOR_REQUEST_ERROR_DATA_ONLY: + sdhci_do_reset(host, SDHCI_RESET_DATA); + break; + } +} + +#define sdhci_reset_for(h, r) sdhci_reset_for_reason((h), SDHCI_RESET_FOR_##r) + static void sdhci_set_default_irqs(struct sdhci_host *host) { host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT | @@ -241,36 +313,103 @@ static void sdhci_set_default_irqs(struct sdhci_host *host) sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); } +static void sdhci_config_dma(struct sdhci_host *host) +{ + u8 ctrl; + u16 ctrl2; + + if (host->version < SDHCI_SPEC_200) + return; + + ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); + + /* + * Always adjust the DMA selection as some controllers + * (e.g. JMicron) can't do PIO properly when the selection + * is ADMA. + */ + ctrl &= ~SDHCI_CTRL_DMA_MASK; + if (!(host->flags & SDHCI_REQ_USE_DMA)) + goto out; + + /* Note if DMA Select is zero then SDMA is selected */ + if (host->flags & SDHCI_USE_ADMA) + ctrl |= SDHCI_CTRL_ADMA32; + + if (host->flags & SDHCI_USE_64_BIT_DMA) { + /* + * If v4 mode, all supported DMA can be 64-bit addressing if + * controller supports 64-bit system address, otherwise only + * ADMA can support 64-bit addressing. + */ + if (host->v4_mode) { + ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2); + ctrl2 |= SDHCI_CTRL_64BIT_ADDR; + sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2); + } else if (host->flags & SDHCI_USE_ADMA) { + /* + * Don't need to undo SDHCI_CTRL_ADMA32 in order to + * set SDHCI_CTRL_ADMA64. + */ + ctrl |= SDHCI_CTRL_ADMA64; + } + } + +out: + sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); +} + static void sdhci_init(struct sdhci_host *host, int soft) { struct mmc_host *mmc = host->mmc; + unsigned long flags; if (soft) - sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); + sdhci_reset_for(host, INIT); else - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); + + if (host->v4_mode) + sdhci_do_enable_v4_mode(host); + spin_lock_irqsave(&host->lock, flags); sdhci_set_default_irqs(host); + spin_unlock_irqrestore(&host->lock, flags); host->cqe_on = false; if (soft) { /* force clock reconfiguration */ host->clock = 0; + host->reinit_uhs = true; mmc->ops->set_ios(mmc, &mmc->ios); } } static void sdhci_reinit(struct sdhci_host *host) { + u32 cd = host->ier & (SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT); + sdhci_init(host, 0); sdhci_enable_card_detection(host); + + /* + * A change to the card detect bits indicates a change in present state, + * refer sdhci_set_card_detection(). A card detect interrupt might have + * been missed while the host controller was being reset, so trigger a + * rescan to check. + */ + if (cd != (host->ier & (SDHCI_INT_CARD_REMOVE | SDHCI_INT_CARD_INSERT))) + mmc_detect_change(host->mmc, msecs_to_jiffies(200)); } static void __sdhci_led_activate(struct sdhci_host *host) { u8 ctrl; + if (host->quirks & SDHCI_QUIRK_NO_LED) + return; + ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); ctrl |= SDHCI_CTRL_LED; sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); @@ -280,6 +419,9 @@ static void __sdhci_led_deactivate(struct sdhci_host *host) { u8 ctrl; + if (host->quirks & SDHCI_QUIRK_NO_LED) + return; + ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); ctrl &= ~SDHCI_CTRL_LED; sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); @@ -309,6 +451,9 @@ static int sdhci_led_register(struct sdhci_host *host) { struct mmc_host *mmc = host->mmc; + if (host->quirks & SDHCI_QUIRK_NO_LED) + return 0; + snprintf(host->led_name, sizeof(host->led_name), "%s::", mmc_hostname(mmc)); @@ -322,6 +467,9 @@ static int sdhci_led_register(struct sdhci_host *host) static void sdhci_led_unregister(struct sdhci_host *host) { + if (host->quirks & SDHCI_QUIRK_NO_LED) + return; + led_classdev_unregister(&host->led); } @@ -356,6 +504,29 @@ static inline void sdhci_led_deactivate(struct sdhci_host *host) #endif +void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq, + unsigned long timeout) +{ + if (sdhci_data_line_cmd(mrq->cmd)) + mod_timer(&host->data_timer, timeout); + else + mod_timer(&host->timer, timeout); +} +EXPORT_SYMBOL_GPL(sdhci_mod_timer); + +static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq) +{ + if (sdhci_data_line_cmd(mrq->cmd)) + timer_delete(&host->data_timer); + else + timer_delete(&host->timer); +} + +static inline bool sdhci_has_requests(struct sdhci_host *host) +{ + return host->cmd || host->data_cmd; +} + /*****************************************************************************\ * * * Core functions * @@ -364,9 +535,8 @@ static inline void sdhci_led_deactivate(struct sdhci_host *host) static void sdhci_read_block_pio(struct sdhci_host *host) { - unsigned long flags; size_t blksize, len, chunk; - u32 uninitialized_var(scratch); + u32 scratch; u8 *buf; DBG("PIO reading\n"); @@ -374,8 +544,6 @@ static void sdhci_read_block_pio(struct sdhci_host *host) blksize = host->data->blksz; chunk = 0; - local_irq_save(flags); - while (blksize) { BUG_ON(!sg_miter_next(&host->sg_miter)); @@ -402,13 +570,10 @@ static void sdhci_read_block_pio(struct sdhci_host *host) } sg_miter_stop(&host->sg_miter); - - local_irq_restore(flags); } static void sdhci_write_block_pio(struct sdhci_host *host) { - unsigned long flags; size_t blksize, len, chunk; u32 scratch; u8 *buf; @@ -419,8 +584,6 @@ static void sdhci_write_block_pio(struct sdhci_host *host) chunk = 0; scratch = 0; - local_irq_save(flags); - while (blksize) { BUG_ON(!sg_miter_next(&host->sg_miter)); @@ -447,8 +610,6 @@ static void sdhci_write_block_pio(struct sdhci_host *host) } sg_miter_stop(&host->sg_miter); - - local_irq_restore(flags); } static void sdhci_transfer_pio(struct sdhci_host *host) @@ -501,8 +662,39 @@ static int sdhci_pre_dma_transfer(struct sdhci_host *host, if (data->host_cookie == COOKIE_PRE_MAPPED) return data->sg_count; - sg_count = dma_map_sg(mmc_dev(host->mmc), data->sg, data->sg_len, - mmc_get_dma_dir(data)); + /* Bounce write requests to the bounce buffer */ + if (host->bounce_buffer) { + unsigned int length = data->blksz * data->blocks; + + if (length > host->bounce_buffer_size) { + pr_err("%s: asked for transfer of %u bytes exceeds bounce buffer %u bytes\n", + mmc_hostname(host->mmc), length, + host->bounce_buffer_size); + return -EIO; + } + if (mmc_get_dma_dir(data) == DMA_TO_DEVICE) { + /* Copy the data to the bounce buffer */ + if (host->ops->copy_to_bounce_buffer) { + host->ops->copy_to_bounce_buffer(host, + data, length); + } else { + sg_copy_to_buffer(data->sg, data->sg_len, + host->bounce_buffer, length); + } + } + /* Switch ownership to the DMA */ + dma_sync_single_for_device(mmc_dev(host->mmc), + host->bounce_addr, + host->bounce_buffer_size, + mmc_get_dma_dir(data)); + /* Just a dummy value */ + sg_count = 1; + } else { + /* Just access the data directly from memory */ + sg_count = dma_map_sg(mmc_dev(host->mmc), + data->sg, data->sg_len, + mmc_get_dma_dir(data)); + } if (sg_count == 0) return -ENOSPC; @@ -513,30 +705,41 @@ static int sdhci_pre_dma_transfer(struct sdhci_host *host, return sg_count; } -static char *sdhci_kmap_atomic(struct scatterlist *sg, unsigned long *flags) +static char *sdhci_kmap_atomic(struct scatterlist *sg) { - local_irq_save(*flags); - return kmap_atomic(sg_page(sg)) + sg->offset; + return kmap_local_page(sg_page(sg)) + sg->offset; } -static void sdhci_kunmap_atomic(void *buffer, unsigned long *flags) +static void sdhci_kunmap_atomic(void *buffer) { - kunmap_atomic(buffer); - local_irq_restore(*flags); + kunmap_local(buffer); } -static void sdhci_adma_write_desc(struct sdhci_host *host, void *desc, - dma_addr_t addr, int len, unsigned cmd) +void sdhci_adma_write_desc(struct sdhci_host *host, void **desc, + dma_addr_t addr, int len, unsigned int cmd) { - struct sdhci_adma2_64_desc *dma_desc = desc; + struct sdhci_adma2_64_desc *dma_desc = *desc; /* 32-bit and 64-bit descriptors have these members in same position */ dma_desc->cmd = cpu_to_le16(cmd); dma_desc->len = cpu_to_le16(len); - dma_desc->addr_lo = cpu_to_le32((u32)addr); + dma_desc->addr_lo = cpu_to_le32(lower_32_bits(addr)); if (host->flags & SDHCI_USE_64_BIT_DMA) - dma_desc->addr_hi = cpu_to_le32((u64)addr >> 32); + dma_desc->addr_hi = cpu_to_le32(upper_32_bits(addr)); + + *desc += host->desc_sz; +} +EXPORT_SYMBOL_GPL(sdhci_adma_write_desc); + +static inline void __sdhci_adma_write_desc(struct sdhci_host *host, + void **desc, dma_addr_t addr, + int len, unsigned int cmd) +{ + if (host->ops->adma_write_desc) + host->ops->adma_write_desc(host, desc, addr, len, cmd); + else + sdhci_adma_write_desc(host, desc, addr, len, cmd); } static void sdhci_adma_mark_end(void *desc) @@ -551,7 +754,6 @@ static void sdhci_adma_table_pre(struct sdhci_host *host, struct mmc_data *data, int sg_count) { struct scatterlist *sg; - unsigned long flags; dma_addr_t addr, align_addr; void *desc, *align; char *buffer; @@ -583,35 +785,43 @@ static void sdhci_adma_table_pre(struct sdhci_host *host, SDHCI_ADMA2_MASK; if (offset) { if (data->flags & MMC_DATA_WRITE) { - buffer = sdhci_kmap_atomic(sg, &flags); + buffer = sdhci_kmap_atomic(sg); memcpy(align, buffer, offset); - sdhci_kunmap_atomic(buffer, &flags); + sdhci_kunmap_atomic(buffer); } /* tran, valid */ - sdhci_adma_write_desc(host, desc, align_addr, offset, - ADMA2_TRAN_VALID); + __sdhci_adma_write_desc(host, &desc, align_addr, + offset, ADMA2_TRAN_VALID); BUG_ON(offset > 65536); align += SDHCI_ADMA2_ALIGN; align_addr += SDHCI_ADMA2_ALIGN; - desc += host->desc_sz; - addr += offset; len -= offset; } - BUG_ON(len > 65536); + /* + * The block layer forces a minimum segment size of PAGE_SIZE, + * so 'len' can be too big here if PAGE_SIZE >= 64KiB. Write + * multiple descriptors, noting that the ADMA table is sized + * for 4KiB chunks anyway, so it will be big enough. + */ + while (len > host->max_adma) { + int n = 32 * 1024; /* 32KiB*/ - if (len) { - /* tran, valid */ - sdhci_adma_write_desc(host, desc, addr, len, - ADMA2_TRAN_VALID); - desc += host->desc_sz; + __sdhci_adma_write_desc(host, &desc, addr, n, ADMA2_TRAN_VALID); + addr += n; + len -= n; } + /* tran, valid */ + if (len) + __sdhci_adma_write_desc(host, &desc, addr, len, + ADMA2_TRAN_VALID); + /* * If this triggers then we have a calculation bug * somewhere. :/ @@ -627,7 +837,7 @@ static void sdhci_adma_table_pre(struct sdhci_host *host, } } else { /* Add a terminating entry - nop, end, valid */ - sdhci_adma_write_desc(host, desc, 0, 0, ADMA2_NOP_END_VALID); + __sdhci_adma_write_desc(host, &desc, 0, 0, ADMA2_NOP_END_VALID); } } @@ -638,7 +848,6 @@ static void sdhci_adma_table_post(struct sdhci_host *host, int i, size; void *align; char *buffer; - unsigned long flags; if (data->flags & MMC_DATA_READ) { bool has_unaligned = false; @@ -661,9 +870,9 @@ static void sdhci_adma_table_post(struct sdhci_host *host, size = SDHCI_ADMA2_ALIGN - (sg_dma_address(sg) & SDHCI_ADMA2_MASK); - buffer = sdhci_kmap_atomic(sg, &flags); + buffer = sdhci_kmap_atomic(sg); memcpy(buffer, align, size); - sdhci_kunmap_atomic(buffer, &flags); + sdhci_kunmap_atomic(buffer); align += SDHCI_ADMA2_ALIGN; } @@ -672,29 +881,39 @@ static void sdhci_adma_table_post(struct sdhci_host *host, } } -static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd) +static void sdhci_set_adma_addr(struct sdhci_host *host, dma_addr_t addr) { - u8 count; - struct mmc_data *data = cmd->data; - unsigned target_timeout, current_timeout; + sdhci_writel(host, lower_32_bits(addr), SDHCI_ADMA_ADDRESS); + if (host->flags & SDHCI_USE_64_BIT_DMA) + sdhci_writel(host, upper_32_bits(addr), SDHCI_ADMA_ADDRESS_HI); +} - /* - * If the host controller provides us with an incorrect timeout - * value, just skip the check and use 0xE. The hardware may take - * longer to time out, but that's much better than having a too-short - * timeout value. - */ - if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL) - return 0xE; +static dma_addr_t sdhci_sdma_address(struct sdhci_host *host) +{ + if (host->bounce_buffer) + return host->bounce_addr; + else + return sg_dma_address(host->data->sg); +} - /* Unspecified timeout, assume max */ - if (!data && !cmd->busy_timeout) - return 0xE; +static void sdhci_set_sdma_addr(struct sdhci_host *host, dma_addr_t addr) +{ + if (host->v4_mode) + sdhci_set_adma_addr(host, addr); + else + sdhci_writel(host, addr, SDHCI_DMA_ADDRESS); +} + +static unsigned int sdhci_target_timeout(struct sdhci_host *host, + struct mmc_command *cmd, + struct mmc_data *data) +{ + unsigned int target_timeout; /* timeout in us */ - if (!data) + if (!data) { target_timeout = cmd->busy_timeout * 1000; - else { + } else { target_timeout = DIV_ROUND_UP(data->timeout_ns, 1000); if (host->clock && data->timeout_clks) { unsigned long long val; @@ -711,6 +930,72 @@ static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd) } } + return target_timeout; +} + +static void sdhci_calc_sw_timeout(struct sdhci_host *host, + struct mmc_command *cmd) +{ + struct mmc_data *data = cmd->data; + struct mmc_host *mmc = host->mmc; + struct mmc_ios *ios = &mmc->ios; + unsigned char bus_width = 1 << ios->bus_width; + unsigned int blksz; + unsigned int freq; + u64 target_timeout; + u64 transfer_time; + + target_timeout = sdhci_target_timeout(host, cmd, data); + target_timeout *= NSEC_PER_USEC; + + if (data) { + blksz = data->blksz; + freq = mmc->actual_clock ? : host->clock; + transfer_time = (u64)blksz * NSEC_PER_SEC * (8 / bus_width); + do_div(transfer_time, freq); + /* multiply by '2' to account for any unknowns */ + transfer_time = transfer_time * 2; + /* calculate timeout for the entire data */ + host->data_timeout = data->blocks * target_timeout + + transfer_time; + } else { + host->data_timeout = target_timeout; + } + + if (host->data_timeout) + host->data_timeout += MMC_CMD_TRANSFER_TIME; +} + +static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd, + bool *too_big) +{ + u8 count; + struct mmc_data *data; + unsigned target_timeout, current_timeout; + + *too_big = false; + + /* + * If the host controller provides us with an incorrect timeout + * value, just skip the check and use the maximum. The hardware may take + * longer to time out, but that's much better than having a too-short + * timeout value. + */ + if (host->quirks & SDHCI_QUIRK_BROKEN_TIMEOUT_VAL) + return host->max_timeout_count; + + /* Unspecified command, assume max */ + if (cmd == NULL) + return host->max_timeout_count; + + data = cmd->data; + /* Unspecified timeout, assume max */ + if (!data && !cmd->busy_timeout) + return host->max_timeout_count; + + /* timeout in us */ + target_timeout = sdhci_target_timeout(host, cmd, data); + /* * Figure out needed cycles. * We do this in steps in order to fit inside a 32 bit int. @@ -726,14 +1011,14 @@ static u8 sdhci_calc_timeout(struct sdhci_host *host, struct mmc_command *cmd) while (current_timeout < target_timeout) { count++; current_timeout <<= 1; - if (count >= 0xF) + if (count > host->max_timeout_count) { + if (!(host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT)) + DBG("Too large timeout 0x%x requested for CMD%d!\n", + count, cmd->opcode); + count = host->max_timeout_count; + *too_big = true; break; - } - - if (count >= 0xF) { - DBG("Too large timeout 0x%x requested for CMD%d!\n", - count, cmd->opcode); - count = 0xE; + } } return count; @@ -749,33 +1034,53 @@ static void sdhci_set_transfer_irqs(struct sdhci_host *host) else host->ier = (host->ier & ~dma_irqs) | pio_irqs; + if (host->flags & (SDHCI_AUTO_CMD23 | SDHCI_AUTO_CMD12)) + host->ier |= SDHCI_INT_AUTO_CMD_ERR; + else + host->ier &= ~SDHCI_INT_AUTO_CMD_ERR; + sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); } -static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd) +void sdhci_set_data_timeout_irq(struct sdhci_host *host, bool enable) { - u8 count; - - if (host->ops->set_timeout) { - host->ops->set_timeout(host, cmd); - } else { - count = sdhci_calc_timeout(host, cmd); - sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL); - } + if (enable) + host->ier |= SDHCI_INT_DATA_TIMEOUT; + else + host->ier &= ~SDHCI_INT_DATA_TIMEOUT; + sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); + sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); } +EXPORT_SYMBOL_GPL(sdhci_set_data_timeout_irq); -static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) +void __sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd) { - u8 ctrl; - struct mmc_data *data = cmd->data; + bool too_big = false; + u8 count = sdhci_calc_timeout(host, cmd, &too_big); + + if (too_big && + host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT) { + sdhci_calc_sw_timeout(host, cmd); + sdhci_set_data_timeout_irq(host, false); + } else if (!(host->ier & SDHCI_INT_DATA_TIMEOUT)) { + sdhci_set_data_timeout_irq(host, true); + } - if (sdhci_data_line_cmd(cmd)) - sdhci_set_timeout(host, cmd); + sdhci_writeb(host, count, SDHCI_TIMEOUT_CONTROL); +} +EXPORT_SYMBOL_GPL(__sdhci_set_timeout); - if (!data) - return; +static void sdhci_set_timeout(struct sdhci_host *host, struct mmc_command *cmd) +{ + if (host->ops->set_timeout) + host->ops->set_timeout(host, cmd); + else + __sdhci_set_timeout(host, cmd); +} +void sdhci_initialize_data(struct sdhci_host *host, struct mmc_data *data) +{ WARN_ON(host->data); /* Sanity checks */ @@ -786,7 +1091,32 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) host->data = data; host->data_early = 0; host->data->bytes_xfered = 0; +} +EXPORT_SYMBOL_GPL(sdhci_initialize_data); +static inline void sdhci_set_block_info(struct sdhci_host *host, + struct mmc_data *data) +{ + /* Set the DMA boundary value and block size */ + sdhci_writew(host, + SDHCI_MAKE_BLKSZ(host->sdma_boundary, data->blksz), + SDHCI_BLOCK_SIZE); + /* + * For Version 4.10 onwards, if v4 mode is enabled, 32-bit Block Count + * can be supported, in that case 16-bit block count register must be 0. + */ + if (host->version >= SDHCI_SPEC_410 && host->v4_mode && + (host->quirks2 & SDHCI_QUIRK2_USE_32BIT_BLK_CNT)) { + if (sdhci_readw(host, SDHCI_BLOCK_COUNT)) + sdhci_writew(host, 0, SDHCI_BLOCK_COUNT); + sdhci_writew(host, data->blocks, SDHCI_32BIT_BLK_CNT); + } else { + sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT); + } +} + +void sdhci_prepare_dma(struct sdhci_host *host, struct mmc_data *data) +{ if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) { struct scatterlist *sg; unsigned int length_mask, offset_mask; @@ -837,6 +1167,8 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) } } + sdhci_config_dma(host); + if (host->flags & SDHCI_REQ_USE_DMA) { int sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED); @@ -849,37 +1181,11 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) host->flags &= ~SDHCI_REQ_USE_DMA; } else if (host->flags & SDHCI_USE_ADMA) { sdhci_adma_table_pre(host, data, sg_cnt); - - sdhci_writel(host, host->adma_addr, SDHCI_ADMA_ADDRESS); - if (host->flags & SDHCI_USE_64_BIT_DMA) - sdhci_writel(host, - (u64)host->adma_addr >> 32, - SDHCI_ADMA_ADDRESS_HI); + sdhci_set_adma_addr(host, host->adma_addr); } else { WARN_ON(sg_cnt != 1); - sdhci_writel(host, sg_dma_address(data->sg), - SDHCI_DMA_ADDRESS); - } - } - - /* - * Always adjust the DMA selection as some controllers - * (e.g. JMicron) can't do PIO properly when the selection - * is ADMA. - */ - if (host->version >= SDHCI_SPEC_200) { - ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); - ctrl &= ~SDHCI_CTRL_DMA_MASK; - if ((host->flags & SDHCI_REQ_USE_DMA) && - (host->flags & SDHCI_USE_ADMA)) { - if (host->flags & SDHCI_USE_64_BIT_DMA) - ctrl |= SDHCI_CTRL_ADMA64; - else - ctrl |= SDHCI_CTRL_ADMA32; - } else { - ctrl |= SDHCI_CTRL_SDMA; + sdhci_set_sdma_addr(host, sdhci_sdma_address(host)); } - sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); } if (!(host->flags & SDHCI_REQ_USE_DMA)) { @@ -895,13 +1201,204 @@ static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) } sdhci_set_transfer_irqs(host); +} +EXPORT_SYMBOL_GPL(sdhci_prepare_dma); - /* Set the DMA boundary value and block size */ - sdhci_writew(host, SDHCI_MAKE_BLKSZ(SDHCI_DEFAULT_BOUNDARY_ARG, - data->blksz), SDHCI_BLOCK_SIZE); - sdhci_writew(host, data->blocks, SDHCI_BLOCK_COUNT); +static void sdhci_prepare_data(struct sdhci_host *host, struct mmc_command *cmd) +{ + struct mmc_data *data = cmd->data; + + sdhci_initialize_data(host, data); + + sdhci_prepare_dma(host, data); + + sdhci_set_block_info(host, data); +} + +#if IS_ENABLED(CONFIG_MMC_SDHCI_EXTERNAL_DMA) + +static int sdhci_external_dma_init(struct sdhci_host *host) +{ + int ret = 0; + struct mmc_host *mmc = host->mmc; + + host->tx_chan = dma_request_chan(mmc_dev(mmc), "tx"); + if (IS_ERR(host->tx_chan)) { + ret = PTR_ERR(host->tx_chan); + if (ret != -EPROBE_DEFER) + pr_warn("Failed to request TX DMA channel.\n"); + host->tx_chan = NULL; + return ret; + } + + host->rx_chan = dma_request_chan(mmc_dev(mmc), "rx"); + if (IS_ERR(host->rx_chan)) { + if (host->tx_chan) { + dma_release_channel(host->tx_chan); + host->tx_chan = NULL; + } + + ret = PTR_ERR(host->rx_chan); + if (ret != -EPROBE_DEFER) + pr_warn("Failed to request RX DMA channel.\n"); + host->rx_chan = NULL; + } + + return ret; +} + +static struct dma_chan *sdhci_external_dma_channel(struct sdhci_host *host, + struct mmc_data *data) +{ + return data->flags & MMC_DATA_WRITE ? host->tx_chan : host->rx_chan; +} + +static int sdhci_external_dma_setup(struct sdhci_host *host, + struct mmc_command *cmd) +{ + int ret, i; + enum dma_transfer_direction dir; + struct dma_async_tx_descriptor *desc; + struct mmc_data *data = cmd->data; + struct dma_chan *chan; + struct dma_slave_config cfg; + dma_cookie_t cookie; + int sg_cnt; + + if (!host->mapbase) + return -EINVAL; + + memset(&cfg, 0, sizeof(cfg)); + cfg.src_addr = host->mapbase + SDHCI_BUFFER; + cfg.dst_addr = host->mapbase + SDHCI_BUFFER; + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = data->blksz / 4; + cfg.dst_maxburst = data->blksz / 4; + + /* Sanity check: all the SG entries must be aligned by block size. */ + for (i = 0; i < data->sg_len; i++) { + if ((data->sg + i)->length % data->blksz) + return -EINVAL; + } + + chan = sdhci_external_dma_channel(host, data); + + ret = dmaengine_slave_config(chan, &cfg); + if (ret) + return ret; + + sg_cnt = sdhci_pre_dma_transfer(host, data, COOKIE_MAPPED); + if (sg_cnt <= 0) + return -EINVAL; + + dir = data->flags & MMC_DATA_WRITE ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; + desc = dmaengine_prep_slave_sg(chan, data->sg, data->sg_len, dir, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) + return -EINVAL; + + desc->callback = NULL; + desc->callback_param = NULL; + + cookie = dmaengine_submit(desc); + if (dma_submit_error(cookie)) + ret = cookie; + + return ret; } +static void sdhci_external_dma_release(struct sdhci_host *host) +{ + if (host->tx_chan) { + dma_release_channel(host->tx_chan); + host->tx_chan = NULL; + } + + if (host->rx_chan) { + dma_release_channel(host->rx_chan); + host->rx_chan = NULL; + } + + sdhci_switch_external_dma(host, false); +} + +static void __sdhci_external_dma_prepare_data(struct sdhci_host *host, + struct mmc_command *cmd) +{ + struct mmc_data *data = cmd->data; + + sdhci_initialize_data(host, data); + + host->flags |= SDHCI_REQ_USE_DMA; + sdhci_set_transfer_irqs(host); + + sdhci_set_block_info(host, data); +} + +static void sdhci_external_dma_prepare_data(struct sdhci_host *host, + struct mmc_command *cmd) +{ + if (!sdhci_external_dma_setup(host, cmd)) { + __sdhci_external_dma_prepare_data(host, cmd); + } else { + sdhci_external_dma_release(host); + pr_err("%s: Cannot use external DMA, switch to the DMA/PIO which standard SDHCI provides.\n", + mmc_hostname(host->mmc)); + sdhci_prepare_data(host, cmd); + } +} + +static void sdhci_external_dma_pre_transfer(struct sdhci_host *host, + struct mmc_command *cmd) +{ + struct dma_chan *chan; + + if (!cmd->data) + return; + + chan = sdhci_external_dma_channel(host, cmd->data); + if (chan) + dma_async_issue_pending(chan); +} + +#else + +static inline int sdhci_external_dma_init(struct sdhci_host *host) +{ + return -EOPNOTSUPP; +} + +static inline void sdhci_external_dma_release(struct sdhci_host *host) +{ +} + +static inline void sdhci_external_dma_prepare_data(struct sdhci_host *host, + struct mmc_command *cmd) +{ + /* This should never happen */ + WARN_ON_ONCE(1); +} + +static inline void sdhci_external_dma_pre_transfer(struct sdhci_host *host, + struct mmc_command *cmd) +{ +} + +static inline struct dma_chan *sdhci_external_dma_channel(struct sdhci_host *host, + struct mmc_data *data) +{ + return NULL; +} + +#endif + +void sdhci_switch_external_dma(struct sdhci_host *host, bool en) +{ + host->use_external_dma = en; +} +EXPORT_SYMBOL_GPL(sdhci_switch_external_dma); + static inline bool sdhci_auto_cmd12(struct sdhci_host *host, struct mmc_request *mrq) { @@ -909,6 +1406,57 @@ static inline bool sdhci_auto_cmd12(struct sdhci_host *host, !mrq->cap_cmd_during_tfr; } +static inline bool sdhci_auto_cmd23(struct sdhci_host *host, + struct mmc_request *mrq) +{ + return mrq->sbc && (host->flags & SDHCI_AUTO_CMD23); +} + +static inline bool sdhci_manual_cmd23(struct sdhci_host *host, + struct mmc_request *mrq) +{ + return mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23); +} + +static inline void sdhci_auto_cmd_select(struct sdhci_host *host, + struct mmc_command *cmd, + u16 *mode) +{ + bool use_cmd12 = sdhci_auto_cmd12(host, cmd->mrq) && + (cmd->opcode != SD_IO_RW_EXTENDED); + bool use_cmd23 = sdhci_auto_cmd23(host, cmd->mrq); + u16 ctrl2; + + /* + * In case of Version 4.10 or later, use of 'Auto CMD Auto + * Select' is recommended rather than use of 'Auto CMD12 + * Enable' or 'Auto CMD23 Enable'. We require Version 4 Mode + * here because some controllers (e.g sdhci-of-dwmshc) expect it. + */ + if (host->version >= SDHCI_SPEC_410 && host->v4_mode && + (use_cmd12 || use_cmd23)) { + *mode |= SDHCI_TRNS_AUTO_SEL; + + ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2); + if (use_cmd23) + ctrl2 |= SDHCI_CMD23_ENABLE; + else + ctrl2 &= ~SDHCI_CMD23_ENABLE; + sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2); + + return; + } + + /* + * If we are sending CMD23, CMD12 never gets sent + * on successful completion (so no Auto-CMD12). + */ + if (use_cmd12) + *mode |= SDHCI_TRNS_AUTO_CMD12; + else if (use_cmd23) + *mode |= SDHCI_TRNS_AUTO_CMD23; +} + static void sdhci_set_transfer_mode(struct sdhci_host *host, struct mmc_command *cmd) { @@ -918,7 +1466,9 @@ static void sdhci_set_transfer_mode(struct sdhci_host *host, if (data == NULL) { if (host->quirks2 & SDHCI_QUIRK2_CLEAR_TRANSFERMODE_REG_BEFORE_CMD) { - sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE); + /* must not clear SDHCI_TRANSFER_MODE when tuning */ + if (!mmc_op_tuning(cmd->opcode)) + sdhci_writew(host, 0x0, SDHCI_TRANSFER_MODE); } else { /* clear Auto CMD settings for no data CMDs */ mode = sdhci_readw(host, SDHCI_TRANSFER_MODE); @@ -935,17 +1485,9 @@ static void sdhci_set_transfer_mode(struct sdhci_host *host, if (mmc_op_multi(cmd->opcode) || data->blocks > 1) { mode = SDHCI_TRNS_BLK_CNT_EN | SDHCI_TRNS_MULTI; - /* - * If we are sending CMD23, CMD12 never gets sent - * on successful completion (so no Auto-CMD12). - */ - if (sdhci_auto_cmd12(host, cmd->mrq) && - (cmd->opcode != SD_IO_RW_EXTENDED)) - mode |= SDHCI_TRNS_AUTO_CMD12; - else if (cmd->mrq->sbc && (host->flags & SDHCI_AUTO_CMD23)) { - mode |= SDHCI_TRNS_AUTO_CMD23; + sdhci_auto_cmd_select(host, cmd, &mode); + if (sdhci_auto_cmd23(host, cmd->mrq)) sdhci_writel(host, cmd->mrq->sbc->arg, SDHCI_ARGUMENT2); - } } if (data->flags & MMC_DATA_READ) @@ -956,17 +1498,17 @@ static void sdhci_set_transfer_mode(struct sdhci_host *host, sdhci_writew(host, mode, SDHCI_TRANSFER_MODE); } -static bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq) +bool sdhci_needs_reset(struct sdhci_host *host, struct mmc_request *mrq) { return (!(host->flags & SDHCI_DEVICE_DEAD) && ((mrq->cmd && mrq->cmd->error) || (mrq->sbc && mrq->sbc->error) || - (mrq->data && ((mrq->data->error && !mrq->data->stop) || - (mrq->data->stop && mrq->data->stop->error))) || + (mrq->data && mrq->data->stop && mrq->data->stop->error) || (host->quirks & SDHCI_QUIRK_RESET_AFTER_REQUEST))); } +EXPORT_SYMBOL_GPL(sdhci_needs_reset); -static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq) +static void sdhci_set_mrq_done(struct sdhci_host *host, struct mmc_request *mrq) { int i; @@ -985,11 +1527,9 @@ static void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq) } WARN_ON(i >= SDHCI_MAX_MRQS); - - tasklet_schedule(&host->finish_tasklet); } -static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq) +void __sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq) { if (host->cmd && host->cmd->mrq == mrq) host->cmd = NULL; @@ -997,16 +1537,33 @@ static void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq) if (host->data_cmd && host->data_cmd->mrq == mrq) host->data_cmd = NULL; + if (host->deferred_cmd && host->deferred_cmd->mrq == mrq) + host->deferred_cmd = NULL; + if (host->data && host->data->mrq == mrq) host->data = NULL; if (sdhci_needs_reset(host, mrq)) host->pending_reset = true; + sdhci_set_mrq_done(host, mrq); + + sdhci_del_timer(host, mrq); + + if (!sdhci_has_requests(host)) + sdhci_led_deactivate(host); +} +EXPORT_SYMBOL_GPL(__sdhci_finish_mrq); + +void sdhci_finish_mrq(struct sdhci_host *host, struct mmc_request *mrq) +{ __sdhci_finish_mrq(host, mrq); + + queue_work(host->complete_wq, &host->complete_work); } +EXPORT_SYMBOL_GPL(sdhci_finish_mrq); -static void sdhci_finish_data(struct sdhci_host *host) +void __sdhci_finish_data_common(struct sdhci_host *host, bool defer_reset) { struct mmc_command *data_cmd = host->data_cmd; struct mmc_data *data = host->data; @@ -1014,6 +1571,19 @@ static void sdhci_finish_data(struct sdhci_host *host) host->data = NULL; host->data_cmd = NULL; + /* + * The controller needs a reset of internal state machines upon error + * conditions. + */ + if (data->error) { + if (defer_reset) + host->pending_reset = true; + else if (!host->cmd || host->cmd == data_cmd) + sdhci_reset_for(host, REQUEST_ERROR); + else + sdhci_reset_for(host, REQUEST_ERROR_DATA_ONLY); + } + if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) == (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) sdhci_adma_table_post(host, data); @@ -1029,61 +1599,58 @@ static void sdhci_finish_data(struct sdhci_host *host) data->bytes_xfered = 0; else data->bytes_xfered = data->blksz * data->blocks; +} +EXPORT_SYMBOL_GPL(__sdhci_finish_data_common); + +static void __sdhci_finish_data(struct sdhci_host *host, bool sw_data_timeout) +{ + struct mmc_data *data = host->data; + + __sdhci_finish_data_common(host, false); /* * Need to send CMD12 if - - * a) open-ended multiblock transfer (no CMD23) + * a) open-ended multiblock transfer not using auto CMD12 (no CMD23) * b) error in multiblock transfer */ if (data->stop && - (data->error || - !data->mrq->sbc)) { - - /* - * The controller needs a reset of internal state machines - * upon error conditions. - */ - if (data->error) { - if (!host->cmd || host->cmd == data_cmd) - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); - } - + ((!data->mrq->sbc && !sdhci_auto_cmd12(host, data->mrq)) || + data->error)) { /* * 'cap_cmd_during_tfr' request must not use the command line * after mmc_command_done() has been called. It is upper layer's * responsibility to send the stop command if required. */ if (data->mrq->cap_cmd_during_tfr) { - sdhci_finish_mrq(host, data->mrq); + __sdhci_finish_mrq(host, data->mrq); } else { /* Avoid triggering warning in sdhci_send_command() */ host->cmd = NULL; - sdhci_send_command(host, data->stop); + if (!sdhci_send_command(host, data->stop)) { + if (sw_data_timeout) { + /* + * This is anyway a sw data timeout, so + * give up now. + */ + data->stop->error = -EIO; + __sdhci_finish_mrq(host, data->mrq); + } else { + WARN_ON(host->deferred_cmd); + host->deferred_cmd = data->stop; + } + } } } else { - sdhci_finish_mrq(host, data->mrq); + __sdhci_finish_mrq(host, data->mrq); } } -static void sdhci_mod_timer(struct sdhci_host *host, struct mmc_request *mrq, - unsigned long timeout) -{ - if (sdhci_data_line_cmd(mrq->cmd)) - mod_timer(&host->data_timer, timeout); - else - mod_timer(&host->timer, timeout); -} - -static void sdhci_del_timer(struct sdhci_host *host, struct mmc_request *mrq) +static void sdhci_finish_data(struct sdhci_host *host) { - if (sdhci_data_line_cmd(mrq->cmd)) - del_timer(&host->data_timer); - else - del_timer(&host->timer); + __sdhci_finish_data(host, false); } -void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) +static bool sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) { int flags; u32 mask; @@ -1098,9 +1665,6 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) cmd->opcode == MMC_STOP_TRANSMISSION) cmd->flags |= MMC_RSP_BUSY; - /* Wait max 10 ms */ - timeout = 10; - mask = SDHCI_CMD_INHIBIT; if (sdhci_data_line_cmd(cmd)) mask |= SDHCI_DATA_INHIBIT; @@ -1110,44 +1674,36 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) if (cmd->mrq->data && (cmd == cmd->mrq->data->stop)) mask &= ~SDHCI_DATA_INHIBIT; - while (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) { - if (timeout == 0) { - pr_err("%s: Controller never released inhibit bit(s).\n", - mmc_hostname(host->mmc)); - sdhci_dumpregs(host); - cmd->error = -EIO; - sdhci_finish_mrq(host, cmd->mrq); - return; - } - timeout--; - mdelay(1); - } - - timeout = jiffies; - if (!cmd->data && cmd->busy_timeout > 9000) - timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ; - else - timeout += 10 * HZ; - sdhci_mod_timer(host, cmd->mrq, timeout); + if (sdhci_readl(host, SDHCI_PRESENT_STATE) & mask) + return false; host->cmd = cmd; + host->data_timeout = 0; if (sdhci_data_line_cmd(cmd)) { WARN_ON(host->data_cmd); host->data_cmd = cmd; + sdhci_set_timeout(host, cmd); } - sdhci_prepare_data(host, cmd); + if (cmd->data) { + if (host->use_external_dma) + sdhci_external_dma_prepare_data(host, cmd); + else + sdhci_prepare_data(host, cmd); + } sdhci_writel(host, cmd->arg, SDHCI_ARGUMENT); sdhci_set_transfer_mode(host, cmd); if ((cmd->flags & MMC_RSP_136) && (cmd->flags & MMC_RSP_BUSY)) { - pr_err("%s: Unsupported response type!\n", - mmc_hostname(host->mmc)); - cmd->error = -EINVAL; - sdhci_finish_mrq(host, cmd->mrq); - return; + WARN_ONCE(1, "Unsupported response type!\n"); + /* + * This does not happen in practice because 136-bit response + * commands never have busy waiting, so rather than complicate + * the error path, just remove busy waiting and continue. + */ + cmd->flags &= ~MMC_RSP_BUSY; } if (!(cmd->flags & MMC_RSP_PRESENT)) @@ -1165,32 +1721,109 @@ void sdhci_send_command(struct sdhci_host *host, struct mmc_command *cmd) flags |= SDHCI_CMD_INDEX; /* CMD19 is special in that the Data Present Select should be set */ - if (cmd->data || cmd->opcode == MMC_SEND_TUNING_BLOCK || - cmd->opcode == MMC_SEND_TUNING_BLOCK_HS200) + if (cmd->data || mmc_op_tuning(cmd->opcode)) flags |= SDHCI_CMD_DATA; + timeout = jiffies; + if (host->data_timeout) + timeout += nsecs_to_jiffies(host->data_timeout); + else if (!cmd->data && cmd->busy_timeout > 9000) + timeout += DIV_ROUND_UP(cmd->busy_timeout, 1000) * HZ + HZ; + else + timeout += 10 * HZ; + sdhci_mod_timer(host, cmd->mrq, timeout); + + if (host->use_external_dma) + sdhci_external_dma_pre_transfer(host, cmd); + sdhci_writew(host, SDHCI_MAKE_CMD(cmd->opcode, flags), SDHCI_COMMAND); + + return true; +} + +bool sdhci_present_error(struct sdhci_host *host, + struct mmc_command *cmd, bool present) +{ + if (!present || host->flags & SDHCI_DEVICE_DEAD) { + cmd->error = -ENOMEDIUM; + return true; + } + + return false; +} +EXPORT_SYMBOL_GPL(sdhci_present_error); + +static bool sdhci_send_command_retry(struct sdhci_host *host, + struct mmc_command *cmd, + unsigned long flags) + __releases(host->lock) + __acquires(host->lock) +{ + struct mmc_command *deferred_cmd = host->deferred_cmd; + int timeout = 10; /* Approx. 10 ms */ + bool present; + + while (!sdhci_send_command(host, cmd)) { + if (!timeout--) { + pr_err("%s: Controller never released inhibit bit(s).\n", + mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); + sdhci_dumpregs(host); + cmd->error = -EIO; + return false; + } + + spin_unlock_irqrestore(&host->lock, flags); + + usleep_range(1000, 1250); + + present = host->mmc->ops->get_cd(host->mmc); + + spin_lock_irqsave(&host->lock, flags); + + /* A deferred command might disappear, handle that */ + if (cmd == deferred_cmd && cmd != host->deferred_cmd) + return true; + + if (sdhci_present_error(host, cmd, present)) + return false; + } + + if (cmd == host->deferred_cmd) + host->deferred_cmd = NULL; + + return true; +} + +static void sdhci_read_rsp_136(struct sdhci_host *host, struct mmc_command *cmd) +{ + int i, reg; + + for (i = 0; i < 4; i++) { + reg = SDHCI_RESPONSE + (3 - i) * 4; + cmd->resp[i] = sdhci_readl(host, reg); + } + + if (host->quirks2 & SDHCI_QUIRK2_RSP_136_HAS_CRC) + return; + + /* CRC is stripped so we need to do some shifting */ + for (i = 0; i < 4; i++) { + cmd->resp[i] <<= 8; + if (i != 3) + cmd->resp[i] |= cmd->resp[i + 1] >> 24; + } } -EXPORT_SYMBOL_GPL(sdhci_send_command); static void sdhci_finish_command(struct sdhci_host *host) { struct mmc_command *cmd = host->cmd; - int i; host->cmd = NULL; if (cmd->flags & MMC_RSP_PRESENT) { if (cmd->flags & MMC_RSP_136) { - /* CRC is stripped so we need to do some shifting. */ - for (i = 0;i < 4;i++) { - cmd->resp[i] = sdhci_readl(host, - SDHCI_RESPONSE + (3-i)*4) << 8; - if (i != 3) - cmd->resp[i] |= - sdhci_readb(host, - SDHCI_RESPONSE + (3-i)*4-1); - } + sdhci_read_rsp_136(host, cmd); } else { cmd->resp[0] = sdhci_readl(host, SDHCI_RESPONSE); } @@ -1221,7 +1854,10 @@ static void sdhci_finish_command(struct sdhci_host *host) /* Finished CMD23, now send actual command. */ if (cmd == cmd->mrq->sbc) { - sdhci_send_command(host, cmd->mrq->cmd); + if (!sdhci_send_command(host, cmd->mrq->cmd)) { + WARN_ON(host->deferred_cmd); + host->deferred_cmd = cmd->mrq->cmd; + } } else { /* Processed actual command. */ @@ -1229,7 +1865,7 @@ static void sdhci_finish_command(struct sdhci_host *host) sdhci_finish_data(host); if (!cmd->data) - sdhci_finish_mrq(host, cmd->mrq); + __sdhci_finish_mrq(host, cmd->mrq); } } @@ -1238,6 +1874,10 @@ static u16 sdhci_get_preset_value(struct sdhci_host *host) u16 preset = 0; switch (host->timing) { + case MMC_TIMING_MMC_HS: + case MMC_TIMING_SD_HS: + preset = sdhci_readw(host, SDHCI_PRESET_FOR_HIGH_SPEED); + break; case MMC_TIMING_UHS_SDR12: preset = sdhci_readw(host, SDHCI_PRESET_FOR_SDR12); break; @@ -1258,6 +1898,12 @@ static u16 sdhci_get_preset_value(struct sdhci_host *host) case MMC_TIMING_MMC_HS400: preset = sdhci_readw(host, SDHCI_PRESET_FOR_HS400); break; + case MMC_TIMING_UHS2_SPEED_A: + case MMC_TIMING_UHS2_SPEED_A_HD: + case MMC_TIMING_UHS2_SPEED_B: + case MMC_TIMING_UHS2_SPEED_B_HD: + preset = sdhci_readw(host, SDHCI_PRESET_FOR_UHS2); + break; default: pr_warn("%s: Invalid UHS-I mode selected\n", mmc_hostname(host->mmc)); @@ -1281,10 +1927,9 @@ u16 sdhci_calc_clk(struct sdhci_host *host, unsigned int clock, clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL); pre_val = sdhci_get_preset_value(host); - div = (pre_val & SDHCI_PRESET_SDCLK_FREQ_MASK) - >> SDHCI_PRESET_SDCLK_FREQ_SHIFT; + div = FIELD_GET(SDHCI_PRESET_SDCLK_FREQ_MASK, pre_val); if (host->clk_mul && - (pre_val & SDHCI_PRESET_CLKGEN_SEL_MASK)) { + (pre_val & SDHCI_PRESET_CLKGEN_SEL)) { clk = SDHCI_PROG_CLOCK_MODE; real_div = div + 1; clk_mul = host->clk_mul; @@ -1367,19 +2012,48 @@ void sdhci_enable_clk(struct sdhci_host *host, u16 clk) clk |= SDHCI_CLOCK_INT_EN; sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL); - /* Wait max 20 ms */ - timeout = ktime_add_ms(ktime_get(), 20); - while (!((clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL)) - & SDHCI_CLOCK_INT_STABLE)) { - if (ktime_after(ktime_get(), timeout)) { + /* Wait max 150 ms */ + timeout = ktime_add_ms(ktime_get(), 150); + while (1) { + bool timedout = ktime_after(ktime_get(), timeout); + + clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL); + if (clk & SDHCI_CLOCK_INT_STABLE) + break; + if (timedout) { pr_err("%s: Internal clock never stabilised.\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); sdhci_dumpregs(host); return; } udelay(10); } + if (host->version >= SDHCI_SPEC_410 && host->v4_mode) { + clk |= SDHCI_CLOCK_PLL_EN; + clk &= ~SDHCI_CLOCK_INT_STABLE; + sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL); + + /* Wait max 150 ms */ + timeout = ktime_add_ms(ktime_get(), 150); + while (1) { + bool timedout = ktime_after(ktime_get(), timeout); + + clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL); + if (clk & SDHCI_CLOCK_INT_STABLE) + break; + if (timedout) { + pr_err("%s: PLL clock never stabilised.\n", + mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, CTRL_TIMEOUT); + sdhci_dumpregs(host); + return; + } + udelay(10); + } + } + clk |= SDHCI_CLOCK_CARD_EN; sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL); } @@ -1414,28 +2088,46 @@ static void sdhci_set_power_reg(struct sdhci_host *host, unsigned char mode, sdhci_writeb(host, 0, SDHCI_POWER_CONTROL); } +unsigned short sdhci_get_vdd_value(unsigned short vdd) +{ + switch (1 << vdd) { + case MMC_VDD_165_195: + /* + * Without a regulator, SDHCI does not support 2.0v + * so we only get here if the driver deliberately + * added the 2.0v range to ocr_avail. Map it to 1.8v + * for the purpose of turning on the power. + */ + case MMC_VDD_20_21: + return SDHCI_POWER_180; + case MMC_VDD_29_30: + case MMC_VDD_30_31: + return SDHCI_POWER_300; + case MMC_VDD_32_33: + case MMC_VDD_33_34: + /* + * 3.4V ~ 3.6V are valid only for those platforms where it's + * known that the voltage range is supported by hardware. + */ + case MMC_VDD_34_35: + case MMC_VDD_35_36: + return SDHCI_POWER_330; + default: + return 0; + } +} +EXPORT_SYMBOL_GPL(sdhci_get_vdd_value); + void sdhci_set_power_noreg(struct sdhci_host *host, unsigned char mode, unsigned short vdd) { u8 pwr = 0; if (mode != MMC_POWER_OFF) { - switch (1 << vdd) { - case MMC_VDD_165_195: - pwr = SDHCI_POWER_180; - break; - case MMC_VDD_29_30: - case MMC_VDD_30_31: - pwr = SDHCI_POWER_300; - break; - case MMC_VDD_32_33: - case MMC_VDD_33_34: - pwr = SDHCI_POWER_330; - break; - default: + pwr = sdhci_get_vdd_value(vdd); + if (!pwr) { WARN(1, "%s: Invalid vdd %#x\n", mmc_hostname(host->mmc), vdd); - break; } } @@ -1491,19 +2183,37 @@ void sdhci_set_power(struct sdhci_host *host, unsigned char mode, } EXPORT_SYMBOL_GPL(sdhci_set_power); +/* + * Some controllers need to configure a valid bus voltage on their power + * register regardless of whether an external regulator is taking care of power + * supply. This helper function takes care of it if set as the controller's + * sdhci_ops.set_power callback. + */ +void sdhci_set_power_and_bus_voltage(struct sdhci_host *host, + unsigned char mode, + unsigned short vdd) +{ + if (!IS_ERR(host->mmc->supply.vmmc)) { + struct mmc_host *mmc = host->mmc; + + mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, vdd); + } + sdhci_set_power_noreg(host, mode, vdd); +} +EXPORT_SYMBOL_GPL(sdhci_set_power_and_bus_voltage); + /*****************************************************************************\ * * * MMC callbacks * * * \*****************************************************************************/ -static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq) +void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq) { - struct sdhci_host *host; - int present; + struct sdhci_host *host = mmc_priv(mmc); + struct mmc_command *cmd; unsigned long flags; - - host = mmc_priv(mmc); + bool present; /* Firstly check card presence */ present = mmc->ops->get_cd(mmc); @@ -1512,30 +2222,57 @@ static void sdhci_request(struct mmc_host *mmc, struct mmc_request *mrq) sdhci_led_activate(host); - /* - * Ensure we don't send the STOP for non-SET_BLOCK_COUNTED - * requests if Auto-CMD12 is enabled. - */ - if (sdhci_auto_cmd12(host, mrq)) { - if (mrq->stop) { - mrq->data->stop = NULL; - mrq->stop = NULL; - } - } + if (sdhci_present_error(host, mrq->cmd, present)) + goto out_finish; - if (!present || host->flags & SDHCI_DEVICE_DEAD) { - mrq->cmd->error = -ENOMEDIUM; + cmd = sdhci_manual_cmd23(host, mrq) ? mrq->sbc : mrq->cmd; + + if (!sdhci_send_command_retry(host, cmd, flags)) + goto out_finish; + + spin_unlock_irqrestore(&host->lock, flags); + + return; + +out_finish: + sdhci_finish_mrq(host, mrq); + spin_unlock_irqrestore(&host->lock, flags); +} +EXPORT_SYMBOL_GPL(sdhci_request); + +int sdhci_request_atomic(struct mmc_host *mmc, struct mmc_request *mrq) +{ + struct sdhci_host *host = mmc_priv(mmc); + struct mmc_command *cmd; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&host->lock, flags); + + if (sdhci_present_error(host, mrq->cmd, true)) { sdhci_finish_mrq(host, mrq); - } else { - if (mrq->sbc && !(host->flags & SDHCI_AUTO_CMD23)) - sdhci_send_command(host, mrq->sbc); - else - sdhci_send_command(host, mrq->cmd); + goto out_finish; } - mmiowb(); + cmd = sdhci_manual_cmd23(host, mrq) ? mrq->sbc : mrq->cmd; + + /* + * The HSQ may send a command in interrupt context without polling + * the busy signaling, which means we should return BUSY if controller + * has not released inhibit bits to allow HSQ trying to send request + * again in non-atomic context. So we should not finish this request + * here. + */ + if (!sdhci_send_command(host, cmd)) + ret = -EBUSY; + else + sdhci_led_activate(host); + +out_finish: spin_unlock_irqrestore(&host->lock, flags); + return ret; } +EXPORT_SYMBOL_GPL(sdhci_request_atomic); void sdhci_set_bus_width(struct sdhci_host *host, int width) { @@ -1544,10 +2281,9 @@ void sdhci_set_bus_width(struct sdhci_host *host, int width) ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); if (width == MMC_BUS_WIDTH_8) { ctrl &= ~SDHCI_CTRL_4BITBUS; - if (host->version >= SDHCI_SPEC_300) - ctrl |= SDHCI_CTRL_8BITBUS; + ctrl |= SDHCI_CTRL_8BITBUS; } else { - if (host->version >= SDHCI_SPEC_300) + if (host->mmc->caps & MMC_CAP_8_BIT_DATA) ctrl &= ~SDHCI_CTRL_8BITBUS; if (width == MMC_BUS_WIDTH_4) ctrl |= SDHCI_CTRL_4BITBUS; @@ -1583,20 +2319,40 @@ void sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned timing) } EXPORT_SYMBOL_GPL(sdhci_set_uhs_signaling); -void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) +static bool sdhci_timing_has_preset(unsigned char timing) { - struct sdhci_host *host = mmc_priv(mmc); - u8 ctrl; + switch (timing) { + case MMC_TIMING_UHS_SDR12: + case MMC_TIMING_UHS_SDR25: + case MMC_TIMING_UHS_SDR50: + case MMC_TIMING_UHS_SDR104: + case MMC_TIMING_UHS_DDR50: + case MMC_TIMING_MMC_DDR52: + return true; + } + return false; +} - if (ios->power_mode == MMC_POWER_UNDEFINED) - return; +static bool sdhci_preset_needed(struct sdhci_host *host, unsigned char timing) +{ + return !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) && + sdhci_timing_has_preset(timing); +} - if (host->flags & SDHCI_DEVICE_DEAD) { - if (!IS_ERR(mmc->supply.vmmc) && - ios->power_mode == MMC_POWER_OFF) - mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0); - return; - } +static bool sdhci_presetable_values_change(struct sdhci_host *host, struct mmc_ios *ios) +{ + /* + * Preset Values are: Driver Strength, Clock Generator and SDCLK/RCLK + * Frequency. Check if preset values need to be enabled, or the Driver + * Strength needs updating. Note, clock changes are handled separately. + */ + return !host->preset_enabled && + (sdhci_preset_needed(host, ios->timing) || host->drv_type != ios->drv_type); +} + +void sdhci_set_ios_common(struct mmc_host *mmc, struct mmc_ios *ios) +{ + struct sdhci_host *host = mmc_priv(mmc); /* * Reset the chip on each power off. @@ -1611,6 +2367,31 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) (ios->power_mode == MMC_POWER_UP) && !(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN)) sdhci_enable_preset_value(host, false); +} +EXPORT_SYMBOL_GPL(sdhci_set_ios_common); + +void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) +{ + struct sdhci_host *host = mmc_priv(mmc); + bool reinit_uhs = host->reinit_uhs; + bool turning_on_clk; + u8 ctrl; + + host->reinit_uhs = false; + + if (ios->power_mode == MMC_POWER_UNDEFINED) + return; + + if (host->flags & SDHCI_DEVICE_DEAD) { + if (!IS_ERR(mmc->supply.vmmc) && + ios->power_mode == MMC_POWER_OFF) + mmc_regulator_set_ocr(mmc, mmc->supply.vmmc, 0); + return; + } + + turning_on_clk = ios->clock != host->clock && ios->clock && !host->clock; + + sdhci_set_ios_common(mmc, ios); if (!ios->clock || ios->clock != host->clock) { host->ops->set_clock(host, ios->clock); @@ -1618,14 +2399,14 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) if (host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK && host->clock) { - host->timeout_clk = host->mmc->actual_clock ? - host->mmc->actual_clock / 1000 : + host->timeout_clk = mmc->actual_clock ? + mmc->actual_clock / 1000 : host->clock / 1000; - host->mmc->max_busy_timeout = + mmc->max_busy_timeout = host->ops->get_max_timeout_count ? host->ops->get_max_timeout_count(host) : 1 << 27; - host->mmc->max_busy_timeout /= host->timeout_clk; + mmc->max_busy_timeout /= host->timeout_clk; } } @@ -1639,27 +2420,52 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) host->ops->set_bus_width(host, ios->bus_width); + /* + * Special case to avoid multiple clock changes during voltage + * switching. + */ + if (!reinit_uhs && + turning_on_clk && + host->timing == ios->timing && + host->version >= SDHCI_SPEC_300 && + !sdhci_presetable_values_change(host, ios)) + return; + ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); - if ((ios->timing == MMC_TIMING_SD_HS || - ios->timing == MMC_TIMING_MMC_HS || - ios->timing == MMC_TIMING_MMC_HS400 || - ios->timing == MMC_TIMING_MMC_HS200 || - ios->timing == MMC_TIMING_MMC_DDR52 || - ios->timing == MMC_TIMING_UHS_SDR50 || - ios->timing == MMC_TIMING_UHS_SDR104 || - ios->timing == MMC_TIMING_UHS_DDR50 || - ios->timing == MMC_TIMING_UHS_SDR25) - && !(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) - ctrl |= SDHCI_CTRL_HISPD; - else - ctrl &= ~SDHCI_CTRL_HISPD; + if (!(host->quirks & SDHCI_QUIRK_NO_HISPD_BIT)) { + if (ios->timing == MMC_TIMING_SD_HS || + ios->timing == MMC_TIMING_MMC_HS || + ios->timing == MMC_TIMING_MMC_HS400 || + ios->timing == MMC_TIMING_MMC_HS200 || + ios->timing == MMC_TIMING_MMC_DDR52 || + ios->timing == MMC_TIMING_UHS_SDR50 || + ios->timing == MMC_TIMING_UHS_SDR104 || + ios->timing == MMC_TIMING_UHS_DDR50 || + ios->timing == MMC_TIMING_UHS_SDR25) + ctrl |= SDHCI_CTRL_HISPD; + else + ctrl &= ~SDHCI_CTRL_HISPD; + } if (host->version >= SDHCI_SPEC_300) { u16 clk, ctrl_2; + /* + * According to SDHCI Spec v3.00, if the Preset Value + * Enable in the Host Control 2 register is set, we + * need to reset SD Clock Enable before changing High + * Speed Enable to avoid generating clock glitches. + */ + clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL); + if (clk & SDHCI_CLOCK_CARD_EN) { + clk &= ~SDHCI_CLOCK_CARD_EN; + sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL); + } + + sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); + if (!host->preset_enabled) { - sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); /* * We only need to set Driver Strength if the * preset value enable is not set. @@ -1681,62 +2487,26 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios) } sdhci_writew(host, ctrl_2, SDHCI_HOST_CONTROL2); - } else { - /* - * According to SDHC Spec v3.00, if the Preset Value - * Enable in the Host Control 2 register is set, we - * need to reset SD Clock Enable before changing High - * Speed Enable to avoid generating clock gliches. - */ - - /* Reset SD Clock Enable */ - clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL); - clk &= ~SDHCI_CLOCK_CARD_EN; - sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL); - - sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); - - /* Re-enable SD Clock */ - host->ops->set_clock(host, host->clock); + host->drv_type = ios->drv_type; } - /* Reset SD Clock Enable */ - clk = sdhci_readw(host, SDHCI_CLOCK_CONTROL); - clk &= ~SDHCI_CLOCK_CARD_EN; - sdhci_writew(host, clk, SDHCI_CLOCK_CONTROL); - host->ops->set_uhs_signaling(host, ios->timing); host->timing = ios->timing; - if (!(host->quirks2 & SDHCI_QUIRK2_PRESET_VALUE_BROKEN) && - ((ios->timing == MMC_TIMING_UHS_SDR12) || - (ios->timing == MMC_TIMING_UHS_SDR25) || - (ios->timing == MMC_TIMING_UHS_SDR50) || - (ios->timing == MMC_TIMING_UHS_SDR104) || - (ios->timing == MMC_TIMING_UHS_DDR50) || - (ios->timing == MMC_TIMING_MMC_DDR52))) { + if (sdhci_preset_needed(host, ios->timing)) { u16 preset; sdhci_enable_preset_value(host, true); preset = sdhci_get_preset_value(host); - ios->drv_type = (preset & SDHCI_PRESET_DRV_MASK) - >> SDHCI_PRESET_DRV_SHIFT; + ios->drv_type = FIELD_GET(SDHCI_PRESET_DRV_MASK, + preset); + host->drv_type = ios->drv_type; } /* Re-enable SD Clock */ host->ops->set_clock(host, host->clock); } else sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); - - /* - * Some (ENE) controllers go apeshit on some ios operation, - * signalling timeout and CRC errors even on CMD0. Resetting - * it on each ios seems to solve the problem. - */ - if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS) - sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); - - mmiowb(); } EXPORT_SYMBOL_GPL(sdhci_set_ios); @@ -1749,7 +2519,7 @@ static int sdhci_get_cd(struct mmc_host *mmc) return 0; /* If nonremovable, assume that the card is always present. */ - if (!mmc_card_is_removable(host->mmc)) + if (!mmc_card_is_removable(mmc)) return 1; /* @@ -1767,48 +2537,53 @@ static int sdhci_get_cd(struct mmc_host *mmc) return !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT); } -static int sdhci_check_ro(struct sdhci_host *host) +int sdhci_get_cd_nogpio(struct mmc_host *mmc) { + struct sdhci_host *host = mmc_priv(mmc); unsigned long flags; - int is_readonly; + int ret = 0; spin_lock_irqsave(&host->lock, flags); if (host->flags & SDHCI_DEVICE_DEAD) - is_readonly = 0; - else if (host->ops->get_ro) - is_readonly = host->ops->get_ro(host); - else - is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE) - & SDHCI_WRITE_PROTECT); + goto out; + ret = !!(sdhci_readl(host, SDHCI_PRESENT_STATE) & SDHCI_CARD_PRESENT); +out: spin_unlock_irqrestore(&host->lock, flags); - /* This quirk needs to be replaced by a callback-function later */ - return host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT ? - !is_readonly : is_readonly; + return ret; } +EXPORT_SYMBOL_GPL(sdhci_get_cd_nogpio); -#define SAMPLE_COUNT 5 - -static int sdhci_get_ro(struct mmc_host *mmc) +int sdhci_get_ro(struct mmc_host *mmc) { struct sdhci_host *host = mmc_priv(mmc); - int i, ro_count; - - if (!(host->quirks & SDHCI_QUIRK_UNSTABLE_RO_DETECT)) - return sdhci_check_ro(host); + bool allow_invert = false; + int is_readonly; - ro_count = 0; - for (i = 0; i < SAMPLE_COUNT; i++) { - if (sdhci_check_ro(host)) { - if (++ro_count > SAMPLE_COUNT / 2) - return 1; - } - msleep(30); + if (host->flags & SDHCI_DEVICE_DEAD) { + is_readonly = 0; + } else if (host->ops->get_ro) { + is_readonly = host->ops->get_ro(host); + } else if (mmc_host_can_gpio_ro(mmc)) { + is_readonly = mmc_gpio_get_ro(mmc); + /* Do not invert twice */ + allow_invert = !(mmc->caps2 & MMC_CAP2_RO_ACTIVE_HIGH); + } else { + is_readonly = !(sdhci_readl(host, SDHCI_PRESENT_STATE) + & SDHCI_WRITE_PROTECT); + allow_invert = true; } - return 0; + + if (is_readonly >= 0 && + allow_invert && + (host->quirks & SDHCI_QUIRK_INVERTED_WRITE_PROTECT)) + is_readonly = !is_readonly; + + return is_readonly; } +EXPORT_SYMBOL_GPL(sdhci_get_ro); static void sdhci_hw_reset(struct mmc_host *mmc) { @@ -1828,7 +2603,6 @@ static void sdhci_enable_sdio_irq_nolock(struct sdhci_host *host, int enable) sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); - mmiowb(); } } @@ -1838,22 +2612,27 @@ void sdhci_enable_sdio_irq(struct mmc_host *mmc, int enable) unsigned long flags; if (enable) - pm_runtime_get_noresume(host->mmc->parent); + pm_runtime_get_noresume(mmc_dev(mmc)); spin_lock_irqsave(&host->lock, flags); - if (enable) - host->flags |= SDHCI_SDIO_IRQ_ENABLED; - else - host->flags &= ~SDHCI_SDIO_IRQ_ENABLED; - sdhci_enable_sdio_irq_nolock(host, enable); spin_unlock_irqrestore(&host->lock, flags); if (!enable) - pm_runtime_put_noidle(host->mmc->parent); + pm_runtime_put_noidle(mmc_dev(mmc)); } EXPORT_SYMBOL_GPL(sdhci_enable_sdio_irq); +static void sdhci_ack_sdio_irq(struct mmc_host *mmc) +{ + struct sdhci_host *host = mmc_priv(mmc); + unsigned long flags; + + spin_lock_irqsave(&host->lock, flags); + sdhci_enable_sdio_irq_nolock(host, true); + spin_unlock_irqrestore(&host->lock, flags); +} + int sdhci_start_signal_voltage_switch(struct mmc_host *mmc, struct mmc_ios *ios) { @@ -1880,7 +2659,7 @@ int sdhci_start_signal_voltage_switch(struct mmc_host *mmc, if (!IS_ERR(mmc->supply.vqmmc)) { ret = mmc_regulator_set_vqmmc(mmc, ios); - if (ret) { + if (ret < 0) { pr_warn("%s: Switching to 3.3V signalling voltage failed\n", mmc_hostname(mmc)); return -EIO; @@ -1894,7 +2673,7 @@ int sdhci_start_signal_voltage_switch(struct mmc_host *mmc, if (!(ctrl & SDHCI_CTRL_VDD_180)) return 0; - pr_warn("%s: 3.3V regulator output did not became stable\n", + pr_warn("%s: 3.3V regulator output did not become stable\n", mmc_hostname(mmc)); return -EAGAIN; @@ -1903,7 +2682,7 @@ int sdhci_start_signal_voltage_switch(struct mmc_host *mmc, return -EINVAL; if (!IS_ERR(mmc->supply.vqmmc)) { ret = mmc_regulator_set_vqmmc(mmc, ios); - if (ret) { + if (ret < 0) { pr_warn("%s: Switching to 1.8V signalling voltage failed\n", mmc_hostname(mmc)); return -EIO; @@ -1926,7 +2705,7 @@ int sdhci_start_signal_voltage_switch(struct mmc_host *mmc, if (ctrl & SDHCI_CTRL_VDD_180) return 0; - pr_warn("%s: 1.8V regulator output did not became stable\n", + pr_warn("%s: 1.8V regulator output did not become stable\n", mmc_hostname(mmc)); return -EAGAIN; @@ -1935,7 +2714,7 @@ int sdhci_start_signal_voltage_switch(struct mmc_host *mmc, return -EINVAL; if (!IS_ERR(mmc->supply.vqmmc)) { ret = mmc_regulator_set_vqmmc(mmc, ios); - if (ret) { + if (ret < 0) { pr_warn("%s: Switching to 1.2V signalling voltage failed\n", mmc_hostname(mmc)); return -EIO; @@ -1972,7 +2751,7 @@ static int sdhci_prepare_hs400_tuning(struct mmc_host *mmc, struct mmc_ios *ios) return 0; } -static void sdhci_start_tuning(struct sdhci_host *host) +void sdhci_start_tuning(struct sdhci_host *host) { u16 ctrl; @@ -1995,14 +2774,16 @@ static void sdhci_start_tuning(struct sdhci_host *host) sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_INT_ENABLE); sdhci_writel(host, SDHCI_INT_DATA_AVAIL, SDHCI_SIGNAL_ENABLE); } +EXPORT_SYMBOL_GPL(sdhci_start_tuning); -static void sdhci_end_tuning(struct sdhci_host *host) +void sdhci_end_tuning(struct sdhci_host *host) { sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); } +EXPORT_SYMBOL_GPL(sdhci_end_tuning); -static void sdhci_reset_tuning(struct sdhci_host *host) +void sdhci_reset_tuning(struct sdhci_host *host) { u16 ctrl; @@ -2011,18 +2792,19 @@ static void sdhci_reset_tuning(struct sdhci_host *host) ctrl &= ~SDHCI_CTRL_EXEC_TUNING; sdhci_writew(host, ctrl, SDHCI_HOST_CONTROL2); } +EXPORT_SYMBOL_GPL(sdhci_reset_tuning); -static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode) +void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode) { sdhci_reset_tuning(host); - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); + sdhci_reset_for(host, TUNING_ABORT); sdhci_end_tuning(host); - mmc_abort_tuning(host->mmc, opcode); + mmc_send_abort_tuning(host->mmc, opcode); } +EXPORT_SYMBOL_GPL(sdhci_abort_tuning); /* * We use sdhci_send_tuning() because mmc_send_tuning() is not a good fit. SDHCI @@ -2031,12 +2813,13 @@ static void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode) * interrupt setup is different to other commands and there is no timeout * interrupt so special handling is needed. */ -static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode) +void sdhci_send_tuning(struct sdhci_host *host, u32 opcode) { struct mmc_host *mmc = host->mmc; struct mmc_command cmd = {}; struct mmc_request mrq = {}; unsigned long flags; + u32 b = host->sdma_boundary; spin_lock_irqsave(&host->lock, flags); @@ -2052,9 +2835,9 @@ static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode) */ if (cmd.opcode == MMC_SEND_TUNING_BLOCK_HS200 && mmc->ios.bus_width == MMC_BUS_WIDTH_8) - sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 128), SDHCI_BLOCK_SIZE); + sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 128), SDHCI_BLOCK_SIZE); else - sdhci_writew(host, SDHCI_MAKE_BLKSZ(7, 64), SDHCI_BLOCK_SIZE); + sdhci_writew(host, SDHCI_MAKE_BLKSZ(b, 64), SDHCI_BLOCK_SIZE); /* * The tuning block is sent by the card to the host controller. @@ -2064,7 +2847,11 @@ static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode) */ sdhci_writew(host, SDHCI_TRNS_READ, SDHCI_TRANSFER_MODE); - sdhci_send_command(host, &cmd); + if (!sdhci_send_command_retry(host, &cmd, flags)) { + spin_unlock_irqrestore(&host->lock, flags); + host->tuning_done = 0; + return; + } host->cmd = NULL; @@ -2072,7 +2859,6 @@ static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode) host->tuning_done = 0; - mmiowb(); spin_unlock_irqrestore(&host->lock, flags); /* Wait for Buffer Read Ready interrupt */ @@ -2080,43 +2866,47 @@ static void sdhci_send_tuning(struct sdhci_host *host, u32 opcode) msecs_to_jiffies(50)); } +EXPORT_SYMBOL_GPL(sdhci_send_tuning); -static void __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode) +int __sdhci_execute_tuning(struct sdhci_host *host, u32 opcode) { int i; /* * Issue opcode repeatedly till Execute Tuning is set to 0 or the number - * of loops reaches 40 times. + * of loops reaches tuning loop count. */ - for (i = 0; i < MAX_TUNING_LOOP; i++) { + for (i = 0; i < host->tuning_loop_count; i++) { u16 ctrl; sdhci_send_tuning(host, opcode); if (!host->tuning_done) { - pr_info("%s: Tuning timeout, falling back to fixed sampling clock\n", - mmc_hostname(host->mmc)); + pr_debug("%s: Tuning timeout, falling back to fixed sampling clock\n", + mmc_hostname(host->mmc)); sdhci_abort_tuning(host, opcode); - return; + return -ETIMEDOUT; } + /* Spec does not require a delay between tuning cycles */ + if (host->tuning_delay > 0) + mdelay(host->tuning_delay); + ctrl = sdhci_readw(host, SDHCI_HOST_CONTROL2); if (!(ctrl & SDHCI_CTRL_EXEC_TUNING)) { if (ctrl & SDHCI_CTRL_TUNED_CLK) - return; /* Success! */ + return 0; /* Success! */ break; } - /* Spec does not require a delay between tuning cycles */ - if (host->tuning_delay > 0) - mdelay(host->tuning_delay); } pr_info("%s: Tuning failed, falling back to fixed sampling clock\n", mmc_hostname(host->mmc)); sdhci_reset_tuning(host); + return -EAGAIN; } +EXPORT_SYMBOL_GPL(__sdhci_execute_tuning); int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) { @@ -2159,7 +2949,7 @@ int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) case MMC_TIMING_UHS_SDR50: if (host->flags & SDHCI_SDR50_NEEDS_TUNING) break; - /* FALLTHROUGH */ + fallthrough; default: goto out; @@ -2170,14 +2960,14 @@ int sdhci_execute_tuning(struct mmc_host *mmc, u32 opcode) goto out; } - host->mmc->retune_period = tuning_count; + mmc->retune_period = tuning_count; if (host->tuning_delay < 0) host->tuning_delay = opcode == MMC_SEND_TUNING_BLOCK; sdhci_start_tuning(host); - __sdhci_execute_tuning(host, opcode); + host->tuning_err = __sdhci_execute_tuning(host, opcode); sdhci_end_tuning(host); out: @@ -2187,7 +2977,7 @@ out: } EXPORT_SYMBOL_GPL(sdhci_execute_tuning); -static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable) +void sdhci_enable_preset_value(struct sdhci_host *host, bool enable) { /* Host Controller v3.00 defines preset value registers */ if (host->version < SDHCI_SPEC_300) @@ -2215,15 +3005,15 @@ static void sdhci_enable_preset_value(struct sdhci_host *host, bool enable) host->preset_enabled = enable; } } +EXPORT_SYMBOL_GPL(sdhci_enable_preset_value); static void sdhci_post_req(struct mmc_host *mmc, struct mmc_request *mrq, int err) { - struct sdhci_host *host = mmc_priv(mmc); struct mmc_data *data = mrq->data; if (data->host_cookie != COOKIE_UNMAPPED) - dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len, + dma_unmap_sg(mmc_dev(mmc), data->sg, data->sg_len, mmc_get_dma_dir(data)); data->host_cookie = COOKIE_UNMAPPED; @@ -2235,15 +3025,15 @@ static void sdhci_pre_req(struct mmc_host *mmc, struct mmc_request *mrq) mrq->data->host_cookie = COOKIE_UNMAPPED; - if (host->flags & SDHCI_REQ_USE_DMA) + /* + * No pre-mapping in the pre hook if we're using the bounce buffer, + * for that we would need two bounce buffers since one buffer is + * in flight when this is getting called. + */ + if (host->flags & SDHCI_REQ_USE_DMA && !host->bounce_buffer) sdhci_pre_dma_transfer(host, mrq->data, COOKIE_PRE_MAPPED); } -static inline bool sdhci_has_requests(struct sdhci_host *host) -{ - return host->cmd || host->data_cmd; -} - static void sdhci_error_out_mrqs(struct sdhci_host *host, int err) { if (host->data_cmd) { @@ -2274,12 +3064,11 @@ static void sdhci_card_event(struct mmc_host *mmc) /* Check sdhci_has_requests() first in case we are runtime suspended */ if (sdhci_has_requests(host) && !present) { pr_err("%s: Card removed during transfer!\n", - mmc_hostname(host->mmc)); + mmc_hostname(mmc)); pr_err("%s: Resetting controller.\n", - mmc_hostname(host->mmc)); + mmc_hostname(mmc)); - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); + sdhci_reset_for(host, CARD_REMOVED); sdhci_error_out_mrqs(host, -ENOMEDIUM); } @@ -2294,8 +3083,9 @@ static const struct mmc_host_ops sdhci_ops = { .set_ios = sdhci_set_ios, .get_cd = sdhci_get_cd, .get_ro = sdhci_get_ro, - .hw_reset = sdhci_hw_reset, + .card_hw_reset = sdhci_hw_reset, .enable_sdio_irq = sdhci_enable_sdio_irq, + .ack_sdio_irq = sdhci_ack_sdio_irq, .start_signal_voltage_switch = sdhci_start_signal_voltage_switch, .prepare_hs400_tuning = sdhci_prepare_hs400_tuning, .execute_tuning = sdhci_execute_tuning, @@ -2305,10 +3095,57 @@ static const struct mmc_host_ops sdhci_ops = { /*****************************************************************************\ * * - * Tasklets * + * Request done * * * \*****************************************************************************/ +void sdhci_request_done_dma(struct sdhci_host *host, struct mmc_request *mrq) +{ + struct mmc_data *data = mrq->data; + + if (data && data->host_cookie == COOKIE_MAPPED) { + if (host->bounce_buffer) { + /* + * On reads, copy the bounced data into the + * sglist + */ + if (mmc_get_dma_dir(data) == DMA_FROM_DEVICE) { + unsigned int length = data->bytes_xfered; + + if (length > host->bounce_buffer_size) { + pr_err("%s: bounce buffer is %u bytes but DMA claims to have transferred %u bytes\n", + mmc_hostname(host->mmc), + host->bounce_buffer_size, + data->bytes_xfered); + /* Cap it down and continue */ + length = host->bounce_buffer_size; + } + dma_sync_single_for_cpu(mmc_dev(host->mmc), + host->bounce_addr, + host->bounce_buffer_size, + DMA_FROM_DEVICE); + sg_copy_from_buffer(data->sg, + data->sg_len, + host->bounce_buffer, + length); + } else { + /* No copying, just switch ownership */ + dma_sync_single_for_cpu(mmc_dev(host->mmc), + host->bounce_addr, + host->bounce_buffer_size, + mmc_get_dma_dir(data)); + } + } else { + /* Unmap the raw data */ + dma_unmap_sg(mmc_dev(host->mmc), data->sg, + data->sg_len, + mmc_get_dma_dir(data)); + } + data->host_cookie = COOKIE_UNMAPPED; + } +} +EXPORT_SYMBOL_GPL(sdhci_request_done_dma); + static bool sdhci_request_done(struct sdhci_host *host) { unsigned long flags; @@ -2328,23 +3165,6 @@ static bool sdhci_request_done(struct sdhci_host *host) return true; } - sdhci_del_timer(host, mrq); - - /* - * Always unmap the data buffers if they were mapped by - * sdhci_prepare_data() whenever we finish with a request. - * This avoids leaking DMA mappings on error. - */ - if (host->flags & SDHCI_REQ_USE_DMA) { - struct mmc_data *data = mrq->data; - - if (data && data->host_cookie == COOKIE_MAPPED) { - dma_unmap_sg(mmc_dev(host->mmc), data->sg, data->sg_len, - mmc_get_dma_dir(data)); - data->host_cookie = COOKIE_UNMAPPED; - } - } - /* * The controller needs a reset of internal state machines * upon error conditions. @@ -2366,63 +3186,83 @@ static bool sdhci_request_done(struct sdhci_host *host) /* This is to force an update */ host->ops->set_clock(host, host->clock); - /* Spec says we should do both at the same time, but Ricoh - controllers do not like that. */ - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); + sdhci_reset_for(host, REQUEST_ERROR); host->pending_reset = false; } - if (!sdhci_has_requests(host)) - sdhci_led_deactivate(host); + /* + * Always unmap the data buffers if they were mapped by + * sdhci_prepare_data() whenever we finish with a request. + * This avoids leaking DMA mappings on error. + */ + if (host->flags & SDHCI_REQ_USE_DMA) { + struct mmc_data *data = mrq->data; + + if (host->use_external_dma && data && + (mrq->cmd->error || data->error)) { + struct dma_chan *chan = sdhci_external_dma_channel(host, data); + + host->mrqs_done[i] = NULL; + spin_unlock_irqrestore(&host->lock, flags); + dmaengine_terminate_sync(chan); + spin_lock_irqsave(&host->lock, flags); + sdhci_set_mrq_done(host, mrq); + } + + sdhci_request_done_dma(host, mrq); + } host->mrqs_done[i] = NULL; - mmiowb(); spin_unlock_irqrestore(&host->lock, flags); - mmc_request_done(host->mmc, mrq); + if (host->ops->request_done) + host->ops->request_done(host, mrq); + else + mmc_request_done(host->mmc, mrq); return false; } -static void sdhci_tasklet_finish(unsigned long param) +void sdhci_complete_work(struct work_struct *work) { - struct sdhci_host *host = (struct sdhci_host *)param; + struct sdhci_host *host = container_of(work, struct sdhci_host, + complete_work); while (!sdhci_request_done(host)) ; } +EXPORT_SYMBOL_GPL(sdhci_complete_work); -static void sdhci_timeout_timer(unsigned long data) +static void sdhci_timeout_timer(struct timer_list *t) { struct sdhci_host *host; unsigned long flags; - host = (struct sdhci_host*)data; + host = timer_container_of(host, t, timer); spin_lock_irqsave(&host->lock, flags); if (host->cmd && !sdhci_data_line_cmd(host->cmd)) { pr_err("%s: Timeout waiting for hardware cmd interrupt.\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, REQ_TIMEOUT); sdhci_dumpregs(host); host->cmd->error = -ETIMEDOUT; sdhci_finish_mrq(host, host->cmd->mrq); } - mmiowb(); spin_unlock_irqrestore(&host->lock, flags); } -static void sdhci_timeout_data_timer(unsigned long data) +static void sdhci_timeout_data_timer(struct timer_list *t) { struct sdhci_host *host; unsigned long flags; - host = (struct sdhci_host *)data; + host = timer_container_of(host, t, data_timer); spin_lock_irqsave(&host->lock, flags); @@ -2430,11 +3270,13 @@ static void sdhci_timeout_data_timer(unsigned long data) (host->cmd && sdhci_data_line_cmd(host->cmd))) { pr_err("%s: Timeout waiting for hardware interrupt.\n", mmc_hostname(host->mmc)); + sdhci_err_stats_inc(host, REQ_TIMEOUT); sdhci_dumpregs(host); if (host->data) { host->data->error = -ETIMEDOUT; - sdhci_finish_data(host); + __sdhci_finish_data(host, true); + queue_work(host->complete_wq, &host->complete_work); } else if (host->data_cmd) { host->data_cmd->error = -ETIMEDOUT; sdhci_finish_mrq(host, host->data_cmd->mrq); @@ -2444,7 +3286,6 @@ static void sdhci_timeout_data_timer(unsigned long data) } } - mmiowb(); spin_unlock_irqrestore(&host->lock, flags); } @@ -2454,8 +3295,23 @@ static void sdhci_timeout_data_timer(unsigned long data) * * \*****************************************************************************/ -static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask) +static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask, u32 *intmask_p) { + /* Handle auto-CMD12 error */ + if (intmask & SDHCI_INT_AUTO_CMD_ERR && host->data_cmd) { + struct mmc_request *mrq = host->data_cmd->mrq; + u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS); + int data_err_bit = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ? + SDHCI_INT_DATA_TIMEOUT : + SDHCI_INT_DATA_CRC; + + /* Treat auto-CMD12 error the same as data error */ + if (!mrq->sbc && (host->flags & SDHCI_AUTO_CMD12)) { + *intmask_p |= data_err_bit; + return; + } + } + if (!host->cmd) { /* * SDHCI recovers from errors by resetting the cmd and data @@ -2466,46 +3322,59 @@ static void sdhci_cmd_irq(struct sdhci_host *host, u32 intmask) return; pr_err("%s: Got command interrupt 0x%08x even though no command operation was in progress.\n", mmc_hostname(host->mmc), (unsigned)intmask); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); return; } if (intmask & (SDHCI_INT_TIMEOUT | SDHCI_INT_CRC | SDHCI_INT_END_BIT | SDHCI_INT_INDEX)) { - if (intmask & SDHCI_INT_TIMEOUT) + if (intmask & SDHCI_INT_TIMEOUT) { host->cmd->error = -ETIMEDOUT; - else + sdhci_err_stats_inc(host, CMD_TIMEOUT); + } else { host->cmd->error = -EILSEQ; - - /* - * If this command initiates a data phase and a response - * CRC error is signalled, the card can start transferring - * data - the card may have received the command without - * error. We must not terminate the mmc_request early. - * - * If the card did not receive the command or returned an - * error which prevented it sending data, the data phase - * will time out. - */ + if (!mmc_op_tuning(host->cmd->opcode)) + sdhci_err_stats_inc(host, CMD_CRC); + } + /* Treat data command CRC error the same as data CRC error */ if (host->cmd->data && (intmask & (SDHCI_INT_CRC | SDHCI_INT_TIMEOUT)) == SDHCI_INT_CRC) { host->cmd = NULL; + *intmask_p |= SDHCI_INT_DATA_CRC; return; } - sdhci_finish_mrq(host, host->cmd->mrq); + __sdhci_finish_mrq(host, host->cmd->mrq); return; } + /* Handle auto-CMD23 error */ + if (intmask & SDHCI_INT_AUTO_CMD_ERR) { + struct mmc_request *mrq = host->cmd->mrq; + u16 auto_cmd_status = sdhci_readw(host, SDHCI_AUTO_CMD_STATUS); + int err = (auto_cmd_status & SDHCI_AUTO_CMD_TIMEOUT) ? + -ETIMEDOUT : + -EILSEQ; + + sdhci_err_stats_inc(host, AUTO_CMD); + + if (sdhci_auto_cmd23(host, mrq)) { + mrq->sbc->error = err; + __sdhci_finish_mrq(host, mrq); + return; + } + } + if (intmask & SDHCI_INT_RESPONSE) sdhci_finish_command(host); } -#ifdef CONFIG_MMC_DEBUG static void sdhci_adma_show_error(struct sdhci_host *host) { void *desc = host->adma_table; + dma_addr_t dma = host->adma_addr; sdhci_dumpregs(host); @@ -2513,36 +3382,38 @@ static void sdhci_adma_show_error(struct sdhci_host *host) struct sdhci_adma2_64_desc *dma_desc = desc; if (host->flags & SDHCI_USE_64_BIT_DMA) - DBG("%p: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n", - desc, le32_to_cpu(dma_desc->addr_hi), + SDHCI_DUMP("%08llx: DMA 0x%08x%08x, LEN 0x%04x, Attr=0x%02x\n", + (unsigned long long)dma, + le32_to_cpu(dma_desc->addr_hi), le32_to_cpu(dma_desc->addr_lo), le16_to_cpu(dma_desc->len), le16_to_cpu(dma_desc->cmd)); else - DBG("%p: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n", - desc, le32_to_cpu(dma_desc->addr_lo), + SDHCI_DUMP("%08llx: DMA 0x%08x, LEN 0x%04x, Attr=0x%02x\n", + (unsigned long long)dma, + le32_to_cpu(dma_desc->addr_lo), le16_to_cpu(dma_desc->len), le16_to_cpu(dma_desc->cmd)); desc += host->desc_sz; + dma += host->desc_sz; if (dma_desc->cmd & cpu_to_le16(ADMA2_END)) break; } } -#else -static void sdhci_adma_show_error(struct sdhci_host *host) { } -#endif static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) { - u32 command; - - /* CMD19 generates _only_ Buffer Read Ready interrupt */ - if (intmask & SDHCI_INT_DATA_AVAIL) { - command = SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)); - if (command == MMC_SEND_TUNING_BLOCK || - command == MMC_SEND_TUNING_BLOCK_HS200) { + /* + * CMD19 generates _only_ Buffer Read Ready interrupt if + * use sdhci_send_tuning. + * Need to exclude this case: PIO mode and use mmc_send_tuning, + * If not, sdhci_transfer_pio will never be called, make the + * SDHCI_INT_DATA_AVAIL always there, stuck in irq storm. + */ + if (intmask & SDHCI_INT_DATA_AVAIL && !host->data) { + if (mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) { host->tuning_done = 1; wake_up(&host->buf_ready_int); return; @@ -2561,7 +3432,8 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) if (intmask & SDHCI_INT_DATA_TIMEOUT) { host->data_cmd = NULL; data_cmd->error = -ETIMEDOUT; - sdhci_finish_mrq(host, data_cmd->mrq); + sdhci_err_stats_inc(host, CMD_TIMEOUT); + __sdhci_finish_mrq(host, data_cmd->mrq); return; } if (intmask & SDHCI_INT_DATA_END) { @@ -2574,7 +3446,7 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) if (host->cmd == data_cmd) return; - sdhci_finish_mrq(host, data_cmd->mrq); + __sdhci_finish_mrq(host, data_cmd->mrq); return; } } @@ -2589,22 +3461,36 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) pr_err("%s: Got data interrupt 0x%08x even though no data operation was in progress.\n", mmc_hostname(host->mmc), (unsigned)intmask); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); return; } - if (intmask & SDHCI_INT_DATA_TIMEOUT) + if (intmask & SDHCI_INT_DATA_TIMEOUT) { host->data->error = -ETIMEDOUT; - else if (intmask & SDHCI_INT_DATA_END_BIT) + sdhci_err_stats_inc(host, DAT_TIMEOUT); + } else if (intmask & SDHCI_INT_DATA_END_BIT) { host->data->error = -EILSEQ; - else if ((intmask & SDHCI_INT_DATA_CRC) && + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) + sdhci_err_stats_inc(host, DAT_CRC); + } else if ((intmask & (SDHCI_INT_DATA_CRC | SDHCI_INT_TUNING_ERROR)) && SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)) - != MMC_BUS_TEST_R) + != MMC_BUS_TEST_R) { host->data->error = -EILSEQ; - else if (intmask & SDHCI_INT_ADMA_ERROR) { - pr_err("%s: ADMA error\n", mmc_hostname(host->mmc)); + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) + sdhci_err_stats_inc(host, DAT_CRC); + if (intmask & SDHCI_INT_TUNING_ERROR) { + u16 ctrl2 = sdhci_readw(host, SDHCI_HOST_CONTROL2); + + ctrl2 &= ~SDHCI_CTRL_TUNED_CLK; + sdhci_writew(host, ctrl2, SDHCI_HOST_CONTROL2); + } + } else if (intmask & SDHCI_INT_ADMA_ERROR) { + pr_err("%s: ADMA error: 0x%08x\n", mmc_hostname(host->mmc), + intmask); sdhci_adma_show_error(host); + sdhci_err_stats_inc(host, ADMA); host->data->error = -EIO; if (host->ops->adma_workaround) host->ops->adma_workaround(host, intmask); @@ -2626,19 +3512,20 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) * some controllers are faulty, don't trust them. */ if (intmask & SDHCI_INT_DMA_END) { - u32 dmastart, dmanow; - dmastart = sg_dma_address(host->data->sg); + dma_addr_t dmastart, dmanow; + + dmastart = sdhci_sdma_address(host); dmanow = dmastart + host->data->bytes_xfered; /* * Force update to the next DMA block boundary. */ dmanow = (dmanow & - ~(SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) + + ~((dma_addr_t)SDHCI_DEFAULT_BOUNDARY_SIZE - 1)) + SDHCI_DEFAULT_BOUNDARY_SIZE; host->data->bytes_xfered = dmanow - dmastart; - DBG("DMA base 0x%08x, transferred 0x%06x bytes, next 0x%08x\n", - dmastart, host->data->bytes_xfered, dmanow); - sdhci_writel(host, dmanow, SDHCI_DMA_ADDRESS); + DBG("DMA base %pad, transferred 0x%06x bytes, next %pad\n", + &dmastart, host->data->bytes_xfered, &dmanow); + sdhci_set_sdma_addr(host, dmanow); } if (intmask & SDHCI_INT_DATA_END) { @@ -2656,16 +3543,28 @@ static void sdhci_data_irq(struct sdhci_host *host, u32 intmask) } } +static inline bool sdhci_defer_done(struct sdhci_host *host, + struct mmc_request *mrq) +{ + struct mmc_data *data = mrq->data; + + return host->pending_reset || host->always_defer_done || + ((host->flags & SDHCI_REQ_USE_DMA) && data && + data->host_cookie == COOKIE_MAPPED); +} + static irqreturn_t sdhci_irq(int irq, void *dev_id) { + struct mmc_request *mrqs_done[SDHCI_MAX_MRQS] = {0}; irqreturn_t result = IRQ_NONE; struct sdhci_host *host = dev_id; u32 intmask, mask, unexpected = 0; int max_loops = 16; + int i; spin_lock(&host->lock); - if (host->runtime_suspended && !sdhci_sdio_irq_enabled(host)) { + if (host->runtime_suspended) { spin_unlock(&host->lock); return IRQ_NONE; } @@ -2721,7 +3620,7 @@ static irqreturn_t sdhci_irq(int irq, void *dev_id) } if (intmask & SDHCI_INT_CMD_MASK) - sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK); + sdhci_cmd_irq(host, intmask & SDHCI_INT_CMD_MASK, &intmask); if (intmask & SDHCI_INT_DATA_MASK) sdhci_data_irq(host, intmask & SDHCI_INT_DATA_MASK); @@ -2736,8 +3635,7 @@ static irqreturn_t sdhci_irq(int irq, void *dev_id) if ((intmask & SDHCI_INT_CARD_INT) && (host->ier & SDHCI_INT_CARD_INT)) { sdhci_enable_sdio_irq_nolock(host, false); - host->thread_isr |= SDHCI_INT_CARD_INT; - result = IRQ_WAKE_THREAD; + sdio_signal_irq(host->mmc); } intmask &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE | @@ -2755,27 +3653,67 @@ cont: intmask = sdhci_readl(host, SDHCI_INT_STATUS); } while (intmask && --max_loops); + + /* Determine if mrqs can be completed immediately */ + for (i = 0; i < SDHCI_MAX_MRQS; i++) { + struct mmc_request *mrq = host->mrqs_done[i]; + + if (!mrq) + continue; + + if (sdhci_defer_done(host, mrq)) { + result = IRQ_WAKE_THREAD; + } else { + mrqs_done[i] = mrq; + host->mrqs_done[i] = NULL; + } + } out: + if (host->deferred_cmd) + result = IRQ_WAKE_THREAD; + spin_unlock(&host->lock); + /* Process mrqs ready for immediate completion */ + for (i = 0; i < SDHCI_MAX_MRQS; i++) { + if (!mrqs_done[i]) + continue; + + if (host->ops->request_done) + host->ops->request_done(host, mrqs_done[i]); + else + mmc_request_done(host->mmc, mrqs_done[i]); + } + if (unexpected) { pr_err("%s: Unexpected interrupt 0x%08x.\n", mmc_hostname(host->mmc), unexpected); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); } return result; } -static irqreturn_t sdhci_thread_irq(int irq, void *dev_id) +irqreturn_t sdhci_thread_irq(int irq, void *dev_id) { struct sdhci_host *host = dev_id; + struct mmc_command *cmd; unsigned long flags; u32 isr; + while (!sdhci_request_done(host)) + ; + spin_lock_irqsave(&host->lock, flags); + isr = host->thread_isr; host->thread_isr = 0; + + cmd = host->deferred_cmd; + if (cmd && !sdhci_send_command_retry(host, cmd, flags)) + sdhci_finish_mrq(host, cmd->mrq); + spin_unlock_irqrestore(&host->lock, flags); if (isr & (SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE)) { @@ -2785,17 +3723,9 @@ static irqreturn_t sdhci_thread_irq(int irq, void *dev_id) mmc_detect_change(mmc, msecs_to_jiffies(200)); } - if (isr & SDHCI_INT_CARD_INT) { - sdio_run_irqs(host->mmc); - - spin_lock_irqsave(&host->lock, flags); - if (host->flags & SDHCI_SDIO_IRQ_ENABLED) - sdhci_enable_sdio_irq_nolock(host, true); - spin_unlock_irqrestore(&host->lock, flags); - } - - return isr ? IRQ_HANDLED : IRQ_NONE; + return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(sdhci_thread_irq); /*****************************************************************************\ * * @@ -2804,6 +3734,14 @@ static irqreturn_t sdhci_thread_irq(int irq, void *dev_id) \*****************************************************************************/ #ifdef CONFIG_PM + +static bool sdhci_cd_irq_can_wakeup(struct sdhci_host *host) +{ + return mmc_card_is_removable(host->mmc) && + !(host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) && + !mmc_host_can_gpio_cd(host->mmc); +} + /* * To enable wakeup events, the corresponding events have to be enabled in * the Interrupt Status Enable register too. See 'Table 1-6: Wakeup Signal @@ -2812,27 +3750,41 @@ static irqreturn_t sdhci_thread_irq(int irq, void *dev_id) * sdhci_disable_irq_wakeups() since it will be set by * sdhci_enable_card_detection() or sdhci_init(). */ -void sdhci_enable_irq_wakeups(struct sdhci_host *host) +bool sdhci_enable_irq_wakeups(struct sdhci_host *host) { + u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE | + SDHCI_WAKE_ON_INT; + u32 irq_val = 0; + u8 wake_val = 0; u8 val; - u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE - | SDHCI_WAKE_ON_INT; - u32 irq_val = SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE | - SDHCI_INT_CARD_INT; - val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL); - val |= mask ; - /* Avoid fake wake up */ - if (host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) { - val &= ~(SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE); - irq_val &= ~(SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE); + if (sdhci_cd_irq_can_wakeup(host)) { + wake_val |= SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE; + irq_val |= SDHCI_INT_CARD_INSERT | SDHCI_INT_CARD_REMOVE; } + + if (mmc_card_wake_sdio_irq(host->mmc)) { + wake_val |= SDHCI_WAKE_ON_INT; + irq_val |= SDHCI_INT_CARD_INT; + } + + if (!irq_val) + return false; + + val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL); + val &= ~mask; + val |= wake_val; sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL); + sdhci_writel(host, irq_val, SDHCI_INT_ENABLE); + + host->irq_wake_enabled = !enable_irq_wake(host->irq); + + return host->irq_wake_enabled; } EXPORT_SYMBOL_GPL(sdhci_enable_irq_wakeups); -static void sdhci_disable_irq_wakeups(struct sdhci_host *host) +void sdhci_disable_irq_wakeups(struct sdhci_host *host) { u8 val; u8 mask = SDHCI_WAKE_ON_INSERT | SDHCI_WAKE_ON_REMOVE @@ -2841,7 +3793,12 @@ static void sdhci_disable_irq_wakeups(struct sdhci_host *host) val = sdhci_readb(host, SDHCI_WAKE_UP_CONTROL); val &= ~mask; sdhci_writeb(host, val, SDHCI_WAKE_UP_CONTROL); + + disable_irq_wake(host->irq); + + host->irq_wake_enabled = false; } +EXPORT_SYMBOL_GPL(sdhci_disable_irq_wakeups); int sdhci_suspend_host(struct sdhci_host *host) { @@ -2849,15 +3806,14 @@ int sdhci_suspend_host(struct sdhci_host *host) mmc_retune_timer_stop(host->mmc); - if (!device_may_wakeup(mmc_dev(host->mmc))) { + if (!device_may_wakeup(mmc_dev(host->mmc)) || + !sdhci_enable_irq_wakeups(host)) { host->ier = 0; sdhci_writel(host, 0, SDHCI_INT_ENABLE); sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE); free_irq(host->irq, host); - } else { - sdhci_enable_irq_wakeups(host); - enable_irq_wake(host->irq); } + return 0; } @@ -2873,27 +3829,26 @@ int sdhci_resume_host(struct sdhci_host *host) host->ops->enable_dma(host); } - if ((host->mmc->pm_flags & MMC_PM_KEEP_POWER) && + if ((mmc->pm_flags & MMC_PM_KEEP_POWER) && (host->quirks2 & SDHCI_QUIRK2_HOST_OFF_CARD_ON)) { /* Card keeps power but host controller does not */ sdhci_init(host, 0); host->pwr = 0; host->clock = 0; + host->reinit_uhs = true; mmc->ops->set_ios(mmc, &mmc->ios); } else { - sdhci_init(host, (host->mmc->pm_flags & MMC_PM_KEEP_POWER)); - mmiowb(); + sdhci_init(host, (mmc->pm_flags & MMC_PM_KEEP_POWER)); } - if (!device_may_wakeup(mmc_dev(host->mmc))) { + if (host->irq_wake_enabled) { + sdhci_disable_irq_wakeups(host); + } else { ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq, IRQF_SHARED, - mmc_hostname(host->mmc), host); + mmc_hostname(mmc), host); if (ret) return ret; - } else { - sdhci_disable_irq_wakeups(host); - disable_irq_wake(host->irq); } sdhci_enable_card_detection(host); @@ -2903,7 +3858,7 @@ int sdhci_resume_host(struct sdhci_host *host) EXPORT_SYMBOL_GPL(sdhci_resume_host); -int sdhci_runtime_suspend_host(struct sdhci_host *host) +void sdhci_runtime_suspend_host(struct sdhci_host *host) { unsigned long flags; @@ -2920,12 +3875,10 @@ int sdhci_runtime_suspend_host(struct sdhci_host *host) spin_lock_irqsave(&host->lock, flags); host->runtime_suspended = true; spin_unlock_irqrestore(&host->lock, flags); - - return 0; } EXPORT_SYMBOL_GPL(sdhci_runtime_suspend_host); -int sdhci_runtime_resume_host(struct sdhci_host *host) +void sdhci_runtime_resume_host(struct sdhci_host *host, int soft_reset) { struct mmc_host *mmc = host->mmc; unsigned long flags; @@ -2936,12 +3889,14 @@ int sdhci_runtime_resume_host(struct sdhci_host *host) host->ops->enable_dma(host); } - sdhci_init(host, 0); + sdhci_init(host, soft_reset); - if (mmc->ios.power_mode != MMC_POWER_UNDEFINED) { + if (mmc->ios.power_mode != MMC_POWER_UNDEFINED && + mmc->ios.power_mode != MMC_POWER_OFF) { /* Force clock and power re-program */ host->pwr = 0; host->clock = 0; + host->reinit_uhs = true; mmc->ops->start_signal_voltage_switch(mmc, &mmc->ios); mmc->ops->set_ios(mmc, &mmc->ios); @@ -2962,15 +3917,13 @@ int sdhci_runtime_resume_host(struct sdhci_host *host) host->runtime_suspended = false; /* Enable SDIO IRQ */ - if (host->flags & SDHCI_SDIO_IRQ_ENABLED) + if (sdio_irq_claimed(mmc)) sdhci_enable_sdio_irq_nolock(host, true); /* Enable Card Detection */ sdhci_enable_card_detection(host); spin_unlock_irqrestore(&host->lock, flags); - - return 0; } EXPORT_SYMBOL_GPL(sdhci_runtime_resume_host); @@ -2992,17 +3945,24 @@ void sdhci_cqe_enable(struct mmc_host *mmc) ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); ctrl &= ~SDHCI_CTRL_DMA_MASK; - if (host->flags & SDHCI_USE_64_BIT_DMA) + /* + * Host from V4.10 supports ADMA3 DMA type. + * ADMA3 performs integrated descriptor which is more suitable + * for cmd queuing to fetch both command and transfer descriptors. + */ + if (host->v4_mode && (host->caps1 & SDHCI_CAN_DO_ADMA3)) + ctrl |= SDHCI_CTRL_ADMA3; + else if (host->flags & SDHCI_USE_64_BIT_DMA) ctrl |= SDHCI_CTRL_ADMA64; else ctrl |= SDHCI_CTRL_ADMA32; sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL); - sdhci_writew(host, SDHCI_MAKE_BLKSZ(SDHCI_DEFAULT_BOUNDARY_ARG, 512), + sdhci_writew(host, SDHCI_MAKE_BLKSZ(host->sdma_boundary, 512), SDHCI_BLOCK_SIZE); /* Set maximum timeout */ - sdhci_writeb(host, 0xE, SDHCI_TIMEOUT_CONTROL); + sdhci_set_timeout(host, NULL); host->ier = host->cqe_ier; @@ -3015,7 +3975,6 @@ void sdhci_cqe_enable(struct mmc_host *mmc) mmc_hostname(mmc), host->ier, sdhci_readl(host, SDHCI_INT_STATUS)); - mmiowb(); spin_unlock_irqrestore(&host->lock, flags); } EXPORT_SYMBOL_GPL(sdhci_cqe_enable); @@ -3031,16 +3990,13 @@ void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery) host->cqe_on = false; - if (recovery) { - sdhci_do_reset(host, SDHCI_RESET_CMD); - sdhci_do_reset(host, SDHCI_RESET_DATA); - } + if (recovery) + sdhci_reset_for(host, CQE_RECOVERY); pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n", mmc_hostname(mmc), host->ier, sdhci_readl(host, SDHCI_INT_STATUS)); - mmiowb(); spin_unlock_irqrestore(&host->lock, flags); } EXPORT_SYMBOL_GPL(sdhci_cqe_disable); @@ -3053,20 +4009,27 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error, if (!host->cqe_on) return false; - if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC)) + if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC)) { *cmd_error = -EILSEQ; - else if (intmask & SDHCI_INT_TIMEOUT) + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) + sdhci_err_stats_inc(host, CMD_CRC); + } else if (intmask & SDHCI_INT_TIMEOUT) { *cmd_error = -ETIMEDOUT; - else + sdhci_err_stats_inc(host, CMD_TIMEOUT); + } else *cmd_error = 0; - if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC)) + if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC | SDHCI_INT_TUNING_ERROR)) { *data_error = -EILSEQ; - else if (intmask & SDHCI_INT_DATA_TIMEOUT) + if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND)))) + sdhci_err_stats_inc(host, DAT_CRC); + } else if (intmask & SDHCI_INT_DATA_TIMEOUT) { *data_error = -ETIMEDOUT; - else if (intmask & SDHCI_INT_ADMA_ERROR) + sdhci_err_stats_inc(host, DAT_TIMEOUT); + } else if (intmask & SDHCI_INT_ADMA_ERROR) { *data_error = -EIO; - else + sdhci_err_stats_inc(host, ADMA); + } else *data_error = 0; /* Clear selected interrupts. */ @@ -3082,6 +4045,7 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error, sdhci_writel(host, intmask, SDHCI_INT_STATUS); pr_err("%s: CQE: Unexpected interrupt 0x%08x.\n", mmc_hostname(host->mmc), intmask); + sdhci_err_stats_inc(host, UNEXPECTED_IRQ); sdhci_dumpregs(host); } @@ -3103,7 +4067,7 @@ struct sdhci_host *sdhci_alloc_host(struct device *dev, WARN_ON(dev == NULL); - mmc = mmc_alloc_host(sizeof(struct sdhci_host) + priv_size, dev); + mmc = devm_mmc_alloc_host(dev, sizeof(struct sdhci_host) + priv_size); if (!mmc) return ERR_PTR(-ENOMEM); @@ -3118,6 +4082,22 @@ struct sdhci_host *sdhci_alloc_host(struct device *dev, host->cqe_err_ier = SDHCI_CQE_INT_ERR_MASK; host->tuning_delay = -1; + host->tuning_loop_count = MAX_TUNING_LOOP; + + host->sdma_boundary = SDHCI_DEFAULT_BOUNDARY_ARG; + + /* + * The DMA table descriptor count is calculated as the maximum + * number of segments times 2, to allow for an alignment + * descriptor for each segment, plus 1 for a nop end descriptor. + */ + host->adma_table_cnt = SDHCI_MAX_SEGS * 2 + 1; + host->max_adma = 65536; + + host->max_timeout_count = 0xE; + + host->complete_work_fn = sdhci_complete_work; + host->thread_irq_fn = sdhci_thread_irq; return host; } @@ -3154,7 +4134,8 @@ static int sdhci_set_dma_mask(struct sdhci_host *host) return ret; } -void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1) +void __sdhci_read_caps(struct sdhci_host *host, const u16 *ver, + const u32 *caps, const u32 *caps1) { u16 v; u64 dt_caps_mask = 0; @@ -3171,19 +4152,19 @@ void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1) if (debug_quirks2) host->quirks2 = debug_quirks2; - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); + + if (host->v4_mode) + sdhci_do_enable_v4_mode(host); - of_property_read_u64(mmc_dev(host->mmc)->of_node, - "sdhci-caps-mask", &dt_caps_mask); - of_property_read_u64(mmc_dev(host->mmc)->of_node, - "sdhci-caps", &dt_caps); + device_property_read_u64(mmc_dev(host->mmc), + "sdhci-caps-mask", &dt_caps_mask); + device_property_read_u64(mmc_dev(host->mmc), + "sdhci-caps", &dt_caps); v = ver ? *ver : sdhci_readw(host, SDHCI_HOST_VERSION); host->version = (v & SDHCI_SPEC_VER_MASK) >> SDHCI_SPEC_VER_SHIFT; - if (host->quirks & SDHCI_QUIRK_MISSING_CAPS) - return; - if (caps) { host->caps = *caps; } else { @@ -3205,6 +4186,83 @@ void __sdhci_read_caps(struct sdhci_host *host, u16 *ver, u32 *caps, u32 *caps1) } EXPORT_SYMBOL_GPL(__sdhci_read_caps); +static void sdhci_allocate_bounce_buffer(struct sdhci_host *host) +{ + struct mmc_host *mmc = host->mmc; + unsigned int max_blocks; + unsigned int bounce_size; + int ret; + + /* + * Cap the bounce buffer at 64KB. Using a bigger bounce buffer + * has diminishing returns, this is probably because SD/MMC + * cards are usually optimized to handle this size of requests. + */ + bounce_size = SZ_64K; + /* + * Adjust downwards to maximum request size if this is less + * than our segment size, else hammer down the maximum + * request size to the maximum buffer size. + */ + if (mmc->max_req_size < bounce_size) + bounce_size = mmc->max_req_size; + max_blocks = bounce_size / 512; + + /* + * When we just support one segment, we can get significant + * speedups by the help of a bounce buffer to group scattered + * reads/writes together. + */ + host->bounce_buffer = devm_kmalloc(mmc_dev(mmc), + bounce_size, + GFP_KERNEL); + if (!host->bounce_buffer) { + pr_err("%s: failed to allocate %u bytes for bounce buffer, falling back to single segments\n", + mmc_hostname(mmc), + bounce_size); + /* + * Exiting with zero here makes sure we proceed with + * mmc->max_segs == 1. + */ + return; + } + + host->bounce_addr = dma_map_single(mmc_dev(mmc), + host->bounce_buffer, + bounce_size, + DMA_BIDIRECTIONAL); + ret = dma_mapping_error(mmc_dev(mmc), host->bounce_addr); + if (ret) { + devm_kfree(mmc_dev(mmc), host->bounce_buffer); + host->bounce_buffer = NULL; + /* Again fall back to max_segs == 1 */ + return; + } + + host->bounce_buffer_size = bounce_size; + + /* Lie about this since we're bouncing */ + mmc->max_segs = max_blocks; + mmc->max_seg_size = bounce_size; + mmc->max_req_size = bounce_size; + + pr_info("%s bounce up to %u segments into one, max segment size %u bytes\n", + mmc_hostname(mmc), max_blocks, bounce_size); +} + +static inline bool sdhci_can_64bit_dma(struct sdhci_host *host) +{ + /* + * According to SD Host Controller spec v4.10, bit[27] added from + * version 4.10 in Capabilities Register is used as 64-bit System + * Address support for V4 mode. + */ + if (host->version >= SDHCI_SPEC_410 && host->v4_mode) + return host->caps & SDHCI_CAN_64BIT_V4; + + return host->caps & SDHCI_CAN_64BIT; +} + int sdhci_setup_host(struct sdhci_host *host) { struct mmc_host *mmc; @@ -3212,7 +4270,8 @@ int sdhci_setup_host(struct sdhci_host *host) unsigned int ocr_avail; unsigned int override_timeout_clk; u32 max_clk; - int ret; + int ret = 0; + bool enable_vqmmc = false; WARN_ON(host == NULL); if (host == NULL) @@ -3226,15 +4285,25 @@ int sdhci_setup_host(struct sdhci_host *host) * the host can take the appropriate action if regulators are not * available. */ - ret = mmc_regulator_get_supply(mmc); - if (ret == -EPROBE_DEFER) - return ret; + if (!mmc->supply.vqmmc) { + ret = mmc_regulator_get_supply(mmc); + if (ret) + return ret; + enable_vqmmc = true; + } + + DBG("Version: 0x%08x | Present: 0x%08x\n", + sdhci_readw(host, SDHCI_HOST_VERSION), + sdhci_readl(host, SDHCI_PRESENT_STATE)); + DBG("Caps: 0x%08x | Caps_1: 0x%08x\n", + sdhci_readl(host, SDHCI_CAPABILITIES), + sdhci_readl(host, SDHCI_CAPABILITIES_1)); sdhci_read_caps(host); override_timeout_clk = host->timeout_clk; - if (host->version > SDHCI_SPEC_300) { + if (host->version > SDHCI_SPEC_420) { pr_err("%s: Unknown controller version (%d). You may experience problems.\n", mmc_hostname(mmc), host->version); } @@ -3262,18 +4331,29 @@ int sdhci_setup_host(struct sdhci_host *host) host->flags &= ~SDHCI_USE_ADMA; } - /* - * It is assumed that a 64-bit capable device has set a 64-bit DMA mask - * and *must* do 64-bit DMA. A driver has the opportunity to change - * that during the first call to ->enable_dma(). Similarly - * SDHCI_QUIRK2_BROKEN_64_BIT_DMA must be left to the drivers to - * implement. - */ - if (host->caps & SDHCI_CAN_64BIT) + if (sdhci_can_64bit_dma(host)) host->flags |= SDHCI_USE_64_BIT_DMA; + if (host->use_external_dma) { + ret = sdhci_external_dma_init(host); + if (ret == -EPROBE_DEFER) + goto unreg; + /* + * Fall back to use the DMA/PIO integrated in standard SDHCI + * instead of external DMA devices. + */ + else if (ret) + sdhci_switch_external_dma(host, false); + /* Disable internal DMA sources */ + else + host->flags &= ~(SDHCI_USE_SDMA | SDHCI_USE_ADMA); + } + if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) { - ret = sdhci_set_dma_mask(host); + if (host->ops->set_dma_mask) + ret = host->ops->set_dma_mask(host); + else + ret = sdhci_set_dma_mask(host); if (!ret && host->ops->enable_dma) ret = host->ops->enable_dma(host); @@ -3287,33 +4367,30 @@ int sdhci_setup_host(struct sdhci_host *host) } } - /* SDMA does not support 64-bit DMA */ - if (host->flags & SDHCI_USE_64_BIT_DMA) + /* SDMA does not support 64-bit DMA if v4 mode not set */ + if ((host->flags & SDHCI_USE_64_BIT_DMA) && !host->v4_mode) host->flags &= ~SDHCI_USE_SDMA; if (host->flags & SDHCI_USE_ADMA) { dma_addr_t dma; void *buf; - /* - * The DMA descriptor table size is calculated as the maximum - * number of segments times 2, to allow for an alignment - * descriptor for each segment, plus 1 for a nop end descriptor, - * all multipled by the descriptor size. - */ - if (host->flags & SDHCI_USE_64_BIT_DMA) { - host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) * - SDHCI_ADMA2_64_DESC_SZ; - host->desc_sz = SDHCI_ADMA2_64_DESC_SZ; - } else { - host->adma_table_sz = (SDHCI_MAX_SEGS * 2 + 1) * - SDHCI_ADMA2_32_DESC_SZ; - host->desc_sz = SDHCI_ADMA2_32_DESC_SZ; - } + if (!(host->flags & SDHCI_USE_64_BIT_DMA)) + host->alloc_desc_sz = SDHCI_ADMA2_32_DESC_SZ; + else if (!host->alloc_desc_sz) + host->alloc_desc_sz = SDHCI_ADMA2_64_DESC_SZ(host); + + host->desc_sz = host->alloc_desc_sz; + host->adma_table_sz = host->adma_table_cnt * host->desc_sz; host->align_buffer_sz = SDHCI_MAX_SEGS * SDHCI_ADMA2_ALIGN; - buf = dma_alloc_coherent(mmc_dev(mmc), host->align_buffer_sz + - host->adma_table_sz, &dma, GFP_KERNEL); + /* + * Use zalloc to zero the reserved high 32-bits of 128-bit + * descriptors so that they never need to be written. + */ + buf = dma_alloc_coherent(mmc_dev(mmc), + host->align_buffer_sz + host->adma_table_sz, + &dma, GFP_KERNEL); if (!buf) { pr_warn("%s: Unable to allocate ADMA buffers - falling back to standard DMA\n", mmc_hostname(mmc)); @@ -3345,11 +4422,9 @@ int sdhci_setup_host(struct sdhci_host *host) } if (host->version >= SDHCI_SPEC_300) - host->max_clk = (host->caps & SDHCI_CLOCK_V3_BASE_MASK) - >> SDHCI_CLOCK_BASE_SHIFT; + host->max_clk = FIELD_GET(SDHCI_CLOCK_V3_BASE_MASK, host->caps); else - host->max_clk = (host->caps & SDHCI_CLOCK_BASE_MASK) - >> SDHCI_CLOCK_BASE_SHIFT; + host->max_clk = FIELD_GET(SDHCI_CLOCK_BASE_MASK, host->caps); host->max_clk *= 1000000; if (host->max_clk == 0 || host->quirks & @@ -3367,8 +4442,7 @@ int sdhci_setup_host(struct sdhci_host *host) * In case of Host Controller v3.00, find out whether clock * multiplier is supported. */ - host->clk_mul = (host->caps1 & SDHCI_CLOCK_MUL_MASK) >> - SDHCI_CLOCK_MUL_SHIFT; + host->clk_mul = FIELD_GET(SDHCI_CLOCK_MUL_MASK, host->caps1); /* * In case the value in Clock Multiplier is 0, then programmable @@ -3387,11 +4461,13 @@ int sdhci_setup_host(struct sdhci_host *host) if (host->ops->get_min_clock) mmc->f_min = host->ops->get_min_clock(host); else if (host->version >= SDHCI_SPEC_300) { - if (host->clk_mul) { - mmc->f_min = (host->max_clk * host->clk_mul) / 1024; + if (host->clk_mul) max_clk = host->max_clk * host->clk_mul; - } else - mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300; + /* + * Divided Clock Mode minimum clock rate is always less than + * Programmable Clock Mode minimum clock rate. + */ + mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_300; } else mmc->f_min = host->max_clk / SDHCI_MAX_DIV_SPEC_200; @@ -3399,8 +4475,7 @@ int sdhci_setup_host(struct sdhci_host *host) mmc->f_max = max_clk; if (!(host->quirks & SDHCI_QUIRK_DATA_TIMEOUT_USES_SDCLK)) { - host->timeout_clk = (host->caps & SDHCI_TIMEOUT_CLK_MASK) >> - SDHCI_TIMEOUT_CLK_SHIFT; + host->timeout_clk = FIELD_GET(SDHCI_TIMEOUT_CLK_MASK, host->caps); if (host->caps & SDHCI_TIMEOUT_CLK_UNIT) host->timeout_clk *= 1000; @@ -3426,16 +4501,23 @@ int sdhci_setup_host(struct sdhci_host *host) mmc->max_busy_timeout /= host->timeout_clk; } - mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_ERASE | MMC_CAP_CMD23; + if (host->quirks2 & SDHCI_QUIRK2_DISABLE_HW_TIMEOUT && + !host->ops->get_max_timeout_count) + mmc->max_busy_timeout = 0; + + mmc->caps |= MMC_CAP_SDIO_IRQ | MMC_CAP_CMD23; mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD; if (host->quirks & SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12) host->flags |= SDHCI_AUTO_CMD12; - /* Auto-CMD23 stuff only works in ADMA or PIO. */ + /* + * For v3 mode, Auto-CMD23 stuff only works in ADMA or PIO. + * For v4 mode, SDMA may use Auto-CMD23 as well. + */ if ((host->version >= SDHCI_SPEC_300) && ((host->flags & SDHCI_USE_ADMA) || - !(host->flags & SDHCI_USE_SDMA)) && + !(host->flags & SDHCI_USE_SDMA) || host->v4_mode) && !(host->quirks2 & SDHCI_QUIRK2_ACMD23_BROKEN)) { host->flags |= SDHCI_AUTO_CMD23; DBG("Auto-CMD23 available\n"); @@ -3461,27 +4543,48 @@ int sdhci_setup_host(struct sdhci_host *host) if ((host->quirks & SDHCI_QUIRK_BROKEN_CARD_DETECTION) && mmc_card_is_removable(mmc) && - mmc_gpio_get_cd(host->mmc) < 0) + mmc_gpio_get_cd(mmc) < 0) mmc->caps |= MMC_CAP_NEEDS_POLL; - /* If vqmmc regulator and no 1.8V signalling, then there's no UHS */ if (!IS_ERR(mmc->supply.vqmmc)) { - ret = regulator_enable(mmc->supply.vqmmc); + if (enable_vqmmc) { + ret = regulator_enable(mmc->supply.vqmmc); + host->sdhci_core_to_disable_vqmmc = !ret; + } + + /* If vqmmc provides no 1.8V signalling, then there's no UHS */ if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 1700000, 1950000)) host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 | SDHCI_SUPPORT_DDR50); + + /* In eMMC case vqmmc might be a fixed 1.8V regulator */ + if (!regulator_is_supported_voltage(mmc->supply.vqmmc, 2700000, + 3600000)) + host->flags &= ~SDHCI_SIGNALING_330; + if (ret) { pr_warn("%s: Failed to enable vqmmc regulator: %d\n", mmc_hostname(mmc), ret); mmc->supply.vqmmc = ERR_PTR(-EINVAL); } + } if (host->quirks2 & SDHCI_QUIRK2_NO_1_8_V) { host->caps1 &= ~(SDHCI_SUPPORT_SDR104 | SDHCI_SUPPORT_SDR50 | SDHCI_SUPPORT_DDR50); + /* + * The SDHCI controller in a SoC might support HS200/HS400 + * (indicated using mmc-hs200-1_8v/mmc-hs400-1_8v dt property), + * but if the board is modeled such that the IO lines are not + * connected to 1.8v then HS200/HS400 cannot be supported. + * Disable HS200/HS400 if the board does not have 1.8v connected + * to the IO lines. (Applicable for other modes in 1.8v) + */ + mmc->caps2 &= ~(MMC_CAP2_HSX00_1_8V | MMC_CAP2_HS400_ES); + mmc->caps &= ~(MMC_CAP_1_8V_DDR | MMC_CAP_UHS); } /* Any UHS-I mode in caps implies SDR12 and SDR25 support. */ @@ -3528,8 +4631,8 @@ int sdhci_setup_host(struct sdhci_host *host) mmc->caps |= MMC_CAP_DRIVER_TYPE_D; /* Initial value for re-tuning timer count */ - host->tuning_count = (host->caps1 & SDHCI_RETUNING_TIMER_COUNT_MASK) >> - SDHCI_RETUNING_TIMER_COUNT_SHIFT; + host->tuning_count = FIELD_GET(SDHCI_RETUNING_TIMER_COUNT_MASK, + host->caps1); /* * In case Re-tuning Timer is not disabled, the actual value of @@ -3539,8 +4642,7 @@ int sdhci_setup_host(struct sdhci_host *host) host->tuning_count = 1 << (host->tuning_count - 1); /* Re-tuning mode supported by the Host Controller */ - host->tuning_mode = (host->caps1 & SDHCI_RETUNING_MODE_MASK) >> - SDHCI_RETUNING_MODE_SHIFT; + host->tuning_mode = FIELD_GET(SDHCI_RETUNING_MODE_MASK, host->caps1); ocr_avail = 0; @@ -3562,35 +4664,32 @@ int sdhci_setup_host(struct sdhci_host *host) curr = min_t(u32, curr, SDHCI_MAX_CURRENT_LIMIT); max_current_caps = - (curr << SDHCI_MAX_CURRENT_330_SHIFT) | - (curr << SDHCI_MAX_CURRENT_300_SHIFT) | - (curr << SDHCI_MAX_CURRENT_180_SHIFT); + FIELD_PREP(SDHCI_MAX_CURRENT_330_MASK, curr) | + FIELD_PREP(SDHCI_MAX_CURRENT_300_MASK, curr) | + FIELD_PREP(SDHCI_MAX_CURRENT_180_MASK, curr); } } if (host->caps & SDHCI_CAN_VDD_330) { ocr_avail |= MMC_VDD_32_33 | MMC_VDD_33_34; - mmc->max_current_330 = ((max_current_caps & - SDHCI_MAX_CURRENT_330_MASK) >> - SDHCI_MAX_CURRENT_330_SHIFT) * - SDHCI_MAX_CURRENT_MULTIPLIER; + mmc->max_current_330 = FIELD_GET(SDHCI_MAX_CURRENT_330_MASK, + max_current_caps) * + SDHCI_MAX_CURRENT_MULTIPLIER; } if (host->caps & SDHCI_CAN_VDD_300) { ocr_avail |= MMC_VDD_29_30 | MMC_VDD_30_31; - mmc->max_current_300 = ((max_current_caps & - SDHCI_MAX_CURRENT_300_MASK) >> - SDHCI_MAX_CURRENT_300_SHIFT) * - SDHCI_MAX_CURRENT_MULTIPLIER; + mmc->max_current_300 = FIELD_GET(SDHCI_MAX_CURRENT_300_MASK, + max_current_caps) * + SDHCI_MAX_CURRENT_MULTIPLIER; } if (host->caps & SDHCI_CAN_VDD_180) { ocr_avail |= MMC_VDD_165_195; - mmc->max_current_180 = ((max_current_caps & - SDHCI_MAX_CURRENT_180_MASK) >> - SDHCI_MAX_CURRENT_180_SHIFT) * - SDHCI_MAX_CURRENT_MULTIPLIER; + mmc->max_current_180 = FIELD_GET(SDHCI_MAX_CURRENT_180_MASK, + max_current_caps) * + SDHCI_MAX_CURRENT_MULTIPLIER; } /* If OCR set by host, use it instead. */ @@ -3633,22 +4732,25 @@ int sdhci_setup_host(struct sdhci_host *host) spin_lock_init(&host->lock); /* + * Maximum number of sectors in one transfer. Limited by SDMA boundary + * size (512KiB). Note some tuning modes impose a 4MiB limit, but this + * is less anyway. + */ + mmc->max_req_size = 524288; + + /* * Maximum number of segments. Depends on if the hardware * can do scatter/gather or not. */ - if (host->flags & SDHCI_USE_ADMA) + if (host->flags & SDHCI_USE_ADMA) { mmc->max_segs = SDHCI_MAX_SEGS; - else if (host->flags & SDHCI_USE_SDMA) + } else if (host->flags & SDHCI_USE_SDMA) { mmc->max_segs = 1; - else /* PIO */ + mmc->max_req_size = min_t(size_t, mmc->max_req_size, + dma_max_mapping_size(mmc_dev(mmc))); + } else { /* PIO */ mmc->max_segs = SDHCI_MAX_SEGS; - - /* - * Maximum number of sectors in one transfer. Limited by SDMA boundary - * size (512KiB). Note some tuning modes impose a 4MiB limit, but this - * is less anyway. - */ - mmc->max_req_size = 524288; + } /* * Maximum segment size. Could be one segment with the maximum number @@ -3656,10 +4758,27 @@ int sdhci_setup_host(struct sdhci_host *host) * be larger than 64 KiB though. */ if (host->flags & SDHCI_USE_ADMA) { - if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC) + if (host->quirks & SDHCI_QUIRK_BROKEN_ADMA_ZEROLEN_DESC) { + host->max_adma = 65532; /* 32-bit alignment */ mmc->max_seg_size = 65535; - else + /* + * sdhci_adma_table_pre() expects to define 1 DMA + * descriptor per segment, so the maximum segment size + * is set accordingly. SDHCI allows up to 64KiB per DMA + * descriptor (16-bit field), but some controllers do + * not support "zero means 65536" reducing the maximum + * for them to 65535. That is a problem if PAGE_SIZE is + * 64KiB because the block layer does not support + * max_seg_size < PAGE_SIZE, however + * sdhci_adma_table_pre() has a workaround to handle + * that case, and split the descriptor. Refer also + * comment in sdhci_adma_table_pre(). + */ + if (mmc->max_seg_size < PAGE_SIZE) + mmc->max_seg_size = PAGE_SIZE; + } else { mmc->max_seg_size = 65536; + } } else { mmc->max_seg_size = mmc->max_req_size; } @@ -3687,10 +4806,14 @@ int sdhci_setup_host(struct sdhci_host *host) */ mmc->max_blk_count = (host->quirks & SDHCI_QUIRK_NO_MULTIBLOCK) ? 1 : 65535; + if (mmc->max_segs == 1) + /* This may alter mmc->*_blk_* parameters */ + sdhci_allocate_bounce_buffer(host); + return 0; unreg: - if (!IS_ERR(mmc->supply.vqmmc)) + if (host->sdhci_core_to_disable_vqmmc) regulator_disable(mmc->supply.vqmmc); undma: if (host->align_buffer) @@ -3708,13 +4831,17 @@ void sdhci_cleanup_host(struct sdhci_host *host) { struct mmc_host *mmc = host->mmc; - if (!IS_ERR(mmc->supply.vqmmc)) + if (host->sdhci_core_to_disable_vqmmc) regulator_disable(mmc->supply.vqmmc); if (host->align_buffer) dma_free_coherent(mmc_dev(mmc), host->align_buffer_sz + host->adma_table_sz, host->align_buffer, host->align_addr); + + if (host->use_external_dma) + sdhci_external_dma_release(host); + host->adma_table = NULL; host->align_buffer = NULL; } @@ -3722,35 +4849,37 @@ EXPORT_SYMBOL_GPL(sdhci_cleanup_host); int __sdhci_add_host(struct sdhci_host *host) { + unsigned int flags = WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_HIGHPRI; struct mmc_host *mmc = host->mmc; int ret; - /* - * Init tasklets. - */ - tasklet_init(&host->finish_tasklet, - sdhci_tasklet_finish, (unsigned long)host); + if ((mmc->caps2 & MMC_CAP2_CQE) && + (host->quirks & SDHCI_QUIRK_BROKEN_CQE)) { + mmc->caps2 &= ~MMC_CAP2_CQE; + mmc->cqe_ops = NULL; + } + + host->complete_wq = alloc_workqueue("sdhci", flags, 0); + if (!host->complete_wq) + return -ENOMEM; - setup_timer(&host->timer, sdhci_timeout_timer, (unsigned long)host); - setup_timer(&host->data_timer, sdhci_timeout_data_timer, - (unsigned long)host); + INIT_WORK(&host->complete_work, host->complete_work_fn); + + timer_setup(&host->timer, sdhci_timeout_timer, 0); + timer_setup(&host->data_timer, sdhci_timeout_data_timer, 0); init_waitqueue_head(&host->buf_ready_int); sdhci_init(host, 0); - ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq, + ret = request_threaded_irq(host->irq, sdhci_irq, host->thread_irq_fn, IRQF_SHARED, mmc_hostname(mmc), host); if (ret) { pr_err("%s: Failed to request IRQ %d: %d\n", mmc_hostname(mmc), host->irq, ret); - goto untasklet; + goto unwq; } -#ifdef CONFIG_MMC_DEBUG - sdhci_dumpregs(host); -#endif - ret = sdhci_led_register(host); if (ret) { pr_err("%s: Failed to register LED device: %d\n", @@ -3758,14 +4887,13 @@ int __sdhci_add_host(struct sdhci_host *host) goto unirq; } - mmiowb(); - ret = mmc_add_host(mmc); if (ret) goto unled; pr_info("%s: SDHCI controller on %s [%s] using %s\n", mmc_hostname(mmc), host->hw_name, dev_name(mmc_dev(mmc)), + host->use_external_dma ? "External DMA" : (host->flags & SDHCI_USE_ADMA) ? (host->flags & SDHCI_USE_64_BIT_DMA) ? "ADMA 64-bit" : "ADMA" : (host->flags & SDHCI_USE_SDMA) ? "DMA" : "PIO"); @@ -3777,12 +4905,12 @@ int __sdhci_add_host(struct sdhci_host *host) unled: sdhci_led_unregister(host); unirq: - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); sdhci_writel(host, 0, SDHCI_INT_ENABLE); sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE); free_irq(host->irq, host); -untasklet: - tasklet_kill(&host->finish_tasklet); +unwq: + destroy_workqueue(host->complete_wq); return ret; } @@ -3835,18 +4963,18 @@ void sdhci_remove_host(struct sdhci_host *host, int dead) sdhci_led_unregister(host); if (!dead) - sdhci_do_reset(host, SDHCI_RESET_ALL); + sdhci_reset_for_all(host); sdhci_writel(host, 0, SDHCI_INT_ENABLE); sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE); free_irq(host->irq, host); - del_timer_sync(&host->timer); - del_timer_sync(&host->data_timer); + timer_delete_sync(&host->timer); + timer_delete_sync(&host->data_timer); - tasklet_kill(&host->finish_tasklet); + destroy_workqueue(host->complete_wq); - if (!IS_ERR(mmc->supply.vqmmc)) + if (host->sdhci_core_to_disable_vqmmc) regulator_disable(mmc->supply.vqmmc); if (host->align_buffer) @@ -3854,19 +4982,15 @@ void sdhci_remove_host(struct sdhci_host *host, int dead) host->adma_table_sz, host->align_buffer, host->align_addr); + if (host->use_external_dma) + sdhci_external_dma_release(host); + host->adma_table = NULL; host->align_buffer = NULL; } EXPORT_SYMBOL_GPL(sdhci_remove_host); -void sdhci_free_host(struct sdhci_host *host) -{ - mmc_free_host(host->mmc); -} - -EXPORT_SYMBOL_GPL(sdhci_free_host); - /*****************************************************************************\ * * * Driver init/exit * |