diff options
Diffstat (limited to 'drivers/ufs/core/ufshcd.c')
| -rw-r--r-- | drivers/ufs/core/ufshcd.c | 4772 |
1 files changed, 2965 insertions, 1807 deletions
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 3a1c4d31e010..040a0ceb170a 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -20,12 +20,15 @@ #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/pm_opp.h> #include <linux/regulator/consumer.h> #include <linux/sched/clock.h> +#include <linux/iopoll.h> #include <scsi/scsi_cmnd.h> #include <scsi/scsi_dbg.h> #include <scsi/scsi_driver.h> #include <scsi/scsi_eh.h> +#include <scsi/scsi_tcq.h> #include "ufshcd-priv.h" #include <ufs/ufs_quirks.h> #include <ufs/unipro.h> @@ -34,18 +37,20 @@ #include "ufs-fault-injection.h" #include "ufs_bsg.h" #include "ufshcd-crypto.h" -#include "ufshpb.h" -#include <asm/unaligned.h> +#include <linux/unaligned.h> #define CREATE_TRACE_POINTS -#include <trace/events/ufs.h> +#include "ufs_trace.h" #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ UTP_TASK_REQ_COMPL |\ UFSHCD_ERROR_MASK) -/* UIC command timeout, unit: ms */ -#define UIC_CMD_TIMEOUT 500 +/* UIC command timeout, unit: ms */ +enum { + UIC_CMD_TIMEOUT_DEFAULT = 500, + UIC_CMD_TIMEOUT_MAX = 5000, +}; /* NOP OUT retries waiting for NOP IN response */ #define NOP_OUT_RETRIES 10 /* Timeout after 50 msecs if NOP OUT hangs without response */ @@ -54,7 +59,14 @@ /* Query request retries */ #define QUERY_REQ_RETRIES 3 /* Query request timeout */ -#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */ +enum { + QUERY_REQ_TIMEOUT_MIN = 1, + QUERY_REQ_TIMEOUT_DEFAULT = 1500, + QUERY_REQ_TIMEOUT_MAX = 30000 +}; + +/* Advanced RPMB request timeout */ +#define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */ /* Task management command timeout */ #define TM_CMD_TIMEOUT 100 /* msecs */ @@ -89,6 +101,57 @@ /* Polling time to wait for fDeviceInit */ #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */ +/* Default RTC update every 10 seconds */ +#define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC) + +/* bMaxNumOfRTT is equal to two after device manufacturing */ +#define DEFAULT_MAX_NUM_RTT 2 + +/* UFSHC 4.0 compliant HC support this mode. */ +static bool use_mcq_mode = true; + +static bool is_mcq_supported(struct ufs_hba *hba) +{ + return hba->mcq_sup && use_mcq_mode; +} + +module_param(use_mcq_mode, bool, 0644); +MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); + +static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT; + +static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT, + UIC_CMD_TIMEOUT_MAX); +} + +static const struct kernel_param_ops uic_cmd_timeout_ops = { + .set = uic_cmd_timeout_set, + .get = param_get_uint, +}; + +module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644); +MODULE_PARM_DESC(uic_cmd_timeout, + "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 5 seconds inclusively"); + +static unsigned int dev_cmd_timeout = QUERY_REQ_TIMEOUT_DEFAULT; + +static int dev_cmd_timeout_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, QUERY_REQ_TIMEOUT_MIN, + QUERY_REQ_TIMEOUT_MAX); +} + +static const struct kernel_param_ops dev_cmd_timeout_ops = { + .set = dev_cmd_timeout_set, + .get = param_get_uint, +}; + +module_param_cb(dev_cmd_timeout, &dev_cmd_timeout_ops, &dev_cmd_timeout, 0644); +MODULE_PARM_DESC(dev_cmd_timeout, + "UFS Device command timeout in milliseconds. Defaults to 1.5s. Supported values range from 1ms to 30 seconds inclusively"); + #define ufshcd_toggle_vreg(_dev, _vreg, _on) \ ({ \ int _ret; \ @@ -137,9 +200,6 @@ EXPORT_SYMBOL_GPL(ufshcd_dump_regs); enum { UFSHCD_MAX_CHANNEL = 0, UFSHCD_MAX_ID = 1, - UFSHCD_NUM_RESERVED = 1, - UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED, - UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED, }; static const char *const ufshcd_state_name[] = { @@ -215,16 +275,27 @@ ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, return UFS_PM_LVL_0; } +static bool ufshcd_has_pending_tasks(struct ufs_hba *hba) +{ + return hba->outstanding_tasks || hba->active_uic_cmd || + hba->uic_async_done; +} + +static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba) +{ + return scsi_host_busy(hba->host) || ufshcd_has_pending_tasks(hba); +} + static const struct ufs_dev_quirk ufs_fixups[] = { /* UFS cards deviations table */ { .wmanufacturerid = UFS_VENDOR_MICRON, .model = UFS_ANY_MODEL, - .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | - UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ }, + .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, { .wmanufacturerid = UFS_VENDOR_SAMSUNG, .model = UFS_ANY_MODEL, .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | + UFS_DEVICE_QUIRK_PA_HIBER8TIME | UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, { .wmanufacturerid = UFS_VENDOR_SKHYNIX, .model = UFS_ANY_MODEL, @@ -241,6 +312,9 @@ static const struct ufs_dev_quirk ufs_fixups[] = { { .wmanufacturerid = UFS_VENDOR_TOSHIBA, .model = "THGLF2G9D8KBADG", .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, + { .wmanufacturerid = UFS_VENDOR_TOSHIBA, + .model = "THGJFJT1E45BATP", + .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, {} }; @@ -250,14 +324,15 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba); static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); static void ufshcd_hba_exit(struct ufs_hba *hba); +static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params); static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params); static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); static void ufshcd_resume_clkscaling(struct ufs_hba *hba); static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); -static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba); -static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up); +static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, + bool scale_up); static irqreturn_t ufshcd_intr(int irq, void *__hba); static int ufshcd_change_power_mode(struct ufs_hba *hba, struct ufs_pa_layer_attr *pwr_mode); @@ -265,27 +340,56 @@ static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on); static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on); static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba, struct ufs_vreg *vreg); -static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag); static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, bool enable); static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); -static inline void ufshcd_enable_irq(struct ufs_hba *hba) +void ufshcd_enable_irq(struct ufs_hba *hba) { if (!hba->is_irq_enabled) { enable_irq(hba->irq); hba->is_irq_enabled = true; } } +EXPORT_SYMBOL_GPL(ufshcd_enable_irq); -static inline void ufshcd_disable_irq(struct ufs_hba *hba) +void ufshcd_disable_irq(struct ufs_hba *hba) { if (hba->is_irq_enabled) { disable_irq(hba->irq); hba->is_irq_enabled = false; } } +EXPORT_SYMBOL_GPL(ufshcd_disable_irq); + +/** + * ufshcd_enable_intr - enable interrupts + * @hba: per adapter instance + * @intrs: interrupt bits + */ +void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) +{ + u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + u32 new_val = old_val | intrs; + + if (new_val != old_val) + ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); +} + +/** + * ufshcd_disable_intr - disable interrupts + * @hba: per adapter instance + * @intrs: interrupt bits + */ +static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) +{ + u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + u32 new_val = old_val & ~intrs; + + if (new_val != old_val) + ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); +} static void ufshcd_configure_wb(struct ufs_hba *hba) { @@ -300,22 +404,11 @@ static void ufshcd_configure_wb(struct ufs_hba *hba) ufshcd_wb_toggle_buf_flush(hba, true); } -static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba) -{ - if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt)) - scsi_unblock_requests(hba->host); -} - -static void ufshcd_scsi_block_requests(struct ufs_hba *hba) -{ - if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1) - scsi_block_requests(hba->host); -} - -static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, +static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, + struct ufshcd_lrb *lrb, enum ufs_trace_str_t str_t) { - struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr; + struct utp_upiu_req *rq = lrb->ucd_req_ptr; struct utp_upiu_header *header; if (!trace_ufshcd_upiu_enabled()) @@ -324,9 +417,9 @@ static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, if (str_t == UFS_CMD_SEND) header = &rq->header; else - header = &hba->lrb[tag].ucd_rsp_ptr->header; + header = &lrb->ucd_rsp_ptr->header; - trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb, + trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, UFS_TSF_CDB); } @@ -337,7 +430,7 @@ static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, if (!trace_ufshcd_upiu_enabled()) return; - trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header, + trace_ufshcd_upiu(hba, str_t, &rq_rsp->header, &rq_rsp->qr, UFS_TSF_OSF); } @@ -350,12 +443,12 @@ static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, return; if (str_t == UFS_TM_SEND) - trace_ufshcd_upiu(dev_name(hba->dev), str_t, + trace_ufshcd_upiu(hba, str_t, &descp->upiu_req.req_header, &descp->upiu_req.input_param1, UFS_TSF_TM_INPUT); else - trace_ufshcd_upiu(dev_name(hba->dev), str_t, + trace_ufshcd_upiu(hba, str_t, &descp->upiu_rsp.rsp_header, &descp->upiu_rsp.output_param1, UFS_TSF_TM_OUTPUT); @@ -375,28 +468,27 @@ static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, else cmd = ufshcd_readl(hba, REG_UIC_COMMAND); - trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd, + trace_ufshcd_uic_command(hba, str_t, cmd, ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); } -static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, +static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, enum ufs_trace_str_t str_t) { u64 lba = 0; u8 opcode = 0, group_id = 0; - u32 intr, doorbell; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; - struct scsi_cmnd *cmd = lrbp->cmd; + u32 doorbell = 0; + u32 intr; + u32 hwq_id = 0; struct request *rq = scsi_cmd_to_rq(cmd); + unsigned int tag = rq->tag; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int transfer_len = -1; - if (!cmd) - return; - /* trace UPIU also */ - ufshcd_add_cmd_upiu_trace(hba, tag, str_t); + ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); if (!trace_ufshcd_command_enabled()) return; @@ -410,7 +502,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); lba = scsi_get_lba(cmd); if (opcode == WRITE_10) - group_id = lrbp->cmd->cmnd[6]; + group_id = cmd->cmnd[6]; } else if (opcode == UNMAP) { /* * The number of Bytes to be unmapped beginning with the lba. @@ -420,9 +512,16 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, } intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); - doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); - trace_ufshcd_command(dev_name(hba->dev), str_t, tag, - doorbell, transfer_len, intr, lba, opcode, group_id); + + if (hba->mcq_enabled) { + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); + + hwq_id = hwq->id; + } else { + doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); + } + trace_ufshcd_command(cmd->device, hba, str_t, tag, doorbell, hwq_id, + transfer_len, intr, lba, opcode, group_id); } static void ufshcd_print_clk_freqs(struct ufs_hba *hba) @@ -496,49 +595,69 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba) ufshcd_vops_dbg_register_dump(hba); } -static -void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt) +static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, + bool pr_prdt) { - const struct ufshcd_lrb *lrbp; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; int prdt_length; - int tag; - for_each_set_bit(tag, &bitmap, hba->nutrs) { - lrbp = &hba->lrb[tag]; + if (hba->monitor.enabled) { + dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, + div_u64(lrbp->issue_time_stamp_local_clock, 1000)); + dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", tag, + div_u64(lrbp->compl_time_stamp_local_clock, 1000)); + } + dev_err(hba->dev, + "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", + tag, (u64)lrbp->utrd_dma_addr); + + ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, + sizeof(struct utp_transfer_req_desc)); + dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, + (u64)lrbp->ucd_req_dma_addr); + ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, + sizeof(struct utp_upiu_req)); + dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, + (u64)lrbp->ucd_rsp_dma_addr); + ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, + sizeof(struct utp_upiu_rsp)); + + prdt_length = le16_to_cpu( + lrbp->utr_descriptor_ptr->prd_table_length); + if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) + prdt_length /= ufshcd_sg_entry_size(hba); - dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", - tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000)); - dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", - tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000)); - dev_err(hba->dev, - "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", - tag, (u64)lrbp->utrd_dma_addr); - - ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr, - sizeof(struct utp_transfer_req_desc)); - dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag, - (u64)lrbp->ucd_req_dma_addr); - ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr, - sizeof(struct utp_upiu_req)); - dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag, - (u64)lrbp->ucd_rsp_dma_addr); - ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr, - sizeof(struct utp_upiu_rsp)); - - prdt_length = le16_to_cpu( - lrbp->utr_descriptor_ptr->prd_table_length); - if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) - prdt_length /= sizeof(struct ufshcd_sg_entry); + dev_err(hba->dev, + "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", + tag, prdt_length, + (u64)lrbp->ucd_prdt_dma_addr); - dev_err(hba->dev, - "UPIU[%d] - PRDT - %d entries phys@0x%llx\n", - tag, prdt_length, - (u64)lrbp->ucd_prdt_dma_addr); + if (pr_prdt) + ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, + ufshcd_sg_entry_size(hba) * prdt_length); +} - if (pr_prdt) - ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr, - sizeof(struct ufshcd_sg_entry) * prdt_length); - } +static bool ufshcd_print_tr_iter(struct request *req, void *priv) +{ + struct scsi_device *sdev = req->q->queuedata; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + + if (!blk_mq_is_reserved_rq(req)) + ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); + + return true; +} + +/** + * ufshcd_print_trs_all - print trs for all started requests. + * @hba: per-adapter instance. + * @pr_prdt: need to print prdt or not. + */ +static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt) +{ + blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt); } static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap) @@ -558,8 +677,8 @@ static void ufshcd_print_host_state(struct ufs_hba *hba) const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); - dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n", - hba->outstanding_reqs, hba->outstanding_tasks); + dev_err(hba->dev, "%d outstanding reqs, tasks=0x%lx\n", + scsi_host_busy(hba->host), hba->outstanding_tasks); dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", hba->saved_err, hba->saved_uic_err); dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", @@ -573,9 +692,6 @@ static void ufshcd_print_host_state(struct ufs_hba *hba) "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), hba->ufs_stats.hibern8_exit_cnt); - dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n", - div_u64(hba->ufs_stats.last_intr_ts, 1000), - hba->ufs_stats.last_intr_status); dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", hba->eh_flags, hba->req_abort_count); dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", @@ -632,6 +748,8 @@ static void ufshcd_device_reset(struct ufs_hba *hba) hba->dev_info.wb_enabled = false; hba->dev_info.wb_buf_flush_enabled = false; } + if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) + hba->dev_info.rtc_time_baseline = 0; } if (err != -EOPNOTSUPP) ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); @@ -658,41 +776,28 @@ EXPORT_SYMBOL_GPL(ufshcd_delay_us); * @interval_us: polling interval in microseconds * @timeout_ms: timeout in milliseconds * - * Return: - * -ETIMEDOUT on error, zero on success. + * Return: -ETIMEDOUT on error, zero on success. */ static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, - u32 val, unsigned long interval_us, - unsigned long timeout_ms) + u32 val, unsigned long interval_us, + unsigned long timeout_ms) { - int err = 0; - unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + u32 v; - /* ignore bits that we don't intend to wait on */ - val = val & mask; + val &= mask; /* ignore bits that we don't intend to wait on */ - while ((ufshcd_readl(hba, reg) & mask) != val) { - usleep_range(interval_us, interval_us + 50); - if (time_after(jiffies, timeout)) { - if ((ufshcd_readl(hba, reg) & mask) != val) - err = -ETIMEDOUT; - break; - } - } - - return err; + return read_poll_timeout(ufshcd_readl, v, (v & mask) == val, + interval_us, timeout_ms * 1000, false, hba, reg); } /** * ufshcd_get_intr_mask - Get the interrupt bit mask * @hba: Pointer to adapter instance * - * Returns interrupt bit mask per version + * Return: interrupt bit mask per version */ static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) { - if (hba->ufs_version == ufshci_version(1, 0)) - return INTERRUPT_MASK_ALL_VER_10; if (hba->ufs_version <= ufshci_version(2, 0)) return INTERRUPT_MASK_ALL_VER_11; @@ -703,7 +808,7 @@ static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) * ufshcd_get_ufs_version - Get the UFS version supported by the HBA * @hba: Pointer to adapter instance * - * Returns UFSHCI version supported by the controller + * Return: UFSHCI version supported by the controller */ static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) { @@ -730,7 +835,7 @@ static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) * the host controller * @hba: pointer to adapter instance * - * Returns true if device present, false if no device detected + * Return: true if device present, false if no device detected */ static inline bool ufshcd_is_device_present(struct ufs_hba *hba) { @@ -740,13 +845,19 @@ static inline bool ufshcd_is_device_present(struct ufs_hba *hba) /** * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status * @lrbp: pointer to local command reference block + * @cqe: pointer to the completion queue entry * * This function is used to get the OCS field from UTRD - * Returns the OCS field in the UTRD + * + * Return: the OCS field in the UTRD. */ -static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp) +static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, + struct cq_entry *cqe) { - return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; + if (cqe) + return cqe->overall_status & MASK_OCS; + + return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS; } /** @@ -791,7 +902,7 @@ static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY * @reg: Register value of host controller status * - * Returns integer, 0 on Success and positive value if failed + * Return: 0 on success; a positive value if failed. */ static inline int ufshcd_get_lists_status(u32 reg) { @@ -803,7 +914,8 @@ static inline int ufshcd_get_lists_status(u32 reg) * @hba: Pointer to adapter instance * * This function gets the result of UIC command completion - * Returns 0 on success, non zero value on error + * + * Return: 0 on success; non-zero value on error. */ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) { @@ -816,7 +928,8 @@ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) * @hba: Pointer to adapter instance * * This function gets UIC command argument3 - * Returns 0 on success, non zero value on error + * + * Return: 0 on success; non-zero value on error. */ static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) { @@ -826,38 +939,13 @@ static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) /** * ufshcd_get_req_rsp - returns the TR response transaction type * @ucd_rsp_ptr: pointer to response UPIU - */ -static inline int -ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) -{ - return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; -} - -/** - * ufshcd_get_rsp_upiu_result - Get the result from response UPIU - * @ucd_rsp_ptr: pointer to response UPIU - * - * This function gets the response status and scsi_status from response UPIU - * Returns the response result code. - */ -static inline int -ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) -{ - return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; -} - -/* - * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length - * from response UPIU - * @ucd_rsp_ptr: pointer to response UPIU * - * Return the data segment length. + * Return: UPIU type. */ -static inline unsigned int -ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) +static inline enum upiu_response_transaction +ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) { - return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & - MASK_RSP_UPIU_DATA_SEG_LEN; + return ucd_rsp_ptr->header.transaction_code; } /** @@ -867,12 +955,11 @@ ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) * The function checks if the device raised an exception event indicated in * the Device Information field of response UPIU. * - * Returns true if exception is raised, false otherwise. + * Return: true if exception is raised, false otherwise. */ static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) { - return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & - MASK_RSP_EXCEPTION_EVENT; + return ucd_rsp_ptr->header.device_information & 1; } /** @@ -943,44 +1030,68 @@ static inline void ufshcd_hba_start(struct ufs_hba *hba) * ufshcd_is_hba_active - Get controller state * @hba: per adapter instance * - * Returns true if and only if the controller is active. + * Return: true if and only if the controller is active. */ -static inline bool ufshcd_is_hba_active(struct ufs_hba *hba) +bool ufshcd_is_hba_active(struct ufs_hba *hba) { return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; } +EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); -u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba) +/** + * ufshcd_pm_qos_init - initialize PM QoS request + * @hba: per adapter instance + */ +void ufshcd_pm_qos_init(struct ufs_hba *hba) { - /* HCI version 1.0 and 1.1 supports UniPro 1.41 */ - if (hba->ufs_version <= ufshci_version(1, 1)) - return UFS_UNIPRO_VER_1_41; - else - return UFS_UNIPRO_VER_1_6; + guard(mutex)(&hba->pm_qos_mutex); + + if (hba->pm_qos_enabled) + return; + + cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE); + + if (cpu_latency_qos_request_active(&hba->pm_qos_req)) + hba->pm_qos_enabled = true; +} + +/** + * ufshcd_pm_qos_exit - remove request from PM QoS + * @hba: per adapter instance + */ +void ufshcd_pm_qos_exit(struct ufs_hba *hba) +{ + guard(mutex)(&hba->pm_qos_mutex); + + if (!hba->pm_qos_enabled) + return; + + cpu_latency_qos_remove_request(&hba->pm_qos_req); + hba->pm_qos_enabled = false; } -EXPORT_SYMBOL(ufshcd_get_local_unipro_ver); -static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba) +/** + * ufshcd_pm_qos_update - update PM QoS request + * @hba: per adapter instance + * @on: If True, vote for perf PM QoS mode otherwise power save mode + */ +void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) { - /* - * If both host and device support UniPro ver1.6 or later, PA layer - * parameters tuning happens during link startup itself. - * - * We can manually tune PA layer parameters if either host or device - * doesn't support UniPro ver 1.6 or later. But to keep manual tuning - * logic simple, we will only do manual tuning if local unipro version - * doesn't support ver1.6 or later. - */ - return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6; + guard(mutex)(&hba->pm_qos_mutex); + + if (!hba->pm_qos_enabled) + return; + + cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE); } +EXPORT_SYMBOL_GPL(ufshcd_pm_qos_update); /** * ufshcd_set_clk_freq - set UFS controller clock frequencies * @hba: per adapter instance * @scale_up: If True, set max possible frequency othewise set low frequency * - * Returns 0 if successful - * Returns < 0 for any other errors + * Return: 0 if successful; < 0 upon failure. */ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) { @@ -1004,7 +1115,7 @@ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) clki->max_freq, ret); break; } - trace_ufshcd_clk_scaling(dev_name(hba->dev), + trace_ufshcd_clk_scaling(hba, "scaled up", clki->name, clki->curr_freq, clki->max_freq); @@ -1022,7 +1133,7 @@ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) clki->min_freq, ret); break; } - trace_ufshcd_clk_scaling(dev_name(hba->dev), + trace_ufshcd_clk_scaling(hba, "scaled down", clki->name, clki->curr_freq, clki->min_freq); @@ -1037,33 +1148,97 @@ out: return ret; } +int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, + struct dev_pm_opp *opp, void *data, + bool scaling_down) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct list_head *head = &hba->clk_list_head; + struct ufs_clk_info *clki; + unsigned long freq; + u8 idx = 0; + int ret; + + list_for_each_entry(clki, head, list) { + if (!IS_ERR_OR_NULL(clki->clk)) { + freq = dev_pm_opp_get_freq_indexed(opp, idx++); + + /* Do not set rate for clocks having frequency as 0 */ + if (!freq) + continue; + + ret = clk_set_rate(clki->clk, freq); + if (ret) { + dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n", + __func__, clki->name, freq, ret); + return ret; + } + + trace_ufshcd_clk_scaling(hba, + (scaling_down ? "scaled down" : "scaled up"), + clki->name, hba->clk_scaling.target_freq, freq); + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks); + +static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq) +{ + struct dev_pm_opp *opp; + int ret; + + opp = dev_pm_opp_find_freq_floor_indexed(hba->dev, + &freq, 0); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(hba->dev, opp); + dev_pm_opp_put(opp); + + return ret; +} + /** * ufshcd_scale_clks - scale up or scale down UFS controller clocks * @hba: per adapter instance + * @freq: frequency to scale * @scale_up: True if scaling up and false if scaling down * - * Returns 0 if successful - * Returns < 0 for any other errors + * Return: 0 if successful; < 0 upon failure. */ -static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) +static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, + bool scale_up) { int ret = 0; ktime_t start = ktime_get(); - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE); + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, PRE_CHANGE); if (ret) goto out; - ret = ufshcd_set_clk_freq(hba, scale_up); + if (hba->use_pm_opp) + ret = ufshcd_opp_set_rate(hba, freq); + else + ret = ufshcd_set_clk_freq(hba, scale_up); if (ret) goto out; - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE); - if (ret) - ufshcd_set_clk_freq(hba, !scale_up); + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, POST_CHANGE); + if (ret) { + if (hba->use_pm_opp) + ufshcd_opp_set_rate(hba, + hba->devfreq->previous_freq); + else + ufshcd_set_clk_freq(hba, !scale_up); + goto out; + } + + ufshcd_pm_qos_update(hba, scale_up); out: - trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + trace_ufshcd_profile_clk_scaling(hba, (scale_up ? "up" : "down"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); return ret; @@ -1072,12 +1247,13 @@ out: /** * ufshcd_is_devfreq_scaling_required - check if scaling is required or not * @hba: per adapter instance + * @freq: frequency to scale * @scale_up: True if scaling up and false if scaling down * - * Returns true if scaling is required, false otherwise. + * Return: true if scaling is required, false otherwise. */ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, - bool scale_up) + unsigned long freq, bool scale_up) { struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; @@ -1085,6 +1261,9 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, if (list_empty(head)) return false; + if (hba->use_pm_opp) + return freq != hba->clk_scaling.target_freq; + list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk)) { if (scale_up && clki->max_freq) { @@ -1111,28 +1290,34 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, */ static u32 ufshcd_pending_cmds(struct ufs_hba *hba) { - const struct scsi_device *sdev; + struct scsi_device *sdev; + unsigned long flags; u32 pending = 0; - lockdep_assert_held(hba->host->host_lock); + spin_lock_irqsave(hba->host->host_lock, flags); __shost_for_each_device(sdev, hba->host) - pending += sbitmap_weight(&sdev->budget_map); + pending += scsi_device_busy(sdev); + spin_unlock_irqrestore(hba->host->host_lock, flags); return pending; } -static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, +/* + * Wait until all pending SCSI commands and TMFs have finished or the timeout + * has expired. + * + * Return: 0 upon success; -EBUSY upon timeout. + */ +static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, u64 wait_timeout_us) { - unsigned long flags; int ret = 0; u32 tm_doorbell; u32 tr_pending; bool timeout = false, do_last_check = false; ktime_t start; - ufshcd_hold(hba, false); - spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_hold(hba); /* * Wait for all the outstanding tasks/transfer requests. * Verify by checking the doorbell registers are clear. @@ -1153,8 +1338,7 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, break; } - spin_unlock_irqrestore(hba->host->host_lock, flags); - schedule(); + io_schedule_timeout(msecs_to_jiffies(20)); if (ktime_to_us(ktime_sub(ktime_get(), start)) > wait_timeout_us) { timeout = true; @@ -1165,7 +1349,6 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, */ do_last_check = true; } - spin_lock_irqsave(hba->host->host_lock, flags); } while (tm_doorbell || tr_pending); if (timeout) { @@ -1175,7 +1358,6 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, ret = -EBUSY; } out: - spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_release(hba); return ret; } @@ -1183,19 +1365,28 @@ out: /** * ufshcd_scale_gear - scale up/down UFS gear * @hba: per adapter instance + * @target_gear: target gear to scale to * @scale_up: True for scaling up gear and false for scaling down * - * Returns 0 for success, - * Returns -EBUSY if scaling can't happen at this time - * Returns non-zero for any other errors + * Return: 0 for success; -EBUSY if scaling can't happen at this time; + * non-zero for any other errors. */ -static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) +static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up) { int ret = 0; struct ufs_pa_layer_attr new_pwr_info; + if (target_gear) { + new_pwr_info = hba->pwr_info; + new_pwr_info.gear_tx = target_gear; + new_pwr_info.gear_rx = target_gear; + + goto config_pwr_mode; + } + + /* Legacy gear scaling, in case vops_freq_to_gear_speed() is not implemented */ if (scale_up) { - memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info, + memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info, sizeof(struct ufs_pa_layer_attr)); } else { memcpy(&new_pwr_info, &hba->pwr_info, @@ -1204,7 +1395,7 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear || hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) { /* save the current power mode */ - memcpy(&hba->clk_scaling.saved_pwr_info.info, + memcpy(&hba->clk_scaling.saved_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); @@ -1214,6 +1405,7 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) } } +config_pwr_mode: /* check if the power mode needs to be changed or not? */ ret = ufshcd_config_pwr_mode(hba, &new_pwr_info); if (ret) @@ -1225,90 +1417,104 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) return ret; } -static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba) +/* + * Wait until all pending SCSI commands and TMFs have finished or the timeout + * has expired. + * + * Return: 0 upon success; -EBUSY upon timeout. + */ +static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) { - #define DOORBELL_CLR_TOUT_US (1000 * 1000) /* 1 sec */ int ret = 0; /* * make sure that there are no outstanding requests when * clock scaling is in progress */ - ufshcd_scsi_block_requests(hba); + mutex_lock(&hba->host->scan_mutex); + blk_mq_quiesce_tagset(&hba->host->tag_set); mutex_lock(&hba->wb_mutex); down_write(&hba->clk_scaling_lock); if (!hba->clk_scaling.is_allowed || - ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) { + ufshcd_wait_for_pending_cmds(hba, timeout_us)) { ret = -EBUSY; up_write(&hba->clk_scaling_lock); mutex_unlock(&hba->wb_mutex); - ufshcd_scsi_unblock_requests(hba); + blk_mq_unquiesce_tagset(&hba->host->tag_set); + mutex_unlock(&hba->host->scan_mutex); goto out; } /* let's not get into low power until clock scaling is completed */ - ufshcd_hold(hba, false); + ufshcd_hold(hba); out: return ret; } -static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up) +static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err) { up_write(&hba->clk_scaling_lock); - /* Enable Write Booster if we have scaled up else disable it */ + /* Enable Write Booster if current gear requires it else disable it */ if (ufshcd_enable_wb_if_scaling_up(hba) && !err) - ufshcd_wb_toggle(hba, scale_up); + ufshcd_wb_toggle(hba, hba->pwr_info.gear_rx >= hba->clk_scaling.wb_gear); mutex_unlock(&hba->wb_mutex); - ufshcd_scsi_unblock_requests(hba); + blk_mq_unquiesce_tagset(&hba->host->tag_set); + mutex_unlock(&hba->host->scan_mutex); ufshcd_release(hba); } /** * ufshcd_devfreq_scale - scale up/down UFS clocks and gear * @hba: per adapter instance + * @freq: frequency to scale * @scale_up: True for scaling up and false for scalin down * - * Returns 0 for success, - * Returns -EBUSY if scaling can't happen at this time - * Returns non-zero for any other errors + * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero + * for any other errors. */ -static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up) +static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, + bool scale_up) { + u32 old_gear = hba->pwr_info.gear_rx; + u32 new_gear = 0; int ret = 0; - ret = ufshcd_clock_scaling_prepare(hba); + new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq); + + ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); if (ret) return ret; /* scale down the gear before scaling down clocks */ if (!scale_up) { - ret = ufshcd_scale_gear(hba, false); + ret = ufshcd_scale_gear(hba, new_gear, false); if (ret) goto out_unprepare; } - ret = ufshcd_scale_clks(hba, scale_up); + ret = ufshcd_scale_clks(hba, freq, scale_up); if (ret) { if (!scale_up) - ufshcd_scale_gear(hba, true); + ufshcd_scale_gear(hba, old_gear, true); goto out_unprepare; } /* scale up the gear after scaling up clocks */ if (scale_up) { - ret = ufshcd_scale_gear(hba, true); + ret = ufshcd_scale_gear(hba, new_gear, true); if (ret) { - ufshcd_scale_clks(hba, false); + ufshcd_scale_clks(hba, hba->devfreq->previous_freq, + false); goto out_unprepare; } } out_unprepare: - ufshcd_clock_scaling_unprepare(hba, ret, scale_up); + ufshcd_clock_scaling_unprepare(hba, ret); return ret; } @@ -1316,32 +1522,31 @@ static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_scaling.suspend_work); - unsigned long irq_flags; - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (hba->clk_scaling.active_reqs || + hba->clk_scaling.is_suspended) + return; + + hba->clk_scaling.is_suspended = true; + hba->clk_scaling.window_start_t = 0; } - hba->clk_scaling.is_suspended = true; - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - __ufshcd_suspend_clkscaling(hba); + devfreq_suspend_device(hba->devfreq); } static void ufshcd_clk_scaling_resume_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_scaling.resume_work); - unsigned long irq_flags; - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (!hba->clk_scaling.is_suspended) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (!hba->clk_scaling.is_suspended) + return; + hba->clk_scaling.is_suspended = false; } - hba->clk_scaling.is_suspended = false; - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); devfreq_resume_device(hba->devfreq); } @@ -1352,52 +1557,74 @@ static int ufshcd_devfreq_target(struct device *dev, int ret = 0; struct ufs_hba *hba = dev_get_drvdata(dev); ktime_t start; - bool scale_up, sched_clk_scaling_suspend_work = false; + bool scale_up = false, sched_clk_scaling_suspend_work = false; struct list_head *clk_list = &hba->clk_list_head; struct ufs_clk_info *clki; - unsigned long irq_flags; if (!ufshcd_is_clkscaling_supported(hba)) return -EINVAL; - clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); - /* Override with the closest supported frequency */ - *freq = (unsigned long) clk_round_rate(clki->clk, *freq); - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (ufshcd_eh_in_progress(hba)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return 0; - } + if (hba->use_pm_opp) { + struct dev_pm_opp *opp; - if (!hba->clk_scaling.active_reqs) - sched_clk_scaling_suspend_work = true; + /* Get the recommended frequency from OPP framework */ + opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(opp)) + return PTR_ERR(opp); - if (list_empty(clk_list)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - goto out; + dev_pm_opp_put(opp); + } else { + /* Override with the closest supported frequency */ + clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, + list); + *freq = (unsigned long) clk_round_rate(clki->clk, *freq); } - /* Decide based on the rounded-off frequency and update */ - scale_up = *freq == clki->max_freq; - if (!scale_up) - *freq = clki->min_freq; - /* Update the frequency */ - if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - ret = 0; - goto out; /* no state change required */ + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (ufshcd_eh_in_progress(hba)) + return 0; + + /* Skip scaling clock when clock scaling is suspended */ + if (hba->clk_scaling.is_suspended) { + dev_warn(hba->dev, "clock scaling is suspended, skip"); + return 0; + } + + if (!hba->clk_scaling.active_reqs) + sched_clk_scaling_suspend_work = true; + + if (list_empty(clk_list)) + goto out; + + /* Decide based on the target or rounded-off frequency and update */ + if (hba->use_pm_opp) + scale_up = *freq > hba->clk_scaling.target_freq; + else + scale_up = *freq == clki->max_freq; + + if (!hba->use_pm_opp && !scale_up) + *freq = clki->min_freq; + + /* Update the frequency */ + if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { + ret = 0; + goto out; /* no state change required */ + } } - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); start = ktime_get(); - ret = ufshcd_devfreq_scale(hba, scale_up); + ret = ufshcd_devfreq_scale(hba, *freq, scale_up); + if (!ret) + hba->clk_scaling.target_freq = *freq; - trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + trace_ufshcd_profile_clk_scaling(hba, (scale_up ? "up" : "down"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); out: - if (sched_clk_scaling_suspend_work) + if (sched_clk_scaling_suspend_work && + (!scale_up || hba->clk_scaling.suspend_on_no_request)) queue_work(hba->clk_scaling.workq, &hba->clk_scaling.suspend_work); @@ -1409,9 +1636,6 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_clk_scaling *scaling = &hba->clk_scaling; - unsigned long flags; - struct list_head *clk_list = &hba->clk_list_head; - struct ufs_clk_info *clki; ktime_t curr_t; if (!ufshcd_is_clkscaling_supported(hba)) @@ -1419,36 +1643,44 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, memset(stat, 0, sizeof(*stat)); - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + curr_t = ktime_get(); if (!scaling->window_start_t) goto start_window; - clki = list_first_entry(clk_list, struct ufs_clk_info, list); /* * If current frequency is 0, then the ondemand governor considers * there's no initial frequency set. And it always requests to set * to max. frequency. */ - stat->current_frequency = clki->curr_freq; + if (hba->use_pm_opp) { + stat->current_frequency = hba->clk_scaling.target_freq; + } else { + struct list_head *clk_list = &hba->clk_list_head; + struct ufs_clk_info *clki; + + clki = list_first_entry(clk_list, struct ufs_clk_info, list); + stat->current_frequency = clki->curr_freq; + } + if (scaling->is_busy_started) scaling->tot_busy_t += ktime_us_delta(curr_t, scaling->busy_start_t); - stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); stat->busy_time = scaling->tot_busy_t; start_window: scaling->window_start_t = curr_t; scaling->tot_busy_t = 0; - if (hba->outstanding_reqs) { + if (scaling->active_reqs) { scaling->busy_start_t = curr_t; scaling->is_busy_started = true; } else { scaling->busy_start_t = 0; scaling->is_busy_started = false; } - spin_unlock_irqrestore(hba->host->host_lock, flags); + return 0; } @@ -1463,9 +1695,11 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba) if (list_empty(clk_list)) return 0; - clki = list_first_entry(clk_list, struct ufs_clk_info, list); - dev_pm_opp_add(hba->dev, clki->min_freq, 0); - dev_pm_opp_add(hba->dev, clki->max_freq, 0); + if (!hba->use_pm_opp) { + clki = list_first_entry(clk_list, struct ufs_clk_info, list); + dev_pm_opp_add(hba->dev, clki->min_freq, 0); + dev_pm_opp_add(hba->dev, clki->max_freq, 0); + } ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile, &hba->vps->ondemand_data); @@ -1477,8 +1711,10 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba) ret = PTR_ERR(devfreq); dev_err(hba->dev, "Unable to register with devfreq %d\n", ret); - dev_pm_opp_remove(hba->dev, clki->min_freq); - dev_pm_opp_remove(hba->dev, clki->max_freq); + if (!hba->use_pm_opp) { + dev_pm_opp_remove(hba->dev, clki->min_freq); + dev_pm_opp_remove(hba->dev, clki->max_freq); + } return ret; } @@ -1490,7 +1726,6 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba) static void ufshcd_devfreq_remove(struct ufs_hba *hba) { struct list_head *clk_list = &hba->clk_list_head; - struct ufs_clk_info *clki; if (!hba->devfreq) return; @@ -1498,51 +1733,46 @@ static void ufshcd_devfreq_remove(struct ufs_hba *hba) devfreq_remove_device(hba->devfreq); hba->devfreq = NULL; - clki = list_first_entry(clk_list, struct ufs_clk_info, list); - dev_pm_opp_remove(hba->dev, clki->min_freq); - dev_pm_opp_remove(hba->dev, clki->max_freq); -} + if (!hba->use_pm_opp) { + struct ufs_clk_info *clki; -static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba) -{ - unsigned long flags; - - devfreq_suspend_device(hba->devfreq); - spin_lock_irqsave(hba->host->host_lock, flags); - hba->clk_scaling.window_start_t = 0; - spin_unlock_irqrestore(hba->host->host_lock, flags); + clki = list_first_entry(clk_list, struct ufs_clk_info, list); + dev_pm_opp_remove(hba->dev, clki->min_freq); + dev_pm_opp_remove(hba->dev, clki->max_freq); + } } static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) { - unsigned long flags; bool suspend = false; cancel_work_sync(&hba->clk_scaling.suspend_work); cancel_work_sync(&hba->clk_scaling.resume_work); - spin_lock_irqsave(hba->host->host_lock, flags); - if (!hba->clk_scaling.is_suspended) { - suspend = true; - hba->clk_scaling.is_suspended = true; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (!hba->clk_scaling.is_suspended) { + suspend = true; + hba->clk_scaling.is_suspended = true; + hba->clk_scaling.window_start_t = 0; + } } - spin_unlock_irqrestore(hba->host->host_lock, flags); if (suspend) - __ufshcd_suspend_clkscaling(hba); + devfreq_suspend_device(hba->devfreq); } static void ufshcd_resume_clkscaling(struct ufs_hba *hba) { - unsigned long flags; bool resume = false; - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_scaling.is_suspended) { - resume = true; - hba->clk_scaling.is_suspended = false; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (hba->clk_scaling.is_suspended) { + resume = true; + hba->clk_scaling.is_suspended = false; + } } - spin_unlock_irqrestore(hba->host->host_lock, flags); if (resume) devfreq_resume_device(hba->devfreq); @@ -1560,6 +1790,8 @@ static ssize_t ufshcd_clkscale_enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_clk_info *clki; + unsigned long freq; u32 value; int err = 0; @@ -1577,20 +1809,31 @@ static ssize_t ufshcd_clkscale_enable_store(struct device *dev, goto out; ufshcd_rpm_get_sync(hba); - ufshcd_hold(hba, false); + ufshcd_hold(hba); hba->clk_scaling.is_enabled = value; if (value) { ufshcd_resume_clkscaling(hba); - } else { - ufshcd_suspend_clkscaling(hba); - err = ufshcd_devfreq_scale(hba, true); - if (err) - dev_err(hba->dev, "%s: failed to scale clocks up %d\n", - __func__, err); + goto out_rel; } + clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); + freq = clki->max_freq; + + ufshcd_suspend_clkscaling(hba); + + if (!ufshcd_is_devfreq_scaling_required(hba, freq, true)) + goto out_rel; + + err = ufshcd_devfreq_scale(hba, freq, true); + if (err) + dev_err(hba->dev, "%s: failed to scale clocks up %d\n", + __func__, err); + else + hba->clk_scaling.target_freq = freq; + +out_rel: ufshcd_release(hba); ufshcd_rpm_put_sync(hba); out: @@ -1617,22 +1860,25 @@ static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba) static void ufshcd_init_clk_scaling(struct ufs_hba *hba) { - char wq_name[sizeof("ufs_clkscaling_00")]; - if (!ufshcd_is_clkscaling_supported(hba)) return; if (!hba->clk_scaling.min_gear) hba->clk_scaling.min_gear = UFS_HS_G1; + if (!hba->clk_scaling.wb_gear) + /* Use intermediate gear speed HS_G3 as the default wb_gear */ + hba->clk_scaling.wb_gear = UFS_HS_G3; + INIT_WORK(&hba->clk_scaling.suspend_work, ufshcd_clk_scaling_suspend_work); INIT_WORK(&hba->clk_scaling.resume_work, ufshcd_clk_scaling_resume_work); - snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d", - hba->host->host_no); - hba->clk_scaling.workq = create_singlethread_workqueue(wq_name); + spin_lock_init(&hba->clk_scaling.lock); + + hba->clk_scaling.workq = alloc_ordered_workqueue( + "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no); hba->clk_scaling.is_initialized = true; } @@ -1651,19 +1897,16 @@ static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) static void ufshcd_ungate_work(struct work_struct *work) { int ret; - unsigned long flags; struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_gating.ungate_work); cancel_delayed_work_sync(&hba->clk_gating.gate_work); - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_gating.state == CLKS_ON) { - spin_unlock_irqrestore(hba->host->host_lock, flags); - goto unblock_reqs; + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { + if (hba->clk_gating.state == CLKS_ON) + return; } - spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_hba_vreg_set_hpm(hba); ufshcd_setup_clocks(hba, true); @@ -1683,26 +1926,22 @@ static void ufshcd_ungate_work(struct work_struct *work) } hba->clk_gating.is_suspended = false; } -unblock_reqs: - ufshcd_scsi_unblock_requests(hba); } /** * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release. * Also, exit from hibern8 mode and set the link as active. * @hba: per adapter instance - * @async: This indicates whether caller should ungate clocks asynchronously. */ -int ufshcd_hold(struct ufs_hba *hba, bool async) +void ufshcd_hold(struct ufs_hba *hba) { - int rc = 0; bool flush_result; unsigned long flags; if (!ufshcd_is_clkgating_allowed(hba) || !hba->clk_gating.is_initialized) - goto out; - spin_lock_irqsave(hba->host->host_lock, flags); + return; + spin_lock_irqsave(&hba->clk_gating.lock, flags); hba->clk_gating.active_reqs++; start: @@ -1718,23 +1957,18 @@ start: */ if (ufshcd_can_hibern8_during_gating(hba) && ufshcd_is_link_hibern8(hba)) { - if (async) { - rc = -EAGAIN; - hba->clk_gating.active_reqs--; - break; - } - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); flush_result = flush_work(&hba->clk_gating.ungate_work); if (hba->clk_gating.is_suspended && !flush_result) - goto out; - spin_lock_irqsave(hba->host->host_lock, flags); + return; + spin_lock_irqsave(&hba->clk_gating.lock, flags); goto start; } break; case REQ_CLKS_OFF: if (cancel_delayed_work(&hba->clk_gating.gate_work)) { hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); break; } @@ -1746,36 +1980,27 @@ start: fallthrough; case CLKS_OFF: hba->clk_gating.state = REQ_CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); - if (queue_work(hba->clk_gating.clk_gating_workq, - &hba->clk_gating.ungate_work)) - ufshcd_scsi_block_requests(hba); + queue_work(hba->clk_gating.clk_gating_workq, + &hba->clk_gating.ungate_work); /* * fall through to check if we should wait for this * work to be done or not. */ fallthrough; case REQ_CLKS_ON: - if (async) { - rc = -EAGAIN; - hba->clk_gating.active_reqs--; - break; - } - - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); flush_work(&hba->clk_gating.ungate_work); /* Make sure state is CLKS_ON before returning */ - spin_lock_irqsave(hba->host->host_lock, flags); + spin_lock_irqsave(&hba->clk_gating.lock, flags); goto start; default: dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", __func__, hba->clk_gating.state); break; } - spin_unlock_irqrestore(hba->host->host_lock, flags); -out: - return rc; + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_hold); @@ -1783,31 +2008,32 @@ static void ufshcd_gate_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_gating.gate_work.work); - unsigned long flags; int ret; - spin_lock_irqsave(hba->host->host_lock, flags); - /* - * In case you are here to cancel this work the gating state - * would be marked as REQ_CLKS_ON. In this case save time by - * skipping the gating work and exit after changing the clock - * state to CLKS_ON. - */ - if (hba->clk_gating.is_suspended || - (hba->clk_gating.state != REQ_CLKS_OFF)) { - hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), - hba->clk_gating.state); - goto rel_lock; - } + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { + /* + * In case you are here to cancel this work the gating state + * would be marked as REQ_CLKS_ON. In this case save time by + * skipping the gating work and exit after changing the clock + * state to CLKS_ON. + */ + if (hba->clk_gating.is_suspended || + hba->clk_gating.state != REQ_CLKS_OFF) { + hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(hba, + hba->clk_gating.state); + return; + } - if (hba->clk_gating.active_reqs - || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL - || hba->outstanding_reqs || hba->outstanding_tasks - || hba->active_uic_cmd || hba->uic_async_done) - goto rel_lock; + if (hba->clk_gating.active_reqs) + return; + } - spin_unlock_irqrestore(hba->host->host_lock, flags); + scoped_guard(spinlock_irqsave, hba->host->host_lock) { + if (ufshcd_is_ufs_dev_busy(hba) || + hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) + return; + } /* put the link into hibern8 mode before turning off clocks */ if (ufshcd_can_hibern8_during_gating(hba)) { @@ -1816,9 +2042,9 @@ static void ufshcd_gate_work(struct work_struct *work) hba->clk_gating.state = CLKS_ON; dev_err(hba->dev, "%s: hibern8 enter failed %d\n", __func__, ret); - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); - goto out; + return; } ufshcd_set_link_hibern8(hba); } @@ -1838,35 +2064,36 @@ static void ufshcd_gate_work(struct work_struct *work) * prevent from doing cancel work multiple times when there are * new requests arriving before the current cancel work is done. */ - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); if (hba->clk_gating.state == REQ_CLKS_OFF) { hba->clk_gating.state = CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); } -rel_lock: - spin_unlock_irqrestore(hba->host->host_lock, flags); -out: - return; } -/* host lock must be held before calling this variant */ static void __ufshcd_release(struct ufs_hba *hba) { + lockdep_assert_held(&hba->clk_gating.lock); + if (!ufshcd_is_clkgating_allowed(hba)) return; hba->clk_gating.active_reqs--; if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || - hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL || - hba->outstanding_tasks || !hba->clk_gating.is_initialized || - hba->active_uic_cmd || hba->uic_async_done || + !hba->clk_gating.is_initialized || hba->clk_gating.state == CLKS_OFF) return; + scoped_guard(spinlock_irqsave, hba->host->host_lock) { + if (ufshcd_has_pending_tasks(hba) || + hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) + return; + } + hba->clk_gating.state = REQ_CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state); + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); queue_delayed_work(hba->clk_gating.clk_gating_workq, &hba->clk_gating.gate_work, msecs_to_jiffies(hba->clk_gating.delay_ms)); @@ -1874,11 +2101,8 @@ static void __ufshcd_release(struct ufs_hba *hba) void ufshcd_release(struct ufs_hba *hba) { - unsigned long flags; - - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); __ufshcd_release(hba); - spin_unlock_irqrestore(hba->host->host_lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_release); @@ -1893,11 +2117,9 @@ static ssize_t ufshcd_clkgate_delay_show(struct device *dev, void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) { struct ufs_hba *hba = dev_get_drvdata(dev); - unsigned long flags; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); hba->clk_gating.delay_ms = value; - spin_unlock_irqrestore(hba->host->host_lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); @@ -1925,7 +2147,6 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); - unsigned long flags; u32 value; if (kstrtou32(buf, 0, &value)) @@ -1933,9 +2154,10 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, value = !!value; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); + if (value == hba->clk_gating.is_enabled) - goto out; + return count; if (value) __ufshcd_release(hba); @@ -1943,8 +2165,7 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, hba->clk_gating.active_reqs++; hba->clk_gating.is_enabled = value; -out: - spin_unlock_irqrestore(hba->host->host_lock, flags); + return count; } @@ -1977,8 +2198,6 @@ static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba) static void ufshcd_init_clk_gating(struct ufs_hba *hba) { - char wq_name[sizeof("ufs_clk_gating_00")]; - if (!ufshcd_is_clkgating_allowed(hba)) return; @@ -1988,10 +2207,9 @@ static void ufshcd_init_clk_gating(struct ufs_hba *hba) INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); - snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d", - hba->host->host_no); - hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name, - WQ_MEM_RECLAIM | WQ_HIGHPRI); + hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue( + "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI, + hba->host->host_no); ufshcd_init_clk_gating_sysfs(hba); @@ -2007,7 +2225,7 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) ufshcd_remove_clk_gating_sysfs(hba); /* Ungate the clock if necessary. */ - ufshcd_hold(hba, false); + ufshcd_hold(hba); hba->clk_gating.is_initialized = false; ufshcd_release(hba); @@ -2017,20 +2235,20 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) { bool queue_resume_work = false; - ktime_t curr_t = ktime_get(); - unsigned long flags; + ktime_t curr_t; if (!ufshcd_is_clkscaling_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); + curr_t = ktime_get(); + + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + if (!hba->clk_scaling.active_reqs++) queue_resume_work = true; - if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) { - spin_unlock_irqrestore(hba->host->host_lock, flags); + if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) return; - } if (queue_resume_work) queue_work(hba->clk_scaling.workq, @@ -2046,26 +2264,24 @@ static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) hba->clk_scaling.busy_start_t = curr_t; hba->clk_scaling.is_busy_started = true; } - spin_unlock_irqrestore(hba->host->host_lock, flags); } static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) { struct ufs_clk_scaling *scaling = &hba->clk_scaling; - unsigned long flags; if (!ufshcd_is_clkscaling_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + hba->clk_scaling.active_reqs--; - if (!hba->outstanding_reqs && scaling->is_busy_started) { + if (!scaling->active_reqs && scaling->is_busy_started) { scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), scaling->busy_start_t)); scaling->busy_start_t = 0; scaling->is_busy_started = false; } - spin_unlock_irqrestore(hba->host->host_lock, flags); } static inline int ufshcd_monitor_opcode2dir(u8 opcode) @@ -2078,20 +2294,21 @@ static inline int ufshcd_monitor_opcode2dir(u8 opcode) return -EINVAL; } +/* Must only be called for SCSI commands. */ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { const struct ufs_hba_monitor *m = &hba->monitor; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); - return (m->enabled && lrbp && lrbp->cmd && - (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) && - ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp)); + return m->enabled && + (!m->chunk_size || m->chunk_size == cmd->sdb.length) && + ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp); } -static void ufshcd_start_monitor(struct ufs_hba *hba, - const struct ufshcd_lrb *lrbp) +static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); @@ -2100,14 +2317,15 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, spin_unlock_irqrestore(hba->host->host_lock, flags); } -static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp) +static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); + struct request *req = scsi_cmd_to_rq(cmd); + const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { - const struct request *req = scsi_cmd_to_rq(lrbp->cmd); struct ufs_hba_monitor *m = &hba->monitor; ktime_t now, inc, lat; @@ -2132,45 +2350,73 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb * spin_unlock_irqrestore(hba->host->host_lock, flags); } +/* Returns %true for SCSI commands and %false for device management commands. */ +static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) +{ + return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)); +} + /** * ufshcd_send_command - Send SCSI or device management commands * @hba: per adapter instance - * @task_tag: Task tag of the command + * @cmd: SCSI command or device management command pointer + * @hwq: pointer to hardware queue instance */ -static inline -void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) +static inline void ufshcd_send_command(struct ufs_hba *hba, + struct scsi_cmnd *cmd, + struct ufs_hw_queue *hwq) { - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; unsigned long flags; - lrbp->issue_time_stamp = ktime_get(); - lrbp->issue_time_stamp_local_clock = local_clock(); - lrbp->compl_time_stamp = ktime_set(0, 0); - lrbp->compl_time_stamp_local_clock = 0; - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); - ufshcd_clk_scaling_start_busy(hba); - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) - ufshcd_start_monitor(hba, lrbp); - - spin_lock_irqsave(&hba->outstanding_lock, flags); - if (hba->vops && hba->vops->setup_xfer_req) - hba->vops->setup_xfer_req(hba, task_tag, !!lrbp->cmd); - __set_bit(task_tag, &hba->outstanding_reqs); - ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); - spin_unlock_irqrestore(&hba->outstanding_lock, flags); + if (hba->monitor.enabled) { + lrbp->issue_time_stamp = ktime_get(); + lrbp->issue_time_stamp_local_clock = local_clock(); + lrbp->compl_time_stamp = ktime_set(0, 0); + lrbp->compl_time_stamp_local_clock = 0; + } + if (ufshcd_is_scsi_cmd(cmd)) { + ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); + ufshcd_clk_scaling_start_busy(hba); + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) + ufshcd_start_monitor(hba, cmd); + } + + if (hba->mcq_enabled) { + int utrd_size = sizeof(struct utp_transfer_req_desc); + struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr; + struct utp_transfer_req_desc *dest; + + spin_lock(&hwq->sq_lock); + dest = hwq->sqe_base_addr + hwq->sq_tail_slot; + memcpy(dest, src, utrd_size); + ufshcd_inc_sq_tail(hwq); + spin_unlock(&hwq->sq_lock); + } else { + spin_lock_irqsave(&hba->outstanding_lock, flags); + if (hba->vops && hba->vops->setup_xfer_req) + hba->vops->setup_xfer_req(hba, tag, + ufshcd_is_scsi_cmd(cmd)); + __set_bit(tag, &hba->outstanding_reqs); + ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); + spin_unlock_irqrestore(&hba->outstanding_lock, flags); + } } /** * ufshcd_copy_sense_data - Copy sense data in case of check condition - * @lrbp: pointer to local reference block + * @cmd: SCSI command */ -static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) +static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) { - u8 *const sense_buffer = lrbp->cmd->sense_buffer; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u8 *const sense_buffer = cmd->sense_buffer; + u16 resp_len; int len; - if (sense_buffer && - ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { + resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length); + if (sense_buffer && resp_len) { int len_to_copy; len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); @@ -2186,6 +2432,8 @@ static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) * descriptor * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) @@ -2203,8 +2451,8 @@ int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) u16 buf_len; /* data segment length */ - resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & - MASK_QUERY_DATA_SEG_LEN; + resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header + .data_segment_length); buf_len = be16_to_cpu( hba->dev_cmd.query.request.upiu_req.length); if (likely(buf_len >= resp_len)) { @@ -2231,32 +2479,55 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) int err; hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); - if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS) - hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT; /* nutrs and nutmrs are 0 based values */ - hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1; + hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; hba->nutmrs = ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; - hba->reserved_slot = hba->nutrs - 1; + + hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; /* Read crypto capabilities */ err = ufshcd_hba_init_crypto_capabilities(hba); - if (err) + if (err) { dev_err(hba->dev, "crypto setup failed\n"); + return err; + } - return err; + /* + * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and + * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which + * means we can simply read values regardless of version. + */ + hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities); + /* + * 0h: legacy single doorbell support is available + * 1h: indicate that legacy single doorbell support has been removed + */ + if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP)) + hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities); + else + hba->lsdb_sup = true; + + hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); + + return 0; } /** * ufshcd_ready_for_uic_cmd - Check if controller is ready * to accept UIC commands * @hba: per adapter instance - * Return true on success, else false + * + * Return: true on success, else false. */ static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) { - return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY; + u32 val; + int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY, + 500, uic_cmd_timeout * 1000, false, hba, + REG_CONTROLLER_STATUS); + return ret == 0; } /** @@ -2264,7 +2535,8 @@ static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) * @hba: Pointer to adapter instance * * This function gets the UPMCRS field of HCS register - * Returns value of UPMCRS field + * + * Return: value of UPMCRS field. */ static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) { @@ -2302,7 +2574,7 @@ ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) * @hba: per adapter instance * @uic_cmd: UIC command * - * Returns 0 only if success. + * Return: 0 only if success. */ static int ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) @@ -2313,7 +2585,7 @@ ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) lockdep_assert_held(&hba->uic_cmd_mutex); if (wait_for_completion_timeout(&uic_cmd->done, - msecs_to_jiffies(UIC_CMD_TIMEOUT))) { + msecs_to_jiffies(uic_cmd_timeout))) { ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; } else { ret = -ETIMEDOUT; @@ -2339,16 +2611,13 @@ ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result * @hba: per adapter instance * @uic_cmd: UIC command - * @completion: initialize the completion only if this is set to true * - * Returns 0 only if success. + * Return: 0 if successful; < 0 upon failure. */ static int -__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, - bool completion) +__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { lockdep_assert_held(&hba->uic_cmd_mutex); - lockdep_assert_held(hba->host->host_lock); if (!ufshcd_ready_for_uic_cmd(hba)) { dev_err(hba->dev, @@ -2356,10 +2625,9 @@ __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, return -EIO; } - if (completion) - init_completion(&uic_cmd->done); + init_completion(&uic_cmd->done); - uic_cmd->cmd_active = 1; + uic_cmd->cmd_active = true; ufshcd_dispatch_uic_cmd(hba, uic_cmd); return 0; @@ -2370,23 +2638,25 @@ __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, * @hba: per adapter instance * @uic_cmd: UIC command * - * Returns 0 only if success. + * Return: 0 only if success. */ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { - int ret; unsigned long flags; + int ret; if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) return 0; - ufshcd_hold(hba, false); + ufshcd_hold(hba); mutex_lock(&hba->uic_cmd_mutex); ufshcd_add_delay_before_dme_cmd(hba); spin_lock_irqsave(hba->host->host_lock, flags); - ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true); + ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); spin_unlock_irqrestore(hba->host->host_lock, flags); + + ret = __ufshcd_send_uic_cmd(hba, uic_cmd); if (!ret) ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); @@ -2397,38 +2667,30 @@ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) } /** - * ufshcd_map_sg - Map scatter-gather list to prdt - * @hba: per adapter instance - * @lrbp: pointer to local reference block - * - * Returns 0 in case of success, non-zero value in case of failure + * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format) + * @hba: per-adapter instance + * @lrbp: pointer to local reference block + * @sg_entries: The number of sg lists actually used + * @sg_list: Pointer to SG list */ -static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries, + struct scatterlist *sg_list) { - struct ufshcd_sg_entry *prd_table; + struct ufshcd_sg_entry *prd; struct scatterlist *sg; - struct scsi_cmnd *cmd; - int sg_segments; int i; - cmd = lrbp->cmd; - sg_segments = scsi_dma_map(cmd); - if (sg_segments < 0) - return sg_segments; - - if (sg_segments) { + if (sg_entries) { if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) lrbp->utr_descriptor_ptr->prd_table_length = - cpu_to_le16((sg_segments * - sizeof(struct ufshcd_sg_entry))); + cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba)); else - lrbp->utr_descriptor_ptr->prd_table_length = - cpu_to_le16(sg_segments); + lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries); - prd_table = lrbp->ucd_prdt_ptr; + prd = lrbp->ucd_prdt_ptr; - scsi_for_each_sg(cmd, sg, sg_segments, i) { + for_each_sg(sg_list, sg, sg_entries, i) { const unsigned int len = sg_dma_len(sg); /* @@ -2439,75 +2701,58 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) * 11b to indicate Dword granularity. A value of '3' * indicates 4 bytes, '7' indicates 8 bytes, etc." */ - WARN_ONCE(len > 256 * 1024, "len = %#x\n", len); - prd_table[i].size = cpu_to_le32(len - 1); - prd_table[i].addr = cpu_to_le64(sg->dma_address); - prd_table[i].reserved = 0; + WARN_ONCE(len > SZ_256K, "len = %#x\n", len); + prd->size = cpu_to_le32(len - 1); + prd->addr = cpu_to_le64(sg->dma_address); + prd->reserved = 0; + prd = (void *)prd + ufshcd_sg_entry_size(hba); } } else { lrbp->utr_descriptor_ptr->prd_table_length = 0; } - - return 0; } /** - * ufshcd_enable_intr - enable interrupts + * ufshcd_map_sg - Map scatter-gather list to prdt * @hba: per adapter instance - * @intrs: interrupt bits + * @cmd: SCSI command + * + * Return: 0 in case of success, non-zero value in case of failure. */ -static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) +static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); - - if (hba->ufs_version == ufshci_version(1, 0)) { - u32 rw; - rw = set & INTERRUPT_MASK_RW_VER_10; - set = rw | ((set ^ intrs) & intrs); - } else { - set |= intrs; - } - - ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); -} + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + int sg_segments = scsi_dma_map(cmd); -/** - * ufshcd_disable_intr - disable interrupts - * @hba: per adapter instance - * @intrs: interrupt bits - */ -static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) -{ - u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + if (sg_segments < 0) + return sg_segments; - if (hba->ufs_version == ufshci_version(1, 0)) { - u32 rw; - rw = (set & INTERRUPT_MASK_RW_VER_10) & - ~(intrs & INTERRUPT_MASK_RW_VER_10); - set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10); + ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); - } else { - set &= ~intrs; - } - - ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); + return ufshcd_crypto_fill_prdt(hba, cmd); } /** - * ufshcd_prepare_req_desc_hdr() - Fills the requests header + * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request * descriptor according to request + * @hba: per adapter instance * @lrbp: pointer to local reference block * @upiu_flags: flags required in the header * @cmd_dir: requests data direction + * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments) */ -static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, - u8 *upiu_flags, enum dma_data_direction cmd_dir) +static void +ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, + u8 *upiu_flags, enum dma_data_direction cmd_dir, + int ehs_length) { struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; - u32 data_direction; - u32 dword_0; - u32 dword_1 = 0; - u32 dword_3 = 0; + struct request_desc_header *h = &req_desc->header; + enum utp_data_direction data_direction; + + lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + + *h = (typeof(*h)){ }; if (cmd_dir == DMA_FROM_DEVICE) { data_direction = UTP_DEVICE_TO_HOST; @@ -2520,25 +2765,22 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, *upiu_flags = UPIU_CMD_FLAGS_NONE; } - dword_0 = data_direction | (lrbp->command_type - << UPIU_COMMAND_TYPE_OFFSET); + h->command_type = lrbp->command_type; + h->data_direction = data_direction; + h->ehs_length = ehs_length; + if (lrbp->intr_cmd) - dword_0 |= UTP_REQ_DESC_INT_CMD; + h->interrupt = 1; /* Prepare crypto related dwords */ - ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3); + ufshcd_prepare_req_desc_hdr_crypto(lrbp, h); - /* Transfer request descriptor header fields */ - req_desc->header.dword_0 = cpu_to_le32(dword_0); - req_desc->header.dword_1 = cpu_to_le32(dword_1); /* * assigning invalid value for command status. Controller * updates OCS on command completion, with the command * status */ - req_desc->header.dword_2 = - cpu_to_le32(OCS_INVALID_COMMAND_STATUS); - req_desc->header.dword_3 = cpu_to_le32(dword_3); + h->ocs = OCS_INVALID_COMMAND_STATUS; req_desc->prd_table_length = 0; } @@ -2546,62 +2788,64 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, /** * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, * for scsi commands - * @lrbp: local reference block pointer + * @cmd: SCSI command * @upiu_flags: flags */ -static -void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) +static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, + u8 upiu_flags) { - struct scsi_cmnd *cmd = lrbp->cmd; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; unsigned short cdb_len; - /* command descriptor fields */ - ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( - UPIU_TRANSACTION_COMMAND, upiu_flags, - lrbp->lun, lrbp->task_tag); - ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( - UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_COMMAND, + .flags = upiu_flags, + .lun = lrbp->lun, + .task_tag = tag, + .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, + }; - /* Total EHS length and Data segment length will be zero */ - ucd_req_ptr->header.dword_2 = 0; + WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag); ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE); - memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE); memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len); memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); } /** - * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc, - * for query requsts + * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request * @hba: UFS hba - * @lrbp: local reference block pointer + * @cmd: SCSI command pointer * @upiu_flags: flags */ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, u8 upiu_flags) + struct scsi_cmnd *cmd, u8 upiu_flags) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + const int tag = scsi_cmd_to_rq(cmd)->tag; struct ufs_query *query = &hba->dev_cmd.query; u16 len = be16_to_cpu(query->request.upiu_req.length); /* Query request header */ - ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( - UPIU_TRANSACTION_QUERY_REQ, upiu_flags, - lrbp->lun, lrbp->task_tag); - ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( - 0, query->request.query_func, 0, 0); - - /* Data segment length only need for WRITE_DESC */ - if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) - ucd_req_ptr->header.dword_2 = - UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len); - else - ucd_req_ptr->header.dword_2 = 0; + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_QUERY_REQ, + .flags = upiu_flags, + .lun = lrbp->lun, + .task_tag = tag, + .query_function = query->request.query_func, + /* Data segment length only need for WRITE_DESC */ + .data_segment_length = + query->request.upiu_req.opcode == + UPIU_QUERY_OPCODE_WRITE_DESC ? + cpu_to_be16(len) : + 0, + }; /* Copy the Query Request buffer as is */ memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, @@ -2610,52 +2854,48 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, /* Copy the Descriptor */ if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) memcpy(ucd_req_ptr + 1, query->descriptor, len); - - memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); } -static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) +static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + const int tag = scsi_cmd_to_rq(cmd)->tag; memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); - /* command descriptor fields */ - ucd_req_ptr->header.dword_0 = - UPIU_HEADER_DWORD( - UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); - /* clear rest of the fields of basic header */ - ucd_req_ptr->header.dword_1 = 0; - ucd_req_ptr->header.dword_2 = 0; - - memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_NOP_OUT, + .task_tag = tag, + }; } /** * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) * for Device Management Purposes * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command pointer + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); u8 upiu_flags; int ret = 0; - if (hba->ufs_version <= ufshci_version(1, 1)) - lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; - else - lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); - ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE); if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) - ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags); + ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags); else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) - ufshcd_prepare_utp_nop_upiu(lrbp); + ufshcd_prepare_utp_nop_upiu(cmd); else ret = -EINVAL; + memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); + return ret; } @@ -2663,34 +2903,76 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) * for SCSI Purposes * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command */ -static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); + unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); u8 upiu_flags; - int ret = 0; - if (hba->ufs_version <= ufshci_version(1, 1)) - lrbp->command_type = UTP_CMD_TYPE_SCSI; - else - lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, + cmd->sc_data_direction, 0); + if (ioprio_class == IOPRIO_CLASS_RT) + upiu_flags |= UPIU_CMD_FLAGS_CP; + ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); +} - if (likely(lrbp->cmd)) { - ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, - lrbp->cmd->sc_data_direction); - ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); - } else { - ret = -EINVAL; - } +static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) +{ + const int i = scsi_cmd_to_rq(cmd)->tag; + struct utp_transfer_cmd_desc *cmd_descp = + (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); + struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; + dma_addr_t cmd_desc_element_addr = + hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); + u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); + u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); + struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); - return ret; + lrb->utr_descriptor_ptr = utrdlp + i; + lrb->utrd_dma_addr = + hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc); + lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; + lrb->ucd_req_dma_addr = cmd_desc_element_addr; + lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; + lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; + lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; + lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; +} + +static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + u8 lun, int tag) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + ufshcd_init_lrb(hba, cmd); + + memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); + + lrbp->lun = lun; + ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? + scsi_cmd_to_rq(cmd) : NULL, lrbp); +} + +static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + u8 lun, int tag) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + __ufshcd_setup_cmd(hba, cmd, lun, tag); + lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); + lrbp->req_abort_skip = false; + + ufshcd_comp_scsi_upiu(hba, cmd); } /** * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID * @upiu_wlun_id: UPIU W-LUN id * - * Returns SCSI W-LUN id + * Return: SCSI W-LUN id. */ static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) { @@ -2709,46 +2991,38 @@ static inline bool is_device_wlun(struct scsi_device *sdev) */ static void ufshcd_map_queues(struct Scsi_Host *shost) { - int i; + struct ufs_hba *hba = shost_priv(shost); + int i, queue_offset = 0; + + if (!is_mcq_supported(hba)) { + hba->nr_queues[HCTX_TYPE_DEFAULT] = 1; + hba->nr_queues[HCTX_TYPE_READ] = 0; + hba->nr_queues[HCTX_TYPE_POLL] = 1; + hba->nr_hw_queues = 1; + } for (i = 0; i < shost->nr_maps; i++) { struct blk_mq_queue_map *map = &shost->tag_set.map[i]; - switch (i) { - case HCTX_TYPE_DEFAULT: - case HCTX_TYPE_POLL: - map->nr_queues = 1; - break; - case HCTX_TYPE_READ: - map->nr_queues = 0; + map->nr_queues = hba->nr_queues[i]; + if (!map->nr_queues) continue; - default: - WARN_ON_ONCE(true); - } - map->queue_offset = 0; + map->queue_offset = queue_offset; + if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba)) + map->queue_offset = 0; + blk_mq_map_queues(map); + queue_offset += map->nr_queues; } } -static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) +/* + * The only purpose of this function is to make the SCSI core skip the memset() + * call for the private command data. + */ +static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd) { - struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr; - struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; - dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr + - i * sizeof(struct utp_transfer_cmd_desc); - u16 response_offset = offsetof(struct utp_transfer_cmd_desc, - response_upiu); - u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table); - - lrb->utr_descriptor_ptr = utrdlp + i; - lrb->utrd_dma_addr = hba->utrdl_dma_addr + - i * sizeof(struct utp_transfer_req_desc); - lrb->ucd_req_ptr = (struct utp_upiu_req *)(cmd_descp + i); - lrb->ucd_req_dma_addr = cmd_desc_element_addr; - lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp[i].response_upiu; - lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; - lrb->ucd_prdt_ptr = cmd_descp[i].prd_table; - lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; + return 0; } /** @@ -2756,22 +3030,14 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) * @host: SCSI host pointer * @cmd: command from SCSI Midlayer * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) { struct ufs_hba *hba = shost_priv(host); int tag = scsi_cmd_to_rq(cmd)->tag; - struct ufshcd_lrb *lrbp; int err = 0; - - WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag); - - /* - * Allows the UFS error handler to wait for prior ufshcd_queuecommand() - * calls. - */ - rcu_read_lock(); + struct ufs_hw_queue *hwq = NULL; switch (hba->ufshcd_state) { case UFSHCD_STATE_OPERATIONAL: @@ -2818,42 +3084,24 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) hba->req_abort_count = 0; - err = ufshcd_hold(hba, true); - if (err) { - err = SCSI_MLQUEUE_HOST_BUSY; - goto out; - } - WARN_ON(ufshcd_is_clkgating_allowed(hba) && - (hba->clk_gating.state != CLKS_ON)); - - lrbp = &hba->lrb[tag]; - WARN_ON(lrbp->cmd); - lrbp->cmd = cmd; - lrbp->task_tag = tag; - lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); - lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); - - ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp); - - lrbp->req_abort_skip = false; + ufshcd_hold(hba); - ufshpb_prep(hba, lrbp); + ufshcd_setup_scsi_cmd(hba, cmd, + ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); - ufshcd_comp_scsi_upiu(hba, lrbp); - - err = ufshcd_map_sg(hba, lrbp); + err = ufshcd_map_sg(hba, cmd); if (err) { - lrbp->cmd = NULL; ufshcd_release(hba); goto out; } - ufshcd_send_command(hba, tag); + if (hba->mcq_enabled) + hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); -out: - rcu_read_unlock(); + ufshcd_send_command(hba, cmd, hwq); - if (ufs_trigger_eh()) { +out: + if (ufs_trigger_eh(hba)) { unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); @@ -2864,32 +3112,81 @@ out: return err; } -static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) +static int ufshcd_queue_reserved_command(struct Scsi_Host *host, + struct scsi_cmnd *cmd) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); + struct ufs_hba *hba = shost_priv(host); + struct ufs_hw_queue *hwq = + hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL; + + ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); + ufshcd_send_command(hba, cmd, hwq); + return 0; +} + +static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + enum dev_cmd_type cmd_type, u8 lun, int tag) { - lrbp->cmd = NULL; - lrbp->task_tag = tag; - lrbp->lun = 0; /* device management cmd is not specific to any LUN */ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + __ufshcd_setup_cmd(hba, cmd, lun, tag); lrbp->intr_cmd = true; /* No interrupt aggregation */ - ufshcd_prepare_lrbp_crypto(NULL, lrbp); hba->dev_cmd.type = cmd_type; +} + +/* + * Return: 0 upon success; < 0 upon failure. + */ +static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + enum dev_cmd_type cmd_type, int tag) +{ + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); - return ufshcd_compose_devman_upiu(hba, lrbp); + return ufshcd_compose_devman_upiu(hba, cmd); } /* - * Clear all the requests from the controller for which a bit has been set in - * @mask and wait until the controller confirms that these requests have been - * cleared. + * Check with the block layer if the command is inflight + * @cmd: command to check. + * + * Return: true if command is inflight; false if not. */ -static int ufshcd_clear_cmds(struct ufs_hba *hba, u32 mask) +bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) { - unsigned long flags; + return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT; +} + +/* + * Clear the pending command in the controller and wait until + * the controller confirms that the command has been cleared. + * @hba: per adapter instance + * @task_tag: The tag number of the command to be cleared. + */ +static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) +{ + u32 mask; + int err; + + if (hba->mcq_enabled) { + /* + * MCQ mode. Clean up the MCQ resources similar to + * what the ufshcd_utrl_clear() does for SDB mode. + */ + err = ufshcd_mcq_sq_cleanup(hba, task_tag); + if (err) { + dev_err(hba->dev, "%s: failed tag=%d. err=%d\n", + __func__, task_tag, err); + return err; + } + return 0; + } + + mask = 1U << task_tag; /* clear outstanding transaction before retry */ - spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_utrl_clear(hba, mask); - spin_unlock_irqrestore(hba->host->host_lock, flags); /* * wait for h/w to clear corresponding bit in door-bell. @@ -2899,26 +3196,17 @@ static int ufshcd_clear_cmds(struct ufs_hba *hba, u32 mask) mask, ~mask, 1000, 1000); } -static int -ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) -{ - struct ufs_query_res *query_res = &hba->dev_cmd.query.response; - - /* Get the UPIU response */ - query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> - UPIU_RSP_CODE_OFFSET; - return query_res->response; -} - /** * ufshcd_dev_cmd_completion() - handles device management command responses * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { - int resp; + enum upiu_response_transaction resp; int err = 0; hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); @@ -2932,17 +3220,30 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) __func__, resp); } break; - case UPIU_TRANSACTION_QUERY_RSP: - err = ufshcd_check_query_response(hba, lrbp); - if (!err) + case UPIU_TRANSACTION_QUERY_RSP: { + u8 response = lrbp->ucd_rsp_ptr->header.response; + + if (response == 0) { err = ufshcd_copy_query_response(hba, lrbp); + } else { + err = -EINVAL; + dev_err(hba->dev, "%s: unexpected response in Query RSP: %x\n", + __func__, response); + } break; + } case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ err = -EPERM; dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", __func__); break; + case UPIU_TRANSACTION_RESPONSE: + if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) { + err = -EINVAL; + dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp); + } + break; default: err = -EINVAL; dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", @@ -2950,68 +3251,58 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) break; } + WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); return err; } -static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, int max_timeout) +static void ufshcd_dev_man_lock(struct ufs_hba *hba) { - unsigned long time_left = msecs_to_jiffies(max_timeout); - unsigned long flags; - bool pending; - int err; + ufshcd_hold(hba); + mutex_lock(&hba->dev_cmd.lock); + down_read(&hba->clk_scaling_lock); +} -retry: - time_left = wait_for_completion_timeout(hba->dev_cmd.complete, - time_left); +static void ufshcd_dev_man_unlock(struct ufs_hba *hba) +{ + up_read(&hba->clk_scaling_lock); + mutex_unlock(&hba->dev_cmd.lock); + ufshcd_release(hba); +} - if (likely(time_left)) { - /* - * The completion handler called complete() and the caller of - * this function still owns the @lrbp tag so the code below does - * not trigger any race conditions. - */ - hba->dev_cmd.complete = NULL; - err = ufshcd_get_tr_ocs(lrbp); - if (!err) - err = ufshcd_dev_cmd_completion(hba, lrbp); - } else { - err = -ETIMEDOUT; - dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", - __func__, lrbp->task_tag); - if (ufshcd_clear_cmds(hba, 1U << lrbp->task_tag) == 0) { - /* successfully cleared the command, retry if needed */ - err = -EAGAIN; - /* - * Since clearing the command succeeded we also need to - * clear the task tag bit from the outstanding_reqs - * variable. - */ - spin_lock_irqsave(&hba->outstanding_lock, flags); - pending = test_bit(lrbp->task_tag, - &hba->outstanding_reqs); - if (pending) { - hba->dev_cmd.complete = NULL; - __clear_bit(lrbp->task_tag, - &hba->outstanding_reqs); - } - spin_unlock_irqrestore(&hba->outstanding_lock, flags); +static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba) +{ + /* + * The caller must hold this lock to guarantee that the NOWAIT + * allocation will succeed. + */ + lockdep_assert_held(&hba->dev_cmd.lock); - if (!pending) { - /* - * The completion handler ran while we tried to - * clear the command. - */ - time_left = 1; - goto retry; - } - } else { - dev_err(hba->dev, "%s: failed to clear tag %d\n", - __func__, lrbp->task_tag); - } - } + return scsi_get_internal_cmd( + hba->host->pseudo_sdev, DMA_TO_DEVICE, + BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); +} - return err; +static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd) +{ + scsi_put_internal_cmd(cmd); +} + +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ +static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + const u32 tag, int timeout) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); + blk_status_t sts; + + rq->timeout = timeout; + sts = blk_execute_rq(rq, true); + if (sts != BLK_STS_OK) + return blk_status_to_errno(sts); + return lrbp->utr_descriptor_ptr->header.ocs; } /** @@ -3020,39 +3311,39 @@ retry: * @cmd_type: specifies the type (NOP, Query...) * @timeout: timeout in milliseconds * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + * * NOTE: Since there is only one available tag for device management commands, * it is expected you hold the hba->dev_cmd.lock mutex. */ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, int timeout) { - DECLARE_COMPLETION_ONSTACK(wait); - const u32 tag = hba->reserved_slot; - struct ufshcd_lrb *lrbp; + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u32 tag; int err; - /* Protects use of hba->reserved_slot. */ + /* Protects use of hba->dev_cmd. */ lockdep_assert_held(&hba->dev_cmd.lock); - down_read(&hba->clk_scaling_lock); + if (WARN_ON_ONCE(!cmd)) + return -ENOMEM; + + tag = scsi_cmd_to_rq(cmd)->tag; - lrbp = &hba->lrb[tag]; - WARN_ON(lrbp->cmd); - err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); + err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); if (unlikely(err)) goto out; - hba->dev_cmd.complete = &wait; - - ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - - ufshcd_send_command(hba, tag); - err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); - ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, - (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); + if (err == 0) + err = ufshcd_dev_cmd_completion(hba, lrbp); out: - up_read(&hba->clk_scaling_lock); + ufshcd_put_dev_mgmt_cmd(cmd); + return err; } @@ -3080,6 +3371,10 @@ static inline void ufshcd_init_query(struct ufs_hba *hba, (*request)->upiu_req.selector = selector; } +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ static int ufshcd_query_flag_retry(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) { @@ -3111,7 +3406,8 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba, * @index: flag index to access * @flag_res: the flag value after the query request completes * - * Returns 0 for success, non-zero in case of failure + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) @@ -3119,12 +3415,12 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, struct ufs_query_req *request = NULL; struct ufs_query_res *response = NULL; int err, selector = 0; - int timeout = QUERY_REQ_TIMEOUT; + int timeout = dev_cmd_timeout; BUG_ON(!hba); - ufshcd_hold(hba, false); - mutex_lock(&hba->dev_cmd.lock); + ufshcd_dev_man_lock(hba); + ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); @@ -3166,8 +3462,7 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, MASK_QUERY_UPIU_FLAG_LOC) & 0x1; out_unlock: - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); return err; } @@ -3180,8 +3475,9 @@ out_unlock: * @selector: selector field * @attr_val: the attribute value after the query request completes * - * Returns 0 for success, non-zero in case of failure -*/ + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) { @@ -3197,9 +3493,8 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, return -EINVAL; } - ufshcd_hold(hba, false); + ufshcd_dev_man_lock(hba); - mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); @@ -3218,7 +3513,7 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, goto out_unlock; } - err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); if (err) { dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", @@ -3229,8 +3524,7 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, *attr_val = be32_to_cpu(response->upiu_res.value); out_unlock: - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); return err; } @@ -3245,8 +3539,9 @@ out_unlock: * @attr_val: the attribute value after the query request * completes * - * Returns 0 for success, non-zero in case of failure -*/ + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ int ufshcd_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) @@ -3271,6 +3566,10 @@ int ufshcd_query_attr_retry(struct ufs_hba *hba, return ret; } +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ static int __ufshcd_query_descriptor(struct ufs_hba *hba, enum query_opcode opcode, enum desc_idn idn, u8 index, u8 selector, u8 *desc_buf, int *buf_len) @@ -3293,9 +3592,8 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, return -EINVAL; } - ufshcd_hold(hba, false); + ufshcd_dev_man_lock(hba); - mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); hba->dev_cmd.query.descriptor = desc_buf; @@ -3316,7 +3614,7 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, goto out_unlock; } - err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); if (err) { dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", @@ -3328,8 +3626,7 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, out_unlock: hba->dev_cmd.query.descriptor = NULL; - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); return err; } @@ -3343,9 +3640,11 @@ out_unlock: * @desc_buf: the buffer that contains the descriptor * @buf_len: length parameter passed to the device * - * Returns 0 for success, non-zero in case of failure. * The buf_len parameter will contain, on return, the length parameter * received on the response. + * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode, @@ -3367,37 +3666,6 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, } /** - * ufshcd_map_desc_id_to_length - map descriptor IDN to its length - * @hba: Pointer to adapter instance - * @desc_id: descriptor idn value - * @desc_len: mapped desc length (out) - */ -void ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id, - int *desc_len) -{ - if (desc_id >= QUERY_DESC_IDN_MAX || desc_id == QUERY_DESC_IDN_RFU_0 || - desc_id == QUERY_DESC_IDN_RFU_1) - *desc_len = 0; - else - *desc_len = hba->desc_size[desc_id]; -} -EXPORT_SYMBOL(ufshcd_map_desc_id_to_length); - -static void ufshcd_update_desc_length(struct ufs_hba *hba, - enum desc_idn desc_id, int desc_index, - unsigned char desc_len) -{ - if (hba->desc_size[desc_id] == QUERY_DESC_MAX_SIZE && - desc_id != QUERY_DESC_IDN_STRING && desc_index != UFS_RPMB_UNIT) - /* For UFS 3.1, the normal unit descriptor is 10 bytes larger - * than the RPMB unit, however, both descriptors share the same - * desc_idn, to cover both unit descriptors with one length, we - * choose the normal unit descriptor length by desc_index. - */ - hba->desc_size[desc_id] = desc_len; -} - -/** * ufshcd_read_desc_param - read the specified descriptor parameter * @hba: Pointer to adapter instance * @desc_id: descriptor idn value @@ -3406,7 +3674,8 @@ static void ufshcd_update_desc_length(struct ufs_hba *hba, * @param_read_buf: pointer to buffer where parameter would be read * @param_size: sizeof(param_read_buf) * - * Return 0 in case of success, non-zero otherwise + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id, @@ -3417,26 +3686,13 @@ int ufshcd_read_desc_param(struct ufs_hba *hba, { int ret; u8 *desc_buf; - int buff_len; + int buff_len = QUERY_DESC_MAX_SIZE; bool is_kmalloc = true; /* Safety check */ if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) return -EINVAL; - /* Get the length of descriptor */ - ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len); - if (!buff_len) { - dev_err(hba->dev, "%s: Failed to get desc length\n", __func__); - return -EINVAL; - } - - if (param_offset >= buff_len) { - dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n", - __func__, param_offset, desc_id, buff_len); - return -EINVAL; - } - /* Check whether we need temp memory */ if (param_offset != 0 || param_size < buff_len) { desc_buf = kzalloc(buff_len, GFP_KERNEL); @@ -3449,15 +3705,24 @@ int ufshcd_read_desc_param(struct ufs_hba *hba, /* Request for full descriptor */ ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, - desc_id, desc_index, 0, - desc_buf, &buff_len); - + desc_id, desc_index, 0, + desc_buf, &buff_len); if (ret) { dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n", __func__, desc_id, desc_index, param_offset, ret); goto out; } + /* Update descriptor length */ + buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; + + if (param_offset >= buff_len) { + dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n", + __func__, param_offset, desc_id, buff_len); + ret = -EINVAL; + goto out; + } + /* Sanity check */ if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n", @@ -3466,10 +3731,6 @@ int ufshcd_read_desc_param(struct ufs_hba *hba, goto out; } - /* Update descriptor length */ - buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET]; - ufshcd_update_desc_length(hba, desc_id, desc_index, buff_len); - if (is_kmalloc) { /* Make sure we don't copy more data than available */ if (param_offset >= buff_len) @@ -3509,16 +3770,14 @@ static inline char ufshcd_remove_non_printable(u8 ch) * @desc_index: descriptor index * @buf: pointer to buffer where descriptor would be read, * the caller should free the memory. - * @ascii: if true convert from unicode to ascii characters - * null terminated string. + * @fmt: if %SD_ASCII_STD, convert from UTF-16 to ASCII * * Return: * * string size on success. * * -ENOMEM: on allocation failure * * -EINVAL: on a wrong parameter */ -int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, - u8 **buf, bool ascii) +int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt) { struct uc_string_id *uc_str; u8 *str; @@ -3547,7 +3806,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, goto out; } - if (ascii) { + if (fmt == SD_ASCII_STD) { ssize_t ascii_len; int i; /* remove header and divide by 2 to move from UTF16 to UTF8 */ @@ -3564,7 +3823,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, */ ret = utf16s_to_utf8s(uc_str->uc, uc_str->len - QUERY_DESC_HDR_SIZE, - UTF16_BIG_ENDIAN, str, ascii_len); + UTF16_BIG_ENDIAN, str, ascii_len - 1); /* replace non-printable or non-ASCII characters with spaces */ for (i = 0; i < ret; i++) @@ -3573,7 +3832,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, str[ret++] = '\0'; } else { - str = kmemdup(uc_str, uc_str->len, GFP_KERNEL); + str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); if (!str) { ret = -ENOMEM; goto out; @@ -3594,7 +3853,7 @@ out: * @param_read_buf: pointer to buffer where parameter would be read * @param_size: sizeof(param_read_buf) * - * Return 0 in case of success, non-zero otherwise + * Return: 0 in case of success; < 0 upon failure. */ static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, int lun, @@ -3649,14 +3908,14 @@ static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) * (UTMRDL) * 4. Allocate memory for local reference block(lrb). * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_memory_alloc(struct ufs_hba *hba) { size_t utmrdl_size, utrdl_size, ucdl_size; /* Allocate memory for UTP command descriptors */ - ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs); + ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs; hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev, ucdl_size, &hba->ucdl_dma_addr, @@ -3664,12 +3923,9 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) /* * UFSHCI requires UTP command descriptor to be 128 byte aligned. - * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE - * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will - * be aligned to 128 bytes as well */ if (!hba->ucdl_base_addr || - WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) { + WARN_ON(hba->ucdl_dma_addr & (128 - 1))) { dev_err(hba->dev, "Command Descriptor Memory allocation failed\n"); goto out; @@ -3677,7 +3933,7 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) /* * Allocate memory for UTP Transfer descriptors - * UFSHCI requires 1024 byte alignment of UTRD + * UFSHCI requires 1KB alignment of UTRD */ utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs); hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev, @@ -3685,15 +3941,23 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) &hba->utrdl_dma_addr, GFP_KERNEL); if (!hba->utrdl_base_addr || - WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) { + WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) { dev_err(hba->dev, "Transfer Descriptor Memory allocation failed\n"); goto out; } /* + * Skip utmrdl allocation; it may have been + * allocated during first pass and not released during + * MCQ memory allocation. + * See ufshcd_release_sdb_queue() and ufshcd_config_mcq() + */ + if (hba->utmrdl_base_addr) + goto skip_utmrdl; + /* * Allocate memory for UTP Task Management descriptors - * UFSHCI requires 1024 byte alignment of UTMRD + * UFSHCI requires 1KB alignment of UTMRD */ utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs; hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev, @@ -3701,20 +3965,13 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) &hba->utmrdl_dma_addr, GFP_KERNEL); if (!hba->utmrdl_base_addr || - WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) { + WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) { dev_err(hba->dev, "Task Management Descriptor Memory allocation failed\n"); goto out; } - /* Allocate memory for local reference block */ - hba->lrb = devm_kcalloc(hba->dev, - hba->nutrs, sizeof(struct ufshcd_lrb), - GFP_KERNEL); - if (!hba->lrb) { - dev_err(hba->dev, "LRB Memory allocation failed\n"); - goto out; - } +skip_utmrdl: return 0; out: return -ENOMEM; @@ -3750,17 +4007,15 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table); - cmd_desc_size = sizeof(struct utp_transfer_cmd_desc); + cmd_desc_size = ufshcd_get_ucd_size(hba); cmd_desc_dma_addr = hba->ucdl_dma_addr; for (i = 0; i < hba->nutrs; i++) { /* Configure UTRD with command descriptor base address */ cmd_desc_element_addr = (cmd_desc_dma_addr + (cmd_desc_size * i)); - utrdlp[i].command_desc_base_addr_lo = - cpu_to_le32(lower_32_bits(cmd_desc_element_addr)); - utrdlp[i].command_desc_base_addr_hi = - cpu_to_le32(upper_32_bits(cmd_desc_element_addr)); + utrdlp[i].command_desc_base_addr = + cpu_to_le64(cmd_desc_element_addr); /* Response upiu and prdt offset should be in double words */ if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) { @@ -3778,8 +4033,6 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) utrdlp[i].response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); } - - ufshcd_init_lrb(hba, &hba->lrb[i], i); } } @@ -3792,15 +4045,15 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) * Once the Unipro links are up, the device connected to the controller * is detected. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_dme_link_startup(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_LINK_STARTUP, + }; int ret; - uic_cmd.command = UIC_CMD_DME_LINK_STARTUP; - ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_dbg(hba->dev, @@ -3814,15 +4067,15 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba) * DME_RESET command is issued in order to reset UniPro stack. * This function now deals with cold reset. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ -static int ufshcd_dme_reset(struct ufs_hba *hba) +int ufshcd_dme_reset(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_RESET, + }; int ret; - uic_cmd.command = UIC_CMD_DME_RESET; - ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_err(hba->dev, @@ -3830,6 +4083,7 @@ static int ufshcd_dme_reset(struct ufs_hba *hba) return ret; } +EXPORT_SYMBOL_GPL(ufshcd_dme_reset); int ufshcd_dme_configure_adapt(struct ufs_hba *hba, int agreed_gear, @@ -3853,15 +4107,15 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); * * DME_ENABLE command is issued in order to enable UniPro stack. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ -static int ufshcd_dme_enable(struct ufs_hba *hba) +int ufshcd_dme_enable(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_ENABLE, + }; int ret; - uic_cmd.command = UIC_CMD_DME_ENABLE; - ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_err(hba->dev, @@ -3869,6 +4123,7 @@ static int ufshcd_dme_enable(struct ufs_hba *hba) return ret; } +EXPORT_SYMBOL_GPL(ufshcd_dme_enable); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) { @@ -3894,11 +4149,16 @@ static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US - delta; else - return; /* no more delay required */ + min_sleep_time_us = 0; /* no more delay required */ } - /* allow sleep for extra 50us if needed */ - usleep_range(min_sleep_time_us, min_sleep_time_us + 50); + if (min_sleep_time_us > 0) { + /* allow sleep for extra 50us if needed */ + usleep_range(min_sleep_time_us, min_sleep_time_us + 50); + } + + /* update the last_dme_cmd_tstamp */ + hba->last_dme_cmd_tstamp = ktime_get(); } /** @@ -3909,12 +4169,17 @@ static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) * @mib_val: setting value as uic command argument3 * @peer: indicate whether peer or local * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set, u32 mib_val, u8 peer) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET, + .argument1 = attr_sel, + .argument2 = UIC_ARG_ATTR_TYPE(attr_set), + .argument3 = mib_val, + }; static const char *const action[] = { "dme-set", "dme-peer-set" @@ -3923,12 +4188,6 @@ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, int ret; int retries = UFS_UIC_COMMAND_RETRIES; - uic_cmd.command = peer ? - UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET; - uic_cmd.argument1 = attr_sel; - uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set); - uic_cmd.argument3 = mib_val; - do { /* for peer attributes we retry upon failure */ ret = ufshcd_send_uic_cmd(hba, &uic_cmd); @@ -3953,12 +4212,15 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); * @mib_val: the value of the attribute as returned by the UIC command * @peer: indicate whether peer or local * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, u32 *mib_val, u8 peer) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET, + .argument1 = attr_sel, + }; static const char *const action[] = { "dme-get", "dme-peer-get" @@ -3992,10 +4254,6 @@ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, } } - uic_cmd.command = peer ? - UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET; - uic_cmd.argument1 = attr_sel; - do { /* for peer attributes we retry upon failure */ ret = ufshcd_send_uic_cmd(hba, &uic_cmd); @@ -4009,8 +4267,8 @@ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, get, UIC_GET_ATTR_ID(attr_sel), UFS_UIC_COMMAND_RETRIES - retries); - if (mib_val && !ret) - *mib_val = uic_cmd.argument3; + if (mib_val) + *mib_val = ret == 0 ? uic_cmd.argument3 : 0; if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) && pwr_mode_change) @@ -4021,6 +4279,30 @@ out: EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); /** + * ufshcd_dme_rmw - get modify set a DME attribute + * @hba: per adapter instance + * @mask: indicates which bits to clear from the value that has been read + * @val: actual value to write + * @attr: dme attribute + */ +int ufshcd_dme_rmw(struct ufs_hba *hba, u32 mask, + u32 val, u32 attr) +{ + u32 cfg = 0; + int err; + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(attr), &cfg); + if (err) + return err; + + cfg &= ~mask; + cfg |= (val & mask); + + return ufshcd_dme_set(hba, UIC_ARG_MIB(attr), cfg); +} +EXPORT_SYMBOL_GPL(ufshcd_dme_rmw); + +/** * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power * state) and waits for it to take effect. * @@ -4034,7 +4316,7 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); * addition to normal UIC command completion Status (UCCS). This function only * returns after the relevant status bits indicate the completion. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) { @@ -4042,7 +4324,6 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) unsigned long flags; u8 status; int ret; - bool reenable_intr = false; mutex_lock(&hba->uic_cmd_mutex); ufshcd_add_delay_before_dme_cmd(hba); @@ -4053,17 +4334,9 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) goto out_unlock; } hba->uic_async_done = &uic_async_done; - if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) { - ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); - /* - * Make sure UIC command completion interrupt is disabled before - * issuing UIC command. - */ - wmb(); - reenable_intr = true; - } - ret = __ufshcd_send_uic_cmd(hba, cmd, false); + ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); spin_unlock_irqrestore(hba->host->host_lock, flags); + ret = __ufshcd_send_uic_cmd(hba, cmd); if (ret) { dev_err(hba->dev, "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", @@ -4072,7 +4345,7 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) } if (!wait_for_completion_timeout(hba->uic_async_done, - msecs_to_jiffies(UIC_CMD_TIMEOUT))) { + msecs_to_jiffies(uic_cmd_timeout))) { dev_err(hba->dev, "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", cmd->command, cmd->argument3); @@ -4105,9 +4378,7 @@ out: spin_lock_irqsave(hba->host->host_lock, flags); hba->active_uic_cmd = NULL; hba->uic_async_done = NULL; - if (reenable_intr) - ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); - if (ret) { + if (ret && !hba->pm_op_in_progress) { ufshcd_set_link_broken(hba); ufshcd_schedule_eh_work(hba); } @@ -4115,6 +4386,39 @@ out_unlock: spin_unlock_irqrestore(hba->host->host_lock, flags); mutex_unlock(&hba->uic_cmd_mutex); + /* + * If the h8 exit fails during the runtime resume process, it becomes + * stuck and cannot be recovered through the error handler. To fix + * this, use link recovery instead of the error handler. + */ + if (ret && hba->pm_op_in_progress) + ret = ufshcd_link_recovery(hba); + + return ret; +} + +/** + * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result + * @hba: per adapter instance + * @uic_cmd: UIC command + * + * Return: 0 only if success. + */ +int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) +{ + int ret; + + if (uic_cmd->argument1 != UIC_ARG_MIB(PA_PWRMODE) || + uic_cmd->command != UIC_CMD_DME_SET) + return ufshcd_send_uic_cmd(hba, uic_cmd); + + if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) + return 0; + + ufshcd_hold(hba); + ret = ufshcd_uic_pwr_ctrl(hba, uic_cmd); + ufshcd_release(hba); + return ret; } @@ -4124,11 +4428,15 @@ out_unlock: * @hba: per adapter instance * @mode: powr mode value * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_SET, + .argument1 = UIC_ARG_MIB(PA_PWRMODE), + .argument3 = mode, + }; int ret; if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) { @@ -4141,10 +4449,7 @@ int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) } } - uic_cmd.command = UIC_CMD_DME_SET; - uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE); - uic_cmd.argument3 = mode; - ufshcd_hold(hba, false); + ufshcd_hold(hba); ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); ufshcd_release(hba); @@ -4184,15 +4489,16 @@ EXPORT_SYMBOL_GPL(ufshcd_link_recovery); int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) { - int ret; - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_HIBER_ENTER, + }; ktime_t start = ktime_get(); + int ret; ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); - uic_cmd.command = UIC_CMD_DME_HIBER_ENTER; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); - trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter", + trace_ufshcd_profile_hibern8(hba, "enter", ktime_to_us(ktime_sub(ktime_get(), start)), ret); if (ret) @@ -4208,15 +4514,16 @@ EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter); int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_HIBER_EXIT, + }; int ret; ktime_t start = ktime_get(); ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); - uic_cmd.command = UIC_CMD_DME_HIBER_EXIT; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); - trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit", + trace_ufshcd_profile_hibern8(hba, "exit", ktime_to_us(ktime_sub(ktime_get(), start)), ret); if (ret) { @@ -4233,40 +4540,32 @@ int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) } EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit); -void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) +static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba) { - unsigned long flags; - bool update = false; - if (!ufshcd_is_auto_hibern8_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->ahit != ahit) { - hba->ahit = ahit; - update = true; - } - spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); +} - if (update && - !pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { +void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) +{ + const u32 cur_ahit = READ_ONCE(hba->ahit); + + if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit) + return; + + WRITE_ONCE(hba->ahit, ahit); + if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { ufshcd_rpm_get_sync(hba); - ufshcd_hold(hba, false); - ufshcd_auto_hibern8_enable(hba); + ufshcd_hold(hba); + ufshcd_configure_auto_hibern8(hba); ufshcd_release(hba); ufshcd_rpm_put_sync(hba); } } EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update); -void ufshcd_auto_hibern8_enable(struct ufs_hba *hba) -{ - if (!ufshcd_is_auto_hibern8_supported(hba)) - return; - - ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); -} - /** * ufshcd_init_pwr_info - setting the POR (power on reset) * values in hba power info @@ -4276,8 +4575,8 @@ static void ufshcd_init_pwr_info(struct ufs_hba *hba) { hba->pwr_info.gear_rx = UFS_PWM_G1; hba->pwr_info.gear_tx = UFS_PWM_G1; - hba->pwr_info.lane_rx = 1; - hba->pwr_info.lane_tx = 1; + hba->pwr_info.lane_rx = UFS_LANE_1; + hba->pwr_info.lane_tx = UFS_LANE_1; hba->pwr_info.pwr_rx = SLOWAUTO_MODE; hba->pwr_info.pwr_tx = SLOWAUTO_MODE; hba->pwr_info.hs_rate = 0; @@ -4286,6 +4585,8 @@ static void ufshcd_init_pwr_info(struct ufs_hba *hba) /** * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device * @hba: per-adapter instance + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) { @@ -4317,6 +4618,14 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) return -EINVAL; } + if (pwr_info->lane_rx != pwr_info->lane_tx) { + dev_err(hba->dev, "%s: asymmetric connected lanes. rx=%d, tx=%d\n", + __func__, + pwr_info->lane_rx, + pwr_info->lane_tx); + return -EINVAL; + } + /* * First, get the maximum gears of HS speed. * If a zero value, it means there is no HSGEAR capability. @@ -4429,9 +4738,6 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba, dev_err(hba->dev, "%s: power mode change failed %d\n", __func__, ret); } else { - ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, - pwr_mode); - memcpy(&hba->pwr_info, pwr_mode, sizeof(struct ufs_pa_layer_attr)); } @@ -4443,6 +4749,8 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba, * ufshcd_config_pwr_mode - configure a new power mode * @hba: per-adapter instance * @desired_pwr_mode: desired power configuration + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_config_pwr_mode(struct ufs_hba *hba, struct ufs_pa_layer_attr *desired_pwr_mode) @@ -4458,6 +4766,10 @@ int ufshcd_config_pwr_mode(struct ufs_hba *hba, ret = ufshcd_change_power_mode(hba, &final_params); + if (!ret) + ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, + &final_params); + return ret; } EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); @@ -4467,6 +4779,9 @@ EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); * @hba: per-adapter instance * * Set fDeviceInit flag and poll until device toggles it. + * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_complete_dev_init(struct ufs_hba *hba) { @@ -4517,7 +4832,7 @@ out: * 3. Program UTRL and UTMRL base address * 4. Configure run-stop-registers * - * Returns 0 on success, non-zero value on failure + * Return: 0 if successful; < 0 upon failure. */ int ufshcd_make_hba_operational(struct ufs_hba *hba) { @@ -4544,12 +4859,6 @@ int ufshcd_make_hba_operational(struct ufs_hba *hba) REG_UTP_TASK_REQ_LIST_BASE_H); /* - * Make sure base address and interrupt setup are updated before - * enabling the run/stop registers below. - */ - wmb(); - - /* * UCRDY, UTMRLDY and UTRLRDY bits must be 1 */ reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); @@ -4571,20 +4880,14 @@ EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); */ void ufshcd_hba_stop(struct ufs_hba *hba) { - unsigned long flags; int err; - /* - * Obtain the host lock to prevent that the controller is disabled - * while the UFS interrupt handler is active on another CPU. - */ - spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_disable_irq(hba); ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); - spin_unlock_irqrestore(hba->host->host_lock, flags); - err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, CONTROLLER_DISABLE, 10, 1); + ufshcd_enable_irq(hba); if (err) dev_err(hba->dev, "%s: Controller disable failed\n", __func__); } @@ -4598,55 +4901,48 @@ EXPORT_SYMBOL_GPL(ufshcd_hba_stop); * sequence kicks off. When controller is ready it will set * the Host Controller Enable bit to 1. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_hba_execute_hce(struct ufs_hba *hba) { - int retry_outer = 3; - int retry_inner; + int retry; -start: - if (ufshcd_is_hba_active(hba)) - /* change controller state to "reset state" */ - ufshcd_hba_stop(hba); + for (retry = 3; retry > 0; retry--) { + if (ufshcd_is_hba_active(hba)) + /* change controller state to "reset state" */ + ufshcd_hba_stop(hba); - /* UniPro link is disabled at this point */ - ufshcd_set_link_off(hba); + /* UniPro link is disabled at this point */ + ufshcd_set_link_off(hba); - ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); + ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); - /* start controller initialization sequence */ - ufshcd_hba_start(hba); + /* start controller initialization sequence */ + ufshcd_hba_start(hba); - /* - * To initialize a UFS host controller HCE bit must be set to 1. - * During initialization the HCE bit value changes from 1->0->1. - * When the host controller completes initialization sequence - * it sets the value of HCE bit to 1. The same HCE bit is read back - * to check if the controller has completed initialization sequence. - * So without this delay the value HCE = 1, set in the previous - * instruction might be read back. - * This delay can be changed based on the controller. - */ - ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); + /* + * To initialize a UFS host controller HCE bit must be set to 1. + * During initialization the HCE bit value changes from 1->0->1. + * When the host controller completes initialization sequence + * it sets the value of HCE bit to 1. The same HCE bit is read back + * to check if the controller has completed initialization sequence. + * So without this delay the value HCE = 1, set in the previous + * instruction might be read back. + * This delay can be changed based on the controller. + */ + ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); - /* wait for the host controller to complete initialization */ - retry_inner = 50; - while (!ufshcd_is_hba_active(hba)) { - if (retry_inner) { - retry_inner--; - } else { - dev_err(hba->dev, - "Controller enable failed\n"); - if (retry_outer) { - retry_outer--; - goto start; - } - return -EIO; - } - usleep_range(1000, 1100); + /* wait for the host controller to complete initialization */ + if (!ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, + CONTROLLER_ENABLE, 1000, 50)) + break; + + dev_err(hba->dev, "Enabling the controller failed\n"); } + if (!retry) + return -EIO; + /* enable UIC related interrupts */ ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); @@ -4688,7 +4984,7 @@ EXPORT_SYMBOL_GPL(ufshcd_hba_enable); static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) { - int tx_lanes = 0, i, err = 0; + int tx_lanes, i, err = 0; if (!peer) ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), @@ -4743,7 +5039,7 @@ EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); * ufshcd_link_startup - Initialize unipro link startup * @hba: per adapter instance * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_link_startup(struct ufs_hba *hba) { @@ -4755,7 +5051,8 @@ static int ufshcd_link_startup(struct ufs_hba *hba) * If UFS device isn't active then we will have to issue link startup * 2 times to make sure the device state move to active. */ - if (!ufshcd_is_ufs_dev_active(hba)) + if (!(hba->quirks & UFSHCD_QUIRK_PERFORM_LINK_STARTUP_ONCE) && + !ufshcd_is_ufs_dev_active(hba)) link_startup_again = true; link_startup: @@ -4820,12 +5117,8 @@ link_startup: ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); ret = ufshcd_make_hba_operational(hba); out: - if (ret) { + if (ret) dev_err(hba->dev, "link startup failed %d\n", ret); - ufshcd_print_host_state(hba); - ufshcd_print_pwr_info(hba); - ufshcd_print_evt_hist(hba); - } return ret; } @@ -4838,14 +5131,17 @@ out: * If the UTP layer at the device side is not initialized, it may * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. + * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_verify_dev_init(struct ufs_hba *hba) { int err = 0; int retries; - ufshcd_hold(hba, false); - mutex_lock(&hba->dev_cmd.lock); + ufshcd_dev_man_lock(hba); + for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, hba->nop_out_timeout); @@ -4855,8 +5151,8 @@ static int ufshcd_verify_dev_init(struct ufs_hba *hba) dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); } - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + + ufshcd_dev_man_unlock(hba); if (err) dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); @@ -4907,7 +5203,7 @@ static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev) */ static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) { - int len = hba->desc_size[QUERY_DESC_IDN_UNIT]; + int len = QUERY_DESC_MAX_SIZE; u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun); u8 lun_qdepth = hba->nutrs; u8 *desc_buf; @@ -4942,6 +5238,17 @@ static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP) hba->dev_info.is_lu_power_on_wp = true; + /* In case of RPMB LU, check if advanced RPMB mode is enabled, and get region size */ + if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN) { + if (desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) + hba->dev_info.b_advanced_rpmb_en = true; + hba->dev_info.rpmb_region_size[0] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION0_SIZE]; + hba->dev_info.rpmb_region_size[1] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION1_SIZE]; + hba->dev_info.rpmb_region_size[2] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION2_SIZE]; + hba->dev_info.rpmb_region_size[3] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION3_SIZE]; + } + + kfree(desc_buf); set_qdepth: /* @@ -4953,12 +5260,12 @@ set_qdepth: } /** - * ufshcd_slave_alloc - handle initial SCSI device configurations + * ufshcd_sdev_init - handle initial SCSI device configurations * @sdev: pointer to SCSI device * - * Returns success + * Return: success. */ -static int ufshcd_slave_alloc(struct scsi_device *sdev) +static int ufshcd_sdev_init(struct scsi_device *sdev) { struct ufs_hba *hba; @@ -4992,46 +5299,29 @@ static int ufshcd_slave_alloc(struct scsi_device *sdev) * @depth: required depth to set * * Change queue depth and make sure the max. limits are not crossed. + * + * Return: new queue depth. */ static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) { return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); } -static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev) -{ - /* skip well-known LU */ - if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || - !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) - return; - - ufshpb_destroy_lu(hba, sdev); -} - -static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev) -{ - /* skip well-known LU */ - if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || - !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) - return; - - ufshpb_init_hpb_lu(hba, sdev); -} - /** - * ufshcd_slave_configure - adjust SCSI device configurations + * ufshcd_sdev_configure - adjust SCSI device configurations * @sdev: pointer to SCSI device + * @lim: queue limits + * + * Return: 0 (success). */ -static int ufshcd_slave_configure(struct scsi_device *sdev) +static int ufshcd_sdev_configure(struct scsi_device *sdev, + struct queue_limits *lim) { struct ufs_hba *hba = shost_priv(sdev->host); struct request_queue *q = sdev->request_queue; - ufshcd_hpb_configure(hba, sdev); + lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1; - blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); - if (hba->quirks & UFSHCD_QUIRK_ALIGN_SG_WITH_PAGE_SIZE) - blk_queue_update_dma_alignment(q, PAGE_SIZE - 1); /* * Block runtime-pm until all consumers are added. * Refer ufshcd_setup_links(). @@ -5047,24 +5337,25 @@ static int ufshcd_slave_configure(struct scsi_device *sdev) */ sdev->silence_suspend = 1; + if (hba->vops && hba->vops->config_scsi_dev) + hba->vops->config_scsi_dev(sdev); + ufshcd_crypto_register(hba, q); return 0; } /** - * ufshcd_slave_destroy - remove SCSI device configurations + * ufshcd_sdev_destroy - remove SCSI device configurations * @sdev: pointer to SCSI device */ -static void ufshcd_slave_destroy(struct scsi_device *sdev) +static void ufshcd_sdev_destroy(struct scsi_device *sdev) { struct ufs_hba *hba; unsigned long flags; hba = shost_priv(sdev->host); - ufshcd_hpb_destroy(hba, sdev); - /* Drop the reference as it won't be needed anymore */ if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { spin_lock_irqsave(hba->host->host_lock, flags); @@ -5095,19 +5386,18 @@ static void ufshcd_slave_destroy(struct scsi_device *sdev) /** * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status - * @lrbp: pointer to local reference block of completed command + * @cmd: SCSI command * @scsi_status: SCSI command status * - * Returns value base on SCSI command status + * Return: value base on SCSI command status. */ -static inline int -ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) +static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) { int result = 0; switch (scsi_status) { case SAM_STAT_CHECK_CONDITION: - ufshcd_copy_sense_data(lrbp); + ufshcd_copy_sense_data(cmd); fallthrough; case SAM_STAT_GOOD: result |= DID_OK << 16 | scsi_status; @@ -5115,7 +5405,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) case SAM_STAT_TASK_SET_FULL: case SAM_STAT_BUSY: case SAM_STAT_TASK_ABORTED: - ufshcd_copy_sense_data(lrbp); + ufshcd_copy_sense_data(cmd); result |= scsi_status; break; default: @@ -5129,44 +5419,52 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) /** * ufshcd_transfer_rsp_status - Get overall status of the response * @hba: per adapter instance - * @lrbp: pointer to local reference block of completed command + * @cmd: SCSI command + * @cqe: pointer to the completion queue entry * - * Returns result of the command to notify SCSI midlayer + * Return: result of the command to notify SCSI midlayer. */ -static inline int -ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, + struct scsi_cmnd *cmd, + struct cq_entry *cqe) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; int result = 0; int scsi_status; enum utp_ocs ocs; + u8 upiu_flags; + u32 resid; + + upiu_flags = lrbp->ucd_rsp_ptr->header.flags; + resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count); + /* + * Test !overflow instead of underflow to support UFS devices that do + * not set either flag. + */ + if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) + scsi_set_resid(cmd, resid); /* overall command status of utrd */ - ocs = ufshcd_get_tr_ocs(lrbp); + ocs = ufshcd_get_tr_ocs(lrbp, cqe); if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { - if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) & - MASK_RSP_UPIU_RESULT) + if (lrbp->ucd_rsp_ptr->header.response || + lrbp->ucd_rsp_ptr->header.status) ocs = OCS_SUCCESS; } switch (ocs) { case OCS_SUCCESS: - result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); - switch (result) { + switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) { case UPIU_TRANSACTION_RESPONSE: /* - * get the response UPIU result to extract - * the SCSI command status - */ - result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); - - /* * get the result based on SCSI status response * to notify the SCSI midlayer of the command status */ - scsi_status = result & MASK_SCSI_STATUS; - result = ufshcd_scsi_cmd_status(lrbp, scsi_status); + scsi_status = lrbp->ucd_rsp_ptr->header.status; + result = ufshcd_scsi_cmd_status(cmd, scsi_status); /* * Currently we are only supporting BKOPs exception @@ -5185,9 +5483,6 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) /* Flushed in suspend */ schedule_work(&hba->eeh_work); - - if (scsi_status == SAM_STAT_GOOD) - ufshpb_rsp_upiu(hba, lrbp); break; case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ @@ -5204,10 +5499,10 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) } break; case OCS_ABORTED: - result |= DID_ABORT << 16; - break; case OCS_INVALID_COMMAND_STATUS: result |= DID_REQUEUE << 16; + dev_warn(hba->dev, "OCS %s from controller for tag %d\n", + ocs == OCS_ABORTED ? "aborted" : "invalid", tag); break; case OCS_INVALID_CMD_TABLE_ATTR: case OCS_INVALID_PRDT_ATTR: @@ -5220,17 +5515,19 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) case OCS_GENERAL_CRYPTO_ERROR: default: result |= DID_ERROR << 16; - dev_err(hba->dev, - "OCS error from controller = %x for tag %d\n", - ocs, lrbp->task_tag); + dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", + ocs, tag); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); break; } /* end of switch */ if ((host_byte(result) != DID_OK) && - (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) - ufshcd_print_trs(hba, 1 << lrbp->task_tag, true); + (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { + if (cqe) + ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); + ufshcd_print_tr(hba, cmd, true); + } return result; } @@ -5257,89 +5554,109 @@ static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, * @hba: per adapter instance * @intr_status: interrupt status generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) { irqreturn_t retval = IRQ_NONE; + struct uic_command *cmd; + + guard(spinlock_irqsave)(hba->host->host_lock); + cmd = hba->active_uic_cmd; + if (!cmd) + return retval; - spin_lock(hba->host->host_lock); if (ufshcd_is_auto_hibern8_error(hba, intr_status)) hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status); - if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) { - hba->active_uic_cmd->argument2 |= - ufshcd_get_uic_cmd_result(hba); - hba->active_uic_cmd->argument3 = - ufshcd_get_dme_attr_val(hba); + if (intr_status & UIC_COMMAND_COMPL) { + cmd->argument2 |= ufshcd_get_uic_cmd_result(hba); + cmd->argument3 = ufshcd_get_dme_attr_val(hba); if (!hba->uic_async_done) - hba->active_uic_cmd->cmd_active = 0; - complete(&hba->active_uic_cmd->done); + cmd->cmd_active = false; + complete(&cmd->done); retval = IRQ_HANDLED; } - if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) { - hba->active_uic_cmd->cmd_active = 0; + if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) { + cmd->cmd_active = false; complete(hba->uic_async_done); retval = IRQ_HANDLED; } if (retval == IRQ_HANDLED) - ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd, - UFS_CMD_COMP); - spin_unlock(hba->host->host_lock); + ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP); + return retval; } /* Release the resources allocated for processing a SCSI command. */ -static void ufshcd_release_scsi_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) +void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - struct scsi_cmnd *cmd = lrbp->cmd; - scsi_dma_unmap(cmd); - lrbp->cmd = NULL; /* Mark the command as completed. */ + ufshcd_crypto_clear_prdt(hba, cmd); ufshcd_release(hba); ufshcd_clk_scaling_update_busy(hba); } /** - * __ufshcd_transfer_req_compl - handle SCSI and query command completion + * ufshcd_compl_one_cqe - handle a completion queue entry * @hba: per adapter instance - * @completed_reqs: bitmask that indicates which requests to complete + * @task_tag: the task tag of the request to be completed + * @cqe: pointer to the completion queue entry */ -static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, - unsigned long completed_reqs) +void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, + struct cq_entry *cqe) { - struct ufshcd_lrb *lrbp; - struct scsi_cmnd *cmd; - int index; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + enum utp_ocs ocs; + + if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", + le64_to_cpu(cqe->command_desc_base_addr))) + return; - for_each_set_bit(index, &completed_reqs, hba->nutrs) { - lrbp = &hba->lrb[index]; + if (hba->monitor.enabled) { lrbp->compl_time_stamp = ktime_get(); lrbp->compl_time_stamp_local_clock = local_clock(); - cmd = lrbp->cmd; - if (cmd) { - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) - ufshcd_update_monitor(hba, lrbp); - ufshcd_add_command_trace(hba, index, UFS_CMD_COMP); - cmd->result = ufshcd_transfer_rsp_status(hba, lrbp); - ufshcd_release_scsi_cmd(hba, lrbp); - /* Do not touch lrbp after scsi done */ - scsi_done(cmd); - } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE || - lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) { - if (hba->dev_cmd.complete) { - ufshcd_add_command_trace(hba, index, - UFS_DEV_COMP); - complete(hba->dev_cmd.complete); - ufshcd_clk_scaling_update_busy(hba); - } + } + if (ufshcd_is_scsi_cmd(cmd)) { + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) + ufshcd_update_monitor(hba, cmd); + ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); + cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); + ufshcd_release_scsi_cmd(hba, cmd); + } else { + if (cqe) { + ocs = cqe->overall_status & MASK_OCS; + lrbp->utr_descriptor_ptr->header.ocs = ocs; + } else { + ocs = lrbp->utr_descriptor_ptr->header.ocs; } + ufshcd_add_query_upiu_trace( + hba, + ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR, + (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); + cmd->result = 0; } + /* Do not touch lrbp after scsi_done() has been called. */ + scsi_done(cmd); +} + +/** + * __ufshcd_transfer_req_compl - handle SCSI and query command completion + * @hba: per adapter instance + * @completed_reqs: bitmask that indicates which requests to complete + */ +static void __ufshcd_transfer_req_compl(struct ufs_hba *hba, + unsigned long completed_reqs) +{ + int tag; + + for_each_set_bit(tag, &completed_reqs, hba->nutrs) + ufshcd_compl_one_cqe(hba, tag, NULL); } /* Any value that is not an existing queue number is fine for this constant. */ @@ -5353,7 +5670,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba, int tag; for_each_set_bit(tag, completed_reqs, hba->nutrs) { - struct scsi_cmnd *cmd = hba->lrb[tag].cmd; + struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); if (!cmd) continue; @@ -5363,7 +5680,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba, } /* - * Returns > 0 if one or more commands have been completed or 0 if no + * Return: > 0 if one or more commands have been completed or 0 if no * requests have been completed. */ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) @@ -5371,6 +5688,13 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) struct ufs_hba *hba = shost_priv(shost); unsigned long completed_reqs, flags; u32 tr_doorbell; + struct ufs_hw_queue *hwq; + + if (hba->mcq_enabled) { + hwq = &hba->uhq[queue_num]; + + return ufshcd_mcq_poll_cqe_lock(hba, hwq); + } spin_lock_irqsave(&hba->outstanding_lock, flags); tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); @@ -5391,11 +5715,72 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) return completed_reqs != 0; } +static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) +{ + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); + struct scsi_device *sdev = rq->q->queuedata; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); + + if (blk_mq_is_reserved_rq(rq) || !hwq) + return true; + + ufshcd_mcq_compl_all_cqes_lock(hba, hwq); + + /* + * For those cmds of which the cqes are not present in the cq, complete + * them explicitly. + */ + scoped_guard(spinlock_irqsave, &hwq->cq_lock) { + if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { + set_host_byte(cmd, DID_REQUEUE); + ufshcd_release_scsi_cmd(hba, cmd); + scsi_done(cmd); + } + } + + return true; +} + +static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) +{ + struct scsi_device *sdev = rq->q->queuedata; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); + + if (!blk_mq_is_reserved_rq(rq) && hwq) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + + return true; +} + +/** + * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is + * invoked from the error handler context or ufshcd_host_reset_and_restore() + * to complete the pending transfers and free the resources associated with + * the scsi command. + * + * @hba: per adapter instance + * @force_compl: This flag is set to true when invoked + * from ufshcd_host_reset_and_restore() in which case it requires special + * handling because the host controller has been reset by ufshcd_hba_stop(). + */ +static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, + bool force_compl) +{ + blk_mq_tagset_busy_iter(&hba->host->tag_set, + force_compl ? ufshcd_mcq_force_compl_one : + ufshcd_mcq_compl_one, + NULL); +} + /** * ufshcd_transfer_req_compl - handle SCSI and query command completion * @hba: per adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -5412,7 +5797,7 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) ufshcd_reset_intr_aggr(hba); - if (ufs_fail_completion()) + if (ufs_fail_completion(hba)) return IRQ_HANDLED; /* @@ -5472,7 +5857,7 @@ int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, * Disables exception event in the device so that the EVENT_ALERT * bit is not set. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) { @@ -5487,7 +5872,7 @@ static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) * Enable corresponding exception event in the device to allow * device to alert host in critical scenarios. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) { @@ -5503,7 +5888,8 @@ static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) * as the device is allowed to manage its own way of handling background * operations. * - * Returns zero on success, non-zero on failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) { @@ -5521,7 +5907,7 @@ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = true; - trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled"); + trace_ufshcd_auto_bkops_state(hba, "Enabled"); /* No need of URGENT_BKOPS exception from the device */ err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); @@ -5542,7 +5928,8 @@ out: * host is idle so that BKOPS are managed effectively without any negative * impacts. * - * Returns zero on success, non-zero on failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) { @@ -5572,7 +5959,7 @@ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = false; - trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled"); + trace_ufshcd_auto_bkops_state(hba, "Disabled"); hba->is_urgent_bkops_lvl_checked = false; out: return err; @@ -5611,24 +5998,23 @@ static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) /** * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status * @hba: per-adapter instance - * @status: bkops_status value * * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn * flag in the device to permit background operations if the device - * bkops_status is greater than or equal to "status" argument passed to - * this function, disable otherwise. + * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl", + * disable otherwise. * - * Returns 0 for success, non-zero in case of failure. + * Return: 0 for success, non-zero in case of failure. * * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag * to know whether auto bkops is enabled or disabled after this function * returns control to it. */ -static int ufshcd_bkops_ctrl(struct ufs_hba *hba, - enum bkops_status status) +static int ufshcd_bkops_ctrl(struct ufs_hba *hba) { - int err; + enum bkops_status status = hba->urgent_bkops_lvl; u32 curr_status = 0; + int err; err = ufshcd_get_bkops_status(hba, &curr_status); if (err) { @@ -5650,21 +6036,6 @@ out: return err; } -/** - * ufshcd_urgent_bkops - handle urgent bkops exception event - * @hba: per-adapter instance - * - * Enable fBackgroundOpsEn flag in the device to permit background - * operations. - * - * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled - * and negative error value for any other failure. - */ -static int ufshcd_urgent_bkops(struct ufs_hba *hba) -{ - return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl); -} - static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) { return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, @@ -5708,22 +6079,44 @@ out: __func__, err); } -static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status) +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ +int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) { - u32 value; + struct utp_upiu_query_v4_0 *upiu_resp; + struct ufs_query_req *request = NULL; + struct ufs_query_res *response = NULL; + int err; - if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, - QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value)) - return; + if (hba->dev_info.wspecversion < 0x410) + return -EOPNOTSUPP; - dev_info(hba->dev, "exception Tcase %d\n", value - 80); + ufshcd_hold(hba); + mutex_lock(&hba->dev_cmd.lock); - ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); + ufshcd_init_query(hba, &request, &response, + UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0); - /* - * A placeholder for the platform vendors to add whatever additional - * steps required - */ + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); + + if (err) { + dev_err(hba->dev, "%s: failed to read device level exception %d\n", + __func__, err); + goto out; + } + + upiu_resp = (struct utp_upiu_query_v4_0 *)response; + *exception_id = get_unaligned_be64(&upiu_resp->osf3); +out: + mutex_unlock(&hba->dev_cmd.lock); + ufshcd_release(hba); + + return err; } static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) @@ -5796,7 +6189,22 @@ int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) return ret; } -static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba, +int ufshcd_wb_set_resize_en(struct ufs_hba *hba, enum wb_resize_en en_mode) +{ + int ret; + u8 index; + + index = ufshcd_wb_get_query_index(hba); + ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_WB_BUF_RESIZE_EN, index, 0, &en_mode); + if (ret) + dev_err(hba->dev, "%s: Enable WB buf resize operation failed %d\n", + __func__, ret); + + return ret; +} + +static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba, u32 avail_buf) { u32 cur_buf; @@ -5878,15 +6286,13 @@ static bool ufshcd_wb_need_flush(struct ufs_hba *hba) } /* - * The ufs device needs the vcc to be ON to flush. * With user-space reduction enabled, it's enough to enable flush * by checking only the available buffer. The threshold * defined here is > 90% full. * With user-space preserved enabled, the current-buffer * should be checked too because the wb buffer size can reduce * when disk tends to be full. This info is provided by current - * buffer (dCurrentWriteBoosterBufferSize). There's no point in - * keeping vcc on when current buffer is empty. + * buffer (dCurrentWriteBoosterBufferSize). */ index = ufshcd_wb_get_query_index(hba); ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, @@ -5901,7 +6307,7 @@ static bool ufshcd_wb_need_flush(struct ufs_hba *hba) if (!hba->dev_info.b_presrv_uspc_en) return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); - return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf); + return ufshcd_wb_curr_buff_threshold_check(hba, avail_buf); } static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) @@ -5933,31 +6339,42 @@ static void ufshcd_exception_event_handler(struct work_struct *work) u32 status = 0; hba = container_of(work, struct ufs_hba, eeh_work); - ufshcd_scsi_block_requests(hba); err = ufshcd_get_ee_status(hba, &status); if (err) { dev_err(hba->dev, "%s: failed to get exception status %d\n", __func__, err); - goto out; + return; } - trace_ufshcd_exception_event(dev_name(hba->dev), status); + trace_ufshcd_exception_event(hba, status); if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) ufshcd_bkops_exception_event_handler(hba); if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) - ufshcd_temp_exception_event_handler(hba, status); + ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); + + if (status & hba->ee_drv_mask & MASK_EE_HEALTH_CRITICAL) { + hba->critical_health_count++; + sysfs_notify(&hba->dev->kobj, NULL, "critical_health"); + } + + if (status & hba->ee_drv_mask & MASK_EE_DEV_LVL_EXCEPTION) { + atomic_inc(&hba->dev_lvl_exception_count); + sysfs_notify(&hba->dev->kobj, NULL, "device_lvl_exception_count"); + } ufs_debugfs_exception_event(hba, status); -out: - ufshcd_scsi_unblock_requests(hba); } /* Complete requests that have door-bell cleared */ -static void ufshcd_complete_requests(struct ufs_hba *hba) +static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) { - ufshcd_transfer_req_compl(hba); + if (hba->mcq_enabled) + ufshcd_mcq_compl_pending_transfer(hba, force_compl); + else + ufshcd_transfer_req_compl(hba); + ufshcd_tmc_handler(hba); } @@ -5966,7 +6383,7 @@ static void ufshcd_complete_requests(struct ufs_hba *hba) * to recover from the DL NAC errors or not. * @hba: per-adapter instance * - * Returns true if error handling is required, false otherwise + * Return: true if error handling is required, false otherwise. */ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) { @@ -6054,13 +6471,14 @@ void ufshcd_schedule_eh_work(struct ufs_hba *hba) } } -static void ufshcd_force_error_recovery(struct ufs_hba *hba) +void ufshcd_force_error_recovery(struct ufs_hba *hba) { spin_lock_irq(hba->host->host_lock); hba->force_reset = true; ufshcd_schedule_eh_work(hba); spin_unlock_irq(hba->host->host_lock); } +EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery); static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) { @@ -6101,28 +6519,26 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) ufshcd_setup_vreg(hba, true); ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq); ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2); - ufshcd_hold(hba, false); + ufshcd_hold(hba); if (!ufshcd_is_clkgating_allowed(hba)) ufshcd_setup_clocks(hba, true); - ufshcd_release(hba); pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; ufshcd_vops_resume(hba, pm_op); } else { - ufshcd_hold(hba, false); + ufshcd_hold(hba); if (ufshcd_is_clkscaling_supported(hba) && hba->clk_scaling.is_enabled) ufshcd_suspend_clkscaling(hba); ufshcd_clk_scaling_allow(hba, false); } - ufshcd_scsi_block_requests(hba); - /* Drain ufshcd_queuecommand() */ - synchronize_rcu(); + /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ + blk_mq_quiesce_tagset(&hba->host->tag_set); cancel_work_sync(&hba->eeh_work); } static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) { - ufshcd_scsi_unblock_requests(hba); + blk_mq_unquiesce_tagset(&hba->host->tag_set); ufshcd_release(hba); if (ufshcd_is_clkscaling_supported(hba)) ufshcd_clk_scaling_suspend(hba, false); @@ -6193,36 +6609,52 @@ static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) return false; } +static bool ufshcd_abort_one(struct request *rq, void *priv) +{ + int *ret = priv; + u32 tag = rq->tag; + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); + struct scsi_device *sdev = cmd->device; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + + if (blk_mq_is_reserved_rq(rq)) + return true; + + *ret = ufshcd_try_to_abort_task(hba, tag); + dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, + ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, + *ret ? "failed" : "succeeded"); + + return *ret == 0; +} + +/** + * ufshcd_abort_all - Abort all pending commands. + * @hba: Host bus adapter pointer. + * + * Return: true if and only if the host controller needs to be reset. + */ static bool ufshcd_abort_all(struct ufs_hba *hba) { - bool needs_reset = false; - int tag, ret; + int tag, ret = 0; - /* Clear pending transfer requests */ - for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) { - ret = ufshcd_try_to_abort_task(hba, tag); - dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, - ret ? "failed" : "succeeded"); - if (ret) { - needs_reset = true; - goto out; - } - } + blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret); + if (ret) + goto out; /* Clear pending task management requests */ for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { - if (ufshcd_clear_tm_cmd(hba, tag)) { - needs_reset = true; + ret = ufshcd_clear_tm_cmd(hba, tag); + if (ret) goto out; - } } out: /* Complete the requests that are cleared by s/w */ - ufshcd_complete_requests(hba); + ufshcd_complete_requests(hba, false); - return needs_reset; + return ret != 0; } /** @@ -6241,12 +6673,26 @@ static void ufshcd_err_handler(struct work_struct *work) hba = container_of(work, struct ufs_hba, eh_work); dev_info(hba->dev, - "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n", + "%s started; HBA state %s; powered %d; shutting down %d; saved_err = 0x%x; saved_uic_err = 0x%x; force_reset = %d%s\n", __func__, ufshcd_state_name[hba->ufshcd_state], hba->is_powered, hba->shutting_down, hba->saved_err, hba->saved_uic_err, hba->force_reset, ufshcd_is_link_broken(hba) ? "; link is broken" : ""); + /* + * Use ufshcd_rpm_get_noresume() here to safely perform link recovery + * even if an error occurs during runtime suspend or runtime resume. + * This avoids potential deadlocks that could happen if we tried to + * resume the device while a PM operation is already in progress. + */ + ufshcd_rpm_get_noresume(hba); + if (hba->pm_op_in_progress) { + ufshcd_link_recovery(hba); + ufshcd_rpm_put(hba); + return; + } + ufshcd_rpm_put(hba); + down(&hba->host_sem); spin_lock_irqsave(hba->host->host_lock, flags); if (ufshcd_err_handling_should_stop(hba)) { @@ -6256,11 +6702,16 @@ static void ufshcd_err_handler(struct work_struct *work) up(&hba->host_sem); return; } - ufshcd_set_eh_in_progress(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_err_handling_prepare(hba); + + spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_set_eh_in_progress(hba); + spin_unlock_irqrestore(hba->host->host_lock, flags); + /* Complete requests that have door-bell cleared by h/w */ - ufshcd_complete_requests(hba); + ufshcd_complete_requests(hba, false); spin_lock_irqsave(hba->host->host_lock, flags); again: needs_restore = false; @@ -6275,7 +6726,8 @@ again: if (ufshcd_err_handling_should_stop(hba)) goto skip_err_handling; - if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { + if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) && + !hba->force_reset) { bool ret; spin_unlock_irqrestore(hba->host->host_lock, flags); @@ -6296,7 +6748,7 @@ again: ufshcd_print_pwr_info(hba); ufshcd_print_evt_hist(hba); ufshcd_print_tmrs(hba, hba->outstanding_tasks); - ufshcd_print_trs(hba, hba->outstanding_reqs, pr_prdt); + ufshcd_print_trs_all(hba, pr_prdt); spin_lock_irqsave(hba->host->host_lock, flags); } @@ -6409,7 +6861,7 @@ skip_err_handling: * ufshcd_update_uic_error - check and set fatal UIC error flags. * @hba: per-adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6502,7 +6954,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) * @hba: per-adapter instance * @intr_status: interrupt status generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6511,7 +6963,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) bool queue_eh_work = false; irqreturn_t retval = IRQ_NONE; - spin_lock(hba->host->host_lock); + guard(spinlock_irqsave)(hba->host->host_lock); hba->errors |= UFSHCD_ERROR_MASK & intr_status; if (hba->errors & INT_FATAL_ERRORS) { @@ -6570,7 +7022,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) */ hba->errors = 0; hba->uic_error = 0; - spin_unlock(hba->host->host_lock); + return retval; } @@ -6578,7 +7030,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) * ufshcd_tmc_handler - handle task management function completion * @hba: per adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6604,11 +7056,45 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) } /** + * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events + * @hba: per adapter instance + * + * Return: IRQ_HANDLED if interrupt is handled. + */ +static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) +{ + struct ufs_hw_queue *hwq; + unsigned long outstanding_cqs; + unsigned int nr_queues; + int i, ret; + u32 events; + + ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs); + if (ret) + outstanding_cqs = (1U << hba->nr_hw_queues) - 1; + + /* Exclude the poll queues */ + nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; + for_each_set_bit(i, &outstanding_cqs, nr_queues) { + hwq = &hba->uhq[i]; + + events = ufshcd_mcq_read_cqis(hba, i); + if (events) + ufshcd_mcq_write_cqis(hba, events, i); + + if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + } + + return IRQ_HANDLED; +} + +/** * ufshcd_sl_intr - Interrupt service routine * @hba: per adapter instance * @intr_status: contains interrupts generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6628,28 +7114,29 @@ static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) if (intr_status & UTP_TRANSFER_REQ_COMPL) retval |= ufshcd_transfer_req_compl(hba); + if (intr_status & MCQ_CQ_EVENT_STATUS) + retval |= ufshcd_handle_mcq_cq_events(hba); + return retval; } /** - * ufshcd_intr - Main interrupt service routine + * ufshcd_threaded_intr - Threaded interrupt service routine * @irq: irq number * @__hba: pointer to adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ -static irqreturn_t ufshcd_intr(int irq, void *__hba) +static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) { - u32 intr_status, enabled_intr_status = 0; + u32 last_intr_status, intr_status, enabled_intr_status = 0; irqreturn_t retval = IRQ_NONE; struct ufs_hba *hba = __hba; int retries = hba->nutrs; - intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); - hba->ufs_stats.last_intr_status = intr_status; - hba->ufs_stats.last_intr_ts = local_clock(); + last_intr_status = intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); /* * There could be max of hba->nutrs reqs in flight and in worst case @@ -6673,7 +7160,7 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba) dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n", __func__, intr_status, - hba->ufs_stats.last_intr_status, + last_intr_status, enabled_intr_status); ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); } @@ -6681,18 +7168,43 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba) return retval; } +/** + * ufshcd_intr - Main interrupt service routine + * @irq: irq number + * @__hba: pointer to adapter instance + * + * Return: + * IRQ_HANDLED - If interrupt is valid + * IRQ_WAKE_THREAD - If handling is moved to threaded handled + * IRQ_NONE - If invalid interrupt + */ +static irqreturn_t ufshcd_intr(int irq, void *__hba) +{ + struct ufs_hba *hba = __hba; + u32 intr_status, enabled_intr_status; + + /* Move interrupt handling to thread when MCQ & ESI are not enabled */ + if (!hba->mcq_enabled || !hba->mcq_esi_enabled) + return IRQ_WAKE_THREAD; + + intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); + enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + + ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); + + /* Directly handle interrupts since MCQ ESI handlers does the hard job */ + return ufshcd_sl_intr(hba, enabled_intr_status); +} + static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) { int err = 0; u32 mask = 1 << tag; - unsigned long flags; if (!test_bit(tag, &hba->outstanding_tasks)) goto out; - spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_utmrl_clear(hba, tag); - spin_unlock_irqrestore(hba->host->host_lock, flags); /* poll for max. 1 sec to clear door bell register by h/w */ err = ufshcd_wait_for_register(hba, @@ -6700,7 +7212,7 @@ static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) mask, 0, 1000, 1000); dev_err(hba->dev, "Clearing task management function with tag %d %s\n", - tag, err ? "succeeded" : "failed"); + tag, err < 0 ? "failed" : "succeeded"); out: return err; @@ -6724,28 +7236,24 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, return PTR_ERR(req); req->end_io_data = &wait; - ufshcd_hold(hba, false); + ufshcd_hold(hba); spin_lock_irqsave(host->host_lock, flags); task_tag = req->tag; - WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n", - task_tag); hba->tmf_rqs[req->tag] = req; - treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag); + treq->upiu_req.req_header.task_tag = task_tag; memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); - /* send command to the controller */ __set_bit(task_tag, &hba->outstanding_tasks); - ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); - /* Make sure that doorbell is committed immediately */ - wmb(); - spin_unlock_irqrestore(host->host_lock, flags); + /* send command to the controller */ + ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); + ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND); /* wait until the task management command is completed */ @@ -6785,23 +7293,23 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, * @tm_function: task management function opcode * @tm_response: task management service response return value * - * Returns non-zero value on error, zero on success. + * Return: non-zero value on error, zero on success. */ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, u8 tm_function, u8 *tm_response) { - struct utp_task_req_desc treq = { { 0 }, }; + struct utp_task_req_desc treq = { }; enum utp_ocs ocs_value; int err; /* Configure task request descriptor */ - treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); - treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); + treq.header.interrupt = 1; + treq.header.ocs = OCS_INVALID_COMMAND_STATUS; /* Configure task request UPIU */ - treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) | - cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24); - treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16); + treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ; + treq.upiu_req.req_header.lun = lun_id; + treq.upiu_req.req_header.tm_function = tm_function; /* * The host shall provide the same value for LUN field in the basic @@ -6814,7 +7322,7 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, if (err == -ETIMEDOUT) return err; - ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; + ocs_value = treq.header.ocs & MASK_OCS; if (ocs_value != OCS_SUCCESS) dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, ocs_value); @@ -6840,6 +7348,8 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, * * Since there is only one available tag for device management commands, * the caller is expected to hold the hba->dev_cmd.lock mutex. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, @@ -6848,35 +7358,26 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, enum query_opcode desc_op) { - DECLARE_COMPLETION_ONSTACK(wait); - const u32 tag = hba->reserved_slot; - struct ufshcd_lrb *lrbp; + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u32 tag; int err = 0; u8 upiu_flags; - /* Protects use of hba->reserved_slot. */ + /* Protects use of hba->dev_cmd. */ lockdep_assert_held(&hba->dev_cmd.lock); - down_read(&hba->clk_scaling_lock); + if (WARN_ON_ONCE(!cmd)) + return -ENOMEM; - lrbp = &hba->lrb[tag]; - WARN_ON(lrbp->cmd); - lrbp->cmd = NULL; - lrbp->task_tag = tag; - lrbp->lun = 0; - lrbp->intr_cmd = true; - ufshcd_prepare_lrbp_crypto(NULL, lrbp); - hba->dev_cmd.type = cmd_type; + tag = scsi_cmd_to_rq(cmd)->tag; - if (hba->ufs_version <= ufshci_version(1, 1)) - lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; - else - lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); - /* update the task tag in the request upiu */ - req_upiu->header.dword_0 |= cpu_to_be32(tag); + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); - ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE); + /* update the task tag in the request upiu */ + req_upiu->header.task_tag = tag; /* just copy the upiu request as it is */ memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); @@ -6891,24 +7392,16 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); - hba->dev_cmd.complete = &wait; - - ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - - ufshcd_send_command(hba, tag); - /* - * ignore the returning value here - ufshcd_check_query_response is - * bound to fail since dev_cmd.query and dev_cmd.type were left empty. - * read the response directly ignoring all errors. - */ - ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); + if (err) + goto put_dev_mgmt_cmd; /* just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); - u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & - MASK_QUERY_DATA_SEG_LEN; + u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header + .data_segment_length); if (*buff_len >= resp_len) { memcpy(desc_buff, descp, resp_len); @@ -6921,10 +7414,10 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, err = -EINVAL; } } - ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, - (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); - up_read(&hba->clk_scaling_lock); +put_dev_mgmt_cmd: + ufshcd_put_dev_mgmt_cmd(cmd); + return err; } @@ -6942,37 +7435,37 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, * Management requests. * It is up to the caller to fill the upiu conent properly, as it will * be copied without any further input validations. + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, struct utp_upiu_req *rsp_upiu, - int msgcode, + enum upiu_request_transaction msgcode, u8 *desc_buff, int *buff_len, enum query_opcode desc_op) { int err; enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; - struct utp_task_req_desc treq = { { 0 }, }; + struct utp_task_req_desc treq = { }; enum utp_ocs ocs_value; - u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC; + u8 tm_f = req_upiu->header.tm_function; switch (msgcode) { case UPIU_TRANSACTION_NOP_OUT: cmd_type = DEV_CMD_TYPE_NOP; fallthrough; case UPIU_TRANSACTION_QUERY_REQ: - ufshcd_hold(hba, false); - mutex_lock(&hba->dev_cmd.lock); + ufshcd_dev_man_lock(hba); err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu, desc_buff, buff_len, cmd_type, desc_op); - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); break; case UPIU_TRANSACTION_TASK_REQ: - treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); - treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); + treq.header.interrupt = 1; + treq.header.ocs = OCS_INVALID_COMMAND_STATUS; memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); @@ -6980,7 +7473,7 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, if (err == -ETIMEDOUT) break; - ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; + ocs_value = treq.header.ocs & MASK_OCS; if (ocs_value != OCS_SUCCESS) { dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, ocs_value); @@ -7000,17 +7493,141 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, } /** + * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request + * @hba: per adapter instance + * @req_upiu: upiu request + * @rsp_upiu: upiu reply + * @req_ehs: EHS field which contains Advanced RPMB Request Message + * @rsp_ehs: EHS field which returns Advanced RPMB Response Message + * @sg_cnt: The number of sg lists actually used + * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation + * @dir: DMA direction + * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ +int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, + struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, + struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, + enum dma_data_direction dir) +{ + struct scsi_cmnd *cmd; + struct ufshcd_lrb *lrbp; + u32 tag; + int err = 0; + int result; + u8 upiu_flags; + u8 *ehs_data; + u16 ehs_len; + int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; + + ufshcd_dev_man_lock(hba); + + cmd = ufshcd_get_dev_mgmt_cmd(hba); + + if (WARN_ON_ONCE(!cmd)) { + err = -ENOMEM; + goto unlock; + } + + lrbp = scsi_cmd_priv(cmd); + tag = scsi_cmd_to_rq(cmd)->tag; + + ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, + tag); + + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); + + /* update the task tag */ + req_upiu->header.task_tag = tag; + + /* copy the UPIU(contains CDB) request as it is */ + memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); + /* Copy EHS, starting with byte32, immediately after the CDB package */ + memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs)); + + if (dir != DMA_NONE && sg_list) + ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list); + + memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); + + err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); + if (err) + goto put_dev_mgmt_cmd; + + err = ufshcd_dev_cmd_completion(hba, lrbp); + if (!err) { + /* Just copy the upiu response as it is */ + memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); + /* Get the response UPIU result */ + result = (lrbp->ucd_rsp_ptr->header.response << 8) | + lrbp->ucd_rsp_ptr->header.status; + + ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length; + /* + * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data + * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB + * Message is 02h + */ + if (ehs_len == 2 && rsp_ehs) { + /* + * ucd_rsp_ptr points to a buffer with a length of 512 bytes + * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32 + */ + ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE; + memcpy(rsp_ehs, ehs_data, ehs_len * 32); + } + } + +put_dev_mgmt_cmd: + ufshcd_put_dev_mgmt_cmd(cmd); + +unlock: + ufshcd_dev_man_unlock(hba); + + return err ? : result; +} + +static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) +{ + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); + struct scsi_device *sdev = cmd->device; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + const u64 lun = *(u64 *)priv; + const u32 tag = req->tag; + + if (blk_mq_is_reserved_rq(req) || sdev->lun != lun) + return true; + + if (ufshcd_clear_cmd(hba, tag) < 0) { + dev_err(hba->dev, "%s: failed to clear request %d\n", __func__, + tag); + return true; + } + + if (hba->mcq_enabled) { + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req); + + if (hwq) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + return true; + } + + ufshcd_compl_one_cqe(hba, tag, NULL); + return true; +} + +/** * ufshcd_eh_device_reset_handler() - Reset a single logical unit. * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) { - unsigned long flags, pending_reqs = 0, not_cleared = 0; struct Scsi_Host *host; struct ufs_hba *hba; - u32 pos; int err; u8 resp = 0xF, lun; @@ -7019,33 +7636,16 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); - if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { - if (!err) - err = resp; - goto out; - } - - /* clear the commands that were pending for corresponding LUN */ - spin_lock_irqsave(&hba->outstanding_lock, flags); - for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) - if (hba->lrb[pos].lun == lun) - __set_bit(pos, &pending_reqs); - hba->outstanding_reqs &= ~pending_reqs; - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - if (ufshcd_clear_cmds(hba, pending_reqs) < 0) { - spin_lock_irqsave(&hba->outstanding_lock, flags); - not_cleared = pending_reqs & - ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); - hba->outstanding_reqs |= not_cleared; - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - dev_err(hba->dev, "%s: failed to clear requests %#lx\n", - __func__, not_cleared); + if (err) { + } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { + err = resp; + } else { + /* clear the commands that were pending for corresponding LUN */ + blk_mq_tagset_busy_iter(&hba->host->tag_set, + ufshcd_clear_lu_cmds, + &cmd->device->lun); } - __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared); -out: hba->req_abort_count = 0; ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); if (!err) { @@ -7059,11 +7659,12 @@ out: static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) { - struct ufshcd_lrb *lrbp; int tag; for_each_set_bit(tag, &bitmap, hba->nutrs) { - lrbp = &hba->lrb[tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + lrbp->req_abort_skip = true; } } @@ -7071,7 +7672,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) /** * ufshcd_try_to_abort_task - abort a specific task * @hba: Pointer to adapter instance - * @tag: Task tag/index to be aborted + * @tag: Tag of the task to be aborted * * Abort the pending command in device by sending UFS_ABORT_TASK task management * command, and in host controller by clearing the door-bell register. There can @@ -7079,19 +7680,19 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is * really issued and then try to abort it. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ -static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) +int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) { - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; - int err = 0; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + int err; int poll_cnt; u8 resp = 0xF; - u32 reg; for (poll_cnt = 100; poll_cnt; poll_cnt--) { - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, - UFS_QUERY_TASK, &resp); + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK, + &resp); if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { /* cmd pending in the device */ dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", @@ -7102,50 +7703,43 @@ static int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) * cmd not pending in the device, check if it is * in transition. */ - dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n", + dev_info( + hba->dev, + "%s: cmd with tag %d not pending in the device.\n", __func__, tag); - reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); - if (reg & (1 << tag)) { - /* sleep for max. 200us to stabilize */ - usleep_range(100, 200); - continue; + if (!ufshcd_cmd_inflight(cmd)) { + dev_info(hba->dev, + "%s: cmd with tag=%d completed.\n", + __func__, tag); + return 0; } - /* command completed already */ - dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n", - __func__, tag); - goto out; + usleep_range(100, 200); } else { dev_err(hba->dev, "%s: no response from device. tag = %d, err %d\n", __func__, tag, err); - if (!err) - err = resp; /* service response error */ - goto out; + return err ? : resp; } } - if (!poll_cnt) { - err = -EBUSY; - goto out; - } + if (!poll_cnt) + return -EBUSY; - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, - UFS_ABORT_TASK, &resp); + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp); if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { if (!err) { err = resp; /* service response error */ dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", __func__, tag, err); } - goto out; + return err; } - err = ufshcd_clear_cmds(hba, 1U << tag); + err = ufshcd_clear_cmd(hba, tag); if (err) dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", __func__, tag, err); -out: return err; } @@ -7153,29 +7747,31 @@ out: * ufshcd_abort - scsi host template eh_abort_handler callback * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_abort(struct scsi_cmnd *cmd) { struct Scsi_Host *host = cmd->device->host; struct ufs_hba *hba = shost_priv(host); - int tag = scsi_cmd_to_rq(cmd)->tag; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct request *rq = scsi_cmd_to_rq(cmd); + int tag = rq->tag; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); unsigned long flags; int err = FAILED; bool outstanding; u32 reg; - WARN_ONCE(tag < 0, "Invalid tag %d\n", tag); + ufshcd_hold(hba); - ufshcd_hold(hba, false); - reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); - /* If command is already aborted/completed, return FAILED. */ - if (!(test_bit(tag, &hba->outstanding_reqs))) { - dev_err(hba->dev, - "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", - __func__, tag, hba->outstanding_reqs, reg); - goto release; + if (!hba->mcq_enabled) { + reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); + if (!test_bit(tag, &hba->outstanding_reqs)) { + /* If command is already aborted/completed, return FAILED. */ + dev_err(hba->dev, + "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n", + __func__, tag, hba->outstanding_reqs, reg); + goto release; + } } /* Print Transfer Request of aborted task */ @@ -7188,19 +7784,21 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) * to reduce repeated printouts. For other aborted requests only print * basic details. */ - scsi_print_command(cmd); + if (ufshcd_is_scsi_cmd(cmd)) + scsi_print_command(cmd); if (!hba->req_abort_count) { ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); - ufshcd_print_trs(hba, 1 << tag, true); + ufshcd_print_tr(hba, cmd, true); } else { - ufshcd_print_trs(hba, 1 << tag, false); + ufshcd_print_tr(hba, cmd, false); } hba->req_abort_count++; - if (!(reg & (1 << tag))) { + if (!hba->mcq_enabled && !(reg & (1 << tag))) { + /* only execute this code in single doorbell mode */ dev_err(hba->dev, "%s: cmd was completed, but without a notifying intr, tag = %d", __func__, tag); @@ -7226,6 +7824,12 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } + if (hba->mcq_enabled) { + /* MCQ mode. Branch off to handle abort for mcq mode */ + err = ufshcd_mcq_abort(cmd); + goto release; + } + /* Skip task abort in case previous aborts failed and report failure */ if (lrbp->req_abort_skip) { dev_err(hba->dev, "%s: skipping abort\n", __func__); @@ -7233,7 +7837,10 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } - err = ufshcd_try_to_abort_task(hba, tag); + if (blk_mq_is_reserved_rq(rq)) + err = ufshcd_clear_cmd(hba, tag); + else + err = ufshcd_try_to_abort_task(hba, tag); if (err) { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); @@ -7250,7 +7857,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (outstanding) - ufshcd_release_scsi_cmd(hba, lrbp); + ufshcd_release_scsi_cmd(hba, cmd); err = SUCCESS; @@ -7261,6 +7868,29 @@ release: } /** + * ufshcd_process_probe_result - Process the ufshcd_probe_hba() result. + * @hba: UFS host controller instance. + * @probe_start: time when the ufshcd_probe_hba() call started. + * @ret: ufshcd_probe_hba() return value. + */ +static void ufshcd_process_probe_result(struct ufs_hba *hba, + ktime_t probe_start, int ret) +{ + unsigned long flags; + + spin_lock_irqsave(hba->host->host_lock, flags); + if (ret) + hba->ufshcd_state = UFSHCD_STATE_ERROR; + else if (hba->ufshcd_state == UFSHCD_STATE_RESET) + hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; + spin_unlock_irqrestore(hba->host->host_lock, flags); + + trace_ufshcd_init(hba, ret, + ktime_to_us(ktime_sub(ktime_get(), probe_start)), + hba->curr_dev_pwr_mode, hba->uic_link_state); +} + +/** * ufshcd_host_reset_and_restore - reset and restore host controller * @hba: per-adapter instance * @@ -7268,7 +7898,7 @@ release: * local and remote (device) Uni-Pro stack and the attributes * are reset to default state. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) { @@ -7278,20 +7908,26 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) * Stop the host controller and complete the requests * cleared by h/w */ - ufshpb_toggle_state(hba, HPB_PRESENT, HPB_RESET); ufshcd_hba_stop(hba); hba->silence_err_logs = true; - ufshcd_complete_requests(hba); + ufshcd_complete_requests(hba, true); hba->silence_err_logs = false; /* scale up clocks to max frequency before full reinitialization */ - ufshcd_scale_clks(hba, true); + if (ufshcd_is_clkscaling_supported(hba)) + ufshcd_scale_clks(hba, ULONG_MAX, true); err = ufshcd_hba_enable(hba); /* Establish the link again and restore the device */ - if (!err) - err = ufshcd_probe_hba(hba, false); + if (!err) { + ktime_t probe_start = ktime_get(); + + err = ufshcd_device_init(hba, /*init_dev_params=*/false); + if (!err) + err = ufshcd_probe_hba(hba, false); + ufshcd_process_probe_result(hba, probe_start, err); + } if (err) dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); @@ -7306,7 +7942,7 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) * Reset and recover device, host and re-establish link. This * is helpful to recover the communication in fatal error conditions. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ static int ufshcd_reset_and_restore(struct ufs_hba *hba) { @@ -7364,7 +8000,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba) * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) { @@ -7374,6 +8010,19 @@ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) hba = shost_priv(cmd->device->host); + /* + * If runtime PM sent SSU and got a timeout, scsi_error_handler is + * stuck in this function waiting for flush_work(&hba->eh_work). And + * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do + * ufshcd_link_recovery instead of eh_work to prevent deadlock. + */ + if (hba->pm_op_in_progress) { + if (ufshcd_link_recovery(hba)) + err = FAILED; + + return err; + } + spin_lock_irqsave(hba->host->host_lock, flags); hba->force_reset = true; ufshcd_schedule_eh_work(hba); @@ -7396,7 +8045,7 @@ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) * @start_scan: row at the desc table to start scan from * @buff: power descriptor buffer * - * Returns calculated max ICC level for specific regulator + * Return: calculated max ICC level for specific regulator. */ static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, const char *buff) @@ -7441,12 +8090,11 @@ static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, * In case regulators are not initialized we'll return 0 * @hba: per-adapter instance * @desc_buf: power descriptor buffer to extract ICC levels from. - * @len: length of desc_buff * - * Returns calculated ICC level + * Return: calculated ICC level. */ static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, - const u8 *desc_buf, int len) + const u8 *desc_buf) { u32 icc_level = 0; @@ -7488,25 +8136,23 @@ out: static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba) { int ret; - int buff_len = hba->desc_size[QUERY_DESC_IDN_POWER]; u8 *desc_buf; u32 icc_level; - desc_buf = kmalloc(buff_len, GFP_KERNEL); + desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); if (!desc_buf) return; ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0, - desc_buf, buff_len); + desc_buf, QUERY_DESC_MAX_SIZE); if (ret) { dev_err(hba->dev, - "%s: Failed reading power descriptor.len = %d ret = %d", - __func__, buff_len, ret); + "%s: Failed reading power descriptor ret = %d", + __func__, ret); goto out; } - icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf, - buff_len); + icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf); dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level); ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, @@ -7523,11 +8169,13 @@ out: static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) { + struct Scsi_Host *shost = sdev->host; + scsi_autopm_get_device(sdev); blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); if (sdev->rpm_autosuspend) pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, - RPM_AUTOSUSPEND_DELAY_MS); + shost->rpm_autosuspend_delay); scsi_autopm_put_device(sdev); } @@ -7554,7 +8202,7 @@ static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) * This function adds scsi device instances for each of all well known LUs * (except "REPORT LUNS" LU). * - * Returns zero on success (all required W-LUs are added successfully), + * Return: zero on success (all required W-LUs are added successfully), * non-zero error value on failure (if failed to add any of the required W-LU). */ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) @@ -7575,8 +8223,11 @@ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); if (IS_ERR(sdev_rpmb)) { ret = PTR_ERR(sdev_rpmb); + hba->ufs_rpmb_wlun = NULL; + dev_err(hba->dev, "%s: RPMB WLUN not found\n", __func__); goto remove_ufs_device_wlun; } + hba->ufs_rpmb_wlun = sdev_rpmb; ufshcd_blk_pm_runtime_init(sdev_rpmb); scsi_device_put(sdev_rpmb); @@ -7616,10 +8267,6 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES))) goto wb_disabled; - if (hba->desc_size[QUERY_DESC_IDN_DEVICE] < - DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP + 4) - goto wb_disabled; - ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); @@ -7633,6 +8280,9 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) */ dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; + dev_info->ext_wb_sup = get_unaligned_be16(desc_buf + + DEVICE_DESC_PARAM_EXT_WB_SUP); + dev_info->b_presrv_uspc_en = desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; @@ -7690,6 +8340,54 @@ static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) } } +static void ufshcd_device_lvl_exception_probe(struct ufs_hba *hba, u8 *desc_buf) +{ + u32 ext_ufs_feature; + + if (hba->dev_info.wspecversion < 0x410) + return; + + ext_ufs_feature = get_unaligned_be32(desc_buf + + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); + if (!(ext_ufs_feature & UFS_DEV_LVL_EXCEPTION_SUP)) + return; + + atomic_set(&hba->dev_lvl_exception_count, 0); + ufshcd_enable_ee(hba, MASK_EE_DEV_LVL_EXCEPTION); +} + +static void ufshcd_set_rtt(struct ufs_hba *hba) +{ + struct ufs_dev_info *dev_info = &hba->dev_info; + u32 rtt = 0; + u32 dev_rtt = 0; + int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ? + hba->vops->max_num_rtt : hba->nortt; + + /* RTT override makes sense only for UFS-4.0 and above */ + if (dev_info->wspecversion < 0x400) + return; + + if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) { + dev_err(hba->dev, "failed reading bMaxNumOfRTT\n"); + return; + } + + /* do not override if it was already written */ + if (dev_rtt != DEFAULT_MAX_NUM_RTT) + return; + + rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap); + + if (rtt == dev_rtt) + return; + + if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt)) + dev_err(hba->dev, "failed writing bMaxNumOfRTT\n"); +} + void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, const struct ufs_dev_quirk *fixups) { @@ -7719,22 +8417,161 @@ static void ufs_fixup_device_setup(struct ufs_hba *hba) ufshcd_vops_fixup_dev_quirks(hba); } +static void ufshcd_update_rtc(struct ufs_hba *hba) +{ + struct timespec64 ts64; + int err; + u32 val; + + ktime_get_real_ts64(&ts64); + + if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) { + dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__); + return; + } + + /* + * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond + * 2146 is required, it is recommended to choose the relative RTC mode. + */ + val = ts64.tv_sec - hba->dev_info.rtc_time_baseline; + + /* Skip update RTC if RPM state is not RPM_ACTIVE */ + if (ufshcd_rpm_get_if_active(hba) <= 0) + return; + + err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED, + 0, 0, &val); + ufshcd_rpm_put(hba); + + if (err) + dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err); + else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) + hba->dev_info.rtc_time_baseline = ts64.tv_sec; +} + +static void ufshcd_rtc_work(struct work_struct *work) +{ + struct ufs_hba *hba; + + hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work); + + /* Update RTC only when there are no requests in progress and UFSHCI is operational */ + if (!ufshcd_is_ufs_dev_busy(hba) && + hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL && + !hba->clk_gating.active_reqs) + ufshcd_update_rtc(hba); + + if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period) + schedule_delayed_work(&hba->ufs_rtc_update_work, + msecs_to_jiffies(hba->dev_info.rtc_update_period)); +} + +static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf) +{ + u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]); + struct ufs_dev_info *dev_info = &hba->dev_info; + + if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) { + dev_info->rtc_type = UFS_RTC_ABSOLUTE; + + /* + * The concept of measuring time in Linux as the number of seconds elapsed since + * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st + * 2010 00:00, here we need to adjust ABS baseline. + */ + dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) - + mktime64(1970, 1, 1, 0, 0, 0); + } else { + dev_info->rtc_type = UFS_RTC_RELATIVE; + dev_info->rtc_time_baseline = 0; + } + + /* + * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state + * how to calculate the specific update period for each time unit. And we disable periodic + * RTC update work, let user configure by sysfs node according to specific circumstance. + */ + dev_info->rtc_update_period = 0; +} + +/** + * ufshcd_create_device_id - Generate unique device identifier string + * @hba: per-adapter instance + * @desc_buf: device descriptor buffer + * + * Creates a unique device ID string combining manufacturer ID, spec version, + * model name, serial number (as hex), device version, and manufacture date. + * + * Returns: Allocated device ID string on success, NULL on failure + */ +static char *ufshcd_create_device_id(struct ufs_hba *hba, u8 *desc_buf) +{ + struct ufs_dev_info *dev_info = &hba->dev_info; + u16 manufacture_date; + u16 device_version; + u8 *serial_number; + char *serial_hex; + char *device_id; + u8 serial_index; + int serial_len; + int ret; + + serial_index = desc_buf[DEVICE_DESC_PARAM_SN]; + + ret = ufshcd_read_string_desc(hba, serial_index, &serial_number, SD_RAW); + if (ret < 0) { + dev_err(hba->dev, "Failed reading Serial Number. err = %d\n", ret); + return NULL; + } + + device_version = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_DEV_VER]); + manufacture_date = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_MANF_DATE]); + + serial_len = ret; + /* Allocate buffer for hex string: 2 chars per byte + null terminator */ + serial_hex = kzalloc(serial_len * 2 + 1, GFP_KERNEL); + if (!serial_hex) { + kfree(serial_number); + return NULL; + } + + bin2hex(serial_hex, serial_number, serial_len); + + /* + * Device ID format is ABI with secure world - do not change without firmware + * coordination. + */ + device_id = kasprintf(GFP_KERNEL, "%04X-%04X-%s-%s-%04X-%04X", + dev_info->wmanufacturerid, dev_info->wspecversion, + dev_info->model, serial_hex, device_version, + manufacture_date); + + kfree(serial_hex); + kfree(serial_number); + + if (!device_id) + dev_warn(hba->dev, "Failed to allocate unique device ID\n"); + + return device_id; +} + static int ufs_get_device_desc(struct ufs_hba *hba) { + struct ufs_dev_info *dev_info = &hba->dev_info; + struct Scsi_Host *shost = hba->host; int err; u8 model_index; - u8 b_ufs_feature_sup; u8 *desc_buf; - struct ufs_dev_info *dev_info = &hba->dev_info; - desc_buf = kmalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); + desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); if (!desc_buf) { err = -ENOMEM; goto out; } err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf, - hba->desc_size[QUERY_DESC_IDN_DEVICE]); + QUERY_DESC_MAX_SIZE); if (err) { dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", __func__, err); @@ -7751,25 +8588,27 @@ static int ufs_get_device_desc(struct ufs_hba *hba) /* getting Specification Version in big endian format */ dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; - b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT]; + dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; - model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; - - if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION && - (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) { - bool hpb_en = false; + /* + * According to the UFS standard, the UFS device queue depth + * (bQueueDepth) must be in the range 1..255 if the shared queueing + * architecture is supported. bQueueDepth is zero if the shared queueing + * architecture is not supported. + */ + if (dev_info->bqueuedepth) + shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) - + UFSHCD_NUM_RESERVED; + else + shost->cmd_per_lun = shost->can_queue; - ufshpb_get_dev_info(hba, desc_buf); + dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; - if (!ufshpb_is_legacy(hba)) - err = ufshcd_query_flag_retry(hba, - UPIU_QUERY_OPCODE_READ_FLAG, - QUERY_FLAG_IDN_HPB_EN, 0, - &hpb_en); + dev_info->hid_sup = get_unaligned_be32(desc_buf + + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP) & + UFS_DEV_HID_SUPPORT; - if (ufshpb_is_legacy(hba) || (!err && hpb_en)) - dev_info->hpb_enabled = true; - } + model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; err = ufshcd_read_string_desc(hba, model_index, &dev_info->model, SD_ASCII_STD); @@ -7779,6 +8618,9 @@ static int ufs_get_device_desc(struct ufs_hba *hba) goto out; } + /* Generate unique device ID */ + dev_info->device_id = ufshcd_create_device_id(hba, desc_buf); + hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] + desc_buf[DEVICE_DESC_PARAM_NUM_WLU]; @@ -7788,6 +8630,15 @@ static int ufs_get_device_desc(struct ufs_hba *hba) ufshcd_temp_notif_probe(hba, desc_buf); + if (dev_info->wspecversion >= 0x410) { + hba->critical_health_count = 0; + ufshcd_enable_ee(hba, MASK_EE_HEALTH_CRITICAL); + } + + ufs_init_rtc(hba, desc_buf); + + ufshcd_device_lvl_exception_probe(hba, desc_buf); + /* * ufshcd_read_string_desc returns size of the string * reset the error value @@ -7805,83 +8656,8 @@ static void ufs_put_device_desc(struct ufs_hba *hba) kfree(dev_info->model); dev_info->model = NULL; -} - -/** - * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro - * @hba: per-adapter instance - * - * PA_TActivate parameter can be tuned manually if UniPro version is less than - * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's - * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce - * the hibern8 exit latency. - * - * Returns zero on success, non-zero error value on failure. - */ -static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba) -{ - int ret = 0; - u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate; - - ret = ufshcd_dme_peer_get(hba, - UIC_ARG_MIB_SEL( - RX_MIN_ACTIVATETIME_CAPABILITY, - UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), - &peer_rx_min_activatetime); - if (ret) - goto out; - - /* make sure proper unit conversion is applied */ - tuned_pa_tactivate = - ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US) - / PA_TACTIVATE_TIME_UNIT_US); - ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), - tuned_pa_tactivate); - -out: - return ret; -} - -/** - * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro - * @hba: per-adapter instance - * - * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than - * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's - * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY. - * This optimal value can help reduce the hibern8 exit latency. - * - * Returns zero on success, non-zero error value on failure. - */ -static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba) -{ - int ret = 0; - u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0; - u32 max_hibern8_time, tuned_pa_hibern8time; - - ret = ufshcd_dme_get(hba, - UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY, - UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)), - &local_tx_hibern8_time_cap); - if (ret) - goto out; - - ret = ufshcd_dme_peer_get(hba, - UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY, - UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), - &peer_rx_hibern8_time_cap); - if (ret) - goto out; - - max_hibern8_time = max(local_tx_hibern8_time_cap, - peer_rx_hibern8_time_cap); - /* make sure proper unit conversion is applied */ - tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US) - / PA_HIBERN8_TIME_UNIT_US); - ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), - tuned_pa_hibern8time); -out: - return ret; + kfree(dev_info->device_id); + dev_info->device_id = NULL; } /** @@ -7893,7 +8669,7 @@ out: * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk * for such devices. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) { @@ -7954,13 +8730,33 @@ out: return ret; } -static void ufshcd_tune_unipro_params(struct ufs_hba *hba) +/** + * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. + * @hba: per-adapter instance + * + * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter + * to ensure proper hibernation timing. This function retrieves the current + * PA_HIBERN8TIME value and increments it by 100us. + */ +static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) { - if (ufshcd_is_unipro_pa_params_tuning_req(hba)) { - ufshcd_tune_pa_tactivate(hba); - ufshcd_tune_pa_hibern8time(hba); + u32 pa_h8time; + int ret; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); + if (ret) { + dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); + return; } + /* Increment by 1 to increase hibernation time by 100 µs */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); + if (ret) + dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); +} + +static void ufshcd_tune_unipro_params(struct ufs_hba *hba) +{ ufshcd_vops_apply_dev_quirks(hba); if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) @@ -7969,6 +8765,9 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) ufshcd_quirk_tune_host_pa_tactivate(hba); + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) + ufshcd_quirk_override_pa_h8time(hba); } static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) @@ -7981,18 +8780,16 @@ static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) static int ufshcd_device_geo_params_init(struct ufs_hba *hba) { int err; - size_t buff_len; u8 *desc_buf; - buff_len = hba->desc_size[QUERY_DESC_IDN_GEOMETRY]; - desc_buf = kmalloc(buff_len, GFP_KERNEL); + desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); if (!desc_buf) { err = -ENOMEM; goto out; } err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0, - desc_buf, buff_len); + desc_buf, QUERY_DESC_MAX_SIZE); if (err) { dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n", __func__, err); @@ -8004,9 +8801,7 @@ static int ufshcd_device_geo_params_init(struct ufs_hba *hba) else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) hba->dev_info.max_lu_supported = 8; - if (hba->desc_size[QUERY_DESC_IDN_GEOMETRY] >= - GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS) - ufshpb_get_geo_info(hba, desc_buf); + hba->dev_info.rpmb_io_size = desc_buf[GEOMETRY_DESC_PARAM_RPMB_RW_SIZE]; out: kfree(desc_buf); @@ -8089,11 +8884,7 @@ out: static int ufshcd_device_params_init(struct ufs_hba *hba) { bool flag; - int ret, i; - - /* Init device descriptor sizes */ - for (i = 0; i < QUERY_DESC_IDN_MAX; i++) - hba->desc_size[i] = QUERY_DESC_MAX_SIZE; + int ret; /* Init UFS geometry descriptor related parameters */ ret = ufshcd_device_geo_params_init(hba); @@ -8108,6 +8899,8 @@ static int ufshcd_device_params_init(struct ufs_hba *hba) goto out; } + ufshcd_set_rtt(hba); + ufshcd_get_ref_clk_gating_wait(hba); if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, @@ -8123,9 +8916,44 @@ out: return ret; } +static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) +{ + int err; + struct ufs_query_req *request = NULL; + struct ufs_query_res *response = NULL; + struct ufs_dev_info *dev_info = &hba->dev_info; + struct utp_upiu_query_v4_0 *upiu_data; + + if (dev_info->wspecversion < 0x400 || + hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT) + return; + + ufshcd_dev_man_lock(hba); + + ufshcd_init_query(hba, &request, &response, + UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_TIMESTAMP, 0, 0); + + request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; + + upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req; + + put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3); + + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); + + if (err) + dev_err(hba->dev, "%s: failed to set timestamp %d\n", + __func__, err); + + ufshcd_dev_man_unlock(hba); +} + /** * ufshcd_add_lus - probe and add UFS logical units * @hba: per-adapter instance + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_add_lus(struct ufs_hba *hba) { @@ -8138,10 +8966,9 @@ static int ufshcd_add_lus(struct ufs_hba *hba) /* Initialize devfreq after UFS device is detected */ if (ufshcd_is_clkscaling_supported(hba)) { - memcpy(&hba->clk_scaling.saved_pwr_info.info, + memcpy(&hba->clk_scaling.saved_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); - hba->clk_scaling.saved_pwr_info.is_valid = true; hba->clk_scaling.is_allowed = true; ret = ufshcd_devfreq_init(hba); @@ -8152,36 +8979,137 @@ static int ufshcd_add_lus(struct ufs_hba *hba) ufshcd_init_clk_scaling_sysfs(hba); } + /* + * The RTC update code accesses the hba->ufs_device_wlun->sdev_gendev + * pointer and hence must only be started after the WLUN pointer has + * been initialized by ufshcd_scsi_add_wlus(). + */ + schedule_delayed_work(&hba->ufs_rtc_update_work, + msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); + ufs_bsg_probe(hba); - ufshpb_init(hba); scsi_scan_host(hba->host); - pm_runtime_put_sync(hba->dev); + ufs_rpmb_probe(hba); out: return ret; } -/** - * ufshcd_probe_hba - probe hba to detect device and initialize it - * @hba: per-adapter instance - * @init_dev_params: whether or not to call ufshcd_device_params_init(). - * - * Execute link-startup and verify device initialization - */ -static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) +/* SDB - Single Doorbell */ +static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) +{ + size_t ucdl_size, utrdl_size; + + ucdl_size = ufshcd_get_ucd_size(hba) * nutrs; + dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr, + hba->ucdl_dma_addr); + + utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; + dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, + hba->utrdl_dma_addr); +} + +static int ufshcd_alloc_mcq(struct ufs_hba *hba) { int ret; - unsigned long flags; - ktime_t start = ktime_get(); + int old_nutrs = hba->nutrs; + + ret = ufshcd_get_hba_mac(hba); + if (ret < 0) + return ret; + + hba->nutrs = ret; + ret = ufshcd_mcq_init(hba); + if (ret) + goto err; + + /* + * Previously allocated memory for nutrs may not be enough in MCQ mode. + * Number of supported tags in MCQ mode may be larger than SDB mode. + */ + if (hba->nutrs != old_nutrs) { + ufshcd_release_sdb_queue(hba, old_nutrs); + ret = ufshcd_memory_alloc(hba); + if (ret) + goto err; + ufshcd_host_memory_configure(hba); + } + + ret = ufshcd_mcq_memory_alloc(hba); + if (ret) + goto err; + + hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; + + return 0; +err: + hba->nutrs = old_nutrs; + return ret; +} + +static void ufshcd_config_mcq(struct ufs_hba *hba) +{ + int ret; + + ret = ufshcd_mcq_vops_config_esi(hba); + hba->mcq_esi_enabled = !ret; + dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); + + ufshcd_mcq_make_queues_operational(hba); + ufshcd_mcq_config_mac(hba, hba->nutrs); + + dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", + hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT], + hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL], + hba->nutrs); +} + +static int ufshcd_post_device_init(struct ufs_hba *hba) +{ + int ret; + + ufshcd_tune_unipro_params(hba); + + /* UFS device is also active now */ + ufshcd_set_ufs_dev_active(hba); + ufshcd_force_reset_auto_bkops(hba); + + ufshcd_set_timestamp_attr(hba); + + if (!hba->max_pwr_info.is_valid) + return 0; + + /* + * Set the right value to bRefClkFreq before attempting to + * switch to HS gears. + */ + if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) + ufshcd_set_dev_ref_clk(hba); + /* Gear up to HS gear. */ + ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); + if (ret) { + dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", + __func__, ret); + return ret; + } + + return 0; +} + +static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) +{ + int ret; + + WARN_ON_ONCE(!hba->scsi_host_added); hba->ufshcd_state = UFSHCD_STATE_RESET; ret = ufshcd_link_startup(hba); if (ret) - goto out; + return ret; if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) - goto out; + return ret; /* Debug counters initialization */ ufshcd_clear_dbg_ufs_stats(hba); @@ -8189,15 +9117,21 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) /* UniPro link is active now */ ufshcd_set_link_active(hba); + /* Reconfigure MCQ upon reset */ + if (hba->mcq_enabled && !init_dev_params) { + ufshcd_config_mcq(hba); + ufshcd_mcq_enable(hba); + } + /* Verify device initialization by sending NOP OUT UPIU */ ret = ufshcd_verify_dev_init(hba); if (ret) - goto out; + return ret; /* Initiate UFS initialization, and waiting until completion */ ret = ufshcd_complete_dev_init(hba); if (ret) - goto out; + return ret; /* * Initialize UFS device parameters used by driver, these @@ -8206,32 +9140,52 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) if (init_dev_params) { ret = ufshcd_device_params_init(hba); if (ret) - goto out; + return ret; + if (is_mcq_supported(hba) && + hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { + ufshcd_config_mcq(hba); + ufshcd_mcq_enable(hba); + } } - ufshcd_tune_unipro_params(hba); + return ufshcd_post_device_init(hba); +} - /* UFS device is also active now */ - ufshcd_set_ufs_dev_active(hba); - ufshcd_force_reset_auto_bkops(hba); +/** + * ufshcd_probe_hba - probe hba to detect device and initialize it + * @hba: per-adapter instance + * @init_dev_params: whether or not to call ufshcd_device_params_init(). + * + * Execute link-startup and verify device initialization + * + * Return: 0 upon success; < 0 upon failure. + */ +static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) +{ + int ret; - /* Gear up to HS gear if supported */ - if (hba->max_pwr_info.is_valid) { - /* - * Set the right value to bRefClkFreq before attempting to - * switch to HS gears. - */ - if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) - ufshcd_set_dev_ref_clk(hba); - ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); + if (!hba->pm_op_in_progress && + (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) { + /* Reset the device and controller before doing reinit */ + ufshcd_device_reset(hba); + ufs_put_device_desc(hba); + ufshcd_hba_stop(hba); + ret = ufshcd_hba_enable(hba); if (ret) { - dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", - __func__, ret); - goto out; + dev_err(hba->dev, "Host controller enable failed\n"); + ufshcd_print_evt_hist(hba); + ufshcd_print_host_state(hba); + return ret; } - ufshcd_print_pwr_info(hba); + + /* Reinit the device */ + ret = ufshcd_device_init(hba, init_dev_params); + if (ret) + return ret; } + ufshcd_print_pwr_info(hba); + /* * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec) * and for removable UFS card as well, hence always set the parameter. @@ -8245,22 +9199,9 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) if (hba->ee_usr_mask) ufshcd_write_ee_control(hba); - /* Enable Auto-Hibernate if configured */ - ufshcd_auto_hibern8_enable(hba); - - ufshpb_toggle_state(hba, HPB_RESET, HPB_PRESENT); -out: - spin_lock_irqsave(hba->host->host_lock, flags); - if (ret) - hba->ufshcd_state = UFSHCD_STATE_ERROR; - else if (hba->ufshcd_state == UFSHCD_STATE_RESET) - hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; - spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_configure_auto_hibern8(hba); - trace_ufshcd_init(dev_name(hba->dev), ret, - ktime_to_us(ktime_sub(ktime_get(), start)), - hba->curr_dev_pwr_mode, hba->uic_link_state); - return ret; + return 0; } /** @@ -8271,26 +9212,26 @@ out: static void ufshcd_async_scan(void *data, async_cookie_t cookie) { struct ufs_hba *hba = (struct ufs_hba *)data; + ktime_t probe_start; int ret; down(&hba->host_sem); /* Initialize hba, detect and initialize UFS device */ + probe_start = ktime_get(); ret = ufshcd_probe_hba(hba, true); + ufshcd_process_probe_result(hba, probe_start, ret); up(&hba->host_sem); if (ret) goto out; /* Probe and add UFS logical units */ ret = ufshcd_add_lus(hba); + out: - /* - * If we failed to initialize the device or the device is not - * present, turn off the power/clocks etc. - */ - if (ret) { - pm_runtime_put_sync(hba->dev); - ufshcd_hba_exit(hba); - } + pm_runtime_put_sync(hba->dev); + + if (ret) + dev_err(hba->dev, "%s failed: %d\n", __func__, ret); } static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) @@ -8312,16 +9253,12 @@ static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", __func__, hba->outstanding_tasks); - return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; + return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; } static const struct attribute_group *ufshcd_driver_groups[] = { &ufs_sysfs_unit_descriptor_group, &ufs_sysfs_lun_attributes_group, -#ifdef CONFIG_SCSI_UFS_HPB - &ufs_sysfs_hpb_stat_group, - &ufs_sysfs_hpb_param_group, -#endif NULL, }; @@ -8335,16 +9272,20 @@ static struct ufs_hba_variant_params ufs_hba_vps = { .ondemand_data.downdifferential = 5, }; -static struct scsi_host_template ufshcd_driver_template = { +static const struct scsi_host_template ufshcd_driver_template = { .module = THIS_MODULE, .name = UFSHCD, .proc_name = UFSHCD, .map_queues = ufshcd_map_queues, + .cmd_size = sizeof(struct ufshcd_lrb), + .init_cmd_priv = ufshcd_init_cmd_priv, .queuecommand = ufshcd_queuecommand, + .queue_reserved_command = ufshcd_queue_reserved_command, + .nr_reserved_cmds = UFSHCD_NUM_RESERVED, .mq_poll = ufshcd_poll, - .slave_alloc = ufshcd_slave_alloc, - .slave_configure = ufshcd_slave_configure, - .slave_destroy = ufshcd_slave_destroy, + .sdev_init = ufshcd_sdev_init, + .sdev_configure = ufshcd_sdev_configure, + .sdev_destroy = ufshcd_sdev_destroy, .change_queue_depth = ufshcd_change_queue_depth, .eh_abort_handler = ufshcd_abort, .eh_device_reset_handler = ufshcd_eh_device_reset_handler, @@ -8352,15 +9293,12 @@ static struct scsi_host_template ufshcd_driver_template = { .eh_timed_out = ufshcd_eh_timed_out, .this_id = -1, .sg_tablesize = SG_ALL, - .cmd_per_lun = UFSHCD_CMD_PER_LUN, - .can_queue = UFSHCD_CAN_QUEUE, .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, - .max_sectors = (1 << 20) / SECTOR_SIZE, /* 1 MiB */ + .max_sectors = SZ_1M / SECTOR_SIZE, .max_host_blocked = 1, .track_queue_depth = 1, + .skip_settle_delay = 1, .sdev_groups = ufshcd_driver_groups, - .dma_boundary = PAGE_SIZE - 1, - .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS, }; static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, @@ -8533,7 +9471,6 @@ static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) int ret = 0; struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; - unsigned long flags; ktime_t start = ktime_get(); bool clk_state_changed = false; @@ -8575,22 +9512,23 @@ static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) if (ret) return ret; + if (!ufshcd_is_clkscaling_supported(hba)) + ufshcd_pm_qos_update(hba, on); out: if (ret) { list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) clk_disable_unprepare(clki->clk); } - } else if (!ret && on) { - spin_lock_irqsave(hba->host->host_lock, flags); - hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + } else if (!ret && on && hba->clk_gating.is_initialized) { + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) + hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); - spin_unlock_irqrestore(hba->host->host_lock, flags); } if (clk_state_changed) - trace_ufshcd_profile_clk_gating(dev_name(hba->dev), + trace_ufshcd_profile_clk_gating(hba, (on ? "on" : "off"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); return ret; @@ -8652,6 +9590,17 @@ static int ufshcd_init_clocks(struct ufs_hba *hba) dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, clki->name, clk_get_rate(clki->clk)); } + + /* Set Max. frequency for all clocks */ + if (hba->use_pm_opp) { + ret = ufshcd_opp_set_rate(hba, ULONG_MAX); + if (ret) { + dev_err(hba->dev, "%s: failed to set OPP: %d", __func__, + ret); + goto out; + } + } + out: return ret; } @@ -8665,8 +9614,9 @@ static int ufshcd_variant_hba_init(struct ufs_hba *hba) err = ufshcd_vops_init(hba); if (err) - dev_err(hba->dev, "%s: variant %s init failed err %d\n", - __func__, ufshcd_get_var_name(hba), err); + dev_err_probe(hba->dev, err, + "%s: variant %s init failed with err %d\n", + __func__, ufshcd_get_var_name(hba), err); out: return err; } @@ -8722,6 +9672,7 @@ static int ufshcd_hba_init(struct ufs_hba *hba) goto out_disable_vreg; ufs_debugfs_hba_init(hba); + ufs_fault_inject_hba_init(hba); hba->is_powered = true; goto out; @@ -8739,6 +9690,7 @@ out: static void ufshcd_hba_exit(struct ufs_hba *hba) { if (hba->is_powered) { + ufshcd_pm_qos_exit(hba); ufshcd_exit_clk_scaling(hba); ufshcd_exit_clk_gating(hba); if (hba->eh_wq) @@ -8757,34 +9709,26 @@ static int ufshcd_execute_start_stop(struct scsi_device *sdev, enum ufs_dev_pwr_mode pwr_mode, struct scsi_sense_hdr *sshdr) { - unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; - struct request *req; - struct scsi_cmnd *scmd; - int ret; - - req = scsi_alloc_request(sdev->request_queue, REQ_OP_DRV_IN, - BLK_MQ_REQ_PM); - if (IS_ERR(req)) - return PTR_ERR(req); - - scmd = blk_mq_rq_to_pdu(req); - scmd->cmd_len = COMMAND_SIZE(cdb[0]); - memcpy(scmd->cmnd, cdb, scmd->cmd_len); - scmd->allowed = 0/*retries*/; - scmd->flags |= SCMD_FAIL_IF_RECOVERING; - req->timeout = 1 * HZ; - req->rq_flags |= RQF_PM | RQF_QUIET; - - blk_execute_rq(req, /*at_head=*/true); - - if (sshdr) - scsi_normalize_sense(scmd->sense_buffer, scmd->sense_len, - sshdr); - ret = scmd->result; - - blk_mq_free_request(req); + const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; + struct scsi_failure failure_defs[] = { + { + .allowed = 2, + .result = SCMD_FAILURE_RESULT_ANY, + }, + }; + struct scsi_failures failures = { + .failure_definitions = failure_defs, + }; + const struct scsi_exec_args args = { + .failures = &failures, + .sshdr = sshdr, + .req_flags = BLK_MQ_REQ_PM, + .scmd_flags = SCMD_FAIL_IF_RECOVERING, + }; - return ret; + return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL, + /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0, + &args); } /** @@ -8793,8 +9737,8 @@ static int ufshcd_execute_start_stop(struct scsi_device *sdev, * @hba: per adapter instance * @pwr_mode: device power mode to set * - * Returns 0 if requested power mode is set successfully - * Returns < 0 if failed to set the requested power mode + * Return: 0 if requested power mode is set successfully; + * < 0 if failed to set the requested power mode. */ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, enum ufs_dev_pwr_mode pwr_mode) @@ -8802,7 +9746,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, struct scsi_sense_hdr sshdr; struct scsi_device *sdp; unsigned long flags; - int ret, retries; + int ret; spin_lock_irqsave(hba->host->host_lock, flags); sdp = hba->ufs_device_wlun; @@ -8828,15 +9772,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, * callbacks hence set the RQF_PM flag so that it doesn't resume the * already suspended childs. */ - for (retries = 3; retries > 0; --retries) { - ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); - /* - * scsi_execute() only returns a negative value if the request - * queue is dying. - */ - if (ret <= 0) - break; - } + ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); if (ret) { sdev_printk(KERN_WARNING, sdp, "START_STOP failed for power mode: %d, result %x\n", @@ -8956,11 +9892,11 @@ static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) } /* - * Some UFS devices require delay after VCC power rail is turned-off. + * All UFS devices require delay after VCC power rail is turned-off. */ - if (vcc_off && hba->vreg_info.vcc && - hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM) - usleep_range(5000, 5100); + if (vcc_off && hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on) + usleep_range(hba->vcc_off_delay_us, + hba->vcc_off_delay_us + 100); } #ifdef CONFIG_PM @@ -9024,13 +9960,11 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) req_link_state = UIC_LINK_OFF_STATE; } - ufshpb_suspend(hba); - /* * If we can't transition into any of the low power modes * just gate the clocks. */ - ufshcd_hold(hba, false); + ufshcd_hold(hba); hba->clk_gating.is_suspended = true; if (ufshcd_is_clkscaling_supported(hba)) @@ -9047,7 +9981,10 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) /* UFS device & link must be active before we enter in this function */ if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { - ret = -EINVAL; + /* Wait err handler finish or trigger err recovery */ + if (!ufshcd_eh_in_progress(hba)) + ufshcd_force_error_recovery(hba); + ret = -EBUSY; goto enable_scaling; } @@ -9058,9 +9995,17 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) * allow background operations if bkops status shows * that performance might be impacted. */ - ret = ufshcd_urgent_bkops(hba); - if (ret) + ret = ufshcd_bkops_ctrl(hba); + if (ret) { + /* + * If return err in suspend flow, IO will hang. + * Trigger error handler and break suspend for + * error recovery. + */ + ufshcd_force_error_recovery(hba); + ret = -EBUSY; goto enable_scaling; + } } else { /* make sure that auto bkops is disabled */ ufshcd_disable_auto_bkops(hba); @@ -9132,6 +10077,8 @@ vops_suspend: ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); if (ret) goto set_link_active; + + cancel_delayed_work_sync(&hba->ufs_rtc_update_work); goto out; set_link_active: @@ -9171,7 +10118,6 @@ out: ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); hba->clk_gating.is_suspended = false; ufshcd_release(hba); - ufshpb_resume(hba); } hba->pm_op_in_progress = false; return ret; @@ -9226,6 +10172,9 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE); if (ret) goto set_old_link_state; + ufshcd_set_timestamp_attr(hba); + schedule_delayed_work(&hba->ufs_rtc_update_work, + msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); } if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) @@ -9235,7 +10184,7 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) * If BKOPs operations are urgently needed at this moment then * keep auto-bkops enabled or else disable it. */ - ufshcd_urgent_bkops(hba); + ufshcd_bkops_ctrl(hba); if (hba->ee_usr_mask) ufshcd_write_ee_control(hba); @@ -9248,10 +10197,8 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); } - /* Enable Auto-Hibernate if configured */ - ufshcd_auto_hibern8_enable(hba); + ufshcd_configure_auto_hibern8(hba); - ufshpb_resume(hba); goto out; set_old_link_state: @@ -9281,7 +10228,7 @@ static int ufshcd_wl_runtime_suspend(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); - trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret, + trace_ufshcd_wl_runtime_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9301,7 +10248,7 @@ static int ufshcd_wl_runtime_resume(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); - trace_ufshcd_wl_runtime_resume(dev_name(dev), ret, + trace_ufshcd_wl_runtime_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9333,7 +10280,7 @@ static int ufshcd_wl_suspend(struct device *dev) out: if (!ret) hba->is_sys_suspended = true; - trace_ufshcd_wl_suspend(dev_name(dev), ret, + trace_ufshcd_wl_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9356,7 +10303,7 @@ static int ufshcd_wl_resume(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); out: - trace_ufshcd_wl_resume(dev_name(dev), ret, + trace_ufshcd_wl_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); if (!ret) @@ -9367,34 +10314,14 @@ out: } #endif -static void ufshcd_wl_shutdown(struct device *dev) -{ - struct scsi_device *sdev = to_scsi_device(dev); - struct ufs_hba *hba; - - hba = shost_priv(sdev->host); - - down(&hba->host_sem); - hba->shutting_down = true; - up(&hba->host_sem); - - /* Turn on everything while shutting down */ - ufshcd_rpm_get_sync(hba); - scsi_device_quiesce(sdev); - shost_for_each_device(sdev, hba->host) { - if (sdev == hba->ufs_device_wlun) - continue; - scsi_device_quiesce(sdev); - } - __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); -} - /** * ufshcd_suspend - helper function for suspend operations * @hba: per adapter instance * * This function will put disable irqs, turn off clocks * and set vreg and hba-vreg in lpm mode. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_suspend(struct ufs_hba *hba) { @@ -9414,13 +10341,14 @@ static int ufshcd_suspend(struct ufs_hba *hba) } if (ufshcd_is_clkgating_allowed(hba)) { hba->clk_gating.state = CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); } ufshcd_vreg_set_lpm(hba); /* Put the host controller in low power mode if possible */ ufshcd_hba_vreg_set_lpm(hba); + ufshcd_pm_qos_update(hba, false); return ret; } @@ -9432,7 +10360,7 @@ static int ufshcd_suspend(struct ufs_hba *hba) * This function basically turns on the regulators, clocks and * irqs of the hba. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ static int ufshcd_resume(struct ufs_hba *hba) { @@ -9453,6 +10381,7 @@ static int ufshcd_resume(struct ufs_hba *hba) /* enable the host irq as host controller would be active soon */ ufshcd_enable_irq(hba); + goto out; disable_vreg: @@ -9472,7 +10401,7 @@ out: * Executed before putting the system into a sleep state in which the contents * of main memory are preserved. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_system_suspend(struct device *dev) { @@ -9485,7 +10414,7 @@ int ufshcd_system_suspend(struct device *dev) ret = ufshcd_suspend(hba); out: - trace_ufshcd_system_suspend(dev_name(hba->dev), ret, + trace_ufshcd_system_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -9499,7 +10428,7 @@ EXPORT_SYMBOL(ufshcd_system_suspend); * Executed after waking the system up from a sleep state in which the contents * of main memory were preserved. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_system_resume(struct device *dev) { @@ -9513,7 +10442,7 @@ int ufshcd_system_resume(struct device *dev) ret = ufshcd_resume(hba); out: - trace_ufshcd_system_resume(dev_name(hba->dev), ret, + trace_ufshcd_system_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9529,7 +10458,7 @@ EXPORT_SYMBOL(ufshcd_system_resume); * * Check the description of ufshcd_suspend() function for more details. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_runtime_suspend(struct device *dev) { @@ -9539,7 +10468,7 @@ int ufshcd_runtime_suspend(struct device *dev) ret = ufshcd_suspend(hba); - trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret, + trace_ufshcd_runtime_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -9555,6 +10484,8 @@ EXPORT_SYMBOL(ufshcd_runtime_suspend); * * 1. Turn on all the controller related clocks * 2. Turn ON VCC rail + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_runtime_resume(struct device *dev) { @@ -9564,7 +10495,7 @@ int ufshcd_runtime_resume(struct device *dev) ret = ufshcd_resume(hba); - trace_ufshcd_runtime_resume(dev_name(hba->dev), ret, + trace_ufshcd_runtime_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -9572,25 +10503,36 @@ int ufshcd_runtime_resume(struct device *dev) EXPORT_SYMBOL(ufshcd_runtime_resume); #endif /* CONFIG_PM */ -/** - * ufshcd_shutdown - shutdown routine - * @hba: per adapter instance - * - * This function would turn off both UFS device and UFS hba - * regulators. It would also disable clocks. - * - * Returns 0 always to allow force shutdown even in case of errors. - */ -int ufshcd_shutdown(struct ufs_hba *hba) +static void ufshcd_wl_shutdown(struct device *dev) { + struct scsi_device *sdev = to_scsi_device(dev); + struct ufs_hba *hba = shost_priv(sdev->host); + + down(&hba->host_sem); + hba->shutting_down = true; + up(&hba->host_sem); + + /* Turn on everything while shutting down */ + ufshcd_rpm_get_sync(hba); + scsi_device_quiesce(sdev); + shost_for_each_device(sdev, hba->host) { + if (sdev == hba->ufs_device_wlun) + continue; + mutex_lock(&sdev->state_mutex); + scsi_device_set_state(sdev, SDEV_OFFLINE); + mutex_unlock(&sdev->state_mutex); + } + __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); + + /* + * Next, turn off the UFS controller and the UFS regulators. Disable + * clocks. + */ if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba)) ufshcd_suspend(hba); hba->is_powered = false; - /* allow force shutdown even in case of errors */ - return 0; } -EXPORT_SYMBOL(ufshcd_shutdown); /** * ufshcd_remove - de-allocate SCSI host and host memory space @@ -9603,12 +10545,14 @@ void ufshcd_remove(struct ufs_hba *hba) ufshcd_rpm_get_sync(hba); ufs_hwmon_remove(hba); ufs_bsg_remove(hba); - ufshpb_remove(hba); + ufs_rpmb_remove(hba); ufs_sysfs_remove_nodes(hba->dev); + cancel_delayed_work_sync(&hba->ufs_rtc_update_work); blk_mq_destroy_queue(hba->tmf_queue); blk_put_queue(hba->tmf_queue); blk_mq_free_tag_set(&hba->tmf_tag_set); - scsi_remove_host(hba->host); + if (hba->scsi_host_added) + scsi_remove_host(hba->host); /* disable interrupts */ ufshcd_disable_intr(hba, hba->intr_mask); ufshcd_hba_stop(hba); @@ -9616,25 +10560,64 @@ void ufshcd_remove(struct ufs_hba *hba) } EXPORT_SYMBOL_GPL(ufshcd_remove); -/** - * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA) - * @hba: pointer to Host Bus Adapter (HBA) - */ -void ufshcd_dealloc_host(struct ufs_hba *hba) +#ifdef CONFIG_PM_SLEEP +int ufshcd_system_freeze(struct device *dev) +{ + + return ufshcd_system_suspend(dev); + +} +EXPORT_SYMBOL_GPL(ufshcd_system_freeze); + +int ufshcd_system_restore(struct device *dev) +{ + + struct ufs_hba *hba = dev_get_drvdata(dev); + int ret; + + ret = ufshcd_system_resume(dev); + if (ret) + return ret; + + /* Configure UTRL and UTMRL base address registers */ + ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr), + REG_UTP_TRANSFER_REQ_LIST_BASE_L); + ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr), + REG_UTP_TRANSFER_REQ_LIST_BASE_H); + ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr), + REG_UTP_TASK_REQ_LIST_BASE_L); + ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr), + REG_UTP_TASK_REQ_LIST_BASE_H); + /* + * Make sure that UTRL and UTMRL base address registers + * are updated with the latest queue addresses. Only after + * updating these addresses, we can queue the new commands. + */ + ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H); + + return 0; + +} +EXPORT_SYMBOL_GPL(ufshcd_system_restore); + +int ufshcd_system_thaw(struct device *dev) { - scsi_host_put(hba->host); + return ufshcd_system_resume(dev); } -EXPORT_SYMBOL_GPL(ufshcd_dealloc_host); +EXPORT_SYMBOL_GPL(ufshcd_system_thaw); +#endif /* CONFIG_PM_SLEEP */ /** * ufshcd_set_dma_mask - Set dma mask based on the controller * addressing capability * @hba: per adapter instance * - * Returns 0 for success, non-zero for failure + * Return: 0 for success, non-zero for failure. */ static int ufshcd_set_dma_mask(struct ufs_hba *hba) { + if (hba->vops && hba->vops->set_dma_mask) + return hba->vops->set_dma_mask(hba); if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) return 0; @@ -9643,10 +10626,25 @@ static int ufshcd_set_dma_mask(struct ufs_hba *hba) } /** + * ufshcd_devres_release - devres cleanup handler, invoked during release of + * hba->dev + * @host: pointer to SCSI host + */ +static void ufshcd_devres_release(void *host) +{ + scsi_host_put(host); +} + +/** * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) * @dev: pointer to device handle * @hba_handle: driver private handle - * Returns 0 on success, non-zero value on failure + * + * Return: 0 on success, non-zero value on failure. + * + * NOTE: There is no corresponding ufshcd_dealloc_host() because this function + * keeps track of its allocations using devres and deallocates everything on + * device removal automatically. */ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) { @@ -9668,12 +10666,19 @@ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) err = -ENOMEM; goto out_error; } + + err = devm_add_action_or_reset(dev, ufshcd_devres_release, + host); + if (err) + return err; + host->nr_maps = HCTX_TYPE_POLL + 1; hba = shost_priv(host); hba->host = host; hba->dev = dev; hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL; hba->nop_out_timeout = NOP_OUT_TIMEOUT; + ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry)); INIT_LIST_HEAD(&hba->clk_list_head); spin_lock_init(&hba->outstanding_lock); @@ -9696,19 +10701,89 @@ static const struct blk_mq_ops ufshcd_tmf_ops = { .queue_rq = ufshcd_queue_tmf, }; +static int ufshcd_add_scsi_host(struct ufs_hba *hba) +{ + int err; + + WARN_ON_ONCE(!hba->host->can_queue); + WARN_ON_ONCE(!hba->host->cmd_per_lun); + + if (is_mcq_supported(hba)) { + ufshcd_mcq_enable(hba); + err = ufshcd_alloc_mcq(hba); + if (!err) { + ufshcd_config_mcq(hba); + } else { + /* Continue with SDB mode */ + ufshcd_mcq_disable(hba); + use_mcq_mode = false; + dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", + err); + } + } + if (!is_mcq_supported(hba) && !hba->lsdb_sup) { + dev_err(hba->dev, + "%s: failed to initialize (legacy doorbell mode not supported)\n", + __func__); + return -EINVAL; + } + + err = scsi_add_host(hba->host, hba->dev); + if (err) { + dev_err(hba->dev, "scsi_add_host failed\n"); + return err; + } + hba->scsi_host_added = true; + + hba->tmf_tag_set = (struct blk_mq_tag_set) { + .nr_hw_queues = 1, + .queue_depth = hba->nutmrs, + .ops = &ufshcd_tmf_ops, + }; + err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); + if (err < 0) + goto remove_scsi_host; + hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL); + if (IS_ERR(hba->tmf_queue)) { + err = PTR_ERR(hba->tmf_queue); + goto free_tmf_tag_set; + } + hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, + sizeof(*hba->tmf_rqs), GFP_KERNEL); + if (!hba->tmf_rqs) { + err = -ENOMEM; + goto free_tmf_queue; + } + + return 0; + +free_tmf_queue: + blk_mq_destroy_queue(hba->tmf_queue); + blk_put_queue(hba->tmf_queue); + +free_tmf_tag_set: + blk_mq_free_tag_set(&hba->tmf_tag_set); + +remove_scsi_host: + if (hba->scsi_host_added) + scsi_remove_host(hba->host); + + return err; +} + /** * ufshcd_init - Driver initialization routine * @hba: per-adapter instance * @mmio_base: base register address * @irq: Interrupt line of device - * Returns 0 on success, non-zero value on failure + * + * Return: 0 on success; < 0 on failure. */ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) { int err; struct Scsi_Host *host = hba->host; struct device *dev = hba->dev; - char eh_wq_name[sizeof("ufs_eh_wq_00")]; /* * dev_set_drvdata() must be called before any callbacks are registered @@ -9728,6 +10803,37 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) hba->irq = irq; hba->vps = &ufs_hba_vps; + /* + * Initialize clk_gating.lock early since it is being used in + * ufshcd_setup_clocks() + */ + spin_lock_init(&hba->clk_gating.lock); + + /* Initialize mutex for PM QoS request synchronization */ + mutex_init(&hba->pm_qos_mutex); + + /* + * Set the default power management level for runtime and system PM. + * Host controller drivers can override them in their + * 'ufs_hba_variant_ops::init' callback. + * + * Default power saving mode is to keep UFS link in Hibern8 state + * and UFS device in sleep state. + */ + hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + + /* + * Most ufs devices require 1ms delay after vcc is powered off before + * it can be powered on again. Set the default to 2ms. The platform + * drivers can override this setting as needed. + */ + hba->vcc_off_delay_us = 2000; + err = ufshcd_hba_init(hba); if (err) goto out_error; @@ -9760,19 +10866,27 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_host_memory_configure(hba); host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; - host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED; + /* + * Set the queue depth for WLUNs. ufs_get_device_desc() will increase + * host->cmd_per_lun to a larger value. + */ + host->cmd_per_lun = 1; host->max_id = UFSHCD_MAX_ID; host->max_lun = UFS_MAX_LUNS; host->max_channel = UFSHCD_MAX_CHANNEL; host->unique_id = host->host_no; host->max_cmd_len = UFS_CDB_SIZE; + host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); + + /* Use default RPM delay if host not set */ + if (host->rpm_autosuspend_delay == 0) + host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; hba->max_pwr_info.is_valid = false; /* Initialize work queues */ - snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d", - hba->host->host_no); - hba->eh_wq = create_singlethread_workqueue(eh_wq_name); + hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM, + hba->host->host_no); if (!hba->eh_wq) { dev_err(hba->dev, "%s: failed to create eh workqueue\n", __func__); @@ -9794,6 +10908,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) mutex_init(&hba->ee_ctrl_mutex); mutex_init(&hba->wb_mutex); + init_rwsem(&hba->clk_scaling_lock); ufshcd_init_clk_gating(hba); @@ -9812,10 +10927,11 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) * Make sure that UFS interrupts are disabled and any pending interrupt * status is cleared before registering UFS interrupt handler. */ - mb(); + ufshcd_readl(hba, REG_INTERRUPT_ENABLE); /* IRQ registration */ - err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); + err = devm_request_threaded_irq(dev, irq, ufshcd_intr, ufshcd_threaded_intr, + IRQF_ONESHOT | IRQF_SHARED, UFSHCD, hba); if (err) { dev_err(hba->dev, "request irq failed\n"); goto out_disable; @@ -9823,33 +10939,6 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) hba->is_irq_enabled = true; } - err = scsi_add_host(host, hba->dev); - if (err) { - dev_err(hba->dev, "scsi_add_host failed\n"); - goto out_disable; - } - - hba->tmf_tag_set = (struct blk_mq_tag_set) { - .nr_hw_queues = 1, - .queue_depth = hba->nutmrs, - .ops = &ufshcd_tmf_ops, - .flags = BLK_MQ_F_NO_SCHED, - }; - err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); - if (err < 0) - goto out_remove_scsi_host; - hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set); - if (IS_ERR(hba->tmf_queue)) { - err = PTR_ERR(hba->tmf_queue); - goto free_tmf_tag_set; - } - hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, - sizeof(*hba->tmf_rqs), GFP_KERNEL); - if (!hba->tmf_rqs) { - err = -ENOMEM; - goto free_tmf_queue; - } - /* Reset the attached device */ ufshcd_device_reset(hba); @@ -9861,23 +10950,11 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) dev_err(hba->dev, "Host controller enable failed\n"); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); - goto free_tmf_queue; + goto out_disable; } - /* - * Set the default power management level for runtime and system PM. - * Default power saving mode is to keep UFS link in Hibern8 state - * and UFS device in sleep state. - */ - hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( - UFS_SLEEP_PWR_MODE, - UIC_LINK_HIBERN8_STATE); - hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( - UFS_SLEEP_PWR_MODE, - UIC_LINK_HIBERN8_STATE); - - INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, - ufshcd_rpm_dev_flush_recheck_work); + INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); + INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work); /* Set the default auto-hiberate idle timer value to 150 ms */ if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) { @@ -9885,9 +10962,13 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); } + err = ufshcd_add_scsi_host(hba); + if (err) + goto out_disable; + /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); - atomic_set(&hba->scsi_block_reqs_cnt, 0); + /* * We are assuming that device wasn't put in sleep/power-down * state exclusively during the boot stage before kernel. @@ -9896,24 +10977,57 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) */ ufshcd_set_ufs_dev_active(hba); - async_schedule(ufshcd_async_scan, hba); + /* Initialize hba, detect and initialize UFS device */ + ktime_t probe_start = ktime_get(); + + hba->ufshcd_state = UFSHCD_STATE_RESET; + + err = ufshcd_link_startup(hba); + if (err) + goto out_disable; + + if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) + goto initialized; + + /* Debug counters initialization */ + ufshcd_clear_dbg_ufs_stats(hba); + + /* UniPro link is active now */ + ufshcd_set_link_active(hba); + + /* Verify device initialization by sending NOP OUT UPIU */ + err = ufshcd_verify_dev_init(hba); + if (err) + goto out_disable; + + /* Initiate UFS initialization, and waiting until completion */ + err = ufshcd_complete_dev_init(hba); + if (err) + goto out_disable; + + err = ufshcd_device_params_init(hba); + if (err) + goto out_disable; + + err = ufshcd_post_device_init(hba); + +initialized: + ufshcd_process_probe_result(hba, probe_start, err); + if (err) + goto out_disable; + ufs_sysfs_add_nodes(hba->dev); + async_schedule(ufshcd_async_scan, hba); device_enable_async_suspend(dev); + ufshcd_pm_qos_init(hba); return 0; -free_tmf_queue: - blk_mq_destroy_queue(hba->tmf_queue); - blk_put_queue(hba->tmf_queue); -free_tmf_tag_set: - blk_mq_free_tag_set(&hba->tmf_tag_set); -out_remove_scsi_host: - scsi_remove_host(hba->host); out_disable: hba->is_irq_enabled = false; ufshcd_hba_exit(hba); out_error: - return err; + return err > 0 ? -EIO : err; } EXPORT_SYMBOL_GPL(ufshcd_init); @@ -10029,6 +11143,53 @@ static const struct dev_pm_ops ufshcd_wl_pm_ops = { SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) }; +static void ufshcd_check_header_layout(void) +{ + /* + * gcc compilers before version 10 cannot do constant-folding for + * sub-byte bitfields. Hence skip the layout checks for gcc 9 and + * before. + */ + if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000) + return; + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .cci = 3})[0] != 3); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .ehs_length = 2})[1] != 2); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .enable_crypto = 1})[2] + != 0x80); + + BUILD_BUG_ON((((u8 *)&(struct request_desc_header){ + .command_type = 5, + .data_direction = 3, + .interrupt = 1, + })[3]) != ((5 << 4) | (3 << 1) | 1)); + + BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ + .dunl = cpu_to_le32(0xdeadbeef)})[1] != + cpu_to_le32(0xdeadbeef)); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .ocs = 4})[8] != 4); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .cds = 5})[9] != 5); + + BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ + .dunu = cpu_to_le32(0xbadcafe)})[3] != + cpu_to_le32(0xbadcafe)); + + BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ + .iid = 0xf })[4] != 0xf0); + + BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ + .command_set_type = 0xf })[4] != 0xf); +} + /* * ufs_dev_wlun_template - describes ufs device wlun * ufs-device wlun - used to send pm commands @@ -10042,7 +11203,6 @@ static const struct dev_pm_ops ufshcd_wl_pm_ops = { static struct scsi_driver ufs_dev_wlun_template = { .gendrv = { .name = "ufs_device_wlun", - .owner = THIS_MODULE, .probe = ufshcd_wl_probe, .remove = ufshcd_wl_remove, .pm = &ufshcd_wl_pm_ops, @@ -10054,10 +11214,7 @@ static int __init ufshcd_core_init(void) { int ret; - /* Verify that there are no gaps in struct utp_transfer_cmd_desc. */ - static_assert(sizeof(struct utp_transfer_cmd_desc) == - 2 * ALIGNED_UPIU_SIZE + - SG_ALL * sizeof(struct ufshcd_sg_entry)); + ufshcd_check_header_layout(); ufs_debugfs_init(); @@ -10079,4 +11236,5 @@ module_exit(ufshcd_core_exit); MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); MODULE_DESCRIPTION("Generic UFS host controller driver Core"); +MODULE_SOFTDEP("pre: governor_simpleondemand"); MODULE_LICENSE("GPL"); |