diff options
Diffstat (limited to 'drivers/ufs/core/ufshcd.c')
| -rw-r--r-- | drivers/ufs/core/ufshcd.c | 4055 |
1 files changed, 2307 insertions, 1748 deletions
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 129446775796..040a0ceb170a 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -20,12 +20,15 @@ #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/module.h> +#include <linux/pm_opp.h> #include <linux/regulator/consumer.h> #include <linux/sched/clock.h> +#include <linux/iopoll.h> #include <scsi/scsi_cmnd.h> #include <scsi/scsi_dbg.h> #include <scsi/scsi_driver.h> #include <scsi/scsi_eh.h> +#include <scsi/scsi_tcq.h> #include "ufshcd-priv.h" #include <ufs/ufs_quirks.h> #include <ufs/unipro.h> @@ -34,24 +37,20 @@ #include "ufs-fault-injection.h" #include "ufs_bsg.h" #include "ufshcd-crypto.h" -#include "ufshpb.h" -#include <asm/unaligned.h> +#include <linux/unaligned.h> #define CREATE_TRACE_POINTS -#include <trace/events/ufs.h> +#include "ufs_trace.h" #define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ UTP_TASK_REQ_COMPL |\ UFSHCD_ERROR_MASK) -#define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\ - UFSHCD_ERROR_MASK |\ - MCQ_CQ_EVENT_STATUS) - - /* UIC command timeout, unit: ms */ -#define UIC_CMD_TIMEOUT 500 - +enum { + UIC_CMD_TIMEOUT_DEFAULT = 500, + UIC_CMD_TIMEOUT_MAX = 5000, +}; /* NOP OUT retries waiting for NOP IN response */ #define NOP_OUT_RETRIES 10 /* Timeout after 50 msecs if NOP OUT hangs without response */ @@ -60,7 +59,11 @@ /* Query request retries */ #define QUERY_REQ_RETRIES 3 /* Query request timeout */ -#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */ +enum { + QUERY_REQ_TIMEOUT_MIN = 1, + QUERY_REQ_TIMEOUT_DEFAULT = 1500, + QUERY_REQ_TIMEOUT_MAX = 30000 +}; /* Advanced RPMB request timeout */ #define ADVANCED_RPMB_REQ_TIMEOUT 3000 /* 3 seconds */ @@ -98,6 +101,12 @@ /* Polling time to wait for fDeviceInit */ #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */ +/* Default RTC update every 10 seconds */ +#define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC) + +/* bMaxNumOfRTT is equal to two after device manufacturing */ +#define DEFAULT_MAX_NUM_RTT 2 + /* UFSHC 4.0 compliant HC support this mode. */ static bool use_mcq_mode = true; @@ -109,6 +118,40 @@ static bool is_mcq_supported(struct ufs_hba *hba) module_param(use_mcq_mode, bool, 0644); MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default"); +static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT; + +static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT, + UIC_CMD_TIMEOUT_MAX); +} + +static const struct kernel_param_ops uic_cmd_timeout_ops = { + .set = uic_cmd_timeout_set, + .get = param_get_uint, +}; + +module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644); +MODULE_PARM_DESC(uic_cmd_timeout, + "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 5 seconds inclusively"); + +static unsigned int dev_cmd_timeout = QUERY_REQ_TIMEOUT_DEFAULT; + +static int dev_cmd_timeout_set(const char *val, const struct kernel_param *kp) +{ + return param_set_uint_minmax(val, kp, QUERY_REQ_TIMEOUT_MIN, + QUERY_REQ_TIMEOUT_MAX); +} + +static const struct kernel_param_ops dev_cmd_timeout_ops = { + .set = dev_cmd_timeout_set, + .get = param_get_uint, +}; + +module_param_cb(dev_cmd_timeout, &dev_cmd_timeout_ops, &dev_cmd_timeout, 0644); +MODULE_PARM_DESC(dev_cmd_timeout, + "UFS Device command timeout in milliseconds. Defaults to 1.5s. Supported values range from 1ms to 30 seconds inclusively"); + #define ufshcd_toggle_vreg(_dev, _vreg, _on) \ ({ \ int _ret; \ @@ -157,8 +200,6 @@ EXPORT_SYMBOL_GPL(ufshcd_dump_regs); enum { UFSHCD_MAX_CHANNEL = 0, UFSHCD_MAX_ID = 1, - UFSHCD_CMD_PER_LUN = 32 - UFSHCD_NUM_RESERVED, - UFSHCD_CAN_QUEUE = 32 - UFSHCD_NUM_RESERVED, }; static const char *const ufshcd_state_name[] = { @@ -234,16 +275,27 @@ ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state, return UFS_PM_LVL_0; } +static bool ufshcd_has_pending_tasks(struct ufs_hba *hba) +{ + return hba->outstanding_tasks || hba->active_uic_cmd || + hba->uic_async_done; +} + +static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba) +{ + return scsi_host_busy(hba->host) || ufshcd_has_pending_tasks(hba); +} + static const struct ufs_dev_quirk ufs_fixups[] = { /* UFS cards deviations table */ { .wmanufacturerid = UFS_VENDOR_MICRON, .model = UFS_ANY_MODEL, - .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | - UFS_DEVICE_QUIRK_SWAP_L2P_ENTRY_FOR_HPB_READ }, + .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, { .wmanufacturerid = UFS_VENDOR_SAMSUNG, .model = UFS_ANY_MODEL, .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM | UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE | + UFS_DEVICE_QUIRK_PA_HIBER8TIME | UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS }, { .wmanufacturerid = UFS_VENDOR_SKHYNIX, .model = UFS_ANY_MODEL, @@ -260,6 +312,9 @@ static const struct ufs_dev_quirk ufs_fixups[] = { { .wmanufacturerid = UFS_VENDOR_TOSHIBA, .model = "THGLF2G9D8KBADG", .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE }, + { .wmanufacturerid = UFS_VENDOR_TOSHIBA, + .model = "THGJFJT1E45BATP", + .quirk = UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT }, {} }; @@ -269,14 +324,15 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba); static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd); static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag); static void ufshcd_hba_exit(struct ufs_hba *hba); +static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params); static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params); static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba); static int ufshcd_host_reset_and_restore(struct ufs_hba *hba); static void ufshcd_resume_clkscaling(struct ufs_hba *hba); static void ufshcd_suspend_clkscaling(struct ufs_hba *hba); -static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba); -static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up); +static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, + bool scale_up); static irqreturn_t ufshcd_intr(int irq, void *__hba); static int ufshcd_change_power_mode(struct ufs_hba *hba, struct ufs_pa_layer_attr *pwr_mode); @@ -289,21 +345,51 @@ static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba, static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba); static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba); -static inline void ufshcd_enable_irq(struct ufs_hba *hba) +void ufshcd_enable_irq(struct ufs_hba *hba) { if (!hba->is_irq_enabled) { enable_irq(hba->irq); hba->is_irq_enabled = true; } } +EXPORT_SYMBOL_GPL(ufshcd_enable_irq); -static inline void ufshcd_disable_irq(struct ufs_hba *hba) +void ufshcd_disable_irq(struct ufs_hba *hba) { if (hba->is_irq_enabled) { disable_irq(hba->irq); hba->is_irq_enabled = false; } } +EXPORT_SYMBOL_GPL(ufshcd_disable_irq); + +/** + * ufshcd_enable_intr - enable interrupts + * @hba: per adapter instance + * @intrs: interrupt bits + */ +void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) +{ + u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + u32 new_val = old_val | intrs; + + if (new_val != old_val) + ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); +} + +/** + * ufshcd_disable_intr - disable interrupts + * @hba: per adapter instance + * @intrs: interrupt bits + */ +static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) +{ + u32 old_val = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + u32 new_val = old_val & ~intrs; + + if (new_val != old_val) + ufshcd_writel(hba, new_val, REG_INTERRUPT_ENABLE); +} static void ufshcd_configure_wb(struct ufs_hba *hba) { @@ -318,22 +404,11 @@ static void ufshcd_configure_wb(struct ufs_hba *hba) ufshcd_wb_toggle_buf_flush(hba, true); } -static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba) -{ - if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt)) - scsi_unblock_requests(hba->host); -} - -static void ufshcd_scsi_block_requests(struct ufs_hba *hba) -{ - if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1) - scsi_block_requests(hba->host); -} - -static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, +static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, + struct ufshcd_lrb *lrb, enum ufs_trace_str_t str_t) { - struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr; + struct utp_upiu_req *rq = lrb->ucd_req_ptr; struct utp_upiu_header *header; if (!trace_ufshcd_upiu_enabled()) @@ -342,9 +417,9 @@ static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag, if (str_t == UFS_CMD_SEND) header = &rq->header; else - header = &hba->lrb[tag].ucd_rsp_ptr->header; + header = &lrb->ucd_rsp_ptr->header; - trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb, + trace_ufshcd_upiu(hba, str_t, header, &rq->sc.cdb, UFS_TSF_CDB); } @@ -355,7 +430,7 @@ static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, if (!trace_ufshcd_upiu_enabled()) return; - trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header, + trace_ufshcd_upiu(hba, str_t, &rq_rsp->header, &rq_rsp->qr, UFS_TSF_OSF); } @@ -368,12 +443,12 @@ static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag, return; if (str_t == UFS_TM_SEND) - trace_ufshcd_upiu(dev_name(hba->dev), str_t, + trace_ufshcd_upiu(hba, str_t, &descp->upiu_req.req_header, &descp->upiu_req.input_param1, UFS_TSF_TM_INPUT); else - trace_ufshcd_upiu(dev_name(hba->dev), str_t, + trace_ufshcd_upiu(hba, str_t, &descp->upiu_rsp.rsp_header, &descp->upiu_rsp.output_param1, UFS_TSF_TM_OUTPUT); @@ -393,30 +468,27 @@ static void ufshcd_add_uic_command_trace(struct ufs_hba *hba, else cmd = ufshcd_readl(hba, REG_UIC_COMMAND); - trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd, + trace_ufshcd_uic_command(hba, str_t, cmd, ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1), ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2), ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3)); } -static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, +static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd, enum ufs_trace_str_t str_t) { u64 lba = 0; u8 opcode = 0, group_id = 0; u32 doorbell = 0; u32 intr; - int hwq_id = -1; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; - struct scsi_cmnd *cmd = lrbp->cmd; + u32 hwq_id = 0; struct request *rq = scsi_cmd_to_rq(cmd); + unsigned int tag = rq->tag; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int transfer_len = -1; - if (!cmd) - return; - /* trace UPIU also */ - ufshcd_add_cmd_upiu_trace(hba, tag, str_t); + ufshcd_add_cmd_upiu_trace(hba, lrbp, str_t); if (!trace_ufshcd_command_enabled()) return; @@ -430,7 +502,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len); lba = scsi_get_lba(cmd); if (opcode == WRITE_10) - group_id = lrbp->cmd->cmnd[6]; + group_id = cmd->cmnd[6]; } else if (opcode == UNMAP) { /* * The number of Bytes to be unmapped beginning with the lba. @@ -441,15 +513,15 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag, intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS); - if (is_mcq_enabled(hba)) { + if (hba->mcq_enabled) { struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); hwq_id = hwq->id; } else { doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); } - trace_ufshcd_command(dev_name(hba->dev), str_t, tag, - doorbell, hwq_id, transfer_len, intr, lba, opcode, group_id); + trace_ufshcd_command(cmd->device, hba, str_t, tag, doorbell, hwq_id, + transfer_len, intr, lba, opcode, group_id); } static void ufshcd_print_clk_freqs(struct ufs_hba *hba) @@ -523,18 +595,19 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba) ufshcd_vops_dbg_register_dump(hba); } -static -void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt) +static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd, + bool pr_prdt) { - const struct ufshcd_lrb *lrbp; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; int prdt_length; - lrbp = &hba->lrb[tag]; - - dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", - tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000)); - dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", - tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000)); + if (hba->monitor.enabled) { + dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n", tag, + div_u64(lrbp->issue_time_stamp_local_clock, 1000)); + dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n", tag, + div_u64(lrbp->compl_time_stamp_local_clock, 1000)); + } dev_err(hba->dev, "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n", tag, (u64)lrbp->utrd_dma_addr); @@ -571,7 +644,8 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv) struct Scsi_Host *shost = sdev->host; struct ufs_hba *hba = shost_priv(shost); - ufshcd_print_tr(hba, req->tag, *(bool *)priv); + if (!blk_mq_is_reserved_rq(req)) + ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv); return true; } @@ -603,8 +677,8 @@ static void ufshcd_print_host_state(struct ufs_hba *hba) const struct scsi_device *sdev_ufs = hba->ufs_device_wlun; dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state); - dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n", - hba->outstanding_reqs, hba->outstanding_tasks); + dev_err(hba->dev, "%d outstanding reqs, tasks=0x%lx\n", + scsi_host_busy(hba->host), hba->outstanding_tasks); dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n", hba->saved_err, hba->saved_uic_err); dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n", @@ -618,9 +692,6 @@ static void ufshcd_print_host_state(struct ufs_hba *hba) "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n", div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000), hba->ufs_stats.hibern8_exit_cnt); - dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n", - div_u64(hba->ufs_stats.last_intr_ts, 1000), - hba->ufs_stats.last_intr_status); dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n", hba->eh_flags, hba->req_abort_count); dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n", @@ -677,6 +748,8 @@ static void ufshcd_device_reset(struct ufs_hba *hba) hba->dev_info.wb_enabled = false; hba->dev_info.wb_buf_flush_enabled = false; } + if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) + hba->dev_info.rtc_time_baseline = 0; } if (err != -EOPNOTSUPP) ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err); @@ -703,41 +776,28 @@ EXPORT_SYMBOL_GPL(ufshcd_delay_us); * @interval_us: polling interval in microseconds * @timeout_ms: timeout in milliseconds * - * Return: - * -ETIMEDOUT on error, zero on success. + * Return: -ETIMEDOUT on error, zero on success. */ static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, - u32 val, unsigned long interval_us, - unsigned long timeout_ms) + u32 val, unsigned long interval_us, + unsigned long timeout_ms) { - int err = 0; - unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms); + u32 v; - /* ignore bits that we don't intend to wait on */ - val = val & mask; + val &= mask; /* ignore bits that we don't intend to wait on */ - while ((ufshcd_readl(hba, reg) & mask) != val) { - usleep_range(interval_us, interval_us + 50); - if (time_after(jiffies, timeout)) { - if ((ufshcd_readl(hba, reg) & mask) != val) - err = -ETIMEDOUT; - break; - } - } - - return err; + return read_poll_timeout(ufshcd_readl, v, (v & mask) == val, + interval_us, timeout_ms * 1000, false, hba, reg); } /** * ufshcd_get_intr_mask - Get the interrupt bit mask * @hba: Pointer to adapter instance * - * Returns interrupt bit mask per version + * Return: interrupt bit mask per version */ static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) { - if (hba->ufs_version == ufshci_version(1, 0)) - return INTERRUPT_MASK_ALL_VER_10; if (hba->ufs_version <= ufshci_version(2, 0)) return INTERRUPT_MASK_ALL_VER_11; @@ -748,7 +808,7 @@ static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) * ufshcd_get_ufs_version - Get the UFS version supported by the HBA * @hba: Pointer to adapter instance * - * Returns UFSHCI version supported by the controller + * Return: UFSHCI version supported by the controller */ static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) { @@ -775,7 +835,7 @@ static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) * the host controller * @hba: pointer to adapter instance * - * Returns true if device present, false if no device detected + * Return: true if device present, false if no device detected */ static inline bool ufshcd_is_device_present(struct ufs_hba *hba) { @@ -788,15 +848,16 @@ static inline bool ufshcd_is_device_present(struct ufs_hba *hba) * @cqe: pointer to the completion queue entry * * This function is used to get the OCS field from UTRD - * Returns the OCS field in the UTRD + * + * Return: the OCS field in the UTRD. */ static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp, struct cq_entry *cqe) { if (cqe) - return le32_to_cpu(cqe->status) & MASK_OCS; + return cqe->overall_status & MASK_OCS; - return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS; + return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS; } /** @@ -841,7 +902,7 @@ static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos) * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY * @reg: Register value of host controller status * - * Returns integer, 0 on Success and positive value if failed + * Return: 0 on success; a positive value if failed. */ static inline int ufshcd_get_lists_status(u32 reg) { @@ -853,7 +914,8 @@ static inline int ufshcd_get_lists_status(u32 reg) * @hba: Pointer to adapter instance * * This function gets the result of UIC command completion - * Returns 0 on success, non zero value on error + * + * Return: 0 on success; non-zero value on error. */ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) { @@ -866,7 +928,8 @@ static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) * @hba: Pointer to adapter instance * * This function gets UIC command argument3 - * Returns 0 on success, non zero value on error + * + * Return: 0 on success; non-zero value on error. */ static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) { @@ -876,38 +939,13 @@ static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) /** * ufshcd_get_req_rsp - returns the TR response transaction type * @ucd_rsp_ptr: pointer to response UPIU - */ -static inline int -ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) -{ - return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; -} - -/** - * ufshcd_get_rsp_upiu_result - Get the result from response UPIU - * @ucd_rsp_ptr: pointer to response UPIU * - * This function gets the response status and scsi_status from response UPIU - * Returns the response result code. + * Return: UPIU type. */ -static inline int -ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) -{ - return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; -} - -/* - * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length - * from response UPIU - * @ucd_rsp_ptr: pointer to response UPIU - * - * Return the data segment length. - */ -static inline unsigned int -ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) +static inline enum upiu_response_transaction +ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) { - return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & - MASK_RSP_UPIU_DATA_SEG_LEN; + return ucd_rsp_ptr->header.transaction_code; } /** @@ -917,12 +955,11 @@ ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr) * The function checks if the device raised an exception event indicated in * the Device Information field of response UPIU. * - * Returns true if exception is raised, false otherwise. + * Return: true if exception is raised, false otherwise. */ static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr) { - return be32_to_cpu(ucd_rsp_ptr->header.dword_2) & - MASK_RSP_EXCEPTION_EVENT; + return ucd_rsp_ptr->header.device_information & 1; } /** @@ -993,44 +1030,68 @@ static inline void ufshcd_hba_start(struct ufs_hba *hba) * ufshcd_is_hba_active - Get controller state * @hba: per adapter instance * - * Returns true if and only if the controller is active. + * Return: true if and only if the controller is active. */ -static inline bool ufshcd_is_hba_active(struct ufs_hba *hba) +bool ufshcd_is_hba_active(struct ufs_hba *hba) { return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE; } +EXPORT_SYMBOL_GPL(ufshcd_is_hba_active); -u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba) +/** + * ufshcd_pm_qos_init - initialize PM QoS request + * @hba: per adapter instance + */ +void ufshcd_pm_qos_init(struct ufs_hba *hba) { - /* HCI version 1.0 and 1.1 supports UniPro 1.41 */ - if (hba->ufs_version <= ufshci_version(1, 1)) - return UFS_UNIPRO_VER_1_41; - else - return UFS_UNIPRO_VER_1_6; + guard(mutex)(&hba->pm_qos_mutex); + + if (hba->pm_qos_enabled) + return; + + cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE); + + if (cpu_latency_qos_request_active(&hba->pm_qos_req)) + hba->pm_qos_enabled = true; } -EXPORT_SYMBOL(ufshcd_get_local_unipro_ver); -static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba) +/** + * ufshcd_pm_qos_exit - remove request from PM QoS + * @hba: per adapter instance + */ +void ufshcd_pm_qos_exit(struct ufs_hba *hba) { - /* - * If both host and device support UniPro ver1.6 or later, PA layer - * parameters tuning happens during link startup itself. - * - * We can manually tune PA layer parameters if either host or device - * doesn't support UniPro ver 1.6 or later. But to keep manual tuning - * logic simple, we will only do manual tuning if local unipro version - * doesn't support ver1.6 or later. - */ - return ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6; + guard(mutex)(&hba->pm_qos_mutex); + + if (!hba->pm_qos_enabled) + return; + + cpu_latency_qos_remove_request(&hba->pm_qos_req); + hba->pm_qos_enabled = false; } /** + * ufshcd_pm_qos_update - update PM QoS request + * @hba: per adapter instance + * @on: If True, vote for perf PM QoS mode otherwise power save mode + */ +void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on) +{ + guard(mutex)(&hba->pm_qos_mutex); + + if (!hba->pm_qos_enabled) + return; + + cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE); +} +EXPORT_SYMBOL_GPL(ufshcd_pm_qos_update); + +/** * ufshcd_set_clk_freq - set UFS controller clock frequencies * @hba: per adapter instance * @scale_up: If True, set max possible frequency othewise set low frequency * - * Returns 0 if successful - * Returns < 0 for any other errors + * Return: 0 if successful; < 0 upon failure. */ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) { @@ -1054,7 +1115,7 @@ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) clki->max_freq, ret); break; } - trace_ufshcd_clk_scaling(dev_name(hba->dev), + trace_ufshcd_clk_scaling(hba, "scaled up", clki->name, clki->curr_freq, clki->max_freq); @@ -1072,7 +1133,7 @@ static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up) clki->min_freq, ret); break; } - trace_ufshcd_clk_scaling(dev_name(hba->dev), + trace_ufshcd_clk_scaling(hba, "scaled down", clki->name, clki->curr_freq, clki->min_freq); @@ -1087,33 +1148,97 @@ out: return ret; } +int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table, + struct dev_pm_opp *opp, void *data, + bool scaling_down) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct list_head *head = &hba->clk_list_head; + struct ufs_clk_info *clki; + unsigned long freq; + u8 idx = 0; + int ret; + + list_for_each_entry(clki, head, list) { + if (!IS_ERR_OR_NULL(clki->clk)) { + freq = dev_pm_opp_get_freq_indexed(opp, idx++); + + /* Do not set rate for clocks having frequency as 0 */ + if (!freq) + continue; + + ret = clk_set_rate(clki->clk, freq); + if (ret) { + dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n", + __func__, clki->name, freq, ret); + return ret; + } + + trace_ufshcd_clk_scaling(hba, + (scaling_down ? "scaled down" : "scaled up"), + clki->name, hba->clk_scaling.target_freq, freq); + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks); + +static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq) +{ + struct dev_pm_opp *opp; + int ret; + + opp = dev_pm_opp_find_freq_floor_indexed(hba->dev, + &freq, 0); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(hba->dev, opp); + dev_pm_opp_put(opp); + + return ret; +} + /** * ufshcd_scale_clks - scale up or scale down UFS controller clocks * @hba: per adapter instance + * @freq: frequency to scale * @scale_up: True if scaling up and false if scaling down * - * Returns 0 if successful - * Returns < 0 for any other errors + * Return: 0 if successful; < 0 upon failure. */ -static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up) +static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq, + bool scale_up) { int ret = 0; ktime_t start = ktime_get(); - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE); + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, PRE_CHANGE); if (ret) goto out; - ret = ufshcd_set_clk_freq(hba, scale_up); + if (hba->use_pm_opp) + ret = ufshcd_opp_set_rate(hba, freq); + else + ret = ufshcd_set_clk_freq(hba, scale_up); if (ret) goto out; - ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE); - if (ret) - ufshcd_set_clk_freq(hba, !scale_up); + ret = ufshcd_vops_clk_scale_notify(hba, scale_up, freq, POST_CHANGE); + if (ret) { + if (hba->use_pm_opp) + ufshcd_opp_set_rate(hba, + hba->devfreq->previous_freq); + else + ufshcd_set_clk_freq(hba, !scale_up); + goto out; + } + + ufshcd_pm_qos_update(hba, scale_up); out: - trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + trace_ufshcd_profile_clk_scaling(hba, (scale_up ? "up" : "down"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); return ret; @@ -1122,12 +1247,13 @@ out: /** * ufshcd_is_devfreq_scaling_required - check if scaling is required or not * @hba: per adapter instance + * @freq: frequency to scale * @scale_up: True if scaling up and false if scaling down * - * Returns true if scaling is required, false otherwise. + * Return: true if scaling is required, false otherwise. */ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, - bool scale_up) + unsigned long freq, bool scale_up) { struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; @@ -1135,6 +1261,9 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, if (list_empty(head)) return false; + if (hba->use_pm_opp) + return freq != hba->clk_scaling.target_freq; + list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk)) { if (scale_up && clki->max_freq) { @@ -1161,12 +1290,14 @@ static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba, */ static u32 ufshcd_pending_cmds(struct ufs_hba *hba) { - const struct scsi_device *sdev; + struct scsi_device *sdev; + unsigned long flags; u32 pending = 0; - lockdep_assert_held(hba->host->host_lock); + spin_lock_irqsave(hba->host->host_lock, flags); __shost_for_each_device(sdev, hba->host) - pending += sbitmap_weight(&sdev->budget_map); + pending += scsi_device_busy(sdev); + spin_unlock_irqrestore(hba->host->host_lock, flags); return pending; } @@ -1177,10 +1308,9 @@ static u32 ufshcd_pending_cmds(struct ufs_hba *hba) * * Return: 0 upon success; -EBUSY upon timeout. */ -static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, +static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, u64 wait_timeout_us) { - unsigned long flags; int ret = 0; u32 tm_doorbell; u32 tr_pending; @@ -1188,7 +1318,6 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, ktime_t start; ufshcd_hold(hba); - spin_lock_irqsave(hba->host->host_lock, flags); /* * Wait for all the outstanding tasks/transfer requests. * Verify by checking the doorbell registers are clear. @@ -1209,7 +1338,6 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, break; } - spin_unlock_irqrestore(hba->host->host_lock, flags); io_schedule_timeout(msecs_to_jiffies(20)); if (ktime_to_us(ktime_sub(ktime_get(), start)) > wait_timeout_us) { @@ -1221,7 +1349,6 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, */ do_last_check = true; } - spin_lock_irqsave(hba->host->host_lock, flags); } while (tm_doorbell || tr_pending); if (timeout) { @@ -1231,7 +1358,6 @@ static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba, ret = -EBUSY; } out: - spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_release(hba); return ret; } @@ -1239,17 +1365,26 @@ out: /** * ufshcd_scale_gear - scale up/down UFS gear * @hba: per adapter instance + * @target_gear: target gear to scale to * @scale_up: True for scaling up gear and false for scaling down * - * Returns 0 for success, - * Returns -EBUSY if scaling can't happen at this time - * Returns non-zero for any other errors + * Return: 0 for success; -EBUSY if scaling can't happen at this time; + * non-zero for any other errors. */ -static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) +static int ufshcd_scale_gear(struct ufs_hba *hba, u32 target_gear, bool scale_up) { int ret = 0; struct ufs_pa_layer_attr new_pwr_info; + if (target_gear) { + new_pwr_info = hba->pwr_info; + new_pwr_info.gear_tx = target_gear; + new_pwr_info.gear_rx = target_gear; + + goto config_pwr_mode; + } + + /* Legacy gear scaling, in case vops_freq_to_gear_speed() is not implemented */ if (scale_up) { memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info, sizeof(struct ufs_pa_layer_attr)); @@ -1270,6 +1405,7 @@ static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up) } } +config_pwr_mode: /* check if the power mode needs to be changed or not? */ ret = ufshcd_config_pwr_mode(hba, &new_pwr_info); if (ret) @@ -1294,16 +1430,18 @@ static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us) * make sure that there are no outstanding requests when * clock scaling is in progress */ - ufshcd_scsi_block_requests(hba); + mutex_lock(&hba->host->scan_mutex); + blk_mq_quiesce_tagset(&hba->host->tag_set); mutex_lock(&hba->wb_mutex); down_write(&hba->clk_scaling_lock); if (!hba->clk_scaling.is_allowed || - ufshcd_wait_for_doorbell_clr(hba, timeout_us)) { + ufshcd_wait_for_pending_cmds(hba, timeout_us)) { ret = -EBUSY; up_write(&hba->clk_scaling_lock); mutex_unlock(&hba->wb_mutex); - ufshcd_scsi_unblock_requests(hba); + blk_mq_unquiesce_tagset(&hba->host->tag_set); + mutex_unlock(&hba->host->scan_mutex); goto out; } @@ -1314,62 +1452,69 @@ out: return ret; } -static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up) +static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err) { up_write(&hba->clk_scaling_lock); - /* Enable Write Booster if we have scaled up else disable it */ + /* Enable Write Booster if current gear requires it else disable it */ if (ufshcd_enable_wb_if_scaling_up(hba) && !err) - ufshcd_wb_toggle(hba, scale_up); + ufshcd_wb_toggle(hba, hba->pwr_info.gear_rx >= hba->clk_scaling.wb_gear); mutex_unlock(&hba->wb_mutex); - ufshcd_scsi_unblock_requests(hba); + blk_mq_unquiesce_tagset(&hba->host->tag_set); + mutex_unlock(&hba->host->scan_mutex); ufshcd_release(hba); } /** * ufshcd_devfreq_scale - scale up/down UFS clocks and gear * @hba: per adapter instance + * @freq: frequency to scale * @scale_up: True for scaling up and false for scalin down * - * Returns 0 for success, - * Returns -EBUSY if scaling can't happen at this time - * Returns non-zero for any other errors + * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero + * for any other errors. */ -static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up) +static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq, + bool scale_up) { + u32 old_gear = hba->pwr_info.gear_rx; + u32 new_gear = 0; int ret = 0; + new_gear = ufshcd_vops_freq_to_gear_speed(hba, freq); + ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC); if (ret) return ret; /* scale down the gear before scaling down clocks */ if (!scale_up) { - ret = ufshcd_scale_gear(hba, false); + ret = ufshcd_scale_gear(hba, new_gear, false); if (ret) goto out_unprepare; } - ret = ufshcd_scale_clks(hba, scale_up); + ret = ufshcd_scale_clks(hba, freq, scale_up); if (ret) { if (!scale_up) - ufshcd_scale_gear(hba, true); + ufshcd_scale_gear(hba, old_gear, true); goto out_unprepare; } /* scale up the gear after scaling up clocks */ if (scale_up) { - ret = ufshcd_scale_gear(hba, true); + ret = ufshcd_scale_gear(hba, new_gear, true); if (ret) { - ufshcd_scale_clks(hba, false); + ufshcd_scale_clks(hba, hba->devfreq->previous_freq, + false); goto out_unprepare; } } out_unprepare: - ufshcd_clock_scaling_unprepare(hba, ret, scale_up); + ufshcd_clock_scaling_unprepare(hba, ret); return ret; } @@ -1377,32 +1522,31 @@ static void ufshcd_clk_scaling_suspend_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_scaling.suspend_work); - unsigned long irq_flags; - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (hba->clk_scaling.active_reqs || + hba->clk_scaling.is_suspended) + return; + + hba->clk_scaling.is_suspended = true; + hba->clk_scaling.window_start_t = 0; } - hba->clk_scaling.is_suspended = true; - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - __ufshcd_suspend_clkscaling(hba); + devfreq_suspend_device(hba->devfreq); } static void ufshcd_clk_scaling_resume_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_scaling.resume_work); - unsigned long irq_flags; - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (!hba->clk_scaling.is_suspended) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (!hba->clk_scaling.is_suspended) + return; + hba->clk_scaling.is_suspended = false; } - hba->clk_scaling.is_suspended = false; - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); devfreq_resume_device(hba->devfreq); } @@ -1413,52 +1557,74 @@ static int ufshcd_devfreq_target(struct device *dev, int ret = 0; struct ufs_hba *hba = dev_get_drvdata(dev); ktime_t start; - bool scale_up, sched_clk_scaling_suspend_work = false; + bool scale_up = false, sched_clk_scaling_suspend_work = false; struct list_head *clk_list = &hba->clk_list_head; struct ufs_clk_info *clki; - unsigned long irq_flags; if (!ufshcd_is_clkscaling_supported(hba)) return -EINVAL; - clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); - /* Override with the closest supported frequency */ - *freq = (unsigned long) clk_round_rate(clki->clk, *freq); - spin_lock_irqsave(hba->host->host_lock, irq_flags); - if (ufshcd_eh_in_progress(hba)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - return 0; - } + if (hba->use_pm_opp) { + struct dev_pm_opp *opp; - if (!hba->clk_scaling.active_reqs) - sched_clk_scaling_suspend_work = true; + /* Get the recommended frequency from OPP framework */ + opp = devfreq_recommended_opp(dev, freq, flags); + if (IS_ERR(opp)) + return PTR_ERR(opp); - if (list_empty(clk_list)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - goto out; + dev_pm_opp_put(opp); + } else { + /* Override with the closest supported frequency */ + clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, + list); + *freq = (unsigned long) clk_round_rate(clki->clk, *freq); } - /* Decide based on the rounded-off frequency and update */ - scale_up = *freq == clki->max_freq; - if (!scale_up) - *freq = clki->min_freq; - /* Update the frequency */ - if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) { - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); - ret = 0; - goto out; /* no state change required */ + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (ufshcd_eh_in_progress(hba)) + return 0; + + /* Skip scaling clock when clock scaling is suspended */ + if (hba->clk_scaling.is_suspended) { + dev_warn(hba->dev, "clock scaling is suspended, skip"); + return 0; + } + + if (!hba->clk_scaling.active_reqs) + sched_clk_scaling_suspend_work = true; + + if (list_empty(clk_list)) + goto out; + + /* Decide based on the target or rounded-off frequency and update */ + if (hba->use_pm_opp) + scale_up = *freq > hba->clk_scaling.target_freq; + else + scale_up = *freq == clki->max_freq; + + if (!hba->use_pm_opp && !scale_up) + *freq = clki->min_freq; + + /* Update the frequency */ + if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) { + ret = 0; + goto out; /* no state change required */ + } } - spin_unlock_irqrestore(hba->host->host_lock, irq_flags); start = ktime_get(); - ret = ufshcd_devfreq_scale(hba, scale_up); + ret = ufshcd_devfreq_scale(hba, *freq, scale_up); + if (!ret) + hba->clk_scaling.target_freq = *freq; - trace_ufshcd_profile_clk_scaling(dev_name(hba->dev), + trace_ufshcd_profile_clk_scaling(hba, (scale_up ? "up" : "down"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); out: - if (sched_clk_scaling_suspend_work) + if (sched_clk_scaling_suspend_work && + (!scale_up || hba->clk_scaling.suspend_on_no_request)) queue_work(hba->clk_scaling.workq, &hba->clk_scaling.suspend_work); @@ -1470,9 +1636,6 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, { struct ufs_hba *hba = dev_get_drvdata(dev); struct ufs_clk_scaling *scaling = &hba->clk_scaling; - unsigned long flags; - struct list_head *clk_list = &hba->clk_list_head; - struct ufs_clk_info *clki; ktime_t curr_t; if (!ufshcd_is_clkscaling_supported(hba)) @@ -1480,22 +1643,30 @@ static int ufshcd_devfreq_get_dev_status(struct device *dev, memset(stat, 0, sizeof(*stat)); - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + curr_t = ktime_get(); if (!scaling->window_start_t) goto start_window; - clki = list_first_entry(clk_list, struct ufs_clk_info, list); /* * If current frequency is 0, then the ondemand governor considers * there's no initial frequency set. And it always requests to set * to max. frequency. */ - stat->current_frequency = clki->curr_freq; + if (hba->use_pm_opp) { + stat->current_frequency = hba->clk_scaling.target_freq; + } else { + struct list_head *clk_list = &hba->clk_list_head; + struct ufs_clk_info *clki; + + clki = list_first_entry(clk_list, struct ufs_clk_info, list); + stat->current_frequency = clki->curr_freq; + } + if (scaling->is_busy_started) scaling->tot_busy_t += ktime_us_delta(curr_t, scaling->busy_start_t); - stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t); stat->busy_time = scaling->tot_busy_t; start_window: @@ -1509,7 +1680,7 @@ start_window: scaling->busy_start_t = 0; scaling->is_busy_started = false; } - spin_unlock_irqrestore(hba->host->host_lock, flags); + return 0; } @@ -1524,9 +1695,11 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba) if (list_empty(clk_list)) return 0; - clki = list_first_entry(clk_list, struct ufs_clk_info, list); - dev_pm_opp_add(hba->dev, clki->min_freq, 0); - dev_pm_opp_add(hba->dev, clki->max_freq, 0); + if (!hba->use_pm_opp) { + clki = list_first_entry(clk_list, struct ufs_clk_info, list); + dev_pm_opp_add(hba->dev, clki->min_freq, 0); + dev_pm_opp_add(hba->dev, clki->max_freq, 0); + } ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile, &hba->vps->ondemand_data); @@ -1538,8 +1711,10 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba) ret = PTR_ERR(devfreq); dev_err(hba->dev, "Unable to register with devfreq %d\n", ret); - dev_pm_opp_remove(hba->dev, clki->min_freq); - dev_pm_opp_remove(hba->dev, clki->max_freq); + if (!hba->use_pm_opp) { + dev_pm_opp_remove(hba->dev, clki->min_freq); + dev_pm_opp_remove(hba->dev, clki->max_freq); + } return ret; } @@ -1551,7 +1726,6 @@ static int ufshcd_devfreq_init(struct ufs_hba *hba) static void ufshcd_devfreq_remove(struct ufs_hba *hba) { struct list_head *clk_list = &hba->clk_list_head; - struct ufs_clk_info *clki; if (!hba->devfreq) return; @@ -1559,51 +1733,46 @@ static void ufshcd_devfreq_remove(struct ufs_hba *hba) devfreq_remove_device(hba->devfreq); hba->devfreq = NULL; - clki = list_first_entry(clk_list, struct ufs_clk_info, list); - dev_pm_opp_remove(hba->dev, clki->min_freq); - dev_pm_opp_remove(hba->dev, clki->max_freq); -} - -static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba) -{ - unsigned long flags; + if (!hba->use_pm_opp) { + struct ufs_clk_info *clki; - devfreq_suspend_device(hba->devfreq); - spin_lock_irqsave(hba->host->host_lock, flags); - hba->clk_scaling.window_start_t = 0; - spin_unlock_irqrestore(hba->host->host_lock, flags); + clki = list_first_entry(clk_list, struct ufs_clk_info, list); + dev_pm_opp_remove(hba->dev, clki->min_freq); + dev_pm_opp_remove(hba->dev, clki->max_freq); + } } static void ufshcd_suspend_clkscaling(struct ufs_hba *hba) { - unsigned long flags; bool suspend = false; cancel_work_sync(&hba->clk_scaling.suspend_work); cancel_work_sync(&hba->clk_scaling.resume_work); - spin_lock_irqsave(hba->host->host_lock, flags); - if (!hba->clk_scaling.is_suspended) { - suspend = true; - hba->clk_scaling.is_suspended = true; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (!hba->clk_scaling.is_suspended) { + suspend = true; + hba->clk_scaling.is_suspended = true; + hba->clk_scaling.window_start_t = 0; + } } - spin_unlock_irqrestore(hba->host->host_lock, flags); if (suspend) - __ufshcd_suspend_clkscaling(hba); + devfreq_suspend_device(hba->devfreq); } static void ufshcd_resume_clkscaling(struct ufs_hba *hba) { - unsigned long flags; bool resume = false; - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_scaling.is_suspended) { - resume = true; - hba->clk_scaling.is_suspended = false; + scoped_guard(spinlock_irqsave, &hba->clk_scaling.lock) + { + if (hba->clk_scaling.is_suspended) { + resume = true; + hba->clk_scaling.is_suspended = false; + } } - spin_unlock_irqrestore(hba->host->host_lock, flags); if (resume) devfreq_resume_device(hba->devfreq); @@ -1621,6 +1790,8 @@ static ssize_t ufshcd_clkscale_enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_clk_info *clki; + unsigned long freq; u32 value; int err = 0; @@ -1644,14 +1815,25 @@ static ssize_t ufshcd_clkscale_enable_store(struct device *dev, if (value) { ufshcd_resume_clkscaling(hba); - } else { - ufshcd_suspend_clkscaling(hba); - err = ufshcd_devfreq_scale(hba, true); - if (err) - dev_err(hba->dev, "%s: failed to scale clocks up %d\n", - __func__, err); + goto out_rel; } + clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list); + freq = clki->max_freq; + + ufshcd_suspend_clkscaling(hba); + + if (!ufshcd_is_devfreq_scaling_required(hba, freq, true)) + goto out_rel; + + err = ufshcd_devfreq_scale(hba, freq, true); + if (err) + dev_err(hba->dev, "%s: failed to scale clocks up %d\n", + __func__, err); + else + hba->clk_scaling.target_freq = freq; + +out_rel: ufshcd_release(hba); ufshcd_rpm_put_sync(hba); out: @@ -1678,22 +1860,25 @@ static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba) static void ufshcd_init_clk_scaling(struct ufs_hba *hba) { - char wq_name[sizeof("ufs_clkscaling_00")]; - if (!ufshcd_is_clkscaling_supported(hba)) return; if (!hba->clk_scaling.min_gear) hba->clk_scaling.min_gear = UFS_HS_G1; + if (!hba->clk_scaling.wb_gear) + /* Use intermediate gear speed HS_G3 as the default wb_gear */ + hba->clk_scaling.wb_gear = UFS_HS_G3; + INIT_WORK(&hba->clk_scaling.suspend_work, ufshcd_clk_scaling_suspend_work); INIT_WORK(&hba->clk_scaling.resume_work, ufshcd_clk_scaling_resume_work); - snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d", - hba->host->host_no); - hba->clk_scaling.workq = create_singlethread_workqueue(wq_name); + spin_lock_init(&hba->clk_scaling.lock); + + hba->clk_scaling.workq = alloc_ordered_workqueue( + "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no); hba->clk_scaling.is_initialized = true; } @@ -1712,19 +1897,16 @@ static void ufshcd_exit_clk_scaling(struct ufs_hba *hba) static void ufshcd_ungate_work(struct work_struct *work) { int ret; - unsigned long flags; struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_gating.ungate_work); cancel_delayed_work_sync(&hba->clk_gating.gate_work); - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->clk_gating.state == CLKS_ON) { - spin_unlock_irqrestore(hba->host->host_lock, flags); - return; + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { + if (hba->clk_gating.state == CLKS_ON) + return; } - spin_unlock_irqrestore(hba->host->host_lock, flags); ufshcd_hba_vreg_set_hpm(hba); ufshcd_setup_clocks(hba, true); @@ -1759,7 +1941,7 @@ void ufshcd_hold(struct ufs_hba *hba) if (!ufshcd_is_clkgating_allowed(hba) || !hba->clk_gating.is_initialized) return; - spin_lock_irqsave(hba->host->host_lock, flags); + spin_lock_irqsave(&hba->clk_gating.lock, flags); hba->clk_gating.active_reqs++; start: @@ -1775,18 +1957,18 @@ start: */ if (ufshcd_can_hibern8_during_gating(hba) && ufshcd_is_link_hibern8(hba)) { - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); flush_result = flush_work(&hba->clk_gating.ungate_work); if (hba->clk_gating.is_suspended && !flush_result) return; - spin_lock_irqsave(hba->host->host_lock, flags); + spin_lock_irqsave(&hba->clk_gating.lock, flags); goto start; } break; case REQ_CLKS_OFF: if (cancel_delayed_work(&hba->clk_gating.gate_work)) { hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); break; } @@ -1798,7 +1980,7 @@ start: fallthrough; case CLKS_OFF: hba->clk_gating.state = REQ_CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); queue_work(hba->clk_gating.clk_gating_workq, &hba->clk_gating.ungate_work); @@ -1808,17 +1990,17 @@ start: */ fallthrough; case REQ_CLKS_ON: - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); flush_work(&hba->clk_gating.ungate_work); /* Make sure state is CLKS_ON before returning */ - spin_lock_irqsave(hba->host->host_lock, flags); + spin_lock_irqsave(&hba->clk_gating.lock, flags); goto start; default: dev_err(hba->dev, "%s: clk gating is in invalid state %d\n", __func__, hba->clk_gating.state); break; } - spin_unlock_irqrestore(hba->host->host_lock, flags); + spin_unlock_irqrestore(&hba->clk_gating.lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_hold); @@ -1826,31 +2008,32 @@ static void ufshcd_gate_work(struct work_struct *work) { struct ufs_hba *hba = container_of(work, struct ufs_hba, clk_gating.gate_work.work); - unsigned long flags; int ret; - spin_lock_irqsave(hba->host->host_lock, flags); - /* - * In case you are here to cancel this work the gating state - * would be marked as REQ_CLKS_ON. In this case save time by - * skipping the gating work and exit after changing the clock - * state to CLKS_ON. - */ - if (hba->clk_gating.is_suspended || - (hba->clk_gating.state != REQ_CLKS_OFF)) { - hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), - hba->clk_gating.state); - goto rel_lock; - } + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) { + /* + * In case you are here to cancel this work the gating state + * would be marked as REQ_CLKS_ON. In this case save time by + * skipping the gating work and exit after changing the clock + * state to CLKS_ON. + */ + if (hba->clk_gating.is_suspended || + hba->clk_gating.state != REQ_CLKS_OFF) { + hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(hba, + hba->clk_gating.state); + return; + } - if (hba->clk_gating.active_reqs - || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL - || hba->outstanding_reqs || hba->outstanding_tasks - || hba->active_uic_cmd || hba->uic_async_done) - goto rel_lock; + if (hba->clk_gating.active_reqs) + return; + } - spin_unlock_irqrestore(hba->host->host_lock, flags); + scoped_guard(spinlock_irqsave, hba->host->host_lock) { + if (ufshcd_is_ufs_dev_busy(hba) || + hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) + return; + } /* put the link into hibern8 mode before turning off clocks */ if (ufshcd_can_hibern8_during_gating(hba)) { @@ -1859,9 +2042,9 @@ static void ufshcd_gate_work(struct work_struct *work) hba->clk_gating.state = CLKS_ON; dev_err(hba->dev, "%s: hibern8 enter failed %d\n", __func__, ret); - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); - goto out; + return; } ufshcd_set_link_hibern8(hba); } @@ -1881,35 +2064,36 @@ static void ufshcd_gate_work(struct work_struct *work) * prevent from doing cancel work multiple times when there are * new requests arriving before the current cancel work is done. */ - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); if (hba->clk_gating.state == REQ_CLKS_OFF) { hba->clk_gating.state = CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); } -rel_lock: - spin_unlock_irqrestore(hba->host->host_lock, flags); -out: - return; } -/* host lock must be held before calling this variant */ static void __ufshcd_release(struct ufs_hba *hba) { + lockdep_assert_held(&hba->clk_gating.lock); + if (!ufshcd_is_clkgating_allowed(hba)) return; hba->clk_gating.active_reqs--; if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended || - hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL || - hba->outstanding_tasks || !hba->clk_gating.is_initialized || - hba->active_uic_cmd || hba->uic_async_done || + !hba->clk_gating.is_initialized || hba->clk_gating.state == CLKS_OFF) return; + scoped_guard(spinlock_irqsave, hba->host->host_lock) { + if (ufshcd_has_pending_tasks(hba) || + hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) + return; + } + hba->clk_gating.state = REQ_CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state); + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); queue_delayed_work(hba->clk_gating.clk_gating_workq, &hba->clk_gating.gate_work, msecs_to_jiffies(hba->clk_gating.delay_ms)); @@ -1917,11 +2101,8 @@ static void __ufshcd_release(struct ufs_hba *hba) void ufshcd_release(struct ufs_hba *hba) { - unsigned long flags; - - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); __ufshcd_release(hba); - spin_unlock_irqrestore(hba->host->host_lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_release); @@ -1936,11 +2117,9 @@ static ssize_t ufshcd_clkgate_delay_show(struct device *dev, void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value) { struct ufs_hba *hba = dev_get_drvdata(dev); - unsigned long flags; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); hba->clk_gating.delay_ms = value; - spin_unlock_irqrestore(hba->host->host_lock, flags); } EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set); @@ -1968,7 +2147,6 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct ufs_hba *hba = dev_get_drvdata(dev); - unsigned long flags; u32 value; if (kstrtou32(buf, 0, &value)) @@ -1976,9 +2154,10 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, value = !!value; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_gating.lock); + if (value == hba->clk_gating.is_enabled) - goto out; + return count; if (value) __ufshcd_release(hba); @@ -1986,8 +2165,7 @@ static ssize_t ufshcd_clkgate_enable_store(struct device *dev, hba->clk_gating.active_reqs++; hba->clk_gating.is_enabled = value; -out: - spin_unlock_irqrestore(hba->host->host_lock, flags); + return count; } @@ -2020,8 +2198,6 @@ static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba) static void ufshcd_init_clk_gating(struct ufs_hba *hba) { - char wq_name[sizeof("ufs_clk_gating_00")]; - if (!ufshcd_is_clkgating_allowed(hba)) return; @@ -2031,10 +2207,9 @@ static void ufshcd_init_clk_gating(struct ufs_hba *hba) INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work); INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work); - snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d", - hba->host->host_no); - hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name, - WQ_MEM_RECLAIM | WQ_HIGHPRI); + hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue( + "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI, + hba->host->host_no); ufshcd_init_clk_gating_sysfs(hba); @@ -2060,20 +2235,20 @@ static void ufshcd_exit_clk_gating(struct ufs_hba *hba) static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) { bool queue_resume_work = false; - ktime_t curr_t = ktime_get(); - unsigned long flags; + ktime_t curr_t; if (!ufshcd_is_clkscaling_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); + curr_t = ktime_get(); + + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + if (!hba->clk_scaling.active_reqs++) queue_resume_work = true; - if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) { - spin_unlock_irqrestore(hba->host->host_lock, flags); + if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) return; - } if (queue_resume_work) queue_work(hba->clk_scaling.workq, @@ -2089,18 +2264,17 @@ static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba) hba->clk_scaling.busy_start_t = curr_t; hba->clk_scaling.is_busy_started = true; } - spin_unlock_irqrestore(hba->host->host_lock, flags); } static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) { struct ufs_clk_scaling *scaling = &hba->clk_scaling; - unsigned long flags; if (!ufshcd_is_clkscaling_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); + guard(spinlock_irqsave)(&hba->clk_scaling.lock); + hba->clk_scaling.active_reqs--; if (!scaling->active_reqs && scaling->is_busy_started) { scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(), @@ -2108,7 +2282,6 @@ static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba) scaling->busy_start_t = 0; scaling->is_busy_started = false; } - spin_unlock_irqrestore(hba->host->host_lock, flags); } static inline int ufshcd_monitor_opcode2dir(u8 opcode) @@ -2121,20 +2294,21 @@ static inline int ufshcd_monitor_opcode2dir(u8 opcode) return -EINVAL; } +/* Must only be called for SCSI commands. */ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { const struct ufs_hba_monitor *m = &hba->monitor; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); - return (m->enabled && lrbp && lrbp->cmd && - (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) && - ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp)); + return m->enabled && + (!m->chunk_size || m->chunk_size == cmd->sdb.length) && + ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp); } -static void ufshcd_start_monitor(struct ufs_hba *hba, - const struct ufshcd_lrb *lrbp) +static void ufshcd_start_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); @@ -2143,14 +2317,15 @@ static void ufshcd_start_monitor(struct ufs_hba *hba, spin_unlock_irqrestore(hba->host->host_lock, flags); } -static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp) +static void ufshcd_update_monitor(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd); + struct request *req = scsi_cmd_to_rq(cmd); + const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]); unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) { - const struct request *req = scsi_cmd_to_rq(lrbp->cmd); struct ufs_hba_monitor *m = &hba->monitor; ktime_t now, inc, lat; @@ -2175,60 +2350,73 @@ static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb * spin_unlock_irqrestore(hba->host->host_lock, flags); } +/* Returns %true for SCSI commands and %false for device management commands. */ +static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd) +{ + return !blk_mq_is_reserved_rq(scsi_cmd_to_rq(cmd)); +} + /** * ufshcd_send_command - Send SCSI or device management commands * @hba: per adapter instance - * @task_tag: Task tag of the command + * @cmd: SCSI command or device management command pointer * @hwq: pointer to hardware queue instance */ -static inline -void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag, - struct ufs_hw_queue *hwq) +static inline void ufshcd_send_command(struct ufs_hba *hba, + struct scsi_cmnd *cmd, + struct ufs_hw_queue *hwq) { - struct ufshcd_lrb *lrbp = &hba->lrb[task_tag]; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; unsigned long flags; - lrbp->issue_time_stamp = ktime_get(); - lrbp->issue_time_stamp_local_clock = local_clock(); - lrbp->compl_time_stamp = ktime_set(0, 0); - lrbp->compl_time_stamp_local_clock = 0; - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND); - ufshcd_clk_scaling_start_busy(hba); - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) - ufshcd_start_monitor(hba, lrbp); + if (hba->monitor.enabled) { + lrbp->issue_time_stamp = ktime_get(); + lrbp->issue_time_stamp_local_clock = local_clock(); + lrbp->compl_time_stamp = ktime_set(0, 0); + lrbp->compl_time_stamp_local_clock = 0; + } + if (ufshcd_is_scsi_cmd(cmd)) { + ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND); + ufshcd_clk_scaling_start_busy(hba); + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) + ufshcd_start_monitor(hba, cmd); + } - if (is_mcq_enabled(hba)) { + if (hba->mcq_enabled) { int utrd_size = sizeof(struct utp_transfer_req_desc); struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr; - struct utp_transfer_req_desc *dest = hwq->sqe_base_addr + hwq->sq_tail_slot; + struct utp_transfer_req_desc *dest; spin_lock(&hwq->sq_lock); + dest = hwq->sqe_base_addr + hwq->sq_tail_slot; memcpy(dest, src, utrd_size); ufshcd_inc_sq_tail(hwq); spin_unlock(&hwq->sq_lock); } else { spin_lock_irqsave(&hba->outstanding_lock, flags); if (hba->vops && hba->vops->setup_xfer_req) - hba->vops->setup_xfer_req(hba, lrbp->task_tag, - !!lrbp->cmd); - __set_bit(lrbp->task_tag, &hba->outstanding_reqs); - ufshcd_writel(hba, 1 << lrbp->task_tag, - REG_UTP_TRANSFER_REQ_DOOR_BELL); + hba->vops->setup_xfer_req(hba, tag, + ufshcd_is_scsi_cmd(cmd)); + __set_bit(tag, &hba->outstanding_reqs); + ufshcd_writel(hba, 1 << tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); spin_unlock_irqrestore(&hba->outstanding_lock, flags); } } /** * ufshcd_copy_sense_data - Copy sense data in case of check condition - * @lrbp: pointer to local reference block + * @cmd: SCSI command */ -static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) +static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd) { - u8 *const sense_buffer = lrbp->cmd->sense_buffer; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u8 *const sense_buffer = cmd->sense_buffer; + u16 resp_len; int len; - if (sense_buffer && - ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) { + resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length); + if (sense_buffer && resp_len) { int len_to_copy; len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len); @@ -2244,6 +2432,8 @@ static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp) * descriptor * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) @@ -2261,8 +2451,8 @@ int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) u16 buf_len; /* data segment length */ - resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & - MASK_QUERY_DATA_SEG_LEN; + resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header + .data_segment_length); buf_len = be16_to_cpu( hba->dev_cmd.query.request.upiu_req.length); if (likely(buf_len >= resp_len)) { @@ -2289,14 +2479,13 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) int err; hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); - if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS) - hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT; /* nutrs and nutmrs are 0 based values */ - hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1; + hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1; hba->nutmrs = ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1; - hba->reserved_slot = hba->nutrs - 1; + + hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1; /* Read crypto capabilities */ err = ufshcd_hba_init_crypto_capabilities(hba); @@ -2305,13 +2494,22 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) return err; } + /* + * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and + * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which + * means we can simply read values regardless of version. + */ hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities); - if (!hba->mcq_sup) - return 0; + /* + * 0h: legacy single doorbell support is available + * 1h: indicate that legacy single doorbell support has been removed + */ + if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP)) + hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities); + else + hba->lsdb_sup = true; hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP); - hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT, - hba->mcq_capabilities); return 0; } @@ -2320,11 +2518,16 @@ static inline int ufshcd_hba_capabilities(struct ufs_hba *hba) * ufshcd_ready_for_uic_cmd - Check if controller is ready * to accept UIC commands * @hba: per adapter instance - * Return true on success, else false + * + * Return: true on success, else false. */ static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) { - return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY; + u32 val; + int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY, + 500, uic_cmd_timeout * 1000, false, hba, + REG_CONTROLLER_STATUS); + return ret == 0; } /** @@ -2332,7 +2535,8 @@ static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) * @hba: Pointer to adapter instance * * This function gets the UPMCRS field of HCS register - * Returns value of UPMCRS field + * + * Return: value of UPMCRS field. */ static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) { @@ -2370,7 +2574,7 @@ ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) * @hba: per adapter instance * @uic_cmd: UIC command * - * Returns 0 only if success. + * Return: 0 only if success. */ static int ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) @@ -2381,7 +2585,7 @@ ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) lockdep_assert_held(&hba->uic_cmd_mutex); if (wait_for_completion_timeout(&uic_cmd->done, - msecs_to_jiffies(UIC_CMD_TIMEOUT))) { + msecs_to_jiffies(uic_cmd_timeout))) { ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT; } else { ret = -ETIMEDOUT; @@ -2407,16 +2611,13 @@ ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result * @hba: per adapter instance * @uic_cmd: UIC command - * @completion: initialize the completion only if this is set to true * - * Returns 0 only if success. + * Return: 0 if successful; < 0 upon failure. */ static int -__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, - bool completion) +__ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { lockdep_assert_held(&hba->uic_cmd_mutex); - lockdep_assert_held(hba->host->host_lock); if (!ufshcd_ready_for_uic_cmd(hba)) { dev_err(hba->dev, @@ -2424,10 +2625,9 @@ __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, return -EIO; } - if (completion) - init_completion(&uic_cmd->done); + init_completion(&uic_cmd->done); - uic_cmd->cmd_active = 1; + uic_cmd->cmd_active = true; ufshcd_dispatch_uic_cmd(hba, uic_cmd); return 0; @@ -2438,12 +2638,12 @@ __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd, * @hba: per adapter instance * @uic_cmd: UIC command * - * Returns 0 only if success. + * Return: 0 only if success. */ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) { - int ret; unsigned long flags; + int ret; if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) return 0; @@ -2453,8 +2653,10 @@ int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) ufshcd_add_delay_before_dme_cmd(hba); spin_lock_irqsave(hba->host->host_lock, flags); - ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true); + ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); spin_unlock_irqrestore(hba->host->host_lock, flags); + + ret = __ufshcd_send_uic_cmd(hba, uic_cmd); if (!ret) ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd); @@ -2513,13 +2715,13 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int /** * ufshcd_map_sg - Map scatter-gather list to prdt * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command * - * Returns 0 in case of success, non-zero value in case of failure + * Return: 0 in case of success, non-zero value in case of failure. */ -static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - struct scsi_cmnd *cmd = lrbp->cmd; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); int sg_segments = scsi_dma_map(cmd); if (sg_segments < 0) @@ -2527,67 +2729,30 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd)); - return 0; -} - -/** - * ufshcd_enable_intr - enable interrupts - * @hba: per adapter instance - * @intrs: interrupt bits - */ -static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) -{ - u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); - - if (hba->ufs_version == ufshci_version(1, 0)) { - u32 rw; - rw = set & INTERRUPT_MASK_RW_VER_10; - set = rw | ((set ^ intrs) & intrs); - } else { - set |= intrs; - } - - ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); -} - -/** - * ufshcd_disable_intr - disable interrupts - * @hba: per adapter instance - * @intrs: interrupt bits - */ -static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs) -{ - u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); - - if (hba->ufs_version == ufshci_version(1, 0)) { - u32 rw; - rw = (set & INTERRUPT_MASK_RW_VER_10) & - ~(intrs & INTERRUPT_MASK_RW_VER_10); - set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10); - - } else { - set &= ~intrs; - } - - ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); + return ufshcd_crypto_fill_prdt(hba, cmd); } /** * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request * descriptor according to request + * @hba: per adapter instance * @lrbp: pointer to local reference block * @upiu_flags: flags required in the header * @cmd_dir: requests data direction * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments) */ -static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u8 *upiu_flags, - enum dma_data_direction cmd_dir, int ehs_length) +static void +ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, + u8 *upiu_flags, enum dma_data_direction cmd_dir, + int ehs_length) { struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr; - u32 data_direction; - u32 dword_0; - u32 dword_1 = 0; - u32 dword_3 = 0; + struct request_desc_header *h = &req_desc->header; + enum utp_data_direction data_direction; + + lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + + *h = (typeof(*h)){ }; if (cmd_dir == DMA_FROM_DEVICE) { data_direction = UTP_DEVICE_TO_HOST; @@ -2600,25 +2765,22 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u8 *upiu_flags, *upiu_flags = UPIU_CMD_FLAGS_NONE; } - dword_0 = data_direction | (lrbp->command_type << UPIU_COMMAND_TYPE_OFFSET) | - ehs_length << 8; + h->command_type = lrbp->command_type; + h->data_direction = data_direction; + h->ehs_length = ehs_length; + if (lrbp->intr_cmd) - dword_0 |= UTP_REQ_DESC_INT_CMD; + h->interrupt = 1; /* Prepare crypto related dwords */ - ufshcd_prepare_req_desc_hdr_crypto(lrbp, &dword_0, &dword_1, &dword_3); + ufshcd_prepare_req_desc_hdr_crypto(lrbp, h); - /* Transfer request descriptor header fields */ - req_desc->header.dword_0 = cpu_to_le32(dword_0); - req_desc->header.dword_1 = cpu_to_le32(dword_1); /* * assigning invalid value for command status. Controller * updates OCS on command completion, with the command * status */ - req_desc->header.dword_2 = - cpu_to_le32(OCS_INVALID_COMMAND_STATUS); - req_desc->header.dword_3 = cpu_to_le32(dword_3); + h->ocs = OCS_INVALID_COMMAND_STATUS; req_desc->prd_table_length = 0; } @@ -2626,30 +2788,30 @@ static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp, u8 *upiu_flags, /** * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc, * for scsi commands - * @lrbp: local reference block pointer + * @cmd: SCSI command * @upiu_flags: flags */ -static -void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) +static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd, + u8 upiu_flags) { - struct scsi_cmnd *cmd = lrbp->cmd; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; unsigned short cdb_len; - /* command descriptor fields */ - ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( - UPIU_TRANSACTION_COMMAND, upiu_flags, - lrbp->lun, lrbp->task_tag); - ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( - UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_COMMAND, + .flags = upiu_flags, + .lun = lrbp->lun, + .task_tag = tag, + .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI, + }; - /* Total EHS length and Data segment length will be zero */ - ucd_req_ptr->header.dword_2 = 0; + WARN_ON_ONCE(ucd_req_ptr->header.task_tag != tag); ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length); cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE); - memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE); memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len); memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); @@ -2658,29 +2820,32 @@ void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags) /** * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request * @hba: UFS hba - * @lrbp: local reference block pointer + * @cmd: SCSI command pointer * @upiu_flags: flags */ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, u8 upiu_flags) + struct scsi_cmnd *cmd, u8 upiu_flags) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + const int tag = scsi_cmd_to_rq(cmd)->tag; struct ufs_query *query = &hba->dev_cmd.query; u16 len = be16_to_cpu(query->request.upiu_req.length); /* Query request header */ - ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD( - UPIU_TRANSACTION_QUERY_REQ, upiu_flags, - lrbp->lun, lrbp->task_tag); - ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD( - 0, query->request.query_func, 0, 0); - - /* Data segment length only need for WRITE_DESC */ - if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) - ucd_req_ptr->header.dword_2 = - UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len); - else - ucd_req_ptr->header.dword_2 = 0; + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_QUERY_REQ, + .flags = upiu_flags, + .lun = lrbp->lun, + .task_tag = tag, + .query_function = query->request.query_func, + /* Data segment length only need for WRITE_DESC */ + .data_segment_length = + query->request.upiu_req.opcode == + UPIU_QUERY_OPCODE_WRITE_DESC ? + cpu_to_be16(len) : + 0, + }; /* Copy the Query Request buffer as is */ memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, @@ -2689,52 +2854,48 @@ static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, /* Copy the Descriptor */ if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) memcpy(ucd_req_ptr + 1, query->descriptor, len); - - memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); } -static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp) +static inline void ufshcd_prepare_utp_nop_upiu(struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr; + const int tag = scsi_cmd_to_rq(cmd)->tag; memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); - /* command descriptor fields */ - ucd_req_ptr->header.dword_0 = - UPIU_HEADER_DWORD( - UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag); - /* clear rest of the fields of basic header */ - ucd_req_ptr->header.dword_1 = 0; - ucd_req_ptr->header.dword_2 = 0; - - memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); + ucd_req_ptr->header = (struct utp_upiu_header){ + .transaction_code = UPIU_TRANSACTION_NOP_OUT, + .task_tag = tag, + }; } /** * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU) * for Device Management Purposes * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command pointer + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) + struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); u8 upiu_flags; int ret = 0; - if (hba->ufs_version <= ufshci_version(1, 1)) - lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; - else - lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); - ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE, 0); if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY) - ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags); + ufshcd_prepare_utp_query_req_upiu(hba, cmd, upiu_flags); else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP) - ufshcd_prepare_utp_nop_upiu(lrbp); + ufshcd_prepare_utp_nop_upiu(cmd); else ret = -EINVAL; + memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); + return ret; } @@ -2742,33 +2903,76 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba, * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU) * for SCSI Purposes * @hba: per adapter instance - * @lrbp: pointer to local reference block + * @cmd: SCSI command */ -static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) +static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); + unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); u8 upiu_flags; - int ret = 0; - if (hba->ufs_version <= ufshci_version(1, 1)) - lrbp->command_type = UTP_CMD_TYPE_SCSI; - else - lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, + cmd->sc_data_direction, 0); + if (ioprio_class == IOPRIO_CLASS_RT) + upiu_flags |= UPIU_CMD_FLAGS_CP; + ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags); +} - if (likely(lrbp->cmd)) { - ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0); - ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags); - } else { - ret = -EINVAL; - } +static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd) +{ + const int i = scsi_cmd_to_rq(cmd)->tag; + struct utp_transfer_cmd_desc *cmd_descp = + (void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba); + struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; + dma_addr_t cmd_desc_element_addr = + hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba); + u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset); + u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset); + struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd); - return ret; + lrb->utr_descriptor_ptr = utrdlp + i; + lrb->utrd_dma_addr = + hba->utrdl_dma_addr + i * sizeof(struct utp_transfer_req_desc); + lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; + lrb->ucd_req_dma_addr = cmd_desc_element_addr; + lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; + lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; + lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; + lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; +} + +static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + u8 lun, int tag) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + ufshcd_init_lrb(hba, cmd); + + memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr)); + + lrbp->lun = lun; + ufshcd_prepare_lrbp_crypto(ufshcd_is_scsi_cmd(cmd) ? + scsi_cmd_to_rq(cmd) : NULL, lrbp); +} + +static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + u8 lun, int tag) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + __ufshcd_setup_cmd(hba, cmd, lun, tag); + lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); + lrbp->req_abort_skip = false; + + ufshcd_comp_scsi_upiu(hba, cmd); } /** * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID * @upiu_wlun_id: UPIU W-LUN id * - * Returns SCSI W-LUN id + * Return: SCSI W-LUN id. */ static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id) { @@ -2812,26 +3016,13 @@ static void ufshcd_map_queues(struct Scsi_Host *shost) } } -static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) +/* + * The only purpose of this function is to make the SCSI core skip the memset() + * call for the private command data. + */ +static int ufshcd_init_cmd_priv(struct Scsi_Host *host, struct scsi_cmnd *cmd) { - struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr + - i * ufshcd_get_ucd_size(hba); - struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr; - dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr + - i * ufshcd_get_ucd_size(hba); - u16 response_offset = offsetof(struct utp_transfer_cmd_desc, - response_upiu); - u16 prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table); - - lrb->utr_descriptor_ptr = utrdlp + i; - lrb->utrd_dma_addr = hba->utrdl_dma_addr + - i * sizeof(struct utp_transfer_req_desc); - lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu; - lrb->ucd_req_dma_addr = cmd_desc_element_addr; - lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu; - lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset; - lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table; - lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset; + return 0; } /** @@ -2839,18 +3030,15 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i) * @host: SCSI host pointer * @cmd: command from SCSI Midlayer * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) { struct ufs_hba *hba = shost_priv(host); int tag = scsi_cmd_to_rq(cmd)->tag; - struct ufshcd_lrb *lrbp; int err = 0; struct ufs_hw_queue *hwq = NULL; - WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n", tag); - switch (hba->ufshcd_state) { case UFSHCD_STATE_OPERATIONAL: break; @@ -2898,33 +3086,22 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd) ufshcd_hold(hba); - lrbp = &hba->lrb[tag]; - lrbp->cmd = cmd; - lrbp->task_tag = tag; - lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); - lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba); - - ufshcd_prepare_lrbp_crypto(scsi_cmd_to_rq(cmd), lrbp); - - lrbp->req_abort_skip = false; - - ufshpb_prep(hba, lrbp); - - ufshcd_comp_scsi_upiu(hba, lrbp); + ufshcd_setup_scsi_cmd(hba, cmd, + ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag); - err = ufshcd_map_sg(hba, lrbp); + err = ufshcd_map_sg(hba, cmd); if (err) { ufshcd_release(hba); goto out; } - if (is_mcq_enabled(hba)) + if (hba->mcq_enabled) hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd)); - ufshcd_send_command(hba, tag, hwq); + ufshcd_send_command(hba, cmd, hwq); out: - if (ufs_trigger_eh()) { + if (ufs_trigger_eh(hba)) { unsigned long flags; spin_lock_irqsave(hba->host->host_lock, flags); @@ -2935,37 +3112,50 @@ out: return err; } -static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag) +static int ufshcd_queue_reserved_command(struct Scsi_Host *host, + struct scsi_cmnd *cmd) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); + struct ufs_hba *hba = shost_priv(host); + struct ufs_hw_queue *hwq = + hba->mcq_enabled ? ufshcd_mcq_req_to_hwq(hba, rq) : NULL; + + ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); + ufshcd_send_command(hba, cmd, hwq); + return 0; +} + +static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + enum dev_cmd_type cmd_type, u8 lun, int tag) { - lrbp->cmd = NULL; - lrbp->task_tag = tag; - lrbp->lun = 0; /* device management cmd is not specific to any LUN */ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + + __ufshcd_setup_cmd(hba, cmd, lun, tag); lrbp->intr_cmd = true; /* No interrupt aggregation */ - ufshcd_prepare_lrbp_crypto(NULL, lrbp); hba->dev_cmd.type = cmd_type; +} + +/* + * Return: 0 upon success; < 0 upon failure. + */ +static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + enum dev_cmd_type cmd_type, int tag) +{ + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); - return ufshcd_compose_devman_upiu(hba, lrbp); + return ufshcd_compose_devman_upiu(hba, cmd); } /* * Check with the block layer if the command is inflight * @cmd: command to check. * - * Returns true if command is inflight; false if not. + * Return: true if command is inflight; false if not. */ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) { - struct request *rq; - - if (!cmd) - return false; - - rq = scsi_cmd_to_rq(cmd); - if (!blk_mq_request_started(rq)) - return false; - - return true; + return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT; } /* @@ -2976,11 +3166,10 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd) */ static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) { - u32 mask = 1U << task_tag; - unsigned long flags; + u32 mask; int err; - if (is_mcq_enabled(hba)) { + if (hba->mcq_enabled) { /* * MCQ mode. Clean up the MCQ resources similar to * what the ufshcd_utrl_clear() does for SDB mode. @@ -2994,10 +3183,10 @@ static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) return 0; } + mask = 1U << task_tag; + /* clear outstanding transaction before retry */ - spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_utrl_clear(hba, mask); - spin_unlock_irqrestore(hba->host->host_lock, flags); /* * wait for h/w to clear corresponding bit in door-bell. @@ -3007,26 +3196,17 @@ static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag) mask, ~mask, 1000, 1000); } -static int -ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) -{ - struct ufs_query_res *query_res = &hba->dev_cmd.query.response; - - /* Get the UPIU response */ - query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >> - UPIU_RSP_CODE_OFFSET; - return query_res->response; -} - /** * ufshcd_dev_cmd_completion() - handles device management command responses * @hba: per adapter instance * @lrbp: pointer to local reference block + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) { - int resp; + enum upiu_response_transaction resp; int err = 0; hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); @@ -3040,11 +3220,18 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) __func__, resp); } break; - case UPIU_TRANSACTION_QUERY_RSP: - err = ufshcd_check_query_response(hba, lrbp); - if (!err) + case UPIU_TRANSACTION_QUERY_RSP: { + u8 response = lrbp->ucd_rsp_ptr->header.response; + + if (response == 0) { err = ufshcd_copy_query_response(hba, lrbp); + } else { + err = -EINVAL; + dev_err(hba->dev, "%s: unexpected response in Query RSP: %x\n", + __func__, response); + } break; + } case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ err = -EPERM; @@ -3064,93 +3251,58 @@ ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) break; } + WARN_ONCE(err > 0, "Incorrect return value %d > 0\n", err); return err; } -static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp, int max_timeout) +static void ufshcd_dev_man_lock(struct ufs_hba *hba) { - unsigned long time_left = msecs_to_jiffies(max_timeout); - unsigned long flags; - bool pending; - int err; - -retry: - time_left = wait_for_completion_timeout(hba->dev_cmd.complete, - time_left); - - if (likely(time_left)) { - /* - * The completion handler called complete() and the caller of - * this function still owns the @lrbp tag so the code below does - * not trigger any race conditions. - */ - hba->dev_cmd.complete = NULL; - err = ufshcd_get_tr_ocs(lrbp, NULL); - if (!err) - err = ufshcd_dev_cmd_completion(hba, lrbp); - } else { - err = -ETIMEDOUT; - dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n", - __func__, lrbp->task_tag); + ufshcd_hold(hba); + mutex_lock(&hba->dev_cmd.lock); + down_read(&hba->clk_scaling_lock); +} - /* MCQ mode */ - if (is_mcq_enabled(hba)) { - err = ufshcd_clear_cmd(hba, lrbp->task_tag); - hba->dev_cmd.complete = NULL; - return err; - } +static void ufshcd_dev_man_unlock(struct ufs_hba *hba) +{ + up_read(&hba->clk_scaling_lock); + mutex_unlock(&hba->dev_cmd.lock); + ufshcd_release(hba); +} - /* SDB mode */ - if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) { - /* successfully cleared the command, retry if needed */ - err = -EAGAIN; - /* - * Since clearing the command succeeded we also need to - * clear the task tag bit from the outstanding_reqs - * variable. - */ - spin_lock_irqsave(&hba->outstanding_lock, flags); - pending = test_bit(lrbp->task_tag, - &hba->outstanding_reqs); - if (pending) { - hba->dev_cmd.complete = NULL; - __clear_bit(lrbp->task_tag, - &hba->outstanding_reqs); - } - spin_unlock_irqrestore(&hba->outstanding_lock, flags); +static struct scsi_cmnd *ufshcd_get_dev_mgmt_cmd(struct ufs_hba *hba) +{ + /* + * The caller must hold this lock to guarantee that the NOWAIT + * allocation will succeed. + */ + lockdep_assert_held(&hba->dev_cmd.lock); - if (!pending) { - /* - * The completion handler ran while we tried to - * clear the command. - */ - time_left = 1; - goto retry; - } - } else { - dev_err(hba->dev, "%s: failed to clear tag %d\n", - __func__, lrbp->task_tag); + return scsi_get_internal_cmd( + hba->host->pseudo_sdev, DMA_TO_DEVICE, + BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT); +} - spin_lock_irqsave(&hba->outstanding_lock, flags); - pending = test_bit(lrbp->task_tag, - &hba->outstanding_reqs); - if (pending) - hba->dev_cmd.complete = NULL; - spin_unlock_irqrestore(&hba->outstanding_lock, flags); +static void ufshcd_put_dev_mgmt_cmd(struct scsi_cmnd *cmd) +{ + scsi_put_internal_cmd(cmd); +} - if (!pending) { - /* - * The completion handler ran while we tried to - * clear the command. - */ - time_left = 1; - goto retry; - } - } - } +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ +static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd, + const u32 tag, int timeout) +{ + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + struct request *rq = scsi_cmd_to_rq(cmd); + blk_status_t sts; - return err; + rq->timeout = timeout; + sts = blk_execute_rq(rq, true); + if (sts != BLK_STS_OK) + return blk_status_to_errno(sts); + return lrbp->utr_descriptor_ptr->header.ocs; } /** @@ -3159,39 +3311,39 @@ retry: * @cmd_type: specifies the type (NOP, Query...) * @timeout: timeout in milliseconds * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + * * NOTE: Since there is only one available tag for device management commands, * it is expected you hold the hba->dev_cmd.lock mutex. */ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, int timeout) { - DECLARE_COMPLETION_ONSTACK(wait); - const u32 tag = hba->reserved_slot; - struct ufshcd_lrb *lrbp; + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u32 tag; int err; - /* Protects use of hba->reserved_slot. */ + /* Protects use of hba->dev_cmd. */ lockdep_assert_held(&hba->dev_cmd.lock); - down_read(&hba->clk_scaling_lock); + if (WARN_ON_ONCE(!cmd)) + return -ENOMEM; + + tag = scsi_cmd_to_rq(cmd)->tag; - lrbp = &hba->lrb[tag]; - lrbp->cmd = NULL; - err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag); + err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag); if (unlikely(err)) goto out; - hba->dev_cmd.complete = &wait; - - ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - - ufshcd_send_command(hba, tag, hba->dev_cmd_queue); - err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout); - ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, - (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, timeout); + if (err == 0) + err = ufshcd_dev_cmd_completion(hba, lrbp); out: - up_read(&hba->clk_scaling_lock); + ufshcd_put_dev_mgmt_cmd(cmd); + return err; } @@ -3219,6 +3371,10 @@ static inline void ufshcd_init_query(struct ufs_hba *hba, (*request)->upiu_req.selector = selector; } +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ static int ufshcd_query_flag_retry(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) { @@ -3250,7 +3406,8 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba, * @index: flag index to access * @flag_res: the flag value after the query request completes * - * Returns 0 for success, non-zero in case of failure + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res) @@ -3258,12 +3415,12 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, struct ufs_query_req *request = NULL; struct ufs_query_res *response = NULL; int err, selector = 0; - int timeout = QUERY_REQ_TIMEOUT; + int timeout = dev_cmd_timeout; BUG_ON(!hba); - ufshcd_hold(hba); - mutex_lock(&hba->dev_cmd.lock); + ufshcd_dev_man_lock(hba); + ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); @@ -3305,8 +3462,7 @@ int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, MASK_QUERY_UPIU_FLAG_LOC) & 0x1; out_unlock: - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); return err; } @@ -3319,8 +3475,9 @@ out_unlock: * @selector: selector field * @attr_val: the attribute value after the query request completes * - * Returns 0 for success, non-zero in case of failure -*/ + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) { @@ -3336,9 +3493,8 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, return -EINVAL; } - ufshcd_hold(hba); + ufshcd_dev_man_lock(hba); - mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); @@ -3357,7 +3513,7 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, goto out_unlock; } - err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); if (err) { dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", @@ -3368,8 +3524,7 @@ int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode, *attr_val = be32_to_cpu(response->upiu_res.value); out_unlock: - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); return err; } @@ -3384,8 +3539,9 @@ out_unlock: * @attr_val: the attribute value after the query request * completes * - * Returns 0 for success, non-zero in case of failure -*/ + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ int ufshcd_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector, u32 *attr_val) @@ -3410,6 +3566,10 @@ int ufshcd_query_attr_retry(struct ufs_hba *hba, return ret; } +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ static int __ufshcd_query_descriptor(struct ufs_hba *hba, enum query_opcode opcode, enum desc_idn idn, u8 index, u8 selector, u8 *desc_buf, int *buf_len) @@ -3432,9 +3592,8 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, return -EINVAL; } - ufshcd_hold(hba); + ufshcd_dev_man_lock(hba); - mutex_lock(&hba->dev_cmd.lock); ufshcd_init_query(hba, &request, &response, opcode, idn, index, selector); hba->dev_cmd.query.descriptor = desc_buf; @@ -3455,7 +3614,7 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, goto out_unlock; } - err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); if (err) { dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", @@ -3467,8 +3626,7 @@ static int __ufshcd_query_descriptor(struct ufs_hba *hba, out_unlock: hba->dev_cmd.query.descriptor = NULL; - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); return err; } @@ -3482,9 +3640,11 @@ out_unlock: * @desc_buf: the buffer that contains the descriptor * @buf_len: length parameter passed to the device * - * Returns 0 for success, non-zero in case of failure. * The buf_len parameter will contain, on return, the length parameter * received on the response. + * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode, @@ -3514,7 +3674,8 @@ int ufshcd_query_descriptor_retry(struct ufs_hba *hba, * @param_read_buf: pointer to buffer where parameter would be read * @param_size: sizeof(param_read_buf) * - * Return 0 in case of success, non-zero otherwise + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id, @@ -3609,16 +3770,14 @@ static inline char ufshcd_remove_non_printable(u8 ch) * @desc_index: descriptor index * @buf: pointer to buffer where descriptor would be read, * the caller should free the memory. - * @ascii: if true convert from unicode to ascii characters - * null terminated string. + * @fmt: if %SD_ASCII_STD, convert from UTF-16 to ASCII * * Return: * * string size on success. * * -ENOMEM: on allocation failure * * -EINVAL: on a wrong parameter */ -int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, - u8 **buf, bool ascii) +int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, u8 **buf, enum ufs_descr_fmt fmt) { struct uc_string_id *uc_str; u8 *str; @@ -3647,7 +3806,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, goto out; } - if (ascii) { + if (fmt == SD_ASCII_STD) { ssize_t ascii_len; int i; /* remove header and divide by 2 to move from UTF16 to UTF8 */ @@ -3664,7 +3823,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, */ ret = utf16s_to_utf8s(uc_str->uc, uc_str->len - QUERY_DESC_HDR_SIZE, - UTF16_BIG_ENDIAN, str, ascii_len); + UTF16_BIG_ENDIAN, str, ascii_len - 1); /* replace non-printable or non-ASCII characters with spaces */ for (i = 0; i < ret; i++) @@ -3673,7 +3832,7 @@ int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index, str[ret++] = '\0'; } else { - str = kmemdup(uc_str, uc_str->len, GFP_KERNEL); + str = kmemdup(uc_str->uc, uc_str->len, GFP_KERNEL); if (!str) { ret = -ENOMEM; goto out; @@ -3694,7 +3853,7 @@ out: * @param_read_buf: pointer to buffer where parameter would be read * @param_size: sizeof(param_read_buf) * - * Return 0 in case of success, non-zero otherwise + * Return: 0 in case of success; < 0 upon failure. */ static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba, int lun, @@ -3749,7 +3908,7 @@ static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba) * (UTMRDL) * 4. Allocate memory for local reference block(lrb). * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_memory_alloc(struct ufs_hba *hba) { @@ -3813,14 +3972,6 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba) } skip_utmrdl: - /* Allocate memory for local reference block */ - hba->lrb = devm_kcalloc(hba->dev, - hba->nutrs, sizeof(struct ufshcd_lrb), - GFP_KERNEL); - if (!hba->lrb) { - dev_err(hba->dev, "LRB Memory allocation failed\n"); - goto out; - } return 0; out: return -ENOMEM; @@ -3882,8 +4033,6 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) utrdlp[i].response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); } - - ufshcd_init_lrb(hba, &hba->lrb[i], i); } } @@ -3896,15 +4045,15 @@ static void ufshcd_host_memory_configure(struct ufs_hba *hba) * Once the Unipro links are up, the device connected to the controller * is detected. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_dme_link_startup(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_LINK_STARTUP, + }; int ret; - uic_cmd.command = UIC_CMD_DME_LINK_STARTUP; - ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_dbg(hba->dev, @@ -3918,15 +4067,15 @@ static int ufshcd_dme_link_startup(struct ufs_hba *hba) * DME_RESET command is issued in order to reset UniPro stack. * This function now deals with cold reset. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ -static int ufshcd_dme_reset(struct ufs_hba *hba) +int ufshcd_dme_reset(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_RESET, + }; int ret; - uic_cmd.command = UIC_CMD_DME_RESET; - ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_err(hba->dev, @@ -3934,6 +4083,7 @@ static int ufshcd_dme_reset(struct ufs_hba *hba) return ret; } +EXPORT_SYMBOL_GPL(ufshcd_dme_reset); int ufshcd_dme_configure_adapt(struct ufs_hba *hba, int agreed_gear, @@ -3957,15 +4107,15 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt); * * DME_ENABLE command is issued in order to enable UniPro stack. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ -static int ufshcd_dme_enable(struct ufs_hba *hba) +int ufshcd_dme_enable(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_ENABLE, + }; int ret; - uic_cmd.command = UIC_CMD_DME_ENABLE; - ret = ufshcd_send_uic_cmd(hba, &uic_cmd); if (ret) dev_err(hba->dev, @@ -3973,6 +4123,7 @@ static int ufshcd_dme_enable(struct ufs_hba *hba) return ret; } +EXPORT_SYMBOL_GPL(ufshcd_dme_enable); static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) { @@ -3998,11 +4149,16 @@ static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US - delta; else - return; /* no more delay required */ + min_sleep_time_us = 0; /* no more delay required */ } - /* allow sleep for extra 50us if needed */ - usleep_range(min_sleep_time_us, min_sleep_time_us + 50); + if (min_sleep_time_us > 0) { + /* allow sleep for extra 50us if needed */ + usleep_range(min_sleep_time_us, min_sleep_time_us + 50); + } + + /* update the last_dme_cmd_tstamp */ + hba->last_dme_cmd_tstamp = ktime_get(); } /** @@ -4013,12 +4169,17 @@ static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba) * @mib_val: setting value as uic command argument3 * @peer: indicate whether peer or local * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set, u32 mib_val, u8 peer) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET, + .argument1 = attr_sel, + .argument2 = UIC_ARG_ATTR_TYPE(attr_set), + .argument3 = mib_val, + }; static const char *const action[] = { "dme-set", "dme-peer-set" @@ -4027,12 +4188,6 @@ int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, int ret; int retries = UFS_UIC_COMMAND_RETRIES; - uic_cmd.command = peer ? - UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET; - uic_cmd.argument1 = attr_sel; - uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set); - uic_cmd.argument3 = mib_val; - do { /* for peer attributes we retry upon failure */ ret = ufshcd_send_uic_cmd(hba, &uic_cmd); @@ -4057,12 +4212,15 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr); * @mib_val: the value of the attribute as returned by the UIC command * @peer: indicate whether peer or local * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, u32 *mib_val, u8 peer) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET, + .argument1 = attr_sel, + }; static const char *const action[] = { "dme-get", "dme-peer-get" @@ -4096,10 +4254,6 @@ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, } } - uic_cmd.command = peer ? - UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET; - uic_cmd.argument1 = attr_sel; - do { /* for peer attributes we retry upon failure */ ret = ufshcd_send_uic_cmd(hba, &uic_cmd); @@ -4113,8 +4267,8 @@ int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, get, UIC_GET_ATTR_ID(attr_sel), UFS_UIC_COMMAND_RETRIES - retries); - if (mib_val && !ret) - *mib_val = uic_cmd.argument3; + if (mib_val) + *mib_val = ret == 0 ? uic_cmd.argument3 : 0; if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE) && pwr_mode_change) @@ -4125,6 +4279,30 @@ out: EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); /** + * ufshcd_dme_rmw - get modify set a DME attribute + * @hba: per adapter instance + * @mask: indicates which bits to clear from the value that has been read + * @val: actual value to write + * @attr: dme attribute + */ +int ufshcd_dme_rmw(struct ufs_hba *hba, u32 mask, + u32 val, u32 attr) +{ + u32 cfg = 0; + int err; + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(attr), &cfg); + if (err) + return err; + + cfg &= ~mask; + cfg |= (val & mask); + + return ufshcd_dme_set(hba, UIC_ARG_MIB(attr), cfg); +} +EXPORT_SYMBOL_GPL(ufshcd_dme_rmw); + +/** * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power * state) and waits for it to take effect. * @@ -4138,7 +4316,7 @@ EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr); * addition to normal UIC command completion Status (UCCS). This function only * returns after the relevant status bits indicate the completion. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) { @@ -4146,7 +4324,6 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) unsigned long flags; u8 status; int ret; - bool reenable_intr = false; mutex_lock(&hba->uic_cmd_mutex); ufshcd_add_delay_before_dme_cmd(hba); @@ -4157,17 +4334,9 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) goto out_unlock; } hba->uic_async_done = &uic_async_done; - if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) { - ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); - /* - * Make sure UIC command completion interrupt is disabled before - * issuing UIC command. - */ - wmb(); - reenable_intr = true; - } - ret = __ufshcd_send_uic_cmd(hba, cmd, false); + ufshcd_disable_intr(hba, UIC_COMMAND_COMPL); spin_unlock_irqrestore(hba->host->host_lock, flags); + ret = __ufshcd_send_uic_cmd(hba, cmd); if (ret) { dev_err(hba->dev, "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", @@ -4176,7 +4345,7 @@ static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) } if (!wait_for_completion_timeout(hba->uic_async_done, - msecs_to_jiffies(UIC_CMD_TIMEOUT))) { + msecs_to_jiffies(uic_cmd_timeout))) { dev_err(hba->dev, "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n", cmd->command, cmd->argument3); @@ -4209,9 +4378,7 @@ out: spin_lock_irqsave(hba->host->host_lock, flags); hba->active_uic_cmd = NULL; hba->uic_async_done = NULL; - if (reenable_intr) - ufshcd_enable_intr(hba, UIC_COMMAND_COMPL); - if (ret) { + if (ret && !hba->pm_op_in_progress) { ufshcd_set_link_broken(hba); ufshcd_schedule_eh_work(hba); } @@ -4219,6 +4386,39 @@ out_unlock: spin_unlock_irqrestore(hba->host->host_lock, flags); mutex_unlock(&hba->uic_cmd_mutex); + /* + * If the h8 exit fails during the runtime resume process, it becomes + * stuck and cannot be recovered through the error handler. To fix + * this, use link recovery instead of the error handler. + */ + if (ret && hba->pm_op_in_progress) + ret = ufshcd_link_recovery(hba); + + return ret; +} + +/** + * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result + * @hba: per adapter instance + * @uic_cmd: UIC command + * + * Return: 0 only if success. + */ +int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) +{ + int ret; + + if (uic_cmd->argument1 != UIC_ARG_MIB(PA_PWRMODE) || + uic_cmd->command != UIC_CMD_DME_SET) + return ufshcd_send_uic_cmd(hba, uic_cmd); + + if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD) + return 0; + + ufshcd_hold(hba); + ret = ufshcd_uic_pwr_ctrl(hba, uic_cmd); + ufshcd_release(hba); + return ret; } @@ -4228,11 +4428,15 @@ out_unlock: * @hba: per adapter instance * @mode: powr mode value * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_SET, + .argument1 = UIC_ARG_MIB(PA_PWRMODE), + .argument3 = mode, + }; int ret; if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) { @@ -4245,9 +4449,6 @@ int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) } } - uic_cmd.command = UIC_CMD_DME_SET; - uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE); - uic_cmd.argument3 = mode; ufshcd_hold(hba); ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); ufshcd_release(hba); @@ -4288,15 +4489,16 @@ EXPORT_SYMBOL_GPL(ufshcd_link_recovery); int ufshcd_uic_hibern8_enter(struct ufs_hba *hba) { - int ret; - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_HIBER_ENTER, + }; ktime_t start = ktime_get(); + int ret; ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE); - uic_cmd.command = UIC_CMD_DME_HIBER_ENTER; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); - trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter", + trace_ufshcd_profile_hibern8(hba, "enter", ktime_to_us(ktime_sub(ktime_get(), start)), ret); if (ret) @@ -4312,15 +4514,16 @@ EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter); int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) { - struct uic_command uic_cmd = {0}; + struct uic_command uic_cmd = { + .command = UIC_CMD_DME_HIBER_EXIT, + }; int ret; ktime_t start = ktime_get(); ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE); - uic_cmd.command = UIC_CMD_DME_HIBER_EXIT; ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); - trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit", + trace_ufshcd_profile_hibern8(hba, "exit", ktime_to_us(ktime_sub(ktime_get(), start)), ret); if (ret) { @@ -4337,40 +4540,32 @@ int ufshcd_uic_hibern8_exit(struct ufs_hba *hba) } EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit); -void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) +static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba) { - unsigned long flags; - bool update = false; - if (!ufshcd_is_auto_hibern8_supported(hba)) return; - spin_lock_irqsave(hba->host->host_lock, flags); - if (hba->ahit != ahit) { - hba->ahit = ahit; - update = true; - } - spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); +} + +void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit) +{ + const u32 cur_ahit = READ_ONCE(hba->ahit); + + if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit) + return; - if (update && - !pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { + WRITE_ONCE(hba->ahit, ahit); + if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) { ufshcd_rpm_get_sync(hba); ufshcd_hold(hba); - ufshcd_auto_hibern8_enable(hba); + ufshcd_configure_auto_hibern8(hba); ufshcd_release(hba); ufshcd_rpm_put_sync(hba); } } EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update); -void ufshcd_auto_hibern8_enable(struct ufs_hba *hba) -{ - if (!ufshcd_is_auto_hibern8_supported(hba)) - return; - - ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); -} - /** * ufshcd_init_pwr_info - setting the POR (power on reset) * values in hba power info @@ -4380,8 +4575,8 @@ static void ufshcd_init_pwr_info(struct ufs_hba *hba) { hba->pwr_info.gear_rx = UFS_PWM_G1; hba->pwr_info.gear_tx = UFS_PWM_G1; - hba->pwr_info.lane_rx = 1; - hba->pwr_info.lane_tx = 1; + hba->pwr_info.lane_rx = UFS_LANE_1; + hba->pwr_info.lane_tx = UFS_LANE_1; hba->pwr_info.pwr_rx = SLOWAUTO_MODE; hba->pwr_info.pwr_tx = SLOWAUTO_MODE; hba->pwr_info.hs_rate = 0; @@ -4390,6 +4585,8 @@ static void ufshcd_init_pwr_info(struct ufs_hba *hba) /** * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device * @hba: per-adapter instance + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) { @@ -4421,6 +4618,14 @@ static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) return -EINVAL; } + if (pwr_info->lane_rx != pwr_info->lane_tx) { + dev_err(hba->dev, "%s: asymmetric connected lanes. rx=%d, tx=%d\n", + __func__, + pwr_info->lane_rx, + pwr_info->lane_tx); + return -EINVAL; + } + /* * First, get the maximum gears of HS speed. * If a zero value, it means there is no HSGEAR capability. @@ -4533,9 +4738,6 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba, dev_err(hba->dev, "%s: power mode change failed %d\n", __func__, ret); } else { - ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, - pwr_mode); - memcpy(&hba->pwr_info, pwr_mode, sizeof(struct ufs_pa_layer_attr)); } @@ -4547,6 +4749,8 @@ static int ufshcd_change_power_mode(struct ufs_hba *hba, * ufshcd_config_pwr_mode - configure a new power mode * @hba: per-adapter instance * @desired_pwr_mode: desired power configuration + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_config_pwr_mode(struct ufs_hba *hba, struct ufs_pa_layer_attr *desired_pwr_mode) @@ -4562,6 +4766,10 @@ int ufshcd_config_pwr_mode(struct ufs_hba *hba, ret = ufshcd_change_power_mode(hba, &final_params); + if (!ret) + ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL, + &final_params); + return ret; } EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); @@ -4571,6 +4779,9 @@ EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode); * @hba: per-adapter instance * * Set fDeviceInit flag and poll until device toggles it. + * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_complete_dev_init(struct ufs_hba *hba) { @@ -4621,7 +4832,7 @@ out: * 3. Program UTRL and UTMRL base address * 4. Configure run-stop-registers * - * Returns 0 on success, non-zero value on failure + * Return: 0 if successful; < 0 upon failure. */ int ufshcd_make_hba_operational(struct ufs_hba *hba) { @@ -4648,12 +4859,6 @@ int ufshcd_make_hba_operational(struct ufs_hba *hba) REG_UTP_TASK_REQ_LIST_BASE_H); /* - * Make sure base address and interrupt setup are updated before - * enabling the run/stop registers below. - */ - wmb(); - - /* * UCRDY, UTMRLDY and UTRLRDY bits must be 1 */ reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); @@ -4675,20 +4880,14 @@ EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational); */ void ufshcd_hba_stop(struct ufs_hba *hba) { - unsigned long flags; int err; - /* - * Obtain the host lock to prevent that the controller is disabled - * while the UFS interrupt handler is active on another CPU. - */ - spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_disable_irq(hba); ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); - spin_unlock_irqrestore(hba->host->host_lock, flags); - err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, CONTROLLER_DISABLE, 10, 1); + ufshcd_enable_irq(hba); if (err) dev_err(hba->dev, "%s: Controller disable failed\n", __func__); } @@ -4702,55 +4901,48 @@ EXPORT_SYMBOL_GPL(ufshcd_hba_stop); * sequence kicks off. When controller is ready it will set * the Host Controller Enable bit to 1. * - * Returns 0 on success, non-zero value on failure + * Return: 0 on success, non-zero value on failure. */ static int ufshcd_hba_execute_hce(struct ufs_hba *hba) { - int retry_outer = 3; - int retry_inner; + int retry; -start: - if (ufshcd_is_hba_active(hba)) - /* change controller state to "reset state" */ - ufshcd_hba_stop(hba); + for (retry = 3; retry > 0; retry--) { + if (ufshcd_is_hba_active(hba)) + /* change controller state to "reset state" */ + ufshcd_hba_stop(hba); - /* UniPro link is disabled at this point */ - ufshcd_set_link_off(hba); + /* UniPro link is disabled at this point */ + ufshcd_set_link_off(hba); - ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); + ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE); - /* start controller initialization sequence */ - ufshcd_hba_start(hba); + /* start controller initialization sequence */ + ufshcd_hba_start(hba); - /* - * To initialize a UFS host controller HCE bit must be set to 1. - * During initialization the HCE bit value changes from 1->0->1. - * When the host controller completes initialization sequence - * it sets the value of HCE bit to 1. The same HCE bit is read back - * to check if the controller has completed initialization sequence. - * So without this delay the value HCE = 1, set in the previous - * instruction might be read back. - * This delay can be changed based on the controller. - */ - ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); + /* + * To initialize a UFS host controller HCE bit must be set to 1. + * During initialization the HCE bit value changes from 1->0->1. + * When the host controller completes initialization sequence + * it sets the value of HCE bit to 1. The same HCE bit is read back + * to check if the controller has completed initialization sequence. + * So without this delay the value HCE = 1, set in the previous + * instruction might be read back. + * This delay can be changed based on the controller. + */ + ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100); - /* wait for the host controller to complete initialization */ - retry_inner = 50; - while (!ufshcd_is_hba_active(hba)) { - if (retry_inner) { - retry_inner--; - } else { - dev_err(hba->dev, - "Controller enable failed\n"); - if (retry_outer) { - retry_outer--; - goto start; - } - return -EIO; - } - usleep_range(1000, 1100); + /* wait for the host controller to complete initialization */ + if (!ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE, + CONTROLLER_ENABLE, 1000, 50)) + break; + + dev_err(hba->dev, "Enabling the controller failed\n"); } + if (!retry) + return -EIO; + /* enable UIC related interrupts */ ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); @@ -4792,7 +4984,7 @@ EXPORT_SYMBOL_GPL(ufshcd_hba_enable); static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) { - int tx_lanes = 0, i, err = 0; + int tx_lanes, i, err = 0; if (!peer) ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), @@ -4847,7 +5039,7 @@ EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist); * ufshcd_link_startup - Initialize unipro link startup * @hba: per adapter instance * - * Returns 0 for success, non-zero in case of failure + * Return: 0 for success, non-zero in case of failure. */ static int ufshcd_link_startup(struct ufs_hba *hba) { @@ -4859,7 +5051,8 @@ static int ufshcd_link_startup(struct ufs_hba *hba) * If UFS device isn't active then we will have to issue link startup * 2 times to make sure the device state move to active. */ - if (!ufshcd_is_ufs_dev_active(hba)) + if (!(hba->quirks & UFSHCD_QUIRK_PERFORM_LINK_STARTUP_ONCE) && + !ufshcd_is_ufs_dev_active(hba)) link_startup_again = true; link_startup: @@ -4924,12 +5117,8 @@ link_startup: ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER); ret = ufshcd_make_hba_operational(hba); out: - if (ret) { + if (ret) dev_err(hba->dev, "link startup failed %d\n", ret); - ufshcd_print_host_state(hba); - ufshcd_print_pwr_info(hba); - ufshcd_print_evt_hist(hba); - } return ret; } @@ -4942,14 +5131,17 @@ out: * If the UTP layer at the device side is not initialized, it may * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations. + * + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_verify_dev_init(struct ufs_hba *hba) { int err = 0; int retries; - ufshcd_hold(hba); - mutex_lock(&hba->dev_cmd.lock); + ufshcd_dev_man_lock(hba); + for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, hba->nop_out_timeout); @@ -4959,8 +5151,8 @@ static int ufshcd_verify_dev_init(struct ufs_hba *hba) dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); } - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + + ufshcd_dev_man_unlock(hba); if (err) dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); @@ -5046,10 +5238,15 @@ static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev) desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP) hba->dev_info.is_lu_power_on_wp = true; - /* In case of RPMB LU, check if advanced RPMB mode is enabled */ - if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN && - desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) - hba->dev_info.b_advanced_rpmb_en = true; + /* In case of RPMB LU, check if advanced RPMB mode is enabled, and get region size */ + if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN) { + if (desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4)) + hba->dev_info.b_advanced_rpmb_en = true; + hba->dev_info.rpmb_region_size[0] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION0_SIZE]; + hba->dev_info.rpmb_region_size[1] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION1_SIZE]; + hba->dev_info.rpmb_region_size[2] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION2_SIZE]; + hba->dev_info.rpmb_region_size[3] = desc_buf[RPMB_UNIT_DESC_PARAM_REGION3_SIZE]; + } kfree(desc_buf); @@ -5063,12 +5260,12 @@ set_qdepth: } /** - * ufshcd_slave_alloc - handle initial SCSI device configurations + * ufshcd_sdev_init - handle initial SCSI device configurations * @sdev: pointer to SCSI device * - * Returns success + * Return: success. */ -static int ufshcd_slave_alloc(struct scsi_device *sdev) +static int ufshcd_sdev_init(struct scsi_device *sdev) { struct ufs_hba *hba; @@ -5102,46 +5299,29 @@ static int ufshcd_slave_alloc(struct scsi_device *sdev) * @depth: required depth to set * * Change queue depth and make sure the max. limits are not crossed. + * + * Return: new queue depth. */ static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth) { return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue)); } -static void ufshcd_hpb_destroy(struct ufs_hba *hba, struct scsi_device *sdev) -{ - /* skip well-known LU */ - if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || - !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) - return; - - ufshpb_destroy_lu(hba, sdev); -} - -static void ufshcd_hpb_configure(struct ufs_hba *hba, struct scsi_device *sdev) -{ - /* skip well-known LU */ - if ((sdev->lun >= UFS_UPIU_MAX_UNIT_NUM_ID) || - !(hba->dev_info.hpb_enabled) || !ufshpb_is_allowed(hba)) - return; - - ufshpb_init_hpb_lu(hba, sdev); -} - /** - * ufshcd_slave_configure - adjust SCSI device configurations + * ufshcd_sdev_configure - adjust SCSI device configurations * @sdev: pointer to SCSI device + * @lim: queue limits + * + * Return: 0 (success). */ -static int ufshcd_slave_configure(struct scsi_device *sdev) +static int ufshcd_sdev_configure(struct scsi_device *sdev, + struct queue_limits *lim) { struct ufs_hba *hba = shost_priv(sdev->host); struct request_queue *q = sdev->request_queue; - ufshcd_hpb_configure(hba, sdev); + lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1; - blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); - if (hba->quirks & UFSHCD_QUIRK_4KB_DMA_ALIGNMENT) - blk_queue_update_dma_alignment(q, SZ_4K - 1); /* * Block runtime-pm until all consumers are added. * Refer ufshcd_setup_links(). @@ -5157,24 +5337,25 @@ static int ufshcd_slave_configure(struct scsi_device *sdev) */ sdev->silence_suspend = 1; + if (hba->vops && hba->vops->config_scsi_dev) + hba->vops->config_scsi_dev(sdev); + ufshcd_crypto_register(hba, q); return 0; } /** - * ufshcd_slave_destroy - remove SCSI device configurations + * ufshcd_sdev_destroy - remove SCSI device configurations * @sdev: pointer to SCSI device */ -static void ufshcd_slave_destroy(struct scsi_device *sdev) +static void ufshcd_sdev_destroy(struct scsi_device *sdev) { struct ufs_hba *hba; unsigned long flags; hba = shost_priv(sdev->host); - ufshcd_hpb_destroy(hba, sdev); - /* Drop the reference as it won't be needed anymore */ if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) { spin_lock_irqsave(hba->host->host_lock, flags); @@ -5205,19 +5386,18 @@ static void ufshcd_slave_destroy(struct scsi_device *sdev) /** * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status - * @lrbp: pointer to local reference block of completed command + * @cmd: SCSI command * @scsi_status: SCSI command status * - * Returns value base on SCSI command status + * Return: value base on SCSI command status. */ -static inline int -ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) +static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status) { int result = 0; switch (scsi_status) { case SAM_STAT_CHECK_CONDITION: - ufshcd_copy_sense_data(lrbp); + ufshcd_copy_sense_data(cmd); fallthrough; case SAM_STAT_GOOD: result |= DID_OK << 16 | scsi_status; @@ -5225,7 +5405,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) case SAM_STAT_TASK_SET_FULL: case SAM_STAT_BUSY: case SAM_STAT_TASK_ABORTED: - ufshcd_copy_sense_data(lrbp); + ufshcd_copy_sense_data(cmd); result |= scsi_status; break; default: @@ -5239,49 +5419,52 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status) /** * ufshcd_transfer_rsp_status - Get overall status of the response * @hba: per adapter instance - * @lrbp: pointer to local reference block of completed command + * @cmd: SCSI command * @cqe: pointer to the completion queue entry * - * Returns result of the command to notify SCSI midlayer + * Return: result of the command to notify SCSI midlayer. */ -static inline int -ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, - struct cq_entry *cqe) +static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba, + struct scsi_cmnd *cmd, + struct cq_entry *cqe) { + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + const int tag = scsi_cmd_to_rq(cmd)->tag; int result = 0; int scsi_status; enum utp_ocs ocs; + u8 upiu_flags; + u32 resid; - scsi_set_resid(lrbp->cmd, - be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count)); + upiu_flags = lrbp->ucd_rsp_ptr->header.flags; + resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count); + /* + * Test !overflow instead of underflow to support UFS devices that do + * not set either flag. + */ + if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW)) + scsi_set_resid(cmd, resid); /* overall command status of utrd */ ocs = ufshcd_get_tr_ocs(lrbp, cqe); if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) { - if (be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_1) & - MASK_RSP_UPIU_RESULT) + if (lrbp->ucd_rsp_ptr->header.response || + lrbp->ucd_rsp_ptr->header.status) ocs = OCS_SUCCESS; } switch (ocs) { case OCS_SUCCESS: - result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr); hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0); - switch (result) { + switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) { case UPIU_TRANSACTION_RESPONSE: /* - * get the response UPIU result to extract - * the SCSI command status - */ - result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); - - /* * get the result based on SCSI status response * to notify the SCSI midlayer of the command status */ - scsi_status = result & MASK_SCSI_STATUS; - result = ufshcd_scsi_cmd_status(lrbp, scsi_status); + scsi_status = lrbp->ucd_rsp_ptr->header.status; + result = ufshcd_scsi_cmd_status(cmd, scsi_status); /* * Currently we are only supporting BKOPs exception @@ -5300,9 +5483,6 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, ufshcd_is_exception_event(lrbp->ucd_rsp_ptr)) /* Flushed in suspend */ schedule_work(&hba->eeh_work); - - if (scsi_status == SAM_STAT_GOOD) - ufshpb_rsp_upiu(hba, lrbp); break; case UPIU_TRANSACTION_REJECT_UPIU: /* TODO: handle Reject UPIU Response */ @@ -5319,10 +5499,10 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, } break; case OCS_ABORTED: - result |= DID_ABORT << 16; - break; case OCS_INVALID_COMMAND_STATUS: result |= DID_REQUEUE << 16; + dev_warn(hba->dev, "OCS %s from controller for tag %d\n", + ocs == OCS_ABORTED ? "aborted" : "invalid", tag); break; case OCS_INVALID_CMD_TABLE_ATTR: case OCS_INVALID_PRDT_ATTR: @@ -5335,17 +5515,19 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, case OCS_GENERAL_CRYPTO_ERROR: default: result |= DID_ERROR << 16; - dev_err(hba->dev, - "OCS error from controller = %x for tag %d\n", - ocs, lrbp->task_tag); + dev_err(hba->dev, "OCS error from controller = %x for tag %d\n", + ocs, tag); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); break; } /* end of switch */ if ((host_byte(result) != DID_OK) && - (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) - ufshcd_print_tr(hba, lrbp->task_tag, true); + (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs) { + if (cqe) + ufshcd_hex_dump("UPIU CQE: ", cqe, sizeof(struct cq_entry)); + ufshcd_print_tr(hba, cmd, true); + } return result; } @@ -5372,49 +5554,49 @@ static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba, * @hba: per adapter instance * @intr_status: interrupt status generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status) { irqreturn_t retval = IRQ_NONE; + struct uic_command *cmd; + + guard(spinlock_irqsave)(hba->host->host_lock); + cmd = hba->active_uic_cmd; + if (!cmd) + return retval; - spin_lock(hba->host->host_lock); if (ufshcd_is_auto_hibern8_error(hba, intr_status)) hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status); - if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) { - hba->active_uic_cmd->argument2 |= - ufshcd_get_uic_cmd_result(hba); - hba->active_uic_cmd->argument3 = - ufshcd_get_dme_attr_val(hba); + if (intr_status & UIC_COMMAND_COMPL) { + cmd->argument2 |= ufshcd_get_uic_cmd_result(hba); + cmd->argument3 = ufshcd_get_dme_attr_val(hba); if (!hba->uic_async_done) - hba->active_uic_cmd->cmd_active = 0; - complete(&hba->active_uic_cmd->done); + cmd->cmd_active = false; + complete(&cmd->done); retval = IRQ_HANDLED; } - if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done) { - hba->active_uic_cmd->cmd_active = 0; + if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) { + cmd->cmd_active = false; complete(hba->uic_async_done); retval = IRQ_HANDLED; } if (retval == IRQ_HANDLED) - ufshcd_add_uic_command_trace(hba, hba->active_uic_cmd, - UFS_CMD_COMP); - spin_unlock(hba->host->host_lock); + ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP); + return retval; } /* Release the resources allocated for processing a SCSI command. */ -void ufshcd_release_scsi_cmd(struct ufs_hba *hba, - struct ufshcd_lrb *lrbp) +void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd) { - struct scsi_cmnd *cmd = lrbp->cmd; - scsi_dma_unmap(cmd); + ufshcd_crypto_clear_prdt(hba, cmd); ufshcd_release(hba); ufshcd_clk_scaling_update_busy(hba); } @@ -5428,33 +5610,39 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba, void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag, struct cq_entry *cqe) { - struct ufshcd_lrb *lrbp; - struct scsi_cmnd *cmd; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); enum utp_ocs ocs; - lrbp = &hba->lrb[task_tag]; - lrbp->compl_time_stamp = ktime_get(); - cmd = lrbp->cmd; - if (cmd) { - if (unlikely(ufshcd_should_inform_monitor(hba, lrbp))) - ufshcd_update_monitor(hba, lrbp); - ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP); - cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe); - ufshcd_release_scsi_cmd(hba, lrbp); - /* Do not touch lrbp after scsi done */ - scsi_done(cmd); - } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE || - lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) { - if (hba->dev_cmd.complete) { - if (cqe) { - ocs = le32_to_cpu(cqe->status) & MASK_OCS; - lrbp->utr_descriptor_ptr->header.dword_2 = - cpu_to_le32(ocs); - } - complete(hba->dev_cmd.complete); - ufshcd_clk_scaling_update_busy(hba); + if (WARN_ONCE(!cmd, "cqe->command_desc_base_addr = %#llx\n", + le64_to_cpu(cqe->command_desc_base_addr))) + return; + + if (hba->monitor.enabled) { + lrbp->compl_time_stamp = ktime_get(); + lrbp->compl_time_stamp_local_clock = local_clock(); + } + if (ufshcd_is_scsi_cmd(cmd)) { + if (unlikely(ufshcd_should_inform_monitor(hba, cmd))) + ufshcd_update_monitor(hba, cmd); + ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP); + cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe); + ufshcd_release_scsi_cmd(hba, cmd); + } else { + if (cqe) { + ocs = cqe->overall_status & MASK_OCS; + lrbp->utr_descriptor_ptr->header.ocs = ocs; + } else { + ocs = lrbp->utr_descriptor_ptr->header.ocs; } + ufshcd_add_query_upiu_trace( + hba, + ocs == OCS_SUCCESS ? UFS_QUERY_COMP : UFS_QUERY_ERR, + (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); + cmd->result = 0; } + /* Do not touch lrbp after scsi_done() has been called. */ + scsi_done(cmd); } /** @@ -5482,7 +5670,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba, int tag; for_each_set_bit(tag, completed_reqs, hba->nutrs) { - struct scsi_cmnd *cmd = hba->lrb[tag].cmd; + struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag); if (!cmd) continue; @@ -5492,7 +5680,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba, } /* - * Returns > 0 if one or more commands have been completed or 0 if no + * Return: > 0 if one or more commands have been completed or 0 if no * requests have been completed. */ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) @@ -5502,7 +5690,7 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) u32 tr_doorbell; struct ufs_hw_queue *hwq; - if (is_mcq_enabled(hba)) { + if (hba->mcq_enabled) { hwq = &hba->uhq[queue_num]; return ufshcd_mcq_poll_cqe_lock(hba, hwq); @@ -5527,6 +5715,47 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) return completed_reqs != 0; } +static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) +{ + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); + struct scsi_device *sdev = rq->q->queuedata; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); + + if (blk_mq_is_reserved_rq(rq) || !hwq) + return true; + + ufshcd_mcq_compl_all_cqes_lock(hba, hwq); + + /* + * For those cmds of which the cqes are not present in the cq, complete + * them explicitly. + */ + scoped_guard(spinlock_irqsave, &hwq->cq_lock) { + if (!test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { + set_host_byte(cmd, DID_REQUEUE); + ufshcd_release_scsi_cmd(hba, cmd); + scsi_done(cmd); + } + } + + return true; +} + +static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) +{ + struct scsi_device *sdev = rq->q->queuedata; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); + + if (!blk_mq_is_reserved_rq(rq) && hwq) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + + return true; +} + /** * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is * invoked from the error handler context or ufshcd_host_reset_and_restore() @@ -5541,48 +5770,17 @@ static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num) static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, bool force_compl) { - struct ufs_hw_queue *hwq; - struct ufshcd_lrb *lrbp; - struct scsi_cmnd *cmd; - unsigned long flags; - u32 hwq_num, utag; - int tag; - - for (tag = 0; tag < hba->nutrs; tag++) { - lrbp = &hba->lrb[tag]; - cmd = lrbp->cmd; - if (!ufshcd_cmd_inflight(cmd) || - test_bit(SCMD_STATE_COMPLETE, &cmd->state)) - continue; - - utag = blk_mq_unique_tag(scsi_cmd_to_rq(cmd)); - hwq_num = blk_mq_unique_tag_to_hwq(utag); - hwq = &hba->uhq[hwq_num]; - - if (force_compl) { - ufshcd_mcq_compl_all_cqes_lock(hba, hwq); - /* - * For those cmds of which the cqes are not present - * in the cq, complete them explicitly. - */ - if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) { - spin_lock_irqsave(&hwq->cq_lock, flags); - set_host_byte(cmd, DID_REQUEUE); - ufshcd_release_scsi_cmd(hba, lrbp); - scsi_done(cmd); - spin_unlock_irqrestore(&hwq->cq_lock, flags); - } - } else { - ufshcd_mcq_poll_cqe_lock(hba, hwq); - } - } + blk_mq_tagset_busy_iter(&hba->host->tag_set, + force_compl ? ufshcd_mcq_force_compl_one : + ufshcd_mcq_compl_one, + NULL); } /** * ufshcd_transfer_req_compl - handle SCSI and query command completion * @hba: per adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -5599,7 +5797,7 @@ static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR)) ufshcd_reset_intr_aggr(hba); - if (ufs_fail_completion()) + if (ufs_fail_completion(hba)) return IRQ_HANDLED; /* @@ -5659,7 +5857,7 @@ int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask, * Disables exception event in the device so that the EVENT_ALERT * bit is not set. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) { @@ -5674,7 +5872,7 @@ static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask) * Enable corresponding exception event in the device to allow * device to alert host in critical scenarios. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) { @@ -5690,7 +5888,8 @@ static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask) * as the device is allowed to manage its own way of handling background * operations. * - * Returns zero on success, non-zero on failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) { @@ -5708,7 +5907,7 @@ static int ufshcd_enable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = true; - trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled"); + trace_ufshcd_auto_bkops_state(hba, "Enabled"); /* No need of URGENT_BKOPS exception from the device */ err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS); @@ -5729,7 +5928,8 @@ out: * host is idle so that BKOPS are managed effectively without any negative * impacts. * - * Returns zero on success, non-zero on failure. + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) { @@ -5759,7 +5959,7 @@ static int ufshcd_disable_auto_bkops(struct ufs_hba *hba) } hba->auto_bkops_enabled = false; - trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled"); + trace_ufshcd_auto_bkops_state(hba, "Disabled"); hba->is_urgent_bkops_lvl_checked = false; out: return err; @@ -5798,24 +5998,23 @@ static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status) /** * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status * @hba: per-adapter instance - * @status: bkops_status value * * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn * flag in the device to permit background operations if the device - * bkops_status is greater than or equal to "status" argument passed to - * this function, disable otherwise. + * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl", + * disable otherwise. * - * Returns 0 for success, non-zero in case of failure. + * Return: 0 for success, non-zero in case of failure. * * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag * to know whether auto bkops is enabled or disabled after this function * returns control to it. */ -static int ufshcd_bkops_ctrl(struct ufs_hba *hba, - enum bkops_status status) +static int ufshcd_bkops_ctrl(struct ufs_hba *hba) { - int err; + enum bkops_status status = hba->urgent_bkops_lvl; u32 curr_status = 0; + int err; err = ufshcd_get_bkops_status(hba, &curr_status); if (err) { @@ -5837,21 +6036,6 @@ out: return err; } -/** - * ufshcd_urgent_bkops - handle urgent bkops exception event - * @hba: per-adapter instance - * - * Enable fBackgroundOpsEn flag in the device to permit background - * operations. - * - * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled - * and negative error value for any other failure. - */ -static int ufshcd_urgent_bkops(struct ufs_hba *hba) -{ - return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl); -} - static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status) { return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, @@ -5895,22 +6079,44 @@ out: __func__, err); } -static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status) +/* + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. + */ +int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id) { - u32 value; + struct utp_upiu_query_v4_0 *upiu_resp; + struct ufs_query_req *request = NULL; + struct ufs_query_res *response = NULL; + int err; - if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, - QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value)) - return; + if (hba->dev_info.wspecversion < 0x410) + return -EOPNOTSUPP; - dev_info(hba->dev, "exception Tcase %d\n", value - 80); + ufshcd_hold(hba); + mutex_lock(&hba->dev_cmd.lock); - ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); + ufshcd_init_query(hba, &request, &response, + UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0); - /* - * A placeholder for the platform vendors to add whatever additional - * steps required - */ + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); + + if (err) { + dev_err(hba->dev, "%s: failed to read device level exception %d\n", + __func__, err); + goto out; + } + + upiu_resp = (struct utp_upiu_query_v4_0 *)response; + *exception_id = get_unaligned_be64(&upiu_resp->osf3); +out: + mutex_unlock(&hba->dev_cmd.lock); + ufshcd_release(hba); + + return err; } static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn) @@ -5983,7 +6189,22 @@ int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable) return ret; } -static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba, +int ufshcd_wb_set_resize_en(struct ufs_hba *hba, enum wb_resize_en en_mode) +{ + int ret; + u8 index; + + index = ufshcd_wb_get_query_index(hba); + ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_WB_BUF_RESIZE_EN, index, 0, &en_mode); + if (ret) + dev_err(hba->dev, "%s: Enable WB buf resize operation failed %d\n", + __func__, ret); + + return ret; +} + +static bool ufshcd_wb_curr_buff_threshold_check(struct ufs_hba *hba, u32 avail_buf) { u32 cur_buf; @@ -6065,15 +6286,13 @@ static bool ufshcd_wb_need_flush(struct ufs_hba *hba) } /* - * The ufs device needs the vcc to be ON to flush. * With user-space reduction enabled, it's enough to enable flush * by checking only the available buffer. The threshold * defined here is > 90% full. * With user-space preserved enabled, the current-buffer * should be checked too because the wb buffer size can reduce * when disk tends to be full. This info is provided by current - * buffer (dCurrentWriteBoosterBufferSize). There's no point in - * keeping vcc on when current buffer is empty. + * buffer (dCurrentWriteBoosterBufferSize). */ index = ufshcd_wb_get_query_index(hba); ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, @@ -6088,7 +6307,7 @@ static bool ufshcd_wb_need_flush(struct ufs_hba *hba) if (!hba->dev_info.b_presrv_uspc_en) return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10); - return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf); + return ufshcd_wb_curr_buff_threshold_check(hba, avail_buf); } static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work) @@ -6120,31 +6339,38 @@ static void ufshcd_exception_event_handler(struct work_struct *work) u32 status = 0; hba = container_of(work, struct ufs_hba, eeh_work); - ufshcd_scsi_block_requests(hba); err = ufshcd_get_ee_status(hba, &status); if (err) { dev_err(hba->dev, "%s: failed to get exception status %d\n", __func__, err); - goto out; + return; } - trace_ufshcd_exception_event(dev_name(hba->dev), status); + trace_ufshcd_exception_event(hba, status); if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS) ufshcd_bkops_exception_event_handler(hba); if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP) - ufshcd_temp_exception_event_handler(hba, status); + ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP); + + if (status & hba->ee_drv_mask & MASK_EE_HEALTH_CRITICAL) { + hba->critical_health_count++; + sysfs_notify(&hba->dev->kobj, NULL, "critical_health"); + } + + if (status & hba->ee_drv_mask & MASK_EE_DEV_LVL_EXCEPTION) { + atomic_inc(&hba->dev_lvl_exception_count); + sysfs_notify(&hba->dev->kobj, NULL, "device_lvl_exception_count"); + } ufs_debugfs_exception_event(hba, status); -out: - ufshcd_scsi_unblock_requests(hba); } /* Complete requests that have door-bell cleared */ static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) { - if (is_mcq_enabled(hba)) + if (hba->mcq_enabled) ufshcd_mcq_compl_pending_transfer(hba, force_compl); else ufshcd_transfer_req_compl(hba); @@ -6157,7 +6383,7 @@ static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl) * to recover from the DL NAC errors or not. * @hba: per-adapter instance * - * Returns true if error handling is required, false otherwise + * Return: true if error handling is required, false otherwise. */ static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba) { @@ -6245,13 +6471,14 @@ void ufshcd_schedule_eh_work(struct ufs_hba *hba) } } -static void ufshcd_force_error_recovery(struct ufs_hba *hba) +void ufshcd_force_error_recovery(struct ufs_hba *hba) { spin_lock_irq(hba->host->host_lock); hba->force_reset = true; ufshcd_schedule_eh_work(hba); spin_unlock_irq(hba->host->host_lock); } +EXPORT_SYMBOL_GPL(ufshcd_force_error_recovery); static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow) { @@ -6295,7 +6522,6 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) ufshcd_hold(hba); if (!ufshcd_is_clkgating_allowed(hba)) ufshcd_setup_clocks(hba, true); - ufshcd_release(hba); pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM; ufshcd_vops_resume(hba, pm_op); } else { @@ -6305,15 +6531,14 @@ static void ufshcd_err_handling_prepare(struct ufs_hba *hba) ufshcd_suspend_clkscaling(hba); ufshcd_clk_scaling_allow(hba, false); } - ufshcd_scsi_block_requests(hba); /* Wait for ongoing ufshcd_queuecommand() calls to finish. */ - blk_mq_wait_quiesce_done(&hba->host->tag_set); + blk_mq_quiesce_tagset(&hba->host->tag_set); cancel_work_sync(&hba->eeh_work); } static void ufshcd_err_handling_unprepare(struct ufs_hba *hba) { - ufshcd_scsi_unblock_requests(hba); + blk_mq_unquiesce_tagset(&hba->host->tag_set); ufshcd_release(hba); if (ufshcd_is_clkscaling_supported(hba)) ufshcd_clk_scaling_suspend(hba, false); @@ -6384,54 +6609,52 @@ static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba) return false; } +static bool ufshcd_abort_one(struct request *rq, void *priv) +{ + int *ret = priv; + u32 tag = rq->tag; + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq); + struct scsi_device *sdev = cmd->device; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + + if (blk_mq_is_reserved_rq(rq)) + return true; + + *ret = ufshcd_try_to_abort_task(hba, tag); + dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, + ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1, + *ret ? "failed" : "succeeded"); + + return *ret == 0; +} + +/** + * ufshcd_abort_all - Abort all pending commands. + * @hba: Host bus adapter pointer. + * + * Return: true if and only if the host controller needs to be reset. + */ static bool ufshcd_abort_all(struct ufs_hba *hba) { - bool needs_reset = false; - int tag, ret; + int tag, ret = 0; - if (is_mcq_enabled(hba)) { - struct ufshcd_lrb *lrbp; - int tag; + blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret); + if (ret) + goto out; - for (tag = 0; tag < hba->nutrs; tag++) { - lrbp = &hba->lrb[tag]; - if (!ufshcd_cmd_inflight(lrbp->cmd)) - continue; - ret = ufshcd_try_to_abort_task(hba, tag); - dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, - ret ? "failed" : "succeeded"); - if (ret) { - needs_reset = true; - goto out; - } - } - } else { - /* Clear pending transfer requests */ - for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) { - ret = ufshcd_try_to_abort_task(hba, tag); - dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag, - hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1, - ret ? "failed" : "succeeded"); - if (ret) { - needs_reset = true; - goto out; - } - } - } /* Clear pending task management requests */ for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) { - if (ufshcd_clear_tm_cmd(hba, tag)) { - needs_reset = true; + ret = ufshcd_clear_tm_cmd(hba, tag); + if (ret) goto out; - } } out: /* Complete the requests that are cleared by s/w */ ufshcd_complete_requests(hba, false); - return needs_reset; + return ret != 0; } /** @@ -6450,12 +6673,26 @@ static void ufshcd_err_handler(struct work_struct *work) hba = container_of(work, struct ufs_hba, eh_work); dev_info(hba->dev, - "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n", + "%s started; HBA state %s; powered %d; shutting down %d; saved_err = 0x%x; saved_uic_err = 0x%x; force_reset = %d%s\n", __func__, ufshcd_state_name[hba->ufshcd_state], hba->is_powered, hba->shutting_down, hba->saved_err, hba->saved_uic_err, hba->force_reset, ufshcd_is_link_broken(hba) ? "; link is broken" : ""); + /* + * Use ufshcd_rpm_get_noresume() here to safely perform link recovery + * even if an error occurs during runtime suspend or runtime resume. + * This avoids potential deadlocks that could happen if we tried to + * resume the device while a PM operation is already in progress. + */ + ufshcd_rpm_get_noresume(hba); + if (hba->pm_op_in_progress) { + ufshcd_link_recovery(hba); + ufshcd_rpm_put(hba); + return; + } + ufshcd_rpm_put(hba); + down(&hba->host_sem); spin_lock_irqsave(hba->host->host_lock, flags); if (ufshcd_err_handling_should_stop(hba)) { @@ -6465,9 +6702,14 @@ static void ufshcd_err_handler(struct work_struct *work) up(&hba->host_sem); return; } - ufshcd_set_eh_in_progress(hba); spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_err_handling_prepare(hba); + + spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_set_eh_in_progress(hba); + spin_unlock_irqrestore(hba->host->host_lock, flags); + /* Complete requests that have door-bell cleared by h/w */ ufshcd_complete_requests(hba, false); spin_lock_irqsave(hba->host->host_lock, flags); @@ -6484,7 +6726,8 @@ again: if (ufshcd_err_handling_should_stop(hba)) goto skip_err_handling; - if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) { + if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) && + !hba->force_reset) { bool ret; spin_unlock_irqrestore(hba->host->host_lock, flags); @@ -6618,7 +6861,7 @@ skip_err_handling: * ufshcd_update_uic_error - check and set fatal UIC error flags. * @hba: per-adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6711,7 +6954,7 @@ static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba) * @hba: per-adapter instance * @intr_status: interrupt status generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6720,7 +6963,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) bool queue_eh_work = false; irqreturn_t retval = IRQ_NONE; - spin_lock(hba->host->host_lock); + guard(spinlock_irqsave)(hba->host->host_lock); hba->errors |= UFSHCD_ERROR_MASK & intr_status; if (hba->errors & INT_FATAL_ERRORS) { @@ -6779,7 +7022,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) */ hba->errors = 0; hba->uic_error = 0; - spin_unlock(hba->host->host_lock); + return retval; } @@ -6787,7 +7030,7 @@ static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status) * ufshcd_tmc_handler - handle task management function completion * @hba: per adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6816,7 +7059,7 @@ static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba) * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events * @hba: per adapter instance * - * Returns IRQ_HANDLED if interrupt is handled + * Return: IRQ_HANDLED if interrupt is handled. */ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) { @@ -6851,7 +7094,7 @@ static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba) * @hba: per adapter instance * @intr_status: contains interrupts generated by the controller * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ @@ -6878,24 +7121,22 @@ static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) } /** - * ufshcd_intr - Main interrupt service routine + * ufshcd_threaded_intr - Threaded interrupt service routine * @irq: irq number * @__hba: pointer to adapter instance * - * Returns + * Return: * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ -static irqreturn_t ufshcd_intr(int irq, void *__hba) +static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) { - u32 intr_status, enabled_intr_status = 0; + u32 last_intr_status, intr_status, enabled_intr_status = 0; irqreturn_t retval = IRQ_NONE; struct ufs_hba *hba = __hba; int retries = hba->nutrs; - intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); - hba->ufs_stats.last_intr_status = intr_status; - hba->ufs_stats.last_intr_ts = local_clock(); + last_intr_status = intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); /* * There could be max of hba->nutrs reqs in flight and in worst case @@ -6919,7 +7160,7 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba) dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n", __func__, intr_status, - hba->ufs_stats.last_intr_status, + last_intr_status, enabled_intr_status); ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: "); } @@ -6927,18 +7168,43 @@ static irqreturn_t ufshcd_intr(int irq, void *__hba) return retval; } +/** + * ufshcd_intr - Main interrupt service routine + * @irq: irq number + * @__hba: pointer to adapter instance + * + * Return: + * IRQ_HANDLED - If interrupt is valid + * IRQ_WAKE_THREAD - If handling is moved to threaded handled + * IRQ_NONE - If invalid interrupt + */ +static irqreturn_t ufshcd_intr(int irq, void *__hba) +{ + struct ufs_hba *hba = __hba; + u32 intr_status, enabled_intr_status; + + /* Move interrupt handling to thread when MCQ & ESI are not enabled */ + if (!hba->mcq_enabled || !hba->mcq_esi_enabled) + return IRQ_WAKE_THREAD; + + intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); + enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + + ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); + + /* Directly handle interrupts since MCQ ESI handlers does the hard job */ + return ufshcd_sl_intr(hba, enabled_intr_status); +} + static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) { int err = 0; u32 mask = 1 << tag; - unsigned long flags; if (!test_bit(tag, &hba->outstanding_tasks)) goto out; - spin_lock_irqsave(hba->host->host_lock, flags); ufshcd_utmrl_clear(hba, tag); - spin_unlock_irqrestore(hba->host->host_lock, flags); /* poll for max. 1 sec to clear door bell register by h/w */ err = ufshcd_wait_for_register(hba, @@ -6946,7 +7212,7 @@ static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) mask, 0, 1000, 1000); dev_err(hba->dev, "Clearing task management function with tag %d %s\n", - tag, err ? "succeeded" : "failed"); + tag, err < 0 ? "failed" : "succeeded"); out: return err; @@ -6975,23 +7241,19 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, spin_lock_irqsave(host->host_lock, flags); task_tag = req->tag; - WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag %d\n", - task_tag); hba->tmf_rqs[req->tag] = req; - treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag); + treq->upiu_req.req_header.task_tag = task_tag; memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq)); ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function); - /* send command to the controller */ __set_bit(task_tag, &hba->outstanding_tasks); - ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); - /* Make sure that doorbell is committed immediately */ - wmb(); - spin_unlock_irqrestore(host->host_lock, flags); + /* send command to the controller */ + ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL); + ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND); /* wait until the task management command is completed */ @@ -7031,23 +7293,23 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba, * @tm_function: task management function opcode * @tm_response: task management service response return value * - * Returns non-zero value on error, zero on success. + * Return: non-zero value on error, zero on success. */ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, u8 tm_function, u8 *tm_response) { - struct utp_task_req_desc treq = { { 0 }, }; + struct utp_task_req_desc treq = { }; enum utp_ocs ocs_value; int err; /* Configure task request descriptor */ - treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); - treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); + treq.header.interrupt = 1; + treq.header.ocs = OCS_INVALID_COMMAND_STATUS; /* Configure task request UPIU */ - treq.upiu_req.req_header.dword_0 = cpu_to_be32(lun_id << 8) | - cpu_to_be32(UPIU_TRANSACTION_TASK_REQ << 24); - treq.upiu_req.req_header.dword_1 = cpu_to_be32(tm_function << 16); + treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ; + treq.upiu_req.req_header.lun = lun_id; + treq.upiu_req.req_header.tm_function = tm_function; /* * The host shall provide the same value for LUN field in the basic @@ -7060,7 +7322,7 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, if (err == -ETIMEDOUT) return err; - ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; + ocs_value = treq.header.ocs & MASK_OCS; if (ocs_value != OCS_SUCCESS) dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, ocs_value); @@ -7086,6 +7348,8 @@ static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id, * * Since there is only one available tag for device management commands, * the caller is expected to hold the hba->dev_cmd.lock mutex. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, @@ -7094,34 +7358,26 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, enum query_opcode desc_op) { - DECLARE_COMPLETION_ONSTACK(wait); - const u32 tag = hba->reserved_slot; - struct ufshcd_lrb *lrbp; + struct scsi_cmnd *cmd = ufshcd_get_dev_mgmt_cmd(hba); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + u32 tag; int err = 0; u8 upiu_flags; - /* Protects use of hba->reserved_slot. */ + /* Protects use of hba->dev_cmd. */ lockdep_assert_held(&hba->dev_cmd.lock); - down_read(&hba->clk_scaling_lock); + if (WARN_ON_ONCE(!cmd)) + return -ENOMEM; - lrbp = &hba->lrb[tag]; - lrbp->cmd = NULL; - lrbp->task_tag = tag; - lrbp->lun = 0; - lrbp->intr_cmd = true; - ufshcd_prepare_lrbp_crypto(NULL, lrbp); - hba->dev_cmd.type = cmd_type; + tag = scsi_cmd_to_rq(cmd)->tag; - if (hba->ufs_version <= ufshci_version(1, 1)) - lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE; - else - lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag); - /* update the task tag in the request upiu */ - req_upiu->header.dword_0 |= cpu_to_be32(tag); + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0); - ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE, 0); + /* update the task tag in the request upiu */ + req_upiu->header.task_tag = tag; /* just copy the upiu request as it is */ memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); @@ -7136,24 +7392,16 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); - hba->dev_cmd.complete = &wait; - - ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr); - - ufshcd_send_command(hba, tag, hba->dev_cmd_queue); - /* - * ignore the returning value here - ufshcd_check_query_response is - * bound to fail since dev_cmd.query and dev_cmd.type were left empty. - * read the response directly ignoring all errors. - */ - ufshcd_wait_for_dev_cmd(hba, lrbp, QUERY_REQ_TIMEOUT); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, dev_cmd_timeout); + if (err) + goto put_dev_mgmt_cmd; /* just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) { u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu); - u16 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) & - MASK_QUERY_DATA_SEG_LEN; + u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header + .data_segment_length); if (*buff_len >= resp_len) { memcpy(desc_buff, descp, resp_len); @@ -7166,10 +7414,10 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, err = -EINVAL; } } - ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP, - (struct utp_upiu_req *)lrbp->ucd_rsp_ptr); - up_read(&hba->clk_scaling_lock); +put_dev_mgmt_cmd: + ufshcd_put_dev_mgmt_cmd(cmd); + return err; } @@ -7187,37 +7435,37 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba, * Management requests. * It is up to the caller to fill the upiu conent properly, as it will * be copied without any further input validations. + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, struct utp_upiu_req *rsp_upiu, - int msgcode, + enum upiu_request_transaction msgcode, u8 *desc_buff, int *buff_len, enum query_opcode desc_op) { int err; enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY; - struct utp_task_req_desc treq = { { 0 }, }; + struct utp_task_req_desc treq = { }; enum utp_ocs ocs_value; - u8 tm_f = be32_to_cpu(req_upiu->header.dword_1) >> 16 & MASK_TM_FUNC; + u8 tm_f = req_upiu->header.tm_function; switch (msgcode) { case UPIU_TRANSACTION_NOP_OUT: cmd_type = DEV_CMD_TYPE_NOP; fallthrough; case UPIU_TRANSACTION_QUERY_REQ: - ufshcd_hold(hba); - mutex_lock(&hba->dev_cmd.lock); + ufshcd_dev_man_lock(hba); err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu, desc_buff, buff_len, cmd_type, desc_op); - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); break; case UPIU_TRANSACTION_TASK_REQ: - treq.header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD); - treq.header.dword_2 = cpu_to_le32(OCS_INVALID_COMMAND_STATUS); + treq.header.interrupt = 1; + treq.header.ocs = OCS_INVALID_COMMAND_STATUS; memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu)); @@ -7225,7 +7473,7 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, if (err == -ETIMEDOUT) break; - ocs_value = le32_to_cpu(treq.header.dword_2) & MASK_OCS; + ocs_value = treq.header.ocs & MASK_OCS; if (ocs_value != OCS_SUCCESS) { dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__, ocs_value); @@ -7255,43 +7503,43 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba, * @sg_list: Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation * @dir: DMA direction * - * Returns zero on success, non-zero on failure + * Return: 0 upon success; > 0 in case the UFS device reported an OCS error; + * < 0 if another error occurred. */ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu, struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs, struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list, enum dma_data_direction dir) { - DECLARE_COMPLETION_ONSTACK(wait); - const u32 tag = hba->reserved_slot; + struct scsi_cmnd *cmd; struct ufshcd_lrb *lrbp; + u32 tag; int err = 0; int result; u8 upiu_flags; u8 *ehs_data; u16 ehs_len; + int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0; - /* Protects use of hba->reserved_slot. */ - ufshcd_hold(hba); - mutex_lock(&hba->dev_cmd.lock); - down_read(&hba->clk_scaling_lock); + ufshcd_dev_man_lock(hba); - lrbp = &hba->lrb[tag]; - lrbp->cmd = NULL; - lrbp->task_tag = tag; - lrbp->lun = UFS_UPIU_RPMB_WLUN; + cmd = ufshcd_get_dev_mgmt_cmd(hba); - lrbp->intr_cmd = true; - ufshcd_prepare_lrbp_crypto(NULL, lrbp); - hba->dev_cmd.type = DEV_CMD_TYPE_RPMB; + if (WARN_ON_ONCE(!cmd)) { + err = -ENOMEM; + goto unlock; + } - /* Advanced RPMB starts from UFS 4.0, so its command type is UTP_CMD_TYPE_UFS_STORAGE */ - lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE; + lrbp = scsi_cmd_priv(cmd); + tag = scsi_cmd_to_rq(cmd)->tag; + + ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, + tag); - ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, dir, 2); + ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs); - /* update the task tag and LUN in the request upiu */ - req_upiu->header.dword_0 |= cpu_to_be32(upiu_flags << 16 | UFS_UPIU_RPMB_WLUN << 8 | tag); + /* update the task tag */ + req_upiu->header.task_tag = tag; /* copy the UPIU(contains CDB) request as it is */ memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr)); @@ -7303,19 +7551,19 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); - hba->dev_cmd.complete = &wait; - - ufshcd_send_command(hba, tag, hba->dev_cmd_queue); - - err = ufshcd_wait_for_dev_cmd(hba, lrbp, ADVANCED_RPMB_REQ_TIMEOUT); + err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT); + if (err) + goto put_dev_mgmt_cmd; + err = ufshcd_dev_cmd_completion(hba, lrbp); if (!err) { /* Just copy the upiu response as it is */ memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu)); /* Get the response UPIU result */ - result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr); + result = (lrbp->ucd_rsp_ptr->header.response << 8) | + lrbp->ucd_rsp_ptr->header.status; - ehs_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) >> 24; + ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length; /* * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB @@ -7331,26 +7579,55 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r } } - up_read(&hba->clk_scaling_lock); - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); +put_dev_mgmt_cmd: + ufshcd_put_dev_mgmt_cmd(cmd); + +unlock: + ufshcd_dev_man_unlock(hba); + return err ? : result; } +static bool ufshcd_clear_lu_cmds(struct request *req, void *priv) +{ + struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req); + struct scsi_device *sdev = cmd->device; + struct Scsi_Host *shost = sdev->host; + struct ufs_hba *hba = shost_priv(shost); + const u64 lun = *(u64 *)priv; + const u32 tag = req->tag; + + if (blk_mq_is_reserved_rq(req) || sdev->lun != lun) + return true; + + if (ufshcd_clear_cmd(hba, tag) < 0) { + dev_err(hba->dev, "%s: failed to clear request %d\n", __func__, + tag); + return true; + } + + if (hba->mcq_enabled) { + struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, req); + + if (hwq) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + return true; + } + + ufshcd_compl_one_cqe(hba, tag, NULL); + return true; +} + /** * ufshcd_eh_device_reset_handler() - Reset a single logical unit. * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) { - unsigned long flags, pending_reqs = 0, not_cleared = 0; struct Scsi_Host *host; struct ufs_hba *hba; - struct ufs_hw_queue *hwq; - struct ufshcd_lrb *lrbp; - u32 pos, not_cleared_mask = 0; int err; u8 resp = 0xF, lun; @@ -7359,50 +7636,16 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd) lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun); err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp); - if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { - if (!err) - err = resp; - goto out; - } - - if (is_mcq_enabled(hba)) { - for (pos = 0; pos < hba->nutrs; pos++) { - lrbp = &hba->lrb[pos]; - if (ufshcd_cmd_inflight(lrbp->cmd) && - lrbp->lun == lun) { - ufshcd_clear_cmd(hba, pos); - hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd)); - ufshcd_mcq_poll_cqe_lock(hba, hwq); - } - } - err = 0; - goto out; - } - - /* clear the commands that were pending for corresponding LUN */ - spin_lock_irqsave(&hba->outstanding_lock, flags); - for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) - if (hba->lrb[pos].lun == lun) - __set_bit(pos, &pending_reqs); - hba->outstanding_reqs &= ~pending_reqs; - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - for_each_set_bit(pos, &pending_reqs, hba->nutrs) { - if (ufshcd_clear_cmd(hba, pos) < 0) { - spin_lock_irqsave(&hba->outstanding_lock, flags); - not_cleared = 1U << pos & - ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); - hba->outstanding_reqs |= not_cleared; - not_cleared_mask |= not_cleared; - spin_unlock_irqrestore(&hba->outstanding_lock, flags); - - dev_err(hba->dev, "%s: failed to clear request %d\n", - __func__, pos); - } + if (err) { + } else if (resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { + err = resp; + } else { + /* clear the commands that were pending for corresponding LUN */ + blk_mq_tagset_busy_iter(&hba->host->tag_set, + ufshcd_clear_lu_cmds, + &cmd->device->lun); } - __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask); -out: hba->req_abort_count = 0; ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err); if (!err) { @@ -7416,11 +7659,12 @@ out: static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) { - struct ufshcd_lrb *lrbp; int tag; for_each_set_bit(tag, &bitmap, hba->nutrs) { - lrbp = &hba->lrb[tag]; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + lrbp->req_abort_skip = true; } } @@ -7428,7 +7672,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) /** * ufshcd_try_to_abort_task - abort a specific task * @hba: Pointer to adapter instance - * @tag: Task tag/index to be aborted + * @tag: Tag of the task to be aborted * * Abort the pending command in device by sending UFS_ABORT_TASK task management * command, and in host controller by clearing the door-bell register. There can @@ -7436,19 +7680,19 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap) * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is * really issued and then try to abort it. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) { - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; - int err = 0; + struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag); + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); + int err; int poll_cnt; u8 resp = 0xF; - u32 reg; for (poll_cnt = 100; poll_cnt; poll_cnt--) { - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, - UFS_QUERY_TASK, &resp); + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_QUERY_TASK, + &resp); if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) { /* cmd pending in the device */ dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n", @@ -7459,56 +7703,36 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) * cmd not pending in the device, check if it is * in transition. */ - dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n", + dev_info( + hba->dev, + "%s: cmd with tag %d not pending in the device.\n", __func__, tag); - if (is_mcq_enabled(hba)) { - /* MCQ mode */ - if (ufshcd_cmd_inflight(lrbp->cmd)) { - /* sleep for max. 200us same delay as in SDB mode */ - usleep_range(100, 200); - continue; - } - /* command completed already */ - dev_err(hba->dev, "%s: cmd at tag=%d is cleared.\n", - __func__, tag); - goto out; - } - - /* Single Doorbell Mode */ - reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); - if (reg & (1 << tag)) { - /* sleep for max. 200us to stabilize */ - usleep_range(100, 200); - continue; + if (!ufshcd_cmd_inflight(cmd)) { + dev_info(hba->dev, + "%s: cmd with tag=%d completed.\n", + __func__, tag); + return 0; } - /* command completed already */ - dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n", - __func__, tag); - goto out; + usleep_range(100, 200); } else { dev_err(hba->dev, "%s: no response from device. tag = %d, err %d\n", __func__, tag, err); - if (!err) - err = resp; /* service response error */ - goto out; + return err ? : resp; } } - if (!poll_cnt) { - err = -EBUSY; - goto out; - } + if (!poll_cnt) + return -EBUSY; - err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag, - UFS_ABORT_TASK, &resp); + err = ufshcd_issue_tm_cmd(hba, lrbp->lun, tag, UFS_ABORT_TASK, &resp); if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) { if (!err) { err = resp; /* service response error */ dev_err(hba->dev, "%s: issued. tag = %d, err %d\n", __func__, tag, err); } - goto out; + return err; } err = ufshcd_clear_cmd(hba, tag); @@ -7516,7 +7740,6 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag) dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n", __func__, tag, err); -out: return err; } @@ -7524,24 +7747,23 @@ out: * ufshcd_abort - scsi host template eh_abort_handler callback * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_abort(struct scsi_cmnd *cmd) { struct Scsi_Host *host = cmd->device->host; struct ufs_hba *hba = shost_priv(host); - int tag = scsi_cmd_to_rq(cmd)->tag; - struct ufshcd_lrb *lrbp = &hba->lrb[tag]; + struct request *rq = scsi_cmd_to_rq(cmd); + int tag = rq->tag; + struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd); unsigned long flags; int err = FAILED; bool outstanding; u32 reg; - WARN_ONCE(tag < 0, "Invalid tag %d\n", tag); - ufshcd_hold(hba); - if (!is_mcq_enabled(hba)) { + if (!hba->mcq_enabled) { reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL); if (!test_bit(tag, &hba->outstanding_reqs)) { /* If command is already aborted/completed, return FAILED. */ @@ -7562,19 +7784,20 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) * to reduce repeated printouts. For other aborted requests only print * basic details. */ - scsi_print_command(cmd); + if (ufshcd_is_scsi_cmd(cmd)) + scsi_print_command(cmd); if (!hba->req_abort_count) { ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); ufshcd_print_pwr_info(hba); - ufshcd_print_tr(hba, tag, true); + ufshcd_print_tr(hba, cmd, true); } else { - ufshcd_print_tr(hba, tag, false); + ufshcd_print_tr(hba, cmd, false); } hba->req_abort_count++; - if (!is_mcq_enabled(hba) && !(reg & (1 << tag))) { + if (!hba->mcq_enabled && !(reg & (1 << tag))) { /* only execute this code in single doorbell mode */ dev_err(hba->dev, "%s: cmd was completed, but without a notifying intr, tag = %d", @@ -7601,7 +7824,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } - if (is_mcq_enabled(hba)) { + if (hba->mcq_enabled) { /* MCQ mode. Branch off to handle abort for mcq mode */ err = ufshcd_mcq_abort(cmd); goto release; @@ -7614,7 +7837,10 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) goto release; } - err = ufshcd_try_to_abort_task(hba, tag); + if (blk_mq_is_reserved_rq(rq)) + err = ufshcd_clear_cmd(hba, tag); + else + err = ufshcd_try_to_abort_task(hba, tag); if (err) { dev_err(hba->dev, "%s: failed with err %d\n", __func__, err); ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs); @@ -7631,7 +7857,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd) spin_unlock_irqrestore(&hba->outstanding_lock, flags); if (outstanding) - ufshcd_release_scsi_cmd(hba, lrbp); + ufshcd_release_scsi_cmd(hba, cmd); err = SUCCESS; @@ -7642,6 +7868,29 @@ release: } /** + * ufshcd_process_probe_result - Process the ufshcd_probe_hba() result. + * @hba: UFS host controller instance. + * @probe_start: time when the ufshcd_probe_hba() call started. + * @ret: ufshcd_probe_hba() return value. + */ +static void ufshcd_process_probe_result(struct ufs_hba *hba, + ktime_t probe_start, int ret) +{ + unsigned long flags; + + spin_lock_irqsave(hba->host->host_lock, flags); + if (ret) + hba->ufshcd_state = UFSHCD_STATE_ERROR; + else if (hba->ufshcd_state == UFSHCD_STATE_RESET) + hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; + spin_unlock_irqrestore(hba->host->host_lock, flags); + + trace_ufshcd_init(hba, ret, + ktime_to_us(ktime_sub(ktime_get(), probe_start)), + hba->curr_dev_pwr_mode, hba->uic_link_state); +} + +/** * ufshcd_host_reset_and_restore - reset and restore host controller * @hba: per-adapter instance * @@ -7649,7 +7898,7 @@ release: * local and remote (device) Uni-Pro stack and the attributes * are reset to default state. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) { @@ -7659,20 +7908,26 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) * Stop the host controller and complete the requests * cleared by h/w */ - ufshpb_toggle_state(hba, HPB_PRESENT, HPB_RESET); ufshcd_hba_stop(hba); hba->silence_err_logs = true; ufshcd_complete_requests(hba, true); hba->silence_err_logs = false; /* scale up clocks to max frequency before full reinitialization */ - ufshcd_scale_clks(hba, true); + if (ufshcd_is_clkscaling_supported(hba)) + ufshcd_scale_clks(hba, ULONG_MAX, true); err = ufshcd_hba_enable(hba); /* Establish the link again and restore the device */ - if (!err) - err = ufshcd_probe_hba(hba, false); + if (!err) { + ktime_t probe_start = ktime_get(); + + err = ufshcd_device_init(hba, /*init_dev_params=*/false); + if (!err) + err = ufshcd_probe_hba(hba, false); + ufshcd_process_probe_result(hba, probe_start, err); + } if (err) dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err); @@ -7687,7 +7942,7 @@ static int ufshcd_host_reset_and_restore(struct ufs_hba *hba) * Reset and recover device, host and re-establish link. This * is helpful to recover the communication in fatal error conditions. * - * Returns zero on success, non-zero on failure + * Return: zero on success, non-zero on failure. */ static int ufshcd_reset_and_restore(struct ufs_hba *hba) { @@ -7745,7 +8000,7 @@ static int ufshcd_reset_and_restore(struct ufs_hba *hba) * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer * @cmd: SCSI command pointer * - * Returns SUCCESS/FAILED + * Return: SUCCESS or FAILED. */ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) { @@ -7755,6 +8010,19 @@ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) hba = shost_priv(cmd->device->host); + /* + * If runtime PM sent SSU and got a timeout, scsi_error_handler is + * stuck in this function waiting for flush_work(&hba->eh_work). And + * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do + * ufshcd_link_recovery instead of eh_work to prevent deadlock. + */ + if (hba->pm_op_in_progress) { + if (ufshcd_link_recovery(hba)) + err = FAILED; + + return err; + } + spin_lock_irqsave(hba->host->host_lock, flags); hba->force_reset = true; ufshcd_schedule_eh_work(hba); @@ -7777,7 +8045,7 @@ static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd) * @start_scan: row at the desc table to start scan from * @buff: power descriptor buffer * - * Returns calculated max ICC level for specific regulator + * Return: calculated max ICC level for specific regulator. */ static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, const char *buff) @@ -7823,7 +8091,7 @@ static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, * @hba: per-adapter instance * @desc_buf: power descriptor buffer to extract ICC levels from. * - * Returns calculated ICC level + * Return: calculated ICC level. */ static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba, const u8 *desc_buf) @@ -7901,11 +8169,13 @@ out: static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) { + struct Scsi_Host *shost = sdev->host; + scsi_autopm_get_device(sdev); blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev); if (sdev->rpm_autosuspend) pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev, - RPM_AUTOSUSPEND_DELAY_MS); + shost->rpm_autosuspend_delay); scsi_autopm_put_device(sdev); } @@ -7932,7 +8202,7 @@ static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev) * This function adds scsi device instances for each of all well known LUs * (except "REPORT LUNS" LU). * - * Returns zero on success (all required W-LUs are added successfully), + * Return: zero on success (all required W-LUs are added successfully), * non-zero error value on failure (if failed to add any of the required W-LU). */ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) @@ -7953,8 +8223,11 @@ static int ufshcd_scsi_add_wlus(struct ufs_hba *hba) ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL); if (IS_ERR(sdev_rpmb)) { ret = PTR_ERR(sdev_rpmb); + hba->ufs_rpmb_wlun = NULL; + dev_err(hba->dev, "%s: RPMB WLUN not found\n", __func__); goto remove_ufs_device_wlun; } + hba->ufs_rpmb_wlun = sdev_rpmb; ufshcd_blk_pm_runtime_init(sdev_rpmb); scsi_device_put(sdev_rpmb); @@ -8007,6 +8280,9 @@ static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf) */ dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE]; + dev_info->ext_wb_sup = get_unaligned_be16(desc_buf + + DEVICE_DESC_PARAM_EXT_WB_SUP); + dev_info->b_presrv_uspc_en = desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN]; @@ -8064,29 +8340,52 @@ static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf) } } -static void ufshcd_ext_iid_probe(struct ufs_hba *hba, u8 *desc_buf) +static void ufshcd_device_lvl_exception_probe(struct ufs_hba *hba, u8 *desc_buf) { - struct ufs_dev_info *dev_info = &hba->dev_info; u32 ext_ufs_feature; - u32 ext_iid_en = 0; - int err; - /* Only UFS-4.0 and above may support EXT_IID */ - if (dev_info->wspecversion < 0x400) - goto out; + if (hba->dev_info.wspecversion < 0x410) + return; ext_ufs_feature = get_unaligned_be32(desc_buf + - DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); - if (!(ext_ufs_feature & UFS_DEV_EXT_IID_SUP)) - goto out; + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP); + if (!(ext_ufs_feature & UFS_DEV_LVL_EXCEPTION_SUP)) + return; - err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, - QUERY_ATTR_IDN_EXT_IID_EN, 0, 0, &ext_iid_en); - if (err) - dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err); + atomic_set(&hba->dev_lvl_exception_count, 0); + ufshcd_enable_ee(hba, MASK_EE_DEV_LVL_EXCEPTION); +} -out: - dev_info->b_ext_iid_en = ext_iid_en; +static void ufshcd_set_rtt(struct ufs_hba *hba) +{ + struct ufs_dev_info *dev_info = &hba->dev_info; + u32 rtt = 0; + u32 dev_rtt = 0; + int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ? + hba->vops->max_num_rtt : hba->nortt; + + /* RTT override makes sense only for UFS-4.0 and above */ + if (dev_info->wspecversion < 0x400) + return; + + if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) { + dev_err(hba->dev, "failed reading bMaxNumOfRTT\n"); + return; + } + + /* do not override if it was already written */ + if (dev_rtt != DEFAULT_MAX_NUM_RTT) + return; + + rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap); + + if (rtt == dev_rtt) + return; + + if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, + QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt)) + dev_err(hba->dev, "failed writing bMaxNumOfRTT\n"); } void ufshcd_fixup_dev_quirks(struct ufs_hba *hba, @@ -8118,13 +8417,152 @@ static void ufs_fixup_device_setup(struct ufs_hba *hba) ufshcd_vops_fixup_dev_quirks(hba); } +static void ufshcd_update_rtc(struct ufs_hba *hba) +{ + struct timespec64 ts64; + int err; + u32 val; + + ktime_get_real_ts64(&ts64); + + if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) { + dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__); + return; + } + + /* + * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond + * 2146 is required, it is recommended to choose the relative RTC mode. + */ + val = ts64.tv_sec - hba->dev_info.rtc_time_baseline; + + /* Skip update RTC if RPM state is not RPM_ACTIVE */ + if (ufshcd_rpm_get_if_active(hba) <= 0) + return; + + err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED, + 0, 0, &val); + ufshcd_rpm_put(hba); + + if (err) + dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err); + else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE) + hba->dev_info.rtc_time_baseline = ts64.tv_sec; +} + +static void ufshcd_rtc_work(struct work_struct *work) +{ + struct ufs_hba *hba; + + hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work); + + /* Update RTC only when there are no requests in progress and UFSHCI is operational */ + if (!ufshcd_is_ufs_dev_busy(hba) && + hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL && + !hba->clk_gating.active_reqs) + ufshcd_update_rtc(hba); + + if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period) + schedule_delayed_work(&hba->ufs_rtc_update_work, + msecs_to_jiffies(hba->dev_info.rtc_update_period)); +} + +static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf) +{ + u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]); + struct ufs_dev_info *dev_info = &hba->dev_info; + + if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) { + dev_info->rtc_type = UFS_RTC_ABSOLUTE; + + /* + * The concept of measuring time in Linux as the number of seconds elapsed since + * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st + * 2010 00:00, here we need to adjust ABS baseline. + */ + dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) - + mktime64(1970, 1, 1, 0, 0, 0); + } else { + dev_info->rtc_type = UFS_RTC_RELATIVE; + dev_info->rtc_time_baseline = 0; + } + + /* + * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state + * how to calculate the specific update period for each time unit. And we disable periodic + * RTC update work, let user configure by sysfs node according to specific circumstance. + */ + dev_info->rtc_update_period = 0; +} + +/** + * ufshcd_create_device_id - Generate unique device identifier string + * @hba: per-adapter instance + * @desc_buf: device descriptor buffer + * + * Creates a unique device ID string combining manufacturer ID, spec version, + * model name, serial number (as hex), device version, and manufacture date. + * + * Returns: Allocated device ID string on success, NULL on failure + */ +static char *ufshcd_create_device_id(struct ufs_hba *hba, u8 *desc_buf) +{ + struct ufs_dev_info *dev_info = &hba->dev_info; + u16 manufacture_date; + u16 device_version; + u8 *serial_number; + char *serial_hex; + char *device_id; + u8 serial_index; + int serial_len; + int ret; + + serial_index = desc_buf[DEVICE_DESC_PARAM_SN]; + + ret = ufshcd_read_string_desc(hba, serial_index, &serial_number, SD_RAW); + if (ret < 0) { + dev_err(hba->dev, "Failed reading Serial Number. err = %d\n", ret); + return NULL; + } + + device_version = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_DEV_VER]); + manufacture_date = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_MANF_DATE]); + + serial_len = ret; + /* Allocate buffer for hex string: 2 chars per byte + null terminator */ + serial_hex = kzalloc(serial_len * 2 + 1, GFP_KERNEL); + if (!serial_hex) { + kfree(serial_number); + return NULL; + } + + bin2hex(serial_hex, serial_number, serial_len); + + /* + * Device ID format is ABI with secure world - do not change without firmware + * coordination. + */ + device_id = kasprintf(GFP_KERNEL, "%04X-%04X-%s-%s-%04X-%04X", + dev_info->wmanufacturerid, dev_info->wspecversion, + dev_info->model, serial_hex, device_version, + manufacture_date); + + kfree(serial_hex); + kfree(serial_number); + + if (!device_id) + dev_warn(hba->dev, "Failed to allocate unique device ID\n"); + + return device_id; +} + static int ufs_get_device_desc(struct ufs_hba *hba) { + struct ufs_dev_info *dev_info = &hba->dev_info; + struct Scsi_Host *shost = hba->host; int err; u8 model_index; - u8 b_ufs_feature_sup; u8 *desc_buf; - struct ufs_dev_info *dev_info = &hba->dev_info; desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL); if (!desc_buf) { @@ -8151,25 +8589,26 @@ static int ufs_get_device_desc(struct ufs_hba *hba) dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 | desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1]; dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH]; - b_ufs_feature_sup = desc_buf[DEVICE_DESC_PARAM_UFS_FEAT]; - - model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; - if (dev_info->wspecversion >= UFS_DEV_HPB_SUPPORT_VERSION && - (b_ufs_feature_sup & UFS_DEV_HPB_SUPPORT)) { - bool hpb_en = false; + /* + * According to the UFS standard, the UFS device queue depth + * (bQueueDepth) must be in the range 1..255 if the shared queueing + * architecture is supported. bQueueDepth is zero if the shared queueing + * architecture is not supported. + */ + if (dev_info->bqueuedepth) + shost->cmd_per_lun = min(hba->nutrs, dev_info->bqueuedepth) - + UFSHCD_NUM_RESERVED; + else + shost->cmd_per_lun = shost->can_queue; - ufshpb_get_dev_info(hba, desc_buf); + dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP]; - if (!ufshpb_is_legacy(hba)) - err = ufshcd_query_flag_retry(hba, - UPIU_QUERY_OPCODE_READ_FLAG, - QUERY_FLAG_IDN_HPB_EN, 0, - &hpb_en); + dev_info->hid_sup = get_unaligned_be32(desc_buf + + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP) & + UFS_DEV_HID_SUPPORT; - if (ufshpb_is_legacy(hba) || (!err && hpb_en)) - dev_info->hpb_enabled = true; - } + model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; err = ufshcd_read_string_desc(hba, model_index, &dev_info->model, SD_ASCII_STD); @@ -8179,6 +8618,9 @@ static int ufs_get_device_desc(struct ufs_hba *hba) goto out; } + /* Generate unique device ID */ + dev_info->device_id = ufshcd_create_device_id(hba, desc_buf); + hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] + desc_buf[DEVICE_DESC_PARAM_NUM_WLU]; @@ -8188,8 +8630,14 @@ static int ufs_get_device_desc(struct ufs_hba *hba) ufshcd_temp_notif_probe(hba, desc_buf); - if (hba->ext_iid_sup) - ufshcd_ext_iid_probe(hba, desc_buf); + if (dev_info->wspecversion >= 0x410) { + hba->critical_health_count = 0; + ufshcd_enable_ee(hba, MASK_EE_HEALTH_CRITICAL); + } + + ufs_init_rtc(hba, desc_buf); + + ufshcd_device_lvl_exception_probe(hba, desc_buf); /* * ufshcd_read_string_desc returns size of the string @@ -8208,83 +8656,8 @@ static void ufs_put_device_desc(struct ufs_hba *hba) kfree(dev_info->model); dev_info->model = NULL; -} - -/** - * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro - * @hba: per-adapter instance - * - * PA_TActivate parameter can be tuned manually if UniPro version is less than - * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's - * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce - * the hibern8 exit latency. - * - * Returns zero on success, non-zero error value on failure. - */ -static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba) -{ - int ret = 0; - u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate; - - ret = ufshcd_dme_peer_get(hba, - UIC_ARG_MIB_SEL( - RX_MIN_ACTIVATETIME_CAPABILITY, - UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), - &peer_rx_min_activatetime); - if (ret) - goto out; - - /* make sure proper unit conversion is applied */ - tuned_pa_tactivate = - ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US) - / PA_TACTIVATE_TIME_UNIT_US); - ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), - tuned_pa_tactivate); - -out: - return ret; -} - -/** - * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro - * @hba: per-adapter instance - * - * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than - * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's - * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY. - * This optimal value can help reduce the hibern8 exit latency. - * - * Returns zero on success, non-zero error value on failure. - */ -static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba) -{ - int ret = 0; - u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0; - u32 max_hibern8_time, tuned_pa_hibern8time; - - ret = ufshcd_dme_get(hba, - UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY, - UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)), - &local_tx_hibern8_time_cap); - if (ret) - goto out; - - ret = ufshcd_dme_peer_get(hba, - UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY, - UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), - &peer_rx_hibern8_time_cap); - if (ret) - goto out; - - max_hibern8_time = max(local_tx_hibern8_time_cap, - peer_rx_hibern8_time_cap); - /* make sure proper unit conversion is applied */ - tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US) - / PA_HIBERN8_TIME_UNIT_US); - ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), - tuned_pa_hibern8time); -out: - return ret; + kfree(dev_info->device_id); + dev_info->device_id = NULL; } /** @@ -8296,7 +8669,7 @@ out: * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk * for such devices. * - * Returns zero on success, non-zero error value on failure. + * Return: zero on success, non-zero error value on failure. */ static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba) { @@ -8357,13 +8730,33 @@ out: return ret; } -static void ufshcd_tune_unipro_params(struct ufs_hba *hba) +/** + * ufshcd_quirk_override_pa_h8time - Ensures proper adjustment of PA_HIBERN8TIME. + * @hba: per-adapter instance + * + * Some UFS devices require specific adjustments to the PA_HIBERN8TIME parameter + * to ensure proper hibernation timing. This function retrieves the current + * PA_HIBERN8TIME value and increments it by 100us. + */ +static void ufshcd_quirk_override_pa_h8time(struct ufs_hba *hba) { - if (ufshcd_is_unipro_pa_params_tuning_req(hba)) { - ufshcd_tune_pa_tactivate(hba); - ufshcd_tune_pa_hibern8time(hba); + u32 pa_h8time; + int ret; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), &pa_h8time); + if (ret) { + dev_err(hba->dev, "Failed to get PA_HIBERN8TIME: %d\n", ret); + return; } + /* Increment by 1 to increase hibernation time by 100 µs */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), pa_h8time + 1); + if (ret) + dev_err(hba->dev, "Failed updating PA_HIBERN8TIME: %d\n", ret); +} + +static void ufshcd_tune_unipro_params(struct ufs_hba *hba) +{ ufshcd_vops_apply_dev_quirks(hba); if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE) @@ -8372,6 +8765,9 @@ static void ufshcd_tune_unipro_params(struct ufs_hba *hba) if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE) ufshcd_quirk_tune_host_pa_tactivate(hba); + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_HIBER8TIME) + ufshcd_quirk_override_pa_h8time(hba); } static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba) @@ -8405,9 +8801,7 @@ static int ufshcd_device_geo_params_init(struct ufs_hba *hba) else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0) hba->dev_info.max_lu_supported = 8; - if (desc_buf[QUERY_DESC_LENGTH_OFFSET] >= - GEOMETRY_DESC_PARAM_HPB_MAX_ACTIVE_REGS) - ufshpb_get_geo_info(hba, desc_buf); + hba->dev_info.rpmb_io_size = desc_buf[GEOMETRY_DESC_PARAM_RPMB_RW_SIZE]; out: kfree(desc_buf); @@ -8505,6 +8899,8 @@ static int ufshcd_device_params_init(struct ufs_hba *hba) goto out; } + ufshcd_set_rtt(hba); + ufshcd_get_ref_clk_gating_wait(hba); if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, @@ -8528,12 +8924,11 @@ static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) struct ufs_dev_info *dev_info = &hba->dev_info; struct utp_upiu_query_v4_0 *upiu_data; - if (dev_info->wspecversion < 0x400) + if (dev_info->wspecversion < 0x400 || + hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT) return; - ufshcd_hold(hba); - - mutex_lock(&hba->dev_cmd.lock); + ufshcd_dev_man_lock(hba); ufshcd_init_query(hba, &request, &response, UPIU_QUERY_OPCODE_WRITE_ATTR, @@ -8545,19 +8940,20 @@ static void ufshcd_set_timestamp_attr(struct ufs_hba *hba) put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3); - err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout); if (err) dev_err(hba->dev, "%s: failed to set timestamp %d\n", __func__, err); - mutex_unlock(&hba->dev_cmd.lock); - ufshcd_release(hba); + ufshcd_dev_man_unlock(hba); } /** * ufshcd_add_lus - probe and add UFS logical units * @hba: per-adapter instance + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_add_lus(struct ufs_hba *hba) { @@ -8583,10 +8979,17 @@ static int ufshcd_add_lus(struct ufs_hba *hba) ufshcd_init_clk_scaling_sysfs(hba); } + /* + * The RTC update code accesses the hba->ufs_device_wlun->sdev_gendev + * pointer and hence must only be started after the WLUN pointer has + * been initialized by ufshcd_scsi_add_wlus(). + */ + schedule_delayed_work(&hba->ufs_rtc_update_work, + msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); + ufs_bsg_probe(hba); - ufshpb_init(hba); scsi_scan_host(hba->host); - pm_runtime_put_sync(hba->dev); + ufs_rpmb_probe(hba); out: return ret; @@ -8604,8 +9007,6 @@ static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs) utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs; dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr, hba->utrdl_dma_addr); - - devm_kfree(hba->dev, hba->lrb); } static int ufshcd_alloc_mcq(struct ufs_hba *hba) @@ -8613,7 +9014,7 @@ static int ufshcd_alloc_mcq(struct ufs_hba *hba) int ret; int old_nutrs = hba->nutrs; - ret = ufshcd_mcq_decide_queue_depth(hba); + ret = ufshcd_get_hba_mac(hba); if (ret < 0) return ret; @@ -8638,6 +9039,8 @@ static int ufshcd_alloc_mcq(struct ufs_hba *hba) if (ret) goto err; + hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; + return 0; err: hba->nutrs = old_nutrs; @@ -8647,36 +9050,57 @@ err: static void ufshcd_config_mcq(struct ufs_hba *hba) { int ret; - u32 intrs; ret = ufshcd_mcq_vops_config_esi(hba); + hba->mcq_esi_enabled = !ret; dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : ""); - intrs = UFSHCD_ENABLE_MCQ_INTRS; - if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR) - intrs &= ~MCQ_CQ_EVENT_STATUS; - ufshcd_enable_intr(hba, intrs); ufshcd_mcq_make_queues_operational(hba); ufshcd_mcq_config_mac(hba, hba->nutrs); - hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; - hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED; - - /* Select MCQ mode */ - ufshcd_writel(hba, ufshcd_readl(hba, REG_UFS_MEM_CFG) | 0x1, - REG_UFS_MEM_CFG); - hba->mcq_enabled = true; - dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n", hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT], hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL], hba->nutrs); } +static int ufshcd_post_device_init(struct ufs_hba *hba) +{ + int ret; + + ufshcd_tune_unipro_params(hba); + + /* UFS device is also active now */ + ufshcd_set_ufs_dev_active(hba); + ufshcd_force_reset_auto_bkops(hba); + + ufshcd_set_timestamp_attr(hba); + + if (!hba->max_pwr_info.is_valid) + return 0; + + /* + * Set the right value to bRefClkFreq before attempting to + * switch to HS gears. + */ + if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) + ufshcd_set_dev_ref_clk(hba); + /* Gear up to HS gear. */ + ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); + if (ret) { + dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", + __func__, ret); + return ret; + } + + return 0; +} + static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) { int ret; - struct Scsi_Host *host = hba->host; + + WARN_ON_ONCE(!hba->scsi_host_added); hba->ufshcd_state = UFSHCD_STATE_RESET; @@ -8694,8 +9118,10 @@ static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) ufshcd_set_link_active(hba); /* Reconfigure MCQ upon reset */ - if (is_mcq_enabled(hba) && !init_dev_params) + if (hba->mcq_enabled && !init_dev_params) { ufshcd_config_mcq(hba); + ufshcd_mcq_enable(hba); + } /* Verify device initialization by sending NOP OUT UPIU */ ret = ufshcd_verify_dev_init(hba); @@ -8715,53 +9141,14 @@ static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) ret = ufshcd_device_params_init(hba); if (ret) return ret; - if (is_mcq_supported(hba) && !hba->scsi_host_added) { - ret = ufshcd_alloc_mcq(hba); - if (!ret) { - ufshcd_config_mcq(hba); - } else { - /* Continue with SDB mode */ - use_mcq_mode = false; - dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", - ret); - } - ret = scsi_add_host(host, hba->dev); - if (ret) { - dev_err(hba->dev, "scsi_add_host failed\n"); - return ret; - } - hba->scsi_host_added = true; - } else if (is_mcq_supported(hba)) { - /* UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is set */ + if (is_mcq_supported(hba) && + hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { ufshcd_config_mcq(hba); + ufshcd_mcq_enable(hba); } } - ufshcd_tune_unipro_params(hba); - - /* UFS device is also active now */ - ufshcd_set_ufs_dev_active(hba); - ufshcd_force_reset_auto_bkops(hba); - - ufshcd_set_timestamp_attr(hba); - - /* Gear up to HS gear if supported */ - if (hba->max_pwr_info.is_valid) { - /* - * Set the right value to bRefClkFreq before attempting to - * switch to HS gears. - */ - if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL) - ufshcd_set_dev_ref_clk(hba); - ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info); - if (ret) { - dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", - __func__, ret); - return ret; - } - } - - return 0; + return ufshcd_post_device_init(hba); } /** @@ -8770,34 +9157,31 @@ static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params) * @init_dev_params: whether or not to call ufshcd_device_params_init(). * * Execute link-startup and verify device initialization + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) { - ktime_t start = ktime_get(); - unsigned long flags; int ret; - ret = ufshcd_device_init(hba, init_dev_params); - if (ret) - goto out; - - if (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) { + if (!hba->pm_op_in_progress && + (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) { /* Reset the device and controller before doing reinit */ ufshcd_device_reset(hba); + ufs_put_device_desc(hba); ufshcd_hba_stop(hba); - ufshcd_vops_reinit_notify(hba); ret = ufshcd_hba_enable(hba); if (ret) { dev_err(hba->dev, "Host controller enable failed\n"); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); - goto out; + return ret; } /* Reinit the device */ ret = ufshcd_device_init(hba, init_dev_params); if (ret) - goto out; + return ret; } ufshcd_print_pwr_info(hba); @@ -8815,22 +9199,9 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params) if (hba->ee_usr_mask) ufshcd_write_ee_control(hba); - /* Enable Auto-Hibernate if configured */ - ufshcd_auto_hibern8_enable(hba); - - ufshpb_toggle_state(hba, HPB_RESET, HPB_PRESENT); -out: - spin_lock_irqsave(hba->host->host_lock, flags); - if (ret) - hba->ufshcd_state = UFSHCD_STATE_ERROR; - else if (hba->ufshcd_state == UFSHCD_STATE_RESET) - hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL; - spin_unlock_irqrestore(hba->host->host_lock, flags); + ufshcd_configure_auto_hibern8(hba); - trace_ufshcd_init(dev_name(hba->dev), ret, - ktime_to_us(ktime_sub(ktime_get(), start)), - hba->curr_dev_pwr_mode, hba->uic_link_state); - return ret; + return 0; } /** @@ -8841,26 +9212,26 @@ out: static void ufshcd_async_scan(void *data, async_cookie_t cookie) { struct ufs_hba *hba = (struct ufs_hba *)data; + ktime_t probe_start; int ret; down(&hba->host_sem); /* Initialize hba, detect and initialize UFS device */ + probe_start = ktime_get(); ret = ufshcd_probe_hba(hba, true); + ufshcd_process_probe_result(hba, probe_start, ret); up(&hba->host_sem); if (ret) goto out; /* Probe and add UFS logical units */ ret = ufshcd_add_lus(hba); + out: - /* - * If we failed to initialize the device or the device is not - * present, turn off the power/clocks etc. - */ - if (ret) { - pm_runtime_put_sync(hba->dev); - ufshcd_hba_exit(hba); - } + pm_runtime_put_sync(hba->dev); + + if (ret) + dev_err(hba->dev, "%s failed: %d\n", __func__, ret); } static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) @@ -8882,16 +9253,12 @@ static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd) dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n", __func__, hba->outstanding_tasks); - return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; + return scsi_host_busy(hba->host) ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE; } static const struct attribute_group *ufshcd_driver_groups[] = { &ufs_sysfs_unit_descriptor_group, &ufs_sysfs_lun_attributes_group, -#ifdef CONFIG_SCSI_UFS_HPB - &ufs_sysfs_hpb_stat_group, - &ufs_sysfs_hpb_param_group, -#endif NULL, }; @@ -8910,11 +9277,15 @@ static const struct scsi_host_template ufshcd_driver_template = { .name = UFSHCD, .proc_name = UFSHCD, .map_queues = ufshcd_map_queues, + .cmd_size = sizeof(struct ufshcd_lrb), + .init_cmd_priv = ufshcd_init_cmd_priv, .queuecommand = ufshcd_queuecommand, + .queue_reserved_command = ufshcd_queue_reserved_command, + .nr_reserved_cmds = UFSHCD_NUM_RESERVED, .mq_poll = ufshcd_poll, - .slave_alloc = ufshcd_slave_alloc, - .slave_configure = ufshcd_slave_configure, - .slave_destroy = ufshcd_slave_destroy, + .sdev_init = ufshcd_sdev_init, + .sdev_configure = ufshcd_sdev_configure, + .sdev_destroy = ufshcd_sdev_destroy, .change_queue_depth = ufshcd_change_queue_depth, .eh_abort_handler = ufshcd_abort, .eh_device_reset_handler = ufshcd_eh_device_reset_handler, @@ -8922,15 +9293,12 @@ static const struct scsi_host_template ufshcd_driver_template = { .eh_timed_out = ufshcd_eh_timed_out, .this_id = -1, .sg_tablesize = SG_ALL, - .cmd_per_lun = UFSHCD_CMD_PER_LUN, - .can_queue = UFSHCD_CAN_QUEUE, .max_segment_size = PRDT_DATA_BYTE_COUNT_MAX, .max_sectors = SZ_1M / SECTOR_SIZE, .max_host_blocked = 1, .track_queue_depth = 1, .skip_settle_delay = 1, .sdev_groups = ufshcd_driver_groups, - .rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS, }; static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg, @@ -9103,7 +9471,6 @@ static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) int ret = 0; struct ufs_clk_info *clki; struct list_head *head = &hba->clk_list_head; - unsigned long flags; ktime_t start = ktime_get(); bool clk_state_changed = false; @@ -9145,22 +9512,23 @@ static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on) if (ret) return ret; + if (!ufshcd_is_clkscaling_supported(hba)) + ufshcd_pm_qos_update(hba, on); out: if (ret) { list_for_each_entry(clki, head, list) { if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled) clk_disable_unprepare(clki->clk); } - } else if (!ret && on) { - spin_lock_irqsave(hba->host->host_lock, flags); - hba->clk_gating.state = CLKS_ON; - trace_ufshcd_clk_gating(dev_name(hba->dev), + } else if (!ret && on && hba->clk_gating.is_initialized) { + scoped_guard(spinlock_irqsave, &hba->clk_gating.lock) + hba->clk_gating.state = CLKS_ON; + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); - spin_unlock_irqrestore(hba->host->host_lock, flags); } if (clk_state_changed) - trace_ufshcd_profile_clk_gating(dev_name(hba->dev), + trace_ufshcd_profile_clk_gating(hba, (on ? "on" : "off"), ktime_to_us(ktime_sub(ktime_get(), start)), ret); return ret; @@ -9222,6 +9590,17 @@ static int ufshcd_init_clocks(struct ufs_hba *hba) dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__, clki->name, clk_get_rate(clki->clk)); } + + /* Set Max. frequency for all clocks */ + if (hba->use_pm_opp) { + ret = ufshcd_opp_set_rate(hba, ULONG_MAX); + if (ret) { + dev_err(hba->dev, "%s: failed to set OPP: %d", __func__, + ret); + goto out; + } + } + out: return ret; } @@ -9235,8 +9614,9 @@ static int ufshcd_variant_hba_init(struct ufs_hba *hba) err = ufshcd_vops_init(hba); if (err) - dev_err(hba->dev, "%s: variant %s init failed err %d\n", - __func__, ufshcd_get_var_name(hba), err); + dev_err_probe(hba->dev, err, + "%s: variant %s init failed with err %d\n", + __func__, ufshcd_get_var_name(hba), err); out: return err; } @@ -9292,6 +9672,7 @@ static int ufshcd_hba_init(struct ufs_hba *hba) goto out_disable_vreg; ufs_debugfs_hba_init(hba); + ufs_fault_inject_hba_init(hba); hba->is_powered = true; goto out; @@ -9309,6 +9690,7 @@ out: static void ufshcd_hba_exit(struct ufs_hba *hba) { if (hba->is_powered) { + ufshcd_pm_qos_exit(hba); ufshcd_exit_clk_scaling(hba); ufshcd_exit_clk_gating(hba); if (hba->eh_wq) @@ -9328,7 +9710,17 @@ static int ufshcd_execute_start_stop(struct scsi_device *sdev, struct scsi_sense_hdr *sshdr) { const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 }; + struct scsi_failure failure_defs[] = { + { + .allowed = 2, + .result = SCMD_FAILURE_RESULT_ANY, + }, + }; + struct scsi_failures failures = { + .failure_definitions = failure_defs, + }; const struct scsi_exec_args args = { + .failures = &failures, .sshdr = sshdr, .req_flags = BLK_MQ_REQ_PM, .scmd_flags = SCMD_FAIL_IF_RECOVERING, @@ -9345,8 +9737,8 @@ static int ufshcd_execute_start_stop(struct scsi_device *sdev, * @hba: per adapter instance * @pwr_mode: device power mode to set * - * Returns 0 if requested power mode is set successfully - * Returns < 0 if failed to set the requested power mode + * Return: 0 if requested power mode is set successfully; + * < 0 if failed to set the requested power mode. */ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, enum ufs_dev_pwr_mode pwr_mode) @@ -9354,7 +9746,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, struct scsi_sense_hdr sshdr; struct scsi_device *sdp; unsigned long flags; - int ret, retries; + int ret; spin_lock_irqsave(hba->host->host_lock, flags); sdp = hba->ufs_device_wlun; @@ -9380,15 +9772,7 @@ static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba, * callbacks hence set the RQF_PM flag so that it doesn't resume the * already suspended childs. */ - for (retries = 3; retries > 0; --retries) { - ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); - /* - * scsi_execute() only returns a negative value if the request - * queue is dying. - */ - if (ret <= 0) - break; - } + ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr); if (ret) { sdev_printk(KERN_WARNING, sdp, "START_STOP failed for power mode: %d, result %x\n", @@ -9508,11 +9892,11 @@ static void ufshcd_vreg_set_lpm(struct ufs_hba *hba) } /* - * Some UFS devices require delay after VCC power rail is turned-off. + * All UFS devices require delay after VCC power rail is turned-off. */ - if (vcc_off && hba->vreg_info.vcc && - hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM) - usleep_range(5000, 5100); + if (vcc_off && hba->vreg_info.vcc && !hba->vreg_info.vcc->always_on) + usleep_range(hba->vcc_off_delay_us, + hba->vcc_off_delay_us + 100); } #ifdef CONFIG_PM @@ -9576,8 +9960,6 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) req_link_state = UIC_LINK_OFF_STATE; } - ufshpb_suspend(hba); - /* * If we can't transition into any of the low power modes * just gate the clocks. @@ -9599,7 +9981,10 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) /* UFS device & link must be active before we enter in this function */ if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) { - ret = -EINVAL; + /* Wait err handler finish or trigger err recovery */ + if (!ufshcd_eh_in_progress(hba)) + ufshcd_force_error_recovery(hba); + ret = -EBUSY; goto enable_scaling; } @@ -9610,7 +9995,7 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) * allow background operations if bkops status shows * that performance might be impacted. */ - ret = ufshcd_urgent_bkops(hba); + ret = ufshcd_bkops_ctrl(hba); if (ret) { /* * If return err in suspend flow, IO will hang. @@ -9692,6 +10077,8 @@ vops_suspend: ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE); if (ret) goto set_link_active; + + cancel_delayed_work_sync(&hba->ufs_rtc_update_work); goto out; set_link_active: @@ -9731,7 +10118,6 @@ out: ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret); hba->clk_gating.is_suspended = false; ufshcd_release(hba); - ufshpb_resume(hba); } hba->pm_op_in_progress = false; return ret; @@ -9787,6 +10173,8 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (ret) goto set_old_link_state; ufshcd_set_timestamp_attr(hba); + schedule_delayed_work(&hba->ufs_rtc_update_work, + msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); } if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) @@ -9796,7 +10184,7 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) * If BKOPs operations are urgently needed at this moment then * keep auto-bkops enabled or else disable it. */ - ufshcd_urgent_bkops(hba); + ufshcd_bkops_ctrl(hba); if (hba->ee_usr_mask) ufshcd_write_ee_control(hba); @@ -9809,10 +10197,8 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) cancel_delayed_work(&hba->rpm_dev_flush_recheck_work); } - /* Enable Auto-Hibernate if configured */ - ufshcd_auto_hibern8_enable(hba); + ufshcd_configure_auto_hibern8(hba); - ufshpb_resume(hba); goto out; set_old_link_state: @@ -9842,7 +10228,7 @@ static int ufshcd_wl_runtime_suspend(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); - trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret, + trace_ufshcd_wl_runtime_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9862,7 +10248,7 @@ static int ufshcd_wl_runtime_resume(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); - trace_ufshcd_wl_runtime_resume(dev_name(dev), ret, + trace_ufshcd_wl_runtime_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9894,7 +10280,7 @@ static int ufshcd_wl_suspend(struct device *dev) out: if (!ret) hba->is_sys_suspended = true; - trace_ufshcd_wl_suspend(dev_name(dev), ret, + trace_ufshcd_wl_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -9917,7 +10303,7 @@ static int ufshcd_wl_resume(struct device *dev) if (ret) dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret); out: - trace_ufshcd_wl_resume(dev_name(dev), ret, + trace_ufshcd_wl_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); if (!ret) @@ -9934,6 +10320,8 @@ out: * * This function will put disable irqs, turn off clocks * and set vreg and hba-vreg in lpm mode. + * + * Return: 0 upon success; < 0 upon failure. */ static int ufshcd_suspend(struct ufs_hba *hba) { @@ -9953,13 +10341,14 @@ static int ufshcd_suspend(struct ufs_hba *hba) } if (ufshcd_is_clkgating_allowed(hba)) { hba->clk_gating.state = CLKS_OFF; - trace_ufshcd_clk_gating(dev_name(hba->dev), + trace_ufshcd_clk_gating(hba, hba->clk_gating.state); } ufshcd_vreg_set_lpm(hba); /* Put the host controller in low power mode if possible */ ufshcd_hba_vreg_set_lpm(hba); + ufshcd_pm_qos_update(hba, false); return ret; } @@ -9971,7 +10360,7 @@ static int ufshcd_suspend(struct ufs_hba *hba) * This function basically turns on the regulators, clocks and * irqs of the hba. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ static int ufshcd_resume(struct ufs_hba *hba) { @@ -10012,7 +10401,7 @@ out: * Executed before putting the system into a sleep state in which the contents * of main memory are preserved. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_system_suspend(struct device *dev) { @@ -10025,7 +10414,7 @@ int ufshcd_system_suspend(struct device *dev) ret = ufshcd_suspend(hba); out: - trace_ufshcd_system_suspend(dev_name(hba->dev), ret, + trace_ufshcd_system_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -10039,7 +10428,7 @@ EXPORT_SYMBOL(ufshcd_system_suspend); * Executed after waking the system up from a sleep state in which the contents * of main memory were preserved. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_system_resume(struct device *dev) { @@ -10053,7 +10442,7 @@ int ufshcd_system_resume(struct device *dev) ret = ufshcd_resume(hba); out: - trace_ufshcd_system_resume(dev_name(hba->dev), ret, + trace_ufshcd_system_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); @@ -10069,7 +10458,7 @@ EXPORT_SYMBOL(ufshcd_system_resume); * * Check the description of ufshcd_suspend() function for more details. * - * Returns 0 for success and non-zero for failure + * Return: 0 for success and non-zero for failure. */ int ufshcd_runtime_suspend(struct device *dev) { @@ -10079,7 +10468,7 @@ int ufshcd_runtime_suspend(struct device *dev) ret = ufshcd_suspend(hba); - trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret, + trace_ufshcd_runtime_suspend(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -10095,6 +10484,8 @@ EXPORT_SYMBOL(ufshcd_runtime_suspend); * * 1. Turn on all the controller related clocks * 2. Turn ON VCC rail + * + * Return: 0 upon success; < 0 upon failure. */ int ufshcd_runtime_resume(struct device *dev) { @@ -10104,7 +10495,7 @@ int ufshcd_runtime_resume(struct device *dev) ret = ufshcd_resume(hba); - trace_ufshcd_runtime_resume(dev_name(hba->dev), ret, + trace_ufshcd_runtime_resume(hba, ret, ktime_to_us(ktime_sub(ktime_get(), start)), hba->curr_dev_pwr_mode, hba->uic_link_state); return ret; @@ -10127,7 +10518,9 @@ static void ufshcd_wl_shutdown(struct device *dev) shost_for_each_device(sdev, hba->host) { if (sdev == hba->ufs_device_wlun) continue; - scsi_device_quiesce(sdev); + mutex_lock(&sdev->state_mutex); + scsi_device_set_state(sdev, SDEV_OFFLINE); + mutex_unlock(&sdev->state_mutex); } __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM); @@ -10152,12 +10545,14 @@ void ufshcd_remove(struct ufs_hba *hba) ufshcd_rpm_get_sync(hba); ufs_hwmon_remove(hba); ufs_bsg_remove(hba); - ufshpb_remove(hba); + ufs_rpmb_remove(hba); ufs_sysfs_remove_nodes(hba->dev); + cancel_delayed_work_sync(&hba->ufs_rtc_update_work); blk_mq_destroy_queue(hba->tmf_queue); blk_put_queue(hba->tmf_queue); blk_mq_free_tag_set(&hba->tmf_tag_set); - scsi_remove_host(hba->host); + if (hba->scsi_host_added) + scsi_remove_host(hba->host); /* disable interrupts */ ufshcd_disable_intr(hba, hba->intr_mask); ufshcd_hba_stop(hba); @@ -10198,10 +10593,7 @@ int ufshcd_system_restore(struct device *dev) * are updated with the latest queue addresses. Only after * updating these addresses, we can queue the new commands. */ - mb(); - - /* Resuming from hibernate, assume that link was OFF */ - ufshcd_set_link_off(hba); + ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H); return 0; @@ -10216,24 +10608,16 @@ EXPORT_SYMBOL_GPL(ufshcd_system_thaw); #endif /* CONFIG_PM_SLEEP */ /** - * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA) - * @hba: pointer to Host Bus Adapter (HBA) - */ -void ufshcd_dealloc_host(struct ufs_hba *hba) -{ - scsi_host_put(hba->host); -} -EXPORT_SYMBOL_GPL(ufshcd_dealloc_host); - -/** * ufshcd_set_dma_mask - Set dma mask based on the controller * addressing capability * @hba: per adapter instance * - * Returns 0 for success, non-zero for failure + * Return: 0 for success, non-zero for failure. */ static int ufshcd_set_dma_mask(struct ufs_hba *hba) { + if (hba->vops && hba->vops->set_dma_mask) + return hba->vops->set_dma_mask(hba); if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) { if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64))) return 0; @@ -10242,10 +10626,25 @@ static int ufshcd_set_dma_mask(struct ufs_hba *hba) } /** + * ufshcd_devres_release - devres cleanup handler, invoked during release of + * hba->dev + * @host: pointer to SCSI host + */ +static void ufshcd_devres_release(void *host) +{ + scsi_host_put(host); +} + +/** * ufshcd_alloc_host - allocate Host Bus Adapter (HBA) * @dev: pointer to device handle * @hba_handle: driver private handle - * Returns 0 on success, non-zero value on failure + * + * Return: 0 on success, non-zero value on failure. + * + * NOTE: There is no corresponding ufshcd_dealloc_host() because this function + * keeps track of its allocations using devres and deallocates everything on + * device removal automatically. */ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) { @@ -10267,6 +10666,12 @@ int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle) err = -ENOMEM; goto out_error; } + + err = devm_add_action_or_reset(dev, ufshcd_devres_release, + host); + if (err) + return err; + host->nr_maps = HCTX_TYPE_POLL + 1; hba = shost_priv(host); hba->host = host; @@ -10296,19 +10701,89 @@ static const struct blk_mq_ops ufshcd_tmf_ops = { .queue_rq = ufshcd_queue_tmf, }; +static int ufshcd_add_scsi_host(struct ufs_hba *hba) +{ + int err; + + WARN_ON_ONCE(!hba->host->can_queue); + WARN_ON_ONCE(!hba->host->cmd_per_lun); + + if (is_mcq_supported(hba)) { + ufshcd_mcq_enable(hba); + err = ufshcd_alloc_mcq(hba); + if (!err) { + ufshcd_config_mcq(hba); + } else { + /* Continue with SDB mode */ + ufshcd_mcq_disable(hba); + use_mcq_mode = false; + dev_err(hba->dev, "MCQ mode is disabled, err=%d\n", + err); + } + } + if (!is_mcq_supported(hba) && !hba->lsdb_sup) { + dev_err(hba->dev, + "%s: failed to initialize (legacy doorbell mode not supported)\n", + __func__); + return -EINVAL; + } + + err = scsi_add_host(hba->host, hba->dev); + if (err) { + dev_err(hba->dev, "scsi_add_host failed\n"); + return err; + } + hba->scsi_host_added = true; + + hba->tmf_tag_set = (struct blk_mq_tag_set) { + .nr_hw_queues = 1, + .queue_depth = hba->nutmrs, + .ops = &ufshcd_tmf_ops, + }; + err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); + if (err < 0) + goto remove_scsi_host; + hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL); + if (IS_ERR(hba->tmf_queue)) { + err = PTR_ERR(hba->tmf_queue); + goto free_tmf_tag_set; + } + hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, + sizeof(*hba->tmf_rqs), GFP_KERNEL); + if (!hba->tmf_rqs) { + err = -ENOMEM; + goto free_tmf_queue; + } + + return 0; + +free_tmf_queue: + blk_mq_destroy_queue(hba->tmf_queue); + blk_put_queue(hba->tmf_queue); + +free_tmf_tag_set: + blk_mq_free_tag_set(&hba->tmf_tag_set); + +remove_scsi_host: + if (hba->scsi_host_added) + scsi_remove_host(hba->host); + + return err; +} + /** * ufshcd_init - Driver initialization routine * @hba: per-adapter instance * @mmio_base: base register address * @irq: Interrupt line of device - * Returns 0 on success, non-zero value on failure + * + * Return: 0 on success; < 0 on failure. */ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) { int err; struct Scsi_Host *host = hba->host; struct device *dev = hba->dev; - char eh_wq_name[sizeof("ufs_eh_wq_00")]; /* * dev_set_drvdata() must be called before any callbacks are registered @@ -10328,6 +10803,37 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) hba->irq = irq; hba->vps = &ufs_hba_vps; + /* + * Initialize clk_gating.lock early since it is being used in + * ufshcd_setup_clocks() + */ + spin_lock_init(&hba->clk_gating.lock); + + /* Initialize mutex for PM QoS request synchronization */ + mutex_init(&hba->pm_qos_mutex); + + /* + * Set the default power management level for runtime and system PM. + * Host controller drivers can override them in their + * 'ufs_hba_variant_ops::init' callback. + * + * Default power saving mode is to keep UFS link in Hibern8 state + * and UFS device in sleep state. + */ + hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( + UFS_SLEEP_PWR_MODE, + UIC_LINK_HIBERN8_STATE); + + /* + * Most ufs devices require 1ms delay after vcc is powered off before + * it can be powered on again. Set the default to 2ms. The platform + * drivers can override this setting as needed. + */ + hba->vcc_off_delay_us = 2000; + err = ufshcd_hba_init(hba); if (err) goto out_error; @@ -10360,7 +10866,11 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_host_memory_configure(hba); host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED; - host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED; + /* + * Set the queue depth for WLUNs. ufs_get_device_desc() will increase + * host->cmd_per_lun to a larger value. + */ + host->cmd_per_lun = 1; host->max_id = UFSHCD_MAX_ID; host->max_lun = UFS_MAX_LUNS; host->max_channel = UFSHCD_MAX_CHANNEL; @@ -10368,12 +10878,15 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) host->max_cmd_len = UFS_CDB_SIZE; host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING); + /* Use default RPM delay if host not set */ + if (host->rpm_autosuspend_delay == 0) + host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS; + hba->max_pwr_info.is_valid = false; /* Initialize work queues */ - snprintf(eh_wq_name, sizeof(eh_wq_name), "ufs_eh_wq_%d", - hba->host->host_no); - hba->eh_wq = create_singlethread_workqueue(eh_wq_name); + hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM, + hba->host->host_no); if (!hba->eh_wq) { dev_err(hba->dev, "%s: failed to create eh workqueue\n", __func__); @@ -10395,6 +10908,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) mutex_init(&hba->ee_ctrl_mutex); mutex_init(&hba->wb_mutex); + init_rwsem(&hba->clk_scaling_lock); ufshcd_init_clk_gating(hba); @@ -10413,10 +10927,11 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) * Make sure that UFS interrupts are disabled and any pending interrupt * status is cleared before registering UFS interrupt handler. */ - mb(); + ufshcd_readl(hba, REG_INTERRUPT_ENABLE); /* IRQ registration */ - err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); + err = devm_request_threaded_irq(dev, irq, ufshcd_intr, ufshcd_threaded_intr, + IRQF_ONESHOT | IRQF_SHARED, UFSHCD, hba); if (err) { dev_err(hba->dev, "request irq failed\n"); goto out_disable; @@ -10424,35 +10939,6 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) hba->is_irq_enabled = true; } - if (!is_mcq_supported(hba)) { - err = scsi_add_host(host, hba->dev); - if (err) { - dev_err(hba->dev, "scsi_add_host failed\n"); - goto out_disable; - } - } - - hba->tmf_tag_set = (struct blk_mq_tag_set) { - .nr_hw_queues = 1, - .queue_depth = hba->nutmrs, - .ops = &ufshcd_tmf_ops, - .flags = BLK_MQ_F_NO_SCHED, - }; - err = blk_mq_alloc_tag_set(&hba->tmf_tag_set); - if (err < 0) - goto out_remove_scsi_host; - hba->tmf_queue = blk_mq_init_queue(&hba->tmf_tag_set); - if (IS_ERR(hba->tmf_queue)) { - err = PTR_ERR(hba->tmf_queue); - goto free_tmf_tag_set; - } - hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs, - sizeof(*hba->tmf_rqs), GFP_KERNEL); - if (!hba->tmf_rqs) { - err = -ENOMEM; - goto free_tmf_queue; - } - /* Reset the attached device */ ufshcd_device_reset(hba); @@ -10464,23 +10950,11 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) dev_err(hba->dev, "Host controller enable failed\n"); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); - goto free_tmf_queue; + goto out_disable; } - /* - * Set the default power management level for runtime and system PM. - * Default power saving mode is to keep UFS link in Hibern8 state - * and UFS device in sleep state. - */ - hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( - UFS_SLEEP_PWR_MODE, - UIC_LINK_HIBERN8_STATE); - hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state( - UFS_SLEEP_PWR_MODE, - UIC_LINK_HIBERN8_STATE); - - INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, - ufshcd_rpm_dev_flush_recheck_work); + INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); + INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work); /* Set the default auto-hiberate idle timer value to 150 ms */ if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) { @@ -10488,9 +10962,13 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3); } + err = ufshcd_add_scsi_host(hba); + if (err) + goto out_disable; + /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); - atomic_set(&hba->scsi_block_reqs_cnt, 0); + /* * We are assuming that device wasn't put in sleep/power-down * state exclusively during the boot stage before kernel. @@ -10499,24 +10977,57 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) */ ufshcd_set_ufs_dev_active(hba); - async_schedule(ufshcd_async_scan, hba); + /* Initialize hba, detect and initialize UFS device */ + ktime_t probe_start = ktime_get(); + + hba->ufshcd_state = UFSHCD_STATE_RESET; + + err = ufshcd_link_startup(hba); + if (err) + goto out_disable; + + if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION) + goto initialized; + + /* Debug counters initialization */ + ufshcd_clear_dbg_ufs_stats(hba); + + /* UniPro link is active now */ + ufshcd_set_link_active(hba); + + /* Verify device initialization by sending NOP OUT UPIU */ + err = ufshcd_verify_dev_init(hba); + if (err) + goto out_disable; + + /* Initiate UFS initialization, and waiting until completion */ + err = ufshcd_complete_dev_init(hba); + if (err) + goto out_disable; + + err = ufshcd_device_params_init(hba); + if (err) + goto out_disable; + + err = ufshcd_post_device_init(hba); + +initialized: + ufshcd_process_probe_result(hba, probe_start, err); + if (err) + goto out_disable; + ufs_sysfs_add_nodes(hba->dev); + async_schedule(ufshcd_async_scan, hba); device_enable_async_suspend(dev); + ufshcd_pm_qos_init(hba); return 0; -free_tmf_queue: - blk_mq_destroy_queue(hba->tmf_queue); - blk_put_queue(hba->tmf_queue); -free_tmf_tag_set: - blk_mq_free_tag_set(&hba->tmf_tag_set); -out_remove_scsi_host: - scsi_remove_host(hba->host); out_disable: hba->is_irq_enabled = false; ufshcd_hba_exit(hba); out_error: - return err; + return err > 0 ? -EIO : err; } EXPORT_SYMBOL_GPL(ufshcd_init); @@ -10632,6 +11143,53 @@ static const struct dev_pm_ops ufshcd_wl_pm_ops = { SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL) }; +static void ufshcd_check_header_layout(void) +{ + /* + * gcc compilers before version 10 cannot do constant-folding for + * sub-byte bitfields. Hence skip the layout checks for gcc 9 and + * before. + */ + if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000) + return; + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .cci = 3})[0] != 3); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .ehs_length = 2})[1] != 2); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .enable_crypto = 1})[2] + != 0x80); + + BUILD_BUG_ON((((u8 *)&(struct request_desc_header){ + .command_type = 5, + .data_direction = 3, + .interrupt = 1, + })[3]) != ((5 << 4) | (3 << 1) | 1)); + + BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ + .dunl = cpu_to_le32(0xdeadbeef)})[1] != + cpu_to_le32(0xdeadbeef)); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .ocs = 4})[8] != 4); + + BUILD_BUG_ON(((u8 *)&(struct request_desc_header){ + .cds = 5})[9] != 5); + + BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){ + .dunu = cpu_to_le32(0xbadcafe)})[3] != + cpu_to_le32(0xbadcafe)); + + BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ + .iid = 0xf })[4] != 0xf0); + + BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){ + .command_set_type = 0xf })[4] != 0xf); +} + /* * ufs_dev_wlun_template - describes ufs device wlun * ufs-device wlun - used to send pm commands @@ -10645,7 +11203,6 @@ static const struct dev_pm_ops ufshcd_wl_pm_ops = { static struct scsi_driver ufs_dev_wlun_template = { .gendrv = { .name = "ufs_device_wlun", - .owner = THIS_MODULE, .probe = ufshcd_wl_probe, .remove = ufshcd_wl_remove, .pm = &ufshcd_wl_pm_ops, @@ -10657,6 +11214,8 @@ static int __init ufshcd_core_init(void) { int ret; + ufshcd_check_header_layout(); + ufs_debugfs_init(); ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv); |