diff options
Diffstat (limited to 'drivers/scsi/mpt3sas/mpt3sas_base.c')
| -rw-r--r-- | drivers/scsi/mpt3sas/mpt3sas_base.c | 3481 |
1 files changed, 2831 insertions, 650 deletions
diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 0a6cb8f0680c..0d652db8fe24 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -60,7 +60,6 @@ #include <linux/ktime.h> #include <linux/kthread.h> #include <asm/page.h> /* To get host page size per arch */ -#include <linux/aer.h> #include "mpt3sas_base.h" @@ -74,32 +73,73 @@ static MPT_CALLBACK mpt_callbacks[MPT_MAX_CALLBACKS]; #define MAX_HBA_QUEUE_DEPTH 30000 #define MAX_CHAIN_DEPTH 100000 static int max_queue_depth = -1; -module_param(max_queue_depth, int, 0); +module_param(max_queue_depth, int, 0444); MODULE_PARM_DESC(max_queue_depth, " max controller queue depth "); static int max_sgl_entries = -1; -module_param(max_sgl_entries, int, 0); +module_param(max_sgl_entries, int, 0444); MODULE_PARM_DESC(max_sgl_entries, " max sg entries "); static int msix_disable = -1; -module_param(msix_disable, int, 0); +module_param(msix_disable, int, 0444); MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)"); static int smp_affinity_enable = 1; -module_param(smp_affinity_enable, int, S_IRUGO); +module_param(smp_affinity_enable, int, 0444); MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)"); static int max_msix_vectors = -1; -module_param(max_msix_vectors, int, 0); +module_param(max_msix_vectors, int, 0444); MODULE_PARM_DESC(max_msix_vectors, " max msix vectors"); +static int irqpoll_weight = -1; +module_param(irqpoll_weight, int, 0444); +MODULE_PARM_DESC(irqpoll_weight, + "irq poll weight (default= one fourth of HBA queue depth)"); + static int mpt3sas_fwfault_debug; MODULE_PARM_DESC(mpt3sas_fwfault_debug, " enable detection of firmware fault and halt firmware - (default=0)"); +static int perf_mode = -1; +module_param(perf_mode, int, 0444); +MODULE_PARM_DESC(perf_mode, + "Performance mode (only for Aero/Sea Generation), options:\n\t\t" + "0 - balanced: high iops mode is enabled &\n\t\t" + "interrupt coalescing is enabled only on high iops queues,\n\t\t" + "1 - iops: high iops mode is disabled &\n\t\t" + "interrupt coalescing is enabled on all queues,\n\t\t" + "2 - latency: high iops mode is disabled &\n\t\t" + "interrupt coalescing is enabled on all queues with timeout value 0xA,\n" + "\t\tdefault - default perf_mode is 'balanced'" + ); + +static int poll_queues; +module_param(poll_queues, int, 0444); +MODULE_PARM_DESC(poll_queues, "Number of queues to be use for io_uring poll mode.\n\t\t" + "This parameter is effective only if host_tagset_enable=1. &\n\t\t" + "when poll_queues are enabled then &\n\t\t" + "perf_mode is set to latency mode. &\n\t\t" + ); + +enum mpt3sas_perf_mode { + MPT_PERF_MODE_DEFAULT = -1, + MPT_PERF_MODE_BALANCED = 0, + MPT_PERF_MODE_IOPS = 1, + MPT_PERF_MODE_LATENCY = 2, +}; + +static int +_base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, + u32 ioc_state, int timeout); static int _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc); +static void +_base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc); + +static u32 +_base_readl_ext_retry(const void __iomem *addr); /** * mpt3sas_base_check_cmd_timeout - Function @@ -111,7 +151,7 @@ _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc); * @mpi_request:mf request pointer. * @sz: size of buffer. * - * @Returns - 1/0 Reset to be done or Not + * Return: 1/0 Reset to be done or Not */ u8 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc, @@ -158,13 +198,13 @@ module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug, /** * _base_readl_aero - retry readl for max three times. - * @addr - MPT Fusion system interface register address + * @addr: MPT Fusion system interface register address * * Retry the readl() for max three times if it gets zero value * while reading the system interface register. */ static inline u32 -_base_readl_aero(const volatile void __iomem *addr) +_base_readl_aero(const void __iomem *addr) { u32 i = 0, ret_val; @@ -176,8 +216,22 @@ _base_readl_aero(const volatile void __iomem *addr) return ret_val; } +static u32 +_base_readl_ext_retry(const void __iomem *addr) +{ + u32 i, ret_val; + + for (i = 0 ; i < 30 ; i++) { + ret_val = readl(addr); + if (ret_val != 0) + break; + } + + return ret_val; +} + static inline u32 -_base_readl(const volatile void __iomem *addr) +_base_readl(const void __iomem *addr) { return readl(addr); } @@ -381,7 +435,7 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc, { Mpi2SGESimple32_t *sgel, *sgel_next; u32 sgl_flags, sge_chain_count = 0; - bool is_write = 0; + bool is_write = false; u16 i = 0; void __iomem *buffer_iomem; phys_addr_t buffer_iomem_phys; @@ -410,7 +464,7 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc, return; /* From smid we can get scsi_cmd, once we have sg_scmd, - * we just need to get sg_virt and sg_next to get virual + * we just need to get sg_virt and sg_next to get virtual * address associated with sgel->Address. */ @@ -450,7 +504,7 @@ static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc, if (le32_to_cpu(sgel->FlagsLength) & (MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT)) - is_write = 1; + is_write = true; for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) { @@ -567,6 +621,71 @@ static int mpt3sas_remove_dead_ioc_func(void *arg) } /** + * _base_sync_drv_fw_timestamp - Sync Drive-Fw TimeStamp. + * @ioc: Per Adapter Object + * + * Return: nothing. + */ +static void _base_sync_drv_fw_timestamp(struct MPT3SAS_ADAPTER *ioc) +{ + Mpi26IoUnitControlRequest_t *mpi_request; + Mpi26IoUnitControlReply_t *mpi_reply; + u16 smid; + ktime_t current_time; + u64 TimeStamp = 0; + u8 issue_reset = 0; + + mutex_lock(&ioc->scsih_cmds.mutex); + if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) { + ioc_err(ioc, "scsih_cmd in use %s\n", __func__); + goto out; + } + ioc->scsih_cmds.status = MPT3_CMD_PENDING; + smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx); + if (!smid) { + ioc_err(ioc, "Failed obtaining a smid %s\n", __func__); + ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; + goto out; + } + mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); + ioc->scsih_cmds.smid = smid; + memset(mpi_request, 0, sizeof(Mpi26IoUnitControlRequest_t)); + mpi_request->Function = MPI2_FUNCTION_IO_UNIT_CONTROL; + mpi_request->Operation = MPI26_CTRL_OP_SET_IOC_PARAMETER; + mpi_request->IOCParameter = MPI26_SET_IOC_PARAMETER_SYNC_TIMESTAMP; + current_time = ktime_get_real(); + TimeStamp = ktime_to_ms(current_time); + mpi_request->Reserved7 = cpu_to_le32(TimeStamp >> 32); + mpi_request->IOCParameterValue = cpu_to_le32(TimeStamp & 0xFFFFFFFF); + init_completion(&ioc->scsih_cmds.done); + ioc->put_smid_default(ioc, smid); + dinitprintk(ioc, ioc_info(ioc, + "Io Unit Control Sync TimeStamp (sending), @time %lld ms\n", + TimeStamp)); + wait_for_completion_timeout(&ioc->scsih_cmds.done, + MPT3SAS_TIMESYNC_TIMEOUT_SECONDS*HZ); + if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) { + mpt3sas_check_cmd_timeout(ioc, + ioc->scsih_cmds.status, mpi_request, + sizeof(Mpi2SasIoUnitControlRequest_t)/4, issue_reset); + goto issue_host_reset; + } + if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) { + mpi_reply = ioc->scsih_cmds.reply; + dinitprintk(ioc, ioc_info(ioc, + "Io Unit Control sync timestamp (complete): ioc_status(0x%04x), loginfo(0x%08x)\n", + le16_to_cpu(mpi_reply->IOCStatus), + le32_to_cpu(mpi_reply->IOCLogInfo))); + } +issue_host_reset: + if (issue_reset) + mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); + ioc->scsih_cmds.status = MPT3_CMD_NOT_USED; +out: + mutex_unlock(&ioc->scsih_cmds.mutex); +} + +/** * _base_fault_reset_work - workq handling ioc fault conditions * @work: input argument, used to derive ioc * @@ -584,7 +703,8 @@ _base_fault_reset_work(struct work_struct *work) spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); - if (ioc->shost_recovery || ioc->pci_error_recovery) + if ((ioc->shost_recovery && (ioc->ioc_coredump_loop == 0)) || + ioc->pci_error_recovery) goto rearm_timer; spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); @@ -608,11 +728,12 @@ _base_fault_reset_work(struct work_struct *work) /* * Call _scsih_flush_pending_cmds callback so that we flush all - * pending commands back to OS. This call is required to aovid + * pending commands back to OS. This call is required to avoid * deadlock at block layer. Dead IOC will fail to do diag reset, * and this call is safe since dead ioc will never return any * command back from HW. */ + mpt3sas_base_pause_mq_polling(ioc); ioc->schedule_dead_ioc_flush_running_cmds(ioc); /* * Set remove_host flag early since kernel thread will @@ -631,21 +752,70 @@ _base_fault_reset_work(struct work_struct *work) return; /* don't rearm timer */ } - ioc->non_operational_loop = 0; + if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) { + u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ? + ioc->manu_pg11.CoreDumpTOSec : + MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS; + + timeout /= (FAULT_POLLING_INTERVAL/1000); + + if (ioc->ioc_coredump_loop == 0) { + mpt3sas_print_coredump_info(ioc, + doorbell & MPI2_DOORBELL_DATA_MASK); + /* do not accept any IOs and disable the interrupts */ + spin_lock_irqsave( + &ioc->ioc_reset_in_progress_lock, flags); + ioc->shost_recovery = 1; + spin_unlock_irqrestore( + &ioc->ioc_reset_in_progress_lock, flags); + mpt3sas_base_mask_interrupts(ioc); + mpt3sas_base_pause_mq_polling(ioc); + _base_clear_outstanding_commands(ioc); + } + + ioc_info(ioc, "%s: CoreDump loop %d.", + __func__, ioc->ioc_coredump_loop); + + /* Wait until CoreDump completes or times out */ + if (ioc->ioc_coredump_loop++ < timeout) { + spin_lock_irqsave( + &ioc->ioc_reset_in_progress_lock, flags); + goto rearm_timer; + } + } + if (ioc->ioc_coredump_loop) { + if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_COREDUMP) + ioc_err(ioc, "%s: CoreDump completed. LoopCount: %d", + __func__, ioc->ioc_coredump_loop); + else + ioc_err(ioc, "%s: CoreDump Timed out. LoopCount: %d", + __func__, ioc->ioc_coredump_loop); + ioc->ioc_coredump_loop = MPT3SAS_COREDUMP_LOOP_DONE; + } + ioc->non_operational_loop = 0; if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) { rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER); ioc_warn(ioc, "%s: hard reset: %s\n", __func__, rc == 0 ? "success" : "failed"); doorbell = mpt3sas_base_get_iocstate(ioc, 0); - if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) - mpt3sas_base_fault_info(ioc, doorbell & + if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { + mpt3sas_print_fault_code(ioc, doorbell & + MPI2_DOORBELL_DATA_MASK); + } else if ((doorbell & MPI2_IOC_STATE_MASK) == + MPI2_IOC_STATE_COREDUMP) + mpt3sas_print_coredump_info(ioc, doorbell & MPI2_DOORBELL_DATA_MASK); if (rc && (doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) return; /* don't rearm timer */ } - + ioc->ioc_coredump_loop = 0; + if (ioc->time_sync_interval && + ++ioc->timestamp_update_count >= ioc->time_sync_interval) { + ioc->timestamp_update_count = 0; + _base_sync_drv_fw_timestamp(ioc); + } spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); rearm_timer: if (ioc->fault_reset_work_q) @@ -669,14 +839,15 @@ mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc) if (ioc->fault_reset_work_q) return; + ioc->timestamp_update_count = 0; /* initialize fault polling */ INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work); snprintf(ioc->fault_reset_work_q_name, sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status", ioc->driver_name, ioc->id); - ioc->fault_reset_work_q = - create_singlethread_workqueue(ioc->fault_reset_work_q_name); + ioc->fault_reset_work_q = alloc_ordered_workqueue( + "%s", WQ_MEM_RECLAIM, ioc->fault_reset_work_q_name); if (!ioc->fault_reset_work_q) { ioc_err(ioc, "%s: failed (line=%d)\n", __func__, __LINE__); return; @@ -718,12 +889,56 @@ mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc) * @fault_code: fault code */ void -mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code) +mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code) { ioc_err(ioc, "fault_state(0x%04x)!\n", fault_code); } /** + * mpt3sas_base_coredump_info - verbose translation of firmware CoreDump state + * @ioc: per adapter object + * @fault_code: fault code + * + * Return: nothing. + */ +void +mpt3sas_base_coredump_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code) +{ + ioc_err(ioc, "coredump_state(0x%04x)!\n", fault_code); +} + +/** + * mpt3sas_base_wait_for_coredump_completion - Wait until coredump + * completes or times out + * @ioc: per adapter object + * @caller: caller function name + * + * Return: 0 for success, non-zero for failure. + */ +int +mpt3sas_base_wait_for_coredump_completion(struct MPT3SAS_ADAPTER *ioc, + const char *caller) +{ + u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ? + ioc->manu_pg11.CoreDumpTOSec : + MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS; + + int ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_FAULT, + timeout); + + if (ioc_state) + ioc_err(ioc, + "%s: CoreDump timed out. (ioc_state=0x%x)\n", + caller, ioc_state); + else + ioc_info(ioc, + "%s: CoreDump completed. (ioc_state=0x%x)\n", + caller, ioc_state); + + return ioc_state; +} + +/** * mpt3sas_halt_firmware - halt's mpt controller firmware * @ioc: per adapter object * @@ -742,10 +957,15 @@ mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc) dump_stack(); - doorbell = ioc->base_readl(&ioc->chip->Doorbell); - if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) - mpt3sas_base_fault_info(ioc , doorbell); - else { + doorbell = ioc->base_readl_ext_retry(&ioc->chip->Doorbell); + if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { + mpt3sas_print_fault_code(ioc, doorbell & + MPI2_DOORBELL_DATA_MASK); + } else if ((doorbell & MPI2_IOC_STATE_MASK) == + MPI2_IOC_STATE_COREDUMP) { + mpt3sas_print_coredump_info(ioc, doorbell & + MPI2_DOORBELL_DATA_MASK); + } else { writel(0xC0FFEE00, &ioc->chip->Doorbell); ioc_err(ioc, "Firmware is halted due to command timeout\n"); } @@ -781,6 +1001,20 @@ _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply, if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE) return; + /* + * Older Firmware version doesn't support driver trigger pages. + * So, skip displaying 'config invalid type' type + * of error message. + */ + if (request_hdr->Function == MPI2_FUNCTION_CONFIG) { + Mpi2ConfigRequest_t *rqst = (Mpi2ConfigRequest_t *)request_hdr; + + if ((rqst->ExtPageType == + MPI2_CONFIG_EXTPAGETYPE_DRIVER_PERSISTENT_TRIGGER) && + !(ioc->logging_level & MPT_DEBUG_CONFIG)) { + return; + } + } switch (ioc_status) { @@ -839,7 +1073,7 @@ _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply, desc = "config no defaults"; break; case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT: - desc = "config cant commit"; + desc = "config can't commit"; break; /**************************************************************************** @@ -968,6 +1202,11 @@ _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply, ioc->sge_size; func_str = "nvme_encapsulated"; break; + case MPI2_FUNCTION_MCTP_PASSTHROUGH: + frame_sz = sizeof(Mpi26MctpPassthroughRequest_t) + + ioc->sge_size; + func_str = "mctp_passthru"; + break; default: frame_sz = 32; func_str = "unknown"; @@ -1103,7 +1342,7 @@ _base_display_event_data(struct MPT3SAS_ADAPTER *ioc, * @log_info: log info */ static void -_base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info) +_base_sas_log_info(struct MPT3SAS_ADAPTER *ioc, u32 log_info) { union loginfo_type { u32 loginfo; @@ -1151,11 +1390,11 @@ _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info) } /** - * _base_display_reply_info - + * _base_display_reply_info - handle reply descriptors depending on IOC Status * @ioc: per adapter object * @smid: system request message index * @msix_index: MSIX table index supplied by the OS - * @reply: reply message frame(lower 32bit addr) + * @reply: reply message frame (lower 32bit addr) */ static void _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, @@ -1175,13 +1414,19 @@ _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, if ((ioc_status & MPI2_IOCSTATUS_MASK) && (ioc->logging_level & MPT_DEBUG_REPLY)) { - _base_sas_ioc_info(ioc , mpi_reply, + _base_sas_ioc_info(ioc, mpi_reply, mpt3sas_base_get_msg_frame(ioc, smid)); } if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) { loginfo = le32_to_cpu(mpi_reply->IOCLogInfo); - _base_sas_log_info(ioc, loginfo); + if (ioc->logging_level & MPT_DEBUG_REPLY) + _base_sas_log_info(ioc, loginfo); + else { + if (!((ioc_status & MPI2_IOCSTATUS_MASK) & + MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)) + _base_sas_log_info(ioc, loginfo); + } } if (ioc_status || loginfo) { @@ -1277,7 +1522,7 @@ _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply) ack_request->EventContext = mpi_reply->EventContext; ack_request->VF_ID = 0; /* TODO */ ack_request->VP_ID = 0; - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); out: @@ -1340,13 +1585,62 @@ _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid) } /** - * _base_mask_interrupts - disable interrupts + * mpt3sas_base_pause_mq_polling - pause polling on the mq poll queues + * when driver is flushing out the IOs. + * @ioc: per adapter object + * + * Pause polling on the mq poll (io uring) queues when driver is flushing + * out the IOs. Otherwise we may see the race condition of completing the same + * IO from two paths. + * + * Returns nothing. + */ +void +mpt3sas_base_pause_mq_polling(struct MPT3SAS_ADAPTER *ioc) +{ + int iopoll_q_count = + ioc->reply_queue_count - ioc->iopoll_q_start_index; + int qid; + + for (qid = 0; qid < iopoll_q_count; qid++) + atomic_set(&ioc->io_uring_poll_queues[qid].pause, 1); + + /* + * wait for current poll to complete. + */ + for (qid = 0; qid < iopoll_q_count; qid++) { + while (atomic_read(&ioc->io_uring_poll_queues[qid].busy)) { + cpu_relax(); + udelay(500); + } + } +} + +/** + * mpt3sas_base_resume_mq_polling - Resume polling on mq poll queues. + * @ioc: per adapter object + * + * Returns nothing. + */ +void +mpt3sas_base_resume_mq_polling(struct MPT3SAS_ADAPTER *ioc) +{ + int iopoll_q_count = + ioc->reply_queue_count - ioc->iopoll_q_start_index; + int qid; + + for (qid = 0; qid < iopoll_q_count; qid++) + atomic_set(&ioc->io_uring_poll_queues[qid].pause, 0); +} + +/** + * mpt3sas_base_mask_interrupts - disable interrupts * @ioc: per adapter object * * Disabling ResetIRQ, Reply and Doorbell Interrupts */ -static void -_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc) +void +mpt3sas_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc) { u32 him_register; @@ -1358,13 +1652,13 @@ _base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc) } /** - * _base_unmask_interrupts - enable interrupts + * mpt3sas_base_unmask_interrupts - enable interrupts * @ioc: per adapter object * * Enabling only Reply Interrupts */ -static void -_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc) +void +mpt3sas_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc) { u32 him_register; @@ -1382,20 +1676,30 @@ union reply_descriptor { } u; }; +static u32 base_mod64(u64 dividend, u32 divisor) +{ + u32 remainder; + + if (!divisor) + pr_err("mpt3sas: DIVISOR is zero, in div fn\n"); + remainder = do_div(dividend, divisor); + return remainder; +} + /** - * _base_interrupt - MPT adapter (IOC) specific interrupt handler. - * @irq: irq number (not used) - * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure + * _base_process_reply_queue - Process reply descriptors from reply + * descriptor post queue. + * @reply_q: per IRQ's reply queue object. * - * Return: IRQ_HANDLED if processed, else IRQ_NONE. + * Return: number of reply descriptors processed from reply + * descriptor queue. */ -static irqreturn_t -_base_interrupt(int irq, void *bus_id) +static int +_base_process_reply_queue(struct adapter_reply_queue *reply_q) { - struct adapter_reply_queue *reply_q = bus_id; union reply_descriptor rd; - u32 completed_cmds; - u8 request_desript_type; + u64 completed_cmds; + u8 request_descript_type; u16 smid; u8 cb_idx; u32 reply; @@ -1404,21 +1708,18 @@ _base_interrupt(int irq, void *bus_id) Mpi2ReplyDescriptorsUnion_t *rpf; u8 rc; - if (ioc->mask_interrupts) - return IRQ_NONE; - + completed_cmds = 0; if (!atomic_add_unless(&reply_q->busy, 1, 1)) - return IRQ_NONE; + return completed_cmds; rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index]; - request_desript_type = rpf->Default.ReplyFlags + request_descript_type = rpf->Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; - if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) { + if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) { atomic_dec(&reply_q->busy); - return IRQ_NONE; + return completed_cmds; } - completed_cmds = 0; cb_idx = 0xFF; do { rd.word = le64_to_cpu(rpf->Words); @@ -1426,11 +1727,11 @@ _base_interrupt(int irq, void *bus_id) goto out; reply = 0; smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1); - if (request_desript_type == + if (request_descript_type == MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS || - request_desript_type == + request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS || - request_desript_type == + request_descript_type == MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) { cb_idx = _base_get_cb_idx(ioc, smid); if ((likely(cb_idx < MPT_MAX_CALLBACKS)) && @@ -1440,7 +1741,7 @@ _base_interrupt(int irq, void *bus_id) if (rc) mpt3sas_base_free_smid(ioc, smid); } - } else if (request_desript_type == + } else if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) { reply = le32_to_cpu( rpf->AddressReply.ReplyFrameAddress); @@ -1486,7 +1787,7 @@ _base_interrupt(int irq, void *bus_id) (reply_q->reply_post_host_index == (ioc->reply_post_queue_depth - 1)) ? 0 : reply_q->reply_post_host_index + 1; - request_desript_type = + request_descript_type = reply_q->reply_post_free[reply_q->reply_post_host_index]. Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; completed_cmds++; @@ -1495,7 +1796,7 @@ _base_interrupt(int irq, void *bus_id) * So that FW can find enough entries to post the Reply * Descriptors in the reply descriptor post queue. */ - if (completed_cmds > ioc->hba_queue_depth/3) { + if (completed_cmds >= ioc->thresh_hold) { if (ioc->combined_reply_queue) { writel(reply_q->reply_post_host_index | ((msix_index & 7) << @@ -1507,9 +1808,15 @@ _base_interrupt(int irq, void *bus_id) MPI2_RPHI_MSIX_INDEX_SHIFT), &ioc->chip->ReplyPostHostIndex); } - completed_cmds = 1; + if (!reply_q->is_iouring_poll_q && + !reply_q->irq_poll_scheduled) { + reply_q->irq_poll_scheduled = true; + irq_poll_sched(&reply_q->irqpoll); + } + atomic_dec(&reply_q->busy); + return completed_cmds; } - if (request_desript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) + if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) goto out; if (!reply_q->reply_post_host_index) rpf = reply_q->reply_post_free; @@ -1521,14 +1828,14 @@ _base_interrupt(int irq, void *bus_id) if (!completed_cmds) { atomic_dec(&reply_q->busy); - return IRQ_NONE; + return completed_cmds; } if (ioc->is_warpdrive) { writel(reply_q->reply_post_host_index, ioc->reply_post_host_index[msix_index]); atomic_dec(&reply_q->busy); - return IRQ_HANDLED; + return completed_cmds; } /* Update Reply Post Host Index. @@ -1555,7 +1862,118 @@ _base_interrupt(int irq, void *bus_id) MPI2_RPHI_MSIX_INDEX_SHIFT), &ioc->chip->ReplyPostHostIndex); atomic_dec(&reply_q->busy); - return IRQ_HANDLED; + return completed_cmds; +} + +/** + * mpt3sas_blk_mq_poll - poll the blk mq poll queue + * @shost: Scsi_Host object + * @queue_num: hw ctx queue number + * + * Return number of entries that has been processed from poll queue. + */ +int mpt3sas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num) +{ + struct MPT3SAS_ADAPTER *ioc = + (struct MPT3SAS_ADAPTER *)shost->hostdata; + struct adapter_reply_queue *reply_q; + int num_entries = 0; + int qid = queue_num - ioc->iopoll_q_start_index; + + if (atomic_read(&ioc->io_uring_poll_queues[qid].pause) || + !atomic_add_unless(&ioc->io_uring_poll_queues[qid].busy, 1, 1)) + return 0; + + reply_q = ioc->io_uring_poll_queues[qid].reply_q; + + num_entries = _base_process_reply_queue(reply_q); + atomic_dec(&ioc->io_uring_poll_queues[qid].busy); + + return num_entries; +} + +/** + * _base_interrupt - MPT adapter (IOC) specific interrupt handler. + * @irq: irq number (not used) + * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure + * + * Return: IRQ_HANDLED if processed, else IRQ_NONE. + */ +static irqreturn_t +_base_interrupt(int irq, void *bus_id) +{ + struct adapter_reply_queue *reply_q = bus_id; + struct MPT3SAS_ADAPTER *ioc = reply_q->ioc; + + if (ioc->mask_interrupts) + return IRQ_NONE; + if (reply_q->irq_poll_scheduled) + return IRQ_HANDLED; + return ((_base_process_reply_queue(reply_q) > 0) ? + IRQ_HANDLED : IRQ_NONE); +} + +/** + * _base_irqpoll - IRQ poll callback handler + * @irqpoll: irq_poll object + * @budget: irq poll weight + * + * Return: number of reply descriptors processed + */ +static int +_base_irqpoll(struct irq_poll *irqpoll, int budget) +{ + struct adapter_reply_queue *reply_q; + int num_entries = 0; + + reply_q = container_of(irqpoll, struct adapter_reply_queue, + irqpoll); + if (reply_q->irq_line_enable) { + disable_irq_nosync(reply_q->os_irq); + reply_q->irq_line_enable = false; + } + num_entries = _base_process_reply_queue(reply_q); + if (num_entries < budget) { + irq_poll_complete(irqpoll); + reply_q->irq_poll_scheduled = false; + reply_q->irq_line_enable = true; + enable_irq(reply_q->os_irq); + /* + * Go for one more round of processing the + * reply descriptor post queue in case the HBA + * Firmware has posted some reply descriptors + * while reenabling the IRQ. + */ + _base_process_reply_queue(reply_q); + } + + return num_entries; +} + +/** + * _base_init_irqpolls - initliaze IRQ polls + * @ioc: per adapter object + * + * Return: nothing + */ +static void +_base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc) +{ + struct adapter_reply_queue *reply_q, *next; + + if (list_empty(&ioc->reply_queue_list)) + return; + + list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) { + if (reply_q->is_iouring_poll_q) + continue; + irq_poll_init(&reply_q->irqpoll, + ioc->hba_queue_depth/4, _base_irqpoll); + reply_q->irq_poll_scheduled = false; + reply_q->irq_line_enable = true; + reply_q->os_irq = pci_irq_vector(ioc->pdev, + reply_q->msix_index); + } } /** @@ -1574,12 +1992,14 @@ _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc) /** * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts * @ioc: per adapter object - * Context: non ISR conext + * @poll: poll over reply descriptor pools incase interrupt for + * timed-out SCSI command got delayed + * Context: non-ISR context * * Called when a Task Management request has completed. */ void -mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc) +mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll) { struct adapter_reply_queue *reply_q; @@ -1596,7 +2016,31 @@ mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc) /* TMs are on msix_index == 0 */ if (reply_q->msix_index == 0) continue; + + if (reply_q->is_iouring_poll_q) { + _base_process_reply_queue(reply_q); + continue; + } + synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index)); + if (reply_q->irq_poll_scheduled) { + /* Calling irq_poll_disable will wait for any pending + * callbacks to have completed. + */ + irq_poll_disable(&reply_q->irqpoll); + irq_poll_enable(&reply_q->irqpoll); + /* check how the scheduled poll has ended, + * clean up only if necessary + */ + if (reply_q->irq_poll_scheduled) { + reply_q->irq_poll_scheduled = false; + reply_q->irq_line_enable = true; + enable_irq(reply_q->os_irq); + } + } + + if (poll) + _base_process_reply_queue(reply_q); } } @@ -1783,7 +2227,16 @@ _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge, /** * _base_build_nvme_prp - This function is called for NVMe end devices to build - * a native SGL (NVMe PRP). The native SGL is built starting in the first PRP + * a native SGL (NVMe PRP). + * @ioc: per adapter object + * @smid: system request message index for getting asscociated SGL + * @nvme_encap_request: the NVMe request msg frame pointer + * @data_out_dma: physical address for WRITES + * @data_out_sz: data xfer size for WRITES + * @data_in_dma: physical address for READS + * @data_in_sz: data xfer size for READS + * + * The native SGL is built starting in the first PRP * entry of the NVMe message (PRP1). If the data buffer is small enough to be * described entirely using PRP1, then PRP2 is not used. If needed, PRP2 is * used to describe a larger data buffer. If the data buffer is too large to @@ -1812,7 +2265,7 @@ _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge, * Each 64-bit PRP entry comprises an address and an offset field. The address * always points at the beginning of a 4KB physical memory page, and the offset * describes where within that 4KB page the memory segment begins. Only the - * first element in a PRP list may contain a non-zero offest, implying that all + * first element in a PRP list may contain a non-zero offset, implying that all * memory segments following the first begin at the start of a 4KB page. * * Each PRP element normally describes 4KB of physical memory, with exceptions @@ -1826,14 +2279,6 @@ _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge, * Since PRP entries lack any indication of size, the overall data buffer length * is used to determine where the end of the data memory buffer is located, and * how many PRP entries are required to describe it. - * - * @ioc: per adapter object - * @smid: system request message index for getting asscociated SGL - * @nvme_encap_request: the NVMe request msg frame pointer - * @data_out_dma: physical address for WRITES - * @data_out_sz: data xfer size for WRITES - * @data_in_dma: physical address for READS - * @data_in_sz: data xfer size for READS */ static void _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid, @@ -1990,8 +2435,8 @@ _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid, } /** - * base_make_prp_nvme - - * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only + * base_make_prp_nvme - Prepare PRPs (Physical Region Page) - + * SGLs specific to NVMe drives only * * @ioc: per adapter object * @scmd: SCSI command from the mid-layer @@ -2137,6 +2582,11 @@ base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc, bool build_prp = true; data_length = scsi_bufflen(scmd); + if (pcie_device && + (mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info))) { + build_prp = false; + return build_prp; + } /* If Datalenth is <= 16K and number of SGE’s entries are <= 2 * we built IEEE SGL @@ -2171,16 +2621,12 @@ _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc, /* Get the SG list pointer and info. */ sges_left = scsi_dma_map(scmd); - if (sges_left < 0) { - sdev_printk(KERN_ERR, scmd->device, - "scsi_dma_map failed: request for %d bytes!\n", - scsi_bufflen(scmd)); + if (sges_left < 0) return 1; - } /* Check if we need to build a native SG list. */ - if (base_is_prp_possible(ioc, pcie_device, - scmd, sges_left) == 0) { + if (!base_is_prp_possible(ioc, pcie_device, + scmd, sges_left)) { /* We built a native SG list, just return. */ goto out; } @@ -2236,6 +2682,22 @@ _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr) _base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1); } +static inline int _base_scsi_dma_map(struct scsi_cmnd *cmd) +{ + /* + * Some firmware versions byte-swap the REPORT ZONES command reply from + * ATA-ZAC devices by directly accessing in the host buffer. This does + * not respect the default command DMA direction and causes IOMMU page + * faults on some architectures with an IOMMU enforcing write mappings + * (e.g. AMD hosts). Avoid such issue by making the report zones buffer + * mapping bi-directional. + */ + if (cmd->cmnd[0] == ZBC_IN && cmd->cmnd[1] == ZI_REPORT_ZONES) + cmd->sc_data_direction = DMA_BIDIRECTIONAL; + + return scsi_dma_map(cmd); +} + /** * _base_build_sg_scmd - main sg creation routine * pcie_device is unused here! @@ -2282,13 +2744,9 @@ _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc, sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT; sg_scmd = scsi_sglist(scmd); - sges_left = scsi_dma_map(scmd); - if (sges_left < 0) { - sdev_printk(KERN_ERR, scmd->device, - "scsi_dma_map failed: request for %d bytes!\n", - scsi_bufflen(scmd)); + sges_left = _base_scsi_dma_map(scmd); + if (sges_left < 0) return -ENOMEM; - } sg_local = &mpi_request->SGL; sges_in_segment = ioc->max_sges_in_main_message; @@ -2430,13 +2888,9 @@ _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc, } sg_scmd = scsi_sglist(scmd); - sges_left = scsi_dma_map(scmd); - if (sges_left < 0) { - sdev_printk(KERN_ERR, scmd->device, - "scsi_dma_map failed: request for %d bytes!\n", - scsi_bufflen(scmd)); + sges_left = _base_scsi_dma_map(scmd); + if (sges_left < 0) return -ENOMEM; - } sg_local = &mpi_request->SGL; sges_in_segment = (ioc->request_sz - @@ -2578,56 +3032,43 @@ _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge, static int _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev) { - u64 required_mask, coherent_mask; struct sysinfo s; + u64 coherent_dma_mask, dma_mask; + + if (ioc->is_mcpu_endpoint || sizeof(dma_addr_t) == 4) { + ioc->dma_mask = 32; + coherent_dma_mask = dma_mask = DMA_BIT_MASK(32); + /* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */ + } else if (ioc->hba_mpi_version_belonged > MPI2_VERSION) { + ioc->dma_mask = 63; + coherent_dma_mask = dma_mask = DMA_BIT_MASK(63); + } else { + ioc->dma_mask = 64; + coherent_dma_mask = dma_mask = DMA_BIT_MASK(64); + } - if (ioc->is_mcpu_endpoint) - goto try_32bit; - - required_mask = dma_get_required_mask(&pdev->dev); - if (sizeof(dma_addr_t) == 4 || required_mask == 32) - goto try_32bit; - - if (ioc->dma_mask) - coherent_mask = DMA_BIT_MASK(64); - else - coherent_mask = DMA_BIT_MASK(32); - - if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) || - dma_set_coherent_mask(&pdev->dev, coherent_mask)) - goto try_32bit; - - ioc->base_add_sg_single = &_base_add_sg_single_64; - ioc->sge_size = sizeof(Mpi2SGESimple64_t); - ioc->dma_mask = 64; - goto out; + if (ioc->use_32bit_dma) + coherent_dma_mask = DMA_BIT_MASK(32); - try_32bit: - if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) + if (dma_set_mask(&pdev->dev, dma_mask) || + dma_set_coherent_mask(&pdev->dev, coherent_dma_mask)) return -ENODEV; - ioc->base_add_sg_single = &_base_add_sg_single_32; - ioc->sge_size = sizeof(Mpi2SGESimple32_t); - ioc->dma_mask = 32; - out: + if (ioc->dma_mask > 32) { + ioc->base_add_sg_single = &_base_add_sg_single_64; + ioc->sge_size = sizeof(Mpi2SGESimple64_t); + } else { + ioc->base_add_sg_single = &_base_add_sg_single_32; + ioc->sge_size = sizeof(Mpi2SGESimple32_t); + } + si_meminfo(&s); ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n", - ioc->dma_mask, convert_to_kb(s.totalram)); + ioc->dma_mask, convert_to_kb(s.totalram)); return 0; } -static int -_base_change_consistent_dma_mask(struct MPT3SAS_ADAPTER *ioc, - struct pci_dev *pdev) -{ - if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64))) { - if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32))) - return -ENODEV; - } - return 0; -} - /** * _base_check_enable_msix - checks MSIX capabable. * @ioc: per adapter object @@ -2675,14 +3116,15 @@ _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc) } /** - * _base_free_irq - free irq + * mpt3sas_base_free_irq - free irq * @ioc: per adapter object * * Freeing respective reply_queue from the list. */ -static void -_base_free_irq(struct MPT3SAS_ADAPTER *ioc) +void +mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc) { + unsigned int irq; struct adapter_reply_queue *reply_q, *next; if (list_empty(&ioc->reply_queue_list)) @@ -2690,6 +3132,15 @@ _base_free_irq(struct MPT3SAS_ADAPTER *ioc) list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) { list_del(&reply_q->list); + if (reply_q->is_iouring_poll_q) { + kfree(reply_q); + continue; + } + + if (ioc->smp_affinity_enable) { + irq = pci_irq_vector(ioc->pdev, reply_q->msix_index); + irq_update_affinity_hint(irq, NULL); + } free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index), reply_q); kfree(reply_q); @@ -2708,7 +3159,7 @@ _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index) { struct pci_dev *pdev = ioc->pdev; struct adapter_reply_queue *reply_q; - int r; + int r, qid; reply_q = kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL); if (!reply_q) { @@ -2720,6 +3171,17 @@ _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index) reply_q->msix_index = index; atomic_set(&reply_q->busy, 0); + + if (index >= ioc->iopoll_q_start_index) { + qid = index - ioc->iopoll_q_start_index; + snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-mq-poll%d", + ioc->driver_name, ioc->id, qid); + reply_q->is_iouring_poll_q = 1; + ioc->io_uring_poll_queues[qid].reply_q = reply_q; + goto out; + } + + if (ioc->msix_enable) snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d", ioc->driver_name, ioc->id, index); @@ -2734,7 +3196,7 @@ _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index) kfree(reply_q); return -EBUSY; } - +out: INIT_LIST_HEAD(&reply_q->list); list_add_tail(&reply_q->list, &ioc->reply_queue_list); return 0; @@ -2745,19 +3207,22 @@ _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index) * @ioc: per adapter object * * The enduser would need to set the affinity via /proc/irq/#/smp_affinity - * - * It would nice if we could call irq_set_affinity, however it is not - * an exported symbol */ static void _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) { - unsigned int cpu, nr_cpus, nr_msix, index = 0; + unsigned int cpu, nr_cpus, nr_msix, index = 0, irq; struct adapter_reply_queue *reply_q; + int iopoll_q_count = ioc->reply_queue_count - + ioc->iopoll_q_start_index; + const struct cpumask *mask; if (!_base_is_controller_msix_enabled(ioc)) return; + if (ioc->msix_load_balance) + return; + memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz); nr_cpus = num_online_cpus(); @@ -2766,14 +3231,34 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) if (!nr_msix) return; - if (smp_affinity_enable) { + if (ioc->smp_affinity_enable) { + + /* + * set irq affinity to local numa node for those irqs + * corresponding to high iops queues. + */ + if (ioc->high_iops_queues) { + mask = cpumask_of_node(dev_to_node(&ioc->pdev->dev)); + for (index = 0; index < ioc->high_iops_queues; + index++) { + irq = pci_irq_vector(ioc->pdev, index); + irq_set_affinity_and_hint(irq, mask); + } + } + list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { - const cpumask_t *mask = pci_irq_get_affinity(ioc->pdev, - reply_q->msix_index); + const cpumask_t *mask; + + if (reply_q->msix_index < ioc->high_iops_queues || + reply_q->msix_index >= ioc->iopoll_q_start_index) + continue; + + mask = pci_irq_get_affinity(ioc->pdev, + reply_q->msix_index); if (!mask) { ioc_warn(ioc, "no affinity for msi %x\n", reply_q->msix_index); - continue; + goto fall_back; } for_each_cpu_and(cpu, mask, cpu_online_mask) { @@ -2784,12 +3269,19 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) } return; } + +fall_back: cpu = cpumask_first(cpu_online_mask); + nr_msix -= (ioc->high_iops_queues - iopoll_q_count); + index = 0; list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { - unsigned int i, group = nr_cpus / nr_msix; + if (reply_q->msix_index < ioc->high_iops_queues || + reply_q->msix_index >= ioc->iopoll_q_start_index) + continue; + if (cpu >= nr_cpus) break; @@ -2805,17 +3297,101 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) } /** - * _base_disable_msix - disables msix + * _base_check_and_enable_high_iops_queues - enable high iops mode * @ioc: per adapter object + * @hba_msix_vector_count: msix vectors supported by HBA + * + * Enable high iops queues only if + * - HBA is a SEA/AERO controller and + * - MSI-Xs vector supported by the HBA is 128 and + * - total CPU count in the system >=16 and + * - loaded driver with default max_msix_vectors module parameter and + * - system booted in non kdump mode * + * Return: nothing. */ static void -_base_disable_msix(struct MPT3SAS_ADAPTER *ioc) +_base_check_and_enable_high_iops_queues(struct MPT3SAS_ADAPTER *ioc, + int hba_msix_vector_count) +{ + u16 lnksta, speed; + + /* + * Disable high iops queues if io uring poll queues are enabled. + */ + if (perf_mode == MPT_PERF_MODE_IOPS || + perf_mode == MPT_PERF_MODE_LATENCY || + ioc->io_uring_poll_queues) { + ioc->high_iops_queues = 0; + return; + } + + if (perf_mode == MPT_PERF_MODE_DEFAULT) { + + pcie_capability_read_word(ioc->pdev, PCI_EXP_LNKSTA, &lnksta); + speed = lnksta & PCI_EXP_LNKSTA_CLS; + + if (speed < 0x4) { + ioc->high_iops_queues = 0; + return; + } + } + + if (!reset_devices && ioc->is_aero_ioc && + hba_msix_vector_count == MPT3SAS_GEN35_MAX_MSIX_QUEUES && + num_online_cpus() >= MPT3SAS_HIGH_IOPS_REPLY_QUEUES && + max_msix_vectors == -1) + ioc->high_iops_queues = MPT3SAS_HIGH_IOPS_REPLY_QUEUES; + else + ioc->high_iops_queues = 0; +} + +/** + * mpt3sas_base_disable_msix - disables msix + * @ioc: per adapter object + * + */ +void +mpt3sas_base_disable_msix(struct MPT3SAS_ADAPTER *ioc) { if (!ioc->msix_enable) return; - pci_disable_msix(ioc->pdev); + pci_free_irq_vectors(ioc->pdev); ioc->msix_enable = 0; + kfree(ioc->io_uring_poll_queues); +} + +/** + * _base_alloc_irq_vectors - allocate msix vectors + * @ioc: per adapter object + * + */ +static int +_base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc) +{ + int i, irq_flags = PCI_IRQ_MSIX; + struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues }; + struct irq_affinity *descp = &desc; + /* + * Don't allocate msix vectors for poll_queues. + * msix_vectors is always within a range of FW supported reply queue. + */ + int nr_msix_vectors = ioc->iopoll_q_start_index; + + + if (ioc->smp_affinity_enable) + irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES; + else + descp = NULL; + + ioc_info(ioc, " %d %d %d\n", ioc->high_iops_queues, + ioc->reply_queue_count, nr_msix_vectors); + + i = pci_alloc_irq_vectors_affinity(ioc->pdev, + ioc->high_iops_queues, + nr_msix_vectors, irq_flags, descp); + + return i; } /** @@ -2829,7 +3405,9 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc) int r; int i, local_max_msix_vectors; u8 try_msix = 0; - unsigned int irq_flags = PCI_IRQ_MSIX; + int iopoll_q_count = 0; + + ioc->msix_load_balance = false; if (msix_disable == -1 || msix_disable == 0) try_msix = 1; @@ -2840,56 +3418,131 @@ _base_enable_msix(struct MPT3SAS_ADAPTER *ioc) if (_base_check_enable_msix(ioc) != 0) goto try_ioapic; - ioc->reply_queue_count = min_t(int, ioc->cpu_count, - ioc->msix_vector_count); + ioc_info(ioc, "MSI-X vectors supported: %d\n", ioc->msix_vector_count); + pr_info("\t no of cores: %d, max_msix_vectors: %d\n", + ioc->cpu_count, max_msix_vectors); - ioc_info(ioc, "MSI-X vectors supported: %d, no of cores: %d, max_msix_vectors: %d\n", - ioc->msix_vector_count, ioc->cpu_count, max_msix_vectors); + ioc->reply_queue_count = + min_t(int, ioc->cpu_count, ioc->msix_vector_count); if (!ioc->rdpq_array_enable && max_msix_vectors == -1) local_max_msix_vectors = (reset_devices) ? 1 : 8; else local_max_msix_vectors = max_msix_vectors; - if (local_max_msix_vectors > 0) - ioc->reply_queue_count = min_t(int, local_max_msix_vectors, - ioc->reply_queue_count); - else if (local_max_msix_vectors == 0) + if (local_max_msix_vectors == 0) goto try_ioapic; - if (ioc->msix_vector_count < ioc->cpu_count) - smp_affinity_enable = 0; + /* + * Enable msix_load_balance only if combined reply queue mode is + * disabled on SAS3 & above generation HBA devices. + */ + if (!ioc->combined_reply_queue && + ioc->hba_mpi_version_belonged != MPI2_VERSION) { + ioc_info(ioc, + "combined ReplyQueue is off, Enabling msix load balance\n"); + ioc->msix_load_balance = true; + } + + /* + * smp affinity setting is not need when msix load balance + * is enabled. + */ + if (ioc->msix_load_balance) + ioc->smp_affinity_enable = 0; + + if (!ioc->smp_affinity_enable || ioc->reply_queue_count <= 1) + ioc->shost->host_tagset = 0; + + /* + * Enable io uring poll queues only if host_tagset is enabled. + */ + if (ioc->shost->host_tagset) + iopoll_q_count = poll_queues; + + if (iopoll_q_count) { + ioc->io_uring_poll_queues = kcalloc(iopoll_q_count, + sizeof(struct io_uring_poll_queue), GFP_KERNEL); + if (!ioc->io_uring_poll_queues) + iopoll_q_count = 0; + } + + if (ioc->is_aero_ioc) + _base_check_and_enable_high_iops_queues(ioc, + ioc->msix_vector_count); + + /* + * Add high iops queues count to reply queue count if high iops queues + * are enabled. + */ + ioc->reply_queue_count = min_t(int, + ioc->reply_queue_count + ioc->high_iops_queues, + ioc->msix_vector_count); + + /* + * Adjust the reply queue count incase reply queue count + * exceeds the user provided MSIx vectors count. + */ + if (local_max_msix_vectors > 0) + ioc->reply_queue_count = min_t(int, local_max_msix_vectors, + ioc->reply_queue_count); + /* + * Add io uring poll queues count to reply queues count + * if io uring is enabled in driver. + */ + if (iopoll_q_count) { + if (ioc->reply_queue_count < (iopoll_q_count + MPT3_MIN_IRQS)) + iopoll_q_count = 0; + ioc->reply_queue_count = min_t(int, + ioc->reply_queue_count + iopoll_q_count, + ioc->msix_vector_count); + } - if (smp_affinity_enable) - irq_flags |= PCI_IRQ_AFFINITY; + /* + * Starting index of io uring poll queues in reply queue list. + */ + ioc->iopoll_q_start_index = + ioc->reply_queue_count - iopoll_q_count; - r = pci_alloc_irq_vectors(ioc->pdev, 1, ioc->reply_queue_count, - irq_flags); + r = _base_alloc_irq_vectors(ioc); if (r < 0) { - dfailprintk(ioc, - ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", - r)); + ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", r); goto try_ioapic; } + /* + * Adjust the reply queue count if the allocated + * MSIx vectors is less then the requested number + * of MSIx vectors. + */ + if (r < ioc->iopoll_q_start_index) { + ioc->reply_queue_count = r + iopoll_q_count; + ioc->iopoll_q_start_index = + ioc->reply_queue_count - iopoll_q_count; + } + ioc->msix_enable = 1; - ioc->reply_queue_count = r; for (i = 0; i < ioc->reply_queue_count; i++) { r = _base_request_irq(ioc, i); if (r) { - _base_free_irq(ioc); - _base_disable_msix(ioc); + mpt3sas_base_free_irq(ioc); + mpt3sas_base_disable_msix(ioc); goto try_ioapic; } } + ioc_info(ioc, "High IOPs queues : %s\n", + ioc->high_iops_queues ? "enabled" : "disabled"); + return 0; /* failback to io_apic interrupt routing */ try_ioapic: - + ioc->high_iops_queues = 0; + ioc_info(ioc, "High IOPs queues : disabled\n"); ioc->reply_queue_count = 1; - r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY); + ioc->iopoll_q_start_index = ioc->reply_queue_count - 0; + r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_INTX); if (r < 0) { dfailprintk(ioc, ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n", @@ -2911,8 +3564,8 @@ mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc) dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); - _base_free_irq(ioc); - _base_disable_msix(ioc); + mpt3sas_base_free_irq(ioc); + mpt3sas_base_disable_msix(ioc); kfree(ioc->replyPostRegisterIndex); ioc->replyPostRegisterIndex = NULL; @@ -2925,11 +3578,49 @@ mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc) if (pci_is_enabled(pdev)) { pci_release_selected_regions(ioc->pdev, ioc->bars); - pci_disable_pcie_error_reporting(pdev); pci_disable_device(pdev); } } +static int +_base_diag_reset(struct MPT3SAS_ADAPTER *ioc); + +/** + * mpt3sas_base_check_for_fault_and_issue_reset - check if IOC is in fault state + * and if it is in fault state then issue diag reset. + * @ioc: per adapter object + * + * Return: 0 for success, non-zero for failure. + */ +int +mpt3sas_base_check_for_fault_and_issue_reset(struct MPT3SAS_ADAPTER *ioc) +{ + u32 ioc_state; + int rc = -EFAULT; + + dinitprintk(ioc, pr_info("%s\n", __func__)); + if (ioc->pci_error_recovery) + return 0; + ioc_state = mpt3sas_base_get_iocstate(ioc, 0); + dhsprintk(ioc, pr_info("%s: ioc_state(0x%08x)\n", __func__, ioc_state)); + + if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { + mpt3sas_print_fault_code(ioc, ioc_state & + MPI2_DOORBELL_DATA_MASK); + mpt3sas_base_mask_interrupts(ioc); + rc = _base_diag_reset(ioc); + } else if ((ioc_state & MPI2_IOC_STATE_MASK) == + MPI2_IOC_STATE_COREDUMP) { + mpt3sas_print_coredump_info(ioc, ioc_state & + MPI2_DOORBELL_DATA_MASK); + mpt3sas_base_wait_for_coredump_completion(ioc, __func__); + mpt3sas_base_mask_interrupts(ioc); + rc = _base_diag_reset(ioc); + } + + return rc; +} + /** * mpt3sas_base_map_resources - map in controller resources (io/irq/memap) * @ioc: per adapter object @@ -2942,10 +3633,11 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) struct pci_dev *pdev = ioc->pdev; u32 memap_sz; u32 pio_sz; - int i, r = 0; + int i, r = 0, rc; u64 pio_chip = 0; phys_addr_t chip_phys = 0; struct adapter_reply_queue *reply_q; + int iopoll_q_count = 0; dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); @@ -2965,9 +3657,6 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) goto out_fail; } -/* AER (Advanced Error Reporting) hooks */ - pci_enable_pcie_error_reporting(pdev); - pci_set_master(pdev); @@ -2995,16 +3684,20 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) } if (ioc->chip == NULL) { - ioc_err(ioc, "unable to map adapter memory! or resource not found\n"); + ioc_err(ioc, + "unable to map adapter memory! or resource not found\n"); r = -EINVAL; goto out_fail; } - _base_mask_interrupts(ioc); + mpt3sas_base_mask_interrupts(ioc); r = _base_get_ioc_facts(ioc); - if (r) - goto out_fail; + if (r) { + rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); + if (rc || (_base_get_ioc_facts(ioc))) + goto out_fail; + } if (!ioc->rdpq_array_enable_assigned) { ioc->rdpq_array_enable = ioc->rdpq_array_capable; @@ -3015,6 +3708,14 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) if (r) goto out_fail; + iopoll_q_count = ioc->reply_queue_count - ioc->iopoll_q_start_index; + for (i = 0; i < iopoll_q_count; i++) { + atomic_set(&ioc->io_uring_poll_queues[i].busy, 0); + atomic_set(&ioc->io_uring_poll_queues[i].pause, 0); + } + + if (!ioc->is_driver_loading) + _base_init_irqpolls(ioc); /* Use the Combined reply queue feature only for SAS3 C0 & higher * revision HBAs and also only when reply queue count is greater than 8 */ @@ -3029,17 +3730,18 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) ioc->combined_reply_index_count, sizeof(resource_size_t *), GFP_KERNEL); if (!ioc->replyPostRegisterIndex) { - dfailprintk(ioc, - ioc_warn(ioc, "allocation for reply Post Register Index failed!!!\n")); + ioc_err(ioc, + "allocation for replyPostRegisterIndex failed!\n"); r = -ENOMEM; goto out_fail; } for (i = 0; i < ioc->combined_reply_index_count; i++) { - ioc->replyPostRegisterIndex[i] = (resource_size_t *) - ((u8 __force *)&ioc->chip->Doorbell + - MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET + - (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET)); + ioc->replyPostRegisterIndex[i] = + (resource_size_t __iomem *) + ((u8 __force *)&ioc->chip->Doorbell + + MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET + + (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET)); } } @@ -3054,11 +3756,18 @@ mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc) * 4))); } - list_for_each_entry(reply_q, &ioc->reply_queue_list, list) + list_for_each_entry(reply_q, &ioc->reply_queue_list, list) { + if (reply_q->msix_index >= ioc->iopoll_q_start_index) { + pr_info("%s: enabled: index: %d\n", + reply_q->name, reply_q->msix_index); + continue; + } + pr_info("%s: %s enabled: IRQ %d\n", reply_q->name, ioc->msix_enable ? "PCI-MSI-X" : "IO-APIC", pci_irq_vector(ioc->pdev, reply_q->msix_index)); + } ioc_info(ioc, "iomem(%pap), mapped(0x%p), size(%d)\n", &chip_phys, ioc->chip, memap_sz); @@ -3155,13 +3864,65 @@ mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr) return ioc->reply + (phys_addr - (u32)ioc->reply_dma); } +/** + * _base_get_msix_index - get the msix index + * @ioc: per adapter object + * @scmd: scsi_cmnd object + * + * Return: msix index of general reply queues, + * i.e. reply queue on which IO request's reply + * should be posted by the HBA firmware. + */ static inline u8 -_base_get_msix_index(struct MPT3SAS_ADAPTER *ioc) +_base_get_msix_index(struct MPT3SAS_ADAPTER *ioc, + struct scsi_cmnd *scmd) { + /* Enables reply_queue load balancing */ + if (ioc->msix_load_balance) + return ioc->reply_queue_count ? + base_mod64(atomic64_add_return(1, + &ioc->total_io_cnt), ioc->reply_queue_count) : 0; + + if (scmd && ioc->shost->nr_hw_queues > 1) { + u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd)); + + return blk_mq_unique_tag_to_hwq(tag) + + ioc->high_iops_queues; + } + return ioc->cpu_msix_table[raw_smp_processor_id()]; } /** + * _base_get_high_iops_msix_index - get the msix index of + * high iops queues + * @ioc: per adapter object + * @scmd: scsi_cmnd object + * + * Return: msix index of high iops reply queues. + * i.e. high iops reply queue on which IO request's + * reply should be posted by the HBA firmware. + */ +static inline u8 +_base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc, + struct scsi_cmnd *scmd) +{ + /** + * Round robin the IO interrupts among the high iops + * reply queues in terms of batch count 16 when outstanding + * IOs on the target device is >=8. + */ + + if (scsi_device_busy(scmd->device) > MPT3SAS_DEVICE_HIGH_IOPS_DEPTH) + return base_mod64(( + atomic64_add_return(1, &ioc->high_iops_outstanding) / + MPT3SAS_HIGH_IOPS_BATCH_COUNT), + MPT3SAS_HIGH_IOPS_REPLY_QUEUES); + + return _base_get_msix_index(ioc, scmd); +} + +/** * mpt3sas_base_get_smid - obtain a free smid from internal queue * @ioc: per adapter object * @cb_idx: callback index @@ -3204,13 +3965,28 @@ mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx, struct scsi_cmnd *scmd) { struct scsiio_tracker *request = scsi_cmd_priv(scmd); - unsigned int tag = scmd->request->tag; u16 smid; + u32 tag, unique_tag; + + unique_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd)); + tag = blk_mq_unique_tag_to_tag(unique_tag); + + /* + * Store hw queue number corresponding to the tag. + * This hw queue number is used later to determine + * the unique_tag using the logic below. This unique_tag + * is used to retrieve the scmd pointer corresponding + * to tag using scsi_host_find_tag() API. + * + * tag = smid - 1; + * unique_tag = ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag; + */ + ioc->io_queue_num[tag] = blk_mq_unique_tag_to_hwq(unique_tag); smid = tag + 1; request->cb_idx = cb_idx; - request->msix_io = _base_get_msix_index(ioc); request->smid = smid; + request->scmd = scmd; INIT_LIST_HEAD(&request->chain_list); return smid; } @@ -3264,6 +4040,7 @@ void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc, return; st->cb_idx = 0xFF; st->direct_io = 0; + st->scmd = NULL; atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0); st->smid = 0; } @@ -3281,14 +4058,21 @@ mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid) if (smid < ioc->hi_priority_smid) { struct scsiio_tracker *st; + void *request; st = _get_st_from_smid(ioc, smid); if (!st) { _base_recovery_check(ioc); return; } + + /* Clear MPI request frame */ + request = mpt3sas_base_get_msg_frame(ioc, smid); + memset(request, 0, ioc->request_sz); + mpt3sas_base_clear_st(ioc, st); _base_recovery_check(ioc); + ioc->io_queue_num[smid - 1] = 0; return; } @@ -3327,7 +4111,6 @@ _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr, spin_lock_irqsave(writeq_lock, flags); __raw_writel((u32)(b), addr); __raw_writel((u32)(b >> 32), (addr + 4)); - mmiowb(); spin_unlock_irqrestore(writeq_lock, flags); } @@ -3358,13 +4141,37 @@ _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock) #endif /** + * _base_set_and_get_msix_index - get the msix index and assign to msix_io + * variable of scsi tracker + * @ioc: per adapter object + * @smid: system request message index + * + * Return: msix index. + */ +static u8 +_base_set_and_get_msix_index(struct MPT3SAS_ADAPTER *ioc, u16 smid) +{ + struct scsiio_tracker *st = NULL; + + if (smid < ioc->hi_priority_smid) + st = _get_st_from_smid(ioc, smid); + + if (st == NULL) + return _base_get_msix_index(ioc, NULL); + + st->msix_io = ioc->get_msix_index_for_smlio(ioc, st->scmd); + return st->msix_io; +} + +/** * _base_put_smid_mpi_ep_scsi_io - send SCSI_IO request to firmware * @ioc: per adapter object * @smid: system request message index * @handle: device handle */ static void -_base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) +_base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, + u16 smid, u16 handle) { Mpi2RequestDescriptorUnion_t descriptor; u64 *request = (u64 *)&descriptor; @@ -3377,7 +4184,7 @@ _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) _base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp, ioc->request_sz); descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; - descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc); + descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.SCSIIO.SMID = cpu_to_le16(smid); descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); descriptor.SCSIIO.LMID = 0; @@ -3399,7 +4206,7 @@ _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; - descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc); + descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.SCSIIO.SMID = cpu_to_le16(smid); descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); descriptor.SCSIIO.LMID = 0; @@ -3408,13 +4215,13 @@ _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) } /** - * mpt3sas_base_put_smid_fast_path - send fast path request to firmware + * _base_put_smid_fast_path - send fast path request to firmware * @ioc: per adapter object * @smid: system request message index * @handle: device handle */ -void -mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, +static void +_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle) { Mpi2RequestDescriptorUnion_t descriptor; @@ -3422,7 +4229,7 @@ mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, descriptor.SCSIIO.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; - descriptor.SCSIIO.MSIxIndex = _base_get_msix_index(ioc); + descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.SCSIIO.SMID = cpu_to_le16(smid); descriptor.SCSIIO.DevHandle = cpu_to_le16(handle); descriptor.SCSIIO.LMID = 0; @@ -3431,13 +4238,13 @@ mpt3sas_base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid, } /** - * mpt3sas_base_put_smid_hi_priority - send Task Management request to firmware + * _base_put_smid_hi_priority - send Task Management request to firmware * @ioc: per adapter object * @smid: system request message index - * @msix_task: msix_task will be same as msix of IO incase of task abort else 0. + * @msix_task: msix_task will be same as msix of IO in case of task abort else 0 */ -void -mpt3sas_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid, +static void +_base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 msix_task) { Mpi2RequestDescriptorUnion_t descriptor; @@ -3486,7 +4293,7 @@ mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid) descriptor.Default.RequestFlags = MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED; - descriptor.Default.MSIxIndex = _base_get_msix_index(ioc); + descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.Default.SMID = cpu_to_le16(smid); descriptor.Default.LMID = 0; descriptor.Default.DescriptorTypeDependent = 0; @@ -3495,12 +4302,12 @@ mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid) } /** - * mpt3sas_base_put_smid_default - Default, primarily used for config pages + * _base_put_smid_default - Default, primarily used for config pages * @ioc: per adapter object * @smid: system request message index */ -void -mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) +static void +_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) { Mpi2RequestDescriptorUnion_t descriptor; void *mpi_req_iomem; @@ -3518,7 +4325,7 @@ mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) } request = (u64 *)&descriptor; descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; - descriptor.Default.MSIxIndex = _base_get_msix_index(ioc); + descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); descriptor.Default.SMID = cpu_to_le16(smid); descriptor.Default.LMID = 0; descriptor.Default.DescriptorTypeDependent = 0; @@ -3532,6 +4339,95 @@ mpt3sas_base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid) } /** + * _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using + * Atomic Request Descriptor + * @ioc: per adapter object + * @smid: system request message index + * @handle: device handle, unused in this function, for function type match + * + * Return: nothing. + */ +static void +_base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, + u16 handle) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO; + descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** + * _base_put_smid_fast_path_atomic - send fast path request to firmware + * using Atomic Request Descriptor + * @ioc: per adapter object + * @smid: system request message index + * @handle: device handle, unused in this function, for function type match + * Return: nothing + */ +static void +_base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, + u16 handle) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO; + descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** + * _base_put_smid_hi_priority_atomic - send Task Management request to + * firmware using Atomic Request Descriptor + * @ioc: per adapter object + * @smid: system request message index + * @msix_task: msix_task will be same as msix of IO in case of task abort else 0 + * + * Return: nothing. + */ +static void +_base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid, + u16 msix_task) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY; + descriptor.MSIxIndex = msix_task; + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** + * _base_put_smid_default_atomic - Default, primarily used for config pages + * use Atomic Request Descriptor + * @ioc: per adapter object + * @smid: system request message index + * + * Return: nothing. + */ +static void +_base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid) +{ + Mpi26AtomicRequestDescriptor_t descriptor; + u32 *request = (u32 *)&descriptor; + + descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; + descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid); + descriptor.SMID = cpu_to_le16(smid); + + writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost); +} + +/** * _base_display_OEMs_branding - Display branding string * @ioc: per adapter object */ @@ -3563,6 +4459,7 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; case MPI2_MFGPAGE_DEVID_SAS2308_2: switch (ioc->pdev->subsystem_device) { case MPT2SAS_INTEL_RS25GB008_SSDID: @@ -3598,6 +4495,7 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; case MPI25_MFGPAGE_DEVID_SAS3008: switch (ioc->pdev->subsystem_device) { case MPT3SAS_INTEL_RMS3JC080_SSDID: @@ -3742,6 +4640,7 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; case MPI2_MFGPAGE_DEVID_SAS2308_2: switch (ioc->pdev->subsystem_device) { case MPT2SAS_HP_2_4_INTERNAL_SSDID: @@ -3765,11 +4664,13 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) ioc->pdev->subsystem_device); break; } + break; default: ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n", ioc->pdev->subsystem_device); break; } + break; default: break; } @@ -3785,10 +4686,12 @@ _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc) static int _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc) { - Mpi2FWImageHeader_t *FWImgHdr; + Mpi2FWImageHeader_t *fw_img_hdr; + Mpi26ComponentImageHeader_t *cmp_img_hdr; Mpi25FWUploadRequest_t *mpi_request; Mpi2FWUploadReply_t mpi_reply; - int r = 0; + int r = 0, issue_diag_reset = 0; + u32 package_version = 0; void *fwpkg_data = NULL; dma_addr_t fwpkg_data_dma; u16 smid, ioc_status; @@ -3805,7 +4708,8 @@ _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc) fwpkg_data = dma_alloc_coherent(&ioc->pdev->dev, data_length, &fwpkg_data_dma, GFP_KERNEL); if (!fwpkg_data) { - ioc_err(ioc, "failure at %s:%d/%s()!\n", + ioc_err(ioc, + "Memory allocation for fwpkg data failed at %s:%d/%s()!\n", __FILE__, __LINE__, __func__); return -ENOMEM; } @@ -3827,7 +4731,7 @@ _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc) ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma, data_length); init_completion(&ioc->base_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); /* Wait for 15 seconds */ wait_for_completion_timeout(&ioc->base_cmds.done, FW_IMG_HDR_READ_TIMEOUT*HZ); @@ -3836,7 +4740,7 @@ _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc) ioc_err(ioc, "%s: timeout\n", __func__); _debug_dump_mf(mpi_request, sizeof(Mpi25FWUploadRequest_t)/4); - r = -ETIME; + issue_diag_reset = 1; } else { memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t)); if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) { @@ -3845,14 +4749,26 @@ _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc) ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK; if (ioc_status == MPI2_IOCSTATUS_SUCCESS) { - FWImgHdr = (Mpi2FWImageHeader_t *)fwpkg_data; - if (FWImgHdr->PackageVersion.Word) { - ioc_info(ioc, "FW Package Version (%02d.%02d.%02d.%02d)\n", - FWImgHdr->PackageVersion.Struct.Major, - FWImgHdr->PackageVersion.Struct.Minor, - FWImgHdr->PackageVersion.Struct.Unit, - FWImgHdr->PackageVersion.Struct.Dev); - } + fw_img_hdr = (Mpi2FWImageHeader_t *)fwpkg_data; + if (le32_to_cpu(fw_img_hdr->Signature) == + MPI26_IMAGE_HEADER_SIGNATURE0_MPI26) { + cmp_img_hdr = + (Mpi26ComponentImageHeader_t *) + (fwpkg_data); + package_version = + le32_to_cpu( + cmp_img_hdr->ApplicationSpecific); + } else + package_version = + le32_to_cpu( + fw_img_hdr->PackageVersion.Word); + if (package_version) + ioc_info(ioc, + "FW Package Ver(%02d.%02d.%02d.%02d)\n", + ((package_version) & 0xFF000000) >> 24, + ((package_version) & 0x00FF0000) >> 16, + ((package_version) & 0x0000FF00) >> 8, + (package_version) & 0x000000FF); } else { _debug_dump_mf(&mpi_reply, sizeof(Mpi2FWUploadReply_t)/4); @@ -3864,34 +4780,35 @@ out: if (fwpkg_data) dma_free_coherent(&ioc->pdev->dev, data_length, fwpkg_data, fwpkg_data_dma); + if (issue_diag_reset) { + if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED) + return -EFAULT; + if (mpt3sas_base_check_for_fault_and_issue_reset(ioc)) + return -EFAULT; + r = -EAGAIN; + } return r; } /** - * _base_display_ioc_capabilities - Disply IOC's capabilities. + * _base_display_ioc_capabilities - Display IOC's capabilities. * @ioc: per adapter object */ static void _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc) { int i = 0; - char desc[16]; + char desc[17] = {0}; u32 iounit_pg1_flags; - u32 bios_version; - bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion); - strncpy(desc, ioc->manu_pg0.ChipName, 16); - ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n", + memtostr(desc, ioc->manu_pg0.ChipName); + ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x)\n", desc, (ioc->facts.FWVersion.Word & 0xFF000000) >> 24, (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16, (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8, ioc->facts.FWVersion.Word & 0x000000FF, - ioc->pdev->revision, - (bios_version & 0xFF000000) >> 24, - (bios_version & 0x00FF0000) >> 16, - (bios_version & 0x0000FF00) >> 8, - bios_version & 0x000000FF); + ioc->pdev->revision); _base_display_OEMs_branding(ioc); @@ -3968,6 +4885,12 @@ _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc) i++; } + if (ioc->facts.IOCCapabilities & + MPI26_IOCFACTS_CAPABILITY_MCTP_PASSTHRU) { + pr_cont("%sMCTP Passthru", i ? "," : ""); + i++; + } + iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags); if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) { pr_cont("%sNCQ", i ? "," : ""); @@ -4003,8 +4926,7 @@ mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc, if (!num_phys) return; - sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys * - sizeof(Mpi2SasIOUnit1PhyData_t)); + sz = struct_size(sas_iounit_pg1, PhyData, num_phys); sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL); if (!sas_iounit_pg1) { ioc_err(ioc, "failure at %s:%d/%s()!\n", @@ -4067,31 +4989,664 @@ out: } /** + * _base_update_ioc_page1_inlinewith_perf_mode - Update IOC Page1 fields + * according to performance mode. + * @ioc : per adapter object + * + * Return: zero on success; otherwise return EAGAIN error code asking the + * caller to retry. + */ +static int +_base_update_ioc_page1_inlinewith_perf_mode(struct MPT3SAS_ADAPTER *ioc) +{ + Mpi2IOCPage1_t ioc_pg1; + Mpi2ConfigReply_t mpi_reply; + int rc; + + rc = mpt3sas_config_get_ioc_pg1(ioc, &mpi_reply, &ioc->ioc_pg1_copy); + if (rc) + return rc; + memcpy(&ioc_pg1, &ioc->ioc_pg1_copy, sizeof(Mpi2IOCPage1_t)); + + switch (perf_mode) { + case MPT_PERF_MODE_DEFAULT: + case MPT_PERF_MODE_BALANCED: + if (ioc->high_iops_queues) { + ioc_info(ioc, + "Enable interrupt coalescing only for first\t" + "%d reply queues\n", + MPT3SAS_HIGH_IOPS_REPLY_QUEUES); + /* + * If 31st bit is zero then interrupt coalescing is + * enabled for all reply descriptor post queues. + * If 31st bit is set to one then user can + * enable/disable interrupt coalescing on per reply + * descriptor post queue group(8) basis. So to enable + * interrupt coalescing only on first reply descriptor + * post queue group 31st bit and zero th bit is enabled. + */ + ioc_pg1.ProductSpecific = cpu_to_le32(0x80000000 | + ((1 << MPT3SAS_HIGH_IOPS_REPLY_QUEUES/8) - 1)); + rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); + if (rc) + return rc; + ioc_info(ioc, "performance mode: balanced\n"); + return 0; + } + fallthrough; + case MPT_PERF_MODE_LATENCY: + /* + * Enable interrupt coalescing on all reply queues + * with timeout value 0xA + */ + ioc_pg1.CoalescingTimeout = cpu_to_le32(0xa); + ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING); + ioc_pg1.ProductSpecific = 0; + rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); + if (rc) + return rc; + ioc_info(ioc, "performance mode: latency\n"); + break; + case MPT_PERF_MODE_IOPS: + /* + * Enable interrupt coalescing on all reply queues. + */ + ioc_info(ioc, + "performance mode: iops with coalescing timeout: 0x%x\n", + le32_to_cpu(ioc_pg1.CoalescingTimeout)); + ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING); + ioc_pg1.ProductSpecific = 0; + rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1); + if (rc) + return rc; + break; + } + return 0; +} + +/** + * _base_get_event_diag_triggers - get event diag trigger values from + * persistent pages + * @ioc : per adapter object + * + * Return: nothing. + */ +static int +_base_get_event_diag_triggers(struct MPT3SAS_ADAPTER *ioc) +{ + Mpi26DriverTriggerPage2_t trigger_pg2; + struct SL_WH_EVENT_TRIGGER_T *event_tg; + MPI26_DRIVER_MPI_EVENT_TRIGGER_ENTRY *mpi_event_tg; + Mpi2ConfigReply_t mpi_reply; + int r = 0, i = 0; + u16 count = 0; + u16 ioc_status; + + r = mpt3sas_config_get_driver_trigger_pg2(ioc, &mpi_reply, + &trigger_pg2); + if (r) + return r; + + ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & + MPI2_IOCSTATUS_MASK; + if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { + dinitprintk(ioc, + ioc_err(ioc, + "%s: Failed to get trigger pg2, ioc_status(0x%04x)\n", + __func__, ioc_status)); + return 0; + } + + if (le16_to_cpu(trigger_pg2.NumMPIEventTrigger)) { + count = le16_to_cpu(trigger_pg2.NumMPIEventTrigger); + count = min_t(u16, NUM_VALID_ENTRIES, count); + ioc->diag_trigger_event.ValidEntries = count; + + event_tg = &ioc->diag_trigger_event.EventTriggerEntry[0]; + mpi_event_tg = &trigger_pg2.MPIEventTriggers[0]; + for (i = 0; i < count; i++) { + event_tg->EventValue = le16_to_cpu( + mpi_event_tg->MPIEventCode); + event_tg->LogEntryQualifier = le16_to_cpu( + mpi_event_tg->MPIEventCodeSpecific); + event_tg++; + mpi_event_tg++; + } + } + return 0; +} + +/** + * _base_get_scsi_diag_triggers - get scsi diag trigger values from + * persistent pages + * @ioc : per adapter object + * + * Return: 0 on success; otherwise return failure status. + */ +static int +_base_get_scsi_diag_triggers(struct MPT3SAS_ADAPTER *ioc) +{ + Mpi26DriverTriggerPage3_t trigger_pg3; + struct SL_WH_SCSI_TRIGGER_T *scsi_tg; + MPI26_DRIVER_SCSI_SENSE_TRIGGER_ENTRY *mpi_scsi_tg; + Mpi2ConfigReply_t mpi_reply; + int r = 0, i = 0; + u16 count = 0; + u16 ioc_status; + + r = mpt3sas_config_get_driver_trigger_pg3(ioc, &mpi_reply, + &trigger_pg3); + if (r) + return r; + + ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & + MPI2_IOCSTATUS_MASK; + if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { + dinitprintk(ioc, + ioc_err(ioc, + "%s: Failed to get trigger pg3, ioc_status(0x%04x)\n", + __func__, ioc_status)); + return 0; + } + + if (le16_to_cpu(trigger_pg3.NumSCSISenseTrigger)) { + count = le16_to_cpu(trigger_pg3.NumSCSISenseTrigger); + count = min_t(u16, NUM_VALID_ENTRIES, count); + ioc->diag_trigger_scsi.ValidEntries = count; + + scsi_tg = &ioc->diag_trigger_scsi.SCSITriggerEntry[0]; + mpi_scsi_tg = &trigger_pg3.SCSISenseTriggers[0]; + for (i = 0; i < count; i++) { + scsi_tg->ASCQ = mpi_scsi_tg->ASCQ; + scsi_tg->ASC = mpi_scsi_tg->ASC; + scsi_tg->SenseKey = mpi_scsi_tg->SenseKey; + + scsi_tg++; + mpi_scsi_tg++; + } + } + return 0; +} + +/** + * _base_get_mpi_diag_triggers - get mpi diag trigger values from + * persistent pages + * @ioc : per adapter object + * + * Return: 0 on success; otherwise return failure status. + */ +static int +_base_get_mpi_diag_triggers(struct MPT3SAS_ADAPTER *ioc) +{ + Mpi26DriverTriggerPage4_t trigger_pg4; + struct SL_WH_MPI_TRIGGER_T *status_tg; + MPI26_DRIVER_IOCSTATUS_LOGINFO_TRIGGER_ENTRY *mpi_status_tg; + Mpi2ConfigReply_t mpi_reply; + int r = 0, i = 0; + u16 count = 0; + u16 ioc_status; + + r = mpt3sas_config_get_driver_trigger_pg4(ioc, &mpi_reply, + &trigger_pg4); + if (r) + return r; + + ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & + MPI2_IOCSTATUS_MASK; + if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { + dinitprintk(ioc, + ioc_err(ioc, + "%s: Failed to get trigger pg4, ioc_status(0x%04x)\n", + __func__, ioc_status)); + return 0; + } + + if (le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger)) { + count = le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger); + count = min_t(u16, NUM_VALID_ENTRIES, count); + ioc->diag_trigger_mpi.ValidEntries = count; + + status_tg = &ioc->diag_trigger_mpi.MPITriggerEntry[0]; + mpi_status_tg = &trigger_pg4.IOCStatusLoginfoTriggers[0]; + + for (i = 0; i < count; i++) { + status_tg->IOCStatus = le16_to_cpu( + mpi_status_tg->IOCStatus); + status_tg->IocLogInfo = le32_to_cpu( + mpi_status_tg->LogInfo); + + status_tg++; + mpi_status_tg++; + } + } + return 0; +} + +/** + * _base_get_master_diag_triggers - get master diag trigger values from + * persistent pages + * @ioc : per adapter object + * + * Return: nothing. + */ +static int +_base_get_master_diag_triggers(struct MPT3SAS_ADAPTER *ioc) +{ + Mpi26DriverTriggerPage1_t trigger_pg1; + Mpi2ConfigReply_t mpi_reply; + int r; + u16 ioc_status; + + r = mpt3sas_config_get_driver_trigger_pg1(ioc, &mpi_reply, + &trigger_pg1); + if (r) + return r; + + ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & + MPI2_IOCSTATUS_MASK; + if (ioc_status != MPI2_IOCSTATUS_SUCCESS) { + dinitprintk(ioc, + ioc_err(ioc, + "%s: Failed to get trigger pg1, ioc_status(0x%04x)\n", + __func__, ioc_status)); + return 0; + } + + if (le16_to_cpu(trigger_pg1.NumMasterTrigger)) + ioc->diag_trigger_master.MasterData |= + le32_to_cpu( + trigger_pg1.MasterTriggers[0].MasterTriggerFlags); + return 0; +} + +/** + * _base_check_for_trigger_pages_support - checks whether HBA FW supports + * driver trigger pages or not + * @ioc : per adapter object + * @trigger_flags : address where trigger page0's TriggerFlags value is copied + * + * Return: trigger flags mask if HBA FW supports driver trigger pages; + * otherwise returns %-EFAULT if driver trigger pages are not supported by FW or + * return EAGAIN if diag reset occurred due to FW fault and asking the + * caller to retry the command. + * + */ +static int +_base_check_for_trigger_pages_support(struct MPT3SAS_ADAPTER *ioc, u32 *trigger_flags) +{ + Mpi26DriverTriggerPage0_t trigger_pg0; + int r = 0; + Mpi2ConfigReply_t mpi_reply; + u16 ioc_status; + + r = mpt3sas_config_get_driver_trigger_pg0(ioc, &mpi_reply, + &trigger_pg0); + if (r) + return r; + + ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & + MPI2_IOCSTATUS_MASK; + if (ioc_status != MPI2_IOCSTATUS_SUCCESS) + return -EFAULT; + + *trigger_flags = le16_to_cpu(trigger_pg0.TriggerFlags); + return 0; +} + +/** + * _base_get_diag_triggers - Retrieve diag trigger values from + * persistent pages. + * @ioc : per adapter object + * + * Return: zero on success; otherwise return EAGAIN error codes + * asking the caller to retry. + */ +static int +_base_get_diag_triggers(struct MPT3SAS_ADAPTER *ioc) +{ + int trigger_flags; + int r; + + /* + * Default setting of master trigger. + */ + ioc->diag_trigger_master.MasterData = + (MASTER_TRIGGER_FW_FAULT + MASTER_TRIGGER_ADAPTER_RESET); + + r = _base_check_for_trigger_pages_support(ioc, &trigger_flags); + if (r) { + if (r == -EAGAIN) + return r; + /* + * Don't go for error handling when FW doesn't support + * driver trigger pages. + */ + return 0; + } + + ioc->supports_trigger_pages = 1; + + /* + * Retrieve master diag trigger values from driver trigger pg1 + * if master trigger bit enabled in TriggerFlags. + */ + if ((u16)trigger_flags & + MPI26_DRIVER_TRIGGER0_FLAG_MASTER_TRIGGER_VALID) { + r = _base_get_master_diag_triggers(ioc); + if (r) + return r; + } + + /* + * Retrieve event diag trigger values from driver trigger pg2 + * if event trigger bit enabled in TriggerFlags. + */ + if ((u16)trigger_flags & + MPI26_DRIVER_TRIGGER0_FLAG_MPI_EVENT_TRIGGER_VALID) { + r = _base_get_event_diag_triggers(ioc); + if (r) + return r; + } + + /* + * Retrieve scsi diag trigger values from driver trigger pg3 + * if scsi trigger bit enabled in TriggerFlags. + */ + if ((u16)trigger_flags & + MPI26_DRIVER_TRIGGER0_FLAG_SCSI_SENSE_TRIGGER_VALID) { + r = _base_get_scsi_diag_triggers(ioc); + if (r) + return r; + } + /* + * Retrieve mpi error diag trigger values from driver trigger pg4 + * if loginfo trigger bit enabled in TriggerFlags. + */ + if ((u16)trigger_flags & + MPI26_DRIVER_TRIGGER0_FLAG_LOGINFO_TRIGGER_VALID) { + r = _base_get_mpi_diag_triggers(ioc); + if (r) + return r; + } + return 0; +} + +/** + * _base_update_diag_trigger_pages - Update the driver trigger pages after + * online FW update, in case updated FW supports driver + * trigger pages. + * @ioc : per adapter object + * + * Return: nothing. + */ +static void +_base_update_diag_trigger_pages(struct MPT3SAS_ADAPTER *ioc) +{ + + if (ioc->diag_trigger_master.MasterData) + mpt3sas_config_update_driver_trigger_pg1(ioc, + &ioc->diag_trigger_master, 1); + + if (ioc->diag_trigger_event.ValidEntries) + mpt3sas_config_update_driver_trigger_pg2(ioc, + &ioc->diag_trigger_event, 1); + + if (ioc->diag_trigger_scsi.ValidEntries) + mpt3sas_config_update_driver_trigger_pg3(ioc, + &ioc->diag_trigger_scsi, 1); + + if (ioc->diag_trigger_mpi.ValidEntries) + mpt3sas_config_update_driver_trigger_pg4(ioc, + &ioc->diag_trigger_mpi, 1); +} + +/** + * _base_assign_fw_reported_qd - Get FW reported QD for SAS/SATA devices. + * - On failure set default QD values. + * @ioc : per adapter object + * + * Returns 0 for success, non-zero for failure. + * + */ +static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc) +{ + Mpi2ConfigReply_t mpi_reply; + Mpi2SasIOUnitPage1_t sas_iounit_pg1; + Mpi26PCIeIOUnitPage1_t pcie_iounit_pg1; + u16 depth; + int rc = 0; + + ioc->max_wideport_qd = MPT3SAS_SAS_QUEUE_DEPTH; + ioc->max_narrowport_qd = MPT3SAS_SAS_QUEUE_DEPTH; + ioc->max_sata_qd = MPT3SAS_SATA_QUEUE_DEPTH; + ioc->max_nvme_qd = MPT3SAS_NVME_QUEUE_DEPTH; + if (!ioc->is_gen35_ioc) + goto out; + /* sas iounit page 1 */ + rc = mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply, + &sas_iounit_pg1, sizeof(Mpi2SasIOUnitPage1_t)); + if (rc) { + pr_err("%s: failure at %s:%d/%s()!\n", + ioc->name, __FILE__, __LINE__, __func__); + goto out; + } + + depth = le16_to_cpu(sas_iounit_pg1.SASWideMaxQueueDepth); + ioc->max_wideport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH); + + depth = le16_to_cpu(sas_iounit_pg1.SASNarrowMaxQueueDepth); + ioc->max_narrowport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH); + + depth = sas_iounit_pg1.SATAMaxQDepth; + ioc->max_sata_qd = (depth ? depth : MPT3SAS_SATA_QUEUE_DEPTH); + + /* pcie iounit page 1 */ + rc = mpt3sas_config_get_pcie_iounit_pg1(ioc, &mpi_reply, + &pcie_iounit_pg1, sizeof(Mpi26PCIeIOUnitPage1_t)); + if (rc) { + pr_err("%s: failure at %s:%d/%s()!\n", + ioc->name, __FILE__, __LINE__, __func__); + goto out; + } + ioc->max_nvme_qd = (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) ? + (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) : + MPT3SAS_NVME_QUEUE_DEPTH; +out: + dinitprintk(ioc, pr_err( + "MaxWidePortQD: 0x%x MaxNarrowPortQD: 0x%x MaxSataQD: 0x%x MaxNvmeQD: 0x%x\n", + ioc->max_wideport_qd, ioc->max_narrowport_qd, + ioc->max_sata_qd, ioc->max_nvme_qd)); + return rc; +} + +/** + * mpt3sas_atto_validate_nvram - validate the ATTO nvram read from mfg pg1 + * + * @ioc : per adapter object + * @n : ptr to the ATTO nvram structure + * Return: 0 for success, non-zero for failure. + */ +static int +mpt3sas_atto_validate_nvram(struct MPT3SAS_ADAPTER *ioc, + struct ATTO_SAS_NVRAM *n) +{ + int r = -EINVAL; + union ATTO_SAS_ADDRESS *s1; + u32 len; + u8 *pb; + u8 ckSum; + + /* validate nvram checksum */ + pb = (u8 *) n; + ckSum = ATTO_SASNVR_CKSUM_SEED; + len = sizeof(struct ATTO_SAS_NVRAM); + + while (len--) + ckSum = ckSum + pb[len]; + + if (ckSum) { + ioc_err(ioc, "Invalid ATTO NVRAM checksum\n"); + return r; + } + + s1 = (union ATTO_SAS_ADDRESS *) n->SasAddr; + + if (n->Signature[0] != 'E' + || n->Signature[1] != 'S' + || n->Signature[2] != 'A' + || n->Signature[3] != 'S') + ioc_err(ioc, "Invalid ATTO NVRAM signature\n"); + else if (n->Version > ATTO_SASNVR_VERSION) + ioc_info(ioc, "Invalid ATTO NVRAM version"); + else if ((n->SasAddr[7] & (ATTO_SAS_ADDR_ALIGN - 1)) + || s1->b[0] != 0x50 + || s1->b[1] != 0x01 + || s1->b[2] != 0x08 + || (s1->b[3] & 0xF0) != 0x60 + || ((s1->b[3] & 0x0F) | le32_to_cpu(s1->d[1])) == 0) { + ioc_err(ioc, "Invalid ATTO SAS address\n"); + } else + r = 0; + return r; +} + +/** + * mpt3sas_atto_get_sas_addr - get the ATTO SAS address from mfg page 1 + * + * @ioc : per adapter object + * @sas_addr : return sas address + * Return: 0 for success, non-zero for failure. + */ +static int +mpt3sas_atto_get_sas_addr(struct MPT3SAS_ADAPTER *ioc, union ATTO_SAS_ADDRESS *sas_addr) +{ + Mpi2ManufacturingPage1_t mfg_pg1; + Mpi2ConfigReply_t mpi_reply; + struct ATTO_SAS_NVRAM *nvram; + int r; + __be64 addr; + + r = mpt3sas_config_get_manufacturing_pg1(ioc, &mpi_reply, &mfg_pg1); + if (r) { + ioc_err(ioc, "Failed to read manufacturing page 1\n"); + return r; + } + + /* validate nvram */ + nvram = (struct ATTO_SAS_NVRAM *) mfg_pg1.VPD; + r = mpt3sas_atto_validate_nvram(ioc, nvram); + if (r) + return r; + + addr = *((__be64 *) nvram->SasAddr); + sas_addr->q = cpu_to_le64(be64_to_cpu(addr)); + return r; +} + +/** + * mpt3sas_atto_init - perform initializaion for ATTO branded + * adapter. + * @ioc : per adapter object + *5 + * Return: 0 for success, non-zero for failure. + */ +static int +mpt3sas_atto_init(struct MPT3SAS_ADAPTER *ioc) +{ + int sz = 0; + Mpi2BiosPage4_t *bios_pg4 = NULL; + Mpi2ConfigReply_t mpi_reply; + int r; + int ix; + union ATTO_SAS_ADDRESS sas_addr; + union ATTO_SAS_ADDRESS temp; + union ATTO_SAS_ADDRESS bias; + + r = mpt3sas_atto_get_sas_addr(ioc, &sas_addr); + if (r) + return r; + + /* get header first to get size */ + r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, NULL, 0); + if (r) { + ioc_err(ioc, "Failed to read ATTO bios page 4 header.\n"); + return r; + } + + sz = mpi_reply.Header.PageLength * sizeof(u32); + bios_pg4 = kzalloc(sz, GFP_KERNEL); + if (!bios_pg4) { + ioc_err(ioc, "Failed to allocate memory for ATTO bios page.\n"); + return -ENOMEM; + } + + /* read bios page 4 */ + r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, bios_pg4, sz); + if (r) { + ioc_err(ioc, "Failed to read ATTO bios page 4\n"); + goto out; + } + + /* Update bios page 4 with the ATTO WWID */ + bias.q = sas_addr.q; + bias.b[7] += ATTO_SAS_ADDR_DEVNAME_BIAS; + + for (ix = 0; ix < bios_pg4->NumPhys; ix++) { + temp.q = sas_addr.q; + temp.b[7] += ix; + bios_pg4->Phy[ix].ReassignmentWWID = temp.q; + bios_pg4->Phy[ix].ReassignmentDeviceName = bias.q; + } + r = mpt3sas_config_set_bios_pg4(ioc, &mpi_reply, bios_pg4, sz); + +out: + kfree(bios_pg4); + return r; +} + +/** * _base_static_config_pages - static start of day config pages * @ioc: per adapter object */ -static void +static int _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc) { + Mpi2IOUnitPage8_t iounit_pg8; Mpi2ConfigReply_t mpi_reply; u32 iounit_pg1_flags; - + int tg_flags = 0; + int rc; ioc->nvme_abort_timeout = 30; - mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, &ioc->manu_pg0); - if (ioc->ir_firmware) - mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply, + + rc = mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply, + &ioc->manu_pg0); + if (rc) + return rc; + if (ioc->ir_firmware) { + rc = mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply, &ioc->manu_pg10); + if (rc) + return rc; + } + + if (ioc->pdev->vendor == MPI2_MFGPAGE_VENDORID_ATTO) { + rc = mpt3sas_atto_init(ioc); + if (rc) + return rc; + } /* * Ensure correct T10 PI operation if vendor left EEDPTagMode * flag unset in NVDATA. */ - mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, &ioc->manu_pg11); + rc = mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply, + &ioc->manu_pg11); + if (rc) + return rc; if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) { - pr_err("%s: overriding NVDATA EEDPTagMode setting\n", + pr_err("%s: overriding NVDATA EEDPTagMode setting from 0 to 1\n", ioc->name); - ioc->manu_pg11.EEDPTagMode &= ~0x3; - ioc->manu_pg11.EEDPTagMode |= 0x1; + ioc->manu_pg11.EEDPTagMode = 0x1; mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply, &ioc->manu_pg11); } @@ -4107,13 +5662,54 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc) else ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO; } + ioc->time_sync_interval = + ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_MASK; + if (ioc->time_sync_interval) { + if (ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_UNIT_MASK) + ioc->time_sync_interval = + ioc->time_sync_interval * SECONDS_PER_HOUR; + else + ioc->time_sync_interval = + ioc->time_sync_interval * SECONDS_PER_MIN; + dinitprintk(ioc, ioc_info(ioc, + "Driver-FW TimeSync interval is %d seconds. ManuPg11 TimeSync Unit is in %s\n", + ioc->time_sync_interval, (ioc->manu_pg11.TimeSyncInterval & + MPT3SAS_TIMESYNC_UNIT_MASK) ? "Hour" : "Minute")); + } else { + if (ioc->is_gen35_ioc) + ioc_warn(ioc, + "TimeSync Interval in Manuf page-11 is not enabled. Periodic Time-Sync will be disabled\n"); + } + rc = _base_assign_fw_reported_qd(ioc); + if (rc) + return rc; + + /* + * ATTO doesn't use bios page 2 and 3 for bios settings. + */ + if (ioc->pdev->vendor == MPI2_MFGPAGE_VENDORID_ATTO) + ioc->bios_pg3.BiosVersion = 0; + else { + rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2); + if (rc) + return rc; + rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3); + if (rc) + return rc; + } - mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2); - mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3); - mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8); - mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0); - mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); - mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8); + rc = mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8); + if (rc) + return rc; + rc = mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0); + if (rc) + return rc; + rc = mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); + if (rc) + return rc; + rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &iounit_pg8); + if (rc) + return rc; _base_display_ioc_capabilities(ioc); /* @@ -4129,17 +5725,50 @@ _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc) iounit_pg1_flags |= MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING; ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags); - mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); + rc = mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1); + if (rc) + return rc; - if (ioc->iounit_pg8.NumSensors) - ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors; + if (iounit_pg8.NumSensors) + ioc->temp_sensors_count = iounit_pg8.NumSensors; + if (ioc->is_aero_ioc) { + rc = _base_update_ioc_page1_inlinewith_perf_mode(ioc); + if (rc) + return rc; + } + if (ioc->is_gen35_ioc) { + if (ioc->is_driver_loading) { + rc = _base_get_diag_triggers(ioc); + if (rc) + return rc; + } else { + /* + * In case of online HBA FW update operation, + * check whether updated FW supports the driver trigger + * pages or not. + * - If previous FW has not supported driver trigger + * pages and newer FW supports them then update these + * pages with current diag trigger values. + * - If previous FW has supported driver trigger pages + * and new FW doesn't support them then disable + * support_trigger_pages flag. + */ + _base_check_for_trigger_pages_support(ioc, &tg_flags); + if (!ioc->supports_trigger_pages && tg_flags != -EFAULT) + _base_update_diag_trigger_pages(ioc); + else if (ioc->supports_trigger_pages && + tg_flags == -EFAULT) + ioc->supports_trigger_pages = 0; + } + } + return 0; } /** * mpt3sas_free_enclosure_list - release memory * @ioc: per adapter object * - * Free memory allocated during encloure add. + * Free memory allocated during enclosure add. */ void mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc) @@ -4165,8 +5794,9 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) { int i = 0; int j = 0; + int dma_alloc_count = 0; struct chain_tracker *ct; - struct reply_post_struct *rps; + int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1; dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); @@ -4208,29 +5838,34 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) } if (ioc->reply_post) { - do { - rps = &ioc->reply_post[i]; - if (rps->reply_post_free) { - dma_pool_free( - ioc->reply_post_free_dma_pool, - rps->reply_post_free, - rps->reply_post_free_dma); - dexitprintk(ioc, - ioc_info(ioc, "reply_post_free_pool(0x%p): free\n", - rps->reply_post_free)); - rps->reply_post_free = NULL; + dma_alloc_count = DIV_ROUND_UP(count, + RDPQ_MAX_INDEX_IN_ONE_CHUNK); + for (i = 0; i < count; i++) { + if (i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0 + && dma_alloc_count) { + if (ioc->reply_post[i].reply_post_free) { + dma_pool_free( + ioc->reply_post_free_dma_pool, + ioc->reply_post[i].reply_post_free, + ioc->reply_post[i].reply_post_free_dma); + dexitprintk(ioc, ioc_info(ioc, + "reply_post_free_pool(0x%p): free\n", + ioc->reply_post[i].reply_post_free)); + ioc->reply_post[i].reply_post_free = + NULL; + } + --dma_alloc_count; } - } while (ioc->rdpq_array_enable && - (++i < ioc->reply_queue_count)); + } + dma_pool_destroy(ioc->reply_post_free_dma_pool); if (ioc->reply_post_free_array && ioc->rdpq_array_enable) { dma_pool_free(ioc->reply_post_free_array_dma_pool, - ioc->reply_post_free_array, - ioc->reply_post_free_array_dma); + ioc->reply_post_free_array, + ioc->reply_post_free_array_dma); ioc->reply_post_free_array = NULL; } dma_pool_destroy(ioc->reply_post_free_array_dma_pool); - dma_pool_destroy(ioc->reply_post_free_dma_pool); kfree(ioc->reply_post); } @@ -4239,10 +5874,12 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) dma_pool_free(ioc->pcie_sgl_dma_pool, ioc->pcie_sg_lookup[i].pcie_sgl, ioc->pcie_sg_lookup[i].pcie_sgl_dma); + ioc->pcie_sg_lookup[i].pcie_sgl = NULL; } - if (ioc->pcie_sgl_dma_pool) - dma_pool_destroy(ioc->pcie_sgl_dma_pool); + dma_pool_destroy(ioc->pcie_sgl_dma_pool); } + kfree(ioc->pcie_sg_lookup); + ioc->pcie_sg_lookup = NULL; if (ioc->config_page) { dexitprintk(ioc, @@ -4253,7 +5890,9 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) } kfree(ioc->hpr_lookup); + ioc->hpr_lookup = NULL; kfree(ioc->internal_lookup); + ioc->internal_lookup = NULL; if (ioc->chain_lookup) { for (i = 0; i < ioc->scsiio_depth; i++) { for (j = ioc->chains_per_prp_buffer; @@ -4270,33 +5909,378 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) kfree(ioc->chain_lookup); ioc->chain_lookup = NULL; } + + kfree(ioc->io_queue_num); + ioc->io_queue_num = NULL; } /** - * is_MSB_are_same - checks whether all reply queues in a set are + * mpt3sas_check_same_4gb_region - checks whether all reply queues in a set are * having same upper 32bits in their base memory address. - * @reply_pool_start_address: Base address of a reply queue set + * @start_address: Base address of a reply queue set * @pool_sz: Size of single Reply Descriptor Post Queues pool size * * Return: 1 if reply queues in a set have a same upper 32bits in their base * memory address, else 0. */ - static int -is_MSB_are_same(long reply_pool_start_address, u32 pool_sz) +mpt3sas_check_same_4gb_region(dma_addr_t start_address, u32 pool_sz) { - long reply_pool_end_address; + dma_addr_t end_address; - reply_pool_end_address = reply_pool_start_address + pool_sz; + end_address = start_address + pool_sz - 1; - if (upper_32_bits(reply_pool_start_address) == - upper_32_bits(reply_pool_end_address)) + if (upper_32_bits(start_address) == upper_32_bits(end_address)) return 1; else return 0; } /** + * _base_reduce_hba_queue_depth- Retry with reduced queue depth + * @ioc: Adapter object + * + * Return: 0 for success, non-zero for failure. + **/ +static inline int +_base_reduce_hba_queue_depth(struct MPT3SAS_ADAPTER *ioc) +{ + int reduce_sz = 64; + + if ((ioc->hba_queue_depth - reduce_sz) > + (ioc->internal_depth + INTERNAL_SCSIIO_CMDS_COUNT)) { + ioc->hba_queue_depth -= reduce_sz; + return 0; + } else + return -ENOMEM; +} + +/** + * _base_allocate_pcie_sgl_pool - Allocating DMA'able memory + * for pcie sgl pools. + * @ioc: Adapter object + * @sz: DMA Pool size + * + * Return: 0 for success, non-zero for failure. + */ + +static int +_base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) +{ + int i = 0, j = 0; + struct chain_tracker *ct; + + ioc->pcie_sgl_dma_pool = + dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz, + ioc->page_size, 0); + if (!ioc->pcie_sgl_dma_pool) { + ioc_err(ioc, "PCIe SGL pool: dma_pool_create failed\n"); + return -ENOMEM; + } + + ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz; + ioc->chains_per_prp_buffer = + min(ioc->chains_per_prp_buffer, ioc->chains_needed_per_io); + for (i = 0; i < ioc->scsiio_depth; i++) { + ioc->pcie_sg_lookup[i].pcie_sgl = + dma_pool_alloc(ioc->pcie_sgl_dma_pool, GFP_KERNEL, + &ioc->pcie_sg_lookup[i].pcie_sgl_dma); + if (!ioc->pcie_sg_lookup[i].pcie_sgl) { + ioc_err(ioc, "PCIe SGL pool: dma_pool_alloc failed\n"); + return -EAGAIN; + } + + if (!mpt3sas_check_same_4gb_region( + ioc->pcie_sg_lookup[i].pcie_sgl_dma, sz)) { + ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n", + ioc->pcie_sg_lookup[i].pcie_sgl, + (unsigned long long) + ioc->pcie_sg_lookup[i].pcie_sgl_dma); + ioc->use_32bit_dma = true; + return -EAGAIN; + } + + for (j = 0; j < ioc->chains_per_prp_buffer; j++) { + ct = &ioc->chain_lookup[i].chains_per_smid[j]; + ct->chain_buffer = + ioc->pcie_sg_lookup[i].pcie_sgl + + (j * ioc->chain_segment_sz); + ct->chain_buffer_dma = + ioc->pcie_sg_lookup[i].pcie_sgl_dma + + (j * ioc->chain_segment_sz); + } + } + dinitprintk(ioc, ioc_info(ioc, + "PCIe sgl pool depth(%d), element_size(%d), pool_size(%d kB)\n", + ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024)); + dinitprintk(ioc, ioc_info(ioc, + "Number of chains can fit in a PRP page(%d)\n", + ioc->chains_per_prp_buffer)); + return 0; +} + +/** + * _base_allocate_chain_dma_pool - Allocating DMA'able memory + * for chain dma pool. + * @ioc: Adapter object + * @sz: DMA Pool size + * + * Return: 0 for success, non-zero for failure. + */ +static int +_base_allocate_chain_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) +{ + int i = 0, j = 0; + struct chain_tracker *ctr; + + ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev, + ioc->chain_segment_sz, 16, 0); + if (!ioc->chain_dma_pool) + return -ENOMEM; + + for (i = 0; i < ioc->scsiio_depth; i++) { + for (j = ioc->chains_per_prp_buffer; + j < ioc->chains_needed_per_io; j++) { + ctr = &ioc->chain_lookup[i].chains_per_smid[j]; + ctr->chain_buffer = dma_pool_alloc(ioc->chain_dma_pool, + GFP_KERNEL, &ctr->chain_buffer_dma); + if (!ctr->chain_buffer) + return -EAGAIN; + if (!mpt3sas_check_same_4gb_region( + ctr->chain_buffer_dma, ioc->chain_segment_sz)) { + ioc_err(ioc, + "Chain buffers are not in same 4G !!! Chain buff (0x%p) dma = (0x%llx)\n", + ctr->chain_buffer, + (unsigned long long)ctr->chain_buffer_dma); + ioc->use_32bit_dma = true; + return -EAGAIN; + } + } + } + dinitprintk(ioc, ioc_info(ioc, + "chain_lookup depth (%d), frame_size(%d), pool_size(%d kB)\n", + ioc->scsiio_depth, ioc->chain_segment_sz, ((ioc->scsiio_depth * + (ioc->chains_needed_per_io - ioc->chains_per_prp_buffer) * + ioc->chain_segment_sz))/1024)); + return 0; +} + +/** + * _base_allocate_sense_dma_pool - Allocating DMA'able memory + * for sense dma pool. + * @ioc: Adapter object + * @sz: DMA Pool size + * Return: 0 for success, non-zero for failure. + */ +static int +_base_allocate_sense_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) +{ + ioc->sense_dma_pool = + dma_pool_create("sense pool", &ioc->pdev->dev, sz, 4, 0); + if (!ioc->sense_dma_pool) + return -ENOMEM; + ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, + GFP_KERNEL, &ioc->sense_dma); + if (!ioc->sense) + return -EAGAIN; + if (!mpt3sas_check_same_4gb_region(ioc->sense_dma, sz)) { + dinitprintk(ioc, pr_err( + "Bad Sense Pool! sense (0x%p) sense_dma = (0x%llx)\n", + ioc->sense, (unsigned long long) ioc->sense_dma)); + ioc->use_32bit_dma = true; + return -EAGAIN; + } + ioc_info(ioc, + "sense pool(0x%p) - dma(0x%llx): depth(%d), element_size(%d), pool_size (%d kB)\n", + ioc->sense, (unsigned long long)ioc->sense_dma, + ioc->scsiio_depth, SCSI_SENSE_BUFFERSIZE, sz/1024); + return 0; +} + +/** + * _base_allocate_reply_pool - Allocating DMA'able memory + * for reply pool. + * @ioc: Adapter object + * @sz: DMA Pool size + * Return: 0 for success, non-zero for failure. + */ +static int +_base_allocate_reply_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) +{ + /* reply pool, 4 byte align */ + ioc->reply_dma_pool = dma_pool_create("reply pool", + &ioc->pdev->dev, sz, 4, 0); + if (!ioc->reply_dma_pool) + return -ENOMEM; + ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL, + &ioc->reply_dma); + if (!ioc->reply) + return -EAGAIN; + if (!mpt3sas_check_same_4gb_region(ioc->reply_dma, sz)) { + dinitprintk(ioc, pr_err( + "Bad Reply Pool! Reply (0x%p) Reply dma = (0x%llx)\n", + ioc->reply, (unsigned long long) ioc->reply_dma)); + ioc->use_32bit_dma = true; + return -EAGAIN; + } + ioc->reply_dma_min_address = (u32)(ioc->reply_dma); + ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz; + ioc_info(ioc, + "reply pool(0x%p) - dma(0x%llx): depth(%d), frame_size(%d), pool_size(%d kB)\n", + ioc->reply, (unsigned long long)ioc->reply_dma, + ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024); + return 0; +} + +/** + * _base_allocate_reply_free_dma_pool - Allocating DMA'able memory + * for reply free dma pool. + * @ioc: Adapter object + * @sz: DMA Pool size + * Return: 0 for success, non-zero for failure. + */ +static int +_base_allocate_reply_free_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz) +{ + /* reply free queue, 16 byte align */ + ioc->reply_free_dma_pool = dma_pool_create( + "reply_free pool", &ioc->pdev->dev, sz, 16, 0); + if (!ioc->reply_free_dma_pool) + return -ENOMEM; + ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool, + GFP_KERNEL, &ioc->reply_free_dma); + if (!ioc->reply_free) + return -EAGAIN; + if (!mpt3sas_check_same_4gb_region(ioc->reply_free_dma, sz)) { + dinitprintk(ioc, + pr_err("Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n", + ioc->reply_free, (unsigned long long) ioc->reply_free_dma)); + ioc->use_32bit_dma = true; + return -EAGAIN; + } + memset(ioc->reply_free, 0, sz); + dinitprintk(ioc, ioc_info(ioc, + "reply_free pool(0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n", + ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024)); + dinitprintk(ioc, ioc_info(ioc, + "reply_free_dma (0x%llx)\n", + (unsigned long long)ioc->reply_free_dma)); + return 0; +} + +/** + * _base_allocate_reply_post_free_array - Allocating DMA'able memory + * for reply post free array. + * @ioc: Adapter object + * @reply_post_free_array_sz: DMA Pool size + * Return: 0 for success, non-zero for failure. + */ + +static int +_base_allocate_reply_post_free_array(struct MPT3SAS_ADAPTER *ioc, + u32 reply_post_free_array_sz) +{ + ioc->reply_post_free_array_dma_pool = + dma_pool_create("reply_post_free_array pool", + &ioc->pdev->dev, reply_post_free_array_sz, 16, 0); + if (!ioc->reply_post_free_array_dma_pool) + return -ENOMEM; + ioc->reply_post_free_array = + dma_pool_alloc(ioc->reply_post_free_array_dma_pool, + GFP_KERNEL, &ioc->reply_post_free_array_dma); + if (!ioc->reply_post_free_array) + return -EAGAIN; + if (!mpt3sas_check_same_4gb_region(ioc->reply_post_free_array_dma, + reply_post_free_array_sz)) { + dinitprintk(ioc, pr_err( + "Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n", + ioc->reply_free, + (unsigned long long) ioc->reply_free_dma)); + ioc->use_32bit_dma = true; + return -EAGAIN; + } + return 0; +} +/** + * base_alloc_rdpq_dma_pool - Allocating DMA'able memory + * for reply queues. + * @ioc: per adapter object + * @sz: DMA Pool size + * Return: 0 for success, non-zero for failure. + */ +static int +base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz) +{ + int i = 0; + u32 dma_alloc_count = 0; + int reply_post_free_sz = ioc->reply_post_queue_depth * + sizeof(Mpi2DefaultReplyDescriptor_t); + int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1; + + ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct), + GFP_KERNEL); + if (!ioc->reply_post) + return -ENOMEM; + /* + * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and + * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should + * be within 4GB boundary i.e reply queues in a set must have same + * upper 32-bits in their memory address. so here driver is allocating + * the DMA'able memory for reply queues according. + * Driver uses limitation of + * VENTURA_SERIES to manage INVADER_SERIES as well. + */ + dma_alloc_count = DIV_ROUND_UP(count, + RDPQ_MAX_INDEX_IN_ONE_CHUNK); + ioc->reply_post_free_dma_pool = + dma_pool_create("reply_post_free pool", + &ioc->pdev->dev, sz, 16, 0); + if (!ioc->reply_post_free_dma_pool) + return -ENOMEM; + for (i = 0; i < count; i++) { + if ((i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0) && dma_alloc_count) { + ioc->reply_post[i].reply_post_free = + dma_pool_zalloc(ioc->reply_post_free_dma_pool, + GFP_KERNEL, + &ioc->reply_post[i].reply_post_free_dma); + if (!ioc->reply_post[i].reply_post_free) + return -ENOMEM; + /* + * Each set of RDPQ pool must satisfy 4gb boundary + * restriction. + * 1) Check if allocated resources for RDPQ pool are in + * the same 4GB range. + * 2) If #1 is true, continue with 64 bit DMA. + * 3) If #1 is false, return 1. which means free all the + * resources and set DMA mask to 32 and allocate. + */ + if (!mpt3sas_check_same_4gb_region( + ioc->reply_post[i].reply_post_free_dma, sz)) { + dinitprintk(ioc, + ioc_err(ioc, "bad Replypost free pool(0x%p)" + "reply_post_free_dma = (0x%llx)\n", + ioc->reply_post[i].reply_post_free, + (unsigned long long) + ioc->reply_post[i].reply_post_free_dma)); + return -EAGAIN; + } + dma_alloc_count--; + + } else { + ioc->reply_post[i].reply_post_free = + (Mpi2ReplyDescriptorsUnion_t *) + ((long)ioc->reply_post[i-1].reply_post_free + + reply_post_free_sz); + ioc->reply_post[i].reply_post_free_dma = + (dma_addr_t) + (ioc->reply_post[i-1].reply_post_free_dma + + reply_post_free_sz); + } + } + return 0; +} + +/** * _base_allocate_memory_pools - allocate start of day memory pools * @ioc: per adapter object * @@ -4310,11 +6294,12 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) u16 chains_needed_per_io; u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz; u32 retry_sz; + u32 rdpq_sz = 0, sense_sz = 0; u16 max_request_credit, nvme_blocks_needed; unsigned short sg_tablesize; u16 sge_size; - int i, j; - struct chain_tracker *ct; + int i; + int ret = 0, rc = 0; dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); @@ -4456,64 +6441,40 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64; } - dinitprintk(ioc, - ioc_info(ioc, "scatter gather: sge_in_main_msg(%d), sge_per_chain(%d), sge_per_io(%d), chains_per_io(%d)\n", - ioc->max_sges_in_main_message, - ioc->max_sges_in_chain_message, - ioc->shost->sg_tablesize, - ioc->chains_needed_per_io)); + ioc_info(ioc, + "scatter gather: sge_in_main_msg(%d), sge_per_chain(%d), " + "sge_per_io(%d), chains_per_io(%d)\n", + ioc->max_sges_in_main_message, + ioc->max_sges_in_chain_message, + ioc->shost->sg_tablesize, + ioc->chains_needed_per_io); /* reply post queue, 16 byte align */ reply_post_free_sz = ioc->reply_post_queue_depth * sizeof(Mpi2DefaultReplyDescriptor_t); - - sz = reply_post_free_sz; - if (_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable) - sz *= ioc->reply_queue_count; - - ioc->reply_post = kcalloc((ioc->rdpq_array_enable) ? - (ioc->reply_queue_count):1, - sizeof(struct reply_post_struct), GFP_KERNEL); - - if (!ioc->reply_post) { - ioc_err(ioc, "reply_post_free pool: kcalloc failed\n"); - goto out; - } - ioc->reply_post_free_dma_pool = dma_pool_create("reply_post_free pool", - &ioc->pdev->dev, sz, 16, 0); - if (!ioc->reply_post_free_dma_pool) { - ioc_err(ioc, "reply_post_free pool: dma_pool_create failed\n"); - goto out; - } - i = 0; - do { - ioc->reply_post[i].reply_post_free = - dma_pool_zalloc(ioc->reply_post_free_dma_pool, - GFP_KERNEL, - &ioc->reply_post[i].reply_post_free_dma); - if (!ioc->reply_post[i].reply_post_free) { - ioc_err(ioc, "reply_post_free pool: dma_pool_alloc failed\n"); - goto out; - } - dinitprintk(ioc, - ioc_info(ioc, "reply post free pool (0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n", - ioc->reply_post[i].reply_post_free, - ioc->reply_post_queue_depth, - 8, sz / 1024)); - dinitprintk(ioc, - ioc_info(ioc, "reply_post_free_dma = (0x%llx)\n", - (u64)ioc->reply_post[i].reply_post_free_dma)); - total_sz += sz; - } while (ioc->rdpq_array_enable && (++i < ioc->reply_queue_count)); - - if (ioc->dma_mask == 64) { - if (_base_change_consistent_dma_mask(ioc, ioc->pdev) != 0) { - ioc_warn(ioc, "no suitable consistent DMA mask for %s\n", - pci_name(ioc->pdev)); - goto out; + rdpq_sz = reply_post_free_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK; + if ((_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable) + || (ioc->reply_queue_count < RDPQ_MAX_INDEX_IN_ONE_CHUNK)) + rdpq_sz = reply_post_free_sz * ioc->reply_queue_count; + ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz); + if (ret == -EAGAIN) { + /* + * Free allocated bad RDPQ memory pools. + * Change dma coherent mask to 32 bit and reallocate RDPQ + */ + _base_release_memory_pools(ioc); + ioc->use_32bit_dma = true; + if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) { + ioc_err(ioc, + "32 DMA mask failed %s\n", pci_name(ioc->pdev)); + return -ENODEV; } - } - + if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz)) + return -ENOMEM; + } else if (ret == -ENOMEM) + return -ENOMEM; + total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 : + DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK)); ioc->scsiio_depth = ioc->hba_queue_depth - ioc->hi_priority_depth - ioc->internal_depth; @@ -4525,7 +6486,6 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) ioc_info(ioc, "scsi host: can_queue depth (%d)\n", ioc->shost->can_queue)); - /* contiguous pool for request and chains, 16 byte align, one extra " * "frame for smid=0 */ @@ -4570,15 +6530,13 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth * ioc->request_sz); - dinitprintk(ioc, - ioc_info(ioc, "request pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n", - ioc->request, ioc->hba_queue_depth, - ioc->request_sz, - (ioc->hba_queue_depth * ioc->request_sz) / 1024)); + ioc_info(ioc, + "request pool(0x%p) - dma(0x%llx): " + "depth(%d), frame_size(%d), pool_size(%d kB)\n", + ioc->request, (unsigned long long) ioc->request_dma, + ioc->hba_queue_depth, ioc->request_sz, + (ioc->hba_queue_depth * ioc->request_sz) / 1024); - dinitprintk(ioc, - ioc_info(ioc, "request pool: dma(0x%llx)\n", - (unsigned long long)ioc->request_dma)); total_sz += sz; dinitprintk(ioc, @@ -4627,6 +6585,11 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n", ioc->internal, ioc->internal_depth, ioc->internal_smid)); + + ioc->io_queue_num = kcalloc(ioc->scsiio_depth, + sizeof(u16), GFP_KERNEL); + if (!ioc->io_queue_num) + goto out; /* * The number of NVMe page sized blocks needed is: * (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1 @@ -4640,6 +6603,7 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) * be required for NVMe PRP's, only each set of NVMe blocks will be * contiguous, so a new set is allocated for each possible I/O. */ + ioc->chains_per_prp_buffer = 0; if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) { nvme_blocks_needed = @@ -4654,192 +6618,62 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) goto out; } sz = nvme_blocks_needed * ioc->page_size; - ioc->pcie_sgl_dma_pool = - dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz, 16, 0); - if (!ioc->pcie_sgl_dma_pool) { - ioc_info(ioc, "PCIe SGL pool: dma_pool_create failed\n"); - goto out; - } - - ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz; - ioc->chains_per_prp_buffer = min(ioc->chains_per_prp_buffer, - ioc->chains_needed_per_io); - - for (i = 0; i < ioc->scsiio_depth; i++) { - ioc->pcie_sg_lookup[i].pcie_sgl = dma_pool_alloc( - ioc->pcie_sgl_dma_pool, GFP_KERNEL, - &ioc->pcie_sg_lookup[i].pcie_sgl_dma); - if (!ioc->pcie_sg_lookup[i].pcie_sgl) { - ioc_info(ioc, "PCIe SGL pool: dma_pool_alloc failed\n"); - goto out; - } - for (j = 0; j < ioc->chains_per_prp_buffer; j++) { - ct = &ioc->chain_lookup[i].chains_per_smid[j]; - ct->chain_buffer = - ioc->pcie_sg_lookup[i].pcie_sgl + - (j * ioc->chain_segment_sz); - ct->chain_buffer_dma = - ioc->pcie_sg_lookup[i].pcie_sgl_dma + - (j * ioc->chain_segment_sz); - } - } - - dinitprintk(ioc, - ioc_info(ioc, "PCIe sgl pool depth(%d), element_size(%d), pool_size(%d kB)\n", - ioc->scsiio_depth, sz, - (sz * ioc->scsiio_depth) / 1024)); - dinitprintk(ioc, - ioc_info(ioc, "Number of chains can fit in a PRP page(%d)\n", - ioc->chains_per_prp_buffer)); + rc = _base_allocate_pcie_sgl_pool(ioc, sz); + if (rc == -ENOMEM) + return -ENOMEM; + else if (rc == -EAGAIN) + goto try_32bit_dma; total_sz += sz * ioc->scsiio_depth; } - ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev, - ioc->chain_segment_sz, 16, 0); - if (!ioc->chain_dma_pool) { - ioc_err(ioc, "chain_dma_pool: dma_pool_create failed\n"); - goto out; - } - for (i = 0; i < ioc->scsiio_depth; i++) { - for (j = ioc->chains_per_prp_buffer; - j < ioc->chains_needed_per_io; j++) { - ct = &ioc->chain_lookup[i].chains_per_smid[j]; - ct->chain_buffer = dma_pool_alloc( - ioc->chain_dma_pool, GFP_KERNEL, - &ct->chain_buffer_dma); - if (!ct->chain_buffer) { - ioc_err(ioc, "chain_lookup: pci_pool_alloc failed\n"); - _base_release_memory_pools(ioc); - goto out; - } - } - total_sz += ioc->chain_segment_sz; - } - + rc = _base_allocate_chain_dma_pool(ioc, ioc->chain_segment_sz); + if (rc == -ENOMEM) + return -ENOMEM; + else if (rc == -EAGAIN) + goto try_32bit_dma; + total_sz += ioc->chain_segment_sz * ((ioc->chains_needed_per_io - + ioc->chains_per_prp_buffer) * ioc->scsiio_depth); dinitprintk(ioc, - ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n", - ioc->chain_depth, ioc->chain_segment_sz, - (ioc->chain_depth * ioc->chain_segment_sz) / 1024)); - + ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n", + ioc->chain_depth, ioc->chain_segment_sz, + (ioc->chain_depth * ioc->chain_segment_sz) / 1024)); /* sense buffers, 4 byte align */ - sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE; - ioc->sense_dma_pool = dma_pool_create("sense pool", &ioc->pdev->dev, sz, - 4, 0); - if (!ioc->sense_dma_pool) { - ioc_err(ioc, "sense pool: dma_pool_create failed\n"); - goto out; - } - ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL, - &ioc->sense_dma); - if (!ioc->sense) { - ioc_err(ioc, "sense pool: dma_pool_alloc failed\n"); - goto out; - } - /* sense buffer requires to be in same 4 gb region. - * Below function will check the same. - * In case of failure, new pci pool will be created with updated - * alignment. Older allocation and pool will be destroyed. - * Alignment will be used such a way that next allocation if - * success, will always meet same 4gb region requirement. - * Actual requirement is not alignment, but we need start and end of - * DMA address must have same upper 32 bit address. - */ - if (!is_MSB_are_same((long)ioc->sense, sz)) { - //Release Sense pool & Reallocate - dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma); - dma_pool_destroy(ioc->sense_dma_pool); - ioc->sense = NULL; - - ioc->sense_dma_pool = - dma_pool_create("sense pool", &ioc->pdev->dev, sz, - roundup_pow_of_two(sz), 0); - if (!ioc->sense_dma_pool) { - ioc_err(ioc, "sense pool: pci_pool_create failed\n"); - goto out; - } - ioc->sense = dma_pool_alloc(ioc->sense_dma_pool, GFP_KERNEL, - &ioc->sense_dma); - if (!ioc->sense) { - ioc_err(ioc, "sense pool: pci_pool_alloc failed\n"); - goto out; - } - } - dinitprintk(ioc, - ioc_info(ioc, "sense pool(0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n", - ioc->sense, ioc->scsiio_depth, - SCSI_SENSE_BUFFERSIZE, sz / 1024)); - dinitprintk(ioc, - ioc_info(ioc, "sense_dma(0x%llx)\n", - (unsigned long long)ioc->sense_dma)); - total_sz += sz; - + sense_sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE; + rc = _base_allocate_sense_dma_pool(ioc, sense_sz); + if (rc == -ENOMEM) + return -ENOMEM; + else if (rc == -EAGAIN) + goto try_32bit_dma; + total_sz += sense_sz; /* reply pool, 4 byte align */ sz = ioc->reply_free_queue_depth * ioc->reply_sz; - ioc->reply_dma_pool = dma_pool_create("reply pool", &ioc->pdev->dev, sz, - 4, 0); - if (!ioc->reply_dma_pool) { - ioc_err(ioc, "reply pool: dma_pool_create failed\n"); - goto out; - } - ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL, - &ioc->reply_dma); - if (!ioc->reply) { - ioc_err(ioc, "reply pool: dma_pool_alloc failed\n"); - goto out; - } - ioc->reply_dma_min_address = (u32)(ioc->reply_dma); - ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz; - dinitprintk(ioc, - ioc_info(ioc, "reply pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB)\n", - ioc->reply, ioc->reply_free_queue_depth, - ioc->reply_sz, sz / 1024)); - dinitprintk(ioc, - ioc_info(ioc, "reply_dma(0x%llx)\n", - (unsigned long long)ioc->reply_dma)); + rc = _base_allocate_reply_pool(ioc, sz); + if (rc == -ENOMEM) + return -ENOMEM; + else if (rc == -EAGAIN) + goto try_32bit_dma; total_sz += sz; /* reply free queue, 16 byte align */ sz = ioc->reply_free_queue_depth * 4; - ioc->reply_free_dma_pool = dma_pool_create("reply_free pool", - &ioc->pdev->dev, sz, 16, 0); - if (!ioc->reply_free_dma_pool) { - ioc_err(ioc, "reply_free pool: dma_pool_create failed\n"); - goto out; - } - ioc->reply_free = dma_pool_zalloc(ioc->reply_free_dma_pool, GFP_KERNEL, - &ioc->reply_free_dma); - if (!ioc->reply_free) { - ioc_err(ioc, "reply_free pool: dma_pool_alloc failed\n"); - goto out; - } - dinitprintk(ioc, - ioc_info(ioc, "reply_free pool(0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n", - ioc->reply_free, ioc->reply_free_queue_depth, - 4, sz / 1024)); + rc = _base_allocate_reply_free_dma_pool(ioc, sz); + if (rc == -ENOMEM) + return -ENOMEM; + else if (rc == -EAGAIN) + goto try_32bit_dma; dinitprintk(ioc, ioc_info(ioc, "reply_free_dma (0x%llx)\n", (unsigned long long)ioc->reply_free_dma)); total_sz += sz; - if (ioc->rdpq_array_enable) { reply_post_free_array_sz = ioc->reply_queue_count * sizeof(Mpi2IOCInitRDPQArrayEntry); - ioc->reply_post_free_array_dma_pool = - dma_pool_create("reply_post_free_array pool", - &ioc->pdev->dev, reply_post_free_array_sz, 16, 0); - if (!ioc->reply_post_free_array_dma_pool) { - dinitprintk(ioc, - ioc_info(ioc, "reply_post_free_array pool: dma_pool_create failed\n")); - goto out; - } - ioc->reply_post_free_array = - dma_pool_alloc(ioc->reply_post_free_array_dma_pool, - GFP_KERNEL, &ioc->reply_post_free_array_dma); - if (!ioc->reply_post_free_array) { - dinitprintk(ioc, - ioc_info(ioc, "reply_post_free_array pool: dma_pool_alloc failed\n")); - goto out; - } + rc = _base_allocate_reply_post_free_array(ioc, + reply_post_free_array_sz); + if (rc == -ENOMEM) + return -ENOMEM; + else if (rc == -EAGAIN) + goto try_32bit_dma; } ioc->config_page_sz = 512; ioc->config_page = dma_alloc_coherent(&ioc->pdev->dev, @@ -4848,12 +6682,10 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) ioc_err(ioc, "config page: dma_pool_alloc failed\n"); goto out; } - dinitprintk(ioc, - ioc_info(ioc, "config page(0x%p): size(%d)\n", - ioc->config_page, ioc->config_page_sz)); - dinitprintk(ioc, - ioc_info(ioc, "config_page_dma(0x%llx)\n", - (unsigned long long)ioc->config_page_dma)); + + ioc_info(ioc, "config page(0x%p) - dma(0x%llx): size(%d)\n", + ioc->config_page, (unsigned long long)ioc->config_page_dma, + ioc->config_page_sz); total_sz += ioc->config_page_sz; ioc_info(ioc, "Allocated physical memory: size(%d kB)\n", @@ -4864,6 +6696,19 @@ _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc) ioc->shost->sg_tablesize); return 0; +try_32bit_dma: + _base_release_memory_pools(ioc); + if (ioc->use_32bit_dma && (ioc->dma_mask > 32)) { + /* Change dma coherent mask to 32 bit and reallocate */ + if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) { + pr_err("Setting 32 bit coherent DMA mask Failed %s\n", + pci_name(ioc->pdev)); + return -ENODEV; + } + } else if (_base_reduce_hba_queue_depth(ioc) != 0) + return -ENOMEM; + goto retry_allocation; + out: return -ENOMEM; } @@ -4881,7 +6726,7 @@ mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked) { u32 s, sc; - s = ioc->base_readl(&ioc->chip->Doorbell); + s = ioc->base_readl_ext_retry(&ioc->chip->Doorbell); sc = s & MPI2_IOC_STATE_MASK; return cooked ? sc : s; } @@ -4908,6 +6753,8 @@ _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout) return 0; if (count && current_state == MPI2_IOC_STATE_FAULT) break; + if (count && current_state == MPI2_IOC_STATE_COREDUMP) + break; usleep_range(1000, 1500); count++; @@ -4917,16 +6764,32 @@ _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout) } /** + * _base_dump_reg_set - This function will print hexdump of register set. + * @ioc: per adapter object + * + * Return: nothing. + */ +static inline void +_base_dump_reg_set(struct MPT3SAS_ADAPTER *ioc) +{ + unsigned int i, sz = 256; + u32 __iomem *reg = (u32 __iomem *)ioc->chip; + + ioc_info(ioc, "System Register set:\n"); + for (i = 0; i < (sz / sizeof(u32)); i++) + pr_info("%08x: %08x\n", (i * 4), readl(®[i])); +} + +/** * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by * a write to the doorbell) * @ioc: per adapter object + * @timeout: timeout in seconds * * Return: 0 for success, non-zero for failure. * * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell. */ -static int -_base_diag_reset(struct MPT3SAS_ADAPTER *ioc); static int _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout) @@ -5008,10 +6871,15 @@ _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout) __func__, count, timeout)); return 0; } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { - doorbell = ioc->base_readl(&ioc->chip->Doorbell); + doorbell = ioc->base_readl_ext_retry(&ioc->chip->Doorbell); if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { - mpt3sas_base_fault_info(ioc , doorbell); + mpt3sas_print_fault_code(ioc, doorbell); + return -EFAULT; + } + if ((doorbell & MPI2_IOC_STATE_MASK) == + MPI2_IOC_STATE_COREDUMP) { + mpt3sas_print_coredump_info(ioc, doorbell); return -EFAULT; } } else if (int_status == 0xFFFFFFFF) @@ -5043,7 +6911,7 @@ _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout) count = 0; cntdn = 1000 * timeout; do { - doorbell_reg = ioc->base_readl(&ioc->chip->Doorbell); + doorbell_reg = ioc->base_readl_ext_retry(&ioc->chip->Doorbell); if (!(doorbell_reg & MPI2_DOORBELL_USED)) { dhsprintk(ioc, ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n", @@ -5073,6 +6941,7 @@ _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout) { u32 ioc_state; int r = 0; + unsigned long flags; if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) { ioc_err(ioc, "%s: unknown reset_type\n", __func__); @@ -5091,6 +6960,7 @@ _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout) r = -EFAULT; goto out; } + ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout); if (ioc_state) { ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", @@ -5099,6 +6969,26 @@ _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout) goto out; } out: + if (r != 0) { + ioc_state = mpt3sas_base_get_iocstate(ioc, 0); + spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); + /* + * Wait for IOC state CoreDump to clear only during + * HBA initialization & release time. + */ + if ((ioc_state & MPI2_IOC_STATE_MASK) == + MPI2_IOC_STATE_COREDUMP && (ioc->is_driver_loading == 1 || + ioc->fault_reset_work_q == NULL)) { + spin_unlock_irqrestore( + &ioc->ioc_reset_in_progress_lock, flags); + mpt3sas_print_coredump_info(ioc, ioc_state); + mpt3sas_base_wait_for_coredump_completion(ioc, + __func__); + spin_lock_irqsave( + &ioc->ioc_reset_in_progress_lock, flags); + } + spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); + } ioc_info(ioc, "message unit reset: %s\n", r == 0 ? "SUCCESS" : "FAILED"); return r; @@ -5107,11 +6997,11 @@ _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout) /** * mpt3sas_wait_for_ioc - IOC's operational state is checked here. * @ioc: per adapter object - * @wait_count: timeout in seconds + * @timeout: timeout in seconds * * Return: Waits up to timeout seconds for the IOC to * become operational. Returns 0 if IOC is present - * and operational; otherwise returns -EFAULT. + * and operational; otherwise returns %-EFAULT. */ int @@ -5124,6 +7014,17 @@ mpt3sas_wait_for_ioc(struct MPT3SAS_ADAPTER *ioc, int timeout) ioc_state = mpt3sas_base_get_iocstate(ioc, 1); if (ioc_state == MPI2_IOC_STATE_OPERATIONAL) break; + + /* + * Watchdog thread will be started after IOC Initialization, so + * no need to wait here for IOC state to become operational + * when IOC Initialization is on. Instead the driver will + * return ETIME status, so that calling function can issue + * diag reset operation and retry the command. + */ + if (ioc->is_driver_loading) + return -ETIME; + ssleep(1); ioc_info(ioc, "%s: waiting for operational state(count=%d)\n", __func__, ++wait_state_count); @@ -5156,11 +7057,12 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, int i; u8 failed; __le32 *mfp; + int ret_val; /* make sure doorbell is not in use */ - if ((ioc->base_readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) { + if ((ioc->base_readl_ext_retry(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) { ioc_err(ioc, "doorbell is in use (line=%d)\n", __LINE__); - return -EFAULT; + goto doorbell_diag_reset; } /* clear pending doorbell interrupts from previous state changes */ @@ -5207,7 +7109,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, } /* read the first two 16-bits, it gives the total length of the reply */ - reply[0] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) + reply[0] = le16_to_cpu(ioc->base_readl_ext_retry(&ioc->chip->Doorbell) & MPI2_DOORBELL_DATA_MASK); writel(0, &ioc->chip->HostInterruptStatus); if ((_base_wait_for_doorbell_int(ioc, 5))) { @@ -5215,7 +7117,7 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, __LINE__); return -EFAULT; } - reply[1] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell) + reply[1] = le16_to_cpu(ioc->base_readl_ext_retry(&ioc->chip->Doorbell) & MPI2_DOORBELL_DATA_MASK); writel(0, &ioc->chip->HostInterruptStatus); @@ -5226,10 +7128,10 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, return -EFAULT; } if (i >= reply_bytes/2) /* overflow case */ - ioc->base_readl(&ioc->chip->Doorbell); + ioc->base_readl_ext_retry(&ioc->chip->Doorbell); else reply[i] = le16_to_cpu( - ioc->base_readl(&ioc->chip->Doorbell) + ioc->base_readl_ext_retry(&ioc->chip->Doorbell) & MPI2_DOORBELL_DATA_MASK); writel(0, &ioc->chip->HostInterruptStatus); } @@ -5246,10 +7148,14 @@ _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes, mfp = (__le32 *)reply; pr_info("\toffset:data\n"); for (i = 0; i < reply_bytes/4; i++) - pr_info("\t[0x%02x]:%08x\n", i*4, + ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4, le32_to_cpu(mfp[i])); } return 0; + +doorbell_diag_reset: + ret_val = _base_diag_reset(ioc); + return ret_val; } /** @@ -5306,7 +7212,7 @@ mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc, mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) ioc->ioc_link_reset_in_progress = 1; init_completion(&ioc->base_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->base_cmds.done, msecs_to_jiffies(10000)); if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET || @@ -5314,10 +7220,9 @@ mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc, ioc->ioc_link_reset_in_progress) ioc->ioc_link_reset_in_progress = 0; if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { - issue_reset = - mpt3sas_base_check_cmd_timeout(ioc, - ioc->base_cmds.status, mpi_request, - sizeof(Mpi2SasIoUnitControlRequest_t)/4); + mpt3sas_check_cmd_timeout(ioc, ioc->base_cmds.status, + mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t)/4, + issue_reset); goto issue_host_reset; } if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) @@ -5383,16 +7288,16 @@ mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc, ioc->base_cmds.status = MPT3_CMD_PENDING; request = mpt3sas_base_get_msg_frame(ioc, smid); ioc->base_cmds.smid = smid; + memset(request, 0, ioc->request_sz); memcpy(request, mpi_request, sizeof(Mpi2SepReply_t)); init_completion(&ioc->base_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->base_cmds.done, msecs_to_jiffies(10000)); if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { - issue_reset = - mpt3sas_base_check_cmd_timeout(ioc, - ioc->base_cmds.status, mpi_request, - sizeof(Mpi2SepRequest_t)/4); + mpt3sas_check_cmd_timeout(ioc, + ioc->base_cmds.status, mpi_request, + sizeof(Mpi2SepRequest_t)/4, issue_reset); goto issue_host_reset; } if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) @@ -5491,9 +7396,15 @@ _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout) } if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { - mpt3sas_base_fault_info(ioc, ioc_state & + mpt3sas_print_fault_code(ioc, ioc_state & MPI2_DOORBELL_DATA_MASK); goto issue_diag_reset; + } else if ((ioc_state & MPI2_IOC_STATE_MASK) == + MPI2_IOC_STATE_COREDUMP) { + ioc_info(ioc, + "%s: Skipping the diag reset here. (ioc_state=0x%x)\n", + __func__, ioc_state); + return -EFAULT; } ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout); @@ -5504,7 +7415,9 @@ _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout) return -EFAULT; } - issue_diag_reset: + return 0; + +issue_diag_reset: rc = _base_diag_reset(ioc); return rc; } @@ -5568,6 +7481,9 @@ _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc) if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices)) ioc->rdpq_array_capable = 1; + if ((facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ) + && ioc->is_aero_ioc) + ioc->atomic_desc_capable = 1; facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word); facts->IOCRequestFrameSize = le16_to_cpu(mpi_reply.IOCRequestFrameSize); @@ -5669,6 +7585,12 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc) cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma); } + /* + * Set the flag to enable CoreDump state feature in IOC firmware. + */ + mpi_request.ConfigurationFlags |= + cpu_to_le16(MPI26_IOCINIT_CFGFLAGS_COREDUMP_ENABLE); + /* This time stamp specifies number of milliseconds * since epoch ~ midnight January 1, 1970. */ @@ -5680,15 +7602,15 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc) int i; mfp = (__le32 *)&mpi_request; - pr_info("\toffset:data\n"); + ioc_info(ioc, "\toffset:data\n"); for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++) - pr_info("\t[0x%02x]:%08x\n", i*4, + ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4, le32_to_cpu(mfp[i])); } r = _base_handshake_req_reply_wait(ioc, sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request, - sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 10); + sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 30); if (r != 0) { ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r); @@ -5702,6 +7624,8 @@ _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc) r = -EIO; } + /* Reset TimeSync Counter*/ + ioc->timestamp_update_count = 0; return r; } @@ -5740,7 +7664,8 @@ mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, if (ioc_status != MPI2_IOCSTATUS_SUCCESS) ioc->port_enable_failed = 1; - if (ioc->is_driver_loading) { + if (ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE_ASYNC) { + ioc->port_enable_cmds.status &= ~MPT3_CMD_COMPLETE_ASYNC; if (ioc_status == MPI2_IOCSTATUS_SUCCESS) { mpt3sas_port_enable_complete(ioc); return 1; @@ -5789,7 +7714,7 @@ _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc) mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; init_completion(&ioc->port_enable_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ); if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) { ioc_err(ioc, "%s: timeout\n", __func__); @@ -5841,14 +7766,15 @@ mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc) ioc_err(ioc, "%s: failed obtaining a smid\n", __func__); return -EAGAIN; } - + ioc->drv_internal_flags |= MPT_DRV_INTERNAL_FIRST_PE_ISSUED; ioc->port_enable_cmds.status = MPT3_CMD_PENDING; + ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE_ASYNC; mpi_request = mpt3sas_base_get_msg_frame(ioc, smid); ioc->port_enable_cmds.smid = smid; memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t)); mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE; - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); return 0; } @@ -5939,7 +7865,7 @@ _base_event_notification(struct MPT3SAS_ADAPTER *ioc) Mpi2EventNotificationRequest_t *mpi_request; u16 smid; int r = 0; - int i; + int i, issue_diag_reset = 0; dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); @@ -5964,7 +7890,7 @@ _base_event_notification(struct MPT3SAS_ADAPTER *ioc) mpi_request->EventMasks[i] = cpu_to_le32(ioc->event_masks[i]); init_completion(&ioc->base_cmds.done); - mpt3sas_base_put_smid_default(ioc, smid); + ioc->put_smid_default(ioc, smid); wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ); if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) { ioc_err(ioc, "%s: timeout\n", __func__); @@ -5973,10 +7899,19 @@ _base_event_notification(struct MPT3SAS_ADAPTER *ioc) if (ioc->base_cmds.status & MPT3_CMD_RESET) r = -EFAULT; else - r = -ETIME; + issue_diag_reset = 1; + } else dinitprintk(ioc, ioc_info(ioc, "%s: complete\n", __func__)); ioc->base_cmds.status = MPT3_CMD_NOT_USED; + + if (issue_diag_reset) { + if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED) + return -EFAULT; + if (mpt3sas_base_check_for_fault_and_issue_reset(ioc)) + return -EFAULT; + r = -EAGAIN; + } return r; } @@ -6018,24 +7953,22 @@ mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type) } /** - * _base_diag_reset - the "big hammer" start of day reset - * @ioc: per adapter object - * - * Return: 0 for success, non-zero for failure. - */ -static int -_base_diag_reset(struct MPT3SAS_ADAPTER *ioc) -{ - u32 host_diagnostic; - u32 ioc_state; - u32 count; - u32 hcb_size; +* mpt3sas_base_unlock_and_get_host_diagnostic- enable Host Diagnostic Register writes +* @ioc: per adapter object +* @host_diagnostic: host diagnostic register content +* +* Return: 0 for success, non-zero for failure. +*/ - ioc_info(ioc, "sending diag reset !!\n"); - - drsprintk(ioc, ioc_info(ioc, "clear interrupts\n")); +int +mpt3sas_base_unlock_and_get_host_diagnostic(struct MPT3SAS_ADAPTER *ioc, + u32 *host_diagnostic) +{ + u32 count; + *host_diagnostic = 0; count = 0; + do { /* Write magic sequence to WriteSequence register * Loop until in diagnostic mode @@ -6052,43 +7985,90 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc) /* wait 100 msec */ msleep(100); - if (count++ > 20) - goto out; + if (count++ > 20) { + ioc_info(ioc, + "Stop writing magic sequence after 20 retries\n"); + _base_dump_reg_set(ioc); + return -EFAULT; + } - host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic); + *host_diagnostic = ioc->base_readl_ext_retry(&ioc->chip->HostDiagnostic); drsprintk(ioc, - ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n", - count, host_diagnostic)); + ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n", + count, *host_diagnostic)); - } while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0); + } while ((*host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0); + return 0; +} - hcb_size = ioc->base_readl(&ioc->chip->HCBSize); +/** + * mpt3sas_base_lock_host_diagnostic: Disable Host Diagnostic Register writes + * @ioc: per adapter object + */ + +void +mpt3sas_base_lock_host_diagnostic(struct MPT3SAS_ADAPTER *ioc) +{ + drsprintk(ioc, ioc_info(ioc, "disable writes to the diagnostic register\n")); + writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence); +} + +/** + * _base_diag_reset - the "big hammer" start of day reset + * @ioc: per adapter object + * + * Return: 0 for success, non-zero for failure. + */ +static int +_base_diag_reset(struct MPT3SAS_ADAPTER *ioc) +{ + u32 host_diagnostic; + u32 ioc_state; + u32 count; + u32 hcb_size; + + ioc_info(ioc, "sending diag reset !!\n"); + + pci_cfg_access_lock(ioc->pdev); + drsprintk(ioc, ioc_info(ioc, "clear interrupts\n")); + + mutex_lock(&ioc->hostdiag_unlock_mutex); + if (mpt3sas_base_unlock_and_get_host_diagnostic(ioc, &host_diagnostic)) + goto unlock; + + hcb_size = ioc->base_readl(&ioc->chip->HCBSize); drsprintk(ioc, ioc_info(ioc, "diag reset: issued\n")); writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER, &ioc->chip->HostDiagnostic); - /*This delay allows the chip PCIe hardware time to finish reset tasks*/ + /* This delay allows the chip PCIe hardware time to finish reset tasks */ msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000); /* Approximately 300 second max wait */ for (count = 0; count < (300000000 / - MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) { + MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) { - host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic); + host_diagnostic = ioc->base_readl_ext_retry(&ioc->chip->HostDiagnostic); - if (host_diagnostic == 0xFFFFFFFF) - goto out; + if (host_diagnostic == 0xFFFFFFFF) { + ioc_info(ioc, + "Invalid host diagnostic register value\n"); + _base_dump_reg_set(ioc); + goto unlock; + } if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER)) break; - msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000); + /* Wait to pass the second read delay window */ + msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC/1000); } if (host_diagnostic & MPI2_DIAG_HCB_MODE) { drsprintk(ioc, - ioc_info(ioc, "restart the adapter assuming the HCB Address points to good F/W\n")); + ioc_info(ioc, "restart the adapter assuming the\n" + "HCB Address points to good F/W\n")); host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK; host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW; writel(host_diagnostic, &ioc->chip->HostDiagnostic); @@ -6102,35 +8082,40 @@ _base_diag_reset(struct MPT3SAS_ADAPTER *ioc) writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET, &ioc->chip->HostDiagnostic); - drsprintk(ioc, - ioc_info(ioc, "disable writes to the diagnostic register\n")); - writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence); + mpt3sas_base_lock_host_diagnostic(ioc); + mutex_unlock(&ioc->hostdiag_unlock_mutex); drsprintk(ioc, ioc_info(ioc, "Wait for FW to go to the READY state\n")); ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20); if (ioc_state) { ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n", __func__, ioc_state); - goto out; + _base_dump_reg_set(ioc); + goto fail; } + pci_cfg_access_unlock(ioc->pdev); ioc_info(ioc, "diag reset: SUCCESS\n"); return 0; - out: +unlock: + mutex_unlock(&ioc->hostdiag_unlock_mutex); + +fail: + pci_cfg_access_unlock(ioc->pdev); ioc_err(ioc, "diag reset: FAILED\n"); return -EFAULT; } /** - * _base_make_ioc_ready - put controller in READY state + * mpt3sas_base_make_ioc_ready - put controller in READY state * @ioc: per adapter object * @type: FORCE_BIG_HAMMER or SOFT_RESET * * Return: 0 for success, non-zero for failure. */ -static int -_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type) +int +mpt3sas_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type) { u32 ioc_state; int rc; @@ -6165,16 +8150,33 @@ _base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type) return 0; if (ioc_state & MPI2_DOORBELL_USED) { - dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n")); + ioc_info(ioc, "unexpected doorbell active!\n"); goto issue_diag_reset; } if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { - mpt3sas_base_fault_info(ioc, ioc_state & + mpt3sas_print_fault_code(ioc, ioc_state & MPI2_DOORBELL_DATA_MASK); goto issue_diag_reset; } + if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) { + /* + * if host reset is invoked while watch dog thread is waiting + * for IOC state to be changed to Fault state then driver has + * to wait here for CoreDump state to clear otherwise reset + * will be issued to the FW and FW move the IOC state to + * reset state without copying the FW logs to coredump region. + */ + if (ioc->ioc_coredump_loop != MPT3SAS_COREDUMP_LOOP_DONE) { + mpt3sas_print_coredump_info(ioc, ioc_state & + MPI2_DOORBELL_DATA_MASK); + mpt3sas_base_wait_for_coredump_completion(ioc, + __func__); + } + goto issue_diag_reset; + } + if (type == FORCE_BIG_HAMMER) goto issue_diag_reset; @@ -6198,7 +8200,7 @@ _base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type) static int _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc) { - int r, i, index; + int r, i, index, rc; unsigned long flags; u32 reply_address; u16 smid; @@ -6301,8 +8303,19 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc) skip_init_reply_post_free_queue: r = _base_send_ioc_init(ioc); - if (r) - return r; + if (r) { + /* + * No need to check IOC state for fault state & issue + * diag reset during host reset. This check is need + * only during driver load time. + */ + if (!ioc->is_driver_loading) + return r; + + rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); + if (rc || (_base_send_ioc_init(ioc))) + return r; + } /* initialize reply free host index */ ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1; @@ -6325,7 +8338,7 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc) skip_init_reply_post_host_index: - _base_unmask_interrupts(ioc); + mpt3sas_base_unmask_interrupts(ioc); if (ioc->hba_mpi_version_belonged != MPI2_VERSION) { r = _base_display_fwpkg_version(ioc); @@ -6333,12 +8346,15 @@ _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc) return r; } - _base_static_config_pages(ioc); + r = _base_static_config_pages(ioc); + if (r) + return r; + r = _base_event_notification(ioc); if (r) return r; - if (ioc->is_driver_loading) { + if (!ioc->shost_recovery) { if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier == 0x80) { @@ -6374,9 +8390,9 @@ mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc) /* synchronizing freeing resource with pci_access_mutex lock */ mutex_lock(&ioc->pci_access_mutex); if (ioc->chip_phys && ioc->chip) { - _base_mask_interrupts(ioc); + mpt3sas_base_mask_interrupts(ioc); ioc->shost_recovery = 1; - _base_make_ioc_ready(ioc, SOFT_RESET); + mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET); ioc->shost_recovery = 0; } @@ -6394,7 +8410,7 @@ mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc) int mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) { - int r, i; + int r, i, rc; int cpu_id, last_cpu_id = 0; dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__)); @@ -6407,8 +8423,7 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL); ioc->reply_queue_count = 1; if (!ioc->cpu_msix_table) { - dfailprintk(ioc, - ioc_info(ioc, "allocation for cpu_msix_table failed!!!\n")); + ioc_info(ioc, "Allocation for cpu_msix_table failed!!!\n"); r = -ENOMEM; goto out_free_resources; } @@ -6417,40 +8432,49 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz, sizeof(resource_size_t *), GFP_KERNEL); if (!ioc->reply_post_host_index) { - dfailprintk(ioc, - ioc_info(ioc, "allocation for reply_post_host_index failed!!!\n")); + ioc_info(ioc, "Allocation for reply_post_host_index failed!!!\n"); r = -ENOMEM; goto out_free_resources; } } + ioc->smp_affinity_enable = smp_affinity_enable; + ioc->rdpq_array_enable_assigned = 0; - ioc->dma_mask = 0; - if (ioc->is_aero_ioc) + ioc->use_32bit_dma = false; + ioc->dma_mask = 64; + if (ioc->is_aero_ioc) { ioc->base_readl = &_base_readl_aero; - else + ioc->base_readl_ext_retry = &_base_readl_ext_retry; + } else { ioc->base_readl = &_base_readl; + ioc->base_readl_ext_retry = &_base_readl; + } r = mpt3sas_base_map_resources(ioc); if (r) goto out_free_resources; pci_set_drvdata(ioc->pdev, ioc->shost); r = _base_get_ioc_facts(ioc); - if (r) - goto out_free_resources; + if (r) { + rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); + if (rc || (_base_get_ioc_facts(ioc))) + goto out_free_resources; + } switch (ioc->hba_mpi_version_belonged) { case MPI2_VERSION: ioc->build_sg_scmd = &_base_build_sg_scmd; ioc->build_sg = &_base_build_sg; ioc->build_zero_len_sge = &_base_build_zero_len_sge; + ioc->get_msix_index_for_smlio = &_base_get_msix_index; break; case MPI25_VERSION: case MPI26_VERSION: /* * In SAS3.0, * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and - * Target Status - all require the IEEE formated scatter gather + * Target Status - all require the IEEE formatted scatter gather * elements. */ ioc->build_sg_scmd = &_base_build_sg_scmd_ieee; @@ -6458,15 +8482,30 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) ioc->build_nvme_prp = &_base_build_nvme_prp; ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee; ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t); - + if (ioc->high_iops_queues) + ioc->get_msix_index_for_smlio = + &_base_get_high_iops_msix_index; + else + ioc->get_msix_index_for_smlio = &_base_get_msix_index; break; } - - if (ioc->is_mcpu_endpoint) - ioc->put_smid_scsi_io = &_base_put_smid_mpi_ep_scsi_io; - else - ioc->put_smid_scsi_io = &_base_put_smid_scsi_io; - + if (ioc->atomic_desc_capable) { + ioc->put_smid_default = &_base_put_smid_default_atomic; + ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic; + ioc->put_smid_fast_path = + &_base_put_smid_fast_path_atomic; + ioc->put_smid_hi_priority = + &_base_put_smid_hi_priority_atomic; + } else { + ioc->put_smid_default = &_base_put_smid_default; + ioc->put_smid_fast_path = &_base_put_smid_fast_path; + ioc->put_smid_hi_priority = &_base_put_smid_hi_priority; + if (ioc->is_mcpu_endpoint) + ioc->put_smid_scsi_io = + &_base_put_smid_mpi_ep_scsi_io; + else + ioc->put_smid_scsi_io = &_base_put_smid_scsi_io; + } /* * These function pointers for other requests that don't * the require IEEE scatter gather elements. @@ -6476,7 +8515,7 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) ioc->build_sg_mpi = &_base_build_sg; ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge; - r = _base_make_ioc_ready(ioc, SOFT_RESET); + r = mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET); if (r) goto out_free_resources; @@ -6489,20 +8528,35 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) { r = _base_get_port_facts(ioc, i); - if (r) - goto out_free_resources; + if (r) { + rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc); + if (rc || (_base_get_port_facts(ioc, i))) + goto out_free_resources; + } } r = _base_allocate_memory_pools(ioc); if (r) goto out_free_resources; + if (irqpoll_weight > 0) + ioc->thresh_hold = irqpoll_weight; + else + ioc->thresh_hold = ioc->hba_queue_depth/4; + + _base_init_irqpolls(ioc); init_waitqueue_head(&ioc->reset_wq); /* allocate memory pd handle bitmask list */ ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8); if (ioc->facts.MaxDevHandle % 8) ioc->pd_handles_sz++; + /* + * pd_handles_sz should have, at least, the minimal room for + * set_bit()/test_bit(), otherwise out-of-memory touch may occur. + */ + ioc->pd_handles_sz = ALIGN(ioc->pd_handles_sz, sizeof(unsigned long)); + ioc->pd_handles = kzalloc(ioc->pd_handles_sz, GFP_KERNEL); if (!ioc->pd_handles) { @@ -6520,16 +8574,27 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8); if (ioc->facts.MaxDevHandle % 8) ioc->pend_os_device_add_sz++; + + /* + * pend_os_device_add_sz should have, at least, the minimal room for + * set_bit()/test_bit(), otherwise out-of-memory may occur. + */ + ioc->pend_os_device_add_sz = ALIGN(ioc->pend_os_device_add_sz, + sizeof(unsigned long)); ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz, GFP_KERNEL); - if (!ioc->pend_os_device_add) + if (!ioc->pend_os_device_add) { + r = -ENOMEM; goto out_free_resources; + } ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz; ioc->device_remove_in_progress = kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL); - if (!ioc->device_remove_in_progress) + if (!ioc->device_remove_in_progress) { + r = -ENOMEM; goto out_free_resources; + } ioc->fwfault_debug = mpt3sas_fwfault_debug; @@ -6603,10 +8668,21 @@ mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc) } } r = _base_make_ioc_operational(ioc); - if (r) - goto out_free_resources; + if (r == -EAGAIN) { + r = _base_make_ioc_operational(ioc); + if (r) + goto out_free_resources; + } + + /* + * Copy current copy of IOCFacts in prev_fw_facts + * and it will be used during online firmware upgrade. + */ + memcpy(&ioc->prev_fw_facts, &ioc->facts, + sizeof(struct mpt3sas_facts)); ioc->non_operational_loop = 0; + ioc->ioc_coredump_loop = 0; ioc->got_task_abort_from_ioctl = 0; return 0; @@ -6688,14 +8764,14 @@ static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc) } /** - * _base_after_reset_handler - after reset handler + * _base_clear_outstanding_mpt_commands - clears outstanding mpt commands * @ioc: per adapter object */ -static void _base_after_reset_handler(struct MPT3SAS_ADAPTER *ioc) +static void +_base_clear_outstanding_mpt_commands(struct MPT3SAS_ADAPTER *ioc) { - mpt3sas_scsih_after_reset_handler(ioc); - mpt3sas_ctl_after_reset_handler(ioc); - dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_AFTER_RESET\n", __func__)); + dtmprintk(ioc, + ioc_info(ioc, "%s: clear outstanding mpt cmds\n", __func__)); if (ioc->transport_cmds.status & MPT3_CMD_PENDING) { ioc->transport_cmds.status |= MPT3_CMD_RESET; mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid); @@ -6714,8 +8790,6 @@ static void _base_after_reset_handler(struct MPT3SAS_ADAPTER *ioc) ioc->start_scan_failed = MPI2_IOCSTATUS_INTERNAL_ERROR; ioc->start_scan = 0; - ioc->port_enable_cmds.status = - MPT3_CMD_NOT_USED; } else { complete(&ioc->port_enable_cmds.done); } @@ -6729,6 +8803,17 @@ static void _base_after_reset_handler(struct MPT3SAS_ADAPTER *ioc) } /** + * _base_clear_outstanding_commands - clear all outstanding commands + * @ioc: per adapter object + */ +static void _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc) +{ + mpt3sas_scsih_clear_outstanding_scsi_tm_commands(ioc); + mpt3sas_ctl_clear_outstanding_ioctls(ioc); + _base_clear_outstanding_mpt_commands(ioc); +} + +/** * _base_reset_done_handler - reset done handler * @ioc: per adapter object */ @@ -6768,6 +8853,90 @@ mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc) } /** + * _base_check_ioc_facts_changes - Look for increase/decrease of IOCFacts + * attributes during online firmware upgrade and update the corresponding + * IOC variables accordingly. + * + * @ioc: Pointer to MPT_ADAPTER structure + */ +static int +_base_check_ioc_facts_changes(struct MPT3SAS_ADAPTER *ioc) +{ + u16 pd_handles_sz; + void *pd_handles = NULL, *blocking_handles = NULL; + void *pend_os_device_add = NULL, *device_remove_in_progress = NULL; + struct mpt3sas_facts *old_facts = &ioc->prev_fw_facts; + + if (ioc->facts.MaxDevHandle > old_facts->MaxDevHandle) { + pd_handles_sz = (ioc->facts.MaxDevHandle / 8); + if (ioc->facts.MaxDevHandle % 8) + pd_handles_sz++; + + /* + * pd_handles should have, at least, the minimal room for + * set_bit()/test_bit(), otherwise out-of-memory touch may + * occur. + */ + pd_handles_sz = ALIGN(pd_handles_sz, sizeof(unsigned long)); + pd_handles = krealloc(ioc->pd_handles, pd_handles_sz, + GFP_KERNEL); + if (!pd_handles) { + ioc_info(ioc, + "Unable to allocate the memory for pd_handles of sz: %d\n", + pd_handles_sz); + return -ENOMEM; + } + memset(pd_handles + ioc->pd_handles_sz, 0, + (pd_handles_sz - ioc->pd_handles_sz)); + ioc->pd_handles = pd_handles; + + blocking_handles = krealloc(ioc->blocking_handles, + pd_handles_sz, GFP_KERNEL); + if (!blocking_handles) { + ioc_info(ioc, + "Unable to allocate the memory for " + "blocking_handles of sz: %d\n", + pd_handles_sz); + return -ENOMEM; + } + memset(blocking_handles + ioc->pd_handles_sz, 0, + (pd_handles_sz - ioc->pd_handles_sz)); + ioc->blocking_handles = blocking_handles; + ioc->pd_handles_sz = pd_handles_sz; + + pend_os_device_add = krealloc(ioc->pend_os_device_add, + pd_handles_sz, GFP_KERNEL); + if (!pend_os_device_add) { + ioc_info(ioc, + "Unable to allocate the memory for pend_os_device_add of sz: %d\n", + pd_handles_sz); + return -ENOMEM; + } + memset(pend_os_device_add + ioc->pend_os_device_add_sz, 0, + (pd_handles_sz - ioc->pend_os_device_add_sz)); + ioc->pend_os_device_add = pend_os_device_add; + ioc->pend_os_device_add_sz = pd_handles_sz; + + device_remove_in_progress = krealloc( + ioc->device_remove_in_progress, pd_handles_sz, GFP_KERNEL); + if (!device_remove_in_progress) { + ioc_info(ioc, + "Unable to allocate the memory for device_remove_in_progress of sz: %d\n", + pd_handles_sz); + return -ENOMEM; + } + memset(device_remove_in_progress + + ioc->device_remove_in_progress_sz, 0, + (pd_handles_sz - ioc->device_remove_in_progress_sz)); + ioc->device_remove_in_progress = device_remove_in_progress; + ioc->device_remove_in_progress_sz = pd_handles_sz; + } + + memcpy(&ioc->prev_fw_facts, &ioc->facts, sizeof(struct mpt3sas_facts)); + return 0; +} + +/** * mpt3sas_base_hard_reset_handler - reset controller * @ioc: Pointer to MPT_ADAPTER structure * @type: FORCE_BIG_HAMMER or SOFT_RESET @@ -6807,16 +8976,22 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, MPT3_DIAG_BUFFER_IS_RELEASED))) { is_trigger = 1; ioc_state = mpt3sas_base_get_iocstate(ioc, 0); - if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) + if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT || + (ioc_state & MPI2_IOC_STATE_MASK) == + MPI2_IOC_STATE_COREDUMP) { is_fault = 1; + ioc->htb_rel.trigger_info_dwords[1] = + (ioc_state & MPI2_DOORBELL_DATA_MASK); + } } _base_pre_reset_handler(ioc); mpt3sas_wait_for_commands_to_complete(ioc); - _base_mask_interrupts(ioc); - r = _base_make_ioc_ready(ioc, type); + mpt3sas_base_mask_interrupts(ioc); + mpt3sas_base_pause_mq_polling(ioc); + r = mpt3sas_base_make_ioc_ready(ioc, type); if (r) goto out; - _base_after_reset_handler(ioc); + _base_clear_outstanding_commands(ioc); /* If this hard reset is called while port enable is active, then * there is no reason to call make_ioc_operational @@ -6830,6 +9005,13 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, if (r) goto out; + r = _base_check_ioc_facts_changes(ioc); + if (r) { + ioc_info(ioc, + "Some of the parameters got changed in this new firmware" + " image and it requires system reboot\n"); + goto out; + } if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable) panic("%s: Issue occurred with flashing controller firmware." "Please reboot the system and ensure that the correct" @@ -6840,15 +9022,14 @@ mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc, _base_reset_done_handler(ioc); out: - dtmprintk(ioc, - ioc_info(ioc, "%s: %s\n", - __func__, r == 0 ? "SUCCESS" : "FAILED")); + ioc_info(ioc, "%s: %s\n", __func__, r == 0 ? "SUCCESS" : "FAILED"); spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags); ioc->shost_recovery = 0; spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags); ioc->ioc_reset_count++; mutex_unlock(&ioc->reset_in_progress_mutex); + mpt3sas_base_resume_mq_polling(ioc); out_unlocked: if ((r == 0) && is_trigger) { |