diff options
Diffstat (limited to 'drivers/scsi/megaraid/megaraid_sas_fusion.c')
| -rw-r--r-- | drivers/scsi/megaraid/megaraid_sas_fusion.c | 2529 |
1 files changed, 1695 insertions, 834 deletions
diff --git a/drivers/scsi/megaraid/megaraid_sas_fusion.c b/drivers/scsi/megaraid/megaraid_sas_fusion.c index 985510628f56..a6794f49e9fa 100644 --- a/drivers/scsi/megaraid/megaraid_sas_fusion.c +++ b/drivers/scsi/megaraid/megaraid_sas_fusion.c @@ -1,34 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Linux MegaRAID driver for SAS based RAID controllers * * Copyright (c) 2009-2013 LSI Corporation - * Copyright (c) 2013-2014 Avago Technologies - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. + * Copyright (c) 2013-2016 Avago Technologies + * Copyright (c) 2016-2018 Broadcom Inc. * * FILE: megaraid_sas_fusion.c * - * Authors: Avago Technologies + * Authors: Broadcom Inc. * Sumant Patro * Adam Radford - * Kashyap Desai <kashyap.desai@avagotech.com> - * Sumit Saxena <sumit.saxena@avagotech.com> - * - * Send feedback to: megaraidlinux.pdl@avagotech.com + * Kashyap Desai <kashyap.desai@broadcom.com> + * Sumit Saxena <sumit.saxena@broadcom.com> * - * Mail to: Avago Technologies, 350 West Trimble Road, Building 90, - * San Jose, California 95131 + * Send feedback to: megaraidlinux.pdl@broadcom.com */ #include <linux/kernel.h> @@ -48,6 +34,8 @@ #include <linux/mutex.h> #include <linux/poll.h> #include <linux/vmalloc.h> +#include <linux/workqueue.h> +#include <linux/irq_poll.h> #include <scsi/scsi.h> #include <scsi/scsi_cmnd.h> @@ -60,9 +48,6 @@ #include "megaraid_sas.h" -extern void megasas_free_cmds(struct megasas_instance *instance); -extern struct megasas_cmd *megasas_get_cmd(struct megasas_instance - *instance); extern void megasas_complete_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd, u8 alt_status); @@ -70,40 +55,113 @@ int wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd, int seconds); -void -megasas_return_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd); -int megasas_alloc_cmds(struct megasas_instance *instance); -int -megasas_clear_intr_fusion(struct megasas_register_set __iomem *regs); int -megasas_issue_polled(struct megasas_instance *instance, - struct megasas_cmd *cmd); -void -megasas_check_and_restore_queue_depth(struct megasas_instance *instance); +megasas_clear_intr_fusion(struct megasas_instance *instance); int megasas_transition_to_ready(struct megasas_instance *instance, int ocr); -void megaraid_sas_kill_hba(struct megasas_instance *instance); extern u32 megasas_dbg_lvl; -void megasas_sriov_heartbeat_handler(unsigned long instance_addr); int megasas_sriov_start_heartbeat(struct megasas_instance *instance, int initial); -void megasas_start_timer(struct megasas_instance *instance, - struct timer_list *timer, - void *fn, unsigned long interval); extern struct megasas_mgmt_info megasas_mgmt_info; extern unsigned int resetwaittime; extern unsigned int dual_qdepth_disable; static void megasas_free_rdpq_fusion(struct megasas_instance *instance); static void megasas_free_reply_fusion(struct megasas_instance *instance); +static inline +void megasas_configure_queue_sizes(struct megasas_instance *instance); +static void megasas_fusion_crash_dump(struct megasas_instance *instance); + +/** + * megasas_adp_reset_wait_for_ready - initiate chip reset and wait for + * controller to come to ready state + * @instance: adapter's soft state + * @do_adp_reset: If true, do a chip reset + * @ocr_context: If called from OCR context this will + * be set to 1, else 0 + * + * This function initiates a chip reset followed by a wait for controller to + * transition to ready state. + * During this, driver will block all access to PCI config space from userspace + */ +int +megasas_adp_reset_wait_for_ready(struct megasas_instance *instance, + bool do_adp_reset, + int ocr_context) +{ + int ret = FAILED; + + /* + * Block access to PCI config space from userspace + * when diag reset is initiated from driver + */ + if (megasas_dbg_lvl & OCR_DEBUG) + dev_info(&instance->pdev->dev, + "Block access to PCI config space %s %d\n", + __func__, __LINE__); + + pci_cfg_access_lock(instance->pdev); + + if (do_adp_reset) { + if (instance->instancet->adp_reset + (instance, instance->reg_set)) + goto out; + } + + /* Wait for FW to become ready */ + if (megasas_transition_to_ready(instance, ocr_context)) { + dev_warn(&instance->pdev->dev, + "Failed to transition controller to ready for scsi%d.\n", + instance->host->host_no); + goto out; + } + ret = SUCCESS; +out: + if (megasas_dbg_lvl & OCR_DEBUG) + dev_info(&instance->pdev->dev, + "Unlock access to PCI config space %s %d\n", + __func__, __LINE__); + + pci_cfg_access_unlock(instance->pdev); + return ret; +} + +/** + * megasas_check_same_4gb_region - check if allocation + * crosses same 4GB boundary or not + * @instance: adapter's soft instance + * @start_addr: start address of DMA allocation + * @size: size of allocation in bytes + * @return: true : allocation does not cross same + * 4GB boundary + * false: allocation crosses same + * 4GB boundary + */ +static inline bool megasas_check_same_4gb_region + (struct megasas_instance *instance, dma_addr_t start_addr, size_t size) +{ + dma_addr_t end_addr; + + end_addr = start_addr + size; + + if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) { + dev_err(&instance->pdev->dev, + "Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n", + (unsigned long long)start_addr, + (unsigned long long)end_addr); + return false; + } + + return true; +} /** * megasas_enable_intr_fusion - Enables interrupts - * @regs: MFI register set + * @instance: adapter's soft instance */ -void +static void megasas_enable_intr_fusion(struct megasas_instance *instance) { struct megasas_register_set __iomem *regs; @@ -117,35 +175,39 @@ megasas_enable_intr_fusion(struct megasas_instance *instance) writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask); /* Dummy readl to force pci flush */ - readl(®s->outbound_intr_mask); + dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n", + __func__, readl(®s->outbound_intr_mask)); } /** * megasas_disable_intr_fusion - Disables interrupt - * @regs: MFI register set + * @instance: adapter's soft instance */ -void +static void megasas_disable_intr_fusion(struct megasas_instance *instance) { u32 mask = 0xFFFFFFFF; - u32 status; struct megasas_register_set __iomem *regs; regs = instance->reg_set; instance->mask_interrupts = 1; writel(mask, ®s->outbound_intr_mask); /* Dummy readl to force pci flush */ - status = readl(®s->outbound_intr_mask); + dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n", + __func__, readl(®s->outbound_intr_mask)); } int -megasas_clear_intr_fusion(struct megasas_register_set __iomem *regs) +megasas_clear_intr_fusion(struct megasas_instance *instance) { u32 status; + struct megasas_register_set __iomem *regs; + regs = instance->reg_set; /* * Check if it is our interrupt */ - status = readl(®s->outbound_intr_status); + status = megasas_readl(instance, + ®s->outbound_intr_status); if (status & 1) { writel(status, ®s->outbound_intr_status); @@ -158,9 +220,44 @@ megasas_clear_intr_fusion(struct megasas_register_set __iomem *regs) return 1; } +static inline void +megasas_sdev_busy_inc(struct megasas_instance *instance, + struct scsi_cmnd *scmd) +{ + if (instance->perf_mode == MR_BALANCED_PERF_MODE) { + struct MR_PRIV_DEVICE *mr_device_priv_data = + scmd->device->hostdata; + atomic_inc(&mr_device_priv_data->sdev_priv_busy); + } +} + +static inline void +megasas_sdev_busy_dec(struct megasas_instance *instance, + struct scsi_cmnd *scmd) +{ + if (instance->perf_mode == MR_BALANCED_PERF_MODE) { + struct MR_PRIV_DEVICE *mr_device_priv_data = + scmd->device->hostdata; + atomic_dec(&mr_device_priv_data->sdev_priv_busy); + } +} + +static inline int +megasas_sdev_busy_read(struct megasas_instance *instance, + struct scsi_cmnd *scmd) +{ + if (instance->perf_mode == MR_BALANCED_PERF_MODE) { + struct MR_PRIV_DEVICE *mr_device_priv_data = + scmd->device->hostdata; + return atomic_read(&mr_device_priv_data->sdev_priv_busy); + } + return 0; +} + /** * megasas_get_cmd_fusion - Get a command from the free pool * @instance: Adapter soft state + * @blk_tag: Command tag * * Returns a blk_tag indexed mpt frame */ @@ -188,48 +285,56 @@ inline void megasas_return_cmd_fusion(struct megasas_instance *instance, } /** + * megasas_write_64bit_req_desc - PCI writes 64bit request descriptor + * @instance: Adapter soft state + * @req_desc: 64bit Request descriptor + */ +static void +megasas_write_64bit_req_desc(struct megasas_instance *instance, + union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc) +{ +#if defined(writeq) && defined(CONFIG_64BIT) + u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) | + le32_to_cpu(req_desc->u.low)); + writeq(req_data, &instance->reg_set->inbound_low_queue_port); +#else + unsigned long flags; + spin_lock_irqsave(&instance->hba_lock, flags); + writel(le32_to_cpu(req_desc->u.low), + &instance->reg_set->inbound_low_queue_port); + writel(le32_to_cpu(req_desc->u.high), + &instance->reg_set->inbound_high_queue_port); + spin_unlock_irqrestore(&instance->hba_lock, flags); +#endif +} + +/** * megasas_fire_cmd_fusion - Sends command to the FW * @instance: Adapter soft state * @req_desc: 32bit or 64bit Request descriptor * - * Perform PCI Write. Ventura supports 32 bit Descriptor. - * Prior to Ventura (12G) MR controller supports 64 bit Descriptor. + * Perform PCI Write. AERO SERIES supports 32 bit Descriptor. + * Prior to AERO_SERIES support 64 bit Descriptor. */ - static void megasas_fire_cmd_fusion(struct megasas_instance *instance, union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc) { - if (instance->is_ventura) + if (instance->atomic_desc_support) writel(le32_to_cpu(req_desc->u.low), &instance->reg_set->inbound_single_queue_port); - else { -#if defined(writeq) && defined(CONFIG_64BIT) - u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) | - le32_to_cpu(req_desc->u.low)); - - writeq(req_data, &instance->reg_set->inbound_low_queue_port); -#else - unsigned long flags; - spin_lock_irqsave(&instance->hba_lock, flags); - writel(le32_to_cpu(req_desc->u.low), - &instance->reg_set->inbound_low_queue_port); - writel(le32_to_cpu(req_desc->u.high), - &instance->reg_set->inbound_high_queue_port); - mmiowb(); - spin_unlock_irqrestore(&instance->hba_lock, flags); -#endif - } + else + megasas_write_64bit_req_desc(instance, req_desc); } /** * megasas_fusion_update_can_queue - Do all Adapter Queue depth related calculations here - * @instance: Adapter soft state - * fw_boot_context: Whether this function called during probe or after OCR + * @instance: Adapter soft state + * @fw_boot_context: Whether this function called during probe or after OCR * * This function is only for fusion controllers. * Update host can queue, if firmware downgrade max supported firmware commands. - * Firmware upgrade case will be skiped because underlying firmware has + * Firmware upgrade case will be skipped because underlying firmware has * more resource than exposed to the OS. * */ @@ -238,24 +343,22 @@ megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_c { u16 cur_max_fw_cmds = 0; u16 ldio_threshold = 0; - struct megasas_register_set __iomem *reg_set; - reg_set = instance->reg_set; - - /* ventura FW does not fill outbound_scratch_pad_3 with queue depth */ - if (!instance->is_ventura) + /* ventura FW does not fill outbound_scratch_pad_2 with queue depth */ + if (instance->adapter_type < VENTURA_SERIES) cur_max_fw_cmds = - readl(&instance->reg_set->outbound_scratch_pad_3) & 0x00FFFF; + megasas_readl(instance, + &instance->reg_set->outbound_scratch_pad_2) & 0x00FFFF; if (dual_qdepth_disable || !cur_max_fw_cmds) - cur_max_fw_cmds = instance->instancet->read_fw_status_reg(reg_set) & 0x00FFFF; + cur_max_fw_cmds = instance->instancet->read_fw_status_reg(instance) & 0x00FFFF; else ldio_threshold = - (instance->instancet->read_fw_status_reg(reg_set) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS; + (instance->instancet->read_fw_status_reg(instance) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS; dev_info(&instance->pdev->dev, - "Current firmware maximum commands: %d\t LDIO threshold: %d\n", - cur_max_fw_cmds, ldio_threshold); + "Current firmware supports maximum commands: %d\t LDIO threshold: %d\n", + cur_max_fw_cmds, ldio_threshold); if (fw_boot_context == OCR_CONTEXT) { cur_max_fw_cmds = cur_max_fw_cmds - 1; @@ -270,10 +373,6 @@ megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_c instance->max_fw_cmds = cur_max_fw_cmds; instance->ldio_threshold = ldio_threshold; - if (!instance->is_rdpq) - instance->max_fw_cmds = - min_t(u16, instance->max_fw_cmds, 1024); - if (reset_devices) instance->max_fw_cmds = min(instance->max_fw_cmds, (u16)MEGASAS_KDUMP_QUEUE_DEPTH); @@ -283,20 +382,36 @@ megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_c * does not exceed max cmds that the FW can support */ instance->max_fw_cmds = instance->max_fw_cmds-1; - - instance->max_scsi_cmds = instance->max_fw_cmds - - (MEGASAS_FUSION_INTERNAL_CMDS + - MEGASAS_FUSION_IOCTL_CMDS); - instance->cur_can_queue = instance->max_scsi_cmds; - instance->host->can_queue = instance->cur_can_queue; } +} - if (instance->is_ventura) - instance->max_mpt_cmds = - instance->max_fw_cmds * RAID_1_PEER_CMDS; - else - instance->max_mpt_cmds = instance->max_fw_cmds; +static inline void +megasas_get_msix_index(struct megasas_instance *instance, + struct scsi_cmnd *scmd, + struct megasas_cmd_fusion *cmd, + u8 data_arms) +{ + if (instance->perf_mode == MR_BALANCED_PERF_MODE && + (megasas_sdev_busy_read(instance, scmd) > + (data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))) { + cmd->request_desc->SCSIIO.MSIxIndex = + mega_mod64((atomic64_add_return(1, &instance->high_iops_outstanding) / + MR_HIGH_IOPS_BATCH_COUNT), instance->low_latency_index_start); + } else if (instance->msix_load_balance) { + cmd->request_desc->SCSIIO.MSIxIndex = + (mega_mod64(atomic64_add_return(1, &instance->total_io_count), + instance->msix_vectors)); + } else if (instance->host->nr_hw_queues > 1) { + u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd)); + + cmd->request_desc->SCSIIO.MSIxIndex = blk_mq_unique_tag_to_hwq(tag) + + instance->low_latency_index_start; + } else { + cmd->request_desc->SCSIIO.MSIxIndex = + instance->reply_map[raw_smp_processor_id()]; + } } + /** * megasas_free_cmds_fusion - Free all the cmds in the free cmd pool * @instance: Adapter soft state @@ -308,25 +423,31 @@ megasas_free_cmds_fusion(struct megasas_instance *instance) struct fusion_context *fusion = instance->ctrl_context; struct megasas_cmd_fusion *cmd; - /* SG, Sense */ - for (i = 0; i < instance->max_mpt_cmds; i++) { - cmd = fusion->cmd_list[i]; - if (cmd) { - if (cmd->sg_frame) - pci_pool_free(fusion->sg_dma_pool, cmd->sg_frame, - cmd->sg_frame_phys_addr); - if (cmd->sense) - pci_pool_free(fusion->sense_dma_pool, cmd->sense, - cmd->sense_phys_addr); + if (fusion->sense) + dma_pool_free(fusion->sense_dma_pool, fusion->sense, + fusion->sense_phys_addr); + + /* SG */ + if (fusion->cmd_list) { + for (i = 0; i < instance->max_mpt_cmds; i++) { + cmd = fusion->cmd_list[i]; + if (cmd) { + if (cmd->sg_frame) + dma_pool_free(fusion->sg_dma_pool, + cmd->sg_frame, + cmd->sg_frame_phys_addr); + } + kfree(cmd); } + kfree(fusion->cmd_list); } if (fusion->sg_dma_pool) { - pci_pool_destroy(fusion->sg_dma_pool); + dma_pool_destroy(fusion->sg_dma_pool); fusion->sg_dma_pool = NULL; } if (fusion->sense_dma_pool) { - pci_pool_destroy(fusion->sense_dma_pool); + dma_pool_destroy(fusion->sense_dma_pool); fusion->sense_dma_pool = NULL; } @@ -343,20 +464,13 @@ megasas_free_cmds_fusion(struct megasas_instance *instance) fusion->request_alloc_sz, fusion->req_frames_desc, fusion->req_frames_desc_phys); if (fusion->io_request_frames) - pci_pool_free(fusion->io_request_frames_pool, + dma_pool_free(fusion->io_request_frames_pool, fusion->io_request_frames, fusion->io_request_frames_phys); if (fusion->io_request_frames_pool) { - pci_pool_destroy(fusion->io_request_frames_pool); + dma_pool_destroy(fusion->io_request_frames_pool); fusion->io_request_frames_pool = NULL; } - - - /* cmd_list */ - for (i = 0; i < instance->max_mpt_cmds; i++) - kfree(fusion->cmd_list[i]); - - kfree(fusion->cmd_list); } /** @@ -370,19 +484,21 @@ static int megasas_create_sg_sense_fusion(struct megasas_instance *instance) u16 max_cmd; struct fusion_context *fusion; struct megasas_cmd_fusion *cmd; + int sense_sz; + u32 offset; fusion = instance->ctrl_context; max_cmd = instance->max_fw_cmds; - + sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE; fusion->sg_dma_pool = - pci_pool_create("mr_sg", instance->pdev, + dma_pool_create("mr_sg", &instance->pdev->dev, instance->max_chain_frame_sz, MR_DEFAULT_NVME_PAGE_SIZE, 0); /* SCSI_SENSE_BUFFERSIZE = 96 bytes */ fusion->sense_dma_pool = - pci_pool_create("mr_sense", instance->pdev, - SCSI_SENSE_BUFFERSIZE, 64, 0); + dma_pool_create("mr_sense", &instance->pdev->dev, + sense_sz, 64, 0); if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) { dev_err(&instance->pdev->dev, @@ -390,17 +506,64 @@ static int megasas_create_sg_sense_fusion(struct megasas_instance *instance) return -ENOMEM; } + fusion->sense = dma_pool_alloc(fusion->sense_dma_pool, + GFP_KERNEL, &fusion->sense_phys_addr); + if (!fusion->sense) { + dev_err(&instance->pdev->dev, + "failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + + /* sense buffer, request frame and reply desc pool requires to be in + * same 4 gb region. Below function will check this. + * In case of failure, new pci pool will be created with updated + * alignment. + * Older allocation and pool will be destroyed. + * Alignment will be used such a way that next allocation if success, + * will always meet same 4gb region requirement. + * Actual requirement is not alignment, but we need start and end of + * DMA address must have same upper 32 bit address. + */ + + if (!megasas_check_same_4gb_region(instance, fusion->sense_phys_addr, + sense_sz)) { + dma_pool_free(fusion->sense_dma_pool, fusion->sense, + fusion->sense_phys_addr); + fusion->sense = NULL; + dma_pool_destroy(fusion->sense_dma_pool); + + fusion->sense_dma_pool = + dma_pool_create("mr_sense_align", &instance->pdev->dev, + sense_sz, roundup_pow_of_two(sense_sz), + 0); + if (!fusion->sense_dma_pool) { + dev_err(&instance->pdev->dev, + "Failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + fusion->sense = dma_pool_alloc(fusion->sense_dma_pool, + GFP_KERNEL, + &fusion->sense_phys_addr); + if (!fusion->sense) { + dev_err(&instance->pdev->dev, + "failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + } + /* * Allocate and attach a frame to each of the commands in cmd_list */ for (i = 0; i < max_cmd; i++) { cmd = fusion->cmd_list[i]; - cmd->sg_frame = pci_pool_alloc(fusion->sg_dma_pool, + cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool, GFP_KERNEL, &cmd->sg_frame_phys_addr); - cmd->sense = pci_pool_alloc(fusion->sense_dma_pool, - GFP_KERNEL, &cmd->sense_phys_addr); - if (!cmd->sg_frame || !cmd->sense) { + offset = SCSI_SENSE_BUFFERSIZE * i; + cmd->sense = (u8 *)fusion->sense + offset; + cmd->sense_phys_addr = fusion->sense_phys_addr + offset; + + if (!cmd->sg_frame) { dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); return -ENOMEM; @@ -410,19 +573,16 @@ static int megasas_create_sg_sense_fusion(struct megasas_instance *instance) /* create sense buffer for the raid 1/10 fp */ for (i = max_cmd; i < instance->max_mpt_cmds; i++) { cmd = fusion->cmd_list[i]; - cmd->sense = pci_pool_alloc(fusion->sense_dma_pool, - GFP_KERNEL, &cmd->sense_phys_addr); - if (!cmd->sense) { - dev_err(&instance->pdev->dev, - "Failed from %s %d\n", __func__, __LINE__); - return -ENOMEM; - } + offset = SCSI_SENSE_BUFFERSIZE * i; + cmd->sense = (u8 *)fusion->sense + offset; + cmd->sense_phys_addr = fusion->sense_phys_addr + offset; + } return 0; } -int +static int megasas_alloc_cmdlist_fusion(struct megasas_instance *instance) { u32 max_mpt_cmd, i, j; @@ -438,7 +598,7 @@ megasas_alloc_cmdlist_fusion(struct megasas_instance *instance) * commands. */ fusion->cmd_list = - kzalloc(sizeof(struct megasas_cmd_fusion *) * max_mpt_cmd, + kcalloc(max_mpt_cmd, sizeof(struct megasas_cmd_fusion *), GFP_KERNEL); if (!fusion->cmd_list) { dev_err(&instance->pdev->dev, @@ -461,25 +621,17 @@ megasas_alloc_cmdlist_fusion(struct megasas_instance *instance) return 0; } -int + +static int megasas_alloc_request_fusion(struct megasas_instance *instance) { struct fusion_context *fusion; fusion = instance->ctrl_context; - fusion->req_frames_desc = - dma_alloc_coherent(&instance->pdev->dev, - fusion->request_alloc_sz, - &fusion->req_frames_desc_phys, GFP_KERNEL); - if (!fusion->req_frames_desc) { - dev_err(&instance->pdev->dev, - "Failed from %s %d\n", __func__, __LINE__); - return -ENOMEM; - } - +retry_alloc: fusion->io_request_frames_pool = - pci_pool_create("mr_ioreq", instance->pdev, + dma_pool_create("mr_ioreq", &instance->pdev->dev, fusion->io_frames_alloc_sz, 16, 0); if (!fusion->io_request_frames_pool) { @@ -489,17 +641,70 @@ megasas_alloc_request_fusion(struct megasas_instance *instance) } fusion->io_request_frames = - pci_pool_alloc(fusion->io_request_frames_pool, - GFP_KERNEL, &fusion->io_request_frames_phys); + dma_pool_alloc(fusion->io_request_frames_pool, + GFP_KERNEL | __GFP_NOWARN, + &fusion->io_request_frames_phys); if (!fusion->io_request_frames) { + if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) { + instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT; + dma_pool_destroy(fusion->io_request_frames_pool); + megasas_configure_queue_sizes(instance); + goto retry_alloc; + } else { + dev_err(&instance->pdev->dev, + "Failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + } + + if (!megasas_check_same_4gb_region(instance, + fusion->io_request_frames_phys, + fusion->io_frames_alloc_sz)) { + dma_pool_free(fusion->io_request_frames_pool, + fusion->io_request_frames, + fusion->io_request_frames_phys); + fusion->io_request_frames = NULL; + dma_pool_destroy(fusion->io_request_frames_pool); + + fusion->io_request_frames_pool = + dma_pool_create("mr_ioreq_align", + &instance->pdev->dev, + fusion->io_frames_alloc_sz, + roundup_pow_of_two(fusion->io_frames_alloc_sz), + 0); + + if (!fusion->io_request_frames_pool) { + dev_err(&instance->pdev->dev, + "Failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + + fusion->io_request_frames = + dma_pool_alloc(fusion->io_request_frames_pool, + GFP_KERNEL | __GFP_NOWARN, + &fusion->io_request_frames_phys); + + if (!fusion->io_request_frames) { + dev_err(&instance->pdev->dev, + "Failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + } + + fusion->req_frames_desc = + dma_alloc_coherent(&instance->pdev->dev, + fusion->request_alloc_sz, + &fusion->req_frames_desc_phys, GFP_KERNEL); + if (!fusion->req_frames_desc) { dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); return -ENOMEM; } + return 0; } -int +static int megasas_alloc_reply_fusion(struct megasas_instance *instance) { int i, count; @@ -508,8 +713,10 @@ megasas_alloc_reply_fusion(struct megasas_instance *instance) fusion = instance->ctrl_context; count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; + count += instance->iopoll_q_count; + fusion->reply_frames_desc_pool = - pci_pool_create("mr_reply", instance->pdev, + dma_pool_create("mr_reply", &instance->pdev->dev, fusion->reply_alloc_sz * count, 16, 0); if (!fusion->reply_frames_desc_pool) { @@ -519,13 +726,48 @@ megasas_alloc_reply_fusion(struct megasas_instance *instance) } fusion->reply_frames_desc[0] = - pci_pool_alloc(fusion->reply_frames_desc_pool, + dma_pool_alloc(fusion->reply_frames_desc_pool, GFP_KERNEL, &fusion->reply_frames_desc_phys[0]); if (!fusion->reply_frames_desc[0]) { dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); return -ENOMEM; } + + if (!megasas_check_same_4gb_region(instance, + fusion->reply_frames_desc_phys[0], + (fusion->reply_alloc_sz * count))) { + dma_pool_free(fusion->reply_frames_desc_pool, + fusion->reply_frames_desc[0], + fusion->reply_frames_desc_phys[0]); + fusion->reply_frames_desc[0] = NULL; + dma_pool_destroy(fusion->reply_frames_desc_pool); + + fusion->reply_frames_desc_pool = + dma_pool_create("mr_reply_align", + &instance->pdev->dev, + fusion->reply_alloc_sz * count, + roundup_pow_of_two(fusion->reply_alloc_sz * count), + 0); + + if (!fusion->reply_frames_desc_pool) { + dev_err(&instance->pdev->dev, + "Failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + + fusion->reply_frames_desc[0] = + dma_pool_alloc(fusion->reply_frames_desc_pool, + GFP_KERNEL, + &fusion->reply_frames_desc_phys[0]); + + if (!fusion->reply_frames_desc[0]) { + dev_err(&instance->pdev->dev, + "Failed from %s %d\n", __func__, __LINE__); + return -ENOMEM; + } + } + reply_desc = fusion->reply_frames_desc[0]; for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++) reply_desc->Words = cpu_to_le64(ULLONG_MAX); @@ -541,53 +783,128 @@ megasas_alloc_reply_fusion(struct megasas_instance *instance) return 0; } -int +static int megasas_alloc_rdpq_fusion(struct megasas_instance *instance) { - int i, j, count; + int i, j, k, msix_count; struct fusion_context *fusion; union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc; + union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT]; + dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT]; + u8 dma_alloc_count, abs_index; + u32 chunk_size, array_size, offset; fusion = instance->ctrl_context; + chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK; + array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * + MAX_MSIX_QUEUES_FUSION; - fusion->rdpq_virt = pci_alloc_consistent(instance->pdev, - sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION, - &fusion->rdpq_phys); + fusion->rdpq_virt = dma_alloc_coherent(&instance->pdev->dev, + array_size, &fusion->rdpq_phys, + GFP_KERNEL); if (!fusion->rdpq_virt) { dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); return -ENOMEM; } - memset(fusion->rdpq_virt, 0, - sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION); - count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; - fusion->reply_frames_desc_pool = pci_pool_create("mr_rdpq", - instance->pdev, fusion->reply_alloc_sz, 16, 0); - - if (!fusion->reply_frames_desc_pool) { + msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; + msix_count += instance->iopoll_q_count; + + fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq", + &instance->pdev->dev, + chunk_size, 16, 0); + fusion->reply_frames_desc_pool_align = + dma_pool_create("mr_rdpq_align", + &instance->pdev->dev, + chunk_size, + roundup_pow_of_two(chunk_size), + 0); + + if (!fusion->reply_frames_desc_pool || + !fusion->reply_frames_desc_pool_align) { dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); return -ENOMEM; } - for (i = 0; i < count; i++) { - fusion->reply_frames_desc[i] = - pci_pool_alloc(fusion->reply_frames_desc_pool, - GFP_KERNEL, &fusion->reply_frames_desc_phys[i]); - if (!fusion->reply_frames_desc[i]) { +/* + * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and + * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should be + * within 4GB boundary and also reply queues in a set must have same + * upper 32-bits in their memory address. so here driver is allocating the + * DMA'able memory for reply queues according. Driver uses limitation of + * VENTURA_SERIES to manage INVADER_SERIES as well. + */ + dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK); + + for (i = 0; i < dma_alloc_count; i++) { + rdpq_chunk_virt[i] = + dma_pool_alloc(fusion->reply_frames_desc_pool, + GFP_KERNEL, &rdpq_chunk_phys[i]); + if (!rdpq_chunk_virt[i]) { dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); return -ENOMEM; } + /* reply desc pool requires to be in same 4 gb region. + * Below function will check this. + * In case of failure, new pci pool will be created with updated + * alignment. + * For RDPQ buffers, driver always allocate two separate pci pool. + * Alignment will be used such a way that next allocation if + * success, will always meet same 4gb region requirement. + * rdpq_tracker keep track of each buffer's physical, + * virtual address and pci pool descriptor. It will help driver + * while freeing the resources. + * + */ + if (!megasas_check_same_4gb_region(instance, rdpq_chunk_phys[i], + chunk_size)) { + dma_pool_free(fusion->reply_frames_desc_pool, + rdpq_chunk_virt[i], + rdpq_chunk_phys[i]); + + rdpq_chunk_virt[i] = + dma_pool_alloc(fusion->reply_frames_desc_pool_align, + GFP_KERNEL, &rdpq_chunk_phys[i]); + if (!rdpq_chunk_virt[i]) { + dev_err(&instance->pdev->dev, + "Failed from %s %d\n", + __func__, __LINE__); + return -ENOMEM; + } + fusion->rdpq_tracker[i].dma_pool_ptr = + fusion->reply_frames_desc_pool_align; + } else { + fusion->rdpq_tracker[i].dma_pool_ptr = + fusion->reply_frames_desc_pool; + } - fusion->rdpq_virt[i].RDPQBaseAddress = - cpu_to_le64(fusion->reply_frames_desc_phys[i]); + fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i]; + fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i]; + } - reply_desc = fusion->reply_frames_desc[i]; - for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++) - reply_desc->Words = cpu_to_le64(ULLONG_MAX); + for (k = 0; k < dma_alloc_count; k++) { + for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) { + abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i; + + if (abs_index == msix_count) + break; + offset = fusion->reply_alloc_sz * i; + fusion->rdpq_virt[abs_index].RDPQBaseAddress = + cpu_to_le64(rdpq_chunk_phys[k] + offset); + fusion->reply_frames_desc_phys[abs_index] = + rdpq_chunk_phys[k] + offset; + fusion->reply_frames_desc[abs_index] = + (union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset); + + reply_desc = fusion->reply_frames_desc[abs_index]; + for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++) + reply_desc->Words = ULLONG_MAX; + } } + return 0; } @@ -599,18 +916,19 @@ megasas_free_rdpq_fusion(struct megasas_instance *instance) { fusion = instance->ctrl_context; - for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++) { - if (fusion->reply_frames_desc[i]) - pci_pool_free(fusion->reply_frames_desc_pool, - fusion->reply_frames_desc[i], - fusion->reply_frames_desc_phys[i]); + for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) { + if (fusion->rdpq_tracker[i].pool_entry_virt) + dma_pool_free(fusion->rdpq_tracker[i].dma_pool_ptr, + fusion->rdpq_tracker[i].pool_entry_virt, + fusion->rdpq_tracker[i].pool_entry_phys); + } - if (fusion->reply_frames_desc_pool) - pci_pool_destroy(fusion->reply_frames_desc_pool); + dma_pool_destroy(fusion->reply_frames_desc_pool); + dma_pool_destroy(fusion->reply_frames_desc_pool_align); if (fusion->rdpq_virt) - pci_free_consistent(instance->pdev, + dma_free_coherent(&instance->pdev->dev, sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION, fusion->rdpq_virt, fusion->rdpq_phys); } @@ -623,12 +941,11 @@ megasas_free_reply_fusion(struct megasas_instance *instance) { fusion = instance->ctrl_context; if (fusion->reply_frames_desc[0]) - pci_pool_free(fusion->reply_frames_desc_pool, + dma_pool_free(fusion->reply_frames_desc_pool, fusion->reply_frames_desc[0], fusion->reply_frames_desc_phys[0]); - if (fusion->reply_frames_desc_pool) - pci_pool_destroy(fusion->reply_frames_desc_pool); + dma_pool_destroy(fusion->reply_frames_desc_pool); } @@ -649,7 +966,7 @@ megasas_free_reply_fusion(struct megasas_instance *instance) { * and is used as SMID of the cmd. * SMID value range is from 1 to max_fw_cmds. */ -int +static int megasas_alloc_cmds_fusion(struct megasas_instance *instance) { int i; @@ -662,9 +979,6 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance) fusion = instance->ctrl_context; - if (megasas_alloc_cmdlist_fusion(instance)) - goto fail_exit; - if (megasas_alloc_request_fusion(instance)) goto fail_exit; @@ -675,6 +989,8 @@ megasas_alloc_cmds_fusion(struct megasas_instance *instance) if (megasas_alloc_reply_fusion(instance)) goto fail_exit; + if (megasas_alloc_cmdlist_fusion(instance)) + goto fail_exit; /* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */ io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE; @@ -719,6 +1035,7 @@ fail_exit: * wait_and_poll - Issues a polling command * @instance: Adapter soft state * @cmd: Command packet to be issued + * @seconds: Maximum poll time * * For polling, MFI requires the cmd_status to be set to 0xFF before posting. */ @@ -728,17 +1045,22 @@ wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd, { int i; struct megasas_header *frame_hdr = &cmd->frame->hdr; - struct fusion_context *fusion; + u32 status_reg; u32 msecs = seconds * 1000; - fusion = instance->ctrl_context; /* * Wait for cmd_status to change */ for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) { rmb(); msleep(20); + if (!(i % 5000)) { + status_reg = instance->instancet->read_fw_status_reg(instance) + & MFI_STATE_MASK; + if (status_reg == MFI_STATE_FAULT) + break; + } } if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS) @@ -769,23 +1091,35 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) struct megasas_header *frame_hdr; const char *sys_info; MFI_CAPABILITIES *drv_ops; - u32 scratch_pad_2; - unsigned long flags; + u32 scratch_pad_1; + ktime_t time; + bool cur_fw_64bit_dma_capable; + bool cur_intr_coalescing; fusion = instance->ctrl_context; - cmd = megasas_get_cmd(instance); + ioc_init_handle = fusion->ioc_init_request_phys; + IOCInitMessage = fusion->ioc_init_request; - if (!cmd) { - dev_err(&instance->pdev->dev, "Could not allocate cmd for INIT Frame\n"); - ret = 1; - goto fail_get_cmd; - } + cmd = fusion->ioc_init_cmd; + + scratch_pad_1 = megasas_readl + (instance, &instance->reg_set->outbound_scratch_pad_1); - scratch_pad_2 = readl - (&instance->reg_set->outbound_scratch_pad_2); + cur_rdpq_mode = (scratch_pad_1 & MR_RDPQ_MODE_OFFSET) ? 1 : 0; - cur_rdpq_mode = (scratch_pad_2 & MR_RDPQ_MODE_OFFSET) ? 1 : 0; + if (instance->adapter_type == INVADER_SERIES) { + cur_fw_64bit_dma_capable = + (scratch_pad_1 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false; + + if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) { + dev_err(&instance->pdev->dev, "Driver was operating on 64bit " + "DMA mask, but upcoming FW does not support 64bit DMA mask\n"); + megaraid_sas_kill_hba(instance); + ret = 1; + goto fail_fw_init; + } + } if (instance->is_rdpq && !cur_rdpq_mode) { dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*" @@ -794,23 +1128,22 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) goto fail_fw_init; } - instance->fw_sync_cache_support = (scratch_pad_2 & + cur_intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ? + true : false; + + if ((instance->low_latency_index_start == + MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing) + instance->perf_mode = MR_BALANCED_PERF_MODE; + + dev_info(&instance->pdev->dev, "Performance mode :%s (latency index = %d)\n", + MEGASAS_PERF_MODE_2STR(instance->perf_mode), + instance->low_latency_index_start); + + instance->fw_sync_cache_support = (scratch_pad_1 & MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0; dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n", instance->fw_sync_cache_support ? "Yes" : "No"); - IOCInitMessage = - dma_alloc_coherent(&instance->pdev->dev, - sizeof(struct MPI2_IOC_INIT_REQUEST), - &ioc_init_handle, GFP_KERNEL); - - if (!IOCInitMessage) { - dev_err(&instance->pdev->dev, "Could not allocate memory for " - "IOCInitMessage\n"); - ret = 1; - goto fail_fw_init; - } - memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST)); IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT; @@ -826,16 +1159,20 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) IOCInitMessage->MsgFlags = instance->is_rdpq ? MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0; IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys); - IOCInitMessage->HostMSIxVectors = instance->msix_vectors; + IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr)); + IOCInitMessage->HostMSIxVectors = instance->msix_vectors + instance->iopoll_q_count; IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT; + + time = ktime_get_real(); + /* Convert to milliseconds as per FW requirement */ + IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time)); + init_frame = (struct megasas_init_frame *)cmd->frame; - memset(init_frame, 0, MEGAMFI_FRAME_SIZE); + memset(init_frame, 0, IOC_INIT_FRAME_SIZE); frame_hdr = &cmd->frame->hdr; frame_hdr->cmd_status = 0xFF; - frame_hdr->flags = cpu_to_le16( - le16_to_cpu(frame_hdr->flags) | - MFI_FRAME_DONT_POST_IN_REPLY_QUEUE); + frame_hdr->flags |= cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE); init_frame->cmd = MFI_CMD_INIT; init_frame->cmd_status = 0xFF; @@ -843,7 +1180,7 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations); /* driver support Extended MSIX */ - if (fusion->adapter_type >= INVADER_SERIES) + if (instance->adapter_type >= INVADER_SERIES) drv_ops->mfi_capabilities.support_additional_msix = 1; /* driver supports HA / Remote LUN over Fast Path interface */ drv_ops->mfi_capabilities.support_fp_remote_lun = 1; @@ -861,6 +1198,15 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) drv_ops->mfi_capabilities.support_qd_throttling = 1; drv_ops->mfi_capabilities.support_pd_map_target_id = 1; + drv_ops->mfi_capabilities.support_nvme_passthru = 1; + drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1; + + if (reset_devices) + drv_ops->mfi_capabilities.support_memdump = 1; + + if (instance->consistent_mask_64bit) + drv_ops->mfi_capabilities.support_64bit_mode = 1; + /* Convert capability to LE32 */ cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities); @@ -870,8 +1216,8 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) strlen(sys_info) > 64 ? 64 : strlen(sys_info)); instance->system_info_buf->systemIdLength = strlen(sys_info) > 64 ? 64 : strlen(sys_info); - init_frame->system_info_lo = instance->system_info_h; - init_frame->system_info_hi = 0; + init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h)); + init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h)); } init_frame->queue_info_new_phys_addr_hi = @@ -880,6 +1226,22 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) cpu_to_le32(lower_32_bits(ioc_init_handle)); init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST)); + /* + * Each bit in replyqueue_mask represents one group of MSI-x vectors + * (each group has 8 vectors) + */ + switch (instance->perf_mode) { + case MR_BALANCED_PERF_MODE: + init_frame->replyqueue_mask = + cpu_to_le16(~(~0 << instance->low_latency_index_start/8)); + break; + case MR_IOPS_PERF_MODE: + init_frame->replyqueue_mask = + cpu_to_le16(~(~0 << instance->msix_vectors/8)); + break; + } + + req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr)); req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr)); req_desc.MFAIo.RequestFlags = @@ -892,39 +1254,41 @@ megasas_ioc_init_fusion(struct megasas_instance *instance) instance->instancet->disable_intr(instance); for (i = 0; i < (10 * 1000); i += 20) { - if (readl(&instance->reg_set->doorbell) & 1) + if (megasas_readl(instance, &instance->reg_set->doorbell) & 1) msleep(20); else break; } - /* For Ventura also IOC INIT required 64 bit Descriptor write. */ - spin_lock_irqsave(&instance->hba_lock, flags); - writel(le32_to_cpu(req_desc.u.low), - &instance->reg_set->inbound_low_queue_port); - writel(le32_to_cpu(req_desc.u.high), - &instance->reg_set->inbound_high_queue_port); - mmiowb(); - spin_unlock_irqrestore(&instance->hba_lock, flags); + /* For AERO also, IOC_INIT requires 64 bit descriptor write */ + megasas_write_64bit_req_desc(instance, &req_desc); - wait_and_poll(instance, cmd, MFI_POLL_TIMEOUT_SECS); + wait_and_poll(instance, cmd, MFI_IO_TIMEOUT_SECS); frame_hdr = &cmd->frame->hdr; if (frame_hdr->cmd_status != 0) { ret = 1; goto fail_fw_init; } - dev_info(&instance->pdev->dev, "Init cmd success\n"); - ret = 0; + if (instance->adapter_type >= AERO_SERIES) { + scratch_pad_1 = megasas_readl + (instance, &instance->reg_set->outbound_scratch_pad_1); + + instance->atomic_desc_support = + (scratch_pad_1 & MR_ATOMIC_DESCRIPTOR_SUPPORT_OFFSET) ? 1 : 0; + + dev_info(&instance->pdev->dev, "FW supports atomic descriptor\t: %s\n", + instance->atomic_desc_support ? "Yes" : "No"); + } + + return 0; fail_fw_init: - megasas_return_cmd(instance, cmd); - if (IOCInitMessage) - dma_free_coherent(&instance->pdev->dev, - sizeof(struct MPI2_IOC_INIT_REQUEST), - IOCInitMessage, ioc_init_handle); -fail_get_cmd: + dev_err(&instance->pdev->dev, + "Init cmd return status FAILED for SCSI host %d\n", + instance->host->host_no); + return ret; } @@ -940,7 +1304,7 @@ fail_get_cmd: int megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) { int ret = 0; - u32 pd_seq_map_sz; + size_t pd_seq_map_sz; struct megasas_cmd *cmd; struct megasas_dcmd_frame *dcmd; struct fusion_context *fusion = instance->ctrl_context; @@ -949,9 +1313,7 @@ megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) { pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)]; pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)]; - pd_seq_map_sz = sizeof(struct MR_PD_CFG_SEQ_NUM_SYNC) + - (sizeof(struct MR_PD_CFG_SEQ) * - (MAX_PHYSICAL_DEVICES - 1)); + pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES); cmd = megasas_get_cmd(instance); if (!cmd) { @@ -965,6 +1327,15 @@ megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) { memset(pd_sync, 0, pd_seq_map_sz); memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE); + + if (pend) { + dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG; + dcmd->flags = MFI_FRAME_DIR_WRITE; + instance->jbod_seq_cmd = cmd; + } else { + dcmd->flags = MFI_FRAME_DIR_READ; + } + dcmd->cmd = MFI_CMD_DCMD; dcmd->cmd_status = 0xFF; dcmd->sge_count = 1; @@ -972,21 +1343,16 @@ megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) { dcmd->pad_0 = 0; dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz); dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO); - dcmd->sgl.sge32[0].phys_addr = cpu_to_le32(pd_seq_h); - dcmd->sgl.sge32[0].length = cpu_to_le32(pd_seq_map_sz); + + megasas_set_dma_settings(instance, dcmd, pd_seq_h, pd_seq_map_sz); if (pend) { - dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG; - dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_WRITE); - instance->jbod_seq_cmd = cmd; instance->instancet->issue_dcmd(instance, cmd); return 0; } - dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_READ); - /* Below code is only for non pended DCMD */ - if (instance->ctrl_context && !instance->mask_interrupts) + if (!instance->mask_interrupts) ret = megasas_issue_blocked_cmd(instance, cmd, MFI_IO_TIMEOUT_SECS); else @@ -999,8 +1365,10 @@ megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) { ret = -EINVAL; } - if (ret == DCMD_TIMEOUT && instance->ctrl_context) - megaraid_sas_kill_hba(instance); + if (ret == DCMD_TIMEOUT) + dev_warn(&instance->pdev->dev, + "%s DCMD timed out, continue without JBOD sequence map\n", + __func__); if (ret == DCMD_SUCCESS) instance->pd_seq_map_id++; @@ -1067,22 +1435,24 @@ megasas_get_ld_map_info(struct megasas_instance *instance) dcmd->cmd = MFI_CMD_DCMD; dcmd->cmd_status = 0xFF; dcmd->sge_count = 1; - dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_READ); + dcmd->flags = MFI_FRAME_DIR_READ; dcmd->timeout = 0; dcmd->pad_0 = 0; dcmd->data_xfer_len = cpu_to_le32(size_map_info); dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO); - dcmd->sgl.sge32[0].phys_addr = cpu_to_le32(ci_h); - dcmd->sgl.sge32[0].length = cpu_to_le32(size_map_info); - if (instance->ctrl_context && !instance->mask_interrupts) + megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info); + + if (!instance->mask_interrupts) ret = megasas_issue_blocked_cmd(instance, cmd, MFI_IO_TIMEOUT_SECS); else ret = megasas_issue_polled(instance, cmd); - if (ret == DCMD_TIMEOUT && instance->ctrl_context) - megaraid_sas_kill_hba(instance); + if (ret == DCMD_TIMEOUT) + dev_warn(&instance->pdev->dev, + "%s DCMD timed out, RAID map is disabled\n", + __func__); megasas_return_cmd(instance, cmd); @@ -1096,7 +1466,7 @@ megasas_get_map_info(struct megasas_instance *instance) fusion->fast_path_io = 0; if (!megasas_get_ld_map_info(instance)) { - if (MR_ValidateMapInfo(instance)) { + if (MR_ValidateMapInfo(instance, instance->map_id)) { fusion->fast_path_io = 1; return 0; } @@ -1119,7 +1489,6 @@ megasas_sync_map_info(struct megasas_instance *instance) struct megasas_cmd *cmd; struct megasas_dcmd_frame *dcmd; u16 num_lds; - u32 size_sync_info; struct fusion_context *fusion; struct MR_LD_TARGET_SYNC *ci = NULL; struct MR_DRV_RAID_MAP_ALL *map; @@ -1148,8 +1517,6 @@ megasas_sync_map_info(struct megasas_instance *instance) dcmd = &cmd->frame->dcmd; - size_sync_info = sizeof(struct MR_LD_TARGET_SYNC) *num_lds; - memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE); ci = (struct MR_LD_TARGET_SYNC *) @@ -1171,15 +1538,15 @@ megasas_sync_map_info(struct megasas_instance *instance) dcmd->cmd = MFI_CMD_DCMD; dcmd->cmd_status = 0xFF; dcmd->sge_count = 1; - dcmd->flags = cpu_to_le16(MFI_FRAME_DIR_WRITE); + dcmd->flags = MFI_FRAME_DIR_WRITE; dcmd->timeout = 0; dcmd->pad_0 = 0; dcmd->data_xfer_len = cpu_to_le32(size_map_info); dcmd->mbox.b[0] = num_lds; dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG; dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO); - dcmd->sgl.sge32[0].phys_addr = cpu_to_le32(ci_h); - dcmd->sgl.sge32[0].length = cpu_to_le32(size_map_info); + + megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info); instance->map_update_cmd = cmd; @@ -1261,24 +1628,180 @@ megasas_display_intel_branding(struct megasas_instance *instance) } /** + * megasas_allocate_raid_maps - Allocate memory for RAID maps + * @instance: Adapter soft state + * + * return: if success: return 0 + * failed: return -ENOMEM + */ +static inline int megasas_allocate_raid_maps(struct megasas_instance *instance) +{ + struct fusion_context *fusion; + int i = 0; + + fusion = instance->ctrl_context; + + fusion->drv_map_pages = get_order(fusion->drv_map_sz); + + for (i = 0; i < 2; i++) { + fusion->ld_map[i] = NULL; + + fusion->ld_drv_map[i] = (void *) + __get_free_pages(__GFP_ZERO | GFP_KERNEL, + fusion->drv_map_pages); + + if (!fusion->ld_drv_map[i]) { + fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz); + + if (!fusion->ld_drv_map[i]) { + dev_err(&instance->pdev->dev, + "Could not allocate memory for local map" + " size requested: %d\n", + fusion->drv_map_sz); + goto ld_drv_map_alloc_fail; + } + } + } + + for (i = 0; i < 2; i++) { + fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev, + fusion->max_map_sz, + &fusion->ld_map_phys[i], + GFP_KERNEL); + if (!fusion->ld_map[i]) { + dev_err(&instance->pdev->dev, + "Could not allocate memory for map info %s:%d\n", + __func__, __LINE__); + goto ld_map_alloc_fail; + } + } + + return 0; + +ld_map_alloc_fail: + for (i = 0; i < 2; i++) { + if (fusion->ld_map[i]) + dma_free_coherent(&instance->pdev->dev, + fusion->max_map_sz, + fusion->ld_map[i], + fusion->ld_map_phys[i]); + } + +ld_drv_map_alloc_fail: + for (i = 0; i < 2; i++) { + if (fusion->ld_drv_map[i]) { + if (is_vmalloc_addr(fusion->ld_drv_map[i])) + vfree(fusion->ld_drv_map[i]); + else + free_pages((ulong)fusion->ld_drv_map[i], + fusion->drv_map_pages); + } + } + + return -ENOMEM; +} + +/** + * megasas_configure_queue_sizes - Calculate size of request desc queue, + * reply desc queue, + * IO request frame queue, set can_queue. + * @instance: Adapter soft state + * @return: void + */ +static inline +void megasas_configure_queue_sizes(struct megasas_instance *instance) +{ + struct fusion_context *fusion; + u16 max_cmd; + + fusion = instance->ctrl_context; + max_cmd = instance->max_fw_cmds; + + if (instance->adapter_type >= VENTURA_SERIES) + instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS; + else + instance->max_mpt_cmds = instance->max_fw_cmds; + + instance->max_scsi_cmds = instance->max_fw_cmds - instance->max_mfi_cmds; + instance->cur_can_queue = instance->max_scsi_cmds; + instance->host->can_queue = instance->cur_can_queue; + + fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16; + + fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * + instance->max_mpt_cmds; + fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) * + (fusion->reply_q_depth); + fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE + + (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE + * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */ +} + +static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance) +{ + struct fusion_context *fusion; + struct megasas_cmd *cmd; + + fusion = instance->ctrl_context; + + cmd = kzalloc(sizeof(struct megasas_cmd), GFP_KERNEL); + + if (!cmd) { + dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n", + __func__, __LINE__); + return -ENOMEM; + } + + cmd->frame = dma_alloc_coherent(&instance->pdev->dev, + IOC_INIT_FRAME_SIZE, + &cmd->frame_phys_addr, GFP_KERNEL); + + if (!cmd->frame) { + dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n", + __func__, __LINE__); + kfree(cmd); + return -ENOMEM; + } + + fusion->ioc_init_cmd = cmd; + return 0; +} + +/** + * megasas_free_ioc_init_cmd - Free IOC INIT command frame + * @instance: Adapter soft state + */ +static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance) +{ + struct fusion_context *fusion; + + fusion = instance->ctrl_context; + + if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame) + dma_free_coherent(&instance->pdev->dev, + IOC_INIT_FRAME_SIZE, + fusion->ioc_init_cmd->frame, + fusion->ioc_init_cmd->frame_phys_addr); + + kfree(fusion->ioc_init_cmd); +} + +/** * megasas_init_adapter_fusion - Initializes the FW * @instance: Adapter soft state * * This is the main function for initializing firmware. */ -u32 +static u32 megasas_init_adapter_fusion(struct megasas_instance *instance) { - struct megasas_register_set __iomem *reg_set; struct fusion_context *fusion; - u16 max_cmd; - u32 scratch_pad_2; + u32 scratch_pad_1; int i = 0, count; + u32 status_reg; fusion = instance->ctrl_context; - reg_set = instance->reg_set; - megasas_fusion_update_can_queue(instance, PROBE_CONTEXT); /* @@ -1287,32 +1810,23 @@ megasas_init_adapter_fusion(struct megasas_instance *instance) instance->max_mfi_cmds = MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS; - max_cmd = instance->max_fw_cmds; - - fusion->reply_q_depth = 2 * (((max_cmd + 1 + 15)/16)*16); - - fusion->request_alloc_sz = - sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * instance->max_mpt_cmds; - fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) - *(fusion->reply_q_depth); - fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE + - (MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE - * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */ + megasas_configure_queue_sizes(instance); - scratch_pad_2 = readl(&instance->reg_set->outbound_scratch_pad_2); - /* If scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set, + scratch_pad_1 = megasas_readl(instance, + &instance->reg_set->outbound_scratch_pad_1); + /* If scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set, * Firmware support extended IO chain frame which is 4 times more than * legacy Firmware. * Legacy Firmware - Frame size is (8 * 128) = 1K * 1M IO Firmware - Frame size is (8 * 128 * 4) = 4K */ - if (scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK) + if (scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK) instance->max_chain_frame_sz = - ((scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_MASK) >> + ((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >> MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO; else instance->max_chain_frame_sz = - ((scratch_pad_2 & MEGASAS_MAX_CHAIN_SIZE_MASK) >> + ((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >> MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO; if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) { @@ -1343,6 +1857,8 @@ megasas_init_adapter_fusion(struct megasas_instance *instance) sizeof(union MPI2_SGE_IO_UNION))/16; count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; + count += instance->iopoll_q_count; + for (i = 0 ; i < count; i++) fusion->last_reply_idx[i] = 0; @@ -1355,6 +1871,12 @@ megasas_init_adapter_fusion(struct megasas_instance *instance) MEGASAS_FUSION_IOCTL_CMDS); sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS); + for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++) + atomic_set(&fusion->busy_mq_poll[i], 0); + + if (megasas_alloc_ioc_init_frame(instance)) + return 1; + /* * Allocate memory for descriptors * Create a pool of commands @@ -1364,8 +1886,21 @@ megasas_init_adapter_fusion(struct megasas_instance *instance) if (megasas_alloc_cmds_fusion(instance)) goto fail_alloc_cmds; - if (megasas_ioc_init_fusion(instance)) - goto fail_ioc_init; + if (megasas_ioc_init_fusion(instance)) { + status_reg = instance->instancet->read_fw_status_reg(instance); + if (((status_reg & MFI_STATE_MASK) == MFI_STATE_FAULT) && + (status_reg & MFI_RESET_ADAPTER)) { + /* Do a chip reset and then retry IOC INIT once */ + if (megasas_adp_reset_wait_for_ready + (instance, true, 0) == FAILED) + goto fail_ioc_init; + + if (megasas_ioc_init_fusion(instance)) + goto fail_ioc_init; + } else { + goto fail_ioc_init; + } + } megasas_display_intel_branding(instance); if (megasas_get_ctrl_info(instance)) { @@ -1377,63 +1912,121 @@ megasas_init_adapter_fusion(struct megasas_instance *instance) instance->flag_ieee = 1; instance->r1_ldio_hint_default = MR_R1_LDIO_PIGGYBACK_DEFAULT; + instance->threshold_reply_count = instance->max_fw_cmds / 4; fusion->fast_path_io = 0; - fusion->drv_map_pages = get_order(fusion->drv_map_sz); - for (i = 0; i < 2; i++) { - fusion->ld_map[i] = NULL; - fusion->ld_drv_map[i] = (void *)__get_free_pages(GFP_KERNEL, - fusion->drv_map_pages); - if (!fusion->ld_drv_map[i]) { - dev_err(&instance->pdev->dev, "Could not allocate " - "memory for local map info for %d pages\n", - fusion->drv_map_pages); - if (i == 1) - free_pages((ulong)fusion->ld_drv_map[0], - fusion->drv_map_pages); - goto fail_ioc_init; - } - memset(fusion->ld_drv_map[i], 0, - ((1 << PAGE_SHIFT) << fusion->drv_map_pages)); - } - - for (i = 0; i < 2; i++) { - fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev, - fusion->max_map_sz, - &fusion->ld_map_phys[i], - GFP_KERNEL); - if (!fusion->ld_map[i]) { - dev_err(&instance->pdev->dev, "Could not allocate memory " - "for map info\n"); - goto fail_map_info; - } - } + if (megasas_allocate_raid_maps(instance)) + goto fail_ioc_init; if (!megasas_get_map_info(instance)) megasas_sync_map_info(instance); return 0; -fail_map_info: - if (i == 1) - dma_free_coherent(&instance->pdev->dev, fusion->max_map_sz, - fusion->ld_map[0], fusion->ld_map_phys[0]); fail_ioc_init: megasas_free_cmds_fusion(instance); fail_alloc_cmds: megasas_free_cmds(instance); fail_alloc_mfi_cmds: + megasas_free_ioc_init_cmd(instance); return 1; } /** - * map_cmd_status - Maps FW cmd status to OS cmd status - * @cmd : Pointer to cmd - * @status : status of cmd returned by FW - * @ext_status : ext status of cmd returned by FW + * megasas_fault_detect_work - Worker function of + * FW fault handling workqueue. + * @work: FW fault work struct */ +static void +megasas_fault_detect_work(struct work_struct *work) +{ + struct megasas_instance *instance = + container_of(work, struct megasas_instance, + fw_fault_work.work); + u32 fw_state, dma_state, status; + + /* Check the fw state */ + fw_state = instance->instancet->read_fw_status_reg(instance) & + MFI_STATE_MASK; + + if (fw_state == MFI_STATE_FAULT) { + dma_state = instance->instancet->read_fw_status_reg(instance) & + MFI_STATE_DMADONE; + /* Start collecting crash, if DMA bit is done */ + if (instance->crash_dump_drv_support && + instance->crash_dump_app_support && dma_state) { + megasas_fusion_crash_dump(instance); + } else { + if (instance->unload == 0) { + status = megasas_reset_fusion(instance->host, 0); + if (status != SUCCESS) { + dev_err(&instance->pdev->dev, + "Failed from %s %d, do not re-arm timer\n", + __func__, __LINE__); + return; + } + } + } + } + + if (instance->fw_fault_work_q) + queue_delayed_work(instance->fw_fault_work_q, + &instance->fw_fault_work, + msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL)); +} + +int +megasas_fusion_start_watchdog(struct megasas_instance *instance) +{ + /* Check if the Fault WQ is already started */ + if (instance->fw_fault_work_q) + return SUCCESS; + + INIT_DELAYED_WORK(&instance->fw_fault_work, megasas_fault_detect_work); + + snprintf(instance->fault_handler_work_q_name, + sizeof(instance->fault_handler_work_q_name), + "poll_megasas%d_status", instance->host->host_no); + + instance->fw_fault_work_q = alloc_ordered_workqueue( + "%s", WQ_MEM_RECLAIM, instance->fault_handler_work_q_name); + if (!instance->fw_fault_work_q) { + dev_err(&instance->pdev->dev, "Failed from %s %d\n", + __func__, __LINE__); + return FAILED; + } + + queue_delayed_work(instance->fw_fault_work_q, + &instance->fw_fault_work, + msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL)); + + return SUCCESS; +} void +megasas_fusion_stop_watchdog(struct megasas_instance *instance) +{ + struct workqueue_struct *wq; + + if (instance->fw_fault_work_q) { + wq = instance->fw_fault_work_q; + instance->fw_fault_work_q = NULL; + if (!cancel_delayed_work_sync(&instance->fw_fault_work)) + flush_workqueue(wq); + destroy_workqueue(wq); + } +} + +/** + * map_cmd_status - Maps FW cmd status to OS cmd status + * @fusion: fusion context + * @scmd: Pointer to cmd + * @status: status of cmd returned by FW + * @ext_status: ext status of cmd returned by FW + * @data_length: command data length + * @sense: command sense data + */ +static void map_cmd_status(struct fusion_context *fusion, struct scsi_cmnd *scmd, u8 status, u8 ext_status, u32 data_length, u8 *sense) @@ -1450,18 +2043,18 @@ map_cmd_status(struct fusion_context *fusion, case MFI_STAT_SCSI_IO_FAILED: case MFI_STAT_LD_INIT_IN_PROGRESS: - scmd->result = (DID_ERROR << 16) | ext_status; + if (ext_status == 0xf0) + scmd->result = (DID_ERROR << 16) | SAM_STAT_CHECK_CONDITION; + else + scmd->result = (DID_ERROR << 16) | ext_status; break; case MFI_STAT_SCSI_DONE_WITH_ERROR: scmd->result = (DID_OK << 16) | ext_status; if (ext_status == SAM_STAT_CHECK_CONDITION) { - memset(scmd->sense_buffer, 0, - SCSI_SENSE_BUFFERSIZE); memcpy(scmd->sense_buffer, sense, SCSI_SENSE_BUFFERSIZE); - scmd->result |= DRIVER_SENSE << 24; } /* @@ -1509,8 +2102,6 @@ static bool megasas_is_prp_possible(struct megasas_instance *instance, struct scsi_cmnd *scmd, int sge_count) { - struct fusion_context *fusion; - int i; u32 data_length = 0; struct scatterlist *sg_scmd; bool build_prp = false; @@ -1518,7 +2109,6 @@ megasas_is_prp_possible(struct megasas_instance *instance, mr_nvme_pg_size = max_t(u32, instance->nvme_page_size, MR_DEFAULT_NVME_PAGE_SIZE); - fusion = instance->ctrl_context; data_length = scsi_bufflen(scmd); sg_scmd = scsi_sglist(scmd); @@ -1540,66 +2130,6 @@ megasas_is_prp_possible(struct megasas_instance *instance, build_prp = true; } -/* - * Below code detects gaps/holes in IO data buffers. - * What does holes/gaps mean? - * Any SGE except first one in a SGL starts at non NVME page size - * aligned address OR Any SGE except last one in a SGL ends at - * non NVME page size boundary. - * - * Driver has already informed block layer by setting boundary rules for - * bio merging done at NVME page size boundary calling kernel API - * blk_queue_virt_boundary inside slave_config. - * Still there is possibility of IO coming with holes to driver because of - * IO merging done by IO scheduler. - * - * With SCSI BLK MQ enabled, there will be no IO with holes as there is no - * IO scheduling so no IO merging. - * - * With SCSI BLK MQ disabled, IO scheduler may attempt to merge IOs and - * then sending IOs with holes. - * - * Though driver can request block layer to disable IO merging by calling- - * queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, sdev->request_queue) but - * user may tune sysfs parameter- nomerges again to 0 or 1. - * - * If in future IO scheduling is enabled with SCSI BLK MQ, - * this algorithm to detect holes will be required in driver - * for SCSI BLK MQ enabled case as well. - * - * - */ - scsi_for_each_sg(scmd, sg_scmd, sge_count, i) { - if ((i != 0) && (i != (sge_count - 1))) { - if (mega_mod64(sg_dma_len(sg_scmd), mr_nvme_pg_size) || - mega_mod64(sg_dma_address(sg_scmd), - mr_nvme_pg_size)) { - build_prp = false; - atomic_inc(&instance->sge_holes_type1); - break; - } - } - - if ((sge_count > 1) && (i == 0)) { - if ((mega_mod64((sg_dma_address(sg_scmd) + - sg_dma_len(sg_scmd)), - mr_nvme_pg_size))) { - build_prp = false; - atomic_inc(&instance->sge_holes_type2); - break; - } - } - - if ((sge_count > 1) && (i == (sge_count - 1))) { - if (mega_mod64(sg_dma_address(sg_scmd), - mr_nvme_pg_size)) { - build_prp = false; - atomic_inc(&instance->sge_holes_type3); - break; - } - } - } - return build_prp; } @@ -1631,12 +2161,9 @@ megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd, u32 first_prp_len; bool build_prp = false; int data_len = scsi_bufflen(scmd); - struct fusion_context *fusion; u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size, MR_DEFAULT_NVME_PAGE_SIZE); - fusion = instance->ctrl_context; - build_prp = megasas_is_prp_possible(instance, scmd, sge_count); if (!build_prp) @@ -1730,7 +2257,6 @@ megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd, main_chain_element->Length = cpu_to_le32(num_prp_in_chain * sizeof(u64)); - atomic_inc(&instance->prp_sgl); return build_prp; } @@ -1740,7 +2266,7 @@ megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd, * @scp: SCSI command from the mid-layer * @sgl_ptr: SGL to be filled in * @cmd: cmd we are working on - * @sge_count sge count + * @sge_count: sge count * */ static void @@ -1755,7 +2281,7 @@ megasas_make_sgl_fusion(struct megasas_instance *instance, fusion = instance->ctrl_context; - if (fusion->adapter_type >= INVADER_SERIES) { + if (instance->adapter_type >= INVADER_SERIES) { struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr; sgl_ptr_end += fusion->max_sge_in_main_msg - 1; sgl_ptr_end->Flags = 0; @@ -1765,7 +2291,7 @@ megasas_make_sgl_fusion(struct megasas_instance *instance, sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl)); sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl)); sgl_ptr->Flags = 0; - if (fusion->adapter_type >= INVADER_SERIES) + if (instance->adapter_type >= INVADER_SERIES) if (i == sge_count - 1) sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST; sgl_ptr++; @@ -1775,7 +2301,7 @@ megasas_make_sgl_fusion(struct megasas_instance *instance, (sge_count > fusion->max_sge_in_main_msg)) { struct MPI25_IEEE_SGE_CHAIN64 *sg_chain; - if (fusion->adapter_type >= INVADER_SERIES) { + if (instance->adapter_type >= INVADER_SERIES) { if ((le16_to_cpu(cmd->io_request->IoFlags) & MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) != MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) @@ -1791,7 +2317,7 @@ megasas_make_sgl_fusion(struct megasas_instance *instance, sg_chain = sgl_ptr; /* Prepare chain element */ sg_chain->NextChainOffset = 0; - if (fusion->adapter_type >= INVADER_SERIES) + if (instance->adapter_type >= INVADER_SERIES) sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT; else sg_chain->Flags = @@ -1805,7 +2331,6 @@ megasas_make_sgl_fusion(struct megasas_instance *instance, memset(sgl_ptr, 0, instance->max_chain_frame_sz); } } - atomic_inc(&instance->ieee_sgl); } /** @@ -1850,13 +2375,16 @@ int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp, /** * megasas_set_pd_lba - Sets PD LBA - * @cdb: CDB + * @io_request: IO request * @cdb_len: cdb length - * @start_blk: Start block of IO + * @io_info: IO information + * @scp: SCSI command + * @local_map_ptr: Raid map + * @ref_tag: Primary reference tag * * Used to set the PD LBA in CDB for FP IOs */ -void +static void megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len, struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp, struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag) @@ -1876,7 +2404,7 @@ megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len, cdb[0] = MEGASAS_SCSI_VARIABLE_LENGTH_CMD; cdb[7] = MEGASAS_SCSI_ADDL_CDB_LEN; - if (scp->sc_data_direction == PCI_DMA_FROMDEVICE) + if (scp->sc_data_direction == DMA_FROM_DEVICE) cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32; else cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32; @@ -1905,7 +2433,7 @@ megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len, cdb[31] = (u8)(num_blocks & 0xff); /* set SCSI IO EEDPFlags */ - if (scp->sc_data_direction == PCI_DMA_FROMDEVICE) { + if (scp->sc_data_direction == DMA_FROM_DEVICE) { io_request->EEDPFlags = cpu_to_le16( MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG | MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG | @@ -2066,7 +2594,7 @@ static void megasas_stream_detect(struct megasas_instance *instance, if ((io_info->ldStartBlock != current_sd->next_seq_lba) && ((!io_info->isRead) || (!is_read_ahead))) /* - * Once the API availible we need to change this. + * Once the API is available we need to change this. * At this point we are not allowing any gap */ continue; @@ -2110,16 +2638,19 @@ static void megasas_stream_detect(struct megasas_instance *instance, * affinity (cpu of the controller) and raid_flags in the raid context * based on IO type. * + * @fusion: Fusion context * @praid_context: IO RAID context * @raid: LD raid map * @fp_possible: Is fast path possible? * @is_read: Is read IO? + * @scsi_buff_len: SCSI command buffer length * */ static void -megasas_set_raidflag_cpu_affinity(union RAID_CONTEXT_UNION *praid_context, - struct MR_LD_RAID *raid, bool fp_possible, - u8 is_read, u32 scsi_buff_len) +megasas_set_raidflag_cpu_affinity(struct fusion_context *fusion, + union RAID_CONTEXT_UNION *praid_context, + struct MR_LD_RAID *raid, bool fp_possible, + u8 is_read, u32 scsi_buff_len) { u8 cpu_sel = MR_RAID_CTX_CPUSEL_0; struct RAID_CONTEXT_G35 *rctx_g35; @@ -2177,11 +2708,11 @@ megasas_set_raidflag_cpu_affinity(union RAID_CONTEXT_UNION *praid_context, * vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS. * IO Subtype is not bitmap. */ - if ((raid->level == 1) && (!is_read)) { - if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE) - praid_context->raid_context_g35.raid_flags = - (MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT - << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT); + if ((fusion->pcie_bw_limitation) && (raid->level == 1) && (!is_read) && + (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)) { + praid_context->raid_context_g35.raid_flags = + (MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT + << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT); } } @@ -2194,7 +2725,7 @@ megasas_set_raidflag_cpu_affinity(union RAID_CONTEXT_UNION *praid_context, * Prepares the io_request and chain elements (sg_frame) for IO * The IO can be for PD (Fast Path) or LD */ -void +static void megasas_build_ldio_fusion(struct megasas_instance *instance, struct scsi_cmnd *scp, struct megasas_cmd_fusion *cmd) @@ -2204,27 +2735,27 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, u32 start_lba_lo, start_lba_hi, device_id, datalength = 0; u32 scsi_buff_len; struct MPI2_RAID_SCSI_IO_REQUEST *io_request; - union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc; struct IO_REQUEST_INFO io_info; struct fusion_context *fusion; struct MR_DRV_RAID_MAP_ALL *local_map_ptr; u8 *raidLUN; unsigned long spinlock_flags; - union RAID_CONTEXT_UNION *praid_context; struct MR_LD_RAID *raid = NULL; struct MR_PRIV_DEVICE *mrdev_priv; + struct RAID_CONTEXT *rctx; + struct RAID_CONTEXT_G35 *rctx_g35; device_id = MEGASAS_DEV_INDEX(scp); fusion = instance->ctrl_context; io_request = cmd->io_request; - io_request->RaidContext.raid_context.virtual_disk_tgt_id = - cpu_to_le16(device_id); - io_request->RaidContext.raid_context.status = 0; - io_request->RaidContext.raid_context.ex_status = 0; + rctx = &io_request->RaidContext.raid_context; + rctx_g35 = &io_request->RaidContext.raid_context_g35; - req_desc = (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)cmd->request_desc; + rctx->virtual_disk_tgt_id = cpu_to_le16(device_id); + rctx->status = 0; + rctx->ex_status = 0; start_lba_lo = 0; start_lba_hi = 0; @@ -2287,8 +2818,9 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID; scsi_buff_len = scsi_bufflen(scp); io_request->DataLength = cpu_to_le32(scsi_buff_len); + io_info.data_arms = 1; - if (scp->sc_data_direction == PCI_DMA_FROMDEVICE) + if (scp->sc_data_direction == DMA_FROM_DEVICE) io_info.isRead = 1; local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)]; @@ -2298,34 +2830,17 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, raid = MR_LdRaidGet(ld, local_map_ptr); if (!raid || (!fusion->fast_path_io)) { - io_request->RaidContext.raid_context.reg_lock_flags = 0; + rctx->reg_lock_flags = 0; fp_possible = false; } else { - if (MR_BuildRaidContext(instance, &io_info, - &io_request->RaidContext.raid_context, + if (MR_BuildRaidContext(instance, &io_info, rctx, local_map_ptr, &raidLUN)) fp_possible = (io_info.fpOkForIo > 0) ? true : false; } - /* Use raw_smp_processor_id() for now until cmd->request->cpu is CPU - id by default, not CPU group id, otherwise all MSI-X queues won't - be utilized */ - cmd->request_desc->SCSIIO.MSIxIndex = instance->msix_vectors ? - raw_smp_processor_id() % instance->msix_vectors : 0; - - praid_context = &io_request->RaidContext; - - if (instance->is_ventura) { - spin_lock_irqsave(&instance->stream_lock, spinlock_flags); - megasas_stream_detect(instance, cmd, &io_info); - spin_unlock_irqrestore(&instance->stream_lock, spinlock_flags); - /* In ventura if stream detected for a read and it is read ahead - * capable make this IO as LDIO - */ - if (is_stream_detected(&io_request->RaidContext.raid_context_g35) && - io_info.isRead && io_info.ra_capable) - fp_possible = false; + megasas_get_msix_index(instance, scp, cmd, io_info.data_arms); + if (instance->adapter_type >= VENTURA_SERIES) { /* FP for Optimal raid level 1. * All large RAID-1 writes (> 32 KiB, both WT and WB modes) * are built by the driver as LD I/Os. @@ -2341,8 +2856,9 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, (instance->host->can_queue)) { fp_possible = false; atomic_dec(&instance->fw_outstanding); - } else if ((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) || - (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0)) { + } else if (fusion->pcie_bw_limitation && + ((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) || + (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0))) { fp_possible = false; atomic_dec(&instance->fw_outstanding); if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE) @@ -2351,13 +2867,27 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, } } + if (!fp_possible || + (io_info.isRead && io_info.ra_capable)) { + spin_lock_irqsave(&instance->stream_lock, + spinlock_flags); + megasas_stream_detect(instance, cmd, &io_info); + spin_unlock_irqrestore(&instance->stream_lock, + spinlock_flags); + /* In ventura if stream detected for a read and it is + * read ahead capable make this IO as LDIO + */ + if (is_stream_detected(rctx_g35)) + fp_possible = false; + } + /* If raid is NULL, set CPU affinity to default CPU0 */ if (raid) - megasas_set_raidflag_cpu_affinity(praid_context, + megasas_set_raidflag_cpu_affinity(fusion, &io_request->RaidContext, raid, fp_possible, io_info.isRead, scsi_buff_len); else - praid_context->raid_context_g35.routing_flags |= + rctx_g35->routing_flags |= (MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT); } @@ -2368,26 +2898,17 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, cmd->request_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_FP_IO << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT); - if (fusion->adapter_type == INVADER_SERIES) { - if (io_request->RaidContext.raid_context.reg_lock_flags == - REGION_TYPE_UNUSED) - cmd->request_desc->SCSIIO.RequestFlags = - (MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK << - MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT); - io_request->RaidContext.raid_context.type - = MPI2_TYPE_CUDA; - io_request->RaidContext.raid_context.nseg = 0x1; + if (instance->adapter_type == INVADER_SERIES) { + rctx->type = MPI2_TYPE_CUDA; + rctx->nseg = 0x1; io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH); - io_request->RaidContext.raid_context.reg_lock_flags |= + rctx->reg_lock_flags |= (MR_RL_FLAGS_GRANT_DESTINATION_CUDA | MR_RL_FLAGS_SEQ_NUM_ENABLE); - } else if (instance->is_ventura) { - io_request->RaidContext.raid_context_g35.nseg_type |= - (1 << RAID_CONTEXT_NSEG_SHIFT); - io_request->RaidContext.raid_context_g35.nseg_type |= - (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT); - io_request->RaidContext.raid_context_g35.routing_flags |= - (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT); + } else if (instance->adapter_type >= VENTURA_SERIES) { + rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT); + rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT); + rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT); io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH); } @@ -2398,19 +2919,17 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, get_updated_dev_handle(instance, &fusion->load_balance_info[device_id], &io_info, local_map_ptr); - scp->SCp.Status |= MEGASAS_LOAD_BALANCE_FLAG; + megasas_priv(scp)->status |= MEGASAS_LOAD_BALANCE_FLAG; cmd->pd_r1_lb = io_info.pd_after_lb; - if (instance->is_ventura) - io_request->RaidContext.raid_context_g35.span_arm - = io_info.span_arm; + if (instance->adapter_type >= VENTURA_SERIES) + rctx_g35->span_arm = io_info.span_arm; else - io_request->RaidContext.raid_context.span_arm - = io_info.span_arm; + rctx->span_arm = io_info.span_arm; } else - scp->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG; + megasas_priv(scp)->status &= ~MEGASAS_LOAD_BALANCE_FLAG; - if (instance->is_ventura) + if (instance->adapter_type >= VENTURA_SERIES) cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle; else cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID; @@ -2428,31 +2947,26 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, /* populate the LUN field */ memcpy(io_request->LUN, raidLUN, 8); } else { - io_request->RaidContext.raid_context.timeout_value = + rctx->timeout_value = cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec); cmd->request_desc->SCSIIO.RequestFlags = (MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT); - if (fusion->adapter_type == INVADER_SERIES) { + if (instance->adapter_type == INVADER_SERIES) { if (io_info.do_fp_rlbypass || - (io_request->RaidContext.raid_context.reg_lock_flags - == REGION_TYPE_UNUSED)) + (rctx->reg_lock_flags == REGION_TYPE_UNUSED)) cmd->request_desc->SCSIIO.RequestFlags = (MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT); - io_request->RaidContext.raid_context.type - = MPI2_TYPE_CUDA; - io_request->RaidContext.raid_context.reg_lock_flags |= + rctx->type = MPI2_TYPE_CUDA; + rctx->reg_lock_flags |= (MR_RL_FLAGS_GRANT_DESTINATION_CPU0 | - MR_RL_FLAGS_SEQ_NUM_ENABLE); - io_request->RaidContext.raid_context.nseg = 0x1; - } else if (instance->is_ventura) { - io_request->RaidContext.raid_context_g35.routing_flags |= - (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT); - io_request->RaidContext.raid_context_g35.nseg_type |= - (1 << RAID_CONTEXT_NSEG_SHIFT); - io_request->RaidContext.raid_context_g35.nseg_type |= - (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT); + MR_RL_FLAGS_SEQ_NUM_ENABLE); + rctx->nseg = 0x1; + } else if (instance->adapter_type >= VENTURA_SERIES) { + rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT); + rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT); + rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT); } io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST; io_request->DevHandle = cpu_to_le16(device_id); @@ -2463,7 +2977,7 @@ megasas_build_ldio_fusion(struct megasas_instance *instance, /** * megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk * @instance: Adapter soft state - * @scp: SCSI command + * @scmd: SCSI command * @cmd: Command to be prepared * * Prepares the io_request frame for non-rw io cmds for vd. @@ -2498,7 +3012,7 @@ static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance, device_id < instance->fw_supported_vd_count)) { ld = MR_TargetIdToLdGet(device_id, local_map_ptr); - if (ld >= instance->fw_supported_vd_count) + if (ld >= instance->fw_supported_vd_count - 1) fp_possible = 0; else { raid = MR_LdRaidGet(ld, local_map_ptr); @@ -2513,7 +3027,7 @@ static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance, io_request->DevHandle = cpu_to_le16(device_id); io_request->LUN[1] = scmd->device->lun; pRAID_Context->timeout_value = - cpu_to_le16 (scmd->request->timeout / HZ); + cpu_to_le16(scsi_cmd_to_rq(scmd)->timeout / HZ); cmd->request_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT); @@ -2521,7 +3035,7 @@ static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance, /* set RAID context values */ pRAID_Context->config_seq_num = raid->seqNum; - if (!instance->is_ventura) + if (instance->adapter_type < VENTURA_SERIES) pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ; pRAID_Context->timeout_value = cpu_to_le16(raid->fpIoTimeoutForLd); @@ -2551,7 +3065,7 @@ static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance, /** * megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd * @instance: Adapter soft state - * @scp: SCSI command + * @scmd: SCSI command * @cmd: Command to be prepared * @fp_possible: parameter to detect fast path or firmware path io. * @@ -2576,7 +3090,7 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, device_id = MEGASAS_DEV_INDEX(scmd); pd_index = MEGASAS_PD_INDEX(scmd); - os_timeout_value = scmd->request->timeout / HZ; + os_timeout_value = scsi_cmd_to_rq(scmd)->timeout / HZ; mr_device_priv_data = scmd->device->hostdata; cmd->pd_interface = mr_device_priv_data->interface_type; @@ -2592,51 +3106,60 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, << MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT; /* If FW supports PD sequence number */ - if (instance->use_seqnum_jbod_fp && - instance->pd_list[pd_index].driveType == TYPE_DISK) { - /* TgtId must be incremented by 255 as jbod seq number is index - * below raid map - */ - /* More than 256 PD/JBOD support for Ventura */ - if (instance->support_morethan256jbod) - pRAID_Context->virtual_disk_tgt_id = - pd_sync->seq[pd_index].pd_target_id; - else - pRAID_Context->virtual_disk_tgt_id = - cpu_to_le16(device_id + (MAX_PHYSICAL_DEVICES - 1)); - pRAID_Context->config_seq_num = pd_sync->seq[pd_index].seqNum; - io_request->DevHandle = pd_sync->seq[pd_index].devHandle; - if (instance->is_ventura) { - io_request->RaidContext.raid_context_g35.routing_flags |= - (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT); - io_request->RaidContext.raid_context_g35.nseg_type |= - (1 << RAID_CONTEXT_NSEG_SHIFT); - io_request->RaidContext.raid_context_g35.nseg_type |= - (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT); + if (instance->support_seqnum_jbod_fp) { + if (instance->use_seqnum_jbod_fp && + instance->pd_list[pd_index].driveType == TYPE_DISK) { + + /* More than 256 PD/JBOD support for Ventura */ + if (instance->support_morethan256jbod) + pRAID_Context->virtual_disk_tgt_id = + pd_sync->seq[pd_index].pd_target_id; + else + pRAID_Context->virtual_disk_tgt_id = + cpu_to_le16(device_id + + (MAX_PHYSICAL_DEVICES - 1)); + pRAID_Context->config_seq_num = + pd_sync->seq[pd_index].seqNum; + io_request->DevHandle = + pd_sync->seq[pd_index].devHandle; + if (instance->adapter_type >= VENTURA_SERIES) { + io_request->RaidContext.raid_context_g35.routing_flags |= + (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT); + io_request->RaidContext.raid_context_g35.nseg_type |= + (1 << RAID_CONTEXT_NSEG_SHIFT); + io_request->RaidContext.raid_context_g35.nseg_type |= + (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT); + } else { + pRAID_Context->type = MPI2_TYPE_CUDA; + pRAID_Context->nseg = 0x1; + pRAID_Context->reg_lock_flags |= + (MR_RL_FLAGS_SEQ_NUM_ENABLE | + MR_RL_FLAGS_GRANT_DESTINATION_CUDA); + } } else { - pRAID_Context->type = MPI2_TYPE_CUDA; - pRAID_Context->nseg = 0x1; - pRAID_Context->reg_lock_flags |= - (MR_RL_FLAGS_SEQ_NUM_ENABLE|MR_RL_FLAGS_GRANT_DESTINATION_CUDA); + pRAID_Context->virtual_disk_tgt_id = + cpu_to_le16(device_id + + (MAX_PHYSICAL_DEVICES - 1)); + pRAID_Context->config_seq_num = 0; + io_request->DevHandle = cpu_to_le16(0xFFFF); } - } else if (fusion->fast_path_io) { - pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id); - pRAID_Context->config_seq_num = 0; - local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)]; - io_request->DevHandle = - local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl; } else { - /* Want to send all IO via FW path */ pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id); pRAID_Context->config_seq_num = 0; - io_request->DevHandle = cpu_to_le16(0xFFFF); + + if (fusion->fast_path_io) { + local_map_ptr = + fusion->ld_drv_map[(instance->map_id & 1)]; + io_request->DevHandle = + local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl; + } else { + io_request->DevHandle = cpu_to_le16(0xFFFF); + } } cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle; - cmd->request_desc->SCSIIO.MSIxIndex = - instance->msix_vectors ? - (raw_smp_processor_id() % instance->msix_vectors) : 0; + megasas_get_msix_index(instance, scmd, cmd, 1); if (!fp_possible) { /* system pd firmware path */ @@ -2647,6 +3170,9 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value); pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id); } else { + if (os_timeout_value) + os_timeout_value++; + /* system pd Fast Path */ io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST; timeout_limit = (scmd->device->type == TYPE_DISK) ? @@ -2654,7 +3180,7 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, pRAID_Context->timeout_value = cpu_to_le16((os_timeout_value > timeout_limit) ? timeout_limit : os_timeout_value); - if (fusion->adapter_type >= INVADER_SERIES) + if (instance->adapter_type >= INVADER_SERIES) io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH); @@ -2673,13 +3199,14 @@ megasas_build_syspd_fusion(struct megasas_instance *instance, * Invokes helper functions to prepare request frames * and sets flags appropriate for IO/Non-IO cmd */ -int +static int megasas_build_io_fusion(struct megasas_instance *instance, struct scsi_cmnd *scp, struct megasas_cmd_fusion *cmd) { int sge_count; - u8 cmd_type; + u16 pd_index = 0; + u8 drive_type = 0; struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request; struct MR_PRIV_DEVICE *mr_device_priv_data; mr_device_priv_data = scp->device->hostdata; @@ -2703,7 +3230,7 @@ megasas_build_io_fusion(struct megasas_instance *instance, */ io_request->IoFlags = cpu_to_le16(scp->cmd_len); - switch (cmd_type = megasas_cmd_type(scp)) { + switch (megasas_cmd_type(scp)) { case READ_WRITE_LDIO: megasas_build_ldio_fusion(instance, scp, cmd); break; @@ -2714,8 +3241,12 @@ megasas_build_io_fusion(struct megasas_instance *instance, megasas_build_syspd_fusion(instance, scp, cmd, true); break; case NON_READ_WRITE_SYSPDIO: - if (instance->secure_jbod_support || - mr_device_priv_data->is_tm_capable) + pd_index = MEGASAS_PD_INDEX(scp); + drive_type = instance->pd_list[pd_index].driveType; + if ((instance->secure_jbod_support || + mr_device_priv_data->is_tm_capable) || + (instance->adapter_type >= VENTURA_SERIES && + drive_type == TYPE_ENCLOSURE)) megasas_build_syspd_fusion(instance, scp, cmd, false); else megasas_build_syspd_fusion(instance, scp, cmd, true); @@ -2737,7 +3268,7 @@ megasas_build_io_fusion(struct megasas_instance *instance, return 1; } - if (instance->is_ventura) { + if (instance->adapter_type >= VENTURA_SERIES) { set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count); cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags); cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type); @@ -2752,19 +3283,20 @@ megasas_build_io_fusion(struct megasas_instance *instance, io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING); - if (scp->sc_data_direction == PCI_DMA_TODEVICE) + if (scp->sc_data_direction == DMA_TO_DEVICE) io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE); - else if (scp->sc_data_direction == PCI_DMA_FROMDEVICE) + else if (scp->sc_data_direction == DMA_FROM_DEVICE) io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ); io_request->SGLOffset0 = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4; - io_request->SenseBufferLowAddress = cpu_to_le32(cmd->sense_phys_addr); + io_request->SenseBufferLowAddress = + cpu_to_le32(lower_32_bits(cmd->sense_phys_addr)); io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE; cmd->scmd = scp; - scp->SCp.ptr = (char *)cmd; + megasas_priv(scp)->cmd_priv = cmd; return 0; } @@ -2786,9 +3318,9 @@ megasas_get_request_descriptor(struct megasas_instance *instance, u16 index) /* megasas_prepate_secondRaid1_IO * It prepares the raid 1 second IO */ -void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance, - struct megasas_cmd_fusion *cmd, - struct megasas_cmd_fusion *r1_cmd) +static void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance, + struct megasas_cmd_fusion *cmd, + struct megasas_cmd_fusion *r1_cmd) { union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL; struct fusion_context *fusion; @@ -2797,11 +3329,11 @@ void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance, /* copy the io request frame as well as 8 SGEs data for r1 command*/ memcpy(r1_cmd->io_request, cmd->io_request, (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST))); - memcpy(&r1_cmd->io_request->SGL, &cmd->io_request->SGL, + memcpy(r1_cmd->io_request->SGLs, cmd->io_request->SGLs, (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION))); /*sense buffer is different for r1 command*/ r1_cmd->io_request->SenseBufferLowAddress = - cpu_to_le32(r1_cmd->sense_phys_addr); + cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr)); r1_cmd->scmd = cmd->scmd; req_desc2 = megasas_get_request_descriptor(instance, (r1_cmd->index - 1)); @@ -2812,9 +3344,9 @@ void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance, r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle; r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle; r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle; - cmd->io_request->RaidContext.raid_context_g35.smid.peer_smid = + cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid = cpu_to_le16(r1_cmd->index); - r1_cmd->io_request->RaidContext.raid_context_g35.smid.peer_smid = + r1_cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid = cpu_to_le16(cmd->index); /*MSIxIndex of both commands request descriptors should be same*/ r1_cmd->request_desc->SCSIIO.MSIxIndex = @@ -2837,9 +3369,6 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance, struct megasas_cmd_fusion *cmd, *r1_cmd = NULL; union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc; u32 index; - struct fusion_context *fusion; - - fusion = instance->ctrl_context; if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) && instance->ldio_threshold && @@ -2855,7 +3384,7 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance, return SCSI_MLQUEUE_HOST_BUSY; } - cmd = megasas_get_cmd_fusion(instance, scmd->request->tag); + cmd = megasas_get_cmd_fusion(instance, scsi_cmd_to_rq(scmd)->tag); if (!cmd) { atomic_dec(&instance->fw_outstanding); @@ -2896,7 +3425,7 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance, */ if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) { r1_cmd = megasas_get_cmd_fusion(instance, - (scmd->request->tag + instance->max_fw_cmds)); + scsi_cmd_to_rq(scmd)->tag + instance->max_fw_cmds); megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd); } @@ -2905,6 +3434,7 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance, * Issue the command to the FW */ + megasas_sdev_busy_inc(instance, scmd); megasas_fire_cmd_fusion(instance, req_desc); if (r1_cmd) @@ -2918,7 +3448,7 @@ megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance, * megasas_complete_r1_command - * completes R1 FP write commands which has valid peer smid * @instance: Adapter soft state - * @cmd_fusion: MPT command frame + * @cmd: MPT command frame * */ static inline void @@ -2935,7 +3465,7 @@ megasas_complete_r1_command(struct megasas_instance *instance, rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35; fusion = instance->ctrl_context; - peer_smid = le16_to_cpu(rctx_g35->smid.peer_smid); + peer_smid = le16_to_cpu(rctx_g35->flow_specific.peer_smid); r1_cmd = fusion->cmd_list[peer_smid - 1]; scmd_local = cmd->scmd; @@ -2962,20 +3492,60 @@ megasas_complete_r1_command(struct megasas_instance *instance, if (instance->ldio_threshold && megasas_cmd_type(scmd_local) == READ_WRITE_LDIO) atomic_dec(&instance->ldio_outstanding); - scmd_local->SCp.ptr = NULL; + megasas_priv(scmd_local)->cmd_priv = NULL; megasas_return_cmd_fusion(instance, cmd); scsi_dma_unmap(scmd_local); - scmd_local->scsi_done(scmd_local); + megasas_sdev_busy_dec(instance, scmd_local); + scsi_done(scmd_local); } } /** + * access_irq_context: Access to reply processing + * @irq_context: IRQ context + * + * Synchronize access to reply processing. + * + * Return: true on success, false on failure. + */ +static inline +bool access_irq_context(struct megasas_irq_context *irq_context) +{ + if (!irq_context) + return true; + + if (atomic_add_unless(&irq_context->in_used, 1, 1)) + return true; + + return false; +} + +/** + * release_irq_context: Release reply processing + * @irq_context: IRQ context + * + * Release access of reply processing. + * + * Return: Nothing. + */ +static inline +void release_irq_context(struct megasas_irq_context *irq_context) +{ + if (irq_context) + atomic_dec(&irq_context->in_used); +} + +/** * complete_cmd_fusion - Completes command * @instance: Adapter soft state + * @MSIxIndex: MSI number + * @irq_context: IRQ context + * * Completes all commands that is in reply descriptor queue */ -int -complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) +static int +complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex, + struct megasas_irq_context *irq_context) { union MPI2_REPLY_DESCRIPTORS_UNION *desc; struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc; @@ -2998,6 +3568,9 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) return IRQ_HANDLED; + if (!access_irq_context(irq_context)) + return 0; + desc = fusion->reply_frames_desc[MSIxIndex] + fusion->last_reply_idx[MSIxIndex]; @@ -3008,8 +3581,10 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) reply_descript_type = reply_desc->ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; - if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) + if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) { + release_irq_context(irq_context); return IRQ_NONE; + } num_completed = 0; @@ -3041,14 +3616,15 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) case MPI2_FUNCTION_SCSI_IO_REQUEST: /*Fast Path IO.*/ /* Update load balancing info */ if (fusion->load_balance_info && - (cmd_fusion->scmd->SCp.Status & + (megasas_priv(cmd_fusion->scmd)->status & MEGASAS_LOAD_BALANCE_FLAG)) { device_id = MEGASAS_DEV_INDEX(scmd_local); lbinfo = &fusion->load_balance_info[device_id]; atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]); - cmd_fusion->scmd->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG; + megasas_priv(cmd_fusion->scmd)->status &= + ~MEGASAS_LOAD_BALANCE_FLAG; } - //Fall thru and complete IO + fallthrough; /* and complete IO */ case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */ atomic_dec(&instance->fw_outstanding); if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) { @@ -3058,10 +3634,11 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) if (instance->ldio_threshold && (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)) atomic_dec(&instance->ldio_outstanding); - scmd_local->SCp.ptr = NULL; + megasas_priv(scmd_local)->cmd_priv = NULL; megasas_return_cmd_fusion(instance, cmd_fusion); scsi_dma_unmap(scmd_local); - scmd_local->scsi_done(scmd_local); + megasas_sdev_busy_dec(instance, scmd_local); + scsi_done(scmd_local); } else /* Optimal VD - R1 FP command completion. */ megasas_complete_r1_command(instance, cmd_fusion); break; @@ -3108,7 +3685,7 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) * number of reply counts and still there are more replies in reply queue * pending to be completed */ - if (threshold_reply_count >= THRESHOLD_REPLY_COUNT) { + if (threshold_reply_count >= instance->threshold_reply_count) { if (instance->msix_combined) writel(((MSIxIndex & 0x7) << 24) | fusion->last_reply_idx[MSIxIndex], @@ -3118,127 +3695,199 @@ complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex) fusion->last_reply_idx[MSIxIndex], instance->reply_post_host_index_addr[0]); threshold_reply_count = 0; + if (irq_context) { + if (!irq_context->irq_poll_scheduled) { + irq_context->irq_poll_scheduled = true; + irq_context->irq_line_enable = true; + irq_poll_sched(&irq_context->irqpoll); + } + release_irq_context(irq_context); + return num_completed; + } } } - if (!num_completed) - return IRQ_NONE; + if (num_completed) { + wmb(); + if (instance->msix_combined) + writel(((MSIxIndex & 0x7) << 24) | + fusion->last_reply_idx[MSIxIndex], + instance->reply_post_host_index_addr[MSIxIndex/8]); + else + writel((MSIxIndex << 24) | + fusion->last_reply_idx[MSIxIndex], + instance->reply_post_host_index_addr[0]); + megasas_check_and_restore_queue_depth(instance); + } - wmb(); - if (instance->msix_combined) - writel(((MSIxIndex & 0x7) << 24) | - fusion->last_reply_idx[MSIxIndex], - instance->reply_post_host_index_addr[MSIxIndex/8]); - else - writel((MSIxIndex << 24) | - fusion->last_reply_idx[MSIxIndex], - instance->reply_post_host_index_addr[0]); - megasas_check_and_restore_queue_depth(instance); - return IRQ_HANDLED; + release_irq_context(irq_context); + + return num_completed; +} + +int megasas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num) +{ + + struct megasas_instance *instance; + int num_entries = 0; + struct fusion_context *fusion; + + instance = (struct megasas_instance *)shost->hostdata; + + fusion = instance->ctrl_context; + + queue_num = queue_num + instance->low_latency_index_start; + + if (!atomic_add_unless(&fusion->busy_mq_poll[queue_num], 1, 1)) + return 0; + + num_entries = complete_cmd_fusion(instance, queue_num, NULL); + atomic_dec(&fusion->busy_mq_poll[queue_num]); + + return num_entries; } /** - * megasas_sync_irqs - Synchronizes all IRQs owned by adapter + * megasas_enable_irq_poll() - enable irqpoll * @instance: Adapter soft state */ -void megasas_sync_irqs(unsigned long instance_addr) +static void megasas_enable_irq_poll(struct megasas_instance *instance) +{ + u32 count, i; + struct megasas_irq_context *irq_ctx; + + count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; + + for (i = 0; i < count; i++) { + irq_ctx = &instance->irq_context[i]; + irq_poll_enable(&irq_ctx->irqpoll); + } +} + +/** + * megasas_sync_irqs - Synchronizes all IRQs owned by adapter + * @instance_addr: Adapter soft state address + */ +static void megasas_sync_irqs(unsigned long instance_addr) { u32 count, i; struct megasas_instance *instance = (struct megasas_instance *)instance_addr; + struct megasas_irq_context *irq_ctx; count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; - for (i = 0; i < count; i++) + for (i = 0; i < count; i++) { synchronize_irq(pci_irq_vector(instance->pdev, i)); + irq_ctx = &instance->irq_context[i]; + irq_poll_disable(&irq_ctx->irqpoll); + if (irq_ctx->irq_poll_scheduled) { + irq_ctx->irq_poll_scheduled = false; + enable_irq(irq_ctx->os_irq); + complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx); + } + } +} + +/** + * megasas_irqpoll() - process a queue for completed reply descriptors + * @irqpoll: IRQ poll structure associated with queue to poll. + * @budget: Threshold of reply descriptors to process per poll. + * + * Return: The number of entries processed. + */ + +int megasas_irqpoll(struct irq_poll *irqpoll, int budget) +{ + struct megasas_irq_context *irq_ctx; + struct megasas_instance *instance; + int num_entries; + + irq_ctx = container_of(irqpoll, struct megasas_irq_context, irqpoll); + instance = irq_ctx->instance; + + if (irq_ctx->irq_line_enable) { + disable_irq_nosync(irq_ctx->os_irq); + irq_ctx->irq_line_enable = false; + } + + num_entries = complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx); + if (num_entries < budget) { + irq_poll_complete(irqpoll); + irq_ctx->irq_poll_scheduled = false; + enable_irq(irq_ctx->os_irq); + complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx); + } + + return num_entries; } /** * megasas_complete_cmd_dpc_fusion - Completes command - * @instance: Adapter soft state + * @instance_addr: Adapter soft state address * * Tasklet to complete cmds */ -void +static void megasas_complete_cmd_dpc_fusion(unsigned long instance_addr) { struct megasas_instance *instance = (struct megasas_instance *)instance_addr; - unsigned long flags; + struct megasas_irq_context *irq_ctx = NULL; u32 count, MSIxIndex; count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; /* If we have already declared adapter dead, donot complete cmds */ - spin_lock_irqsave(&instance->hba_lock, flags); - if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) { - spin_unlock_irqrestore(&instance->hba_lock, flags); + if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) return; - } - spin_unlock_irqrestore(&instance->hba_lock, flags); - for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) - complete_cmd_fusion(instance, MSIxIndex); + for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) { + irq_ctx = &instance->irq_context[MSIxIndex]; + complete_cmd_fusion(instance, MSIxIndex, irq_ctx); + } } /** * megasas_isr_fusion - isr entry point + * @irq: IRQ number + * @devp: IRQ context */ -irqreturn_t megasas_isr_fusion(int irq, void *devp) +static irqreturn_t megasas_isr_fusion(int irq, void *devp) { struct megasas_irq_context *irq_context = devp; struct megasas_instance *instance = irq_context->instance; - u32 mfiStatus, fw_state, dma_state; + u32 mfiStatus; if (instance->mask_interrupts) return IRQ_NONE; + if (irq_context->irq_poll_scheduled) + return IRQ_HANDLED; + if (!instance->msix_vectors) { - mfiStatus = instance->instancet->clear_intr(instance->reg_set); + mfiStatus = instance->instancet->clear_intr(instance); if (!mfiStatus) return IRQ_NONE; } /* If we are resetting, bail */ if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) { - instance->instancet->clear_intr(instance->reg_set); + instance->instancet->clear_intr(instance); return IRQ_HANDLED; } - if (!complete_cmd_fusion(instance, irq_context->MSIxIndex)) { - instance->instancet->clear_intr(instance->reg_set); - /* If we didn't complete any commands, check for FW fault */ - fw_state = instance->instancet->read_fw_status_reg( - instance->reg_set) & MFI_STATE_MASK; - dma_state = instance->instancet->read_fw_status_reg - (instance->reg_set) & MFI_STATE_DMADONE; - if (instance->crash_dump_drv_support && - instance->crash_dump_app_support) { - /* Start collecting crash, if DMA bit is done */ - if ((fw_state == MFI_STATE_FAULT) && dma_state) - schedule_work(&instance->crash_init); - else if (fw_state == MFI_STATE_FAULT) { - if (instance->unload == 0) - schedule_work(&instance->work_init); - } - } else if (fw_state == MFI_STATE_FAULT) { - dev_warn(&instance->pdev->dev, "Iop2SysDoorbellInt" - "for scsi%d\n", instance->host->host_no); - if (instance->unload == 0) - schedule_work(&instance->work_init); - } - } - - return IRQ_HANDLED; + return complete_cmd_fusion(instance, irq_context->MSIxIndex, irq_context) + ? IRQ_HANDLED : IRQ_NONE; } /** * build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru * @instance: Adapter soft state - * mfi_cmd: megasas_cmd pointer + * @mfi_cmd: megasas_cmd pointer * */ -void +static void build_mpt_mfi_pass_thru(struct megasas_instance *instance, struct megasas_cmd *mfi_cmd) { @@ -3267,7 +3916,7 @@ build_mpt_mfi_pass_thru(struct megasas_instance *instance, io_req = cmd->io_request; - if (fusion->adapter_type >= INVADER_SERIES) { + if (instance->adapter_type >= INVADER_SERIES) { struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL; sgl_ptr_end += fusion->max_sge_in_main_msg - 1; @@ -3287,7 +3936,7 @@ build_mpt_mfi_pass_thru(struct megasas_instance *instance, mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT | MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR; - mpi25_ieee_chain->Length = cpu_to_le32(instance->max_chain_frame_sz); + mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size); } /** @@ -3296,7 +3945,7 @@ build_mpt_mfi_pass_thru(struct megasas_instance *instance, * @cmd: mfi cmd to build * */ -union MEGASAS_REQUEST_DESCRIPTOR_UNION * +static union MEGASAS_REQUEST_DESCRIPTOR_UNION * build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd) { union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL; @@ -3322,7 +3971,7 @@ build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd) * @cmd: mfi cmd pointer * */ -void +static void megasas_issue_dcmd_fusion(struct megasas_instance *instance, struct megasas_cmd *cmd) { @@ -3341,6 +3990,7 @@ megasas_issue_dcmd_fusion(struct megasas_instance *instance, void megasas_release_fusion(struct megasas_instance *instance) { + megasas_free_ioc_init_cmd(instance); megasas_free_cmds(instance); megasas_free_cmds_fusion(instance); @@ -3351,35 +4001,31 @@ megasas_release_fusion(struct megasas_instance *instance) /** * megasas_read_fw_status_reg_fusion - returns the current FW status value - * @regs: MFI register set + * @instance: Adapter soft state */ static u32 -megasas_read_fw_status_reg_fusion(struct megasas_register_set __iomem *regs) +megasas_read_fw_status_reg_fusion(struct megasas_instance *instance) { - return readl(&(regs)->outbound_scratch_pad); + return megasas_readl(instance, &instance->reg_set->outbound_scratch_pad_0); } /** * megasas_alloc_host_crash_buffer - Host buffers for Crash dump collection from Firmware * @instance: Controller's soft instance - * return: Number of allocated host crash buffers + * @return: Number of allocated host crash buffers */ static void megasas_alloc_host_crash_buffer(struct megasas_instance *instance) { unsigned int i; - instance->crash_buf_pages = get_order(CRASH_DMA_BUF_SIZE); for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) { - instance->crash_buf[i] = (void *)__get_free_pages(GFP_KERNEL, - instance->crash_buf_pages); + instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE); if (!instance->crash_buf[i]) { dev_info(&instance->pdev->dev, "Firmware crash dump " "memory allocation failed at index %d\n", i); break; } - memset(instance->crash_buf[i], 0, - ((1 << PAGE_SHIFT) << instance->crash_buf_pages)); } instance->drv_buf_alloc = i; } @@ -3391,12 +4037,9 @@ megasas_alloc_host_crash_buffer(struct megasas_instance *instance) void megasas_free_host_crash_buffer(struct megasas_instance *instance) { - unsigned int i -; + unsigned int i; for (i = 0; i < instance->drv_buf_alloc; i++) { - if (instance->crash_buf[i]) - free_pages((ulong)instance->crash_buf[i], - instance->crash_buf_pages); + vfree(instance->crash_buf[i]); } instance->drv_buf_index = 0; instance->drv_buf_alloc = 0; @@ -3406,6 +4049,7 @@ megasas_free_host_crash_buffer(struct megasas_instance *instance) /** * megasas_adp_reset_fusion - For controller reset + * @instance: Controller's soft instance * @regs: MFI register set */ static int @@ -3424,11 +4068,12 @@ megasas_adp_reset_fusion(struct megasas_instance *instance, writel(MPI2_WRSEQ_6TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset); /* Check that the diag write enable (DRWE) bit is on */ - host_diag = readl(&instance->reg_set->fusion_host_diag); + host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag); retry = 0; while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) { msleep(100); - host_diag = readl(&instance->reg_set->fusion_host_diag); + host_diag = megasas_readl(instance, + &instance->reg_set->fusion_host_diag); if (retry++ == 100) { dev_warn(&instance->pdev->dev, "Host diag unlock failed from %s %d\n", @@ -3445,11 +4090,12 @@ megasas_adp_reset_fusion(struct megasas_instance *instance, msleep(3000); /* Make sure reset adapter bit is cleared */ - host_diag = readl(&instance->reg_set->fusion_host_diag); + host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag); retry = 0; while (host_diag & HOST_DIAG_RESET_ADAPTER) { msleep(100); - host_diag = readl(&instance->reg_set->fusion_host_diag); + host_diag = megasas_readl(instance, + &instance->reg_set->fusion_host_diag); if (retry++ == 1000) { dev_warn(&instance->pdev->dev, "Diag reset adapter never cleared %s %d\n", @@ -3460,14 +4106,14 @@ megasas_adp_reset_fusion(struct megasas_instance *instance, if (host_diag & HOST_DIAG_RESET_ADAPTER) return -1; - abs_state = instance->instancet->read_fw_status_reg(instance->reg_set) + abs_state = instance->instancet->read_fw_status_reg(instance) & MFI_STATE_MASK; retry = 0; while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) { msleep(100); abs_state = instance->instancet-> - read_fw_status_reg(instance->reg_set) & MFI_STATE_MASK; + read_fw_status_reg(instance) & MFI_STATE_MASK; } if (abs_state <= MFI_STATE_FW_INIT) { dev_warn(&instance->pdev->dev, @@ -3481,6 +4127,7 @@ megasas_adp_reset_fusion(struct megasas_instance *instance, /** * megasas_check_reset_fusion - For controller reset check + * @instance: Controller's soft instance * @regs: MFI register set */ static int @@ -3490,21 +4137,68 @@ megasas_check_reset_fusion(struct megasas_instance *instance, return 0; } +/** + * megasas_trigger_snap_dump - Trigger snap dump in FW + * @instance: Soft instance of adapter + */ +static inline void megasas_trigger_snap_dump(struct megasas_instance *instance) +{ + int j; + u32 fw_state, abs_state; + + if (!instance->disableOnlineCtrlReset) { + dev_info(&instance->pdev->dev, "Trigger snap dump\n"); + writel(MFI_ADP_TRIGGER_SNAP_DUMP, + &instance->reg_set->doorbell); + readl(&instance->reg_set->doorbell); + } + + for (j = 0; j < instance->snapdump_wait_time; j++) { + abs_state = instance->instancet->read_fw_status_reg(instance); + fw_state = abs_state & MFI_STATE_MASK; + if (fw_state == MFI_STATE_FAULT) { + dev_printk(KERN_ERR, &instance->pdev->dev, + "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n", + abs_state & MFI_STATE_FAULT_CODE, + abs_state & MFI_STATE_FAULT_SUBCODE, __func__); + return; + } + msleep(1000); + } +} + /* This function waits for outstanding commands on fusion to complete */ -int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, - int reason, int *convert) +static int +megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, + int reason, int *convert) { int i, outstanding, retval = 0, hb_seconds_missed = 0; - u32 fw_state; + u32 fw_state, abs_state; + u32 waittime_for_io_completion; + + waittime_for_io_completion = + min_t(u32, resetwaittime, + (resetwaittime - instance->snapdump_wait_time)); + + if (reason == MFI_IO_TIMEOUT_OCR) { + dev_info(&instance->pdev->dev, + "MFI command is timed out\n"); + megasas_complete_cmd_dpc_fusion((unsigned long)instance); + if (instance->snapdump_wait_time) + megasas_trigger_snap_dump(instance); + retval = 1; + goto out; + } - for (i = 0; i < resetwaittime; i++) { + for (i = 0; i < waittime_for_io_completion; i++) { /* Check if firmware is in fault state */ - fw_state = instance->instancet->read_fw_status_reg( - instance->reg_set) & MFI_STATE_MASK; + abs_state = instance->instancet->read_fw_status_reg(instance); + fw_state = abs_state & MFI_STATE_MASK; if (fw_state == MFI_STATE_FAULT) { - dev_warn(&instance->pdev->dev, "Found FW in FAULT state," - " will reset adapter scsi%d.\n", - instance->host->host_no); + dev_printk(KERN_ERR, &instance->pdev->dev, + "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n", + abs_state & MFI_STATE_FAULT_CODE, + abs_state & MFI_STATE_FAULT_SUBCODE, __func__); megasas_complete_cmd_dpc_fusion((unsigned long)instance); if (instance->requestorId && reason) { dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT" @@ -3518,13 +4212,6 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, goto out; } - if (reason == MFI_IO_TIMEOUT_OCR) { - dev_info(&instance->pdev->dev, - "MFI IO is timed out, initiating OCR\n"); - megasas_complete_cmd_dpc_fusion((unsigned long)instance); - retval = 1; - goto out; - } /* If SR-IOV VF mode & heartbeat timeout, don't wait */ if (instance->requestorId && !reason) { @@ -3556,6 +4243,7 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, } } + megasas_complete_cmd_dpc_fusion((unsigned long)instance); outstanding = atomic_read(&instance->fw_outstanding); if (!outstanding) goto out; @@ -3564,12 +4252,16 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, dev_notice(&instance->pdev->dev, "[%2d]waiting for %d " "commands to complete for scsi%d\n", i, outstanding, instance->host->host_no); - megasas_complete_cmd_dpc_fusion( - (unsigned long)instance); } msleep(1000); } + if (instance->snapdump_wait_time) { + megasas_trigger_snap_dump(instance); + retval = 1; + goto out; + } + if (atomic_read(&instance->fw_outstanding)) { dev_err(&instance->pdev->dev, "pending commands remain after waiting, " "will reset adapter scsi%d.\n", @@ -3577,7 +4269,11 @@ int megasas_wait_for_outstanding_fusion(struct megasas_instance *instance, *convert = 1; retval = 1; } + out: + if (!retval && reason == SCSIIO_TIMEOUT_OCR) + dev_info(&instance->pdev->dev, "IO is completed, no OCR is required\n"); + return retval; } @@ -3589,6 +4285,8 @@ void megasas_reset_reply_desc(struct megasas_instance *instance) fusion = instance->ctrl_context; count = instance->msix_vectors > 0 ? instance->msix_vectors : 1; + count += instance->iopoll_q_count; + for (i = 0 ; i < count ; i++) { fusion->last_reply_idx[i] = 0; reply_desc = fusion->reply_frames_desc[i]; @@ -3601,15 +4299,19 @@ void megasas_reset_reply_desc(struct megasas_instance *instance) * megasas_refire_mgmt_cmd : Re-fire management commands * @instance: Controller's soft instance */ -void megasas_refire_mgmt_cmd(struct megasas_instance *instance) +static void megasas_refire_mgmt_cmd(struct megasas_instance *instance, + bool return_ioctl) { int j; struct megasas_cmd_fusion *cmd_fusion; struct fusion_context *fusion; struct megasas_cmd *cmd_mfi; union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc; + struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req; u16 smid; - bool refire_cmd = 0; + bool refire_cmd = false; + u8 result; + u32 opcode = 0; fusion = instance->ctrl_context; @@ -3620,20 +4322,106 @@ void megasas_refire_mgmt_cmd(struct megasas_instance *instance) cmd_fusion = fusion->cmd_list[j]; cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx]; smid = le16_to_cpu(cmd_mfi->context.smid); + result = REFIRE_CMD; if (!smid) continue; - req_desc = megasas_get_request_descriptor - (instance, smid - 1); - refire_cmd = req_desc && ((cmd_mfi->frame->dcmd.opcode != - cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO)) && - (cmd_mfi->frame->dcmd.opcode != - cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO))) - && !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE); - if (refire_cmd) + + req_desc = megasas_get_request_descriptor(instance, smid - 1); + + switch (cmd_mfi->frame->hdr.cmd) { + case MFI_CMD_DCMD: + opcode = le32_to_cpu(cmd_mfi->frame->dcmd.opcode); + /* Do not refire shutdown command */ + if (opcode == MR_DCMD_CTRL_SHUTDOWN) { + cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK; + result = COMPLETE_CMD; + break; + } + + refire_cmd = ((opcode != MR_DCMD_LD_MAP_GET_INFO)) && + (opcode != MR_DCMD_SYSTEM_PD_MAP_GET_INFO) && + !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE); + + if (!refire_cmd) + result = RETURN_CMD; + + break; + case MFI_CMD_NVME: + if (!instance->support_nvme_passthru) { + cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD; + result = COMPLETE_CMD; + } + + break; + case MFI_CMD_TOOLBOX: + if (!instance->support_pci_lane_margining) { + cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD; + result = COMPLETE_CMD; + } + + break; + default: + break; + } + + if (return_ioctl && cmd_mfi->sync_cmd && + cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT) { + dev_err(&instance->pdev->dev, + "return -EBUSY from %s %d cmd 0x%x opcode 0x%x\n", + __func__, __LINE__, cmd_mfi->frame->hdr.cmd, + le32_to_cpu(cmd_mfi->frame->dcmd.opcode)); + cmd_mfi->cmd_status_drv = DCMD_BUSY; + result = COMPLETE_CMD; + } + + scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *) + cmd_fusion->io_request; + if (scsi_io_req->Function == MPI2_FUNCTION_SCSI_TASK_MGMT) + result = RETURN_CMD; + + switch (result) { + case REFIRE_CMD: megasas_fire_cmd_fusion(instance, req_desc); - else + break; + case RETURN_CMD: + megasas_return_cmd(instance, cmd_mfi); + break; + case COMPLETE_CMD: + megasas_complete_cmd(instance, cmd_mfi, DID_OK); + break; + } + } +} + +/* + * megasas_return_polled_cmds: Return polled mode commands back to the pool + * before initiating an OCR. + * @instance: Controller's soft instance + */ +static void +megasas_return_polled_cmds(struct megasas_instance *instance) +{ + int i; + struct megasas_cmd_fusion *cmd_fusion; + struct fusion_context *fusion; + struct megasas_cmd *cmd_mfi; + + fusion = instance->ctrl_context; + + for (i = instance->max_scsi_cmds; i < instance->max_fw_cmds; i++) { + cmd_fusion = fusion->cmd_list[i]; + cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx]; + + if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) { + if (megasas_dbg_lvl & OCR_DEBUG) + dev_info(&instance->pdev->dev, + "%s %d return cmd 0x%x opcode 0x%x\n", + __func__, __LINE__, cmd_mfi->frame->hdr.cmd, + le32_to_cpu(cmd_mfi->frame->dcmd.opcode)); + cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE; megasas_return_cmd(instance, cmd_mfi); + } } } @@ -3674,8 +4462,8 @@ static int megasas_track_scsiio(struct megasas_instance *instance, /** * megasas_tm_response_code - translation of device response code - * @ioc: per adapter object - * @mpi_reply: MPI reply returned by firmware + * @instance: Controller's soft instance + * @mpi_reply: MPI reply returned by firmware * * Return nothing. */ @@ -3730,9 +4518,9 @@ megasas_tm_response_code(struct megasas_instance *instance, * @device_handle: device handle * @channel: the channel assigned by the OS * @id: the id assigned by the OS - * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c) * @smid_task: smid assigned to the task - * @m_type: TM_MUTEX_ON or TM_MUTEX_OFF + * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c) + * @mr_device_priv_data: private data * Context: user * * MegaRaid use MPT interface for Task Magement request. @@ -3742,7 +4530,8 @@ megasas_tm_response_code(struct megasas_instance *instance, */ static int megasas_issue_tm(struct megasas_instance *instance, u16 device_handle, - uint channel, uint id, u16 smid_task, u8 type) + uint channel, uint id, u16 smid_task, u8 type, + struct MR_PRIV_DEVICE *mr_device_priv_data) { struct MR_TASK_MANAGE_REQUEST *mr_request; struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request; @@ -3750,9 +4539,10 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle, struct megasas_cmd_fusion *cmd_fusion; struct megasas_cmd *cmd_mfi; union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc; - struct fusion_context *fusion; + struct fusion_context *fusion = NULL; struct megasas_cmd_fusion *scsi_lookup; int rc; + int timeout = MEGASAS_DEFAULT_TM_TIMEOUT; struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply; fusion = instance->ctrl_context; @@ -3777,8 +4567,6 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle, cmd_fusion->request_desc = req_desc; req_desc->Words = 0; - scsi_lookup = fusion->cmd_list[smid_task - 1]; - mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request; memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST)); mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest; @@ -3806,12 +4594,20 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle, init_completion(&cmd_fusion->done); megasas_fire_cmd_fusion(instance, req_desc); - timeleft = wait_for_completion_timeout(&cmd_fusion->done, 50 * HZ); + switch (type) { + case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK: + timeout = mr_device_priv_data->task_abort_tmo; + break; + case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET: + timeout = mr_device_priv_data->target_reset_tmo; + break; + } + + timeleft = wait_for_completion_timeout(&cmd_fusion->done, timeout * HZ); if (!timeleft) { dev_err(&instance->pdev->dev, "task mgmt type 0x%x timed out\n", type); - cmd_mfi->flags |= DRV_DCMD_SKIP_REFIRE; mutex_unlock(&instance->reset_mutex); rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR); mutex_lock(&instance->reset_mutex); @@ -3825,14 +4621,15 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle, rc = SUCCESS; switch (type) { case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK: + scsi_lookup = fusion->cmd_list[smid_task - 1]; + if (scsi_lookup->scmd == NULL) break; else { instance->instancet->disable_intr(instance); megasas_sync_irqs((unsigned long)instance); - megasas_complete_cmd_dpc_fusion - ((unsigned long)instance); instance->instancet->enable_intr(instance); + megasas_enable_irq_poll(instance); if (scsi_lookup->scmd == NULL) break; } @@ -3843,11 +4640,10 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle, if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF)) break; instance->instancet->disable_intr(instance); - msleep(1000); - megasas_complete_cmd_dpc_fusion - ((unsigned long)instance); + megasas_sync_irqs((unsigned long)instance); rc = megasas_track_scsiio(instance, id, channel); instance->instancet->enable_intr(instance); + megasas_enable_irq_poll(instance); break; case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET: @@ -3863,7 +4659,7 @@ megasas_issue_tm(struct megasas_instance *instance, u16 device_handle, } /* - * megasas_fusion_smid_lookup : Look for fusion command correpspodning to SCSI + * megasas_fusion_smid_lookup : Look for fusion command corresponding to SCSI * @instance: per adapter struct * * Return Non Zero index, if SMID found in outstanding commands @@ -3940,17 +4736,11 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd) { struct megasas_instance *instance; u16 smid, devhandle; - struct fusion_context *fusion; int ret; struct MR_PRIV_DEVICE *mr_device_priv_data; mr_device_priv_data = scmd->device->hostdata; - instance = (struct megasas_instance *)scmd->device->host->hostdata; - fusion = instance->ctrl_context; - - scmd_printk(KERN_INFO, scmd, "task abort called for scmd(%p)\n", scmd); - scsi_print_command(scmd); if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) { dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL," @@ -3967,7 +4757,6 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd) goto out; } - if (!mr_device_priv_data->is_tm_capable) { ret = FAILED; goto out; @@ -3980,7 +4769,7 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd) if (!smid) { ret = SUCCESS; scmd_printk(KERN_NOTICE, scmd, "Command for which abort is" - " issued is not found in oustanding commands\n"); + " issued is not found in outstanding commands\n"); mutex_unlock(&instance->reset_mutex); goto out; } @@ -3988,26 +4777,30 @@ int megasas_task_abort_fusion(struct scsi_cmnd *scmd) devhandle = megasas_get_tm_devhandle(scmd->device); if (devhandle == (u16)ULONG_MAX) { - ret = SUCCESS; + ret = FAILED; sdev_printk(KERN_INFO, scmd->device, "task abort issued for invalid devhandle\n"); mutex_unlock(&instance->reset_mutex); goto out; } sdev_printk(KERN_INFO, scmd->device, - "attempting task abort! scmd(%p) tm_dev_handle 0x%x\n", + "attempting task abort! scmd(0x%p) tm_dev_handle 0x%x\n", scmd, devhandle); - mr_device_priv_data->tm_busy = 1; + mr_device_priv_data->tm_busy = true; ret = megasas_issue_tm(instance, devhandle, scmd->device->channel, scmd->device->id, smid, - MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK); - mr_device_priv_data->tm_busy = 0; + MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK, + mr_device_priv_data); + mr_device_priv_data->tm_busy = false; mutex_unlock(&instance->reset_mutex); -out: - sdev_printk(KERN_INFO, scmd->device, "task abort: %s scmd(%p)\n", + scmd_printk(KERN_INFO, scmd, "task abort %s!! scmd(0x%p)\n", ((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd); +out: + scsi_print_command(scmd); + if (megasas_dbg_lvl & TM_DEBUG) + megasas_dump_fusion_io(scmd); return ret; } @@ -4025,15 +4818,10 @@ int megasas_reset_target_fusion(struct scsi_cmnd *scmd) struct megasas_instance *instance; int ret = FAILED; u16 devhandle; - struct fusion_context *fusion; struct MR_PRIV_DEVICE *mr_device_priv_data; mr_device_priv_data = scmd->device->hostdata; instance = (struct megasas_instance *)scmd->device->host->hostdata; - fusion = instance->ctrl_context; - - sdev_printk(KERN_INFO, scmd->device, - "target reset called for scmd(%p)\n", scmd); if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) { dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL," @@ -4043,14 +4831,13 @@ int megasas_reset_target_fusion(struct scsi_cmnd *scmd) } if (!mr_device_priv_data) { - sdev_printk(KERN_INFO, scmd->device, "device been deleted! " - "scmd(%p)\n", scmd); + sdev_printk(KERN_INFO, scmd->device, + "device been deleted! scmd: (0x%p)\n", scmd); scmd->result = DID_NO_CONNECT << 16; ret = SUCCESS; goto out; } - if (!mr_device_priv_data->is_tm_capable) { ret = FAILED; goto out; @@ -4060,7 +4847,7 @@ int megasas_reset_target_fusion(struct scsi_cmnd *scmd) devhandle = megasas_get_tm_devhandle(scmd->device); if (devhandle == (u16)ULONG_MAX) { - ret = SUCCESS; + ret = FAILED; sdev_printk(KERN_INFO, scmd->device, "target reset issued for invalid devhandle\n"); mutex_unlock(&instance->reset_mutex); @@ -4068,23 +4855,25 @@ int megasas_reset_target_fusion(struct scsi_cmnd *scmd) } sdev_printk(KERN_INFO, scmd->device, - "attempting target reset! scmd(%p) tm_dev_handle 0x%x\n", + "attempting target reset! scmd(0x%p) tm_dev_handle: 0x%x\n", scmd, devhandle); - mr_device_priv_data->tm_busy = 1; + mr_device_priv_data->tm_busy = true; ret = megasas_issue_tm(instance, devhandle, scmd->device->channel, scmd->device->id, 0, - MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET); - mr_device_priv_data->tm_busy = 0; + MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET, + mr_device_priv_data); + mr_device_priv_data->tm_busy = false; mutex_unlock(&instance->reset_mutex); -out: - scmd_printk(KERN_NOTICE, scmd, "megasas: target reset %s!!\n", + scmd_printk(KERN_NOTICE, scmd, "target reset %s!!\n", (ret == SUCCESS) ? "SUCCESS" : "FAILED"); +out: return ret; } /*SRIOV get other instance in cluster if any*/ -struct megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance) +static struct +megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance) { int i; @@ -4124,10 +4913,14 @@ int megasas_reset_fusion(struct Scsi_Host *shost, int reason) struct megasas_instance *instance; struct megasas_cmd_fusion *cmd_fusion, *r1_cmd; struct fusion_context *fusion; - u32 abs_state, status_reg, reset_adapter; + u32 abs_state, status_reg, reset_adapter, fpio_count = 0; u32 io_timeout_in_crash_mode = 0; struct scsi_cmnd *scmd_local = NULL; struct scsi_device *sdev; + int ret_target_prop = DCMD_FAILED; + bool is_target_prop = false; + bool do_adp_reset = true; + int max_reset_tries = MEGASAS_FUSION_MAX_RESET_TRIES; instance = (struct megasas_instance *)shost->hostdata; fusion = instance->ctrl_context; @@ -4141,7 +4934,7 @@ int megasas_reset_fusion(struct Scsi_Host *shost, int reason) mutex_unlock(&instance->reset_mutex); return FAILED; } - status_reg = instance->instancet->read_fw_status_reg(instance->reg_set); + status_reg = instance->instancet->read_fw_status_reg(instance); abs_state = status_reg & MFI_STATE_MASK; /* IO timeout detected, forcibly put FW in FAULT state */ @@ -4150,7 +4943,7 @@ int megasas_reset_fusion(struct Scsi_Host *shost, int reason) dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, " "forcibly FAULT Firmware\n"); atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT); - status_reg = readl(&instance->reg_set->doorbell); + status_reg = megasas_readl(instance, &instance->reg_set->doorbell); writel(status_reg | MFI_STATE_FORCE_OCR, &instance->reg_set->doorbell); readl(&instance->reg_set->doorbell); @@ -4179,8 +4972,9 @@ int megasas_reset_fusion(struct Scsi_Host *shost, int reason) } if (instance->requestorId && !instance->skip_heartbeat_timer_del) - del_timer_sync(&instance->sriov_heartbeat_timer); + timer_delete_sync(&instance->sriov_heartbeat_timer); set_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags); + set_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags); atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_POLLING); instance->instancet->disable_intr(instance); megasas_sync_irqs((unsigned long)instance); @@ -4194,26 +4988,30 @@ int megasas_reset_fusion(struct Scsi_Host *shost, int reason) if (convert) reason = 0; - if (megasas_dbg_lvl & OCR_LOGS) + if (megasas_dbg_lvl & OCR_DEBUG) dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n"); /* Now return commands back to the OS */ for (i = 0 ; i < instance->max_scsi_cmds; i++) { cmd_fusion = fusion->cmd_list[i]; /*check for extra commands issued by driver*/ - if (instance->is_ventura) { + if (instance->adapter_type >= VENTURA_SERIES) { r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds]; megasas_return_cmd_fusion(instance, r1_cmd); } scmd_local = cmd_fusion->scmd; if (cmd_fusion->scmd) { - if (megasas_dbg_lvl & OCR_LOGS) { + if (megasas_dbg_lvl & OCR_DEBUG) { sdev_printk(KERN_INFO, cmd_fusion->scmd->device, "SMID: 0x%x\n", cmd_fusion->index); - scsi_print_command(cmd_fusion->scmd); + megasas_dump_fusion_io(cmd_fusion->scmd); } + if (cmd_fusion->io_request->Function == + MPI2_FUNCTION_SCSI_IO_REQUEST) + fpio_count++; + scmd_local->result = megasas_check_mpio_paths(instance, scmd_local); @@ -4222,14 +5020,16 @@ int megasas_reset_fusion(struct Scsi_Host *shost, int reason) atomic_dec(&instance->ldio_outstanding); megasas_return_cmd_fusion(instance, cmd_fusion); scsi_dma_unmap(scmd_local); - scmd_local->scsi_done(scmd_local); + scsi_done(scmd_local); } } + dev_info(&instance->pdev->dev, "Outstanding fastpath IOs: %d\n", + fpio_count); + atomic_set(&instance->fw_outstanding, 0); - status_reg = instance->instancet->read_fw_status_reg( - instance->reg_set); + status_reg = instance->instancet->read_fw_status_reg(instance); abs_state = status_reg & MFI_STATE_MASK; reset_adapter = status_reg & MFI_RESET_ADAPTER; if (instance->disableOnlineCtrlReset || @@ -4238,78 +5038,76 @@ int megasas_reset_fusion(struct Scsi_Host *shost, int reason) dev_warn(&instance->pdev->dev, "Reset not supported" ", killing adapter scsi%d.\n", instance->host->host_no); - megaraid_sas_kill_hba(instance); - instance->skip_heartbeat_timer_del = 1; - retval = FAILED; - goto out; + goto kill_hba; } /* Let SR-IOV VF & PF sync up if there was a HB failure */ if (instance->requestorId && !reason) { msleep(MEGASAS_OCR_SETTLE_TIME_VF); - goto transition_to_ready; + do_adp_reset = false; + max_reset_tries = MEGASAS_SRIOV_MAX_RESET_TRIES_VF; } /* Now try to reset the chip */ - for (i = 0; i < MEGASAS_FUSION_MAX_RESET_TRIES; i++) { - - if (instance->instancet->adp_reset - (instance, instance->reg_set)) + for (i = 0; i < max_reset_tries; i++) { + /* + * Do adp reset and wait for + * controller to transition to ready + */ + if (megasas_adp_reset_wait_for_ready(instance, + do_adp_reset, 1) == FAILED) continue; -transition_to_ready: + /* Wait for FW to become ready */ if (megasas_transition_to_ready(instance, 1)) { dev_warn(&instance->pdev->dev, "Failed to transition controller to ready for " "scsi%d.\n", instance->host->host_no); - if (instance->requestorId && !reason) - goto fail_kill_adapter; - else - continue; + continue; } megasas_reset_reply_desc(instance); megasas_fusion_update_can_queue(instance, OCR_CONTEXT); if (megasas_ioc_init_fusion(instance)) { - dev_warn(&instance->pdev->dev, - "megasas_ioc_init_fusion() failed! for " - "scsi%d\n", instance->host->host_no); - if (instance->requestorId && !reason) - goto fail_kill_adapter; - else - continue; + continue; } - megasas_refire_mgmt_cmd(instance); - if (megasas_get_ctrl_info(instance)) { dev_info(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); - megaraid_sas_kill_hba(instance); - retval = FAILED; - goto out; + goto kill_hba; } + + megasas_refire_mgmt_cmd(instance, + (i == (MEGASAS_FUSION_MAX_RESET_TRIES - 1) + ? 1 : 0)); + /* Reset load balance info */ if (fusion->load_balance_info) memset(fusion->load_balance_info, 0, (sizeof(struct LD_LOAD_BALANCE_INFO) * MAX_LOGICAL_DRIVES_EXT)); - if (!megasas_get_map_info(instance)) + if (!megasas_get_map_info(instance)) { megasas_sync_map_info(instance); + } else { + /* + * Return pending polled mode cmds before + * retrying OCR + */ + megasas_return_polled_cmds(instance); + continue; + } megasas_setup_jbod_map(instance); - shost_for_each_device(sdev, shost) - megasas_set_dynamic_target_properties(sdev); - /* reset stream detection array */ - if (instance->is_ventura) { + if (instance->adapter_type >= VENTURA_SERIES) { for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) { memset(fusion->stream_detect_by_ld[j], - 0, sizeof(struct LD_STREAM_DETECT)); - fusion->stream_detect_by_ld[j]->mru_bit_map + 0, sizeof(struct LD_STREAM_DETECT)); + fusion->stream_detect_by_ld[j]->mru_bit_map = MR_STREAM_BITMAP; } } @@ -4317,24 +5115,40 @@ transition_to_ready: clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags); instance->instancet->enable_intr(instance); + megasas_enable_irq_poll(instance); + shost_for_each_device(sdev, shost) { + if ((instance->tgt_prop) && + (instance->nvme_page_size)) + ret_target_prop = megasas_get_target_prop(instance, sdev); + + is_target_prop = (ret_target_prop == DCMD_SUCCESS) ? true : false; + megasas_set_dynamic_target_properties(sdev, NULL, + is_target_prop); + } + + status_reg = instance->instancet->read_fw_status_reg + (instance); + abs_state = status_reg & MFI_STATE_MASK; + if (abs_state != MFI_STATE_OPERATIONAL) { + dev_info(&instance->pdev->dev, + "Adapter is not OPERATIONAL, state 0x%x for scsi:%d\n", + abs_state, instance->host->host_no); + goto out; + } atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL); + dev_info(&instance->pdev->dev, + "Adapter is OPERATIONAL for scsi:%d\n", + instance->host->host_no); + /* Restart SR-IOV heartbeat */ if (instance->requestorId) { if (!megasas_sriov_start_heartbeat(instance, 0)) - megasas_start_timer(instance, - &instance->sriov_heartbeat_timer, - megasas_sriov_heartbeat_handler, - MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF); + megasas_start_timer(instance); else instance->skip_heartbeat_timer_del = 1; } - /* Adapter reset completed successfully */ - dev_warn(&instance->pdev->dev, "Reset " - "successful for scsi%d.\n", - instance->host->host_no); - if (instance->crash_dump_drv_support && instance->crash_dump_app_support) megasas_set_crash_dump_params(instance, @@ -4343,44 +5157,57 @@ transition_to_ready: megasas_set_crash_dump_params(instance, MR_CRASH_BUF_TURN_OFF); + if (instance->snapdump_wait_time) { + megasas_get_snapdump_properties(instance); + dev_info(&instance->pdev->dev, + "Snap dump wait time\t: %d\n", + instance->snapdump_wait_time); + } + retval = SUCCESS; + + /* Adapter reset completed successfully */ + dev_warn(&instance->pdev->dev, + "Reset successful for scsi%d.\n", + instance->host->host_no); + goto out; } -fail_kill_adapter: /* Reset failed, kill the adapter */ dev_warn(&instance->pdev->dev, "Reset failed, killing " "adapter scsi%d.\n", instance->host->host_no); - megaraid_sas_kill_hba(instance); - instance->skip_heartbeat_timer_del = 1; - retval = FAILED; + goto kill_hba; } else { /* For VF: Restart HB timer if we didn't OCR */ if (instance->requestorId) { - megasas_start_timer(instance, - &instance->sriov_heartbeat_timer, - megasas_sriov_heartbeat_handler, - MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF); + megasas_start_timer(instance); } clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags); instance->instancet->enable_intr(instance); + megasas_enable_irq_poll(instance); atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL); + goto out; } +kill_hba: + megaraid_sas_kill_hba(instance); + megasas_enable_irq_poll(instance); + instance->skip_heartbeat_timer_del = 1; + retval = FAILED; out: - clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags); + clear_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags); mutex_unlock(&instance->reset_mutex); return retval; } -/* Fusion Crash dump collection work queue */ -void megasas_fusion_crash_dump_wq(struct work_struct *work) +/* Fusion Crash dump collection */ +static void megasas_fusion_crash_dump(struct megasas_instance *instance) { - struct megasas_instance *instance = - container_of(work, struct megasas_instance, crash_init); u32 status_reg; u8 partial_copy = 0; + int wait = 0; - status_reg = instance->instancet->read_fw_status_reg(instance->reg_set); + status_reg = instance->instancet->read_fw_status_reg(instance); /* * Allocate host crash buffers to copy data from 1 MB DMA crash buffer @@ -4396,8 +5223,8 @@ void megasas_fusion_crash_dump_wq(struct work_struct *work) "crash dump and initiating OCR\n"); status_reg |= MFI_STATE_CRASH_DUMP_DONE; writel(status_reg, - &instance->reg_set->outbound_scratch_pad); - readl(&instance->reg_set->outbound_scratch_pad); + &instance->reg_set->outbound_scratch_pad_0); + readl(&instance->reg_set->outbound_scratch_pad_0); return; } megasas_alloc_host_crash_buffer(instance); @@ -4405,21 +5232,41 @@ void megasas_fusion_crash_dump_wq(struct work_struct *work) "allocated: %d\n", instance->drv_buf_alloc); } - /* - * Driver has allocated max buffers, which can be allocated - * and FW has more crash dump data, then driver will - * ignore the data. - */ - if (instance->drv_buf_index >= (instance->drv_buf_alloc)) { - dev_info(&instance->pdev->dev, "Driver is done copying " - "the buffer: %d\n", instance->drv_buf_alloc); - status_reg |= MFI_STATE_CRASH_DUMP_DONE; - partial_copy = 1; - } else { - memcpy(instance->crash_buf[instance->drv_buf_index], - instance->crash_dump_buf, CRASH_DMA_BUF_SIZE); - instance->drv_buf_index++; - status_reg &= ~MFI_STATE_DMADONE; + while (!(status_reg & MFI_STATE_CRASH_DUMP_DONE) && + (wait < MEGASAS_WATCHDOG_WAIT_COUNT)) { + if (!(status_reg & MFI_STATE_DMADONE)) { + /* + * Next crash dump buffer is not yet DMA'd by FW + * Check after 10ms. Wait for 1 second for FW to + * post the next buffer. If not bail out. + */ + wait++; + msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS); + status_reg = instance->instancet->read_fw_status_reg( + instance); + continue; + } + + wait = 0; + if (instance->drv_buf_index >= instance->drv_buf_alloc) { + dev_info(&instance->pdev->dev, + "Driver is done copying the buffer: %d\n", + instance->drv_buf_alloc); + status_reg |= MFI_STATE_CRASH_DUMP_DONE; + partial_copy = 1; + break; + } else { + memcpy(instance->crash_buf[instance->drv_buf_index], + instance->crash_dump_buf, CRASH_DMA_BUF_SIZE); + instance->drv_buf_index++; + status_reg &= ~MFI_STATE_DMADONE; + } + + writel(status_reg, &instance->reg_set->outbound_scratch_pad_0); + readl(&instance->reg_set->outbound_scratch_pad_0); + + msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS); + status_reg = instance->instancet->read_fw_status_reg(instance); } if (status_reg & MFI_STATE_CRASH_DUMP_DONE) { @@ -4428,13 +5275,10 @@ void megasas_fusion_crash_dump_wq(struct work_struct *work) instance->fw_crash_buffer_size = instance->drv_buf_index; instance->fw_crash_state = AVAILABLE; instance->drv_buf_index = 0; - writel(status_reg, &instance->reg_set->outbound_scratch_pad); - readl(&instance->reg_set->outbound_scratch_pad); + writel(status_reg, &instance->reg_set->outbound_scratch_pad_0); + readl(&instance->reg_set->outbound_scratch_pad_0); if (!partial_copy) megasas_reset_fusion(instance->host, 0); - } else { - writel(status_reg, &instance->reg_set->outbound_scratch_pad); - readl(&instance->reg_set->outbound_scratch_pad); } } @@ -4454,28 +5298,41 @@ megasas_alloc_fusion_context(struct megasas_instance *instance) { struct fusion_context *fusion; - instance->ctrl_context_pages = get_order(sizeof(struct fusion_context)); - instance->ctrl_context = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - instance->ctrl_context_pages); + instance->ctrl_context = kzalloc(sizeof(struct fusion_context), + GFP_KERNEL); if (!instance->ctrl_context) { - /* fall back to using vmalloc for fusion_context */ - instance->ctrl_context = vzalloc(sizeof(struct fusion_context)); - if (!instance->ctrl_context) { - dev_err(&instance->pdev->dev, "Failed from %s %d\n", __func__, __LINE__); - return -ENOMEM; - } + dev_err(&instance->pdev->dev, "Failed from %s %d\n", + __func__, __LINE__); + return -ENOMEM; } fusion = instance->ctrl_context; + fusion->log_to_span_pages = get_order(MAX_LOGICAL_DRIVES_EXT * + sizeof(LD_SPAN_INFO)); + fusion->log_to_span = + (PLD_SPAN_INFO)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + fusion->log_to_span_pages); + if (!fusion->log_to_span) { + fusion->log_to_span = + vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT, + sizeof(LD_SPAN_INFO))); + if (!fusion->log_to_span) { + dev_err(&instance->pdev->dev, "Failed from %s %d\n", + __func__, __LINE__); + return -ENOMEM; + } + } + fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT * sizeof(struct LD_LOAD_BALANCE_INFO)); fusion->load_balance_info = (struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, fusion->load_balance_info_pages); if (!fusion->load_balance_info) { - fusion->load_balance_info = vzalloc(MAX_LOGICAL_DRIVES_EXT * - sizeof(struct LD_LOAD_BALANCE_INFO)); + fusion->load_balance_info = + vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT, + sizeof(struct LD_LOAD_BALANCE_INFO))); if (!fusion->load_balance_info) dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, " "continuing without Load Balance support\n"); @@ -4498,11 +5355,15 @@ megasas_free_fusion_context(struct megasas_instance *instance) fusion->load_balance_info_pages); } - if (is_vmalloc_addr(fusion)) - vfree(fusion); - else - free_pages((ulong)fusion, - instance->ctrl_context_pages); + if (fusion->log_to_span) { + if (is_vmalloc_addr(fusion->log_to_span)) + vfree(fusion->log_to_span); + else + free_pages((ulong)fusion->log_to_span, + fusion->log_to_span_pages); + } + + kfree(fusion); } } |