diff options
Diffstat (limited to 'drivers/scsi/lpfc/lpfc_sli.c')
| -rw-r--r-- | drivers/scsi/lpfc/lpfc_sli.c | 10101 |
1 files changed, 6580 insertions, 3521 deletions
diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 2242e9b3ca12..73d77cfab5f8 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -1,8 +1,8 @@ /******************************************************************* * This file is part of the Emulex Linux Device Driver for * * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2018 Broadcom. All Rights Reserved. The term * - * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * + * Copyright (C) 2017-2025 Broadcom. All Rights Reserved. The term * + * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * * Copyright (C) 2004-2016 Emulex. All rights reserved. * * EMULEX and SLI are trademarks of Emulex. * * www.broadcom.com * @@ -27,6 +27,8 @@ #include <linux/delay.h> #include <linux/slab.h> #include <linux/lockdep.h> +#include <linux/dmi.h> +#include <linux/of.h> #include <scsi/scsi.h> #include <scsi/scsi_cmnd.h> @@ -34,13 +36,11 @@ #include <scsi/scsi_host.h> #include <scsi/scsi_transport_fc.h> #include <scsi/fc/fc_fs.h> -#include <linux/aer.h> +#include <linux/crash_dump.h> #ifdef CONFIG_X86 #include <asm/set_memory.h> #endif -#include <linux/nvme-fc-driver.h> - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_sli.h" @@ -50,7 +50,6 @@ #include "lpfc.h" #include "lpfc_scsi.h" #include "lpfc_nvme.h" -#include "lpfc_nvmet.h" #include "lpfc_crtn.h" #include "lpfc_logmsg.h" #include "lpfc_compat.h" @@ -72,28 +71,155 @@ static int lpfc_sli_issue_mbox_s4(struct lpfc_hba *, LPFC_MBOXQ_t *, uint32_t); static int lpfc_sli4_read_rev(struct lpfc_hba *, LPFC_MBOXQ_t *, uint8_t *, uint32_t *); -static struct lpfc_iocbq *lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *, - struct lpfc_iocbq *); +static struct lpfc_iocbq * +lpfc_sli4_els_preprocess_rspiocbq(struct lpfc_hba *phba, + struct lpfc_iocbq *rspiocbq); static void lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *, struct hbq_dmabuf *); static void lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf); -static int lpfc_sli4_fp_handle_cqe(struct lpfc_hba *, struct lpfc_queue *, - struct lpfc_cqe *); +static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, + struct lpfc_queue *cq, struct lpfc_cqe *cqe); static int lpfc_sli4_post_sgl_list(struct lpfc_hba *, struct list_head *, int); static void lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, - struct lpfc_eqe *eqe, uint32_t qidx); + struct lpfc_queue *eq, + struct lpfc_eqe *eqe, + enum lpfc_poll_mode poll_mode); static bool lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba); static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba); -static int lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, - struct lpfc_sli_ring *pring, - struct lpfc_iocbq *cmdiocb); +static struct lpfc_cqe *lpfc_sli4_cq_get(struct lpfc_queue *q); +static void __lpfc_sli4_consume_cqe(struct lpfc_hba *phba, + struct lpfc_queue *cq, + struct lpfc_cqe *cqe); +static uint16_t lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, + struct lpfc_iocbq *pwqeq, + struct lpfc_sglq *sglq); -static IOCB_t * -lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq) +union lpfc_wqe128 lpfc_iread_cmd_template; +union lpfc_wqe128 lpfc_iwrite_cmd_template; +union lpfc_wqe128 lpfc_icmnd_cmd_template; + +/* Setup WQE templates for IOs */ +void lpfc_wqe_cmd_template(void) { - return &iocbq->iocb; + union lpfc_wqe128 *wqe; + + /* IREAD template */ + wqe = &lpfc_iread_cmd_template; + memset(wqe, 0, sizeof(union lpfc_wqe128)); + + /* Word 0, 1, 2 - BDE is variable */ + + /* Word 3 - cmd_buff_len, payload_offset_len is zero */ + + /* Word 4 - total_xfer_len is variable */ + + /* Word 5 - is zero */ + + /* Word 6 - ctxt_tag, xri_tag is variable */ + + /* Word 7 */ + bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); + bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); + bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); + bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); + + /* Word 8 - abort_tag is variable */ + + /* Word 9 - reqtag is variable */ + + /* Word 10 - dbde, wqes is variable */ + bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); + bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); + bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); + bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); + bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); + + /* Word 11 - pbde is variable */ + bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, COMMAND_DATA_IN); + bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); + bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0); + + /* Word 12 - is zero */ + + /* Word 13, 14, 15 - PBDE is variable */ + + /* IWRITE template */ + wqe = &lpfc_iwrite_cmd_template; + memset(wqe, 0, sizeof(union lpfc_wqe128)); + + /* Word 0, 1, 2 - BDE is variable */ + + /* Word 3 - cmd_buff_len, payload_offset_len is zero */ + + /* Word 4 - total_xfer_len is variable */ + + /* Word 5 - initial_xfer_len is variable */ + + /* Word 6 - ctxt_tag, xri_tag is variable */ + + /* Word 7 */ + bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); + bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); + bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); + bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); + + /* Word 8 - abort_tag is variable */ + + /* Word 9 - reqtag is variable */ + + /* Word 10 - dbde, wqes is variable */ + bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); + bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); + bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); + bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); + bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); + + /* Word 11 - pbde is variable */ + bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, COMMAND_DATA_OUT); + bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); + bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0); + + /* Word 12 - is zero */ + + /* Word 13, 14, 15 - PBDE is variable */ + + /* ICMND template */ + wqe = &lpfc_icmnd_cmd_template; + memset(wqe, 0, sizeof(union lpfc_wqe128)); + + /* Word 0, 1, 2 - BDE is variable */ + + /* Word 3 - payload_offset_len is variable */ + + /* Word 4, 5 - is zero */ + + /* Word 6 - ctxt_tag, xri_tag is variable */ + + /* Word 7 */ + bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); + bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); + bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); + bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); + + /* Word 8 - abort_tag is variable */ + + /* Word 9 - reqtag is variable */ + + /* Word 10 - dbde, wqes is variable */ + bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); + bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); + bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); + bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); + bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); + + /* Word 11 */ + bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, COMMAND_DATA_IN); + bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); + bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); + + /* Word 12, 13, 14, 15 - is zero */ } #if defined(CONFIG_64BIT) && defined(__LITTLE_ENDIAN) @@ -110,7 +236,7 @@ lpfc_get_iocb_from_iocbq(struct lpfc_iocbq *iocbq) * endianness. This function can be called with or without * lock. **/ -void +static void lpfc_sli4_pcimem_bcopy(void *srcp, void *destp, uint32_t cnt) { uint64_t *src = srcp; @@ -150,7 +276,8 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe) /* sanity check on queue memory */ if (unlikely(!q)) return -ENOMEM; - temp_wqe = q->qe[q->host_index].wqe; + + temp_wqe = lpfc_sli4_qe(q, q->host_index); /* If the host has not yet processed the next entry then we are done */ idx = ((q->host_index + 1) % q->entry_count); @@ -160,7 +287,7 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe) } q->WQ_posted++; /* set consumption flag every once in a while */ - if (!((q->host_index + 1) % q->entry_repost)) + if (!((q->host_index + 1) % q->notify_interval)) bf_set(wqe_wqec, &wqe->generic.wqe_com, 1); else bf_set(wqe_wqec, &wqe->generic.wqe_com, 0); @@ -228,31 +355,22 @@ lpfc_sli4_wq_put(struct lpfc_queue *q, union lpfc_wqe128 *wqe) * This routine will update the HBA index of a queue to reflect consumption of * Work Queue Entries by the HBA. When the HBA indicates that it has consumed * an entry the host calls this function to update the queue's internal - * pointers. This routine returns the number of entries that were consumed by - * the HBA. + * pointers. **/ -static uint32_t +static void lpfc_sli4_wq_release(struct lpfc_queue *q, uint32_t index) { - uint32_t released = 0; - /* sanity check on queue memory */ if (unlikely(!q)) - return 0; + return; - if (q->hba_index == index) - return 0; - do { - q->hba_index = ((q->hba_index + 1) % q->entry_count); - released++; - } while (q->hba_index != index); - return released; + q->hba_index = index; } /** * lpfc_sli4_mq_put - Put a Mailbox Queue Entry on an Mailbox Queue * @q: The Mailbox Queue to operate on. - * @wqe: The Mailbox Queue Entry to put on the Work queue. + * @mqe: The Mailbox Queue Entry to put on the Work queue. * * This routine will copy the contents of @mqe to the next available entry on * the @q. This function will then ring the Work Queue Doorbell to signal the @@ -270,7 +388,7 @@ lpfc_sli4_mq_put(struct lpfc_queue *q, struct lpfc_mqe *mqe) /* sanity check on queue memory */ if (unlikely(!q)) return -ENOMEM; - temp_mqe = q->qe[q->host_index].mqe; + temp_mqe = lpfc_sli4_qe(q, q->host_index); /* If the host has not yet processed the next entry then we are done */ if (((q->host_index + 1) % q->entry_count) == q->hba_index) @@ -325,29 +443,16 @@ lpfc_sli4_mq_release(struct lpfc_queue *q) static struct lpfc_eqe * lpfc_sli4_eq_get(struct lpfc_queue *q) { - struct lpfc_hba *phba; struct lpfc_eqe *eqe; - uint32_t idx; /* sanity check on queue memory */ if (unlikely(!q)) return NULL; - phba = q->phba; - eqe = q->qe[q->hba_index].eqe; + eqe = lpfc_sli4_qe(q, q->host_index); /* If the next EQE is not valid then we are done */ if (bf_get_le32(lpfc_eqe_valid, eqe) != q->qe_valid) return NULL; - /* If the host has not yet processed the next entry then we are done */ - idx = ((q->hba_index + 1) % q->entry_count); - if (idx == q->host_index) - return NULL; - - q->hba_index = idx; - /* if the index wrapped around, toggle the valid bit */ - if (phba->sli4_hba.pc_sli4_params.eqav && !q->hba_index) - q->qe_valid = (q->qe_valid) ? 0 : 1; - /* * insert barrier for instruction interlock : data from the hardware @@ -367,7 +472,7 @@ lpfc_sli4_eq_get(struct lpfc_queue *q) * @q: The Event Queue to disable interrupts * **/ -inline void +void lpfc_sli4_eq_clr_intr(struct lpfc_queue *q) { struct lpfc_register doorbell; @@ -386,7 +491,7 @@ lpfc_sli4_eq_clr_intr(struct lpfc_queue *q) * @q: The Event Queue to disable interrupts * **/ -inline void +void lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q) { struct lpfc_register doorbell; @@ -397,44 +502,25 @@ lpfc_sli4_if6_eq_clr_intr(struct lpfc_queue *q) } /** - * lpfc_sli4_eq_release - Indicates the host has finished processing an EQ + * lpfc_sli4_write_eq_db - write EQ DB for eqe's consumed or arm state + * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. + * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Event Queue Entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the EQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of EQEs that were popped. + * This routine will notify the HBA, by ringing the doorbell, that count + * number of EQEs have been processed. The @arm parameter indicates whether + * the queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_eqe *temp_eqe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.eqav) { - temp_eqe = q->qe[q->host_index].eqe; - bf_set_le32(lpfc_eqe_valid, temp_eqe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; @@ -442,7 +528,7 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1); bf_set(lpfc_eqcq_doorbell_eqci, &doorbell, 1); } - bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count); bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_EVENT); bf_set(lpfc_eqcq_doorbell_eqid_hi, &doorbell, (q->queue_id >> LPFC_EQID_HI_FIELD_SHIFT)); @@ -451,60 +537,139 @@ lpfc_sli4_eq_release(struct lpfc_queue *q, bool arm) /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM)) readl(q->phba->sli4_hba.EQDBregaddr); - return released; } /** - * lpfc_sli4_if6_eq_release - Indicates the host has finished processing an EQ + * lpfc_sli4_if6_write_eq_db - write EQ DB for eqe's consumed or arm state + * @phba: adapter with EQ * @q: The Event Queue that the host has completed processing for. + * @count: Number of elements that have been consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Event Queue Entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the EQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of EQEs that were popped. + * This routine will notify the HBA, by ringing the doorbell, that count + * number of EQEs have been processed. The @arm parameter indicates whether + * the queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_if6_write_eq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_eqe *temp_eqe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.eqav) { - temp_eqe = q->qe[q->host_index].eqe; - bf_set_le32(lpfc_eqe_valid, temp_eqe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_if6_eq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_if6_eq_doorbell_num_released, &doorbell, count); bf_set(lpfc_if6_eq_doorbell_eqid, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.EQDBregaddr); /* PCI read to flush PCI pipeline on re-arming for INTx mode */ if ((q->phba->intr_type == INTx) && (arm == LPFC_QUEUE_REARM)) readl(q->phba->sli4_hba.EQDBregaddr); - return released; +} + +static void +__lpfc_sli4_consume_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq, + struct lpfc_eqe *eqe) +{ + if (!phba->sli4_hba.pc_sli4_params.eqav) + bf_set_le32(lpfc_eqe_valid, eqe, 0); + + eq->host_index = ((eq->host_index + 1) % eq->entry_count); + + /* if the index wrapped around, toggle the valid bit */ + if (phba->sli4_hba.pc_sli4_params.eqav && !eq->host_index) + eq->qe_valid = (eq->qe_valid) ? 0 : 1; +} + +static void +lpfc_sli4_eqcq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) +{ + struct lpfc_eqe *eqe = NULL; + u32 eq_count = 0, cq_count = 0; + struct lpfc_cqe *cqe = NULL; + struct lpfc_queue *cq = NULL, *childq = NULL; + int cqid = 0; + + /* walk all the EQ entries and drop on the floor */ + eqe = lpfc_sli4_eq_get(eq); + while (eqe) { + /* Get the reference to the corresponding CQ */ + cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); + cq = NULL; + + list_for_each_entry(childq, &eq->child_list, list) { + if (childq->queue_id == cqid) { + cq = childq; + break; + } + } + /* If CQ is valid, iterate through it and drop all the CQEs */ + if (cq) { + cqe = lpfc_sli4_cq_get(cq); + while (cqe) { + __lpfc_sli4_consume_cqe(phba, cq, cqe); + cq_count++; + cqe = lpfc_sli4_cq_get(cq); + } + /* Clear and re-arm the CQ */ + phba->sli4_hba.sli4_write_cq_db(phba, cq, cq_count, + LPFC_QUEUE_REARM); + cq_count = 0; + } + __lpfc_sli4_consume_eqe(phba, eq, eqe); + eq_count++; + eqe = lpfc_sli4_eq_get(eq); + } + + /* Clear and re-arm the EQ */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, eq_count, LPFC_QUEUE_REARM); +} + +static int +lpfc_sli4_process_eq(struct lpfc_hba *phba, struct lpfc_queue *eq, + u8 rearm, enum lpfc_poll_mode poll_mode) +{ + struct lpfc_eqe *eqe; + int count = 0, consumed = 0; + + if (cmpxchg(&eq->queue_claimed, 0, 1) != 0) + goto rearm_and_exit; + + eqe = lpfc_sli4_eq_get(eq); + while (eqe) { + lpfc_sli4_hba_handle_eqe(phba, eq, eqe, poll_mode); + __lpfc_sli4_consume_eqe(phba, eq, eqe); + + consumed++; + if (!(++count % eq->max_proc_limit)) + break; + + if (!(count % eq->notify_interval)) { + phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, + LPFC_QUEUE_NOARM); + consumed = 0; + } + + eqe = lpfc_sli4_eq_get(eq); + } + eq->EQ_processed += count; + + /* Track the max number of EQEs processed in 1 intr */ + if (count > eq->EQ_max_eqe) + eq->EQ_max_eqe = count; + + xchg(&eq->queue_claimed, 0); + +rearm_and_exit: + /* Always clear the EQ. */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, consumed, rearm); + + return count; } /** @@ -519,28 +684,16 @@ lpfc_sli4_if6_eq_release(struct lpfc_queue *q, bool arm) static struct lpfc_cqe * lpfc_sli4_cq_get(struct lpfc_queue *q) { - struct lpfc_hba *phba; struct lpfc_cqe *cqe; - uint32_t idx; /* sanity check on queue memory */ if (unlikely(!q)) return NULL; - phba = q->phba; - cqe = q->qe[q->hba_index].cqe; + cqe = lpfc_sli4_qe(q, q->host_index); /* If the next CQE is not valid then we are done */ if (bf_get_le32(lpfc_cqe_valid, cqe) != q->qe_valid) return NULL; - /* If the host has not yet processed the next entry then we are done */ - idx = ((q->hba_index + 1) % q->entry_count); - if (idx == q->host_index) - return NULL; - - q->hba_index = idx; - /* if the index wrapped around, toggle the valid bit */ - if (phba->sli4_hba.pc_sli4_params.cqav && !q->hba_index) - q->qe_valid = (q->qe_valid) ? 0 : 1; /* * insert barrier for instruction interlock : data from the hardware @@ -554,113 +707,85 @@ lpfc_sli4_cq_get(struct lpfc_queue *q) return cqe; } +static void +__lpfc_sli4_consume_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, + struct lpfc_cqe *cqe) +{ + if (!phba->sli4_hba.pc_sli4_params.cqav) + bf_set_le32(lpfc_cqe_valid, cqe, 0); + + cq->host_index = ((cq->host_index + 1) % cq->entry_count); + + /* if the index wrapped around, toggle the valid bit */ + if (phba->sli4_hba.pc_sli4_params.cqav && !cq->host_index) + cq->qe_valid = (cq->qe_valid) ? 0 : 1; +} + /** - * lpfc_sli4_cq_release - Indicates the host has finished processing a CQ + * lpfc_sli4_write_cq_db - write cq DB for entries consumed or arm state. + * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. + * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Completion queue entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the CQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of CQEs that were released. + * This routine will notify the HBA, by ringing the doorbell, that the + * CQEs have been processed. The @arm parameter specifies whether the + * queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_cq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_cqe *temp_qe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.cqav) { - temp_qe = q->qe[q->host_index].cqe; - bf_set_le32(lpfc_cqe_valid, temp_qe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_eqcq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_eqcq_doorbell_num_released, &doorbell, count); bf_set(lpfc_eqcq_doorbell_qt, &doorbell, LPFC_QUEUE_TYPE_COMPLETION); bf_set(lpfc_eqcq_doorbell_cqid_hi, &doorbell, (q->queue_id >> LPFC_CQID_HI_FIELD_SHIFT)); bf_set(lpfc_eqcq_doorbell_cqid_lo, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr); - return released; } /** - * lpfc_sli4_if6_cq_release - Indicates the host has finished processing a CQ + * lpfc_sli4_if6_write_cq_db - write cq DB for entries consumed or arm state. + * @phba: the adapter with the CQ * @q: The Completion Queue that the host has completed processing for. + * @count: the number of elements that were consumed * @arm: Indicates whether the host wants to arms this CQ. * - * This routine will mark all Completion queue entries on @q, from the last - * known completed entry to the last entry that was processed, as completed - * by clearing the valid bit for each completion queue entry. Then it will - * notify the HBA, by ringing the doorbell, that the CQEs have been processed. - * The internal host index in the @q will be updated by this routine to indicate - * that the host has finished processing the entries. The @arm parameter - * indicates that the queue should be rearmed when ringing the doorbell. - * - * This function will return the number of CQEs that were released. + * This routine will notify the HBA, by ringing the doorbell, that the + * CQEs have been processed. The @arm parameter specifies whether the + * queue should be rearmed when ringing the doorbell. **/ -uint32_t -lpfc_sli4_if6_cq_release(struct lpfc_queue *q, bool arm) +void +lpfc_sli4_if6_write_cq_db(struct lpfc_hba *phba, struct lpfc_queue *q, + uint32_t count, bool arm) { - uint32_t released = 0; - struct lpfc_hba *phba; - struct lpfc_cqe *temp_qe; struct lpfc_register doorbell; /* sanity check on queue memory */ - if (unlikely(!q)) - return 0; - phba = q->phba; - - /* while there are valid entries */ - while (q->hba_index != q->host_index) { - if (!phba->sli4_hba.pc_sli4_params.cqav) { - temp_qe = q->qe[q->host_index].cqe; - bf_set_le32(lpfc_cqe_valid, temp_qe, 0); - } - released++; - q->host_index = ((q->host_index + 1) % q->entry_count); - } - if (unlikely(released == 0 && !arm)) - return 0; + if (unlikely(!q || (count == 0 && !arm))) + return; /* ring doorbell for number popped */ doorbell.word0 = 0; if (arm) bf_set(lpfc_if6_cq_doorbell_arm, &doorbell, 1); - bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, released); + bf_set(lpfc_if6_cq_doorbell_num_released, &doorbell, count); bf_set(lpfc_if6_cq_doorbell_cqid, &doorbell, q->queue_id); writel(doorbell.word0, q->phba->sli4_hba.CQDBregaddr); - return released; } -/** +/* * lpfc_sli4_rq_put - Put a Receive Buffer Queue Entry on a Receive Queue - * @q: The Header Receive Queue to operate on. - * @wqe: The Receive Queue Entry to put on the Receive queue. * * This routine will copy the contents of @wqe to the next available entry on * the @q. This function will then ring the Receive Queue Doorbell to signal the @@ -684,8 +809,8 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, return -ENOMEM; hq_put_index = hq->host_index; dq_put_index = dq->host_index; - temp_hrqe = hq->qe[hq_put_index].rqe; - temp_drqe = dq->qe[dq_put_index].rqe; + temp_hrqe = lpfc_sli4_qe(hq, hq_put_index); + temp_drqe = lpfc_sli4_qe(dq, dq_put_index); if (hq->type != LPFC_HRQ || dq->type != LPFC_DRQ) return -EINVAL; @@ -703,15 +828,15 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, hq->RQ_buf_posted++; /* Ring The Header Receive Queue Doorbell */ - if (!(hq->host_index % hq->entry_repost)) { + if (!(hq->host_index % hq->notify_interval)) { doorbell.word0 = 0; if (hq->db_format == LPFC_DB_RING_FORMAT) { bf_set(lpfc_rq_db_ring_fm_num_posted, &doorbell, - hq->entry_repost); + hq->notify_interval); bf_set(lpfc_rq_db_ring_fm_id, &doorbell, hq->queue_id); } else if (hq->db_format == LPFC_DB_LIST_FORMAT) { bf_set(lpfc_rq_db_list_fm_num_posted, &doorbell, - hq->entry_repost); + hq->notify_interval); bf_set(lpfc_rq_db_list_fm_index, &doorbell, hq->host_index); bf_set(lpfc_rq_db_list_fm_id, &doorbell, hq->queue_id); @@ -723,9 +848,8 @@ lpfc_sli4_rq_put(struct lpfc_queue *hq, struct lpfc_queue *dq, return hq_put_index; } -/** +/* * lpfc_sli4_rq_release - Updates internal hba index for RQ - * @q: The Header Receive Queue to operate on. * * This routine will update the HBA index of a queue to reflect consumption of * one Receive Queue Entry by the HBA. When the HBA indicates that it has @@ -811,7 +935,7 @@ __lpfc_sli_get_iocbq(struct lpfc_hba *phba) * @phba: Pointer to HBA context object. * @xritag: XRI value. * - * This function clears the sglq pointer from the array of acive + * This function clears the sglq pointer from the array of active * sglq's. The xritag that is passed in is used to index into the * array. Before the xritag can be used it needs to be adjusted * by subtracting the xribase. @@ -833,7 +957,7 @@ __lpfc_clear_active_sglq(struct lpfc_hba *phba, uint16_t xritag) * @phba: Pointer to HBA context object. * @xritag: XRI value. * - * This function returns the sglq pointer from the array of acive + * This function returns the sglq pointer from the array of active * sglq's. The xritag that is passed in is used to index into the * array. Before the xritag can be used it needs to be adjusted * by subtracting the xribase. @@ -863,16 +987,10 @@ lpfc_clr_rrq_active(struct lpfc_hba *phba, { struct lpfc_nodelist *ndlp = NULL; - if ((rrq->vport) && NLP_CHK_NODE_ACT(rrq->ndlp)) + /* Lookup did to verify if did is still active on this vport */ + if (rrq->vport) ndlp = lpfc_findnode_did(rrq->vport, rrq->nlp_DID); - /* The target DID could have been swapped (cable swap) - * we should use the ndlp from the findnode if it is - * available. - */ - if ((!ndlp) && rrq->ndlp) - ndlp = rrq->ndlp; - if (!ndlp) goto out; @@ -908,9 +1026,9 @@ lpfc_handle_rrq_active(struct lpfc_hba *phba) unsigned long iflags; LIST_HEAD(send_rrq); - spin_lock_irqsave(&phba->hbalock, iflags); - phba->hba_flag &= ~HBA_RRQ_ACTIVE; - next_time = jiffies + msecs_to_jiffies(1000 * (phba->fc_ratov + 1)); + clear_bit(HBA_RRQ_ACTIVE, &phba->hba_flag); + next_time = jiffies + secs_to_jiffies(phba->fc_ratov + 1); + spin_lock_irqsave(&phba->rrq_list_lock, iflags); list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) { if (time_after(jiffies, rrq->rrq_stop_time)) @@ -918,16 +1036,16 @@ lpfc_handle_rrq_active(struct lpfc_hba *phba) else if (time_before(rrq->rrq_stop_time, next_time)) next_time = rrq->rrq_stop_time; } - spin_unlock_irqrestore(&phba->hbalock, iflags); + spin_unlock_irqrestore(&phba->rrq_list_lock, iflags); if ((!list_empty(&phba->active_rrq_list)) && - (!(phba->pport->load_flag & FC_UNLOADING))) + (!test_bit(FC_UNLOADING, &phba->pport->load_flag))) mod_timer(&phba->rrq_tmr, next_time); list_for_each_entry_safe(rrq, nextrrq, &send_rrq, list) { list_del(&rrq->list); - if (!rrq->send_rrq) + if (!rrq->send_rrq) { /* this call will free the rrq */ - lpfc_clr_rrq_active(phba, rrq->xritag, rrq); - else if (lpfc_send_rrq(phba, rrq)) { + lpfc_clr_rrq_active(phba, rrq->xritag, rrq); + } else if (lpfc_send_rrq(phba, rrq)) { /* if we send the rrq then the completion handler * will clear the bit in the xribitmap. */ @@ -956,16 +1074,16 @@ lpfc_get_active_rrq(struct lpfc_vport *vport, uint16_t xri, uint32_t did) if (phba->sli_rev != LPFC_SLI_REV4) return NULL; - spin_lock_irqsave(&phba->hbalock, iflags); + spin_lock_irqsave(&phba->rrq_list_lock, iflags); list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) { if (rrq->vport == vport && rrq->xritag == xri && rrq->nlp_DID == did){ list_del(&rrq->list); - spin_unlock_irqrestore(&phba->hbalock, iflags); + spin_unlock_irqrestore(&phba->rrq_list_lock, iflags); return rrq; } } - spin_unlock_irqrestore(&phba->hbalock, iflags); + spin_unlock_irqrestore(&phba->rrq_list_lock, iflags); return NULL; } @@ -993,11 +1111,16 @@ lpfc_cleanup_vports_rrqs(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) lpfc_sli4_vport_delete_els_xri_aborted(vport); lpfc_sli4_vport_delete_fcp_xri_aborted(vport); } - spin_lock_irqsave(&phba->hbalock, iflags); - list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) - if ((rrq->vport == vport) && (!ndlp || rrq->ndlp == ndlp)) + spin_lock_irqsave(&phba->rrq_list_lock, iflags); + list_for_each_entry_safe(rrq, nextrrq, &phba->active_rrq_list, list) { + if (rrq->vport != vport) + continue; + + if (!ndlp || ndlp == lpfc_findnode_did(vport, rrq->nlp_DID)) list_move(&rrq->list, &rrq_list); - spin_unlock_irqrestore(&phba->hbalock, iflags); + + } + spin_unlock_irqrestore(&phba->rrq_list_lock, iflags); list_for_each_entry_safe(rrq, nextrrq, &rrq_list, list) { list_del(&rrq->list); @@ -1009,23 +1132,22 @@ lpfc_cleanup_vports_rrqs(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) * lpfc_test_rrq_active - Test RRQ bit in xri_bitmap. * @phba: Pointer to HBA context object. * @ndlp: Targets nodelist pointer for this exchange. - * @xritag the xri in the bitmap to test. + * @xritag: the xri in the bitmap to test. * - * This function is called with hbalock held. This function - * returns 0 = rrq not active for this xri - * 1 = rrq is valid for this xri. + * This function returns: + * 0 = rrq not active for this xri + * 1 = rrq is valid for this xri. **/ int lpfc_test_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, uint16_t xritag) { - lockdep_assert_held(&phba->hbalock); if (!ndlp) return 0; if (!ndlp->active_rrqs_xri_bitmap) return 0; if (test_bit(xritag, ndlp->active_rrqs_xri_bitmap)) - return 1; + return 1; else return 0; } @@ -1059,19 +1181,13 @@ lpfc_set_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, if (!phba->cfg_enable_rrq) return -EINVAL; - spin_lock_irqsave(&phba->hbalock, iflags); - if (phba->pport->load_flag & FC_UNLOADING) { - phba->hba_flag &= ~HBA_RRQ_ACTIVE; - goto out; + if (test_bit(FC_UNLOADING, &phba->pport->load_flag)) { + clear_bit(HBA_RRQ_ACTIVE, &phba->hba_flag); + goto outnl; } - /* - * set the active bit even if there is no mem available. - */ - if (NLP_CHK_FREE_REQ(ndlp)) - goto out; - - if (ndlp->vport && (ndlp->vport->load_flag & FC_UNLOADING)) + spin_lock_irqsave(&phba->hbalock, iflags); + if (ndlp->vport && test_bit(FC_UNLOADING, &ndlp->vport->load_flag)) goto out; if (!ndlp->active_rrqs_xri_bitmap) @@ -1081,7 +1197,7 @@ lpfc_set_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, goto out; spin_unlock_irqrestore(&phba->hbalock, iflags); - rrq = mempool_alloc(phba->rrq_pool, GFP_KERNEL); + rrq = mempool_alloc(phba->rrq_pool, GFP_ATOMIC); if (!rrq) { lpfc_printf_log(phba, KERN_INFO, LOG_SLI, "3155 Unable to allocate RRQ xri:0x%x rxid:0x%x" @@ -1094,22 +1210,22 @@ lpfc_set_rrq_active(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, else rrq->send_rrq = 0; rrq->xritag = xritag; - rrq->rrq_stop_time = jiffies + - msecs_to_jiffies(1000 * (phba->fc_ratov + 1)); - rrq->ndlp = ndlp; + rrq->rrq_stop_time = jiffies + secs_to_jiffies(phba->fc_ratov + 1); rrq->nlp_DID = ndlp->nlp_DID; rrq->vport = ndlp->vport; rrq->rxid = rxid; - spin_lock_irqsave(&phba->hbalock, iflags); + + spin_lock_irqsave(&phba->rrq_list_lock, iflags); empty = list_empty(&phba->active_rrq_list); list_add_tail(&rrq->list, &phba->active_rrq_list); - phba->hba_flag |= HBA_RRQ_ACTIVE; + spin_unlock_irqrestore(&phba->rrq_list_lock, iflags); + set_bit(HBA_RRQ_ACTIVE, &phba->hba_flag); if (empty) lpfc_worker_wake_up(phba); - spin_unlock_irqrestore(&phba->hbalock, iflags); return 0; out: spin_unlock_irqrestore(&phba->hbalock, iflags); +outnl: lpfc_printf_log(phba, KERN_INFO, LOG_SLI, "2921 Can't set rrq active xri:0x%x rxid:0x%x" " DID:0x%x Send:%d\n", @@ -1120,12 +1236,13 @@ out: /** * __lpfc_sli_get_els_sglq - Allocates an iocb object from sgl pool * @phba: Pointer to HBA context object. - * @piocb: Pointer to the iocbq. + * @piocbq: Pointer to the iocbq. * - * This function is called with the ring lock held. This function - * gets a new driver sglq object from the sglq list. If the - * list is not empty then it is successful, it returns pointer to the newly - * allocated sglq object else it returns NULL. + * The driver calls this function with either the nvme ls ring lock + * or the fc els ring lock held depending on the iocb usage. This function + * gets a new driver sglq object from the sglq list. If the list is not empty + * then it is successful, it returns pointer to the newly allocated sglq + * object else it returns NULL. **/ static struct lpfc_sglq * __lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq) @@ -1133,25 +1250,26 @@ __lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq) struct list_head *lpfc_els_sgl_list = &phba->sli4_hba.lpfc_els_sgl_list; struct lpfc_sglq *sglq = NULL; struct lpfc_sglq *start_sglq = NULL; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_nodelist *ndlp; int found = 0; + u8 cmnd; - lockdep_assert_held(&phba->hbalock); + cmnd = get_job_cmnd(phba, piocbq); - if (piocbq->iocb_flag & LPFC_IO_FCP) { - lpfc_cmd = (struct lpfc_scsi_buf *) piocbq->context1; + if (piocbq->cmd_flag & LPFC_IO_FCP) { + lpfc_cmd = piocbq->io_buf; ndlp = lpfc_cmd->rdata->pnode; - } else if ((piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) && - !(piocbq->iocb_flag & LPFC_IO_LIBDFC)) { - ndlp = piocbq->context_un.ndlp; - } else if (piocbq->iocb_flag & LPFC_IO_LIBDFC) { - if (piocbq->iocb_flag & LPFC_IO_LOOPBACK) + } else if ((cmnd == CMD_GEN_REQUEST64_CR) && + !(piocbq->cmd_flag & LPFC_IO_LIBDFC)) { + ndlp = piocbq->ndlp; + } else if (piocbq->cmd_flag & LPFC_IO_LIBDFC) { + if (piocbq->cmd_flag & LPFC_IO_LOOPBACK) ndlp = NULL; else - ndlp = piocbq->context_un.ndlp; + ndlp = piocbq->ndlp; } else { - ndlp = piocbq->context1; + ndlp = piocbq->ndlp; } spin_lock(&phba->sli4_hba.sgl_list_lock); @@ -1189,7 +1307,7 @@ __lpfc_sli_get_els_sglq(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq) /** * __lpfc_sli_get_nvmet_sglq - Allocates an iocb object from sgl pool * @phba: Pointer to HBA context object. - * @piocb: Pointer to the iocbq. + * @piocbq: Pointer to the iocbq. * * This function is called with the sgl_list lock held. This function * gets a new driver sglq object from the sglq list. If the @@ -1240,8 +1358,8 @@ lpfc_sli_get_iocbq(struct lpfc_hba *phba) * @phba: Pointer to HBA context object. * @iocbq: Pointer to driver iocb object. * - * This function is called with hbalock held to release driver - * iocb object to the iocb pool. The iotag in the iocb object + * This function is called to release the driver iocb object + * to the iocb pool. The iotag in the iocb object * does not change for each use of the iocb object. This function * clears all other fields of the iocb object when it is freed. * The sqlq structure that holds the xritag and phys and virtual @@ -1251,18 +1369,16 @@ lpfc_sli_get_iocbq(struct lpfc_hba *phba) * this IO was aborted then the sglq entry it put on the * lpfc_abts_els_sgl_list until the CQ_ABORTED_XRI is received. If the * IO has good status or fails for any other reason then the sglq - * entry is added to the free list (lpfc_els_sgl_list). + * entry is added to the free list (lpfc_els_sgl_list). The hbalock is + * asserted held in the code path calling this routine. **/ static void __lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq) { struct lpfc_sglq *sglq; - size_t start_clean = offsetof(struct lpfc_iocbq, iocb); unsigned long iflag = 0; struct lpfc_sli_ring *pring; - lockdep_assert_held(&phba->hbalock); - if (iocbq->sli4_xritag == NO_XRI) sglq = NULL; else @@ -1270,7 +1386,7 @@ __lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq) if (sglq) { - if (iocbq->iocb_flag & LPFC_IO_NVMET) { + if (iocbq->cmd_flag & LPFC_IO_NVMET) { spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock, iflag); sglq->state = SGL_FREED; @@ -1282,11 +1398,16 @@ __lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq) goto out; } - pring = phba->sli4_hba.els_wq->pring; - if ((iocbq->iocb_flag & LPFC_EXCHANGE_BUSY) && - (sglq->state != SGL_XRI_ABORTED)) { + if ((iocbq->cmd_flag & LPFC_EXCHANGE_BUSY) && + (!(unlikely(pci_channel_offline(phba->pcidev)))) && + sglq->state != SGL_XRI_ABORTED) { spin_lock_irqsave(&phba->sli4_hba.sgl_list_lock, iflag); + + /* Check if we can get a reference on ndlp */ + if (sglq->ndlp && !lpfc_nlp_get(sglq->ndlp)) + sglq->ndlp = NULL; + list_add(&sglq->list, &phba->sli4_hba.lpfc_abts_els_sgl_list); spin_unlock_irqrestore( @@ -1300,9 +1421,9 @@ __lpfc_sli_release_iocbq_s4(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq) &phba->sli4_hba.lpfc_els_sgl_list); spin_unlock_irqrestore( &phba->sli4_hba.sgl_list_lock, iflag); - + pring = lpfc_phba_elsring(phba); /* Check if TXQ queue needs to be serviced */ - if (!list_empty(&pring->txq)) + if (pring && (!list_empty(&pring->txq))) lpfc_worker_wake_up(phba); } } @@ -1311,10 +1432,10 @@ out: /* * Clean all volatile data fields, preserve iotag and node struct. */ - memset((char *)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean); + memset_startat(iocbq, 0, wqe); iocbq->sli4_lxritag = NO_XRI; iocbq->sli4_xritag = NO_XRI; - iocbq->iocb_flag &= ~(LPFC_IO_NVME | LPFC_IO_NVMET | + iocbq->cmd_flag &= ~(LPFC_IO_NVME | LPFC_IO_NVMET | LPFC_IO_CMF | LPFC_IO_NVME_LS); list_add_tail(&iocbq->list, &phba->lpfc_iocb_list); } @@ -1325,22 +1446,20 @@ out: * @phba: Pointer to HBA context object. * @iocbq: Pointer to driver iocb object. * - * This function is called with hbalock held to release driver - * iocb object to the iocb pool. The iotag in the iocb object - * does not change for each use of the iocb object. This function - * clears all other fields of the iocb object when it is freed. + * This function is called to release the driver iocb object to the + * iocb pool. The iotag in the iocb object does not change for each + * use of the iocb object. This function clears all other fields of + * the iocb object when it is freed. The hbalock is asserted held in + * the code path calling this routine. **/ static void __lpfc_sli_release_iocbq_s3(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq) { - size_t start_clean = offsetof(struct lpfc_iocbq, iocb); - - lockdep_assert_held(&phba->hbalock); /* * Clean all volatile data fields, preserve iotag and node struct. */ - memset((char*)iocbq + start_clean, 0, sizeof(*iocbq) - start_clean); + memset_startat(iocbq, 0, iocb); iocbq->sli4_xritag = NO_XRI; list_add_tail(&iocbq->list, &phba->lpfc_iocb_list); } @@ -1405,12 +1524,23 @@ lpfc_sli_cancel_iocbs(struct lpfc_hba *phba, struct list_head *iocblist, while (!list_empty(iocblist)) { list_remove_head(iocblist, piocb, struct lpfc_iocbq, list); - if (!piocb->iocb_cmpl) + if (piocb->cmd_cmpl) { + if (piocb->cmd_flag & LPFC_IO_NVME) { + lpfc_nvme_cancel_iocb(phba, piocb, + ulpstatus, ulpWord4); + } else { + if (phba->sli_rev == LPFC_SLI_REV4) { + bf_set(lpfc_wcqe_c_status, + &piocb->wcqe_cmpl, ulpstatus); + piocb->wcqe_cmpl.parameter = ulpWord4; + } else { + piocb->iocb.ulpStatus = ulpstatus; + piocb->iocb.un.ulpWord[4] = ulpWord4; + } + (piocb->cmd_cmpl) (phba, piocb, piocb); + } + } else { lpfc_sli_release_iocbq(phba, piocb); - else { - piocb->iocb.ulpStatus = ulpstatus; - piocb->iocb.un.ulpWord[4] = ulpWord4; - (piocb->iocb_cmpl) (phba, piocb, piocb); } } return; @@ -1485,6 +1615,7 @@ lpfc_sli_iocb_cmd_type(uint8_t iocb_cmnd) case DSSCMD_IWRITE64_CX: case DSSCMD_IREAD64_CR: case DSSCMD_IREAD64_CX: + case CMD_SEND_FRAME: type = LPFC_SOL_IOCB; break; case CMD_ABORT_XRI_CN: @@ -1559,7 +1690,7 @@ lpfc_sli_ring_map(struct lpfc_hba *phba) lpfc_config_ring(phba, i, pmb); rc = lpfc_sli_issue_mbox(phba, pmb, MBX_POLL); if (rc != MBX_SUCCESS) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0446 Adapter failed to init (%d), " "mbxCmd x%x CFG_RING, mbxStatus x%x, " "ring %d\n", @@ -1580,7 +1711,8 @@ lpfc_sli_ring_map(struct lpfc_hba *phba) * @pring: Pointer to driver SLI ring object. * @piocb: Pointer to the driver iocb object. * - * This function is called with hbalock held. The function adds the + * The driver calls this function with the hbalock held for SLI3 ports or + * the ring lock held for SLI4 ports. The function adds the * new iocb to txcmplq of the given ring. This function always returns * 0. If this function is called for ELS ring, this function checks if * there is a vport associated with the ELS command. This function also @@ -1590,21 +1722,22 @@ static int lpfc_sli_ringtxcmpl_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_iocbq *piocb) { - lockdep_assert_held(&phba->hbalock); + u32 ulp_command = 0; BUG_ON(!piocb); + ulp_command = get_job_cmnd(phba, piocb); list_add_tail(&piocb->list, &pring->txcmplq); - piocb->iocb_flag |= LPFC_IO_ON_TXCMPLQ; - + piocb->cmd_flag |= LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt++; if ((unlikely(pring->ringno == LPFC_ELS_RING)) && - (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) && - (piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) { + (ulp_command != CMD_ABORT_XRI_WQE) && + (ulp_command != CMD_ABORT_XRI_CN) && + (ulp_command != CMD_CLOSE_XRI_CN)) { BUG_ON(!piocb->vport); - if (!(piocb->vport->load_flag & FC_UNLOADING)) + if (!test_bit(FC_UNLOADING, &piocb->vport->load_flag)) mod_timer(&piocb->vport->els_tmofunc, - jiffies + - msecs_to_jiffies(1000 * (phba->fc_ratov << 1))); + jiffies + secs_to_jiffies(phba->fc_ratov << 1)); } return 0; @@ -1632,6 +1765,285 @@ lpfc_sli_ringtx_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring) } /** + * lpfc_cmf_sync_cmpl - Process a CMF_SYNC_WQE cmpl + * @phba: Pointer to HBA context object. + * @cmdiocb: Pointer to driver command iocb object. + * @rspiocb: Pointer to driver response iocb object. + * + * This routine will inform the driver of any BW adjustments we need + * to make. These changes will be picked up during the next CMF + * timer interrupt. In addition, any BW changes will be logged + * with LOG_CGN_MGMT. + **/ +static void +lpfc_cmf_sync_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, + struct lpfc_iocbq *rspiocb) +{ + union lpfc_wqe128 *wqe; + uint32_t status, info; + struct lpfc_wcqe_complete *wcqe = &rspiocb->wcqe_cmpl; + uint64_t bw, bwdif, slop; + uint64_t pcent, bwpcent; + int asig, afpin, sigcnt, fpincnt; + int wsigmax, wfpinmax, cg, tdp; + char *s; + + /* First check for error */ + status = bf_get(lpfc_wcqe_c_status, wcqe); + if (status) { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6211 CMF_SYNC_WQE Error " + "req_tag x%x status x%x hwstatus x%x " + "tdatap x%x parm x%x\n", + bf_get(lpfc_wcqe_c_request_tag, wcqe), + bf_get(lpfc_wcqe_c_status, wcqe), + bf_get(lpfc_wcqe_c_hw_status, wcqe), + wcqe->total_data_placed, + wcqe->parameter); + goto out; + } + + /* Gather congestion information on a successful cmpl */ + info = wcqe->parameter; + phba->cmf_active_info = info; + + /* See if firmware info count is valid or has changed */ + if (info > LPFC_MAX_CMF_INFO || phba->cmf_info_per_interval == info) + info = 0; + else + phba->cmf_info_per_interval = info; + + tdp = bf_get(lpfc_wcqe_c_cmf_bw, wcqe); + cg = bf_get(lpfc_wcqe_c_cmf_cg, wcqe); + + /* Get BW requirement from firmware */ + bw = (uint64_t)tdp * LPFC_CMF_BLK_SIZE; + if (!bw) { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6212 CMF_SYNC_WQE x%x: NULL bw\n", + bf_get(lpfc_wcqe_c_request_tag, wcqe)); + goto out; + } + + /* Gather information needed for logging if a BW change is required */ + wqe = &cmdiocb->wqe; + asig = bf_get(cmf_sync_asig, &wqe->cmf_sync); + afpin = bf_get(cmf_sync_afpin, &wqe->cmf_sync); + fpincnt = bf_get(cmf_sync_wfpincnt, &wqe->cmf_sync); + sigcnt = bf_get(cmf_sync_wsigcnt, &wqe->cmf_sync); + if (phba->cmf_max_bytes_per_interval != bw || + (asig || afpin || sigcnt || fpincnt)) { + /* Are we increasing or decreasing BW */ + if (phba->cmf_max_bytes_per_interval < bw) { + bwdif = bw - phba->cmf_max_bytes_per_interval; + s = "Increase"; + } else { + bwdif = phba->cmf_max_bytes_per_interval - bw; + s = "Decrease"; + } + + /* What is the change percentage */ + slop = div_u64(phba->cmf_link_byte_count, 200); /*For rounding*/ + pcent = div64_u64(bwdif * 100 + slop, + phba->cmf_link_byte_count); + bwpcent = div64_u64(bw * 100 + slop, + phba->cmf_link_byte_count); + /* Because of bytes adjustment due to shorter timer in + * lpfc_cmf_timer() the cmf_link_byte_count can be shorter and + * may seem like BW is above 100%. + */ + if (bwpcent > 100) + bwpcent = 100; + + if (phba->cmf_max_bytes_per_interval < bw && + bwpcent > 95) + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6208 Congestion bandwidth " + "limits removed\n"); + else if ((phba->cmf_max_bytes_per_interval > bw) && + ((bwpcent + pcent) <= 100) && ((bwpcent + pcent) > 95)) + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6209 Congestion bandwidth " + "limits in effect\n"); + + if (asig) { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6237 BW Threshold %lld%% (%lld): " + "%lld%% %s: Signal Alarm: cg:%d " + "Info:%u\n", + bwpcent, bw, pcent, s, cg, + phba->cmf_active_info); + } else if (afpin) { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6238 BW Threshold %lld%% (%lld): " + "%lld%% %s: FPIN Alarm: cg:%d " + "Info:%u\n", + bwpcent, bw, pcent, s, cg, + phba->cmf_active_info); + } else if (sigcnt) { + wsigmax = bf_get(cmf_sync_wsigmax, &wqe->cmf_sync); + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6239 BW Threshold %lld%% (%lld): " + "%lld%% %s: Signal Warning: " + "Cnt %d Max %d: cg:%d Info:%u\n", + bwpcent, bw, pcent, s, sigcnt, + wsigmax, cg, phba->cmf_active_info); + } else if (fpincnt) { + wfpinmax = bf_get(cmf_sync_wfpinmax, &wqe->cmf_sync); + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6240 BW Threshold %lld%% (%lld): " + "%lld%% %s: FPIN Warning: " + "Cnt %d Max %d: cg:%d Info:%u\n", + bwpcent, bw, pcent, s, fpincnt, + wfpinmax, cg, phba->cmf_active_info); + } else { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6241 BW Threshold %lld%% (%lld): " + "CMF %lld%% %s: cg:%d Info:%u\n", + bwpcent, bw, pcent, s, cg, + phba->cmf_active_info); + } + } else if (info) { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6246 Info Threshold %u\n", info); + } + + /* Save BW change to be picked up during next timer interrupt */ + phba->cmf_last_sync_bw = bw; +out: + lpfc_sli_release_iocbq(phba, cmdiocb); +} + +/** + * lpfc_issue_cmf_sync_wqe - Issue a CMF_SYNC_WQE + * @phba: Pointer to HBA context object. + * @ms: ms to set in WQE interval, 0 means use init op + * @total: Total rcv bytes for this interval + * + * This routine is called every CMF timer interrupt. Its purpose is + * to issue a CMF_SYNC_WQE to the firmware to inform it of any events + * that may indicate we have congestion (FPINs or Signals). Upon + * completion, the firmware will indicate any BW restrictions the + * driver may need to take. + **/ +int +lpfc_issue_cmf_sync_wqe(struct lpfc_hba *phba, u32 ms, u64 total) +{ + union lpfc_wqe128 *wqe; + struct lpfc_iocbq *sync_buf; + unsigned long iflags; + u32 ret_val, cgn_sig_freq; + u32 atot, wtot, max; + u8 warn_sync_period = 0; + + /* First address any alarm / warning activity */ + atot = atomic_xchg(&phba->cgn_sync_alarm_cnt, 0); + wtot = atomic_xchg(&phba->cgn_sync_warn_cnt, 0); + + spin_lock_irqsave(&phba->hbalock, iflags); + + /* ONLY Managed mode will send the CMF_SYNC_WQE to the HBA */ + if (phba->cmf_active_mode != LPFC_CFG_MANAGED || + phba->link_state < LPFC_LINK_UP) { + ret_val = 0; + goto out_unlock; + } + + sync_buf = __lpfc_sli_get_iocbq(phba); + if (!sync_buf) { + lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT, + "6244 No available WQEs for CMF_SYNC_WQE\n"); + ret_val = ENOMEM; + goto out_unlock; + } + + wqe = &sync_buf->wqe; + + /* WQEs are reused. Clear stale data and set key fields to zero */ + memset(wqe, 0, sizeof(*wqe)); + + /* If this is the very first CMF_SYNC_WQE, issue an init operation */ + if (!ms) { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6441 CMF Init %d - CMF_SYNC_WQE\n", + phba->fc_eventTag); + bf_set(cmf_sync_op, &wqe->cmf_sync, 1); /* 1=init */ + bf_set(cmf_sync_interval, &wqe->cmf_sync, LPFC_CMF_INTERVAL); + goto initpath; + } + + bf_set(cmf_sync_op, &wqe->cmf_sync, 0); /* 0=recalc */ + bf_set(cmf_sync_interval, &wqe->cmf_sync, ms); + + /* Check for alarms / warnings */ + if (atot) { + if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) { + /* We hit an Signal alarm condition */ + bf_set(cmf_sync_asig, &wqe->cmf_sync, 1); + } else { + /* We hit a FPIN alarm condition */ + bf_set(cmf_sync_afpin, &wqe->cmf_sync, 1); + } + } else if (wtot) { + if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ONLY || + phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) { + cgn_sig_freq = phba->cgn_sig_freq ? phba->cgn_sig_freq : + lpfc_fabric_cgn_frequency; + /* We hit an Signal warning condition */ + max = LPFC_SEC_TO_MSEC / cgn_sig_freq * + lpfc_acqe_cgn_frequency; + bf_set(cmf_sync_wsigmax, &wqe->cmf_sync, max); + bf_set(cmf_sync_wsigcnt, &wqe->cmf_sync, wtot); + warn_sync_period = lpfc_acqe_cgn_frequency; + } else { + /* We hit a FPIN warning condition */ + bf_set(cmf_sync_wfpinmax, &wqe->cmf_sync, 1); + bf_set(cmf_sync_wfpincnt, &wqe->cmf_sync, 1); + if (phba->cgn_fpin_frequency != LPFC_FPIN_INIT_FREQ) + warn_sync_period = + LPFC_MSECS_TO_SECS(phba->cgn_fpin_frequency); + } + } + + /* Update total read blocks during previous timer interval */ + wqe->cmf_sync.read_bytes = (u32)(total / LPFC_CMF_BLK_SIZE); + +initpath: + bf_set(cmf_sync_ver, &wqe->cmf_sync, LPFC_CMF_SYNC_VER); + wqe->cmf_sync.event_tag = phba->fc_eventTag; + bf_set(cmf_sync_cmnd, &wqe->cmf_sync, CMD_CMF_SYNC_WQE); + + /* Setup reqtag to match the wqe completion. */ + bf_set(cmf_sync_reqtag, &wqe->cmf_sync, sync_buf->iotag); + + bf_set(cmf_sync_qosd, &wqe->cmf_sync, 1); + bf_set(cmf_sync_period, &wqe->cmf_sync, warn_sync_period); + + bf_set(cmf_sync_cmd_type, &wqe->cmf_sync, CMF_SYNC_COMMAND); + bf_set(cmf_sync_wqec, &wqe->cmf_sync, 1); + bf_set(cmf_sync_cqid, &wqe->cmf_sync, LPFC_WQE_CQ_ID_DEFAULT); + + sync_buf->vport = phba->pport; + sync_buf->cmd_cmpl = lpfc_cmf_sync_cmpl; + sync_buf->cmd_dmabuf = NULL; + sync_buf->rsp_dmabuf = NULL; + sync_buf->bpl_dmabuf = NULL; + sync_buf->sli4_xritag = NO_XRI; + + sync_buf->cmd_flag |= LPFC_IO_CMF; + ret_val = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], sync_buf); + if (ret_val) { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "6214 Cannot issue CMF_SYNC_WQE: x%x\n", + ret_val); + __lpfc_sli_release_iocbq(phba, sync_buf); + } +out_unlock: + spin_unlock_irqrestore(&phba->hbalock, iflags); + return ret_val; +} + +/** * lpfc_sli_next_iocb_slot - Get next iocb slot in the ring * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. @@ -1663,7 +2075,7 @@ lpfc_sli_next_iocb_slot (struct lpfc_hba *phba, struct lpfc_sli_ring *pring) pring->sli.sli3.local_getidx = le32_to_cpu(pgp->cmdGetInx); if (unlikely(pring->sli.sli3.local_getidx >= max_cmd_idx)) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0315 Ring %d issue: portCmdGet %d " "is bigger than cmd ring %d\n", pring->ringno, @@ -1773,21 +2185,21 @@ lpfc_sli_next_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq) * @nextiocb: Pointer to driver iocb object which need to be * posted to firmware. * - * This function is called with hbalock held to post a new iocb to - * the firmware. This function copies the new iocb to ring iocb slot and - * updates the ring pointers. It adds the new iocb to txcmplq if there is + * This function is called to post a new iocb to the firmware. This + * function copies the new iocb to ring iocb slot and updates the + * ring pointers. It adds the new iocb to txcmplq if there is * a completion call back for this iocb else the function will free the - * iocb object. + * iocb object. The hbalock is asserted held in the code path calling + * this routine. **/ static void lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, IOCB_t *iocb, struct lpfc_iocbq *nextiocb) { - lockdep_assert_held(&phba->hbalock); /* * Set up an iotag */ - nextiocb->iocb.ulpIoTag = (nextiocb->iocb_cmpl) ? nextiocb->iotag : 0; + nextiocb->iocb.ulpIoTag = (nextiocb->cmd_cmpl) ? nextiocb->iotag : 0; if (pring->ringno == LPFC_ELS_RING) { @@ -1808,9 +2220,9 @@ lpfc_sli_submit_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, /* * If there is no completion routine to call, we can release the * IOCB buffer back right now. For IOCBs, like QUE_RING_BUF, - * that have no rsp ring completion, iocb_cmpl MUST be NULL. + * that have no rsp ring completion, cmd_cmpl MUST be NULL. */ - if (nextiocb->iocb_cmpl) + if (nextiocb->cmd_cmpl) lpfc_sli_ringtxcmpl_put(phba, pring, nextiocb); else __lpfc_sli_release_iocbq(phba, nextiocb); @@ -1949,8 +2361,7 @@ lpfc_sli_next_hbq_slot(struct lpfc_hba *phba, uint32_t hbqno) hbqp->local_hbqGetIdx = getidx; if (unlikely(hbqp->local_hbqGetIdx >= hbqp->entry_count)) { - lpfc_printf_log(phba, KERN_ERR, - LOG_SLI | LOG_VPORT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "1802 HBQ %d: local_hbqGetIdx " "%u is > than hbqp->entry_count %u\n", hbqno, hbqp->local_hbqGetIdx, @@ -2218,10 +2629,8 @@ lpfc_sli_hbqbuf_init_hbqs(struct lpfc_hba *phba, uint32_t qno) lpfc_hbq_defs[qno]->init_count); } -/** +/* * lpfc_sli_hbqbuf_get - Remove the first hbq off of an hbq list - * @phba: Pointer to HBA context object. - * @hbqno: HBQ number. * * This function removes the first hbq buffer on an hbq list and returns a * pointer to that buffer. If it finds no buffers on the list it returns NULL. @@ -2240,7 +2649,7 @@ lpfc_sli_hbqbuf_get(struct list_head *rb_list) /** * lpfc_sli_rqbuf_get - Remove the first dma buffer off of an RQ list * @phba: Pointer to HBA context object. - * @hbqno: HBQ number. + * @hrq: HBQ number. * * This function removes the first RQ buffer on an RQ buffer list and returns a * pointer to that buffer. If it finds no buffers on the list it returns NULL. @@ -2289,7 +2698,7 @@ lpfc_sli_hbqbuf_find(struct lpfc_hba *phba, uint32_t tag) } } spin_unlock_irq(&phba->hbalock); - lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_VPORT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "1803 Bad hbq tag. Data: x%x x%x\n", tag, phba->hbqs[tag >> 16].buffer_count); return NULL; @@ -2428,14 +2837,13 @@ lpfc_sli_wake_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq) */ pmboxq->mbox_flag |= LPFC_MBX_WAKE; spin_lock_irqsave(&phba->hbalock, drvr_flag); - pmbox_done = (struct completion *)pmboxq->context3; + pmbox_done = pmboxq->ctx_u.mbox_wait; if (pmbox_done) complete(pmbox_done); spin_unlock_irqrestore(&phba->hbalock, drvr_flag); return; } - /** * lpfc_sli_def_mbox_cmpl - Default mailbox completion handler * @phba: Pointer to HBA context object. @@ -2456,22 +2864,23 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) uint16_t rpi, vpi; int rc; - mp = (struct lpfc_dmabuf *)(pmb->ctx_buf); - - if (mp) { - lpfc_mbuf_free(phba, mp->virt, mp->phys); - kfree(mp); - } - /* * If a REG_LOGIN succeeded after node is destroyed or node * is in re-discovery driver need to cleanup the RPI. */ - if (!(phba->pport->load_flag & FC_UNLOADING) && + if (!test_bit(FC_UNLOADING, &phba->pport->load_flag) && pmb->u.mb.mbxCommand == MBX_REG_LOGIN64 && !pmb->u.mb.mbxStatus) { + mp = pmb->ctx_buf; + if (mp) { + pmb->ctx_buf = NULL; + lpfc_mbuf_free(phba, mp->virt, mp->phys); + kfree(mp); + } rpi = pmb->u.mb.un.varWords[0]; vpi = pmb->u.mb.un.varRegLogin.vpi; + if (phba->sli_rev == LPFC_SLI_REV4) + vpi -= phba->sli4_hba.max_cfg_param.vpi_base; lpfc_unreg_login(phba, vpi, rpi, pmb); pmb->vport = vport; pmb->mbox_cmpl = lpfc_sli_def_mbox_cmpl; @@ -2481,24 +2890,22 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) } if ((pmb->u.mb.mbxCommand == MBX_REG_VPI) && - !(phba->pport->load_flag & FC_UNLOADING) && + !test_bit(FC_UNLOADING, &phba->pport->load_flag) && !pmb->u.mb.mbxStatus) { shost = lpfc_shost_from_vport(vport); spin_lock_irq(shost->host_lock); vport->vpi_state |= LPFC_VPI_REGISTERED; - vport->fc_flag &= ~FC_VPORT_NEEDS_REG_VPI; spin_unlock_irq(shost->host_lock); + clear_bit(FC_VPORT_NEEDS_REG_VPI, &vport->fc_flag); } if (pmb->u.mb.mbxCommand == MBX_REG_LOGIN64) { - ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp; + ndlp = pmb->ctx_ndlp; lpfc_nlp_put(ndlp); - pmb->ctx_buf = NULL; - pmb->ctx_ndlp = NULL; } if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) { - ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp; + ndlp = pmb->ctx_ndlp; /* Check to see if there are any deferred events to process */ if (ndlp) { @@ -2506,33 +2913,46 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) vport, KERN_INFO, LOG_MBOX | LOG_DISCOVERY, "1438 UNREG cmpl deferred mbox x%x " - "on NPort x%x Data: x%x x%x %p\n", + "on NPort x%x Data: x%lx x%x x%px x%lx x%x\n", ndlp->nlp_rpi, ndlp->nlp_DID, - ndlp->nlp_flag, ndlp->nlp_defer_did, ndlp); + ndlp->nlp_flag, ndlp->nlp_defer_did, + ndlp, vport->load_flag, kref_read(&ndlp->kref)); - if ((ndlp->nlp_flag & NLP_UNREG_INP) && - (ndlp->nlp_defer_did != NLP_EVT_NOTHING_PENDING)) { - ndlp->nlp_flag &= ~NLP_UNREG_INP; + if (test_bit(NLP_UNREG_INP, &ndlp->nlp_flag) && + ndlp->nlp_defer_did != NLP_EVT_NOTHING_PENDING) { + clear_bit(NLP_UNREG_INP, &ndlp->nlp_flag); ndlp->nlp_defer_did = NLP_EVT_NOTHING_PENDING; lpfc_issue_els_plogi(vport, ndlp->nlp_DID, 0); } else { - ndlp->nlp_flag &= ~NLP_UNREG_INP; + clear_bit(NLP_UNREG_INP, &ndlp->nlp_flag); } + + /* The unreg_login mailbox is complete and had a + * reference that has to be released. The PLOGI + * got its own ref. + */ + lpfc_nlp_put(ndlp); + pmb->ctx_ndlp = NULL; } - pmb->ctx_ndlp = NULL; + } + + /* This nlp_put pairs with lpfc_sli4_resume_rpi */ + if (pmb->u.mb.mbxCommand == MBX_RESUME_RPI) { + ndlp = pmb->ctx_ndlp; + lpfc_nlp_put(ndlp); } /* Check security permission status on INIT_LINK mailbox command */ if ((pmb->u.mb.mbxCommand == MBX_INIT_LINK) && (pmb->u.mb.mbxStatus == MBXERR_SEC_NO_PERMISSION)) - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2860 SLI authentication is required " "for INIT_LINK but has not done yet\n"); if (bf_get(lpfc_mqe_command, &pmb->u.mqe) == MBX_SLI4_CONFIG) lpfc_sli4_mbox_cmd_free(phba, pmb); else - mempool_free(pmb, phba->mbox_mem_pool); + lpfc_mbox_rsrc_cleanup(phba, pmb, MBOX_THD_UNLOCKED); } /** * lpfc_sli4_unreg_rpi_cmpl_clr - mailbox completion handler @@ -2541,7 +2961,7 @@ lpfc_sli_def_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) * * This function is the unreg rpi mailbox completion handler. It * frees the memory resources associated with the completed mailbox - * command. An additional refrenece is put on the ndlp to prevent + * command. An additional reference is put on the ndlp to prevent * lpfc_nlp_release from freeing the rpi bit in the bitmask before * the unreg mailbox command completes, this routine puts the * reference back. @@ -2552,6 +2972,7 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) { struct lpfc_vport *vport = pmb->vport; struct lpfc_nodelist *ndlp; + bool unreg_inp; ndlp = pmb->ctx_ndlp; if (pmb->u.mb.mbxCommand == MBX_UNREG_LOGIN) { @@ -2561,38 +2982,44 @@ lpfc_sli4_unreg_rpi_cmpl_clr(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) LPFC_SLI_INTF_IF_TYPE_2)) { if (ndlp) { lpfc_printf_vlog( - vport, KERN_INFO, LOG_MBOX | LOG_SLI, - "0010 UNREG_LOGIN vpi:%x " - "rpi:%x DID:%x defer x%x flg x%x " - "map:%x %p\n", + vport, KERN_INFO, + LOG_MBOX | LOG_SLI | LOG_NODE, + "0010 UNREG_LOGIN vpi:x%x " + "rpi:%x DID:%x defer x%x flg x%lx " + "x%px\n", vport->vpi, ndlp->nlp_rpi, ndlp->nlp_DID, ndlp->nlp_defer_did, ndlp->nlp_flag, - ndlp->nlp_usg_map, ndlp); - ndlp->nlp_flag &= ~NLP_LOGO_ACC; - lpfc_nlp_put(ndlp); + ndlp); + + /* Cleanup the nlp_flag now that the UNREG RPI + * has completed. + */ + unreg_inp = test_and_clear_bit(NLP_UNREG_INP, + &ndlp->nlp_flag); + clear_bit(NLP_LOGO_ACC, &ndlp->nlp_flag); /* Check to see if there are any deferred * events to process */ - if ((ndlp->nlp_flag & NLP_UNREG_INP) && - (ndlp->nlp_defer_did != - NLP_EVT_NOTHING_PENDING)) { + if (unreg_inp && + ndlp->nlp_defer_did != + NLP_EVT_NOTHING_PENDING) { lpfc_printf_vlog( - vport, KERN_INFO, LOG_DISCOVERY, + vport, KERN_INFO, + LOG_MBOX | LOG_SLI | LOG_NODE, "4111 UNREG cmpl deferred " "clr x%x on " - "NPort x%x Data: x%x %p\n", + "NPort x%x Data: x%x x%px\n", ndlp->nlp_rpi, ndlp->nlp_DID, ndlp->nlp_defer_did, ndlp); - ndlp->nlp_flag &= ~NLP_UNREG_INP; ndlp->nlp_defer_did = NLP_EVT_NOTHING_PENDING; lpfc_issue_els_plogi( vport, ndlp->nlp_DID, 0); - } else { - ndlp->nlp_flag &= ~NLP_UNREG_INP; } + + lpfc_nlp_put(ndlp); } } } @@ -2661,10 +3088,11 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) if (lpfc_sli_chk_mbx_command(pmbox->mbxCommand) == MBX_SHUTDOWN) { /* Unknown mailbox command compl */ - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):0323 Unknown Mailbox command " "x%x (x%x/x%x) Cmpl\n", - pmb->vport ? pmb->vport->vpi : 0, + pmb->vport ? pmb->vport->vpi : + LPFC_VPORT_UNKNOWN, pmbox->mbxCommand, lpfc_sli_config_mbox_subsys_get(phba, pmb), @@ -2685,7 +3113,8 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) "(%d):0305 Mbox cmd cmpl " "error - RETRYing Data: x%x " "(x%x/x%x) x%x x%x x%x\n", - pmb->vport ? pmb->vport->vpi : 0, + pmb->vport ? pmb->vport->vpi : + LPFC_VPORT_UNKNOWN, pmbox->mbxCommand, lpfc_sli_config_mbox_subsys_get(phba, pmb), @@ -2693,7 +3122,8 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) pmb), pmbox->mbxStatus, pmbox->un.varWords[0], - pmb->vport->port_state); + pmb->vport ? pmb->vport->port_state : + LPFC_VPORT_UNKNOWN); pmbox->mbxStatus = 0; pmbox->mbxOwner = OWN_HOST; rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT); @@ -2704,7 +3134,7 @@ lpfc_sli_handle_mb_event(struct lpfc_hba *phba) /* Mailbox cmd <cmd> Cmpl <cmpl> */ lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI, - "(%d):0307 Mailbox cmd x%x (x%x/x%x) Cmpl x%p " + "(%d):0307 Mailbox cmd x%x (x%x/x%x) Cmpl %ps " "Data: x%x x%x x%x x%x x%x x%x x%x x%x x%x " "x%x x%x x%x\n", pmb->vport ? pmb->vport->vpi : 0, @@ -2759,6 +3189,144 @@ lpfc_sli_get_buff(struct lpfc_hba *phba, } /** + * lpfc_nvme_unsol_ls_handler - Process an unsolicited event data buffer + * containing a NVME LS request. + * @phba: pointer to lpfc hba data structure. + * @piocb: pointer to the iocbq struct representing the sequence starting + * frame. + * + * This routine initially validates the NVME LS, validates there is a login + * with the port that sent the LS, and then calls the appropriate nvme host + * or target LS request handler. + **/ +static void +lpfc_nvme_unsol_ls_handler(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) +{ + struct lpfc_nodelist *ndlp; + struct lpfc_dmabuf *d_buf; + struct hbq_dmabuf *nvmebuf; + struct fc_frame_header *fc_hdr; + struct lpfc_async_xchg_ctx *axchg = NULL; + char *failwhy = NULL; + uint32_t oxid, sid, did, fctl, size; + int ret = 1; + + d_buf = piocb->cmd_dmabuf; + + nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf); + fc_hdr = nvmebuf->hbuf.virt; + oxid = be16_to_cpu(fc_hdr->fh_ox_id); + sid = sli4_sid_from_fc_hdr(fc_hdr); + did = sli4_did_from_fc_hdr(fc_hdr); + fctl = (fc_hdr->fh_f_ctl[0] << 16 | + fc_hdr->fh_f_ctl[1] << 8 | + fc_hdr->fh_f_ctl[2]); + size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl); + + lpfc_nvmeio_data(phba, "NVME LS RCV: xri x%x sz %d from %06x\n", + oxid, size, sid); + + if (test_bit(FC_UNLOADING, &phba->pport->load_flag)) { + failwhy = "Driver Unloading"; + } else if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) { + failwhy = "NVME FC4 Disabled"; + } else if (!phba->nvmet_support && !phba->pport->localport) { + failwhy = "No Localport"; + } else if (phba->nvmet_support && !phba->targetport) { + failwhy = "No Targetport"; + } else if (unlikely(fc_hdr->fh_r_ctl != FC_RCTL_ELS4_REQ)) { + failwhy = "Bad NVME LS R_CTL"; + } else if (unlikely((fctl & 0x00FF0000) != + (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT))) { + failwhy = "Bad NVME LS F_CTL"; + } else { + axchg = kzalloc(sizeof(*axchg), GFP_ATOMIC); + if (!axchg) + failwhy = "No CTX memory"; + } + + if (unlikely(failwhy)) { + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6154 Drop NVME LS: SID %06X OXID x%X: %s\n", + sid, oxid, failwhy); + goto out_fail; + } + + /* validate the source of the LS is logged in */ + ndlp = lpfc_findnode_did(phba->pport, sid); + if (!ndlp || + ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && + (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, + "6216 NVME Unsol rcv: No ndlp: " + "NPort_ID x%x oxid x%x\n", + sid, oxid); + goto out_fail; + } + + axchg->phba = phba; + axchg->ndlp = ndlp; + axchg->size = size; + axchg->oxid = oxid; + axchg->sid = sid; + axchg->wqeq = NULL; + axchg->state = LPFC_NVME_STE_LS_RCV; + axchg->entry_cnt = 1; + axchg->rqb_buffer = (void *)nvmebuf; + axchg->hdwq = &phba->sli4_hba.hdwq[0]; + axchg->payload = nvmebuf->dbuf.virt; + INIT_LIST_HEAD(&axchg->list); + + if (phba->nvmet_support) { + ret = lpfc_nvmet_handle_lsreq(phba, axchg); + spin_lock_irq(&ndlp->lock); + if (!ret && !(ndlp->fc4_xpt_flags & NLP_XPT_HAS_HH)) { + ndlp->fc4_xpt_flags |= NLP_XPT_HAS_HH; + spin_unlock_irq(&ndlp->lock); + + /* This reference is a single occurrence to hold the + * node valid until the nvmet transport calls + * host_release. + */ + if (!lpfc_nlp_get(ndlp)) + goto out_fail; + + lpfc_printf_log(phba, KERN_ERR, LOG_NODE, + "6206 NVMET unsol ls_req ndlp x%px " + "DID x%x xflags x%x refcnt %d\n", + ndlp, ndlp->nlp_DID, + ndlp->fc4_xpt_flags, + kref_read(&ndlp->kref)); + } else { + spin_unlock_irq(&ndlp->lock); + } + } else { + ret = lpfc_nvme_handle_lsreq(phba, axchg); + } + + /* if zero, LS was successfully handled. If non-zero, LS not handled */ + if (!ret) + return; + +out_fail: + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6155 Drop NVME LS from DID %06X: SID %06X OXID x%X " + "NVMe%s handler failed %d\n", + did, sid, oxid, + (phba->nvmet_support) ? "T" : "I", ret); + + /* recycle receive buffer */ + lpfc_in_buf_free(phba, &nvmebuf->dbuf); + + /* If start of new exchange, abort it */ + if (axchg && (fctl & FC_FC_FIRST_SEQ && !(fctl & FC_FC_EX_CTX))) + ret = lpfc_nvme_unsol_ls_issue_abort(phba, axchg, sid, oxid); + + if (ret) + kfree(axchg); +} + +/** * lpfc_complete_unsol_iocb - Complete an unsolicited sequence * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. @@ -2779,7 +3347,7 @@ lpfc_complete_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, switch (fch_type) { case FC_TYPE_NVME: - lpfc_nvmet_unsol_ls_event(phba, pring, saveq); + lpfc_nvme_unsol_ls_handler(phba, saveq); return 1; default: break; @@ -2806,6 +3374,56 @@ lpfc_complete_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, return 0; } +static void +lpfc_sli_prep_unsol_wqe(struct lpfc_hba *phba, + struct lpfc_iocbq *saveq) +{ + IOCB_t *irsp; + union lpfc_wqe128 *wqe; + u16 i = 0; + + irsp = &saveq->iocb; + wqe = &saveq->wqe; + + /* Fill wcqe with the IOCB status fields */ + bf_set(lpfc_wcqe_c_status, &saveq->wcqe_cmpl, irsp->ulpStatus); + saveq->wcqe_cmpl.word3 = irsp->ulpBdeCount; + saveq->wcqe_cmpl.parameter = irsp->un.ulpWord[4]; + saveq->wcqe_cmpl.total_data_placed = irsp->unsli3.rcvsli3.acc_len; + + /* Source ID */ + bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, irsp->un.rcvels.parmRo); + + /* rx-id of the response frame */ + bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com, irsp->ulpContext); + + /* ox-id of the frame */ + bf_set(wqe_rcvoxid, &wqe->xmit_els_rsp.wqe_com, + irsp->unsli3.rcvsli3.ox_id); + + /* DID */ + bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest, + irsp->un.rcvels.remoteID); + + /* unsol data len */ + for (i = 0; i < irsp->ulpBdeCount; i++) { + struct lpfc_hbq_entry *hbqe = NULL; + + if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) { + if (i == 0) { + hbqe = (struct lpfc_hbq_entry *) + &irsp->un.ulpWord[0]; + saveq->wqe.gen_req.bde.tus.f.bdeSize = + hbqe->bde.tus.f.bdeSize; + } else if (i == 1) { + hbqe = (struct lpfc_hbq_entry *) + &irsp->unsli3.sli3Words[4]; + saveq->unsol_rcv_len = hbqe->bde.tus.f.bdeSize; + } + } + } +} + /** * lpfc_sli_process_unsol_iocb - Unsolicited iocb handler * @phba: Pointer to HBA context object. @@ -2826,11 +3444,13 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, { IOCB_t * irsp; WORD5 * w5p; + dma_addr_t paddr; uint32_t Rctl, Type; struct lpfc_iocbq *iocbq; struct lpfc_dmabuf *dmzbuf; - irsp = &(saveq->iocb); + irsp = &saveq->iocb; + saveq->vport = phba->pport; if (irsp->ulpCommand == CMD_ASYNC_STATUS) { if (pring->lpfc_sli_rcv_async_status) @@ -2848,22 +3468,22 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, } if ((irsp->ulpCommand == CMD_IOCB_RET_XRI64_CX) && - (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED)) { + (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED)) { if (irsp->ulpBdeCount > 0) { dmzbuf = lpfc_sli_get_buff(phba, pring, - irsp->un.ulpWord[3]); + irsp->un.ulpWord[3]); lpfc_in_buf_free(phba, dmzbuf); } if (irsp->ulpBdeCount > 1) { dmzbuf = lpfc_sli_get_buff(phba, pring, - irsp->unsli3.sli3Words[3]); + irsp->unsli3.sli3Words[3]); lpfc_in_buf_free(phba, dmzbuf); } if (irsp->ulpBdeCount > 2) { dmzbuf = lpfc_sli_get_buff(phba, pring, - irsp->unsli3.sli3Words[7]); + irsp->unsli3.sli3Words[7]); lpfc_in_buf_free(phba, dmzbuf); } @@ -2872,9 +3492,9 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, if (phba->sli3_options & LPFC_SLI3_HBQ_ENABLED) { if (irsp->ulpBdeCount != 0) { - saveq->context2 = lpfc_sli_get_buff(phba, pring, + saveq->cmd_dmabuf = lpfc_sli_get_buff(phba, pring, irsp->un.ulpWord[3]); - if (!saveq->context2) + if (!saveq->cmd_dmabuf) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -2884,9 +3504,9 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, irsp->un.ulpWord[3]); } if (irsp->ulpBdeCount == 2) { - saveq->context3 = lpfc_sli_get_buff(phba, pring, + saveq->bpl_dmabuf = lpfc_sli_get_buff(phba, pring, irsp->unsli3.sli3Words[7]); - if (!saveq->context3) + if (!saveq->bpl_dmabuf) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -2896,11 +3516,12 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, irsp->unsli3.sli3Words[7]); } list_for_each_entry(iocbq, &saveq->list, list) { - irsp = &(iocbq->iocb); + irsp = &iocbq->iocb; if (irsp->ulpBdeCount != 0) { - iocbq->context2 = lpfc_sli_get_buff(phba, pring, + iocbq->cmd_dmabuf = lpfc_sli_get_buff(phba, + pring, irsp->un.ulpWord[3]); - if (!iocbq->context2) + if (!iocbq->cmd_dmabuf) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -2910,9 +3531,10 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, irsp->un.ulpWord[3]); } if (irsp->ulpBdeCount == 2) { - iocbq->context3 = lpfc_sli_get_buff(phba, pring, + iocbq->bpl_dmabuf = lpfc_sli_get_buff(phba, + pring, irsp->unsli3.sli3Words[7]); - if (!iocbq->context3) + if (!iocbq->bpl_dmabuf) lpfc_printf_log(phba, KERN_ERR, LOG_SLI, @@ -2923,7 +3545,20 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, irsp->unsli3.sli3Words[7]); } } + } else { + paddr = getPaddr(irsp->un.cont64[0].addrHigh, + irsp->un.cont64[0].addrLow); + saveq->cmd_dmabuf = lpfc_sli_ringpostbuf_get(phba, pring, + paddr); + if (irsp->ulpBdeCount == 2) { + paddr = getPaddr(irsp->un.cont64[1].addrHigh, + irsp->un.cont64[1].addrLow); + saveq->bpl_dmabuf = lpfc_sli_ringpostbuf_get(phba, + pring, + paddr); + } } + if (irsp->ulpBdeCount != 0 && (irsp->ulpCommand == CMD_IOCB_RCV_CONT64_CX || irsp->ulpStatus == IOSTAT_INTERMED_RSP)) { @@ -2941,12 +3576,14 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, if (!found) list_add_tail(&saveq->clist, &pring->iocb_continue_saveq); + if (saveq->iocb.ulpStatus != IOSTAT_INTERMED_RSP) { list_del_init(&iocbq->clist); saveq = iocbq; - irsp = &(saveq->iocb); - } else + irsp = &saveq->iocb; + } else { return 0; + } } if ((irsp->ulpCommand == CMD_RCV_ELS_REQ64_CX) || (irsp->ulpCommand == CMD_RCV_ELS_REQ_CX) || @@ -2969,6 +3606,19 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, } } + if ((phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) && + (irsp->ulpCommand == CMD_IOCB_RCV_ELS64_CX || + irsp->ulpCommand == CMD_IOCB_RCV_SEQ64_CX)) { + if (irsp->unsli3.rcvsli3.vpi == 0xffff) + saveq->vport = phba->pport; + else + saveq->vport = lpfc_find_vport_by_vpid(phba, + irsp->unsli3.rcvsli3.vpi); + } + + /* Prepare WQE with Unsol frame */ + lpfc_sli_prep_unsol_wqe(phba, saveq); + if (!lpfc_complete_unsol_iocb(phba, pring, saveq, Rctl, Type)) lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, "0313 Ring %d handler: unexpected Rctl x%x " @@ -2986,8 +3636,8 @@ lpfc_sli_process_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, * * This function looks up the iocb_lookup table to get the command iocb * corresponding to the given response iocb using the iotag of the - * response iocb. This function is called with the hbalock held - * for sli3 devices or the ring_lock for sli4 devices. + * response iocb. The driver calls this function with the hbalock held + * for SLI3 ports or the ring lock held for SLI4 ports. * This function returns the command iocb object if it finds the command * iocb else returns NULL. **/ @@ -2997,26 +3647,28 @@ lpfc_sli_iocbq_lookup(struct lpfc_hba *phba, struct lpfc_iocbq *prspiocb) { struct lpfc_iocbq *cmd_iocb = NULL; - uint16_t iotag; - lockdep_assert_held(&phba->hbalock); + u16 iotag; - iotag = prspiocb->iocb.ulpIoTag; + if (phba->sli_rev == LPFC_SLI_REV4) + iotag = get_wqe_reqtag(prspiocb); + else + iotag = prspiocb->iocb.ulpIoTag; if (iotag != 0 && iotag <= phba->sli.last_iotag) { cmd_iocb = phba->sli.iocbq_lookup[iotag]; - if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) { + if (cmd_iocb->cmd_flag & LPFC_IO_ON_TXCMPLQ) { /* remove from txcmpl queue list */ list_del_init(&cmd_iocb->list); - cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + cmd_iocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt--; return cmd_iocb; } } - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0317 iotag x%x is out of " - "range: max iotag x%x wd0 x%x\n", - iotag, phba->sli.last_iotag, - *(((uint32_t *) &prspiocb->iocb) + 7)); + "range: max iotag x%x\n", + iotag, phba->sli.last_iotag); return NULL; } @@ -3027,8 +3679,8 @@ lpfc_sli_iocbq_lookup(struct lpfc_hba *phba, * @iotag: IOCB tag. * * This function looks up the iocb_lookup table to get the command iocb - * corresponding to the given iotag. This function is called with the - * hbalock held. + * corresponding to the given iotag. The driver calls this function with + * the ring lock held because this function is an SLI4 port only helper. * This function returns the command iocb object if it finds the command * iocb else returns NULL. **/ @@ -3038,22 +3690,22 @@ lpfc_sli_iocbq_lookup_by_tag(struct lpfc_hba *phba, { struct lpfc_iocbq *cmd_iocb = NULL; - lockdep_assert_held(&phba->hbalock); if (iotag != 0 && iotag <= phba->sli.last_iotag) { cmd_iocb = phba->sli.iocbq_lookup[iotag]; - if (cmd_iocb->iocb_flag & LPFC_IO_ON_TXCMPLQ) { + if (cmd_iocb->cmd_flag & LPFC_IO_ON_TXCMPLQ) { /* remove from txcmpl queue list */ list_del_init(&cmd_iocb->list); - cmd_iocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + cmd_iocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ; + pring->txcmplq_cnt--; return cmd_iocb; } } - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0372 iotag x%x lookup error: max iotag (x%x) " - "iocb_flag x%x\n", + "cmd_flag x%x\n", iotag, phba->sli.last_iotag, - cmd_iocb ? cmd_iocb->iocb_flag : 0xffff); + cmd_iocb ? cmd_iocb->cmd_flag : 0xffff); return NULL; } @@ -3079,10 +3731,9 @@ lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_iocbq *saveq) { struct lpfc_iocbq *cmdiocbp; - int rc = 1; unsigned long iflag; + u32 ulp_command, ulp_status, ulp_word4, ulp_context, iotag; - /* Based on the iotag field, get the cmd IOCB from the txcmplq */ if (phba->sli_rev == LPFC_SLI_REV4) spin_lock_irqsave(&pring->ring_lock, iflag); else @@ -3093,16 +3744,25 @@ lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, else spin_unlock_irqrestore(&phba->hbalock, iflag); + ulp_command = get_job_cmnd(phba, saveq); + ulp_status = get_job_ulpstatus(phba, saveq); + ulp_word4 = get_job_word4(phba, saveq); + ulp_context = get_job_ulpcontext(phba, saveq); + if (phba->sli_rev == LPFC_SLI_REV4) + iotag = get_wqe_reqtag(saveq); + else + iotag = saveq->iocb.ulpIoTag; + if (cmdiocbp) { - if (cmdiocbp->iocb_cmpl) { + ulp_command = get_job_cmnd(phba, cmdiocbp); + if (cmdiocbp->cmd_cmpl) { /* * If an ELS command failed send an event to mgmt * application. */ - if (saveq->iocb.ulpStatus && + if (ulp_status && (pring->ringno == LPFC_ELS_RING) && - (cmdiocbp->iocb.ulpCommand == - CMD_ELS_REQUEST64_CR)) + (ulp_command == CMD_ELS_REQUEST64_CR)) lpfc_send_els_failure_event(phba, cmdiocbp, saveq); @@ -3112,11 +3772,11 @@ lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, */ if (pring->ringno == LPFC_ELS_RING) { if ((phba->sli_rev < LPFC_SLI_REV4) && - (cmdiocbp->iocb_flag & + (cmdiocbp->cmd_flag & LPFC_DRIVER_ABORTED)) { spin_lock_irqsave(&phba->hbalock, iflag); - cmdiocbp->iocb_flag &= + cmdiocbp->cmd_flag &= ~LPFC_DRIVER_ABORTED; spin_unlock_irqrestore(&phba->hbalock, iflag); @@ -3131,12 +3791,12 @@ lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, */ spin_lock_irqsave(&phba->hbalock, iflag); - saveq->iocb_flag |= LPFC_DELAY_MEM_FREE; + saveq->cmd_flag |= LPFC_DELAY_MEM_FREE; spin_unlock_irqrestore(&phba->hbalock, iflag); } if (phba->sli_rev == LPFC_SLI_REV4) { - if (saveq->iocb_flag & + if (saveq->cmd_flag & LPFC_EXCHANGE_BUSY) { /* Set cmdiocb flag for the * exchange busy so sgl (xri) @@ -3146,12 +3806,12 @@ lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, */ spin_lock_irqsave( &phba->hbalock, iflag); - cmdiocbp->iocb_flag |= + cmdiocbp->cmd_flag |= LPFC_EXCHANGE_BUSY; spin_unlock_irqrestore( &phba->hbalock, iflag); } - if (cmdiocbp->iocb_flag & + if (cmdiocbp->cmd_flag & LPFC_DRIVER_ABORTED) { /* * Clear LPFC_DRIVER_ABORTED @@ -3160,34 +3820,34 @@ lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, */ spin_lock_irqsave( &phba->hbalock, iflag); - cmdiocbp->iocb_flag &= + cmdiocbp->cmd_flag &= ~LPFC_DRIVER_ABORTED; spin_unlock_irqrestore( &phba->hbalock, iflag); - cmdiocbp->iocb.ulpStatus = - IOSTAT_LOCAL_REJECT; - cmdiocbp->iocb.un.ulpWord[4] = - IOERR_ABORT_REQUESTED; + set_job_ulpstatus(cmdiocbp, + IOSTAT_LOCAL_REJECT); + set_job_ulpword4(cmdiocbp, + IOERR_ABORT_REQUESTED); /* - * For SLI4, irsiocb contains + * For SLI4, irspiocb contains * NO_XRI in sli_xritag, it * shall not affect releasing * sgl (xri) process. */ - saveq->iocb.ulpStatus = - IOSTAT_LOCAL_REJECT; - saveq->iocb.un.ulpWord[4] = - IOERR_SLI_ABORTED; + set_job_ulpstatus(saveq, + IOSTAT_LOCAL_REJECT); + set_job_ulpword4(saveq, + IOERR_SLI_ABORTED); spin_lock_irqsave( &phba->hbalock, iflag); - saveq->iocb_flag |= + saveq->cmd_flag |= LPFC_DELAY_MEM_FREE; spin_unlock_irqrestore( &phba->hbalock, iflag); } } } - (cmdiocbp->iocb_cmpl) (phba, cmdiocbp, saveq); + cmdiocbp->cmd_cmpl(phba, cmdiocbp, saveq); } else lpfc_sli_release_iocbq(phba, cmdiocbp); } else { @@ -3205,16 +3865,12 @@ lpfc_sli_process_sol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, "0322 Ring %d handler: " "unexpected completion IoTag x%x " "Data: x%x x%x x%x x%x\n", - pring->ringno, - saveq->iocb.ulpIoTag, - saveq->iocb.ulpStatus, - saveq->iocb.un.ulpWord[4], - saveq->iocb.ulpCommand, - saveq->iocb.ulpContext); + pring->ringno, iotag, ulp_status, + ulp_word4, ulp_command, ulp_context); } } - return rc; + return 1; } /** @@ -3235,7 +3891,7 @@ lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring) * Ring <ringno> handler: portRspPut <portRspPut> is bigger than * rsp ring <portRspMax> */ - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0312 Ring %d handler: portRspPut %d " "is bigger than rsp ring %d\n", pring->ringno, le32_to_cpu(pgp->rspPutInx), @@ -3257,7 +3913,7 @@ lpfc_sli_rsp_pointers_error(struct lpfc_hba *phba, struct lpfc_sli_ring *pring) /** * lpfc_poll_eratt - Error attention polling timer timeout handler - * @ptr: Pointer to address of HBA context object. + * @t: Context to fetch pointer to address of HBA context object from. * * This function is invoked by the Error Attention polling timer when the * timer times out. It will check the SLI Error Attention register for @@ -3271,7 +3927,19 @@ void lpfc_poll_eratt(struct timer_list *t) uint32_t eratt = 0; uint64_t sli_intr, cnt; - phba = from_timer(phba, t, eratt_poll); + phba = timer_container_of(phba, t, eratt_poll); + + if (test_bit(FC_UNLOADING, &phba->pport->load_flag)) + return; + + if (phba->sli_rev == LPFC_SLI_REV4 && + !test_bit(HBA_SETUP, &phba->hba_flag)) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "0663 HBA still initializing 0x%lx, restart " + "timer\n", + phba->hba_flag); + goto restart_timer; + } /* Here we will also keep track of interrupts per sec of the hba */ sli_intr = phba->sli.slistat.sli_intr; @@ -3291,14 +3959,16 @@ void lpfc_poll_eratt(struct timer_list *t) /* Check chip HA register for error event */ eratt = lpfc_sli_check_eratt(phba); - if (eratt) + if (eratt) { /* Tell the worker thread there is work to do */ lpfc_worker_wake_up(phba); - else - /* Restart the timer for next eratt poll */ - mod_timer(&phba->eratt_poll, - jiffies + - msecs_to_jiffies(1000 * phba->eratt_poll_interval)); + return; + } + +restart_timer: + /* Restart the timer for next eratt poll */ + mod_timer(&phba->eratt_poll, + jiffies + secs_to_jiffies(phba->eratt_poll_interval)); return; } @@ -3427,12 +4097,11 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba, &rspiocbq); if (unlikely(!cmdiocbq)) break; - if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) - cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED; - if (cmdiocbq->iocb_cmpl) { + if (cmdiocbq->cmd_flag & LPFC_DRIVER_ABORTED) + cmdiocbq->cmd_flag &= ~LPFC_DRIVER_ABORTED; + if (cmdiocbq->cmd_cmpl) { spin_unlock_irqrestore(&phba->hbalock, iflag); - (cmdiocbq->iocb_cmpl)(phba, cmdiocbq, - &rspiocbq); + cmdiocbq->cmd_cmpl(phba, cmdiocbq, &rspiocbq); spin_lock_irqsave(&phba->hbalock, iflag); } break; @@ -3452,7 +4121,7 @@ lpfc_sli_handle_fast_ring_event(struct lpfc_hba *phba, phba->brd_no, adaptermsg); } else { /* Unknown IOCB command */ - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0334 Unknown IOCB command " "Data: x%x, x%x x%x x%x x%x\n", type, irsp->ulpCommand, @@ -3523,152 +4192,159 @@ lpfc_sli_sp_handle_rspiocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_iocbq *rspiocbp) { struct lpfc_iocbq *saveq; - struct lpfc_iocbq *cmdiocbp; + struct lpfc_iocbq *cmdiocb; struct lpfc_iocbq *next_iocb; - IOCB_t *irsp = NULL; + IOCB_t *irsp; uint32_t free_saveq; - uint8_t iocb_cmd_type; + u8 cmd_type; lpfc_iocb_type type; unsigned long iflag; + u32 ulp_status = get_job_ulpstatus(phba, rspiocbp); + u32 ulp_word4 = get_job_word4(phba, rspiocbp); + u32 ulp_command = get_job_cmnd(phba, rspiocbp); int rc; spin_lock_irqsave(&phba->hbalock, iflag); /* First add the response iocb to the countinueq list */ - list_add_tail(&rspiocbp->list, &(pring->iocb_continueq)); + list_add_tail(&rspiocbp->list, &pring->iocb_continueq); pring->iocb_continueq_cnt++; - /* Now, determine whether the list is completed for processing */ - irsp = &rspiocbp->iocb; - if (irsp->ulpLe) { - /* - * By default, the driver expects to free all resources - * associated with this iocb completion. - */ - free_saveq = 1; - saveq = list_get_first(&pring->iocb_continueq, - struct lpfc_iocbq, list); - irsp = &(saveq->iocb); - list_del_init(&pring->iocb_continueq); - pring->iocb_continueq_cnt = 0; + /* + * By default, the driver expects to free all resources + * associated with this iocb completion. + */ + free_saveq = 1; + saveq = list_get_first(&pring->iocb_continueq, + struct lpfc_iocbq, list); + list_del_init(&pring->iocb_continueq); + pring->iocb_continueq_cnt = 0; - pring->stats.iocb_rsp++; + pring->stats.iocb_rsp++; - /* - * If resource errors reported from HBA, reduce - * queuedepths of the SCSI device. - */ - if ((irsp->ulpStatus == IOSTAT_LOCAL_REJECT) && - ((irsp->un.ulpWord[4] & IOERR_PARAM_MASK) == - IOERR_NO_RESOURCES)) { - spin_unlock_irqrestore(&phba->hbalock, iflag); - phba->lpfc_rampdown_queue_depth(phba); - spin_lock_irqsave(&phba->hbalock, iflag); - } + /* + * If resource errors reported from HBA, reduce + * queuedepths of the SCSI device. + */ + if (ulp_status == IOSTAT_LOCAL_REJECT && + ((ulp_word4 & IOERR_PARAM_MASK) == + IOERR_NO_RESOURCES)) { + spin_unlock_irqrestore(&phba->hbalock, iflag); + phba->lpfc_rampdown_queue_depth(phba); + spin_lock_irqsave(&phba->hbalock, iflag); + } - if (irsp->ulpStatus) { - /* Rsp ring <ringno> error: IOCB */ + if (ulp_status) { + /* Rsp ring <ringno> error: IOCB */ + if (phba->sli_rev < LPFC_SLI_REV4) { + irsp = &rspiocbp->iocb; + lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, + "0328 Rsp Ring %d error: ulp_status x%x " + "IOCB Data: " + "x%08x x%08x x%08x x%08x " + "x%08x x%08x x%08x x%08x " + "x%08x x%08x x%08x x%08x " + "x%08x x%08x x%08x x%08x\n", + pring->ringno, ulp_status, + get_job_ulpword(rspiocbp, 0), + get_job_ulpword(rspiocbp, 1), + get_job_ulpword(rspiocbp, 2), + get_job_ulpword(rspiocbp, 3), + get_job_ulpword(rspiocbp, 4), + get_job_ulpword(rspiocbp, 5), + *(((uint32_t *)irsp) + 6), + *(((uint32_t *)irsp) + 7), + *(((uint32_t *)irsp) + 8), + *(((uint32_t *)irsp) + 9), + *(((uint32_t *)irsp) + 10), + *(((uint32_t *)irsp) + 11), + *(((uint32_t *)irsp) + 12), + *(((uint32_t *)irsp) + 13), + *(((uint32_t *)irsp) + 14), + *(((uint32_t *)irsp) + 15)); + } else { lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, - "0328 Rsp Ring %d error: " + "0321 Rsp Ring %d error: " "IOCB Data: " - "x%x x%x x%x x%x " - "x%x x%x x%x x%x " - "x%x x%x x%x x%x " "x%x x%x x%x x%x\n", pring->ringno, - irsp->un.ulpWord[0], - irsp->un.ulpWord[1], - irsp->un.ulpWord[2], - irsp->un.ulpWord[3], - irsp->un.ulpWord[4], - irsp->un.ulpWord[5], - *(((uint32_t *) irsp) + 6), - *(((uint32_t *) irsp) + 7), - *(((uint32_t *) irsp) + 8), - *(((uint32_t *) irsp) + 9), - *(((uint32_t *) irsp) + 10), - *(((uint32_t *) irsp) + 11), - *(((uint32_t *) irsp) + 12), - *(((uint32_t *) irsp) + 13), - *(((uint32_t *) irsp) + 14), - *(((uint32_t *) irsp) + 15)); + rspiocbp->wcqe_cmpl.word0, + rspiocbp->wcqe_cmpl.total_data_placed, + rspiocbp->wcqe_cmpl.parameter, + rspiocbp->wcqe_cmpl.word3); } + } - /* - * Fetch the IOCB command type and call the correct completion - * routine. Solicited and Unsolicited IOCBs on the ELS ring - * get freed back to the lpfc_iocb_list by the discovery - * kernel thread. - */ - iocb_cmd_type = irsp->ulpCommand & CMD_IOCB_MASK; - type = lpfc_sli_iocb_cmd_type(iocb_cmd_type); - switch (type) { - case LPFC_SOL_IOCB: - spin_unlock_irqrestore(&phba->hbalock, iflag); - rc = lpfc_sli_process_sol_iocb(phba, pring, saveq); - spin_lock_irqsave(&phba->hbalock, iflag); - break; - - case LPFC_UNSOL_IOCB: - spin_unlock_irqrestore(&phba->hbalock, iflag); - rc = lpfc_sli_process_unsol_iocb(phba, pring, saveq); - spin_lock_irqsave(&phba->hbalock, iflag); - if (!rc) - free_saveq = 0; - break; - - case LPFC_ABORT_IOCB: - cmdiocbp = NULL; - if (irsp->ulpCommand != CMD_XRI_ABORTED_CX) - cmdiocbp = lpfc_sli_iocbq_lookup(phba, pring, - saveq); - if (cmdiocbp) { - /* Call the specified completion routine */ - if (cmdiocbp->iocb_cmpl) { - spin_unlock_irqrestore(&phba->hbalock, - iflag); - (cmdiocbp->iocb_cmpl)(phba, cmdiocbp, - saveq); - spin_lock_irqsave(&phba->hbalock, - iflag); - } else - __lpfc_sli_release_iocbq(phba, - cmdiocbp); - } - break; - case LPFC_UNKNOWN_IOCB: - if (irsp->ulpCommand == CMD_ADAPTER_MSG) { - char adaptermsg[LPFC_MAX_ADPTMSG]; - memset(adaptermsg, 0, LPFC_MAX_ADPTMSG); - memcpy(&adaptermsg[0], (uint8_t *)irsp, - MAX_MSG_DATA); - dev_warn(&((phba->pcidev)->dev), - "lpfc%d: %s\n", - phba->brd_no, adaptermsg); + /* + * Fetch the iocb command type and call the correct completion + * routine. Solicited and Unsolicited IOCBs on the ELS ring + * get freed back to the lpfc_iocb_list by the discovery + * kernel thread. + */ + cmd_type = ulp_command & CMD_IOCB_MASK; + type = lpfc_sli_iocb_cmd_type(cmd_type); + switch (type) { + case LPFC_SOL_IOCB: + spin_unlock_irqrestore(&phba->hbalock, iflag); + rc = lpfc_sli_process_sol_iocb(phba, pring, saveq); + spin_lock_irqsave(&phba->hbalock, iflag); + break; + case LPFC_UNSOL_IOCB: + spin_unlock_irqrestore(&phba->hbalock, iflag); + rc = lpfc_sli_process_unsol_iocb(phba, pring, saveq); + spin_lock_irqsave(&phba->hbalock, iflag); + if (!rc) + free_saveq = 0; + break; + case LPFC_ABORT_IOCB: + cmdiocb = NULL; + if (ulp_command != CMD_XRI_ABORTED_CX) + cmdiocb = lpfc_sli_iocbq_lookup(phba, pring, + saveq); + if (cmdiocb) { + /* Call the specified completion routine */ + if (cmdiocb->cmd_cmpl) { + spin_unlock_irqrestore(&phba->hbalock, iflag); + cmdiocb->cmd_cmpl(phba, cmdiocb, saveq); + spin_lock_irqsave(&phba->hbalock, iflag); } else { - /* Unknown IOCB command */ - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0335 Unknown IOCB " - "command Data: x%x " - "x%x x%x x%x\n", - irsp->ulpCommand, - irsp->ulpStatus, - irsp->ulpIoTag, - irsp->ulpContext); + __lpfc_sli_release_iocbq(phba, cmdiocb); } - break; } + break; + case LPFC_UNKNOWN_IOCB: + if (ulp_command == CMD_ADAPTER_MSG) { + char adaptermsg[LPFC_MAX_ADPTMSG]; + + memset(adaptermsg, 0, LPFC_MAX_ADPTMSG); + memcpy(&adaptermsg[0], (uint8_t *)&rspiocbp->wqe, + MAX_MSG_DATA); + dev_warn(&((phba->pcidev)->dev), + "lpfc%d: %s\n", + phba->brd_no, adaptermsg); + } else { + /* Unknown command */ + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "0335 Unknown IOCB " + "command Data: x%x " + "x%x x%x x%x\n", + ulp_command, + ulp_status, + get_wqe_reqtag(rspiocbp), + get_job_ulpcontext(phba, rspiocbp)); + } + break; + } - if (free_saveq) { - list_for_each_entry_safe(rspiocbp, next_iocb, - &saveq->list, list) { - list_del_init(&rspiocbp->list); - __lpfc_sli_release_iocbq(phba, rspiocbp); - } - __lpfc_sli_release_iocbq(phba, saveq); + if (free_saveq) { + list_for_each_entry_safe(rspiocbp, next_iocb, + &saveq->list, list) { + list_del_init(&rspiocbp->list); + __lpfc_sli_release_iocbq(phba, rspiocbp); } - rspiocbp = NULL; + __lpfc_sli_release_iocbq(phba, saveq); } + rspiocbp = NULL; spin_unlock_irqrestore(&phba->hbalock, iflag); return rspiocbp; } @@ -3727,7 +4403,7 @@ lpfc_sli_handle_slow_ring_event_s3(struct lpfc_hba *phba, * Ring <ringno> handler: portRspPut <portRspPut> is bigger than * rsp ring <portRspMax> */ - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0303 Ring %d handler: portRspPut %d " "is bigger than rsp ring %d\n", pring->ringno, portRspPut, portRspMax); @@ -3846,9 +4522,7 @@ lpfc_sli_handle_slow_ring_event_s4(struct lpfc_hba *phba, unsigned long iflag; int count = 0; - spin_lock_irqsave(&phba->hbalock, iflag); - phba->hba_flag &= ~HBA_SP_QUEUE_EVT; - spin_unlock_irqrestore(&phba->hbalock, iflag); + clear_bit(HBA_SP_QUEUE_EVT, &phba->hba_flag); while (!list_empty(&phba->sli4_hba.sp_queue_event)) { /* Get the response iocb from the head of work queue */ spin_lock_irqsave(&phba->hbalock, iflag); @@ -3861,8 +4535,8 @@ lpfc_sli_handle_slow_ring_event_s4(struct lpfc_hba *phba, irspiocbq = container_of(cq_event, struct lpfc_iocbq, cq_event); /* Translate ELS WCQE to response IOCBQ */ - irspiocbq = lpfc_sli4_els_wcqe_to_rspiocbq(phba, - irspiocbq); + irspiocbq = lpfc_sli4_els_preprocess_rspiocbq(phba, + irspiocbq); if (irspiocbq) lpfc_sli_sp_handle_rspiocb(phba, pring, irspiocbq); @@ -3898,74 +4572,68 @@ lpfc_sli_handle_slow_ring_event_s4(struct lpfc_hba *phba, void lpfc_sli_abort_iocb_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring) { - LIST_HEAD(completions); + LIST_HEAD(tx_completions); + LIST_HEAD(txcmplq_completions); struct lpfc_iocbq *iocb, *next_iocb; + int offline; if (pring->ringno == LPFC_ELS_RING) { lpfc_fabric_abort_hba(phba); } + offline = pci_channel_offline(phba->pcidev); /* Error everything on txq and txcmplq * First do the txq. */ if (phba->sli_rev >= LPFC_SLI_REV4) { spin_lock_irq(&pring->ring_lock); - list_splice_init(&pring->txq, &completions); + list_splice_init(&pring->txq, &tx_completions); pring->txq_cnt = 0; - spin_unlock_irq(&pring->ring_lock); - spin_lock_irq(&phba->hbalock); - /* Next issue ABTS for everything on the txcmplq */ - list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list) - lpfc_sli_issue_abort_iotag(phba, pring, iocb); - spin_unlock_irq(&phba->hbalock); + if (offline) { + list_splice_init(&pring->txcmplq, + &txcmplq_completions); + } else { + /* Next issue ABTS for everything on the txcmplq */ + list_for_each_entry_safe(iocb, next_iocb, + &pring->txcmplq, list) + lpfc_sli_issue_abort_iotag(phba, pring, + iocb, NULL); + } + spin_unlock_irq(&pring->ring_lock); } else { spin_lock_irq(&phba->hbalock); - list_splice_init(&pring->txq, &completions); + list_splice_init(&pring->txq, &tx_completions); pring->txq_cnt = 0; - /* Next issue ABTS for everything on the txcmplq */ - list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list) - lpfc_sli_issue_abort_iotag(phba, pring, iocb); + if (offline) { + list_splice_init(&pring->txcmplq, &txcmplq_completions); + } else { + /* Next issue ABTS for everything on the txcmplq */ + list_for_each_entry_safe(iocb, next_iocb, + &pring->txcmplq, list) + lpfc_sli_issue_abort_iotag(phba, pring, + iocb, NULL); + } spin_unlock_irq(&phba->hbalock); } + if (offline) { + /* Cancel all the IOCBs from the completions list */ + lpfc_sli_cancel_iocbs(phba, &txcmplq_completions, + IOSTAT_LOCAL_REJECT, IOERR_SLI_ABORTED); + } else { + /* Make sure HBA is alive */ + lpfc_issue_hb_tmo(phba); + } /* Cancel all the IOCBs from the completions list */ - lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT, + lpfc_sli_cancel_iocbs(phba, &tx_completions, IOSTAT_LOCAL_REJECT, IOERR_SLI_ABORTED); } /** - * lpfc_sli_abort_wqe_ring - Abort all iocbs in the ring - * @phba: Pointer to HBA context object. - * @pring: Pointer to driver SLI ring object. - * - * This function aborts all iocbs in the given ring and frees all the iocb - * objects in txq. This function issues an abort iocb for all the iocb commands - * in txcmplq. The iocbs in the txcmplq is not guaranteed to complete before - * the return of this function. The caller is not required to hold any locks. - **/ -void -lpfc_sli_abort_wqe_ring(struct lpfc_hba *phba, struct lpfc_sli_ring *pring) -{ - LIST_HEAD(completions); - struct lpfc_iocbq *iocb, *next_iocb; - - if (pring->ringno == LPFC_ELS_RING) - lpfc_fabric_abort_hba(phba); - - spin_lock_irq(&phba->hbalock); - /* Next issue ABTS for everything on the txcmplq */ - list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list) - lpfc_sli4_abort_nvme_io(phba, pring, iocb); - spin_unlock_irq(&phba->hbalock); -} - - -/** * lpfc_sli_abort_fcp_rings - Abort all iocbs in all FCP rings * @phba: Pointer to HBA context object. - * @pring: Pointer to driver SLI ring object. * * This function aborts all iocbs in FCP rings and frees all the iocb * objects in txq. This function issues an abort iocb for all the iocb commands @@ -3981,8 +4649,8 @@ lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba) /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].io_wq->pring; lpfc_sli_abort_iocb_ring(phba, pring); } } else { @@ -3992,43 +4660,17 @@ lpfc_sli_abort_fcp_rings(struct lpfc_hba *phba) } /** - * lpfc_sli_abort_nvme_rings - Abort all wqes in all NVME rings - * @phba: Pointer to HBA context object. - * - * This function aborts all wqes in NVME rings. This function issues an - * abort wqe for all the outstanding IO commands in txcmplq. The iocbs in - * the txcmplq is not guaranteed to complete before the return of this - * function. The caller is not required to hold any locks. - **/ -void -lpfc_sli_abort_nvme_rings(struct lpfc_hba *phba) -{ - struct lpfc_sli_ring *pring; - uint32_t i; - - if (phba->sli_rev < LPFC_SLI_REV4) - return; - - /* Abort all IO on each NVME ring. */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; - lpfc_sli_abort_wqe_ring(phba, pring); - } -} - - -/** - * lpfc_sli_flush_fcp_rings - flush all iocbs in the fcp ring + * lpfc_sli_flush_io_rings - flush all iocbs in the IO ring * @phba: Pointer to HBA context object. * - * This function flushes all iocbs in the fcp ring and frees all the iocb + * This function flushes all iocbs in the IO ring and frees all the iocb * objects in txq and txcmplq. This function will not issue abort iocbs * for all the iocb commands in txcmplq, they will just be returned with * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI * slot has been permanently disabled. **/ void -lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) +lpfc_sli_flush_io_rings(struct lpfc_hba *phba) { LIST_HEAD(txq); LIST_HEAD(txcmplq); @@ -4037,22 +4679,31 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) uint32_t i; struct lpfc_iocbq *piocb, *next_iocb; - spin_lock_irq(&phba->hbalock); /* Indicate the I/O queues are flushed */ - phba->hba_flag |= HBA_FCP_IOQ_FLUSH; - spin_unlock_irq(&phba->hbalock); + set_bit(HBA_IOQ_FLUSH, &phba->hba_flag); /* Look on all the FCP Rings for the iotag */ if (phba->sli_rev >= LPFC_SLI_REV4) { - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + if (!phba->sli4_hba.hdwq || + !phba->sli4_hba.hdwq[i].io_wq) { + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "7777 hdwq's deleted %lx " + "%lx %x %x\n", + phba->pport->load_flag, + phba->hba_flag, + phba->link_state, + phba->sli.sli_flag); + return; + } + pring = phba->sli4_hba.hdwq[i].io_wq->pring; spin_lock_irq(&pring->ring_lock); /* Retrieve everything on txq */ list_splice_init(&pring->txq, &txq); list_for_each_entry_safe(piocb, next_iocb, &pring->txcmplq, list) - piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + piocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ; /* Retrieve everything on the txcmplq */ list_splice_init(&pring->txcmplq, &txcmplq); pring->txq_cnt = 0; @@ -4063,10 +4714,12 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) lpfc_sli_cancel_iocbs(phba, &txq, IOSTAT_LOCAL_REJECT, IOERR_SLI_DOWN); - /* Flush the txcmpq */ + /* Flush the txcmplq */ lpfc_sli_cancel_iocbs(phba, &txcmplq, IOSTAT_LOCAL_REJECT, IOERR_SLI_DOWN); + if (unlikely(pci_channel_offline(phba->pcidev))) + lpfc_sli4_io_xri_aborted(phba, NULL, 0); } } else { pring = &psli->sli3_ring[LPFC_FCP_RING]; @@ -4076,7 +4729,7 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) list_splice_init(&pring->txq, &txq); list_for_each_entry_safe(piocb, next_iocb, &pring->txcmplq, list) - piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; + piocb->cmd_flag &= ~LPFC_IO_ON_TXCMPLQ; /* Retrieve everything on the txcmplq */ list_splice_init(&pring->txcmplq, &txcmplq); pring->txq_cnt = 0; @@ -4093,55 +4746,6 @@ lpfc_sli_flush_fcp_rings(struct lpfc_hba *phba) } /** - * lpfc_sli_flush_nvme_rings - flush all wqes in the nvme rings - * @phba: Pointer to HBA context object. - * - * This function flushes all wqes in the nvme rings and frees all resources - * in the txcmplq. This function does not issue abort wqes for the IO - * commands in txcmplq, they will just be returned with - * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI - * slot has been permanently disabled. - **/ -void -lpfc_sli_flush_nvme_rings(struct lpfc_hba *phba) -{ - LIST_HEAD(txcmplq); - struct lpfc_sli_ring *pring; - uint32_t i; - struct lpfc_iocbq *piocb, *next_iocb; - - if (phba->sli_rev < LPFC_SLI_REV4) - return; - - /* Hint to other driver operations that a flush is in progress. */ - spin_lock_irq(&phba->hbalock); - phba->hba_flag |= HBA_NVME_IOQ_FLUSH; - spin_unlock_irq(&phba->hbalock); - - /* Cycle through all NVME rings and complete each IO with - * a local driver reason code. This is a flush so no - * abort exchange to FW. - */ - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; - - spin_lock_irq(&pring->ring_lock); - list_for_each_entry_safe(piocb, next_iocb, - &pring->txcmplq, list) - piocb->iocb_flag &= ~LPFC_IO_ON_TXCMPLQ; - /* Retrieve everything on the txcmplq */ - list_splice_init(&pring->txcmplq, &txcmplq); - pring->txcmplq_cnt = 0; - spin_unlock_irq(&pring->ring_lock); - - /* Flush the txcmpq &&&PAE */ - lpfc_sli_cancel_iocbs(phba, &txcmplq, - IOSTAT_LOCAL_REJECT, - IOERR_SLI_DOWN); - } -} - -/** * lpfc_sli_brdready_s3 - Check for sli3 host ready status * @phba: Pointer to HBA context object. * @mask: Bit mask to be checked. @@ -4165,6 +4769,8 @@ lpfc_sli_brdready_s3(struct lpfc_hba *phba, uint32_t mask) if (lpfc_readl(phba->HSregaddr, &status)) return 1; + set_bit(HBA_NEEDS_CFG_PORT, &phba->hba_flag); + /* * Check status register every 100ms for 5 retries, then every * 500ms for 5, then every 2.5 sec for 5, then reset board and @@ -4196,7 +4802,7 @@ lpfc_sli_brdready_s3(struct lpfc_hba *phba, uint32_t mask) /* Check to see if any errors occurred during init */ if ((status & HS_FFERM) || (i >= 20)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2751 Adapter failed to restart, " "status reg x%x, FW Data: A8 x%x AC x%x\n", status, @@ -4242,6 +4848,7 @@ lpfc_sli_brdready_s4(struct lpfc_hba *phba, uint32_t mask) } else phba->sli4_hba.intr_enable = 0; + clear_bit(HBA_SETUP, &phba->hba_flag); return retval; } @@ -4272,7 +4879,7 @@ void lpfc_reset_barrier(struct lpfc_hba *phba) { uint32_t __iomem *resp_buf; uint32_t __iomem *mbox_buf; - volatile uint32_t mbox; + volatile struct MAILBOX_word0 mbox; uint32_t hc_copy, ha_copy, resp_data; int i; uint8_t hdrtype; @@ -4280,7 +4887,7 @@ void lpfc_reset_barrier(struct lpfc_hba *phba) lockdep_assert_held(&phba->hbalock); pci_read_config_byte(phba->pcidev, PCI_HEADER_TYPE, &hdrtype); - if (hdrtype != 0x80 || + if (hdrtype != PCI_HEADER_TYPE_MFD || (FC_JEDEC_ID(phba->vpd.rev.biuRev) != HELIOS_JEDEC_ID && FC_JEDEC_ID(phba->vpd.rev.biuRev) != THOR_JEDEC_ID)) return; @@ -4306,13 +4913,13 @@ void lpfc_reset_barrier(struct lpfc_hba *phba) phba->pport->stopped = 1; } - mbox = 0; - ((MAILBOX_t *)&mbox)->mbxCommand = MBX_KILL_BOARD; - ((MAILBOX_t *)&mbox)->mbxOwner = OWN_CHIP; + mbox.word0 = 0; + mbox.mbxCommand = MBX_KILL_BOARD; + mbox.mbxOwner = OWN_CHIP; writel(BARRIER_TEST_PATTERN, (resp_buf + 1)); mbox_buf = phba->MBslimaddr; - writel(mbox, mbox_buf); + writel(mbox.word0, mbox_buf); for (i = 0; i < 50; i++) { if (lpfc_readl((resp_buf + 1), &resp_data)) @@ -4333,12 +4940,12 @@ void lpfc_reset_barrier(struct lpfc_hba *phba) goto clear_errat; } - ((MAILBOX_t *)&mbox)->mbxOwner = OWN_HOST; + mbox.mbxOwner = OWN_HOST; resp_data = 0; for (i = 0; i < 500; i++) { if (lpfc_readl(resp_buf, &resp_data)) return; - if (resp_data != mbox) + if (resp_data != mbox.word0) mdelay(1); else break; @@ -4418,7 +5025,7 @@ lpfc_sli_brdkill(struct lpfc_hba *phba) if (retval != MBX_SUCCESS) { if (retval != MBX_BUSY) mempool_free(pmb, phba->mbox_mem_pool); - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2752 KILL_BOARD command failed retval %d\n", retval); spin_lock_irq(&phba->hbalock); @@ -4446,7 +5053,7 @@ lpfc_sli_brdkill(struct lpfc_hba *phba) return 1; } - del_timer_sync(&psli->mbox_tmo); + timer_delete_sync(&psli->mbox_tmo); if (ha_copy & HA_ERATT) { writel(HA_ERATT, phba->HAregaddr); phba->pport->stopped = 1; @@ -4493,13 +5100,16 @@ lpfc_sli_brdreset(struct lpfc_hba *phba) /* perform board reset */ phba->fc_eventTag = 0; phba->link_events = 0; + set_bit(HBA_NEEDS_CFG_PORT, &phba->hba_flag); if (phba->pport) { phba->pport->fc_myDID = 0; phba->pport->fc_prevDID = 0; } /* Turn off parity checking and serr during the physical reset */ - pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value); + if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value)) + return -EIO; + pci_write_config_word(phba->pcidev, PCI_COMMAND, (cfg_value & ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR))); @@ -4539,7 +5149,7 @@ lpfc_sli_brdreset(struct lpfc_hba *phba) * checking during resets the device. The caller is not required to hold * any locks. * - * This function returns 0 always. + * This function returns 0 on success else returns negative error code. **/ int lpfc_sli4_brdreset(struct lpfc_hba *phba) @@ -4550,7 +5160,7 @@ lpfc_sli4_brdreset(struct lpfc_hba *phba) /* Reset HBA */ lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "0295 Reset HBA Data: x%x x%x x%x\n", + "0295 Reset HBA Data: x%x x%x x%lx\n", phba->pport->port_state, psli->sli_flag, phba->hba_flag); @@ -4565,18 +5175,17 @@ lpfc_sli4_brdreset(struct lpfc_hba *phba) phba->fcf.fcf_flag = 0; spin_unlock_irq(&phba->hbalock); - /* SLI4 INTF 2: if FW dump is being taken skip INIT_PORT */ - if (phba->hba_flag & HBA_FW_DUMP_OP) { - phba->hba_flag &= ~HBA_FW_DUMP_OP; - return rc; - } - /* Now physically reset the device */ lpfc_printf_log(phba, KERN_INFO, LOG_INIT, "0389 Performing PCI function reset!\n"); /* Turn off parity checking and serr during the physical reset */ - pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value); + if (pci_read_config_word(phba->pcidev, PCI_COMMAND, &cfg_value)) { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "3205 PCI read Config failed\n"); + return -EIO; + } + pci_write_config_word(phba->pcidev, PCI_COMMAND, (cfg_value & ~(PCI_COMMAND_PARITY | PCI_COMMAND_SERR))); @@ -4605,17 +5214,12 @@ lpfc_sli4_brdreset(struct lpfc_hba *phba) static int lpfc_sli_brdrestart_s3(struct lpfc_hba *phba) { - MAILBOX_t *mb; + volatile struct MAILBOX_word0 mb; struct lpfc_sli *psli; - volatile uint32_t word0; void __iomem *to_slim; - uint32_t hba_aer_enabled; spin_lock_irq(&phba->hbalock); - /* Take PCIe device Advanced Error Reporting (AER) state */ - hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED; - psli = &phba->sli; /* Restart HBA */ @@ -4624,24 +5228,23 @@ lpfc_sli_brdrestart_s3(struct lpfc_hba *phba) (phba->pport) ? phba->pport->port_state : 0, psli->sli_flag); - word0 = 0; - mb = (MAILBOX_t *) &word0; - mb->mbxCommand = MBX_RESTART; - mb->mbxHc = 1; + mb.word0 = 0; + mb.mbxCommand = MBX_RESTART; + mb.mbxHc = 1; lpfc_reset_barrier(phba); to_slim = phba->MBslimaddr; - writel(*(uint32_t *) mb, to_slim); + writel(mb.word0, to_slim); readl(to_slim); /* flush */ /* Only skip post after fc_ffinit is completed */ if (phba->pport && phba->pport->port_state) - word0 = 1; /* This is really setting up word1 */ + mb.word0 = 1; /* This is really setting up word1 */ else - word0 = 0; /* This is really setting up word1 */ + mb.word0 = 0; /* This is really setting up word1 */ to_slim = phba->MBslimaddr + sizeof (uint32_t); - writel(*(uint32_t *) mb, to_slim); + writel(mb.word0, to_slim); readl(to_slim); /* flush */ lpfc_sli_brdreset(phba); @@ -4657,10 +5260,6 @@ lpfc_sli_brdrestart_s3(struct lpfc_hba *phba) /* Give the INITFF and Post time to settle. */ mdelay(100); - /* Reset HBA AER if it was enabled, note hba_flag was reset above */ - if (hba_aer_enabled) - pci_disable_pcie_error_reporting(phba->pcidev); - lpfc_hba_down_post(phba); return 0; @@ -4679,7 +5278,6 @@ static int lpfc_sli_brdrestart_s4(struct lpfc_hba *phba) { struct lpfc_sli *psli = &phba->sli; - uint32_t hba_aer_enabled; int rc; /* Restart HBA */ @@ -4687,26 +5285,27 @@ lpfc_sli_brdrestart_s4(struct lpfc_hba *phba) "0296 Restart HBA Data: x%x x%x\n", phba->pport->port_state, psli->sli_flag); - /* Take PCIe device Advanced Error Reporting (AER) state */ - hba_aer_enabled = phba->hba_flag & HBA_AER_ENABLED; + clear_bit(HBA_SETUP, &phba->hba_flag); + lpfc_sli4_queue_unset(phba); rc = lpfc_sli4_brdreset(phba); - if (rc) - return rc; + if (rc) { + phba->link_state = LPFC_HBA_ERROR; + goto hba_down_queue; + } spin_lock_irq(&phba->hbalock); phba->pport->stopped = 0; phba->link_state = LPFC_INIT_START; phba->hba_flag = 0; + /* Preserve FA-PWWN expectation */ + phba->sli4_hba.fawwpn_flag &= LPFC_FAWWPN_FABRIC; spin_unlock_irq(&phba->hbalock); memset(&psli->lnk_stat_offsets, 0, sizeof(psli->lnk_stat_offsets)); psli->stats_start = ktime_get_seconds(); - /* Reset HBA AER if it was enabled, note hba_flag was reset above */ - if (hba_aer_enabled) - pci_disable_pcie_error_reporting(phba->pcidev); - +hba_down_queue: lpfc_hba_down_post(phba); lpfc_sli4_queue_destroy(phba); @@ -4760,7 +5359,7 @@ lpfc_sli_chipset_init(struct lpfc_hba *phba) if (i++ >= 200) { /* Adapter failed to init, timeout, status reg <status> */ - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0436 Adapter failed to init, " "timeout, status reg x%x, " "FW Data: A8 x%x AC x%x\n", status, @@ -4775,7 +5374,7 @@ lpfc_sli_chipset_init(struct lpfc_hba *phba) /* ERROR: During chipset initialization */ /* Adapter failed to init, chipset, status reg <status> */ - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0437 Adapter failed to init, " "chipset, status reg x%x, " "FW Data: A8 x%x AC x%x\n", status, @@ -4806,7 +5405,7 @@ lpfc_sli_chipset_init(struct lpfc_hba *phba) if (status & HS_FFERM) { /* ERROR: During chipset initialization */ /* Adapter failed to init, chipset, status reg <status> */ - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0438 Adapter failed to init, chipset, " "status reg x%x, " "FW Data: A8 x%x AC x%x\n", status, @@ -4816,6 +5415,8 @@ lpfc_sli_chipset_init(struct lpfc_hba *phba) return -EIO; } + set_bit(HBA_NEEDS_CFG_PORT, &phba->hba_flag); + /* Clear all interrupt enable conditions */ writel(0, phba->HCregaddr); readl(phba->HCregaddr); /* flush */ @@ -4951,8 +5552,17 @@ static int lpfc_sli4_rb_setup(struct lpfc_hba *phba) { phba->hbq_in_use = 1; - phba->hbqs[LPFC_ELS_HBQ].entry_count = - lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count; + /** + * Specific case when the MDS diagnostics is enabled and supported. + * The receive buffer count is truncated to manage the incoming + * traffic. + **/ + if (phba->cfg_enable_mds_diags && phba->mds_diags_support) + phba->hbqs[LPFC_ELS_HBQ].entry_count = + lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count >> 1; + else + phba->hbqs[LPFC_ELS_HBQ].entry_count = + lpfc_hbq_defs[LPFC_ELS_HBQ]->entry_count; phba->hbq_count = 1; lpfc_sli_hbqbuf_init_hbqs(phba, LPFC_ELS_HBQ); /* Initially populate or replenish the HBQs */ @@ -5020,7 +5630,7 @@ lpfc_sli_config_port(struct lpfc_hba *phba, int sli_mode) LPFC_SLI3_CRP_ENABLED | LPFC_SLI3_DSS_ENABLED); if (rc != MBX_SUCCESS) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0442 Adapter failed to init, mbxCmd x%x " "CONFIG_PORT, mbxStatus x%x Data: x%x\n", pmb->u.mb.mbxCommand, pmb->u.mb.mbxStatus, 0); @@ -5058,23 +5668,6 @@ lpfc_sli_config_port(struct lpfc_hba *phba, int sli_mode) } else phba->max_vpi = 0; - phba->fips_level = 0; - phba->fips_spec_rev = 0; - if (pmb->u.mb.un.varCfgPort.gdss) { - phba->sli3_options |= LPFC_SLI3_DSS_ENABLED; - phba->fips_level = pmb->u.mb.un.varCfgPort.fips_level; - phba->fips_spec_rev = pmb->u.mb.un.varCfgPort.fips_rev; - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2850 Security Crypto Active. FIPS x%d " - "(Spec Rev: x%d)", - phba->fips_level, phba->fips_spec_rev); - } - if (pmb->u.mb.un.varCfgPort.sec_err) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2856 Config Port Security Crypto " - "Error: x%x ", - pmb->u.mb.un.varCfgPort.sec_err); - } if (pmb->u.mb.un.varCfgPort.gerbm) phba->sli3_options |= LPFC_SLI3_HBQ_ENABLED; if (pmb->u.mb.un.varCfgPort.gcrp) @@ -5087,7 +5680,7 @@ lpfc_sli_config_port(struct lpfc_hba *phba, int sli_mode) if (pmb->u.mb.un.varCfgPort.gbg == 0) { phba->cfg_enable_bg = 0; phba->sli3_options &= ~LPFC_SLI3_BG_ENABLED; - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0443 Adapter did not grant " "BlockGuard\n"); } @@ -5120,64 +5713,18 @@ int lpfc_sli_hba_setup(struct lpfc_hba *phba) { uint32_t rc; - int mode = 3, i; + int i; int longs; - switch (phba->cfg_sli_mode) { - case 2: - if (phba->cfg_enable_npiv) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT, - "1824 NPIV enabled: Override sli_mode " - "parameter (%d) to auto (0).\n", - phba->cfg_sli_mode); - break; - } - mode = 2; - break; - case 0: - case 3: - break; - default: - lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT, - "1819 Unrecognized sli_mode parameter: %d.\n", - phba->cfg_sli_mode); - - break; + /* Enable ISR already does config_port because of config_msi mbx */ + if (test_bit(HBA_NEEDS_CFG_PORT, &phba->hba_flag)) { + rc = lpfc_sli_config_port(phba, LPFC_SLI_REV3); + if (rc) + return -EIO; + clear_bit(HBA_NEEDS_CFG_PORT, &phba->hba_flag); } phba->fcp_embed_io = 0; /* SLI4 FC support only */ - rc = lpfc_sli_config_port(phba, mode); - - if (rc && phba->cfg_sli_mode == 3) - lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_VPORT, - "1820 Unable to select SLI-3. " - "Not supported by adapter.\n"); - if (rc && mode != 2) - rc = lpfc_sli_config_port(phba, 2); - else if (rc && mode == 2) - rc = lpfc_sli_config_port(phba, 3); - if (rc) - goto lpfc_sli_hba_setup_error; - - /* Enable PCIe device Advanced Error Reporting (AER) if configured */ - if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) { - rc = pci_enable_pcie_error_reporting(phba->pcidev); - if (!rc) { - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2709 This device supports " - "Advanced Error Reporting (AER)\n"); - spin_lock_irq(&phba->hbalock); - phba->hba_flag |= HBA_AER_ENABLED; - spin_unlock_irq(&phba->hbalock); - } else { - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2708 This device does not support " - "Advanced Error Reporting (AER): %d\n", - rc); - phba->cfg_aer_support = 0; - } - } - if (phba->sli_rev == 3) { phba->iocb_cmd_size = SLI3_IOCB_CMD_SIZE; phba->iocb_rsp_size = SLI3_IOCB_RSP_SIZE; @@ -5243,7 +5790,7 @@ lpfc_sli_hba_setup(struct lpfc_hba *phba) lpfc_sli_hba_setup_error: phba->link_state = LPFC_HBA_ERROR; - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0445 Firmware initialization failed\n"); return rc; } @@ -5251,7 +5798,7 @@ lpfc_sli_hba_setup_error: /** * lpfc_sli4_read_fcoe_params - Read fcoe params from conf region * @phba: Pointer to HBA context object. - * @mboxq: mailbox pointer. + * * This function issue a dump mailbox command to read config region * 23 and parse the records in the region and populate driver * data structure. @@ -5281,7 +5828,7 @@ lpfc_sli4_read_fcoe_params(struct lpfc_hba *phba) goto out_free_mboxq; } - mp = (struct lpfc_dmabuf *)mboxq->ctx_buf; + mp = mboxq->ctx_buf; rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI, @@ -5306,26 +5853,20 @@ lpfc_sli4_read_fcoe_params(struct lpfc_hba *phba) mboxq->mcqe.trailer); if (rc) { - lpfc_mbuf_free(phba, mp->virt, mp->phys); - kfree(mp); rc = -EIO; goto out_free_mboxq; } data_length = mqe->un.mb_words[5]; if (data_length > DMP_RGN23_SIZE) { - lpfc_mbuf_free(phba, mp->virt, mp->phys); - kfree(mp); rc = -EIO; goto out_free_mboxq; } lpfc_parse_fcoe_conf(phba, mp->virt, data_length); - lpfc_mbuf_free(phba, mp->virt, mp->phys); - kfree(mp); rc = 0; out_free_mboxq: - mempool_free(mboxq, phba->mbox_mem_pool); + lpfc_mbox_rsrc_cleanup(phba, mboxq, MBOX_THD_UNLOCKED); return rc; } @@ -5407,7 +5948,7 @@ lpfc_sli4_read_rev(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, } /** - * lpfc_sli4_retrieve_pport_name - Retrieve SLI4 device physical port name + * lpfc_sli4_get_ctl_attr - Retrieve SLI4 device controller attributes * @phba: pointer to lpfc hba data structure. * * This routine retrieves SLI4 device physical port name this PCI function @@ -5415,42 +5956,32 @@ lpfc_sli4_read_rev(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, * * Return codes * 0 - successful - * otherwise - failed to retrieve physical port name + * otherwise - failed to retrieve controller attributes **/ static int -lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba) +lpfc_sli4_get_ctl_attr(struct lpfc_hba *phba) { LPFC_MBOXQ_t *mboxq; struct lpfc_mbx_get_cntl_attributes *mbx_cntl_attr; struct lpfc_controller_attribute *cntl_attr; - struct lpfc_mbx_get_port_name *get_port_name; void *virtaddr = NULL; uint32_t alloclen, reqlen; uint32_t shdr_status, shdr_add_status; union lpfc_sli4_cfg_shdr *shdr; - char cport_name = 0; int rc; - /* We assume nothing at this point */ - phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL; - phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_NON; - mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mboxq) return -ENOMEM; - /* obtain link type and link number via READ_CONFIG */ - phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL; - lpfc_sli4_read_config(phba); - if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL) - goto retrieve_ppname; - /* obtain link type and link number via COMMON_GET_CNTL_ATTRIBUTES */ + /* Send COMMON_GET_CNTL_ATTRIBUTES mbox cmd */ reqlen = sizeof(struct lpfc_mbx_get_cntl_attributes); alloclen = lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON, LPFC_MBOX_OPCODE_GET_CNTL_ATTRIBUTES, reqlen, LPFC_SLI4_MBX_NEMBED); + if (alloclen < reqlen) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3084 Allocated DMA memory size (%d) is " "less than the requested DMA memory size " "(%d)\n", alloclen, reqlen); @@ -5474,16 +6005,77 @@ lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba) rc = -ENXIO; goto out_free_mboxq; } + cntl_attr = &mbx_cntl_attr->cntl_attr; phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_VAL; phba->sli4_hba.lnk_info.lnk_tp = bf_get(lpfc_cntl_attr_lnk_type, cntl_attr); phba->sli4_hba.lnk_info.lnk_no = bf_get(lpfc_cntl_attr_lnk_numb, cntl_attr); + phba->sli4_hba.flash_id = bf_get(lpfc_cntl_attr_flash_id, cntl_attr); + phba->sli4_hba.asic_rev = bf_get(lpfc_cntl_attr_asic_rev, cntl_attr); + + memcpy(phba->BIOSVersion, cntl_attr->bios_ver_str, + sizeof(phba->BIOSVersion)); + phba->BIOSVersion[sizeof(phba->BIOSVersion) - 1] = '\0'; + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "3086 lnk_type:%d, lnk_numb:%d\n", + "3086 lnk_type:%d, lnk_numb:%d, bios_ver:%s, " + "flash_id: x%02x, asic_rev: x%02x\n", phba->sli4_hba.lnk_info.lnk_tp, - phba->sli4_hba.lnk_info.lnk_no); + phba->sli4_hba.lnk_info.lnk_no, + phba->BIOSVersion, phba->sli4_hba.flash_id, + phba->sli4_hba.asic_rev); +out_free_mboxq: + if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG) + lpfc_sli4_mbox_cmd_free(phba, mboxq); + else + mempool_free(mboxq, phba->mbox_mem_pool); + return rc; +} + +/** + * lpfc_sli4_retrieve_pport_name - Retrieve SLI4 device physical port name + * @phba: pointer to lpfc hba data structure. + * + * This routine retrieves SLI4 device physical port name this PCI function + * is attached to. + * + * Return codes + * 0 - successful + * otherwise - failed to retrieve physical port name + **/ +static int +lpfc_sli4_retrieve_pport_name(struct lpfc_hba *phba) +{ + LPFC_MBOXQ_t *mboxq; + struct lpfc_mbx_get_port_name *get_port_name; + uint32_t shdr_status, shdr_add_status; + union lpfc_sli4_cfg_shdr *shdr; + char cport_name = 0; + int rc; + + /* We assume nothing at this point */ + phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL; + phba->sli4_hba.pport_name_sta = LPFC_SLI4_PPNAME_NON; + + mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); + if (!mboxq) + return -ENOMEM; + /* obtain link type and link number via READ_CONFIG */ + phba->sli4_hba.lnk_info.lnk_dv = LPFC_LNK_DAT_INVAL; + lpfc_sli4_read_config(phba); + + if (phba->sli4_hba.fawwpn_flag & LPFC_FAWWPN_CONFIG) + phba->sli4_hba.fawwpn_flag |= LPFC_FAWWPN_FABRIC; + + if (phba->sli4_hba.lnk_info.lnk_dv == LPFC_LNK_DAT_VAL) + goto retrieve_ppname; + + /* obtain link type and link number via COMMON_GET_CNTL_ATTRIBUTES */ + rc = lpfc_sli4_get_ctl_attr(phba); + if (rc) + goto out_free_mboxq; retrieve_ppname: lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON, @@ -5543,12 +6135,10 @@ retrieve_ppname: } out_free_mboxq: - if (rc != MBX_TIMEOUT) { - if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG) - lpfc_sli4_mbox_cmd_free(phba, mboxq); - else - mempool_free(mboxq, phba->mbox_mem_pool); - } + if (bf_get(lpfc_mqe_command, &mboxq->u.mqe) == MBX_SLI4_CONFIG) + lpfc_sli4_mbox_cmd_free(phba, mboxq); + else + mempool_free(mboxq, phba->mbox_mem_pool); return rc; } @@ -5564,41 +6154,40 @@ lpfc_sli4_arm_cqeq_intr(struct lpfc_hba *phba) { int qidx; struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_queue *eq; - sli4_hba->sli4_cq_release(sli4_hba->mbx_cq, LPFC_QUEUE_REARM); - sli4_hba->sli4_cq_release(sli4_hba->els_cq, LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->mbx_cq, 0, LPFC_QUEUE_REARM); + sli4_hba->sli4_write_cq_db(phba, sli4_hba->els_cq, 0, LPFC_QUEUE_REARM); if (sli4_hba->nvmels_cq) - sli4_hba->sli4_cq_release(sli4_hba->nvmels_cq, - LPFC_QUEUE_REARM); - - if (sli4_hba->fcp_cq) - for (qidx = 0; qidx < phba->cfg_fcp_io_channel; qidx++) - sli4_hba->sli4_cq_release(sli4_hba->fcp_cq[qidx], + sli4_hba->sli4_write_cq_db(phba, sli4_hba->nvmels_cq, 0, + LPFC_QUEUE_REARM); + + if (sli4_hba->hdwq) { + /* Loop thru all Hardware Queues */ + for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) { + qp = &sli4_hba->hdwq[qidx]; + /* ARM the corresponding CQ */ + sli4_hba->sli4_write_cq_db(phba, qp->io_cq, 0, LPFC_QUEUE_REARM); + } - if (sli4_hba->nvme_cq) - for (qidx = 0; qidx < phba->cfg_nvme_io_channel; qidx++) - sli4_hba->sli4_cq_release(sli4_hba->nvme_cq[qidx], - LPFC_QUEUE_REARM); - - if (phba->cfg_fof) - sli4_hba->sli4_cq_release(sli4_hba->oas_cq, LPFC_QUEUE_REARM); - - if (sli4_hba->hba_eq) - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) - sli4_hba->sli4_eq_release(sli4_hba->hba_eq[qidx], - LPFC_QUEUE_REARM); + /* Loop thru all IRQ vectors */ + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { + eq = sli4_hba->hba_eq_hdl[qidx].eq; + /* ARM the corresponding EQ */ + sli4_hba->sli4_write_eq_db(phba, eq, + 0, LPFC_QUEUE_REARM); + } + } if (phba->nvmet_support) { for (qidx = 0; qidx < phba->cfg_nvmet_mrq; qidx++) { - sli4_hba->sli4_cq_release( - sli4_hba->nvmet_cqset[qidx], + sli4_hba->sli4_write_cq_db(phba, + sli4_hba->nvmet_cqset[qidx], 0, LPFC_QUEUE_REARM); } } - - if (phba->cfg_fof) - sli4_hba->sli4_eq_release(sli4_hba->fof_eq, LPFC_QUEUE_REARM); } /** @@ -5623,6 +6212,9 @@ lpfc_sli4_get_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type, struct lpfc_mbx_get_rsrc_extent_info *rsrc_info; LPFC_MBOXQ_t *mbox; + *extnt_count = 0; + *extnt_size = 0; + mbox = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -5656,7 +6248,7 @@ lpfc_sli4_get_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type, rsrc_info = &mbox->u.mqe.un.rsrc_extent_info; if (bf_get(lpfc_mbox_hdr_status, &rsrc_info->header.cfg_shdr.response)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2930 Failed to get resource extents " "Status 0x%x Add'l Status 0x%x\n", bf_get(lpfc_mbox_hdr_status, @@ -5745,10 +6337,10 @@ lpfc_sli4_chk_avail_extnt_rsrc(struct lpfc_hba *phba, uint16_t type) /** * lpfc_sli4_cfg_post_extnts - * @phba: Pointer to HBA context object. - * @extnt_cnt - number of available extents. - * @type - the extent type (rpi, xri, vfi, vpi). - * @emb - buffer to hold either MBX_EMBED or MBX_NEMBED operation. - * @mbox - pointer to the caller's allocated mailbox structure. + * @extnt_cnt: number of available extents. + * @type: the extent type (rpi, xri, vfi, vpi). + * @emb: buffer to hold either MBX_EMBED or MBX_NEMBED operation. + * @mbox: pointer to the caller's allocated mailbox structure. * * This function executes the extents allocation request. It also * takes care of the amount of memory needed to allocate or get the @@ -5794,7 +6386,7 @@ lpfc_sli4_cfg_post_extnts(struct lpfc_hba *phba, uint16_t extnt_cnt, LPFC_MBOX_OPCODE_ALLOC_RSRC_EXTENT, req_len, *emb); if (alloc_len < req_len) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2982 Allocated DMA memory size (x%x) is " "less than the requested DMA memory " "size (x%x)\n", alloc_len, req_len); @@ -5850,7 +6442,7 @@ lpfc_sli4_alloc_extent(struct lpfc_hba *phba, uint16_t type) return -EIO; if ((rsrc_cnt == 0) || (rsrc_size == 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3009 No available Resource Extents " "for resource type 0x%x: Count: 0x%x, " "Size 0x%x\n", type, rsrc_cnt, @@ -6027,11 +6619,8 @@ lpfc_sli4_alloc_extent(struct lpfc_hba *phba, uint16_t type) list_add_tail(&rsrc_blks->list, ext_blk_list); rsrc_start = rsrc_id; if ((type == LPFC_RSC_TYPE_FCOE_XRI) && (j == 0)) { - phba->sli4_hba.scsi_xri_start = rsrc_start + + phba->sli4_hba.io_xri_start = rsrc_start + lpfc_sli4_get_iocb_cnt(phba); - phba->sli4_hba.nvme_xri_start = - phba->sli4_hba.scsi_xri_start + - phba->sli4_hba.scsi_xri_max; } while (rsrc_id < (rsrc_start + rsrc_size)) { @@ -6104,7 +6693,7 @@ lpfc_sli4_dealloc_extent(struct lpfc_hba *phba, uint16_t type) dealloc_rsrc = &mbox->u.mqe.un.dealloc_rsrc_extents; if (bf_get(lpfc_mbox_hdr_status, &dealloc_rsrc->header.cfg_shdr.response)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2919 Failed to release resource extents " "for type %d - Status 0x%x Add'l Status 0x%x. " "Resource memory not released.\n", @@ -6173,6 +6762,7 @@ lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox, uint32_t feature) { uint32_t len; + u32 sig_freq = 0; len = sizeof(struct lpfc_mbx_set_feature) - sizeof(struct lpfc_sli4_cfg_mhdr); @@ -6195,8 +6785,64 @@ lpfc_set_features(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox, mbox->u.mqe.un.set_feature.feature = LPFC_SET_MDS_DIAGS; mbox->u.mqe.un.set_feature.param_len = 8; break; - } + case LPFC_SET_CGN_SIGNAL: + if (phba->cmf_active_mode == LPFC_CFG_OFF) + sig_freq = 0; + else + sig_freq = phba->cgn_sig_freq; + + if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ALARM) { + bf_set(lpfc_mbx_set_feature_CGN_alarm_freq, + &mbox->u.mqe.un.set_feature, sig_freq); + bf_set(lpfc_mbx_set_feature_CGN_warn_freq, + &mbox->u.mqe.un.set_feature, sig_freq); + } + + if (phba->cgn_reg_signal == EDC_CG_SIG_WARN_ONLY) + bf_set(lpfc_mbx_set_feature_CGN_warn_freq, + &mbox->u.mqe.un.set_feature, sig_freq); + if (phba->cmf_active_mode == LPFC_CFG_OFF || + phba->cgn_reg_signal == EDC_CG_SIG_NOTSUPPORTED) + sig_freq = 0; + else + sig_freq = lpfc_acqe_cgn_frequency; + + bf_set(lpfc_mbx_set_feature_CGN_acqe_freq, + &mbox->u.mqe.un.set_feature, sig_freq); + + mbox->u.mqe.un.set_feature.feature = LPFC_SET_CGN_SIGNAL; + mbox->u.mqe.un.set_feature.param_len = 12; + break; + case LPFC_SET_DUAL_DUMP: + bf_set(lpfc_mbx_set_feature_dd, + &mbox->u.mqe.un.set_feature, LPFC_ENABLE_DUAL_DUMP); + bf_set(lpfc_mbx_set_feature_ddquery, + &mbox->u.mqe.un.set_feature, 0); + mbox->u.mqe.un.set_feature.feature = LPFC_SET_DUAL_DUMP; + mbox->u.mqe.un.set_feature.param_len = 4; + break; + case LPFC_SET_ENABLE_MI: + mbox->u.mqe.un.set_feature.feature = LPFC_SET_ENABLE_MI; + mbox->u.mqe.un.set_feature.param_len = 4; + bf_set(lpfc_mbx_set_feature_milunq, &mbox->u.mqe.un.set_feature, + phba->pport->cfg_lun_queue_depth); + bf_set(lpfc_mbx_set_feature_mi, &mbox->u.mqe.un.set_feature, + phba->sli4_hba.pc_sli4_params.mi_ver); + break; + case LPFC_SET_LD_SIGNAL: + mbox->u.mqe.un.set_feature.feature = LPFC_SET_LD_SIGNAL; + mbox->u.mqe.un.set_feature.param_len = 16; + bf_set(lpfc_mbx_set_feature_lds_qry, + &mbox->u.mqe.un.set_feature, LPFC_QUERY_LDS_OP); + break; + case LPFC_SET_ENABLE_CMF: + mbox->u.mqe.un.set_feature.feature = LPFC_SET_ENABLE_CMF; + mbox->u.mqe.un.set_feature.param_len = 4; + bf_set(lpfc_mbx_set_feature_cmf, + &mbox->u.mqe.un.set_feature, 1); + break; + } return; } @@ -6212,11 +6858,16 @@ lpfc_ras_stop_fwlog(struct lpfc_hba *phba) { struct lpfc_ras_fwlog *ras_fwlog = &phba->ras_fwlog; - ras_fwlog->ras_active = false; + spin_lock_irq(&phba->ras_fwlog_lock); + ras_fwlog->state = INACTIVE; + spin_unlock_irq(&phba->ras_fwlog_lock); /* Disable FW logging to host memory */ writel(LPFC_CTL_PDEV_CTL_DDL_RAS, phba->sli4_hba.conf_regs_memmap_p + LPFC_CTL_PDEV_CTL_OFFSET); + + /* Wait 10ms for firmware to stop using DMA buffer */ + usleep_range(10 * 1000, 20 * 1000); } /** @@ -6252,7 +6903,9 @@ lpfc_sli4_ras_dma_free(struct lpfc_hba *phba) ras_fwlog->lwpd.virt = NULL; } - ras_fwlog->ras_active = false; + spin_lock_irq(&phba->ras_fwlog_lock); + ras_fwlog->state = INACTIVE; + spin_unlock_irq(&phba->ras_fwlog_lock); } /** @@ -6283,7 +6936,7 @@ lpfc_sli4_ras_dma_alloc(struct lpfc_hba *phba, &ras_fwlog->lwpd.phys, GFP_KERNEL); if (!ras_fwlog->lwpd.virt) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6185 LWPD Memory Alloc Failed\n"); return -ENOMEM; @@ -6324,7 +6977,7 @@ free_mem: /** * lpfc_sli4_ras_mbox_cmpl: Completion handler for RAS MBX command * @phba: pointer to lpfc hba data structure. - * @pmboxq: pointer to the driver internal queue element for mailbox command. + * @pmb: pointer to the driver internal queue element for mailbox command. * * Completion handler for driver's RAS MBX command to the device. **/ @@ -6344,7 +6997,7 @@ lpfc_sli4_ras_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (mb->mbxStatus != MBX_SUCCESS || shdr_status) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6188 FW LOG mailbox " "completed with status x%x add_status x%x," " mbx status x%x\n", @@ -6354,7 +7007,9 @@ lpfc_sli4_ras_mbox_cmpl(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) goto disable_ras; } - ras_fwlog->ras_active = true; + spin_lock_irq(&phba->ras_fwlog_lock); + ras_fwlog->state = ACTIVE; + spin_unlock_irq(&phba->ras_fwlog_lock); mempool_free(pmb, phba->mbox_mem_pool); return; @@ -6386,6 +7041,10 @@ lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba, uint32_t len = 0, fwlog_buffsize, fwlog_entry_count; int rc = 0; + spin_lock_irq(&phba->ras_fwlog_lock); + ras_fwlog->state = INACTIVE; + spin_unlock_irq(&phba->ras_fwlog_lock); + fwlog_buffsize = (LPFC_RAS_MIN_BUFF_POST_SIZE * phba->cfg_ras_fwlog_buffsize); fwlog_entry_count = (fwlog_buffsize/LPFC_RAS_MAX_ENTRY_SIZE); @@ -6406,7 +7065,7 @@ lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba, /* Setup Mailbox command */ mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6190 RAS MBX Alloc Failed"); rc = -ENOMEM; goto mem_free; @@ -6445,13 +7104,16 @@ lpfc_sli4_ras_fwlog_init(struct lpfc_hba *phba, mbx_fwlog->u.request.lwpd.addr_lo = putPaddrLow(ras_fwlog->lwpd.phys); mbx_fwlog->u.request.lwpd.addr_hi = putPaddrHigh(ras_fwlog->lwpd.phys); + spin_lock_irq(&phba->ras_fwlog_lock); + ras_fwlog->state = REG_INPROGRESS; + spin_unlock_irq(&phba->ras_fwlog_lock); mbox->vport = phba->pport; mbox->mbox_cmpl = lpfc_sli4_ras_mbox_cmpl; rc = lpfc_sli_issue_mbox(phba, mbox, MBX_NOWAIT); if (rc == MBX_NOT_FINISHED) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6191 FW-Log Mailbox failed. " "status %d mbxStatus : x%x", rc, bf_get(lpfc_mqe_status, &mbox->u.mqe)); @@ -6587,7 +7249,7 @@ lpfc_sli4_alloc_resource_identifiers(struct lpfc_hba *phba) /* RPIs. */ count = phba->sli4_hba.max_cfg_param.max_rpi; if (count <= 0) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3279 Invalid provisioning of " "rpi:%d\n", count); rc = -EINVAL; @@ -6615,7 +7277,7 @@ lpfc_sli4_alloc_resource_identifiers(struct lpfc_hba *phba) /* VPIs. */ count = phba->sli4_hba.max_cfg_param.max_vpi; if (count <= 0) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3280 Invalid provisioning of " "vpi:%d\n", count); rc = -EINVAL; @@ -6642,7 +7304,7 @@ lpfc_sli4_alloc_resource_identifiers(struct lpfc_hba *phba) /* XRIs. */ count = phba->sli4_hba.max_cfg_param.max_xri; if (count <= 0) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3281 Invalid provisioning of " "xri:%d\n", count); rc = -EINVAL; @@ -6671,7 +7333,7 @@ lpfc_sli4_alloc_resource_identifiers(struct lpfc_hba *phba) /* VFIs. */ count = phba->sli4_hba.max_cfg_param.max_vfi; if (count <= 0) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3282 Invalid provisioning of " "vfi:%d\n", count); rc = -EINVAL; @@ -6765,7 +7427,7 @@ lpfc_sli4_dealloc_resource_identifiers(struct lpfc_hba *phba) * lpfc_sli4_get_allocated_extnts - Get the port's allocated extents. * @phba: Pointer to HBA context object. * @type: The resource extent type. - * @extnt_count: buffer to hold port extent count response + * @extnt_cnt: buffer to hold port extent count response * @extnt_size: buffer to hold port extent size response. * * This function calls the port to read the host allocated extents @@ -6849,7 +7511,7 @@ lpfc_sli4_get_allocated_extnts(struct lpfc_hba *phba, uint16_t type, LPFC_MBOX_OPCODE_GET_ALLOC_RSRC_EXTENT, req_len, emb); if (alloc_len < req_len) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2983 Allocated DMA memory size (x%x) is " "less than the requested DMA memory " "size (x%x)\n", alloc_len, req_len); @@ -6892,7 +7554,7 @@ lpfc_sli4_get_allocated_extnts(struct lpfc_hba *phba, uint16_t type, } if (bf_get(lpfc_mbox_hdr_status, &shdr->response)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2984 Failed to read allocated resources " "for type %d - Status 0x%x Add'l Status 0x%x.\n", type, @@ -6909,7 +7571,6 @@ lpfc_sli4_get_allocated_extnts(struct lpfc_hba *phba, uint16_t type, /** * lpfc_sli4_repost_sgl_list - Repost the buffers sgl pages as block * @phba: pointer to lpfc hba data structure. - * @pring: Pointer to driver SLI ring object. * @sgl_list: linked link of sgl buffers to post * @cnt: number of linked list buffers * @@ -6930,7 +7591,7 @@ lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba, struct lpfc_sglq *sglq_entry = NULL; struct lpfc_sglq *sglq_entry_next = NULL; struct lpfc_sglq *sglq_entry_first = NULL; - int status, total_cnt; + int status = 0, total_cnt; int post_cnt = 0, num_posted = 0, block_cnt = 0; int last_xritag = NO_XRI; LIST_HEAD(prep_sgl_list); @@ -7047,8 +7708,10 @@ lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba, spin_unlock(&phba->sli4_hba.sgl_list_lock); spin_unlock_irq(&phba->hbalock); } else { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "3161 Failure to post sgl to port.\n"); + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "3161 Failure to post sgl to port,status %x " + "blkcnt %d totalcnt %d postcnt %d\n", + status, block_cnt, total_cnt, post_cnt); return -EIO; } @@ -7056,7 +7719,39 @@ lpfc_sli4_repost_sgl_list(struct lpfc_hba *phba, return total_cnt; } -void +/** + * lpfc_sli4_repost_io_sgl_list - Repost all the allocated nvme buffer sgls + * @phba: pointer to lpfc hba data structure. + * + * This routine walks the list of nvme buffers that have been allocated and + * repost them to the port by using SGL block post. This is needed after a + * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine + * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list + * to the lpfc_io_buf_list. If the repost fails, reject all nvme buffers. + * + * Returns: 0 = success, non-zero failure. + **/ +static int +lpfc_sli4_repost_io_sgl_list(struct lpfc_hba *phba) +{ + LIST_HEAD(post_nblist); + int num_posted, rc = 0; + + /* get all NVME buffers need to repost to a local list */ + lpfc_io_buf_flush(phba, &post_nblist); + + /* post the list of nvme buffer sgls to port if available */ + if (!list_empty(&post_nblist)) { + num_posted = lpfc_sli4_post_io_sgl_list( + phba, &post_nblist, phba->sli4_hba.io_xri_cnt); + /* failed to post any nvme buffer, return error */ + if (num_posted == 0) + rc = -EIO; + } + return rc; +} + +static void lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox) { uint32_t len; @@ -7070,10 +7765,10 @@ lpfc_set_host_data(struct lpfc_hba *phba, LPFC_MBOXQ_t *mbox) mbox->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_OS_DRIVER_VERSION; mbox->u.mqe.un.set_host_data.param_len = LPFC_HOST_OS_DRIVER_VERSION_SIZE; - snprintf(mbox->u.mqe.un.set_host_data.data, + snprintf(mbox->u.mqe.un.set_host_data.un.data, LPFC_HOST_OS_DRIVER_VERSION_SIZE, "Linux %s v"LPFC_DRIVER_VERSION, - (phba->hba_flag & HBA_FCOE_MODE) ? "FCoE" : "FC"); + test_bit(HBA_FCOE_MODE, &phba->hba_flag) ? "FCoE" : "FC"); } int @@ -7088,12 +7783,16 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, struct rqb_dmabuf *rqb_buffer; LIST_HEAD(rqb_buf_list); - spin_lock_irqsave(&phba->hbalock, flags); rqbp = hrq->rqbp; for (i = 0; i < count; i++) { + spin_lock_irqsave(&phba->hbalock, flags); /* IF RQ is already full, don't bother */ - if (rqbp->buffer_count + i >= rqbp->entry_count - 1) + if (rqbp->buffer_count + i >= rqbp->entry_count - 1) { + spin_unlock_irqrestore(&phba->hbalock, flags); break; + } + spin_unlock_irqrestore(&phba->hbalock, flags); + rqb_buffer = rqbp->rqb_alloc_buffer(phba); if (!rqb_buffer) break; @@ -7102,6 +7801,8 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, rqb_buffer->idx = idx; list_add_tail(&rqb_buffer->hbuf.list, &rqb_buf_list); } + + spin_lock_irqsave(&phba->hbalock, flags); while (!list_empty(&rqb_buf_list)) { list_remove_head(&rqb_buf_list, rqb_buffer, struct rqb_dmabuf, hbuf.list); @@ -7112,7 +7813,7 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, drqe.address_hi = putPaddrHigh(rqb_buffer->dbuf.phys); rc = lpfc_sli4_rq_put(hrq, drq, &hrqe, &drqe); if (rc < 0) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6421 Cannot post to HRQ %d: %x %x %x " "DRQ %x %x\n", hrq->queue_id, @@ -7132,6 +7833,685 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, return 1; } +static void +lpfc_mbx_cmpl_read_lds_params(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) +{ + union lpfc_sli4_cfg_shdr *shdr; + u32 shdr_status, shdr_add_status; + + shdr = (union lpfc_sli4_cfg_shdr *) + &pmb->u.mqe.un.sli4_config.header.cfg_shdr; + shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); + shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); + if (shdr_status || shdr_add_status || pmb->u.mb.mbxStatus) { + lpfc_printf_log(phba, KERN_INFO, LOG_LDS_EVENT | LOG_MBOX, + "4622 SET_FEATURE (x%x) mbox failed, " + "status x%x add_status x%x, mbx status x%x\n", + LPFC_SET_LD_SIGNAL, shdr_status, + shdr_add_status, pmb->u.mb.mbxStatus); + phba->degrade_activate_threshold = 0; + phba->degrade_deactivate_threshold = 0; + phba->fec_degrade_interval = 0; + goto out; + } + + phba->degrade_activate_threshold = pmb->u.mqe.un.set_feature.word7; + phba->degrade_deactivate_threshold = pmb->u.mqe.un.set_feature.word8; + phba->fec_degrade_interval = pmb->u.mqe.un.set_feature.word10; + + lpfc_printf_log(phba, KERN_INFO, LOG_LDS_EVENT, + "4624 Success: da x%x dd x%x interval x%x\n", + phba->degrade_activate_threshold, + phba->degrade_deactivate_threshold, + phba->fec_degrade_interval); +out: + mempool_free(pmb, phba->mbox_mem_pool); +} + +int +lpfc_read_lds_params(struct lpfc_hba *phba) +{ + LPFC_MBOXQ_t *mboxq; + int rc; + + mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); + if (!mboxq) + return -ENOMEM; + + lpfc_set_features(phba, mboxq, LPFC_SET_LD_SIGNAL); + mboxq->vport = phba->pport; + mboxq->mbox_cmpl = lpfc_mbx_cmpl_read_lds_params; + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT); + if (rc == MBX_NOT_FINISHED) { + mempool_free(mboxq, phba->mbox_mem_pool); + return -EIO; + } + return 0; +} + +static void +lpfc_mbx_cmpl_cgn_set_ftrs(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmb) +{ + struct lpfc_vport *vport = pmb->vport; + union lpfc_sli4_cfg_shdr *shdr; + u32 shdr_status, shdr_add_status; + u32 sig, acqe; + + /* Two outcomes. (1) Set featurs was successul and EDC negotiation + * is done. (2) Mailbox failed and send FPIN support only. + */ + shdr = (union lpfc_sli4_cfg_shdr *) + &pmb->u.mqe.un.sli4_config.header.cfg_shdr; + shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); + shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); + if (shdr_status || shdr_add_status || pmb->u.mb.mbxStatus) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT, + "2516 CGN SET_FEATURE mbox failed with " + "status x%x add_status x%x, mbx status x%x " + "Reset Congestion to FPINs only\n", + shdr_status, shdr_add_status, + pmb->u.mb.mbxStatus); + /* If there is a mbox error, move on to RDF */ + phba->cgn_reg_signal = EDC_CG_SIG_NOTSUPPORTED; + phba->cgn_reg_fpin = LPFC_CGN_FPIN_WARN | LPFC_CGN_FPIN_ALARM; + goto out; + } + + /* Zero out Congestion Signal ACQE counter */ + phba->cgn_acqe_cnt = 0; + + acqe = bf_get(lpfc_mbx_set_feature_CGN_acqe_freq, + &pmb->u.mqe.un.set_feature); + sig = bf_get(lpfc_mbx_set_feature_CGN_warn_freq, + &pmb->u.mqe.un.set_feature); + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "4620 SET_FEATURES Success: Freq: %ds %dms " + " Reg: x%x x%x\n", acqe, sig, + phba->cgn_reg_signal, phba->cgn_reg_fpin); +out: + mempool_free(pmb, phba->mbox_mem_pool); + + /* Register for FPIN events from the fabric now that the + * EDC common_set_features has completed. + */ + lpfc_issue_els_rdf(vport, 0); +} + +int +lpfc_config_cgn_signal(struct lpfc_hba *phba) +{ + LPFC_MBOXQ_t *mboxq; + u32 rc; + + mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); + if (!mboxq) + goto out_rdf; + + lpfc_set_features(phba, mboxq, LPFC_SET_CGN_SIGNAL); + mboxq->vport = phba->pport; + mboxq->mbox_cmpl = lpfc_mbx_cmpl_cgn_set_ftrs; + + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "4621 SET_FEATURES: FREQ sig x%x acqe x%x: " + "Reg: x%x x%x\n", + phba->cgn_sig_freq, lpfc_acqe_cgn_frequency, + phba->cgn_reg_signal, phba->cgn_reg_fpin); + + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT); + if (rc == MBX_NOT_FINISHED) + goto out; + return 0; + +out: + mempool_free(mboxq, phba->mbox_mem_pool); +out_rdf: + /* If there is a mbox error, move on to RDF */ + phba->cgn_reg_fpin = LPFC_CGN_FPIN_WARN | LPFC_CGN_FPIN_ALARM; + phba->cgn_reg_signal = EDC_CG_SIG_NOTSUPPORTED; + lpfc_issue_els_rdf(phba->pport, 0); + return -EIO; +} + +/** + * lpfc_init_idle_stat_hb - Initialize idle_stat tracking + * @phba: pointer to lpfc hba data structure. + * + * This routine initializes the per-eq idle_stat to dynamically dictate + * polling decisions. + * + * Return codes: + * None + **/ +static void lpfc_init_idle_stat_hb(struct lpfc_hba *phba) +{ + int i; + struct lpfc_sli4_hdw_queue *hdwq; + struct lpfc_queue *eq; + struct lpfc_idle_stat *idle_stat; + u64 wall; + + for_each_present_cpu(i) { + hdwq = &phba->sli4_hba.hdwq[phba->sli4_hba.cpu_map[i].hdwq]; + eq = hdwq->hba_eq; + + /* Skip if we've already handled this eq's primary CPU */ + if (eq->chann != i) + continue; + + idle_stat = &phba->sli4_hba.idle_stat[i]; + + idle_stat->prev_idle = get_cpu_idle_time(i, &wall, 1); + idle_stat->prev_wall = wall; + + if (phba->nvmet_support || + phba->cmf_active_mode != LPFC_CFG_OFF || + phba->intr_type != MSIX) + eq->poll_mode = LPFC_QUEUE_WORK; + else + eq->poll_mode = LPFC_THREADED_IRQ; + } + + if (!phba->nvmet_support && phba->intr_type == MSIX) + schedule_delayed_work(&phba->idle_stat_delay_work, + msecs_to_jiffies(LPFC_IDLE_STAT_DELAY)); +} + +static void lpfc_sli4_dip(struct lpfc_hba *phba) +{ + uint32_t if_type; + + if_type = bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf); + if (if_type == LPFC_SLI_INTF_IF_TYPE_2 || + if_type == LPFC_SLI_INTF_IF_TYPE_6) { + struct lpfc_register reg_data; + + if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr, + ®_data.word0)) + return; + + if (bf_get(lpfc_sliport_status_dip, ®_data)) + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "2904 Firmware Dump Image Present" + " on Adapter"); + } +} + +/** + * lpfc_rx_monitor_create_ring - Initialize ring buffer for rx_monitor + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + * @entries: Number of rx_info_entry objects to allocate in ring + * + * Return: + * 0 - Success + * ENOMEM - Failure to kmalloc + **/ +int lpfc_rx_monitor_create_ring(struct lpfc_rx_info_monitor *rx_monitor, + u32 entries) +{ + rx_monitor->ring = kmalloc_array(entries, sizeof(struct rx_info_entry), + GFP_KERNEL); + if (!rx_monitor->ring) + return -ENOMEM; + + rx_monitor->head_idx = 0; + rx_monitor->tail_idx = 0; + spin_lock_init(&rx_monitor->lock); + rx_monitor->entries = entries; + + return 0; +} + +/** + * lpfc_rx_monitor_destroy_ring - Free ring buffer for rx_monitor + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + * + * Called after cancellation of cmf_timer. + **/ +void lpfc_rx_monitor_destroy_ring(struct lpfc_rx_info_monitor *rx_monitor) +{ + kfree(rx_monitor->ring); + rx_monitor->ring = NULL; + rx_monitor->entries = 0; + rx_monitor->head_idx = 0; + rx_monitor->tail_idx = 0; +} + +/** + * lpfc_rx_monitor_record - Insert an entry into rx_monitor's ring + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + * @entry: Pointer to rx_info_entry + * + * Used to insert an rx_info_entry into rx_monitor's ring. Note that this is a + * deep copy of rx_info_entry not a shallow copy of the rx_info_entry ptr. + * + * This is called from lpfc_cmf_timer, which is in timer/softirq context. + * + * In cases of old data overflow, we do a best effort of FIFO order. + **/ +void lpfc_rx_monitor_record(struct lpfc_rx_info_monitor *rx_monitor, + struct rx_info_entry *entry) +{ + struct rx_info_entry *ring = rx_monitor->ring; + u32 *head_idx = &rx_monitor->head_idx; + u32 *tail_idx = &rx_monitor->tail_idx; + spinlock_t *ring_lock = &rx_monitor->lock; + u32 ring_size = rx_monitor->entries; + + spin_lock(ring_lock); + memcpy(&ring[*tail_idx], entry, sizeof(*entry)); + *tail_idx = (*tail_idx + 1) % ring_size; + + /* Best effort of FIFO saved data */ + if (*tail_idx == *head_idx) + *head_idx = (*head_idx + 1) % ring_size; + + spin_unlock(ring_lock); +} + +/** + * lpfc_rx_monitor_report - Read out rx_monitor's ring + * @phba: Pointer to lpfc_hba object + * @rx_monitor: Pointer to lpfc_rx_info_monitor object + * @buf: Pointer to char buffer that will contain rx monitor info data + * @buf_len: Length buf including null char + * @max_read_entries: Maximum number of entries to read out of ring + * + * Used to dump/read what's in rx_monitor's ring buffer. + * + * If buf is NULL || buf_len == 0, then it is implied that we want to log the + * information to kmsg instead of filling out buf. + * + * Return: + * Number of entries read out of the ring + **/ +u32 lpfc_rx_monitor_report(struct lpfc_hba *phba, + struct lpfc_rx_info_monitor *rx_monitor, char *buf, + u32 buf_len, u32 max_read_entries) +{ + struct rx_info_entry *ring = rx_monitor->ring; + struct rx_info_entry *entry; + u32 *head_idx = &rx_monitor->head_idx; + u32 *tail_idx = &rx_monitor->tail_idx; + spinlock_t *ring_lock = &rx_monitor->lock; + u32 ring_size = rx_monitor->entries; + u32 cnt = 0; + char tmp[DBG_LOG_STR_SZ] = {0}; + bool log_to_kmsg = (!buf || !buf_len) ? true : false; + + if (!log_to_kmsg) { + /* clear the buffer to be sure */ + memset(buf, 0, buf_len); + + scnprintf(buf, buf_len, "\t%-16s%-16s%-16s%-16s%-8s%-8s%-8s" + "%-8s%-8s%-8s%-16s\n", + "MaxBPI", "Tot_Data_CMF", + "Tot_Data_Cmd", "Tot_Data_Cmpl", + "Lat(us)", "Avg_IO", "Max_IO", "Bsy", + "IO_cnt", "Info", "BWutil(ms)"); + } + + /* Needs to be _irq because record is called from timer interrupt + * context + */ + spin_lock_irq(ring_lock); + while (*head_idx != *tail_idx) { + entry = &ring[*head_idx]; + + /* Read out this entry's data. */ + if (!log_to_kmsg) { + /* If !log_to_kmsg, then store to buf. */ + scnprintf(tmp, sizeof(tmp), + "%03d:\t%-16llu%-16llu%-16llu%-16llu%-8llu" + "%-8llu%-8llu%-8u%-8u%-8u%u(%u)\n", + *head_idx, entry->max_bytes_per_interval, + entry->cmf_bytes, entry->total_bytes, + entry->rcv_bytes, entry->avg_io_latency, + entry->avg_io_size, entry->max_read_cnt, + entry->cmf_busy, entry->io_cnt, + entry->cmf_info, entry->timer_utilization, + entry->timer_interval); + + /* Check for buffer overflow */ + if ((strlen(buf) + strlen(tmp)) >= buf_len) + break; + + /* Append entry's data to buffer */ + strlcat(buf, tmp, buf_len); + } else { + lpfc_printf_log(phba, KERN_INFO, LOG_CGN_MGMT, + "4410 %02u: MBPI %llu Xmit %llu " + "Cmpl %llu Lat %llu ASz %llu Info %02u " + "BWUtil %u Int %u slot %u\n", + cnt, entry->max_bytes_per_interval, + entry->total_bytes, entry->rcv_bytes, + entry->avg_io_latency, + entry->avg_io_size, entry->cmf_info, + entry->timer_utilization, + entry->timer_interval, *head_idx); + } + + *head_idx = (*head_idx + 1) % ring_size; + + /* Don't feed more than max_read_entries */ + cnt++; + if (cnt >= max_read_entries) + break; + } + spin_unlock_irq(ring_lock); + + return cnt; +} + +/** + * lpfc_cmf_setup - Initialize idle_stat tracking + * @phba: Pointer to HBA context object. + * + * This is called from HBA setup during driver load or when the HBA + * comes online. this does all the initialization to support CMF and MI. + **/ +static int +lpfc_cmf_setup(struct lpfc_hba *phba) +{ + LPFC_MBOXQ_t *mboxq; + struct lpfc_dmabuf *mp; + struct lpfc_pc_sli4_params *sli4_params; + int rc, cmf, mi_ver; + + rc = lpfc_sli4_refresh_params(phba); + if (unlikely(rc)) + return rc; + + mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); + if (!mboxq) + return -ENOMEM; + + sli4_params = &phba->sli4_hba.pc_sli4_params; + + /* Always try to enable MI feature if we can */ + if (sli4_params->mi_ver) { + lpfc_set_features(phba, mboxq, LPFC_SET_ENABLE_MI); + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); + mi_ver = bf_get(lpfc_mbx_set_feature_mi, + &mboxq->u.mqe.un.set_feature); + + if (rc == MBX_SUCCESS) { + if (mi_ver) { + lpfc_printf_log(phba, + KERN_WARNING, LOG_CGN_MGMT, + "6215 MI is enabled\n"); + sli4_params->mi_ver = mi_ver; + } else { + lpfc_printf_log(phba, + KERN_WARNING, LOG_CGN_MGMT, + "6338 MI is disabled\n"); + sli4_params->mi_ver = 0; + } + } else { + /* mi_ver is already set from GET_SLI4_PARAMETERS */ + lpfc_printf_log(phba, KERN_INFO, + LOG_CGN_MGMT | LOG_INIT, + "6245 Enable MI Mailbox x%x (x%x/x%x) " + "failed, rc:x%x mi:x%x\n", + bf_get(lpfc_mqe_command, &mboxq->u.mqe), + lpfc_sli_config_mbox_subsys_get + (phba, mboxq), + lpfc_sli_config_mbox_opcode_get + (phba, mboxq), + rc, sli4_params->mi_ver); + } + } else { + lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT, + "6217 MI is disabled\n"); + } + + /* Ensure FDMI is enabled for MI if enable_mi is set */ + if (sli4_params->mi_ver) + phba->cfg_fdmi_on = LPFC_FDMI_SUPPORT; + + /* Always try to enable CMF feature if we can */ + if (sli4_params->cmf) { + lpfc_set_features(phba, mboxq, LPFC_SET_ENABLE_CMF); + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); + cmf = bf_get(lpfc_mbx_set_feature_cmf, + &mboxq->u.mqe.un.set_feature); + if (rc == MBX_SUCCESS && cmf) { + lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT, + "6218 CMF is enabled: mode %d\n", + phba->cmf_active_mode); + } else { + lpfc_printf_log(phba, KERN_WARNING, + LOG_CGN_MGMT | LOG_INIT, + "6219 Enable CMF Mailbox x%x (x%x/x%x) " + "failed, rc:x%x dd:x%x\n", + bf_get(lpfc_mqe_command, &mboxq->u.mqe), + lpfc_sli_config_mbox_subsys_get + (phba, mboxq), + lpfc_sli_config_mbox_opcode_get + (phba, mboxq), + rc, cmf); + sli4_params->cmf = 0; + phba->cmf_active_mode = LPFC_CFG_OFF; + goto no_cmf; + } + + /* Allocate Congestion Information Buffer */ + if (!phba->cgn_i) { + mp = kmalloc(sizeof(*mp), GFP_KERNEL); + if (mp) + mp->virt = dma_alloc_coherent + (&phba->pcidev->dev, + sizeof(struct lpfc_cgn_info), + &mp->phys, GFP_KERNEL); + if (!mp || !mp->virt) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "2640 Failed to alloc memory " + "for Congestion Info\n"); + kfree(mp); + sli4_params->cmf = 0; + phba->cmf_active_mode = LPFC_CFG_OFF; + goto no_cmf; + } + phba->cgn_i = mp; + + /* initialize congestion buffer info */ + lpfc_init_congestion_buf(phba); + lpfc_init_congestion_stat(phba); + + /* Zero out Congestion Signal counters */ + atomic64_set(&phba->cgn_acqe_stat.alarm, 0); + atomic64_set(&phba->cgn_acqe_stat.warn, 0); + } + + rc = lpfc_sli4_cgn_params_read(phba); + if (rc < 0) { + lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT | LOG_INIT, + "6242 Error reading Cgn Params (%d)\n", + rc); + /* Ensure CGN Mode is off */ + sli4_params->cmf = 0; + } else if (!rc) { + lpfc_printf_log(phba, KERN_ERR, LOG_CGN_MGMT | LOG_INIT, + "6243 CGN Event empty object.\n"); + /* Ensure CGN Mode is off */ + sli4_params->cmf = 0; + } + } else { +no_cmf: + lpfc_printf_log(phba, KERN_WARNING, LOG_CGN_MGMT, + "6220 CMF is disabled\n"); + } + + /* Only register congestion buffer with firmware if BOTH + * CMF and E2E are enabled. + */ + if (sli4_params->cmf && sli4_params->mi_ver) { + rc = lpfc_reg_congestion_buf(phba); + if (rc) { + dma_free_coherent(&phba->pcidev->dev, + sizeof(struct lpfc_cgn_info), + phba->cgn_i->virt, phba->cgn_i->phys); + kfree(phba->cgn_i); + phba->cgn_i = NULL; + /* Ensure CGN Mode is off */ + phba->cmf_active_mode = LPFC_CFG_OFF; + sli4_params->cmf = 0; + return 0; + } + } + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "6470 Setup MI version %d CMF %d mode %d\n", + sli4_params->mi_ver, sli4_params->cmf, + phba->cmf_active_mode); + + mempool_free(mboxq, phba->mbox_mem_pool); + + /* Initialize atomic counters */ + atomic_set(&phba->cgn_fabric_warn_cnt, 0); + atomic_set(&phba->cgn_fabric_alarm_cnt, 0); + atomic_set(&phba->cgn_sync_alarm_cnt, 0); + atomic_set(&phba->cgn_sync_warn_cnt, 0); + atomic_set(&phba->cgn_driver_evt_cnt, 0); + atomic_set(&phba->cgn_latency_evt_cnt, 0); + atomic64_set(&phba->cgn_latency_evt, 0); + + phba->cmf_interval_rate = LPFC_CMF_INTERVAL; + + /* Allocate RX Monitor Buffer */ + if (!phba->rx_monitor) { + phba->rx_monitor = kzalloc(sizeof(*phba->rx_monitor), + GFP_KERNEL); + + if (!phba->rx_monitor) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "2644 Failed to alloc memory " + "for RX Monitor Buffer\n"); + return -ENOMEM; + } + + /* Instruct the rx_monitor object to instantiate its ring */ + if (lpfc_rx_monitor_create_ring(phba->rx_monitor, + LPFC_MAX_RXMONITOR_ENTRY)) { + kfree(phba->rx_monitor); + phba->rx_monitor = NULL; + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "2645 Failed to alloc memory " + "for RX Monitor's Ring\n"); + return -ENOMEM; + } + } + + return 0; +} + +static int +lpfc_set_host_tm(struct lpfc_hba *phba) +{ + LPFC_MBOXQ_t *mboxq; + uint32_t len, rc; + struct timespec64 cur_time; + struct tm broken; + uint32_t month, day, year; + uint32_t hour, minute, second; + struct lpfc_mbx_set_host_date_time *tm; + + mboxq = (LPFC_MBOXQ_t *)mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); + if (!mboxq) + return -ENOMEM; + + len = sizeof(struct lpfc_mbx_set_host_data) - + sizeof(struct lpfc_sli4_cfg_mhdr); + lpfc_sli4_config(phba, mboxq, LPFC_MBOX_SUBSYSTEM_COMMON, + LPFC_MBOX_OPCODE_SET_HOST_DATA, len, + LPFC_SLI4_MBX_EMBED); + + mboxq->u.mqe.un.set_host_data.param_id = LPFC_SET_HOST_DATE_TIME; + mboxq->u.mqe.un.set_host_data.param_len = + sizeof(struct lpfc_mbx_set_host_date_time); + tm = &mboxq->u.mqe.un.set_host_data.un.tm; + ktime_get_real_ts64(&cur_time); + time64_to_tm(cur_time.tv_sec, 0, &broken); + month = broken.tm_mon + 1; + day = broken.tm_mday; + year = broken.tm_year - 100; + hour = broken.tm_hour; + minute = broken.tm_min; + second = broken.tm_sec; + bf_set(lpfc_mbx_set_host_month, tm, month); + bf_set(lpfc_mbx_set_host_day, tm, day); + bf_set(lpfc_mbx_set_host_year, tm, year); + bf_set(lpfc_mbx_set_host_hour, tm, hour); + bf_set(lpfc_mbx_set_host_min, tm, minute); + bf_set(lpfc_mbx_set_host_sec, tm, second); + + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); + mempool_free(mboxq, phba->mbox_mem_pool); + return rc; +} + +/** + * lpfc_get_platform_uuid - Attempts to extract a platform uuid + * @phba: pointer to lpfc hba data structure. + * + * This routine attempts to first read SMBIOS DMI data for the System + * Information structure offset 08h called System UUID. Else, no platform + * UUID will be advertised. + **/ +static void +lpfc_get_platform_uuid(struct lpfc_hba *phba) +{ + int rc; + const char *uuid; + char pni[17] = {0}; /* 16 characters + '\0' */ + bool is_ff = true, is_00 = true; + u8 i; + + /* First attempt SMBIOS DMI */ + uuid = dmi_get_system_info(DMI_PRODUCT_UUID); + if (uuid) { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "2088 SMBIOS UUID %s\n", + uuid); + } else { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "2099 Could not extract UUID\n"); + } + + if (uuid && uuid_is_valid(uuid)) { + /* Generate PNI from UUID format. + * + * 1.) Extract lower 64 bits from UUID format. + * 2.) Set 3h for NAA Locally Assigned Name Identifier format. + * + * e.g. xxxxxxxx-xxxx-xxxx-yyyy-yyyyyyyyyyyy + * + * extract the yyyy-yyyyyyyyyyyy portion + * final PNI 3yyyyyyyyyyyyyyy + */ + scnprintf(pni, sizeof(pni), "3%c%c%c%s", + uuid[20], uuid[21], uuid[22], &uuid[24]); + + /* Sanitize the converted PNI */ + for (i = 1; i < 16 && (is_ff || is_00); i++) { + if (pni[i] != '0') + is_00 = false; + if (pni[i] != 'f' && pni[i] != 'F') + is_ff = false; + } + + /* Convert from char* to unsigned long */ + rc = kstrtoul(pni, 16, &phba->pni); + if (!rc && !is_ff && !is_00) { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "2100 PNI 0x%016lx\n", phba->pni); + } else { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT, + "2101 PNI %s generation status %d\n", + pni, rc); + phba->pni = 0; + } + } +} + /** * lpfc_sli4_hba_setup - SLI4 device initialization PCI function * @phba: Pointer to HBA context object. @@ -7144,7 +8524,7 @@ lpfc_post_rq_buffer(struct lpfc_hba *phba, struct lpfc_queue *hrq, int lpfc_sli4_hba_setup(struct lpfc_hba *phba) { - int rc, i, cnt; + int rc, i, cnt, len, dd; LPFC_MBOXQ_t *mboxq; struct lpfc_mqe *mqe; uint8_t *vpd; @@ -7154,6 +8534,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) struct lpfc_vport *vport = phba->pport; struct lpfc_dmabuf *mp; struct lpfc_rqb *rqbp; + u32 flg; /* Perform a PCI function reset to start from clean */ rc = lpfc_pci_function_reset(phba); @@ -7167,8 +8548,21 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) else { spin_lock_irq(&phba->hbalock); phba->sli.sli_flag |= LPFC_SLI_ACTIVE; + flg = phba->sli.sli_flag; spin_unlock_irq(&phba->hbalock); + /* Allow a little time after setting SLI_ACTIVE for any polled + * MBX commands to complete via BSG. + */ + for (i = 0; i < 50 && (flg & LPFC_SLI_MBOX_ACTIVE); i++) { + msleep(20); + spin_lock_irq(&phba->hbalock); + flg = phba->sli.sli_flag; + spin_unlock_irq(&phba->hbalock); + } } + clear_bit(HBA_SETUP, &phba->hba_flag); + + lpfc_sli4_dip(phba); /* * Allocate a single mailbox container for initializing the @@ -7195,36 +8589,45 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) mqe = &mboxq->u.mqe; phba->sli_rev = bf_get(lpfc_mbx_rd_rev_sli_lvl, &mqe->un.read_rev); if (bf_get(lpfc_mbx_rd_rev_fcoe, &mqe->un.read_rev)) { - phba->hba_flag |= HBA_FCOE_MODE; + set_bit(HBA_FCOE_MODE, &phba->hba_flag); phba->fcp_embed_io = 0; /* SLI4 FC support only */ } else { - phba->hba_flag &= ~HBA_FCOE_MODE; + clear_bit(HBA_FCOE_MODE, &phba->hba_flag); } + /* Obtain platform UUID, only for SLI4 FC adapters */ + if (!test_bit(HBA_FCOE_MODE, &phba->hba_flag)) + lpfc_get_platform_uuid(phba); + if (bf_get(lpfc_mbx_rd_rev_cee_ver, &mqe->un.read_rev) == LPFC_DCBX_CEE_MODE) - phba->hba_flag |= HBA_FIP_SUPPORT; + set_bit(HBA_FIP_SUPPORT, &phba->hba_flag); else - phba->hba_flag &= ~HBA_FIP_SUPPORT; + clear_bit(HBA_FIP_SUPPORT, &phba->hba_flag); - phba->hba_flag &= ~HBA_FCP_IOQ_FLUSH; + clear_bit(HBA_IOQ_FLUSH, &phba->hba_flag); if (phba->sli_rev != LPFC_SLI_REV4) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0376 READ_REV Error. SLI Level %d " "FCoE enabled %d\n", - phba->sli_rev, phba->hba_flag & HBA_FCOE_MODE); + phba->sli_rev, + test_bit(HBA_FCOE_MODE, &phba->hba_flag) ? 1 : 0); rc = -EIO; kfree(vpd); goto out_free_mbox; } + rc = lpfc_set_host_tm(phba); + lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_INIT, + "6468 Set host date / time: Status x%x:\n", rc); + /* * Continue initialization with default values even if driver failed * to read FCoE param config regions, only read parameters if the * board is FCoE */ - if (phba->hba_flag & HBA_FCOE_MODE && + if (test_bit(HBA_FCOE_MODE, &phba->hba_flag) && lpfc_sli4_read_fcoe_params(phba)) lpfc_printf_log(phba, KERN_WARNING, LOG_MBOX | LOG_INIT, "2570 Failed to read FCoE parameters\n"); @@ -7239,18 +8642,22 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) "3080 Successful retrieving SLI4 device " "physical port name: %s.\n", phba->Port); + rc = lpfc_sli4_get_ctl_attr(phba); + if (!rc) + lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI, + "8351 Successful retrieving SLI4 device " + "CTL ATTR\n"); + /* * Evaluate the read rev and vpd data. Populate the driver * state with the results. If this routine fails, the failure * is not fatal as the driver will use generic values. */ rc = lpfc_parse_vpd(phba, vpd, vpd_size); - if (unlikely(!rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + if (unlikely(!rc)) + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0377 Error %d parsing vpd. " "Using defaults.\n", rc); - rc = 0; - } kfree(vpd); /* Save information as VPD data */ @@ -7292,21 +8699,12 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) phba->vpd.rev.fcphHigh, phba->vpd.rev.fcphLow, phba->vpd.rev.feaLevelHigh, phba->vpd.rev.feaLevelLow); - /* Reset the DFT_LUN_Q_DEPTH to (max xri >> 3) */ - rc = (phba->sli4_hba.max_cfg_param.max_xri >> 3); - if (phba->pport->cfg_lun_queue_depth > rc) { - lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, - "3362 LUN queue depth changed from %d to %d\n", - phba->pport->cfg_lun_queue_depth, rc); - phba->pport->cfg_lun_queue_depth = rc; - } - if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) == LPFC_SLI_INTF_IF_TYPE_0) { lpfc_set_features(phba, mboxq, LPFC_SET_UE_RECOVERY); rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); if (rc == MBX_SUCCESS) { - phba->hba_flag |= HBA_RECOVERABLE_UE; + set_bit(HBA_RECOVERABLE_UE, &phba->hba_flag); /* Set 1Sec interval to detect UE */ phba->eratt_poll_interval = 1; phba->sli4_hba.ue_to_sr = bf_get( @@ -7337,6 +8735,15 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) goto out_free_mbox; } + /* Disable VMID if app header is not supported */ + if (phba->cfg_vmid_app_header && !(bf_get(lpfc_mbx_rq_ftr_rsp_ashdr, + &mqe->un.req_ftrs))) { + bf_set(lpfc_ftr_ashdr, &phba->sli4_hba.sli4_flags, 0); + phba->cfg_vmid_app_header = 0; + lpfc_printf_log(phba, KERN_DEBUG, LOG_SLI, + "1242 vmid feature not supported\n"); + } + /* * The port must support FCP initiator mode as this is the * only mode running in the host. @@ -7348,7 +8755,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) } /* Performance Hints are ONLY for FCoE */ - if (phba->hba_flag & HBA_FCOE_MODE) { + if (test_bit(HBA_FCOE_MODE, &phba->hba_flag)) { if (bf_get(lpfc_mbx_rq_ftr_rsp_perfh, &mqe->un.req_ftrs)) phba->sli3_options |= LPFC_SLI4_PERFH_ENABLED; else @@ -7389,18 +8796,38 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) phba->sli3_options |= (LPFC_SLI3_NPIV_ENABLED | LPFC_SLI3_HBQ_ENABLED); spin_unlock_irq(&phba->hbalock); + /* Always try to enable dual dump feature if we can */ + lpfc_set_features(phba, mboxq, LPFC_SET_DUAL_DUMP); + rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); + dd = bf_get(lpfc_mbx_set_feature_dd, &mboxq->u.mqe.un.set_feature); + if ((rc == MBX_SUCCESS) && (dd == LPFC_ENABLE_DUAL_DUMP)) + lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + "6448 Dual Dump is enabled\n"); + else + lpfc_printf_log(phba, KERN_INFO, LOG_SLI | LOG_INIT, + "6447 Dual Dump Mailbox x%x (x%x/x%x) failed, " + "rc:x%x dd:x%x\n", + bf_get(lpfc_mqe_command, &mboxq->u.mqe), + lpfc_sli_config_mbox_subsys_get( + phba, mboxq), + lpfc_sli_config_mbox_opcode_get( + phba, mboxq), + rc, dd); + /* * Allocate all resources (xri,rpi,vpi,vfi) now. Subsequent * calls depends on these resources to complete port setup. */ rc = lpfc_sli4_alloc_resource_identifiers(phba); if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2920 Failed to alloc Resource IDs " "rc = x%x\n", rc); goto out_free_mbox; } + lpfc_sli4_node_rpi_restore(phba); + lpfc_set_host_data(phba, mboxq); rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); @@ -7420,21 +8847,22 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) mboxq->vport = vport; rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); - mp = (struct lpfc_dmabuf *)mboxq->ctx_buf; + mp = mboxq->ctx_buf; if (rc == MBX_SUCCESS) { memcpy(&vport->fc_sparam, mp->virt, sizeof(struct serv_parm)); rc = 0; } /* - * This memory was allocated by the lpfc_read_sparam routine. Release - * it to the mbuf pool. + * This memory was allocated by the lpfc_read_sparam routine but is + * no longer needed. It is released and ctx_buf NULLed to prevent + * unintended pointer access as the mbox is reused. */ lpfc_mbuf_free(phba, mp->virt, mp->phys); kfree(mp); mboxq->ctx_buf = NULL; if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0382 READ_SPARAM command failed " "status %d, mbxStatus x%x\n", rc, bf_get(lpfc_mqe_status, mqe)); @@ -7452,7 +8880,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) /* Create all the SLI4 queues */ rc = lpfc_sli4_queue_create(phba); if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3089 Failed to allocate queues\n"); rc = -ENODEV; goto out_free_mbox; @@ -7460,9 +8888,9 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) /* Set up all the queues to the device */ rc = lpfc_sli4_queue_setup(phba); if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "0381 Error %d during queue setup.\n ", rc); - goto out_stop_timers; + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "0381 Error %d during queue setup.\n", rc); + goto out_destroy_queue; } /* Initialize the driver internal SLI layer lists. */ lpfc_sli4_setup(phba); @@ -7471,7 +8899,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) /* update host els xri-sgl sizes and mappings */ rc = lpfc_sli4_els_sgl_update(phba); if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "1400 Failed to update xri-sgl size and " "mapping: %d\n", rc); goto out_destroy_queue; @@ -7481,7 +8909,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) rc = lpfc_sli4_repost_sgl_list(phba, &phba->sli4_hba.lpfc_els_sgl_list, phba->sli4_hba.els_xri_cnt); if (unlikely(rc < 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0582 Error %d during els sgl post " "operation\n", rc); rc = -ENODEV; @@ -7493,7 +8921,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) /* update host nvmet xri-sgl sizes and mappings */ rc = lpfc_sli4_nvmet_sgl_update(phba); if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6308 Failed to update nvmet-sgl size " "and mapping: %d\n", rc); goto out_destroy_queue; @@ -7505,7 +8933,7 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) &phba->sli4_hba.lpfc_nvmet_sgl_list, phba->sli4_hba.nvmet_xri_cnt); if (unlikely(rc < 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3117 Error %d during nvmet " "sgl post\n", rc); rc = -ENODEV; @@ -7513,39 +8941,46 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) } phba->sli4_hba.nvmet_xri_cnt = rc; - cnt = phba->cfg_iocb_cnt * 1024; - /* We need 1 iocbq for every SGL, for IO processing */ - cnt += phba->sli4_hba.nvmet_xri_cnt; + /* We allocate an iocbq for every receive context SGL. + * The additional allocation is for abort and ls handling. + */ + cnt = phba->sli4_hba.nvmet_xri_cnt + + phba->sli4_hba.max_cfg_param.max_xri; } else { - /* update host scsi xri-sgl sizes and mappings */ - rc = lpfc_sli4_scsi_sgl_update(phba); + /* update host common xri-sgl sizes and mappings */ + rc = lpfc_sli4_io_sgl_update(phba); if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "6309 Failed to update scsi-sgl size " + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6082 Failed to update nvme-sgl size " "and mapping: %d\n", rc); goto out_destroy_queue; } - /* update host nvme xri-sgl sizes and mappings */ - rc = lpfc_sli4_nvme_sgl_update(phba); + /* register the allocated common sgl pool to the port */ + rc = lpfc_sli4_repost_io_sgl_list(phba); if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "6082 Failed to update nvme-sgl size " - "and mapping: %d\n", rc); + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6116 Error %d during nvme sgl post " + "operation\n", rc); + /* Some NVME buffers were moved to abort nvme list */ + /* A pci function reset will repost them */ + rc = -ENODEV; goto out_destroy_queue; } - - cnt = phba->cfg_iocb_cnt * 1024; + /* Each lpfc_io_buf job structure has an iocbq element. + * This cnt provides for abort, els, ct and ls requests. + */ + cnt = phba->sli4_hba.max_cfg_param.max_xri; } if (!phba->sli.iocbq_lookup) { /* Initialize and populate the iocb list per host */ lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2821 initialize iocb list %d total %d\n", - phba->cfg_iocb_cnt, cnt); + "2821 initialize iocb list with %d entries\n", + cnt); rc = lpfc_init_iocb_list(phba, cnt); if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "1413 Failed to init iocb list.\n"); goto out_destroy_queue; } @@ -7571,48 +9006,17 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) } } - if (phba->cfg_enable_fc4_type & LPFC_ENABLE_FCP) { - /* register the allocated scsi sgl pool to the port */ - rc = lpfc_sli4_repost_scsi_sgl_list(phba); - if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "0383 Error %d during scsi sgl post " - "operation\n", rc); - /* Some Scsi buffers were moved to abort scsi list */ - /* A pci function reset will repost them */ - rc = -ENODEV; - goto out_destroy_queue; - } - } - - if ((phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) && - (phba->nvmet_support == 0)) { - - /* register the allocated nvme sgl pool to the port */ - rc = lpfc_repost_nvme_sgl_list(phba); - if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "6116 Error %d during nvme sgl post " - "operation\n", rc); - /* Some NVME buffers were moved to abort nvme list */ - /* A pci function reset will repost them */ - rc = -ENODEV; - goto out_destroy_queue; - } - } - /* Post the rpi header region to the device. */ rc = lpfc_sli4_post_all_rpi_hdrs(phba); if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0393 Error %d during rpi post operation\n", rc); rc = -ENODEV; - goto out_destroy_queue; + goto out_free_iocblist; } - lpfc_sli4_node_prep(phba); - if (!(phba->hba_flag & HBA_FCOE_MODE)) { + if (!test_bit(HBA_FCOE_MODE, &phba->hba_flag)) { if ((phba->nvmet_support == 0) || (phba->cfg_nvmet_mrq == 1)) { /* * The FC Port needs to register FCFI (index 0) @@ -7650,11 +9054,24 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) lpfc_sli_read_link_ste(phba); } - /* Arm the CQs and then EQs on device */ - lpfc_sli4_arm_cqeq_intr(phba); + /* Don't post more new bufs if repost already recovered + * the nvme sgls. + */ + if (phba->nvmet_support == 0) { + if (phba->sli4_hba.io_xri_cnt == 0) { + len = lpfc_new_io_buf( + phba, phba->sli4_hba.io_xri_max); + if (len == 0) { + rc = -ENOMEM; + goto out_unset_queue; + } - /* Indicate device interrupt mode */ - phba->sli4_hba.intr_enable = 1; + if (phba->cfg_xri_rebalancing) + lpfc_create_multixri_pools(phba); + } + } else { + phba->cfg_xri_rebalancing = 0; + } /* Allow asynchronous mailbox command to go through */ spin_lock_irq(&phba->hbalock); @@ -7670,36 +9087,26 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) /* Start the ELS watchdog timer */ mod_timer(&vport->els_tmofunc, - jiffies + msecs_to_jiffies(1000 * (phba->fc_ratov * 2))); + jiffies + secs_to_jiffies(phba->fc_ratov * 2)); /* Start heart beat timer */ mod_timer(&phba->hb_tmofunc, - jiffies + msecs_to_jiffies(1000 * LPFC_HB_MBOX_INTERVAL)); - phba->hb_outstanding = 0; + jiffies + secs_to_jiffies(LPFC_HB_MBOX_INTERVAL)); + clear_bit(HBA_HBEAT_INP, &phba->hba_flag); + clear_bit(HBA_HBEAT_TMO, &phba->hba_flag); phba->last_completion_time = jiffies; + /* start eq_delay heartbeat */ + if (phba->cfg_auto_imax) + queue_delayed_work(phba->wq, &phba->eq_delay_work, + msecs_to_jiffies(LPFC_EQ_DELAY_MSECS)); + + /* start per phba idle_stat_delay heartbeat */ + lpfc_init_idle_stat_hb(phba); + /* Start error attention (ERATT) polling timer */ mod_timer(&phba->eratt_poll, - jiffies + msecs_to_jiffies(1000 * phba->eratt_poll_interval)); - - /* Enable PCIe device Advanced Error Reporting (AER) if configured */ - if (phba->cfg_aer_support == 1 && !(phba->hba_flag & HBA_AER_ENABLED)) { - rc = pci_enable_pcie_error_reporting(phba->pcidev); - if (!rc) { - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2829 This device supports " - "Advanced Error Reporting (AER)\n"); - spin_lock_irq(&phba->hbalock); - phba->hba_flag |= HBA_AER_ENABLED; - spin_unlock_irq(&phba->hbalock); - } else { - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "2830 This device does not support " - "Advanced Error Reporting (AER)\n"); - phba->cfg_aer_support = 0; - } - rc = 0; - } + jiffies + secs_to_jiffies(phba->eratt_poll_interval)); /* * The port is ready, set the host's link state to LINK_DOWN @@ -7719,35 +9126,53 @@ lpfc_sli4_hba_setup(struct lpfc_hba *phba) phba->trunk_link.link3.state = LPFC_LINK_DOWN; spin_unlock_irq(&phba->hbalock); - if (!(phba->hba_flag & HBA_FCOE_MODE) && - (phba->hba_flag & LINK_DISABLED)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI, + /* Arm the CQs and then EQs on device */ + lpfc_sli4_arm_cqeq_intr(phba); + + /* Indicate device interrupt mode */ + phba->sli4_hba.intr_enable = 1; + + /* Setup CMF after HBA is initialized */ + lpfc_cmf_setup(phba); + + if (!test_bit(HBA_FCOE_MODE, &phba->hba_flag) && + test_bit(LINK_DISABLED, &phba->hba_flag)) { + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3103 Adapter Link is disabled.\n"); lpfc_down_link(phba, mboxq); rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); if (rc != MBX_SUCCESS) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3104 Adapter failed to issue " "DOWN_LINK mbox cmd, rc:x%x\n", rc); - goto out_unset_queue; + goto out_io_buff_free; } } else if (phba->cfg_suppress_link_up == LPFC_INITIALIZE_LINK) { /* don't perform init_link on SLI4 FC port loopback test */ if (!(phba->link_flag & LS_LOOPBACK_MODE)) { rc = phba->lpfc_hba_init_link(phba, MBX_NOWAIT); if (rc) - goto out_unset_queue; + goto out_io_buff_free; } } mempool_free(mboxq, phba->mbox_mem_pool); + + /* Enable RAS FW log support */ + lpfc_sli4_ras_setup(phba); + + set_bit(HBA_SETUP, &phba->hba_flag); return rc; + +out_io_buff_free: + /* Free allocated IO Buffers */ + lpfc_io_free(phba); out_unset_queue: /* Unset all the queues set up in this routine when error out */ lpfc_sli4_queue_unset(phba); -out_destroy_queue: +out_free_iocblist: lpfc_free_iocb_list(phba); +out_destroy_queue: lpfc_sli4_queue_destroy(phba); -out_stop_timers: lpfc_stop_hba_timers(phba); out_free_mbox: mempool_free(mboxq, phba->mbox_mem_pool); @@ -7756,7 +9181,7 @@ out_free_mbox: /** * lpfc_mbox_timeout - Timeout call back function for mbox timer - * @ptr: context object - pointer to hba structure. + * @t: Context to fetch pointer to hba structure from. * * This is the callback function for mailbox timer. The mailbox * timer is armed when a new mailbox command is issued and the timer @@ -7769,7 +9194,7 @@ out_free_mbox: void lpfc_mbox_timeout(struct timer_list *t) { - struct lpfc_hba *phba = from_timer(phba, t, sli.mbox_tmo); + struct lpfc_hba *phba = timer_container_of(phba, t, sli.mbox_tmo); unsigned long iflag; uint32_t tmo_posted; @@ -7810,8 +9235,9 @@ lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba) mcq = phba->sli4_hba.mbx_cq; idx = mcq->hba_index; qe_valid = mcq->qe_valid; - while (bf_get_le32(lpfc_cqe_valid, mcq->qe[idx].cqe) == qe_valid) { - mcqe = (struct lpfc_mcqe *)mcq->qe[idx].cqe; + while (bf_get_le32(lpfc_cqe_valid, + (struct lpfc_cqe *)lpfc_sli4_qe(mcq, idx)) == qe_valid) { + mcqe = (struct lpfc_mcqe *)(lpfc_sli4_qe(mcq, idx)); if (bf_get_le32(lpfc_trailer_completed, mcqe) && (!bf_get_le32(lpfc_trailer_async, mcqe))) { pending_completions = true; @@ -7840,27 +9266,28 @@ lpfc_sli4_mbox_completions_pending(struct lpfc_hba *phba) * and will process all the completions associated with the eq for the * mailbox completion queue. **/ -bool +static bool lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) { struct lpfc_sli4_hba *sli4_hba = &phba->sli4_hba; uint32_t eqidx; struct lpfc_queue *fpeq = NULL; - struct lpfc_eqe *eqe; + struct lpfc_queue *eq; bool mbox_pending; if (unlikely(!phba) || (phba->sli_rev != LPFC_SLI_REV4)) return false; - /* Find the eq associated with the mcq */ - - if (sli4_hba->hba_eq) - for (eqidx = 0; eqidx < phba->io_channel_irqs; eqidx++) - if (sli4_hba->hba_eq[eqidx]->queue_id == - sli4_hba->mbx_cq->assoc_qid) { - fpeq = sli4_hba->hba_eq[eqidx]; + /* Find the EQ associated with the mbox CQ */ + if (sli4_hba->hdwq) { + for (eqidx = 0; eqidx < phba->cfg_irq_chann; eqidx++) { + eq = phba->sli4_hba.hba_eq_hdl[eqidx].eq; + if (eq && eq->queue_id == sli4_hba->mbx_cq->assoc_qid) { + fpeq = eq; break; } + } + } if (!fpeq) return false; @@ -7880,14 +9307,12 @@ lpfc_sli4_process_missed_mbox_completions(struct lpfc_hba *phba) */ if (mbox_pending) - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, eqe, eqidx); - fpeq->EQ_processed++; - } - - /* Always clear and re-arm the EQ */ - - sli4_hba->sli4_eq_release(fpeq, LPFC_QUEUE_REARM); + /* process and rearm the EQ */ + lpfc_sli4_process_eq(phba, fpeq, LPFC_QUEUE_REARM, + LPFC_QUEUE_WORK); + else + /* Always clear and re-arm the EQ */ + sli4_hba->sli4_write_eq_db(phba, fpeq, 0, LPFC_QUEUE_REARM); return mbox_pending; @@ -7909,8 +9334,10 @@ lpfc_mbox_timeout_handler(struct lpfc_hba *phba) struct lpfc_sli *psli = &phba->sli; - /* If the mailbox completed, process the completion and return */ - if (lpfc_sli4_process_missed_mbox_completions(phba)) + /* If the mailbox completed, process the completion */ + lpfc_sli4_process_missed_mbox_completions(phba); + + if (!(psli->sli_flag & LPFC_SLI_ACTIVE)) return; if (pmbox != NULL) @@ -7931,8 +9358,8 @@ lpfc_mbox_timeout_handler(struct lpfc_hba *phba) } /* Mbox cmd <mbxCommand> timeout */ - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, - "0310 Mailbox command x%x timeout Data: x%x x%x x%p\n", + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "0310 Mailbox command x%x timeout Data: x%x x%x x%px\n", mb->mbxCommand, phba->pport->port_state, phba->sli.sli_flag, @@ -7943,6 +9370,7 @@ lpfc_mbox_timeout_handler(struct lpfc_hba *phba) * would get IOCB_ERROR from lpfc_sli_issue_iocb, allowing * it to fail all outstanding SCSI IO. */ + set_bit(MBX_TMO_ERR, &phba->bit_flags); spin_lock_irq(&phba->pport->work_port_lock); phba->pport->work_port_events &= ~WORKER_MBOX_TMO; spin_unlock_irq(&phba->pport->work_port_lock); @@ -7951,9 +9379,7 @@ lpfc_mbox_timeout_handler(struct lpfc_hba *phba) psli->sli_flag &= ~LPFC_SLI_ACTIVE; spin_unlock_irq(&phba->hbalock); - lpfc_sli_abort_fcp_rings(phba); - - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0345 Resetting board due to mailbox timeout\n"); /* Reset the HBA device */ @@ -8037,7 +9463,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, } /* If HBA has a deferred error attention, fail the iocb. */ - if (unlikely(phba->hba_flag & DEFER_ERATT)) { + if (unlikely(test_bit(DEFER_ERATT, &phba->hba_flag))) { spin_unlock_irqrestore(&phba->hbalock, drvr_flag); goto out_not_finished; } @@ -8051,7 +9477,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, spin_unlock_irqrestore(&phba->hbalock, drvr_flag); /* Mbox command <mbxCommand> cannot issue */ - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):0311 Mailbox command x%x cannot " "issue Data: x%x x%x\n", pmbox->vport ? pmbox->vport->vpi : 0, @@ -8063,7 +9489,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, if (lpfc_readl(phba->HCregaddr, &hc_copy) || !(hc_copy & HC_MBINT_ENA)) { spin_unlock_irqrestore(&phba->hbalock, drvr_flag); - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2528 Mailbox command x%x cannot " "issue Data: x%x x%x\n", pmbox->vport ? pmbox->vport->vpi : 0, @@ -8082,7 +9508,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, spin_unlock_irqrestore(&phba->hbalock, drvr_flag); /* Mbox command <mbxCommand> cannot issue */ - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2529 Mailbox command x%x " "cannot issue Data: x%x x%x\n", pmbox->vport ? pmbox->vport->vpi : 0, @@ -8094,7 +9520,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, if (!(psli->sli_flag & LPFC_SLI_ACTIVE)) { spin_unlock_irqrestore(&phba->hbalock, drvr_flag); /* Mbox command <mbxCommand> cannot issue */ - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2530 Mailbox command x%x " "cannot issue Data: x%x x%x\n", pmbox->vport ? pmbox->vport->vpi : 0, @@ -8147,7 +9573,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, psli->sli_flag &= ~LPFC_SLI_MBOX_ACTIVE; spin_unlock_irqrestore(&phba->hbalock, drvr_flag); /* Mbox command <mbxCommand> cannot issue */ - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2531 Mailbox command x%x " "cannot issue Data: x%x x%x\n", pmbox->vport ? pmbox->vport->vpi : 0, @@ -8156,8 +9582,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, goto out_not_finished; } /* timeout active mbox command */ - timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox) * - 1000); + timeout = secs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox)); mod_timer(&psli->mbox_tmo, jiffies + timeout); } @@ -8202,8 +9627,8 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, } /* Copy the mailbox extension data */ - if (pmbox->in_ext_byte_len && pmbox->ctx_buf) { - lpfc_sli_pcimem_bcopy(pmbox->ctx_buf, + if (pmbox->in_ext_byte_len && pmbox->ext_buf) { + lpfc_sli_pcimem_bcopy(pmbox->ext_buf, (uint8_t *)phba->mbox_ext, pmbox->in_ext_byte_len); } @@ -8216,10 +9641,10 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, = MAILBOX_HBA_EXT_OFFSET; /* Copy the mailbox extension data */ - if (pmbox->in_ext_byte_len && pmbox->ctx_buf) + if (pmbox->in_ext_byte_len && pmbox->ext_buf) lpfc_memcpy_to_slim(phba->MBslimaddr + MAILBOX_HBA_EXT_OFFSET, - pmbox->ctx_buf, pmbox->in_ext_byte_len); + pmbox->ext_buf, pmbox->in_ext_byte_len); if (mbx->mbxCommand == MBX_CONFIG_PORT) /* copy command data into host mbox for cmpl */ @@ -8281,8 +9706,7 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, drvr_flag); goto out_not_finished; } - timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox) * - 1000) + jiffies; + timeout = secs_to_jiffies(lpfc_mbox_tmo_val(phba, pmbox)) + jiffies; i = 0; /* Wait for command to complete */ while (((word0 & OWN_CHIP) == OWN_CHIP) || @@ -8342,9 +9766,9 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, lpfc_sli_pcimem_bcopy(phba->mbox, mbx, MAILBOX_CMD_SIZE); /* Copy the mailbox extension data */ - if (pmbox->out_ext_byte_len && pmbox->ctx_buf) { + if (pmbox->out_ext_byte_len && pmbox->ext_buf) { lpfc_sli_pcimem_bcopy(phba->mbox_ext, - pmbox->ctx_buf, + pmbox->ext_buf, pmbox->out_ext_byte_len); } } else { @@ -8352,9 +9776,9 @@ lpfc_sli_issue_mbox_s3(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmbox, lpfc_memcpy_from_slim(mbx, phba->MBslimaddr, MAILBOX_CMD_SIZE); /* Copy the mailbox extension data */ - if (pmbox->out_ext_byte_len && pmbox->ctx_buf) { + if (pmbox->out_ext_byte_len && pmbox->ext_buf) { lpfc_memcpy_from_slim( - pmbox->ctx_buf, + pmbox->ext_buf, phba->MBslimaddr + MAILBOX_HBA_EXT_OFFSET, pmbox->out_ext_byte_len); @@ -8395,8 +9819,11 @@ static int lpfc_sli4_async_mbox_block(struct lpfc_hba *phba) { struct lpfc_sli *psli = &phba->sli; + LPFC_MBOXQ_t *mboxq; int rc = 0; unsigned long timeout = 0; + u32 sli_flag; + u8 cmd, subsys, opcode; /* Mark the asynchronous mailbox command posting as blocked */ spin_lock_irq(&phba->hbalock); @@ -8405,21 +9832,45 @@ lpfc_sli4_async_mbox_block(struct lpfc_hba *phba) * command to be gracefully completed by firmware. */ if (phba->sli.mbox_active) - timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, - phba->sli.mbox_active) * - 1000) + jiffies; + timeout = secs_to_jiffies(lpfc_mbox_tmo_val(phba, + phba->sli.mbox_active)) + jiffies; spin_unlock_irq(&phba->hbalock); /* Make sure the mailbox is really active */ if (timeout) lpfc_sli4_process_missed_mbox_completions(phba); - /* Wait for the outstnading mailbox command to complete */ + /* Wait for the outstanding mailbox command to complete */ while (phba->sli.mbox_active) { /* Check active mailbox complete status every 2ms */ msleep(2); if (time_after(jiffies, timeout)) { - /* Timeout, marked the outstanding cmd not complete */ + /* Timeout, mark the outstanding cmd not complete */ + + /* Sanity check sli.mbox_active has not completed or + * cancelled from another context during last 2ms sleep, + * so take hbalock to be sure before logging. + */ + spin_lock_irq(&phba->hbalock); + if (phba->sli.mbox_active) { + mboxq = phba->sli.mbox_active; + cmd = mboxq->u.mb.mbxCommand; + subsys = lpfc_sli_config_mbox_subsys_get(phba, + mboxq); + opcode = lpfc_sli_config_mbox_opcode_get(phba, + mboxq); + sli_flag = psli->sli_flag; + spin_unlock_irq(&phba->hbalock); + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "2352 Mailbox command x%x " + "(x%x/x%x) sli_flag x%x could " + "not complete\n", + cmd, subsys, opcode, + sli_flag); + } else { + spin_unlock_irq(&phba->hbalock); + } + rc = 1; break; } @@ -8465,7 +9916,7 @@ lpfc_sli4_async_mbox_unblock(struct lpfc_hba *phba) psli->sli_flag &= ~LPFC_SLI_ASYNC_MBX_BLK; spin_unlock_irq(&phba->hbalock); - /* wake up worker thread to post asynchronlous mailbox command */ + /* wake up worker thread to post asynchronous mailbox command */ lpfc_worker_wake_up(phba); } @@ -8478,7 +9929,8 @@ lpfc_sli4_async_mbox_unblock(struct lpfc_hba *phba) * port for twice the regular mailbox command timeout value. * * 0 - no timeout on waiting for bootstrap mailbox register ready. - * MBXERR_ERROR - wait for bootstrap mailbox register timed out. + * MBXERR_ERROR - wait for bootstrap mailbox register timed out or port + * is in an unrecoverable state. **/ static int lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) @@ -8486,15 +9938,31 @@ lpfc_sli4_wait_bmbx_ready(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) uint32_t db_ready; unsigned long timeout; struct lpfc_register bmbx_reg; + struct lpfc_register portstat_reg = {-1}; + + /* Sanity check - there is no point to wait if the port is in an + * unrecoverable state. + */ + if (bf_get(lpfc_sli_intf_if_type, &phba->sli4_hba.sli_intf) >= + LPFC_SLI_INTF_IF_TYPE_2) { + if (lpfc_readl(phba->sli4_hba.u.if_type2.STATUSregaddr, + &portstat_reg.word0) || + lpfc_sli4_unrecoverable_port(&portstat_reg)) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + "3858 Skipping bmbx ready because " + "Port Status x%x\n", + portstat_reg.word0); + return MBXERR_ERROR; + } + } - timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, mboxq) - * 1000) + jiffies; + timeout = secs_to_jiffies(lpfc_mbox_tmo_val(phba, mboxq)) + jiffies; do { bmbx_reg.word0 = readl(phba->sli4_hba.BMBXregaddr); db_ready = bf_get(lpfc_bmbx_rdy, &bmbx_reg); if (!db_ready) - msleep(2); + mdelay(2); if (time_after(jiffies, timeout)) return MBXERR_ERROR; @@ -8538,7 +10006,7 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) spin_lock_irqsave(&phba->hbalock, iflag); if (psli->sli_flag & LPFC_SLI_MBOX_ACTIVE) { spin_unlock_irqrestore(&phba->hbalock, iflag); - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2532 Mailbox command x%x (x%x/x%x) " "cannot issue Data: x%x x%x\n", mboxq->vport ? mboxq->vport->vpi : 0, @@ -8557,7 +10025,6 @@ lpfc_sli4_post_sync_mbox(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) rc = lpfc_sli4_wait_bmbx_ready(phba, mboxq); if (rc) goto exit; - /* * Initialize the bootstrap memory region to avoid stale data areas * in the mailbox post. Then copy the caller's mailbox contents to @@ -8638,7 +10105,7 @@ exit: /** * lpfc_sli_issue_mbox_s4 - Issue an SLI4 mailbox command to firmware * @phba: Pointer to HBA context object. - * @pmbox: Pointer to mailbox object. + * @mboxq: Pointer to mailbox object. * @flag: Flag indicating how the mailbox need to be processed. * * This function is called by discovery code and HBA management code to submit @@ -8660,7 +10127,7 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, rc = lpfc_mbox_dev_check(phba); if (unlikely(rc)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2544 Mailbox command x%x (x%x/x%x) " "cannot issue Data: x%x x%x\n", mboxq->vport ? mboxq->vport->vpi : 0, @@ -8682,7 +10149,7 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, "(%d):2541 Mailbox command x%x " "(x%x/x%x) failure: " "mqe_sta: x%x mcqe_sta: x%x/x%x " - "Data: x%x x%x\n,", + "Data: x%x x%x\n", mboxq->vport ? mboxq->vport->vpi : 0, mboxq->u.mb.mbxCommand, lpfc_sli_config_mbox_subsys_get(phba, @@ -8716,7 +10183,7 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, "(%d):2597 Sync Mailbox command " "x%x (x%x/x%x) failure: " "mqe_sta: x%x mcqe_sta: x%x/x%x " - "Data: x%x x%x\n,", + "Data: x%x x%x\n", mboxq->vport ? mboxq->vport->vpi : 0, mboxq->u.mb.mbxCommand, lpfc_sli_config_mbox_subsys_get(phba, @@ -8734,10 +10201,10 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, return rc; } - /* Now, interrupt mode asynchrous mailbox command */ + /* Now, interrupt mode asynchronous mailbox command */ rc = lpfc_mbox_cmd_check(phba, mboxq); if (rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2543 Mailbox command x%x (x%x/x%x) " "cannot issue Data: x%x x%x\n", mboxq->vport ? mboxq->vport->vpi : 0, @@ -8755,11 +10222,12 @@ lpfc_sli_issue_mbox_s4(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq, spin_unlock_irqrestore(&phba->hbalock, iflags); lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI, "(%d):0354 Mbox cmd issue - Enqueue Data: " - "x%x (x%x/x%x) x%x x%x x%x\n", + "x%x (x%x/x%x) x%x x%x x%x x%x\n", mboxq->vport ? mboxq->vport->vpi : 0xffffff, bf_get(lpfc_mqe_command, &mboxq->u.mqe), lpfc_sli_config_mbox_subsys_get(phba, mboxq), lpfc_sli_config_mbox_opcode_get(phba, mboxq), + mboxq->u.mb.un.varUnregLogin.rpi, phba->pport->port_state, psli->sli_flag, MBX_NOWAIT); /* Wake up worker thread to transport mailbox command from head */ @@ -8805,7 +10273,7 @@ lpfc_sli4_post_async_mbox(struct lpfc_hba *phba) } if (unlikely(phba->sli.mbox_active)) { spin_unlock_irqrestore(&phba->hbalock, iflags); - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0384 There is pending active mailbox cmd\n"); return MBX_NOT_FINISHED; } @@ -8836,7 +10304,7 @@ lpfc_sli4_post_async_mbox(struct lpfc_hba *phba) /* Start timer for the mbox_tmo and log some mailbox post messages */ mod_timer(&psli->mbox_tmo, (jiffies + - msecs_to_jiffies(1000 * lpfc_mbox_tmo_val(phba, mboxq)))); + secs_to_jiffies(lpfc_mbox_tmo_val(phba, mboxq)))); lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI, "(%d):0355 Mailbox cmd x%x (x%x/x%x) issue Data: " @@ -8866,7 +10334,7 @@ lpfc_sli4_post_async_mbox(struct lpfc_hba *phba) /* Post the mailbox command to the port */ rc = lpfc_sli4_mq_put(phba->sli4_hba.mbx_wq, mqe); if (rc != MBX_SUCCESS) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):2533 Mailbox command x%x (x%x/x%x) " "cannot issue Data: x%x x%x\n", mboxq->vport ? mboxq->vport->vpi : 0, @@ -8946,7 +10414,6 @@ lpfc_mbox_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) "1420 Invalid HBA PCI-device group: 0x%x\n", dev_grp); return -ENODEV; - break; } return 0; } @@ -8957,7 +10424,8 @@ lpfc_mbox_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) * @pring: Pointer to driver SLI ring object. * @piocb: Pointer to address of newly added command iocb. * - * This function is called with hbalock held to add a command + * This function is called with hbalock held for SLI3 ports or + * the ring lock held for SLI4 ports to add a command * iocb to the txq when SLI layer cannot submit the command iocb * to the ring. **/ @@ -8965,7 +10433,10 @@ void __lpfc_sli_ringtx_put(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, struct lpfc_iocbq *piocb) { - lockdep_assert_held(&phba->hbalock); + if (phba->sli_rev == LPFC_SLI_REV4) + lockdep_assert_held(&pring->ring_lock); + else + lockdep_assert_held(&phba->hbalock); /* Insert the caller's iocb in the txq tail for later processing. */ list_add_tail(&piocb->list, &pring->txq); } @@ -9036,11 +10507,10 @@ __lpfc_sli_issue_iocb_s3(struct lpfc_hba *phba, uint32_t ring_number, lockdep_assert_held(&phba->hbalock); - if (piocb->iocb_cmpl && (!piocb->vport) && + if (piocb->cmd_cmpl && (!piocb->vport) && (piocb->iocb.ulpCommand != CMD_ABORT_XRI_CN) && (piocb->iocb.ulpCommand != CMD_CLOSE_XRI_CN)) { - lpfc_printf_log(phba, KERN_ERR, - LOG_SLI | LOG_VPORT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "1807 IOCB x%x failed. No vport\n", piocb->iocb.ulpCommand); dump_stack(); @@ -9053,7 +10523,7 @@ __lpfc_sli_issue_iocb_s3(struct lpfc_hba *phba, uint32_t ring_number, return IOCB_ERROR; /* If HBA has a deferred error attention, fail the iocb. */ - if (unlikely(phba->hba_flag & DEFER_ERATT)) + if (unlikely(test_bit(DEFER_ERATT, &phba->hba_flag))) return IOCB_ERROR; /* @@ -9075,25 +10545,15 @@ __lpfc_sli_issue_iocb_s3(struct lpfc_hba *phba, uint32_t ring_number, * can be issued if the link is not up. */ switch (piocb->iocb.ulpCommand) { - case CMD_GEN_REQUEST64_CR: - case CMD_GEN_REQUEST64_CX: - if (!(phba->sli.sli_flag & LPFC_MENLO_MAINT) || - (piocb->iocb.un.genreq64.w5.hcsw.Rctl != - FC_RCTL_DD_UNSOL_CMD) || - (piocb->iocb.un.genreq64.w5.hcsw.Type != - MENLO_TRANSPORT_TYPE)) - - goto iocb_busy; - break; case CMD_QUE_RING_BUF_CN: case CMD_QUE_RING_BUF64_CN: /* * For IOCBs, like QUE_RING_BUF, that have no rsp ring - * completion, iocb_cmpl MUST be 0. + * completion, cmd_cmpl MUST be 0. */ - if (piocb->iocb_cmpl) - piocb->iocb_cmpl = NULL; - /*FALLTHROUGH*/ + if (piocb->cmd_cmpl) + piocb->cmd_cmpl = NULL; + fallthrough; case CMD_CREATE_XRI_CR: case CMD_CLOSE_XRI_CN: case CMD_CLOSE_XRI_CX: @@ -9139,703 +10599,111 @@ __lpfc_sli_issue_iocb_s3(struct lpfc_hba *phba, uint32_t ring_number, } /** - * lpfc_sli4_bpl2sgl - Convert the bpl/bde to a sgl. + * __lpfc_sli_issue_fcp_io_s3 - SLI3 device for sending fcp io iocb * @phba: Pointer to HBA context object. + * @ring_number: SLI ring number to issue wqe on. * @piocb: Pointer to command iocb. - * @sglq: Pointer to the scatter gather queue object. + * @flag: Flag indicating if this command can be put into txq. * - * This routine converts the bpl or bde that is in the IOCB - * to a sgl list for the sli4 hardware. The physical address - * of the bpl/bde is converted back to a virtual address. - * If the IOCB contains a BPL then the list of BDE's is - * converted to sli4_sge's. If the IOCB contains a single - * BDE then it is converted to a single sli_sge. - * The IOCB is still in cpu endianess so the contents of - * the bpl can be used without byte swapping. + * __lpfc_sli_issue_fcp_io_s3 is wrapper function to invoke lockless func to + * send an iocb command to an HBA with SLI-3 interface spec. * - * Returns valid XRI = Success, NO_XRI = Failure. -**/ -static uint16_t -lpfc_sli4_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq, - struct lpfc_sglq *sglq) + * This function takes the hbalock before invoking the lockless version. + * The function will return success after it successfully submit the wqe to + * firmware or after adding to the txq. + **/ +static int +__lpfc_sli_issue_fcp_io_s3(struct lpfc_hba *phba, uint32_t ring_number, + struct lpfc_iocbq *piocb, uint32_t flag) { - uint16_t xritag = NO_XRI; - struct ulp_bde64 *bpl = NULL; - struct ulp_bde64 bde; - struct sli4_sge *sgl = NULL; - struct lpfc_dmabuf *dmabuf; - IOCB_t *icmd; - int numBdes = 0; - int i = 0; - uint32_t offset = 0; /* accumulated offset in the sg request list */ - int inbound = 0; /* number of sg reply entries inbound from firmware */ - - if (!piocbq || !sglq) - return xritag; - - sgl = (struct sli4_sge *)sglq->sgl; - icmd = &piocbq->iocb; - if (icmd->ulpCommand == CMD_XMIT_BLS_RSP64_CX) - return sglq->sli4_xritag; - if (icmd->un.genreq64.bdl.bdeFlags == BUFF_TYPE_BLP_64) { - numBdes = icmd->un.genreq64.bdl.bdeSize / - sizeof(struct ulp_bde64); - /* The addrHigh and addrLow fields within the IOCB - * have not been byteswapped yet so there is no - * need to swap them back. - */ - if (piocbq->context3) - dmabuf = (struct lpfc_dmabuf *)piocbq->context3; - else - return xritag; - - bpl = (struct ulp_bde64 *)dmabuf->virt; - if (!bpl) - return xritag; + unsigned long iflags; + int rc; - for (i = 0; i < numBdes; i++) { - /* Should already be byte swapped. */ - sgl->addr_hi = bpl->addrHigh; - sgl->addr_lo = bpl->addrLow; + spin_lock_irqsave(&phba->hbalock, iflags); + rc = __lpfc_sli_issue_iocb_s3(phba, ring_number, piocb, flag); + spin_unlock_irqrestore(&phba->hbalock, iflags); - sgl->word2 = le32_to_cpu(sgl->word2); - if ((i+1) == numBdes) - bf_set(lpfc_sli4_sge_last, sgl, 1); - else - bf_set(lpfc_sli4_sge_last, sgl, 0); - /* swap the size field back to the cpu so we - * can assign it to the sgl. - */ - bde.tus.w = le32_to_cpu(bpl->tus.w); - sgl->sge_len = cpu_to_le32(bde.tus.f.bdeSize); - /* The offsets in the sgl need to be accumulated - * separately for the request and reply lists. - * The request is always first, the reply follows. - */ - if (piocbq->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) { - /* add up the reply sg entries */ - if (bpl->tus.f.bdeFlags == BUFF_TYPE_BDE_64I) - inbound++; - /* first inbound? reset the offset */ - if (inbound == 1) - offset = 0; - bf_set(lpfc_sli4_sge_offset, sgl, offset); - bf_set(lpfc_sli4_sge_type, sgl, - LPFC_SGE_TYPE_DATA); - offset += bde.tus.f.bdeSize; - } - sgl->word2 = cpu_to_le32(sgl->word2); - bpl++; - sgl++; - } - } else if (icmd->un.genreq64.bdl.bdeFlags == BUFF_TYPE_BDE_64) { - /* The addrHigh and addrLow fields of the BDE have not - * been byteswapped yet so they need to be swapped - * before putting them in the sgl. - */ - sgl->addr_hi = - cpu_to_le32(icmd->un.genreq64.bdl.addrHigh); - sgl->addr_lo = - cpu_to_le32(icmd->un.genreq64.bdl.addrLow); - sgl->word2 = le32_to_cpu(sgl->word2); - bf_set(lpfc_sli4_sge_last, sgl, 1); - sgl->word2 = cpu_to_le32(sgl->word2); - sgl->sge_len = - cpu_to_le32(icmd->un.genreq64.bdl.bdeSize); - } - return sglq->sli4_xritag; + return rc; } /** - * lpfc_sli_iocb2wqe - Convert the IOCB to a work queue entry. + * __lpfc_sli_issue_fcp_io_s4 - SLI4 device for sending fcp io wqe * @phba: Pointer to HBA context object. + * @ring_number: SLI ring number to issue wqe on. * @piocb: Pointer to command iocb. - * @wqe: Pointer to the work queue entry. + * @flag: Flag indicating if this command can be put into txq. * - * This routine converts the iocb command to its Work Queue Entry - * equivalent. The wqe pointer should not have any fields set when - * this routine is called because it will memcpy over them. - * This routine does not set the CQ_ID or the WQEC bits in the - * wqe. + * __lpfc_sli_issue_fcp_io_s4 is used by other functions in the driver to issue + * an wqe command to an HBA with SLI-4 interface spec. * - * Returns: 0 = Success, IOCB_ERROR = Failure. + * This function is a lockless version. The function will return success + * after it successfully submit the wqe to firmware or after adding to the + * txq. **/ static int -lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, - union lpfc_wqe128 *wqe) -{ - uint32_t xmit_len = 0, total_len = 0; - uint8_t ct = 0; - uint32_t fip; - uint32_t abort_tag; - uint8_t command_type = ELS_COMMAND_NON_FIP; - uint8_t cmnd; - uint16_t xritag; - uint16_t abrt_iotag; - struct lpfc_iocbq *abrtiocbq; - struct ulp_bde64 *bpl = NULL; - uint32_t els_id = LPFC_ELS_ID_DEFAULT; - int numBdes, i; - struct ulp_bde64 bde; - struct lpfc_nodelist *ndlp; - uint32_t *pcmd; - uint32_t if_type; - - fip = phba->hba_flag & HBA_FIP_SUPPORT; - /* The fcp commands will set command type */ - if (iocbq->iocb_flag & LPFC_IO_FCP) - command_type = FCP_COMMAND; - else if (fip && (iocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK)) - command_type = ELS_COMMAND_FIP; - else - command_type = ELS_COMMAND_NON_FIP; +__lpfc_sli_issue_fcp_io_s4(struct lpfc_hba *phba, uint32_t ring_number, + struct lpfc_iocbq *piocb, uint32_t flag) +{ + struct lpfc_io_buf *lpfc_cmd = piocb->io_buf; - if (phba->fcp_embed_io) - memset(wqe, 0, sizeof(union lpfc_wqe128)); - /* Some of the fields are in the right position already */ - memcpy(wqe, &iocbq->iocb, sizeof(union lpfc_wqe)); - if (iocbq->iocb.ulpCommand != CMD_SEND_FRAME) { - /* The ct field has moved so reset */ - wqe->generic.wqe_com.word7 = 0; - wqe->generic.wqe_com.word10 = 0; - } - - abort_tag = (uint32_t) iocbq->iotag; - xritag = iocbq->sli4_xritag; - /* words0-2 bpl convert bde */ - if (iocbq->iocb.un.genreq64.bdl.bdeFlags == BUFF_TYPE_BLP_64) { - numBdes = iocbq->iocb.un.genreq64.bdl.bdeSize / - sizeof(struct ulp_bde64); - bpl = (struct ulp_bde64 *) - ((struct lpfc_dmabuf *)iocbq->context3)->virt; - if (!bpl) - return IOCB_ERROR; + lpfc_prep_embed_io(phba, lpfc_cmd); + return lpfc_sli4_issue_wqe(phba, lpfc_cmd->hdwq, piocb); +} - /* Should already be byte swapped. */ - wqe->generic.bde.addrHigh = le32_to_cpu(bpl->addrHigh); - wqe->generic.bde.addrLow = le32_to_cpu(bpl->addrLow); - /* swap the size field back to the cpu so we - * can assign it to the sgl. - */ - wqe->generic.bde.tus.w = le32_to_cpu(bpl->tus.w); - xmit_len = wqe->generic.bde.tus.f.bdeSize; - total_len = 0; - for (i = 0; i < numBdes; i++) { - bde.tus.w = le32_to_cpu(bpl[i].tus.w); - total_len += bde.tus.f.bdeSize; - } - } else - xmit_len = iocbq->iocb.un.fcpi64.bdl.bdeSize; +void +lpfc_prep_embed_io(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_cmd) +{ + struct lpfc_iocbq *piocb = &lpfc_cmd->cur_iocbq; + union lpfc_wqe128 *wqe = &lpfc_cmd->cur_iocbq.wqe; + struct sli4_sge_le *sgl; + u32 type_size; - iocbq->iocb.ulpIoTag = iocbq->iotag; - cmnd = iocbq->iocb.ulpCommand; + /* 128 byte wqe support here */ + sgl = (struct sli4_sge_le *)lpfc_cmd->dma_sgl; - switch (iocbq->iocb.ulpCommand) { - case CMD_ELS_REQUEST64_CR: - if (iocbq->iocb_flag & LPFC_IO_LIBDFC) - ndlp = iocbq->context_un.ndlp; - else - ndlp = (struct lpfc_nodelist *)iocbq->context1; - if (!iocbq->iocb.ulpLe) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "2007 Only Limited Edition cmd Format" - " supported 0x%x\n", - iocbq->iocb.ulpCommand); - return IOCB_ERROR; - } - - wqe->els_req.payload_len = xmit_len; - /* Els_reguest64 has a TMO */ - bf_set(wqe_tmo, &wqe->els_req.wqe_com, - iocbq->iocb.ulpTimeout); - /* Need a VF for word 4 set the vf bit*/ - bf_set(els_req64_vf, &wqe->els_req, 0); - /* And a VFID for word 12 */ - bf_set(els_req64_vfid, &wqe->els_req, 0); - ct = ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l); - bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, - iocbq->iocb.ulpContext); - bf_set(wqe_ct, &wqe->els_req.wqe_com, ct); - bf_set(wqe_pu, &wqe->els_req.wqe_com, 0); - /* CCP CCPE PV PRI in word10 were set in the memcpy */ - if (command_type == ELS_COMMAND_FIP) - els_id = ((iocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK) - >> LPFC_FIP_ELS_ID_SHIFT); - pcmd = (uint32_t *) (((struct lpfc_dmabuf *) - iocbq->context2)->virt); - if_type = bf_get(lpfc_sli_intf_if_type, - &phba->sli4_hba.sli_intf); - if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) { - if (pcmd && (*pcmd == ELS_CMD_FLOGI || - *pcmd == ELS_CMD_SCR || - *pcmd == ELS_CMD_FDISC || - *pcmd == ELS_CMD_LOGO || - *pcmd == ELS_CMD_PLOGI)) { - bf_set(els_req64_sp, &wqe->els_req, 1); - bf_set(els_req64_sid, &wqe->els_req, - iocbq->vport->fc_myDID); - if ((*pcmd == ELS_CMD_FLOGI) && - !(phba->fc_topology == - LPFC_TOPOLOGY_LOOP)) - bf_set(els_req64_sid, &wqe->els_req, 0); - bf_set(wqe_ct, &wqe->els_req.wqe_com, 1); - bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, - phba->vpi_ids[iocbq->vport->vpi]); - } else if (pcmd && iocbq->context1) { - bf_set(wqe_ct, &wqe->els_req.wqe_com, 0); - bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, - phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); - } - } - bf_set(wqe_temp_rpi, &wqe->els_req.wqe_com, - phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); - bf_set(wqe_els_id, &wqe->els_req.wqe_com, els_id); - bf_set(wqe_dbde, &wqe->els_req.wqe_com, 1); - bf_set(wqe_iod, &wqe->els_req.wqe_com, LPFC_WQE_IOD_READ); - bf_set(wqe_qosd, &wqe->els_req.wqe_com, 1); - bf_set(wqe_lenloc, &wqe->els_req.wqe_com, LPFC_WQE_LENLOC_NONE); - bf_set(wqe_ebde_cnt, &wqe->els_req.wqe_com, 0); - wqe->els_req.max_response_payload_len = total_len - xmit_len; - break; - case CMD_XMIT_SEQUENCE64_CX: - bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, - iocbq->iocb.un.ulpWord[3]); - bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, - iocbq->iocb.unsli3.rcvsli3.ox_id); - /* The entire sequence is transmitted for this IOCB */ - xmit_len = total_len; - cmnd = CMD_XMIT_SEQUENCE64_CR; - if (phba->link_flag & LS_LOOPBACK_MODE) - bf_set(wqe_xo, &wqe->xmit_sequence.wge_ctl, 1); - /* fall through */ - case CMD_XMIT_SEQUENCE64_CR: - /* word3 iocb=io_tag32 wqe=reserved */ - wqe->xmit_sequence.rsvd3 = 0; - /* word4 relative_offset memcpy */ - /* word5 r_ctl/df_ctl memcpy */ - bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0); - bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1); - bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, - LPFC_WQE_IOD_WRITE); - bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com, - LPFC_WQE_LENLOC_WORD12); - bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0); - wqe->xmit_sequence.xmit_len = xmit_len; - command_type = OTHER_COMMAND; - break; - case CMD_XMIT_BCAST64_CN: - /* word3 iocb=iotag32 wqe=seq_payload_len */ - wqe->xmit_bcast64.seq_payload_len = xmit_len; - /* word4 iocb=rsvd wqe=rsvd */ - /* word5 iocb=rctl/type/df_ctl wqe=rctl/type/df_ctl memcpy */ - /* word6 iocb=ctxt_tag/io_tag wqe=ctxt_tag/xri */ - bf_set(wqe_ct, &wqe->xmit_bcast64.wqe_com, - ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l)); - bf_set(wqe_dbde, &wqe->xmit_bcast64.wqe_com, 1); - bf_set(wqe_iod, &wqe->xmit_bcast64.wqe_com, LPFC_WQE_IOD_WRITE); - bf_set(wqe_lenloc, &wqe->xmit_bcast64.wqe_com, - LPFC_WQE_LENLOC_WORD3); - bf_set(wqe_ebde_cnt, &wqe->xmit_bcast64.wqe_com, 0); - break; - case CMD_FCP_IWRITE64_CR: - command_type = FCP_COMMAND_DATA_OUT; - /* word3 iocb=iotag wqe=payload_offset_len */ - /* Add the FCP_CMD and FCP_RSP sizes to get the offset */ - bf_set(payload_offset_len, &wqe->fcp_iwrite, - xmit_len + sizeof(struct fcp_rsp)); - bf_set(cmd_buff_len, &wqe->fcp_iwrite, - 0); - /* word4 iocb=parameter wqe=total_xfer_length memcpy */ - /* word5 iocb=initial_xfer_len wqe=initial_xfer_len memcpy */ - bf_set(wqe_erp, &wqe->fcp_iwrite.wqe_com, - iocbq->iocb.ulpFCP2Rcvy); - bf_set(wqe_lnk, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpXS); - /* Always open the exchange */ - bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); - bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, - LPFC_WQE_LENLOC_WORD4); - bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, iocbq->iocb.ulpPU); - bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 1); - if (iocbq->iocb_flag & LPFC_IO_OAS) { - bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1); - bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1); - if (iocbq->priority) { - bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com, - (iocbq->priority << 1)); - } else { - bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com, - (phba->cfg_XLanePriority << 1)); - } - } - /* Note, word 10 is already initialized to 0 */ + if (phba->fcp_embed_io) { + struct fcp_cmnd *fcp_cmnd; + u32 *ptr; - /* Don't set PBDE for Perf hints, just lpfc_enable_pbde */ - if (phba->cfg_enable_pbde) - bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); - else - bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 0); + fcp_cmnd = lpfc_cmd->fcp_cmnd; - if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; - struct sli4_sge *sgl; - struct fcp_cmnd *fcp_cmnd; - uint32_t *ptr; + /* Word 0-2 - FCP_CMND */ + type_size = le32_to_cpu(sgl->sge_len); + type_size |= ULP_BDE64_TYPE_BDE_IMMED; + wqe->generic.bde.tus.w = type_size; + wqe->generic.bde.addrHigh = 0; + wqe->generic.bde.addrLow = 72; /* Word 18 */ - /* 128 byte wqe support here */ + bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); + bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); - lpfc_cmd = iocbq->context1; - sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; - fcp_cmnd = lpfc_cmd->fcp_cmnd; + /* Word 18-29 FCP CMND Payload */ + ptr = &wqe->words[18]; + lpfc_sli_pcimem_bcopy(fcp_cmnd, ptr, le32_to_cpu(sgl->sge_len)); + } else { + /* Word 0-2 - Inline BDE */ + wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; + wqe->generic.bde.tus.f.bdeSize = le32_to_cpu(sgl->sge_len); + wqe->generic.bde.addrHigh = le32_to_cpu(sgl->addr_hi); + wqe->generic.bde.addrLow = le32_to_cpu(sgl->addr_lo); - /* Word 0-2 - FCP_CMND */ - wqe->generic.bde.tus.f.bdeFlags = - BUFF_TYPE_BDE_IMMED; - wqe->generic.bde.tus.f.bdeSize = sgl->sge_len; - wqe->generic.bde.addrHigh = 0; - wqe->generic.bde.addrLow = 88; /* Word 22 */ + /* Word 10 */ + bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); + bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); + } + /* add the VMID tags as per switch response */ + if (unlikely(piocb->cmd_flag & LPFC_IO_VMID)) { + if (phba->pport->vmid_flag & LPFC_VMID_TYPE_PRIO) { + bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1); + bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com, + (piocb->vmid_tag.cs_ctl_vmid)); + } else if (phba->cfg_vmid_app_header) { + bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1); bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); - bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); - - /* Word 22-29 FCP CMND Payload */ - ptr = &wqe->words[22]; - memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd)); - } - break; - case CMD_FCP_IREAD64_CR: - /* word3 iocb=iotag wqe=payload_offset_len */ - /* Add the FCP_CMD and FCP_RSP sizes to get the offset */ - bf_set(payload_offset_len, &wqe->fcp_iread, - xmit_len + sizeof(struct fcp_rsp)); - bf_set(cmd_buff_len, &wqe->fcp_iread, - 0); - /* word4 iocb=parameter wqe=total_xfer_length memcpy */ - /* word5 iocb=initial_xfer_len wqe=initial_xfer_len memcpy */ - bf_set(wqe_erp, &wqe->fcp_iread.wqe_com, - iocbq->iocb.ulpFCP2Rcvy); - bf_set(wqe_lnk, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpXS); - /* Always open the exchange */ - bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); - bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, - LPFC_WQE_LENLOC_WORD4); - bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, iocbq->iocb.ulpPU); - bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 1); - if (iocbq->iocb_flag & LPFC_IO_OAS) { - bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1); - bf_set(wqe_ccpe, &wqe->fcp_iread.wqe_com, 1); - if (iocbq->priority) { - bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com, - (iocbq->priority << 1)); - } else { - bf_set(wqe_ccp, &wqe->fcp_iread.wqe_com, - (phba->cfg_XLanePriority << 1)); - } - } - /* Note, word 10 is already initialized to 0 */ - - /* Don't set PBDE for Perf hints, just lpfc_enable_pbde */ - if (phba->cfg_enable_pbde) - bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); - else - bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 0); - - if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; - struct sli4_sge *sgl; - struct fcp_cmnd *fcp_cmnd; - uint32_t *ptr; - - /* 128 byte wqe support here */ - - lpfc_cmd = iocbq->context1; - sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; - fcp_cmnd = lpfc_cmd->fcp_cmnd; - - /* Word 0-2 - FCP_CMND */ - wqe->generic.bde.tus.f.bdeFlags = - BUFF_TYPE_BDE_IMMED; - wqe->generic.bde.tus.f.bdeSize = sgl->sge_len; - wqe->generic.bde.addrHigh = 0; - wqe->generic.bde.addrLow = 88; /* Word 22 */ - - bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); - bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); - - /* Word 22-29 FCP CMND Payload */ - ptr = &wqe->words[22]; - memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd)); + wqe->words[31] = piocb->vmid_tag.app_id; } - break; - case CMD_FCP_ICMND64_CR: - /* word3 iocb=iotag wqe=payload_offset_len */ - /* Add the FCP_CMD and FCP_RSP sizes to get the offset */ - bf_set(payload_offset_len, &wqe->fcp_icmd, - xmit_len + sizeof(struct fcp_rsp)); - bf_set(cmd_buff_len, &wqe->fcp_icmd, - 0); - /* word3 iocb=IO_TAG wqe=reserved */ - bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); - /* Always open the exchange */ - bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 1); - bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE); - bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); - bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, - LPFC_WQE_LENLOC_NONE); - bf_set(wqe_erp, &wqe->fcp_icmd.wqe_com, - iocbq->iocb.ulpFCP2Rcvy); - if (iocbq->iocb_flag & LPFC_IO_OAS) { - bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1); - bf_set(wqe_ccpe, &wqe->fcp_icmd.wqe_com, 1); - if (iocbq->priority) { - bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com, - (iocbq->priority << 1)); - } else { - bf_set(wqe_ccp, &wqe->fcp_icmd.wqe_com, - (phba->cfg_XLanePriority << 1)); - } - } - /* Note, word 10 is already initialized to 0 */ - - if (phba->fcp_embed_io) { - struct lpfc_scsi_buf *lpfc_cmd; - struct sli4_sge *sgl; - struct fcp_cmnd *fcp_cmnd; - uint32_t *ptr; - - /* 128 byte wqe support here */ - - lpfc_cmd = iocbq->context1; - sgl = (struct sli4_sge *)lpfc_cmd->fcp_bpl; - fcp_cmnd = lpfc_cmd->fcp_cmnd; - - /* Word 0-2 - FCP_CMND */ - wqe->generic.bde.tus.f.bdeFlags = - BUFF_TYPE_BDE_IMMED; - wqe->generic.bde.tus.f.bdeSize = sgl->sge_len; - wqe->generic.bde.addrHigh = 0; - wqe->generic.bde.addrLow = 88; /* Word 22 */ - - bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); - bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); - - /* Word 22-29 FCP CMND Payload */ - ptr = &wqe->words[22]; - memcpy(ptr, fcp_cmnd, sizeof(struct fcp_cmnd)); - } - break; - case CMD_GEN_REQUEST64_CR: - /* For this command calculate the xmit length of the - * request bde. - */ - xmit_len = 0; - numBdes = iocbq->iocb.un.genreq64.bdl.bdeSize / - sizeof(struct ulp_bde64); - for (i = 0; i < numBdes; i++) { - bde.tus.w = le32_to_cpu(bpl[i].tus.w); - if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) - break; - xmit_len += bde.tus.f.bdeSize; - } - /* word3 iocb=IO_TAG wqe=request_payload_len */ - wqe->gen_req.request_payload_len = xmit_len; - /* word4 iocb=parameter wqe=relative_offset memcpy */ - /* word5 [rctl, type, df_ctl, la] copied in memcpy */ - /* word6 context tag copied in memcpy */ - if (iocbq->iocb.ulpCt_h || iocbq->iocb.ulpCt_l) { - ct = ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l); - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "2015 Invalid CT %x command 0x%x\n", - ct, iocbq->iocb.ulpCommand); - return IOCB_ERROR; - } - bf_set(wqe_ct, &wqe->gen_req.wqe_com, 0); - bf_set(wqe_tmo, &wqe->gen_req.wqe_com, iocbq->iocb.ulpTimeout); - bf_set(wqe_pu, &wqe->gen_req.wqe_com, iocbq->iocb.ulpPU); - bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); - bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); - bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); - bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); - bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); - wqe->gen_req.max_response_payload_len = total_len - xmit_len; - command_type = OTHER_COMMAND; - break; - case CMD_XMIT_ELS_RSP64_CX: - ndlp = (struct lpfc_nodelist *)iocbq->context1; - /* words0-2 BDE memcpy */ - /* word3 iocb=iotag32 wqe=response_payload_len */ - wqe->xmit_els_rsp.response_payload_len = xmit_len; - /* word4 */ - wqe->xmit_els_rsp.word4 = 0; - /* word5 iocb=rsvd wge=did */ - bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest, - iocbq->iocb.un.xseq64.xmit_els_remoteID); - - if_type = bf_get(lpfc_sli_intf_if_type, - &phba->sli4_hba.sli_intf); - if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) { - if (iocbq->vport->fc_flag & FC_PT2PT) { - bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1); - bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, - iocbq->vport->fc_myDID); - if (iocbq->vport->fc_myDID == Fabric_DID) { - bf_set(wqe_els_did, - &wqe->xmit_els_rsp.wqe_dest, 0); - } - } - } - bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, - ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l)); - bf_set(wqe_pu, &wqe->xmit_els_rsp.wqe_com, iocbq->iocb.ulpPU); - bf_set(wqe_rcvoxid, &wqe->xmit_els_rsp.wqe_com, - iocbq->iocb.unsli3.rcvsli3.ox_id); - if (!iocbq->iocb.ulpCt_h && iocbq->iocb.ulpCt_l) - bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com, - phba->vpi_ids[iocbq->vport->vpi]); - bf_set(wqe_dbde, &wqe->xmit_els_rsp.wqe_com, 1); - bf_set(wqe_iod, &wqe->xmit_els_rsp.wqe_com, LPFC_WQE_IOD_WRITE); - bf_set(wqe_qosd, &wqe->xmit_els_rsp.wqe_com, 1); - bf_set(wqe_lenloc, &wqe->xmit_els_rsp.wqe_com, - LPFC_WQE_LENLOC_WORD3); - bf_set(wqe_ebde_cnt, &wqe->xmit_els_rsp.wqe_com, 0); - bf_set(wqe_rsp_temp_rpi, &wqe->xmit_els_rsp, - phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); - pcmd = (uint32_t *) (((struct lpfc_dmabuf *) - iocbq->context2)->virt); - if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) { - bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1); - bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, - iocbq->vport->fc_myDID); - bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, 1); - bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com, - phba->vpi_ids[phba->pport->vpi]); - } - command_type = OTHER_COMMAND; - break; - case CMD_CLOSE_XRI_CN: - case CMD_ABORT_XRI_CN: - case CMD_ABORT_XRI_CX: - /* words 0-2 memcpy should be 0 rserved */ - /* port will send abts */ - abrt_iotag = iocbq->iocb.un.acxri.abortContextTag; - if (abrt_iotag != 0 && abrt_iotag <= phba->sli.last_iotag) { - abrtiocbq = phba->sli.iocbq_lookup[abrt_iotag]; - fip = abrtiocbq->iocb_flag & LPFC_FIP_ELS_ID_MASK; - } else - fip = 0; - - if ((iocbq->iocb.ulpCommand == CMD_CLOSE_XRI_CN) || fip) - /* - * The link is down, or the command was ELS_FIP - * so the fw does not need to send abts - * on the wire. - */ - bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); - else - bf_set(abort_cmd_ia, &wqe->abort_cmd, 0); - bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); - /* word5 iocb=CONTEXT_TAG|IO_TAG wqe=reserved */ - wqe->abort_cmd.rsrvd5 = 0; - bf_set(wqe_ct, &wqe->abort_cmd.wqe_com, - ((iocbq->iocb.ulpCt_h << 1) | iocbq->iocb.ulpCt_l)); - abort_tag = iocbq->iocb.un.acxri.abortIoTag; - /* - * The abort handler will send us CMD_ABORT_XRI_CN or - * CMD_CLOSE_XRI_CN and the fw only accepts CMD_ABORT_XRI_CX - */ - bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); - bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); - bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, - LPFC_WQE_LENLOC_NONE); - cmnd = CMD_ABORT_XRI_CX; - command_type = OTHER_COMMAND; - xritag = 0; - break; - case CMD_XMIT_BLS_RSP64_CX: - ndlp = (struct lpfc_nodelist *)iocbq->context1; - /* As BLS ABTS RSP WQE is very different from other WQEs, - * we re-construct this WQE here based on information in - * iocbq from scratch. - */ - memset(wqe, 0, sizeof(union lpfc_wqe)); - /* OX_ID is invariable to who sent ABTS to CT exchange */ - bf_set(xmit_bls_rsp64_oxid, &wqe->xmit_bls_rsp, - bf_get(lpfc_abts_oxid, &iocbq->iocb.un.bls_rsp)); - if (bf_get(lpfc_abts_orig, &iocbq->iocb.un.bls_rsp) == - LPFC_ABTS_UNSOL_INT) { - /* ABTS sent by initiator to CT exchange, the - * RX_ID field will be filled with the newly - * allocated responder XRI. - */ - bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp, - iocbq->sli4_xritag); - } else { - /* ABTS sent by responder to CT exchange, the - * RX_ID field will be filled with the responder - * RX_ID from ABTS. - */ - bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp, - bf_get(lpfc_abts_rxid, &iocbq->iocb.un.bls_rsp)); - } - bf_set(xmit_bls_rsp64_seqcnthi, &wqe->xmit_bls_rsp, 0xffff); - bf_set(wqe_xmit_bls_pt, &wqe->xmit_bls_rsp.wqe_dest, 0x1); - - /* Use CT=VPI */ - bf_set(wqe_els_did, &wqe->xmit_bls_rsp.wqe_dest, - ndlp->nlp_DID); - bf_set(xmit_bls_rsp64_temprpi, &wqe->xmit_bls_rsp, - iocbq->iocb.ulpContext); - bf_set(wqe_ct, &wqe->xmit_bls_rsp.wqe_com, 1); - bf_set(wqe_ctxt_tag, &wqe->xmit_bls_rsp.wqe_com, - phba->vpi_ids[phba->pport->vpi]); - bf_set(wqe_qosd, &wqe->xmit_bls_rsp.wqe_com, 1); - bf_set(wqe_lenloc, &wqe->xmit_bls_rsp.wqe_com, - LPFC_WQE_LENLOC_NONE); - /* Overwrite the pre-set comnd type with OTHER_COMMAND */ - command_type = OTHER_COMMAND; - if (iocbq->iocb.un.xseq64.w5.hcsw.Rctl == FC_RCTL_BA_RJT) { - bf_set(xmit_bls_rsp64_rjt_vspec, &wqe->xmit_bls_rsp, - bf_get(lpfc_vndr_code, &iocbq->iocb.un.bls_rsp)); - bf_set(xmit_bls_rsp64_rjt_expc, &wqe->xmit_bls_rsp, - bf_get(lpfc_rsn_expln, &iocbq->iocb.un.bls_rsp)); - bf_set(xmit_bls_rsp64_rjt_rsnc, &wqe->xmit_bls_rsp, - bf_get(lpfc_rsn_code, &iocbq->iocb.un.bls_rsp)); - } - - break; - case CMD_SEND_FRAME: - bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag); - bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag); - return 0; - case CMD_XRI_ABORTED_CX: - case CMD_CREATE_XRI_CR: /* Do we expect to use this? */ - case CMD_IOCB_FCP_IBIDIR64_CR: /* bidirectional xfer */ - case CMD_FCP_TSEND64_CX: /* Target mode send xfer-ready */ - case CMD_FCP_TRSP64_CX: /* Target mode rcv */ - case CMD_FCP_AUTO_TRSP_CX: /* Auto target rsp */ - default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "2014 Invalid command 0x%x\n", - iocbq->iocb.ulpCommand); - return IOCB_ERROR; - break; } - - if (iocbq->iocb_flag & LPFC_IO_DIF_PASS) - bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_PASSTHRU); - else if (iocbq->iocb_flag & LPFC_IO_DIF_STRIP) - bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_STRIP); - else if (iocbq->iocb_flag & LPFC_IO_DIF_INSERT) - bf_set(wqe_dif, &wqe->generic.wqe_com, LPFC_WQE_DIF_INSERT); - iocbq->iocb_flag &= ~(LPFC_IO_DIF_PASS | LPFC_IO_DIF_STRIP | - LPFC_IO_DIF_INSERT); - bf_set(wqe_xri_tag, &wqe->generic.wqe_com, xritag); - bf_set(wqe_reqtag, &wqe->generic.wqe_com, iocbq->iotag); - wqe->generic.wqe_com.abort_tag = abort_tag; - bf_set(wqe_cmd_type, &wqe->generic.wqe_com, command_type); - bf_set(wqe_cmnd, &wqe->generic.wqe_com, cmnd); - bf_set(wqe_class, &wqe->generic.wqe_com, iocbq->iocb.ulpClass); - bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); - return 0; } /** @@ -9848,7 +10716,7 @@ lpfc_sli4_iocb2wqe(struct lpfc_hba *phba, struct lpfc_iocbq *iocbq, * __lpfc_sli_issue_iocb_s4 is used by other functions in the driver to issue * an iocb command to an HBA with SLI-4 interface spec. * - * This function is called with hbalock held. The function will return success + * This function is called with ringlock held. The function will return success * after it successfully submit the iocb to firmware or after adding to the * txq. **/ @@ -9857,17 +10725,15 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number, struct lpfc_iocbq *piocb, uint32_t flag) { struct lpfc_sglq *sglq; - union lpfc_wqe128 wqe; + union lpfc_wqe128 *wqe; struct lpfc_queue *wq; struct lpfc_sli_ring *pring; + u32 ulp_command = get_job_cmnd(phba, piocb); /* Get the WQ */ - if ((piocb->iocb_flag & LPFC_IO_FCP) || - (piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - if (!phba->cfg_fof || (!(piocb->iocb_flag & LPFC_IO_OAS))) - wq = phba->sli4_hba.fcp_wq[piocb->hba_wqidx]; - else - wq = phba->sli4_hba.oas_wq; + if ((piocb->cmd_flag & LPFC_IO_FCP) || + (piocb->cmd_flag & LPFC_USE_FCPWQIDX)) { + wq = phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq; } else { wq = phba->sli4_hba.els_wq; } @@ -9879,37 +10745,28 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number, * The WQE can be either 64 or 128 bytes, */ - lockdep_assert_held(&phba->hbalock); - + lockdep_assert_held(&pring->ring_lock); + wqe = &piocb->wqe; if (piocb->sli4_xritag == NO_XRI) { - if (piocb->iocb.ulpCommand == CMD_ABORT_XRI_CN || - piocb->iocb.ulpCommand == CMD_CLOSE_XRI_CN) + if (ulp_command == CMD_ABORT_XRI_CX) sglq = NULL; else { - if (!list_empty(&pring->txq)) { + sglq = __lpfc_sli_get_els_sglq(phba, piocb); + if (!sglq) { if (!(flag & SLI_IOCB_RET_IOCB)) { __lpfc_sli_ringtx_put(phba, - pring, piocb); + pring, + piocb); return IOCB_SUCCESS; } else { return IOCB_BUSY; } - } else { - sglq = __lpfc_sli_get_els_sglq(phba, piocb); - if (!sglq) { - if (!(flag & SLI_IOCB_RET_IOCB)) { - __lpfc_sli_ringtx_put(phba, - pring, - piocb); - return IOCB_SUCCESS; - } else - return IOCB_BUSY; - } } } - } else if (piocb->iocb_flag & LPFC_IO_FCP) + } else if (piocb->cmd_flag & LPFC_IO_FCP) { /* These IO's already have an XRI and a mapped sgl. */ sglq = NULL; + } else { /* * This is a continuation of a commandi,(CX) so this @@ -9923,21 +10780,51 @@ __lpfc_sli_issue_iocb_s4(struct lpfc_hba *phba, uint32_t ring_number, if (sglq) { piocb->sli4_lxritag = sglq->sli4_lxritag; piocb->sli4_xritag = sglq->sli4_xritag; - if (NO_XRI == lpfc_sli4_bpl2sgl(phba, piocb, sglq)) + + /* ABTS sent by initiator to CT exchange, the + * RX_ID field will be filled with the newly + * allocated responder XRI. + */ + if (ulp_command == CMD_XMIT_BLS_RSP64_CX && + piocb->abort_bls == LPFC_ABTS_UNSOL_INT) + bf_set(xmit_bls_rsp64_rxid, &wqe->xmit_bls_rsp, + piocb->sli4_xritag); + + bf_set(wqe_xri_tag, &wqe->generic.wqe_com, + piocb->sli4_xritag); + + if (lpfc_wqe_bpl2sgl(phba, piocb, sglq) == NO_XRI) return IOCB_ERROR; } - if (lpfc_sli4_iocb2wqe(phba, piocb, &wqe)) + if (lpfc_sli4_wq_put(wq, wqe)) return IOCB_ERROR; - if (lpfc_sli4_wq_put(wq, &wqe)) - return IOCB_ERROR; lpfc_sli_ringtxcmpl_put(phba, pring, piocb); return 0; } -/** +/* + * lpfc_sli_issue_fcp_io - Wrapper func for issuing fcp i/o + * + * This routine wraps the actual fcp i/o function for issusing WQE for sli-4 + * or IOCB for sli-3 function. + * pointer from the lpfc_hba struct. + * + * Return codes: + * IOCB_ERROR - Error + * IOCB_SUCCESS - Success + * IOCB_BUSY - Busy + **/ +int +lpfc_sli_issue_fcp_io(struct lpfc_hba *phba, uint32_t ring_number, + struct lpfc_iocbq *piocb, uint32_t flag) +{ + return phba->__lpfc_sli_issue_fcp_io(phba, ring_number, piocb, flag); +} + +/* * __lpfc_sli_issue_iocb - Wrapper func of lockless version for issuing iocb * * This routine wraps the actual lockless version for issusing IOCB function @@ -9955,6 +10842,419 @@ __lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, return phba->__lpfc_sli_issue_iocb(phba, ring_number, piocb, flag); } +static void +__lpfc_sli_prep_els_req_rsp_s3(struct lpfc_iocbq *cmdiocbq, + struct lpfc_vport *vport, + struct lpfc_dmabuf *bmp, u16 cmd_size, u32 did, + u32 elscmd, u8 tmo, u8 expect_rsp) +{ + struct lpfc_hba *phba = vport->phba; + IOCB_t *cmd; + + cmd = &cmdiocbq->iocb; + memset(cmd, 0, sizeof(*cmd)); + + cmd->un.elsreq64.bdl.addrHigh = putPaddrHigh(bmp->phys); + cmd->un.elsreq64.bdl.addrLow = putPaddrLow(bmp->phys); + cmd->un.elsreq64.bdl.bdeFlags = BUFF_TYPE_BLP_64; + + if (expect_rsp) { + cmd->un.elsreq64.bdl.bdeSize = (2 * sizeof(struct ulp_bde64)); + cmd->un.elsreq64.remoteID = did; /* DID */ + cmd->ulpCommand = CMD_ELS_REQUEST64_CR; + cmd->ulpTimeout = tmo; + } else { + cmd->un.elsreq64.bdl.bdeSize = sizeof(struct ulp_bde64); + cmd->un.genreq64.xmit_els_remoteID = did; /* DID */ + cmd->ulpCommand = CMD_XMIT_ELS_RSP64_CX; + cmd->ulpPU = PARM_NPIV_DID; + } + cmd->ulpBdeCount = 1; + cmd->ulpLe = 1; + cmd->ulpClass = CLASS3; + + /* If we have NPIV enabled, we want to send ELS traffic by VPI. */ + if (phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) { + if (expect_rsp) { + cmd->un.elsreq64.myID = vport->fc_myDID; + + /* For ELS_REQUEST64_CR, use the VPI by default */ + cmd->ulpContext = phba->vpi_ids[vport->vpi]; + } + + cmd->ulpCt_h = 0; + /* The CT field must be 0=INVALID_RPI for the ECHO cmd */ + if (elscmd == ELS_CMD_ECHO) + cmd->ulpCt_l = 0; /* context = invalid RPI */ + else + cmd->ulpCt_l = 1; /* context = VPI */ + } +} + +static void +__lpfc_sli_prep_els_req_rsp_s4(struct lpfc_iocbq *cmdiocbq, + struct lpfc_vport *vport, + struct lpfc_dmabuf *bmp, u16 cmd_size, u32 did, + u32 elscmd, u8 tmo, u8 expect_rsp) +{ + struct lpfc_hba *phba = vport->phba; + union lpfc_wqe128 *wqe; + struct ulp_bde64_le *bde; + u8 els_id; + + wqe = &cmdiocbq->wqe; + memset(wqe, 0, sizeof(*wqe)); + + /* Word 0 - 2 BDE */ + bde = (struct ulp_bde64_le *)&wqe->generic.bde; + bde->addr_low = cpu_to_le32(putPaddrLow(bmp->phys)); + bde->addr_high = cpu_to_le32(putPaddrHigh(bmp->phys)); + bde->type_size = cpu_to_le32(cmd_size); + bde->type_size |= cpu_to_le32(ULP_BDE64_TYPE_BDE_64); + + if (expect_rsp) { + bf_set(wqe_cmnd, &wqe->els_req.wqe_com, CMD_ELS_REQUEST64_WQE); + + /* Transfer length */ + wqe->els_req.payload_len = cmd_size; + wqe->els_req.max_response_payload_len = FCELSSIZE; + + /* DID */ + bf_set(wqe_els_did, &wqe->els_req.wqe_dest, did); + + /* Word 11 - ELS_ID */ + switch (elscmd) { + case ELS_CMD_PLOGI: + els_id = LPFC_ELS_ID_PLOGI; + break; + case ELS_CMD_FLOGI: + els_id = LPFC_ELS_ID_FLOGI; + break; + case ELS_CMD_LOGO: + els_id = LPFC_ELS_ID_LOGO; + break; + case ELS_CMD_FDISC: + if (!vport->fc_myDID) { + els_id = LPFC_ELS_ID_FDISC; + break; + } + fallthrough; + default: + els_id = LPFC_ELS_ID_DEFAULT; + break; + } + + bf_set(wqe_els_id, &wqe->els_req.wqe_com, els_id); + } else { + /* DID */ + bf_set(wqe_els_did, &wqe->xmit_els_rsp.wqe_dest, did); + + /* Transfer length */ + wqe->xmit_els_rsp.response_payload_len = cmd_size; + + bf_set(wqe_cmnd, &wqe->xmit_els_rsp.wqe_com, + CMD_XMIT_ELS_RSP64_WQE); + } + + bf_set(wqe_tmo, &wqe->generic.wqe_com, tmo); + bf_set(wqe_reqtag, &wqe->generic.wqe_com, cmdiocbq->iotag); + bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3); + + /* If we have NPIV enabled, we want to send ELS traffic by VPI. + * For SLI4, since the driver controls VPIs we also want to include + * all ELS pt2pt protocol traffic as well. + */ + if ((phba->sli3_options & LPFC_SLI3_NPIV_ENABLED) || + test_bit(FC_PT2PT, &vport->fc_flag)) { + if (expect_rsp) { + bf_set(els_req64_sid, &wqe->els_req, vport->fc_myDID); + + /* For ELS_REQUEST64_WQE, use the VPI by default */ + bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, + phba->vpi_ids[vport->vpi]); + } + + /* The CT field must be 0=INVALID_RPI for the ECHO cmd */ + if (elscmd == ELS_CMD_ECHO) + bf_set(wqe_ct, &wqe->generic.wqe_com, 0); + else + bf_set(wqe_ct, &wqe->generic.wqe_com, 1); + } +} + +void +lpfc_sli_prep_els_req_rsp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq, + struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, + u16 cmd_size, u32 did, u32 elscmd, u8 tmo, + u8 expect_rsp) +{ + phba->__lpfc_sli_prep_els_req_rsp(cmdiocbq, vport, bmp, cmd_size, did, + elscmd, tmo, expect_rsp); +} + +static void +__lpfc_sli_prep_gen_req_s3(struct lpfc_iocbq *cmdiocbq, struct lpfc_dmabuf *bmp, + u16 rpi, u32 num_entry, u8 tmo) +{ + IOCB_t *cmd; + + cmd = &cmdiocbq->iocb; + memset(cmd, 0, sizeof(*cmd)); + + cmd->un.genreq64.bdl.addrHigh = putPaddrHigh(bmp->phys); + cmd->un.genreq64.bdl.addrLow = putPaddrLow(bmp->phys); + cmd->un.genreq64.bdl.bdeFlags = BUFF_TYPE_BLP_64; + cmd->un.genreq64.bdl.bdeSize = num_entry * sizeof(struct ulp_bde64); + + cmd->un.genreq64.w5.hcsw.Rctl = FC_RCTL_DD_UNSOL_CTL; + cmd->un.genreq64.w5.hcsw.Type = FC_TYPE_CT; + cmd->un.genreq64.w5.hcsw.Fctl = (SI | LA); + + cmd->ulpContext = rpi; + cmd->ulpClass = CLASS3; + cmd->ulpCommand = CMD_GEN_REQUEST64_CR; + cmd->ulpBdeCount = 1; + cmd->ulpLe = 1; + cmd->ulpOwner = OWN_CHIP; + cmd->ulpTimeout = tmo; +} + +static void +__lpfc_sli_prep_gen_req_s4(struct lpfc_iocbq *cmdiocbq, struct lpfc_dmabuf *bmp, + u16 rpi, u32 num_entry, u8 tmo) +{ + union lpfc_wqe128 *cmdwqe; + struct ulp_bde64_le *bde, *bpl; + u32 xmit_len = 0, total_len = 0, size, type, i; + + cmdwqe = &cmdiocbq->wqe; + memset(cmdwqe, 0, sizeof(*cmdwqe)); + + /* Calculate total_len and xmit_len */ + bpl = (struct ulp_bde64_le *)bmp->virt; + for (i = 0; i < num_entry; i++) { + size = le32_to_cpu(bpl[i].type_size) & ULP_BDE64_SIZE_MASK; + total_len += size; + } + for (i = 0; i < num_entry; i++) { + size = le32_to_cpu(bpl[i].type_size) & ULP_BDE64_SIZE_MASK; + type = le32_to_cpu(bpl[i].type_size) & ULP_BDE64_TYPE_MASK; + if (type != ULP_BDE64_TYPE_BDE_64) + break; + xmit_len += size; + } + + /* Words 0 - 2 */ + bde = (struct ulp_bde64_le *)&cmdwqe->generic.bde; + bde->addr_low = bpl->addr_low; + bde->addr_high = bpl->addr_high; + bde->type_size = cpu_to_le32(xmit_len); + bde->type_size |= cpu_to_le32(ULP_BDE64_TYPE_BDE_64); + + /* Word 3 */ + cmdwqe->gen_req.request_payload_len = xmit_len; + + /* Word 5 */ + bf_set(wqe_type, &cmdwqe->gen_req.wge_ctl, FC_TYPE_CT); + bf_set(wqe_rctl, &cmdwqe->gen_req.wge_ctl, FC_RCTL_DD_UNSOL_CTL); + bf_set(wqe_si, &cmdwqe->gen_req.wge_ctl, 1); + bf_set(wqe_la, &cmdwqe->gen_req.wge_ctl, 1); + + /* Word 6 */ + bf_set(wqe_ctxt_tag, &cmdwqe->gen_req.wqe_com, rpi); + + /* Word 7 */ + bf_set(wqe_tmo, &cmdwqe->gen_req.wqe_com, tmo); + bf_set(wqe_class, &cmdwqe->gen_req.wqe_com, CLASS3); + bf_set(wqe_cmnd, &cmdwqe->gen_req.wqe_com, CMD_GEN_REQUEST64_CR); + bf_set(wqe_ct, &cmdwqe->gen_req.wqe_com, SLI4_CT_RPI); + + /* Word 12 */ + cmdwqe->gen_req.max_response_payload_len = total_len - xmit_len; +} + +void +lpfc_sli_prep_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq, + struct lpfc_dmabuf *bmp, u16 rpi, u32 num_entry, u8 tmo) +{ + phba->__lpfc_sli_prep_gen_req(cmdiocbq, bmp, rpi, num_entry, tmo); +} + +static void +__lpfc_sli_prep_xmit_seq64_s3(struct lpfc_iocbq *cmdiocbq, + struct lpfc_dmabuf *bmp, u16 rpi, u16 ox_id, + u32 num_entry, u8 rctl, u8 last_seq, u8 cr_cx_cmd) +{ + IOCB_t *icmd; + + icmd = &cmdiocbq->iocb; + memset(icmd, 0, sizeof(*icmd)); + + icmd->un.xseq64.bdl.addrHigh = putPaddrHigh(bmp->phys); + icmd->un.xseq64.bdl.addrLow = putPaddrLow(bmp->phys); + icmd->un.xseq64.bdl.bdeFlags = BUFF_TYPE_BLP_64; + icmd->un.xseq64.bdl.bdeSize = (num_entry * sizeof(struct ulp_bde64)); + icmd->un.xseq64.w5.hcsw.Fctl = LA; + if (last_seq) + icmd->un.xseq64.w5.hcsw.Fctl |= LS; + icmd->un.xseq64.w5.hcsw.Dfctl = 0; + icmd->un.xseq64.w5.hcsw.Rctl = rctl; + icmd->un.xseq64.w5.hcsw.Type = FC_TYPE_CT; + + icmd->ulpBdeCount = 1; + icmd->ulpLe = 1; + icmd->ulpClass = CLASS3; + + switch (cr_cx_cmd) { + case CMD_XMIT_SEQUENCE64_CR: + icmd->ulpContext = rpi; + icmd->ulpCommand = CMD_XMIT_SEQUENCE64_CR; + break; + case CMD_XMIT_SEQUENCE64_CX: + icmd->ulpContext = ox_id; + icmd->ulpCommand = CMD_XMIT_SEQUENCE64_CX; + break; + default: + break; + } +} + +static void +__lpfc_sli_prep_xmit_seq64_s4(struct lpfc_iocbq *cmdiocbq, + struct lpfc_dmabuf *bmp, u16 rpi, u16 ox_id, + u32 full_size, u8 rctl, u8 last_seq, u8 cr_cx_cmd) +{ + union lpfc_wqe128 *wqe; + struct ulp_bde64 *bpl; + + wqe = &cmdiocbq->wqe; + memset(wqe, 0, sizeof(*wqe)); + + /* Words 0 - 2 */ + bpl = (struct ulp_bde64 *)bmp->virt; + wqe->xmit_sequence.bde.addrHigh = bpl->addrHigh; + wqe->xmit_sequence.bde.addrLow = bpl->addrLow; + wqe->xmit_sequence.bde.tus.w = bpl->tus.w; + + /* Word 5 */ + bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, last_seq); + bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 1); + bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0); + bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, rctl); + bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_CT); + + /* Word 6 */ + bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com, rpi); + + bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com, + CMD_XMIT_SEQUENCE64_WQE); + + /* Word 7 */ + bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3); + + /* Word 9 */ + bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ox_id); + + if (cmdiocbq->cmd_flag & (LPFC_IO_LIBDFC | LPFC_IO_LOOPBACK)) { + /* Word 10 */ + if (cmdiocbq->cmd_flag & LPFC_IO_VMID) { + bf_set(wqe_appid, &wqe->xmit_sequence.wqe_com, 1); + bf_set(wqe_wqes, &wqe->xmit_sequence.wqe_com, 1); + wqe->words[31] = LOOPBACK_SRC_APPID; + } + + /* Word 12 */ + wqe->xmit_sequence.xmit_len = full_size; + } + else + wqe->xmit_sequence.xmit_len = + wqe->xmit_sequence.bde.tus.f.bdeSize; +} + +void +lpfc_sli_prep_xmit_seq64(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq, + struct lpfc_dmabuf *bmp, u16 rpi, u16 ox_id, + u32 num_entry, u8 rctl, u8 last_seq, u8 cr_cx_cmd) +{ + phba->__lpfc_sli_prep_xmit_seq64(cmdiocbq, bmp, rpi, ox_id, num_entry, + rctl, last_seq, cr_cx_cmd); +} + +static void +__lpfc_sli_prep_abort_xri_s3(struct lpfc_iocbq *cmdiocbq, u16 ulp_context, + u16 iotag, u8 ulp_class, u16 cqid, bool ia, + bool wqec) +{ + IOCB_t *icmd = NULL; + + icmd = &cmdiocbq->iocb; + memset(icmd, 0, sizeof(*icmd)); + + /* Word 5 */ + icmd->un.acxri.abortContextTag = ulp_context; + icmd->un.acxri.abortIoTag = iotag; + + if (ia) { + /* Word 7 */ + icmd->ulpCommand = CMD_CLOSE_XRI_CN; + } else { + /* Word 3 */ + icmd->un.acxri.abortType = ABORT_TYPE_ABTS; + + /* Word 7 */ + icmd->ulpClass = ulp_class; + icmd->ulpCommand = CMD_ABORT_XRI_CN; + } + + /* Word 7 */ + icmd->ulpLe = 1; +} + +static void +__lpfc_sli_prep_abort_xri_s4(struct lpfc_iocbq *cmdiocbq, u16 ulp_context, + u16 iotag, u8 ulp_class, u16 cqid, bool ia, + bool wqec) +{ + union lpfc_wqe128 *wqe; + + wqe = &cmdiocbq->wqe; + memset(wqe, 0, sizeof(*wqe)); + + /* Word 3 */ + bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); + if (ia) + bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); + else + bf_set(abort_cmd_ia, &wqe->abort_cmd, 0); + + /* Word 7 */ + bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_WQE); + + /* Word 8 */ + wqe->abort_cmd.wqe_com.abort_tag = ulp_context; + + /* Word 9 */ + bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, iotag); + + /* Word 10 */ + bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); + + /* Word 11 */ + if (wqec) + bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1); + bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, cqid); + bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND); +} + +void +lpfc_sli_prep_abort_xri(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocbq, + u16 ulp_context, u16 iotag, u8 ulp_class, u16 cqid, + bool ia, bool wqec) +{ + phba->__lpfc_sli_prep_abort_xri(cmdiocbq, ulp_context, iotag, ulp_class, + cqid, ia, wqec); +} + /** * lpfc_sli_api_table_setup - Set up sli api function jump table * @phba: The hba struct for which this call is being executed. @@ -9972,19 +11272,27 @@ lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) case LPFC_PCI_DEV_LP: phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s3; phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s3; + phba->__lpfc_sli_issue_fcp_io = __lpfc_sli_issue_fcp_io_s3; + phba->__lpfc_sli_prep_els_req_rsp = __lpfc_sli_prep_els_req_rsp_s3; + phba->__lpfc_sli_prep_gen_req = __lpfc_sli_prep_gen_req_s3; + phba->__lpfc_sli_prep_xmit_seq64 = __lpfc_sli_prep_xmit_seq64_s3; + phba->__lpfc_sli_prep_abort_xri = __lpfc_sli_prep_abort_xri_s3; break; case LPFC_PCI_DEV_OC: phba->__lpfc_sli_issue_iocb = __lpfc_sli_issue_iocb_s4; phba->__lpfc_sli_release_iocbq = __lpfc_sli_release_iocbq_s4; + phba->__lpfc_sli_issue_fcp_io = __lpfc_sli_issue_fcp_io_s4; + phba->__lpfc_sli_prep_els_req_rsp = __lpfc_sli_prep_els_req_rsp_s4; + phba->__lpfc_sli_prep_gen_req = __lpfc_sli_prep_gen_req_s4; + phba->__lpfc_sli_prep_xmit_seq64 = __lpfc_sli_prep_xmit_seq64_s4; + phba->__lpfc_sli_prep_abort_xri = __lpfc_sli_prep_abort_xri_s4; break; default: lpfc_printf_log(phba, KERN_ERR, LOG_INIT, "1419 Invalid HBA PCI-device group: 0x%x\n", dev_grp); return -ENODEV; - break; } - phba->lpfc_get_iocb_from_iocbq = lpfc_get_iocb_from_iocbq; return 0; } @@ -10001,29 +11309,20 @@ lpfc_sli_api_table_setup(struct lpfc_hba *phba, uint8_t dev_grp) struct lpfc_sli_ring * lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) { - if (piocb->iocb_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) { - if (!(phba->cfg_fof) || - (!(piocb->iocb_flag & LPFC_IO_FOF))) { - if (unlikely(!phba->sli4_hba.fcp_wq)) - return NULL; - /* - * for abort iocb hba_wqidx should already - * be setup based on what work queue we used. - */ - if (!(piocb->iocb_flag & LPFC_USE_FCPWQIDX)) { - piocb->hba_wqidx = - lpfc_sli4_scmd_to_wqidx_distr(phba, - piocb->context1); - piocb->hba_wqidx = piocb->hba_wqidx % - phba->cfg_fcp_io_channel; - } - return phba->sli4_hba.fcp_wq[piocb->hba_wqidx]->pring; - } else { - if (unlikely(!phba->sli4_hba.oas_wq)) - return NULL; - piocb->hba_wqidx = 0; - return phba->sli4_hba.oas_wq->pring; + struct lpfc_io_buf *lpfc_cmd; + + if (piocb->cmd_flag & (LPFC_IO_FCP | LPFC_USE_FCPWQIDX)) { + if (unlikely(!phba->sli4_hba.hdwq)) + return NULL; + /* + * for abort iocb hba_wqidx should already + * be setup based on what work queue we used. + */ + if (!(piocb->cmd_flag & LPFC_USE_FCPWQIDX)) { + lpfc_cmd = piocb->io_buf; + piocb->hba_wqidx = lpfc_cmd->hdwq_no; } + return phba->sli4_hba.hdwq[piocb->hba_wqidx].io_wq->pring; } else { if (unlikely(!phba->sli4_hba.els_wq)) return NULL; @@ -10032,10 +11331,35 @@ lpfc_sli4_calc_ring(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) } } +inline void lpfc_sli4_poll_eq(struct lpfc_queue *eq) +{ + struct lpfc_hba *phba = eq->phba; + + /* + * Unlocking an irq is one of the entry point to check + * for re-schedule, but we are good for io submission + * path as midlayer does a get_cpu to glue us in. Flush + * out the invalidate queue so we can see the updated + * value for flag. + */ + smp_rmb(); + + if (READ_ONCE(eq->mode) == LPFC_EQ_POLL) + /* We will not likely get the completion for the caller + * during this iteration but i guess that's fine. + * Future io's coming on this eq should be able to + * pick it up. As for the case of single io's, they + * will be handled through a sched from polling timer + * function which is currently triggered every 1msec. + */ + lpfc_sli4_process_eq(phba, eq, LPFC_QUEUE_NOARM, + LPFC_QUEUE_WORK); +} + /** * lpfc_sli_issue_iocb - Wrapper function for __lpfc_sli_issue_iocb * @phba: Pointer to HBA context object. - * @pring: Pointer to driver SLI ring object. + * @ring_number: Ring number * @piocb: Pointer to command iocb. * @flag: Flag indicating if this command can be put into txq. * @@ -10049,14 +11373,20 @@ int lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, struct lpfc_iocbq *piocb, uint32_t flag) { - struct lpfc_hba_eq_hdl *hba_eq_hdl; struct lpfc_sli_ring *pring; - struct lpfc_queue *fpeq; - struct lpfc_eqe *eqe; + struct lpfc_queue *eq; unsigned long iflags; - int rc, idx; + int rc; + + /* If the PCI channel is in offline state, do not post iocbs. */ + if (unlikely(pci_channel_offline(phba->pcidev))) + return IOCB_ERROR; if (phba->sli_rev == LPFC_SLI_REV4) { + lpfc_sli_prep_wqe(phba, piocb); + + eq = phba->sli4_hba.hdwq[piocb->hba_wqidx].hba_eq; + pring = lpfc_sli4_calc_ring(phba, piocb); if (unlikely(pring == NULL)) return IOCB_ERROR; @@ -10065,33 +11395,7 @@ lpfc_sli_issue_iocb(struct lpfc_hba *phba, uint32_t ring_number, rc = __lpfc_sli_issue_iocb(phba, ring_number, piocb, flag); spin_unlock_irqrestore(&pring->ring_lock, iflags); - if (lpfc_fcp_look_ahead && (piocb->iocb_flag & LPFC_IO_FCP)) { - idx = piocb->hba_wqidx; - hba_eq_hdl = &phba->sli4_hba.hba_eq_hdl[idx]; - - if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) { - - /* Get associated EQ with this index */ - fpeq = phba->sli4_hba.hba_eq[idx]; - - /* Turn off interrupts from this EQ */ - phba->sli4_hba.sli4_eq_clr_intr(fpeq); - - /* - * Process all the events on FCP EQ - */ - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, - eqe, idx); - fpeq->EQ_processed++; - } - - /* Always clear and re-arm the EQ */ - phba->sli4_hba.sli4_eq_release(fpeq, - LPFC_QUEUE_REARM); - } - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - } + lpfc_sli4_poll_eq(eq); } else { /* For now, SLI2/3 will still use hbalock */ spin_lock_irqsave(&phba->hbalock, iflags); @@ -10147,6 +11451,32 @@ lpfc_extra_ring_setup( struct lpfc_hba *phba) return 0; } +static void +lpfc_sli_post_recovery_event(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp) +{ + unsigned long iflags; + struct lpfc_work_evt *evtp = &ndlp->recovery_evt; + + /* Hold a node reference for outstanding queued work */ + if (!lpfc_nlp_get(ndlp)) + return; + + spin_lock_irqsave(&phba->hbalock, iflags); + if (!list_empty(&evtp->evt_listp)) { + spin_unlock_irqrestore(&phba->hbalock, iflags); + lpfc_nlp_put(ndlp); + return; + } + + evtp->evt_arg1 = ndlp; + evtp->evt = LPFC_EVT_RECOVER_PORT; + list_add_tail(&evtp->evt_listp, &phba->work_list); + spin_unlock_irqrestore(&phba->hbalock, iflags); + + lpfc_worker_wake_up(phba); +} + /* lpfc_sli_abts_err_handler - handle a failed ABTS request from an SLI3 port. * @phba: Pointer to HBA context object. * @iocbq: Pointer to iocb object. @@ -10180,7 +11510,7 @@ lpfc_sli_abts_err_handler(struct lpfc_hba *phba, if (!vport) goto err_exit; ndlp = lpfc_findnode_rpi(vport, rpi); - if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) + if (!ndlp) goto err_exit; if (iocbq->iocb.ulpStatus == IOSTAT_LOCAL_REJECT) @@ -10191,8 +11521,8 @@ lpfc_sli_abts_err_handler(struct lpfc_hba *phba, lpfc_printf_log(phba, KERN_INFO, LOG_SLI, "3095 Event Context not found, no " "action on vpi %d rpi %d status 0x%x, reason 0x%x\n", - iocbq->iocb.ulpContext, iocbq->iocb.ulpStatus, - vpi, rpi); + vpi, rpi, iocbq->iocb.ulpStatus, + iocbq->iocb.ulpContext); } /* lpfc_sli4_abts_err_handler - handle a failed ABTS request from an SLI4 port. @@ -10210,17 +11540,15 @@ lpfc_sli4_abts_err_handler(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, struct sli4_wcqe_xri_aborted *axri) { - struct lpfc_vport *vport; uint32_t ext_status = 0; - if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { + if (!ndlp) { lpfc_printf_log(phba, KERN_INFO, LOG_SLI, "3115 Node Context not found, driver " "ignoring abts err event\n"); return; } - vport = ndlp->vport; lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, "3116 Port generated FCP XRI ABORT event on " "vpi %d rpi %d xri x%x status 0x%x parameter x%x\n", @@ -10237,7 +11565,7 @@ lpfc_sli4_abts_err_handler(struct lpfc_hba *phba, ext_status = axri->parameter & IOERR_PARAM_MASK; if ((bf_get(lpfc_wcqe_xa_status, axri) == IOSTAT_LOCAL_REJECT) && ((ext_status == IOERR_SEQUENCE_TIMEOUT) || (ext_status == 0))) - lpfc_sli_abts_recover_port(vport, ndlp); + lpfc_sli_post_recovery_event(phba, ndlp); } /** @@ -10273,13 +11601,13 @@ lpfc_sli_async_event_handler(struct lpfc_hba * phba, temp_event_data.event_type = FC_REG_TEMPERATURE_EVENT; if (evt_code == ASYNC_TEMP_WARN) { temp_event_data.event_code = LPFC_THRESHOLD_TEMP; - lpfc_printf_log(phba, KERN_ERR, LOG_TEMP, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0347 Adapter is very hot, please take " "corrective action. temperature : %d Celsius\n", (uint32_t) icmd->ulpContext); } else { temp_event_data.event_code = LPFC_NORMAL_TEMP; - lpfc_printf_log(phba, KERN_ERR, LOG_TEMP, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0340 Adapter temperature is OK now. " "temperature : %d Celsius\n", (uint32_t) icmd->ulpContext); @@ -10296,7 +11624,7 @@ lpfc_sli_async_event_handler(struct lpfc_hba * phba, break; default: iocb_w = (uint32_t *) icmd; - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0346 Ring %d handler: unexpected ASYNC_STATUS" " evt_code 0x%x\n" "W0 0x%08x W1 0x%08x W2 0x%08x W3 0x%08x\n" @@ -10506,19 +11834,11 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) INIT_LIST_HEAD(&psli->mboxq); INIT_LIST_HEAD(&psli->mboxq_cmpl); /* Initialize list headers for txq and txcmplq as double linked lists */ - for (i = 0; i < phba->cfg_fcp_io_channel; i++) { - pring = phba->sli4_hba.fcp_wq[i]->pring; - pring->flag = 0; - pring->ringno = LPFC_FCP_RING; - INIT_LIST_HEAD(&pring->txq); - INIT_LIST_HEAD(&pring->txcmplq); - INIT_LIST_HEAD(&pring->iocb_continueq); - spin_lock_init(&pring->ring_lock); - } - for (i = 0; i < phba->cfg_nvme_io_channel; i++) { - pring = phba->sli4_hba.nvme_wq[i]->pring; + for (i = 0; i < phba->cfg_hdw_queue; i++) { + pring = phba->sli4_hba.hdwq[i].io_wq->pring; pring->flag = 0; pring->ringno = LPFC_FCP_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -10527,25 +11847,17 @@ lpfc_sli4_queue_init(struct lpfc_hba *phba) pring = phba->sli4_hba.els_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); spin_lock_init(&pring->ring_lock); - if (phba->cfg_nvme_io_channel) { + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { pring = phba->sli4_hba.nvmels_wq->pring; pring->flag = 0; pring->ringno = LPFC_ELS_RING; - INIT_LIST_HEAD(&pring->txq); - INIT_LIST_HEAD(&pring->txcmplq); - INIT_LIST_HEAD(&pring->iocb_continueq); - spin_lock_init(&pring->ring_lock); - } - - if (phba->cfg_fof) { - pring = phba->sli4_hba.oas_wq->pring; - pring->flag = 0; - pring->ringno = LPFC_FCP_RING; + pring->txcmplq_cnt = 0; INIT_LIST_HEAD(&pring->txq); INIT_LIST_HEAD(&pring->txcmplq); INIT_LIST_HEAD(&pring->iocb_continueq); @@ -10707,7 +12019,8 @@ lpfc_sli_host_down(struct lpfc_vport *vport) &pring->txcmplq, list) { if (iocb->vport != vport) continue; - lpfc_sli_issue_abort_iotag(phba, pring, iocb); + lpfc_sli_issue_abort_iotag(phba, pring, iocb, + NULL); } pring->flag = prev_pring_flag; } @@ -10722,25 +12035,29 @@ lpfc_sli_host_down(struct lpfc_vport *vport) set_bit(LPFC_DATA_READY, &phba->data_flags); } prev_pring_flag = pring->flag; - spin_lock_irq(&pring->ring_lock); + spin_lock(&pring->ring_lock); list_for_each_entry_safe(iocb, next_iocb, &pring->txq, list) { if (iocb->vport != vport) continue; list_move_tail(&iocb->list, &completions); } - spin_unlock_irq(&pring->ring_lock); + spin_unlock(&pring->ring_lock); list_for_each_entry_safe(iocb, next_iocb, &pring->txcmplq, list) { if (iocb->vport != vport) continue; - lpfc_sli_issue_abort_iotag(phba, pring, iocb); + lpfc_sli_issue_abort_iotag(phba, pring, iocb, + NULL); } pring->flag = prev_pring_flag; } } spin_unlock_irqrestore(&phba->hbalock, flags); + /* Make sure HBA is alive */ + lpfc_issue_hb_tmo(phba); + /* Cancel all the IOCBs from the completions list */ lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT, IOERR_SLI_DOWN); @@ -10805,9 +12122,9 @@ lpfc_sli_hba_down(struct lpfc_hba *phba) pring = qp->pring; if (!pring) continue; - spin_lock_irq(&pring->ring_lock); + spin_lock(&pring->ring_lock); list_splice_init(&pring->txq, &completions); - spin_unlock_irq(&pring->ring_lock); + spin_unlock(&pring->ring_lock); if (pring == phba->sli4_hba.els_wq->pring) { pring->flag |= LPFC_DEFERRED_RING_EVENT; /* Set the lpfc data pending flag */ @@ -10838,7 +12155,7 @@ lpfc_sli_hba_down(struct lpfc_hba *phba) local_bh_enable(); /* Return any active mbox cmds */ - del_timer_sync(&psli->mbox_tmo); + timer_delete_sync(&psli->mbox_tmo); spin_lock_irqsave(&phba->pport->work_port_lock, flags); phba->pport->work_port_events &= ~WORKER_MBOX_TMO; @@ -10986,9 +12303,9 @@ lpfc_sli_ring_taggedbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, } spin_unlock_irq(&phba->hbalock); - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0402 Cannot find virtual addr for buffer tag on " - "ring %d Data x%lx x%p x%p x%x\n", + "ring %d Data x%lx x%px x%px x%x\n", pring->ringno, (unsigned long) tag, slp->next, slp->prev, pring->postbufq_cnt); @@ -11030,9 +12347,9 @@ lpfc_sli_ringpostbuf_get(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, } spin_unlock_irq(&phba->hbalock); - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0410 Cannot find virtual addr for mapped buf on " - "ring %d Data x%llx x%p x%p x%x\n", + "ring %d Data x%llx x%px x%px x%x\n", pring->ringno, (unsigned long long)phys, slp->next, slp->prev, pring->postbufq_cnt); return NULL; @@ -11053,48 +12370,32 @@ static void lpfc_sli_abort_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_iocbq *rspiocb) { - IOCB_t *irsp = &rspiocb->iocb; - uint16_t abort_iotag, abort_context; - struct lpfc_iocbq *abort_iocb = NULL; - - if (irsp->ulpStatus) { + u32 ulp_status = get_job_ulpstatus(phba, rspiocb); + u32 ulp_word4 = get_job_word4(phba, rspiocb); + u8 cmnd = get_job_cmnd(phba, cmdiocb); + if (ulp_status) { /* * Assume that the port already completed and returned, or * will return the iocb. Just Log the message. */ - abort_context = cmdiocb->iocb.un.acxri.abortContextTag; - abort_iotag = cmdiocb->iocb.un.acxri.abortIoTag; - - spin_lock_irq(&phba->hbalock); if (phba->sli_rev < LPFC_SLI_REV4) { - if (irsp->ulpCommand == CMD_ABORT_XRI_CX && - irsp->ulpStatus == IOSTAT_LOCAL_REJECT && - irsp->un.ulpWord[4] == IOERR_ABORT_REQUESTED) { - spin_unlock_irq(&phba->hbalock); + if (cmnd == CMD_ABORT_XRI_CX && + ulp_status == IOSTAT_LOCAL_REJECT && + ulp_word4 == IOERR_ABORT_REQUESTED) { goto release_iocb; } - if (abort_iotag != 0 && - abort_iotag <= phba->sli.last_iotag) - abort_iocb = - phba->sli.iocbq_lookup[abort_iotag]; - } else - /* For sli4 the abort_tag is the XRI, - * so the abort routine puts the iotag of the iocb - * being aborted in the context field of the abort - * IOCB. - */ - abort_iocb = phba->sli.iocbq_lookup[abort_context]; - - lpfc_printf_log(phba, KERN_WARNING, LOG_ELS | LOG_SLI, - "0327 Cannot abort els iocb %p " - "with tag %x context %x, abort status %x, " - "abort code %x\n", - abort_iocb, abort_iotag, abort_context, - irsp->ulpStatus, irsp->un.ulpWord[4]); - - spin_unlock_irq(&phba->hbalock); + } } + + lpfc_printf_log(phba, KERN_INFO, LOG_ELS | LOG_SLI, + "0327 Abort els iocb complete x%px with io cmd xri %x " + "abort tag x%x abort status %x abort code %x\n", + cmdiocb, get_job_abtsiotag(phba, cmdiocb), + (phba->sli_rev == LPFC_SLI_REV4) ? + get_wqe_reqtag(cmdiocb) : + cmdiocb->iocb.ulpIoTag, + ulp_status, ulp_word4); release_iocb: lpfc_sli_release_iocbq(phba, cmdiocb); return; @@ -11111,114 +12412,161 @@ release_iocb: * which are aborted. The function frees memory resources used for * the aborted ELS commands. **/ -static void +void lpfc_ignore_els_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_iocbq *rspiocb) { - IOCB_t *irsp = &rspiocb->iocb; + struct lpfc_nodelist *ndlp = cmdiocb->ndlp; + IOCB_t *irsp; + LPFC_MBOXQ_t *mbox; + u32 ulp_command, ulp_status, ulp_word4, iotag; + + ulp_command = get_job_cmnd(phba, cmdiocb); + ulp_status = get_job_ulpstatus(phba, rspiocb); + ulp_word4 = get_job_word4(phba, rspiocb); + + if (phba->sli_rev == LPFC_SLI_REV4) { + iotag = get_wqe_reqtag(cmdiocb); + } else { + irsp = &rspiocb->iocb; + iotag = irsp->ulpIoTag; + + /* It is possible a PLOGI_RJT for NPIV ports to get aborted. + * The MBX_REG_LOGIN64 mbox command is freed back to the + * mbox_mem_pool here. + */ + if (cmdiocb->context_un.mbox) { + mbox = cmdiocb->context_un.mbox; + lpfc_mbox_rsrc_cleanup(phba, mbox, MBOX_THD_UNLOCKED); + cmdiocb->context_un.mbox = NULL; + } + } /* ELS cmd tag <ulpIoTag> completes */ lpfc_printf_log(phba, KERN_INFO, LOG_ELS, - "0139 Ignoring ELS cmd tag x%x completion Data: " - "x%x x%x x%x\n", - irsp->ulpIoTag, irsp->ulpStatus, - irsp->un.ulpWord[4], irsp->ulpTimeout); - if (cmdiocb->iocb.ulpCommand == CMD_GEN_REQUEST64_CR) + "0139 Ignoring ELS cmd code x%x ref cnt x%x Data: " + "x%x x%x x%x x%px\n", + ulp_command, kref_read(&cmdiocb->ndlp->kref), + ulp_status, ulp_word4, iotag, cmdiocb->ndlp); + /* + * Deref the ndlp after free_iocb. sli_release_iocb will access the ndlp + * if exchange is busy. + */ + if (ulp_command == CMD_GEN_REQUEST64_CR) lpfc_ct_free_iocb(phba, cmdiocb); else lpfc_els_free_iocb(phba, cmdiocb); - return; + + lpfc_nlp_put(ndlp); } /** - * lpfc_sli_abort_iotag_issue - Issue abort for a command iocb + * lpfc_sli_issue_abort_iotag - Abort function for a command iocb * @phba: Pointer to HBA context object. * @pring: Pointer to driver SLI ring object. * @cmdiocb: Pointer to driver command iocb object. + * @cmpl: completion function. + * + * This function issues an abort iocb for the provided command iocb. In case + * of unloading, the abort iocb will not be issued to commands on the ELS + * ring. Instead, the callback function shall be changed to those commands + * so that nothing happens when them finishes. This function is called with + * hbalock held andno ring_lock held (SLI4). The function returns IOCB_SUCCESS + * when the command iocb is an abort request. * - * This function issues an abort iocb for the provided command iocb down to - * the port. Other than the case the outstanding command iocb is an abort - * request, this function issues abort out unconditionally. This function is - * called with hbalock held. The function returns 0 when it fails due to - * memory allocation failure or when the command iocb is an abort request. **/ -static int -lpfc_sli_abort_iotag_issue(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, - struct lpfc_iocbq *cmdiocb) +int +lpfc_sli_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, + struct lpfc_iocbq *cmdiocb, void *cmpl) { struct lpfc_vport *vport = cmdiocb->vport; struct lpfc_iocbq *abtsiocbp; - IOCB_t *icmd = NULL; - IOCB_t *iabt = NULL; - int retval; + int retval = IOCB_ERROR; unsigned long iflags; - struct lpfc_nodelist *ndlp; - - lockdep_assert_held(&phba->hbalock); + struct lpfc_nodelist *ndlp = NULL; + u32 ulp_command = get_job_cmnd(phba, cmdiocb); + u16 ulp_context, iotag; + bool ia; /* * There are certain command types we don't want to abort. And we * don't want to abort commands that are already in the process of * being aborted. */ - icmd = &cmdiocb->iocb; - if (icmd->ulpCommand == CMD_ABORT_XRI_CN || - icmd->ulpCommand == CMD_CLOSE_XRI_CN || - (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0) - return 0; + if (ulp_command == CMD_ABORT_XRI_WQE || + ulp_command == CMD_ABORT_XRI_CN || + ulp_command == CMD_CLOSE_XRI_CN || + cmdiocb->cmd_flag & LPFC_DRIVER_ABORTED) + return IOCB_ABORTING; - /* issue ABTS for this IOCB based on iotag */ + if (!pring) { + if (cmdiocb->cmd_flag & LPFC_IO_FABRIC) + cmdiocb->fabric_cmd_cmpl = lpfc_ignore_els_cmpl; + else + cmdiocb->cmd_cmpl = lpfc_ignore_els_cmpl; + return retval; + } + + /* + * Always abort the outstanding WQE and set the IA bit correctly + * for the context. This is necessary for correctly removing + * outstanding ndlp reference counts when the CQE completes with + * the XB bit set. + */ abtsiocbp = __lpfc_sli_get_iocbq(phba); if (abtsiocbp == NULL) - return 0; + return IOCB_NORESOURCE; /* This signals the response to set the correct status * before calling the completion handler */ - cmdiocb->iocb_flag |= LPFC_DRIVER_ABORTED; + cmdiocb->cmd_flag |= LPFC_DRIVER_ABORTED; - iabt = &abtsiocbp->iocb; - iabt->un.acxri.abortType = ABORT_TYPE_ABTS; - iabt->un.acxri.abortContextTag = icmd->ulpContext; if (phba->sli_rev == LPFC_SLI_REV4) { - iabt->un.acxri.abortIoTag = cmdiocb->sli4_xritag; - iabt->un.acxri.abortContextTag = cmdiocb->iotag; + ulp_context = cmdiocb->sli4_xritag; + iotag = abtsiocbp->iotag; } else { - iabt->un.acxri.abortIoTag = icmd->ulpIoTag; + iotag = cmdiocb->iocb.ulpIoTag; if (pring->ringno == LPFC_ELS_RING) { - ndlp = (struct lpfc_nodelist *)(cmdiocb->context1); - iabt->un.acxri.abortContextTag = ndlp->nlp_rpi; + ndlp = cmdiocb->ndlp; + ulp_context = ndlp->nlp_rpi; + } else { + ulp_context = cmdiocb->iocb.ulpContext; } } - iabt->ulpLe = 1; - iabt->ulpClass = icmd->ulpClass; + + /* Just close the exchange under certain conditions. */ + if (test_bit(FC_UNLOADING, &vport->load_flag) || + phba->link_state < LPFC_LINK_UP || + (phba->sli_rev == LPFC_SLI_REV4 && + phba->sli4_hba.link_state.status == LPFC_FC_LA_TYPE_LINK_DOWN) || + (phba->link_flag & LS_EXTERNAL_LOOPBACK)) + ia = true; + else + ia = false; + + lpfc_sli_prep_abort_xri(phba, abtsiocbp, ulp_context, iotag, + cmdiocb->iocb.ulpClass, + LPFC_WQE_CQ_ID_DEFAULT, ia, false); /* ABTS WQE must go to the same WQ as the WQE to be aborted */ abtsiocbp->hba_wqidx = cmdiocb->hba_wqidx; - if (cmdiocb->iocb_flag & LPFC_IO_FCP) - abtsiocbp->iocb_flag |= LPFC_USE_FCPWQIDX; - if (cmdiocb->iocb_flag & LPFC_IO_FOF) - abtsiocbp->iocb_flag |= LPFC_IO_FOF; + if (cmdiocb->cmd_flag & LPFC_IO_FCP) + abtsiocbp->cmd_flag |= (LPFC_IO_FCP | LPFC_USE_FCPWQIDX); - if (phba->link_state >= LPFC_LINK_UP) - iabt->ulpCommand = CMD_ABORT_XRI_CN; - else - iabt->ulpCommand = CMD_CLOSE_XRI_CN; + if (cmdiocb->cmd_flag & LPFC_IO_FOF) + abtsiocbp->cmd_flag |= LPFC_IO_FOF; - abtsiocbp->iocb_cmpl = lpfc_sli_abort_els_cmpl; + if (cmpl) + abtsiocbp->cmd_cmpl = cmpl; + else + abtsiocbp->cmd_cmpl = lpfc_sli_abort_els_cmpl; abtsiocbp->vport = vport; - lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI, - "0339 Abort xri x%x, original iotag x%x, " - "abort cmd iotag x%x\n", - iabt->un.acxri.abortIoTag, - iabt->un.acxri.abortContextTag, - abtsiocbp->iotag); - if (phba->sli_rev == LPFC_SLI_REV4) { pring = lpfc_sli4_calc_ring(phba, abtsiocbp); if (unlikely(pring == NULL)) - return 0; + goto abort_iotag_exit; /* Note: both hbalock and ring_lock need to be set here */ spin_lock_irqsave(&pring->ring_lock, iflags); retval = __lpfc_sli_issue_iocb(phba, pring->ringno, @@ -11229,76 +12577,20 @@ lpfc_sli_abort_iotag_issue(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, abtsiocbp, 0); } - if (retval) - __lpfc_sli_release_iocbq(phba, abtsiocbp); - - /* - * Caller to this routine should check for IOCB_ERROR - * and handle it properly. This routine no longer removes - * iocb off txcmplq and call compl in case of IOCB_ERROR. - */ - return retval; -} - -/** - * lpfc_sli_issue_abort_iotag - Abort function for a command iocb - * @phba: Pointer to HBA context object. - * @pring: Pointer to driver SLI ring object. - * @cmdiocb: Pointer to driver command iocb object. - * - * This function issues an abort iocb for the provided command iocb. In case - * of unloading, the abort iocb will not be issued to commands on the ELS - * ring. Instead, the callback function shall be changed to those commands - * so that nothing happens when them finishes. This function is called with - * hbalock held. The function returns 0 when the command iocb is an abort - * request. - **/ -int -lpfc_sli_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, - struct lpfc_iocbq *cmdiocb) -{ - struct lpfc_vport *vport = cmdiocb->vport; - int retval = IOCB_ERROR; - IOCB_t *icmd = NULL; - - lockdep_assert_held(&phba->hbalock); - - /* - * There are certain command types we don't want to abort. And we - * don't want to abort commands that are already in the process of - * being aborted. - */ - icmd = &cmdiocb->iocb; - if (icmd->ulpCommand == CMD_ABORT_XRI_CN || - icmd->ulpCommand == CMD_CLOSE_XRI_CN || - (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0) - return 0; - - if (!pring) { - if (cmdiocb->iocb_flag & LPFC_IO_FABRIC) - cmdiocb->fabric_iocb_cmpl = lpfc_ignore_els_cmpl; - else - cmdiocb->iocb_cmpl = lpfc_ignore_els_cmpl; - goto abort_iotag_exit; - } +abort_iotag_exit: - /* - * If we're unloading, don't abort iocb on the ELS ring, but change - * the callback so that nothing happens when it finishes. - */ - if ((vport->load_flag & FC_UNLOADING) && - (pring->ringno == LPFC_ELS_RING)) { - if (cmdiocb->iocb_flag & LPFC_IO_FABRIC) - cmdiocb->fabric_iocb_cmpl = lpfc_ignore_els_cmpl; - else - cmdiocb->iocb_cmpl = lpfc_ignore_els_cmpl; - goto abort_iotag_exit; + lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI, + "0339 Abort IO XRI x%x, Original iotag x%x, " + "abort tag x%x Cmdjob : x%px Abortjob : x%px " + "retval x%x : IA %d cmd_cmpl %ps\n", + ulp_context, (phba->sli_rev == LPFC_SLI_REV4) ? + cmdiocb->iotag : iotag, iotag, cmdiocb, abtsiocbp, + retval, ia, abtsiocbp->cmd_cmpl); + if (retval) { + cmdiocb->cmd_flag &= ~LPFC_DRIVER_ABORTED; + __lpfc_sli_release_iocbq(phba, abtsiocbp); } - /* Now, we try to issue the abort to the cmdiocb out */ - retval = lpfc_sli_abort_iotag_issue(phba, pring, cmdiocb); - -abort_iotag_exit: /* * Caller to this routine should check for IOCB_ERROR * and handle it properly. This routine no longer removes @@ -11308,100 +12600,6 @@ abort_iotag_exit: } /** - * lpfc_sli4_abort_nvme_io - Issue abort for a command iocb - * @phba: Pointer to HBA context object. - * @pring: Pointer to driver SLI ring object. - * @cmdiocb: Pointer to driver command iocb object. - * - * This function issues an abort iocb for the provided command iocb down to - * the port. Other than the case the outstanding command iocb is an abort - * request, this function issues abort out unconditionally. This function is - * called with hbalock held. The function returns 0 when it fails due to - * memory allocation failure or when the command iocb is an abort request. - **/ -static int -lpfc_sli4_abort_nvme_io(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, - struct lpfc_iocbq *cmdiocb) -{ - struct lpfc_vport *vport = cmdiocb->vport; - struct lpfc_iocbq *abtsiocbp; - union lpfc_wqe128 *abts_wqe; - int retval; - - /* - * There are certain command types we don't want to abort. And we - * don't want to abort commands that are already in the process of - * being aborted. - */ - if (cmdiocb->iocb.ulpCommand == CMD_ABORT_XRI_CN || - cmdiocb->iocb.ulpCommand == CMD_CLOSE_XRI_CN || - (cmdiocb->iocb_flag & LPFC_DRIVER_ABORTED) != 0) - return 0; - - /* issue ABTS for this io based on iotag */ - abtsiocbp = __lpfc_sli_get_iocbq(phba); - if (abtsiocbp == NULL) - return 0; - - /* This signals the response to set the correct status - * before calling the completion handler - */ - cmdiocb->iocb_flag |= LPFC_DRIVER_ABORTED; - - /* Complete prepping the abort wqe and issue to the FW. */ - abts_wqe = &abtsiocbp->wqe; - - /* Clear any stale WQE contents */ - memset(abts_wqe, 0, sizeof(union lpfc_wqe)); - bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG); - - /* word 7 */ - bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); - bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com, - cmdiocb->iocb.ulpClass); - - /* word 8 - tell the FW to abort the IO associated with this - * outstanding exchange ID. - */ - abts_wqe->abort_cmd.wqe_com.abort_tag = cmdiocb->sli4_xritag; - - /* word 9 - this is the iotag for the abts_wqe completion. */ - bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com, - abtsiocbp->iotag); - - /* word 10 */ - bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1); - bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); - - /* word 11 */ - bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND); - bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1); - bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); - - /* ABTS WQE must go to the same WQ as the WQE to be aborted */ - abtsiocbp->iocb_flag |= LPFC_IO_NVME; - abtsiocbp->vport = vport; - abtsiocbp->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; - retval = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abtsiocbp); - if (retval) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME, - "6147 Failed abts issue_wqe with status x%x " - "for oxid x%x\n", - retval, cmdiocb->sli4_xritag); - lpfc_sli_release_iocbq(phba, abtsiocbp); - return retval; - } - - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME, - "6148 Drv Abort NVME Request Issued for " - "ox_id x%x on reqtag x%x\n", - cmdiocb->sli4_xritag, - abtsiocbp->iotag); - - return retval; -} - -/** * lpfc_sli_hba_iocb_abort - Abort all iocbs to an hba. * @phba: pointer to lpfc HBA data structure. * @@ -11431,15 +12629,55 @@ lpfc_sli_hba_iocb_abort(struct lpfc_hba *phba) } /** - * lpfc_sli_validate_fcp_iocb - find commands associated with a vport or LUN + * lpfc_sli_validate_fcp_iocb_for_abort - filter iocbs appropriate for FCP aborts + * @iocbq: Pointer to iocb object. + * @vport: Pointer to driver virtual port object. + * + * This function acts as an iocb filter for functions which abort FCP iocbs. + * + * Return values + * -ENODEV, if a null iocb or vport ptr is encountered + * -EINVAL, if the iocb is not an FCP I/O, not on the TX cmpl queue, premarked as + * driver already started the abort process, or is an abort iocb itself + * 0, passes criteria for aborting the FCP I/O iocb + **/ +static int +lpfc_sli_validate_fcp_iocb_for_abort(struct lpfc_iocbq *iocbq, + struct lpfc_vport *vport) +{ + u8 ulp_command; + + /* No null ptr vports */ + if (!iocbq || iocbq->vport != vport) + return -ENODEV; + + /* iocb must be for FCP IO, already exists on the TX cmpl queue, + * can't be premarked as driver aborted, nor be an ABORT iocb itself + */ + ulp_command = get_job_cmnd(vport->phba, iocbq); + if (!(iocbq->cmd_flag & LPFC_IO_FCP) || + !(iocbq->cmd_flag & LPFC_IO_ON_TXCMPLQ) || + (iocbq->cmd_flag & LPFC_DRIVER_ABORTED) || + (ulp_command == CMD_ABORT_XRI_CN || + ulp_command == CMD_CLOSE_XRI_CN || + ulp_command == CMD_ABORT_XRI_WQE)) + return -EINVAL; + + return 0; +} + +/** + * lpfc_sli_validate_fcp_iocb - validate commands associated with a SCSI target * @iocbq: Pointer to driver iocb object. * @vport: Pointer to driver virtual port object. * @tgt_id: SCSI ID of the target. * @lun_id: LUN ID of the scsi device. * @ctx_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST * - * This function acts as an iocb filter for functions which abort or count - * all FCP iocbs pending on a lun/SCSI target/SCSI host. It will return + * This function acts as an iocb filter for validating a lun/SCSI target/SCSI + * host. + * + * It will return * 0 if the filtering criteria is met for the given iocb and will return * 1 if the filtering criteria is not met. * If ctx_cmd == LPFC_CTX_LUN, the function returns 0 only if the @@ -11457,17 +12695,10 @@ lpfc_sli_validate_fcp_iocb(struct lpfc_iocbq *iocbq, struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd ctx_cmd) { - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; int rc = 1; - if (iocbq->vport != vport) - return rc; - - if (!(iocbq->iocb_flag & LPFC_IO_FCP) || - !(iocbq->iocb_flag & LPFC_IO_ON_TXCMPLQ)) - return rc; - - lpfc_cmd = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); + lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); if (lpfc_cmd->pCmd == NULL) return rc; @@ -11522,16 +12753,33 @@ lpfc_sli_sum_iocb(struct lpfc_vport *vport, uint16_t tgt_id, uint64_t lun_id, struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *iocbq; int sum, i; + unsigned long iflags; + u8 ulp_command; - spin_lock_irq(&phba->hbalock); + spin_lock_irqsave(&phba->hbalock, iflags); for (i = 1, sum = 0; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; - if (lpfc_sli_validate_fcp_iocb (iocbq, vport, tgt_id, lun_id, - ctx_cmd) == 0) + if (!iocbq || iocbq->vport != vport) + continue; + if (!(iocbq->cmd_flag & LPFC_IO_FCP) || + !(iocbq->cmd_flag & LPFC_IO_ON_TXCMPLQ)) + continue; + + /* Include counting outstanding aborts */ + ulp_command = get_job_cmnd(phba, iocbq); + if (ulp_command == CMD_ABORT_XRI_CN || + ulp_command == CMD_CLOSE_XRI_CN || + ulp_command == CMD_ABORT_XRI_WQE) { + sum++; + continue; + } + + if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id, + ctx_cmd) == 0) sum++; } - spin_unlock_irq(&phba->hbalock); + spin_unlock_irqrestore(&phba->hbalock, iflags); return sum; } @@ -11551,13 +12799,15 @@ lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_iocbq *rspiocb) { lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "3096 ABORT_XRI_CN completing on rpi x%x " + "3096 ABORT_XRI_CX completing on rpi x%x " "original iotag x%x, abort cmd iotag x%x " "status 0x%x, reason 0x%x\n", + (phba->sli_rev == LPFC_SLI_REV4) ? + cmdiocb->sli4_xritag : cmdiocb->iocb.un.acxri.abortContextTag, - cmdiocb->iocb.un.acxri.abortIoTag, - cmdiocb->iotag, rspiocb->iocb.ulpStatus, - rspiocb->iocb.un.ulpWord[4]); + get_job_abtsiotag(phba, cmdiocb), + cmdiocb->iotag, get_job_ulpstatus(phba, rspiocb), + get_job_word4(phba, rspiocb)); lpfc_sli_release_iocbq(phba, cmdiocb); return; } @@ -11565,14 +12815,17 @@ lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, /** * lpfc_sli_abort_iocb - issue abort for all commands on a host/target/LUN * @vport: Pointer to virtual port. - * @pring: Pointer to driver SLI ring object. * @tgt_id: SCSI ID of the target. * @lun_id: LUN ID of the scsi device. * @abort_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST. * * This function sends an abort command for every SCSI command * associated with the given virtual port pending on the ring - * filtered by lpfc_sli_validate_fcp_iocb function. + * filtered by lpfc_sli_validate_fcp_iocb_for_abort and then + * lpfc_sli_validate_fcp_iocb function. The ordering for validation before + * submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort + * followed by lpfc_sli_validate_fcp_iocb. + * * When abort_cmd == LPFC_CTX_LUN, the function sends abort only to the * FCP iocbs associated with lun specified by tgt_id and lun_id * parameters @@ -11580,88 +12833,47 @@ lpfc_sli_abort_fcp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, * FCP iocbs associated with SCSI target specified by tgt_id parameter. * When abort_cmd == LPFC_CTX_HOST, the function sends abort to all * FCP iocbs associated with virtual port. + * The pring used for SLI3 is sli3_ring[LPFC_FCP_RING], for SLI4 + * lpfc_sli4_calc_ring is used. * This function returns number of iocbs it failed to abort. * This function is called with no locks held. **/ int -lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, - uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd abort_cmd) +lpfc_sli_abort_iocb(struct lpfc_vport *vport, u16 tgt_id, u64 lun_id, + lpfc_ctx_cmd abort_cmd) { struct lpfc_hba *phba = vport->phba; + struct lpfc_sli_ring *pring = NULL; struct lpfc_iocbq *iocbq; - struct lpfc_iocbq *abtsiocb; - struct lpfc_sli_ring *pring_s4; - IOCB_t *cmd = NULL; int errcnt = 0, ret_val = 0; + unsigned long iflags; int i; /* all I/Os are in process of being flushed */ - if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) + if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) return errcnt; for (i = 1; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; - if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id, - abort_cmd) != 0) + if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport)) continue; - /* - * If the iocbq is already being aborted, don't take a second - * action, but do count it. - */ - if (iocbq->iocb_flag & LPFC_DRIVER_ABORTED) + if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id, + abort_cmd) != 0) continue; - /* issue ABTS for this IOCB based on iotag */ - abtsiocb = lpfc_sli_get_iocbq(phba); - if (abtsiocb == NULL) { - errcnt++; - continue; + spin_lock_irqsave(&phba->hbalock, iflags); + if (phba->sli_rev == LPFC_SLI_REV3) { + pring = &phba->sli.sli3_ring[LPFC_FCP_RING]; + } else if (phba->sli_rev == LPFC_SLI_REV4) { + pring = lpfc_sli4_calc_ring(phba, iocbq); } - - /* indicate the IO is being aborted by the driver. */ - iocbq->iocb_flag |= LPFC_DRIVER_ABORTED; - - cmd = &iocbq->iocb; - abtsiocb->iocb.un.acxri.abortType = ABORT_TYPE_ABTS; - abtsiocb->iocb.un.acxri.abortContextTag = cmd->ulpContext; - if (phba->sli_rev == LPFC_SLI_REV4) - abtsiocb->iocb.un.acxri.abortIoTag = iocbq->sli4_xritag; - else - abtsiocb->iocb.un.acxri.abortIoTag = cmd->ulpIoTag; - abtsiocb->iocb.ulpLe = 1; - abtsiocb->iocb.ulpClass = cmd->ulpClass; - abtsiocb->vport = vport; - - /* ABTS WQE must go to the same WQ as the WQE to be aborted */ - abtsiocb->hba_wqidx = iocbq->hba_wqidx; - if (iocbq->iocb_flag & LPFC_IO_FCP) - abtsiocb->iocb_flag |= LPFC_USE_FCPWQIDX; - if (iocbq->iocb_flag & LPFC_IO_FOF) - abtsiocb->iocb_flag |= LPFC_IO_FOF; - - if (lpfc_is_link_up(phba)) - abtsiocb->iocb.ulpCommand = CMD_ABORT_XRI_CN; - else - abtsiocb->iocb.ulpCommand = CMD_CLOSE_XRI_CN; - - /* Setup callback routine and issue the command. */ - abtsiocb->iocb_cmpl = lpfc_sli_abort_fcp_cmpl; - if (phba->sli_rev == LPFC_SLI_REV4) { - pring_s4 = lpfc_sli4_calc_ring(phba, iocbq); - if (!pring_s4) - continue; - ret_val = lpfc_sli_issue_iocb(phba, pring_s4->ringno, - abtsiocb, 0); - } else - ret_val = lpfc_sli_issue_iocb(phba, pring->ringno, - abtsiocb, 0); - if (ret_val == IOCB_ERROR) { - lpfc_sli_release_iocbq(phba, abtsiocb); + ret_val = lpfc_sli_issue_abort_iotag(phba, pring, iocbq, + lpfc_sli_abort_fcp_cmpl); + spin_unlock_irqrestore(&phba->hbalock, iflags); + if (ret_val != IOCB_SUCCESS) errcnt++; - continue; - } } return errcnt; @@ -11673,11 +12885,15 @@ lpfc_sli_abort_iocb(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, * @pring: Pointer to driver SLI ring object. * @tgt_id: SCSI ID of the target. * @lun_id: LUN ID of the scsi device. - * @taskmgmt_cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST. + * @cmd: LPFC_CTX_LUN/LPFC_CTX_TGT/LPFC_CTX_HOST. * * This function sends an abort command for every SCSI command * associated with the given virtual port pending on the ring - * filtered by lpfc_sli_validate_fcp_iocb function. + * filtered by lpfc_sli_validate_fcp_iocb_for_abort and then + * lpfc_sli_validate_fcp_iocb function. The ordering for validation before + * submitting abort iocbs must be lpfc_sli_validate_fcp_iocb_for_abort + * followed by lpfc_sli_validate_fcp_iocb. + * * When taskmgmt_cmd == LPFC_CTX_LUN, the function sends abort only to the * FCP iocbs associated with lun specified by tgt_id and lun_id * parameters @@ -11694,86 +12910,120 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, uint16_t tgt_id, uint64_t lun_id, lpfc_ctx_cmd cmd) { struct lpfc_hba *phba = vport->phba; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; struct lpfc_iocbq *abtsiocbq; - struct lpfc_nodelist *ndlp; + struct lpfc_nodelist *ndlp = NULL; struct lpfc_iocbq *iocbq; - IOCB_t *icmd; int sum, i, ret_val; unsigned long iflags; - struct lpfc_sli_ring *pring_s4; - - spin_lock_irqsave(&phba->hbalock, iflags); + struct lpfc_sli_ring *pring_s4 = NULL; + u16 ulp_context, iotag, cqid = LPFC_WQE_CQ_ID_DEFAULT; + bool ia; /* all I/Os are in process of being flushed */ - if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) { - spin_unlock_irqrestore(&phba->hbalock, iflags); + if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) return 0; - } + sum = 0; + spin_lock_irqsave(&phba->hbalock, iflags); for (i = 1; i <= phba->sli.last_iotag; i++) { iocbq = phba->sli.iocbq_lookup[i]; + if (lpfc_sli_validate_fcp_iocb_for_abort(iocbq, vport)) + continue; + if (lpfc_sli_validate_fcp_iocb(iocbq, vport, tgt_id, lun_id, cmd) != 0) continue; + /* Guard against IO completion being called at same time */ + lpfc_cmd = container_of(iocbq, struct lpfc_io_buf, cur_iocbq); + spin_lock(&lpfc_cmd->buf_lock); + + if (!lpfc_cmd->pCmd) { + spin_unlock(&lpfc_cmd->buf_lock); + continue; + } + + if (phba->sli_rev == LPFC_SLI_REV4) { + pring_s4 = + phba->sli4_hba.hdwq[iocbq->hba_wqidx].io_wq->pring; + if (!pring_s4) { + spin_unlock(&lpfc_cmd->buf_lock); + continue; + } + /* Note: both hbalock and ring_lock must be set here */ + spin_lock(&pring_s4->ring_lock); + } + /* * If the iocbq is already being aborted, don't take a second * action, but do count it. */ - if (iocbq->iocb_flag & LPFC_DRIVER_ABORTED) + if ((iocbq->cmd_flag & LPFC_DRIVER_ABORTED) || + !(iocbq->cmd_flag & LPFC_IO_ON_TXCMPLQ)) { + if (phba->sli_rev == LPFC_SLI_REV4) + spin_unlock(&pring_s4->ring_lock); + spin_unlock(&lpfc_cmd->buf_lock); continue; + } /* issue ABTS for this IOCB based on iotag */ abtsiocbq = __lpfc_sli_get_iocbq(phba); - if (abtsiocbq == NULL) + if (!abtsiocbq) { + if (phba->sli_rev == LPFC_SLI_REV4) + spin_unlock(&pring_s4->ring_lock); + spin_unlock(&lpfc_cmd->buf_lock); continue; + } - icmd = &iocbq->iocb; - abtsiocbq->iocb.un.acxri.abortType = ABORT_TYPE_ABTS; - abtsiocbq->iocb.un.acxri.abortContextTag = icmd->ulpContext; - if (phba->sli_rev == LPFC_SLI_REV4) - abtsiocbq->iocb.un.acxri.abortIoTag = - iocbq->sli4_xritag; - else - abtsiocbq->iocb.un.acxri.abortIoTag = icmd->ulpIoTag; - abtsiocbq->iocb.ulpLe = 1; - abtsiocbq->iocb.ulpClass = icmd->ulpClass; - abtsiocbq->vport = vport; - - /* ABTS WQE must go to the same WQ as the WQE to be aborted */ - abtsiocbq->hba_wqidx = iocbq->hba_wqidx; - if (iocbq->iocb_flag & LPFC_IO_FCP) - abtsiocbq->iocb_flag |= LPFC_USE_FCPWQIDX; - if (iocbq->iocb_flag & LPFC_IO_FOF) - abtsiocbq->iocb_flag |= LPFC_IO_FOF; + if (phba->sli_rev == LPFC_SLI_REV4) { + iotag = abtsiocbq->iotag; + ulp_context = iocbq->sli4_xritag; + cqid = lpfc_cmd->hdwq->io_cq_map; + } else { + iotag = iocbq->iocb.ulpIoTag; + if (pring->ringno == LPFC_ELS_RING) { + ndlp = iocbq->ndlp; + ulp_context = ndlp->nlp_rpi; + } else { + ulp_context = iocbq->iocb.ulpContext; + } + } - lpfc_cmd = container_of(iocbq, struct lpfc_scsi_buf, cur_iocbq); ndlp = lpfc_cmd->rdata->pnode; if (lpfc_is_link_up(phba) && - (ndlp && ndlp->nlp_state == NLP_STE_MAPPED_NODE)) - abtsiocbq->iocb.ulpCommand = CMD_ABORT_XRI_CN; + (ndlp && ndlp->nlp_state == NLP_STE_MAPPED_NODE) && + !(phba->link_flag & LS_EXTERNAL_LOOPBACK)) + ia = false; else - abtsiocbq->iocb.ulpCommand = CMD_CLOSE_XRI_CN; + ia = true; + + lpfc_sli_prep_abort_xri(phba, abtsiocbq, ulp_context, iotag, + iocbq->iocb.ulpClass, cqid, + ia, false); + + abtsiocbq->vport = vport; + + /* ABTS WQE must go to the same WQ as the WQE to be aborted */ + abtsiocbq->hba_wqidx = iocbq->hba_wqidx; + if (iocbq->cmd_flag & LPFC_IO_FCP) + abtsiocbq->cmd_flag |= LPFC_USE_FCPWQIDX; + if (iocbq->cmd_flag & LPFC_IO_FOF) + abtsiocbq->cmd_flag |= LPFC_IO_FOF; /* Setup callback routine and issue the command. */ - abtsiocbq->iocb_cmpl = lpfc_sli_abort_fcp_cmpl; + abtsiocbq->cmd_cmpl = lpfc_sli_abort_fcp_cmpl; /* * Indicate the IO is being aborted by the driver and set * the caller's flag into the aborted IO. */ - iocbq->iocb_flag |= LPFC_DRIVER_ABORTED; + iocbq->cmd_flag |= LPFC_DRIVER_ABORTED; if (phba->sli_rev == LPFC_SLI_REV4) { - pring_s4 = lpfc_sli4_calc_ring(phba, abtsiocbq); - if (!pring_s4) - continue; - /* Note: both hbalock and ring_lock must be set here */ - spin_lock(&pring_s4->ring_lock); ret_val = __lpfc_sli_issue_iocb(phba, pring_s4->ringno, abtsiocbq, 0); spin_unlock(&pring_s4->ring_lock); @@ -11782,6 +13032,7 @@ lpfc_sli_abort_taskmgmt(struct lpfc_vport *vport, struct lpfc_sli_ring *pring, abtsiocbq, 0); } + spin_unlock(&lpfc_cmd->buf_lock); if (ret_val == IOCB_ERROR) __lpfc_sli_release_iocbq(phba, abtsiocbq); @@ -11816,10 +13067,11 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, { wait_queue_head_t *pdone_q; unsigned long iflags; - struct lpfc_scsi_buf *lpfc_cmd; + struct lpfc_io_buf *lpfc_cmd; + size_t offset = offsetof(struct lpfc_iocbq, wqe); spin_lock_irqsave(&phba->hbalock, iflags); - if (cmdiocbq->iocb_flag & LPFC_IO_WAKE_TMO) { + if (cmdiocbq->cmd_flag & LPFC_IO_WAKE_TMO) { /* * A time out has occurred for the iocb. If a time out @@ -11828,26 +13080,30 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, */ spin_unlock_irqrestore(&phba->hbalock, iflags); - cmdiocbq->iocb_cmpl = cmdiocbq->wait_iocb_cmpl; - cmdiocbq->wait_iocb_cmpl = NULL; - if (cmdiocbq->iocb_cmpl) - (cmdiocbq->iocb_cmpl)(phba, cmdiocbq, NULL); + cmdiocbq->cmd_cmpl = cmdiocbq->wait_cmd_cmpl; + cmdiocbq->wait_cmd_cmpl = NULL; + if (cmdiocbq->cmd_cmpl) + cmdiocbq->cmd_cmpl(phba, cmdiocbq, NULL); else lpfc_sli_release_iocbq(phba, cmdiocbq); return; } - cmdiocbq->iocb_flag |= LPFC_IO_WAKE; - if (cmdiocbq->context2 && rspiocbq) - memcpy(&((struct lpfc_iocbq *)cmdiocbq->context2)->iocb, - &rspiocbq->iocb, sizeof(IOCB_t)); + /* Copy the contents of the local rspiocb into the caller's buffer. */ + cmdiocbq->cmd_flag |= LPFC_IO_WAKE; + if (cmdiocbq->rsp_iocb && rspiocbq) + memcpy((char *)cmdiocbq->rsp_iocb + offset, + (char *)rspiocbq + offset, sizeof(*rspiocbq) - offset); /* Set the exchange busy flag for task management commands */ - if ((cmdiocbq->iocb_flag & LPFC_IO_FCP) && - !(cmdiocbq->iocb_flag & LPFC_IO_LIBDFC)) { - lpfc_cmd = container_of(cmdiocbq, struct lpfc_scsi_buf, - cur_iocbq); - lpfc_cmd->exch_busy = rspiocbq->iocb_flag & LPFC_EXCHANGE_BUSY; + if ((cmdiocbq->cmd_flag & LPFC_IO_FCP) && + !(cmdiocbq->cmd_flag & LPFC_IO_LIBDFC)) { + lpfc_cmd = container_of(cmdiocbq, struct lpfc_io_buf, + cur_iocbq); + if (rspiocbq && (rspiocbq->cmd_flag & LPFC_EXCHANGE_BUSY)) + lpfc_cmd->flags |= LPFC_SBUF_XBUSY; + else + lpfc_cmd->flags &= ~LPFC_SBUF_XBUSY; } pdone_q = cmdiocbq->context_un.wait_queue; @@ -11863,7 +13119,7 @@ lpfc_sli_wake_iocb_wait(struct lpfc_hba *phba, * @piocbq: Pointer to command iocb. * @flag: Flag to test. * - * This routine grabs the hbalock and then test the iocb_flag to + * This routine grabs the hbalock and then test the cmd_flag to * see if the passed in flag is set. * Returns: * 1 if flag is set. @@ -11877,7 +13133,7 @@ lpfc_chk_iocb_flg(struct lpfc_hba *phba, int ret; spin_lock_irqsave(&phba->hbalock, iflags); - ret = piocbq->iocb_flag & flag; + ret = piocbq->cmd_flag & flag; spin_unlock_irqrestore(&phba->hbalock, iflags); return ret; @@ -11886,20 +13142,20 @@ lpfc_chk_iocb_flg(struct lpfc_hba *phba, /** * lpfc_sli_issue_iocb_wait - Synchronous function to issue iocb commands * @phba: Pointer to HBA context object.. - * @pring: Pointer to sli ring. + * @ring_number: Ring number * @piocb: Pointer to command iocb. * @prspiocbq: Pointer to response iocb. * @timeout: Timeout in number of seconds. * * This function issues the iocb to firmware and waits for the - * iocb to complete. The iocb_cmpl field of the shall be used + * iocb to complete. The cmd_cmpl field of the shall be used * to handle iocbs which time out. If the field is NULL, the * function shall free the iocbq structure. If more clean up is * needed, the caller is expected to provide a completion function * that will provide the needed clean up. If the iocb command is * not completed within timeout seconds, the function will either - * free the iocbq structure (if iocb_cmpl == NULL) or execute the - * completion function set in the iocb_cmpl field and then return + * free the iocbq structure (if cmd_cmpl == NULL) or execute the + * completion function set in the cmd_cmpl field and then return * a status of IOCB_TIMEDOUT. The caller should not free the iocb * resources if this function returns IOCB_TIMEDOUT. * The function waits for the iocb completion using an @@ -11911,7 +13167,7 @@ lpfc_chk_iocb_flg(struct lpfc_hba *phba, * This function assumes that the iocb completions occur while * this function sleep. So, this function cannot be called from * the thread which process iocb completion for this ring. - * This function clears the iocb_flag of the iocb object before + * This function clears the cmd_flag of the iocb object before * issuing the iocb and the iocb completion handler sets this * flag and wakes this thread when the iocb completes. * The contents of the response iocb will be copied to prspiocbq @@ -11937,24 +13193,26 @@ lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba, unsigned long iflags; bool iocb_completed = true; - if (phba->sli_rev >= LPFC_SLI_REV4) + if (phba->sli_rev >= LPFC_SLI_REV4) { + lpfc_sli_prep_wqe(phba, piocb); + pring = lpfc_sli4_calc_ring(phba, piocb); - else + } else pring = &phba->sli.sli3_ring[ring_number]; /* - * If the caller has provided a response iocbq buffer, then context2 + * If the caller has provided a response iocbq buffer, then rsp_iocb * is NULL or its an error. */ if (prspiocbq) { - if (piocb->context2) + if (piocb->rsp_iocb) return IOCB_ERROR; - piocb->context2 = prspiocbq; + piocb->rsp_iocb = prspiocbq; } - piocb->wait_iocb_cmpl = piocb->iocb_cmpl; - piocb->iocb_cmpl = lpfc_sli_wake_iocb_wait; + piocb->wait_cmd_cmpl = piocb->cmd_cmpl; + piocb->cmd_cmpl = lpfc_sli_wake_iocb_wait; piocb->context_un.wait_queue = &done_q; - piocb->iocb_flag &= ~(LPFC_IO_WAKE | LPFC_IO_WAKE_TMO); + piocb->cmd_flag &= ~(LPFC_IO_WAKE | LPFC_IO_WAKE_TMO); if (phba->cfg_poll & DISABLE_FCP_RING_INT) { if (lpfc_readl(phba->HCregaddr, &creg_val)) @@ -11967,12 +13225,12 @@ lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba, retval = lpfc_sli_issue_iocb(phba, ring_number, piocb, SLI_IOCB_RET_IOCB); if (retval == IOCB_SUCCESS) { - timeout_req = msecs_to_jiffies(timeout * 1000); + timeout_req = secs_to_jiffies(timeout); timeleft = wait_event_timeout(done_q, lpfc_chk_iocb_flg(phba, piocb, LPFC_IO_WAKE), timeout_req); spin_lock_irqsave(&phba->hbalock, iflags); - if (!(piocb->iocb_flag & LPFC_IO_WAKE)) { + if (!(piocb->cmd_flag & LPFC_IO_WAKE)) { /* * IOCB timed out. Inform the wake iocb wait @@ -11980,7 +13238,7 @@ lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba, */ iocb_completed = false; - piocb->iocb_flag |= LPFC_IO_WAKE_TMO; + piocb->cmd_flag |= LPFC_IO_WAKE_TMO; } spin_unlock_irqrestore(&phba->hbalock, iflags); if (iocb_completed) { @@ -11992,12 +13250,12 @@ lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba, * completed. Not that it completed successfully. * */ } else if (timeleft == 0) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0338 IOCB wait timeout error - no " "wake response Data x%x\n", timeout); retval = IOCB_TIMEDOUT; } else { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0330 IOCB wake NOT set, " "Data x%x x%lx\n", timeout, (timeleft / jiffies)); @@ -12032,10 +13290,10 @@ lpfc_sli_issue_iocb_wait(struct lpfc_hba *phba, } if (prspiocbq) - piocb->context2 = NULL; + piocb->rsp_iocb = NULL; piocb->context_un.wait_queue = NULL; - piocb->iocb_cmpl = NULL; + piocb->cmd_cmpl = NULL; return retval; } @@ -12077,17 +13335,16 @@ lpfc_sli_issue_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq, /* setup wake call as IOCB callback */ pmboxq->mbox_cmpl = lpfc_sli_wake_mbox_wait; - /* setup context3 field to pass wait_queue pointer to wake function */ + /* setup ctx_u field to pass wait_queue pointer to wake function */ init_completion(&mbox_done); - pmboxq->context3 = &mbox_done; + pmboxq->ctx_u.mbox_wait = &mbox_done; /* now issue the command */ retval = lpfc_sli_issue_mbox(phba, pmboxq, MBX_NOWAIT); if (retval == MBX_BUSY || retval == MBX_SUCCESS) { - wait_for_completion_timeout(&mbox_done, - msecs_to_jiffies(timeout * 1000)); + wait_for_completion_timeout(&mbox_done, secs_to_jiffies(timeout)); spin_lock_irqsave(&phba->hbalock, flag); - pmboxq->context3 = NULL; + pmboxq->ctx_u.mbox_wait = NULL; /* * if LPFC_MBX_WAKE flag is set the mailbox is completed * else do not free the resources. @@ -12106,6 +13363,7 @@ lpfc_sli_issue_mbox_wait(struct lpfc_hba *phba, LPFC_MBOXQ_t *pmboxq, /** * lpfc_sli_mbox_sys_shutdown - shutdown mailbox command sub-system * @phba: Pointer to HBA context. + * @mbx_action: Mailbox shutdown options. * * This function is called to shutdown the driver's mailbox sub-system. * It first marks the mailbox sub-system is in a block state to prevent @@ -12130,7 +13388,7 @@ lpfc_sli_mbox_sys_shutdown(struct lpfc_hba *phba, int mbx_action) lpfc_sli_mbox_sys_flush(phba); return; } - timeout = msecs_to_jiffies(LPFC_MBOX_TMO * 1000) + jiffies; + timeout = secs_to_jiffies(LPFC_MBOX_TMO) + jiffies; /* Disable softirqs, including timers from obtaining phba->hbalock */ local_bh_disable(); @@ -12143,9 +13401,8 @@ lpfc_sli_mbox_sys_shutdown(struct lpfc_hba *phba, int mbx_action) * command to be gracefully completed by firmware. */ if (phba->sli.mbox_active) - timeout = msecs_to_jiffies(lpfc_mbox_tmo_val(phba, - phba->sli.mbox_active) * - 1000) + jiffies; + timeout = secs_to_jiffies(lpfc_mbox_tmo_val(phba, + phba->sli.mbox_active)) + jiffies; spin_unlock_irq(&phba->hbalock); /* Enable softirqs again, done with phba->hbalock */ @@ -12199,7 +13456,7 @@ lpfc_sli_eratt_read(struct lpfc_hba *phba) if ((HS_FFER1 & phba->work_hs) && ((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 | HS_FFER6 | HS_FFER7 | HS_FFER8) & phba->work_hs)) { - phba->hba_flag |= DEFER_ERATT; + set_bit(DEFER_ERATT, &phba->hba_flag); /* Clear all interrupt enable conditions */ writel(0, phba->HCregaddr); readl(phba->HCregaddr); @@ -12208,7 +13465,7 @@ lpfc_sli_eratt_read(struct lpfc_hba *phba) /* Set the driver HA work bitmap */ phba->work_ha |= HA_ERATT; /* Indicate polling handles this ERATT */ - phba->hba_flag |= HBA_ERATT_HANDLED; + set_bit(HBA_ERATT_HANDLED, &phba->hba_flag); return 1; } return 0; @@ -12219,7 +13476,7 @@ unplug_err: /* Set the driver HA work bitmap */ phba->work_ha |= HA_ERATT; /* Indicate polling handles this ERATT */ - phba->hba_flag |= HBA_ERATT_HANDLED; + set_bit(HBA_ERATT_HANDLED, &phba->hba_flag); return 1; } @@ -12240,6 +13497,7 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba) uint32_t uerr_sta_hi, uerr_sta_lo; uint32_t if_type, portsmphr; struct lpfc_register portstat_reg; + u32 logmask; /* * For now, use the SLI4 device internal unrecoverable error @@ -12254,12 +13512,12 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba) &uerr_sta_hi)) { phba->work_hs |= UNPLUG_ERR; phba->work_ha |= HA_ERATT; - phba->hba_flag |= HBA_ERATT_HANDLED; + set_bit(HBA_ERATT_HANDLED, &phba->hba_flag); return 1; } if ((~phba->sli4_hba.ue_mask_lo & uerr_sta_lo) || (~phba->sli4_hba.ue_mask_hi & uerr_sta_hi)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "1423 HBA Unrecoverable error: " "uerr_lo_reg=0x%x, uerr_hi_reg=0x%x, " "ue_mask_lo_reg=0x%x, " @@ -12270,7 +13528,7 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba) phba->work_status[0] = uerr_sta_lo; phba->work_status[1] = uerr_sta_hi; phba->work_ha |= HA_ERATT; - phba->hba_flag |= HBA_ERATT_HANDLED; + set_bit(HBA_ERATT_HANDLED, &phba->hba_flag); return 1; } break; @@ -12282,7 +13540,7 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba) &portsmphr)){ phba->work_hs |= UNPLUG_ERR; phba->work_ha |= HA_ERATT; - phba->hba_flag |= HBA_ERATT_HANDLED; + set_bit(HBA_ERATT_HANDLED, &phba->hba_flag); return 1; } if (bf_get(lpfc_sliport_status_err, &portstat_reg)) { @@ -12290,7 +13548,12 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba) readl(phba->sli4_hba.u.if_type2.ERR1regaddr); phba->work_status[1] = readl(phba->sli4_hba.u.if_type2.ERR2regaddr); - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + logmask = LOG_TRACE_EVENT; + if (phba->work_status[0] == + SLIPORT_ERR1_REG_ERR_CODE_2 && + phba->work_status[1] == SLIPORT_ERR2_REG_FW_RESTART) + logmask = LOG_SLI; + lpfc_printf_log(phba, KERN_ERR, logmask, "2885 Port Status Event: " "port status reg 0x%x, " "port smphr reg 0x%x, " @@ -12300,13 +13563,13 @@ lpfc_sli4_eratt_read(struct lpfc_hba *phba) phba->work_status[0], phba->work_status[1]); phba->work_ha |= HA_ERATT; - phba->hba_flag |= HBA_ERATT_HANDLED; + set_bit(HBA_ERATT_HANDLED, &phba->hba_flag); return 1; } break; case LPFC_SLI_INTF_IF_TYPE_1: default: - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2886 HBA Error Attention on unsupported " "if type %d.", if_type); return 1; @@ -12337,22 +13600,18 @@ lpfc_sli_check_eratt(struct lpfc_hba *phba) return 0; /* Check if interrupt handler handles this ERATT */ - spin_lock_irq(&phba->hbalock); - if (phba->hba_flag & HBA_ERATT_HANDLED) { + if (test_bit(HBA_ERATT_HANDLED, &phba->hba_flag)) /* Interrupt handler has handled ERATT */ - spin_unlock_irq(&phba->hbalock); return 0; - } /* * If there is deferred error attention, do not check for error * attention */ - if (unlikely(phba->hba_flag & DEFER_ERATT)) { - spin_unlock_irq(&phba->hbalock); + if (unlikely(test_bit(DEFER_ERATT, &phba->hba_flag))) return 0; - } + spin_lock_irq(&phba->hbalock); /* If PCI channel is offline, don't process it */ if (unlikely(pci_channel_offline(phba->pcidev))) { spin_unlock_irq(&phba->hbalock); @@ -12370,7 +13629,7 @@ lpfc_sli_check_eratt(struct lpfc_hba *phba) ha_copy = lpfc_sli4_eratt_read(phba); break; default: - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0299 Invalid SLI revision (%d)\n", phba->sli_rev); ha_copy = 0; @@ -12474,19 +13733,17 @@ lpfc_sli_sp_intr_handler(int irq, void *dev_id) ha_copy &= ~HA_ERATT; /* Check the need for handling ERATT in interrupt handler */ if (ha_copy & HA_ERATT) { - if (phba->hba_flag & HBA_ERATT_HANDLED) + if (test_and_set_bit(HBA_ERATT_HANDLED, + &phba->hba_flag)) /* ERATT polling has handled ERATT */ ha_copy &= ~HA_ERATT; - else - /* Indicate interrupt handler handles ERATT */ - phba->hba_flag |= HBA_ERATT_HANDLED; } /* * If there is deferred error attention, do not check for any * interrupt. */ - if (unlikely(phba->hba_flag & DEFER_ERATT)) { + if (unlikely(test_bit(DEFER_ERATT, &phba->hba_flag))) { spin_unlock_irqrestore(&phba->hbalock, iflag); return IRQ_NONE; } @@ -12582,7 +13839,7 @@ lpfc_sli_sp_intr_handler(int irq, void *dev_id) ((HS_FFER2 | HS_FFER3 | HS_FFER4 | HS_FFER5 | HS_FFER6 | HS_FFER7 | HS_FFER8) & phba->work_hs)) { - phba->hba_flag |= DEFER_ERATT; + set_bit(DEFER_ERATT, &phba->hba_flag); /* Clear all interrupt enable conditions */ writel(0, phba->HCregaddr); readl(phba->HCregaddr); @@ -12603,8 +13860,7 @@ lpfc_sli_sp_intr_handler(int irq, void *dev_id) * Stray Mailbox Interrupt, mbxCommand <cmd> * mbxStatus <status> */ - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | - LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "(%d):0304 Stray Mailbox " "Interrupt mbxCommand x%x " "mbxStatus x%x\n", @@ -12617,15 +13873,15 @@ lpfc_sli_sp_intr_handler(int irq, void *dev_id) phba->sli.mbox_active = NULL; spin_unlock_irqrestore(&phba->hbalock, iflag); phba->last_completion_time = jiffies; - del_timer(&phba->sli.mbox_tmo); + timer_delete(&phba->sli.mbox_tmo); if (pmb->mbox_cmpl) { lpfc_sli_pcimem_bcopy(mbox, pmbox, MAILBOX_CMD_SIZE); if (pmb->out_ext_byte_len && - pmb->ctx_buf) + pmb->ext_buf) lpfc_sli_pcimem_bcopy( phba->mbox_ext, - pmb->ctx_buf, + pmb->ext_buf, pmb->out_ext_byte_len); } if (pmb->mbox_flag & LPFC_MBX_IMED_UNREG) { @@ -12639,10 +13895,8 @@ lpfc_sli_sp_intr_handler(int irq, void *dev_id) pmbox->un.varWords[0], 0); if (!pmbox->mbxStatus) { - mp = (struct lpfc_dmabuf *) - (pmb->ctx_buf); - ndlp = (struct lpfc_nodelist *) - pmb->ctx_ndlp; + mp = pmb->ctx_buf; + ndlp = pmb->ctx_ndlp; /* Reg_LOGIN of dflt RPI was * successful. new lets get @@ -12664,7 +13918,7 @@ lpfc_sli_sp_intr_handler(int irq, void *dev_id) if (rc != MBX_BUSY) lpfc_printf_log(phba, KERN_ERR, - LOG_MBOX | LOG_SLI, + LOG_TRACE_EVENT, "0350 rc should have" "been MBX_BUSY\n"); if (rc != MBX_NOT_FINISHED) @@ -12679,7 +13933,21 @@ lpfc_sli_sp_intr_handler(int irq, void *dev_id) spin_unlock_irqrestore( &phba->pport->work_port_lock, iflag); - lpfc_mbox_cmpl_put(phba, pmb); + + /* Do NOT queue MBX_HEARTBEAT to the worker + * thread for processing. + */ + if (pmbox->mbxCommand == MBX_HEARTBEAT) { + /* Process mbox now */ + phba->sli.mbox_active = NULL; + phba->sli.sli_flag &= + ~LPFC_SLI_MBOX_ACTIVE; + if (pmb->mbox_cmpl) + pmb->mbox_cmpl(phba, pmb); + } else { + /* Queue to worker thread to process */ + lpfc_mbox_cmpl_put(phba, pmb); + } } } else spin_unlock_irqrestore(&phba->hbalock, iflag); @@ -12693,8 +13961,9 @@ send_current_mbox: MBX_NOWAIT); } while (rc == MBX_NOT_FINISHED); if (rc != MBX_SUCCESS) - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | - LOG_SLI, "0349 rc should be " + lpfc_printf_log(phba, KERN_ERR, + LOG_TRACE_EVENT, + "0349 rc should be " "MBX_SUCCESS\n"); } @@ -12757,16 +14026,16 @@ lpfc_sli_fp_intr_handler(int irq, void *dev_id) /* Need to read HA REG for FCP ring and other ring events */ if (lpfc_readl(phba->HAregaddr, &ha_copy)) return IRQ_HANDLED; - /* Clear up only attention source related to fast-path */ - spin_lock_irqsave(&phba->hbalock, iflag); + /* * If there is deferred error attention, do not check for * any interrupt. */ - if (unlikely(phba->hba_flag & DEFER_ERATT)) { - spin_unlock_irqrestore(&phba->hbalock, iflag); + if (unlikely(test_bit(DEFER_ERATT, &phba->hba_flag))) return IRQ_NONE; - } + + /* Clear up only attention source related to fast-path */ + spin_lock_irqsave(&phba->hbalock, iflag); writel((ha_copy & (HA_R0_CLR_MSK | HA_R1_CLR_MSK)), phba->HAregaddr); readl(phba->HAregaddr); /* flush */ @@ -12849,18 +14118,15 @@ lpfc_sli_intr_handler(int irq, void *dev_id) spin_unlock(&phba->hbalock); return IRQ_NONE; } else if (phba->ha_copy & HA_ERATT) { - if (phba->hba_flag & HBA_ERATT_HANDLED) + if (test_and_set_bit(HBA_ERATT_HANDLED, &phba->hba_flag)) /* ERATT polling has handled ERATT */ phba->ha_copy &= ~HA_ERATT; - else - /* Indicate interrupt handler handles ERATT */ - phba->hba_flag |= HBA_ERATT_HANDLED; } /* * If there is deferred error attention, do not check for any interrupt. */ - if (unlikely(phba->hba_flag & DEFER_ERATT)) { + if (unlikely(test_bit(DEFER_ERATT, &phba->hba_flag))) { spin_unlock(&phba->hbalock); return IRQ_NONE; } @@ -12919,35 +14185,6 @@ lpfc_sli_intr_handler(int irq, void *dev_id) } /* lpfc_sli_intr_handler */ /** - * lpfc_sli4_fcp_xri_abort_event_proc - Process fcp xri abort event - * @phba: pointer to lpfc hba data structure. - * - * This routine is invoked by the worker thread to process all the pending - * SLI4 FCP abort XRI events. - **/ -void lpfc_sli4_fcp_xri_abort_event_proc(struct lpfc_hba *phba) -{ - struct lpfc_cq_event *cq_event; - - /* First, declare the fcp xri abort event has been handled */ - spin_lock_irq(&phba->hbalock); - phba->hba_flag &= ~FCP_XRI_ABORT_EVENT; - spin_unlock_irq(&phba->hbalock); - /* Now, handle all the fcp xri abort events */ - while (!list_empty(&phba->sli4_hba.sp_fcp_xri_aborted_work_queue)) { - /* Get the first event from the head of the event queue */ - spin_lock_irq(&phba->hbalock); - list_remove_head(&phba->sli4_hba.sp_fcp_xri_aborted_work_queue, - cq_event, struct lpfc_cq_event, list); - spin_unlock_irq(&phba->hbalock); - /* Notify aborted XRI for FCP work queue */ - lpfc_sli4_fcp_xri_aborted(phba, &cq_event->cqe.wcqe_axri); - /* Free the event processed back to the free pool */ - lpfc_sli4_cq_event_release(phba, cq_event); - } -} - -/** * lpfc_sli4_els_xri_abort_event_proc - Process els xri abort event * @phba: pointer to lpfc hba data structure. * @@ -12957,155 +14194,44 @@ void lpfc_sli4_fcp_xri_abort_event_proc(struct lpfc_hba *phba) void lpfc_sli4_els_xri_abort_event_proc(struct lpfc_hba *phba) { struct lpfc_cq_event *cq_event; + unsigned long iflags; /* First, declare the els xri abort event has been handled */ - spin_lock_irq(&phba->hbalock); - phba->hba_flag &= ~ELS_XRI_ABORT_EVENT; - spin_unlock_irq(&phba->hbalock); + clear_bit(ELS_XRI_ABORT_EVENT, &phba->hba_flag); + /* Now, handle all the els xri abort events */ + spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock, iflags); while (!list_empty(&phba->sli4_hba.sp_els_xri_aborted_work_queue)) { /* Get the first event from the head of the event queue */ - spin_lock_irq(&phba->hbalock); list_remove_head(&phba->sli4_hba.sp_els_xri_aborted_work_queue, cq_event, struct lpfc_cq_event, list); - spin_unlock_irq(&phba->hbalock); + spin_unlock_irqrestore(&phba->sli4_hba.els_xri_abrt_list_lock, + iflags); /* Notify aborted XRI for ELS work queue */ lpfc_sli4_els_xri_aborted(phba, &cq_event->cqe.wcqe_axri); + /* Free the event processed back to the free pool */ lpfc_sli4_cq_event_release(phba, cq_event); + spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock, + iflags); } + spin_unlock_irqrestore(&phba->sli4_hba.els_xri_abrt_list_lock, iflags); } /** - * lpfc_sli4_iocb_param_transfer - Transfer pIocbOut and cmpl status to pIocbIn - * @phba: pointer to lpfc hba data structure - * @pIocbIn: pointer to the rspiocbq - * @pIocbOut: pointer to the cmdiocbq - * @wcqe: pointer to the complete wcqe - * - * This routine transfers the fields of a command iocbq to a response iocbq - * by copying all the IOCB fields from command iocbq and transferring the - * completion status information from the complete wcqe. - **/ -static void -lpfc_sli4_iocb_param_transfer(struct lpfc_hba *phba, - struct lpfc_iocbq *pIocbIn, - struct lpfc_iocbq *pIocbOut, - struct lpfc_wcqe_complete *wcqe) -{ - int numBdes, i; - unsigned long iflags; - uint32_t status, max_response; - struct lpfc_dmabuf *dmabuf; - struct ulp_bde64 *bpl, bde; - size_t offset = offsetof(struct lpfc_iocbq, iocb); - - memcpy((char *)pIocbIn + offset, (char *)pIocbOut + offset, - sizeof(struct lpfc_iocbq) - offset); - /* Map WCQE parameters into irspiocb parameters */ - status = bf_get(lpfc_wcqe_c_status, wcqe); - pIocbIn->iocb.ulpStatus = (status & LPFC_IOCB_STATUS_MASK); - if (pIocbOut->iocb_flag & LPFC_IO_FCP) - if (pIocbIn->iocb.ulpStatus == IOSTAT_FCP_RSP_ERROR) - pIocbIn->iocb.un.fcpi.fcpi_parm = - pIocbOut->iocb.un.fcpi.fcpi_parm - - wcqe->total_data_placed; - else - pIocbIn->iocb.un.ulpWord[4] = wcqe->parameter; - else { - pIocbIn->iocb.un.ulpWord[4] = wcqe->parameter; - switch (pIocbOut->iocb.ulpCommand) { - case CMD_ELS_REQUEST64_CR: - dmabuf = (struct lpfc_dmabuf *)pIocbOut->context3; - bpl = (struct ulp_bde64 *)dmabuf->virt; - bde.tus.w = le32_to_cpu(bpl[1].tus.w); - max_response = bde.tus.f.bdeSize; - break; - case CMD_GEN_REQUEST64_CR: - max_response = 0; - if (!pIocbOut->context3) - break; - numBdes = pIocbOut->iocb.un.genreq64.bdl.bdeSize/ - sizeof(struct ulp_bde64); - dmabuf = (struct lpfc_dmabuf *)pIocbOut->context3; - bpl = (struct ulp_bde64 *)dmabuf->virt; - for (i = 0; i < numBdes; i++) { - bde.tus.w = le32_to_cpu(bpl[i].tus.w); - if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) - max_response += bde.tus.f.bdeSize; - } - break; - default: - max_response = wcqe->total_data_placed; - break; - } - if (max_response < wcqe->total_data_placed) - pIocbIn->iocb.un.genreq64.bdl.bdeSize = max_response; - else - pIocbIn->iocb.un.genreq64.bdl.bdeSize = - wcqe->total_data_placed; - } - - /* Convert BG errors for completion status */ - if (status == CQE_STATUS_DI_ERROR) { - pIocbIn->iocb.ulpStatus = IOSTAT_LOCAL_REJECT; - - if (bf_get(lpfc_wcqe_c_bg_edir, wcqe)) - pIocbIn->iocb.un.ulpWord[4] = IOERR_RX_DMA_FAILED; - else - pIocbIn->iocb.un.ulpWord[4] = IOERR_TX_DMA_FAILED; - - pIocbIn->iocb.unsli3.sli3_bg.bgstat = 0; - if (bf_get(lpfc_wcqe_c_bg_ge, wcqe)) /* Guard Check failed */ - pIocbIn->iocb.unsli3.sli3_bg.bgstat |= - BGS_GUARD_ERR_MASK; - if (bf_get(lpfc_wcqe_c_bg_ae, wcqe)) /* App Tag Check failed */ - pIocbIn->iocb.unsli3.sli3_bg.bgstat |= - BGS_APPTAG_ERR_MASK; - if (bf_get(lpfc_wcqe_c_bg_re, wcqe)) /* Ref Tag Check failed */ - pIocbIn->iocb.unsli3.sli3_bg.bgstat |= - BGS_REFTAG_ERR_MASK; - - /* Check to see if there was any good data before the error */ - if (bf_get(lpfc_wcqe_c_bg_tdpv, wcqe)) { - pIocbIn->iocb.unsli3.sli3_bg.bgstat |= - BGS_HI_WATER_MARK_PRESENT_MASK; - pIocbIn->iocb.unsli3.sli3_bg.bghm = - wcqe->total_data_placed; - } - - /* - * Set ALL the error bits to indicate we don't know what - * type of error it is. - */ - if (!pIocbIn->iocb.unsli3.sli3_bg.bgstat) - pIocbIn->iocb.unsli3.sli3_bg.bgstat |= - (BGS_REFTAG_ERR_MASK | BGS_APPTAG_ERR_MASK | - BGS_GUARD_ERR_MASK); - } - - /* Pick up HBA exchange busy condition */ - if (bf_get(lpfc_wcqe_c_xb, wcqe)) { - spin_lock_irqsave(&phba->hbalock, iflags); - pIocbIn->iocb_flag |= LPFC_EXCHANGE_BUSY; - spin_unlock_irqrestore(&phba->hbalock, iflags); - } -} - -/** - * lpfc_sli4_els_wcqe_to_rspiocbq - Get response iocbq from els wcqe + * lpfc_sli4_els_preprocess_rspiocbq - Get response iocbq from els wcqe * @phba: Pointer to HBA context object. - * @wcqe: Pointer to work-queue completion queue entry. + * @irspiocbq: Pointer to work-queue completion queue entry. * * This routine handles an ELS work-queue completion event and construct - * a pseudo response ELS IODBQ from the SLI4 ELS WCQE for the common + * a pseudo response ELS IOCBQ from the SLI4 ELS WCQE for the common * discovery engine to handle. * * Return: Pointer to the receive IOCBQ, NULL otherwise. **/ static struct lpfc_iocbq * -lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba, - struct lpfc_iocbq *irspiocbq) +lpfc_sli4_els_preprocess_rspiocbq(struct lpfc_hba *phba, + struct lpfc_iocbq *irspiocbq) { struct lpfc_sli_ring *pring; struct lpfc_iocbq *cmdiocbq; @@ -13133,12 +14259,18 @@ lpfc_sli4_els_wcqe_to_rspiocbq(struct lpfc_hba *phba, return NULL; } + memcpy(&irspiocbq->wqe, &cmdiocbq->wqe, sizeof(union lpfc_wqe128)); + memcpy(&irspiocbq->wcqe_cmpl, wcqe, sizeof(*wcqe)); + /* Put the iocb back on the txcmplq */ lpfc_sli_ringtxcmpl_put(phba, pring, cmdiocbq); spin_unlock_irqrestore(&pring->ring_lock, iflags); - /* Fake the irspiocbq and copy necessary response information */ - lpfc_sli4_iocb_param_transfer(phba, irspiocbq, cmdiocbq, wcqe); + if (bf_get(lpfc_wcqe_c_xb, wcqe)) { + spin_lock_irqsave(&phba->hbalock, iflags); + irspiocbq->cmd_flag |= LPFC_EXCHANGE_BUSY; + spin_unlock_irqrestore(&phba->hbalock, iflags); + } return irspiocbq; } @@ -13151,7 +14283,7 @@ lpfc_cq_event_setup(struct lpfc_hba *phba, void *entry, int size) /* Allocate a new internal CQ_EVENT entry */ cq_event = lpfc_sli4_cq_event_alloc(phba); if (!cq_event) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0602 Failed to alloc CQ_EVENT entry\n"); return NULL; } @@ -13162,11 +14294,11 @@ lpfc_cq_event_setup(struct lpfc_hba *phba, void *entry, int size) } /** - * lpfc_sli4_sp_handle_async_event - Handle an asynchroous event + * lpfc_sli4_sp_handle_async_event - Handle an asynchronous event * @phba: Pointer to HBA context object. - * @cqe: Pointer to mailbox completion queue entry. + * @mcqe: Pointer to mailbox completion queue entry. * - * This routine process a mailbox completion queue entry with asynchrous + * This routine process a mailbox completion queue entry with asynchronous * event. * * Return: true if work posted to worker thread, otherwise false. @@ -13185,11 +14317,13 @@ lpfc_sli4_sp_handle_async_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe) cq_event = lpfc_cq_event_setup(phba, mcqe, sizeof(struct lpfc_mcqe)); if (!cq_event) return false; - spin_lock_irqsave(&phba->hbalock, iflags); + + spin_lock_irqsave(&phba->sli4_hba.asynce_list_lock, iflags); list_add_tail(&cq_event->list, &phba->sli4_hba.sp_asynce_work_queue); + spin_unlock_irqrestore(&phba->sli4_hba.asynce_list_lock, iflags); + /* Set the async event flag */ - phba->hba_flag |= ASYNC_EVENT; - spin_unlock_irqrestore(&phba->hbalock, iflags); + set_bit(ASYNC_EVENT, &phba->hba_flag); return true; } @@ -13197,7 +14331,7 @@ lpfc_sli4_sp_handle_async_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe) /** * lpfc_sli4_sp_handle_mbox_event - Handle a mailbox completion event * @phba: Pointer to HBA context object. - * @cqe: Pointer to mailbox completion queue entry. + * @mcqe: Pointer to mailbox completion queue entry. * * This routine process a mailbox completion queue entry with mailbox * completion event. @@ -13226,7 +14360,7 @@ lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe) spin_lock_irqsave(&phba->hbalock, iflags); pmb = phba->sli.mbox_active; if (unlikely(!pmb)) { - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "1832 No pending MBOX command to handle\n"); spin_unlock_irqrestore(&phba->hbalock, iflags); goto out_no_mqe_complete; @@ -13239,7 +14373,7 @@ lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe) /* Reset heartbeat timer */ phba->last_completion_time = jiffies; - del_timer(&phba->sli.mbox_tmo); + timer_delete(&phba->sli.mbox_tmo); /* Move mbox data to caller's mailbox region, do endian swapping */ if (pmb->mbox_cmpl && mbox) @@ -13262,21 +14396,32 @@ lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe) mcqe_status, pmbox->un.varWords[0], 0); if (mcqe_status == MB_CQE_STATUS_SUCCESS) { - mp = (struct lpfc_dmabuf *)(pmb->ctx_buf); - ndlp = (struct lpfc_nodelist *)pmb->ctx_ndlp; - /* Reg_LOGIN of dflt RPI was successful. Now lets get - * RID of the PPI using the same mbox buffer. + mp = pmb->ctx_buf; + ndlp = pmb->ctx_ndlp; + + /* Reg_LOGIN of dflt RPI was successful. Mark the + * node as having an UNREG_LOGIN in progress to stop + * an unsolicited PLOGI from the same NPortId from + * starting another mailbox transaction. */ + set_bit(NLP_UNREG_INP, &ndlp->nlp_flag); lpfc_unreg_login(phba, vport->vpi, pmbox->un.varWords[0], pmb); pmb->mbox_cmpl = lpfc_mbx_cmpl_dflt_rpi; pmb->ctx_buf = mp; + + /* No reference taken here. This is a default + * RPI reg/immediate unreg cycle. The reference was + * taken in the reg rpi path and is released when + * this mailbox completes. + */ pmb->ctx_ndlp = ndlp; pmb->vport = vport; rc = lpfc_sli_issue_mbox(phba, pmb, MBX_NOWAIT); if (rc != MBX_BUSY) - lpfc_printf_log(phba, KERN_ERR, LOG_MBOX | - LOG_SLI, "0385 rc should " + lpfc_printf_log(phba, KERN_ERR, + LOG_TRACE_EVENT, + "0385 rc should " "have been MBX_BUSY\n"); if (rc != MBX_NOT_FINISHED) goto send_current_mbox; @@ -13286,7 +14431,26 @@ lpfc_sli4_sp_handle_mbox_event(struct lpfc_hba *phba, struct lpfc_mcqe *mcqe) phba->pport->work_port_events &= ~WORKER_MBOX_TMO; spin_unlock_irqrestore(&phba->pport->work_port_lock, iflags); - /* There is mailbox completion work to do */ + /* Do NOT queue MBX_HEARTBEAT to the worker thread for processing. */ + if (pmbox->mbxCommand == MBX_HEARTBEAT) { + spin_lock_irqsave(&phba->hbalock, iflags); + /* Release the mailbox command posting token */ + phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE; + phba->sli.mbox_active = NULL; + if (bf_get(lpfc_trailer_consumed, mcqe)) + lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq); + spin_unlock_irqrestore(&phba->hbalock, iflags); + + /* Post the next mbox command, if there is one */ + lpfc_sli4_post_async_mbox(phba); + + /* Process cmpl now */ + if (pmb->mbox_cmpl) + pmb->mbox_cmpl(phba, pmb); + return false; + } + + /* There is mailbox completion work to queue to the worker thread */ spin_lock_irqsave(&phba->hbalock, iflags); __lpfc_mbox_cmpl_put(phba, pmb); phba->work_ha |= HA_MBATT; @@ -13299,32 +14463,42 @@ send_current_mbox: phba->sli.sli_flag &= ~LPFC_SLI_MBOX_ACTIVE; /* Setting active mailbox pointer need to be in sync to flag clear */ phba->sli.mbox_active = NULL; + if (bf_get(lpfc_trailer_consumed, mcqe)) + lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq); spin_unlock_irqrestore(&phba->hbalock, iflags); /* Wake up worker thread to post the next pending mailbox command */ lpfc_worker_wake_up(phba); + return workposted; + out_no_mqe_complete: + spin_lock_irqsave(&phba->hbalock, iflags); if (bf_get(lpfc_trailer_consumed, mcqe)) lpfc_sli4_mq_release(phba->sli4_hba.mbx_wq); - return workposted; + spin_unlock_irqrestore(&phba->hbalock, iflags); + return false; } /** * lpfc_sli4_sp_handle_mcqe - Process a mailbox completion queue entry * @phba: Pointer to HBA context object. + * @cq: Pointer to associated CQ * @cqe: Pointer to mailbox completion queue entry. * * This routine process a mailbox completion queue entry, it invokes the - * proper mailbox complete handling or asynchrous event handling routine + * proper mailbox complete handling or asynchronous event handling routine * according to the MCQE's async bit. * * Return: true if work posted to worker thread, otherwise false. **/ static bool -lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_cqe *cqe) +lpfc_sli4_sp_handle_mcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, + struct lpfc_cqe *cqe) { struct lpfc_mcqe mcqe; bool workposted; + cq->CQ_mbox++; + /* Copy the mailbox MCQE and convert endian order as needed */ lpfc_sli4_pcimem_bcopy(cqe, &mcqe, sizeof(struct lpfc_mcqe)); @@ -13355,7 +14529,6 @@ lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, struct lpfc_sli_ring *pring = cq->pring; int txq_cnt = 0; int txcmplq_cnt = 0; - int fcp_txcmplq_cnt = 0; /* Check for response status */ if (unlikely(bf_get(lpfc_wcqe_c_status, wcqe))) { @@ -13375,11 +14548,10 @@ lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, txq_cnt++; if (!list_empty(&pring->txcmplq)) txcmplq_cnt++; - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0387 NO IOCBQ data: txq_cnt=%d iocb_cnt=%d " - "fcp_txcmplq_cnt=%d, els_txcmplq_cnt=%d\n", + "els_txcmplq_cnt=%d\n", txq_cnt, phba->iocb_cnt, - fcp_txcmplq_cnt, txcmplq_cnt); return false; } @@ -13389,8 +14561,8 @@ lpfc_sli4_sp_handle_els_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, spin_lock_irqsave(&phba->hbalock, iflags); list_add_tail(&irspiocbq->cq_event.list, &phba->sli4_hba.sp_queue_event); - phba->hba_flag |= HBA_SP_QUEUE_EVT; spin_unlock_irqrestore(&phba->hbalock, iflags); + set_bit(HBA_SP_QUEUE_EVT, &phba->hba_flag); return true; } @@ -13442,44 +14614,35 @@ lpfc_sli4_sp_handle_abort_xri_wcqe(struct lpfc_hba *phba, unsigned long iflags; switch (cq->subtype) { - case LPFC_FCP: - cq_event = lpfc_cq_event_setup( - phba, wcqe, sizeof(struct sli4_wcqe_xri_aborted)); - if (!cq_event) - return false; - spin_lock_irqsave(&phba->hbalock, iflags); - list_add_tail(&cq_event->list, - &phba->sli4_hba.sp_fcp_xri_aborted_work_queue); - /* Set the fcp xri abort event flag */ - phba->hba_flag |= FCP_XRI_ABORT_EVENT; - spin_unlock_irqrestore(&phba->hbalock, iflags); - workposted = true; + case LPFC_IO: + lpfc_sli4_io_xri_aborted(phba, wcqe, cq->hdwq); + if (phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME) { + /* Notify aborted XRI for NVME work queue */ + if (phba->nvmet_support) + lpfc_sli4_nvmet_xri_aborted(phba, wcqe); + } + workposted = false; break; case LPFC_NVME_LS: /* NVME LS uses ELS resources */ case LPFC_ELS: - cq_event = lpfc_cq_event_setup( - phba, wcqe, sizeof(struct sli4_wcqe_xri_aborted)); - if (!cq_event) - return false; - spin_lock_irqsave(&phba->hbalock, iflags); + cq_event = lpfc_cq_event_setup(phba, wcqe, sizeof(*wcqe)); + if (!cq_event) { + workposted = false; + break; + } + cq_event->hdwq = cq->hdwq; + spin_lock_irqsave(&phba->sli4_hba.els_xri_abrt_list_lock, + iflags); list_add_tail(&cq_event->list, &phba->sli4_hba.sp_els_xri_aborted_work_queue); /* Set the els xri abort event flag */ - phba->hba_flag |= ELS_XRI_ABORT_EVENT; - spin_unlock_irqrestore(&phba->hbalock, iflags); + set_bit(ELS_XRI_ABORT_EVENT, &phba->hba_flag); + spin_unlock_irqrestore(&phba->sli4_hba.els_xri_abrt_list_lock, + iflags); workposted = true; break; - case LPFC_NVME: - /* Notify aborted XRI for NVME work queue */ - if (phba->nvmet_support) - lpfc_sli4_nvmet_xri_aborted(phba, wcqe); - else - lpfc_sli4_nvme_xri_aborted(phba, wcqe); - - workposted = false; - break; default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0603 Invalid CQ subtype %d: " "%08x %08x %08x %08x\n", cq->subtype, wcqe->word0, wcqe->parameter, @@ -13527,9 +14690,9 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe) status = bf_get(lpfc_rcqe_status, rcqe); switch (status) { case FC_STATUS_RQ_BUF_LEN_EXCEEDED: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2537 Receive Frame Truncated!!\n"); - /* fall through */ + fallthrough; case FC_STATUS_RQ_SUCCESS: spin_lock_irqsave(&phba->hbalock, iflags); lpfc_sli4_rq_release(hrq, drq); @@ -13549,22 +14712,26 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe) fc_hdr->fh_r_ctl == FC_RCTL_DD_UNSOL_DATA) { spin_unlock_irqrestore(&phba->hbalock, iflags); /* Handle MDS Loopback frames */ - lpfc_sli4_handle_mds_loopback(phba->pport, dma_buf); + if (!test_bit(FC_UNLOADING, &phba->pport->load_flag)) + lpfc_sli4_handle_mds_loopback(phba->pport, + dma_buf); + else + lpfc_in_buf_free(phba, &dma_buf->dbuf); break; } /* save off the frame for the work thread to process */ list_add_tail(&dma_buf->cq_event.list, &phba->sli4_hba.sp_queue_event); - /* Frame received */ - phba->hba_flag |= HBA_SP_QUEUE_EVT; spin_unlock_irqrestore(&phba->hbalock, iflags); + /* Frame received */ + set_bit(HBA_SP_QUEUE_EVT, &phba->hba_flag); workposted = true; break; case FC_STATUS_INSUFF_BUF_FRM_DISC: if (phba->nvmet_support) { tgtp = phba->targetport->private; - lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_NVME, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6402 RQE Error x%x, posted %d err_cnt " "%d: %x %x %x\n", status, hrq->RQ_buf_posted, @@ -13573,15 +14740,45 @@ lpfc_sli4_sp_handle_rcqe(struct lpfc_hba *phba, struct lpfc_rcqe *rcqe) atomic_read(&tgtp->rcv_fcp_cmd_out), atomic_read(&tgtp->xmt_fcp_release)); } - /* fallthrough */ + fallthrough; case FC_STATUS_INSUFF_BUF_NEED_BUF: hrq->RQ_no_posted_buf++; /* Post more buffers if possible */ + set_bit(HBA_POST_RECEIVE_BUFFER, &phba->hba_flag); + workposted = true; + break; + case FC_STATUS_RQ_DMA_FAILURE: + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "2564 RQE DMA Error x%x, x%08x x%08x x%08x " + "x%08x\n", + status, rcqe->word0, rcqe->word1, + rcqe->word2, rcqe->word3); + + /* If IV set, no further recovery */ + if (bf_get(lpfc_rcqe_iv, rcqe)) + break; + + /* recycle consumed resource */ spin_lock_irqsave(&phba->hbalock, iflags); - phba->hba_flag |= HBA_POST_RECEIVE_BUFFER; + lpfc_sli4_rq_release(hrq, drq); + dma_buf = lpfc_sli_hbqbuf_get(&phba->hbqs[0].hbq_buffer_list); + if (!dma_buf) { + hrq->RQ_no_buf_found++; + spin_unlock_irqrestore(&phba->hbalock, iflags); + break; + } + hrq->RQ_rcv_buf++; + hrq->RQ_buf_posted--; spin_unlock_irqrestore(&phba->hbalock, iflags); - workposted = true; + lpfc_in_buf_free(phba, &dma_buf->dbuf); + break; + default: + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "2565 Unexpected RQE Status x%x, w0-3 x%08x " + "x%08x x%08x x%08x\n", + status, rcqe->word0, rcqe->word1, + rcqe->word2, rcqe->word3); break; } out: @@ -13592,7 +14789,7 @@ out: * lpfc_sli4_sp_handle_cqe - Process a slow path completion queue entry * @phba: Pointer to HBA context object. * @cq: Pointer to the completion queue. - * @wcqe: Pointer to a completion queue entry. + * @cqe: Pointer to a completion queue entry. * * This routine process a slow-path work-queue or receive queue completion queue * entry. @@ -13636,7 +14833,7 @@ lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, (struct lpfc_rcqe *)&cqevt); break; default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0388 Not a valid WCQE code: x%x\n", bf_get(lpfc_cqe_code, &cqevt)); break; @@ -13648,6 +14845,7 @@ lpfc_sli4_sp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, * lpfc_sli4_sp_handle_eqe - Process a slow-path event queue entry * @phba: Pointer to HBA context object. * @eqe: Pointer to fast-path event queue entry. + * @speq: Pointer to slow-path event queue. * * This routine process a event queue entry from the slow-path event queue. * It will check the MajorCode and MinorCode to determine this is for a @@ -13663,6 +14861,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, { struct lpfc_queue *cq = NULL, *childq; uint16_t cqid; + int ret = 0; /* Get the reference to the corresponding CQ */ cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); @@ -13675,7 +14874,7 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, } if (unlikely(!cq)) { if (phba->sli.sli_flag & LPFC_SLI_ACTIVE) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0365 Slow-path CQ identifier " "(%d) does not exist\n", cqid); return; @@ -13684,84 +14883,163 @@ lpfc_sli4_sp_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Save EQ associated with this CQ */ cq->assoc_qp = speq; - if (!queue_work(phba->wq, &cq->spwork)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0390 Cannot schedule soft IRQ " + if (is_kdump_kernel()) + ret = queue_work(phba->wq, &cq->spwork); + else + ret = queue_work_on(cq->chann, phba->wq, &cq->spwork); + + if (!ret) + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "0390 Cannot schedule queue work " "for CQ eqcqid=%d, cqid=%d on CPU %d\n", - cqid, cq->queue_id, smp_processor_id()); + cqid, cq->queue_id, raw_smp_processor_id()); } /** - * lpfc_sli4_sp_process_cq - Process a slow-path event queue entry + * __lpfc_sli4_process_cq - Process elements of a CQ * @phba: Pointer to HBA context object. + * @cq: Pointer to CQ to be processed + * @handler: Routine to process each cqe + * @delay: Pointer to usdelay to set in case of rescheduling of the handler * - * This routine process a event queue entry from the slow-path event queue. - * It will check the MajorCode and MinorCode to determine this is for a - * completion event on a completion queue, if not, an error shall be logged - * and just return. Otherwise, it will get to the corresponding completion - * queue and process all the entries on that completion queue, rearm the - * completion queue, and then return. + * This routine processes completion queue entries in a CQ. While a valid + * queue element is found, the handler is called. During processing checks + * are made for periodic doorbell writes to let the hardware know of + * element consumption. + * + * If the max limit on cqes to process is hit, or there are no more valid + * entries, the loop stops. If we processed a sufficient number of elements, + * meaning there is sufficient load, rather than rearming and generating + * another interrupt, a cq rescheduling delay will be set. A delay of 0 + * indicates no rescheduling. * + * Returns True if work scheduled, False otherwise. **/ -static void -lpfc_sli4_sp_process_cq(struct work_struct *work) +static bool +__lpfc_sli4_process_cq(struct lpfc_hba *phba, struct lpfc_queue *cq, + bool (*handler)(struct lpfc_hba *, struct lpfc_queue *, + struct lpfc_cqe *), unsigned long *delay) { - struct lpfc_queue *cq = - container_of(work, struct lpfc_queue, spwork); - struct lpfc_hba *phba = cq->phba; struct lpfc_cqe *cqe; bool workposted = false; - int ccount = 0; + int count = 0, consumed = 0; + bool arm = true; + + /* default - no reschedule */ + *delay = 0; + + if (cmpxchg(&cq->queue_claimed, 0, 1) != 0) + goto rearm_and_exit; /* Process all the entries to the CQ */ + cq->q_flag = 0; + cqe = lpfc_sli4_cq_get(cq); + while (cqe) { + workposted |= handler(phba, cq, cqe); + __lpfc_sli4_consume_cqe(phba, cq, cqe); + + consumed++; + if (!(++count % cq->max_proc_limit)) + break; + + if (!(count % cq->notify_interval)) { + phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed, + LPFC_QUEUE_NOARM); + consumed = 0; + cq->assoc_qp->q_flag |= HBA_EQ_DELAY_CHK; + } + + if (count == LPFC_NVMET_CQ_NOTIFY) + cq->q_flag |= HBA_NVMET_CQ_NOTIFY; + + cqe = lpfc_sli4_cq_get(cq); + } + if (count >= phba->cfg_cq_poll_threshold) { + *delay = 1; + arm = false; + } + + /* Track the max number of CQEs processed in 1 EQ */ + if (count > cq->CQ_max_cqe) + cq->CQ_max_cqe = count; + + cq->assoc_qp->EQ_cqe_cnt += count; + + /* Catch the no cq entry condition */ + if (unlikely(count == 0)) + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "0369 No entry from completion queue " + "qid=%d\n", cq->queue_id); + + xchg(&cq->queue_claimed, 0); + +rearm_and_exit: + phba->sli4_hba.sli4_write_cq_db(phba, cq, consumed, + arm ? LPFC_QUEUE_REARM : LPFC_QUEUE_NOARM); + + return workposted; +} + +/** + * __lpfc_sli4_sp_process_cq - Process a slow-path event queue entry + * @cq: pointer to CQ to process + * + * This routine calls the cq processing routine with a handler specific + * to the type of queue bound to it. + * + * The CQ routine returns two values: the first is the calling status, + * which indicates whether work was queued to the background discovery + * thread. If true, the routine should wakeup the discovery thread; + * the second is the delay parameter. If non-zero, rather than rearming + * the CQ and yet another interrupt, the CQ handler should be queued so + * that it is processed in a subsequent polling action. The value of + * the delay indicates when to reschedule it. + **/ +static void +__lpfc_sli4_sp_process_cq(struct lpfc_queue *cq) +{ + struct lpfc_hba *phba = cq->phba; + unsigned long delay; + bool workposted = false; + int ret = 0; + + /* Process and rearm the CQ */ switch (cq->type) { case LPFC_MCQ: - while ((cqe = lpfc_sli4_cq_get(cq))) { - workposted |= lpfc_sli4_sp_handle_mcqe(phba, cqe); - if (!(++ccount % cq->entry_repost)) - break; - cq->CQ_mbox++; - } + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_sp_handle_mcqe, + &delay); break; case LPFC_WCQ: - while ((cqe = lpfc_sli4_cq_get(cq))) { - if (cq->subtype == LPFC_FCP || - cq->subtype == LPFC_NVME) { -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - if (phba->ktime_on) - cq->isr_timestamp = ktime_get_ns(); - else - cq->isr_timestamp = 0; -#endif - workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, - cqe); - } else { - workposted |= lpfc_sli4_sp_handle_cqe(phba, cq, - cqe); - } - if (!(++ccount % cq->entry_repost)) - break; - } - - /* Track the max number of CQEs processed in 1 EQ */ - if (ccount > cq->CQ_max_cqe) - cq->CQ_max_cqe = ccount; + if (cq->subtype == LPFC_IO) + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_fp_handle_cqe, + &delay); + else + workposted |= __lpfc_sli4_process_cq(phba, cq, + lpfc_sli4_sp_handle_cqe, + &delay); break; default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0370 Invalid completion queue type (%d)\n", cq->type); return; } - /* Catch the no cq entry condition, log an error */ - if (unlikely(ccount == 0)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0371 No entry from the CQ: identifier " - "(x%x), type (%d)\n", cq->queue_id, cq->type); - - /* In any case, flash and re-arm the RCQ */ - phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM); + if (delay) { + if (is_kdump_kernel()) + ret = queue_delayed_work(phba->wq, &cq->sched_spwork, + delay); + else + ret = queue_delayed_work_on(cq->chann, phba->wq, + &cq->sched_spwork, delay); + if (!ret) + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "0394 Cannot schedule queue work " + "for cqid=%d on CPU %d\n", + cq->queue_id, cq->chann); + } /* wake up worker thread if there are works to be done */ if (workposted) @@ -13769,6 +15047,36 @@ lpfc_sli4_sp_process_cq(struct work_struct *work) } /** + * lpfc_sli4_sp_process_cq - slow-path work handler when started by + * interrupt + * @work: pointer to work element + * + * translates from the work handler and calls the slow-path handler. + **/ +static void +lpfc_sli4_sp_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(work, struct lpfc_queue, spwork); + + __lpfc_sli4_sp_process_cq(cq); +} + +/** + * lpfc_sli4_dly_sp_process_cq - slow-path work handler when started by timer + * @work: pointer to work element + * + * translates from the work handler and calls the slow-path handler. + **/ +static void +lpfc_sli4_dly_sp_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(to_delayed_work(work), + struct lpfc_queue, sched_spwork); + + __lpfc_sli4_sp_process_cq(cq); +} + +/** * lpfc_sli4_fp_handle_fcp_wcqe - Process fast-path work queue completion entry * @phba: Pointer to HBA context object. * @cq: Pointer to associated CQ @@ -13783,7 +15091,6 @@ lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, { struct lpfc_sli_ring *pring = cq->pring; struct lpfc_iocbq *cmdiocbq; - struct lpfc_iocbq irspiocbq; unsigned long iflags; /* Check for response status */ @@ -13797,9 +15104,9 @@ lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, IOERR_NO_RESOURCES)) phba->lpfc_rampdown_queue_depth(phba); - /* Log the error status */ + /* Log the cmpl status */ lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "0373 FCP CQE error: status=x%x: " + "0373 FCP CQE cmpl: status=x%x: " "CQE: %08x %08x %08x %08x\n", bf_get(lpfc_wcqe_c_status, wcqe), wcqe->word0, wcqe->total_data_placed, @@ -13822,36 +15129,31 @@ lpfc_sli4_fp_handle_fcp_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, #ifdef CONFIG_SCSI_LPFC_DEBUG_FS cmdiocbq->isr_timestamp = cq->isr_timestamp; #endif - if (cmdiocbq->iocb_cmpl == NULL) { - if (cmdiocbq->wqe_cmpl) { - if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) { - spin_lock_irqsave(&phba->hbalock, iflags); - cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED; - spin_unlock_irqrestore(&phba->hbalock, iflags); - } + if (bf_get(lpfc_wcqe_c_xb, wcqe)) { + spin_lock_irqsave(&phba->hbalock, iflags); + cmdiocbq->cmd_flag |= LPFC_EXCHANGE_BUSY; + spin_unlock_irqrestore(&phba->hbalock, iflags); + } - /* Pass the cmd_iocb and the wcqe to the upper layer */ - (cmdiocbq->wqe_cmpl)(phba, cmdiocbq, wcqe); - return; + if (cmdiocbq->cmd_cmpl) { + /* For FCP the flag is cleared in cmd_cmpl */ + if (!(cmdiocbq->cmd_flag & LPFC_IO_FCP) && + cmdiocbq->cmd_flag & LPFC_DRIVER_ABORTED) { + spin_lock_irqsave(&phba->hbalock, iflags); + cmdiocbq->cmd_flag &= ~LPFC_DRIVER_ABORTED; + spin_unlock_irqrestore(&phba->hbalock, iflags); } + + /* Pass the cmd_iocb and the wcqe to the upper layer */ + memcpy(&cmdiocbq->wcqe_cmpl, wcqe, + sizeof(struct lpfc_wcqe_complete)); + cmdiocbq->cmd_cmpl(phba, cmdiocbq, cmdiocbq); + } else { lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, "0375 FCP cmdiocb not callback function " "iotag: (%d)\n", bf_get(lpfc_wcqe_c_request_tag, wcqe)); - return; } - - /* Fake the irspiocb and copy necessary response information */ - lpfc_sli4_iocb_param_transfer(phba, &irspiocbq, cmdiocbq, wcqe); - - if (cmdiocbq->iocb_flag & LPFC_DRIVER_ABORTED) { - spin_lock_irqsave(&phba->hbalock, iflags); - cmdiocbq->iocb_flag &= ~LPFC_DRIVER_ABORTED; - spin_unlock_irqrestore(&phba->hbalock, iflags); - } - - /* Pass the cmd_iocb and the rsp state to the upper layer */ - (cmdiocbq->iocb_cmpl)(phba, cmdiocbq, &irspiocbq); } /** @@ -13893,6 +15195,7 @@ lpfc_sli4_fp_handle_rel_wcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, /** * lpfc_sli4_nvmet_handle_rcqe - Process a receive-queue completion queue entry * @phba: Pointer to HBA context object. + * @cq: Pointer to completion queue. * @rcqe: Pointer to receive-queue completion queue entry. * * This routine process a receive-queue completion queue entry. @@ -13937,9 +15240,9 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, status = bf_get(lpfc_rcqe_status, rcqe); switch (status) { case FC_STATUS_RQ_BUF_LEN_EXCEEDED: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6126 Receive Frame Truncated!!\n"); - /* fall through */ + fallthrough; case FC_STATUS_RQ_SUCCESS: spin_lock_irqsave(&phba->hbalock, iflags); lpfc_sli4_rq_release(hrq, drq); @@ -13956,8 +15259,8 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, /* Just some basic sanity checks on FCP Command frame */ fctl = (fc_hdr->fh_f_ctl[0] << 16 | - fc_hdr->fh_f_ctl[1] << 8 | - fc_hdr->fh_f_ctl[2]); + fc_hdr->fh_f_ctl[1] << 8 | + fc_hdr->fh_f_ctl[2]); if (((fctl & (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) != (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) || @@ -13965,19 +15268,19 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, goto drop; if (fc_hdr->fh_type == FC_TYPE_FCP) { - dma_buf->bytes_recv = bf_get(lpfc_rcqe_length, rcqe); + dma_buf->bytes_recv = bf_get(lpfc_rcqe_length, rcqe); lpfc_nvmet_unsol_fcp_event( - phba, idx, dma_buf, - cq->isr_timestamp); + phba, idx, dma_buf, cq->isr_timestamp, + cq->q_flag & HBA_NVMET_CQ_NOTIFY); return false; } drop: - lpfc_in_buf_free(phba, &dma_buf->dbuf); + lpfc_rq_buf_free(phba, &dma_buf->hbuf); break; case FC_STATUS_INSUFF_BUF_FRM_DISC: if (phba->nvmet_support) { tgtp = phba->targetport->private; - lpfc_printf_log(phba, KERN_ERR, LOG_SLI | LOG_NVME, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "6401 RQE Error x%x, posted %d err_cnt " "%d: %x %x %x\n", status, hrq->RQ_buf_posted, @@ -13986,12 +15289,44 @@ drop: atomic_read(&tgtp->rcv_fcp_cmd_out), atomic_read(&tgtp->xmt_fcp_release)); } - /* fallthrough */ + fallthrough; case FC_STATUS_INSUFF_BUF_NEED_BUF: hrq->RQ_no_posted_buf++; /* Post more buffers if possible */ break; + case FC_STATUS_RQ_DMA_FAILURE: + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "2575 RQE DMA Error x%x, x%08x x%08x x%08x " + "x%08x\n", + status, rcqe->word0, rcqe->word1, + rcqe->word2, rcqe->word3); + + /* If IV set, no further recovery */ + if (bf_get(lpfc_rcqe_iv, rcqe)) + break; + + /* recycle consumed resource */ + spin_lock_irqsave(&phba->hbalock, iflags); + lpfc_sli4_rq_release(hrq, drq); + dma_buf = lpfc_sli_rqbuf_get(phba, hrq); + if (!dma_buf) { + hrq->RQ_no_buf_found++; + spin_unlock_irqrestore(&phba->hbalock, iflags); + break; + } + hrq->RQ_rcv_buf++; + hrq->RQ_buf_posted--; + spin_unlock_irqrestore(&phba->hbalock, iflags); + lpfc_rq_buf_free(phba, &dma_buf->hbuf); + break; + default: + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "2576 Unexpected RQE Status x%x, w0-3 x%08x " + "x%08x x%08x x%08x\n", + status, rcqe->word0, rcqe->word1, + rcqe->word2, rcqe->word3); + break; } out: return workposted; @@ -13999,13 +15334,16 @@ out: /** * lpfc_sli4_fp_handle_cqe - Process fast-path work queue completion entry + * @phba: adapter with cq * @cq: Pointer to the completion queue. - * @eqe: Pointer to fast-path completion queue entry. + * @cqe: Pointer to fast-path completion queue entry. * * This routine process a fast-path work queue completion entry from fast-path * event queue for FCP command response completion. + * + * Return: true if work posted to worker thread, otherwise false. **/ -static int +static bool lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, struct lpfc_cqe *cqe) { @@ -14022,10 +15360,7 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, cq->CQ_wq++; /* Process the WQ complete event */ phba->last_completion_time = jiffies; - if ((cq->subtype == LPFC_FCP) || (cq->subtype == LPFC_NVME)) - lpfc_sli4_fp_handle_fcp_wcqe(phba, cq, - (struct lpfc_wcqe_complete *)&wcqe); - if (cq->subtype == LPFC_NVME_LS) + if (cq->subtype == LPFC_IO || cq->subtype == LPFC_NVME_LS) lpfc_sli4_fp_handle_fcp_wcqe(phba, cq, (struct lpfc_wcqe_complete *)&wcqe); break; @@ -14051,7 +15386,7 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, } break; default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0144 Not a valid CQE code: x%x\n", bf_get(lpfc_wcqe_c_code, &wcqe)); break; @@ -14060,9 +15395,72 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, } /** + * __lpfc_sli4_hba_process_cq - Process a fast-path event queue entry + * @cq: Pointer to CQ to be processed + * + * This routine calls the cq processing routine with the handler for + * fast path CQEs. + * + * The CQ routine returns two values: the first is the calling status, + * which indicates whether work was queued to the background discovery + * thread. If true, the routine should wakeup the discovery thread; + * the second is the delay parameter. If non-zero, rather than rearming + * the CQ and yet another interrupt, the CQ handler should be queued so + * that it is processed in a subsequent polling action. The value of + * the delay indicates when to reschedule it. + **/ +static void +__lpfc_sli4_hba_process_cq(struct lpfc_queue *cq) +{ + struct lpfc_hba *phba = cq->phba; + unsigned long delay; + bool workposted = false; + int ret; + + /* process and rearm the CQ */ + workposted |= __lpfc_sli4_process_cq(phba, cq, lpfc_sli4_fp_handle_cqe, + &delay); + + if (delay) { + if (is_kdump_kernel()) + ret = queue_delayed_work(phba->wq, &cq->sched_irqwork, + delay); + else + ret = queue_delayed_work_on(cq->chann, phba->wq, + &cq->sched_irqwork, delay); + if (!ret) + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "0367 Cannot schedule queue work " + "for cqid=%d on CPU %d\n", + cq->queue_id, cq->chann); + } + + /* wake up worker thread if there are works to be done */ + if (workposted) + lpfc_worker_wake_up(phba); +} + +/** + * lpfc_sli4_hba_process_cq - fast-path work handler when started by + * interrupt + * @work: pointer to work element + * + * translates from the work handler and calls the fast-path handler. + **/ +static void +lpfc_sli4_hba_process_cq(struct work_struct *work) +{ + struct lpfc_queue *cq = container_of(work, struct lpfc_queue, irqwork); + + __lpfc_sli4_hba_process_cq(cq); +} + +/** * lpfc_sli4_hba_handle_eqe - Process a fast-path event queue entry * @phba: Pointer to HBA context object. + * @eq: Pointer to the queue structure. * @eqe: Pointer to fast-path event queue entry. + * @poll_mode: poll_mode to execute processing the cq. * * This routine process a event queue entry from the fast-path event queue. * It will check the MajorCode and MinorCode to determine this is for a @@ -14072,14 +15470,16 @@ lpfc_sli4_fp_handle_cqe(struct lpfc_hba *phba, struct lpfc_queue *cq, * completion queue, and then return. **/ static void -lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, - uint32_t qidx) +lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_queue *eq, + struct lpfc_eqe *eqe, enum lpfc_poll_mode poll_mode) { struct lpfc_queue *cq = NULL; + uint32_t qidx = eq->hdwq; uint16_t cqid, id; + int ret; if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0366 Not a valid completion " "event: majorcode=x%x, minorcode=x%x\n", bf_get_le32(lpfc_eqe_major_code, eqe), @@ -14090,6 +15490,14 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Get the reference to the corresponding CQ */ cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); + /* Use the fast lookup method first */ + if (cqid <= phba->sli4_hba.cq_max) { + cq = phba->sli4_hba.cq_lookup[cqid]; + if (cq) + goto work_cq; + } + + /* Next check for NVMET completion */ if (phba->cfg_nvmet_mrq && phba->sli4_hba.nvmet_cqset) { id = phba->sli4_hba.nvmet_cqset[0]->queue_id; if ((cqid >= id) && (cqid < (id + phba->cfg_nvmet_mrq))) { @@ -14099,20 +15507,6 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, } } - if (phba->sli4_hba.nvme_cq_map && - (cqid == phba->sli4_hba.nvme_cq_map[qidx])) { - /* Process NVME / NVMET command completion */ - cq = phba->sli4_hba.nvme_cq[qidx]; - goto process_cq; - } - - if (phba->sli4_hba.fcp_cq_map && - (cqid == phba->sli4_hba.fcp_cq_map[qidx])) { - /* Process FCP command completion */ - cq = phba->sli4_hba.fcp_cq[qidx]; - goto process_cq; - } - if (phba->sli4_hba.nvmels_cq && (cqid == phba->sli4_hba.nvmels_cq->queue_id)) { /* Process NVME unsol rcv */ @@ -14121,243 +15515,61 @@ lpfc_sli4_hba_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe, /* Otherwise this is a Slow path event */ if (cq == NULL) { - lpfc_sli4_sp_handle_eqe(phba, eqe, phba->sli4_hba.hba_eq[qidx]); + lpfc_sli4_sp_handle_eqe(phba, eqe, + phba->sli4_hba.hdwq[qidx].hba_eq); return; } process_cq: if (unlikely(cqid != cq->queue_id)) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0368 Miss-matched fast-path completion " "queue identifier: eqcqid=%d, fcpcqid=%d\n", cqid, cq->queue_id); return; } - /* Save EQ associated with this CQ */ - cq->assoc_qp = phba->sli4_hba.hba_eq[qidx]; - - if (!queue_work(phba->wq, &cq->irqwork)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0363 Cannot schedule soft IRQ " - "for CQ eqcqid=%d, cqid=%d on CPU %d\n", - cqid, cq->queue_id, smp_processor_id()); -} - -/** - * lpfc_sli4_hba_process_cq - Process a fast-path event queue entry - * @phba: Pointer to HBA context object. - * @eqe: Pointer to fast-path event queue entry. - * - * This routine process a event queue entry from the fast-path event queue. - * It will check the MajorCode and MinorCode to determine this is for a - * completion event on a completion queue, if not, an error shall be logged - * and just return. Otherwise, it will get to the corresponding completion - * queue and process all the entries on the completion queue, rearm the - * completion queue, and then return. - **/ -static void -lpfc_sli4_hba_process_cq(struct work_struct *work) -{ - struct lpfc_queue *cq = - container_of(work, struct lpfc_queue, irqwork); - struct lpfc_hba *phba = cq->phba; - struct lpfc_cqe *cqe; - bool workposted = false; - int ccount = 0; +work_cq: +#if defined(CONFIG_SCSI_LPFC_DEBUG_FS) + if (phba->ktime_on) + cq->isr_timestamp = ktime_get_ns(); + else + cq->isr_timestamp = 0; +#endif - /* Process all the entries to the CQ */ - while ((cqe = lpfc_sli4_cq_get(cq))) { -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - if (phba->ktime_on) - cq->isr_timestamp = ktime_get_ns(); + switch (poll_mode) { + case LPFC_THREADED_IRQ: + __lpfc_sli4_hba_process_cq(cq); + break; + case LPFC_QUEUE_WORK: + default: + if (is_kdump_kernel()) + ret = queue_work(phba->wq, &cq->irqwork); else - cq->isr_timestamp = 0; -#endif - workposted |= lpfc_sli4_fp_handle_cqe(phba, cq, cqe); - if (!(++ccount % cq->entry_repost)) - break; + ret = queue_work_on(cq->chann, phba->wq, &cq->irqwork); + if (!ret) + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "0383 Cannot schedule queue work " + "for CQ eqcqid=%d, cqid=%d on CPU %d\n", + cqid, cq->queue_id, + raw_smp_processor_id()); + break; } - - /* Track the max number of CQEs processed in 1 EQ */ - if (ccount > cq->CQ_max_cqe) - cq->CQ_max_cqe = ccount; - cq->assoc_qp->EQ_cqe_cnt += ccount; - - /* Catch the no cq entry condition */ - if (unlikely(ccount == 0)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0369 No entry from fast-path completion " - "queue fcpcqid=%d\n", cq->queue_id); - - /* In any case, flash and re-arm the CQ */ - phba->sli4_hba.sli4_cq_release(cq, LPFC_QUEUE_REARM); - - /* wake up worker thread if there are works to be done */ - if (workposted) - lpfc_worker_wake_up(phba); -} - -static void -lpfc_sli4_eq_flush(struct lpfc_hba *phba, struct lpfc_queue *eq) -{ - struct lpfc_eqe *eqe; - - /* walk all the EQ entries and drop on the floor */ - while ((eqe = lpfc_sli4_eq_get(eq))) - ; - - /* Clear and re-arm the EQ */ - phba->sli4_hba.sli4_eq_release(eq, LPFC_QUEUE_REARM); } - /** - * lpfc_sli4_fof_handle_eqe - Process a Flash Optimized Fabric event queue - * entry - * @phba: Pointer to HBA context object. - * @eqe: Pointer to fast-path event queue entry. + * lpfc_sli4_dly_hba_process_cq - fast-path work handler when started by timer + * @work: pointer to work element * - * This routine process a event queue entry from the Flash Optimized Fabric - * event queue. It will check the MajorCode and MinorCode to determine this - * is for a completion event on a completion queue, if not, an error shall be - * logged and just return. Otherwise, it will get to the corresponding - * completion queue and process all the entries on the completion queue, rearm - * the completion queue, and then return. + * translates from the work handler and calls the fast-path handler. **/ static void -lpfc_sli4_fof_handle_eqe(struct lpfc_hba *phba, struct lpfc_eqe *eqe) +lpfc_sli4_dly_hba_process_cq(struct work_struct *work) { - struct lpfc_queue *cq; - uint16_t cqid; - - if (unlikely(bf_get_le32(lpfc_eqe_major_code, eqe) != 0)) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9147 Not a valid completion " - "event: majorcode=x%x, minorcode=x%x\n", - bf_get_le32(lpfc_eqe_major_code, eqe), - bf_get_le32(lpfc_eqe_minor_code, eqe)); - return; - } - - /* Get the reference to the corresponding CQ */ - cqid = bf_get_le32(lpfc_eqe_resource_id, eqe); - - /* Next check for OAS */ - cq = phba->sli4_hba.oas_cq; - if (unlikely(!cq)) { - if (phba->sli.sli_flag & LPFC_SLI_ACTIVE) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9148 OAS completion queue " - "does not exist\n"); - return; - } - - if (unlikely(cqid != cq->queue_id)) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9149 Miss-matched fast-path compl " - "queue id: eqcqid=%d, fcpcqid=%d\n", - cqid, cq->queue_id); - return; - } - - /* Save EQ associated with this CQ */ - cq->assoc_qp = phba->sli4_hba.fof_eq; + struct lpfc_queue *cq = container_of(to_delayed_work(work), + struct lpfc_queue, sched_irqwork); - /* CQ work will be processed on CPU affinitized to this IRQ */ - if (!queue_work(phba->wq, &cq->irqwork)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "0367 Cannot schedule soft IRQ " - "for CQ eqcqid=%d, cqid=%d on CPU %d\n", - cqid, cq->queue_id, smp_processor_id()); -} - -/** - * lpfc_sli4_fof_intr_handler - HBA interrupt handler to SLI-4 device - * @irq: Interrupt number. - * @dev_id: The device context pointer. - * - * This function is directly called from the PCI layer as an interrupt - * service routine when device with SLI-4 interface spec is enabled with - * MSI-X multi-message interrupt mode and there is a Flash Optimized Fabric - * IOCB ring event in the HBA. However, when the device is enabled with either - * MSI or Pin-IRQ interrupt mode, this function is called as part of the - * device-level interrupt handler. When the PCI slot is in error recovery - * or the HBA is undergoing initialization, the interrupt handler will not - * process the interrupt. The Flash Optimized Fabric ring event are handled in - * the intrrupt context. This function is called without any lock held. - * It gets the hbalock to access and update SLI data structures. Note that, - * the EQ to CQ are one-to-one map such that the EQ index is - * equal to that of CQ index. - * - * This function returns IRQ_HANDLED when interrupt is handled else it - * returns IRQ_NONE. - **/ -irqreturn_t -lpfc_sli4_fof_intr_handler(int irq, void *dev_id) -{ - struct lpfc_hba *phba; - struct lpfc_hba_eq_hdl *hba_eq_hdl; - struct lpfc_queue *eq; - struct lpfc_eqe *eqe; - unsigned long iflag; - int ecount = 0; - - /* Get the driver's phba structure from the dev_id */ - hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id; - phba = hba_eq_hdl->phba; - - if (unlikely(!phba)) - return IRQ_NONE; - - /* Get to the EQ struct associated with this vector */ - eq = phba->sli4_hba.fof_eq; - if (unlikely(!eq)) - return IRQ_NONE; - - /* Check device state for handling interrupt */ - if (unlikely(lpfc_intr_state_check(phba))) { - /* Check again for link_state with lock held */ - spin_lock_irqsave(&phba->hbalock, iflag); - if (phba->link_state < LPFC_LINK_DOWN) - /* Flush, clear interrupt, and rearm the EQ */ - lpfc_sli4_eq_flush(phba, eq); - spin_unlock_irqrestore(&phba->hbalock, iflag); - return IRQ_NONE; - } - - /* - * Process all the event on FCP fast-path EQ - */ - while ((eqe = lpfc_sli4_eq_get(eq))) { - lpfc_sli4_fof_handle_eqe(phba, eqe); - if (!(++ecount % eq->entry_repost)) - break; - eq->EQ_processed++; - } - - /* Track the max number of EQEs processed in 1 intr */ - if (ecount > eq->EQ_max_eqe) - eq->EQ_max_eqe = ecount; - - - if (unlikely(ecount == 0)) { - eq->EQ_no_entry++; - - if (phba->intr_type == MSIX) - /* MSI-X treated interrupt served as no EQ share INT */ - lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, - "9145 MSI-X interrupt with no EQE\n"); - else { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "9146 ISR interrupt with no EQE\n"); - /* Non MSI-X treated on interrupt as EQ share INT */ - return IRQ_NONE; - } - } - /* Always clear and re-arm the fast-path EQ */ - phba->sli4_hba.sli4_eq_release(eq, LPFC_QUEUE_REARM); - return IRQ_HANDLED; + __lpfc_sli4_hba_process_cq(cq); } /** @@ -14383,8 +15595,9 @@ lpfc_sli4_fof_intr_handler(int irq, void *dev_id) * and returns for these events. This function is called without any lock * held. It gets the hbalock to access and update SLI data structures. * - * This function returns IRQ_HANDLED when interrupt is handled else it - * returns IRQ_NONE. + * This function returns IRQ_HANDLED when interrupt is handled, IRQ_WAKE_THREAD + * when interrupt is scheduled to be handled from a threaded irq context, or + * else returns IRQ_NONE. **/ irqreturn_t lpfc_sli4_hba_intr_handler(int irq, void *dev_id) @@ -14392,10 +15605,10 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) struct lpfc_hba *phba; struct lpfc_hba_eq_hdl *hba_eq_hdl; struct lpfc_queue *fpeq; - struct lpfc_eqe *eqe; unsigned long iflag; - int ecount = 0; int hba_eqidx; + int ecount = 0; + struct lpfc_eq_intr_info *eqi; /* Get the driver's phba structure from the dev_id */ hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id; @@ -14404,75 +15617,64 @@ lpfc_sli4_hba_intr_handler(int irq, void *dev_id) if (unlikely(!phba)) return IRQ_NONE; - if (unlikely(!phba->sli4_hba.hba_eq)) + if (unlikely(!phba->sli4_hba.hdwq)) return IRQ_NONE; /* Get to the EQ struct associated with this vector */ - fpeq = phba->sli4_hba.hba_eq[hba_eqidx]; + fpeq = phba->sli4_hba.hba_eq_hdl[hba_eqidx].eq; if (unlikely(!fpeq)) return IRQ_NONE; - if (lpfc_fcp_look_ahead) { - if (atomic_dec_and_test(&hba_eq_hdl->hba_eq_in_use)) - phba->sli4_hba.sli4_eq_clr_intr(fpeq); - else { - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_NONE; - } - } - /* Check device state for handling interrupt */ if (unlikely(lpfc_intr_state_check(phba))) { /* Check again for link_state with lock held */ spin_lock_irqsave(&phba->hbalock, iflag); if (phba->link_state < LPFC_LINK_DOWN) /* Flush, clear interrupt, and rearm the EQ */ - lpfc_sli4_eq_flush(phba, fpeq); + lpfc_sli4_eqcq_flush(phba, fpeq); spin_unlock_irqrestore(&phba->hbalock, iflag); - if (lpfc_fcp_look_ahead) - atomic_inc(&hba_eq_hdl->hba_eq_in_use); return IRQ_NONE; } - /* - * Process all the event on FCP fast-path EQ - */ - while ((eqe = lpfc_sli4_eq_get(fpeq))) { - lpfc_sli4_hba_handle_eqe(phba, eqe, hba_eqidx); - if (!(++ecount % fpeq->entry_repost)) - break; - fpeq->EQ_processed++; - } - - /* Track the max number of EQEs processed in 1 intr */ - if (ecount > fpeq->EQ_max_eqe) - fpeq->EQ_max_eqe = ecount; - - /* Always clear and re-arm the fast-path EQ */ - phba->sli4_hba.sli4_eq_release(fpeq, LPFC_QUEUE_REARM); - - if (unlikely(ecount == 0)) { - fpeq->EQ_no_entry++; - - if (lpfc_fcp_look_ahead) { - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_NONE; + switch (fpeq->poll_mode) { + case LPFC_THREADED_IRQ: + /* CGN mgmt is mutually exclusive from irq processing */ + if (phba->cmf_active_mode == LPFC_CFG_OFF) + return IRQ_WAKE_THREAD; + fallthrough; + case LPFC_QUEUE_WORK: + default: + eqi = this_cpu_ptr(phba->sli4_hba.eq_info); + eqi->icnt++; + + fpeq->last_cpu = raw_smp_processor_id(); + + if (eqi->icnt > LPFC_EQD_ISR_TRIGGER && + fpeq->q_flag & HBA_EQ_DELAY_CHK && + phba->cfg_auto_imax && + fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY && + phba->sli.sli_flag & LPFC_SLI_USE_EQDR) + lpfc_sli4_mod_hba_eq_delay(phba, fpeq, + LPFC_MAX_AUTO_EQ_DELAY); + + /* process and rearm the EQ */ + ecount = lpfc_sli4_process_eq(phba, fpeq, LPFC_QUEUE_REARM, + LPFC_QUEUE_WORK); + + if (unlikely(ecount == 0)) { + fpeq->EQ_no_entry++; + if (phba->intr_type == MSIX) + /* MSI-X treated interrupt served as no EQ share INT */ + lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, + "0358 MSI-X interrupt with no EQE\n"); + else + /* Non MSI-X treated on interrupt as EQ share INT */ + return IRQ_NONE; } - - if (phba->intr_type == MSIX) - /* MSI-X treated interrupt served as no EQ share INT */ - lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, - "0358 MSI-X interrupt with no EQE\n"); - else - /* Non MSI-X treated on interrupt as EQ share INT */ - return IRQ_NONE; } - if (lpfc_fcp_look_ahead) - atomic_inc(&hba_eq_hdl->hba_eq_in_use); - return IRQ_HANDLED; -} /* lpfc_sli4_fp_intr_handler */ +} /* lpfc_sli4_hba_intr_handler */ /** * lpfc_sli4_intr_handler - Device-level interrupt handler for SLI-4 device @@ -14508,23 +15710,127 @@ lpfc_sli4_intr_handler(int irq, void *dev_id) /* * Invoke fast-path host attention interrupt handling as appropriate. */ - for (qidx = 0; qidx < phba->io_channel_irqs; qidx++) { + for (qidx = 0; qidx < phba->cfg_irq_chann; qidx++) { hba_irq_rc = lpfc_sli4_hba_intr_handler(irq, &phba->sli4_hba.hba_eq_hdl[qidx]); if (hba_irq_rc == IRQ_HANDLED) hba_handled |= true; } - if (phba->cfg_fof) { - hba_irq_rc = lpfc_sli4_fof_intr_handler(irq, - &phba->sli4_hba.hba_eq_hdl[qidx]); - if (hba_irq_rc == IRQ_HANDLED) - hba_handled |= true; - } - return (hba_handled == true) ? IRQ_HANDLED : IRQ_NONE; } /* lpfc_sli4_intr_handler */ +void lpfc_sli4_poll_hbtimer(struct timer_list *t) +{ + struct lpfc_hba *phba = timer_container_of(phba, t, cpuhp_poll_timer); + struct lpfc_queue *eq; + + rcu_read_lock(); + + list_for_each_entry_rcu(eq, &phba->poll_list, _poll_list) + lpfc_sli4_poll_eq(eq); + if (!list_empty(&phba->poll_list)) + mod_timer(&phba->cpuhp_poll_timer, + jiffies + msecs_to_jiffies(LPFC_POLL_HB)); + + rcu_read_unlock(); +} + +static inline void lpfc_sli4_add_to_poll_list(struct lpfc_queue *eq) +{ + struct lpfc_hba *phba = eq->phba; + + /* kickstart slowpath processing if needed */ + if (list_empty(&phba->poll_list)) + mod_timer(&phba->cpuhp_poll_timer, + jiffies + msecs_to_jiffies(LPFC_POLL_HB)); + + list_add_rcu(&eq->_poll_list, &phba->poll_list); + synchronize_rcu(); +} + +static inline void lpfc_sli4_remove_from_poll_list(struct lpfc_queue *eq) +{ + struct lpfc_hba *phba = eq->phba; + + /* Disable slowpath processing for this eq. Kick start the eq + * by RE-ARMING the eq's ASAP + */ + list_del_rcu(&eq->_poll_list); + synchronize_rcu(); + + if (list_empty(&phba->poll_list)) + timer_delete_sync(&phba->cpuhp_poll_timer); +} + +void lpfc_sli4_cleanup_poll_list(struct lpfc_hba *phba) +{ + struct lpfc_queue *eq, *next; + + list_for_each_entry_safe(eq, next, &phba->poll_list, _poll_list) + list_del(&eq->_poll_list); + + INIT_LIST_HEAD(&phba->poll_list); + synchronize_rcu(); +} + +static inline void +__lpfc_sli4_switch_eqmode(struct lpfc_queue *eq, uint8_t mode) +{ + if (mode == eq->mode) + return; + /* + * currently this function is only called during a hotplug + * event and the cpu on which this function is executing + * is going offline. By now the hotplug has instructed + * the scheduler to remove this cpu from cpu active mask. + * So we don't need to work about being put aside by the + * scheduler for a high priority process. Yes, the inte- + * rrupts could come but they are known to retire ASAP. + */ + + /* Disable polling in the fastpath */ + WRITE_ONCE(eq->mode, mode); + /* flush out the store buffer */ + smp_wmb(); + + /* + * Add this eq to the polling list and start polling. For + * a grace period both interrupt handler and poller will + * try to process the eq _but_ that's fine. We have a + * synchronization mechanism in place (queue_claimed) to + * deal with it. This is just a draining phase for int- + * errupt handler (not eq's) as we have guranteed through + * barrier that all the CPUs have seen the new CQ_POLLED + * state. which will effectively disable the REARMING of + * the EQ. The whole idea is eq's die off eventually as + * we are not rearming EQ's anymore. + */ + mode ? lpfc_sli4_add_to_poll_list(eq) : + lpfc_sli4_remove_from_poll_list(eq); +} + +void lpfc_sli4_start_polling(struct lpfc_queue *eq) +{ + __lpfc_sli4_switch_eqmode(eq, LPFC_EQ_POLL); +} + +void lpfc_sli4_stop_polling(struct lpfc_queue *eq) +{ + struct lpfc_hba *phba = eq->phba; + + __lpfc_sli4_switch_eqmode(eq, LPFC_EQ_INTERRUPT); + + /* Kick start for the pending io's in h/w. + * Once we switch back to interrupt processing on a eq + * the io path completion will only arm eq's when it + * receives a completion. But since eq's are in disa- + * rmed state it doesn't receive a completion. This + * creates a deadlock scenaro. + */ + phba->sli4_hba.sli4_write_eq_db(phba, eq, 0, LPFC_QUEUE_REARM); +} + /** * lpfc_sli4_queue_free - free a queue structure and associated memory * @queue: The queue structure to free. @@ -14541,6 +15847,9 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue) if (!queue) return; + if (!list_empty(&queue->wq_list)) + list_del(&queue->wq_list); + while (!list_empty(&queue->page_list)) { list_remove_head(&queue->page_list, dmabuf, struct lpfc_dmabuf, list); @@ -14553,8 +15862,8 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue) kfree(queue->rqbp); } - if (!list_empty(&queue->wq_list)) - list_del(&queue->wq_list); + if (!list_empty(&queue->cpu_list)) + list_del(&queue->cpu_list); kfree(queue); return; @@ -14565,7 +15874,8 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue) * @phba: The HBA that this queue is being created on. * @page_size: The size of a queue page * @entry_size: The size of each queue entry for this queue. - * @entry count: The number of entries that this queue will handle. + * @entry_count: The number of entries that this queue will handle. + * @cpu: The cpu that will primarily utilize this queue. * * This function allocates a queue structure and the DMAable memory used for * the host resident queue. This function must be called before creating the @@ -14573,45 +15883,49 @@ lpfc_sli4_queue_free(struct lpfc_queue *queue) **/ struct lpfc_queue * lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size, - uint32_t entry_size, uint32_t entry_count) + uint32_t entry_size, uint32_t entry_count, int cpu) { struct lpfc_queue *queue; struct lpfc_dmabuf *dmabuf; - int x, total_qe_count; - void *dma_pointer; uint32_t hw_page_size = phba->sli4_hba.pc_sli4_params.if_page_sz; + uint16_t x, pgcnt; if (!phba->sli4_hba.pc_sli4_params.supported) hw_page_size = page_size; - queue = kzalloc(sizeof(struct lpfc_queue) + - (sizeof(union sli4_qe) * entry_count), GFP_KERNEL); - if (!queue) - return NULL; - queue->page_count = (ALIGN(entry_size * entry_count, - hw_page_size))/hw_page_size; + pgcnt = ALIGN(entry_size * entry_count, hw_page_size) / hw_page_size; /* If needed, Adjust page count to match the max the adapter supports */ - if (phba->sli4_hba.pc_sli4_params.wqpcnt && - (queue->page_count > phba->sli4_hba.pc_sli4_params.wqpcnt)) - queue->page_count = phba->sli4_hba.pc_sli4_params.wqpcnt; + if (pgcnt > phba->sli4_hba.pc_sli4_params.wqpcnt) + pgcnt = phba->sli4_hba.pc_sli4_params.wqpcnt; + + queue = kzalloc_node(sizeof(*queue) + (sizeof(void *) * pgcnt), + GFP_KERNEL, cpu_to_node(cpu)); + if (!queue) + return NULL; INIT_LIST_HEAD(&queue->list); + INIT_LIST_HEAD(&queue->_poll_list); INIT_LIST_HEAD(&queue->wq_list); INIT_LIST_HEAD(&queue->wqfull_list); INIT_LIST_HEAD(&queue->page_list); INIT_LIST_HEAD(&queue->child_list); + INIT_LIST_HEAD(&queue->cpu_list); /* Set queue parameters now. If the system cannot provide memory * resources, the free routine needs to know what was allocated. */ + queue->page_count = pgcnt; + queue->q_pgs = (void **)&queue[1]; + queue->entry_cnt_per_pg = hw_page_size / entry_size; queue->entry_size = entry_size; queue->entry_count = entry_count; queue->page_size = hw_page_size; queue->phba = phba; - for (x = 0, total_qe_count = 0; x < queue->page_count; x++) { - dmabuf = kzalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); + for (x = 0; x < queue->page_count; x++) { + dmabuf = kzalloc_node(sizeof(*dmabuf), GFP_KERNEL, + dev_to_node(&phba->pcidev->dev)); if (!dmabuf) goto out_fail; dmabuf->virt = dma_alloc_coherent(&phba->pcidev->dev, @@ -14623,18 +15937,15 @@ lpfc_sli4_queue_alloc(struct lpfc_hba *phba, uint32_t page_size, } dmabuf->buffer_tag = x; list_add_tail(&dmabuf->list, &queue->page_list); - /* initialize queue's entry array */ - dma_pointer = dmabuf->virt; - for (; total_qe_count < entry_count && - dma_pointer < (hw_page_size + dmabuf->virt); - total_qe_count++, dma_pointer += entry_size) { - queue->qe[total_qe_count].address = dma_pointer; - } + /* use lpfc_sli4_qe to index a paritcular entry in this page */ + queue->q_pgs[x] = dmabuf->virt; } INIT_WORK(&queue->irqwork, lpfc_sli4_hba_process_cq); INIT_WORK(&queue->spwork, lpfc_sli4_sp_process_cq); + INIT_DELAYED_WORK(&queue->sched_irqwork, lpfc_sli4_dly_hba_process_cq); + INIT_DELAYED_WORK(&queue->sched_spwork, lpfc_sli4_dly_sp_process_cq); - /* entry_repost will be set during q creation */ + /* notify_interval will be set during q creation */ return queue; out_fail: @@ -14671,43 +15982,75 @@ lpfc_dual_chute_pci_bar_map(struct lpfc_hba *phba, uint16_t pci_barset) } /** - * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on FCP EQs - * @phba: HBA structure that indicates port to create a queue on. - * @startq: The starting FCP EQ to modify + * lpfc_modify_hba_eq_delay - Modify Delay Multiplier on EQs + * @phba: HBA structure that EQs are on. + * @startq: The starting EQ index to modify + * @numq: The number of EQs (consecutive indexes) to modify + * @usdelay: amount of delay * - * This function sends an MODIFY_EQ_DELAY mailbox command to the HBA. - * The command allows up to LPFC_MAX_EQ_DELAY_EQID_CNT EQ ID's to be - * updated in one mailbox command. + * This function revises the EQ delay on 1 or more EQs. The EQ delay + * is set either by writing to a register (if supported by the SLI Port) + * or by mailbox command. The mailbox command allows several EQs to be + * updated at once. * - * The @phba struct is used to send mailbox command to HBA. The @startq - * is used to get the starting FCP EQ to change. - * This function is asynchronous and will wait for the mailbox - * command to finish before continuing. + * The @phba struct is used to send a mailbox command to HBA. The @startq + * is used to get the starting EQ index to change. The @numq value is + * used to specify how many consecutive EQ indexes, starting at EQ index, + * are to be changed. This function is asynchronous and will wait for any + * mailbox commands to finish before returning. * - * On success this function will return a zero. If unable to allocate enough - * memory this function will return -ENOMEM. If the queue create mailbox command - * fails this function will return -ENXIO. + * On success this function will return a zero. If unable to allocate + * enough memory this function will return -ENOMEM. If a mailbox command + * fails this function will return -ENXIO. Note: on ENXIO, some EQs may + * have had their delay multipler changed. **/ -int +void lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, - uint32_t numq, uint32_t imax) + uint32_t numq, uint32_t usdelay) { struct lpfc_mbx_modify_eq_delay *eq_delay; LPFC_MBOXQ_t *mbox; struct lpfc_queue *eq; - int cnt, rc, length, status = 0; + int cnt = 0, rc, length; uint32_t shdr_status, shdr_add_status; - uint32_t result, val; + uint32_t dmult; int qidx; union lpfc_sli4_cfg_shdr *shdr; - uint16_t dmult; - if (startq >= phba->io_channel_irqs) - return 0; + if (startq >= phba->cfg_irq_chann) + return; + + if (usdelay > 0xFFFF) { + lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_FCP | LOG_NVME, + "6429 usdelay %d too large. Scaled down to " + "0xFFFF.\n", usdelay); + usdelay = 0xFFFF; + } + + /* set values by EQ_DELAY register if supported */ + if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) { + for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) { + eq = phba->sli4_hba.hba_eq_hdl[qidx].eq; + if (!eq) + continue; + + lpfc_sli4_mod_hba_eq_delay(phba, eq, usdelay); + + if (++cnt >= numq) + break; + } + return; + } + + /* Otherwise, set values by mailbox cmd */ mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); - if (!mbox) - return -ENOMEM; + if (!mbox) { + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6428 Failed allocating mailbox cmd buffer." + " EQ delay was not set.\n"); + return; + } length = (sizeof(struct lpfc_mbx_modify_eq_delay) - sizeof(struct lpfc_sli4_cfg_mhdr)); lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON, @@ -14716,66 +16059,41 @@ lpfc_modify_hba_eq_delay(struct lpfc_hba *phba, uint32_t startq, eq_delay = &mbox->u.mqe.un.eq_delay; /* Calculate delay multiper from maximum interrupt per second */ - result = imax / phba->io_channel_irqs; - if (result > LPFC_DMULT_CONST || result == 0) - dmult = 0; - else - dmult = LPFC_DMULT_CONST/result - 1; + dmult = (usdelay * LPFC_DMULT_CONST) / LPFC_SEC_TO_USEC; + if (dmult) + dmult--; if (dmult > LPFC_DMULT_MAX) dmult = LPFC_DMULT_MAX; - cnt = 0; - for (qidx = startq; qidx < phba->io_channel_irqs; qidx++) { - eq = phba->sli4_hba.hba_eq[qidx]; + for (qidx = startq; qidx < phba->cfg_irq_chann; qidx++) { + eq = phba->sli4_hba.hba_eq_hdl[qidx].eq; if (!eq) continue; - eq->q_mode = imax; + eq->q_mode = usdelay; eq_delay->u.request.eq[cnt].eq_id = eq->queue_id; eq_delay->u.request.eq[cnt].phase = 0; eq_delay->u.request.eq[cnt].delay_multi = dmult; - cnt++; - - /* q_mode is only used for auto_imax */ - if (phba->sli.sli_flag & LPFC_SLI_USE_EQDR) { - /* Use EQ Delay Register method for q_mode */ - - /* Convert for EQ Delay register */ - val = phba->cfg_fcp_imax; - if (val) { - /* First, interrupts per sec per EQ */ - val = phba->cfg_fcp_imax / - phba->io_channel_irqs; - /* us delay between each interrupt */ - val = LPFC_SEC_TO_USEC / val; - } - eq->q_mode = val; - } else { - eq->q_mode = imax; - } - - if (cnt >= numq) + if (++cnt >= numq) break; } eq_delay->u.request.num_eq = cnt; mbox->vport = phba->pport; mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl; - mbox->ctx_buf = NULL; mbox->ctx_ndlp = NULL; rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); shdr = (union lpfc_sli4_cfg_shdr *) &eq_delay->header.cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2512 MODIFY_EQ_DELAY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); - status = -ENXIO; } mempool_free(mbox, phba->mbox_mem_pool); - return status; + return; } /** @@ -14846,12 +16164,14 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax) dmult); switch (eq->entry_count) { default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0360 Unsupported EQ count. (%d)\n", eq->entry_count); - if (eq->entry_count < 256) - return -EINVAL; - /* fall through - otherwise default to smallest count */ + if (eq->entry_count < 256) { + status = -EINVAL; + goto out; + } + fallthrough; /* otherwise default to smallest count */ case 256: bf_set(lpfc_eq_context_count, &eq_create->u.request.context, LPFC_EQ_CNT_256); @@ -14888,7 +16208,7 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax) shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2500 EQ_CREATE mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -14900,18 +16220,85 @@ lpfc_eq_create(struct lpfc_hba *phba, struct lpfc_queue *eq, uint32_t imax) if (eq->queue_id == 0xFFFF) status = -ENXIO; eq->host_index = 0; - eq->hba_index = 0; - eq->entry_repost = LPFC_EQ_REPOST; - + eq->notify_interval = LPFC_EQ_NOTIFY_INTRVL; + eq->max_proc_limit = LPFC_EQ_MAX_PROC_LIMIT; +out: mempool_free(mbox, phba->mbox_mem_pool); return status; } /** + * lpfc_sli4_hba_intr_handler_th - SLI4 HBA threaded interrupt handler + * @irq: Interrupt number. + * @dev_id: The device context pointer. + * + * This routine is a mirror of lpfc_sli4_hba_intr_handler, but executed within + * threaded irq context. + * + * Returns + * IRQ_HANDLED - interrupt is handled + * IRQ_NONE - otherwise + **/ +irqreturn_t lpfc_sli4_hba_intr_handler_th(int irq, void *dev_id) +{ + struct lpfc_hba *phba; + struct lpfc_hba_eq_hdl *hba_eq_hdl; + struct lpfc_queue *fpeq; + int ecount = 0; + int hba_eqidx; + struct lpfc_eq_intr_info *eqi; + + /* Get the driver's phba structure from the dev_id */ + hba_eq_hdl = (struct lpfc_hba_eq_hdl *)dev_id; + phba = hba_eq_hdl->phba; + hba_eqidx = hba_eq_hdl->idx; + + if (unlikely(!phba)) + return IRQ_NONE; + if (unlikely(!phba->sli4_hba.hdwq)) + return IRQ_NONE; + + /* Get to the EQ struct associated with this vector */ + fpeq = phba->sli4_hba.hba_eq_hdl[hba_eqidx].eq; + if (unlikely(!fpeq)) + return IRQ_NONE; + + eqi = per_cpu_ptr(phba->sli4_hba.eq_info, raw_smp_processor_id()); + eqi->icnt++; + + fpeq->last_cpu = raw_smp_processor_id(); + + if (eqi->icnt > LPFC_EQD_ISR_TRIGGER && + fpeq->q_flag & HBA_EQ_DELAY_CHK && + phba->cfg_auto_imax && + fpeq->q_mode != LPFC_MAX_AUTO_EQ_DELAY && + phba->sli.sli_flag & LPFC_SLI_USE_EQDR) + lpfc_sli4_mod_hba_eq_delay(phba, fpeq, LPFC_MAX_AUTO_EQ_DELAY); + + /* process and rearm the EQ */ + ecount = lpfc_sli4_process_eq(phba, fpeq, LPFC_QUEUE_REARM, + LPFC_THREADED_IRQ); + + if (unlikely(ecount == 0)) { + fpeq->EQ_no_entry++; + if (phba->intr_type == MSIX) + /* MSI-X treated interrupt served as no EQ share INT */ + lpfc_printf_log(phba, KERN_WARNING, LOG_SLI, + "3358 MSI-X interrupt with no EQE\n"); + else + /* Non MSI-X treated on interrupt as EQ share INT */ + return IRQ_NONE; + } + return IRQ_HANDLED; +} + +/** * lpfc_cq_create - Create a Completion Queue on the HBA * @phba: HBA structure that indicates port to create a queue on. * @cq: The queue structure to use to create the completion queue. * @eq: The event queue to bind this completion queue to. + * @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc). + * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc). * * This function creates a completion queue, as detailed in @wq, on a port, * described by @phba by sending a CQ_CREATE mailbox command to the HBA. @@ -14982,9 +16369,9 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, LPFC_CQ_CNT_WORD7); break; } - /* fall through */ + fallthrough; default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0361 Unsupported CQ count: " "entry cnt %d sz %d pg cnt %d\n", cq->entry_count, cq->entry_size, @@ -14993,7 +16380,7 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, status = -EINVAL; goto out; } - /* fall through - otherwise default to smallest count */ + fallthrough; /* otherwise default to smallest count */ case 256: bf_set(lpfc_cq_context_count, &cq_create->u.request.context, LPFC_CQ_CNT_256); @@ -15020,7 +16407,7 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2501 CQ_CREATE mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -15039,10 +16426,13 @@ lpfc_cq_create(struct lpfc_hba *phba, struct lpfc_queue *cq, cq->subtype = subtype; cq->queue_id = bf_get(lpfc_mbx_cq_create_q_id, &cq_create->u.response); cq->assoc_qid = eq->queue_id; + cq->assoc_qp = eq; cq->host_index = 0; - cq->hba_index = 0; - cq->entry_repost = LPFC_CQ_REPOST; + cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL; + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, cq->entry_count); + if (cq->queue_id > phba->sli4_hba.cq_max) + phba->sli4_hba.cq_max = cq->queue_id; out: mempool_free(mbox, phba->mbox_mem_pool); return status; @@ -15052,7 +16442,9 @@ out: * lpfc_cq_create_set - Create a set of Completion Queues on the HBA for MRQ * @phba: HBA structure that indicates port to create a queue on. * @cqp: The queue structure array to use to create the completion queues. - * @eqp: The event queue array to bind these completion queues to. + * @hdwq: The hardware queue array with the EQ to bind completion queues to. + * @type: Type of queue (EQ, GCQ, MCQ, WCQ, etc). + * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc). * * This function creates a set of completion queue, s to support MRQ * as detailed in @cqp, on a port, @@ -15072,7 +16464,8 @@ out: **/ int lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, - struct lpfc_queue **eqp, uint32_t type, uint32_t subtype) + struct lpfc_sli4_hdw_queue *hdwq, uint32_t type, + uint32_t subtype) { struct lpfc_queue *cq; struct lpfc_queue *eq; @@ -15087,7 +16480,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, /* sanity check on queue memory */ numcq = phba->cfg_nvmet_mrq; - if (!cqp || !eqp || !numcq) + if (!cqp || !hdwq || !numcq) return -ENODEV; mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); @@ -15101,7 +16494,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, LPFC_MBOX_OPCODE_FCOE_CQ_CREATE_SET, length, LPFC_SLI4_MBX_NEMBED); if (alloclen < length) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3098 Allocated DMA memory size (%d) is " "less than the requested DMA memory size " "(%d)\n", alloclen, length); @@ -15114,7 +16507,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, for (idx = 0; idx < numcq; idx++) { cq = cqp[idx]; - eq = eqp[idx]; + eq = hdwq[idx].hba_eq; if (!cq || !eq) { status = -ENOMEM; goto out; @@ -15145,34 +16538,34 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, case 4096: if (phba->sli4_hba.pc_sli4_params.cqv == LPFC_Q_CREATE_VERSION_2) { - bf_set(lpfc_mbx_cq_create_set_cqe_cnt, + bf_set(lpfc_mbx_cq_create_set_cqe_cnt_lo, &cq_set->u.request, - cq->entry_count); - bf_set(lpfc_mbx_cq_create_set_cqe_cnt, + cq->entry_count); + bf_set(lpfc_mbx_cq_create_set_cqecnt, &cq_set->u.request, LPFC_CQ_CNT_WORD7); break; } - /* fall through */ + fallthrough; default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3118 Bad CQ count. (%d)\n", cq->entry_count); if (cq->entry_count < 256) { status = -EINVAL; goto out; } - /* fall through - otherwise default to smallest */ + fallthrough; /* otherwise default to smallest */ case 256: - bf_set(lpfc_mbx_cq_create_set_cqe_cnt, + bf_set(lpfc_mbx_cq_create_set_cqecnt, &cq_set->u.request, LPFC_CQ_CNT_256); break; case 512: - bf_set(lpfc_mbx_cq_create_set_cqe_cnt, + bf_set(lpfc_mbx_cq_create_set_cqecnt, &cq_set->u.request, LPFC_CQ_CNT_512); break; case 1024: - bf_set(lpfc_mbx_cq_create_set_cqe_cnt, + bf_set(lpfc_mbx_cq_create_set_cqecnt, &cq_set->u.request, LPFC_CQ_CNT_1024); break; } @@ -15247,9 +16640,11 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, cq->type = type; cq->subtype = subtype; cq->assoc_qid = eq->queue_id; + cq->assoc_qp = eq; cq->host_index = 0; - cq->hba_index = 0; - cq->entry_repost = LPFC_CQ_REPOST; + cq->notify_interval = LPFC_CQ_NOTIFY_INTRVL; + cq->max_proc_limit = min(phba->cfg_cq_max_proc_limit, + cq->entry_count); cq->chann = idx; rc = 0; @@ -15271,7 +16666,7 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3119 CQ_CREATE_SET mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -15287,6 +16682,8 @@ lpfc_cq_create_set(struct lpfc_hba *phba, struct lpfc_queue **cqp, for (idx = 0; idx < numcq; idx++) { cq = cqp[idx]; cq->queue_id = rc + idx; + if (cq->queue_id > phba->sli4_hba.cq_max) + phba->sli4_hba.cq_max = cq->queue_id; } out: @@ -15427,14 +16824,14 @@ lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq, cq->queue_id); switch (mq->entry_count) { default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0362 Unsupported MQ count. (%d)\n", mq->entry_count); if (mq->entry_count < 16) { status = -EINVAL; goto out; } - /* fall through - otherwise default to smallest count */ + fallthrough; /* otherwise default to smallest count */ case 16: bf_set(lpfc_mq_context_ring_size, &mq_create_ext->u.request.context, @@ -15483,7 +16880,7 @@ lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2502 MQ_CREATE mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -15499,7 +16896,6 @@ lpfc_mq_create(struct lpfc_hba *phba, struct lpfc_queue *mq, mq->subtype = subtype; mq->host_index = 0; mq->hba_index = 0; - mq->entry_repost = LPFC_MQ_REPOST; /* link the mq onto the parent cq child list */ list_add_tail(&mq->list, &cq->child_list); @@ -15547,8 +16943,10 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, uint16_t pci_barset; uint8_t dpp_barset; uint32_t dpp_offset; - unsigned long pg_addr; uint8_t wq_create_version; +#ifdef CONFIG_X86 + unsigned long pg_addr; +#endif /* sanity check on queue memory */ if (!wq || !cq) @@ -15581,12 +16979,6 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, else wq_create_version = LPFC_Q_CREATE_VERSION_0; - - if (phba->sli4_hba.pc_sli4_params.wqsize & LPFC_WQ_SZ128_SUPPORT) - wq_create_version = LPFC_Q_CREATE_VERSION_1; - else - wq_create_version = LPFC_Q_CREATE_VERSION_0; - switch (wq_create_version) { case LPFC_Q_CREATE_VERSION_1: bf_set(lpfc_mbx_wq_create_wqe_count, &wq_create->u.request_1, @@ -15633,7 +17025,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2503 WQ_CREATE mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -15660,7 +17052,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, &wq_create->u.response); if ((wq->db_format != LPFC_DB_LIST_FORMAT) && (wq->db_format != LPFC_DB_RING_FORMAT)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3265 WQ[%d] doorbell format " "not supported: x%x\n", wq->queue_id, wq->db_format); @@ -15672,7 +17064,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset); if (!bar_memmap_p) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3263 WQ[%d] failed to memmap " "pci barset:x%x\n", wq->queue_id, pci_barset); @@ -15682,7 +17074,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, db_offset = wq_create->u.response.doorbell_offset; if ((db_offset != LPFC_ULP0_WQ_DOORBELL) && (db_offset != LPFC_ULP1_WQ_DOORBELL)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3252 WQ[%d] doorbell offset " "not supported: x%x\n", wq->queue_id, db_offset); @@ -15706,7 +17098,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset); if (!bar_memmap_p) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3267 WQ[%d] failed to memmap " "pci barset:x%x\n", wq->queue_id, pci_barset); @@ -15722,7 +17114,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, dpp_barset); if (!bar_memmap_p) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3268 WQ[%d] failed to memmap " "pci barset:x%x\n", wq->queue_id, dpp_barset); @@ -15738,9 +17130,9 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, wq->queue_id, pci_barset, db_offset, wq->dpp_id, dpp_barset, dpp_offset); +#ifdef CONFIG_X86 /* Enable combined writes for DPP aperture */ pg_addr = (unsigned long)(wq->dpp_regaddr) & PAGE_MASK; -#ifdef CONFIG_X86 rc = set_memory_wc(pg_addr, 1); if (rc) { lpfc_printf_log(phba, KERN_ERR, LOG_INIT, @@ -15765,7 +17157,7 @@ lpfc_wq_create(struct lpfc_hba *phba, struct lpfc_queue *wq, wq->subtype = subtype; wq->host_index = 0; wq->hba_index = 0; - wq->entry_repost = LPFC_RELEASE_NOTIFICATION_INTERVAL; + wq->notify_interval = LPFC_WQ_NOTIFY_INTRVL; /* link the wq onto the parent cq child list */ list_add_tail(&wq->list, &cq->child_list); @@ -15780,6 +17172,7 @@ out: * @hrq: The queue structure to use to create the header receive queue. * @drq: The queue structure to use to create the data receive queue. * @cq: The completion queue to bind this work queue to. + * @subtype: The subtype of the work queue indicating its functionality. * * This function creates a receive buffer queue pair , as detailed in @hrq and * @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command @@ -15846,14 +17239,14 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, } else { switch (hrq->entry_count) { default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2535 Unsupported RQ count. (%d)\n", hrq->entry_count); if (hrq->entry_count < 512) { status = -EINVAL; goto out; } - /* fall through - otherwise default to smallest count */ + fallthrough; /* otherwise default to smallest count */ case 512: bf_set(lpfc_rq_context_rqe_count, &rq_create->u.request.context, @@ -15897,7 +17290,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2504 RQ_CREATE mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -15915,7 +17308,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, &rq_create->u.response); if ((hrq->db_format != LPFC_DB_LIST_FORMAT) && (hrq->db_format != LPFC_DB_RING_FORMAT)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3262 RQ [%d] doorbell format not " "supported: x%x\n", hrq->queue_id, hrq->db_format); @@ -15927,7 +17320,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, &rq_create->u.response); bar_memmap_p = lpfc_dual_chute_pci_bar_map(phba, pci_barset); if (!bar_memmap_p) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3269 RQ[%d] failed to memmap pci " "barset:x%x\n", hrq->queue_id, pci_barset); @@ -15938,7 +17331,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, db_offset = rq_create->u.response.doorbell_offset; if ((db_offset != LPFC_ULP0_RQ_DOORBELL) && (db_offset != LPFC_ULP1_RQ_DOORBELL)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3270 RQ[%d] doorbell offset not " "supported: x%x\n", hrq->queue_id, db_offset); @@ -15959,7 +17352,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, hrq->subtype = subtype; hrq->host_index = 0; hrq->hba_index = 0; - hrq->entry_repost = LPFC_RQ_REPOST; + hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; /* now create the data queue */ lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, @@ -15983,14 +17376,14 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, } else { switch (drq->entry_count) { default: - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2536 Unsupported RQ count. (%d)\n", drq->entry_count); if (drq->entry_count < 512) { status = -EINVAL; goto out; } - /* fall through - otherwise default to smallest count */ + fallthrough; /* otherwise default to smallest count */ case 512: bf_set(lpfc_rq_context_rqe_count, &rq_create->u.request.context, @@ -16052,7 +17445,7 @@ lpfc_rq_create(struct lpfc_hba *phba, struct lpfc_queue *hrq, drq->subtype = subtype; drq->host_index = 0; drq->hba_index = 0; - drq->entry_repost = LPFC_RQ_REPOST; + drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; /* link the header and data RQs onto the parent cq child list */ list_add_tail(&hrq->list, &cq->child_list); @@ -16069,6 +17462,7 @@ out: * @hrqp: The queue structure array to use to create the header receive queues. * @drqp: The queue structure array to use to create the data receive queues. * @cqp: The completion queue array to bind these receive queues to. + * @subtype: Functional purpose of the queue (MBOX, IO, ELS, NVMET, etc). * * This function creates a receive buffer queue pair , as detailed in @hrq and * @drq, on a port, described by @phba by sending a RQ_CREATE mailbox command @@ -16120,7 +17514,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, LPFC_MBOX_OPCODE_FCOE_RQ_CREATE, length, LPFC_SLI4_MBX_NEMBED); if (alloclen < length) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3099 Allocated DMA memory size (%d) is " "less than the requested DMA memory size " "(%d)\n", alloclen, length); @@ -16210,7 +17604,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, hrq->subtype = subtype; hrq->host_index = 0; hrq->hba_index = 0; - hrq->entry_repost = LPFC_RQ_REPOST; + hrq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; drq->db_format = LPFC_DB_RING_FORMAT; drq->db_regaddr = phba->sli4_hba.RQDBregaddr; @@ -16219,7 +17613,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, drq->subtype = subtype; drq->host_index = 0; drq->hba_index = 0; - drq->entry_repost = LPFC_RQ_REPOST; + drq->notify_interval = LPFC_RQ_NOTIFY_INTRVL; list_add_tail(&hrq->list, &cq->child_list); list_add_tail(&drq->list, &cq->child_list); @@ -16230,7 +17624,7 @@ lpfc_mrq_create(struct lpfc_hba *phba, struct lpfc_queue **hrqp, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3120 RQ_CREATE mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -16258,6 +17652,7 @@ out: /** * lpfc_eq_destroy - Destroy an event Queue on the HBA + * @phba: HBA structure that indicates port to destroy a queue on. * @eq: The queue structure associated with the queue to destroy. * * This function destroys a queue, as detailed in @eq by sending an mailbox @@ -16279,6 +17674,10 @@ lpfc_eq_destroy(struct lpfc_hba *phba, struct lpfc_queue *eq) /* sanity check on queue memory */ if (!eq) return -ENODEV; + + if (!(phba->sli.sli_flag & LPFC_SLI_ACTIVE)) + goto list_remove; + mbox = mempool_alloc(eq->phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -16299,21 +17698,24 @@ lpfc_eq_destroy(struct lpfc_hba *phba, struct lpfc_queue *eq) shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2505 EQ_DESTROY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); status = -ENXIO; } + mempool_free(mbox, eq->phba->mbox_mem_pool); +list_remove: /* Remove eq from any list */ list_del_init(&eq->list); - mempool_free(mbox, eq->phba->mbox_mem_pool); + return status; } /** * lpfc_cq_destroy - Destroy a Completion Queue on the HBA + * @phba: HBA structure that indicates port to destroy a queue on. * @cq: The queue structure associated with the queue to destroy. * * This function destroys a queue, as detailed in @cq by sending an mailbox @@ -16335,6 +17737,10 @@ lpfc_cq_destroy(struct lpfc_hba *phba, struct lpfc_queue *cq) /* sanity check on queue memory */ if (!cq) return -ENODEV; + + if (!(phba->sli.sli_flag & LPFC_SLI_ACTIVE)) + goto list_remove; + mbox = mempool_alloc(cq->phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -16354,21 +17760,24 @@ lpfc_cq_destroy(struct lpfc_hba *phba, struct lpfc_queue *cq) shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2506 CQ_DESTROY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); status = -ENXIO; } + mempool_free(mbox, cq->phba->mbox_mem_pool); + +list_remove: /* Remove cq from any list */ list_del_init(&cq->list); - mempool_free(mbox, cq->phba->mbox_mem_pool); return status; } /** * lpfc_mq_destroy - Destroy a Mailbox Queue on the HBA - * @qm: The queue structure associated with the queue to destroy. + * @phba: HBA structure that indicates port to destroy a queue on. + * @mq: The queue structure associated with the queue to destroy. * * This function destroys a queue, as detailed in @mq by sending an mailbox * command, specific to the type of queue, to the HBA. @@ -16389,6 +17798,10 @@ lpfc_mq_destroy(struct lpfc_hba *phba, struct lpfc_queue *mq) /* sanity check on queue memory */ if (!mq) return -ENODEV; + + if (!(phba->sli.sli_flag & LPFC_SLI_ACTIVE)) + goto list_remove; + mbox = mempool_alloc(mq->phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -16408,20 +17821,23 @@ lpfc_mq_destroy(struct lpfc_hba *phba, struct lpfc_queue *mq) shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2507 MQ_DESTROY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); status = -ENXIO; } + mempool_free(mbox, mq->phba->mbox_mem_pool); + +list_remove: /* Remove mq from any list */ list_del_init(&mq->list); - mempool_free(mbox, mq->phba->mbox_mem_pool); return status; } /** * lpfc_wq_destroy - Destroy a Work Queue on the HBA + * @phba: HBA structure that indicates port to destroy a queue on. * @wq: The queue structure associated with the queue to destroy. * * This function destroys a queue, as detailed in @wq by sending an mailbox @@ -16443,6 +17859,10 @@ lpfc_wq_destroy(struct lpfc_hba *phba, struct lpfc_queue *wq) /* sanity check on queue memory */ if (!wq) return -ENODEV; + + if (!(phba->sli.sli_flag & LPFC_SLI_ACTIVE)) + goto list_remove; + mbox = mempool_alloc(wq->phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -16461,23 +17881,27 @@ lpfc_wq_destroy(struct lpfc_hba *phba, struct lpfc_queue *wq) shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2508 WQ_DESTROY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); status = -ENXIO; } + mempool_free(mbox, wq->phba->mbox_mem_pool); + +list_remove: /* Remove wq from any list */ list_del_init(&wq->list); kfree(wq->pring); wq->pring = NULL; - mempool_free(mbox, wq->phba->mbox_mem_pool); return status; } /** * lpfc_rq_destroy - Destroy a Receive Queue on the HBA - * @rq: The queue structure associated with the queue to destroy. + * @phba: HBA structure that indicates port to destroy a queue on. + * @hrq: The queue structure associated with the queue to destroy. + * @drq: The queue structure associated with the queue to destroy. * * This function destroys a queue, as detailed in @rq by sending an mailbox * command, specific to the type of queue, to the HBA. @@ -16499,6 +17923,10 @@ lpfc_rq_destroy(struct lpfc_hba *phba, struct lpfc_queue *hrq, /* sanity check on queue memory */ if (!hrq || !drq) return -ENODEV; + + if (!(phba->sli.sli_flag & LPFC_SLI_ACTIVE)) + goto list_remove; + mbox = mempool_alloc(hrq->phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) return -ENOMEM; @@ -16518,12 +17946,11 @@ lpfc_rq_destroy(struct lpfc_hba *phba, struct lpfc_queue *hrq, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2509 RQ_DESTROY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); - if (rc != MBX_TIMEOUT) - mempool_free(mbox, hrq->phba->mbox_mem_pool); + mempool_free(mbox, hrq->phba->mbox_mem_pool); return -ENXIO; } bf_set(lpfc_mbx_rq_destroy_q_id, &mbox->u.mqe.un.rq_destroy.u.request, @@ -16534,15 +17961,17 @@ lpfc_rq_destroy(struct lpfc_hba *phba, struct lpfc_queue *hrq, shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2510 RQ_DESTROY mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); status = -ENXIO; } + mempool_free(mbox, hrq->phba->mbox_mem_pool); + +list_remove: list_del_init(&hrq->list); list_del_init(&drq->list); - mempool_free(mbox, hrq->phba->mbox_mem_pool); return status; } @@ -16582,7 +18011,7 @@ lpfc_sli4_post_sgl(struct lpfc_hba *phba, union lpfc_sli4_cfg_shdr *shdr; if (xritag == NO_XRI) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0364 Invalid param:\n"); return -EINVAL; } @@ -16620,10 +18049,12 @@ lpfc_sli4_post_sgl(struct lpfc_hba *phba, shdr = (union lpfc_sli4_cfg_shdr *) &post_sgl_pages->header.cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) + if (!phba->sli4_hba.intr_enable) + mempool_free(mbox, phba->mbox_mem_pool); + else if (rc != MBX_TIMEOUT) mempool_free(mbox, phba->mbox_mem_pool); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2511 POST_SGL mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -16654,8 +18085,8 @@ lpfc_sli4_alloc_xri(struct lpfc_hba *phba) * the driver starts at 0 each time. */ spin_lock_irq(&phba->hbalock); - xri = find_next_zero_bit(phba->sli4_hba.xri_bmask, - phba->sli4_hba.max_cfg_param.max_xri, 0); + xri = find_first_zero_bit(phba->sli4_hba.xri_bmask, + phba->sli4_hba.max_cfg_param.max_xri); if (xri >= phba->sli4_hba.max_cfg_param.max_xri) { spin_unlock_irq(&phba->hbalock); return NO_XRI; @@ -16668,8 +18099,9 @@ lpfc_sli4_alloc_xri(struct lpfc_hba *phba) } /** - * lpfc_sli4_free_xri - Release an xri for reuse. + * __lpfc_sli4_free_xri - Release an xri for reuse. * @phba: pointer to lpfc hba data structure. + * @xri: xri to release. * * This routine is invoked to release an xri to the pool of * available rpis maintained by the driver. @@ -16685,6 +18117,7 @@ __lpfc_sli4_free_xri(struct lpfc_hba *phba, int xri) /** * lpfc_sli4_free_xri - Release an xri for reuse. * @phba: pointer to lpfc hba data structure. + * @xri: xri to release. * * This routine is invoked to release an xri to the pool of * available rpis maintained by the driver. @@ -16727,7 +18160,7 @@ lpfc_sli4_next_xritag(struct lpfc_hba *phba) * lpfc_sli4_post_sgl_list - post a block of ELS sgls to the port. * @phba: pointer to lpfc hba data structure. * @post_sgl_list: pointer to els sgl entry list. - * @count: number of els sgl entries on the list. + * @post_cnt: number of els sgl entries on the list. * * This routine is invoked to post a block of driver's sgl pages to the * HBA using non-embedded mailbox command. No Lock is held. This routine @@ -16754,7 +18187,7 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, reqlen = post_cnt * sizeof(struct sgl_page_pairs) + sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t); if (reqlen > SLI4_PAGE_SIZE) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2559 Block sgl registration required DMA " "size (%d) great than a page\n", reqlen); return -ENOMEM; @@ -16770,7 +18203,7 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, LPFC_SLI4_MBX_NEMBED); if (alloclen < reqlen) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "0285 Allocated DMA memory size (%d) is " "less than the requested DMA memory " "size (%d)\n", alloclen, reqlen); @@ -16815,10 +18248,12 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, shdr = (union lpfc_sli4_cfg_shdr *) &sgl->cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) + if (!phba->sli4_hba.intr_enable) + lpfc_sli4_mbox_cmd_free(phba, mbox); + else if (rc != MBX_TIMEOUT) lpfc_sli4_mbox_cmd_free(phba, mbox); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2513 POST_SGL_BLOCK mailbox command failed " "status x%x add_status x%x mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -16828,22 +18263,21 @@ lpfc_sli4_post_sgl_list(struct lpfc_hba *phba, } /** - * lpfc_sli4_post_scsi_sgl_block - post a block of scsi sgl list to firmware + * lpfc_sli4_post_io_sgl_block - post a block of nvme sgl list to firmware * @phba: pointer to lpfc hba data structure. - * @sblist: pointer to scsi buffer list. + * @nblist: pointer to nvme buffer list. * @count: number of scsi buffers on the list. * * This routine is invoked to post a block of @count scsi sgl pages from a - * SCSI buffer list @sblist to the HBA using non-embedded mailbox command. + * SCSI buffer list @nblist to the HBA using non-embedded mailbox command. * No Lock is held. * **/ -int -lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, - struct list_head *sblist, - int count) +static int +lpfc_sli4_post_io_sgl_block(struct lpfc_hba *phba, struct list_head *nblist, + int count) { - struct lpfc_scsi_buf *psb; + struct lpfc_io_buf *lpfc_ncmd; struct lpfc_mbx_post_uembed_sgl_page1 *sgl; struct sgl_page_pairs *sgl_pg_pairs; void *viraddr; @@ -16861,25 +18295,25 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t); if (reqlen > SLI4_PAGE_SIZE) { lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, - "0217 Block sgl registration required DMA " + "6118 Block sgl registration required DMA " "size (%d) great than a page\n", reqlen); return -ENOMEM; } mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "0283 Failed to allocate mbox cmd memory\n"); + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6119 Failed to allocate mbox cmd memory\n"); return -ENOMEM; } /* Allocate DMA memory and set up the non-embedded mailbox command */ alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, - LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen, - LPFC_SLI4_MBX_NEMBED); + LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, + reqlen, LPFC_SLI4_MBX_NEMBED); if (alloclen < reqlen) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, - "2561 Allocated DMA memory size (%d) is " + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6120 Allocated DMA memory size (%d) is " "less than the requested DMA memory " "size (%d)\n", alloclen, reqlen); lpfc_sli4_mbox_cmd_free(phba, mbox); @@ -16894,14 +18328,15 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, sgl_pg_pairs = &sgl->sgl_pg_pairs; pg_pairs = 0; - list_for_each_entry(psb, sblist, list) { + list_for_each_entry(lpfc_ncmd, nblist, list) { /* Set up the sge entry */ sgl_pg_pairs->sgl_pg0_addr_lo = - cpu_to_le32(putPaddrLow(psb->dma_phys_bpl)); + cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl)); sgl_pg_pairs->sgl_pg0_addr_hi = - cpu_to_le32(putPaddrHigh(psb->dma_phys_bpl)); + cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl)); if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE) - pdma_phys_bpl1 = psb->dma_phys_bpl + SGL_PAGE_SIZE; + pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl + + SGL_PAGE_SIZE; else pdma_phys_bpl1 = 0; sgl_pg_pairs->sgl_pg1_addr_lo = @@ -16910,7 +18345,7 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, cpu_to_le32(putPaddrHigh(pdma_phys_bpl1)); /* Keep the first xritag on the list */ if (pg_pairs == 0) - xritag_start = psb->cur_iocbq.sli4_xritag; + xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag; sgl_pg_pairs++; pg_pairs++; } @@ -16919,20 +18354,22 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, /* Perform endian conversion if necessary */ sgl->word0 = cpu_to_le32(sgl->word0); - if (!phba->sli4_hba.intr_enable) + if (!phba->sli4_hba.intr_enable) { rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); - else { + } else { mbox_tmo = lpfc_mbox_tmo_val(phba, mbox); rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); } - shdr = (union lpfc_sli4_cfg_shdr *) &sgl->cfg_shdr; + shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) + if (!phba->sli4_hba.intr_enable) + lpfc_sli4_mbox_cmd_free(phba, mbox); + else if (rc != MBX_TIMEOUT) lpfc_sli4_mbox_cmd_free(phba, mbox); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "2564 POST_SGL_BLOCK mailbox command failed " + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6125 POST_SGL_BLOCK mailbox command failed " "status x%x add_status x%x mbx status x%x\n", shdr_status, shdr_add_status, rc); rc = -ENXIO; @@ -16941,6 +18378,135 @@ lpfc_sli4_post_scsi_sgl_block(struct lpfc_hba *phba, } /** + * lpfc_sli4_post_io_sgl_list - Post blocks of nvme buffer sgls from a list + * @phba: pointer to lpfc hba data structure. + * @post_nblist: pointer to the nvme buffer list. + * @sb_count: number of nvme buffers. + * + * This routine walks a list of nvme buffers that was passed in. It attempts + * to construct blocks of nvme buffer sgls which contains contiguous xris and + * uses the non-embedded SGL block post mailbox commands to post to the port. + * For single NVME buffer sgl with non-contiguous xri, if any, it shall use + * embedded SGL post mailbox command for posting. The @post_nblist passed in + * must be local list, thus no lock is needed when manipulate the list. + * + * Returns: 0 = failure, non-zero number of successfully posted buffers. + **/ +int +lpfc_sli4_post_io_sgl_list(struct lpfc_hba *phba, + struct list_head *post_nblist, int sb_count) +{ + struct lpfc_io_buf *lpfc_ncmd, *lpfc_ncmd_next; + int status, sgl_size; + int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; + dma_addr_t pdma_phys_sgl1; + int last_xritag = NO_XRI; + int cur_xritag; + LIST_HEAD(prep_nblist); + LIST_HEAD(blck_nblist); + LIST_HEAD(nvme_nblist); + + /* sanity check */ + if (sb_count <= 0) + return -EINVAL; + + sgl_size = phba->cfg_sg_dma_buf_size; + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) { + list_del_init(&lpfc_ncmd->list); + block_cnt++; + if ((last_xritag != NO_XRI) && + (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) { + /* a hole in xri block, form a sgl posting block */ + list_splice_init(&prep_nblist, &blck_nblist); + post_cnt = block_cnt - 1; + /* prepare list for next posting block */ + list_add_tail(&lpfc_ncmd->list, &prep_nblist); + block_cnt = 1; + } else { + /* prepare list for next posting block */ + list_add_tail(&lpfc_ncmd->list, &prep_nblist); + /* enough sgls for non-embed sgl mbox command */ + if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { + list_splice_init(&prep_nblist, &blck_nblist); + post_cnt = block_cnt; + block_cnt = 0; + } + } + num_posting++; + last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; + + /* end of repost sgl list condition for NVME buffers */ + if (num_posting == sb_count) { + if (post_cnt == 0) { + /* last sgl posting block */ + list_splice_init(&prep_nblist, &blck_nblist); + post_cnt = block_cnt; + } else if (block_cnt == 1) { + /* last single sgl with non-contiguous xri */ + if (sgl_size > SGL_PAGE_SIZE) + pdma_phys_sgl1 = + lpfc_ncmd->dma_phys_sgl + + SGL_PAGE_SIZE; + else + pdma_phys_sgl1 = 0; + cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; + status = lpfc_sli4_post_sgl( + phba, lpfc_ncmd->dma_phys_sgl, + pdma_phys_sgl1, cur_xritag); + if (status) { + /* Post error. Buffer unavailable. */ + lpfc_ncmd->flags |= + LPFC_SBUF_NOT_POSTED; + } else { + /* Post success. Bffer available. */ + lpfc_ncmd->flags &= + ~LPFC_SBUF_NOT_POSTED; + lpfc_ncmd->status = IOSTAT_SUCCESS; + num_posted++; + } + /* success, put on NVME buffer sgl list */ + list_add_tail(&lpfc_ncmd->list, &nvme_nblist); + } + } + + /* continue until a nembed page worth of sgls */ + if (post_cnt == 0) + continue; + + /* post block of NVME buffer list sgls */ + status = lpfc_sli4_post_io_sgl_block(phba, &blck_nblist, + post_cnt); + + /* don't reset xirtag due to hole in xri block */ + if (block_cnt == 0) + last_xritag = NO_XRI; + + /* reset NVME buffer post count for next round of posting */ + post_cnt = 0; + + /* put posted NVME buffer-sgl posted on NVME buffer sgl list */ + while (!list_empty(&blck_nblist)) { + list_remove_head(&blck_nblist, lpfc_ncmd, + struct lpfc_io_buf, list); + if (status) { + /* Post error. Mark buffer unavailable. */ + lpfc_ncmd->flags |= LPFC_SBUF_NOT_POSTED; + } else { + /* Post success, Mark buffer available. */ + lpfc_ncmd->flags &= ~LPFC_SBUF_NOT_POSTED; + lpfc_ncmd->status = IOSTAT_SUCCESS; + num_posted++; + } + list_add_tail(&lpfc_ncmd->list, &nvme_nblist); + } + } + /* Push NVME buffers with sgl posted to the available list */ + lpfc_io_buf_replenish(phba, &nvme_nblist); + + return num_posted; +} + +/** * lpfc_fc_frame_check - Check that this frame is a valid frame to handle * @phba: pointer to lpfc_hba struct that the frame was received on * @fc_hdr: A pointer to the FC Header data (In Big Endian Format) @@ -16955,8 +18521,11 @@ lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr) { /* make rctl_names static to save stack space */ struct fc_vft_header *fc_vft_hdr; + struct fc_app_header *fc_app_hdr; uint32_t *header = (uint32_t *) fc_hdr; +#define FC_RCTL_MDS_DIAGS 0xF4 + switch (fc_hdr->fh_r_ctl) { case FC_RCTL_DD_UNCAT: /* uncategorized information */ case FC_RCTL_DD_SOL_DATA: /* solicited data */ @@ -16970,7 +18539,6 @@ lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr) case FC_RCTL_ELS_REP: /* extended link services reply */ case FC_RCTL_ELS4_REQ: /* FC-4 ELS request */ case FC_RCTL_ELS4_REP: /* FC-4 ELS reply */ - case FC_RCTL_BA_NOP: /* basic link service NOP */ case FC_RCTL_BA_ABTS: /* basic link service abort */ case FC_RCTL_BA_RMC: /* remove connection */ case FC_RCTL_BA_ACC: /* basic accept */ @@ -16991,6 +18559,7 @@ lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr) fc_vft_hdr = (struct fc_vft_header *)fc_hdr; fc_hdr = &((struct fc_frame_header *)fc_vft_hdr)[1]; return lpfc_fc_frame_check(phba, fc_hdr); + case FC_RCTL_BA_NOP: /* basic link service NOP */ default: goto drop; } @@ -17008,6 +18577,32 @@ lpfc_fc_frame_check(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr) goto drop; } + if (unlikely(phba->link_flag == LS_LOOPBACK_MODE && + phba->cfg_vmid_app_header)) { + /* Application header is 16B device header */ + if (fc_hdr->fh_df_ctl & LPFC_FC_16B_DEVICE_HEADER) { + fc_app_hdr = (struct fc_app_header *) (fc_hdr + 1); + if (be32_to_cpu(fc_app_hdr->src_app_id) != + LOOPBACK_SRC_APPID) { + lpfc_printf_log(phba, KERN_WARNING, + LOG_ELS | LOG_LIBDFC, + "1932 Loopback src app id " + "not matched, app_id:x%x\n", + be32_to_cpu(fc_app_hdr->src_app_id)); + + goto drop; + } + } else { + lpfc_printf_log(phba, KERN_WARNING, + LOG_ELS | LOG_LIBDFC, + "1933 Loopback df_ctl bit not set, " + "df_ctl:x%x\n", + fc_hdr->fh_df_ctl); + + goto drop; + } + } + lpfc_printf_log(phba, KERN_INFO, LOG_ELS, "2538 Received frame rctl:x%x, type:x%x, " "frame Data:%08x %08x %08x %08x %08x %08x %08x\n", @@ -17047,6 +18642,7 @@ lpfc_fc_hdr_get_vfi(struct fc_frame_header *fc_hdr) * @phba: Pointer to the HBA structure to search for the vport on * @fc_hdr: A pointer to the FC Header data (In Big Endian Format) * @fcfi: The FC Fabric ID that the frame came from + * @did: Destination ID to match against * * This function searches the @phba for a vport that matches the content of the * @fc_hdr passed in and the @fcfi. This function uses the @fc_hdr to fetch the @@ -17064,8 +18660,8 @@ lpfc_fc_frame_to_vport(struct lpfc_hba *phba, struct fc_frame_header *fc_hdr, if (did == Fabric_DID) return phba->pport; - if ((phba->pport->fc_flag & FC_PT2PT) && - !(phba->link_state == LPFC_HBA_READY)) + if (test_bit(FC_PT2PT, &phba->pport->fc_flag) && + phba->link_state != LPFC_HBA_READY) return phba->pport; vports = lpfc_create_vport_work_array(phba); @@ -17184,6 +18780,7 @@ lpfc_rcv_seq_check_edtov(struct lpfc_vport *vport) /** * lpfc_fc_frame_add - Adds a frame to the vport's list of received sequences + * @vport: pointer to a vitural port * @dmabuf: pointer to a dmabuf that describes the hdr and data of the FC frame * * This function searches through the existing incomplete sequences that have @@ -17243,7 +18840,6 @@ lpfc_fc_frame_add(struct lpfc_vport *vport, struct hbq_dmabuf *dmabuf) seq_dmabuf->time_stamp = jiffies; lpfc_update_rcv_time_stamp(vport); if (list_empty(&seq_dmabuf->dbuf.list)) { - temp_hdr = dmabuf->hbuf.virt; list_add_tail(&dmabuf->dbuf.list, &seq_dmabuf->dbuf.list); return seq_dmabuf; } @@ -17374,21 +18970,17 @@ lpfc_sli4_seq_abort_rsp_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmd_iocbq, struct lpfc_iocbq *rsp_iocbq) { - struct lpfc_nodelist *ndlp; - if (cmd_iocbq) { - ndlp = (struct lpfc_nodelist *)cmd_iocbq->context1; - lpfc_nlp_put(ndlp); - lpfc_nlp_not_used(ndlp); + lpfc_nlp_put(cmd_iocbq->ndlp); lpfc_sli_release_iocbq(phba, cmd_iocbq); } /* Failure means BLS ABORT RSP did not get delivered to remote node*/ if (rsp_iocbq && rsp_iocbq->iocb.ulpStatus) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3154 BLS ABORT RSP failed, data: x%x/x%x\n", - rsp_iocbq->iocb.ulpStatus, - rsp_iocbq->iocb.un.ulpWord[4]); + get_job_ulpstatus(phba, rsp_iocbq), + get_job_word4(phba, rsp_iocbq)); } /** @@ -17414,8 +19006,9 @@ lpfc_sli4_xri_inrange(struct lpfc_hba *phba, /** * lpfc_sli4_seq_abort_rsp - bls rsp to sequence abort - * @phba: Pointer to HBA context object. + * @vport: pointer to a virtual port. * @fc_hdr: pointer to a FC frame header. + * @aborted: was the partially assembled receive sequence successfully aborted * * This function sends a basic response to a previous unsol sequence abort * event after aborting the sequence handling. @@ -17429,7 +19022,7 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp; uint16_t oxid, rxid, xri, lxri; uint32_t sid, fctl; - IOCB_t *icmd; + union lpfc_wqe128 *icmd; int rc; if (!lpfc_is_link_up(phba)) @@ -17448,17 +19041,8 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, "oxid:x%x SID:x%x\n", oxid, sid); return; } - /* Put ndlp onto pport node list */ + /* Put ndlp onto vport node list */ lpfc_enqueue_node(vport, ndlp); - } else if (!NLP_CHK_NODE_ACT(ndlp)) { - /* re-setup ndlp without removing from node list */ - ndlp = lpfc_enable_node(vport, ndlp, NLP_STE_UNUSED_NODE); - if (!ndlp) { - lpfc_printf_vlog(vport, KERN_WARNING, LOG_ELS, - "3275 Failed to active ndlp found " - "for oxid:x%x SID:x%x\n", oxid, sid); - return; - } } /* Allocate buffer for rsp iocb */ @@ -17466,29 +19050,22 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, if (!ctiocb) return; + icmd = &ctiocb->wqe; + /* Extract the F_CTL field from FC_HDR */ fctl = sli4_fctl_from_fc_hdr(fc_hdr); - icmd = &ctiocb->iocb; - icmd->un.xseq64.bdl.bdeSize = 0; - icmd->un.xseq64.bdl.ulpIoTag32 = 0; - icmd->un.xseq64.w5.hcsw.Dfctl = 0; - icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_ACC; - icmd->un.xseq64.w5.hcsw.Type = FC_TYPE_BLS; - - /* Fill in the rest of iocb fields */ - icmd->ulpCommand = CMD_XMIT_BLS_RSP64_CX; - icmd->ulpBdeCount = 0; - icmd->ulpLe = 1; - icmd->ulpClass = CLASS3; - icmd->ulpContext = phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]; - ctiocb->context1 = lpfc_nlp_get(ndlp); + ctiocb->ndlp = lpfc_nlp_get(ndlp); + if (!ctiocb->ndlp) { + lpfc_sli_release_iocbq(phba, ctiocb); + return; + } - ctiocb->iocb_cmpl = NULL; - ctiocb->vport = phba->pport; - ctiocb->iocb_cmpl = lpfc_sli4_seq_abort_rsp_cmpl; + ctiocb->vport = vport; + ctiocb->cmd_cmpl = lpfc_sli4_seq_abort_rsp_cmpl; ctiocb->sli4_lxritag = NO_XRI; ctiocb->sli4_xritag = NO_XRI; + ctiocb->abort_rctl = FC_RCTL_BA_ACC; if (fctl & FC_FC_EX_CTX) /* Exchange responder sent the abort so we @@ -17508,10 +19085,12 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, */ if ((fctl & FC_FC_EX_CTX) && (lxri > lpfc_sli4_get_iocb_cnt(phba))) { - icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_RJT; - bf_set(lpfc_vndr_code, &icmd->un.bls_rsp, 0); - bf_set(lpfc_rsn_expln, &icmd->un.bls_rsp, FC_BA_RJT_INV_XID); - bf_set(lpfc_rsn_code, &icmd->un.bls_rsp, FC_BA_RJT_UNABLE); + ctiocb->abort_rctl = FC_RCTL_BA_RJT; + bf_set(xmit_bls_rsp64_rjt_vspec, &icmd->xmit_bls_rsp, 0); + bf_set(xmit_bls_rsp64_rjt_expc, &icmd->xmit_bls_rsp, + FC_BA_RJT_INV_XID); + bf_set(xmit_bls_rsp64_rjt_rsnc, &icmd->xmit_bls_rsp, + FC_BA_RJT_UNABLE); } /* If BA_ABTS failed to abort a partially assembled receive sequence, @@ -17519,10 +19098,12 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, * the IOCB for a BA_RJT. */ if (aborted == false) { - icmd->un.xseq64.w5.hcsw.Rctl = FC_RCTL_BA_RJT; - bf_set(lpfc_vndr_code, &icmd->un.bls_rsp, 0); - bf_set(lpfc_rsn_expln, &icmd->un.bls_rsp, FC_BA_RJT_INV_XID); - bf_set(lpfc_rsn_code, &icmd->un.bls_rsp, FC_BA_RJT_UNABLE); + ctiocb->abort_rctl = FC_RCTL_BA_RJT; + bf_set(xmit_bls_rsp64_rjt_vspec, &icmd->xmit_bls_rsp, 0); + bf_set(xmit_bls_rsp64_rjt_expc, &icmd->xmit_bls_rsp, + FC_BA_RJT_INV_XID); + bf_set(xmit_bls_rsp64_rjt_rsnc, &icmd->xmit_bls_rsp, + FC_BA_RJT_UNABLE); } if (fctl & FC_FC_EX_CTX) { @@ -17530,33 +19111,53 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, * of BA_ACC will use OX_ID from ABTS for the XRI_TAG * field and RX_ID from ABTS for RX_ID field. */ - bf_set(lpfc_abts_orig, &icmd->un.bls_rsp, LPFC_ABTS_UNSOL_RSP); + ctiocb->abort_bls = LPFC_ABTS_UNSOL_RSP; + bf_set(xmit_bls_rsp64_rxid, &icmd->xmit_bls_rsp, rxid); } else { /* ABTS sent by initiator to CT exchange, construction * of BA_ACC will need to allocate a new XRI as for the * XRI_TAG field. */ - bf_set(lpfc_abts_orig, &icmd->un.bls_rsp, LPFC_ABTS_UNSOL_INT); + ctiocb->abort_bls = LPFC_ABTS_UNSOL_INT; } - bf_set(lpfc_abts_rxid, &icmd->un.bls_rsp, rxid); - bf_set(lpfc_abts_oxid, &icmd->un.bls_rsp, oxid); + + /* OX_ID is invariable to who sent ABTS to CT exchange */ + bf_set(xmit_bls_rsp64_oxid, &icmd->xmit_bls_rsp, oxid); + bf_set(xmit_bls_rsp64_oxid, &icmd->xmit_bls_rsp, rxid); + + /* Use CT=VPI */ + bf_set(wqe_els_did, &icmd->xmit_bls_rsp.wqe_dest, + ndlp->nlp_DID); + bf_set(xmit_bls_rsp64_temprpi, &icmd->xmit_bls_rsp, + phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); + bf_set(wqe_cmnd, &icmd->generic.wqe_com, CMD_XMIT_BLS_RSP64_CX); /* Xmit CT abts response on exchange <xid> */ lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS, "1200 Send BLS cmd x%x on oxid x%x Data: x%x\n", - icmd->un.xseq64.w5.hcsw.Rctl, oxid, phba->link_state); + ctiocb->abort_rctl, oxid, phba->link_state); rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, ctiocb, 0); if (rc == IOCB_ERROR) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS, + lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, "2925 Failed to issue CT ABTS RSP x%x on " "xri x%x, Data x%x\n", - icmd->un.xseq64.w5.hcsw.Rctl, oxid, + ctiocb->abort_rctl, oxid, phba->link_state); lpfc_nlp_put(ndlp); - ctiocb->context1 = NULL; + ctiocb->ndlp = NULL; lpfc_sli_release_iocbq(phba, ctiocb); } + + /* if only usage of this nodelist is BLS response, release initial ref + * to free ndlp when transmit completes + */ + if (ndlp->nlp_state == NLP_STE_UNUSED_NODE && + !test_bit(NLP_DROPPED, &ndlp->nlp_flag) && + !(ndlp->fc4_xpt_flags & (NVME_XPT_REGD | SCSI_XPT_REGD))) { + set_bit(NLP_DROPPED, &ndlp->nlp_flag); + lpfc_nlp_put(ndlp); + } } /** @@ -17568,7 +19169,7 @@ lpfc_sli4_seq_abort_rsp(struct lpfc_vport *vport, * receive sequence is only partially assembed by the driver, it shall abort * the partially assembled frames for the sequence. Otherwise, if the * unsolicited receive sequence has been completely assembled and passed to - * the Upper Layer Protocol (UPL), it then mark the per oxid status for the + * the Upper Layer Protocol (ULP), it then mark the per oxid status for the * unsolicited sequence has been aborted. After that, it will issue a basic * accept to accept the abort. **/ @@ -17655,7 +19256,7 @@ lpfc_seq_complete(struct hbq_dmabuf *dmabuf) /** * lpfc_prep_seq - Prep sequence for ULP processing * @vport: Pointer to the vport on which this sequence was received - * @dmabuf: pointer to a dmabuf that describes the FC sequence + * @seq_dmabuf: pointer to a dmabuf that describes the FC sequence * * This function takes a sequence, described by a list of frames, and creates * a list of iocbq structures to describe the sequence. This iocbq list will be @@ -17674,7 +19275,6 @@ lpfc_prep_seq(struct lpfc_vport *vport, struct hbq_dmabuf *seq_dmabuf) struct fc_frame_header *fc_hdr; uint32_t sid; uint32_t len, tot_len; - struct ulp_bde64 *pbde; fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt; /* remove from receive buffer list */ @@ -17687,40 +19287,40 @@ lpfc_prep_seq(struct lpfc_vport *vport, struct hbq_dmabuf *seq_dmabuf) first_iocbq = lpfc_sli_get_iocbq(vport->phba); if (first_iocbq) { /* Initialize the first IOCB. */ - first_iocbq->iocb.unsli3.rcvsli3.acc_len = 0; - first_iocbq->iocb.ulpStatus = IOSTAT_SUCCESS; + first_iocbq->wcqe_cmpl.total_data_placed = 0; + bf_set(lpfc_wcqe_c_status, &first_iocbq->wcqe_cmpl, + IOSTAT_SUCCESS); first_iocbq->vport = vport; /* Check FC Header to see what TYPE of frame we are rcv'ing */ if (sli4_type_from_fc_hdr(fc_hdr) == FC_TYPE_ELS) { - first_iocbq->iocb.ulpCommand = CMD_IOCB_RCV_ELS64_CX; - first_iocbq->iocb.un.rcvels.parmRo = - sli4_did_from_fc_hdr(fc_hdr); - first_iocbq->iocb.ulpPU = PARM_NPIV_DID; - } else - first_iocbq->iocb.ulpCommand = CMD_IOCB_RCV_SEQ64_CX; - first_iocbq->iocb.ulpContext = NO_XRI; - first_iocbq->iocb.unsli3.rcvsli3.ox_id = - be16_to_cpu(fc_hdr->fh_ox_id); - /* iocbq is prepped for internal consumption. Physical vpi. */ - first_iocbq->iocb.unsli3.rcvsli3.vpi = - vport->phba->vpi_ids[vport->vpi]; - /* put the first buffer into the first IOCBq */ + bf_set(els_rsp64_sid, &first_iocbq->wqe.xmit_els_rsp, + sli4_did_from_fc_hdr(fc_hdr)); + } + + bf_set(wqe_ctxt_tag, &first_iocbq->wqe.xmit_els_rsp.wqe_com, + NO_XRI); + bf_set(wqe_rcvoxid, &first_iocbq->wqe.xmit_els_rsp.wqe_com, + be16_to_cpu(fc_hdr->fh_ox_id)); + + /* put the first buffer into the first iocb */ tot_len = bf_get(lpfc_rcqe_length, - &seq_dmabuf->cq_event.cqe.rcqe_cmpl); + &seq_dmabuf->cq_event.cqe.rcqe_cmpl); + + first_iocbq->cmd_dmabuf = &seq_dmabuf->dbuf; + first_iocbq->bpl_dmabuf = NULL; + /* Keep track of the BDE count */ + first_iocbq->wcqe_cmpl.word3 = 1; - first_iocbq->context2 = &seq_dmabuf->dbuf; - first_iocbq->context3 = NULL; - first_iocbq->iocb.ulpBdeCount = 1; if (tot_len > LPFC_DATA_BUF_SIZE) - first_iocbq->iocb.un.cont64[0].tus.f.bdeSize = - LPFC_DATA_BUF_SIZE; + first_iocbq->wqe.gen_req.bde.tus.f.bdeSize = + LPFC_DATA_BUF_SIZE; else - first_iocbq->iocb.un.cont64[0].tus.f.bdeSize = tot_len; + first_iocbq->wqe.gen_req.bde.tus.f.bdeSize = tot_len; - first_iocbq->iocb.un.rcvels.remoteID = sid; - - first_iocbq->iocb.unsli3.rcvsli3.acc_len = tot_len; + first_iocbq->wcqe_cmpl.total_data_placed = tot_len; + bf_set(wqe_els_did, &first_iocbq->wqe.xmit_els_rsp.wqe_dest, + sid); } iocbq = first_iocbq; /* @@ -17732,30 +19332,25 @@ lpfc_prep_seq(struct lpfc_vport *vport, struct hbq_dmabuf *seq_dmabuf) lpfc_in_buf_free(vport->phba, d_buf); continue; } - if (!iocbq->context3) { - iocbq->context3 = d_buf; - iocbq->iocb.ulpBdeCount++; + if (!iocbq->bpl_dmabuf) { + iocbq->bpl_dmabuf = d_buf; + iocbq->wcqe_cmpl.word3++; /* We need to get the size out of the right CQE */ hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf); len = bf_get(lpfc_rcqe_length, &hbq_buf->cq_event.cqe.rcqe_cmpl); - pbde = (struct ulp_bde64 *) - &iocbq->iocb.unsli3.sli3Words[4]; - if (len > LPFC_DATA_BUF_SIZE) - pbde->tus.f.bdeSize = LPFC_DATA_BUF_SIZE; - else - pbde->tus.f.bdeSize = len; - - iocbq->iocb.unsli3.rcvsli3.acc_len += len; + iocbq->unsol_rcv_len = len; + iocbq->wcqe_cmpl.total_data_placed += len; tot_len += len; } else { iocbq = lpfc_sli_get_iocbq(vport->phba); if (!iocbq) { if (first_iocbq) { - first_iocbq->iocb.ulpStatus = - IOSTAT_FCP_RSP_ERROR; - first_iocbq->iocb.un.ulpWord[4] = - IOERR_NO_RESOURCES; + bf_set(lpfc_wcqe_c_status, + &first_iocbq->wcqe_cmpl, + IOSTAT_SUCCESS); + first_iocbq->wcqe_cmpl.parameter = + IOERR_NO_RESOURCES; } lpfc_in_buf_free(vport->phba, d_buf); continue; @@ -17764,22 +19359,28 @@ lpfc_prep_seq(struct lpfc_vport *vport, struct hbq_dmabuf *seq_dmabuf) hbq_buf = container_of(d_buf, struct hbq_dmabuf, dbuf); len = bf_get(lpfc_rcqe_length, &hbq_buf->cq_event.cqe.rcqe_cmpl); - iocbq->context2 = d_buf; - iocbq->context3 = NULL; - iocbq->iocb.ulpBdeCount = 1; + iocbq->cmd_dmabuf = d_buf; + iocbq->bpl_dmabuf = NULL; + iocbq->wcqe_cmpl.word3 = 1; + if (len > LPFC_DATA_BUF_SIZE) - iocbq->iocb.un.cont64[0].tus.f.bdeSize = - LPFC_DATA_BUF_SIZE; + iocbq->wqe.xmit_els_rsp.bde.tus.f.bdeSize = + LPFC_DATA_BUF_SIZE; else - iocbq->iocb.un.cont64[0].tus.f.bdeSize = len; + iocbq->wqe.xmit_els_rsp.bde.tus.f.bdeSize = + len; tot_len += len; - iocbq->iocb.unsli3.rcvsli3.acc_len = tot_len; - - iocbq->iocb.un.rcvels.remoteID = sid; + iocbq->wcqe_cmpl.total_data_placed = tot_len; + bf_set(wqe_els_did, &iocbq->wqe.xmit_els_rsp.wqe_dest, + sid); list_add_tail(&iocbq->list, &first_iocbq->list); } } + /* Free the sequence's header buffer */ + if (!first_iocbq) + lpfc_in_buf_free(vport->phba, &seq_dmabuf->dbuf); + return first_iocbq; } @@ -17794,7 +19395,7 @@ lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *vport, fc_hdr = (struct fc_frame_header *)seq_dmabuf->hbuf.virt; iocbq = lpfc_prep_seq(vport, seq_dmabuf); if (!iocbq) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2707 Ring %d handler: Failed to allocate " "iocb Rctl x%x Type x%x received\n", LPFC_ELS_RING, @@ -17804,16 +19405,18 @@ lpfc_sli4_send_seq_to_ulp(struct lpfc_vport *vport, if (!lpfc_complete_unsol_iocb(phba, phba->sli4_hba.els_wq->pring, iocbq, fc_hdr->fh_r_ctl, - fc_hdr->fh_type)) - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + fc_hdr->fh_type)) { + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2540 Ring %d handler: unexpected Rctl " "x%x Type x%x received\n", LPFC_ELS_RING, fc_hdr->fh_r_ctl, fc_hdr->fh_type); + lpfc_in_buf_free(phba, &seq_dmabuf->dbuf); + } /* Free iocb created in lpfc_prep_seq */ list_for_each_entry_safe(curr_iocb, next_iocb, - &iocbq->list, list) { + &iocbq->list, list) { list_del_init(&curr_iocb->list); lpfc_sli_release_iocbq(phba, curr_iocb); } @@ -17824,7 +19427,7 @@ static void lpfc_sli4_mds_loopback_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_iocbq *rspiocb) { - struct lpfc_dmabuf *pcmd = cmdiocb->context2; + struct lpfc_dmabuf *pcmd = cmdiocb->cmd_dmabuf; if (pcmd && pcmd->virt) dma_pool_free(phba->lpfc_drb_pool, pcmd->virt, pcmd->phys); @@ -17840,7 +19443,7 @@ lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport, struct fc_frame_header *fc_hdr; struct lpfc_hba *phba = vport->phba; struct lpfc_iocbq *iocbq = NULL; - union lpfc_wqe *wqe; + union lpfc_wqe128 *pwqe; struct lpfc_dmabuf *pcmd = NULL; uint32_t frame_len; int rc; @@ -17856,8 +19459,8 @@ lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport, spin_lock_irqsave(&phba->hbalock, iflags); list_add_tail(&dmabuf->cq_event.list, &phba->sli4_hba.sp_queue_event); - phba->hba_flag |= HBA_SP_QUEUE_EVT; spin_unlock_irqrestore(&phba->hbalock, iflags); + set_bit(HBA_SP_QUEUE_EVT, &phba->hba_flag); lpfc_worker_wake_up(phba); return; } @@ -17875,34 +19478,46 @@ lpfc_sli4_handle_mds_loopback(struct lpfc_vport *vport, /* copyin the payload */ memcpy(pcmd->virt, dmabuf->dbuf.virt, frame_len); - /* fill in BDE's for command */ - iocbq->iocb.un.xseq64.bdl.addrHigh = putPaddrHigh(pcmd->phys); - iocbq->iocb.un.xseq64.bdl.addrLow = putPaddrLow(pcmd->phys); - iocbq->iocb.un.xseq64.bdl.bdeFlags = BUFF_TYPE_BDE_64; - iocbq->iocb.un.xseq64.bdl.bdeSize = frame_len; - - iocbq->context2 = pcmd; + iocbq->cmd_dmabuf = pcmd; iocbq->vport = vport; - iocbq->iocb_flag &= ~LPFC_FIP_ELS_ID_MASK; - iocbq->iocb_flag |= LPFC_USE_FCPWQIDX; + iocbq->cmd_flag &= ~LPFC_FIP_ELS_ID_MASK; + iocbq->cmd_flag |= LPFC_USE_FCPWQIDX; + iocbq->num_bdes = 0; + + pwqe = &iocbq->wqe; + /* fill in BDE's for command */ + pwqe->gen_req.bde.addrHigh = putPaddrHigh(pcmd->phys); + pwqe->gen_req.bde.addrLow = putPaddrLow(pcmd->phys); + pwqe->gen_req.bde.tus.f.bdeSize = frame_len; + pwqe->gen_req.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; + + pwqe->send_frame.frame_len = frame_len; + pwqe->send_frame.fc_hdr_wd0 = be32_to_cpu(*((__be32 *)fc_hdr)); + pwqe->send_frame.fc_hdr_wd1 = be32_to_cpu(*((__be32 *)fc_hdr + 1)); + pwqe->send_frame.fc_hdr_wd2 = be32_to_cpu(*((__be32 *)fc_hdr + 2)); + pwqe->send_frame.fc_hdr_wd3 = be32_to_cpu(*((__be32 *)fc_hdr + 3)); + pwqe->send_frame.fc_hdr_wd4 = be32_to_cpu(*((__be32 *)fc_hdr + 4)); + pwqe->send_frame.fc_hdr_wd5 = be32_to_cpu(*((__be32 *)fc_hdr + 5)); + + pwqe->generic.wqe_com.word7 = 0; + pwqe->generic.wqe_com.word10 = 0; + + bf_set(wqe_cmnd, &pwqe->generic.wqe_com, CMD_SEND_FRAME); + bf_set(wqe_sof, &pwqe->generic.wqe_com, 0x2E); /* SOF byte */ + bf_set(wqe_eof, &pwqe->generic.wqe_com, 0x41); /* EOF byte */ + bf_set(wqe_lenloc, &pwqe->generic.wqe_com, 1); + bf_set(wqe_xbl, &pwqe->generic.wqe_com, 1); + bf_set(wqe_dbde, &pwqe->generic.wqe_com, 1); + bf_set(wqe_xc, &pwqe->generic.wqe_com, 1); + bf_set(wqe_cmd_type, &pwqe->generic.wqe_com, 0xA); + bf_set(wqe_cqid, &pwqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); + bf_set(wqe_xri_tag, &pwqe->generic.wqe_com, iocbq->sli4_xritag); + bf_set(wqe_reqtag, &pwqe->generic.wqe_com, iocbq->iotag); + bf_set(wqe_class, &pwqe->generic.wqe_com, CLASS3); + pwqe->generic.wqe_com.abort_tag = iocbq->iotag; + + iocbq->cmd_cmpl = lpfc_sli4_mds_loopback_cmpl; - /* - * Setup rest of the iocb as though it were a WQE - * Build the SEND_FRAME WQE - */ - wqe = (union lpfc_wqe *)&iocbq->iocb; - - wqe->send_frame.frame_len = frame_len; - wqe->send_frame.fc_hdr_wd0 = be32_to_cpu(*((uint32_t *)fc_hdr)); - wqe->send_frame.fc_hdr_wd1 = be32_to_cpu(*((uint32_t *)fc_hdr + 1)); - wqe->send_frame.fc_hdr_wd2 = be32_to_cpu(*((uint32_t *)fc_hdr + 2)); - wqe->send_frame.fc_hdr_wd3 = be32_to_cpu(*((uint32_t *)fc_hdr + 3)); - wqe->send_frame.fc_hdr_wd4 = be32_to_cpu(*((uint32_t *)fc_hdr + 4)); - wqe->send_frame.fc_hdr_wd5 = be32_to_cpu(*((uint32_t *)fc_hdr + 5)); - - iocbq->iocb.ulpCommand = CMD_SEND_FRAME; - iocbq->iocb.ulpLe = 1; - iocbq->iocb_cmpl = lpfc_sli4_mds_loopback_cmpl; rc = lpfc_sli_issue_iocb(phba, LPFC_ELS_RING, iocbq, 0); if (rc == IOCB_ERROR) goto exit; @@ -17924,6 +19539,7 @@ exit: /** * lpfc_sli4_handle_received_buffer - Handle received buffers from firmware * @phba: Pointer to HBA context object. + * @dmabuf: Pointer to a dmabuf that describes the FC sequence. * * This function is called with no lock held. This function processes all * the received buffers and gives it to upper layers when a received buffer @@ -17949,7 +19565,10 @@ lpfc_sli4_handle_received_buffer(struct lpfc_hba *phba, fc_hdr->fh_r_ctl == FC_RCTL_DD_UNSOL_DATA) { vport = phba->pport; /* Handle MDS Loopback frames */ - lpfc_sli4_handle_mds_loopback(vport, dmabuf); + if (!test_bit(FC_UNLOADING, &phba->pport->load_flag)) + lpfc_sli4_handle_mds_loopback(vport, dmabuf); + else + lpfc_in_buf_free(phba, &dmabuf->dbuf); return; } @@ -17967,6 +19586,17 @@ lpfc_sli4_handle_received_buffer(struct lpfc_hba *phba, fcfi = bf_get(lpfc_rcqe_fcf_id, &dmabuf->cq_event.cqe.rcqe_cmpl); + if (fc_hdr->fh_r_ctl == 0xF4 && fc_hdr->fh_type == 0xFF) { + vport = phba->pport; + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "2023 MDS Loopback %d bytes\n", + bf_get(lpfc_rcqe_length, + &dmabuf->cq_event.cqe.rcqe_cmpl)); + /* Handle MDS Loopback frames */ + lpfc_sli4_handle_mds_loopback(vport, dmabuf); + return; + } + /* d_id this frame is directed to */ did = sli4_did_from_fc_hdr(fc_hdr); @@ -17985,8 +19615,8 @@ lpfc_sli4_handle_received_buffer(struct lpfc_hba *phba, * The pt2pt protocol allows for discovery frames * to be received without a registered VPI. */ - if (!(vport->fc_flag & FC_PT2PT) || - (phba->link_state == LPFC_HBA_READY)) { + if (!test_bit(FC_PT2PT, &vport->fc_flag) || + phba->link_state == LPFC_HBA_READY) { lpfc_in_buf_free(phba, &dmabuf->dbuf); return; } @@ -18059,7 +19689,7 @@ lpfc_sli4_post_all_rpi_hdrs(struct lpfc_hba *phba) rc = lpfc_sli4_post_rpi_hdr(phba, rpi_page); if (rc != MBX_SUCCESS) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2008 Error %d posting all rpi " "headers\n", rc); rc = -EIO; @@ -18105,7 +19735,7 @@ lpfc_sli4_post_rpi_hdr(struct lpfc_hba *phba, struct lpfc_rpi_hdr *rpi_page) /* The port is notified of the header region via a mailbox command. */ mboxq = (LPFC_MBOXQ_t *) mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mboxq) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2001 Unable to allocate memory for issuing " "SLI_CONFIG_SPECIAL mailbox command\n"); return -ENOMEM; @@ -18132,10 +19762,9 @@ lpfc_sli4_post_rpi_hdr(struct lpfc_hba *phba, struct lpfc_rpi_hdr *rpi_page) shdr = (union lpfc_sli4_cfg_shdr *) &hdr_tmpl->header.cfg_shdr; shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); - if (rc != MBX_TIMEOUT) - mempool_free(mboxq, phba->mbox_mem_pool); + mempool_free(mboxq, phba->mbox_mem_pool); if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2514 POST_RPI_HDR mailbox failed with " "status x%x add_status x%x, mbx status x%x\n", shdr_status, shdr_add_status, rc); @@ -18182,7 +19811,7 @@ lpfc_sli4_alloc_rpi(struct lpfc_hba *phba) max_rpi = phba->sli4_hba.max_cfg_param.max_rpi; rpi_limit = phba->sli4_hba.next_rpi; - rpi = find_next_zero_bit(phba->sli4_hba.rpi_bmask, rpi_limit, 0); + rpi = find_first_zero_bit(phba->sli4_hba.rpi_bmask, rpi_limit); if (rpi >= rpi_limit) rpi = LPFC_RPI_ALLOC_ERROR; else { @@ -18190,8 +19819,9 @@ lpfc_sli4_alloc_rpi(struct lpfc_hba *phba) phba->sli4_hba.max_cfg_param.rpi_used++; phba->sli4_hba.rpi_count++; } - lpfc_printf_log(phba, KERN_INFO, LOG_SLI, - "0001 rpi:%x max:%x lim:%x\n", + lpfc_printf_log(phba, KERN_INFO, + LOG_NODE | LOG_DISCOVERY, + "0001 Allocated rpi:x%x max:x%x lim:x%x\n", (int) rpi, max_rpi, rpi_limit); /* @@ -18224,7 +19854,7 @@ lpfc_sli4_alloc_rpi(struct lpfc_hba *phba) if (rpi_remaining < LPFC_RPI_LOW_WATER_MARK) { rpi_hdr = lpfc_sli4_create_rpi_hdr(phba); if (!rpi_hdr) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2002 Error Could not grow rpi " "count\n"); } else { @@ -18238,8 +19868,9 @@ lpfc_sli4_alloc_rpi(struct lpfc_hba *phba) } /** - * lpfc_sli4_free_rpi - Release an rpi for reuse. + * __lpfc_sli4_free_rpi - Release an rpi for reuse. * @phba: pointer to lpfc hba data structure. + * @rpi: rpi to free * * This routine is invoked to release an rpi to the pool of * available rpis maintained by the driver. @@ -18247,15 +19878,28 @@ lpfc_sli4_alloc_rpi(struct lpfc_hba *phba) static void __lpfc_sli4_free_rpi(struct lpfc_hba *phba, int rpi) { + /* + * if the rpi value indicates a prior unreg has already + * been done, skip the unreg. + */ + if (rpi == LPFC_RPI_ALLOC_ERROR) + return; + if (test_and_clear_bit(rpi, phba->sli4_hba.rpi_bmask)) { phba->sli4_hba.rpi_count--; phba->sli4_hba.max_cfg_param.rpi_used--; + } else { + lpfc_printf_log(phba, KERN_INFO, + LOG_NODE | LOG_DISCOVERY, + "2016 rpi %x not inuse\n", + rpi); } } /** * lpfc_sli4_free_rpi - Release an rpi for reuse. * @phba: pointer to lpfc hba data structure. + * @rpi: rpi to free * * This routine is invoked to release an rpi to the pool of * available rpis maintained by the driver. @@ -18284,15 +19928,20 @@ lpfc_sli4_remove_rpis(struct lpfc_hba *phba) } /** - * lpfc_sli4_resume_rpi - Remove the rpi bitmask region - * @phba: pointer to lpfc hba data structure. + * lpfc_sli4_resume_rpi - Resume traffic relative to an RPI + * @ndlp: pointer to lpfc nodelist data structure. + * @cmpl: completion call-back. + * @iocbq: data to load as mbox ctx_u information * - * This routine is invoked to remove the memory region that - * provided rpi via a bitmask. + * Return codes + * 0 - successful + * -ENOMEM - No available memory + * -EIO - The mailbox failed to complete successfully. **/ int lpfc_sli4_resume_rpi(struct lpfc_nodelist *ndlp, - void (*cmpl)(struct lpfc_hba *, LPFC_MBOXQ_t *), void *arg) + void (*cmpl)(struct lpfc_hba *, LPFC_MBOXQ_t *), + struct lpfc_iocbq *iocbq) { LPFC_MBOXQ_t *mboxq; struct lpfc_hba *phba = ndlp->phba; @@ -18303,21 +19952,35 @@ lpfc_sli4_resume_rpi(struct lpfc_nodelist *ndlp, if (!mboxq) return -ENOMEM; - /* Post all rpi memory regions to the port. */ + /* If cmpl assigned, then this nlp_get pairs with + * lpfc_mbx_cmpl_resume_rpi. + * + * Else cmpl is NULL, then this nlp_get pairs with + * lpfc_sli_def_mbox_cmpl. + */ + if (!lpfc_nlp_get(ndlp)) { + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "2122 %s: Failed to get nlp ref\n", + __func__); + mempool_free(mboxq, phba->mbox_mem_pool); + return -EIO; + } + lpfc_resume_rpi(mboxq, ndlp); if (cmpl) { mboxq->mbox_cmpl = cmpl; - mboxq->ctx_buf = arg; - mboxq->ctx_ndlp = ndlp; + mboxq->ctx_u.save_iocb = iocbq; } else mboxq->mbox_cmpl = lpfc_sli_def_mbox_cmpl; + mboxq->ctx_ndlp = ndlp; mboxq->vport = ndlp->vport; rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT); if (rc == MBX_NOT_FINISHED) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2010 Resume RPI Mailbox failed " "status %d, mbxStatus x%x\n", rc, bf_get(lpfc_mqe_status, &mboxq->u.mqe)); + lpfc_nlp_put(ndlp); mempool_free(mboxq, phba->mbox_mem_pool); return -EIO; } @@ -18349,7 +20012,7 @@ lpfc_sli4_init_vpi(struct lpfc_vport *vport) mbox_tmo = lpfc_mbox_tmo_val(phba, mboxq); rc = lpfc_sli_issue_mbox_wait(phba, mboxq, mbox_tmo); if (rc != MBX_SUCCESS) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_SLI, + lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT, "2022 INIT VPI Mailbox failed " "status %d, mbxStatus x%x\n", rc, bf_get(lpfc_mqe_status, &mboxq->u.mqe)); @@ -18385,7 +20048,7 @@ lpfc_mbx_cmpl_add_fcf_record(struct lpfc_hba *phba, LPFC_MBOXQ_t *mboxq) if ((shdr_status || shdr_add_status) && (shdr_status != STATUS_FCF_IN_USE)) - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2558 ADD_FCF_RECORD mailbox failed with " "status x%x add_status x%x\n", shdr_status, shdr_add_status); @@ -18415,7 +20078,7 @@ lpfc_sli4_add_fcf_record(struct lpfc_hba *phba, struct fcf_record *fcf_record) mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mboxq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2009 Failed to allocate mbox for ADD_FCF cmd\n"); return -ENOMEM; } @@ -18428,7 +20091,7 @@ lpfc_sli4_add_fcf_record(struct lpfc_hba *phba, struct fcf_record *fcf_record) LPFC_MBOX_OPCODE_FCOE_ADD_FCF, req_len, LPFC_SLI4_MBX_NEMBED); if (alloc_len < req_len) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2523 Allocated DMA memory size (x%x) is " "less than the requested DMA memory " "size (x%x)\n", alloc_len, req_len); @@ -18461,7 +20124,7 @@ lpfc_sli4_add_fcf_record(struct lpfc_hba *phba, struct fcf_record *fcf_record) mboxq->mbox_cmpl = lpfc_mbx_cmpl_add_fcf_record; rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT); if (rc == MBX_NOT_FINISHED) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2515 ADD_FCF_RECORD mailbox failed with " "status 0x%x\n", rc); lpfc_sli4_mbox_cmd_free(phba, mboxq); @@ -18534,7 +20197,7 @@ lpfc_sli4_fcf_scan_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index) phba->fcoe_cvl_eventtag_attn = phba->fcoe_cvl_eventtag; mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mboxq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2000 Failed to allocate mbox for " "READ_FCF cmd\n"); error = -ENOMEM; @@ -18550,9 +20213,7 @@ lpfc_sli4_fcf_scan_read_fcf_rec(struct lpfc_hba *phba, uint16_t fcf_index) mboxq->vport = phba->pport; mboxq->mbox_cmpl = lpfc_mbx_cmpl_fcf_scan_read_fcf_rec; - spin_lock_irq(&phba->hbalock); - phba->hba_flag |= FCF_TS_INPROG; - spin_unlock_irq(&phba->hbalock); + set_bit(FCF_TS_INPROG, &phba->hba_flag); rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_NOWAIT); if (rc == MBX_NOT_FINISHED) @@ -18568,9 +20229,7 @@ fail_fcf_scan: if (mboxq) lpfc_sli4_mbox_cmd_free(phba, mboxq); /* FCF scan failed, clear FCF_TS_INPROG flag */ - spin_lock_irq(&phba->hbalock); - phba->hba_flag &= ~FCF_TS_INPROG; - spin_unlock_irq(&phba->hbalock); + clear_bit(FCF_TS_INPROG, &phba->hba_flag); } return error; } @@ -18669,7 +20328,7 @@ fail_fcf_read: /** * lpfc_check_next_fcf_pri_level - * phba pointer to the lpfc_hba struct for this port. + * @phba: pointer to the lpfc_hba struct for this port. * This routine is called from the lpfc_sli4_fcf_rr_next_index_get * routine when the rr_bmask is empty. The FCF indecies are put into the * rr_bmask based on their priority level. Starting from the highest priority @@ -18793,8 +20452,8 @@ next_priority: * have been tested so that we can detect when we should * change the priority level. */ - next_fcf_index = find_next_bit(phba->fcf.fcf_rr_bmask, - LPFC_SLI4_FCF_TBL_INDX_MAX, 0); + next_fcf_index = find_first_bit(phba->fcf.fcf_rr_bmask, + LPFC_SLI4_FCF_TBL_INDX_MAX); } @@ -18834,6 +20493,7 @@ next_priority: /** * lpfc_sli4_fcf_rr_index_set - Set bmask with eligible fcf record index * @phba: pointer to lpfc hba data structure. + * @fcf_index: index into the FCF table to 'set' * * This routine sets the FCF record index in to the eligible bmask for * roundrobin failover search. It checks to make sure that the index @@ -18866,6 +20526,7 @@ lpfc_sli4_fcf_rr_index_set(struct lpfc_hba *phba, uint16_t fcf_index) /** * lpfc_sli4_fcf_rr_index_clear - Clear bmask from eligible fcf record index * @phba: pointer to lpfc hba data structure. + * @fcf_index: index into the FCF table to 'clear' * * This routine clears the FCF record index from the eligible bmask for * roundrobin failover search. It checks to make sure that the index @@ -18903,6 +20564,7 @@ lpfc_sli4_fcf_rr_index_clear(struct lpfc_hba *phba, uint16_t fcf_index) /** * lpfc_mbx_cmpl_redisc_fcf_table - completion routine for rediscover FCF table * @phba: pointer to lpfc hba data structure. + * @mbox: An allocated pointer to type LPFC_MBOXQ_t * * This routine is the completion routine for the rediscover FCF table mailbox * command. If the mailbox command returned failure, it will try to stop the @@ -18977,7 +20639,7 @@ lpfc_sli4_redisc_fcf_table(struct lpfc_hba *phba) mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mbox) { - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2745 Failed to allocate mbox for " "requesting FCF rediscover.\n"); return -ENOMEM; @@ -19052,7 +20714,7 @@ lpfc_sli_get_config_region23(struct lpfc_hba *phba, char *rgn23_data) pmb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!pmb) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2600 failed to allocate mailbox memory\n"); return 0; } @@ -19075,6 +20737,7 @@ lpfc_sli_get_config_region23(struct lpfc_hba *phba, char *rgn23_data) */ if (mb->un.varDmp.word_cnt == 0) break; + if (mb->un.varDmp.word_cnt > DMP_RGN23_SIZE - offset) mb->un.varDmp.word_cnt = DMP_RGN23_SIZE - offset; @@ -19111,7 +20774,7 @@ lpfc_sli4_get_config_region23(struct lpfc_hba *phba, char *rgn23_data) mboxq = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); if (!mboxq) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3105 failed to allocate mailbox memory\n"); return 0; } @@ -19119,7 +20782,7 @@ lpfc_sli4_get_config_region23(struct lpfc_hba *phba, char *rgn23_data) if (lpfc_sli4_dump_cfg_rg23(phba, mboxq)) goto out; mqe = &mboxq->u.mqe; - mp = (struct lpfc_dmabuf *)mboxq->ctx_buf; + mp = mboxq->ctx_buf; rc = lpfc_sli_issue_mbox(phba, mboxq, MBX_POLL); if (rc) goto out; @@ -19132,11 +20795,7 @@ lpfc_sli4_get_config_region23(struct lpfc_hba *phba, char *rgn23_data) } lpfc_sli_pcimem_bcopy((char *)mp->virt, rgn23_data, data_length); out: - mempool_free(mboxq, phba->mbox_mem_pool); - if (mp) { - lpfc_mbuf_free(phba, mp->virt, mp->phys); - kfree(mp); - } + lpfc_mbox_rsrc_cleanup(phba, mboxq, MBOX_THD_UNLOCKED); return data_length; } @@ -19175,7 +20834,7 @@ lpfc_sli_read_link_ste(struct lpfc_hba *phba) /* Check the region signature first */ if (memcmp(&rgn23_data[offset], LPFC_REGION23_SIGNATURE, 4)) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2619 Config region 23 has bad signature\n"); goto out; } @@ -19183,7 +20842,7 @@ lpfc_sli_read_link_ste(struct lpfc_hba *phba) /* Check the data structure version */ if (rgn23_data[offset] != LPFC_REGION23_VERSION) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2620 Config region 23 has bad version\n"); goto out; } @@ -19227,7 +20886,7 @@ lpfc_sli_read_link_ste(struct lpfc_hba *phba) /* This HBA contains PORT_STE configured */ if (!rgn23_data[offset + 2]) - phba->hba_flag |= LINK_DISABLED; + set_bit(LINK_DISABLED, &phba->hba_flag); goto out; } @@ -19239,6 +20898,91 @@ out: } /** + * lpfc_log_fw_write_cmpl - logs firmware write completion status + * @phba: pointer to lpfc hba data structure + * @shdr_status: wr_object rsp's status field + * @shdr_add_status: wr_object rsp's add_status field + * @shdr_add_status_2: wr_object rsp's add_status_2 field + * @shdr_change_status: wr_object rsp's change_status field + * @shdr_csf: wr_object rsp's csf bit + * + * This routine is intended to be called after a firmware write completes. + * It will log next action items to be performed by the user to instantiate + * the newly downloaded firmware or reason for incompatibility. + **/ +static void +lpfc_log_fw_write_cmpl(struct lpfc_hba *phba, u32 shdr_status, + u32 shdr_add_status, u32 shdr_add_status_2, + u32 shdr_change_status, u32 shdr_csf) +{ + lpfc_printf_log(phba, KERN_INFO, LOG_MBOX | LOG_SLI, + "4198 %s: flash_id x%02x, asic_rev x%02x, " + "status x%02x, add_status x%02x, add_status_2 x%02x, " + "change_status x%02x, csf %01x\n", __func__, + phba->sli4_hba.flash_id, phba->sli4_hba.asic_rev, + shdr_status, shdr_add_status, shdr_add_status_2, + shdr_change_status, shdr_csf); + + if (shdr_add_status == LPFC_ADD_STATUS_INCOMPAT_OBJ) { + switch (shdr_add_status_2) { + case LPFC_ADD_STATUS_2_INCOMPAT_FLASH: + lpfc_log_msg(phba, KERN_WARNING, LOG_MBOX | LOG_SLI, + "4199 Firmware write failed: " + "image incompatible with flash x%02x\n", + phba->sli4_hba.flash_id); + break; + case LPFC_ADD_STATUS_2_INCORRECT_ASIC: + lpfc_log_msg(phba, KERN_WARNING, LOG_MBOX | LOG_SLI, + "4200 Firmware write failed: " + "image incompatible with ASIC " + "architecture x%02x\n", + phba->sli4_hba.asic_rev); + break; + default: + lpfc_log_msg(phba, KERN_WARNING, LOG_MBOX | LOG_SLI, + "4210 Firmware write failed: " + "add_status_2 x%02x\n", + shdr_add_status_2); + break; + } + } else if (!shdr_status && !shdr_add_status) { + if (shdr_change_status == LPFC_CHANGE_STATUS_FW_RESET || + shdr_change_status == LPFC_CHANGE_STATUS_PORT_MIGRATION) { + if (shdr_csf) + shdr_change_status = + LPFC_CHANGE_STATUS_PCI_RESET; + } + + switch (shdr_change_status) { + case (LPFC_CHANGE_STATUS_PHYS_DEV_RESET): + lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI, + "3198 Firmware write complete: System " + "reboot required to instantiate\n"); + break; + case (LPFC_CHANGE_STATUS_FW_RESET): + lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI, + "3199 Firmware write complete: " + "Firmware reset required to " + "instantiate\n"); + break; + case (LPFC_CHANGE_STATUS_PORT_MIGRATION): + lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI, + "3200 Firmware write complete: Port " + "Migration or PCI Reset required to " + "instantiate\n"); + break; + case (LPFC_CHANGE_STATUS_PCI_RESET): + lpfc_log_msg(phba, KERN_NOTICE, LOG_MBOX | LOG_SLI, + "3201 Firmware write complete: PCI " + "Reset required to instantiate\n"); + break; + default: + break; + } + } +} + +/** * lpfc_wr_object - write an object to the firmware * @phba: HBA structure that indicates port to create a queue on. * @dmabuf_list: list of dmabufs to write to the port. @@ -19253,7 +20997,7 @@ out: * the offset after the write object mailbox has completed. @size is used to * determine the end of the object and whether the eof bit should be set. * - * Return 0 is successful and offset will contain the the new offset to use + * Return 0 is successful and offset will contain the new offset to use * for the next write. * Return negative value for error cases. **/ @@ -19264,7 +21008,9 @@ lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list, struct lpfc_mbx_wr_object *wr_object; LPFC_MBOXQ_t *mbox; int rc = 0, i = 0; - uint32_t shdr_status, shdr_add_status, shdr_change_status; + int mbox_status = 0; + uint32_t shdr_status, shdr_add_status, shdr_add_status_2; + uint32_t shdr_change_status = 0, shdr_csf = 0; uint32_t mbox_tmo; struct lpfc_dmabuf *dmabuf; uint32_t written = 0; @@ -19308,56 +21054,52 @@ lpfc_wr_object(struct lpfc_hba *phba, struct list_head *dmabuf_list, wr_object->u.request.bde_count = i; bf_set(lpfc_wr_object_write_length, &wr_object->u.request, written); if (!phba->sli4_hba.intr_enable) - rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); + mbox_status = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); else { mbox_tmo = lpfc_mbox_tmo_val(phba, mbox); - rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); + mbox_status = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); } + + /* The mbox status needs to be maintained to detect MBOX_TIMEOUT. */ + rc = mbox_status; + /* The IOCTL status is embedded in the mailbox subheader. */ shdr_status = bf_get(lpfc_mbox_hdr_status, &wr_object->header.cfg_shdr.response); shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &wr_object->header.cfg_shdr.response); + shdr_add_status_2 = bf_get(lpfc_mbox_hdr_add_status_2, + &wr_object->header.cfg_shdr.response); if (check_change_status) { shdr_change_status = bf_get(lpfc_wr_object_change_status, &wr_object->u.response); - switch (shdr_change_status) { - case (LPFC_CHANGE_STATUS_PHYS_DEV_RESET): - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "3198 Firmware write complete: System " - "reboot required to instantiate\n"); - break; - case (LPFC_CHANGE_STATUS_FW_RESET): - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "3199 Firmware write complete: Firmware" - " reset required to instantiate\n"); - break; - case (LPFC_CHANGE_STATUS_PORT_MIGRATION): - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "3200 Firmware write complete: Port " - "Migration or PCI Reset required to " - "instantiate\n"); - break; - case (LPFC_CHANGE_STATUS_PCI_RESET): - lpfc_printf_log(phba, KERN_INFO, LOG_INIT, - "3201 Firmware write complete: PCI " - "Reset required to instantiate\n"); - break; - default: - break; - } + shdr_csf = bf_get(lpfc_wr_object_csf, + &wr_object->u.response); } - if (rc != MBX_TIMEOUT) - mempool_free(mbox, phba->mbox_mem_pool); - if (shdr_status || shdr_add_status || rc) { - lpfc_printf_log(phba, KERN_ERR, LOG_INIT, + + if (shdr_status || shdr_add_status || shdr_add_status_2 || rc) { + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "3025 Write Object mailbox failed with " - "status x%x add_status x%x, mbx status x%x\n", - shdr_status, shdr_add_status, rc); + "status x%x add_status x%x, add_status_2 x%x, " + "mbx status x%x\n", + shdr_status, shdr_add_status, shdr_add_status_2, + rc); rc = -ENXIO; *offset = shdr_add_status; - } else + } else { *offset += wr_object->u.response.actual_write_length; + } + + if (rc || check_change_status) + lpfc_log_fw_write_cmpl(phba, shdr_status, shdr_add_status, + shdr_add_status_2, shdr_change_status, + shdr_csf); + + if (!phba->sli4_hba.intr_enable) + mempool_free(mbox, phba->mbox_mem_pool); + else if (mbox_status != MBX_TIMEOUT) + mempool_free(mbox, phba->mbox_mem_pool); + return rc; } @@ -19375,10 +21117,8 @@ lpfc_cleanup_pending_mbox(struct lpfc_vport *vport) { struct lpfc_hba *phba = vport->phba; LPFC_MBOXQ_t *mb, *nextmb; - struct lpfc_dmabuf *mp; struct lpfc_nodelist *ndlp; struct lpfc_nodelist *act_mbx_ndlp = NULL; - struct Scsi_Host *shost = lpfc_shost_from_vport(vport); LIST_HEAD(mbox_cmd_list); uint8_t restart_loop; @@ -19392,8 +21132,7 @@ lpfc_cleanup_pending_mbox(struct lpfc_vport *vport) (mb->u.mb.mbxCommand != MBX_REG_VPI)) continue; - list_del(&mb->list); - list_add_tail(&mb->list, &mbox_cmd_list); + list_move_tail(&mb->list, &mbox_cmd_list); } /* Clean up active mailbox command with the vport */ mb = phba->sli.mbox_active; @@ -19402,9 +21141,13 @@ lpfc_cleanup_pending_mbox(struct lpfc_vport *vport) (mb->u.mb.mbxCommand == MBX_REG_VPI)) mb->mbox_cmpl = lpfc_sli_def_mbox_cmpl; if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) { - act_mbx_ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp; - /* Put reference count for delayed processing */ + act_mbx_ndlp = mb->ctx_ndlp; + + /* This reference is local to this routine. The + * reference is removed at routine exit. + */ act_mbx_ndlp = lpfc_nlp_get(act_mbx_ndlp); + /* Unregister the RPI when mailbox complete */ mb->mbox_flag |= LPFC_MBX_IMED_UNREG; } @@ -19427,15 +21170,11 @@ lpfc_cleanup_pending_mbox(struct lpfc_vport *vport) mb->mbox_cmpl = lpfc_sli_def_mbox_cmpl; if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) { - ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp; + ndlp = mb->ctx_ndlp; /* Unregister the RPI when mailbox complete */ mb->mbox_flag |= LPFC_MBX_IMED_UNREG; restart_loop = 1; - spin_unlock_irq(&phba->hbalock); - spin_lock(shost->host_lock); - ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL; - spin_unlock(shost->host_lock); - spin_lock_irq(&phba->hbalock); + clear_bit(NLP_IGNR_REG_CMPL, &ndlp->nlp_flag); break; } } @@ -19447,29 +21186,19 @@ lpfc_cleanup_pending_mbox(struct lpfc_vport *vport) while (!list_empty(&mbox_cmd_list)) { list_remove_head(&mbox_cmd_list, mb, LPFC_MBOXQ_t, list); if (mb->u.mb.mbxCommand == MBX_REG_LOGIN64) { - mp = (struct lpfc_dmabuf *)(mb->ctx_buf); - if (mp) { - __lpfc_mbuf_free(phba, mp->virt, mp->phys); - kfree(mp); - } - mb->ctx_buf = NULL; - ndlp = (struct lpfc_nodelist *)mb->ctx_ndlp; + ndlp = mb->ctx_ndlp; mb->ctx_ndlp = NULL; if (ndlp) { - spin_lock(shost->host_lock); - ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL; - spin_unlock(shost->host_lock); + clear_bit(NLP_IGNR_REG_CMPL, &ndlp->nlp_flag); lpfc_nlp_put(ndlp); } } - mempool_free(mb, phba->mbox_mem_pool); + lpfc_mbox_rsrc_cleanup(phba, mb, MBOX_THD_UNLOCKED); } /* Release the ndlp with the cleaned-up active mailbox command */ if (act_mbx_ndlp) { - spin_lock(shost->host_lock); - act_mbx_ndlp->nlp_flag &= ~NLP_IGNR_REG_CMPL; - spin_unlock(shost->host_lock); + clear_bit(NLP_IGNR_REG_CMPL, &act_mbx_ndlp->nlp_flag); lpfc_nlp_put(act_mbx_ndlp); } } @@ -19493,14 +21222,13 @@ lpfc_drain_txq(struct lpfc_hba *phba) struct lpfc_iocbq *piocbq = NULL; unsigned long iflags = 0; char *fail_msg = NULL; - struct lpfc_sglq *sglq; - union lpfc_wqe128 wqe; uint32_t txq_cnt = 0; struct lpfc_queue *wq; + int ret = 0; if (phba->link_flag & LS_MDS_LOOPBACK) { /* MDS WQE are posted only to first WQ*/ - wq = phba->sli4_hba.fcp_wq[0]; + wq = phba->sli4_hba.hdwq[0].io_wq; if (unlikely(!wq)) return 0; pring = wq->pring; @@ -19530,48 +21258,38 @@ lpfc_drain_txq(struct lpfc_hba *phba) piocbq = lpfc_sli_ringtx_get(phba, pring); if (!piocbq) { spin_unlock_irqrestore(&pring->ring_lock, iflags); - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, - "2823 txq empty and txq_cnt is %d\n ", + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "2823 txq empty and txq_cnt is %d\n", txq_cnt); break; } - sglq = __lpfc_sli_get_els_sglq(phba, piocbq); - if (!sglq) { - __lpfc_sli_ringtx_put(phba, pring, piocbq); - spin_unlock_irqrestore(&pring->ring_lock, iflags); - break; - } txq_cnt--; - /* The xri and iocb resources secured, - * attempt to issue request - */ - piocbq->sli4_lxritag = sglq->sli4_lxritag; - piocbq->sli4_xritag = sglq->sli4_xritag; - if (NO_XRI == lpfc_sli4_bpl2sgl(phba, piocbq, sglq)) - fail_msg = "to convert bpl to sgl"; - else if (lpfc_sli4_iocb2wqe(phba, piocbq, &wqe)) - fail_msg = "to convert iocb to wqe"; - else if (lpfc_sli4_wq_put(wq, &wqe)) - fail_msg = " - Wq is full"; - else - lpfc_sli_ringtxcmpl_put(phba, pring, piocbq); + ret = __lpfc_sli_issue_iocb(phba, pring->ringno, piocbq, 0); + if (ret && ret != IOCB_BUSY) { + fail_msg = " - Cannot send IO "; + piocbq->cmd_flag &= ~LPFC_DRIVER_ABORTED; + } if (fail_msg) { + piocbq->cmd_flag |= LPFC_DRIVER_ABORTED; /* Failed means we can't issue and need to cancel */ - lpfc_printf_log(phba, KERN_ERR, LOG_SLI, + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, "2822 IOCB failed %s iotag 0x%x " - "xri 0x%x\n", - fail_msg, - piocbq->iotag, piocbq->sli4_xritag); + "xri 0x%x %d flg x%x\n", + fail_msg, piocbq->iotag, + piocbq->sli4_xritag, ret, + piocbq->cmd_flag); list_add_tail(&piocbq->list, &completions); + fail_msg = NULL; } spin_unlock_irqrestore(&pring->ring_lock, iflags); + if (txq_cnt == 0 || ret == IOCB_BUSY) + break; } - /* Cancel all the IOCBs that cannot be issued */ lpfc_sli_cancel_iocbs(phba, &completions, IOSTAT_LOCAL_REJECT, - IOERR_SLI_ABORTED); + IOERR_SLI_ABORTED); return txq_cnt; } @@ -19579,7 +21297,7 @@ lpfc_drain_txq(struct lpfc_hba *phba) /** * lpfc_wqe_bpl2sgl - Convert the bpl/bde to a sgl. * @phba: Pointer to HBA context object. - * @pwqe: Pointer to command WQE. + * @pwqeq: Pointer to command WQE. * @sglq: Pointer to the scatter gather queue object. * * This routine converts the bpl or bde that is in the WQE @@ -19619,14 +21337,14 @@ lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeq, cmd = bf_get(wqe_cmnd, &wqe->generic.wqe_com); if (cmd == CMD_XMIT_BLS_RSP64_WQE) return sglq->sli4_xritag; - numBdes = pwqeq->rsvd2; + numBdes = pwqeq->num_bdes; if (numBdes) { /* The addrHigh and addrLow fields within the WQE * have not been byteswapped yet so there is no * need to swap them back. */ - if (pwqeq->context3) - dmabuf = (struct lpfc_dmabuf *)pwqeq->context3; + if (pwqeq->bpl_dmabuf) + dmabuf = pwqeq->bpl_dmabuf; else return xritag; @@ -19704,25 +21422,26 @@ lpfc_wqe_bpl2sgl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeq, /** * lpfc_sli4_issue_wqe - Issue an SLI4 Work Queue Entry (WQE) * @phba: Pointer to HBA context object. - * @ring_number: Base sli ring number + * @qp: Pointer to HDW queue. * @pwqe: Pointer to command WQE. **/ int -lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, +lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, struct lpfc_iocbq *pwqe) { union lpfc_wqe128 *wqe = &pwqe->wqe; - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; struct lpfc_queue *wq; struct lpfc_sglq *sglq; struct lpfc_sli_ring *pring; unsigned long iflags; - uint32_t ret = 0; + int ret = 0; /* NVME_LS and NVME_LS ABTS requests. */ - if (pwqe->iocb_flag & LPFC_IO_NVME_LS) { + if (pwqe->cmd_flag & LPFC_IO_NVME_LS) { pring = phba->sli4_hba.nvmels_wq->pring; - spin_lock_irqsave(&pring->ring_lock, iflags); + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); sglq = __lpfc_sli_get_els_sglq(phba, pwqe); if (!sglq) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19744,18 +21463,21 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, lpfc_sli_ringtxcmpl_put(phba, pring, pwqe); spin_unlock_irqrestore(&pring->ring_lock, iflags); + + lpfc_sli4_poll_eq(qp->hba_eq); return 0; } /* NVME_FCREQ and NVME_ABTS requests */ - if (pwqe->iocb_flag & LPFC_IO_NVME) { + if (pwqe->cmd_flag & (LPFC_IO_NVME | LPFC_IO_FCP | LPFC_IO_CMF)) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring; + wq = qp->io_wq; + pring = wq->pring; - spin_lock_irqsave(&pring->ring_lock, iflags); - wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]; - bf_set(wqe_cqid, &wqe->generic.wqe_com, - phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id); + bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); + + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19763,16 +21485,18 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, } lpfc_sli_ringtxcmpl_put(phba, pring, pwqe); spin_unlock_irqrestore(&pring->ring_lock, iflags); + + lpfc_sli4_poll_eq(qp->hba_eq); return 0; } /* NVMET requests */ - if (pwqe->iocb_flag & LPFC_IO_NVMET) { + if (pwqe->cmd_flag & LPFC_IO_NVMET) { /* Get the IO distribution (hba_wqidx) for WQ assignment. */ - pring = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]->pring; + wq = qp->io_wq; + pring = wq->pring; - spin_lock_irqsave(&pring->ring_lock, iflags); - ctxp = pwqe->context2; + ctxp = pwqe->context_un.axchg; sglq = ctxp->ctxbuf->sglq; if (pwqe->sli4_xritag == NO_XRI) { pwqe->sli4_lxritag = sglq->sli4_lxritag; @@ -19780,9 +21504,10 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, } bf_set(wqe_xri_tag, &pwqe->wqe.xmit_bls_rsp.wqe_com, pwqe->sli4_xritag); - wq = phba->sli4_hba.nvme_wq[pwqe->hba_wqidx]; - bf_set(wqe_cqid, &wqe->generic.wqe_com, - phba->sli4_hba.nvme_cq[pwqe->hba_wqidx]->queue_id); + bf_set(wqe_cqid, &wqe->generic.wqe_com, qp->io_cq_map); + + lpfc_qp_spin_lock_irqsave(&pring->ring_lock, iflags, + qp, wq_access); ret = lpfc_sli4_wq_put(wq, wqe); if (ret) { spin_unlock_irqrestore(&pring->ring_lock, iflags); @@ -19790,7 +21515,1341 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, uint32_t ring_number, } lpfc_sli_ringtxcmpl_put(phba, pring, pwqe); spin_unlock_irqrestore(&pring->ring_lock, iflags); + + lpfc_sli4_poll_eq(qp->hba_eq); return 0; } return WQE_ERROR; } + +/** + * lpfc_sli4_issue_abort_iotag - SLI-4 WQE init & issue for the Abort + * @phba: Pointer to HBA context object. + * @cmdiocb: Pointer to driver command iocb object. + * @cmpl: completion function. + * + * Fill the appropriate fields for the abort WQE and call + * internal routine lpfc_sli4_issue_wqe to send the WQE + * This function is called with hbalock held and no ring_lock held. + * + * RETURNS 0 - SUCCESS + **/ + +int +lpfc_sli4_issue_abort_iotag(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, + void *cmpl) +{ + struct lpfc_vport *vport = cmdiocb->vport; + struct lpfc_iocbq *abtsiocb = NULL; + union lpfc_wqe128 *abtswqe; + struct lpfc_io_buf *lpfc_cmd; + int retval = IOCB_ERROR; + u16 xritag = cmdiocb->sli4_xritag; + + /* + * The scsi command can not be in txq and it is in flight because the + * pCmd is still pointing at the SCSI command we have to abort. There + * is no need to search the txcmplq. Just send an abort to the FW. + */ + + abtsiocb = __lpfc_sli_get_iocbq(phba); + if (!abtsiocb) + return WQE_NORESOURCE; + + /* Indicate the IO is being aborted by the driver. */ + cmdiocb->cmd_flag |= LPFC_DRIVER_ABORTED; + + abtswqe = &abtsiocb->wqe; + memset(abtswqe, 0, sizeof(*abtswqe)); + + if (!lpfc_is_link_up(phba) || (phba->link_flag & LS_EXTERNAL_LOOPBACK)) + bf_set(abort_cmd_ia, &abtswqe->abort_cmd, 1); + bf_set(abort_cmd_criteria, &abtswqe->abort_cmd, T_XRI_TAG); + abtswqe->abort_cmd.rsrvd5 = 0; + abtswqe->abort_cmd.wqe_com.abort_tag = xritag; + bf_set(wqe_reqtag, &abtswqe->abort_cmd.wqe_com, abtsiocb->iotag); + bf_set(wqe_cmnd, &abtswqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); + bf_set(wqe_xri_tag, &abtswqe->generic.wqe_com, 0); + bf_set(wqe_qosd, &abtswqe->abort_cmd.wqe_com, 1); + bf_set(wqe_lenloc, &abtswqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); + bf_set(wqe_cmd_type, &abtswqe->abort_cmd.wqe_com, OTHER_COMMAND); + + /* ABTS WQE must go to the same WQ as the WQE to be aborted */ + abtsiocb->hba_wqidx = cmdiocb->hba_wqidx; + abtsiocb->cmd_flag |= LPFC_USE_FCPWQIDX; + if (cmdiocb->cmd_flag & LPFC_IO_FCP) + abtsiocb->cmd_flag |= LPFC_IO_FCP; + if (cmdiocb->cmd_flag & LPFC_IO_NVME) + abtsiocb->cmd_flag |= LPFC_IO_NVME; + if (cmdiocb->cmd_flag & LPFC_IO_FOF) + abtsiocb->cmd_flag |= LPFC_IO_FOF; + abtsiocb->vport = vport; + abtsiocb->cmd_cmpl = cmpl; + + lpfc_cmd = container_of(cmdiocb, struct lpfc_io_buf, cur_iocbq); + retval = lpfc_sli4_issue_wqe(phba, lpfc_cmd->hdwq, abtsiocb); + + lpfc_printf_vlog(vport, KERN_INFO, LOG_SLI | LOG_NVME_ABTS | LOG_FCP, + "0359 Abort xri x%x, original iotag x%x, " + "abort cmd iotag x%x retval x%x\n", + xritag, cmdiocb->iotag, abtsiocb->iotag, retval); + + if (retval) { + cmdiocb->cmd_flag &= ~LPFC_DRIVER_ABORTED; + __lpfc_sli_release_iocbq(phba, abtsiocb); + } + + return retval; +} + +#ifdef LPFC_MXP_STAT +/** + * lpfc_snapshot_mxp - Snapshot pbl, pvt and busy count + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * The purpose of this routine is to take a snapshot of pbl, pvt and busy count + * 15 seconds after a test case is running. + * + * The user should call lpfc_debugfs_multixripools_write before running a test + * case to clear stat_snapshot_taken. Then the user starts a test case. During + * test case is running, stat_snapshot_taken is incremented by 1 every time when + * this routine is called from heartbeat timer. When stat_snapshot_taken is + * equal to LPFC_MXP_SNAPSHOT_TAKEN, a snapshot is taken. + **/ +void lpfc_snapshot_mxp(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + u32 txcmplq_cnt; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + return; + + if (multixri_pool->stat_snapshot_taken == LPFC_MXP_SNAPSHOT_TAKEN) { + pvt_pool = &qp->p_multixri_pool->pvt_pool; + pbl_pool = &qp->p_multixri_pool->pbl_pool; + txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; + + multixri_pool->stat_pbl_count = pbl_pool->count; + multixri_pool->stat_pvt_count = pvt_pool->count; + multixri_pool->stat_busy_count = txcmplq_cnt; + } + + multixri_pool->stat_snapshot_taken++; +} +#endif + +/** + * lpfc_adjust_pvt_pool_count - Adjust private pool count + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine moves some XRIs from private to public pool when private pool + * is not busy. + **/ +void lpfc_adjust_pvt_pool_count(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_multixri_pool *multixri_pool; + u32 io_req_count; + u32 prev_io_req_count; + + multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + if (!multixri_pool) + return; + io_req_count = multixri_pool->io_req_count; + prev_io_req_count = multixri_pool->prev_io_req_count; + + if (prev_io_req_count != io_req_count) { + /* Private pool is busy */ + multixri_pool->prev_io_req_count = io_req_count; + } else { + /* Private pool is not busy. + * Move XRIs from private to public pool. + */ + lpfc_move_xri_pvt_to_pbl(phba, hwqid); + } +} + +/** + * lpfc_adjust_high_watermark - Adjust high watermark + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine sets high watermark as number of outstanding XRIs, + * but make sure the new value is between xri_limit/2 and xri_limit. + **/ +void lpfc_adjust_high_watermark(struct lpfc_hba *phba, u32 hwqid) +{ + u32 new_watermark; + u32 watermark_max; + u32 watermark_min; + u32 xri_limit; + u32 txcmplq_cnt; + u32 abts_io_bufs; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_sli4_hdw_queue *qp; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) + return; + xri_limit = multixri_pool->xri_limit; + + watermark_max = xri_limit; + watermark_min = xri_limit / 2; + + txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; + abts_io_bufs = qp->abts_scsi_io_bufs; + abts_io_bufs += qp->abts_nvme_io_bufs; + + new_watermark = txcmplq_cnt + abts_io_bufs; + new_watermark = min(watermark_max, new_watermark); + new_watermark = max(watermark_min, new_watermark); + multixri_pool->pvt_pool.high_watermark = new_watermark; + +#ifdef LPFC_MXP_STAT + multixri_pool->stat_max_hwm = max(multixri_pool->stat_max_hwm, + new_watermark); +#endif +} + +/** + * lpfc_move_xri_pvt_to_pbl - Move some XRIs from private to public pool + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine is called from hearbeat timer when pvt_pool is idle. + * All free XRIs are moved from private to public pool on hwqid with 2 steps. + * The first step moves (all - low_watermark) amount of XRIs. + * The second step moves the rest of XRIs. + **/ +void lpfc_move_xri_pvt_to_pbl(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct list_head tmp_list; + u32 tmp_count; + + qp = &phba->sli4_hba.hdwq[hwqid]; + pbl_pool = &qp->p_multixri_pool->pbl_pool; + pvt_pool = &qp->p_multixri_pool->pvt_pool; + tmp_count = 0; + + lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, qp, mv_to_pub_pool); + lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_from_pvt_pool); + + if (pvt_pool->count > pvt_pool->low_watermark) { + /* Step 1: move (all - low_watermark) from pvt_pool + * to pbl_pool + */ + + /* Move low watermark of bufs from pvt_pool to tmp_list */ + INIT_LIST_HEAD(&tmp_list); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + list_move_tail(&lpfc_ncmd->list, &tmp_list); + tmp_count++; + if (tmp_count >= pvt_pool->low_watermark) + break; + } + + /* Move all bufs from pvt_pool to pbl_pool */ + list_splice_init(&pvt_pool->list, &pbl_pool->list); + + /* Move all bufs from tmp_list to pvt_pool */ + list_splice(&tmp_list, &pvt_pool->list); + + pbl_pool->count += (pvt_pool->count - tmp_count); + pvt_pool->count = tmp_count; + } else { + /* Step 2: move the rest from pvt_pool to pbl_pool */ + list_splice_init(&pvt_pool->list, &pbl_pool->list); + pbl_pool->count += pvt_pool->count; + pvt_pool->count = 0; + } + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&pbl_pool->lock, iflag); +} + +/** + * _lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool + * @phba: pointer to lpfc hba data structure + * @qp: pointer to HDW queue + * @pbl_pool: specified public free XRI pool + * @pvt_pool: specified private free XRI pool + * @count: number of XRIs to move + * + * This routine tries to move some free common bufs from the specified pbl_pool + * to the specified pvt_pool. It might move less than count XRIs if there's not + * enough in public pool. + * + * Return: + * true - if XRIs are successfully moved from the specified pbl_pool to the + * specified pvt_pool + * false - if the specified pbl_pool is empty or locked by someone else + **/ +static bool +_lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, + struct lpfc_pbl_pool *pbl_pool, + struct lpfc_pvt_pool *pvt_pool, u32 count) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + int ret; + + ret = spin_trylock_irqsave(&pbl_pool->lock, iflag); + if (ret) { + if (pbl_pool->count) { + /* Move a batch of XRIs from public to private pool */ + lpfc_qp_spin_lock(&pvt_pool->lock, qp, mv_to_pvt_pool); + list_for_each_entry_safe(lpfc_ncmd, + lpfc_ncmd_next, + &pbl_pool->list, + list) { + list_move_tail(&lpfc_ncmd->list, + &pvt_pool->list); + pvt_pool->count++; + pbl_pool->count--; + count--; + if (count == 0) + break; + } + + spin_unlock(&pvt_pool->lock); + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + return true; + } + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + } + + return false; +} + +/** + * lpfc_move_xri_pbl_to_pvt - Move some XRIs from public to private pool + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * @count: number of XRIs to move + * + * This routine tries to find some free common bufs in one of public pools with + * Round Robin method. The search always starts from local hwqid, then the next + * HWQ which was found last time (rrb_next_hwqid). Once a public pool is found, + * a batch of free common bufs are moved to private pool on hwqid. + * It might move less than count XRIs if there's not enough in public pool. + **/ +void lpfc_move_xri_pbl_to_pvt(struct lpfc_hba *phba, u32 hwqid, u32 count) +{ + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_multixri_pool *next_multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_sli4_hdw_queue *qp; + u32 next_hwqid; + u32 hwq_count; + int ret; + + qp = &phba->sli4_hba.hdwq[hwqid]; + multixri_pool = qp->p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + pbl_pool = &multixri_pool->pbl_pool; + + /* Check if local pbl_pool is available */ + ret = _lpfc_move_xri_pbl_to_pvt(phba, qp, pbl_pool, pvt_pool, count); + if (ret) { +#ifdef LPFC_MXP_STAT + multixri_pool->local_pbl_hit_count++; +#endif + return; + } + + hwq_count = phba->cfg_hdw_queue; + + /* Get the next hwqid which was found last time */ + next_hwqid = multixri_pool->rrb_next_hwqid; + + do { + /* Go to next hwq */ + next_hwqid = (next_hwqid + 1) % hwq_count; + + next_multixri_pool = + phba->sli4_hba.hdwq[next_hwqid].p_multixri_pool; + pbl_pool = &next_multixri_pool->pbl_pool; + + /* Check if the public free xri pool is available */ + ret = _lpfc_move_xri_pbl_to_pvt( + phba, qp, pbl_pool, pvt_pool, count); + + /* Exit while-loop if success or all hwqid are checked */ + } while (!ret && next_hwqid != multixri_pool->rrb_next_hwqid); + + /* Starting point for the next time */ + multixri_pool->rrb_next_hwqid = next_hwqid; + + if (!ret) { + /* stats: all public pools are empty*/ + multixri_pool->pbl_empty_count++; + } + +#ifdef LPFC_MXP_STAT + if (ret) { + if (next_hwqid == hwqid) + multixri_pool->local_pbl_hit_count++; + else + multixri_pool->other_pbl_hit_count++; + } +#endif +} + +/** + * lpfc_keep_pvt_pool_above_lowwm - Keep pvt_pool above low watermark + * @phba: pointer to lpfc hba data structure. + * @hwqid: belong to which HWQ. + * + * This routine get a batch of XRIs from pbl_pool if pvt_pool is less than + * low watermark. + **/ +void lpfc_keep_pvt_pool_above_lowwm(struct lpfc_hba *phba, u32 hwqid) +{ + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + + multixri_pool = phba->sli4_hba.hdwq[hwqid].p_multixri_pool; + pvt_pool = &multixri_pool->pvt_pool; + + if (pvt_pool->count < pvt_pool->low_watermark) + lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH); +} + +/** + * lpfc_release_io_buf - Return one IO buf back to free pool + * @phba: pointer to lpfc hba data structure. + * @lpfc_ncmd: IO buf to be returned. + * @qp: belong to which HWQ. + * + * This routine returns one IO buf back to free pool. If this is an urgent IO, + * the IO buf is returned to expedite pool. If cfg_xri_rebalancing==1, + * the IO buf is returned to pbl_pool or pvt_pool based on watermark and + * xri_limit. If cfg_xri_rebalancing==0, the IO buf is returned to + * lpfc_io_buf_list_put. + **/ +void lpfc_release_io_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd, + struct lpfc_sli4_hdw_queue *qp) +{ + unsigned long iflag; + struct lpfc_pbl_pool *pbl_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_epd_pool *epd_pool; + u32 txcmplq_cnt; + u32 xri_owned; + u32 xri_limit; + u32 abts_io_bufs; + + /* MUST zero fields if buffer is reused by another protocol */ + lpfc_ncmd->nvmeCmd = NULL; + lpfc_ncmd->cur_iocbq.cmd_cmpl = NULL; + + if (phba->cfg_xpsgl && !phba->nvmet_support && + !list_empty(&lpfc_ncmd->dma_sgl_xtra_list)) + lpfc_put_sgl_per_hdwq(phba, lpfc_ncmd); + + if (!list_empty(&lpfc_ncmd->dma_cmd_rsp_list)) + lpfc_put_cmd_rsp_buf_per_hdwq(phba, lpfc_ncmd); + + if (phba->cfg_xri_rebalancing) { + if (lpfc_ncmd->expedite) { + /* Return to expedite pool */ + epd_pool = &phba->epd_pool; + spin_lock_irqsave(&epd_pool->lock, iflag); + list_add_tail(&lpfc_ncmd->list, &epd_pool->list); + epd_pool->count++; + spin_unlock_irqrestore(&epd_pool->lock, iflag); + return; + } + + /* Avoid invalid access if an IO sneaks in and is being rejected + * just _after_ xri pools are destroyed in lpfc_offline. + * Nothing much can be done at this point. + */ + if (!qp->p_multixri_pool) + return; + + pbl_pool = &qp->p_multixri_pool->pbl_pool; + pvt_pool = &qp->p_multixri_pool->pvt_pool; + + txcmplq_cnt = qp->io_wq->pring->txcmplq_cnt; + abts_io_bufs = qp->abts_scsi_io_bufs; + abts_io_bufs += qp->abts_nvme_io_bufs; + + xri_owned = pvt_pool->count + txcmplq_cnt + abts_io_bufs; + xri_limit = qp->p_multixri_pool->xri_limit; + +#ifdef LPFC_MXP_STAT + if (xri_owned <= xri_limit) + qp->p_multixri_pool->below_limit_count++; + else + qp->p_multixri_pool->above_limit_count++; +#endif + + /* XRI goes to either public or private free xri pool + * based on watermark and xri_limit + */ + if ((pvt_pool->count < pvt_pool->low_watermark) || + (xri_owned < xri_limit && + pvt_pool->count < pvt_pool->high_watermark)) { + lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, + qp, free_pvt_pool); + list_add_tail(&lpfc_ncmd->list, + &pvt_pool->list); + pvt_pool->count++; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + } else { + lpfc_qp_spin_lock_irqsave(&pbl_pool->lock, iflag, + qp, free_pub_pool); + list_add_tail(&lpfc_ncmd->list, + &pbl_pool->list); + pbl_pool->count++; + spin_unlock_irqrestore(&pbl_pool->lock, iflag); + } + } else { + lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_put_lock, iflag, + qp, free_xri); + list_add_tail(&lpfc_ncmd->list, + &qp->lpfc_io_buf_list_put); + qp->put_io_bufs++; + spin_unlock_irqrestore(&qp->io_buf_list_put_lock, + iflag); + } +} + +/** + * lpfc_get_io_buf_from_private_pool - Get one free IO buf from private pool + * @phba: pointer to lpfc hba data structure. + * @qp: pointer to HDW queue + * @pvt_pool: pointer to private pool data structure. + * @ndlp: pointer to lpfc nodelist data structure. + * + * This routine tries to get one free IO buf from private pool. + * + * Return: + * pointer to one free IO buf - if private pool is not empty + * NULL - if private pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_private_pool(struct lpfc_hba *phba, + struct lpfc_sli4_hdw_queue *qp, + struct lpfc_pvt_pool *pvt_pool, + struct lpfc_nodelist *ndlp) +{ + struct lpfc_io_buf *lpfc_ncmd; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + + lpfc_qp_spin_lock_irqsave(&pvt_pool->lock, iflag, qp, alloc_pvt_pool); + list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, + &pvt_pool->list, list) { + if (lpfc_test_rrq_active( + phba, ndlp, lpfc_ncmd->cur_iocbq.sli4_lxritag)) + continue; + list_del(&lpfc_ncmd->list); + pvt_pool->count--; + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + return lpfc_ncmd; + } + spin_unlock_irqrestore(&pvt_pool->lock, iflag); + + return NULL; +} + +/** + * lpfc_get_io_buf_from_expedite_pool - Get one free IO buf from expedite pool + * @phba: pointer to lpfc hba data structure. + * + * This routine tries to get one free IO buf from expedite pool. + * + * Return: + * pointer to one free IO buf - if expedite pool is not empty + * NULL - if expedite pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_expedite_pool(struct lpfc_hba *phba) +{ + struct lpfc_io_buf *lpfc_ncmd = NULL, *iter; + struct lpfc_io_buf *lpfc_ncmd_next; + unsigned long iflag; + struct lpfc_epd_pool *epd_pool; + + epd_pool = &phba->epd_pool; + + spin_lock_irqsave(&epd_pool->lock, iflag); + if (epd_pool->count > 0) { + list_for_each_entry_safe(iter, lpfc_ncmd_next, + &epd_pool->list, list) { + list_del(&iter->list); + epd_pool->count--; + lpfc_ncmd = iter; + break; + } + } + spin_unlock_irqrestore(&epd_pool->lock, iflag); + + return lpfc_ncmd; +} + +/** + * lpfc_get_io_buf_from_multixri_pools - Get one free IO bufs + * @phba: pointer to lpfc hba data structure. + * @ndlp: pointer to lpfc nodelist data structure. + * @hwqid: belong to which HWQ + * @expedite: 1 means this request is urgent. + * + * This routine will do the following actions and then return a pointer to + * one free IO buf. + * + * 1. If private free xri count is empty, move some XRIs from public to + * private pool. + * 2. Get one XRI from private free xri pool. + * 3. If we fail to get one from pvt_pool and this is an expedite request, + * get one free xri from expedite pool. + * + * Note: ndlp is only used on SCSI side for RRQ testing. + * The caller should pass NULL for ndlp on NVME side. + * + * Return: + * pointer to one free IO buf - if private pool is not empty + * NULL - if private pool is empty + **/ +static struct lpfc_io_buf * +lpfc_get_io_buf_from_multixri_pools(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, + int hwqid, int expedite) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_multixri_pool *multixri_pool; + struct lpfc_pvt_pool *pvt_pool; + struct lpfc_io_buf *lpfc_ncmd; + + qp = &phba->sli4_hba.hdwq[hwqid]; + lpfc_ncmd = NULL; + if (!qp) { + lpfc_printf_log(phba, KERN_INFO, + LOG_SLI | LOG_NVME_ABTS | LOG_FCP, + "5556 NULL qp for hwqid x%x\n", hwqid); + return lpfc_ncmd; + } + multixri_pool = qp->p_multixri_pool; + if (!multixri_pool) { + lpfc_printf_log(phba, KERN_INFO, + LOG_SLI | LOG_NVME_ABTS | LOG_FCP, + "5557 NULL multixri for hwqid x%x\n", hwqid); + return lpfc_ncmd; + } + pvt_pool = &multixri_pool->pvt_pool; + if (!pvt_pool) { + lpfc_printf_log(phba, KERN_INFO, + LOG_SLI | LOG_NVME_ABTS | LOG_FCP, + "5558 NULL pvt_pool for hwqid x%x\n", hwqid); + return lpfc_ncmd; + } + multixri_pool->io_req_count++; + + /* If pvt_pool is empty, move some XRIs from public to private pool */ + if (pvt_pool->count == 0) + lpfc_move_xri_pbl_to_pvt(phba, hwqid, XRI_BATCH); + + /* Get one XRI from private free xri pool */ + lpfc_ncmd = lpfc_get_io_buf_from_private_pool(phba, qp, pvt_pool, ndlp); + + if (lpfc_ncmd) { + lpfc_ncmd->hdwq = qp; + lpfc_ncmd->hdwq_no = hwqid; + } else if (expedite) { + /* If we fail to get one from pvt_pool and this is an expedite + * request, get one free xri from expedite pool. + */ + lpfc_ncmd = lpfc_get_io_buf_from_expedite_pool(phba); + } + + return lpfc_ncmd; +} + +static inline struct lpfc_io_buf * +lpfc_io_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, int idx) +{ + struct lpfc_sli4_hdw_queue *qp; + struct lpfc_io_buf *lpfc_cmd, *lpfc_cmd_next; + + qp = &phba->sli4_hba.hdwq[idx]; + list_for_each_entry_safe(lpfc_cmd, lpfc_cmd_next, + &qp->lpfc_io_buf_list_get, list) { + if (lpfc_test_rrq_active(phba, ndlp, + lpfc_cmd->cur_iocbq.sli4_lxritag)) + continue; + + if (lpfc_cmd->flags & LPFC_SBUF_NOT_POSTED) + continue; + + list_del_init(&lpfc_cmd->list); + qp->get_io_bufs--; + lpfc_cmd->hdwq = qp; + lpfc_cmd->hdwq_no = idx; + return lpfc_cmd; + } + return NULL; +} + +/** + * lpfc_get_io_buf - Get one IO buffer from free pool + * @phba: The HBA for which this call is being executed. + * @ndlp: pointer to lpfc nodelist data structure. + * @hwqid: belong to which HWQ + * @expedite: 1 means this request is urgent. + * + * This routine gets one IO buffer from free pool. If cfg_xri_rebalancing==1, + * removes a IO buffer from multiXRI pools. If cfg_xri_rebalancing==0, removes + * a IO buffer from head of @hdwq io_buf_list and returns to caller. + * + * Note: ndlp is only used on SCSI side for RRQ testing. + * The caller should pass NULL for ndlp on NVME side. + * + * Return codes: + * NULL - Error + * Pointer to lpfc_io_buf - Success + **/ +struct lpfc_io_buf *lpfc_get_io_buf(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp, + u32 hwqid, int expedite) +{ + struct lpfc_sli4_hdw_queue *qp; + unsigned long iflag; + struct lpfc_io_buf *lpfc_cmd; + + qp = &phba->sli4_hba.hdwq[hwqid]; + lpfc_cmd = NULL; + if (!qp) { + lpfc_printf_log(phba, KERN_WARNING, + LOG_SLI | LOG_NVME_ABTS | LOG_FCP, + "5555 NULL qp for hwqid x%x\n", hwqid); + return lpfc_cmd; + } + + if (phba->cfg_xri_rebalancing) + lpfc_cmd = lpfc_get_io_buf_from_multixri_pools( + phba, ndlp, hwqid, expedite); + else { + lpfc_qp_spin_lock_irqsave(&qp->io_buf_list_get_lock, iflag, + qp, alloc_xri_get); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || expedite) + lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid); + if (!lpfc_cmd) { + lpfc_qp_spin_lock(&qp->io_buf_list_put_lock, + qp, alloc_xri_put); + list_splice(&qp->lpfc_io_buf_list_put, + &qp->lpfc_io_buf_list_get); + qp->get_io_bufs += qp->put_io_bufs; + INIT_LIST_HEAD(&qp->lpfc_io_buf_list_put); + qp->put_io_bufs = 0; + spin_unlock(&qp->io_buf_list_put_lock); + if (qp->get_io_bufs > LPFC_NVME_EXPEDITE_XRICNT || + expedite) + lpfc_cmd = lpfc_io_buf(phba, ndlp, hwqid); + } + spin_unlock_irqrestore(&qp->io_buf_list_get_lock, iflag); + } + + return lpfc_cmd; +} + +/** + * lpfc_read_object - Retrieve object data from HBA + * @phba: The HBA for which this call is being executed. + * @rdobject: Pathname of object data we want to read. + * @datap: Pointer to where data will be copied to. + * @datasz: size of data area + * + * This routine is limited to object sizes of LPFC_BPL_SIZE (1024) or less. + * The data will be truncated if datasz is not large enough. + * Version 1 is not supported with Embedded mbox cmd, so we must use version 0. + * Returns the actual bytes read from the object. + * + * This routine is hard coded to use a poll completion. Unlike other + * sli4_config mailboxes, it uses lpfc_mbuf memory which is not + * cleaned up in lpfc_sli4_cmd_mbox_free. If this routine is modified + * to use interrupt-based completions, code is needed to fully cleanup + * the memory. + */ +int +lpfc_read_object(struct lpfc_hba *phba, char *rdobject, uint32_t *datap, + uint32_t datasz) +{ + struct lpfc_mbx_read_object *read_object; + LPFC_MBOXQ_t *mbox; + int rc, length, eof, j, byte_cnt = 0; + uint32_t shdr_status, shdr_add_status; + union lpfc_sli4_cfg_shdr *shdr; + struct lpfc_dmabuf *pcmd; + u32 rd_object_name[LPFC_MBX_OBJECT_NAME_LEN_DW] = {0}; + + mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); + if (!mbox) + return -ENOMEM; + length = (sizeof(struct lpfc_mbx_read_object) - + sizeof(struct lpfc_sli4_cfg_mhdr)); + lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_COMMON, + LPFC_MBOX_OPCODE_READ_OBJECT, + length, LPFC_SLI4_MBX_EMBED); + read_object = &mbox->u.mqe.un.read_object; + shdr = (union lpfc_sli4_cfg_shdr *)&read_object->header.cfg_shdr; + + bf_set(lpfc_mbox_hdr_version, &shdr->request, LPFC_Q_CREATE_VERSION_0); + bf_set(lpfc_mbx_rd_object_rlen, &read_object->u.request, datasz); + read_object->u.request.rd_object_offset = 0; + read_object->u.request.rd_object_cnt = 1; + + memset((void *)read_object->u.request.rd_object_name, 0, + LPFC_OBJ_NAME_SZ); + scnprintf((char *)rd_object_name, sizeof(rd_object_name), rdobject); + for (j = 0; j < strlen(rdobject); j++) + read_object->u.request.rd_object_name[j] = + cpu_to_le32(rd_object_name[j]); + + pcmd = kmalloc(sizeof(*pcmd), GFP_KERNEL); + if (pcmd) + pcmd->virt = lpfc_mbuf_alloc(phba, MEM_PRI, &pcmd->phys); + if (!pcmd || !pcmd->virt) { + kfree(pcmd); + mempool_free(mbox, phba->mbox_mem_pool); + return -ENOMEM; + } + memset((void *)pcmd->virt, 0, LPFC_BPL_SIZE); + read_object->u.request.rd_object_hbuf[0].pa_lo = + putPaddrLow(pcmd->phys); + read_object->u.request.rd_object_hbuf[0].pa_hi = + putPaddrHigh(pcmd->phys); + read_object->u.request.rd_object_hbuf[0].length = LPFC_BPL_SIZE; + + mbox->vport = phba->pport; + mbox->mbox_cmpl = lpfc_sli_def_mbox_cmpl; + mbox->ctx_ndlp = NULL; + + rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); + shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); + shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); + + if (shdr_status == STATUS_FAILED && + shdr_add_status == ADD_STATUS_INVALID_OBJECT_NAME) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT, + "4674 No port cfg file in FW.\n"); + byte_cnt = -ENOENT; + } else if (shdr_status || shdr_add_status || rc) { + lpfc_printf_log(phba, KERN_ERR, LOG_INIT | LOG_CGN_MGMT, + "2625 READ_OBJECT mailbox failed with " + "status x%x add_status x%x, mbx status x%x\n", + shdr_status, shdr_add_status, rc); + byte_cnt = -ENXIO; + } else { + /* Success */ + length = read_object->u.response.rd_object_actual_rlen; + eof = bf_get(lpfc_mbx_rd_object_eof, &read_object->u.response); + lpfc_printf_log(phba, KERN_INFO, LOG_INIT | LOG_CGN_MGMT, + "2626 READ_OBJECT Success len %d:%d, EOF %d\n", + length, datasz, eof); + + /* Detect the port config file exists but is empty */ + if (!length && eof) { + byte_cnt = 0; + goto exit; + } + + byte_cnt = length; + lpfc_sli_pcimem_bcopy(pcmd->virt, datap, byte_cnt); + } + + exit: + /* This is an embedded SLI4 mailbox with an external buffer allocated. + * Free the pcmd and then cleanup with the correct routine. + */ + lpfc_mbuf_free(phba, pcmd->virt, pcmd->phys); + kfree(pcmd); + lpfc_sli4_mbox_cmd_free(phba, mbox); + return byte_cnt; +} + +/** + * lpfc_get_sgl_per_hdwq - Get one SGL chunk from hdwq's pool + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure to append the SGL chunk + * + * This routine gets one SGL chunk buffer from hdwq's SGL chunk pool, + * and will allocate an SGL chunk if the pool is empty. + * + * Return codes: + * NULL - Error + * Pointer to sli4_hybrid_sgl - Success + **/ +struct sli4_hybrid_sgl * +lpfc_get_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf) +{ + struct sli4_hybrid_sgl *list_entry = NULL; + struct sli4_hybrid_sgl *tmp = NULL; + struct sli4_hybrid_sgl *allocated_sgl = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->sgl_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(buf_list))) { + /* break off 1 chunk from the sgl_list */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, list_node) { + list_move_tail(&list_entry->list_node, + &lpfc_buf->dma_sgl_xtra_list); + break; + } + } else { + /* allocate more */ + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + tmp = kmalloc_node(sizeof(*tmp), GFP_ATOMIC, + cpu_to_node(hdwq->io_wq->chann)); + if (!tmp) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8353 error kmalloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + return NULL; + } + + tmp->dma_sgl = dma_pool_alloc(phba->lpfc_sg_dma_buf_pool, + GFP_ATOMIC, &tmp->dma_phys_sgl); + if (!tmp->dma_sgl) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8354 error pool_alloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + kfree(tmp); + return NULL; + } + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + list_add_tail(&tmp->list_node, &lpfc_buf->dma_sgl_xtra_list); + } + + allocated_sgl = list_last_entry(&lpfc_buf->dma_sgl_xtra_list, + struct sli4_hybrid_sgl, + list_node); + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + + return allocated_sgl; +} + +/** + * lpfc_put_sgl_per_hdwq - Put one SGL chunk into hdwq pool + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure with the SGL chunk + * + * This routine puts one SGL chunk buffer into hdwq's SGL chunk pool. + * + * Return codes: + * 0 - Success + * -EINVAL - Error + **/ +int +lpfc_put_sgl_per_hdwq(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_buf) +{ + int rc = 0; + struct sli4_hybrid_sgl *list_entry = NULL; + struct sli4_hybrid_sgl *tmp = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->sgl_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(&lpfc_buf->dma_sgl_xtra_list))) { + list_for_each_entry_safe(list_entry, tmp, + &lpfc_buf->dma_sgl_xtra_list, + list_node) { + list_move_tail(&list_entry->list_node, + buf_list); + } + } else { + rc = -EINVAL; + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + return rc; +} + +/** + * lpfc_free_sgl_per_hdwq - Free all SGL chunks of hdwq pool + * @phba: phba object + * @hdwq: hdwq to cleanup sgl buff resources on + * + * This routine frees all SGL chunks of hdwq SGL chunk pool. + * + * Return codes: + * None + **/ +void +lpfc_free_sgl_per_hdwq(struct lpfc_hba *phba, + struct lpfc_sli4_hdw_queue *hdwq) +{ + struct list_head *buf_list = &hdwq->sgl_list; + struct sli4_hybrid_sgl *list_entry = NULL; + struct sli4_hybrid_sgl *tmp = NULL; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + /* Free sgl pool */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, list_node) { + list_del(&list_entry->list_node); + dma_pool_free(phba->lpfc_sg_dma_buf_pool, + list_entry->dma_sgl, + list_entry->dma_phys_sgl); + kfree(list_entry); + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); +} + +/** + * lpfc_get_cmd_rsp_buf_per_hdwq - Get one CMD/RSP buffer from hdwq + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure to attach the CMD/RSP buffer + * + * This routine gets one CMD/RSP buffer from hdwq's CMD/RSP pool, + * and will allocate an CMD/RSP buffer if the pool is empty. + * + * Return codes: + * NULL - Error + * Pointer to fcp_cmd_rsp_buf - Success + **/ +struct fcp_cmd_rsp_buf * +lpfc_get_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba, + struct lpfc_io_buf *lpfc_buf) +{ + struct fcp_cmd_rsp_buf *list_entry = NULL; + struct fcp_cmd_rsp_buf *tmp = NULL; + struct fcp_cmd_rsp_buf *allocated_buf = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->cmd_rsp_buf_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(buf_list))) { + /* break off 1 chunk from the list */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, + list_node) { + list_move_tail(&list_entry->list_node, + &lpfc_buf->dma_cmd_rsp_list); + break; + } + } else { + /* allocate more */ + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + tmp = kmalloc_node(sizeof(*tmp), GFP_ATOMIC, + cpu_to_node(hdwq->io_wq->chann)); + if (!tmp) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8355 error kmalloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + return NULL; + } + + tmp->fcp_cmnd = dma_pool_zalloc(phba->lpfc_cmd_rsp_buf_pool, + GFP_ATOMIC, + &tmp->fcp_cmd_rsp_dma_handle); + + if (!tmp->fcp_cmnd) { + lpfc_printf_log(phba, KERN_INFO, LOG_SLI, + "8356 error pool_alloc memory for HDWQ " + "%d %s\n", + lpfc_buf->hdwq_no, __func__); + kfree(tmp); + return NULL; + } + + tmp->fcp_rsp = (struct fcp_rsp *)((uint8_t *)tmp->fcp_cmnd + + sizeof(struct fcp_cmnd32)); + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + list_add_tail(&tmp->list_node, &lpfc_buf->dma_cmd_rsp_list); + } + + allocated_buf = list_last_entry(&lpfc_buf->dma_cmd_rsp_list, + struct fcp_cmd_rsp_buf, + list_node); + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + + return allocated_buf; +} + +/** + * lpfc_put_cmd_rsp_buf_per_hdwq - Put one CMD/RSP buffer into hdwq pool + * @phba: The HBA for which this call is being executed. + * @lpfc_buf: IO buf structure with the CMD/RSP buf + * + * This routine puts one CMD/RSP buffer into executing CPU's CMD/RSP pool. + * + * Return codes: + * 0 - Success + * -EINVAL - Error + **/ +int +lpfc_put_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba, + struct lpfc_io_buf *lpfc_buf) +{ + int rc = 0; + struct fcp_cmd_rsp_buf *list_entry = NULL; + struct fcp_cmd_rsp_buf *tmp = NULL; + struct lpfc_sli4_hdw_queue *hdwq = lpfc_buf->hdwq; + struct list_head *buf_list = &hdwq->cmd_rsp_buf_list; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + if (likely(!list_empty(&lpfc_buf->dma_cmd_rsp_list))) { + list_for_each_entry_safe(list_entry, tmp, + &lpfc_buf->dma_cmd_rsp_list, + list_node) { + list_move_tail(&list_entry->list_node, + buf_list); + } + } else { + rc = -EINVAL; + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); + return rc; +} + +/** + * lpfc_free_cmd_rsp_buf_per_hdwq - Free all CMD/RSP chunks of hdwq pool + * @phba: phba object + * @hdwq: hdwq to cleanup cmd rsp buff resources on + * + * This routine frees all CMD/RSP buffers of hdwq's CMD/RSP buf pool. + * + * Return codes: + * None + **/ +void +lpfc_free_cmd_rsp_buf_per_hdwq(struct lpfc_hba *phba, + struct lpfc_sli4_hdw_queue *hdwq) +{ + struct list_head *buf_list = &hdwq->cmd_rsp_buf_list; + struct fcp_cmd_rsp_buf *list_entry = NULL; + struct fcp_cmd_rsp_buf *tmp = NULL; + unsigned long iflags; + + spin_lock_irqsave(&hdwq->hdwq_lock, iflags); + + /* Free cmd_rsp buf pool */ + list_for_each_entry_safe(list_entry, tmp, + buf_list, + list_node) { + list_del(&list_entry->list_node); + dma_pool_free(phba->lpfc_cmd_rsp_buf_pool, + list_entry->fcp_cmnd, + list_entry->fcp_cmd_rsp_dma_handle); + kfree(list_entry); + } + + spin_unlock_irqrestore(&hdwq->hdwq_lock, iflags); +} + +/** + * lpfc_sli_prep_wqe - Prepare WQE for the command to be posted + * @phba: phba object + * @job: job entry of the command to be posted. + * + * Fill the common fields of the wqe for each of the command. + * + * Return codes: + * None + **/ +void +lpfc_sli_prep_wqe(struct lpfc_hba *phba, struct lpfc_iocbq *job) +{ + u8 cmnd; + u32 *pcmd; + u32 if_type = 0; + u32 abort_tag; + bool fip; + struct lpfc_nodelist *ndlp = NULL; + union lpfc_wqe128 *wqe = &job->wqe; + u8 command_type = ELS_COMMAND_NON_FIP; + + fip = test_bit(HBA_FIP_SUPPORT, &phba->hba_flag); + /* The fcp commands will set command type */ + if (job->cmd_flag & LPFC_IO_FCP) + command_type = FCP_COMMAND; + else if (fip && (job->cmd_flag & LPFC_FIP_ELS_ID_MASK)) + command_type = ELS_COMMAND_FIP; + else + command_type = ELS_COMMAND_NON_FIP; + + abort_tag = job->iotag; + cmnd = bf_get(wqe_cmnd, &wqe->els_req.wqe_com); + + switch (cmnd) { + case CMD_ELS_REQUEST64_WQE: + ndlp = job->ndlp; + + if_type = bf_get(lpfc_sli_intf_if_type, + &phba->sli4_hba.sli_intf); + if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) { + pcmd = (u32 *)job->cmd_dmabuf->virt; + if (pcmd && (*pcmd == ELS_CMD_FLOGI || + *pcmd == ELS_CMD_SCR || + *pcmd == ELS_CMD_RDF || + *pcmd == ELS_CMD_EDC || + *pcmd == ELS_CMD_RSCN_XMT || + *pcmd == ELS_CMD_FDISC || + *pcmd == ELS_CMD_LOGO || + *pcmd == ELS_CMD_QFPA || + *pcmd == ELS_CMD_UVEM || + *pcmd == ELS_CMD_PLOGI)) { + bf_set(els_req64_sp, &wqe->els_req, 1); + bf_set(els_req64_sid, &wqe->els_req, + job->vport->fc_myDID); + + if ((*pcmd == ELS_CMD_FLOGI) && + !(phba->fc_topology == + LPFC_TOPOLOGY_LOOP)) + bf_set(els_req64_sid, &wqe->els_req, 0); + + bf_set(wqe_ct, &wqe->els_req.wqe_com, 1); + bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, + phba->vpi_ids[job->vport->vpi]); + } else if (pcmd) { + bf_set(wqe_ct, &wqe->els_req.wqe_com, 0); + bf_set(wqe_ctxt_tag, &wqe->els_req.wqe_com, + phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); + } + } + + bf_set(wqe_temp_rpi, &wqe->els_req.wqe_com, + phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); + + bf_set(wqe_dbde, &wqe->els_req.wqe_com, 1); + bf_set(wqe_iod, &wqe->els_req.wqe_com, LPFC_WQE_IOD_READ); + bf_set(wqe_qosd, &wqe->els_req.wqe_com, 1); + bf_set(wqe_lenloc, &wqe->els_req.wqe_com, LPFC_WQE_LENLOC_NONE); + bf_set(wqe_ebde_cnt, &wqe->els_req.wqe_com, 0); + break; + case CMD_XMIT_ELS_RSP64_WQE: + ndlp = job->ndlp; + + /* word4 */ + wqe->xmit_els_rsp.word4 = 0; + + if_type = bf_get(lpfc_sli_intf_if_type, + &phba->sli4_hba.sli_intf); + if (if_type >= LPFC_SLI_INTF_IF_TYPE_2) { + if (test_bit(FC_PT2PT, &job->vport->fc_flag)) { + bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1); + bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, + job->vport->fc_myDID); + if (job->vport->fc_myDID == Fabric_DID) { + bf_set(wqe_els_did, + &wqe->xmit_els_rsp.wqe_dest, 0); + } + } + } + + bf_set(wqe_dbde, &wqe->xmit_els_rsp.wqe_com, 1); + bf_set(wqe_iod, &wqe->xmit_els_rsp.wqe_com, LPFC_WQE_IOD_WRITE); + bf_set(wqe_qosd, &wqe->xmit_els_rsp.wqe_com, 1); + bf_set(wqe_lenloc, &wqe->xmit_els_rsp.wqe_com, + LPFC_WQE_LENLOC_WORD3); + bf_set(wqe_ebde_cnt, &wqe->xmit_els_rsp.wqe_com, 0); + + if (phba->fc_topology == LPFC_TOPOLOGY_LOOP) { + bf_set(els_rsp64_sp, &wqe->xmit_els_rsp, 1); + bf_set(els_rsp64_sid, &wqe->xmit_els_rsp, + job->vport->fc_myDID); + bf_set(wqe_ct, &wqe->xmit_els_rsp.wqe_com, 1); + } + + if (phba->sli_rev == LPFC_SLI_REV4) { + bf_set(wqe_rsp_temp_rpi, &wqe->xmit_els_rsp, + phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); + + if (bf_get(wqe_ct, &wqe->xmit_els_rsp.wqe_com)) + bf_set(wqe_ctxt_tag, &wqe->xmit_els_rsp.wqe_com, + phba->vpi_ids[job->vport->vpi]); + } + command_type = OTHER_COMMAND; + break; + case CMD_GEN_REQUEST64_WQE: + /* Word 10 */ + bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); + bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); + bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); + bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); + bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); + command_type = OTHER_COMMAND; + break; + case CMD_XMIT_SEQUENCE64_WQE: + if (phba->link_flag & LS_LOOPBACK_MODE) + bf_set(wqe_xo, &wqe->xmit_sequence.wge_ctl, 1); + + wqe->xmit_sequence.rsvd3 = 0; + bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0); + bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1); + bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, + LPFC_WQE_IOD_WRITE); + bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com, + LPFC_WQE_LENLOC_WORD12); + bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0); + command_type = OTHER_COMMAND; + break; + case CMD_XMIT_BLS_RSP64_WQE: + bf_set(xmit_bls_rsp64_seqcnthi, &wqe->xmit_bls_rsp, 0xffff); + bf_set(wqe_xmit_bls_pt, &wqe->xmit_bls_rsp.wqe_dest, 0x1); + bf_set(wqe_ct, &wqe->xmit_bls_rsp.wqe_com, 1); + bf_set(wqe_ctxt_tag, &wqe->xmit_bls_rsp.wqe_com, + phba->vpi_ids[phba->pport->vpi]); + bf_set(wqe_qosd, &wqe->xmit_bls_rsp.wqe_com, 1); + bf_set(wqe_lenloc, &wqe->xmit_bls_rsp.wqe_com, + LPFC_WQE_LENLOC_NONE); + /* Overwrite the pre-set comnd type with OTHER_COMMAND */ + command_type = OTHER_COMMAND; + break; + case CMD_FCP_ICMND64_WQE: /* task mgmt commands */ + case CMD_ABORT_XRI_WQE: /* abort iotag */ + case CMD_SEND_FRAME: /* mds loopback */ + /* cases already formatted for sli4 wqe - no chgs necessary */ + return; + default: + dump_stack(); + lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT, + "6207 Invalid command 0x%x\n", + cmnd); + break; + } + + wqe->generic.wqe_com.abort_tag = abort_tag; + bf_set(wqe_reqtag, &wqe->generic.wqe_com, job->iotag); + bf_set(wqe_cmd_type, &wqe->generic.wqe_com, command_type); + bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); +} |