diff options
Diffstat (limited to 'drivers/scsi/fnic/fnic_scsi.c')
| -rw-r--r-- | drivers/scsi/fnic/fnic_scsi.c | 2675 |
1 files changed, 1482 insertions, 1193 deletions
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c index cafbcfb85bfa..75b29a018d1f 100644 --- a/drivers/scsi/fnic/fnic_scsi.c +++ b/drivers/scsi/fnic/fnic_scsi.c @@ -1,19 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2008 Cisco Systems, Inc. All rights reserved. * Copyright 2007 Nuova Systems, Inc. All rights reserved. - * - * This program is free software; you may redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #include <linux/mempool.h> #include <linux/errno.h> @@ -23,6 +11,7 @@ #include <linux/scatterlist.h> #include <linux/skbuff.h> #include <linux/spinlock.h> +#include <linux/etherdevice.h> #include <linux/if_ether.h> #include <linux/if_vlan.h> #include <linux/delay.h> @@ -34,11 +23,13 @@ #include <scsi/scsi_tcq.h> #include <scsi/fc/fc_els.h> #include <scsi/fc/fc_fcoe.h> -#include <scsi/libfc.h> #include <scsi/fc_frame.h> +#include <scsi/scsi_transport_fc.h> #include "fnic_io.h" #include "fnic.h" +static void fnic_cleanup_io(struct fnic *fnic, int exclude_id); + const char *fnic_state_str[] = { [FNIC_IN_FC_MODE] = "FNIC_IN_FC_MODE", [FNIC_IN_FC_TRANS_ETH_MODE] = "FNIC_IN_FC_TRANS_ETH_MODE", @@ -76,6 +67,18 @@ static const char *fcpio_status_str[] = { [FCPIO_LUNMAP_CHNG_PEND] = "FCPIO_LUNHMAP_CHNG_PEND", }; +enum terminate_io_return { + TERM_SUCCESS = 0, + TERM_NO_SC = 1, + TERM_IO_REQ_NOT_FOUND, + TERM_ANOTHER_PORT, + TERM_GSTATE, + TERM_IO_BLOCKED, + TERM_OUT_OF_WQ_DESC, + TERM_TIMED_OUT, + TERM_MISC, +}; + const char *fnic_state_to_str(unsigned int state) { if (state >= ARRAY_SIZE(fnic_state_str) || !fnic_state_str[state]) @@ -101,22 +104,6 @@ static const char *fnic_fcpio_status_to_str(unsigned int status) return fcpio_status_str[status]; } -static void fnic_cleanup_io(struct fnic *fnic, int exclude_id); - -static inline spinlock_t *fnic_io_lock_hash(struct fnic *fnic, - struct scsi_cmnd *sc) -{ - u32 hash = sc->request->tag & (FNIC_IO_LOCKS - 1); - - return &fnic->io_req_lock[hash]; -} - -static inline spinlock_t *fnic_io_lock_tag(struct fnic *fnic, - int tag) -{ - return &fnic->io_req_lock[tag & (FNIC_IO_LOCKS - 1)]; -} - /* * Unmap the data buffer and sense buffer for an io_req, * also unmap and free the device-private scatter/gather list. @@ -139,24 +126,83 @@ static void fnic_release_ioreq_buf(struct fnic *fnic, SCSI_SENSE_BUFFERSIZE, DMA_FROM_DEVICE); } +static bool +fnic_count_portid_ioreqs_iter(struct fnic *fnic, struct scsi_cmnd *sc, + void *data1, void *data2) +{ + u32 *portid = data1; + unsigned int *count = data2; + struct fnic_io_req *io_req = fnic_priv(sc)->io_req; + + if (!io_req || (*portid && (io_req->port_id != *portid))) + return true; + + *count += 1; + return true; +} + +unsigned int fnic_count_ioreqs(struct fnic *fnic, u32 portid) +{ + unsigned int count = 0; + + fnic_scsi_io_iter(fnic, fnic_count_portid_ioreqs_iter, + &portid, &count); + + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "portid = 0x%x count = %u\n", portid, count); + return count; +} + +unsigned int fnic_count_all_ioreqs(struct fnic *fnic) +{ + return fnic_count_ioreqs(fnic, 0); +} + +static bool +fnic_count_lun_ioreqs_iter(struct fnic *fnic, struct scsi_cmnd *sc, + void *data1, void *data2) +{ + struct scsi_device *scsi_device = data1; + unsigned int *count = data2; + + if (sc->device != scsi_device || !fnic_priv(sc)->io_req) + return true; + + *count += 1; + return true; +} + +unsigned int +fnic_count_lun_ioreqs(struct fnic *fnic, struct scsi_device *scsi_device) +{ + unsigned int count = 0; + + fnic_scsi_io_iter(fnic, fnic_count_lun_ioreqs_iter, + scsi_device, &count); + + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "lun = %p count = %u\n", scsi_device, count); + return count; +} + /* Free up Copy Wq descriptors. Called with copy_wq lock held */ -static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq) +static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq, unsigned int hwq) { /* if no Ack received from firmware, then nothing to clean */ - if (!fnic->fw_ack_recd[0]) + if (!fnic->fw_ack_recd[hwq]) return 1; /* * Update desc_available count based on number of freed descriptors * Account for wraparound */ - if (wq->to_clean_index <= fnic->fw_ack_index[0]) - wq->ring.desc_avail += (fnic->fw_ack_index[0] + if (wq->to_clean_index <= fnic->fw_ack_index[hwq]) + wq->ring.desc_avail += (fnic->fw_ack_index[hwq] - wq->to_clean_index + 1); else wq->ring.desc_avail += (wq->ring.desc_count - wq->to_clean_index - + fnic->fw_ack_index[0] + 1); + + fnic->fw_ack_index[hwq] + 1); /* * just bump clean index to ack_index+1 accounting for wraparound @@ -164,15 +210,15 @@ static int free_wq_copy_descs(struct fnic *fnic, struct vnic_wq_copy *wq) * to_clean_index and fw_ack_index, both inclusive */ wq->to_clean_index = - (fnic->fw_ack_index[0] + 1) % wq->ring.desc_count; + (fnic->fw_ack_index[hwq] + 1) % wq->ring.desc_count; /* we have processed the acks received so far */ - fnic->fw_ack_recd[0] = 0; + fnic->fw_ack_recd[hwq] = 0; return 0; } -/** +/* * __fnic_set_state_flags * Sets/Clears bits in fnic's state_flags **/ @@ -180,20 +226,16 @@ void __fnic_set_state_flags(struct fnic *fnic, unsigned long st_flags, unsigned long clearbits) { - struct Scsi_Host *host = fnic->lport->host; - int sh_locked = spin_is_locked(host->host_lock); unsigned long flags = 0; - if (!sh_locked) - spin_lock_irqsave(host->host_lock, flags); + spin_lock_irqsave(&fnic->fnic_lock, flags); if (clearbits) fnic->state_flags &= ~st_flags; else fnic->state_flags |= st_flags; - if (!sh_locked) - spin_unlock_irqrestore(host->host_lock, flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return; } @@ -205,15 +247,14 @@ __fnic_set_state_flags(struct fnic *fnic, unsigned long st_flags, */ int fnic_fw_reset_handler(struct fnic *fnic) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; + struct vnic_wq_copy *wq = &fnic->hw_copy_wq[0]; int ret = 0; unsigned long flags; + unsigned int ioreq_count; /* indicate fwreset to io path */ fnic_set_state_flags(fnic, FNIC_FLAGS_FWRESET); - - skb_queue_purge(&fnic->frame_queue); - skb_queue_purge(&fnic->tx_queue); + ioreq_count = fnic_count_all_ioreqs(fnic); /* wait for io cmpl */ while (atomic_read(&fnic->in_flight)) @@ -222,11 +263,13 @@ int fnic_fw_reset_handler(struct fnic *fnic) spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + free_wq_copy_descs(fnic, wq, 0); if (!vnic_wq_copy_desc_avail(wq)) ret = -EAGAIN; else { + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "ioreq_count: %u\n", ioreq_count); fnic_queue_wq_copy_desc_fw_reset(wq, SCSI_NO_TAG); atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs); if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) > @@ -240,12 +283,12 @@ int fnic_fw_reset_handler(struct fnic *fnic) if (!ret) { atomic64_inc(&fnic->fnic_stats.reset_stats.fw_resets); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Issued fw reset\n"); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Issued fw reset\n"); } else { fnic_clear_state_flags(fnic, FNIC_FLAGS_FWRESET); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Failed to issue fw reset\n"); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "Failed to issue fw reset\n"); } return ret; @@ -258,45 +301,40 @@ int fnic_fw_reset_handler(struct fnic *fnic) */ int fnic_flogi_reg_handler(struct fnic *fnic, u32 fc_id) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; + struct vnic_wq_copy *wq = &fnic->hw_copy_wq[0]; enum fcpio_flogi_reg_format_type format; - struct fc_lport *lp = fnic->lport; u8 gw_mac[ETH_ALEN]; int ret = 0; unsigned long flags; + struct fnic_iport_s *iport = &fnic->iport; spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + free_wq_copy_descs(fnic, wq, 0); if (!vnic_wq_copy_desc_avail(wq)) { ret = -EAGAIN; goto flogi_reg_ioreq_end; } - if (fnic->ctlr.map_dest) { - memset(gw_mac, 0xff, ETH_ALEN); - format = FCPIO_FLOGI_REG_DEF_DEST; - } else { - memcpy(gw_mac, fnic->ctlr.dest_addr, ETH_ALEN); - format = FCPIO_FLOGI_REG_GW_DEST; - } + memcpy(gw_mac, fnic->iport.fcfmac, ETH_ALEN); + format = FCPIO_FLOGI_REG_GW_DEST; - if ((fnic->config.flags & VFCF_FIP_CAPABLE) && !fnic->ctlr.map_dest) { + if (fnic->config.flags & VFCF_FIP_CAPABLE) { fnic_queue_wq_copy_desc_fip_reg(wq, SCSI_NO_TAG, fc_id, gw_mac, - fnic->data_src_addr, - lp->r_a_tov, lp->e_d_tov); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "FLOGI FIP reg issued fcid %x src %pM dest %pM\n", - fc_id, fnic->data_src_addr, gw_mac); + fnic->iport.fpma, + iport->r_a_tov, iport->e_d_tov); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "FLOGI FIP reg issued fcid: 0x%x src %p dest %p\n", + fc_id, fnic->iport.fpma, gw_mac); } else { fnic_queue_wq_copy_desc_flogi_reg(wq, SCSI_NO_TAG, format, fc_id, gw_mac); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "FLOGI reg issued fcid %x map %d dest %pM\n", - fc_id, fnic->ctlr.map_dest, gw_mac); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "FLOGI reg issued fcid 0x%x dest %p\n", + fc_id, gw_mac); } atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs); @@ -318,18 +356,23 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, struct vnic_wq_copy *wq, struct fnic_io_req *io_req, struct scsi_cmnd *sc, - int sg_count) + int sg_count, + uint32_t mqtag, + uint16_t hwq) { struct scatterlist *sg; struct fc_rport *rport = starget_to_rport(scsi_target(sc->device)); - struct fc_rport_libfc_priv *rp = rport->dd_data; struct host_sg_desc *desc; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; unsigned int i; - unsigned long intr_flags; int flags; u8 exch_flags; struct scsi_lun fc_lun; + struct fnic_tport_s *tport; + struct rport_dd_data_s *rdd_data; + + rdd_data = rport->dd_data; + tport = rdd_data->tport; if (sg_count) { /* For each SGE, create a device desc entry */ @@ -366,14 +409,11 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, int_to_scsilun(sc->device->lun, &fc_lun); /* Enqueue the descriptor in the Copy WQ */ - spin_lock_irqsave(&fnic->wq_copy_lock[0], intr_flags); - - if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[hwq]) + free_wq_copy_descs(fnic, wq, hwq); if (unlikely(!vnic_wq_copy_desc_avail(wq))) { - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags); - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, "fnic_queue_wq_copy_desc failure - no descriptors\n"); atomic64_inc(&misc_stats->io_cpwq_alloc_failures); return SCSI_MLQUEUE_HOST_BUSY; @@ -387,10 +427,10 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, exch_flags = 0; if ((fnic->config.flags & VFCF_FCP_SEQ_LVL_ERR) && - (rp->flags & FC_RP_FLAGS_RETRY)) + (tport->tgt_flags & FDLS_FC_RP_FLAGS_RETRY)) exch_flags |= FCPIO_ICMND_SRFLAG_RETRY; - fnic_queue_wq_copy_desc_icmnd_16(wq, sc->request->tag, + fnic_queue_wq_copy_desc_icmnd_16(wq, mqtag, 0, exch_flags, io_req->sgl_cnt, SCSI_SENSE_BUFFERSIZE, io_req->sgl_list_pa, @@ -402,8 +442,8 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, sc->cmnd, sc->cmd_len, scsi_bufflen(sc), fc_lun.scsi_lun, io_req->port_id, - rport->maxframe_size, rp->r_a_tov, - rp->e_d_tov); + tport->max_payload_size, + tport->r_a_tov, tport->e_d_tov); atomic64_inc(&fnic->fnic_stats.fw_stats.active_fw_reqs); if (atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs) > @@ -411,38 +451,34 @@ static inline int fnic_queue_wq_copy_desc(struct fnic *fnic, atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs, atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags); return 0; } -/* - * fnic_queuecommand - * Routine to send a scsi cdb - * Called with host_lock held and interrupts disabled. - */ -static int fnic_queuecommand_lck(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *)) +int fnic_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *sc) { - struct fc_lport *lp = shost_priv(sc->device->host); + struct request *const rq = scsi_cmd_to_rq(sc); + uint32_t mqtag = 0; + void (*done)(struct scsi_cmnd *) = scsi_done; struct fc_rport *rport; struct fnic_io_req *io_req = NULL; - struct fnic *fnic = lport_priv(lp); + struct fnic *fnic = *((struct fnic **) shost_priv(sc->device->host)); + struct fnic_iport_s *iport = NULL; struct fnic_stats *fnic_stats = &fnic->fnic_stats; struct vnic_wq_copy *wq; - int ret; + int ret = 1; u64 cmd_trace; int sg_count = 0; unsigned long flags = 0; unsigned long ptr; - spinlock_t *io_lock = NULL; int io_lock_acquired = 0; - struct fc_rport_libfc_priv *rp; - - if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) - return SCSI_MLQUEUE_HOST_BUSY; + uint16_t hwq = 0; + struct fnic_tport_s *tport = NULL; + struct rport_dd_data_s *rdd_data; + uint16_t lun0_delay = 0; rport = starget_to_rport(scsi_target(sc->device)); if (!rport) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, "returning DID_NO_CONNECT for IO as rport is NULL\n"); sc->result = DID_NO_CONNECT << 16; done(sc); @@ -451,49 +487,98 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc, void (*done)(struct scsi_ ret = fc_remote_port_chkready(rport); if (ret) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, "rport is not ready\n"); - atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); + atomic64_inc(&fnic_stats->misc_stats.tport_not_ready); sc->result = ret; done(sc); return 0; } - rp = rport->dd_data; - if (!rp || rp->rp_state == RPORT_ST_DELETE) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "rport 0x%x removed, returning DID_NO_CONNECT\n", - rport->port_id); + mqtag = blk_mq_unique_tag(rq); + spin_lock_irqsave(&fnic->fnic_lock, flags); + iport = &fnic->iport; - atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); - sc->result = DID_NO_CONNECT<<16; + if (iport->state != FNIC_IPORT_STATE_READY) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "returning DID_NO_CONNECT for IO as iport state: %d\n", + iport->state); + sc->result = DID_NO_CONNECT << 16; done(sc); return 0; } - if (rp->rp_state != RPORT_ST_READY) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "rport 0x%x in state 0x%x, returning DID_IMM_RETRY\n", - rport->port_id, rp->rp_state); + /* fc_remote_port_add() may have added the tport to + * fc_transport but dd_data not yet set + */ + rdd_data = rport->dd_data; + tport = rdd_data->tport; + if (!tport || (rdd_data->iport != iport)) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "dd_data not yet set in SCSI for rport portid: 0x%x\n", + rport->port_id); + tport = fnic_find_tport_by_fcid(iport, rport->port_id); + if (!tport) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "returning DID_BUS_BUSY for IO as tport not found for: 0x%x\n", + rport->port_id); + sc->result = DID_BUS_BUSY << 16; + done(sc); + return 0; + } - sc->result = DID_IMM_RETRY << 16; + /* Re-assign same params as in fnic_fdls_add_tport */ + rport->maxframe_size = FNIC_FC_MAX_PAYLOAD_LEN; + rport->supported_classes = + FC_COS_CLASS3 | FC_RPORT_ROLE_FCP_TARGET; + /* the dd_data is allocated by fctransport of size dd_fcrport_size */ + rdd_data = rport->dd_data; + rdd_data->tport = tport; + rdd_data->iport = iport; + tport->rport = rport; + tport->flags |= FNIC_FDLS_SCSI_REGISTERED; + } + + if ((tport->state != FDLS_TGT_STATE_READY) + && (tport->state != FDLS_TGT_STATE_ADISC)) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "returning DID_NO_CONNECT for IO as tport state: %d\n", + tport->state); + sc->result = DID_NO_CONNECT << 16; done(sc); return 0; } - if (lp->state != LPORT_ST_READY || !(lp->link_up)) + atomic_inc(&fnic->in_flight); + atomic_inc(&tport->in_flight); + + if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { + atomic_dec(&fnic->in_flight); + atomic_dec(&tport->in_flight); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return SCSI_MLQUEUE_HOST_BUSY; + } - atomic_inc(&fnic->in_flight); + if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_FWRESET))) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "fnic flags FW reset: 0x%lx. Returning SCSI_MLQUEUE_HOST_BUSY\n", + fnic->state_flags); + return SCSI_MLQUEUE_HOST_BUSY; + } - /* - * Release host lock, use driver resource specific locks from here. - * Don't re-enable interrupts in case they were disabled prior to the - * caller disabling them. - */ - spin_unlock(lp->host->host_lock); - CMD_STATE(sc) = FNIC_IOREQ_NOT_INITED; - CMD_FLAGS(sc) = FNIC_NO_FLAGS; + if (!tport->lun0_delay) { + lun0_delay = 1; + tport->lun0_delay++; + } + + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + + fnic_priv(sc)->state = FNIC_IOREQ_NOT_INITED; + fnic_priv(sc)->flags = FNIC_NO_FLAGS; /* Get a new io_req for this SCSI IO */ io_req = mempool_alloc(fnic->io_req_pool, GFP_ATOMIC); @@ -508,12 +593,12 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc, void (*done)(struct scsi_ sg_count = scsi_dma_map(sc); if (sg_count < 0) { FNIC_TRACE(fnic_queuecommand, sc->device->host->host_no, - sc->request->tag, sc, 0, sc->cmnd[0], - sg_count, CMD_STATE(sc)); + mqtag, sc, 0, sc->cmnd[0], sg_count, fnic_priv(sc)->state); mempool_free(io_req, fnic->io_req_pool); goto out; } + io_req->tport = tport; /* Determine the type of scatter/gather list we need */ io_req->sgl_cnt = sg_count; io_req->sgl_type = FNIC_SGL_CACHE_DFLT; @@ -544,43 +629,53 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc, void (*done)(struct scsi_ } /* - * Will acquire lock defore setting to IO initialized. + * Will acquire lock before setting to IO initialized. */ - - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); /* initialize rest of io_req */ io_lock_acquired = 1; io_req->port_id = rport->port_id; io_req->start_time = jiffies; - CMD_STATE(sc) = FNIC_IOREQ_CMD_PENDING; - CMD_SP(sc) = (char *)io_req; - CMD_FLAGS(sc) |= FNIC_IO_INITIALIZED; - sc->scsi_done = done; + fnic_priv(sc)->state = FNIC_IOREQ_CMD_PENDING; + fnic_priv(sc)->io_req = io_req; + fnic_priv(sc)->flags |= FNIC_IO_INITIALIZED; + io_req->sc = sc; + + if (fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] != NULL) { + WARN(1, "fnic<%d>: %s: hwq: %d tag 0x%x already exists\n", + fnic->fnic_num, __func__, hwq, blk_mq_unique_tag_to_tag(mqtag)); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return SCSI_MLQUEUE_HOST_BUSY; + } + + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = io_req; + io_req->tag = mqtag; /* create copy wq desc and enqueue it */ - wq = &fnic->wq_copy[0]; - ret = fnic_queue_wq_copy_desc(fnic, wq, io_req, sc, sg_count); + wq = &fnic->hw_copy_wq[hwq]; + atomic64_inc(&fnic_stats->io_stats.ios[hwq]); + ret = fnic_queue_wq_copy_desc(fnic, wq, io_req, sc, sg_count, mqtag, hwq); if (ret) { /* * In case another thread cancelled the request, * refetch the pointer under the lock. */ FNIC_TRACE(fnic_queuecommand, sc->device->host->host_no, - sc->request->tag, sc, 0, 0, 0, - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); - io_req = (struct fnic_io_req *)CMD_SP(sc); - CMD_SP(sc) = NULL; - CMD_STATE(sc) = FNIC_IOREQ_CMD_COMPLETE; - spin_unlock_irqrestore(io_lock, flags); + mqtag, sc, 0, 0, 0, fnic_flags_and_state(sc)); + io_req = fnic_priv(sc)->io_req; + fnic_priv(sc)->io_req = NULL; + if (io_req) + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = NULL; + fnic_priv(sc)->state = FNIC_IOREQ_CMD_COMPLETE; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (io_req) { fnic_release_ioreq_buf(fnic, io_req, sc); mempool_free(io_req, fnic->io_req_pool); } atomic_dec(&fnic->in_flight); - /* acquire host lock before returning to SCSI */ - spin_lock(lp->host->host_lock); + atomic_dec(&tport->in_flight); return ret; } else { atomic64_inc(&fnic_stats->io_stats.active_ios); @@ -591,7 +686,7 @@ static int fnic_queuecommand_lck(struct scsi_cmnd *sc, void (*done)(struct scsi_ atomic64_read(&fnic_stats->io_stats.active_ios)); /* REVISIT: Use per IO lock in the final code */ - CMD_FLAGS(sc) |= FNIC_IO_ISSUED; + fnic_priv(sc)->flags |= FNIC_IO_ISSUED; } out: cmd_trace = ((u64)sc->cmnd[0] << 56 | (u64)sc->cmnd[7] << 40 | @@ -600,21 +695,25 @@ out: sc->cmnd[5]); FNIC_TRACE(fnic_queuecommand, sc->device->host->host_no, - sc->request->tag, sc, io_req, - sg_count, cmd_trace, - (((u64)CMD_FLAGS(sc) >> 32) | CMD_STATE(sc))); + mqtag, sc, io_req, sg_count, cmd_trace, + fnic_flags_and_state(sc)); /* if only we issued IO, will we have the io lock */ if (io_lock_acquired) - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); - /* acquire host lock before returning to SCSI */ - spin_lock(lp->host->host_lock); + atomic_dec(&tport->in_flight); + + if (lun0_delay) { + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "LUN0 delay\n"); + mdelay(LUN0_DELAY_TIME); + } + return ret; } -DEF_SCSI_QCMD(fnic_queuecommand) /* * fnic_fcpio_fw_reset_cmpl_handler @@ -647,52 +746,43 @@ static int fnic_fcpio_fw_reset_cmpl_handler(struct fnic *fnic, if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE) { /* Check status of reset completion */ if (!hdr_status) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "reset cmpl success\n"); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "reset cmpl success\n"); /* Ready to send flogi out */ fnic->state = FNIC_IN_ETH_MODE; } else { - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "fnic fw_reset : failed %s\n", - fnic_fcpio_status_to_str(hdr_status)); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "reset failed with header status: %s\n", + fnic_fcpio_status_to_str(hdr_status)); - /* - * Unable to change to eth mode, cannot send out flogi - * Change state to fc mode, so that subsequent Flogi - * requests from libFC will cause more attempts to - * reset the firmware. Free the cached flogi - */ fnic->state = FNIC_IN_FC_MODE; atomic64_inc(&reset_stats->fw_reset_failures); ret = -1; } } else { - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "Unexpected state %s while processing" - " reset cmpl\n", fnic_state_to_str(fnic->state)); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "Unexpected state while processing reset completion: %s\n", + fnic_state_to_str(fnic->state)); atomic64_inc(&reset_stats->fw_reset_failures); ret = -1; } - /* Thread removing device blocks till firmware reset is complete */ - if (fnic->remove_wait) - complete(fnic->remove_wait); + if (fnic->fw_reset_done) + complete(fnic->fw_reset_done); /* * If fnic is being removed, or fw reset failed * free the flogi frame. Else, send it out */ - if (fnic->remove_wait || ret) { + if (ret) { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - skb_queue_purge(&fnic->tx_queue); + fnic_free_txq(&fnic->tx_queue); goto reset_cmpl_handler_end; } spin_unlock_irqrestore(&fnic->fnic_lock, flags); - fnic_flush_tx(fnic); + queue_work(fnic_event_queue, &fnic->flush_work); reset_cmpl_handler_end: fnic_clear_state_flags(fnic, FNIC_FLAGS_FWRESET); @@ -722,19 +812,19 @@ static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic, /* Check flogi registration completion status */ if (!hdr_status) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "flog reg succeeded\n"); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "FLOGI reg succeeded\n"); fnic->state = FNIC_IN_FC_MODE; } else { FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "fnic flogi reg :failed %s\n", + fnic->host, fnic->fnic_num, + "fnic flogi reg failed: %s\n", fnic_fcpio_status_to_str(hdr_status)); fnic->state = FNIC_IN_ETH_MODE; ret = -1; } } else { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "Unexpected fnic state %s while" " processing flogi reg completion\n", fnic_state_to_str(fnic->state)); @@ -748,7 +838,7 @@ static int fnic_fcpio_flogi_reg_cmpl_handler(struct fnic *fnic, } spin_unlock_irqrestore(&fnic->fnic_lock, flags); - fnic_flush_tx(fnic); + queue_work(fnic_event_queue, &fnic->flush_work); queue_work(fnic_event_queue, &fnic->frame_work); } else { spin_unlock_irqrestore(&fnic->fnic_lock, flags); @@ -791,22 +881,23 @@ static inline void fnic_fcpio_ack_handler(struct fnic *fnic, u16 request_out = desc->u.ack.request_out; unsigned long flags; u64 *ox_id_tag = (u64 *)(void *)desc; + unsigned int wq_index = cq_index; /* mark the ack state */ - wq = &fnic->wq_copy[cq_index - fnic->raw_wq_count - fnic->rq_count]; - spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); + wq = &fnic->hw_copy_wq[cq_index]; + spin_lock_irqsave(&fnic->wq_copy_lock[wq_index], flags); fnic->fnic_stats.misc_stats.last_ack_time = jiffies; if (is_ack_index_in_range(wq, request_out)) { - fnic->fw_ack_index[0] = request_out; - fnic->fw_ack_recd[0] = 1; + fnic->fw_ack_index[wq_index] = request_out; + fnic->fw_ack_recd[wq_index] = 1; } else atomic64_inc( &fnic->fnic_stats.misc_stats.ack_index_out_of_range); - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[wq_index], flags); FNIC_TRACE(fnic_fcpio_ack_handler, - fnic->lport->host->host_no, 0, 0, ox_id_tag[2], ox_id_tag[3], + fnic->host->host_no, 0, 0, ox_id_tag[2], ox_id_tag[3], ox_id_tag[4], ox_id_tag[5]); } @@ -814,12 +905,12 @@ static inline void fnic_fcpio_ack_handler(struct fnic *fnic, * fnic_fcpio_icmnd_cmpl_handler * Routine to handle icmnd completions */ -static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, +static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, unsigned int cq_index, struct fcpio_fw_req *desc) { u8 type; u8 hdr_status; - struct fcpio_tag tag; + struct fcpio_tag ftag; u32 id; u64 xfer_len = 0; struct fcpio_icmnd_cmpl *icmnd_cmpl; @@ -827,33 +918,53 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, struct scsi_cmnd *sc; struct fnic_stats *fnic_stats = &fnic->fnic_stats; unsigned long flags; - spinlock_t *io_lock; u64 cmd_trace; unsigned long start_time; unsigned long io_duration_time; + unsigned int hwq = 0; + unsigned int mqtag = 0; + unsigned int tag = 0; /* Decode the cmpl description to get the io_req id */ - fcpio_header_dec(&desc->hdr, &type, &hdr_status, &tag); - fcpio_tag_id_dec(&tag, &id); + fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag); + fcpio_tag_id_dec(&ftag, &id); icmnd_cmpl = &desc->u.icmnd_cmpl; - if (id >= fnic->fnic_max_tag_id) { - shost_printk(KERN_ERR, fnic->lport->host, - "Tag out of range tag %x hdr status = %s\n", - id, fnic_fcpio_status_to_str(hdr_status)); + mqtag = id; + tag = blk_mq_unique_tag_to_tag(mqtag); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + + if (hwq != cq_index) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hdr status: %s icmnd completion on the wrong queue\n", + fnic_fcpio_status_to_str(hdr_status)); + } + + if (tag >= fnic->fnic_max_tag_id) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hdr status: %s Out of range tag\n", + fnic_fcpio_status_to_str(hdr_status)); return; } + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - sc = scsi_host_find_tag(fnic->lport->host, id); + sc = scsi_host_find_tag(fnic->host, id); WARN_ON_ONCE(!sc); if (!sc) { atomic64_inc(&fnic_stats->io_stats.sc_null); - shost_printk(KERN_ERR, fnic->lport->host, + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + shost_printk(KERN_ERR, fnic->host, "icmnd_cmpl sc is null - " "hdr status = %s tag = 0x%x desc = 0x%p\n", fnic_fcpio_status_to_str(hdr_status), id, desc); FNIC_TRACE(fnic_fcpio_icmnd_cmpl_handler, - fnic->lport->host->host_no, id, + fnic->host->host_no, id, ((u64)icmnd_cmpl->_resvd0[1] << 16 | (u64)icmnd_cmpl->_resvd0[0]), ((u64)hdr_status << 16 | @@ -863,15 +974,20 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, return; } - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + io_req = fnic_priv(sc)->io_req; + if (fnic->sw_copy_wq[hwq].io_req_table[tag] != io_req) { + WARN(1, "%s: %d: hwq: %d mqtag: 0x%x tag: 0x%x io_req tag mismatch\n", + __func__, __LINE__, hwq, mqtag, tag); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return; + } + WARN_ON_ONCE(!io_req); if (!io_req) { atomic64_inc(&fnic_stats->io_stats.ioreq_null); - CMD_FLAGS(sc) |= FNIC_IO_REQ_NULL; - spin_unlock_irqrestore(io_lock, flags); - shost_printk(KERN_ERR, fnic->lport->host, + fnic_priv(sc)->flags |= FNIC_IO_REQ_NULL; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + shost_printk(KERN_ERR, fnic->host, "icmnd_cmpl io_req is null - " "hdr status = %s tag = 0x%x sc 0x%p\n", fnic_fcpio_status_to_str(hdr_status), id, sc); @@ -886,19 +1002,19 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, * if SCSI-ML has already issued abort on this command, * set completion of the IO. The abts path will clean it up */ - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) { + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { /* * set the FNIC_IO_DONE so that this doesn't get * flagged as 'out of order' if it was not aborted */ - CMD_FLAGS(sc) |= FNIC_IO_DONE; - CMD_FLAGS(sc) |= FNIC_IO_ABTS_PENDING; - spin_unlock_irqrestore(io_lock, flags); + fnic_priv(sc)->flags |= FNIC_IO_DONE; + fnic_priv(sc)->flags |= FNIC_IO_ABTS_PENDING; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if(FCPIO_ABORTED == hdr_status) - CMD_FLAGS(sc) |= FNIC_IO_ABORTED; + fnic_priv(sc)->flags |= FNIC_IO_ABORTED; - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, "icmnd_cmpl abts pending " "hdr status = %s tag = 0x%x sc = 0x%p " "scsi_status = %x residual = %d\n", @@ -910,7 +1026,7 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, } /* Mark the IO as complete */ - CMD_STATE(sc) = FNIC_IOREQ_CMD_COMPLETE; + fnic_priv(sc)->state = FNIC_IOREQ_CMD_COMPLETE; icmnd_cmpl = &desc->u.icmnd_cmpl; @@ -918,16 +1034,20 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, case FCPIO_SUCCESS: sc->result = (DID_OK << 16) | icmnd_cmpl->scsi_status; xfer_len = scsi_bufflen(sc); - scsi_set_resid(sc, icmnd_cmpl->residual); - if (icmnd_cmpl->flags & FCPIO_ICMND_CMPL_RESID_UNDER) + if (icmnd_cmpl->flags & FCPIO_ICMND_CMPL_RESID_UNDER) { xfer_len -= icmnd_cmpl->residual; + scsi_set_resid(sc, icmnd_cmpl->residual); + } if (icmnd_cmpl->scsi_status == SAM_STAT_CHECK_CONDITION) atomic64_inc(&fnic_stats->misc_stats.check_condition); if (icmnd_cmpl->scsi_status == SAM_STAT_TASK_SET_FULL) atomic64_inc(&fnic_stats->misc_stats.queue_fulls); + + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "xfer_len: %llu", xfer_len); break; case FCPIO_TIMEOUT: /* request was timed out */ @@ -980,21 +1100,21 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, } /* Break link with the SCSI command */ - CMD_SP(sc) = NULL; - CMD_FLAGS(sc) |= FNIC_IO_DONE; + fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; + fnic_priv(sc)->flags |= FNIC_IO_DONE; + fnic->sw_copy_wq[hwq].io_req_table[tag] = NULL; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (hdr_status != FCPIO_SUCCESS) { atomic64_inc(&fnic_stats->io_stats.io_failures); - shost_printk(KERN_ERR, fnic->lport->host, "hdr status = %s\n", + shost_printk(KERN_ERR, fnic->host, "hdr status = %s\n", fnic_fcpio_status_to_str(hdr_status)); } fnic_release_ioreq_buf(fnic, io_req, sc); - mempool_free(io_req, fnic->io_req_pool); - cmd_trace = ((u64)hdr_status << 56) | (u64)icmnd_cmpl->scsi_status << 48 | (u64)icmnd_cmpl->flags << 40 | (u64)sc->cmnd[0] << 32 | @@ -1006,17 +1126,21 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, ((u64)icmnd_cmpl->_resvd0[1] << 56 | (u64)icmnd_cmpl->_resvd0[0] << 48 | jiffies_to_msecs(jiffies - start_time)), - desc, cmd_trace, - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); + desc, cmd_trace, fnic_flags_and_state(sc)); if (sc->sc_data_direction == DMA_FROM_DEVICE) { - fnic->lport->host_stats.fcp_input_requests++; + fnic_stats->host_stats.fcp_input_requests++; fnic->fcp_input_bytes += xfer_len; } else if (sc->sc_data_direction == DMA_TO_DEVICE) { - fnic->lport->host_stats.fcp_output_requests++; + fnic_stats->host_stats.fcp_output_requests++; fnic->fcp_output_bytes += xfer_len; } else - fnic->lport->host_stats.fcp_control_requests++; + fnic_stats->host_stats.fcp_control_requests++; + + /* Call SCSI completion function to complete the IO */ + scsi_done(sc); + + mempool_free(io_req, fnic->io_req_pool); atomic64_dec(&fnic_stats->io_stats.active_ios); if (atomic64_read(&fnic->io_cmpl_skip)) @@ -1025,7 +1149,8 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, atomic64_inc(&fnic_stats->io_stats.io_completions); - io_duration_time = jiffies_to_msecs(jiffies) - jiffies_to_msecs(io_req->start_time); + io_duration_time = jiffies_to_msecs(jiffies) - + jiffies_to_msecs(start_time); if(io_duration_time <= 10) atomic64_inc(&fnic_stats->io_stats.io_btw_0_to_10_msec); @@ -1045,63 +1170,97 @@ static void fnic_fcpio_icmnd_cmpl_handler(struct fnic *fnic, if(io_duration_time > atomic64_read(&fnic_stats->io_stats.current_max_io_time)) atomic64_set(&fnic_stats->io_stats.current_max_io_time, io_duration_time); } - - /* Call SCSI completion function to complete the IO */ - if (sc->scsi_done) - sc->scsi_done(sc); } /* fnic_fcpio_itmf_cmpl_handler * Routine to handle itmf completions */ -static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, +static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, unsigned int cq_index, struct fcpio_fw_req *desc) { u8 type; u8 hdr_status; - struct fcpio_tag tag; + struct fcpio_tag ftag; u32 id; - struct scsi_cmnd *sc; + struct scsi_cmnd *sc = NULL; struct fnic_io_req *io_req; struct fnic_stats *fnic_stats = &fnic->fnic_stats; struct abort_stats *abts_stats = &fnic->fnic_stats.abts_stats; struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; unsigned long flags; - spinlock_t *io_lock; unsigned long start_time; + unsigned int hwq = cq_index; + unsigned int mqtag; + unsigned int tag; + + fcpio_header_dec(&desc->hdr, &type, &hdr_status, &ftag); + fcpio_tag_id_dec(&ftag, &id); + + mqtag = id & FNIC_TAG_MASK; + tag = blk_mq_unique_tag_to_tag(id & FNIC_TAG_MASK); + hwq = blk_mq_unique_tag_to_hwq(id & FNIC_TAG_MASK); + + if (hwq != cq_index) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hdr status: %s ITMF completion on the wrong queue\n", + fnic_fcpio_status_to_str(hdr_status)); + } + + if (tag > fnic->fnic_max_tag_id) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hdr status: %s Tag out of range\n", + fnic_fcpio_status_to_str(hdr_status)); + return; + } else if ((tag == fnic->fnic_max_tag_id) && !(id & FNIC_TAG_DEV_RST)) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x cq index: %d ", + hwq, mqtag, tag, cq_index); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hdr status: %s Tag out of range\n", + fnic_fcpio_status_to_str(hdr_status)); + return; + } - fcpio_header_dec(&desc->hdr, &type, &hdr_status, &tag); - fcpio_tag_id_dec(&tag, &id); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - if ((id & FNIC_TAG_MASK) >= fnic->fnic_max_tag_id) { - shost_printk(KERN_ERR, fnic->lport->host, - "Tag out of range tag %x hdr status = %s\n", - id, fnic_fcpio_status_to_str(hdr_status)); - return; + /* If it is sg3utils allocated SC then tag_id + * is max_tag_id and SC is retrieved from io_req + */ + if ((mqtag == fnic->fnic_max_tag_id) && (id & FNIC_TAG_DEV_RST)) { + io_req = fnic->sw_copy_wq[hwq].io_req_table[tag]; + if (io_req) + sc = io_req->sc; + } else { + sc = scsi_host_find_tag(fnic->host, id & FNIC_TAG_MASK); } - sc = scsi_host_find_tag(fnic->lport->host, id & FNIC_TAG_MASK); WARN_ON_ONCE(!sc); if (!sc) { atomic64_inc(&fnic_stats->io_stats.sc_null); - shost_printk(KERN_ERR, fnic->lport->host, + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + shost_printk(KERN_ERR, fnic->host, "itmf_cmpl sc is null - hdr status = %s tag = 0x%x\n", - fnic_fcpio_status_to_str(hdr_status), id); + fnic_fcpio_status_to_str(hdr_status), tag); return; } - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + + io_req = fnic_priv(sc)->io_req; WARN_ON_ONCE(!io_req); if (!io_req) { atomic64_inc(&fnic_stats->io_stats.ioreq_null); - spin_unlock_irqrestore(io_lock, flags); - CMD_FLAGS(sc) |= FNIC_IO_ABT_TERM_REQ_NULL; - shost_printk(KERN_ERR, fnic->lport->host, + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_REQ_NULL; + shost_printk(KERN_ERR, fnic->host, "itmf_cmpl io_req is null - " "hdr status = %s tag = 0x%x sc 0x%p\n", - fnic_fcpio_status_to_str(hdr_status), id, sc); + fnic_fcpio_status_to_str(hdr_status), tag, sc); return; } start_time = io_req->start_time; @@ -1109,64 +1268,69 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, if ((id & FNIC_TAG_ABORT) && (id & FNIC_TAG_DEV_RST)) { /* Abort and terminate completion of device reset req */ /* REVISIT : Add asserts about various flags */ - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "dev reset abts cmpl recd. id %x status %s\n", - id, fnic_fcpio_status_to_str(hdr_status)); - CMD_STATE(sc) = FNIC_IOREQ_ABTS_COMPLETE; - CMD_ABTS_STATUS(sc) = hdr_status; - CMD_FLAGS(sc) |= FNIC_DEV_RST_DONE; + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Abt/term completion received\n", + hwq, mqtag, tag, + fnic_fcpio_status_to_str(hdr_status)); + fnic_priv(sc)->state = FNIC_IOREQ_ABTS_COMPLETE; + fnic_priv(sc)->abts_status = hdr_status; + fnic_priv(sc)->flags |= FNIC_DEV_RST_DONE; if (io_req->abts_done) complete(io_req->abts_done); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } else if (id & FNIC_TAG_ABORT) { /* Completion of abort cmd */ + shost_printk(KERN_DEBUG, fnic->host, + "hwq: %d mqtag: 0x%x tag: 0x%x Abort header status: %s\n", + hwq, mqtag, tag, + fnic_fcpio_status_to_str(hdr_status)); switch (hdr_status) { case FCPIO_SUCCESS: break; case FCPIO_TIMEOUT: - if (CMD_FLAGS(sc) & FNIC_IO_ABTS_ISSUED) + if (fnic_priv(sc)->flags & FNIC_IO_ABTS_ISSUED) atomic64_inc(&abts_stats->abort_fw_timeouts); else atomic64_inc( &term_stats->terminate_fw_timeouts); break; case FCPIO_ITMF_REJECTED: - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, "abort reject recd. id %d\n", (int)(id & FNIC_TAG_MASK)); break; case FCPIO_IO_NOT_FOUND: - if (CMD_FLAGS(sc) & FNIC_IO_ABTS_ISSUED) + if (fnic_priv(sc)->flags & FNIC_IO_ABTS_ISSUED) atomic64_inc(&abts_stats->abort_io_not_found); else atomic64_inc( &term_stats->terminate_io_not_found); break; default: - if (CMD_FLAGS(sc) & FNIC_IO_ABTS_ISSUED) + if (fnic_priv(sc)->flags & FNIC_IO_ABTS_ISSUED) atomic64_inc(&abts_stats->abort_failures); else atomic64_inc( &term_stats->terminate_failures); break; } - if (CMD_STATE(sc) != FNIC_IOREQ_ABTS_PENDING) { + if (fnic_priv(sc)->state != FNIC_IOREQ_ABTS_PENDING) { /* This is a late completion. Ignore it */ - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); return; } - CMD_FLAGS(sc) |= FNIC_IO_ABT_TERM_DONE; - CMD_ABTS_STATUS(sc) = hdr_status; + fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_DONE; + fnic_priv(sc)->abts_status = hdr_status; /* If the status is IO not found consider it as success */ if (hdr_status == FCPIO_IO_NOT_FOUND) - CMD_ABTS_STATUS(sc) = FCPIO_SUCCESS; + fnic_priv(sc)->abts_status = FCPIO_SUCCESS; - if (!(CMD_FLAGS(sc) & (FNIC_IO_ABORTED | FNIC_IO_DONE))) + if (!(fnic_priv(sc)->flags & (FNIC_IO_ABORTED | FNIC_IO_DONE))) atomic64_inc(&misc_stats->no_icmnd_itmf_cmpls); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "abts cmpl recd. id %d status %s\n", (int)(id & FNIC_TAG_MASK), fnic_fcpio_status_to_str(hdr_status)); @@ -1178,87 +1342,89 @@ static void fnic_fcpio_itmf_cmpl_handler(struct fnic *fnic, */ if (io_req->abts_done) { complete(io_req->abts_done); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + shost_printk(KERN_INFO, fnic->host, + "hwq: %d mqtag: 0x%x tag: 0x%x Waking up abort thread\n", + hwq, mqtag, tag); } else { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "abts cmpl, completing IO\n"); - CMD_SP(sc) = NULL; + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Completing IO\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); + fnic_priv(sc)->io_req = NULL; sc->result = (DID_ERROR << 16); - - spin_unlock_irqrestore(io_lock, flags); + fnic->sw_copy_wq[hwq].io_req_table[tag] = NULL; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_release_ioreq_buf(fnic, io_req, sc); mempool_free(io_req, fnic->io_req_pool); - if (sc->scsi_done) { - FNIC_TRACE(fnic_fcpio_itmf_cmpl_handler, - sc->device->host->host_no, id, - sc, - jiffies_to_msecs(jiffies - start_time), - desc, - (((u64)hdr_status << 40) | - (u64)sc->cmnd[0] << 32 | - (u64)sc->cmnd[2] << 24 | - (u64)sc->cmnd[3] << 16 | - (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), - (((u64)CMD_FLAGS(sc) << 32) | - CMD_STATE(sc))); - sc->scsi_done(sc); - atomic64_dec(&fnic_stats->io_stats.active_ios); - if (atomic64_read(&fnic->io_cmpl_skip)) - atomic64_dec(&fnic->io_cmpl_skip); - else - atomic64_inc(&fnic_stats->io_stats.io_completions); - } + FNIC_TRACE(fnic_fcpio_itmf_cmpl_handler, + sc->device->host->host_no, id, + sc, + jiffies_to_msecs(jiffies - start_time), + desc, + (((u64)hdr_status << 40) | + (u64)sc->cmnd[0] << 32 | + (u64)sc->cmnd[2] << 24 | + (u64)sc->cmnd[3] << 16 | + (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), + fnic_flags_and_state(sc)); + scsi_done(sc); + atomic64_dec(&fnic_stats->io_stats.active_ios); + if (atomic64_read(&fnic->io_cmpl_skip)) + atomic64_dec(&fnic->io_cmpl_skip); + else + atomic64_inc(&fnic_stats->io_stats.io_completions); } - } else if (id & FNIC_TAG_DEV_RST) { /* Completion of device reset */ - CMD_LR_STATUS(sc) = hdr_status; - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); - CMD_FLAGS(sc) |= FNIC_DEV_RST_ABTS_PENDING; + shost_printk(KERN_INFO, fnic->host, + "hwq: %d mqtag: 0x%x tag: 0x%x DR hst: %s\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); + fnic_priv(sc)->lr_status = hdr_status; + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + fnic_priv(sc)->flags |= FNIC_DEV_RST_ABTS_PENDING; FNIC_TRACE(fnic_fcpio_itmf_cmpl_handler, sc->device->host->host_no, id, sc, jiffies_to_msecs(jiffies - start_time), - desc, 0, - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Terminate pending " - "dev reset cmpl recd. id %d status %s\n", - (int)(id & FNIC_TAG_MASK), - fnic_fcpio_status_to_str(hdr_status)); + desc, 0, fnic_flags_and_state(sc)); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s Terminate pending\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); return; } - if (CMD_FLAGS(sc) & FNIC_DEV_RST_TIMED_OUT) { + if (fnic_priv(sc)->flags & FNIC_DEV_RST_TIMED_OUT) { /* Need to wait for terminate completion */ - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); FNIC_TRACE(fnic_fcpio_itmf_cmpl_handler, sc->device->host->host_no, id, sc, jiffies_to_msecs(jiffies - start_time), - desc, 0, - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + desc, 0, fnic_flags_and_state(sc)); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "dev reset cmpl recd after time out. " "id %d status %s\n", (int)(id & FNIC_TAG_MASK), fnic_fcpio_status_to_str(hdr_status)); return; } - CMD_STATE(sc) = FNIC_IOREQ_CMD_COMPLETE; - CMD_FLAGS(sc) |= FNIC_DEV_RST_DONE; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "dev reset cmpl recd. id %d status %s\n", - (int)(id & FNIC_TAG_MASK), - fnic_fcpio_status_to_str(hdr_status)); + fnic_priv(sc)->state = FNIC_IOREQ_CMD_COMPLETE; + fnic_priv(sc)->flags |= FNIC_DEV_RST_DONE; + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x hst: %s DR completion received\n", + hwq, mqtag, + tag, fnic_fcpio_status_to_str(hdr_status)); if (io_req->dr_done) complete(io_req->dr_done); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } else { - shost_printk(KERN_ERR, fnic->lport->host, - "Unexpected itmf io state %s tag %x\n", - fnic_ioreq_state_to_str(CMD_STATE(sc)), id); - spin_unlock_irqrestore(io_lock, flags); + shost_printk(KERN_ERR, fnic->host, + "%s: Unexpected itmf io state: hwq: %d tag 0x%x %s\n", + __func__, hwq, id, fnic_ioreq_state_to_str(fnic_priv(sc)->state)); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } } @@ -1285,17 +1451,19 @@ static int fnic_fcpio_cmpl_handler(struct vnic_dev *vdev, break; } + cq_index -= fnic->copy_wq_base; + switch (desc->hdr.type) { case FCPIO_ACK: /* fw copied copy wq desc to its queue */ fnic_fcpio_ack_handler(fnic, cq_index, desc); break; case FCPIO_ICMND_CMPL: /* fw completed a command */ - fnic_fcpio_icmnd_cmpl_handler(fnic, desc); + fnic_fcpio_icmnd_cmpl_handler(fnic, cq_index, desc); break; case FCPIO_ITMF_CMPL: /* fw completed itmf (abort cmd, lun reset)*/ - fnic_fcpio_itmf_cmpl_handler(fnic, desc); + fnic_fcpio_itmf_cmpl_handler(fnic, cq_index, desc); break; case FCPIO_FLOGI_REG_CMPL: /* fw completed flogi_reg */ @@ -1308,7 +1476,7 @@ static int fnic_fcpio_cmpl_handler(struct vnic_dev *vdev, break; default: - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "firmware completion type %d\n", desc->hdr.type); break; @@ -1321,104 +1489,156 @@ static int fnic_fcpio_cmpl_handler(struct vnic_dev *vdev, * fnic_wq_copy_cmpl_handler * Routine to process wq copy */ -int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do) +int fnic_wq_copy_cmpl_handler(struct fnic *fnic, int copy_work_to_do, unsigned int cq_index) { - unsigned int wq_work_done = 0; - unsigned int i, cq_index; unsigned int cur_work_done; - - for (i = 0; i < fnic->wq_copy_count; i++) { - cq_index = i + fnic->raw_wq_count + fnic->rq_count; - cur_work_done = vnic_cq_copy_service(&fnic->cq[cq_index], - fnic_fcpio_cmpl_handler, - copy_work_to_do); - wq_work_done += cur_work_done; - } - return wq_work_done; + struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; + u64 start_jiffies = 0; + u64 end_jiffies = 0; + u64 delta_jiffies = 0; + u64 delta_ms = 0; + + start_jiffies = jiffies; + cur_work_done = vnic_cq_copy_service(&fnic->cq[cq_index], + fnic_fcpio_cmpl_handler, + copy_work_to_do); + end_jiffies = jiffies; + delta_jiffies = end_jiffies - start_jiffies; + if (delta_jiffies > (u64) atomic64_read(&misc_stats->max_isr_jiffies)) { + atomic64_set(&misc_stats->max_isr_jiffies, delta_jiffies); + delta_ms = jiffies_to_msecs(delta_jiffies); + atomic64_set(&misc_stats->max_isr_time_ms, delta_ms); + atomic64_set(&misc_stats->corr_work_done, cur_work_done); + } + + return cur_work_done; } -static void fnic_cleanup_io(struct fnic *fnic, int exclude_id) +static bool fnic_cleanup_io_iter(struct scsi_cmnd *sc, void *data) { - int i; + struct request *const rq = scsi_cmd_to_rq(sc); + struct fnic *fnic = data; struct fnic_io_req *io_req; - unsigned long flags = 0; - struct scsi_cmnd *sc; - spinlock_t *io_lock; unsigned long start_time = 0; + unsigned long flags; struct fnic_stats *fnic_stats = &fnic->fnic_stats; + uint16_t hwq = 0; + int tag; + int mqtag; - for (i = 0; i < fnic->fnic_max_tag_id; i++) { - if (i == exclude_id) - continue; - - io_lock = fnic_io_lock_tag(fnic, i); - spin_lock_irqsave(io_lock, flags); - sc = scsi_host_find_tag(fnic->lport->host, i); - if (!sc) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } + mqtag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + tag = blk_mq_unique_tag_to_tag(mqtag); - io_req = (struct fnic_io_req *)CMD_SP(sc); - if ((CMD_FLAGS(sc) & FNIC_DEVICE_RESET) && - !(CMD_FLAGS(sc) & FNIC_DEV_RST_DONE)) { - /* - * We will be here only when FW completes reset - * without sending completions for outstanding ios. - */ - CMD_FLAGS(sc) |= FNIC_DEV_RST_DONE; - if (io_req && io_req->dr_done) - complete(io_req->dr_done); - else if (io_req && io_req->abts_done) - complete(io_req->abts_done); - spin_unlock_irqrestore(io_lock, flags); - continue; - } else if (CMD_FLAGS(sc) & FNIC_DEVICE_RESET) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } - if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); - goto cleanup_scsi_cmd; - } + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - CMD_SP(sc) = NULL; + fnic->sw_copy_wq[hwq].io_req_table[tag] = NULL; - spin_unlock_irqrestore(io_lock, flags); + io_req = fnic_priv(sc)->io_req; + if (!io_req) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d mqtag: 0x%x tag: 0x%x flags: 0x%x No ioreq. Returning\n", + hwq, mqtag, tag, fnic_priv(sc)->flags); + return true; + } + if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && + !(fnic_priv(sc)->flags & FNIC_DEV_RST_DONE)) { /* - * If there is a scsi_cmnd associated with this io_req, then - * free the corresponding state + * We will be here only when FW completes reset + * without sending completions for outstanding ios. */ - start_time = io_req->start_time; - fnic_release_ioreq_buf(fnic, io_req, sc); - mempool_free(io_req, fnic->io_req_pool); + fnic_priv(sc)->flags |= FNIC_DEV_RST_DONE; + if (io_req && io_req->dr_done) + complete(io_req->dr_done); + else if (io_req && io_req->abts_done) + complete(io_req->abts_done); -cleanup_scsi_cmd: - sc->result = DID_TRANSPORT_DISRUPTED << 16; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "%s: sc duration = %lu DID_TRANSPORT_DISRUPTED\n", - __func__, (jiffies - start_time)); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } else if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } - if (atomic64_read(&fnic->io_cmpl_skip)) - atomic64_dec(&fnic->io_cmpl_skip); - else - atomic64_inc(&fnic_stats->io_stats.io_completions); + fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; + start_time = io_req->start_time; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - /* Complete the command to SCSI */ - if (sc->scsi_done) { - FNIC_TRACE(fnic_cleanup_io, - sc->device->host->host_no, i, sc, - jiffies_to_msecs(jiffies - start_time), - 0, ((u64)sc->cmnd[0] << 32 | - (u64)sc->cmnd[2] << 24 | - (u64)sc->cmnd[3] << 16 | - (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); + /* + * If there is a scsi_cmnd associated with this io_req, then + * free the corresponding state + */ + fnic_release_ioreq_buf(fnic, io_req, sc); + mempool_free(io_req, fnic->io_req_pool); + + sc->result = DID_TRANSPORT_DISRUPTED << 16; + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "mqtag: 0x%x tag: 0x%x sc: 0x%p duration = %lu DID_TRANSPORT_DISRUPTED\n", + mqtag, tag, sc, (jiffies - start_time)); + + if (atomic64_read(&fnic->io_cmpl_skip)) + atomic64_dec(&fnic->io_cmpl_skip); + else + atomic64_inc(&fnic_stats->io_stats.io_completions); + + FNIC_TRACE(fnic_cleanup_io, + sc->device->host->host_no, tag, sc, + jiffies_to_msecs(jiffies - start_time), + 0, ((u64) sc->cmnd[0] << 32 | + (u64) sc->cmnd[2] << 24 | + (u64) sc->cmnd[3] << 16 | + (u64) sc->cmnd[4] << 8 | sc->cmnd[5]), + (((u64) fnic_priv(sc)->flags << 32) | fnic_priv(sc)-> + state)); + + /* Complete the command to SCSI */ + scsi_done(sc); + return true; +} + +static void fnic_cleanup_io(struct fnic *fnic, int exclude_id) +{ + unsigned int io_count = 0; + unsigned long flags; + struct fnic_io_req *io_req = NULL; + struct scsi_cmnd *sc = NULL; + + io_count = fnic_count_all_ioreqs(fnic); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "Outstanding ioreq count: %d active io count: %lld Waiting\n", + io_count, + atomic64_read(&fnic->fnic_stats.io_stats.active_ios)); - sc->scsi_done(sc); + scsi_host_busy_iter(fnic->host, + fnic_cleanup_io_iter, fnic); + + /* with sg3utils device reset, SC needs to be retrieved from ioreq */ + spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); + io_req = fnic->sw_copy_wq[0].io_req_table[fnic->fnic_max_tag_id]; + if (io_req) { + sc = io_req->sc; + if (sc) { + if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) + && !(fnic_priv(sc)->flags & FNIC_DEV_RST_DONE)) { + fnic_priv(sc)->flags |= FNIC_DEV_RST_DONE; + if (io_req && io_req->dr_done) + complete(io_req->dr_done); + } } } + spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); + + while ((io_count = fnic_count_all_ioreqs(fnic))) { + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "Outstanding ioreq count: %d active io count: %lld Waiting\n", + io_count, + atomic64_read(&fnic->fnic_stats.io_stats.active_ios)); + + schedule_timeout(msecs_to_jiffies(100)); + } } void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, @@ -1429,8 +1649,8 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, struct fnic_io_req *io_req; struct scsi_cmnd *sc; unsigned long flags; - spinlock_t *io_lock; unsigned long start_time = 0; + uint16_t hwq; /* get the tag reference */ fcpio_tag_id_dec(&desc->hdr.tag, &id); @@ -1439,26 +1659,28 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, if (id >= fnic->fnic_max_tag_id) return; - sc = scsi_host_find_tag(fnic->lport->host, id); + sc = scsi_host_find_tag(fnic->host, id); if (!sc) return; - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); + hwq = blk_mq_unique_tag_to_hwq(id); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); /* Get the IO context which this desc refers to */ - io_req = (struct fnic_io_req *)CMD_SP(sc); + io_req = fnic_priv(sc)->io_req; /* fnic interrupts are turned off by now */ if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto wq_copy_cleanup_scsi_cmd; } - CMD_SP(sc) = NULL; + fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(id)] = NULL; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); start_time = io_req->start_time; fnic_release_ioreq_buf(fnic, io_req, sc); @@ -1466,49 +1688,51 @@ void fnic_wq_copy_cleanup_handler(struct vnic_wq_copy *wq, wq_copy_cleanup_scsi_cmd: sc->result = DID_NO_CONNECT << 16; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "wq_copy_cleanup_handler:" + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "wq_copy_cleanup_handler:" " DID_NO_CONNECT\n"); - if (sc->scsi_done) { - FNIC_TRACE(fnic_wq_copy_cleanup_handler, - sc->device->host->host_no, id, sc, - jiffies_to_msecs(jiffies - start_time), - 0, ((u64)sc->cmnd[0] << 32 | - (u64)sc->cmnd[2] << 24 | (u64)sc->cmnd[3] << 16 | - (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); + FNIC_TRACE(fnic_wq_copy_cleanup_handler, + sc->device->host->host_no, id, sc, + jiffies_to_msecs(jiffies - start_time), + 0, ((u64)sc->cmnd[0] << 32 | + (u64)sc->cmnd[2] << 24 | (u64)sc->cmnd[3] << 16 | + (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), + fnic_flags_and_state(sc)); - sc->scsi_done(sc); - } + scsi_done(sc); } static inline int fnic_queue_abort_io_req(struct fnic *fnic, int tag, u32 task_req, u8 *fc_lun, - struct fnic_io_req *io_req) + struct fnic_io_req *io_req, + unsigned int hwq) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; - struct Scsi_Host *host = fnic->lport->host; + struct vnic_wq_copy *wq = &fnic->hw_copy_wq[hwq]; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; unsigned long flags; + struct fnic_tport_s *tport = io_req->tport; - spin_lock_irqsave(host->host_lock, flags); + spin_lock_irqsave(&fnic->fnic_lock, flags); if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { - spin_unlock_irqrestore(host->host_lock, flags); + atomic_dec(&fnic->in_flight); + atomic_dec(&tport->in_flight); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return 1; } else atomic_inc(&fnic->in_flight); - spin_unlock_irqrestore(host->host_lock, flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); - spin_lock_irqsave(&fnic->wq_copy_lock[0], flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[hwq]) + free_wq_copy_descs(fnic, wq, hwq); if (!vnic_wq_copy_desc_avail(wq)) { - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + atomic_dec(&tport->in_flight); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "fnic_queue_abort_io_req: failure: no descriptors\n"); atomic64_inc(&misc_stats->abts_cpwq_alloc_failures); return 1; @@ -1523,279 +1747,251 @@ static inline int fnic_queue_abort_io_req(struct fnic *fnic, int tag, atomic64_set(&fnic->fnic_stats.fw_stats.max_fw_reqs, atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); return 0; } -static void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id) +struct fnic_rport_abort_io_iter_data { + struct fnic *fnic; + u32 port_id; + int term_cnt; +}; + +static bool fnic_rport_abort_io_iter(struct scsi_cmnd *sc, void *data) { - int tag; - int abt_tag; - int term_cnt = 0; + struct request *const rq = scsi_cmd_to_rq(sc); + struct fnic_rport_abort_io_iter_data *iter_data = data; + struct fnic *fnic = iter_data->fnic; + int abt_tag = 0; struct fnic_io_req *io_req; - spinlock_t *io_lock; - unsigned long flags; - struct scsi_cmnd *sc; struct reset_stats *reset_stats = &fnic->fnic_stats.reset_stats; struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats; struct scsi_lun fc_lun; enum fnic_ioreq_state old_ioreq_state; + uint16_t hwq = 0; + unsigned long flags; - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "fnic_rport_exch_reset called portid 0x%06x\n", - port_id); - - if (fnic->in_remove) - return; - - for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) { - abt_tag = tag; - io_lock = fnic_io_lock_tag(fnic, tag); - spin_lock_irqsave(io_lock, flags); - sc = scsi_host_find_tag(fnic->lport->host, tag); - if (!sc) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } + abt_tag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(abt_tag); - io_req = (struct fnic_io_req *)CMD_SP(sc); + if (!sc) { + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "sc is NULL abt_tag: 0x%x hwq: %d\n", abt_tag, hwq); + return true; + } - if (!io_req || io_req->port_id != port_id) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; + if (!io_req || io_req->port_id != iter_data->port_id) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } - if ((CMD_FLAGS(sc) & FNIC_DEVICE_RESET) && - (!(CMD_FLAGS(sc) & FNIC_DEV_RST_ISSUED))) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_rport_exch_reset dev rst not pending sc 0x%p\n", - sc); - spin_unlock_irqrestore(io_lock, flags); - continue; - } + if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && + !(fnic_priv(sc)->flags & FNIC_DEV_RST_ISSUED)) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d abt_tag: 0x%x flags: 0x%x Device reset is not pending\n", + hwq, abt_tag, fnic_priv(sc)->flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } - /* - * Found IO that is still pending with firmware and - * belongs to rport that went away - */ - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } - if (io_req->abts_done) { - shost_printk(KERN_ERR, fnic->lport->host, - "fnic_rport_exch_reset: io_req->abts_done is set " - "state is %s\n", - fnic_ioreq_state_to_str(CMD_STATE(sc))); - } + /* + * Found IO that is still pending with firmware and + * belongs to rport that went away + */ + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } - if (!(CMD_FLAGS(sc) & FNIC_IO_ISSUED)) { - shost_printk(KERN_ERR, fnic->lport->host, - "rport_exch_reset " - "IO not yet issued %p tag 0x%x flags " - "%x state %d\n", - sc, tag, CMD_FLAGS(sc), CMD_STATE(sc)); - } - old_ioreq_state = CMD_STATE(sc); - CMD_STATE(sc) = FNIC_IOREQ_ABTS_PENDING; - CMD_ABTS_STATUS(sc) = FCPIO_INVALID_CODE; - if (CMD_FLAGS(sc) & FNIC_DEVICE_RESET) { - atomic64_inc(&reset_stats->device_reset_terminates); - abt_tag = (tag | FNIC_TAG_DEV_RST); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_rport_exch_reset dev rst sc 0x%p\n", - sc); - } + if (io_req->abts_done) { + shost_printk(KERN_ERR, fnic->host, + "fnic_rport_exch_reset: io_req->abts_done is set state is %s\n", + fnic_ioreq_state_to_str(fnic_priv(sc)->state)); + } - BUG_ON(io_req->abts_done); + if (!(fnic_priv(sc)->flags & FNIC_IO_ISSUED)) { + shost_printk(KERN_ERR, fnic->host, + "rport_exch_reset IO not yet issued %p abt_tag 0x%x", + sc, abt_tag); + shost_printk(KERN_ERR, fnic->host, + "flags %x state %d\n", fnic_priv(sc)->flags, + fnic_priv(sc)->state); + } + old_ioreq_state = fnic_priv(sc)->state; + fnic_priv(sc)->state = FNIC_IOREQ_ABTS_PENDING; + fnic_priv(sc)->abts_status = FCPIO_INVALID_CODE; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_rport_reset_exch: Issuing abts\n"); + if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) { + atomic64_inc(&reset_stats->device_reset_terminates); + abt_tag |= FNIC_TAG_DEV_RST; + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "dev reset sc 0x%p\n", sc); + } + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "fnic_rport_exch_reset: dev rst sc 0x%p\n", sc); + WARN_ON_ONCE(io_req->abts_done); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "fnic_rport_reset_exch: Issuing abts\n"); - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - /* Now queue the abort command to firmware */ - int_to_scsilun(sc->device->lun, &fc_lun); + /* Queue the abort command to firmware */ + int_to_scsilun(sc->device->lun, &fc_lun); - if (fnic_queue_abort_io_req(fnic, abt_tag, - FCPIO_ITMF_ABT_TASK_TERM, - fc_lun.scsi_lun, io_req)) { - /* - * Revert the cmd state back to old state, if - * it hasn't changed in between. This cmd will get - * aborted later by scsi_eh, or cleaned up during - * lun reset - */ - spin_lock_irqsave(io_lock, flags); - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) - CMD_STATE(sc) = old_ioreq_state; - spin_unlock_irqrestore(io_lock, flags); - } else { - spin_lock_irqsave(io_lock, flags); - if (CMD_FLAGS(sc) & FNIC_DEVICE_RESET) - CMD_FLAGS(sc) |= FNIC_DEV_RST_TERM_ISSUED; - else - CMD_FLAGS(sc) |= FNIC_IO_INTERNAL_TERM_ISSUED; - spin_unlock_irqrestore(io_lock, flags); - atomic64_inc(&term_stats->terminates); - term_cnt++; - } + if (fnic_queue_abort_io_req(fnic, abt_tag, + FCPIO_ITMF_ABT_TASK_TERM, + fc_lun.scsi_lun, io_req, hwq)) { + /* + * Revert the cmd state back to old state, if + * it hasn't changed in between. This cmd will get + * aborted later by scsi_eh, or cleaned up during + * lun reset + */ + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d abt_tag: 0x%x flags: 0x%x Queuing abort failed\n", + hwq, abt_tag, fnic_priv(sc)->flags); + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) + fnic_priv(sc)->state = old_ioreq_state; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + } else { + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) + fnic_priv(sc)->flags |= FNIC_DEV_RST_TERM_ISSUED; + else + fnic_priv(sc)->flags |= FNIC_IO_INTERNAL_TERM_ISSUED; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + atomic64_inc(&term_stats->terminates); + iter_data->term_cnt++; } - if (term_cnt > atomic64_read(&term_stats->max_terminates)) - atomic64_set(&term_stats->max_terminates, term_cnt); + return true; } -void fnic_terminate_rport_io(struct fc_rport *rport) +void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id) { - int tag; - int abt_tag; - int term_cnt = 0; - struct fnic_io_req *io_req; - spinlock_t *io_lock; + unsigned int io_count = 0; unsigned long flags; - struct scsi_cmnd *sc; - struct scsi_lun fc_lun; - struct fc_rport_libfc_priv *rdata; - struct fc_lport *lport; - struct fnic *fnic; - struct fc_rport *cmd_rport; - struct reset_stats *reset_stats; - struct terminate_stats *term_stats; - enum fnic_ioreq_state old_ioreq_state; + struct terminate_stats *term_stats = &fnic->fnic_stats.term_stats; + struct fnic_rport_abort_io_iter_data iter_data = { + .fnic = fnic, + .port_id = port_id, + .term_cnt = 0, + }; - if (!rport) { - printk(KERN_ERR "fnic_terminate_rport_io: rport is NULL\n"); - return; - } - rdata = rport->dd_data; + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "fnic rport exchange reset for tport: 0x%06x\n", + port_id); - if (!rdata) { - printk(KERN_ERR "fnic_terminate_rport_io: rdata is NULL\n"); + if (fnic->in_remove) return; - } - lport = rdata->local_port; - if (!lport) { - printk(KERN_ERR "fnic_terminate_rport_io: lport is NULL\n"); + io_count = fnic_count_ioreqs(fnic, port_id); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "Starting terminates: rport:0x%x portid-io-count: %d active-io-count: %lld\n", + port_id, io_count, + atomic64_read(&fnic->fnic_stats.io_stats.active_ios)); + + spin_lock_irqsave(&fnic->fnic_lock, flags); + /* Bump in_flight counter to hold off fnic_fw_reset_handler. */ + atomic_inc(&fnic->in_flight); + if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { + atomic_dec(&fnic->in_flight); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return; } - fnic = lport_priv(lport); - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, "fnic_terminate_rport_io called" - " wwpn 0x%llx, wwnn0x%llx, rport 0x%p, portid 0x%06x\n", - rport->port_name, rport->node_name, rport, - rport->port_id); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); - if (fnic->in_remove) - return; + scsi_host_busy_iter(fnic->host, fnic_rport_abort_io_iter, + &iter_data); - reset_stats = &fnic->fnic_stats.reset_stats; - term_stats = &fnic->fnic_stats.term_stats; + if (iter_data.term_cnt > atomic64_read(&term_stats->max_terminates)) + atomic64_set(&term_stats->max_terminates, iter_data.term_cnt); - for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) { - abt_tag = tag; - io_lock = fnic_io_lock_tag(fnic, tag); - spin_lock_irqsave(io_lock, flags); - sc = scsi_host_find_tag(fnic->lport->host, tag); - if (!sc) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } + atomic_dec(&fnic->in_flight); - cmd_rport = starget_to_rport(scsi_target(sc->device)); - if (rport != cmd_rport) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } + while ((io_count = fnic_count_ioreqs(fnic, port_id))) + schedule_timeout(msecs_to_jiffies(1000)); - io_req = (struct fnic_io_req *)CMD_SP(sc); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "rport: 0x%x remaining portid-io-count: %d ", + port_id, io_count); +} - if (!io_req || rport != cmd_rport) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } +void fnic_terminate_rport_io(struct fc_rport *rport) +{ + struct fnic_tport_s *tport; + struct rport_dd_data_s *rdd_data; + struct fnic_iport_s *iport = NULL; + struct fnic *fnic = NULL; - if ((CMD_FLAGS(sc) & FNIC_DEVICE_RESET) && - (!(CMD_FLAGS(sc) & FNIC_DEV_RST_ISSUED))) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_terminate_rport_io dev rst not pending sc 0x%p\n", - sc); - spin_unlock_irqrestore(io_lock, flags); - continue; - } - /* - * Found IO that is still pending with firmware and - * belongs to rport that went away - */ - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } - if (io_req->abts_done) { - shost_printk(KERN_ERR, fnic->lport->host, - "fnic_terminate_rport_io: io_req->abts_done is set " - "state is %s\n", - fnic_ioreq_state_to_str(CMD_STATE(sc))); - } - if (!(CMD_FLAGS(sc) & FNIC_IO_ISSUED)) { - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "fnic_terminate_rport_io " - "IO not yet issued %p tag 0x%x flags " - "%x state %d\n", - sc, tag, CMD_FLAGS(sc), CMD_STATE(sc)); - } - old_ioreq_state = CMD_STATE(sc); - CMD_STATE(sc) = FNIC_IOREQ_ABTS_PENDING; - CMD_ABTS_STATUS(sc) = FCPIO_INVALID_CODE; - if (CMD_FLAGS(sc) & FNIC_DEVICE_RESET) { - atomic64_inc(&reset_stats->device_reset_terminates); - abt_tag = (tag | FNIC_TAG_DEV_RST); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_terminate_rport_io dev rst sc 0x%p\n", sc); + if (!rport) { + pr_err("rport is NULL\n"); + return; + } + + rdd_data = rport->dd_data; + if (rdd_data) { + tport = rdd_data->tport; + if (!tport) { + pr_err( + "term rport io called after tport is deleted. Returning 0x%8x\n", + rport->port_id); + } else { + pr_err( + "term rport io called after tport is set 0x%8x\n", + rport->port_id); + pr_err( + "tport maybe rediscovered\n"); + + iport = (struct fnic_iport_s *) tport->iport; + fnic = iport->fnic; + fnic_rport_exch_reset(fnic, rport->port_id); } + } +} - BUG_ON(io_req->abts_done); +/* + * FCP-SCSI specific handling for module unload + * + */ +void fnic_scsi_unload(struct fnic *fnic) +{ + unsigned long flags; - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "fnic_terminate_rport_io: Issuing abts\n"); + /* + * Mark state so that the workqueue thread stops forwarding + * received frames and link events to the local port. ISR and + * other threads that can queue work items will also stop + * creating work items on the fnic workqueue + */ + spin_lock_irqsave(&fnic->fnic_lock, flags); + fnic->iport.state = FNIC_IPORT_STATE_LINK_WAIT; + spin_unlock_irqrestore(&fnic->fnic_lock, flags); - spin_unlock_irqrestore(io_lock, flags); + if (fdls_get_state(&fnic->iport.fabric) != FDLS_STATE_INIT) + fnic_scsi_fcpio_reset(fnic); - /* Now queue the abort command to firmware */ - int_to_scsilun(sc->device->lun, &fc_lun); + spin_lock_irqsave(&fnic->fnic_lock, flags); + fnic->in_remove = 1; + spin_unlock_irqrestore(&fnic->fnic_lock, flags); - if (fnic_queue_abort_io_req(fnic, abt_tag, - FCPIO_ITMF_ABT_TASK_TERM, - fc_lun.scsi_lun, io_req)) { - /* - * Revert the cmd state back to old state, if - * it hasn't changed in between. This cmd will get - * aborted later by scsi_eh, or cleaned up during - * lun reset - */ - spin_lock_irqsave(io_lock, flags); - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) - CMD_STATE(sc) = old_ioreq_state; - spin_unlock_irqrestore(io_lock, flags); - } else { - spin_lock_irqsave(io_lock, flags); - if (CMD_FLAGS(sc) & FNIC_DEVICE_RESET) - CMD_FLAGS(sc) |= FNIC_DEV_RST_TERM_ISSUED; - else - CMD_FLAGS(sc) |= FNIC_IO_INTERNAL_TERM_ISSUED; - spin_unlock_irqrestore(io_lock, flags); - atomic64_inc(&term_stats->terminates); - term_cnt++; - } - } - if (term_cnt > atomic64_read(&term_stats->max_terminates)) - atomic64_set(&term_stats->max_terminates, term_cnt); + fnic_flush_tport_event_list(fnic); + fnic_delete_fcp_tports(fnic); +} +void fnic_scsi_unload_cleanup(struct fnic *fnic) +{ + int hwq = 0; + + fc_remove_host(fnic->host); + scsi_remove_host(fnic->host); + for (hwq = 0; hwq < fnic->wq_copy_count; hwq++) + kfree(fnic->sw_copy_wq[hwq].io_req_table); } /* @@ -1805,11 +2001,13 @@ void fnic_terminate_rport_io(struct fc_rport *rport) */ int fnic_abort_cmd(struct scsi_cmnd *sc) { - struct fc_lport *lp; + struct request *const rq = scsi_cmd_to_rq(sc); + struct fnic_iport_s *iport; + struct fnic_tport_s *tport; struct fnic *fnic; struct fnic_io_req *io_req = NULL; struct fc_rport *rport; - spinlock_t *io_lock; + struct rport_dd_data_s *rdd_data; unsigned long flags; unsigned long start_time = 0; int ret = SUCCESS; @@ -1819,35 +2017,75 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) struct abort_stats *abts_stats; struct terminate_stats *term_stats; enum fnic_ioreq_state old_ioreq_state; - int tag; + int mqtag; unsigned long abt_issued_time; + uint16_t hwq = 0; + DECLARE_COMPLETION_ONSTACK(tm_done); /* Wait for rport to unblock */ fc_block_scsi_eh(sc); /* Get local-port, check ready and link up */ - lp = shost_priv(sc->device->host); + fnic = *((struct fnic **) shost_priv(sc->device->host)); + + spin_lock_irqsave(&fnic->fnic_lock, flags); + iport = &fnic->iport; - fnic = lport_priv(lp); fnic_stats = &fnic->fnic_stats; abts_stats = &fnic->fnic_stats.abts_stats; term_stats = &fnic->fnic_stats.term_stats; rport = starget_to_rport(scsi_target(sc->device)); - tag = sc->request->tag; - FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, - "Abort Cmd called FCID 0x%x, LUN 0x%llx TAG %x flags %x\n", - rport->port_id, sc->device->lun, tag, CMD_FLAGS(sc)); + mqtag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(mqtag); + + fnic_priv(sc)->flags = FNIC_NO_FLAGS; + + rdd_data = rport->dd_data; + tport = rdd_data->tport; + + if (!tport) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "Abort cmd called after tport delete! rport fcid: 0x%x", + rport->port_id); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "lun: %llu hwq: 0x%x mqtag: 0x%x Op: 0x%x flags: 0x%x\n", + sc->device->lun, hwq, mqtag, + sc->cmnd[0], fnic_priv(sc)->flags); + ret = FAILED; + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + goto fnic_abort_cmd_end; + } + + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Abort cmd called rport fcid: 0x%x lun: %llu hwq: 0x%x mqtag: 0x%x", + rport->port_id, sc->device->lun, hwq, mqtag); - CMD_FLAGS(sc) = FNIC_NO_FLAGS; + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Op: 0x%x flags: 0x%x\n", + sc->cmnd[0], + fnic_priv(sc)->flags); - if (lp->state != LPORT_ST_READY || !(lp->link_up)) { + if (iport->state != FNIC_IPORT_STATE_READY) { + atomic64_inc(&fnic_stats->misc_stats.iport_not_ready); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "iport NOT in READY state"); ret = FAILED; + spin_unlock_irqrestore(&fnic->fnic_lock, flags); goto fnic_abort_cmd_end; } + if ((tport->state != FDLS_TGT_STATE_READY) && + (tport->state != FDLS_TGT_STATE_ADISC)) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "tport state: %d\n", tport->state); + ret = FAILED; + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + goto fnic_abort_cmd_end; + } + + spin_unlock_irqrestore(&fnic->fnic_lock, flags); /* * Avoid a race between SCSI issuing the abort and the device * completing the command. @@ -1858,20 +2096,20 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) * happened, the completion wont actually complete the command * and it will be considered as an aborted command * - * The CMD_SP will not be cleared except while holding io_req_lock. + * .io_req will not be cleared except while holding io_req_lock. */ - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + ret = FAILED; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto fnic_abort_cmd_end; } io_req->abts_done = &tm_done; - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto wait_pending; } @@ -1891,19 +2129,20 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) else atomic64_inc(&abts_stats->abort_issued_greater_than_60_sec); - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "CBD Opcode: %02x Abort issued time: %lu msec\n", sc->cmnd[0], abt_issued_time); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "CDB Opcode: 0x%02x Abort issued time: %lu msec\n", + sc->cmnd[0], abt_issued_time); /* * Command is still pending, need to abort it * If the firmware completes the command after this point, * the completion wont be done till mid-layer, since abort * has already started. */ - old_ioreq_state = CMD_STATE(sc); - CMD_STATE(sc) = FNIC_IOREQ_ABTS_PENDING; - CMD_ABTS_STATUS(sc) = FCPIO_INVALID_CODE; + old_ioreq_state = fnic_priv(sc)->state; + fnic_priv(sc)->state = FNIC_IOREQ_ABTS_PENDING; + fnic_priv(sc)->abts_status = FCPIO_INVALID_CODE; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); /* * Check readiness of the remote port. If the path to remote @@ -1913,30 +2152,30 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) if (fc_remote_port_chkready(rport) == 0) task_req = FCPIO_ITMF_ABT_TASK; else { - atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); + atomic64_inc(&fnic_stats->misc_stats.tport_not_ready); task_req = FCPIO_ITMF_ABT_TASK_TERM; } /* Now queue the abort command to firmware */ int_to_scsilun(sc->device->lun, &fc_lun); - if (fnic_queue_abort_io_req(fnic, sc->request->tag, task_req, - fc_lun.scsi_lun, io_req)) { - spin_lock_irqsave(io_lock, flags); - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) - CMD_STATE(sc) = old_ioreq_state; - io_req = (struct fnic_io_req *)CMD_SP(sc); + if (fnic_queue_abort_io_req(fnic, mqtag, task_req, fc_lun.scsi_lun, + io_req, hwq)) { + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) + fnic_priv(sc)->state = old_ioreq_state; + io_req = fnic_priv(sc)->io_req; if (io_req) io_req->abts_done = NULL; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); ret = FAILED; goto fnic_abort_cmd_end; } if (task_req == FCPIO_ITMF_ABT_TASK) { - CMD_FLAGS(sc) |= FNIC_IO_ABTS_ISSUED; + fnic_priv(sc)->flags |= FNIC_IO_ABTS_ISSUED; atomic64_inc(&fnic_stats->abts_stats.aborts); } else { - CMD_FLAGS(sc) |= FNIC_IO_TERM_ISSUED; + fnic_priv(sc)->flags |= FNIC_IO_TERM_ISSUED; atomic64_inc(&fnic_stats->term_stats.terminates); } @@ -1952,43 +2191,43 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) fnic->config.ed_tov)); /* Check the abort status */ - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + io_req = fnic_priv(sc)->io_req; if (!io_req) { atomic64_inc(&fnic_stats->io_stats.ioreq_null); - spin_unlock_irqrestore(io_lock, flags); - CMD_FLAGS(sc) |= FNIC_IO_ABT_TERM_REQ_NULL; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_REQ_NULL; ret = FAILED; goto fnic_abort_cmd_end; } io_req->abts_done = NULL; /* fw did not complete abort, timed out */ - if (CMD_ABTS_STATUS(sc) == FCPIO_INVALID_CODE) { - spin_unlock_irqrestore(io_lock, flags); + if (fnic_priv(sc)->abts_status == FCPIO_INVALID_CODE) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (task_req == FCPIO_ITMF_ABT_TASK) { atomic64_inc(&abts_stats->abort_drv_timeouts); } else { atomic64_inc(&term_stats->terminate_drv_timeouts); } - CMD_FLAGS(sc) |= FNIC_IO_ABT_TERM_TIMED_OUT; + fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_TIMED_OUT; ret = FAILED; goto fnic_abort_cmd_end; } /* IO out of order */ - if (!(CMD_FLAGS(sc) & (FNIC_IO_ABORTED | FNIC_IO_DONE))) { - spin_unlock_irqrestore(io_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Issuing Host reset due to out of order IO\n"); + if (!(fnic_priv(sc)->flags & (FNIC_IO_ABORTED | FNIC_IO_DONE))) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "Issuing host reset due to out of order IO\n"); ret = FAILED; goto fnic_abort_cmd_end; } - CMD_STATE(sc) = FNIC_IOREQ_ABTS_COMPLETE; + fnic_priv(sc)->state = FNIC_IOREQ_ABTS_COMPLETE; start_time = io_req->start_time; /* @@ -1996,40 +2235,40 @@ int fnic_abort_cmd(struct scsi_cmnd *sc) * free the io_req if successful. If abort fails, * Device reset will clean the I/O. */ - if (CMD_ABTS_STATUS(sc) == FCPIO_SUCCESS) - CMD_SP(sc) = NULL; - else { + if (fnic_priv(sc)->abts_status == FCPIO_SUCCESS || + (fnic_priv(sc)->abts_status == FCPIO_ABORTED)) { + fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; + } else { ret = FAILED; - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto fnic_abort_cmd_end; } - spin_unlock_irqrestore(io_lock, flags); + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = NULL; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); fnic_release_ioreq_buf(fnic, io_req, sc); mempool_free(io_req, fnic->io_req_pool); - if (sc->scsi_done) { /* Call SCSI completion function to complete the IO */ - sc->result = (DID_ABORT << 16); - sc->scsi_done(sc); - atomic64_dec(&fnic_stats->io_stats.active_ios); - if (atomic64_read(&fnic->io_cmpl_skip)) - atomic64_dec(&fnic->io_cmpl_skip); - else - atomic64_inc(&fnic_stats->io_stats.io_completions); - } + sc->result = DID_ABORT << 16; + scsi_done(sc); + atomic64_dec(&fnic_stats->io_stats.active_ios); + if (atomic64_read(&fnic->io_cmpl_skip)) + atomic64_dec(&fnic->io_cmpl_skip); + else + atomic64_inc(&fnic_stats->io_stats.io_completions); fnic_abort_cmd_end: - FNIC_TRACE(fnic_abort_cmd, sc->device->host->host_no, - sc->request->tag, sc, + FNIC_TRACE(fnic_abort_cmd, sc->device->host->host_no, mqtag, sc, jiffies_to_msecs(jiffies - start_time), 0, ((u64)sc->cmnd[0] << 32 | (u64)sc->cmnd[2] << 24 | (u64)sc->cmnd[3] << 16 | (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); + fnic_flags_and_state(sc)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "Returning from abort cmd type %x %s\n", task_req, (ret == SUCCESS) ? "SUCCESS" : "FAILED"); @@ -2040,29 +2279,37 @@ static inline int fnic_queue_dr_io_req(struct fnic *fnic, struct scsi_cmnd *sc, struct fnic_io_req *io_req) { - struct vnic_wq_copy *wq = &fnic->wq_copy[0]; - struct Scsi_Host *host = fnic->lport->host; + struct vnic_wq_copy *wq; struct misc_stats *misc_stats = &fnic->fnic_stats.misc_stats; struct scsi_lun fc_lun; int ret = 0; - unsigned long intr_flags; + unsigned long flags; + uint16_t hwq = 0; + uint32_t tag = 0; + struct fnic_tport_s *tport = io_req->tport; - spin_lock_irqsave(host->host_lock, intr_flags); + tag = io_req->tag; + hwq = blk_mq_unique_tag_to_hwq(tag); + wq = &fnic->hw_copy_wq[hwq]; + + spin_lock_irqsave(&fnic->fnic_lock, flags); if (unlikely(fnic_chk_state_flags_locked(fnic, FNIC_FLAGS_IO_BLOCKED))) { - spin_unlock_irqrestore(host->host_lock, intr_flags); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); return FAILED; - } else + } else { atomic_inc(&fnic->in_flight); - spin_unlock_irqrestore(host->host_lock, intr_flags); + atomic_inc(&tport->in_flight); + } + spin_unlock_irqrestore(&fnic->fnic_lock, flags); - spin_lock_irqsave(&fnic->wq_copy_lock[0], intr_flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); - if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[0]) - free_wq_copy_descs(fnic, wq); + if (vnic_wq_copy_desc_avail(wq) <= fnic->wq_copy_desc_low[hwq]) + free_wq_copy_descs(fnic, wq, hwq); if (!vnic_wq_copy_desc_avail(wq)) { - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "queue_dr_io_req failure - no descriptors\n"); atomic64_inc(&misc_stats->devrst_cpwq_alloc_failures); ret = -EAGAIN; @@ -2072,7 +2319,8 @@ static inline int fnic_queue_dr_io_req(struct fnic *fnic, /* fill in the lun info */ int_to_scsilun(sc->device->lun, &fc_lun); - fnic_queue_wq_copy_desc_itmf(wq, sc->request->tag | FNIC_TAG_DEV_RST, + tag |= FNIC_TAG_DEV_RST; + fnic_queue_wq_copy_desc_itmf(wq, tag, 0, FCPIO_ITMF_LUN_RESET, SCSI_NO_TAG, fc_lun.scsi_lun, io_req->port_id, fnic->config.ra_tov, fnic->config.ed_tov); @@ -2084,213 +2332,203 @@ static inline int fnic_queue_dr_io_req(struct fnic *fnic, atomic64_read(&fnic->fnic_stats.fw_stats.active_fw_reqs)); lr_io_req_end: - spin_unlock_irqrestore(&fnic->wq_copy_lock[0], intr_flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); atomic_dec(&fnic->in_flight); + atomic_dec(&tport->in_flight); return ret; } -/* - * Clean up any pending aborts on the lun - * For each outstanding IO on this lun, whose abort is not completed by fw, - * issue a local abort. Wait for abort to complete. Return 0 if all commands - * successfully aborted, 1 otherwise - */ -static int fnic_clean_pending_aborts(struct fnic *fnic, - struct scsi_cmnd *lr_sc, - bool new_sc) +struct fnic_pending_aborts_iter_data { + struct fnic *fnic; + struct scsi_cmnd *lr_sc; + struct scsi_device *lun_dev; + int ret; +}; +static bool fnic_pending_aborts_iter(struct scsi_cmnd *sc, void *data) { - int tag, abt_tag; + struct request *const rq = scsi_cmd_to_rq(sc); + struct fnic_pending_aborts_iter_data *iter_data = data; + struct fnic *fnic = iter_data->fnic; + struct scsi_device *lun_dev = iter_data->lun_dev; + unsigned long abt_tag = 0; + uint16_t hwq = 0; struct fnic_io_req *io_req; - spinlock_t *io_lock; unsigned long flags; - int ret = 0; - struct scsi_cmnd *sc; struct scsi_lun fc_lun; - struct scsi_device *lun_dev = lr_sc->device; DECLARE_COMPLETION_ONSTACK(tm_done); enum fnic_ioreq_state old_ioreq_state; - for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) { - io_lock = fnic_io_lock_tag(fnic, tag); - spin_lock_irqsave(io_lock, flags); - sc = scsi_host_find_tag(fnic->lport->host, tag); - /* - * ignore this lun reset cmd if issued using new SC - * or cmds that do not belong to this lun - */ - if (!sc || ((sc == lr_sc) && new_sc) || sc->device != lun_dev) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } + if (sc == iter_data->lr_sc || sc->device != lun_dev) + return true; - io_req = (struct fnic_io_req *)CMD_SP(sc); + abt_tag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(abt_tag); - if (!io_req || sc->device != lun_dev) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; + if (!io_req) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } - /* - * Found IO that is still pending with firmware and - * belongs to the LUN that we are resetting - */ - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Found IO in %s on lun\n", - fnic_ioreq_state_to_str(CMD_STATE(sc))); + /* + * Found IO that is still pending with firmware and + * belongs to the LUN that we are resetting + */ + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "Found IO in %s on lun\n", + fnic_ioreq_state_to_str(fnic_priv(sc)->state)); + + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } + if ((fnic_priv(sc)->flags & FNIC_DEVICE_RESET) && + (!(fnic_priv(sc)->flags & FNIC_DEV_RST_ISSUED))) { + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "dev rst not pending sc 0x%p\n", sc); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; + } + + if (io_req->abts_done) + shost_printk(KERN_ERR, fnic->host, + "%s: io_req->abts_done is set state is %s\n", + __func__, fnic_ioreq_state_to_str(fnic_priv(sc)->state)); + old_ioreq_state = fnic_priv(sc)->state; + /* + * Any pending IO issued prior to reset is expected to be + * in abts pending state, if not we need to set + * FNIC_IOREQ_ABTS_PENDING to indicate the IO is abort pending. + * When IO is completed, the IO will be handed over and + * handled in this function. + */ + fnic_priv(sc)->state = FNIC_IOREQ_ABTS_PENDING; - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) { - spin_unlock_irqrestore(io_lock, flags); - continue; - } - if ((CMD_FLAGS(sc) & FNIC_DEVICE_RESET) && - (!(CMD_FLAGS(sc) & FNIC_DEV_RST_ISSUED))) { - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "%s dev rst not pending sc 0x%p\n", __func__, - sc); - spin_unlock_irqrestore(io_lock, flags); - continue; - } + BUG_ON(io_req->abts_done); - if (io_req->abts_done) - shost_printk(KERN_ERR, fnic->lport->host, - "%s: io_req->abts_done is set state is %s\n", - __func__, fnic_ioreq_state_to_str(CMD_STATE(sc))); - old_ioreq_state = CMD_STATE(sc); - /* - * Any pending IO issued prior to reset is expected to be - * in abts pending state, if not we need to set - * FNIC_IOREQ_ABTS_PENDING to indicate the IO is abort pending. - * When IO is completed, the IO will be handed over and - * handled in this function. - */ - CMD_STATE(sc) = FNIC_IOREQ_ABTS_PENDING; + if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) { + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "dev rst sc 0x%p\n", sc); + } - BUG_ON(io_req->abts_done); + fnic_priv(sc)->abts_status = FCPIO_INVALID_CODE; + io_req->abts_done = &tm_done; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - abt_tag = tag; - if (CMD_FLAGS(sc) & FNIC_DEVICE_RESET) { - abt_tag |= FNIC_TAG_DEV_RST; - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "%s: dev rst sc 0x%p\n", __func__, sc); - } + /* Now queue the abort command to firmware */ + int_to_scsilun(sc->device->lun, &fc_lun); - CMD_ABTS_STATUS(sc) = FCPIO_INVALID_CODE; - io_req->abts_done = &tm_done; - spin_unlock_irqrestore(io_lock, flags); + if (fnic_queue_abort_io_req(fnic, abt_tag, + FCPIO_ITMF_ABT_TASK_TERM, + fc_lun.scsi_lun, io_req, hwq)) { + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; + if (io_req) + io_req->abts_done = NULL; + if (fnic_priv(sc)->state == FNIC_IOREQ_ABTS_PENDING) + fnic_priv(sc)->state = old_ioreq_state; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + iter_data->ret = FAILED; + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "hwq: %d abt_tag: 0x%lx Abort could not be queued\n", + hwq, abt_tag); + return false; + } else { + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + if (fnic_priv(sc)->flags & FNIC_DEVICE_RESET) + fnic_priv(sc)->flags |= FNIC_DEV_RST_TERM_ISSUED; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + } + fnic_priv(sc)->flags |= FNIC_IO_INTERNAL_TERM_ISSUED; - /* Now queue the abort command to firmware */ - int_to_scsilun(sc->device->lun, &fc_lun); + wait_for_completion_timeout(&tm_done, msecs_to_jiffies + (fnic->config.ed_tov)); - if (fnic_queue_abort_io_req(fnic, abt_tag, - FCPIO_ITMF_ABT_TASK_TERM, - fc_lun.scsi_lun, io_req)) { - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); - if (io_req) - io_req->abts_done = NULL; - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) - CMD_STATE(sc) = old_ioreq_state; - spin_unlock_irqrestore(io_lock, flags); - ret = 1; - goto clean_pending_aborts_end; - } else { - spin_lock_irqsave(io_lock, flags); - if (CMD_FLAGS(sc) & FNIC_DEVICE_RESET) - CMD_FLAGS(sc) |= FNIC_DEV_RST_TERM_ISSUED; - spin_unlock_irqrestore(io_lock, flags); - } - CMD_FLAGS(sc) |= FNIC_IO_INTERNAL_TERM_ISSUED; + /* Recheck cmd state to check if it is now aborted */ + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; + if (!io_req) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_REQ_NULL; + return true; + } - wait_for_completion_timeout(&tm_done, - msecs_to_jiffies - (fnic->config.ed_tov)); + io_req->abts_done = NULL; - /* Recheck cmd state to check if it is now aborted */ - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); - if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); - CMD_FLAGS(sc) |= FNIC_IO_ABT_TERM_REQ_NULL; - continue; - } + /* if abort is still pending with fw, fail */ + if (fnic_priv(sc)->abts_status == FCPIO_INVALID_CODE) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + fnic_priv(sc)->flags |= FNIC_IO_ABT_TERM_DONE; + iter_data->ret = FAILED; + return false; + } + fnic_priv(sc)->state = FNIC_IOREQ_ABTS_COMPLETE; - io_req->abts_done = NULL; + /* original sc used for lr is handled by dev reset code */ + if (sc != iter_data->lr_sc) { + fnic_priv(sc)->io_req = NULL; + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(abt_tag)] = NULL; + } + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - /* if abort is still pending with fw, fail */ - if (CMD_ABTS_STATUS(sc) == FCPIO_INVALID_CODE) { - spin_unlock_irqrestore(io_lock, flags); - CMD_FLAGS(sc) |= FNIC_IO_ABT_TERM_DONE; - ret = 1; - goto clean_pending_aborts_end; - } - CMD_STATE(sc) = FNIC_IOREQ_ABTS_COMPLETE; + /* original sc used for lr is handled by dev reset code */ + if (sc != iter_data->lr_sc) { + fnic_release_ioreq_buf(fnic, io_req, sc); + mempool_free(io_req, fnic->io_req_pool); + } - /* original sc used for lr is handled by dev reset code */ - if (sc != lr_sc) - CMD_SP(sc) = NULL; - spin_unlock_irqrestore(io_lock, flags); + /* + * Any IO is returned during reset, it needs to call scsi_done + * to return the scsi_cmnd to upper layer. + */ + /* Set result to let upper SCSI layer retry */ + sc->result = DID_RESET << 16; + scsi_done(sc); - /* original sc used for lr is handled by dev reset code */ - if (sc != lr_sc) { - fnic_release_ioreq_buf(fnic, io_req, sc); - mempool_free(io_req, fnic->io_req_pool); - } + return true; +} - /* - * Any IO is returned during reset, it needs to call scsi_done - * to return the scsi_cmnd to upper layer. - */ - if (sc->scsi_done) { - /* Set result to let upper SCSI layer retry */ - sc->result = DID_RESET << 16; - sc->scsi_done(sc); - } - } +/* + * Clean up any pending aborts on the lun + * For each outstanding IO on this lun, whose abort is not completed by fw, + * issue a local abort. Wait for abort to complete. Return 0 if all commands + * successfully aborted, 1 otherwise + */ +static int fnic_clean_pending_aborts(struct fnic *fnic, + struct scsi_cmnd *lr_sc, + bool new_sc) +{ + int ret = 0; + struct fnic_pending_aborts_iter_data iter_data = { + .fnic = fnic, + .lun_dev = lr_sc->device, + .ret = SUCCESS, + }; + + iter_data.lr_sc = lr_sc; + + scsi_host_busy_iter(fnic->host, + fnic_pending_aborts_iter, &iter_data); + if (iter_data.ret == FAILED) { + ret = iter_data.ret; + goto clean_pending_aborts_end; + } schedule_timeout(msecs_to_jiffies(2 * fnic->config.ed_tov)); /* walk again to check, if IOs are still pending in fw */ if (fnic_is_abts_pending(fnic, lr_sc)) - ret = FAILED; + ret = 1; clean_pending_aborts_end: + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "exit status: %d\n", ret); return ret; } -/** - * fnic_scsi_host_start_tag - * Allocates tagid from host's tag list - **/ -static inline int -fnic_scsi_host_start_tag(struct fnic *fnic, struct scsi_cmnd *sc) -{ - struct request_queue *q = sc->request->q; - struct request *dummy; - - dummy = blk_mq_alloc_request(q, REQ_OP_WRITE, BLK_MQ_REQ_NOWAIT); - if (IS_ERR(dummy)) - return SCSI_NO_TAG; - - sc->tag = sc->request->tag = dummy->tag; - sc->host_scribble = (unsigned char *)dummy; - - return dummy->tag; -} - -/** - * fnic_scsi_host_end_tag - * frees tag allocated by fnic_scsi_host_start_tag. - **/ -static inline void -fnic_scsi_host_end_tag(struct fnic *fnic, struct scsi_cmnd *sc) -{ - struct request *dummy = (struct request *)sc->host_scribble; - - blk_mq_free_request(dummy); -} - /* * SCSI Eh thread issues a Lun Reset when one or more commands on a LUN * fail to get aborted. It calls driver's eh_device_reset with a SCSI command @@ -2298,67 +2536,102 @@ fnic_scsi_host_end_tag(struct fnic *fnic, struct scsi_cmnd *sc) */ int fnic_device_reset(struct scsi_cmnd *sc) { - struct fc_lport *lp; + struct request *rq = scsi_cmd_to_rq(sc); struct fnic *fnic; struct fnic_io_req *io_req = NULL; struct fc_rport *rport; int status; + int count = 0; int ret = FAILED; - spinlock_t *io_lock; unsigned long flags; unsigned long start_time = 0; struct scsi_lun fc_lun; struct fnic_stats *fnic_stats; struct reset_stats *reset_stats; - int tag = 0; + int mqtag = rq->tag; DECLARE_COMPLETION_ONSTACK(tm_done); - int tag_gen_flag = 0; /*to track tags allocated by fnic driver*/ bool new_sc = 0; + uint16_t hwq = 0; + struct fnic_iport_s *iport = NULL; + struct rport_dd_data_s *rdd_data; + struct fnic_tport_s *tport; + u32 old_soft_reset_count; + u32 old_link_down_cnt; + int exit_dr = 0; /* Wait for rport to unblock */ fc_block_scsi_eh(sc); /* Get local-port, check ready and link up */ - lp = shost_priv(sc->device->host); + fnic = *((struct fnic **) shost_priv(sc->device->host)); + iport = &fnic->iport; - fnic = lport_priv(lp); fnic_stats = &fnic->fnic_stats; - reset_stats = &fnic->fnic_stats.reset_stats; + reset_stats = &fnic_stats->reset_stats; atomic64_inc(&reset_stats->device_resets); rport = starget_to_rport(scsi_target(sc->device)); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Device reset called FCID 0x%x, LUN 0x%llx sc 0x%p\n", - rport->port_id, sc->device->lun, sc); - if (lp->state != LPORT_ST_READY || !(lp->link_up)) + spin_lock_irqsave(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "fcid: 0x%x lun: %llu hwq: %d mqtag: 0x%x flags: 0x%x Device reset\n", + rport->port_id, sc->device->lun, hwq, mqtag, + fnic_priv(sc)->flags); + + rdd_data = rport->dd_data; + tport = rdd_data->tport; + if (!tport) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "Dev rst called after tport delete! rport fcid: 0x%x lun: %llu\n", + rport->port_id, sc->device->lun); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + goto fnic_device_reset_end; + } + + if (iport->state != FNIC_IPORT_STATE_READY) { + atomic64_inc(&fnic_stats->misc_stats.iport_not_ready); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "iport NOT in READY state"); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); goto fnic_device_reset_end; + } + + if ((tport->state != FDLS_TGT_STATE_READY) && + (tport->state != FDLS_TGT_STATE_ADISC)) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "tport state: %d\n", tport->state); + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + goto fnic_device_reset_end; + } + spin_unlock_irqrestore(&fnic->fnic_lock, flags); /* Check if remote port up */ if (fc_remote_port_chkready(rport)) { - atomic64_inc(&fnic_stats->misc_stats.rport_not_ready); + atomic64_inc(&fnic_stats->misc_stats.tport_not_ready); goto fnic_device_reset_end; } - CMD_FLAGS(sc) = FNIC_DEVICE_RESET; - /* Allocate tag if not present */ + fnic_priv(sc)->flags = FNIC_DEVICE_RESET; - tag = sc->request->tag; - if (unlikely(tag < 0)) { + if (unlikely(mqtag < 0)) { /* - * Really should fix the midlayer to pass in a proper - * request for ioctls... + * For device reset issued through sg3utils, we let + * only one LUN_RESET to go through and use a special + * tag equal to max_tag_id so that we don't have to allocate + * or free it. It won't interact with tags + * allocated by mid layer. */ - tag = fnic_scsi_host_start_tag(fnic, sc); - if (unlikely(tag == SCSI_NO_TAG)) - goto fnic_device_reset_end; - tag_gen_flag = 1; + mutex_lock(&fnic->sgreset_mutex); + mqtag = fnic->fnic_max_tag_id; new_sc = 1; + } else { + mqtag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(mqtag); } - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; /* * If there is a io_req attached to this command, then use it, @@ -2367,34 +2640,49 @@ int fnic_device_reset(struct scsi_cmnd *sc) if (!io_req) { io_req = mempool_alloc(fnic->io_req_pool, GFP_ATOMIC); if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); goto fnic_device_reset_end; } memset(io_req, 0, sizeof(*io_req)); io_req->port_id = rport->port_id; - CMD_SP(sc) = (char *)io_req; + io_req->tag = mqtag; + fnic_priv(sc)->io_req = io_req; + io_req->tport = tport; + io_req->sc = sc; + + if (fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] != NULL) + WARN(1, "fnic<%d>: %s: tag 0x%x already exists\n", + fnic->fnic_num, __func__, blk_mq_unique_tag_to_tag(mqtag)); + + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(mqtag)] = + io_req; } io_req->dr_done = &tm_done; - CMD_STATE(sc) = FNIC_IOREQ_CMD_PENDING; - CMD_LR_STATUS(sc) = FCPIO_INVALID_CODE; - spin_unlock_irqrestore(io_lock, flags); + fnic_priv(sc)->state = FNIC_IOREQ_CMD_PENDING; + fnic_priv(sc)->lr_status = FCPIO_INVALID_CODE; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, "TAG %x\n", tag); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "TAG %x\n", mqtag); /* * issue the device reset, if enqueue failed, clean up the ioreq * and break assoc with scsi cmd */ if (fnic_queue_dr_io_req(fnic, sc, io_req)) { - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; if (io_req) io_req->dr_done = NULL; goto fnic_device_reset_clean; } - spin_lock_irqsave(io_lock, flags); - CMD_FLAGS(sc) |= FNIC_DEV_RST_ISSUED; - spin_unlock_irqrestore(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + fnic_priv(sc)->flags |= FNIC_DEV_RST_ISSUED; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + + spin_lock_irqsave(&fnic->fnic_lock, flags); + old_link_down_cnt = iport->fnic->link_down_cnt; + old_soft_reset_count = fnic->soft_reset_count; + spin_unlock_irqrestore(&fnic->fnic_lock, flags); /* * Wait on the local completion for LUN reset. The io_req may be @@ -2403,17 +2691,42 @@ int fnic_device_reset(struct scsi_cmnd *sc) wait_for_completion_timeout(&tm_done, msecs_to_jiffies(FNIC_LUN_RESET_TIMEOUT)); - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + /* + * Wake up can be due to the following reasons: + * 1) The device reset completed from target. + * 2) Device reset timed out. + * 3) A link-down/host_reset may have happened in between. + * 4) The device reset was aborted and io_req->dr_done was called. + */ + + exit_dr = 0; + spin_lock_irqsave(&fnic->fnic_lock, flags); + if ((old_link_down_cnt != fnic->link_down_cnt) || + (fnic->reset_in_progress) || + (fnic->soft_reset_count != old_soft_reset_count) || + (iport->state != FNIC_IPORT_STATE_READY)) + exit_dr = 1; + + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; if (!io_req) { - spin_unlock_irqrestore(io_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "io_req is null tag 0x%x sc 0x%p\n", tag, sc); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "io_req is null mqtag 0x%x sc 0x%p\n", mqtag, sc); goto fnic_device_reset_end; } + + if (exit_dr) { + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "Host reset called for fnic. Exit device reset\n"); + io_req->dr_done = NULL; + goto fnic_device_reset_clean; + } io_req->dr_done = NULL; - status = CMD_LR_STATUS(sc); + status = fnic_priv(sc)->lr_status; /* * If lun reset not completed, bail out with failed. io_req @@ -2421,64 +2734,22 @@ int fnic_device_reset(struct scsi_cmnd *sc) */ if (status == FCPIO_INVALID_CODE) { atomic64_inc(&reset_stats->device_reset_timeouts); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "Device reset timed out\n"); - CMD_FLAGS(sc) |= FNIC_DEV_RST_TIMED_OUT; - spin_unlock_irqrestore(io_lock, flags); + fnic_priv(sc)->flags |= FNIC_DEV_RST_TIMED_OUT; int_to_scsilun(sc->device->lun, &fc_lun); - /* - * Issue abort and terminate on device reset request. - * If q'ing of terminate fails, retry it after a delay. - */ - while (1) { - spin_lock_irqsave(io_lock, flags); - if (CMD_FLAGS(sc) & FNIC_DEV_RST_TERM_ISSUED) { - spin_unlock_irqrestore(io_lock, flags); - break; - } - spin_unlock_irqrestore(io_lock, flags); - if (fnic_queue_abort_io_req(fnic, - tag | FNIC_TAG_DEV_RST, - FCPIO_ITMF_ABT_TASK_TERM, - fc_lun.scsi_lun, io_req)) { - wait_for_completion_timeout(&tm_done, - msecs_to_jiffies(FNIC_ABT_TERM_DELAY_TIMEOUT)); - } else { - spin_lock_irqsave(io_lock, flags); - CMD_FLAGS(sc) |= FNIC_DEV_RST_TERM_ISSUED; - CMD_STATE(sc) = FNIC_IOREQ_ABTS_PENDING; - io_req->abts_done = &tm_done; - spin_unlock_irqrestore(io_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Abort and terminate issued on Device reset " - "tag 0x%x sc 0x%p\n", tag, sc); - break; - } - } - while (1) { - spin_lock_irqsave(io_lock, flags); - if (!(CMD_FLAGS(sc) & FNIC_DEV_RST_DONE)) { - spin_unlock_irqrestore(io_lock, flags); - wait_for_completion_timeout(&tm_done, - msecs_to_jiffies(FNIC_LUN_RESET_TIMEOUT)); - break; - } else { - io_req = (struct fnic_io_req *)CMD_SP(sc); - io_req->abts_done = NULL; - goto fnic_device_reset_clean; - } - } + goto fnic_device_reset_clean; } else { - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); } /* Completed, but not successful, clean up the io_req, return fail */ if (status != FCPIO_SUCCESS) { - spin_lock_irqsave(io_lock, flags); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); FNIC_SCSI_DBG(KERN_DEBUG, - fnic->lport->host, + fnic->host, fnic->fnic_num, "Device reset completed - failed\n"); - io_req = (struct fnic_io_req *)CMD_SP(sc); + io_req = fnic_priv(sc)->io_req; goto fnic_device_reset_clean; } @@ -2490,26 +2761,28 @@ int fnic_device_reset(struct scsi_cmnd *sc) * succeeds */ if (fnic_clean_pending_aborts(fnic, sc, new_sc)) { - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Device reset failed" - " since could not abort all IOs\n"); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, + "Device reset failed: Cannot abort all IOs\n"); goto fnic_device_reset_clean; } /* Clean lun reset command */ - spin_lock_irqsave(io_lock, flags); - io_req = (struct fnic_io_req *)CMD_SP(sc); + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + io_req = fnic_priv(sc)->io_req; if (io_req) /* Completed, and successful */ ret = SUCCESS; fnic_device_reset_clean: - if (io_req) - CMD_SP(sc) = NULL; + if (io_req) { + fnic_priv(sc)->io_req = NULL; + io_req->sc = NULL; + fnic->sw_copy_wq[hwq].io_req_table[blk_mq_unique_tag_to_tag(io_req->tag)] = NULL; + } - spin_unlock_irqrestore(io_lock, flags); + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); if (io_req) { start_time = io_req->start_time; @@ -2517,20 +2790,40 @@ fnic_device_reset_clean: mempool_free(io_req, fnic->io_req_pool); } + /* + * If link-event is seen while LUN reset is issued we need + * to complete the LUN reset here + */ + if (!new_sc) { + sc->result = DID_RESET << 16; + scsi_done(sc); + } + fnic_device_reset_end: - FNIC_TRACE(fnic_device_reset, sc->device->host->host_no, - sc->request->tag, sc, + FNIC_TRACE(fnic_device_reset, sc->device->host->host_no, rq->tag, sc, jiffies_to_msecs(jiffies - start_time), 0, ((u64)sc->cmnd[0] << 32 | (u64)sc->cmnd[2] << 24 | (u64)sc->cmnd[3] << 16 | (u64)sc->cmnd[4] << 8 | sc->cmnd[5]), - (((u64)CMD_FLAGS(sc) << 32) | CMD_STATE(sc))); + fnic_flags_and_state(sc)); - /* free tag if it is allocated */ - if (unlikely(tag_gen_flag)) - fnic_scsi_host_end_tag(fnic, sc); + if (new_sc) { + fnic->sgreset_sc = NULL; + mutex_unlock(&fnic->sgreset_mutex); + } - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, + while ((ret == SUCCESS) && fnic_count_lun_ioreqs(fnic, sc->device)) { + if (count >= 2) { + ret = FAILED; + break; + } + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "Cannot clean up all IOs for the LUN\n"); + schedule_timeout(msecs_to_jiffies(1000)); + count++; + } + + FNIC_SCSI_DBG(KERN_DEBUG, fnic->host, fnic->fnic_num, "Returning from device reset %s\n", (ret == SUCCESS) ? "SUCCESS" : "FAILED"); @@ -2541,68 +2834,78 @@ fnic_device_reset_end: return ret; } -/* Clean up all IOs, clean up libFC local port */ -int fnic_reset(struct Scsi_Host *shost) +static void fnic_post_flogo_linkflap(struct fnic *fnic) +{ + unsigned long flags; + + fnic_fdls_link_status_change(fnic, 0); + spin_lock_irqsave(&fnic->fnic_lock, flags); + + if (fnic->link_status) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + fnic_fdls_link_status_change(fnic, 1); + return; + } + spin_unlock_irqrestore(&fnic->fnic_lock, flags); +} + +/* Logout from all the targets and simulate link flap */ +void fnic_reset(struct Scsi_Host *shost) { - struct fc_lport *lp; struct fnic *fnic; - int ret = 0; struct reset_stats *reset_stats; - lp = shost_priv(shost); - fnic = lport_priv(lp); + fnic = *((struct fnic **) shost_priv(shost)); reset_stats = &fnic->fnic_stats.reset_stats; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_reset called\n"); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Issuing fnic reset\n"); atomic64_inc(&reset_stats->fnic_resets); + fnic_post_flogo_linkflap(fnic); - /* - * Reset local port, this will clean up libFC exchanges, - * reset remote port sessions, and if link is up, begin flogi - */ - ret = fc_lport_reset(lp); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Returning from fnic reset"); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "Returning from fnic reset %s\n", - (ret == 0) ? - "SUCCESS" : "FAILED"); + atomic64_inc(&reset_stats->fnic_reset_completions); +} - if (ret == 0) - atomic64_inc(&reset_stats->fnic_reset_completions); - else - atomic64_inc(&reset_stats->fnic_reset_failures); +int fnic_issue_fc_host_lip(struct Scsi_Host *shost) +{ + int ret = 0; + struct fnic *fnic = *((struct fnic **) shost_priv(shost)); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "FC host lip issued"); + + ret = fnic_host_reset(shost); return ret; } -/* - * SCSI Error handling calls driver's eh_host_reset if all prior - * error handling levels return FAILED. If host reset completes - * successfully, and if link is up, then Fabric login begins. - * - * Host Reset is the highest level of error recovery. If this fails, then - * host is offlined by SCSI. - * - */ -int fnic_host_reset(struct scsi_cmnd *sc) +int fnic_host_reset(struct Scsi_Host *shost) { - int ret; + int ret = SUCCESS; unsigned long wait_host_tmo; - struct Scsi_Host *shost = sc->device->host; - struct fc_lport *lp = shost_priv(shost); - struct fnic *fnic = lport_priv(lp); + struct fnic *fnic = *((struct fnic **) shost_priv(shost)); unsigned long flags; + struct fnic_iport_s *iport = &fnic->iport; spin_lock_irqsave(&fnic->fnic_lock, flags); - if (fnic->internal_reset_inprogress == 0) { - fnic->internal_reset_inprogress = 1; + if (fnic->reset_in_progress == NOT_IN_PROGRESS) { + fnic->reset_in_progress = IN_PROGRESS; } else { spin_unlock_irqrestore(&fnic->fnic_lock, flags); - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "host reset in progress skipping another host reset\n"); - return SUCCESS; + wait_for_completion_timeout(&fnic->reset_completion_wait, + msecs_to_jiffies(10000)); + + spin_lock_irqsave(&fnic->fnic_lock, flags); + if (fnic->reset_in_progress == IN_PROGRESS) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_WARNING, fnic->host, fnic->fnic_num, + "Firmware reset in progress. Skipping another host reset\n"); + return SUCCESS; + } + fnic->reset_in_progress = IN_PROGRESS; } spin_unlock_irqrestore(&fnic->fnic_lock, flags); @@ -2611,193 +2914,179 @@ int fnic_host_reset(struct scsi_cmnd *sc) * scsi-ml tries to send a TUR to every device if host reset is * successful, so before returning to scsi, fabric should be up */ - ret = (fnic_reset(shost) == 0) ? SUCCESS : FAILED; - if (ret == SUCCESS) { + fnic_reset(shost); + + spin_lock_irqsave(&fnic->fnic_lock, flags); + fnic->reset_in_progress = NOT_IN_PROGRESS; + complete(&fnic->reset_completion_wait); + fnic->soft_reset_count++; + + /* wait till the link is up */ + if (fnic->link_status) { wait_host_tmo = jiffies + FNIC_HOST_RESET_SETTLE_TIME * HZ; ret = FAILED; while (time_before(jiffies, wait_host_tmo)) { - if ((lp->state == LPORT_ST_READY) && - (lp->link_up)) { + if (iport->state != FNIC_IPORT_STATE_READY + && fnic->link_status) { + spin_unlock_irqrestore(&fnic->fnic_lock, flags); + ssleep(1); + spin_lock_irqsave(&fnic->fnic_lock, flags); + } else { ret = SUCCESS; break; } - ssleep(1); } } - - spin_lock_irqsave(&fnic->fnic_lock, flags); - fnic->internal_reset_inprogress = 0; spin_unlock_irqrestore(&fnic->fnic_lock, flags); + + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "host reset return status: %d\n", ret); return ret; } -/* - * This fxn is called from libFC when host is removed - */ -void fnic_scsi_abort_io(struct fc_lport *lp) +static bool fnic_abts_pending_iter(struct scsi_cmnd *sc, void *data) { - int err = 0; + struct request *const rq = scsi_cmd_to_rq(sc); + struct fnic_pending_aborts_iter_data *iter_data = data; + struct fnic *fnic = iter_data->fnic; + int cmd_state; + struct fnic_io_req *io_req; unsigned long flags; - enum fnic_state old_state; - struct fnic *fnic = lport_priv(lp); - DECLARE_COMPLETION_ONSTACK(remove_wait); + uint16_t hwq = 0; + int tag; - /* Issue firmware reset for fnic, wait for reset to complete */ -retry_fw_reset: - spin_lock_irqsave(&fnic->fnic_lock, flags); - if (unlikely(fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)) { - /* fw reset is in progress, poll for its completion */ - spin_unlock_irqrestore(&fnic->fnic_lock, flags); - schedule_timeout(msecs_to_jiffies(100)); - goto retry_fw_reset; - } + tag = blk_mq_unique_tag(rq); + hwq = blk_mq_unique_tag_to_hwq(tag); - fnic->remove_wait = &remove_wait; - old_state = fnic->state; - fnic->state = FNIC_IN_FC_TRANS_ETH_MODE; - fnic_update_mac_locked(fnic, fnic->ctlr.ctl_src_addr); - spin_unlock_irqrestore(&fnic->fnic_lock, flags); + /* + * ignore this lun reset cmd or cmds that do not belong to + * this lun + */ + if (iter_data->lr_sc && sc == iter_data->lr_sc) + return true; + if (iter_data->lun_dev && sc->device != iter_data->lun_dev) + return true; - err = fnic_fw_reset_handler(fnic); - if (err) { - spin_lock_irqsave(&fnic->fnic_lock, flags); - if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE) - fnic->state = old_state; - fnic->remove_wait = NULL; - spin_unlock_irqrestore(&fnic->fnic_lock, flags); - return; + spin_lock_irqsave(&fnic->wq_copy_lock[hwq], flags); + + io_req = fnic_priv(sc)->io_req; + if (!io_req) { + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + return true; } - /* Wait for firmware reset to complete */ - wait_for_completion_timeout(&remove_wait, - msecs_to_jiffies(FNIC_RMDEVICE_TIMEOUT)); + /* + * Found IO that is still pending with firmware and + * belongs to the LUN that we are resetting + */ + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "hwq: %d tag: 0x%x Found IO in state: %s on lun\n", + hwq, tag, + fnic_ioreq_state_to_str(fnic_priv(sc)->state)); + cmd_state = fnic_priv(sc)->state; + spin_unlock_irqrestore(&fnic->wq_copy_lock[hwq], flags); + if (cmd_state == FNIC_IOREQ_ABTS_PENDING) + iter_data->ret = 1; + + return iter_data->ret ? false : true; +} - spin_lock_irqsave(&fnic->fnic_lock, flags); - fnic->remove_wait = NULL; - FNIC_SCSI_DBG(KERN_DEBUG, fnic->lport->host, - "fnic_scsi_abort_io %s\n", - (fnic->state == FNIC_IN_ETH_MODE) ? - "SUCCESS" : "FAILED"); - spin_unlock_irqrestore(&fnic->fnic_lock, flags); +/* + * fnic_is_abts_pending() is a helper function that + * walks through tag map to check if there is any IOs pending,if there is one, + * then it returns 1 (true), otherwise 0 (false) + * if @lr_sc is non NULL, then it checks IOs specific to particular LUN, + * otherwise, it checks for all IOs. + */ +int fnic_is_abts_pending(struct fnic *fnic, struct scsi_cmnd *lr_sc) +{ + struct fnic_pending_aborts_iter_data iter_data = { + .fnic = fnic, + .lun_dev = NULL, + .ret = 0, + }; + + if (lr_sc) { + iter_data.lun_dev = lr_sc->device; + iter_data.lr_sc = lr_sc; + } + + /* walk again to check, if IOs are still pending in fw */ + scsi_host_busy_iter(fnic->host, + fnic_abts_pending_iter, &iter_data); + return iter_data.ret; } /* - * This fxn called from libFC to clean up driver IO state on link down + * SCSI Error handling calls driver's eh_host_reset if all prior + * error handling levels return FAILED. If host reset completes + * successfully, and if link is up, then Fabric login begins. + * + * Host Reset is the highest level of error recovery. If this fails, then + * host is offlined by SCSI. + * */ -void fnic_scsi_cleanup(struct fc_lport *lp) +int fnic_eh_host_reset_handler(struct scsi_cmnd *sc) +{ + int ret = 0; + struct Scsi_Host *shost = sc->device->host; + struct fnic *fnic = *((struct fnic **) shost_priv(shost)); + + FNIC_SCSI_DBG(KERN_ERR, fnic->host, fnic->fnic_num, + "SCSI error handling: fnic host reset"); + + ret = fnic_host_reset(shost); + return ret; +} + + +void fnic_scsi_fcpio_reset(struct fnic *fnic) { unsigned long flags; enum fnic_state old_state; - struct fnic *fnic = lport_priv(lp); + struct fnic_iport_s *iport = &fnic->iport; + DECLARE_COMPLETION_ONSTACK(fw_reset_done); + int time_remain; /* issue fw reset */ -retry_fw_reset: spin_lock_irqsave(&fnic->fnic_lock, flags); if (unlikely(fnic->state == FNIC_IN_FC_TRANS_ETH_MODE)) { /* fw reset is in progress, poll for its completion */ spin_unlock_irqrestore(&fnic->fnic_lock, flags); - schedule_timeout(msecs_to_jiffies(100)); - goto retry_fw_reset; + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "fnic is in unexpected state: %d for fw_reset\n", + fnic->state); + return; } + old_state = fnic->state; fnic->state = FNIC_IN_FC_TRANS_ETH_MODE; - fnic_update_mac_locked(fnic, fnic->ctlr.ctl_src_addr); + + fnic_update_mac_locked(fnic, iport->hwmac); + fnic->fw_reset_done = &fw_reset_done; spin_unlock_irqrestore(&fnic->fnic_lock, flags); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Issuing fw reset\n"); if (fnic_fw_reset_handler(fnic)) { spin_lock_irqsave(&fnic->fnic_lock, flags); if (fnic->state == FNIC_IN_FC_TRANS_ETH_MODE) fnic->state = old_state; spin_unlock_irqrestore(&fnic->fnic_lock, flags); - } - -} - -void fnic_empty_scsi_cleanup(struct fc_lport *lp) -{ -} - -void fnic_exch_mgr_reset(struct fc_lport *lp, u32 sid, u32 did) -{ - struct fnic *fnic = lport_priv(lp); - - /* Non-zero sid, nothing to do */ - if (sid) - goto call_fc_exch_mgr_reset; - - if (did) { - fnic_rport_exch_reset(fnic, did); - goto call_fc_exch_mgr_reset; - } - - /* - * sid = 0, did = 0 - * link down or device being removed - */ - if (!fnic->in_remove) - fnic_scsi_cleanup(lp); - else - fnic_scsi_abort_io(lp); - - /* call libFC exch mgr reset to reset its exchanges */ -call_fc_exch_mgr_reset: - fc_exch_mgr_reset(lp, sid, did); - -} - -/* - * fnic_is_abts_pending() is a helper function that - * walks through tag map to check if there is any IOs pending,if there is one, - * then it returns 1 (true), otherwise 0 (false) - * if @lr_sc is non NULL, then it checks IOs specific to particular LUN, - * otherwise, it checks for all IOs. - */ -int fnic_is_abts_pending(struct fnic *fnic, struct scsi_cmnd *lr_sc) -{ - int tag; - struct fnic_io_req *io_req; - spinlock_t *io_lock; - unsigned long flags; - int ret = 0; - struct scsi_cmnd *sc; - struct scsi_device *lun_dev = NULL; - - if (lr_sc) - lun_dev = lr_sc->device; - - /* walk again to check, if IOs are still pending in fw */ - for (tag = 0; tag < fnic->fnic_max_tag_id; tag++) { - sc = scsi_host_find_tag(fnic->lport->host, tag); - /* - * ignore this lun reset cmd or cmds that do not belong to - * this lun - */ - if (!sc || (lr_sc && (sc->device != lun_dev || sc == lr_sc))) - continue; - - io_lock = fnic_io_lock_hash(fnic, sc); - spin_lock_irqsave(io_lock, flags); - - io_req = (struct fnic_io_req *)CMD_SP(sc); - - if (!io_req || sc->device != lun_dev) { - spin_unlock_irqrestore(io_lock, flags); - continue; + } else { + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Waiting for fw completion\n"); + time_remain = wait_for_completion_timeout(&fw_reset_done, + msecs_to_jiffies(FNIC_FW_RESET_TIMEOUT)); + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "Woken up after fw completion timeout\n"); + if (time_remain == 0) { + FNIC_SCSI_DBG(KERN_INFO, fnic->host, fnic->fnic_num, + "FW reset completion timed out after %d ms)\n", + FNIC_FW_RESET_TIMEOUT); } - - /* - * Found IO that is still pending with firmware and - * belongs to the LUN that we are resetting - */ - FNIC_SCSI_DBG(KERN_INFO, fnic->lport->host, - "Found IO in %s on lun\n", - fnic_ioreq_state_to_str(CMD_STATE(sc))); - - if (CMD_STATE(sc) == FNIC_IOREQ_ABTS_PENDING) - ret = 1; - spin_unlock_irqrestore(io_lock, flags); + atomic64_inc(&fnic->fnic_stats.reset_stats.fw_reset_timeouts); } - - return ret; + fnic->fw_reset_done = NULL; } |