diff options
| -rw-r--r-- | drivers/scsi/lpfc/lpfc_crtn.h | 1 | ||||
| -rw-r--r-- | drivers/scsi/lpfc/lpfc_hw4.h | 1 | ||||
| -rw-r--r-- | drivers/scsi/lpfc/lpfc_init.c | 1 | ||||
| -rw-r--r-- | drivers/scsi/lpfc/lpfc_nvme.c | 327 |
4 files changed, 203 insertions, 127 deletions
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 14a86b5b51e4..c7df22683e85 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -565,6 +565,7 @@ void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba); void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_wcqe_complete *abts_cmpl); +void lpfc_nvme_cmd_template(void); extern int lpfc_enable_nvmet_cnt; extern unsigned long long lpfc_enable_nvmet[]; extern int lpfc_no_hba_reset_cnt; diff --git a/drivers/scsi/lpfc/lpfc_hw4.h b/drivers/scsi/lpfc/lpfc_hw4.h index a2f372d14eaa..98b80559c215 100644 --- a/drivers/scsi/lpfc/lpfc_hw4.h +++ b/drivers/scsi/lpfc/lpfc_hw4.h @@ -4183,6 +4183,7 @@ struct wqe_common { #define wqe_iod_SHIFT 13 #define wqe_iod_MASK 0x00000001 #define wqe_iod_WORD word10 +#define LPFC_WQE_IOD_NONE 0 #define LPFC_WQE_IOD_WRITE 0 #define LPFC_WQE_IOD_READ 1 #define wqe_dbde_SHIFT 14 diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 50bc6c6efa87..68adea8e0a04 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -12583,6 +12583,7 @@ lpfc_init(void) fc_release_transport(lpfc_transport_template); return -ENOMEM; } + lpfc_nvme_cmd_template(); /* Initialize in case vector mapping is needed */ lpfc_used_cpu = NULL; diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 6d215f27448f..52dd9479b538 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -65,6 +65,136 @@ lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *); static struct nvme_fc_port_template lpfc_nvme_template; +union lpfc_wqe128 lpfc_iread_cmd_template; +union lpfc_wqe128 lpfc_iwrite_cmd_template; +union lpfc_wqe128 lpfc_icmnd_cmd_template; + +/* Setup WQE templates for NVME IOs */ +void +lpfc_nvme_cmd_template() +{ + union lpfc_wqe128 *wqe; + + /* IREAD template */ + wqe = &lpfc_iread_cmd_template; + memset(wqe, 0, sizeof(union lpfc_wqe128)); + + /* Word 0, 1, 2 - BDE is variable */ + + /* Word 3 - cmd_buff_len, payload_offset_len is zero */ + + /* Word 4 - total_xfer_len is variable */ + + /* Word 5 - is zero */ + + /* Word 6 - ctxt_tag, xri_tag is variable */ + + /* Word 7 */ + bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); + bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); + bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); + bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); + + /* Word 8 - abort_tag is variable */ + + /* Word 9 - reqtag is variable */ + + /* Word 10 - dbde, wqes is variable */ + bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); + bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); + bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); + bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); + bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); + bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); + + /* Word 11 - pbde is variable */ + bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); + bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); + bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); + + /* Word 12 - is zero */ + + /* Word 13, 14, 15 - PBDE is variable */ + + /* IWRITE template */ + wqe = &lpfc_iwrite_cmd_template; + memset(wqe, 0, sizeof(union lpfc_wqe128)); + + /* Word 0, 1, 2 - BDE is variable */ + + /* Word 3 - cmd_buff_len, payload_offset_len is zero */ + + /* Word 4 - total_xfer_len is variable */ + + /* Word 5 - initial_xfer_len is variable */ + + /* Word 6 - ctxt_tag, xri_tag is variable */ + + /* Word 7 */ + bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); + bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); + bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); + bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); + + /* Word 8 - abort_tag is variable */ + + /* Word 9 - reqtag is variable */ + + /* Word 10 - dbde, wqes is variable */ + bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); + bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); + bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); + bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); + bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); + bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); + + /* Word 11 - pbde is variable */ + bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); + bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); + bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); + + /* Word 12 - is zero */ + + /* Word 13, 14, 15 - PBDE is variable */ + + /* ICMND template */ + wqe = &lpfc_icmnd_cmd_template; + memset(wqe, 0, sizeof(union lpfc_wqe128)); + + /* Word 0, 1, 2 - BDE is variable */ + + /* Word 3 - payload_offset_len is variable */ + + /* Word 4, 5 - is zero */ + + /* Word 6 - ctxt_tag, xri_tag is variable */ + + /* Word 7 */ + bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); + bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); + bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); + bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); + + /* Word 8 - abort_tag is variable */ + + /* Word 9 - reqtag is variable */ + + /* Word 10 - dbde, wqes is variable */ + bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); + bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); + bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); + bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); + bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); + bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); + + /* Word 11 */ + bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); + bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); + bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); + + /* Word 12, 13, 14, 15 - is zero */ +} + /** * lpfc_nvme_create_queue - * @lpfc_pnvme: Pointer to the driver's nvme instance data @@ -612,7 +742,7 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, } /* Fix up the existing sgls for NVME IO. */ -static void +static inline void lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, struct lpfc_nvme_buf *lpfc_ncmd, struct nvmefc_fcp_req *nCmd) @@ -648,6 +778,37 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, wqe->generic.bde.tus.f.bdeSize = 56; wqe->generic.bde.addrHigh = 0; wqe->generic.bde.addrLow = 64; /* Word 16 */ + + /* Word 10 - dbde is 0, wqes is 1 in template */ + + /* + * Embed the payload in the last half of the WQE + * WQE words 16-30 get the NVME CMD IU payload + * + * WQE words 16-19 get payload Words 1-4 + * WQE words 20-21 get payload Words 6-7 + * WQE words 22-29 get payload Words 16-23 + */ + wptr = &wqe->words[16]; /* WQE ptr */ + dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ + dptr++; /* Skip Word 0 in payload */ + + *wptr++ = *dptr++; /* Word 1 */ + *wptr++ = *dptr++; /* Word 2 */ + *wptr++ = *dptr++; /* Word 3 */ + *wptr++ = *dptr++; /* Word 4 */ + dptr++; /* Skip Word 5 in payload */ + *wptr++ = *dptr++; /* Word 6 */ + *wptr++ = *dptr++; /* Word 7 */ + dptr += 8; /* Skip Words 8-15 in payload */ + *wptr++ = *dptr++; /* Word 16 */ + *wptr++ = *dptr++; /* Word 17 */ + *wptr++ = *dptr++; /* Word 18 */ + *wptr++ = *dptr++; /* Word 19 */ + *wptr++ = *dptr++; /* Word 20 */ + *wptr++ = *dptr++; /* Word 21 */ + *wptr++ = *dptr++; /* Word 22 */ + *wptr = *dptr; /* Word 23 */ } else { sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); @@ -657,6 +818,10 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; wqe->generic.bde.addrHigh = sgl->addr_hi; wqe->generic.bde.addrLow = sgl->addr_lo; + + /* Word 10 */ + bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); + bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); } sgl++; @@ -671,50 +836,6 @@ lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, bf_set(lpfc_sli4_sge_last, sgl, 1); sgl->word2 = cpu_to_le32(sgl->word2); sgl->sge_len = cpu_to_le32(nCmd->rsplen); - - /* Word 3 */ - bf_set(payload_offset_len, &wqe->fcp_icmd, - (nCmd->rsplen + nCmd->cmdlen)); - - /* Word 10 */ - bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); - - if (!phba->cfg_nvme_embed_cmd) { - bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); - bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 0); - return; - } - bf_set(wqe_dbde, &wqe->generic.wqe_com, 0); - bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); - - /* - * Embed the payload in the last half of the WQE - * WQE words 16-30 get the NVME CMD IU payload - * - * WQE words 16-19 get payload Words 1-4 - * WQE words 20-21 get payload Words 6-7 - * WQE words 22-29 get payload Words 16-23 - */ - wptr = &wqe->words[16]; /* WQE ptr */ - dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ - dptr++; /* Skip Word 0 in payload */ - - *wptr++ = *dptr++; /* Word 1 */ - *wptr++ = *dptr++; /* Word 2 */ - *wptr++ = *dptr++; /* Word 3 */ - *wptr++ = *dptr++; /* Word 4 */ - dptr++; /* Skip Word 5 in payload */ - *wptr++ = *dptr++; /* Word 6 */ - *wptr++ = *dptr++; /* Word 7 */ - dptr += 8; /* Skip Words 8-15 in payload */ - *wptr++ = *dptr++; /* Word 16 */ - *wptr++ = *dptr++; /* Word 17 */ - *wptr++ = *dptr++; /* Word 18 */ - *wptr++ = *dptr++; /* Word 19 */ - *wptr++ = *dptr++; /* Word 20 */ - *wptr++ = *dptr++; /* Word 21 */ - *wptr++ = *dptr++; /* Word 22 */ - *wptr = *dptr; /* Word 23 */ } #ifdef CONFIG_SCSI_LPFC_DEBUG_FS @@ -1057,9 +1178,16 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, * There are three possibilities here - use scatter-gather segment, use * the single mapping, or neither. */ - wqe->fcp_iwrite.initial_xfer_len = 0; if (nCmd->sg_cnt) { if (nCmd->io_dir == NVMEFC_FCP_WRITE) { + /* From the iwrite template, initialize words 7 - 11 */ + memcpy(&wqe->words[7], + &lpfc_iwrite_cmd_template.words[7], + sizeof(uint32_t) * 5); + + /* Word 4 */ + wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; + /* Word 5 */ if ((phba->cfg_nvme_enable_fb) && (pnode->nlp_flag & NLP_FIRSTBURST)) { @@ -1070,69 +1198,28 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, else wqe->fcp_iwrite.initial_xfer_len = pnode->nvme_fb_size; + } else { + wqe->fcp_iwrite.initial_xfer_len = 0; } - - /* Word 7 */ - bf_set(wqe_cmnd, &wqe->generic.wqe_com, - CMD_FCP_IWRITE64_WQE); - bf_set(wqe_pu, &wqe->generic.wqe_com, - PARM_READ_CHECK); - - /* Word 10 */ - bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); - bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, - LPFC_WQE_IOD_WRITE); - bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, - LPFC_WQE_LENLOC_WORD4); - if (phba->cfg_nvme_oas) - bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1); - - /* Word 11 */ - bf_set(wqe_cmd_type, &wqe->generic.wqe_com, - NVME_WRITE_CMD); - atomic_inc(&phba->fc4NvmeOutputRequests); } else { - /* Word 7 */ - bf_set(wqe_cmnd, &wqe->generic.wqe_com, - CMD_FCP_IREAD64_WQE); - bf_set(wqe_pu, &wqe->generic.wqe_com, - PARM_READ_CHECK); - - /* Word 10 */ - bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); - bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, - LPFC_WQE_IOD_READ); - bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, - LPFC_WQE_LENLOC_WORD4); - if (phba->cfg_nvme_oas) - bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1); - - /* Word 11 */ - bf_set(wqe_cmd_type, &wqe->generic.wqe_com, - NVME_READ_CMD); + /* From the iread template, initialize words 7 - 11 */ + memcpy(&wqe->words[7], + &lpfc_iread_cmd_template.words[7], + sizeof(uint32_t) * 5); + + /* Word 4 */ + wqe->fcp_iread.total_xfer_len = nCmd->payload_length; + + /* Word 5 */ + wqe->fcp_iread.rsrvd5 = 0; atomic_inc(&phba->fc4NvmeInputRequests); } } else { - /* Word 4 */ - wqe->fcp_icmd.rsrvd4 = 0; - - /* Word 7 */ - bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_FCP_ICMND64_WQE); - bf_set(wqe_pu, &wqe->generic.wqe_com, 0); - - /* Word 10 */ - bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); - bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE); - bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, - LPFC_WQE_LENLOC_NONE); - if (phba->cfg_nvme_oas) - bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1); - - /* Word 11 */ - bf_set(wqe_cmd_type, &wqe->generic.wqe_com, NVME_READ_CMD); - + /* From the icmnd template, initialize words 4 - 11 */ + memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], + sizeof(uint32_t) * 8); atomic_inc(&phba->fc4NvmeControlRequests); } /* @@ -1140,25 +1227,21 @@ lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, * of the nvme_cmnd request_buffer */ + /* Word 3 */ + bf_set(payload_offset_len, &wqe->fcp_icmd, + (nCmd->rsplen + nCmd->cmdlen)); + /* Word 6 */ bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); - /* Word 7 */ - /* Preserve Class data in the ndlp. */ - bf_set(wqe_class, &wqe->generic.wqe_com, - (pnode->nlp_fcp_info & 0x0f)); - /* Word 8 */ wqe->generic.wqe_com.abort_tag = pwqeq->iotag; /* Word 9 */ bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); - /* Word 11 */ - bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); - pwqeq->vport = vport; return 0; } @@ -1269,12 +1352,14 @@ lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, le32_to_cpu(first_data_sgl->sge_len); bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; bde->tus.w = cpu_to_le32(bde->tus.w); - bf_set(wqe_pbde, &wqe->generic.wqe_com, 1); - } else + /* wqe_pbde is 1 in template */ + } else { + memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); - + } } else { bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); + memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); /* For this clause to be valid, the payload_length * and sg_cnt must zero. @@ -1287,12 +1372,6 @@ lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, return 1; } } - - /* - * Due to difference in data length between DIF/non-DIF paths, - * we need to set word 4 of WQE here - */ - wqe->fcp_iread.total_xfer_len = nCmd->payload_length; return 0; } @@ -2175,14 +2254,8 @@ lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc) lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd; - /* Word 7 */ - bf_set(wqe_erp, &wqe->generic.wqe_com, 0); - /* NVME upper layers will time things out, if needed */ - bf_set(wqe_tmo, &wqe->generic.wqe_com, 0); - - /* Word 10 */ - bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0); - bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); + /* Initialize WQE */ + memset(wqe, 0, sizeof(union lpfc_wqe)); /* add the nvme buffer to a post list */ list_add_tail(&lpfc_ncmd->list, &post_nblist); |