diff options
Diffstat (limited to 'drivers/target/target_core_transport.c')
| -rw-r--r-- | drivers/target/target_core_transport.c | 1091 |
1 files changed, 701 insertions, 390 deletions
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index ef9e75b359d4..e8b7955d40f2 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /******************************************************************************* * Filename: target_core_transport.c * @@ -7,20 +8,6 @@ * * Nicholas A. Bellinger <nab@kernel.org> * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * ******************************************************************************/ #include <linux/net.h> @@ -35,7 +22,7 @@ #include <linux/module.h> #include <linux/ratelimit.h> #include <linux/vmalloc.h> -#include <asm/unaligned.h> +#include <linux/unaligned.h> #include <net/sock.h> #include <net/tcp.h> #include <scsi/scsi_proto.h> @@ -54,6 +41,7 @@ #include <trace/events/target.h> static struct workqueue_struct *target_completion_wq; +static struct workqueue_struct *target_submission_wq; static struct kmem_cache *se_sess_cache; struct kmem_cache *se_ua_cache; struct kmem_cache *t10_pr_reg_cache; @@ -138,12 +126,19 @@ int init_se_kmem_caches(void) } target_completion_wq = alloc_workqueue("target_completion", - WQ_MEM_RECLAIM, 0); + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!target_completion_wq) goto out_free_lba_map_mem_cache; + target_submission_wq = alloc_workqueue("target_submission", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!target_submission_wq) + goto out_free_completion_wq; + return 0; +out_free_completion_wq: + destroy_workqueue(target_completion_wq); out_free_lba_map_mem_cache: kmem_cache_destroy(t10_alua_lba_map_mem_cache); out_free_lba_map_cache: @@ -166,6 +161,7 @@ out: void release_se_kmem_caches(void) { + destroy_workqueue(target_submission_wq); destroy_workqueue(target_completion_wq); kmem_cache_destroy(se_sess_cache); kmem_cache_destroy(se_ua_cache); @@ -224,12 +220,53 @@ void transport_subsystem_check_init(void) sub_api_initialized = 1; } -static void target_release_sess_cmd_refcnt(struct percpu_ref *ref) +static void target_release_cmd_refcnt(struct percpu_ref *ref) { - struct se_session *sess = container_of(ref, typeof(*sess), cmd_count); + struct target_cmd_counter *cmd_cnt = container_of(ref, + typeof(*cmd_cnt), + refcnt); + wake_up(&cmd_cnt->refcnt_wq); +} + +struct target_cmd_counter *target_alloc_cmd_counter(void) +{ + struct target_cmd_counter *cmd_cnt; + int rc; + + cmd_cnt = kzalloc(sizeof(*cmd_cnt), GFP_KERNEL); + if (!cmd_cnt) + return NULL; + + init_completion(&cmd_cnt->stop_done); + init_waitqueue_head(&cmd_cnt->refcnt_wq); + atomic_set(&cmd_cnt->stopped, 0); + + rc = percpu_ref_init(&cmd_cnt->refcnt, target_release_cmd_refcnt, 0, + GFP_KERNEL); + if (rc) + goto free_cmd_cnt; - wake_up(&sess->cmd_list_wq); + return cmd_cnt; + +free_cmd_cnt: + kfree(cmd_cnt); + return NULL; +} +EXPORT_SYMBOL_GPL(target_alloc_cmd_counter); + +void target_free_cmd_counter(struct target_cmd_counter *cmd_cnt) +{ + /* + * Drivers like loop do not call target_stop_session during session + * shutdown so we have to drop the ref taken at init time here. + */ + if (!atomic_read(&cmd_cnt->stopped)) + percpu_ref_put(&cmd_cnt->refcnt); + + percpu_ref_exit(&cmd_cnt->refcnt); + kfree(cmd_cnt); } +EXPORT_SYMBOL_GPL(target_free_cmd_counter); /** * transport_init_session - initialize a session object @@ -237,15 +274,11 @@ static void target_release_sess_cmd_refcnt(struct percpu_ref *ref) * * The caller must have zero-initialized @se_sess before calling this function. */ -int transport_init_session(struct se_session *se_sess) +void transport_init_session(struct se_session *se_sess) { INIT_LIST_HEAD(&se_sess->sess_list); INIT_LIST_HEAD(&se_sess->sess_acl_list); - INIT_LIST_HEAD(&se_sess->sess_cmd_list); spin_lock_init(&se_sess->sess_cmd_lock); - init_waitqueue_head(&se_sess->cmd_list_wq); - return percpu_ref_init(&se_sess->cmd_count, - target_release_sess_cmd_refcnt, 0, GFP_KERNEL); } EXPORT_SYMBOL(transport_init_session); @@ -256,7 +289,6 @@ EXPORT_SYMBOL(transport_init_session); struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops) { struct se_session *se_sess; - int ret; se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL); if (!se_sess) { @@ -264,11 +296,7 @@ struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops) " se_sess_cache\n"); return ERR_PTR(-ENOMEM); } - ret = transport_init_session(se_sess); - if (ret < 0) { - kmem_cache_free(se_sess_cache, se_sess); - return ERR_PTR(ret); - } + transport_init_session(se_sess); se_sess->sup_prot_ops = sup_prot_ops; return se_sess; @@ -434,8 +462,13 @@ target_setup_session(struct se_portal_group *tpg, int (*callback)(struct se_portal_group *, struct se_session *, void *)) { + struct target_cmd_counter *cmd_cnt; struct se_session *sess; + int rc; + cmd_cnt = target_alloc_cmd_counter(); + if (!cmd_cnt) + return ERR_PTR(-ENOMEM); /* * If the fabric driver is using percpu-ida based pre allocation * of I/O descriptor tags, go ahead and perform that setup now.. @@ -445,29 +478,38 @@ target_setup_session(struct se_portal_group *tpg, else sess = transport_alloc_session(prot_op); - if (IS_ERR(sess)) - return sess; + if (IS_ERR(sess)) { + rc = PTR_ERR(sess); + goto free_cnt; + } + sess->cmd_cnt = cmd_cnt; sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg, (unsigned char *)initiatorname); if (!sess->se_node_acl) { - transport_free_session(sess); - return ERR_PTR(-EACCES); + rc = -EACCES; + goto free_sess; } /* * Go ahead and perform any remaining fabric setup that is * required before transport_register_session(). */ if (callback != NULL) { - int rc = callback(tpg, sess, private); - if (rc) { - transport_free_session(sess); - return ERR_PTR(rc); - } + rc = callback(tpg, sess, private); + if (rc) + goto free_sess; } transport_register_session(tpg, sess->se_node_acl, sess, private); return sess; + +free_sess: + transport_free_session(sess); + return ERR_PTR(rc); + +free_cnt: + target_free_cmd_counter(cmd_cnt); + return ERR_PTR(rc); } EXPORT_SYMBOL(target_setup_session); @@ -592,11 +634,21 @@ void transport_free_session(struct se_session *se_sess) sbitmap_queue_free(&se_sess->sess_tag_pool); kvfree(se_sess->sess_cmd_map); } - percpu_ref_exit(&se_sess->cmd_count); + if (se_sess->cmd_cnt) + target_free_cmd_counter(se_sess->cmd_cnt); kmem_cache_free(se_sess_cache, se_sess); } EXPORT_SYMBOL(transport_free_session); +static int target_release_res(struct se_device *dev, void *data) +{ + struct se_session *sess = data; + + if (dev->reservation_holder == sess) + target_release_reservation(dev); + return 0; +} + void transport_deregister_session(struct se_session *se_sess) { struct se_portal_group *se_tpg = se_sess->se_tpg; @@ -613,6 +665,12 @@ void transport_deregister_session(struct se_session *se_sess) se_sess->fabric_sess_ptr = NULL; spin_unlock_irqrestore(&se_tpg->session_lock, flags); + /* + * Since the session is being removed, release SPC-2 + * reservations held by the session that is disappearing. + */ + target_for_each_device(target_release_res, se_sess); + pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", se_tpg->se_tpg_tfo->fabric_name); /* @@ -643,14 +701,29 @@ static void target_remove_from_state_list(struct se_cmd *cmd) if (!dev) return; - spin_lock_irqsave(&dev->execute_task_lock, flags); + spin_lock_irqsave(&dev->queues[cmd->cpuid].lock, flags); if (cmd->state_active) { list_del(&cmd->state_list); cmd->state_active = false; } - spin_unlock_irqrestore(&dev->execute_task_lock, flags); + spin_unlock_irqrestore(&dev->queues[cmd->cpuid].lock, flags); } +static void target_remove_from_tmr_list(struct se_cmd *cmd) +{ + struct se_device *dev = NULL; + unsigned long flags; + + if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) + dev = cmd->se_tmr_req->tmr_dev; + + if (dev) { + spin_lock_irqsave(&dev->se_tmr_lock, flags); + if (cmd->se_tmr_req->tmr_dev) + list_del_init(&cmd->se_tmr_req->tmr_list); + spin_unlock_irqrestore(&dev->se_tmr_lock, flags); + } +} /* * This function is called by the target core after the target core has * finished processing a SCSI command or SCSI TMF. Both the regular command @@ -662,13 +735,6 @@ static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) { unsigned long flags; - target_remove_from_state_list(cmd); - - /* - * Clear struct se_cmd->se_lun before the handoff to FE. - */ - cmd->se_lun = NULL; - spin_lock_irqsave(&cmd->t_state_lock, flags); /* * Determine if frontend context caller is requesting the stopping of @@ -703,16 +769,23 @@ static void transport_lun_remove_cmd(struct se_cmd *cmd) if (!lun) return; + target_remove_from_state_list(cmd); + target_remove_from_tmr_list(cmd); + if (cmpxchg(&cmd->lun_ref_active, true, false)) percpu_ref_put(&lun->lun_ref); + + /* + * Clear struct se_cmd->se_lun before the handoff to FE. + */ + cmd->se_lun = NULL; } static void target_complete_failure_work(struct work_struct *work) { struct se_cmd *cmd = container_of(work, struct se_cmd, work); - transport_generic_request_failure(cmd, - TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); + transport_generic_request_failure(cmd, cmd->sense_reason); } /* @@ -830,15 +903,18 @@ static bool target_cmd_interrupted(struct se_cmd *cmd) } /* May be called from interrupt context so must not sleep. */ -void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) +void target_complete_cmd_with_sense(struct se_cmd *cmd, u8 scsi_status, + sense_reason_t sense_reason) { - int success; + struct se_wwn *wwn = cmd->se_sess->se_tpg->se_tpg_wwn; + int success, cpu; unsigned long flags; if (target_cmd_interrupted(cmd)) return; cmd->scsi_status = scsi_status; + cmd->sense_reason = sense_reason; spin_lock_irqsave(&cmd->t_state_lock, flags); switch (cmd->scsi_status) { @@ -859,18 +935,27 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) INIT_WORK(&cmd->work, success ? target_complete_ok_work : target_complete_failure_work); - if (cmd->se_cmd_flags & SCF_USE_CPUID) - queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work); + + if (!wwn || wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID) + cpu = cmd->cpuid; else - queue_work(target_completion_wq, &cmd->work); + cpu = wwn->cmd_compl_affinity; + + queue_work_on(cpu, target_completion_wq, &cmd->work); +} +EXPORT_SYMBOL(target_complete_cmd_with_sense); + +void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) +{ + target_complete_cmd_with_sense(cmd, scsi_status, scsi_status ? + TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE : + TCM_NO_SENSE); } EXPORT_SYMBOL(target_complete_cmd); -void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) +void target_set_cmd_data_length(struct se_cmd *cmd, int length) { - if ((scsi_status == SAM_STAT_GOOD || - cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) && - length < cmd->data_length) { + if (length < cmd->data_length) { if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { cmd->residual_count += cmd->data_length - length; } else { @@ -880,6 +965,15 @@ void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int len cmd->data_length = length; } +} +EXPORT_SYMBOL(target_set_cmd_data_length); + +void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) +{ + if (scsi_status == SAM_STAT_GOOD || + cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) { + target_set_cmd_data_length(cmd, length); + } target_complete_cmd(cmd, scsi_status); } @@ -890,12 +984,13 @@ static void target_add_to_state_list(struct se_cmd *cmd) struct se_device *dev = cmd->se_dev; unsigned long flags; - spin_lock_irqsave(&dev->execute_task_lock, flags); + spin_lock_irqsave(&dev->queues[cmd->cpuid].lock, flags); if (!cmd->state_active) { - list_add_tail(&cmd->state_list, &dev->state_list); + list_add_tail(&cmd->state_list, + &dev->queues[cmd->cpuid].state_list); cmd->state_active = true; } - spin_unlock_irqrestore(&dev->execute_task_lock, flags); + spin_unlock_irqrestore(&dev->queues[cmd->cpuid].lock, flags); } /* @@ -1274,6 +1369,19 @@ target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev, return TCM_NO_SENSE; } +/** + * target_cmd_size_check - Check whether there will be a residual. + * @cmd: SCSI command. + * @size: Data buffer size derived from CDB. The data buffer size provided by + * the SCSI transport driver is available in @cmd->data_length. + * + * Compare the data buffer size from the CDB with the data buffer limit from the transport + * header. Set @cmd->residual_count and SCF_OVERFLOW_BIT or SCF_UNDERFLOW_BIT if necessary. + * + * Note: target drivers set @cmd->data_length by calling __target_init_cmd(). + * + * Return: TCM_NO_SENSE + */ sense_reason_t target_cmd_size_check(struct se_cmd *cmd, unsigned int size) { @@ -1286,12 +1394,32 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) " %u does not match SCSI CDB Length: %u for SAM Opcode:" " 0x%02x\n", cmd->se_tfo->fabric_name, cmd->data_length, size, cmd->t_task_cdb[0]); + /* + * For READ command for the overflow case keep the existing + * fabric provided ->data_length. Otherwise for the underflow + * case, reset ->data_length to the smaller SCSI expected data + * transfer length. + */ + if (size > cmd->data_length) { + cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; + cmd->residual_count = (size - cmd->data_length); + } else { + cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; + cmd->residual_count = (cmd->data_length - size); + /* + * Do not truncate ->data_length for WRITE command to + * dump all payload + */ + if (cmd->data_direction == DMA_FROM_DEVICE) { + cmd->data_length = size; + } + } if (cmd->data_direction == DMA_TO_DEVICE) { if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) { pr_err_ratelimited("Rejecting underflow/overflow" " for WRITE data CDB\n"); - return TCM_INVALID_CDB_FIELD; + return TCM_INVALID_FIELD_IN_COMMAND_IU; } /* * Some fabric drivers like iscsi-target still expect to @@ -1305,31 +1433,6 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) return TCM_INVALID_CDB_FIELD; } } - /* - * Reject READ_* or WRITE_* with overflow/underflow for - * type SCF_SCSI_DATA_CDB. - */ - if (dev->dev_attrib.block_size != 512) { - pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" - " CDB on non 512-byte sector setup subsystem" - " plugin: %s\n", dev->transport->name); - /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ - return TCM_INVALID_CDB_FIELD; - } - /* - * For the overflow case keep the existing fabric provided - * ->data_length. Otherwise for the underflow case, reset - * ->data_length to the smaller SCSI expected data transfer - * length. - */ - if (size > cmd->data_length) { - cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; - cmd->residual_count = (size - cmd->data_length); - } else { - cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; - cmd->residual_count = (cmd->data_length - size); - cmd->data_length = size; - } } return target_check_max_data_sg_nents(cmd, dev, size); @@ -1342,18 +1445,15 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) * * Preserves the value of @cmd->tag. */ -void transport_init_se_cmd( - struct se_cmd *cmd, - const struct target_core_fabric_ops *tfo, - struct se_session *se_sess, - u32 data_length, - int data_direction, - int task_attr, - unsigned char *sense_buffer) +void __target_init_cmd(struct se_cmd *cmd, + const struct target_core_fabric_ops *tfo, + struct se_session *se_sess, u32 data_length, + int data_direction, int task_attr, + unsigned char *sense_buffer, u64 unpacked_lun, + struct target_cmd_counter *cmd_cnt) { INIT_LIST_HEAD(&cmd->se_delayed_node); INIT_LIST_HEAD(&cmd->se_qf_node); - INIT_LIST_HEAD(&cmd->se_cmd_list); INIT_LIST_HEAD(&cmd->state_list); init_completion(&cmd->t_transport_stop_comp); cmd->free_compl = NULL; @@ -1362,16 +1462,22 @@ void transport_init_se_cmd( INIT_WORK(&cmd->work, NULL); kref_init(&cmd->cmd_kref); + cmd->t_task_cdb = &cmd->__t_task_cdb[0]; cmd->se_tfo = tfo; cmd->se_sess = se_sess; cmd->data_length = data_length; cmd->data_direction = data_direction; cmd->sam_task_attr = task_attr; cmd->sense_buffer = sense_buffer; + cmd->orig_fe_lun = unpacked_lun; + cmd->cmd_cnt = cmd_cnt; + + if (!(cmd->se_cmd_flags & SCF_USE_CPUID)) + cmd->cpuid = raw_smp_processor_id(); cmd->state_active = false; } -EXPORT_SYMBOL(transport_init_se_cmd); +EXPORT_SYMBOL(__target_init_cmd); static sense_reason_t transport_check_alloc_task_attr(struct se_cmd *cmd) @@ -1382,7 +1488,7 @@ transport_check_alloc_task_attr(struct se_cmd *cmd) * Check if SAM Task Attribute emulation is enabled for this * struct se_device storage object */ - if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) + if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) return 0; if (cmd->sam_task_attr == TCM_ACA_TAG) { @@ -1395,9 +1501,8 @@ transport_check_alloc_task_attr(struct se_cmd *cmd) } sense_reason_t -target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) +target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb, gfp_t gfp) { - struct se_device *dev = cmd->se_dev; sense_reason_t ret; /* @@ -1408,7 +1513,8 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) pr_err("Received SCSI CDB with command_size: %d that" " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); - return TCM_INVALID_CDB_FIELD; + ret = TCM_INVALID_CDB_FIELD; + goto err; } /* * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, @@ -1416,30 +1522,47 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) * setup the pointer from __t_task_cdb to t_task_cdb. */ if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { - cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), - GFP_KERNEL); + cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), gfp); if (!cmd->t_task_cdb) { pr_err("Unable to allocate cmd->t_task_cdb" " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", scsi_command_size(cdb), (unsigned long)sizeof(cmd->__t_task_cdb)); - return TCM_OUT_OF_RESOURCES; + ret = TCM_OUT_OF_RESOURCES; + goto err; } - } else - cmd->t_task_cdb = &cmd->__t_task_cdb[0]; + } /* * Copy the original CDB into cmd-> */ memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); trace_target_sequencer_start(cmd); + return 0; + +err: + /* + * Copy the CDB here to allow trace_target_cmd_complete() to + * print the cdb to the trace buffers. + */ + memcpy(cmd->t_task_cdb, cdb, min(scsi_command_size(cdb), + (unsigned int)TCM_MAX_COMMAND_SIZE)); + return ret; +} +EXPORT_SYMBOL(target_cmd_init_cdb); + +sense_reason_t +target_cmd_parse_cdb(struct se_cmd *cmd) +{ + struct se_device *dev = cmd->se_dev; + sense_reason_t ret; ret = dev->transport->parse_cdb(cmd); if (ret == TCM_UNSUPPORTED_SCSI_OPCODE) - pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", - cmd->se_tfo->fabric_name, - cmd->se_sess->se_node_acl->initiatorname, - cmd->t_task_cdb[0]); + pr_debug_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", + cmd->se_tfo->fabric_name, + cmd->se_sess->se_node_acl->initiatorname, + cmd->t_task_cdb[0]); if (ret) return ret; @@ -1448,31 +1571,55 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) return ret; cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; - atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus); + /* + * If this is the xcopy_lun then we won't have lun_stats since we + * can't export them. + */ + if (cmd->se_lun->lun_stats) + this_cpu_inc(cmd->se_lun->lun_stats->cmd_pdus); return 0; } -EXPORT_SYMBOL(target_setup_cmd_from_cdb); +EXPORT_SYMBOL(target_cmd_parse_cdb); -/* - * Used by fabric module frontends to queue tasks directly. - * May only be used from process context. - */ -int transport_handle_cdb_direct( - struct se_cmd *cmd) +static int __target_submit(struct se_cmd *cmd) { sense_reason_t ret; + might_sleep(); + + /* + * Check if we need to delay processing because of ALUA + * Active/NonOptimized primary access state.. + */ + core_alua_check_nonop_delay(cmd); + + if (cmd->t_data_nents != 0) { + /* + * This is primarily a hack for udev and tcm loop which sends + * INQUIRYs with a single page and expects the data to be + * cleared. + */ + if (!(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && + cmd->data_direction == DMA_FROM_DEVICE) { + struct scatterlist *sgl = cmd->t_data_sg; + unsigned char *buf = NULL; + + BUG_ON(!sgl); + + buf = kmap_local_page(sg_page(sgl)); + if (buf) { + memset(buf + sgl->offset, 0, sgl->length); + kunmap_local(buf); + } + } + } + if (!cmd->se_lun) { dump_stack(); pr_err("cmd->se_lun is NULL\n"); return -EINVAL; } - if (in_interrupt()) { - dump_stack(); - pr_err("transport_generic_handle_cdb cannot be called" - " from interrupt context\n"); - return -EINVAL; - } + /* * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that * outstanding descriptors are handled correctly during shutdown via @@ -1494,7 +1641,6 @@ int transport_handle_cdb_direct( transport_generic_request_failure(cmd, ret); return 0; } -EXPORT_SYMBOL(transport_handle_cdb_direct); sense_reason_t transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, @@ -1524,93 +1670,105 @@ transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, } /** - * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized - * se_cmd + use pre-allocated SGL memory. - * - * @se_cmd: command descriptor to submit + * target_init_cmd - initialize se_cmd + * @se_cmd: command descriptor to init * @se_sess: associated se_sess for endpoint - * @cdb: pointer to SCSI CDB * @sense: pointer to SCSI sense buffer * @unpacked_lun: unpacked LUN to reference for struct se_lun * @data_length: fabric expected data transfer length * @task_attr: SAM task attribute * @data_dir: DMA data direction * @flags: flags for command submission from target_sc_flags_tables - * @sgl: struct scatterlist memory for unidirectional mapping - * @sgl_count: scatterlist count for unidirectional mapping - * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping - * @sgl_bidi_count: scatterlist count for bidirectional READ mapping - * @sgl_prot: struct scatterlist memory protection information - * @sgl_prot_count: scatterlist count for protection information * * Task tags are supported if the caller has set @se_cmd->tag. * - * Returns non zero to signal active I/O shutdown failure. All other - * setup exceptions will be returned as a SCSI CHECK_CONDITION response, - * but still return zero here. + * Returns: + * - less than zero to signal active I/O shutdown failure. + * - zero on success. * - * This may only be called from process context, and also currently - * assumes internal allocation of fabric payload buffer by target-core. + * If the fabric driver calls target_stop_session, then it must check the + * return code and handle failures. This will never fail for other drivers, + * and the return code can be ignored. */ -int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, - unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, - u32 data_length, int task_attr, int data_dir, int flags, - struct scatterlist *sgl, u32 sgl_count, - struct scatterlist *sgl_bidi, u32 sgl_bidi_count, - struct scatterlist *sgl_prot, u32 sgl_prot_count) +int target_init_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, + unsigned char *sense, u64 unpacked_lun, + u32 data_length, int task_attr, int data_dir, int flags) { struct se_portal_group *se_tpg; - sense_reason_t rc; - int ret; se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); BUG_ON(se_cmd->se_tfo || se_cmd->se_sess); - BUG_ON(in_interrupt()); - /* - * Initialize se_cmd for target operation. From this point - * exceptions are handled by sending exception status via - * target_core_fabric_ops->queue_status() callback - */ - transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - data_length, data_dir, task_attr, sense); if (flags & TARGET_SCF_USE_CPUID) se_cmd->se_cmd_flags |= SCF_USE_CPUID; - else - se_cmd->cpuid = WORK_CPU_UNBOUND; + /* + * Signal bidirectional data payloads to target-core + */ + if (flags & TARGET_SCF_BIDI_OP) + se_cmd->se_cmd_flags |= SCF_BIDI; if (flags & TARGET_SCF_UNKNOWN_SIZE) se_cmd->unknown_data_length = 1; /* - * Obtain struct se_cmd->cmd_kref reference and add new cmd to - * se_sess->sess_cmd_list. A second kref_get here is necessary - * for fabrics using TARGET_SCF_ACK_KREF that expect a second - * kref_put() to happen during fabric packet acknowledgement. + * Initialize se_cmd for target operation. From this point + * exceptions are handled by sending exception status via + * target_core_fabric_ops->queue_status() callback */ - ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); - if (ret) - return ret; + __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, data_length, + data_dir, task_attr, sense, unpacked_lun, + se_sess->cmd_cnt); + /* - * Signal bidirectional data payloads to target-core + * Obtain struct se_cmd->cmd_kref reference. A second kref_get here is + * necessary for fabrics using TARGET_SCF_ACK_KREF that expect a second + * kref_put() to happen during fabric packet acknowledgement. */ - if (flags & TARGET_SCF_BIDI_OP) - se_cmd->se_cmd_flags |= SCF_BIDI; + return target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); +} +EXPORT_SYMBOL_GPL(target_init_cmd); + +/** + * target_submit_prep - prepare cmd for submission + * @se_cmd: command descriptor to prep + * @cdb: pointer to SCSI CDB + * @sgl: struct scatterlist memory for unidirectional mapping + * @sgl_count: scatterlist count for unidirectional mapping + * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping + * @sgl_bidi_count: scatterlist count for bidirectional READ mapping + * @sgl_prot: struct scatterlist memory protection information + * @sgl_prot_count: scatterlist count for protection information + * @gfp: gfp allocation type + * + * Returns: + * - less than zero to signal failure. + * - zero on success. + * + * If failure is returned, lio will the callers queue_status to complete + * the cmd. + */ +int target_submit_prep(struct se_cmd *se_cmd, unsigned char *cdb, + struct scatterlist *sgl, u32 sgl_count, + struct scatterlist *sgl_bidi, u32 sgl_bidi_count, + struct scatterlist *sgl_prot, u32 sgl_prot_count, + gfp_t gfp) +{ + sense_reason_t rc; + + rc = target_cmd_init_cdb(se_cmd, cdb, gfp); + if (rc) + goto send_cc_direct; + /* * Locate se_lun pointer and attach it to struct se_cmd */ - rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); - if (rc) { - transport_send_check_condition_and_sense(se_cmd, rc, 0); - target_put_sess_cmd(se_cmd); - return 0; - } + rc = transport_lookup_cmd_lun(se_cmd); + if (rc) + goto send_cc_direct; - rc = target_setup_cmd_from_cdb(se_cmd, cdb); - if (rc != 0) { - transport_generic_request_failure(se_cmd, rc); - return 0; - } + rc = target_cmd_parse_cdb(se_cmd); + if (rc != 0) + goto generic_fail; /* * Save pointers for SGLs containing protection information, @@ -1630,45 +1788,24 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess if (sgl_count != 0) { BUG_ON(!sgl); - /* - * A work-around for tcm_loop as some userspace code via - * scsi-generic do not memset their associated read buffers, - * so go ahead and do that here for type non-data CDBs. Also - * note that this is currently guaranteed to be a single SGL - * for this case by target core in target_setup_cmd_from_cdb() - * -> transport_generic_cmd_sequencer(). - */ - if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && - se_cmd->data_direction == DMA_FROM_DEVICE) { - unsigned char *buf = NULL; - - if (sgl) - buf = kmap(sg_page(sgl)) + sgl->offset; - - if (buf) { - memset(buf, 0, sgl->length); - kunmap(sg_page(sgl)); - } - } - rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, sgl_bidi, sgl_bidi_count); - if (rc != 0) { - transport_generic_request_failure(se_cmd, rc); - return 0; - } + if (rc != 0) + goto generic_fail; } - /* - * Check if we need to delay processing because of ALUA - * Active/NonOptimized primary access state.. - */ - core_alua_check_nonop_delay(se_cmd); - - transport_handle_cdb_direct(se_cmd); return 0; + +send_cc_direct: + transport_send_check_condition_and_sense(se_cmd, rc, 0); + target_put_sess_cmd(se_cmd); + return -EIO; + +generic_fail: + transport_generic_request_failure(se_cmd, rc); + return -EIO; } -EXPORT_SYMBOL(target_submit_cmd_map_sgls); +EXPORT_SYMBOL_GPL(target_submit_prep); /** * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd @@ -1685,57 +1822,146 @@ EXPORT_SYMBOL(target_submit_cmd_map_sgls); * * Task tags are supported if the caller has set @se_cmd->tag. * - * Returns non zero to signal active I/O shutdown failure. All other - * setup exceptions will be returned as a SCSI CHECK_CONDITION response, - * but still return zero here. - * * This may only be called from process context, and also currently * assumes internal allocation of fabric payload buffer by target-core. * * It also assumes interal target core SGL memory allocation. + * + * This function must only be used by drivers that do their own + * sync during shutdown and does not use target_stop_session. If there + * is a failure this function will call into the fabric driver's + * queue_status with a CHECK_CONDITION. */ -int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, +void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, u32 data_length, int task_attr, int data_dir, int flags) { - return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, - unpacked_lun, data_length, task_attr, data_dir, - flags, NULL, 0, NULL, 0, NULL, 0); + int rc; + + rc = target_init_cmd(se_cmd, se_sess, sense, unpacked_lun, data_length, + task_attr, data_dir, flags); + WARN(rc, "Invalid target_submit_cmd use. Driver must not use target_stop_session or call target_init_cmd directly.\n"); + if (rc) + return; + + if (target_submit_prep(se_cmd, cdb, NULL, 0, NULL, 0, NULL, 0, + GFP_KERNEL)) + return; + + target_submit(se_cmd); } EXPORT_SYMBOL(target_submit_cmd); -static void target_complete_tmr_failure(struct work_struct *work) + +static struct se_dev_plug *target_plug_device(struct se_device *se_dev) { - struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); + struct se_dev_plug *se_plug; - se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; - se_cmd->se_tfo->queue_tm_rsp(se_cmd); + if (!se_dev->transport->plug_device) + return NULL; - transport_lun_remove_cmd(se_cmd); - transport_cmd_check_stop_to_fabric(se_cmd); + se_plug = se_dev->transport->plug_device(se_dev); + if (!se_plug) + return NULL; + + se_plug->se_dev = se_dev; + /* + * We have a ref to the lun at this point, but the cmds could + * complete before we unplug, so grab a ref to the se_device so we + * can call back into the backend. + */ + config_group_get(&se_dev->dev_group); + return se_plug; } -static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag, - u64 *unpacked_lun) +static void target_unplug_device(struct se_dev_plug *se_plug) { - struct se_cmd *se_cmd; - unsigned long flags; - bool ret = false; + struct se_device *se_dev = se_plug->se_dev; - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) { - if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) - continue; + se_dev->transport->unplug_device(se_plug); + config_group_put(&se_dev->dev_group); +} - if (se_cmd->tag == tag) { - *unpacked_lun = se_cmd->orig_fe_lun; - ret = true; - break; +void target_queued_submit_work(struct work_struct *work) +{ + struct se_cmd_queue *sq = container_of(work, struct se_cmd_queue, work); + struct se_cmd *se_cmd, *next_cmd; + struct se_dev_plug *se_plug = NULL; + struct se_device *se_dev = NULL; + struct llist_node *cmd_list; + + cmd_list = llist_del_all(&sq->cmd_list); + if (!cmd_list) + /* Previous call took what we were queued to submit */ + return; + + cmd_list = llist_reverse_order(cmd_list); + llist_for_each_entry_safe(se_cmd, next_cmd, cmd_list, se_cmd_list) { + if (!se_dev) { + se_dev = se_cmd->se_dev; + se_plug = target_plug_device(se_dev); } + + __target_submit(se_cmd); } - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - return ret; + if (se_plug) + target_unplug_device(se_plug); +} + +/** + * target_queue_submission - queue the cmd to run on the LIO workqueue + * @se_cmd: command descriptor to submit + */ +static void target_queue_submission(struct se_cmd *se_cmd) +{ + struct se_device *se_dev = se_cmd->se_dev; + int cpu = se_cmd->cpuid; + struct se_cmd_queue *sq; + + sq = &se_dev->queues[cpu].sq; + llist_add(&se_cmd->se_cmd_list, &sq->cmd_list); + queue_work_on(cpu, target_submission_wq, &sq->work); +} + +/** + * target_submit - perform final initialization and submit cmd to LIO core + * @se_cmd: command descriptor to submit + * + * target_submit_prep or something similar must have been called on the cmd, + * and this must be called from process context. + */ +int target_submit(struct se_cmd *se_cmd) +{ + const struct target_core_fabric_ops *tfo = se_cmd->se_sess->se_tpg->se_tpg_tfo; + struct se_dev_attrib *da = &se_cmd->se_dev->dev_attrib; + u8 submit_type; + + if (da->submit_type == TARGET_FABRIC_DEFAULT_SUBMIT) + submit_type = tfo->default_submit_type; + else if (da->submit_type == TARGET_DIRECT_SUBMIT && + tfo->direct_submit_supp) + submit_type = TARGET_DIRECT_SUBMIT; + else + submit_type = TARGET_QUEUE_SUBMIT; + + if (submit_type == TARGET_DIRECT_SUBMIT) + return __target_submit(se_cmd); + + target_queue_submission(se_cmd); + return 0; +} +EXPORT_SYMBOL_GPL(target_submit); + +static void target_complete_tmr_failure(struct work_struct *work) +{ + struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); + + se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; + se_cmd->se_tfo->queue_tm_rsp(se_cmd); + + transport_lun_remove_cmd(se_cmd); + transport_cmd_check_stop_to_fabric(se_cmd); } /** @@ -1766,8 +1992,9 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); - transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - 0, DMA_NONE, TCM_SIMPLE_TAG, sense); + __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, + 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun, + se_sess->cmd_cnt); /* * FIXME: Currently expect caller to handle se_cmd->se_tmr_req * allocation failure. @@ -1785,17 +2012,8 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, core_tmr_release_req(se_cmd->se_tmr_req); return ret; } - /* - * If this is ABORT_TASK with no explicit fabric provided LUN, - * go ahead and search active session tags for a match to figure - * out unpacked_lun for the original se_cmd. - */ - if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) { - if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun)) - goto failure; - } - ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); + ret = transport_lookup_tmr_lun(se_cmd); if (ret) goto failure; @@ -1851,7 +2069,6 @@ void transport_generic_request_failure(struct se_cmd *cmd, case TCM_ADDRESS_OUT_OF_RANGE: case TCM_CHECK_CONDITION_ABORT_CMD: case TCM_CHECK_CONDITION_UNIT_ATTENTION: - case TCM_CHECK_CONDITION_NOT_READY: case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: @@ -1860,6 +2077,11 @@ void transport_generic_request_failure(struct se_cmd *cmd, case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE: case TCM_TOO_MANY_SEGMENT_DESCS: case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE: + case TCM_INVALID_FIELD_IN_COMMAND_IU: + case TCM_ALUA_TG_PT_STANDBY: + case TCM_ALUA_TG_PT_UNAVAILABLE: + case TCM_ALUA_STATE_TRANSITION: + case TCM_ALUA_OFFLINE: break; case TCM_OUT_OF_RESOURCES: cmd->scsi_status = SAM_STAT_TASK_SET_FULL; @@ -1883,7 +2105,8 @@ void transport_generic_request_failure(struct se_cmd *cmd, * See spc4r17, section 7.4.6 Control Mode Page, Table 349 */ if (cmd->se_sess && - cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) { + cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl + == TARGET_UA_INTLCK_CTRL_ESTABLISH_UA) { target_ua_allocate_lun(cmd->se_sess->se_node_acl, cmd->orig_fe_lun, 0x2C, ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS); @@ -1995,8 +2218,9 @@ static int target_write_prot_action(struct se_cmd *cmd) static bool target_handle_task_attr(struct se_cmd *cmd) { struct se_device *dev = cmd->se_dev; + unsigned long flags; - if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) + if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) return false; cmd->se_cmd_flags |= SCF_TASK_ATTR_SET; @@ -2011,35 +2235,45 @@ static bool target_handle_task_attr(struct se_cmd *cmd) cmd->t_task_cdb[0]); return false; case TCM_ORDERED_TAG: - atomic_inc_mb(&dev->dev_ordered_sync); - pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n", cmd->t_task_cdb[0]); - - /* - * Execute an ORDERED command if no other older commands - * exist that need to be completed first. - */ - if (!atomic_read(&dev->simple_cmds)) - return false; break; default: /* * For SIMPLE and UNTAGGED Task Attribute commands */ - atomic_inc_mb(&dev->simple_cmds); +retry: + if (percpu_ref_tryget_live(&dev->non_ordered)) + return false; + break; } - if (atomic_read(&dev->dev_ordered_sync) == 0) - return false; + spin_lock_irqsave(&dev->delayed_cmd_lock, flags); + if (cmd->sam_task_attr == TCM_SIMPLE_TAG && + !percpu_ref_is_dying(&dev->non_ordered)) { + spin_unlock_irqrestore(&dev->delayed_cmd_lock, flags); + /* We raced with the last ordered completion so retry. */ + goto retry; + } else if (!percpu_ref_is_dying(&dev->non_ordered)) { + percpu_ref_kill(&dev->non_ordered); + } + + spin_lock(&cmd->t_state_lock); + cmd->transport_state &= ~CMD_T_SENT; + spin_unlock(&cmd->t_state_lock); - spin_lock(&dev->delayed_cmd_lock); list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list); - spin_unlock(&dev->delayed_cmd_lock); + spin_unlock_irqrestore(&dev->delayed_cmd_lock, flags); pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn", cmd->t_task_cdb[0], cmd->sam_task_attr); + /* + * We may have no non ordered cmds when this function started or we + * could have raced with the last simple/head cmd completing, so kick + * the delayed handler here. + */ + schedule_work(&dev->delayed_cmd_work); return true; } @@ -2056,19 +2290,14 @@ void target_execute_cmd(struct se_cmd *cmd) spin_lock_irq(&cmd->t_state_lock); cmd->t_state = TRANSPORT_PROCESSING; - cmd->transport_state &= ~CMD_T_PRE_EXECUTE; cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT; spin_unlock_irq(&cmd->t_state_lock); if (target_write_prot_action(cmd)) return; - if (target_handle_task_attr(cmd)) { - spin_lock_irq(&cmd->t_state_lock); - cmd->transport_state &= ~CMD_T_SENT; - spin_unlock_irq(&cmd->t_state_lock); + if (target_handle_task_attr(cmd)) return; - } __target_execute_cmd(cmd, true); } @@ -2078,29 +2307,59 @@ EXPORT_SYMBOL(target_execute_cmd); * Process all commands up to the last received ORDERED task attribute which * requires another blocking boundary */ -static void target_restart_delayed_cmds(struct se_device *dev) +void target_do_delayed_work(struct work_struct *work) { - for (;;) { + struct se_device *dev = container_of(work, struct se_device, + delayed_cmd_work); + + spin_lock(&dev->delayed_cmd_lock); + while (!dev->ordered_sync_in_progress) { struct se_cmd *cmd; - spin_lock(&dev->delayed_cmd_lock); - if (list_empty(&dev->delayed_cmd_list)) { - spin_unlock(&dev->delayed_cmd_lock); + /* + * We can be woken up early/late due to races or the + * extra wake up we do when adding commands to the list. + * We check for both cases here. + */ + if (list_empty(&dev->delayed_cmd_list) || + !percpu_ref_is_zero(&dev->non_ordered)) break; - } cmd = list_entry(dev->delayed_cmd_list.next, struct se_cmd, se_delayed_node); + cmd->se_cmd_flags |= SCF_TASK_ORDERED_SYNC; + cmd->transport_state |= CMD_T_SENT; + + dev->ordered_sync_in_progress = true; + list_del(&cmd->se_delayed_node); spin_unlock(&dev->delayed_cmd_lock); - cmd->transport_state |= CMD_T_SENT; - __target_execute_cmd(cmd, true); - - if (cmd->sam_task_attr == TCM_ORDERED_TAG) - break; + spin_lock(&dev->delayed_cmd_lock); } + spin_unlock(&dev->delayed_cmd_lock); +} + +static void transport_complete_ordered_sync(struct se_cmd *cmd) +{ + struct se_device *dev = cmd->se_dev; + unsigned long flags; + + spin_lock_irqsave(&dev->delayed_cmd_lock, flags); + dev->dev_cur_ordered_id++; + + pr_debug("Incremented dev_cur_ordered_id: %u for type %d\n", + dev->dev_cur_ordered_id, cmd->sam_task_attr); + + dev->ordered_sync_in_progress = false; + + if (list_empty(&dev->delayed_cmd_list)) + percpu_ref_resurrect(&dev->non_ordered); + else + schedule_work(&dev->delayed_cmd_work); + + spin_unlock_irqrestore(&dev->delayed_cmd_lock, flags); } /* @@ -2111,30 +2370,28 @@ static void transport_complete_task_attr(struct se_cmd *cmd) { struct se_device *dev = cmd->se_dev; - if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) + if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) return; if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET)) - goto restart; - - if (cmd->sam_task_attr == TCM_SIMPLE_TAG) { - atomic_dec_mb(&dev->simple_cmds); - dev->dev_cur_ordered_id++; - } else if (cmd->sam_task_attr == TCM_HEAD_TAG) { - dev->dev_cur_ordered_id++; - pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n", - dev->dev_cur_ordered_id); - } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) { - atomic_dec_mb(&dev->dev_ordered_sync); - - dev->dev_cur_ordered_id++; - pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n", - dev->dev_cur_ordered_id); - } + return; + cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET; -restart: - target_restart_delayed_cmds(dev); + if (cmd->se_cmd_flags & SCF_TASK_ORDERED_SYNC) { + transport_complete_ordered_sync(cmd); + return; + } + + switch (cmd->sam_task_attr) { + case TCM_SIMPLE_TAG: + percpu_ref_put(&dev->non_ordered); + break; + case TCM_ORDERED_TAG: + /* All ordered should have been executed as sync */ + WARN_ON(1); + break; + } } static void transport_complete_qf(struct se_cmd *cmd) @@ -2187,7 +2444,7 @@ static void transport_complete_qf(struct se_cmd *cmd) ret = cmd->se_tfo->queue_data_in(cmd); break; } - /* fall through */ + fallthrough; case DMA_NONE: queue_status: trace_target_cmd_complete(cmd); @@ -2345,8 +2602,9 @@ queue_rsp: !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL)) goto queue_status; - atomic_long_add(cmd->data_length, - &cmd->se_lun->lun_stats.tx_data_octets); + if (cmd->se_lun->lun_stats) + this_cpu_add(cmd->se_lun->lun_stats->tx_data_octets, + cmd->data_length); /* * Perform READ_STRIP of PI using software emulation when * backend had PI enabled, if the transport will not be @@ -2369,20 +2627,22 @@ queue_rsp: goto queue_full; break; case DMA_TO_DEVICE: - atomic_long_add(cmd->data_length, - &cmd->se_lun->lun_stats.rx_data_octets); + if (cmd->se_lun->lun_stats) + this_cpu_add(cmd->se_lun->lun_stats->rx_data_octets, + cmd->data_length); /* * Check if we need to send READ payload for BIDI-COMMAND */ if (cmd->se_cmd_flags & SCF_BIDI) { - atomic_long_add(cmd->data_length, - &cmd->se_lun->lun_stats.tx_data_octets); + if (cmd->se_lun->lun_stats) + this_cpu_add(cmd->se_lun->lun_stats->tx_data_octets, + cmd->data_length); ret = cmd->se_tfo->queue_data_in(cmd); if (ret) goto queue_full; break; } - /* fall through */ + fallthrough; case DMA_NONE: queue_status: trace_target_cmd_complete(cmd); @@ -2738,14 +2998,12 @@ int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) EXPORT_SYMBOL(transport_generic_free_cmd); /** - * target_get_sess_cmd - Add command to active ->sess_cmd_list + * target_get_sess_cmd - Verify the session is accepting cmds and take ref * @se_cmd: command descriptor to add * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() */ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) { - struct se_session *se_sess = se_cmd->se_sess; - unsigned long flags; int ret = 0; /* @@ -2754,23 +3012,18 @@ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) * invocations before se_cmd descriptor release. */ if (ack_kref) { - if (!kref_get_unless_zero(&se_cmd->cmd_kref)) - return -EINVAL; - + kref_get(&se_cmd->cmd_kref); se_cmd->se_cmd_flags |= SCF_ACK_KREF; } - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - if (se_sess->sess_tearing_down) { - ret = -ESHUTDOWN; - goto out; + /* + * Users like xcopy do not use counters since they never do a stop + * and wait. + */ + if (se_cmd->cmd_cnt) { + if (!percpu_ref_tryget_live(&se_cmd->cmd_cnt->refcnt)) + ret = -ESHUTDOWN; } - se_cmd->transport_state |= CMD_T_PRE_EXECUTE; - list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list); - percpu_ref_get(&se_sess->cmd_count); -out: - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - if (ret && ack_kref) target_put_sess_cmd(se_cmd); @@ -2791,16 +3044,9 @@ static void target_free_cmd_mem(struct se_cmd *cmd) static void target_release_cmd_kref(struct kref *kref) { struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); - struct se_session *se_sess = se_cmd->se_sess; + struct target_cmd_counter *cmd_cnt = se_cmd->cmd_cnt; struct completion *free_compl = se_cmd->free_compl; struct completion *abrt_compl = se_cmd->abrt_compl; - unsigned long flags; - - if (se_sess) { - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - list_del_init(&se_cmd->se_cmd_list); - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - } target_free_cmd_mem(se_cmd); se_cmd->se_tfo->release_cmd(se_cmd); @@ -2809,7 +3055,8 @@ static void target_release_cmd_kref(struct kref *kref) if (abrt_compl) complete(abrt_compl); - percpu_ref_put(&se_sess->cmd_count); + if (cmd_cnt) + percpu_ref_put(&cmd_cnt->refcnt); } /** @@ -2898,6 +3145,7 @@ static const char *target_tmf_name(enum tcm_tmreq_table tmf) case TMR_LUN_RESET: return "LUN_RESET"; case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET"; case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET"; + case TMR_LUN_RESET_PRO: return "LUN_RESET_PRO"; case TMR_UNKNOWN: break; } return "(?)"; @@ -2927,41 +3175,66 @@ void target_show_cmd(const char *pfx, struct se_cmd *cmd) } EXPORT_SYMBOL(target_show_cmd); +static void target_stop_cmd_counter_confirm(struct percpu_ref *ref) +{ + struct target_cmd_counter *cmd_cnt = container_of(ref, + struct target_cmd_counter, + refcnt); + complete_all(&cmd_cnt->stop_done); +} + /** - * target_sess_cmd_list_set_waiting - Set sess_tearing_down so no new commands are queued. - * @se_sess: session to flag + * target_stop_cmd_counter - Stop new IO from being added to the counter. + * @cmd_cnt: counter to stop */ -void target_sess_cmd_list_set_waiting(struct se_session *se_sess) +void target_stop_cmd_counter(struct target_cmd_counter *cmd_cnt) { - unsigned long flags; - - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - se_sess->sess_tearing_down = 1; - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + pr_debug("Stopping command counter.\n"); + if (!atomic_cmpxchg(&cmd_cnt->stopped, 0, 1)) + percpu_ref_kill_and_confirm(&cmd_cnt->refcnt, + target_stop_cmd_counter_confirm); +} +EXPORT_SYMBOL_GPL(target_stop_cmd_counter); - percpu_ref_kill(&se_sess->cmd_count); +/** + * target_stop_session - Stop new IO from being queued on the session. + * @se_sess: session to stop + */ +void target_stop_session(struct se_session *se_sess) +{ + target_stop_cmd_counter(se_sess->cmd_cnt); } -EXPORT_SYMBOL(target_sess_cmd_list_set_waiting); +EXPORT_SYMBOL(target_stop_session); /** - * target_wait_for_sess_cmds - Wait for outstanding commands - * @se_sess: session to wait for active I/O + * target_wait_for_cmds - Wait for outstanding cmds. + * @cmd_cnt: counter to wait for active I/O for. */ -void target_wait_for_sess_cmds(struct se_session *se_sess) +void target_wait_for_cmds(struct target_cmd_counter *cmd_cnt) { - struct se_cmd *cmd; int ret; - WARN_ON_ONCE(!se_sess->sess_tearing_down); + WARN_ON_ONCE(!atomic_read(&cmd_cnt->stopped)); do { - ret = wait_event_timeout(se_sess->cmd_list_wq, - percpu_ref_is_zero(&se_sess->cmd_count), - 180 * HZ); - list_for_each_entry(cmd, &se_sess->sess_cmd_list, se_cmd_list) - target_show_cmd("session shutdown: still waiting for ", - cmd); + pr_debug("Waiting for running cmds to complete.\n"); + ret = wait_event_timeout(cmd_cnt->refcnt_wq, + percpu_ref_is_zero(&cmd_cnt->refcnt), + 180 * HZ); } while (ret <= 0); + + wait_for_completion(&cmd_cnt->stop_done); + pr_debug("Waiting for cmds done.\n"); +} +EXPORT_SYMBOL_GPL(target_wait_for_cmds); + +/** + * target_wait_for_sess_cmds - Wait for outstanding commands + * @se_sess: session to wait for active I/O + */ +void target_wait_for_sess_cmds(struct se_session *se_sess) +{ + target_wait_for_cmds(se_sess->cmd_cnt); } EXPORT_SYMBOL(target_wait_for_sess_cmds); @@ -2981,9 +3254,7 @@ __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop, __releases(&cmd->t_state_lock) __acquires(&cmd->t_state_lock) { - - assert_spin_locked(&cmd->t_state_lock); - WARN_ON_ONCE(!irqs_disabled()); + lockdep_assert_held(&cmd->t_state_lock); if (fabric_stop) cmd->transport_state |= CMD_T_FABRIC_STOP; @@ -3044,14 +3315,14 @@ bool transport_wait_for_tasks(struct se_cmd *cmd) } EXPORT_SYMBOL(transport_wait_for_tasks); -struct sense_info { +struct sense_detail { u8 key; u8 asc; u8 ascq; - bool add_sector_info; + bool add_sense_info; }; -static const struct sense_info sense_info_table[] = { +static const struct sense_detail sense_detail_table[] = { [TCM_NO_SENSE] = { .key = NOT_READY }, @@ -3139,31 +3410,29 @@ static const struct sense_info sense_info_table[] = { [TCM_CHECK_CONDITION_UNIT_ATTENTION] = { .key = UNIT_ATTENTION, }, - [TCM_CHECK_CONDITION_NOT_READY] = { - .key = NOT_READY, - }, [TCM_MISCOMPARE_VERIFY] = { .key = MISCOMPARE, .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */ .ascq = 0x00, + .add_sense_info = true, }, [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = { .key = ABORTED_COMMAND, .asc = 0x10, .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */ - .add_sector_info = true, + .add_sense_info = true, }, [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = { .key = ABORTED_COMMAND, .asc = 0x10, .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */ - .add_sector_info = true, + .add_sense_info = true, }, [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = { .key = ABORTED_COMMAND, .asc = 0x10, .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */ - .add_sector_info = true, + .add_sense_info = true, }, [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = { .key = COPY_ABORTED, @@ -3196,6 +3465,31 @@ static const struct sense_info sense_info_table[] = { .asc = 0x55, .ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */ }, + [TCM_INVALID_FIELD_IN_COMMAND_IU] = { + .key = ILLEGAL_REQUEST, + .asc = 0x0e, + .ascq = 0x03, /* INVALID FIELD IN COMMAND INFORMATION UNIT */ + }, + [TCM_ALUA_TG_PT_STANDBY] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_TG_PT_STANDBY, + }, + [TCM_ALUA_TG_PT_UNAVAILABLE] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_TG_PT_UNAVAILABLE, + }, + [TCM_ALUA_STATE_TRANSITION] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_STATE_TRANSITION, + }, + [TCM_ALUA_OFFLINE] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_OFFLINE, + }, }; /** @@ -3211,42 +3505,39 @@ static const struct sense_info sense_info_table[] = { */ static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason) { - const struct sense_info *si; + const struct sense_detail *sd; u8 *buffer = cmd->sense_buffer; int r = (__force int)reason; u8 key, asc, ascq; bool desc_format = target_sense_desc_format(cmd->se_dev); - if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key) - si = &sense_info_table[r]; + if (r < ARRAY_SIZE(sense_detail_table) && sense_detail_table[r].key) + sd = &sense_detail_table[r]; else - si = &sense_info_table[(__force int) + sd = &sense_detail_table[(__force int) TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE]; - key = si->key; + key = sd->key; if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) { if (!core_scsi3_ua_for_check_condition(cmd, &key, &asc, &ascq)) { cmd->scsi_status = SAM_STAT_BUSY; return; } - } else if (si->asc == 0) { - WARN_ON_ONCE(cmd->scsi_asc == 0); - asc = cmd->scsi_asc; - ascq = cmd->scsi_ascq; } else { - asc = si->asc; - ascq = si->ascq; + WARN_ON_ONCE(sd->asc == 0); + asc = sd->asc; + ascq = sd->ascq; } cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; cmd->scsi_status = SAM_STAT_CHECK_CONDITION; cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; scsi_build_sense_buffer(desc_format, buffer, key, asc, ascq); - if (si->add_sector_info) + if (sd->add_sense_info) WARN_ON_ONCE(scsi_set_sense_information(buffer, cmd->scsi_sense_length, - cmd->bad_sector) < 0); + cmd->sense_info) < 0); } int @@ -3273,6 +3564,22 @@ transport_send_check_condition_and_sense(struct se_cmd *cmd, } EXPORT_SYMBOL(transport_send_check_condition_and_sense); +/** + * target_send_busy - Send SCSI BUSY status back to the initiator + * @cmd: SCSI command for which to send a BUSY reply. + * + * Note: Only call this function if target_submit_cmd*() failed. + */ +int target_send_busy(struct se_cmd *cmd) +{ + WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB); + + cmd->scsi_status = SAM_STAT_BUSY; + trace_target_cmd_complete(cmd); + return cmd->se_tfo->queue_status(cmd); +} +EXPORT_SYMBOL(target_send_busy); + static void target_tmr_work(struct work_struct *work) { struct se_cmd *cmd = container_of(work, struct se_cmd, work); @@ -3297,8 +3604,7 @@ static void target_tmr_work(struct work_struct *work) tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE : TMR_FUNCTION_REJECTED; if (tmr->response == TMR_FUNCTION_COMPLETE) { - target_ua_allocate_lun(cmd->se_sess->se_node_acl, - cmd->orig_fe_lun, 0x29, + target_dev_ua_allocate(dev, 0x29, ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED); } break; @@ -3320,6 +3626,7 @@ static void target_tmr_work(struct work_struct *work) cmd->se_tfo->queue_tm_rsp(cmd); + transport_lun_remove_cmd(cmd); transport_cmd_check_stop_to_fabric(cmd); return; @@ -3333,6 +3640,10 @@ int transport_generic_handle_tmr( unsigned long flags; bool aborted = false; + spin_lock_irqsave(&cmd->se_dev->se_tmr_lock, flags); + list_add_tail(&cmd->se_tmr_req->tmr_list, &cmd->se_dev->dev_tmr_list); + spin_unlock_irqrestore(&cmd->se_dev->se_tmr_lock, flags); + spin_lock_irqsave(&cmd->t_state_lock, flags); if (cmd->transport_state & CMD_T_ABORTED) { aborted = true; |