diff options
Diffstat (limited to 'drivers/target/target_core_transport.c')
| -rw-r--r-- | drivers/target/target_core_transport.c | 1830 |
1 files changed, 1050 insertions, 780 deletions
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 97fed9a298bd..e8b7955d40f2 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /******************************************************************************* * Filename: target_core_transport.c * @@ -7,20 +8,6 @@ * * Nicholas A. Bellinger <nab@kernel.org> * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * ******************************************************************************/ #include <linux/net.h> @@ -35,7 +22,7 @@ #include <linux/module.h> #include <linux/ratelimit.h> #include <linux/vmalloc.h> -#include <asm/unaligned.h> +#include <linux/unaligned.h> #include <net/sock.h> #include <net/tcp.h> #include <scsi/scsi_proto.h> @@ -54,6 +41,7 @@ #include <trace/events/target.h> static struct workqueue_struct *target_completion_wq; +static struct workqueue_struct *target_submission_wq; static struct kmem_cache *se_sess_cache; struct kmem_cache *se_ua_cache; struct kmem_cache *t10_pr_reg_cache; @@ -64,10 +52,9 @@ struct kmem_cache *t10_alua_lba_map_cache; struct kmem_cache *t10_alua_lba_map_mem_cache; static void transport_complete_task_attr(struct se_cmd *cmd); -static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason); +static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason); static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev, int err, bool write_pending); -static int transport_put_cmd(struct se_cmd *cmd); static void target_complete_ok_work(struct work_struct *work); int init_se_kmem_caches(void) @@ -139,12 +126,19 @@ int init_se_kmem_caches(void) } target_completion_wq = alloc_workqueue("target_completion", - WQ_MEM_RECLAIM, 0); + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!target_completion_wq) goto out_free_lba_map_mem_cache; + target_submission_wq = alloc_workqueue("target_submission", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!target_submission_wq) + goto out_free_completion_wq; + return 0; +out_free_completion_wq: + destroy_workqueue(target_completion_wq); out_free_lba_map_mem_cache: kmem_cache_destroy(t10_alua_lba_map_mem_cache); out_free_lba_map_cache: @@ -167,6 +161,7 @@ out: void release_se_kmem_caches(void) { + destroy_workqueue(target_submission_wq); destroy_workqueue(target_completion_wq); kmem_cache_destroy(se_sess_cache); kmem_cache_destroy(se_ua_cache); @@ -206,26 +201,92 @@ void transport_subsystem_check_init(void) if (sub_api_initialized) return; - ret = request_module("target_core_iblock"); + ret = IS_ENABLED(CONFIG_TCM_IBLOCK) && request_module("target_core_iblock"); if (ret != 0) pr_err("Unable to load target_core_iblock\n"); - ret = request_module("target_core_file"); + ret = IS_ENABLED(CONFIG_TCM_FILEIO) && request_module("target_core_file"); if (ret != 0) pr_err("Unable to load target_core_file\n"); - ret = request_module("target_core_pscsi"); + ret = IS_ENABLED(CONFIG_TCM_PSCSI) && request_module("target_core_pscsi"); if (ret != 0) pr_err("Unable to load target_core_pscsi\n"); - ret = request_module("target_core_user"); + ret = IS_ENABLED(CONFIG_TCM_USER2) && request_module("target_core_user"); if (ret != 0) pr_err("Unable to load target_core_user\n"); sub_api_initialized = 1; } -struct se_session *transport_init_session(enum target_prot_op sup_prot_ops) +static void target_release_cmd_refcnt(struct percpu_ref *ref) +{ + struct target_cmd_counter *cmd_cnt = container_of(ref, + typeof(*cmd_cnt), + refcnt); + wake_up(&cmd_cnt->refcnt_wq); +} + +struct target_cmd_counter *target_alloc_cmd_counter(void) +{ + struct target_cmd_counter *cmd_cnt; + int rc; + + cmd_cnt = kzalloc(sizeof(*cmd_cnt), GFP_KERNEL); + if (!cmd_cnt) + return NULL; + + init_completion(&cmd_cnt->stop_done); + init_waitqueue_head(&cmd_cnt->refcnt_wq); + atomic_set(&cmd_cnt->stopped, 0); + + rc = percpu_ref_init(&cmd_cnt->refcnt, target_release_cmd_refcnt, 0, + GFP_KERNEL); + if (rc) + goto free_cmd_cnt; + + return cmd_cnt; + +free_cmd_cnt: + kfree(cmd_cnt); + return NULL; +} +EXPORT_SYMBOL_GPL(target_alloc_cmd_counter); + +void target_free_cmd_counter(struct target_cmd_counter *cmd_cnt) +{ + /* + * Drivers like loop do not call target_stop_session during session + * shutdown so we have to drop the ref taken at init time here. + */ + if (!atomic_read(&cmd_cnt->stopped)) + percpu_ref_put(&cmd_cnt->refcnt); + + percpu_ref_exit(&cmd_cnt->refcnt); + kfree(cmd_cnt); +} +EXPORT_SYMBOL_GPL(target_free_cmd_counter); + +/** + * transport_init_session - initialize a session object + * @se_sess: Session object pointer. + * + * The caller must have zero-initialized @se_sess before calling this function. + */ +void transport_init_session(struct se_session *se_sess) +{ + INIT_LIST_HEAD(&se_sess->sess_list); + INIT_LIST_HEAD(&se_sess->sess_acl_list); + spin_lock_init(&se_sess->sess_cmd_lock); +} +EXPORT_SYMBOL(transport_init_session); + +/** + * transport_alloc_session - allocate a session object and initialize it + * @sup_prot_ops: bitmask that defines which T10-PI modes are supported. + */ +struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops) { struct se_session *se_sess; @@ -235,33 +296,34 @@ struct se_session *transport_init_session(enum target_prot_op sup_prot_ops) " se_sess_cache\n"); return ERR_PTR(-ENOMEM); } - INIT_LIST_HEAD(&se_sess->sess_list); - INIT_LIST_HEAD(&se_sess->sess_acl_list); - INIT_LIST_HEAD(&se_sess->sess_cmd_list); - INIT_LIST_HEAD(&se_sess->sess_wait_list); - spin_lock_init(&se_sess->sess_cmd_lock); + transport_init_session(se_sess); se_sess->sup_prot_ops = sup_prot_ops; return se_sess; } -EXPORT_SYMBOL(transport_init_session); +EXPORT_SYMBOL(transport_alloc_session); +/** + * transport_alloc_session_tags - allocate target driver private data + * @se_sess: Session pointer. + * @tag_num: Maximum number of in-flight commands between initiator and target. + * @tag_size: Size in bytes of the private data a target driver associates with + * each command. + */ int transport_alloc_session_tags(struct se_session *se_sess, unsigned int tag_num, unsigned int tag_size) { int rc; - se_sess->sess_cmd_map = kzalloc(tag_num * tag_size, - GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL); + se_sess->sess_cmd_map = kvcalloc(tag_size, tag_num, + GFP_KERNEL | __GFP_RETRY_MAYFAIL); if (!se_sess->sess_cmd_map) { - se_sess->sess_cmd_map = vzalloc(tag_num * tag_size); - if (!se_sess->sess_cmd_map) { - pr_err("Unable to allocate se_sess->sess_cmd_map\n"); - return -ENOMEM; - } + pr_err("Unable to allocate se_sess->sess_cmd_map\n"); + return -ENOMEM; } - rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num); + rc = sbitmap_queue_init_node(&se_sess->sess_tag_pool, tag_num, -1, + false, GFP_KERNEL, NUMA_NO_NODE); if (rc < 0) { pr_err("Unable to init se_sess->sess_tag_pool," " tag_num: %u\n", tag_num); @@ -274,9 +336,16 @@ int transport_alloc_session_tags(struct se_session *se_sess, } EXPORT_SYMBOL(transport_alloc_session_tags); -struct se_session *transport_init_session_tags(unsigned int tag_num, - unsigned int tag_size, - enum target_prot_op sup_prot_ops) +/** + * transport_init_session_tags - allocate a session and target driver private data + * @tag_num: Maximum number of in-flight commands between initiator and target. + * @tag_size: Size in bytes of the private data a target driver associates with + * each command. + * @sup_prot_ops: bitmask that defines which T10-PI modes are supported. + */ +static struct se_session * +transport_init_session_tags(unsigned int tag_num, unsigned int tag_size, + enum target_prot_op sup_prot_ops) { struct se_session *se_sess; int rc; @@ -292,7 +361,7 @@ struct se_session *transport_init_session_tags(unsigned int tag_num, return ERR_PTR(-EINVAL); } - se_sess = transport_init_session(sup_prot_ops); + se_sess = transport_alloc_session(sup_prot_ops); if (IS_ERR(se_sess)) return se_sess; @@ -304,7 +373,6 @@ struct se_session *transport_init_session_tags(unsigned int tag_num, return se_sess; } -EXPORT_SYMBOL(transport_init_session_tags); /* * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called. @@ -317,6 +385,7 @@ void __transport_register_session( { const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo; unsigned char buf[PR_REG_ISID_LEN]; + unsigned long flags; se_sess->se_tpg = se_tpg; se_sess->fabric_sess_ptr = fabric_sess_ptr; @@ -353,7 +422,7 @@ void __transport_register_session( se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]); } - spin_lock_irq(&se_nacl->nacl_sess_lock); + spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags); /* * The se_nacl->nacl_sess pointer will be set to the * last active I_T Nexus for each struct se_node_acl. @@ -362,12 +431,12 @@ void __transport_register_session( list_add_tail(&se_sess->sess_acl_list, &se_nacl->acl_sess_list); - spin_unlock_irq(&se_nacl->nacl_sess_lock); + spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); } list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list); pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n", - se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr); + se_tpg->se_tpg_tfo->fabric_name, se_sess->fabric_sess_ptr); } EXPORT_SYMBOL(__transport_register_session); @@ -386,15 +455,20 @@ void transport_register_session( EXPORT_SYMBOL(transport_register_session); struct se_session * -target_alloc_session(struct se_portal_group *tpg, +target_setup_session(struct se_portal_group *tpg, unsigned int tag_num, unsigned int tag_size, enum target_prot_op prot_op, const char *initiatorname, void *private, int (*callback)(struct se_portal_group *, struct se_session *, void *)) { + struct target_cmd_counter *cmd_cnt; struct se_session *sess; + int rc; + cmd_cnt = target_alloc_cmd_counter(); + if (!cmd_cnt) + return ERR_PTR(-ENOMEM); /* * If the fabric driver is using percpu-ida based pre allocation * of I/O descriptor tags, go ahead and perform that setup now.. @@ -402,33 +476,42 @@ target_alloc_session(struct se_portal_group *tpg, if (tag_num != 0) sess = transport_init_session_tags(tag_num, tag_size, prot_op); else - sess = transport_init_session(prot_op); + sess = transport_alloc_session(prot_op); - if (IS_ERR(sess)) - return sess; + if (IS_ERR(sess)) { + rc = PTR_ERR(sess); + goto free_cnt; + } + sess->cmd_cnt = cmd_cnt; sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg, (unsigned char *)initiatorname); if (!sess->se_node_acl) { - transport_free_session(sess); - return ERR_PTR(-EACCES); + rc = -EACCES; + goto free_sess; } /* * Go ahead and perform any remaining fabric setup that is * required before transport_register_session(). */ if (callback != NULL) { - int rc = callback(tpg, sess, private); - if (rc) { - transport_free_session(sess); - return ERR_PTR(rc); - } + rc = callback(tpg, sess, private); + if (rc) + goto free_sess; } transport_register_session(tpg, sess->se_node_acl, sess, private); return sess; + +free_sess: + transport_free_session(sess); + return ERR_PTR(rc); + +free_cnt: + target_free_cmd_counter(cmd_cnt); + return ERR_PTR(rc); } -EXPORT_SYMBOL(target_alloc_session); +EXPORT_SYMBOL(target_setup_session); ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page) { @@ -466,7 +549,7 @@ static void target_complete_nacl(struct kref *kref) } mutex_lock(&se_tpg->acl_node_mutex); - list_del(&nacl->acl_list); + list_del_init(&nacl->acl_list); mutex_unlock(&se_tpg->acl_node_mutex); core_tpg_wait_for_nacl_pr_ref(nacl); @@ -538,7 +621,7 @@ void transport_free_session(struct se_session *se_sess) spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags); if (se_nacl->dynamic_stop) - list_del(&se_nacl->acl_list); + list_del_init(&se_nacl->acl_list); } mutex_unlock(&se_tpg->acl_node_mutex); @@ -548,13 +631,24 @@ void transport_free_session(struct se_session *se_sess) target_put_nacl(se_nacl); } if (se_sess->sess_cmd_map) { - percpu_ida_destroy(&se_sess->sess_tag_pool); + sbitmap_queue_free(&se_sess->sess_tag_pool); kvfree(se_sess->sess_cmd_map); } + if (se_sess->cmd_cnt) + target_free_cmd_counter(se_sess->cmd_cnt); kmem_cache_free(se_sess_cache, se_sess); } EXPORT_SYMBOL(transport_free_session); +static int target_release_res(struct se_device *dev, void *data) +{ + struct se_session *sess = data; + + if (dev->reservation_holder == sess) + target_release_reservation(dev); + return 0; +} + void transport_deregister_session(struct se_session *se_sess) { struct se_portal_group *se_tpg = se_sess->se_tpg; @@ -571,8 +665,14 @@ void transport_deregister_session(struct se_session *se_sess) se_sess->fabric_sess_ptr = NULL; spin_unlock_irqrestore(&se_tpg->session_lock, flags); + /* + * Since the session is being removed, release SPC-2 + * reservations held by the session that is disappearing. + */ + target_for_each_device(target_release_res, se_sess); + pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", - se_tpg->se_tpg_tfo->get_fabric_name()); + se_tpg->se_tpg_tfo->fabric_name); /* * If last kref is dropping now for an explicit NodeACL, awake sleeping * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group @@ -586,6 +686,13 @@ void transport_deregister_session(struct se_session *se_sess) } EXPORT_SYMBOL(transport_deregister_session); +void target_remove_session(struct se_session *se_sess) +{ + transport_deregister_session_configfs(se_sess); + transport_deregister_session(se_sess); +} +EXPORT_SYMBOL(target_remove_session); + static void target_remove_from_state_list(struct se_cmd *cmd) { struct se_device *dev = cmd->se_dev; @@ -594,24 +701,39 @@ static void target_remove_from_state_list(struct se_cmd *cmd) if (!dev) return; - spin_lock_irqsave(&dev->execute_task_lock, flags); + spin_lock_irqsave(&dev->queues[cmd->cpuid].lock, flags); if (cmd->state_active) { list_del(&cmd->state_list); cmd->state_active = false; } - spin_unlock_irqrestore(&dev->execute_task_lock, flags); + spin_unlock_irqrestore(&dev->queues[cmd->cpuid].lock, flags); } -static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) +static void target_remove_from_tmr_list(struct se_cmd *cmd) { + struct se_device *dev = NULL; unsigned long flags; - target_remove_from_state_list(cmd); + if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) + dev = cmd->se_tmr_req->tmr_dev; - /* - * Clear struct se_cmd->se_lun before the handoff to FE. - */ - cmd->se_lun = NULL; + if (dev) { + spin_lock_irqsave(&dev->se_tmr_lock, flags); + if (cmd->se_tmr_req->tmr_dev) + list_del_init(&cmd->se_tmr_req->tmr_list); + spin_unlock_irqrestore(&dev->se_tmr_lock, flags); + } +} +/* + * This function is called by the target core after the target core has + * finished processing a SCSI command or SCSI TMF. Both the regular command + * processing code and the code for aborting commands can call this + * function. CMD_T_STOP is set if and only if another thread is waiting + * inside transport_wait_for_tasks() for t_transport_stop_comp. + */ +static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) +{ + unsigned long flags; spin_lock_irqsave(&cmd->t_state_lock, flags); /* @@ -647,38 +769,23 @@ static void transport_lun_remove_cmd(struct se_cmd *cmd) if (!lun) return; + target_remove_from_state_list(cmd); + target_remove_from_tmr_list(cmd); + if (cmpxchg(&cmd->lun_ref_active, true, false)) percpu_ref_put(&lun->lun_ref); -} -int transport_cmd_finish_abort(struct se_cmd *cmd, int remove) -{ - bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF); - int ret = 0; - - if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) - transport_lun_remove_cmd(cmd); /* - * Allow the fabric driver to unmap any resources before - * releasing the descriptor via TFO->release_cmd() + * Clear struct se_cmd->se_lun before the handoff to FE. */ - if (remove) - cmd->se_tfo->aborted_task(cmd); - - if (transport_cmd_check_stop_to_fabric(cmd)) - return 1; - if (remove && ack_kref) - ret = transport_put_cmd(cmd); - - return ret; + cmd->se_lun = NULL; } static void target_complete_failure_work(struct work_struct *work) { struct se_cmd *cmd = container_of(work, struct se_cmd, work); - transport_generic_request_failure(cmd, - TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); + transport_generic_request_failure(cmd, cmd->sense_reason); } /* @@ -722,13 +829,92 @@ void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense) } EXPORT_SYMBOL(transport_copy_sense_to_cmd); -void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) +static void target_handle_abort(struct se_cmd *cmd) { - struct se_device *dev = cmd->se_dev; - int success; + bool tas = cmd->transport_state & CMD_T_TAS; + bool ack_kref = cmd->se_cmd_flags & SCF_ACK_KREF; + int ret; + + pr_debug("tag %#llx: send_abort_response = %d\n", cmd->tag, tas); + + if (tas) { + if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) { + cmd->scsi_status = SAM_STAT_TASK_ABORTED; + pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n", + cmd->t_task_cdb[0], cmd->tag); + trace_target_cmd_complete(cmd); + ret = cmd->se_tfo->queue_status(cmd); + if (ret) { + transport_handle_queue_full(cmd, cmd->se_dev, + ret, false); + return; + } + } else { + cmd->se_tmr_req->response = TMR_FUNCTION_REJECTED; + cmd->se_tfo->queue_tm_rsp(cmd); + } + } else { + /* + * Allow the fabric driver to unmap any resources before + * releasing the descriptor via TFO->release_cmd(). + */ + cmd->se_tfo->aborted_task(cmd); + if (ack_kref) + WARN_ON_ONCE(target_put_sess_cmd(cmd) != 0); + /* + * To do: establish a unit attention condition on the I_T + * nexus associated with cmd. See also the paragraph "Aborting + * commands" in SAM. + */ + } + + WARN_ON_ONCE(kref_read(&cmd->cmd_kref) == 0); + + transport_lun_remove_cmd(cmd); + + transport_cmd_check_stop_to_fabric(cmd); +} + +static void target_abort_work(struct work_struct *work) +{ + struct se_cmd *cmd = container_of(work, struct se_cmd, work); + + target_handle_abort(cmd); +} + +static bool target_cmd_interrupted(struct se_cmd *cmd) +{ + int post_ret; + + if (cmd->transport_state & CMD_T_ABORTED) { + if (cmd->transport_complete_callback) + cmd->transport_complete_callback(cmd, false, &post_ret); + INIT_WORK(&cmd->work, target_abort_work); + queue_work(target_completion_wq, &cmd->work); + return true; + } else if (cmd->transport_state & CMD_T_STOP) { + if (cmd->transport_complete_callback) + cmd->transport_complete_callback(cmd, false, &post_ret); + complete_all(&cmd->t_transport_stop_comp); + return true; + } + + return false; +} + +/* May be called from interrupt context so must not sleep. */ +void target_complete_cmd_with_sense(struct se_cmd *cmd, u8 scsi_status, + sense_reason_t sense_reason) +{ + struct se_wwn *wwn = cmd->se_sess->se_tpg->se_tpg_wwn; + int success, cpu; unsigned long flags; + if (target_cmd_interrupted(cmd)) + return; + cmd->scsi_status = scsi_status; + cmd->sense_reason = sense_reason; spin_lock_irqsave(&cmd->t_state_lock, flags); switch (cmd->scsi_status) { @@ -743,44 +929,33 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) break; } - /* - * Check for case where an explicit ABORT_TASK has been received - * and transport_wait_for_tasks() will be waiting for completion.. - */ - if (cmd->transport_state & CMD_T_ABORTED || - cmd->transport_state & CMD_T_STOP) { - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - /* - * If COMPARE_AND_WRITE was stopped by __transport_wait_for_tasks(), - * release se_device->caw_sem obtained by sbc_compare_and_write() - * since target_complete_ok_work() or target_complete_failure_work() - * won't be called to invoke the normal CAW completion callbacks. - */ - if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) { - up(&dev->caw_sem); - } - complete_all(&cmd->t_transport_stop_comp); - return; - } else if (!success) { - INIT_WORK(&cmd->work, target_complete_failure_work); - } else { - INIT_WORK(&cmd->work, target_complete_ok_work); - } - cmd->t_state = TRANSPORT_COMPLETE; cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE); spin_unlock_irqrestore(&cmd->t_state_lock, flags); - if (cmd->se_cmd_flags & SCF_USE_CPUID) - queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work); + INIT_WORK(&cmd->work, success ? target_complete_ok_work : + target_complete_failure_work); + + if (!wwn || wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID) + cpu = cmd->cpuid; else - queue_work(target_completion_wq, &cmd->work); + cpu = wwn->cmd_compl_affinity; + + queue_work_on(cpu, target_completion_wq, &cmd->work); +} +EXPORT_SYMBOL(target_complete_cmd_with_sense); + +void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) +{ + target_complete_cmd_with_sense(cmd, scsi_status, scsi_status ? + TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE : + TCM_NO_SENSE); } EXPORT_SYMBOL(target_complete_cmd); -void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) +void target_set_cmd_data_length(struct se_cmd *cmd, int length) { - if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) { + if (length < cmd->data_length) { if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { cmd->residual_count += cmd->data_length - length; } else { @@ -790,6 +965,15 @@ void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int len cmd->data_length = length; } +} +EXPORT_SYMBOL(target_set_cmd_data_length); + +void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) +{ + if (scsi_status == SAM_STAT_GOOD || + cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) { + target_set_cmd_data_length(cmd, length); + } target_complete_cmd(cmd, scsi_status); } @@ -800,12 +984,13 @@ static void target_add_to_state_list(struct se_cmd *cmd) struct se_device *dev = cmd->se_dev; unsigned long flags; - spin_lock_irqsave(&dev->execute_task_lock, flags); + spin_lock_irqsave(&dev->queues[cmd->cpuid].lock, flags); if (!cmd->state_active) { - list_add_tail(&cmd->state_list, &dev->state_list); + list_add_tail(&cmd->state_list, + &dev->queues[cmd->cpuid].state_list); cmd->state_active = true; } - spin_unlock_irqrestore(&dev->execute_task_lock, flags); + spin_unlock_irqrestore(&dev->queues[cmd->cpuid].lock, flags); } /* @@ -830,7 +1015,7 @@ void target_qf_do_work(struct work_struct *work) atomic_dec_mb(&dev->dev_qf_count); pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue" - " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd, + " context: %s\n", cmd->se_tfo->fabric_name, cmd, (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" : (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING" : "UNKNOWN"); @@ -1184,6 +1369,19 @@ target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev, return TCM_NO_SENSE; } +/** + * target_cmd_size_check - Check whether there will be a residual. + * @cmd: SCSI command. + * @size: Data buffer size derived from CDB. The data buffer size provided by + * the SCSI transport driver is available in @cmd->data_length. + * + * Compare the data buffer size from the CDB with the data buffer limit from the transport + * header. Set @cmd->residual_count and SCF_OVERFLOW_BIT or SCF_UNDERFLOW_BIT if necessary. + * + * Note: target drivers set @cmd->data_length by calling __target_init_cmd(). + * + * Return: TCM_NO_SENSE + */ sense_reason_t target_cmd_size_check(struct se_cmd *cmd, unsigned int size) { @@ -1194,14 +1392,34 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) } else if (size != cmd->data_length) { pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:" " %u does not match SCSI CDB Length: %u for SAM Opcode:" - " 0x%02x\n", cmd->se_tfo->get_fabric_name(), + " 0x%02x\n", cmd->se_tfo->fabric_name, cmd->data_length, size, cmd->t_task_cdb[0]); + /* + * For READ command for the overflow case keep the existing + * fabric provided ->data_length. Otherwise for the underflow + * case, reset ->data_length to the smaller SCSI expected data + * transfer length. + */ + if (size > cmd->data_length) { + cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; + cmd->residual_count = (size - cmd->data_length); + } else { + cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; + cmd->residual_count = (cmd->data_length - size); + /* + * Do not truncate ->data_length for WRITE command to + * dump all payload + */ + if (cmd->data_direction == DMA_FROM_DEVICE) { + cmd->data_length = size; + } + } if (cmd->data_direction == DMA_TO_DEVICE) { if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) { pr_err_ratelimited("Rejecting underflow/overflow" " for WRITE data CDB\n"); - return TCM_INVALID_CDB_FIELD; + return TCM_INVALID_FIELD_IN_COMMAND_IU; } /* * Some fabric drivers like iscsi-target still expect to @@ -1215,31 +1433,6 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) return TCM_INVALID_CDB_FIELD; } } - /* - * Reject READ_* or WRITE_* with overflow/underflow for - * type SCF_SCSI_DATA_CDB. - */ - if (dev->dev_attrib.block_size != 512) { - pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" - " CDB on non 512-byte sector setup subsystem" - " plugin: %s\n", dev->transport->name); - /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ - return TCM_INVALID_CDB_FIELD; - } - /* - * For the overflow case keep the existing fabric provided - * ->data_length. Otherwise for the underflow case, reset - * ->data_length to the smaller SCSI expected data transfer - * length. - */ - if (size > cmd->data_length) { - cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; - cmd->residual_count = (size - cmd->data_length); - } else { - cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; - cmd->residual_count = (cmd->data_length - size); - cmd->data_length = size; - } } return target_check_max_data_sg_nents(cmd, dev, size); @@ -1252,35 +1445,39 @@ target_cmd_size_check(struct se_cmd *cmd, unsigned int size) * * Preserves the value of @cmd->tag. */ -void transport_init_se_cmd( - struct se_cmd *cmd, - const struct target_core_fabric_ops *tfo, - struct se_session *se_sess, - u32 data_length, - int data_direction, - int task_attr, - unsigned char *sense_buffer) +void __target_init_cmd(struct se_cmd *cmd, + const struct target_core_fabric_ops *tfo, + struct se_session *se_sess, u32 data_length, + int data_direction, int task_attr, + unsigned char *sense_buffer, u64 unpacked_lun, + struct target_cmd_counter *cmd_cnt) { INIT_LIST_HEAD(&cmd->se_delayed_node); INIT_LIST_HEAD(&cmd->se_qf_node); - INIT_LIST_HEAD(&cmd->se_cmd_list); INIT_LIST_HEAD(&cmd->state_list); init_completion(&cmd->t_transport_stop_comp); - init_completion(&cmd->cmd_wait_comp); + cmd->free_compl = NULL; + cmd->abrt_compl = NULL; spin_lock_init(&cmd->t_state_lock); INIT_WORK(&cmd->work, NULL); kref_init(&cmd->cmd_kref); + cmd->t_task_cdb = &cmd->__t_task_cdb[0]; cmd->se_tfo = tfo; cmd->se_sess = se_sess; cmd->data_length = data_length; cmd->data_direction = data_direction; cmd->sam_task_attr = task_attr; cmd->sense_buffer = sense_buffer; + cmd->orig_fe_lun = unpacked_lun; + cmd->cmd_cnt = cmd_cnt; + + if (!(cmd->se_cmd_flags & SCF_USE_CPUID)) + cmd->cpuid = raw_smp_processor_id(); cmd->state_active = false; } -EXPORT_SYMBOL(transport_init_se_cmd); +EXPORT_SYMBOL(__target_init_cmd); static sense_reason_t transport_check_alloc_task_attr(struct se_cmd *cmd) @@ -1291,7 +1488,7 @@ transport_check_alloc_task_attr(struct se_cmd *cmd) * Check if SAM Task Attribute emulation is enabled for this * struct se_device storage object */ - if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) + if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) return 0; if (cmd->sam_task_attr == TCM_ACA_TAG) { @@ -1304,9 +1501,8 @@ transport_check_alloc_task_attr(struct se_cmd *cmd) } sense_reason_t -target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) +target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb, gfp_t gfp) { - struct se_device *dev = cmd->se_dev; sense_reason_t ret; /* @@ -1317,7 +1513,8 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) pr_err("Received SCSI CDB with command_size: %d that" " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); - return TCM_INVALID_CDB_FIELD; + ret = TCM_INVALID_CDB_FIELD; + goto err; } /* * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, @@ -1325,30 +1522,47 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) * setup the pointer from __t_task_cdb to t_task_cdb. */ if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) { - cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), - GFP_KERNEL); + cmd->t_task_cdb = kzalloc(scsi_command_size(cdb), gfp); if (!cmd->t_task_cdb) { pr_err("Unable to allocate cmd->t_task_cdb" " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", scsi_command_size(cdb), (unsigned long)sizeof(cmd->__t_task_cdb)); - return TCM_OUT_OF_RESOURCES; + ret = TCM_OUT_OF_RESOURCES; + goto err; } - } else - cmd->t_task_cdb = &cmd->__t_task_cdb[0]; + } /* * Copy the original CDB into cmd-> */ memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); trace_target_sequencer_start(cmd); + return 0; + +err: + /* + * Copy the CDB here to allow trace_target_cmd_complete() to + * print the cdb to the trace buffers. + */ + memcpy(cmd->t_task_cdb, cdb, min(scsi_command_size(cdb), + (unsigned int)TCM_MAX_COMMAND_SIZE)); + return ret; +} +EXPORT_SYMBOL(target_cmd_init_cdb); + +sense_reason_t +target_cmd_parse_cdb(struct se_cmd *cmd) +{ + struct se_device *dev = cmd->se_dev; + sense_reason_t ret; ret = dev->transport->parse_cdb(cmd); if (ret == TCM_UNSUPPORTED_SCSI_OPCODE) - pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", - cmd->se_tfo->get_fabric_name(), - cmd->se_sess->se_node_acl->initiatorname, - cmd->t_task_cdb[0]); + pr_debug_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n", + cmd->se_tfo->fabric_name, + cmd->se_sess->se_node_acl->initiatorname, + cmd->t_task_cdb[0]); if (ret) return ret; @@ -1357,31 +1571,55 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) return ret; cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; - atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus); + /* + * If this is the xcopy_lun then we won't have lun_stats since we + * can't export them. + */ + if (cmd->se_lun->lun_stats) + this_cpu_inc(cmd->se_lun->lun_stats->cmd_pdus); return 0; } -EXPORT_SYMBOL(target_setup_cmd_from_cdb); +EXPORT_SYMBOL(target_cmd_parse_cdb); -/* - * Used by fabric module frontends to queue tasks directly. - * May only be used from process context. - */ -int transport_handle_cdb_direct( - struct se_cmd *cmd) +static int __target_submit(struct se_cmd *cmd) { sense_reason_t ret; + might_sleep(); + + /* + * Check if we need to delay processing because of ALUA + * Active/NonOptimized primary access state.. + */ + core_alua_check_nonop_delay(cmd); + + if (cmd->t_data_nents != 0) { + /* + * This is primarily a hack for udev and tcm loop which sends + * INQUIRYs with a single page and expects the data to be + * cleared. + */ + if (!(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && + cmd->data_direction == DMA_FROM_DEVICE) { + struct scatterlist *sgl = cmd->t_data_sg; + unsigned char *buf = NULL; + + BUG_ON(!sgl); + + buf = kmap_local_page(sg_page(sgl)); + if (buf) { + memset(buf + sgl->offset, 0, sgl->length); + kunmap_local(buf); + } + } + } + if (!cmd->se_lun) { dump_stack(); pr_err("cmd->se_lun is NULL\n"); return -EINVAL; } - if (in_interrupt()) { - dump_stack(); - pr_err("transport_generic_handle_cdb cannot be called" - " from interrupt context\n"); - return -EINVAL; - } + /* * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that * outstanding descriptors are handled correctly during shutdown via @@ -1403,7 +1641,6 @@ int transport_handle_cdb_direct( transport_generic_request_failure(cmd, ret); return 0; } -EXPORT_SYMBOL(transport_handle_cdb_direct); sense_reason_t transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, @@ -1432,94 +1669,106 @@ transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, return 0; } -/* - * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized - * se_cmd + use pre-allocated SGL memory. - * - * @se_cmd: command descriptor to submit +/** + * target_init_cmd - initialize se_cmd + * @se_cmd: command descriptor to init * @se_sess: associated se_sess for endpoint - * @cdb: pointer to SCSI CDB * @sense: pointer to SCSI sense buffer * @unpacked_lun: unpacked LUN to reference for struct se_lun * @data_length: fabric expected data transfer length - * @task_addr: SAM task attribute + * @task_attr: SAM task attribute * @data_dir: DMA data direction * @flags: flags for command submission from target_sc_flags_tables - * @sgl: struct scatterlist memory for unidirectional mapping - * @sgl_count: scatterlist count for unidirectional mapping - * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping - * @sgl_bidi_count: scatterlist count for bidirectional READ mapping - * @sgl_prot: struct scatterlist memory protection information - * @sgl_prot_count: scatterlist count for protection information * * Task tags are supported if the caller has set @se_cmd->tag. * - * Returns non zero to signal active I/O shutdown failure. All other - * setup exceptions will be returned as a SCSI CHECK_CONDITION response, - * but still return zero here. + * Returns: + * - less than zero to signal active I/O shutdown failure. + * - zero on success. * - * This may only be called from process context, and also currently - * assumes internal allocation of fabric payload buffer by target-core. + * If the fabric driver calls target_stop_session, then it must check the + * return code and handle failures. This will never fail for other drivers, + * and the return code can be ignored. */ -int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, - unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, - u32 data_length, int task_attr, int data_dir, int flags, - struct scatterlist *sgl, u32 sgl_count, - struct scatterlist *sgl_bidi, u32 sgl_bidi_count, - struct scatterlist *sgl_prot, u32 sgl_prot_count) +int target_init_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, + unsigned char *sense, u64 unpacked_lun, + u32 data_length, int task_attr, int data_dir, int flags) { struct se_portal_group *se_tpg; - sense_reason_t rc; - int ret; se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); BUG_ON(se_cmd->se_tfo || se_cmd->se_sess); - BUG_ON(in_interrupt()); - /* - * Initialize se_cmd for target operation. From this point - * exceptions are handled by sending exception status via - * target_core_fabric_ops->queue_status() callback - */ - transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - data_length, data_dir, task_attr, sense); if (flags & TARGET_SCF_USE_CPUID) se_cmd->se_cmd_flags |= SCF_USE_CPUID; - else - se_cmd->cpuid = WORK_CPU_UNBOUND; + /* + * Signal bidirectional data payloads to target-core + */ + if (flags & TARGET_SCF_BIDI_OP) + se_cmd->se_cmd_flags |= SCF_BIDI; if (flags & TARGET_SCF_UNKNOWN_SIZE) se_cmd->unknown_data_length = 1; /* - * Obtain struct se_cmd->cmd_kref reference and add new cmd to - * se_sess->sess_cmd_list. A second kref_get here is necessary - * for fabrics using TARGET_SCF_ACK_KREF that expect a second - * kref_put() to happen during fabric packet acknowledgement. + * Initialize se_cmd for target operation. From this point + * exceptions are handled by sending exception status via + * target_core_fabric_ops->queue_status() callback */ - ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); - if (ret) - return ret; + __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, data_length, + data_dir, task_attr, sense, unpacked_lun, + se_sess->cmd_cnt); + /* - * Signal bidirectional data payloads to target-core + * Obtain struct se_cmd->cmd_kref reference. A second kref_get here is + * necessary for fabrics using TARGET_SCF_ACK_KREF that expect a second + * kref_put() to happen during fabric packet acknowledgement. */ - if (flags & TARGET_SCF_BIDI_OP) - se_cmd->se_cmd_flags |= SCF_BIDI; + return target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); +} +EXPORT_SYMBOL_GPL(target_init_cmd); + +/** + * target_submit_prep - prepare cmd for submission + * @se_cmd: command descriptor to prep + * @cdb: pointer to SCSI CDB + * @sgl: struct scatterlist memory for unidirectional mapping + * @sgl_count: scatterlist count for unidirectional mapping + * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping + * @sgl_bidi_count: scatterlist count for bidirectional READ mapping + * @sgl_prot: struct scatterlist memory protection information + * @sgl_prot_count: scatterlist count for protection information + * @gfp: gfp allocation type + * + * Returns: + * - less than zero to signal failure. + * - zero on success. + * + * If failure is returned, lio will the callers queue_status to complete + * the cmd. + */ +int target_submit_prep(struct se_cmd *se_cmd, unsigned char *cdb, + struct scatterlist *sgl, u32 sgl_count, + struct scatterlist *sgl_bidi, u32 sgl_bidi_count, + struct scatterlist *sgl_prot, u32 sgl_prot_count, + gfp_t gfp) +{ + sense_reason_t rc; + + rc = target_cmd_init_cdb(se_cmd, cdb, gfp); + if (rc) + goto send_cc_direct; + /* * Locate se_lun pointer and attach it to struct se_cmd */ - rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); - if (rc) { - transport_send_check_condition_and_sense(se_cmd, rc, 0); - target_put_sess_cmd(se_cmd); - return 0; - } + rc = transport_lookup_cmd_lun(se_cmd); + if (rc) + goto send_cc_direct; - rc = target_setup_cmd_from_cdb(se_cmd, cdb); - if (rc != 0) { - transport_generic_request_failure(se_cmd, rc); - return 0; - } + rc = target_cmd_parse_cdb(se_cmd); + if (rc != 0) + goto generic_fail; /* * Save pointers for SGLs containing protection information, @@ -1539,47 +1788,26 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess if (sgl_count != 0) { BUG_ON(!sgl); - /* - * A work-around for tcm_loop as some userspace code via - * scsi-generic do not memset their associated read buffers, - * so go ahead and do that here for type non-data CDBs. Also - * note that this is currently guaranteed to be a single SGL - * for this case by target core in target_setup_cmd_from_cdb() - * -> transport_generic_cmd_sequencer(). - */ - if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && - se_cmd->data_direction == DMA_FROM_DEVICE) { - unsigned char *buf = NULL; - - if (sgl) - buf = kmap(sg_page(sgl)) + sgl->offset; - - if (buf) { - memset(buf, 0, sgl->length); - kunmap(sg_page(sgl)); - } - } - rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, sgl_bidi, sgl_bidi_count); - if (rc != 0) { - transport_generic_request_failure(se_cmd, rc); - return 0; - } + if (rc != 0) + goto generic_fail; } - /* - * Check if we need to delay processing because of ALUA - * Active/NonOptimized primary access state.. - */ - core_alua_check_nonop_delay(se_cmd); - - transport_handle_cdb_direct(se_cmd); return 0; + +send_cc_direct: + transport_send_check_condition_and_sense(se_cmd, rc, 0); + target_put_sess_cmd(se_cmd); + return -EIO; + +generic_fail: + transport_generic_request_failure(se_cmd, rc); + return -EIO; } -EXPORT_SYMBOL(target_submit_cmd_map_sgls); +EXPORT_SYMBOL_GPL(target_submit_prep); -/* +/** * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd * * @se_cmd: command descriptor to submit @@ -1588,63 +1816,152 @@ EXPORT_SYMBOL(target_submit_cmd_map_sgls); * @sense: pointer to SCSI sense buffer * @unpacked_lun: unpacked LUN to reference for struct se_lun * @data_length: fabric expected data transfer length - * @task_addr: SAM task attribute + * @task_attr: SAM task attribute * @data_dir: DMA data direction * @flags: flags for command submission from target_sc_flags_tables * * Task tags are supported if the caller has set @se_cmd->tag. * - * Returns non zero to signal active I/O shutdown failure. All other - * setup exceptions will be returned as a SCSI CHECK_CONDITION response, - * but still return zero here. - * * This may only be called from process context, and also currently * assumes internal allocation of fabric payload buffer by target-core. * * It also assumes interal target core SGL memory allocation. + * + * This function must only be used by drivers that do their own + * sync during shutdown and does not use target_stop_session. If there + * is a failure this function will call into the fabric driver's + * queue_status with a CHECK_CONDITION. */ -int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, +void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, unsigned char *cdb, unsigned char *sense, u64 unpacked_lun, u32 data_length, int task_attr, int data_dir, int flags) { - return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, - unpacked_lun, data_length, task_attr, data_dir, - flags, NULL, 0, NULL, 0, NULL, 0); + int rc; + + rc = target_init_cmd(se_cmd, se_sess, sense, unpacked_lun, data_length, + task_attr, data_dir, flags); + WARN(rc, "Invalid target_submit_cmd use. Driver must not use target_stop_session or call target_init_cmd directly.\n"); + if (rc) + return; + + if (target_submit_prep(se_cmd, cdb, NULL, 0, NULL, 0, NULL, 0, + GFP_KERNEL)) + return; + + target_submit(se_cmd); } EXPORT_SYMBOL(target_submit_cmd); -static void target_complete_tmr_failure(struct work_struct *work) + +static struct se_dev_plug *target_plug_device(struct se_device *se_dev) { - struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); + struct se_dev_plug *se_plug; - se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; - se_cmd->se_tfo->queue_tm_rsp(se_cmd); + if (!se_dev->transport->plug_device) + return NULL; - transport_lun_remove_cmd(se_cmd); - transport_cmd_check_stop_to_fabric(se_cmd); + se_plug = se_dev->transport->plug_device(se_dev); + if (!se_plug) + return NULL; + + se_plug->se_dev = se_dev; + /* + * We have a ref to the lun at this point, but the cmds could + * complete before we unplug, so grab a ref to the se_device so we + * can call back into the backend. + */ + config_group_get(&se_dev->dev_group); + return se_plug; } -static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag, - u64 *unpacked_lun) +static void target_unplug_device(struct se_dev_plug *se_plug) { - struct se_cmd *se_cmd; - unsigned long flags; - bool ret = false; + struct se_device *se_dev = se_plug->se_dev; - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) { - if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) - continue; + se_dev->transport->unplug_device(se_plug); + config_group_put(&se_dev->dev_group); +} - if (se_cmd->tag == tag) { - *unpacked_lun = se_cmd->orig_fe_lun; - ret = true; - break; +void target_queued_submit_work(struct work_struct *work) +{ + struct se_cmd_queue *sq = container_of(work, struct se_cmd_queue, work); + struct se_cmd *se_cmd, *next_cmd; + struct se_dev_plug *se_plug = NULL; + struct se_device *se_dev = NULL; + struct llist_node *cmd_list; + + cmd_list = llist_del_all(&sq->cmd_list); + if (!cmd_list) + /* Previous call took what we were queued to submit */ + return; + + cmd_list = llist_reverse_order(cmd_list); + llist_for_each_entry_safe(se_cmd, next_cmd, cmd_list, se_cmd_list) { + if (!se_dev) { + se_dev = se_cmd->se_dev; + se_plug = target_plug_device(se_dev); } + + __target_submit(se_cmd); } - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - return ret; + if (se_plug) + target_unplug_device(se_plug); +} + +/** + * target_queue_submission - queue the cmd to run on the LIO workqueue + * @se_cmd: command descriptor to submit + */ +static void target_queue_submission(struct se_cmd *se_cmd) +{ + struct se_device *se_dev = se_cmd->se_dev; + int cpu = se_cmd->cpuid; + struct se_cmd_queue *sq; + + sq = &se_dev->queues[cpu].sq; + llist_add(&se_cmd->se_cmd_list, &sq->cmd_list); + queue_work_on(cpu, target_submission_wq, &sq->work); +} + +/** + * target_submit - perform final initialization and submit cmd to LIO core + * @se_cmd: command descriptor to submit + * + * target_submit_prep or something similar must have been called on the cmd, + * and this must be called from process context. + */ +int target_submit(struct se_cmd *se_cmd) +{ + const struct target_core_fabric_ops *tfo = se_cmd->se_sess->se_tpg->se_tpg_tfo; + struct se_dev_attrib *da = &se_cmd->se_dev->dev_attrib; + u8 submit_type; + + if (da->submit_type == TARGET_FABRIC_DEFAULT_SUBMIT) + submit_type = tfo->default_submit_type; + else if (da->submit_type == TARGET_DIRECT_SUBMIT && + tfo->direct_submit_supp) + submit_type = TARGET_DIRECT_SUBMIT; + else + submit_type = TARGET_QUEUE_SUBMIT; + + if (submit_type == TARGET_DIRECT_SUBMIT) + return __target_submit(se_cmd); + + target_queue_submission(se_cmd); + return 0; +} +EXPORT_SYMBOL_GPL(target_submit); + +static void target_complete_tmr_failure(struct work_struct *work) +{ + struct se_cmd *se_cmd = container_of(work, struct se_cmd, work); + + se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; + se_cmd->se_tfo->queue_tm_rsp(se_cmd); + + transport_lun_remove_cmd(se_cmd); + transport_cmd_check_stop_to_fabric(se_cmd); } /** @@ -1655,7 +1972,7 @@ static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag, * @se_sess: associated se_sess for endpoint * @sense: pointer to SCSI sense buffer * @unpacked_lun: unpacked LUN to reference for struct se_lun - * @fabric_context: fabric context for TMR req + * @fabric_tmr_ptr: fabric context for TMR req * @tm_type: Type of TM request * @gfp: gfp type for caller * @tag: referenced task tag for TMR_ABORT_TASK @@ -1675,8 +1992,9 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); - transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - 0, DMA_NONE, TCM_SIMPLE_TAG, sense); + __target_init_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, + 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun, + se_sess->cmd_cnt); /* * FIXME: Currently expect caller to handle se_cmd->se_tmr_req * allocation failure. @@ -1694,17 +2012,8 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, core_tmr_release_req(se_cmd->se_tmr_req); return ret; } - /* - * If this is ABORT_TASK with no explicit fabric provided LUN, - * go ahead and search active session tags for a match to figure - * out unpacked_lun for the original se_cmd. - */ - if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) { - if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun)) - goto failure; - } - ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); + ret = transport_lookup_tmr_lun(se_cmd); if (ret) goto failure; @@ -1728,10 +2037,7 @@ EXPORT_SYMBOL(target_submit_tmr); void transport_generic_request_failure(struct se_cmd *cmd, sense_reason_t sense_reason) { - int ret = 0, post_ret = 0; - - if (transport_check_aborted_status(cmd, 1)) - return; + int ret = 0, post_ret; pr_debug("-----[ Storage Engine Exception; sense_reason %d\n", sense_reason); @@ -1741,14 +2047,16 @@ void transport_generic_request_failure(struct se_cmd *cmd, * For SAM Task Attribute emulation for failed struct se_cmd */ transport_complete_task_attr(cmd); - /* - * Handle special case for COMPARE_AND_WRITE failure, where the - * callback is expected to drop the per device ->caw_sem. - */ - if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && - cmd->transport_complete_callback) + + if (cmd->transport_complete_callback) cmd->transport_complete_callback(cmd, false, &post_ret); + if (cmd->transport_state & CMD_T_ABORTED) { + INIT_WORK(&cmd->work, target_abort_work); + queue_work(target_completion_wq, &cmd->work); + return; + } + switch (sense_reason) { case TCM_NON_EXISTENT_LUN: case TCM_UNSUPPORTED_SCSI_OPCODE: @@ -1761,7 +2069,6 @@ void transport_generic_request_failure(struct se_cmd *cmd, case TCM_ADDRESS_OUT_OF_RANGE: case TCM_CHECK_CONDITION_ABORT_CMD: case TCM_CHECK_CONDITION_UNIT_ATTENTION: - case TCM_CHECK_CONDITION_NOT_READY: case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: @@ -1770,10 +2077,18 @@ void transport_generic_request_failure(struct se_cmd *cmd, case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE: case TCM_TOO_MANY_SEGMENT_DESCS: case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE: + case TCM_INVALID_FIELD_IN_COMMAND_IU: + case TCM_ALUA_TG_PT_STANDBY: + case TCM_ALUA_TG_PT_UNAVAILABLE: + case TCM_ALUA_STATE_TRANSITION: + case TCM_ALUA_OFFLINE: break; case TCM_OUT_OF_RESOURCES: - sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - break; + cmd->scsi_status = SAM_STAT_TASK_SET_FULL; + goto queue_status; + case TCM_LUN_BUSY: + cmd->scsi_status = SAM_STAT_BUSY; + goto queue_status; case TCM_RESERVATION_CONFLICT: /* * No SENSE Data payload for this case, set SCSI Status @@ -1790,16 +2105,14 @@ void transport_generic_request_failure(struct se_cmd *cmd, * See spc4r17, section 7.4.6 Control Mode Page, Table 349 */ if (cmd->se_sess && - cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) { + cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl + == TARGET_UA_INTLCK_CTRL_ESTABLISH_UA) { target_ua_allocate_lun(cmd->se_sess->se_node_acl, cmd->orig_fe_lun, 0x2C, ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS); } - trace_target_cmd_complete(cmd); - ret = cmd->se_tfo->queue_status(cmd); - if (ret) - goto queue_full; - goto check_stop; + + goto queue_status; default: pr_err("Unknown transport error for CDB 0x%02x: %d\n", cmd->t_task_cdb[0], sense_reason); @@ -1816,6 +2129,11 @@ check_stop: transport_cmd_check_stop_to_fabric(cmd); return; +queue_status: + trace_target_cmd_complete(cmd); + ret = cmd->se_tfo->queue_status(cmd); + if (!ret) + goto check_stop; queue_full: transport_handle_queue_full(cmd, cmd->se_dev, ret, false); } @@ -1900,8 +2218,9 @@ static int target_write_prot_action(struct se_cmd *cmd) static bool target_handle_task_attr(struct se_cmd *cmd) { struct se_device *dev = cmd->se_dev; + unsigned long flags; - if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) + if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) return false; cmd->se_cmd_flags |= SCF_TASK_ATTR_SET; @@ -1916,62 +2235,60 @@ static bool target_handle_task_attr(struct se_cmd *cmd) cmd->t_task_cdb[0]); return false; case TCM_ORDERED_TAG: - atomic_inc_mb(&dev->dev_ordered_sync); - pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n", cmd->t_task_cdb[0]); - - /* - * Execute an ORDERED command if no other older commands - * exist that need to be completed first. - */ - if (!atomic_read(&dev->simple_cmds)) - return false; break; default: /* * For SIMPLE and UNTAGGED Task Attribute commands */ - atomic_inc_mb(&dev->simple_cmds); +retry: + if (percpu_ref_tryget_live(&dev->non_ordered)) + return false; + break; } - if (atomic_read(&dev->dev_ordered_sync) == 0) - return false; + spin_lock_irqsave(&dev->delayed_cmd_lock, flags); + if (cmd->sam_task_attr == TCM_SIMPLE_TAG && + !percpu_ref_is_dying(&dev->non_ordered)) { + spin_unlock_irqrestore(&dev->delayed_cmd_lock, flags); + /* We raced with the last ordered completion so retry. */ + goto retry; + } else if (!percpu_ref_is_dying(&dev->non_ordered)) { + percpu_ref_kill(&dev->non_ordered); + } + + spin_lock(&cmd->t_state_lock); + cmd->transport_state &= ~CMD_T_SENT; + spin_unlock(&cmd->t_state_lock); - spin_lock(&dev->delayed_cmd_lock); list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list); - spin_unlock(&dev->delayed_cmd_lock); + spin_unlock_irqrestore(&dev->delayed_cmd_lock, flags); pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn", cmd->t_task_cdb[0], cmd->sam_task_attr); + /* + * We may have no non ordered cmds when this function started or we + * could have raced with the last simple/head cmd completing, so kick + * the delayed handler here. + */ + schedule_work(&dev->delayed_cmd_work); return true; } -static int __transport_check_aborted_status(struct se_cmd *, int); - void target_execute_cmd(struct se_cmd *cmd) { /* * Determine if frontend context caller is requesting the stopping of * this command for frontend exceptions. * - * If the received CDB has aleady been aborted stop processing it here. + * If the received CDB has already been aborted stop processing it here. */ - spin_lock_irq(&cmd->t_state_lock); - if (__transport_check_aborted_status(cmd, 1)) { - spin_unlock_irq(&cmd->t_state_lock); + if (target_cmd_interrupted(cmd)) return; - } - if (cmd->transport_state & CMD_T_STOP) { - pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", - __func__, __LINE__, cmd->tag); - - spin_unlock_irq(&cmd->t_state_lock); - complete_all(&cmd->t_transport_stop_comp); - return; - } + spin_lock_irq(&cmd->t_state_lock); cmd->t_state = TRANSPORT_PROCESSING; cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT; spin_unlock_irq(&cmd->t_state_lock); @@ -1979,12 +2296,8 @@ void target_execute_cmd(struct se_cmd *cmd) if (target_write_prot_action(cmd)) return; - if (target_handle_task_attr(cmd)) { - spin_lock_irq(&cmd->t_state_lock); - cmd->transport_state &= ~CMD_T_SENT; - spin_unlock_irq(&cmd->t_state_lock); + if (target_handle_task_attr(cmd)) return; - } __target_execute_cmd(cmd, true); } @@ -1994,27 +2307,59 @@ EXPORT_SYMBOL(target_execute_cmd); * Process all commands up to the last received ORDERED task attribute which * requires another blocking boundary */ -static void target_restart_delayed_cmds(struct se_device *dev) +void target_do_delayed_work(struct work_struct *work) { - for (;;) { + struct se_device *dev = container_of(work, struct se_device, + delayed_cmd_work); + + spin_lock(&dev->delayed_cmd_lock); + while (!dev->ordered_sync_in_progress) { struct se_cmd *cmd; - spin_lock(&dev->delayed_cmd_lock); - if (list_empty(&dev->delayed_cmd_list)) { - spin_unlock(&dev->delayed_cmd_lock); + /* + * We can be woken up early/late due to races or the + * extra wake up we do when adding commands to the list. + * We check for both cases here. + */ + if (list_empty(&dev->delayed_cmd_list) || + !percpu_ref_is_zero(&dev->non_ordered)) break; - } cmd = list_entry(dev->delayed_cmd_list.next, struct se_cmd, se_delayed_node); + cmd->se_cmd_flags |= SCF_TASK_ORDERED_SYNC; + cmd->transport_state |= CMD_T_SENT; + + dev->ordered_sync_in_progress = true; + list_del(&cmd->se_delayed_node); spin_unlock(&dev->delayed_cmd_lock); __target_execute_cmd(cmd, true); - - if (cmd->sam_task_attr == TCM_ORDERED_TAG) - break; + spin_lock(&dev->delayed_cmd_lock); } + spin_unlock(&dev->delayed_cmd_lock); +} + +static void transport_complete_ordered_sync(struct se_cmd *cmd) +{ + struct se_device *dev = cmd->se_dev; + unsigned long flags; + + spin_lock_irqsave(&dev->delayed_cmd_lock, flags); + dev->dev_cur_ordered_id++; + + pr_debug("Incremented dev_cur_ordered_id: %u for type %d\n", + dev->dev_cur_ordered_id, cmd->sam_task_attr); + + dev->ordered_sync_in_progress = false; + + if (list_empty(&dev->delayed_cmd_list)) + percpu_ref_resurrect(&dev->non_ordered); + else + schedule_work(&dev->delayed_cmd_work); + + spin_unlock_irqrestore(&dev->delayed_cmd_lock, flags); } /* @@ -2025,28 +2370,28 @@ static void transport_complete_task_attr(struct se_cmd *cmd) { struct se_device *dev = cmd->se_dev; - if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) + if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH) return; if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET)) - goto restart; + return; - if (cmd->sam_task_attr == TCM_SIMPLE_TAG) { - atomic_dec_mb(&dev->simple_cmds); - dev->dev_cur_ordered_id++; - } else if (cmd->sam_task_attr == TCM_HEAD_TAG) { - dev->dev_cur_ordered_id++; - pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n", - dev->dev_cur_ordered_id); - } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) { - atomic_dec_mb(&dev->dev_ordered_sync); + cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET; - dev->dev_cur_ordered_id++; - pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n", - dev->dev_cur_ordered_id); + if (cmd->se_cmd_flags & SCF_TASK_ORDERED_SYNC) { + transport_complete_ordered_sync(cmd); + return; + } + + switch (cmd->sam_task_attr) { + case TCM_SIMPLE_TAG: + percpu_ref_put(&dev->non_ordered); + break; + case TCM_ORDERED_TAG: + /* All ordered should have been executed as sync */ + WARN_ON(1); + break; } -restart: - target_restart_delayed_cmds(dev); } static void transport_complete_qf(struct se_cmd *cmd) @@ -2067,19 +2412,28 @@ static void transport_complete_qf(struct se_cmd *cmd) if (cmd->scsi_status) goto queue_status; - cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; - cmd->scsi_status = SAM_STAT_CHECK_CONDITION; - cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); goto queue_status; } - if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) + /* + * Check if we need to send a sense buffer from + * the struct se_cmd in question. We do NOT want + * to take this path of the IO has been marked as + * needing to be treated like a "normal read". This + * is the case if it's a tape read, and either the + * FM, EOM, or ILI bits are set, but there is no + * sense data. + */ + if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) && + cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) goto queue_status; switch (cmd->data_direction) { case DMA_FROM_DEVICE: - if (cmd->scsi_status) + /* queue status if not treating this as a normal read */ + if (cmd->scsi_status && + !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL)) goto queue_status; trace_target_cmd_complete(cmd); @@ -2090,7 +2444,7 @@ static void transport_complete_qf(struct se_cmd *cmd) ret = cmd->se_tfo->queue_data_in(cmd); break; } - /* Fall through for DMA_TO_DEVICE */ + fallthrough; case DMA_NONE: queue_status: trace_target_cmd_complete(cmd); @@ -2184,9 +2538,15 @@ static void target_complete_ok_work(struct work_struct *work) /* * Check if we need to send a sense buffer from - * the struct se_cmd in question. + * the struct se_cmd in question. We do NOT want + * to take this path of the IO has been marked as + * needing to be treated like a "normal read". This + * is the case if it's a tape read, and either the + * FM, EOM, or ILI bits are set, but there is no + * sense data. */ - if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { + if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) && + cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { WARN_ON(!cmd->scsi_status); ret = transport_send_check_condition_and_sense( cmd, 0, 1); @@ -2228,11 +2588,23 @@ static void target_complete_ok_work(struct work_struct *work) queue_rsp: switch (cmd->data_direction) { case DMA_FROM_DEVICE: - if (cmd->scsi_status) + /* + * if this is a READ-type IO, but SCSI status + * is set, then skip returning data and just + * return the status -- unless this IO is marked + * as needing to be treated as a normal read, + * in which case we want to go ahead and return + * the data. This happens, for example, for tape + * reads with the FM, EOM, or ILI bits set, with + * no sense data. + */ + if (cmd->scsi_status && + !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL)) goto queue_status; - atomic_long_add(cmd->data_length, - &cmd->se_lun->lun_stats.tx_data_octets); + if (cmd->se_lun->lun_stats) + this_cpu_add(cmd->se_lun->lun_stats->tx_data_octets, + cmd->data_length); /* * Perform READ_STRIP of PI using software emulation when * backend had PI enabled, if the transport will not be @@ -2255,20 +2627,22 @@ queue_rsp: goto queue_full; break; case DMA_TO_DEVICE: - atomic_long_add(cmd->data_length, - &cmd->se_lun->lun_stats.rx_data_octets); + if (cmd->se_lun->lun_stats) + this_cpu_add(cmd->se_lun->lun_stats->rx_data_octets, + cmd->data_length); /* * Check if we need to send READ payload for BIDI-COMMAND */ if (cmd->se_cmd_flags & SCF_BIDI) { - atomic_long_add(cmd->data_length, - &cmd->se_lun->lun_stats.tx_data_octets); + if (cmd->se_lun->lun_stats) + this_cpu_add(cmd->se_lun->lun_stats->tx_data_octets, + cmd->data_length); ret = cmd->se_tfo->queue_data_in(cmd); if (ret) goto queue_full; break; } - /* Fall through for DMA_TO_DEVICE */ + fallthrough; case DMA_NONE: queue_status: trace_target_cmd_complete(cmd); @@ -2293,13 +2667,7 @@ queue_full: void target_free_sgl(struct scatterlist *sgl, int nents) { - struct scatterlist *sg; - int count; - - for_each_sg(sgl, sg, nents, count) - __free_page(sg_page(sg)); - - kfree(sgl); + sgl_free_n_order(sgl, nents, 0); } EXPORT_SYMBOL(target_free_sgl); @@ -2352,22 +2720,6 @@ static inline void transport_free_pages(struct se_cmd *cmd) cmd->t_bidi_data_nents = 0; } -/** - * transport_put_cmd - release a reference to a command - * @cmd: command to release - * - * This routine releases our reference to the command and frees it if possible. - */ -static int transport_put_cmd(struct se_cmd *cmd) -{ - BUG_ON(!cmd->se_tfo); - /* - * If this cmd has been setup with target_get_sess_cmd(), drop - * the kref and call ->release_cmd() in kref callback. - */ - return target_put_sess_cmd(cmd); -} - void *transport_kmap_data_sg(struct se_cmd *cmd) { struct scatterlist *sg = cmd->t_data_sg; @@ -2423,42 +2775,10 @@ int target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length, bool zero_page, bool chainable) { - struct scatterlist *sg; - struct page *page; - gfp_t zero_flag = (zero_page) ? __GFP_ZERO : 0; - unsigned int nalloc, nent; - int i = 0; - - nalloc = nent = DIV_ROUND_UP(length, PAGE_SIZE); - if (chainable) - nalloc++; - sg = kmalloc_array(nalloc, sizeof(struct scatterlist), GFP_KERNEL); - if (!sg) - return -ENOMEM; - - sg_init_table(sg, nalloc); + gfp_t gfp = GFP_KERNEL | (zero_page ? __GFP_ZERO : 0); - while (length) { - u32 page_len = min_t(u32, length, PAGE_SIZE); - page = alloc_page(GFP_KERNEL | zero_flag); - if (!page) - goto out; - - sg_set_page(&sg[i], page, page_len, 0); - length -= page_len; - i++; - } - *sgl = sg; - *nents = nent; - return 0; - -out: - while (i > 0) { - i--; - __free_page(sg_page(&sg[i])); - } - kfree(sg); - return -ENOMEM; + *sgl = sgl_alloc_order(length, 0, chainable, gfp, nents); + return *sgl ? 0 : -ENOMEM; } EXPORT_SYMBOL(target_alloc_sgl); @@ -2483,7 +2803,7 @@ transport_generic_new_cmd(struct se_cmd *cmd) } /* - * Determine is the TCM fabric module has already allocated physical + * Determine if the TCM fabric module has already allocated physical * memory, and is directly calling transport_generic_map_mem_to_cmd() * beforehand. */ @@ -2543,7 +2863,8 @@ transport_generic_new_cmd(struct se_cmd *cmd) * Determine if frontend context caller is requesting the stopping of * this command for frontend exceptions. */ - if (cmd->transport_state & CMD_T_STOP) { + if (cmd->transport_state & CMD_T_STOP && + !cmd->se_tfo->write_pending_must_be_called) { pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n", __func__, __LINE__, cmd->tag); @@ -2570,7 +2891,20 @@ EXPORT_SYMBOL(transport_generic_new_cmd); static void transport_write_pending_qf(struct se_cmd *cmd) { + unsigned long flags; int ret; + bool stop; + + spin_lock_irqsave(&cmd->t_state_lock, flags); + stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED)); + spin_unlock_irqrestore(&cmd->t_state_lock, flags); + + if (stop) { + pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n", + __func__, __LINE__, cmd->tag); + complete_all(&cmd->t_transport_stop_comp); + return; + } ret = cmd->se_tfo->write_pending(cmd); if (ret) { @@ -2593,20 +2927,53 @@ static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas) spin_unlock_irqrestore(&cmd->t_state_lock, flags); } +/* + * Call target_put_sess_cmd() and wait until target_release_cmd_kref(@cmd) has + * finished. + */ +void target_put_cmd_and_wait(struct se_cmd *cmd) +{ + DECLARE_COMPLETION_ONSTACK(compl); + + WARN_ON_ONCE(cmd->abrt_compl); + cmd->abrt_compl = &compl; + target_put_sess_cmd(cmd); + wait_for_completion(&compl); +} + +/* + * This function is called by frontend drivers after processing of a command + * has finished. + * + * The protocol for ensuring that either the regular frontend command + * processing flow or target_handle_abort() code drops one reference is as + * follows: + * - Calling .queue_data_in(), .queue_status() or queue_tm_rsp() will cause + * the frontend driver to call this function synchronously or asynchronously. + * That will cause one reference to be dropped. + * - During regular command processing the target core sets CMD_T_COMPLETE + * before invoking one of the .queue_*() functions. + * - The code that aborts commands skips commands and TMFs for which + * CMD_T_COMPLETE has been set. + * - CMD_T_ABORTED is set atomically after the CMD_T_COMPLETE check for + * commands that will be aborted. + * - If the CMD_T_ABORTED flag is set but CMD_T_TAS has not been set + * transport_generic_free_cmd() skips its call to target_put_sess_cmd(). + * - For aborted commands for which CMD_T_TAS has been set .queue_status() will + * be called and will drop a reference. + * - For aborted commands for which CMD_T_TAS has not been set .aborted_task() + * will be called. target_handle_abort() will drop the final reference. + */ int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) { + DECLARE_COMPLETION_ONSTACK(compl); int ret = 0; bool aborted = false, tas = false; - if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) { - if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) - target_wait_free_cmd(cmd, &aborted, &tas); + if (wait_for_tasks) + target_wait_free_cmd(cmd, &aborted, &tas); - if (!aborted || tas) - ret = transport_put_cmd(cmd); - } else { - if (wait_for_tasks) - target_wait_free_cmd(cmd, &aborted, &tas); + if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) { /* * Handle WRITE failure case where transport_generic_new_cmd() * has already added se_cmd to state_list, but fabric has @@ -2617,34 +2984,26 @@ int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) if (cmd->se_lun) transport_lun_remove_cmd(cmd); - - if (!aborted || tas) - ret = transport_put_cmd(cmd); } - /* - * If the task has been internally aborted due to TMR ABORT_TASK - * or LUN_RESET, target_core_tmr.c is responsible for performing - * the remaining calls to target_put_sess_cmd(), and not the - * callers of this function. - */ + if (aborted) + cmd->free_compl = &compl; + ret = target_put_sess_cmd(cmd); if (aborted) { pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag); - wait_for_completion(&cmd->cmd_wait_comp); - cmd->se_tfo->release_cmd(cmd); + wait_for_completion(&compl); ret = 1; } return ret; } EXPORT_SYMBOL(transport_generic_free_cmd); -/* target_get_sess_cmd - Add command to active ->sess_cmd_list +/** + * target_get_sess_cmd - Verify the session is accepting cmds and take ref * @se_cmd: command descriptor to add * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() */ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) { - struct se_session *se_sess = se_cmd->se_sess; - unsigned long flags; int ret = 0; /* @@ -2653,21 +3012,18 @@ int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref) * invocations before se_cmd descriptor release. */ if (ack_kref) { - if (!kref_get_unless_zero(&se_cmd->cmd_kref)) - return -EINVAL; - + kref_get(&se_cmd->cmd_kref); se_cmd->se_cmd_flags |= SCF_ACK_KREF; } - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - if (se_sess->sess_tearing_down) { - ret = -ESHUTDOWN; - goto out; + /* + * Users like xcopy do not use counters since they never do a stop + * and wait. + */ + if (se_cmd->cmd_cnt) { + if (!percpu_ref_tryget_live(&se_cmd->cmd_cnt->refcnt)) + ret = -ESHUTDOWN; } - list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list); -out: - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - if (ret && ack_kref) target_put_sess_cmd(se_cmd); @@ -2688,31 +3044,19 @@ static void target_free_cmd_mem(struct se_cmd *cmd) static void target_release_cmd_kref(struct kref *kref) { struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); - struct se_session *se_sess = se_cmd->se_sess; - unsigned long flags; - bool fabric_stop; - - if (se_sess) { - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - - spin_lock(&se_cmd->t_state_lock); - fabric_stop = (se_cmd->transport_state & CMD_T_FABRIC_STOP) && - (se_cmd->transport_state & CMD_T_ABORTED); - spin_unlock(&se_cmd->t_state_lock); - - if (se_cmd->cmd_wait_set || fabric_stop) { - list_del_init(&se_cmd->se_cmd_list); - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - target_free_cmd_mem(se_cmd); - complete(&se_cmd->cmd_wait_comp); - return; - } - list_del_init(&se_cmd->se_cmd_list); - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - } + struct target_cmd_counter *cmd_cnt = se_cmd->cmd_cnt; + struct completion *free_compl = se_cmd->free_compl; + struct completion *abrt_compl = se_cmd->abrt_compl; target_free_cmd_mem(se_cmd); se_cmd->se_tfo->release_cmd(se_cmd); + if (free_compl) + complete(free_compl); + if (abrt_compl) + complete(abrt_compl); + + if (cmd_cnt) + percpu_ref_put(&cmd_cnt->refcnt); } /** @@ -2801,6 +3145,7 @@ static const char *target_tmf_name(enum tcm_tmreq_table tmf) case TMR_LUN_RESET: return "LUN_RESET"; case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET"; case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET"; + case TMR_LUN_RESET_PRO: return "LUN_RESET_PRO"; case TMR_UNKNOWN: break; } return "(?)"; @@ -2830,112 +3175,76 @@ void target_show_cmd(const char *pfx, struct se_cmd *cmd) } EXPORT_SYMBOL(target_show_cmd); -/* target_sess_cmd_list_set_waiting - Flag all commands in - * sess_cmd_list to complete cmd_wait_comp. Set - * sess_tearing_down so no more commands are queued. - * @se_sess: session to flag - */ -void target_sess_cmd_list_set_waiting(struct se_session *se_sess) +static void target_stop_cmd_counter_confirm(struct percpu_ref *ref) { - struct se_cmd *se_cmd, *tmp_cmd; - unsigned long flags; - int rc; - - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - if (se_sess->sess_tearing_down) { - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - return; - } - se_sess->sess_tearing_down = 1; - list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list); - - list_for_each_entry_safe(se_cmd, tmp_cmd, - &se_sess->sess_wait_list, se_cmd_list) { - rc = kref_get_unless_zero(&se_cmd->cmd_kref); - if (rc) { - se_cmd->cmd_wait_set = 1; - spin_lock(&se_cmd->t_state_lock); - se_cmd->transport_state |= CMD_T_FABRIC_STOP; - spin_unlock(&se_cmd->t_state_lock); - } else - list_del_init(&se_cmd->se_cmd_list); - } - - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + struct target_cmd_counter *cmd_cnt = container_of(ref, + struct target_cmd_counter, + refcnt); + complete_all(&cmd_cnt->stop_done); } -EXPORT_SYMBOL(target_sess_cmd_list_set_waiting); -/* target_wait_for_sess_cmds - Wait for outstanding descriptors - * @se_sess: session to wait for active I/O +/** + * target_stop_cmd_counter - Stop new IO from being added to the counter. + * @cmd_cnt: counter to stop */ -void target_wait_for_sess_cmds(struct se_session *se_sess) +void target_stop_cmd_counter(struct target_cmd_counter *cmd_cnt) { - struct se_cmd *se_cmd, *tmp_cmd; - unsigned long flags; - bool tas; - - list_for_each_entry_safe(se_cmd, tmp_cmd, - &se_sess->sess_wait_list, se_cmd_list) { - pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:" - " %d\n", se_cmd, se_cmd->t_state, - se_cmd->se_tfo->get_cmd_state(se_cmd)); - - spin_lock_irqsave(&se_cmd->t_state_lock, flags); - tas = (se_cmd->transport_state & CMD_T_TAS); - spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); + pr_debug("Stopping command counter.\n"); + if (!atomic_cmpxchg(&cmd_cnt->stopped, 0, 1)) + percpu_ref_kill_and_confirm(&cmd_cnt->refcnt, + target_stop_cmd_counter_confirm); +} +EXPORT_SYMBOL_GPL(target_stop_cmd_counter); - if (!target_put_sess_cmd(se_cmd)) { - if (tas) - target_put_sess_cmd(se_cmd); - } +/** + * target_stop_session - Stop new IO from being queued on the session. + * @se_sess: session to stop + */ +void target_stop_session(struct se_session *se_sess) +{ + target_stop_cmd_counter(se_sess->cmd_cnt); +} +EXPORT_SYMBOL(target_stop_session); - wait_for_completion(&se_cmd->cmd_wait_comp); - pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d" - " fabric state: %d\n", se_cmd, se_cmd->t_state, - se_cmd->se_tfo->get_cmd_state(se_cmd)); +/** + * target_wait_for_cmds - Wait for outstanding cmds. + * @cmd_cnt: counter to wait for active I/O for. + */ +void target_wait_for_cmds(struct target_cmd_counter *cmd_cnt) +{ + int ret; - se_cmd->se_tfo->release_cmd(se_cmd); - } + WARN_ON_ONCE(!atomic_read(&cmd_cnt->stopped)); - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); - WARN_ON(!list_empty(&se_sess->sess_cmd_list)); - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + do { + pr_debug("Waiting for running cmds to complete.\n"); + ret = wait_event_timeout(cmd_cnt->refcnt_wq, + percpu_ref_is_zero(&cmd_cnt->refcnt), + 180 * HZ); + } while (ret <= 0); + wait_for_completion(&cmd_cnt->stop_done); + pr_debug("Waiting for cmds done.\n"); } -EXPORT_SYMBOL(target_wait_for_sess_cmds); +EXPORT_SYMBOL_GPL(target_wait_for_cmds); -static void target_lun_confirm(struct percpu_ref *ref) +/** + * target_wait_for_sess_cmds - Wait for outstanding commands + * @se_sess: session to wait for active I/O + */ +void target_wait_for_sess_cmds(struct se_session *se_sess) { - struct se_lun *lun = container_of(ref, struct se_lun, lun_ref); - - complete(&lun->lun_ref_comp); + target_wait_for_cmds(se_sess->cmd_cnt); } +EXPORT_SYMBOL(target_wait_for_sess_cmds); +/* + * Prevent that new percpu_ref_tryget_live() calls succeed and wait until + * all references to the LUN have been released. Called during LUN shutdown. + */ void transport_clear_lun_ref(struct se_lun *lun) { - /* - * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop - * the initial reference and schedule confirm kill to be - * executed after one full RCU grace period has completed. - */ - percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm); - /* - * The first completion waits for percpu_ref_switch_to_atomic_rcu() - * to call target_lun_confirm after lun->lun_ref has been marked - * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t - * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref - * fails for all new incoming I/O. - */ - wait_for_completion(&lun->lun_ref_comp); - /* - * The second completion waits for percpu_ref_put_many() to - * invoke ->release() after lun->lun_ref has switched to - * atomic_t mode, and lun->lun_ref.count has reached zero. - * - * At this point all target-core lun->lun_ref references have - * been dropped via transport_lun_remove_cmd(), and it's safe - * to proceed with the remaining LUN shutdown. - */ + percpu_ref_kill(&lun->lun_ref); wait_for_completion(&lun->lun_shutdown_comp); } @@ -2945,9 +3254,7 @@ __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop, __releases(&cmd->t_state_lock) __acquires(&cmd->t_state_lock) { - - assert_spin_locked(&cmd->t_state_lock); - WARN_ON_ONCE(!irqs_disabled()); + lockdep_assert_held(&cmd->t_state_lock); if (fabric_stop) cmd->transport_state |= CMD_T_FABRIC_STOP; @@ -3008,14 +3315,14 @@ bool transport_wait_for_tasks(struct se_cmd *cmd) } EXPORT_SYMBOL(transport_wait_for_tasks); -struct sense_info { +struct sense_detail { u8 key; u8 asc; u8 ascq; - bool add_sector_info; + bool add_sense_info; }; -static const struct sense_info sense_info_table[] = { +static const struct sense_detail sense_detail_table[] = { [TCM_NO_SENSE] = { .key = NOT_READY }, @@ -3103,31 +3410,29 @@ static const struct sense_info sense_info_table[] = { [TCM_CHECK_CONDITION_UNIT_ATTENTION] = { .key = UNIT_ATTENTION, }, - [TCM_CHECK_CONDITION_NOT_READY] = { - .key = NOT_READY, - }, [TCM_MISCOMPARE_VERIFY] = { .key = MISCOMPARE, .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */ .ascq = 0x00, + .add_sense_info = true, }, [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = { .key = ABORTED_COMMAND, .asc = 0x10, .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */ - .add_sector_info = true, + .add_sense_info = true, }, [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = { .key = ABORTED_COMMAND, .asc = 0x10, .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */ - .add_sector_info = true, + .add_sense_info = true, }, [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = { .key = ABORTED_COMMAND, .asc = 0x10, .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */ - .add_sector_info = true, + .add_sense_info = true, }, [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = { .key = COPY_ABORTED, @@ -3145,41 +3450,94 @@ static const struct sense_info sense_info_table[] = { .key = NOT_READY, .asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */ }, + [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = { + /* + * From spc4r22 section5.7.7,5.7.8 + * If a PERSISTENT RESERVE OUT command with a REGISTER service action + * or a REGISTER AND IGNORE EXISTING KEY service action or + * REGISTER AND MOVE service actionis attempted, + * but there are insufficient device server resources to complete the + * operation, then the command shall be terminated with CHECK CONDITION + * status, with the sense key set to ILLEGAL REQUEST,and the additonal + * sense code set to INSUFFICIENT REGISTRATION RESOURCES. + */ + .key = ILLEGAL_REQUEST, + .asc = 0x55, + .ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */ + }, + [TCM_INVALID_FIELD_IN_COMMAND_IU] = { + .key = ILLEGAL_REQUEST, + .asc = 0x0e, + .ascq = 0x03, /* INVALID FIELD IN COMMAND INFORMATION UNIT */ + }, + [TCM_ALUA_TG_PT_STANDBY] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_TG_PT_STANDBY, + }, + [TCM_ALUA_TG_PT_UNAVAILABLE] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_TG_PT_UNAVAILABLE, + }, + [TCM_ALUA_STATE_TRANSITION] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_STATE_TRANSITION, + }, + [TCM_ALUA_OFFLINE] = { + .key = NOT_READY, + .asc = 0x04, + .ascq = ASCQ_04H_ALUA_OFFLINE, + }, }; -static int translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason) +/** + * translate_sense_reason - translate a sense reason into T10 key, asc and ascq + * @cmd: SCSI command in which the resulting sense buffer or SCSI status will + * be stored. + * @reason: LIO sense reason code. If this argument has the value + * TCM_CHECK_CONDITION_UNIT_ATTENTION, try to dequeue a unit attention. If + * dequeuing a unit attention fails due to multiple commands being processed + * concurrently, set the command status to BUSY. + * + * Return: 0 upon success or -EINVAL if the sense buffer is too small. + */ +static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason) { - const struct sense_info *si; + const struct sense_detail *sd; u8 *buffer = cmd->sense_buffer; int r = (__force int)reason; - u8 asc, ascq; + u8 key, asc, ascq; bool desc_format = target_sense_desc_format(cmd->se_dev); - if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key) - si = &sense_info_table[r]; + if (r < ARRAY_SIZE(sense_detail_table) && sense_detail_table[r].key) + sd = &sense_detail_table[r]; else - si = &sense_info_table[(__force int) + sd = &sense_detail_table[(__force int) TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE]; + key = sd->key; if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) { - core_scsi3_ua_for_check_condition(cmd, &asc, &ascq); - WARN_ON_ONCE(asc == 0); - } else if (si->asc == 0) { - WARN_ON_ONCE(cmd->scsi_asc == 0); - asc = cmd->scsi_asc; - ascq = cmd->scsi_ascq; + if (!core_scsi3_ua_for_check_condition(cmd, &key, &asc, + &ascq)) { + cmd->scsi_status = SAM_STAT_BUSY; + return; + } } else { - asc = si->asc; - ascq = si->ascq; + WARN_ON_ONCE(sd->asc == 0); + asc = sd->asc; + ascq = sd->ascq; } - scsi_build_sense_buffer(desc_format, buffer, si->key, asc, ascq); - if (si->add_sector_info) - return scsi_set_sense_information(buffer, - cmd->scsi_sense_length, - cmd->bad_sector); - - return 0; + cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; + cmd->scsi_status = SAM_STAT_CHECK_CONDITION; + cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; + scsi_build_sense_buffer(desc_format, buffer, key, asc, ascq); + if (sd->add_sense_info) + WARN_ON_ONCE(scsi_set_sense_information(buffer, + cmd->scsi_sense_length, + cmd->sense_info) < 0); } int @@ -3188,6 +3546,8 @@ transport_send_check_condition_and_sense(struct se_cmd *cmd, { unsigned long flags; + WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB); + spin_lock_irqsave(&cmd->t_state_lock, flags); if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) { spin_unlock_irqrestore(&cmd->t_state_lock, flags); @@ -3196,130 +3556,39 @@ transport_send_check_condition_and_sense(struct se_cmd *cmd, cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION; spin_unlock_irqrestore(&cmd->t_state_lock, flags); - if (!from_transport) { - int rc; - - cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; - cmd->scsi_status = SAM_STAT_CHECK_CONDITION; - cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; - rc = translate_sense_reason(cmd, reason); - if (rc) - return rc; - } + if (!from_transport) + translate_sense_reason(cmd, reason); trace_target_cmd_complete(cmd); return cmd->se_tfo->queue_status(cmd); } EXPORT_SYMBOL(transport_send_check_condition_and_sense); -static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status) - __releases(&cmd->t_state_lock) - __acquires(&cmd->t_state_lock) -{ - int ret; - - assert_spin_locked(&cmd->t_state_lock); - WARN_ON_ONCE(!irqs_disabled()); - - if (!(cmd->transport_state & CMD_T_ABORTED)) - return 0; - /* - * If cmd has been aborted but either no status is to be sent or it has - * already been sent, just return - */ - if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) { - if (send_status) - cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; - return 1; - } - - pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:" - " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag); - - cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS; - cmd->scsi_status = SAM_STAT_TASK_ABORTED; - trace_target_cmd_complete(cmd); - - spin_unlock_irq(&cmd->t_state_lock); - ret = cmd->se_tfo->queue_status(cmd); - if (ret) - transport_handle_queue_full(cmd, cmd->se_dev, ret, false); - spin_lock_irq(&cmd->t_state_lock); - - return 1; -} - -int transport_check_aborted_status(struct se_cmd *cmd, int send_status) -{ - int ret; - - spin_lock_irq(&cmd->t_state_lock); - ret = __transport_check_aborted_status(cmd, send_status); - spin_unlock_irq(&cmd->t_state_lock); - - return ret; -} -EXPORT_SYMBOL(transport_check_aborted_status); - -void transport_send_task_abort(struct se_cmd *cmd) +/** + * target_send_busy - Send SCSI BUSY status back to the initiator + * @cmd: SCSI command for which to send a BUSY reply. + * + * Note: Only call this function if target_submit_cmd*() failed. + */ +int target_send_busy(struct se_cmd *cmd) { - unsigned long flags; - int ret; - - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) { - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - return; - } - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - - /* - * If there are still expected incoming fabric WRITEs, we wait - * until until they have completed before sending a TASK_ABORTED - * response. This response with TASK_ABORTED status will be - * queued back to fabric module by transport_check_aborted_status(). - */ - if (cmd->data_direction == DMA_TO_DEVICE) { - if (cmd->se_tfo->write_pending_status(cmd) != 0) { - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) { - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - goto send_abort; - } - cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - return; - } - } -send_abort: - cmd->scsi_status = SAM_STAT_TASK_ABORTED; - - transport_lun_remove_cmd(cmd); - - pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n", - cmd->t_task_cdb[0], cmd->tag); + WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB); + cmd->scsi_status = SAM_STAT_BUSY; trace_target_cmd_complete(cmd); - ret = cmd->se_tfo->queue_status(cmd); - if (ret) - transport_handle_queue_full(cmd, cmd->se_dev, ret, false); + return cmd->se_tfo->queue_status(cmd); } +EXPORT_SYMBOL(target_send_busy); static void target_tmr_work(struct work_struct *work) { struct se_cmd *cmd = container_of(work, struct se_cmd, work); struct se_device *dev = cmd->se_dev; struct se_tmr_req *tmr = cmd->se_tmr_req; - unsigned long flags; int ret; - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->transport_state & CMD_T_ABORTED) { - tmr->response = TMR_FUNCTION_REJECTED; - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - goto check_stop; - } - spin_unlock_irqrestore(&cmd->t_state_lock, flags); + if (cmd->transport_state & CMD_T_ABORTED) + goto aborted; switch (tmr->function) { case TMR_ABORT_TASK: @@ -3335,8 +3604,7 @@ static void target_tmr_work(struct work_struct *work) tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE : TMR_FUNCTION_REJECTED; if (tmr->response == TMR_FUNCTION_COMPLETE) { - target_ua_allocate_lun(cmd->se_sess->se_node_acl, - cmd->orig_fe_lun, 0x29, + target_dev_ua_allocate(dev, 0x29, ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED); } break; @@ -3347,24 +3615,23 @@ static void target_tmr_work(struct work_struct *work) tmr->response = TMR_FUNCTION_REJECTED; break; default: - pr_err("Uknown TMR function: 0x%02x.\n", + pr_err("Unknown TMR function: 0x%02x.\n", tmr->function); tmr->response = TMR_FUNCTION_REJECTED; break; } - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->transport_state & CMD_T_ABORTED) { - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - goto check_stop; - } - spin_unlock_irqrestore(&cmd->t_state_lock, flags); + if (cmd->transport_state & CMD_T_ABORTED) + goto aborted; cmd->se_tfo->queue_tm_rsp(cmd); -check_stop: transport_lun_remove_cmd(cmd); transport_cmd_check_stop_to_fabric(cmd); + return; + +aborted: + target_handle_abort(cmd); } int transport_generic_handle_tmr( @@ -3373,6 +3640,10 @@ int transport_generic_handle_tmr( unsigned long flags; bool aborted = false; + spin_lock_irqsave(&cmd->se_dev->se_tmr_lock, flags); + list_add_tail(&cmd->se_tmr_req->tmr_list, &cmd->se_dev->dev_tmr_list); + spin_unlock_irqrestore(&cmd->se_dev->se_tmr_lock, flags); + spin_lock_irqsave(&cmd->t_state_lock, flags); if (cmd->transport_state & CMD_T_ABORTED) { aborted = true; @@ -3383,16 +3654,15 @@ int transport_generic_handle_tmr( spin_unlock_irqrestore(&cmd->t_state_lock, flags); if (aborted) { - pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d" - "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function, - cmd->se_tmr_req->ref_task_tag, cmd->tag); - transport_lun_remove_cmd(cmd); - transport_cmd_check_stop_to_fabric(cmd); + pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d ref_tag: %llu tag: %llu\n", + cmd->se_tmr_req->function, + cmd->se_tmr_req->ref_task_tag, cmd->tag); + target_handle_abort(cmd); return 0; } INIT_WORK(&cmd->work, target_tmr_work); - queue_work(cmd->se_dev->tmr_wq, &cmd->work); + schedule_work(&cmd->work); return 0; } EXPORT_SYMBOL(transport_generic_handle_tmr); |