diff options
Diffstat (limited to 'drivers/target/iscsi/iscsi_target.c')
| -rw-r--r-- | drivers/target/iscsi/iscsi_target.c | 940 |
1 files changed, 514 insertions, 426 deletions
diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index bd15a564fe24..a2dde08c8a62 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /******************************************************************************* * This file contains main functions related to the iSCSI Target Core Driver. * @@ -5,18 +6,9 @@ * * Author: Nicholas A. Bellinger <nab@linux-iscsi.org> * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. ******************************************************************************/ -#include <crypto/hash.h> +#include <linux/crc32c.h> #include <linux/string.h> #include <linux/kthread.h> #include <linux/completion.h> @@ -25,7 +17,7 @@ #include <linux/idr.h> #include <linux/delay.h> #include <linux/sched/signal.h> -#include <asm/unaligned.h> +#include <linux/unaligned.h> #include <linux/inet.h> #include <net/ipv6.h> #include <scsi/scsi_proto.h> @@ -34,6 +26,7 @@ #include <target/target_core_base.h> #include <target/target_core_fabric.h> +#include <target/target_core_backend.h> #include <target/iscsi/iscsi_target_core.h> #include "iscsi_target_parameters.h" #include "iscsi_target_seq_pdu_list.h" @@ -67,7 +60,7 @@ struct kmem_cache *lio_dr_cache; struct kmem_cache *lio_ooo_cache; struct kmem_cache *lio_r2t_cache; -static int iscsit_handle_immediate_data(struct iscsi_cmd *, +static int iscsit_handle_immediate_data(struct iscsit_cmd *, struct iscsi_scsi_req *, u32); struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf) @@ -308,9 +301,6 @@ bool iscsit_check_np_match( return false; } -/* - * Called with mutex np_lock held - */ static struct iscsi_np *iscsit_get_np( struct sockaddr_storage *sockaddr, int network_transport) @@ -318,6 +308,8 @@ static struct iscsi_np *iscsit_get_np( struct iscsi_np *np; bool match; + lockdep_assert_held(&np_lock); + list_for_each_entry(np, &g_np_list, np_list) { spin_lock_bh(&np->np_thread_lock); if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { @@ -372,8 +364,6 @@ struct iscsi_np *iscsit_add_np( init_completion(&np->np_restart_comp); INIT_LIST_HEAD(&np->np_list); - timer_setup(&np->np_login_timer, iscsi_handle_login_thread_timeout, 0); - ret = iscsi_target_setup_login_socket(np, sockaddr); if (ret != 0) { kfree(np); @@ -481,19 +471,18 @@ int iscsit_del_np(struct iscsi_np *np) return 0; } -static void iscsit_get_rx_pdu(struct iscsi_conn *); +static void iscsit_get_rx_pdu(struct iscsit_conn *); -int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd) +int iscsit_queue_rsp(struct iscsit_conn *conn, struct iscsit_cmd *cmd) { return iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); } EXPORT_SYMBOL(iscsit_queue_rsp); -void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) +void iscsit_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd) { spin_lock_bh(&conn->cmd_lock); - if (!list_empty(&cmd->i_conn_node) && - !(cmd->se_cmd.transport_state & CMD_T_FABRIC_STOP)) + if (!list_empty(&cmd->i_conn_node)) list_del_init(&cmd->i_conn_node); spin_unlock_bh(&conn->cmd_lock); @@ -501,12 +490,12 @@ void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) } EXPORT_SYMBOL(iscsit_aborted_task); -static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *, - u32, u32, const void *, void *); -static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *); +static u32 iscsit_crc_buf(const void *buf, u32 payload_length, + u32 padding, const void *pad_bytes); +static void iscsit_tx_thread_wait_for_tcp(struct iscsit_conn *); static int -iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, const void *data_buf, u32 data_buf_len) { struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu; @@ -521,9 +510,7 @@ iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, if (conn->conn_ops->HeaderDigest) { u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr, - ISCSI_HDR_LEN, 0, NULL, - header_digest); + *header_digest = iscsit_crc_buf(hdr, ISCSI_HDR_LEN, 0, NULL); iov[0].iov_len += ISCSI_CRC_LEN; tx_size += ISCSI_CRC_LEN; @@ -548,11 +535,9 @@ iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } if (conn->conn_ops->DataDigest) { - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, - data_buf, data_buf_len, - padding, &cmd->pad_bytes, - &cmd->data_crc); - + cmd->data_crc = iscsit_crc_buf(data_buf, data_buf_len, + padding, + &cmd->pad_bytes); iov[niov].iov_base = &cmd->data_crc; iov[niov++].iov_len = ISCSI_CRC_LEN; tx_size += ISCSI_CRC_LEN; @@ -574,12 +559,13 @@ iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, return 0; } -static int iscsit_map_iovec(struct iscsi_cmd *, struct kvec *, u32, u32); -static void iscsit_unmap_iovec(struct iscsi_cmd *); -static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *, - u32, u32, u32, u8 *); +static int iscsit_map_iovec(struct iscsit_cmd *cmd, struct kvec *iov, int nvec, + u32 data_offset, u32 data_length); +static void iscsit_unmap_iovec(struct iscsit_cmd *); +static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length, + u32 padding, const u8 *pad_bytes); static int -iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, const struct iscsi_datain *datain) { struct kvec *iov; @@ -594,10 +580,8 @@ iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, if (conn->conn_ops->HeaderDigest) { u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; - iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu, - ISCSI_HDR_LEN, 0, NULL, - header_digest); - + *header_digest = iscsit_crc_buf(cmd->pdu, ISCSI_HDR_LEN, 0, + NULL); iov[0].iov_len += ISCSI_CRC_LEN; tx_size += ISCSI_CRC_LEN; @@ -605,7 +589,8 @@ iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, *header_digest); } - iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], + iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count], + cmd->orig_iov_data_count - (iov_count + 2), datain->offset, datain->length); if (iov_ret < 0) return -1; @@ -623,12 +608,8 @@ iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } if (conn->conn_ops->DataDigest) { - cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash, - cmd, datain->offset, - datain->length, - cmd->padding, - cmd->pad_bytes); - + cmd->data_crc = iscsit_crc_sglist(cmd, datain->length, + cmd->padding, cmd->pad_bytes); iov[iov_count].iov_base = &cmd->data_crc; iov[iov_count++].iov_len = ISCSI_CRC_LEN; tx_size += ISCSI_CRC_LEN; @@ -652,7 +633,7 @@ iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, return 0; } -static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +static int iscsit_xmit_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd, struct iscsi_datain_req *dr, const void *buf, u32 buf_len) { @@ -662,7 +643,7 @@ static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); } -static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn) +static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsit_conn *conn) { return TARGET_PROT_NORMAL; } @@ -710,13 +691,19 @@ static int __init iscsi_target_init_module(void) if (!iscsit_global->ts_bitmap) goto configfs_out; + if (!zalloc_cpumask_var(&iscsit_global->allowed_cpumask, GFP_KERNEL)) { + pr_err("Unable to allocate iscsit_global->allowed_cpumask\n"); + goto bitmap_out; + } + cpumask_setall(iscsit_global->allowed_cpumask); + lio_qr_cache = kmem_cache_create("lio_qr_cache", sizeof(struct iscsi_queue_req), __alignof__(struct iscsi_queue_req), 0, NULL); if (!lio_qr_cache) { pr_err("Unable to kmem_cache_create() for" " lio_qr_cache\n"); - goto bitmap_out; + goto cpumask_out; } lio_dr_cache = kmem_cache_create("lio_dr_cache", @@ -761,6 +748,8 @@ dr_out: kmem_cache_destroy(lio_dr_cache); qr_out: kmem_cache_destroy(lio_qr_cache); +cpumask_out: + free_cpumask_var(iscsit_global->allowed_cpumask); bitmap_out: vfree(iscsit_global->ts_bitmap); configfs_out: @@ -790,16 +779,17 @@ static void __exit iscsi_target_cleanup_module(void) target_unregister_template(&iscsi_ops); + free_cpumask_var(iscsit_global->allowed_cpumask); vfree(iscsit_global->ts_bitmap); kfree(iscsit_global); } int iscsit_add_reject( - struct iscsi_conn *conn, + struct iscsit_conn *conn, u8 reason, unsigned char *buf) { - struct iscsi_cmd *cmd; + struct iscsit_cmd *cmd; cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); if (!cmd) @@ -827,12 +817,12 @@ int iscsit_add_reject( EXPORT_SYMBOL(iscsit_add_reject); static int iscsit_add_reject_from_cmd( - struct iscsi_cmd *cmd, + struct iscsit_cmd *cmd, u8 reason, bool add_to_conn, unsigned char *buf) { - struct iscsi_conn *conn; + struct iscsit_conn *conn; const bool do_put = cmd->se_cmd.se_tfo != NULL; if (!cmd->conn) { @@ -871,13 +861,13 @@ static int iscsit_add_reject_from_cmd( return -1; } -static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason, +static int iscsit_add_reject_cmd(struct iscsit_cmd *cmd, u8 reason, unsigned char *buf) { return iscsit_add_reject_from_cmd(cmd, reason, true, buf); } -int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf) +int iscsit_reject_cmd(struct iscsit_cmd *cmd, u8 reason, unsigned char *buf) { return iscsit_add_reject_from_cmd(cmd, reason, false, buf); } @@ -887,13 +877,10 @@ EXPORT_SYMBOL(iscsit_reject_cmd); * Map some portion of the allocated scatterlist to an iovec, suitable for * kernel sockets to copy data in/out. */ -static int iscsit_map_iovec( - struct iscsi_cmd *cmd, - struct kvec *iov, - u32 data_offset, - u32 data_length) +static int iscsit_map_iovec(struct iscsit_cmd *cmd, struct kvec *iov, int nvec, + u32 data_offset, u32 data_length) { - u32 i = 0; + u32 i = 0, orig_data_length = data_length; struct scatterlist *sg; unsigned int page_off; @@ -902,9 +889,12 @@ static int iscsit_map_iovec( */ u32 ent = data_offset / PAGE_SIZE; + if (!data_length) + return 0; + if (ent >= cmd->se_cmd.t_data_nents) { pr_err("Initial page entry out-of-bounds\n"); - return -1; + goto overflow; } sg = &cmd->se_cmd.t_data_sg[ent]; @@ -914,7 +904,12 @@ static int iscsit_map_iovec( cmd->first_data_sg_off = page_off; while (data_length) { - u32 cur_len = min_t(u32, data_length, sg->length - page_off); + u32 cur_len; + + if (WARN_ON_ONCE(!sg || i >= nvec)) + goto overflow; + + cur_len = min_t(u32, data_length, sg->length - page_off); iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off; iov[i].iov_len = cur_len; @@ -928,9 +923,19 @@ static int iscsit_map_iovec( cmd->kmapped_nents = i; return i; + +overflow: + pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n", + data_offset, orig_data_length, i, nvec); + for_each_sg(cmd->se_cmd.t_data_sg, sg, + cmd->se_cmd.t_data_nents, i) { + pr_err("[%d] off %d len %d\n", + i, sg->offset, sg->length); + } + return -1; } -static void iscsit_unmap_iovec(struct iscsi_cmd *cmd) +static void iscsit_unmap_iovec(struct iscsit_cmd *cmd) { u32 i; struct scatterlist *sg; @@ -941,10 +946,10 @@ static void iscsit_unmap_iovec(struct iscsi_cmd *cmd) kunmap(sg_page(&sg[i])); } -static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) +static void iscsit_ack_from_expstatsn(struct iscsit_conn *conn, u32 exp_statsn) { LIST_HEAD(ack_list); - struct iscsi_cmd *cmd, *cmd_p; + struct iscsit_cmd *cmd, *cmd_p; conn->exp_statsn = exp_statsn; @@ -971,7 +976,7 @@ static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) } } -static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) +static int iscsit_allocate_iovecs(struct iscsit_cmd *cmd) { u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE)); @@ -984,12 +989,14 @@ static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) return 0; } -int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +int iscsit_setup_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, unsigned char *buf) { int data_direction, payload_length; + struct iscsi_ecdb_ahdr *ecdb_ahdr; struct iscsi_scsi_req *hdr; int iscsi_task_attr; + unsigned char *cdb; int sam_task_attr; atomic_long_inc(&conn->sess->cmd_pdus); @@ -1090,6 +1097,27 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, ISCSI_REASON_BOOKMARK_INVALID, buf); } + cdb = hdr->cdb; + + if (hdr->hlength) { + ecdb_ahdr = (struct iscsi_ecdb_ahdr *) (hdr + 1); + if (ecdb_ahdr->ahstype != ISCSI_AHSTYPE_CDB) { + pr_err("Additional Header Segment type %d not supported!\n", + ecdb_ahdr->ahstype); + return iscsit_add_reject_cmd(cmd, + ISCSI_REASON_CMD_NOT_SUPPORTED, buf); + } + + cdb = kmalloc(be16_to_cpu(ecdb_ahdr->ahslength) + 15, + GFP_KERNEL); + if (cdb == NULL) + return iscsit_add_reject_cmd(cmd, + ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); + memcpy(cdb, hdr->cdb, ISCSI_CDB_SIZE); + memcpy(cdb + ISCSI_CDB_SIZE, ecdb_ahdr->ecdb, + be16_to_cpu(ecdb_ahdr->ahslength) - 1); + } + data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE : (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE : DMA_NONE; @@ -1137,9 +1165,12 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, struct iscsi_datain_req *dr; dr = iscsit_allocate_datain_req(); - if (!dr) + if (!dr) { + if (cdb != hdr->cdb) + kfree(cdb); return iscsit_add_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); + } iscsit_attach_datain_req(cmd, dr); } @@ -1147,10 +1178,11 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, /* * Initialize struct se_cmd descriptor from target_core_mod infrastructure */ - transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, - conn->sess->se_sess, be32_to_cpu(hdr->data_length), - cmd->data_direction, sam_task_attr, - cmd->sense_buffer + 2); + __target_init_cmd(&cmd->se_cmd, &iscsi_ops, + conn->sess->se_sess, be32_to_cpu(hdr->data_length), + cmd->data_direction, sam_task_attr, + cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun), + conn->cmd_cnt); pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, @@ -1159,23 +1191,30 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, target_get_sess_cmd(&cmd->se_cmd, true); - cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd, - scsilun_to_int(&hdr->lun)); - if (cmd->sense_reason) - goto attach_cmd; - - /* only used for printks or comparing with ->ref_task_tag */ cmd->se_cmd.tag = (__force u32)cmd->init_task_tag; - cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb); + cmd->sense_reason = target_cmd_init_cdb(&cmd->se_cmd, cdb, + GFP_KERNEL); + + if (cdb != hdr->cdb) + kfree(cdb); + if (cmd->sense_reason) { if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) { return iscsit_add_reject_cmd(cmd, - ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); + ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); } goto attach_cmd; } + cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd); + if (cmd->sense_reason) + goto attach_cmd; + + cmd->sense_reason = target_cmd_parse_cdb(&cmd->se_cmd); + if (cmd->sense_reason) + goto attach_cmd; + if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) { return iscsit_add_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); @@ -1185,17 +1224,11 @@ attach_cmd: spin_lock_bh(&conn->cmd_lock); list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list); spin_unlock_bh(&conn->cmd_lock); - /* - * Check if we need to delay processing because of ALUA - * Active/NonOptimized primary access state.. - */ - core_alua_check_nonop_delay(&cmd->se_cmd); - return 0; } EXPORT_SYMBOL(iscsit_setup_scsi_cmd); -void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *cmd) +void iscsit_set_unsolicited_dataout(struct iscsit_cmd *cmd) { iscsit_set_dataout_sequence_values(cmd); @@ -1203,9 +1236,9 @@ void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *cmd) iscsit_start_dataout_timer(cmd, cmd->conn); spin_unlock_bh(&cmd->dataout_timeout_lock); } -EXPORT_SYMBOL(iscsit_set_unsoliticed_dataout); +EXPORT_SYMBOL(iscsit_set_unsolicited_dataout); -int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +int iscsit_process_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, struct iscsi_scsi_req *hdr) { int cmdsn_ret = 0; @@ -1237,7 +1270,7 @@ int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, */ if (!cmd->immediate_data) { if (!cmd->sense_reason && cmd->unsolicited_data) - iscsit_set_unsoliticed_dataout(cmd); + iscsit_set_unsolicited_dataout(cmd); if (!cmd->sense_reason) return 0; @@ -1265,31 +1298,31 @@ int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, EXPORT_SYMBOL(iscsit_process_scsi_cmd); static int -iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, +iscsit_get_immediate_data(struct iscsit_cmd *cmd, struct iscsi_scsi_req *hdr, bool dump_payload) { int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; + int rc; + /* * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. */ - if (dump_payload) - goto after_immediate_data; - /* - * Check for underflow case where both EDTL and immediate data payload - * exceeds what is presented by CDB's TRANSFER LENGTH, and what has - * already been set in target_cmd_size_check() as se_cmd->data_length. - * - * For this special case, fail the command and dump the immediate data - * payload. - */ - if (cmd->first_burst_len > cmd->se_cmd.data_length) { - cmd->sense_reason = TCM_INVALID_CDB_FIELD; - goto after_immediate_data; + if (dump_payload) { + u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done, + cmd->first_burst_len); + + pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n", + cmd->se_cmd.data_length, cmd->write_data_done, + cmd->first_burst_len, length); + rc = iscsit_dump_data_payload(cmd->conn, length, 1); + pr_debug("Finished dumping immediate data\n"); + if (rc < 0) + immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER; + } else { + immed_ret = iscsit_handle_immediate_data(cmd, hdr, + cmd->first_burst_len); } - immed_ret = iscsit_handle_immediate_data(cmd, hdr, - cmd->first_burst_len); -after_immediate_data: if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { /* * A PDU/CmdSN carrying Immediate Data passed @@ -1302,14 +1335,11 @@ after_immediate_data: return -1; if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { - int rc; - - rc = iscsit_dump_data_payload(cmd->conn, - cmd->first_burst_len, 1); target_put_sess_cmd(&cmd->se_cmd); - return rc; + + return 0; } else if (cmd->unsolicited_data) - iscsit_set_unsoliticed_dataout(cmd); + iscsit_set_unsolicited_dataout(cmd); } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { /* @@ -1332,7 +1362,7 @@ after_immediate_data: } static int -iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_handle_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, unsigned char *buf) { struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; @@ -1364,69 +1394,50 @@ iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, return iscsit_get_immediate_data(cmd, hdr, dump_payload); } -static u32 iscsit_do_crypto_hash_sg( - struct ahash_request *hash, - struct iscsi_cmd *cmd, - u32 data_offset, - u32 data_length, - u32 padding, - u8 *pad_bytes) +static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length, + u32 padding, const u8 *pad_bytes) { - u32 data_crc; - struct scatterlist *sg; - unsigned int page_off; - - crypto_ahash_init(hash); - - sg = cmd->first_data_sg; - page_off = cmd->first_data_sg_off; + struct scatterlist *sg = cmd->first_data_sg; + unsigned int page_off = cmd->first_data_sg_off; + u32 crc = ~0; while (data_length) { - u32 cur_len = min_t(u32, data_length, (sg->length - page_off)); + u32 cur_len = min_t(u32, data_length, sg->length - page_off); + const void *virt; - ahash_request_set_crypt(hash, sg, NULL, cur_len); - crypto_ahash_update(hash); + virt = kmap_local_page(sg_page(sg)) + sg->offset + page_off; + crc = crc32c(crc, virt, cur_len); + kunmap_local(virt); - data_length -= cur_len; - page_off = 0; /* iscsit_map_iovec has already checked for invalid sg pointers */ sg = sg_next(sg); - } - - if (padding) { - struct scatterlist pad_sg; - sg_init_one(&pad_sg, pad_bytes, padding); - ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc, - padding); - crypto_ahash_finup(hash); - } else { - ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0); - crypto_ahash_final(hash); + page_off = 0; + data_length -= cur_len; } - return data_crc; + if (padding) + crc = crc32c(crc, pad_bytes, padding); + + return ~crc; } -static void iscsit_do_crypto_hash_buf(struct ahash_request *hash, - const void *buf, u32 payload_length, u32 padding, - const void *pad_bytes, void *data_crc) +static u32 iscsit_crc_buf(const void *buf, u32 payload_length, + u32 padding, const void *pad_bytes) { - struct scatterlist sg[2]; + u32 crc = ~0; - sg_init_table(sg, ARRAY_SIZE(sg)); - sg_set_buf(sg, buf, payload_length); - if (padding) - sg_set_buf(sg + 1, pad_bytes, padding); + crc = crc32c(crc, buf, payload_length); - ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding); + if (padding) + crc = crc32c(crc, pad_bytes, padding); - crypto_ahash_digest(hash); + return ~crc; } int -__iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, - struct iscsi_cmd *cmd, u32 payload_length, +__iscsit_check_dataout_hdr(struct iscsit_conn *conn, void *buf, + struct iscsit_cmd *cmd, u32 payload_length, bool *success) { struct iscsi_data *hdr = buf; @@ -1529,11 +1540,11 @@ __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, EXPORT_SYMBOL(__iscsit_check_dataout_hdr); int -iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, - struct iscsi_cmd **out_cmd) +iscsit_check_dataout_hdr(struct iscsit_conn *conn, void *buf, + struct iscsit_cmd **out_cmd) { struct iscsi_data *hdr = buf; - struct iscsi_cmd *cmd; + struct iscsit_cmd *cmd; u32 payload_length = ntoh24(hdr->dlength); int rc; bool success = false; @@ -1564,19 +1575,21 @@ iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf, EXPORT_SYMBOL(iscsit_check_dataout_hdr); static int -iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, struct iscsi_data *hdr) { struct kvec *iov; u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0; - u32 payload_length = ntoh24(hdr->dlength); + u32 payload_length; int iov_ret, data_crc_failed = 0; + payload_length = min_t(u32, cmd->se_cmd.data_length, + ntoh24(hdr->dlength)); rx_size += payload_length; iov = &cmd->iov_data[0]; - iov_ret = iscsit_map_iovec(cmd, iov, be32_to_cpu(hdr->offset), - payload_length); + iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2, + be32_to_cpu(hdr->offset), payload_length); if (iov_ret < 0) return -1; @@ -1596,6 +1609,7 @@ iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, rx_size += ISCSI_CRC_LEN; } + WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); iscsit_unmap_iovec(cmd); @@ -1606,11 +1620,8 @@ iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, if (conn->conn_ops->DataDigest) { u32 data_crc; - data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, - be32_to_cpu(hdr->offset), - payload_length, padding, - cmd->pad_bytes); - + data_crc = iscsit_crc_sglist(cmd, payload_length, padding, + cmd->pad_bytes); if (checksum != data_crc) { pr_err("ITT: 0x%08x, Offset: %u, Length: %u," " DataSN: 0x%08x, CRC32C DataDigest 0x%08x" @@ -1629,10 +1640,10 @@ iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } int -iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr, +iscsit_check_dataout_payload(struct iscsit_cmd *cmd, struct iscsi_data *hdr, bool data_crc_failed) { - struct iscsi_conn *conn = cmd->conn; + struct iscsit_conn *conn = cmd->conn; int rc, ooo_cmdsn; /* * Increment post receive data and CRC values or perform @@ -1667,9 +1678,9 @@ iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr, } EXPORT_SYMBOL(iscsit_check_dataout_payload); -static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) +static int iscsit_handle_data_out(struct iscsit_conn *conn, unsigned char *buf) { - struct iscsi_cmd *cmd = NULL; + struct iscsit_cmd *cmd = NULL; struct iscsi_data *hdr = (struct iscsi_data *)buf; int rc; bool data_crc_failed = false; @@ -1689,7 +1700,7 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed); } -int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +int iscsit_setup_nop_out(struct iscsit_conn *conn, struct iscsit_cmd *cmd, struct iscsi_nopout *hdr) { u32 payload_length = ntoh24(hdr->dlength); @@ -1737,7 +1748,7 @@ int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, * This is not a response to a Unsolicited NopIN, which means * it can either be a NOPOUT ping request (with a valid ITT), * or a NOPOUT not requesting a NOPIN (with a reserved ITT). - * Either way, make sure we allocate an struct iscsi_cmd, as both + * Either way, make sure we allocate an struct iscsit_cmd, as both * can contain ping data. */ if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { @@ -1756,10 +1767,10 @@ int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } EXPORT_SYMBOL(iscsit_setup_nop_out); -int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +int iscsit_process_nop_out(struct iscsit_conn *conn, struct iscsit_cmd *cmd, struct iscsi_nopout *hdr) { - struct iscsi_cmd *cmd_p = NULL; + struct iscsit_cmd *cmd_p = NULL; int cmdsn_ret = 0; /* * Initiator is expecting a NopIN ping reply.. @@ -1818,7 +1829,7 @@ int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } EXPORT_SYMBOL(iscsit_process_nop_out); -static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +static int iscsit_handle_nop_out(struct iscsit_conn *conn, struct iscsit_cmd *cmd, unsigned char *buf) { unsigned char *ping_data = NULL; @@ -1861,6 +1872,7 @@ static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, rx_size += ISCSI_CRC_LEN; } + WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc)); rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size); if (rx_got != rx_size) { ret = -1; @@ -1868,10 +1880,8 @@ static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } if (conn->conn_ops->DataDigest) { - iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data, - payload_length, padding, - cmd->pad_bytes, &data_crc); - + data_crc = iscsit_crc_buf(ping_data, payload_length, + padding, cmd->pad_bytes); if (checksum != data_crc) { pr_err("Ping data CRC32C DataDigest" " 0x%08x does not match computed 0x%08x\n", @@ -1902,7 +1912,7 @@ static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, ping_data[payload_length] = '\0'; /* - * Attach ping data to struct iscsi_cmd->buf_ptr. + * Attach ping data to struct iscsit_cmd->buf_ptr. */ cmd->buf_ptr = ping_data; cmd->buf_ptr_size = payload_length; @@ -1944,7 +1954,7 @@ static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf) } int -iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_handle_task_mgt_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, unsigned char *buf) { struct se_tmr_req *se_tmr; @@ -1989,9 +1999,11 @@ iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, buf); } - transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, - conn->sess->se_sess, 0, DMA_NONE, - TCM_SIMPLE_TAG, cmd->sense_buffer + 2); + __target_init_cmd(&cmd->se_cmd, &iscsi_ops, + conn->sess->se_sess, 0, DMA_NONE, + TCM_SIMPLE_TAG, cmd->sense_buffer + 2, + scsilun_to_int(&hdr->lun), + conn->cmd_cnt); target_get_sess_cmd(&cmd->se_cmd, true); @@ -2029,8 +2041,7 @@ iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN */ if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { - ret = transport_lookup_tmr_lun(&cmd->se_cmd, - scsilun_to_int(&hdr->lun)); + ret = transport_lookup_tmr_lun(&cmd->se_cmd); if (ret < 0) { se_tmr->response = ISCSI_TMF_RSP_NO_LUN; goto attach; @@ -2125,7 +2136,7 @@ EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd); /* #warning FIXME: Support Text Command parameters besides SendTargets */ int -iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_setup_text_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, struct iscsi_text *hdr) { u32 payload_length = ntoh24(hdr->dlength); @@ -2165,7 +2176,7 @@ iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, EXPORT_SYMBOL(iscsit_setup_text_cmd); int -iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_process_text_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, struct iscsi_text *hdr) { unsigned char *text_in = cmd->text_in_ptr, *text_ptr; @@ -2180,24 +2191,22 @@ iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } goto empty_sendtargets; } - if (strncmp("SendTargets", text_in, 11) != 0) { + if (strncmp("SendTargets=", text_in, 12) != 0) { pr_err("Received Text Data that is not" " SendTargets, cannot continue.\n"); goto reject; } + /* '=' confirmed in strncmp */ text_ptr = strchr(text_in, '='); - if (!text_ptr) { - pr_err("No \"=\" separator found in Text Data," - " cannot continue.\n"); - goto reject; - } - if (!strncmp("=All", text_ptr, 4)) { + BUG_ON(!text_ptr); + if (!strncmp("=All", text_ptr, 5)) { cmd->cmd_flags |= ICF_SENDTARGETS_ALL; } else if (!strncmp("=iqn.", text_ptr, 5) || !strncmp("=eui.", text_ptr, 5)) { cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE; } else { - pr_err("Unable to locate valid SendTargets=%s value\n", text_ptr); + pr_err("Unable to locate valid SendTargets%s value\n", + text_ptr); goto reject; } @@ -2226,7 +2235,7 @@ reject: EXPORT_SYMBOL(iscsit_process_text_cmd); static int -iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_handle_text_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, unsigned char *buf) { struct iscsi_text *hdr = (struct iscsi_text *)buf; @@ -2241,43 +2250,38 @@ iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, rx_size = payload_length; if (payload_length) { u32 checksum = 0, data_crc = 0; - u32 padding = 0, pad_bytes = 0; + u32 padding = 0; int niov = 0, rx_got; - struct kvec iov[3]; + struct kvec iov[2]; - text_in = kzalloc(payload_length, GFP_KERNEL); + rx_size = ALIGN(payload_length, 4); + text_in = kzalloc(rx_size, GFP_KERNEL); if (!text_in) goto reject; cmd->text_in_ptr = text_in; - memset(iov, 0, 3 * sizeof(struct kvec)); + memset(iov, 0, sizeof(iov)); iov[niov].iov_base = text_in; - iov[niov++].iov_len = payload_length; + iov[niov++].iov_len = rx_size; - padding = ((-payload_length) & 3); - if (padding != 0) { - iov[niov].iov_base = &pad_bytes; - iov[niov++].iov_len = padding; - rx_size += padding; + padding = rx_size - payload_length; + if (padding) pr_debug("Receiving %u additional bytes" " for padding.\n", padding); - } if (conn->conn_ops->DataDigest) { iov[niov].iov_base = &checksum; iov[niov++].iov_len = ISCSI_CRC_LEN; rx_size += ISCSI_CRC_LEN; } + WARN_ON_ONCE(niov > ARRAY_SIZE(iov)); rx_got = rx_data(conn, &iov[0], niov, rx_size); if (rx_got != rx_size) goto reject; if (conn->conn_ops->DataDigest) { - iscsit_do_crypto_hash_buf(conn->conn_rx_hash, text_in, - payload_length, padding, - &pad_bytes, &data_crc); - + data_crc = iscsit_crc_buf(text_in, rx_size, 0, NULL); if (checksum != data_crc) { pr_err("Text data CRC32C DataDigest" " 0x%08x does not match computed" @@ -2317,10 +2321,10 @@ reject: return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf); } -int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +int iscsit_logout_closesession(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { - struct iscsi_conn *conn_p; - struct iscsi_session *sess = conn->sess; + struct iscsit_conn *conn_p; + struct iscsit_session *sess = conn->sess; pr_debug("Received logout request CLOSESESSION on CID: %hu" " for SID: %u.\n", conn->cid, conn->sess->sid); @@ -2347,10 +2351,10 @@ int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn) return 0; } -int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +int iscsit_logout_closeconnection(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { - struct iscsi_conn *l_conn; - struct iscsi_session *sess = conn->sess; + struct iscsit_conn *l_conn; + struct iscsit_session *sess = conn->sess; pr_debug("Received logout request CLOSECONNECTION for CID:" " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); @@ -2395,9 +2399,9 @@ int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn return 0; } -int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +int iscsit_logout_removeconnforrecovery(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { - struct iscsi_session *sess = conn->sess; + struct iscsit_session *sess = conn->sess; pr_debug("Received explicit REMOVECONNFORRECOVERY logout for" " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid); @@ -2425,7 +2429,7 @@ int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn } int -iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, +iscsit_handle_logout_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd, unsigned char *buf) { int cmdsn_ret, logout_remove = 0; @@ -2506,7 +2510,7 @@ iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, EXPORT_SYMBOL(iscsit_handle_logout_cmd); int iscsit_handle_snack( - struct iscsi_conn *conn, + struct iscsit_conn *conn, unsigned char *buf) { struct iscsi_snack *hdr; @@ -2560,7 +2564,7 @@ int iscsit_handle_snack( } EXPORT_SYMBOL(iscsit_handle_snack); -static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) +static void iscsit_rx_thread_wait_for_tcp(struct iscsit_conn *conn) { if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { @@ -2571,22 +2575,42 @@ static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) } static int iscsit_handle_immediate_data( - struct iscsi_cmd *cmd, + struct iscsit_cmd *cmd, struct iscsi_scsi_req *hdr, u32 length) { int iov_ret, rx_got = 0, rx_size = 0; u32 checksum, iov_count = 0, padding = 0; - struct iscsi_conn *conn = cmd->conn; + struct iscsit_conn *conn = cmd->conn; struct kvec *iov; + void *overflow_buf = NULL; - iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length); + BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length); + rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length); + iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, + cmd->orig_iov_data_count - 2, + cmd->write_data_done, rx_size); if (iov_ret < 0) return IMMEDIATE_DATA_CANNOT_RECOVER; - rx_size = length; iov_count = iov_ret; iov = &cmd->iov_data[0]; + if (rx_size < length) { + /* + * Special case: length of immediate data exceeds the data + * buffer size derived from the CDB. + */ + overflow_buf = kmalloc(length - rx_size, GFP_KERNEL); + if (!overflow_buf) { + iscsit_unmap_iovec(cmd); + return IMMEDIATE_DATA_CANNOT_RECOVER; + } + cmd->overflow_buf = overflow_buf; + iov[iov_count].iov_base = overflow_buf; + iov[iov_count].iov_len = length - rx_size; + iov_count++; + rx_size = length; + } padding = ((-length) & 3); if (padding != 0) { @@ -2601,6 +2625,7 @@ static int iscsit_handle_immediate_data( rx_size += ISCSI_CRC_LEN; } + WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count); rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size); iscsit_unmap_iovec(cmd); @@ -2613,10 +2638,8 @@ static int iscsit_handle_immediate_data( if (conn->conn_ops->DataDigest) { u32 data_crc; - data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd, - cmd->write_data_done, length, padding, - cmd->pad_bytes); - + data_crc = iscsit_crc_sglist(cmd, length, padding, + cmd->pad_bytes); if (checksum != data_crc) { pr_err("ImmediateData CRC32C DataDigest 0x%08x" " does not match computed 0x%08x\n", checksum, @@ -2655,17 +2678,16 @@ static int iscsit_handle_immediate_data( return IMMEDIATE_DATA_NORMAL_OPERATION; } -/* - * Called with sess->conn_lock held. - */ /* #warning iscsi_build_conn_drop_async_message() only sends out on connections with active network interface */ -static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn) +static void iscsit_build_conn_drop_async_message(struct iscsit_conn *conn) { - struct iscsi_cmd *cmd; - struct iscsi_conn *conn_p; + struct iscsit_cmd *cmd; + struct iscsit_conn *conn_p; bool found = false; + lockdep_assert_held(&conn->sess->conn_lock); + /* * Only send a Asynchronous Message on connections whos network * interface is still functional. @@ -2700,8 +2722,8 @@ static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn) } static int iscsit_send_conn_drop_async_message( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) + struct iscsit_cmd *cmd, + struct iscsit_conn *conn) { struct iscsi_async *hdr; @@ -2729,7 +2751,7 @@ static int iscsit_send_conn_drop_async_message( return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0); } -static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) +static void iscsit_tx_thread_wait_for_tcp(struct iscsit_conn *conn) { if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) || (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) { @@ -2740,7 +2762,7 @@ static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) } void -iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, +iscsit_build_datain_pdu(struct iscsit_cmd *cmd, struct iscsit_conn *conn, struct iscsi_datain *datain, struct iscsi_data_rsp *hdr, bool set_statsn) { @@ -2785,7 +2807,7 @@ iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, } EXPORT_SYMBOL(iscsit_build_datain_pdu); -static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +static int iscsit_send_datain(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0]; struct iscsi_datain datain; @@ -2846,12 +2868,12 @@ static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn) } int -iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, +iscsit_build_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn, struct iscsi_logout_rsp *hdr) { - struct iscsi_conn *logout_conn = NULL; + struct iscsit_conn *logout_conn = NULL; struct iscsi_conn_recovery *cr = NULL; - struct iscsi_session *sess = conn->sess; + struct iscsit_session *sess = conn->sess; /* * The actual shutting down of Sessions and/or Connections * for CLOSESESSION and CLOSECONNECTION Logout Requests @@ -2941,7 +2963,7 @@ iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, EXPORT_SYMBOL(iscsit_build_logout_rsp); static int -iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +iscsit_send_logout(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { int rc; @@ -2954,7 +2976,7 @@ iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn) } void -iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, +iscsit_build_nopin_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn, struct iscsi_nopin *hdr, bool nopout_response) { hdr->opcode = ISCSI_OP_NOOP_IN; @@ -2985,8 +3007,8 @@ EXPORT_SYMBOL(iscsit_build_nopin_rsp); * Unsolicited NOPIN, either requesting a response or not. */ static int iscsit_send_unsolicited_nopin( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn, + struct iscsit_cmd *cmd, + struct iscsit_conn *conn, int want_response) { struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; @@ -3010,15 +3032,15 @@ static int iscsit_send_unsolicited_nopin( } static int -iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +iscsit_send_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; iscsit_build_nopin_rsp(cmd, conn, hdr, true); /* - * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr. - * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size. + * NOPOUT Ping Data is attached to struct iscsit_cmd->buf_ptr. + * NOPOUT DataSegmentLength is at struct iscsit_cmd->buf_ptr_size. */ pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size); @@ -3028,8 +3050,8 @@ iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn) } static int iscsit_send_r2t( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) + struct iscsit_cmd *cmd, + struct iscsit_conn *conn) { struct iscsi_r2t *r2t; struct iscsi_r2t_rsp *hdr; @@ -3085,8 +3107,8 @@ static int iscsit_send_r2t( * connection recovery. */ int iscsit_build_r2ts_for_cmd( - struct iscsi_conn *conn, - struct iscsi_cmd *cmd, + struct iscsit_conn *conn, + struct iscsit_cmd *cmd, bool recovery) { int first_r2t = 1; @@ -3126,6 +3148,12 @@ int iscsit_build_r2ts_for_cmd( else xfer_len = conn->sess->sess_ops->MaxBurstLength; } + + if ((s32)xfer_len < 0) { + cmd->cmd_flags |= ICF_SENT_LAST_R2T; + break; + } + cmd->r2t_offset += xfer_len; if (cmd->r2t_offset == cmd->se_cmd.data_length) @@ -3162,7 +3190,7 @@ int iscsit_build_r2ts_for_cmd( } EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd); -void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, +void iscsit_build_rsp_pdu(struct iscsit_cmd *cmd, struct iscsit_conn *conn, bool inc_stat_sn, struct iscsi_scsi_rsp *hdr) { if (inc_stat_sn) @@ -3196,7 +3224,7 @@ void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, } EXPORT_SYMBOL(iscsit_build_rsp_pdu); -static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +static int iscsit_send_response(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0]; bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS); @@ -3253,7 +3281,7 @@ static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) } void -iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, +iscsit_build_task_mgt_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn, struct iscsi_tm_rsp *hdr) { struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req; @@ -3276,7 +3304,7 @@ iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, EXPORT_SYMBOL(iscsit_build_task_mgt_rsp); static int -iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +iscsit_send_task_mgt_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) { struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0]; @@ -3288,12 +3316,12 @@ iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) #define SENDTARGETS_BUF_LIMIT 32768U static int -iscsit_build_sendtargets_response(struct iscsi_cmd *cmd, +iscsit_build_sendtargets_response(struct iscsit_cmd *cmd, enum iscsit_transport_type network_transport, int skip_bytes, bool *completed) { char *payload = NULL; - struct iscsi_conn *conn = cmd->conn; + struct iscsit_conn *conn = cmd->conn; struct iscsi_portal_group *tpg; struct iscsi_tiqn *tiqn; struct iscsi_tpg_np *tpg_np; @@ -3391,7 +3419,7 @@ iscsit_build_sendtargets_response(struct iscsi_cmd *cmd, } } - if (inet_addr_is_any((struct sockaddr *)&np->np_sockaddr)) + if (inet_addr_is_any(&np->np_sockaddr)) sockaddr = &conn->local_sockaddr; else sockaddr = &np->np_sockaddr; @@ -3438,7 +3466,7 @@ eob: } int -iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, +iscsit_build_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn, struct iscsi_text_rsp *hdr, enum iscsit_transport_type network_transport) { @@ -3488,8 +3516,8 @@ iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, EXPORT_SYMBOL(iscsit_build_text_rsp); static int iscsit_send_text_rsp( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) + struct iscsit_cmd *cmd, + struct iscsit_conn *conn) { struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu; int text_length; @@ -3505,7 +3533,7 @@ static int iscsit_send_text_rsp( } void -iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn, +iscsit_build_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn, struct iscsi_reject *hdr) { hdr->opcode = ISCSI_OP_REJECT; @@ -3522,8 +3550,8 @@ iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn, EXPORT_SYMBOL(iscsit_build_reject); static int iscsit_send_reject( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) + struct iscsit_cmd *cmd, + struct iscsit_conn *conn) { struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0]; @@ -3537,9 +3565,11 @@ static int iscsit_send_reject( ISCSI_HDR_LEN); } -void iscsit_thread_get_cpumask(struct iscsi_conn *conn) +void iscsit_thread_get_cpumask(struct iscsit_conn *conn) { int ord, cpu; + cpumask_var_t conn_allowed_cpumask; + /* * bitmap_id is assigned from iscsit_global->ts_bitmap from * within iscsit_start_kthreads() @@ -3548,12 +3578,28 @@ void iscsit_thread_get_cpumask(struct iscsi_conn *conn) * iSCSI connection's RX/TX threads will be scheduled to * execute upon. */ - ord = conn->bitmap_id % cpumask_weight(cpu_online_mask); - for_each_online_cpu(cpu) { - if (ord-- == 0) { - cpumask_set_cpu(cpu, conn->conn_cpumask); - return; + if (!zalloc_cpumask_var(&conn_allowed_cpumask, GFP_KERNEL)) { + ord = conn->bitmap_id % cpumask_weight(cpu_online_mask); + for_each_online_cpu(cpu) { + if (ord-- == 0) { + cpumask_set_cpu(cpu, conn->conn_cpumask); + return; + } } + } else { + cpumask_and(conn_allowed_cpumask, iscsit_global->allowed_cpumask, + cpu_online_mask); + + cpumask_clear(conn->conn_cpumask); + ord = conn->bitmap_id % cpumask_weight(conn_allowed_cpumask); + for_each_cpu(cpu, conn_allowed_cpumask) { + if (ord-- == 0) { + cpumask_set_cpu(cpu, conn->conn_cpumask); + free_cpumask_var(conn_allowed_cpumask); + return; + } + } + free_cpumask_var(conn_allowed_cpumask); } /* * This should never be reached.. @@ -3562,8 +3608,64 @@ void iscsit_thread_get_cpumask(struct iscsi_conn *conn) cpumask_setall(conn->conn_cpumask); } +static void iscsit_thread_reschedule(struct iscsit_conn *conn) +{ + /* + * If iscsit_global->allowed_cpumask modified, reschedule iSCSI + * connection's RX/TX threads update conn->allowed_cpumask. + */ + if (!cpumask_equal(iscsit_global->allowed_cpumask, + conn->allowed_cpumask)) { + iscsit_thread_get_cpumask(conn); + conn->conn_tx_reset_cpumask = 1; + conn->conn_rx_reset_cpumask = 1; + cpumask_copy(conn->allowed_cpumask, + iscsit_global->allowed_cpumask); + } +} + +void iscsit_thread_check_cpumask( + struct iscsit_conn *conn, + struct task_struct *p, + int mode) +{ + /* + * The TX and RX threads maybe call iscsit_thread_check_cpumask() + * at the same time. The RX thread might be faster and return from + * iscsit_thread_reschedule() with conn_rx_reset_cpumask set to 0. + * Then the TX thread sets it back to 1. + * The next time the RX thread loops, it sees conn_rx_reset_cpumask + * set to 1 and calls set_cpus_allowed_ptr() again and set it to 0. + */ + iscsit_thread_reschedule(conn); + + /* + * mode == 1 signals iscsi_target_tx_thread() usage. + * mode == 0 signals iscsi_target_rx_thread() usage. + */ + if (mode == 1) { + if (!conn->conn_tx_reset_cpumask) + return; + } else { + if (!conn->conn_rx_reset_cpumask) + return; + } + + /* + * Update the CPU mask for this single kthread so that + * both TX and RX kthreads are scheduled to run on the + * same CPU. + */ + set_cpus_allowed_ptr(p, conn->conn_cpumask); + if (mode == 1) + conn->conn_tx_reset_cpumask = 0; + else + conn->conn_rx_reset_cpumask = 0; +} +EXPORT_SYMBOL(iscsit_thread_check_cpumask); + int -iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) +iscsit_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) { int ret; @@ -3607,11 +3709,11 @@ err: EXPORT_SYMBOL(iscsit_immediate_queue); static int -iscsit_handle_immediate_queue(struct iscsi_conn *conn) +iscsit_handle_immediate_queue(struct iscsit_conn *conn) { struct iscsit_transport *t = conn->conn_transport; struct iscsi_queue_req *qr; - struct iscsi_cmd *cmd; + struct iscsit_cmd *cmd; u8 state; int ret; @@ -3630,7 +3732,7 @@ iscsit_handle_immediate_queue(struct iscsi_conn *conn) } int -iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) +iscsit_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) { int ret; @@ -3706,7 +3808,7 @@ check_rsp_state: case ISTATE_SEND_LOGOUTRSP: if (!iscsit_logout_post_handler(cmd, conn)) return -ECONNRESET; - /* fall through */ + fallthrough; case ISTATE_SEND_STATUS: case ISTATE_SEND_ASYNCMSG: case ISTATE_SEND_NOPIN: @@ -3736,11 +3838,11 @@ err: } EXPORT_SYMBOL(iscsit_response_queue); -static int iscsit_handle_response_queue(struct iscsi_conn *conn) +static int iscsit_handle_response_queue(struct iscsit_conn *conn) { struct iscsit_transport *t = conn->conn_transport; struct iscsi_queue_req *qr; - struct iscsi_cmd *cmd; + struct iscsit_cmd *cmd; u8 state; int ret; @@ -3760,7 +3862,7 @@ static int iscsit_handle_response_queue(struct iscsi_conn *conn) int iscsi_target_tx_thread(void *arg) { int ret = 0; - struct iscsi_conn *conn = arg; + struct iscsit_conn *conn = arg; bool conn_freed = false; /* @@ -3815,10 +3917,10 @@ out: return 0; } -static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf) +static int iscsi_target_rx_opcode(struct iscsit_conn *conn, unsigned char *buf) { struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf; - struct iscsi_cmd *cmd; + struct iscsit_cmd *cmd; int ret = 0; switch (hdr->opcode & ISCSI_OPCODE_MASK) { @@ -3892,7 +3994,7 @@ reject: return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); } -static bool iscsi_target_check_conn_state(struct iscsi_conn *conn) +static bool iscsi_target_check_conn_state(struct iscsit_conn *conn) { bool ret; @@ -3903,11 +4005,12 @@ static bool iscsi_target_check_conn_state(struct iscsi_conn *conn) return ret; } -static void iscsit_get_rx_pdu(struct iscsi_conn *conn) +static void iscsit_get_rx_pdu(struct iscsit_conn *conn) { int ret; - u8 *buffer, opcode; + u8 *buffer, *tmp_buf, opcode; u32 checksum = 0, digest = 0; + struct iscsi_hdr *hdr; struct kvec iov; buffer = kcalloc(ISCSI_HDR_LEN, sizeof(*buffer), GFP_KERNEL); @@ -3932,6 +4035,25 @@ static void iscsit_get_rx_pdu(struct iscsi_conn *conn) break; } + hdr = (struct iscsi_hdr *) buffer; + if (hdr->hlength) { + iov.iov_len = hdr->hlength * 4; + tmp_buf = krealloc(buffer, + ISCSI_HDR_LEN + iov.iov_len, + GFP_KERNEL); + if (!tmp_buf) + break; + + buffer = tmp_buf; + iov.iov_base = &buffer[ISCSI_HDR_LEN]; + + ret = rx_data(conn, &iov, 1, iov.iov_len); + if (ret != iov.iov_len) { + iscsit_rx_thread_wait_for_tcp(conn); + break; + } + } + if (conn->conn_ops->HeaderDigest) { iov.iov_base = &digest; iov.iov_len = ISCSI_CRC_LEN; @@ -3942,10 +4064,8 @@ static void iscsit_get_rx_pdu(struct iscsi_conn *conn) break; } - iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer, - ISCSI_HDR_LEN, 0, NULL, - &checksum); - + checksum = iscsit_crc_buf(buffer, ISCSI_HDR_LEN, 0, + NULL); if (digest != checksum) { pr_err("HeaderDigest CRC32C failed," " received 0x%08x, computed 0x%08x\n", @@ -3988,7 +4108,7 @@ static void iscsit_get_rx_pdu(struct iscsi_conn *conn) int iscsi_target_rx_thread(void *arg) { int rc; - struct iscsi_conn *conn = arg; + struct iscsit_conn *conn = arg; bool conn_freed = false; /* @@ -4023,11 +4143,11 @@ out: return 0; } -static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) +static void iscsit_release_commands_from_conn(struct iscsit_conn *conn) { LIST_HEAD(tmp_list); - struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL; - struct iscsi_session *sess = conn->sess; + struct iscsit_cmd *cmd = NULL, *cmd_tmp = NULL; + struct iscsit_session *sess = conn->sess; /* * We expect this function to only ever be called from either RX or TX * thread context via iscsit_close_connection() once the other context @@ -4036,14 +4156,36 @@ static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) spin_lock_bh(&conn->cmd_lock); list_splice_init(&conn->conn_cmd_list, &tmp_list); - list_for_each_entry(cmd, &tmp_list, i_conn_node) { + list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) { struct se_cmd *se_cmd = &cmd->se_cmd; - if (se_cmd->se_tfo != NULL) { - spin_lock(&se_cmd->t_state_lock); + if (!se_cmd->se_tfo) + continue; + + spin_lock_irq(&se_cmd->t_state_lock); + if (se_cmd->transport_state & CMD_T_ABORTED) { + if (!(se_cmd->transport_state & CMD_T_TAS)) + /* + * LIO's abort path owns the cleanup for this, + * so put it back on the list and let + * aborted_task handle it. + */ + list_move_tail(&cmd->i_conn_node, + &conn->conn_cmd_list); + } else { se_cmd->transport_state |= CMD_T_FABRIC_STOP; - spin_unlock(&se_cmd->t_state_lock); } + + if (cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { + /* + * We never submitted the cmd to LIO core, so we have + * to tell LIO to perform the completion process. + */ + spin_unlock_irq(&se_cmd->t_state_lock); + target_complete_cmd(&cmd->se_cmd, SAM_STAT_TASK_ABORTED); + continue; + } + spin_unlock_irq(&se_cmd->t_state_lock); } spin_unlock_bh(&conn->cmd_lock); @@ -4054,12 +4196,22 @@ static void iscsit_release_commands_from_conn(struct iscsi_conn *conn) iscsit_free_cmd(cmd, true); } + + /* + * Wait on commands that were cleaned up via the aborted_task path. + * LLDs that implement iscsit_wait_conn will already have waited for + * commands. + */ + if (!conn->conn_transport->iscsit_wait_conn) { + target_stop_cmd_counter(conn->cmd_cnt); + target_wait_for_cmds(conn->cmd_cnt); + } } static void iscsit_stop_timers_for_cmds( - struct iscsi_conn *conn) + struct iscsit_conn *conn) { - struct iscsi_cmd *cmd; + struct iscsit_cmd *cmd; spin_lock_bh(&conn->cmd_lock); list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { @@ -4070,10 +4222,10 @@ static void iscsit_stop_timers_for_cmds( } int iscsit_close_connection( - struct iscsi_conn *conn) + struct iscsit_conn *conn) { int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT); - struct iscsi_session *sess = conn->sess; + struct iscsit_session *sess = conn->sess; pr_debug("Closing iSCSI connection CID %hu on SID:" " %u\n", conn->cid, sess->sid); @@ -4086,7 +4238,7 @@ int iscsit_close_connection( * However for iser-target, isert_wait4logout() is using conn_logout_comp * to signal logout response TX interrupt completion. Go ahead and skip * this for iser since isert_rx_opcode() does not wait on logout failure, - * and to avoid iscsi_conn pointer dereference in iser-target code. + * and to avoid iscsit_conn pointer dereference in iser-target code. */ if (!conn->conn_transport->rdma_shutdown) complete(&conn->conn_logout_comp); @@ -4111,8 +4263,8 @@ int iscsit_close_connection( spin_unlock(&iscsit_global->ts_bitmap_lock); iscsit_stop_timers_for_cmds(conn); - iscsit_stop_nopin_response_timer(conn); iscsit_stop_nopin_timer(conn); + iscsit_stop_nopin_response_timer(conn); if (conn->conn_transport->iscsit_wait_conn) conn->conn_transport->iscsit_wait_conn(conn); @@ -4124,7 +4276,7 @@ int iscsit_close_connection( * * During normal operation clear the out of order commands (but * do not free the struct iscsi_ooo_cmdsn's) and release all - * struct iscsi_cmds. + * struct iscsit_cmds. */ if (atomic_read(&conn->connection_recovery)) { iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn); @@ -4200,15 +4352,6 @@ int iscsit_close_connection( */ iscsit_check_conn_usage_count(conn); - ahash_request_free(conn->conn_tx_hash); - if (conn->conn_rx_hash) { - struct crypto_ahash *tfm; - - tfm = crypto_ahash_reqtfm(conn->conn_rx_hash); - ahash_request_free(conn->conn_rx_hash); - crypto_free_ahash(tfm); - } - if (conn->sock) sock_release(conn->sock); @@ -4221,7 +4364,7 @@ int iscsit_close_connection( spin_lock_bh(&sess->conn_lock); atomic_dec(&sess->nconn); - pr_debug("Decremented iSCSI connection count to %hu from node:" + pr_debug("Decremented iSCSI connection count to %d from node:" " %s\n", atomic_read(&sess->nconn), sess->sess_ops->InitiatorName); /* @@ -4270,30 +4413,37 @@ int iscsit_close_connection( if (!atomic_read(&sess->session_reinstatement) && atomic_read(&sess->session_fall_back_to_erl0)) { spin_unlock_bh(&sess->conn_lock); - iscsit_close_session(sess); + complete_all(&sess->session_wait_comp); + iscsit_close_session(sess, true); return 0; } else if (atomic_read(&sess->session_logout)) { pr_debug("Moving to TARG_SESS_STATE_FREE.\n"); sess->session_state = TARG_SESS_STATE_FREE; - spin_unlock_bh(&sess->conn_lock); - if (atomic_read(&sess->sleep_on_sess_wait_comp)) - complete(&sess->session_wait_comp); + if (atomic_read(&sess->session_close)) { + spin_unlock_bh(&sess->conn_lock); + complete_all(&sess->session_wait_comp); + iscsit_close_session(sess, true); + } else { + spin_unlock_bh(&sess->conn_lock); + } return 0; } else { pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); sess->session_state = TARG_SESS_STATE_FAILED; - if (!atomic_read(&sess->session_continuation)) { - spin_unlock_bh(&sess->conn_lock); + if (!atomic_read(&sess->session_continuation)) iscsit_start_time2retain_handler(sess); - } else - spin_unlock_bh(&sess->conn_lock); - if (atomic_read(&sess->sleep_on_sess_wait_comp)) - complete(&sess->session_wait_comp); + if (atomic_read(&sess->session_close)) { + spin_unlock_bh(&sess->conn_lock); + complete_all(&sess->session_wait_comp); + iscsit_close_session(sess, true); + } else { + spin_unlock_bh(&sess->conn_lock); + } return 0; } @@ -4303,7 +4453,7 @@ int iscsit_close_connection( * If the iSCSI Session for the iSCSI Initiator Node exists, * forcefully shutdown the iSCSI NEXUS. */ -int iscsit_close_session(struct iscsi_session *sess) +int iscsit_close_session(struct iscsit_session *sess, bool can_sleep) { struct iscsi_portal_group *tpg = sess->tpg; struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; @@ -4321,6 +4471,9 @@ int iscsit_close_session(struct iscsi_session *sess) iscsit_stop_time2retain_timer(sess); spin_unlock_bh(&se_tpg->session_lock); + if (sess->sess_ops->ErrorRecoveryLevel == 2) + iscsit_free_connection_recovery_entries(sess); + /* * transport_deregister_session_configfs() will clear the * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context @@ -4336,22 +4489,14 @@ int iscsit_close_session(struct iscsi_session *sess) * time2retain handler) and contain and active session usage count we * restart the timer and exit. */ - if (!in_interrupt()) { - if (iscsit_check_session_usage_count(sess) == 1) - iscsit_stop_session(sess, 1, 1); - } else { - if (iscsit_check_session_usage_count(sess) == 2) { - atomic_set(&sess->session_logout, 0); - iscsit_start_time2retain_handler(sess); - return 0; - } + if (iscsit_check_session_usage_count(sess, can_sleep)) { + atomic_set(&sess->session_logout, 0); + iscsit_start_time2retain_handler(sess); + return 0; } transport_deregister_session(sess->se_sess); - if (sess->sess_ops->ErrorRecoveryLevel == 2) - iscsit_free_connection_recovery_entries(sess); - iscsit_free_all_ooo_cmdsns(sess); spin_lock_bh(&se_tpg->session_lock); @@ -4376,9 +4521,9 @@ int iscsit_close_session(struct iscsi_session *sess) } static void iscsit_logout_post_handler_closesession( - struct iscsi_conn *conn) + struct iscsit_conn *conn) { - struct iscsi_session *sess = conn->sess; + struct iscsit_session *sess = conn->sess; int sleep = 1; /* * Traditional iscsi/tcp will invoke this logic from TX thread @@ -4399,13 +4544,13 @@ static void iscsit_logout_post_handler_closesession( complete(&conn->conn_logout_comp); iscsit_dec_conn_usage_count(conn); + atomic_set(&sess->session_close, 1); iscsit_stop_session(sess, sleep, sleep); iscsit_dec_session_usage_count(sess); - iscsit_close_session(sess); } static void iscsit_logout_post_handler_samecid( - struct iscsi_conn *conn) + struct iscsit_conn *conn) { int sleep = 1; @@ -4423,11 +4568,11 @@ static void iscsit_logout_post_handler_samecid( } static void iscsit_logout_post_handler_diffcid( - struct iscsi_conn *conn, + struct iscsit_conn *conn, u16 cid) { - struct iscsi_conn *l_conn; - struct iscsi_session *sess = conn->sess; + struct iscsit_conn *l_conn; + struct iscsit_session *sess = conn->sess; bool conn_found = false; if (!sess) @@ -4462,8 +4607,8 @@ static void iscsit_logout_post_handler_diffcid( * Return of 0 causes the TX thread to restart. */ int iscsit_logout_post_handler( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) + struct iscsit_cmd *cmd, + struct iscsit_conn *conn) { int ret = 0; @@ -4476,7 +4621,6 @@ int iscsit_logout_post_handler( iscsit_logout_post_handler_closesession(conn); break; } - ret = 0; break; case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION: if (conn->cid == cmd->logout_cid) { @@ -4487,7 +4631,6 @@ int iscsit_logout_post_handler( iscsit_logout_post_handler_samecid(conn); break; } - ret = 0; } else { switch (cmd->logout_response) { case ISCSI_LOGOUT_SUCCESS: @@ -4521,76 +4664,16 @@ int iscsit_logout_post_handler( } EXPORT_SYMBOL(iscsit_logout_post_handler); -void iscsit_fail_session(struct iscsi_session *sess) -{ - struct iscsi_conn *conn; - - spin_lock_bh(&sess->conn_lock); - list_for_each_entry(conn, &sess->sess_conn_list, conn_list) { - pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n"); - conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT; - } - spin_unlock_bh(&sess->conn_lock); - - pr_debug("Moving to TARG_SESS_STATE_FAILED.\n"); - sess->session_state = TARG_SESS_STATE_FAILED; -} - -int iscsit_free_session(struct iscsi_session *sess) -{ - u16 conn_count = atomic_read(&sess->nconn); - struct iscsi_conn *conn, *conn_tmp = NULL; - int is_last; - - spin_lock_bh(&sess->conn_lock); - atomic_set(&sess->sleep_on_sess_wait_comp, 1); - - list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, - conn_list) { - if (conn_count == 0) - break; - - if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) { - is_last = 1; - } else { - iscsit_inc_conn_usage_count(conn_tmp); - is_last = 0; - } - iscsit_inc_conn_usage_count(conn); - - spin_unlock_bh(&sess->conn_lock); - iscsit_cause_connection_reinstatement(conn, 1); - spin_lock_bh(&sess->conn_lock); - - iscsit_dec_conn_usage_count(conn); - if (is_last == 0) - iscsit_dec_conn_usage_count(conn_tmp); - - conn_count--; - } - - if (atomic_read(&sess->nconn)) { - spin_unlock_bh(&sess->conn_lock); - wait_for_completion(&sess->session_wait_comp); - } else - spin_unlock_bh(&sess->conn_lock); - - iscsit_close_session(sess); - return 0; -} - void iscsit_stop_session( - struct iscsi_session *sess, + struct iscsit_session *sess, int session_sleep, int connection_sleep) { u16 conn_count = atomic_read(&sess->nconn); - struct iscsi_conn *conn, *conn_tmp = NULL; + struct iscsit_conn *conn, *conn_tmp = NULL; int is_last; spin_lock_bh(&sess->conn_lock); - if (session_sleep) - atomic_set(&sess->sleep_on_sess_wait_comp, 1); if (connection_sleep) { list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list, @@ -4629,7 +4712,7 @@ void iscsit_stop_session( int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) { - struct iscsi_session *sess; + struct iscsit_session *sess; struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; struct se_session *se_sess, *se_sess_tmp; LIST_HEAD(free_list); @@ -4643,17 +4726,20 @@ int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list, sess_list) { - sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; + sess = (struct iscsit_session *)se_sess->fabric_sess_ptr; spin_lock(&sess->conn_lock); if (atomic_read(&sess->session_fall_back_to_erl0) || atomic_read(&sess->session_logout) || + atomic_read(&sess->session_close) || (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) { spin_unlock(&sess->conn_lock); continue; } + iscsit_inc_session_usage_count(sess); atomic_set(&sess->session_reinstatement, 1); atomic_set(&sess->session_fall_back_to_erl0, 1); + atomic_set(&sess->session_close, 1); spin_unlock(&sess->conn_lock); list_move_tail(&se_sess->sess_list, &free_list); @@ -4661,9 +4747,11 @@ int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force) spin_unlock_bh(&se_tpg->session_lock); list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) { - sess = (struct iscsi_session *)se_sess->fabric_sess_ptr; + sess = (struct iscsit_session *)se_sess->fabric_sess_ptr; - iscsit_free_session(sess); + list_del_init(&se_sess->sess_list); + iscsit_stop_session(sess, 1, 1); + iscsit_dec_session_usage_count(sess); session_count++; } |