Print this page
NEX-9981 Deadman timer panic from idm_refcnt_wait_ref thread while offlining iSCSI targets
Reviewed by: Evan Layton <evan.layton@nexenta.com>
Reviewed by: Sanjay Nadkarni <sanjay.nadkarni@nexenta.com>
Reviewed by: Rob Gittins <rob.gittins@nexenta.com>


   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  *

  25  * iSCSI command interfaces
  26  */
  27 
  28 #include "iscsi.h"
  29 
  30 /* internal interfaces */
  31 static void iscsi_cmd_state_free(iscsi_cmd_t *icmdp,
  32     iscsi_cmd_event_t event, void *arg);
  33 static void iscsi_cmd_state_pending(iscsi_cmd_t *icmdp,
  34     iscsi_cmd_event_t event, void *arg);
  35 static void iscsi_cmd_state_active(iscsi_cmd_t *icmdp,
  36     iscsi_cmd_event_t event, void *arg);
  37 static void iscsi_cmd_state_aborting(iscsi_cmd_t *icmdp,
  38     iscsi_cmd_event_t event, void *arg);
  39 static void iscsi_cmd_state_idm_aborting(iscsi_cmd_t *icmdp,
  40     iscsi_cmd_event_t event, void *arg);
  41 static void iscsi_cmd_state_completed(iscsi_cmd_t *icmdp,
  42     iscsi_cmd_event_t event, void *arg);
  43 static char *iscsi_cmd_state_str(iscsi_cmd_state_t state);
  44 static char *iscsi_cmd_event_str(iscsi_cmd_event_t event);


 779                         ASSERT(t_icmdp != NULL);
 780                         mutex_enter(&t_icmdp->cmd_mutex);
 781                         t_icmdp->cmd_un.scsi.abort_icmdp = NULL;
 782                         if (t_icmdp->cmd_state != ISCSI_CMD_STATE_COMPLETED) {
 783                                 iscsi_dequeue_active_cmd(
 784                                     t_icmdp->cmd_conn, t_icmdp);
 785                                 mutex_enter(
 786                                     &icp->conn_queue_idm_aborting.mutex);
 787                                 iscsi_enqueue_idm_aborting_cmd(
 788                                     t_icmdp->cmd_conn,
 789                                     t_icmdp);
 790                                 mutex_exit(&icp->conn_queue_idm_aborting.mutex);
 791 
 792                                 /*
 793                                  * Complete abort processing after IDM
 794                                  * calls us back.  Set the status to use
 795                                  * when we complete the command.
 796                                  */
 797                                 ISCSI_CMD_SET_REASON_STAT(
 798                                     t_icmdp, CMD_TIMEOUT, STAT_ABORTED);
 799                                 idm_task_abort(icp->conn_ic, t_icmdp->cmd_itp,
 800                                     AT_TASK_MGMT_ABORT);
 801                         } else {
 802                                 cv_broadcast(&t_icmdp->cmd_completion);
 803                         }
 804                         mutex_exit(&t_icmdp->cmd_mutex);
 805                         icmdp->cmd_un.abort.icmdp = NULL;
 806 
 807                         icmdp->cmd_misc_flags |=
 808                             ISCSI_CMD_MISCFLAG_FREE;
 809 
 810                         break;
 811                 case ISCSI_CMD_TYPE_RESET:
 812                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
 813                         iscsi_sess_release_itt(isp, icmdp);
 814                         mutex_exit(&isp->sess_cmdsn_mutex);
 815 
 816                         /*
 817                          * Complete the abort/reset command.
 818                          */
 819                         if (icmdp->cmd_un.reset.response !=
 820                             SCSI_TCP_TM_RESP_COMPLETE) {


 925                                     (t_icmdp->cmd_state !=
 926                                     ISCSI_CMD_STATE_IDM_ABORTING) &&
 927                                     (t_icmdp->cmd_state !=
 928                                     ISCSI_CMD_STATE_COMPLETED)) {
 929 
 930                                         iscsi_dequeue_active_cmd(
 931                                             t_icmdp->cmd_conn, t_icmdp);
 932                                         mutex_enter(&icp->
 933                                             conn_queue_idm_aborting.mutex);
 934                                         iscsi_enqueue_idm_aborting_cmd(
 935                                             t_icmdp->cmd_conn,  t_icmdp);
 936                                         mutex_exit(&icp->
 937                                             conn_queue_idm_aborting.mutex);
 938                                         /*
 939                                          * Complete abort processing after IDM
 940                                          * calls us back.  Set the status to use
 941                                          * when we complete the command.
 942                                          */
 943                                         ISCSI_CMD_SET_REASON_STAT(t_icmdp,
 944                                             CMD_TIMEOUT, STAT_TIMEOUT);
 945                                         idm_task_abort(icp->conn_ic,
 946                                             t_icmdp->cmd_itp,
 947                                             AT_TASK_MGMT_ABORT);
 948                                 } else {
 949                                         cv_broadcast(&t_icmdp->cmd_completion);
 950                                 }
 951                                 mutex_exit(&t_icmdp->cmd_mutex);
 952                         } else {
 953                                 t_icmdp->cmd_un.scsi.abort_icmdp = NULL;
 954                         }
 955                         icmdp->cmd_un.abort.icmdp = NULL;
 956                         icmdp->cmd_misc_flags |=
 957                             ISCSI_CMD_MISCFLAG_FREE;
 958                         break;
 959 
 960                 case ISCSI_CMD_TYPE_RESET:
 961                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
 962 
 963                         mutex_enter(&isp->sess_cmdsn_mutex);
 964                         iscsi_sess_release_itt(isp, icmdp);
 965                         iscsi_dequeue_active_cmd(icmdp->cmd_conn, icmdp);


1010                 default:
1011                         ASSERT(FALSE);
1012                 }
1013 
1014                 ASSERT(!mutex_owned(&isp->sess_cmdsn_mutex));
1015                 break;
1016 
1017         /* -E7: Connection has encountered a problem */
1018         case ISCSI_CMD_EVENT_E7:
1019                 mutex_enter(&isp->sess_cmdsn_mutex);
1020                 iscsi_dequeue_active_cmd(icmdp->cmd_conn, icmdp);
1021 
1022                 switch (icmdp->cmd_type) {
1023                 case ISCSI_CMD_TYPE_SCSI:
1024                         mutex_exit(&isp->sess_cmdsn_mutex);
1025                         mutex_enter(&icp->conn_queue_idm_aborting.mutex);
1026                         iscsi_enqueue_idm_aborting_cmd(icmdp->cmd_conn, icmdp);
1027                         mutex_exit(&icp->conn_queue_idm_aborting.mutex);
1028                         ISCSI_CMD_SET_REASON_STAT(icmdp,
1029                             CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
1030                         idm_task_abort(icp->conn_ic, icmdp->cmd_itp,
1031                             AT_TASK_MGMT_ABORT);
1032                         break;
1033 
1034                 case ISCSI_CMD_TYPE_NOP:
1035                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
1036                         iscsi_sess_release_itt(isp, icmdp);
1037                         mutex_exit(&isp->sess_cmdsn_mutex);
1038 
1039                         icmdp->cmd_misc_flags |=
1040                             ISCSI_CMD_MISCFLAG_FREE;
1041                         break;
1042 
1043                 case ISCSI_CMD_TYPE_ABORT:
1044                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
1045                         iscsi_sess_release_itt(isp, icmdp);
1046                         mutex_exit(&isp->sess_cmdsn_mutex);
1047 
1048                         mutex_enter(&icmdp->cmd_un.abort.icmdp->cmd_mutex);
1049                         icmdp->cmd_un.abort.icmdp->
1050                             cmd_un.scsi.abort_icmdp = NULL;


1191                 mutex_enter(&icmdp->cmd_conn->conn_queue_idm_aborting.mutex);
1192                 iscsi_enqueue_idm_aborting_cmd(icmdp->cmd_conn, icmdp);
1193                 mutex_exit(&icmdp->cmd_conn->conn_queue_idm_aborting.mutex);
1194 
1195                 /*
1196                  * Since we are in "aborting" state there is another command
1197                  * representing the abort of this command.  This command
1198                  * will cleanup at some indeterminate time after the call
1199                  * to idm_task_abort so we can't leave the abort request
1200                  * active.  An E10 event to the abort command will cause
1201                  * it to complete immediately.
1202                  */
1203                 if ((a_icmdp = icmdp->cmd_un.scsi.abort_icmdp) != NULL) {
1204                         iscsi_cmd_state_machine(a_icmdp,
1205                             ISCSI_CMD_EVENT_E10, arg);
1206                 }
1207 
1208                 ISCSI_CMD_SET_REASON_STAT(icmdp,
1209                     CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
1210 
1211                 idm_task_abort(icmdp->cmd_conn->conn_ic, icmdp->cmd_itp,
1212                     AT_TASK_MGMT_ABORT);
1213                 break;
1214 
1215         /* -E9: IDM is no longer processing this command */
1216         case ISCSI_CMD_EVENT_E9:
1217                 iscsi_dequeue_active_cmd(icmdp->cmd_conn, icmdp);
1218 
1219                 iscsi_task_cleanup(ISCSI_OP_SCSI_RSP, icmdp);
1220                 iscsi_sess_release_scsi_itt(icmdp);
1221 
1222                 ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR,
1223                     icmdp->cmd_un.scsi.pkt_stat);
1224                 iscsi_enqueue_completed_cmd(isp, icmdp);
1225                 break;
1226 
1227         /* All other events are invalid for this state */
1228         default:
1229                 ASSERT(FALSE);
1230         }
1231 }




   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  *
  25  * Copyright 2017 Nexenta Systems, Inc. All rights reserved.
  26  * iSCSI command interfaces
  27  */
  28 
  29 #include "iscsi.h"
  30 
  31 /* internal interfaces */
  32 static void iscsi_cmd_state_free(iscsi_cmd_t *icmdp,
  33     iscsi_cmd_event_t event, void *arg);
  34 static void iscsi_cmd_state_pending(iscsi_cmd_t *icmdp,
  35     iscsi_cmd_event_t event, void *arg);
  36 static void iscsi_cmd_state_active(iscsi_cmd_t *icmdp,
  37     iscsi_cmd_event_t event, void *arg);
  38 static void iscsi_cmd_state_aborting(iscsi_cmd_t *icmdp,
  39     iscsi_cmd_event_t event, void *arg);
  40 static void iscsi_cmd_state_idm_aborting(iscsi_cmd_t *icmdp,
  41     iscsi_cmd_event_t event, void *arg);
  42 static void iscsi_cmd_state_completed(iscsi_cmd_t *icmdp,
  43     iscsi_cmd_event_t event, void *arg);
  44 static char *iscsi_cmd_state_str(iscsi_cmd_state_t state);
  45 static char *iscsi_cmd_event_str(iscsi_cmd_event_t event);


 780                         ASSERT(t_icmdp != NULL);
 781                         mutex_enter(&t_icmdp->cmd_mutex);
 782                         t_icmdp->cmd_un.scsi.abort_icmdp = NULL;
 783                         if (t_icmdp->cmd_state != ISCSI_CMD_STATE_COMPLETED) {
 784                                 iscsi_dequeue_active_cmd(
 785                                     t_icmdp->cmd_conn, t_icmdp);
 786                                 mutex_enter(
 787                                     &icp->conn_queue_idm_aborting.mutex);
 788                                 iscsi_enqueue_idm_aborting_cmd(
 789                                     t_icmdp->cmd_conn,
 790                                     t_icmdp);
 791                                 mutex_exit(&icp->conn_queue_idm_aborting.mutex);
 792 
 793                                 /*
 794                                  * Complete abort processing after IDM
 795                                  * calls us back.  Set the status to use
 796                                  * when we complete the command.
 797                                  */
 798                                 ISCSI_CMD_SET_REASON_STAT(
 799                                     t_icmdp, CMD_TIMEOUT, STAT_ABORTED);
 800                                 (void) idm_task_abort(icp->conn_ic,
 801                                     t_icmdp->cmd_itp, AT_TASK_MGMT_ABORT);
 802                         } else {
 803                                 cv_broadcast(&t_icmdp->cmd_completion);
 804                         }
 805                         mutex_exit(&t_icmdp->cmd_mutex);
 806                         icmdp->cmd_un.abort.icmdp = NULL;
 807 
 808                         icmdp->cmd_misc_flags |=
 809                             ISCSI_CMD_MISCFLAG_FREE;
 810 
 811                         break;
 812                 case ISCSI_CMD_TYPE_RESET:
 813                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
 814                         iscsi_sess_release_itt(isp, icmdp);
 815                         mutex_exit(&isp->sess_cmdsn_mutex);
 816 
 817                         /*
 818                          * Complete the abort/reset command.
 819                          */
 820                         if (icmdp->cmd_un.reset.response !=
 821                             SCSI_TCP_TM_RESP_COMPLETE) {


 926                                     (t_icmdp->cmd_state !=
 927                                     ISCSI_CMD_STATE_IDM_ABORTING) &&
 928                                     (t_icmdp->cmd_state !=
 929                                     ISCSI_CMD_STATE_COMPLETED)) {
 930 
 931                                         iscsi_dequeue_active_cmd(
 932                                             t_icmdp->cmd_conn, t_icmdp);
 933                                         mutex_enter(&icp->
 934                                             conn_queue_idm_aborting.mutex);
 935                                         iscsi_enqueue_idm_aborting_cmd(
 936                                             t_icmdp->cmd_conn,  t_icmdp);
 937                                         mutex_exit(&icp->
 938                                             conn_queue_idm_aborting.mutex);
 939                                         /*
 940                                          * Complete abort processing after IDM
 941                                          * calls us back.  Set the status to use
 942                                          * when we complete the command.
 943                                          */
 944                                         ISCSI_CMD_SET_REASON_STAT(t_icmdp,
 945                                             CMD_TIMEOUT, STAT_TIMEOUT);
 946                                         (void) idm_task_abort(icp->conn_ic,
 947                                             t_icmdp->cmd_itp,
 948                                             AT_TASK_MGMT_ABORT);
 949                                 } else {
 950                                         cv_broadcast(&t_icmdp->cmd_completion);
 951                                 }
 952                                 mutex_exit(&t_icmdp->cmd_mutex);
 953                         } else {
 954                                 t_icmdp->cmd_un.scsi.abort_icmdp = NULL;
 955                         }
 956                         icmdp->cmd_un.abort.icmdp = NULL;
 957                         icmdp->cmd_misc_flags |=
 958                             ISCSI_CMD_MISCFLAG_FREE;
 959                         break;
 960 
 961                 case ISCSI_CMD_TYPE_RESET:
 962                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
 963 
 964                         mutex_enter(&isp->sess_cmdsn_mutex);
 965                         iscsi_sess_release_itt(isp, icmdp);
 966                         iscsi_dequeue_active_cmd(icmdp->cmd_conn, icmdp);


1011                 default:
1012                         ASSERT(FALSE);
1013                 }
1014 
1015                 ASSERT(!mutex_owned(&isp->sess_cmdsn_mutex));
1016                 break;
1017 
1018         /* -E7: Connection has encountered a problem */
1019         case ISCSI_CMD_EVENT_E7:
1020                 mutex_enter(&isp->sess_cmdsn_mutex);
1021                 iscsi_dequeue_active_cmd(icmdp->cmd_conn, icmdp);
1022 
1023                 switch (icmdp->cmd_type) {
1024                 case ISCSI_CMD_TYPE_SCSI:
1025                         mutex_exit(&isp->sess_cmdsn_mutex);
1026                         mutex_enter(&icp->conn_queue_idm_aborting.mutex);
1027                         iscsi_enqueue_idm_aborting_cmd(icmdp->cmd_conn, icmdp);
1028                         mutex_exit(&icp->conn_queue_idm_aborting.mutex);
1029                         ISCSI_CMD_SET_REASON_STAT(icmdp,
1030                             CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
1031                         (void) idm_task_abort(icp->conn_ic, icmdp->cmd_itp,
1032                             AT_TASK_MGMT_ABORT);
1033                         break;
1034 
1035                 case ISCSI_CMD_TYPE_NOP:
1036                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
1037                         iscsi_sess_release_itt(isp, icmdp);
1038                         mutex_exit(&isp->sess_cmdsn_mutex);
1039 
1040                         icmdp->cmd_misc_flags |=
1041                             ISCSI_CMD_MISCFLAG_FREE;
1042                         break;
1043 
1044                 case ISCSI_CMD_TYPE_ABORT:
1045                         icmdp->cmd_state = ISCSI_CMD_STATE_FREE;
1046                         iscsi_sess_release_itt(isp, icmdp);
1047                         mutex_exit(&isp->sess_cmdsn_mutex);
1048 
1049                         mutex_enter(&icmdp->cmd_un.abort.icmdp->cmd_mutex);
1050                         icmdp->cmd_un.abort.icmdp->
1051                             cmd_un.scsi.abort_icmdp = NULL;


1192                 mutex_enter(&icmdp->cmd_conn->conn_queue_idm_aborting.mutex);
1193                 iscsi_enqueue_idm_aborting_cmd(icmdp->cmd_conn, icmdp);
1194                 mutex_exit(&icmdp->cmd_conn->conn_queue_idm_aborting.mutex);
1195 
1196                 /*
1197                  * Since we are in "aborting" state there is another command
1198                  * representing the abort of this command.  This command
1199                  * will cleanup at some indeterminate time after the call
1200                  * to idm_task_abort so we can't leave the abort request
1201                  * active.  An E10 event to the abort command will cause
1202                  * it to complete immediately.
1203                  */
1204                 if ((a_icmdp = icmdp->cmd_un.scsi.abort_icmdp) != NULL) {
1205                         iscsi_cmd_state_machine(a_icmdp,
1206                             ISCSI_CMD_EVENT_E10, arg);
1207                 }
1208 
1209                 ISCSI_CMD_SET_REASON_STAT(icmdp,
1210                     CMD_TRAN_ERR, icmdp->cmd_un.scsi.pkt_stat);
1211 
1212                 (void) idm_task_abort(icmdp->cmd_conn->conn_ic, icmdp->cmd_itp,
1213                     AT_TASK_MGMT_ABORT);
1214                 break;
1215 
1216         /* -E9: IDM is no longer processing this command */
1217         case ISCSI_CMD_EVENT_E9:
1218                 iscsi_dequeue_active_cmd(icmdp->cmd_conn, icmdp);
1219 
1220                 iscsi_task_cleanup(ISCSI_OP_SCSI_RSP, icmdp);
1221                 iscsi_sess_release_scsi_itt(icmdp);
1222 
1223                 ISCSI_CMD_SET_REASON_STAT(icmdp, CMD_TRAN_ERR,
1224                     icmdp->cmd_un.scsi.pkt_stat);
1225                 iscsi_enqueue_completed_cmd(isp, icmdp);
1226                 break;
1227 
1228         /* All other events are invalid for this state */
1229         default:
1230                 ASSERT(FALSE);
1231         }
1232 }