Print this page
NEX-3856 panic is occurred in module "fct" due to a NULL pointer dereference
Reviewed by: Rob Gittins <rob.gittins@nexenta.com>
Reviewed by: Steve Peng <steve.peng@nexenta.com>
NEX-3277 Panic of both nodes in failover time (FC clients)
        Reviewed by: Rick McNeal <rick.mcneal@nexenta.com>
        Reviewed by: Gordon Ross <gordon.ross@nexenta.com>
NEX-2787 Multiple comstar / fibre channel / qlt threads stuck waiting on locks with a spinning interrupt thread
Reviewed by: Rob Gittins <rob.gittins@nexenta.com>
Approved by: Jean McCormack <jean.mccormack@nexenta.com>


   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.

  23  * Use is subject to license terms.
  24  */
  25 #ifndef _FCT_IMPL_H
  26 #define _FCT_IMPL_H
  27 
  28 #ifdef  __cplusplus
  29 extern "C" {
  30 #endif
  31 
  32 #define RSCN_OPTION_VERIFY      0x0001
  33 
  34 typedef enum fct_li_state {
  35         LI_STATE_DO_FLOGI = 0,  /* FLOGI handled by FCA */
  36         LI_STATE_FINI_TOPOLOGY, /* Finalize topology */
  37         LI_STATE_N2N_PLOGI,     /* In case of a N2N connection */
  38 
  39         LI_STATE_DO_FCLOGIN,    /* Login into 0xFFFFFD */
  40         LI_STATE_DO_SCR,        /* State change registration */
  41 
  42         LI_STATE_DO_NSLOGIN,    /* Login into 0xFFFFFC */


  55 #define LI_STATE_FLAG_CMD_RETCHECK      0x40
  56 /* Link init cmd is still outstanding */
  57 #define LI_STATE_FLAG_CMD_WAITING       0x80
  58 /* Flag to indicate that link info is not available yet */
  59 #define LI_STATE_FLAG_NO_LI_YET         0x100
  60 
  61 #define FCT_MAX_CACHED_CMDS     256
  62 #define USEC_ELS_TIMEOUT        (10 * 1000 * 1000)
  63 #define USEC_SOL_TIMEOUT        (10 * 1000 * 1000)
  64 #define USEC_DEREG_RP_TIMEOUT   (25 * 1000 * 1000)
  65 #define USEC_DEREG_RP_INTERVAL  (2 * 1000 * 1000)
  66 
  67 struct fct_i_cmd;
  68 typedef void (* fct_icmd_cb_t)(struct fct_i_cmd *icmd);
  69 typedef struct fct_i_cmd {
  70         fct_cmd_t               *icmd_cmd;
  71         uint32_t                 icmd_alloc_size;
  72         fct_struct_id_t          icmd_struct_id;
  73         uint32_t                 icmd_flags;
  74         clock_t                  icmd_start_time;
  75         struct fct_i_cmd        *icmd_next;     /* iport_abort_queue and irp */
  76         struct fct_i_cmd        *icmd_solcmd_next;      /* iport_solcmd_queue */
  77         fct_icmd_cb_t            icmd_cb;
  78         void                    *icmd_cb_private;
  79 } fct_i_cmd_t;
  80 
  81 /*
  82  * icmd_flags
  83  */
  84 #define ICMD_SESSION_AFFECTING          0x0002
  85 #define ICMD_IN_IRP_QUEUE               0x0004
  86 #define ICMD_BEING_ABORTED              0x0008
  87 #define ICMD_KNOWN_TO_FCA               0x0020
  88 #define ICMD_FCA_ABORT_CALLED           0x0040
  89 #define ICMD_CMD_COMPLETE               0x0080
  90 #define ICMD_IN_TRANSITION              0x0100
  91 #define ICMD_ABTS_RECEIVED              0x0200
  92 #define ICMD_IMPLICIT                   0x0400
  93 #define ICMD_IMPLICIT_CMD_HAS_RESOURCE  0x0800
  94 /* High order are debug flags */
  95 #define ICMD_ELS_PROCESSING_STARTED     0x80000000


  99  * 1) it's new. We need send it to FCA. ICMD_SOLCMD_NEW is set
 100  * 2) it's running. We are waiting for completion.
 101  * 3) it's completed. We need free it. ICMD_CMD_COMPLETE is set
 102  * ICMD_SOLCMD_NEW and ICMD_CMD_COMPLETE should not be set in the same time
 103  */
 104 #define ICMD_IN_SOLCMD_QUEUE            0x010000
 105 #define ICMD_SOLCMD_NEW                 0x020000
 106 
 107 typedef struct fct_i_remote_port {
 108         fct_remote_port_t               *irp_rp;
 109         uint32_t                        irp_alloc_size;
 110         fct_struct_id_t                 irp_struct_id;
 111         krwlock_t                       irp_lock;
 112 
 113         /* For queueing to local port */
 114         struct fct_i_remote_port        *irp_next;
 115 
 116         /* For queueing to handle elses */
 117         struct fct_i_remote_port        *irp_discovery_next;
 118 
 119         fct_i_cmd_t                     *irp_els_list;
 120 
 121         /*
 122          * sa stands for session affecting, nsa is non session affecting.
 123          * The els counts only represent elses under progress not the ones
 124          * that are terminated. active_xchg_count covers everything including
 125          * the ones waiting to be terminated.
 126          */
 127         uint16_t                        irp_sa_elses_count;
 128         uint16_t                        irp_nsa_elses_count;
 129         uint16_t                        irp_fcp_xchg_count;
 130         uint16_t                        irp_nonfcp_xchg_count;
 131 
 132         uint32_t                        irp_flags;
 133         clock_t                         irp_deregister_timer;
 134         uint32_t                        irp_dereg_count;
 135 
 136         uint32_t                        irp_portid;
 137         uint8_t                         irp_id[24];
 138         uint32_t                        irp_rcvd_prli_params;
 139         uint32_t                        irp_sent_prli_params;
 140 
 141         /*
 142          * Most HBAs will only register symbolic node name instead of port name,
 143          * so we use SNN as session alias.
 144          */
 145         stmf_scsi_session_t             *irp_session;
 146         char                            *irp_snn;

 147 
 148         /* items will be filled in ns cmd */
 149         uint8_t                         irp_fc4types[32]; /* FC-4 types */
 150         char                            *irp_spn;       /* port symbolic name */

 151         uint32_t                        irp_cos;        /* class of service */
 152 
 153         uint32_t                        irp_rscn_counter;
 154 } fct_i_remote_port_t;
 155 
 156 /*
 157  * structure used for fct_rls_cb() callback private data
 158  */
 159 typedef struct fct_rls_cb_data {
 160         struct fct_port_link_status     *fct_link_status;
 161         fct_status_t                    fct_els_res;
 162 } fct_rls_cb_data_t;
 163 
 164 /*
 165  * irp flags
 166  */
 167 #define IRP_PLOGI_DONE                  0x0001
 168 #define IRP_PRLI_DONE                   0x0002
 169 #define IRP_IN_DISCOVERY_QUEUE          0x0004
 170 #define IRP_FCP_CLEANUP                 0x0008


 215         uint64_t                iport_last_change;
 216 
 217         /*
 218          * These variables are used to manage fct_cmd_t cache for SCSI traffic
 219          */
 220         /*
 221          * Total # of cmds allocated by the driver. Some of which are free
 222          * and sitting on iport_cached_cmdlist. And some are executing.
 223          */
 224         uint32_t                iport_total_alloced_ncmds;
 225 
 226         /*
 227          * Max active cmds in last interval (10 or 30 seconds)
 228          */
 229         uint32_t                iport_max_active_ncmds;
 230 
 231         /*
 232          * # of free cmds sitting on the iport_cached_cmdlist
 233          */
 234         uint32_t                iport_cached_ncmds;
 235         struct fct_i_cmd        *iport_cached_cmdlist;
 236         kmutex_t                iport_cached_cmd_lock;
 237 
 238         /*
 239          * To release free cmds periodically
 240          */
 241         clock_t                 iport_cmdcheck_clock;
 242 
 243         uint16_t                iport_task_green_limit;
 244         uint16_t                iport_task_yellow_limit;
 245         uint16_t                iport_task_red_limit;
 246         /* cmd slots */
 247         uint16_t                iport_nslots_free;
 248 
 249         /* upper 16 bits is just a counter to avoid ABA issues */
 250         uint32_t                iport_next_free_slot;
 251 
 252         uint8_t                 iport_login_retry; /* for flogi and N2N plogi */
 253         uint8_t                 iport_link_old_topology;
 254         uint8_t                 iport_link_cleanup_retry;
 255         clock_t                 iport_li_cmd_timeout; /* for li state m/c */
 256         fct_cmd_slot_t          *iport_cmd_slots;
 257 
 258         /* worker thread data */
 259         ddi_taskq_t             *iport_worker_taskq;
 260         kmutex_t                iport_worker_lock;
 261         kcondvar_t              iport_worker_cv;
 262         struct fct_i_event      *iport_event_head;
 263         struct fct_i_event      *iport_event_tail;
 264         struct fct_i_cmd        *iport_abort_queue;
 265         struct fct_i_cmd        **iport_ppicmd_term;
 266 
 267         /* link initialization */
 268         fct_status_t            iport_li_comp_status;
 269         enum fct_li_state       iport_li_state;
 270 
 271         /* solicited cmd link */
 272         struct fct_i_cmd        *iport_solcmd_queue;
 273 
 274         /* rpwe = remote port with pending els(es) */
 275         fct_i_remote_port_t     *iport_rpwe_head;
 276         fct_i_remote_port_t     *iport_rpwe_tail;
 277         kstat_t                 *iport_kstat_portstat;
 278         ksema_t                 iport_rls_sema;
 279         fct_rls_cb_data_t       iport_rls_cb_data;
 280 } fct_i_local_port_t;
 281 
 282 #define IPORT_FLOGI_DONE(iport) PORT_FLOGI_DONE(&(iport)->iport_link_info)
 283 
 284 /*
 285  * iport flags


 418     void *arg, uint8_t *buf, uint32_t *bufsizep);
 419 void fct_event_handler(stmf_local_port_t *lport, int eventid,
 420     void *arg, uint32_t flags);
 421 uint16_t fct_alloc_cmd_slot(fct_i_local_port_t *iport, fct_cmd_t *cmd);
 422 void fct_post_to_discovery_queue(fct_i_local_port_t *iport,
 423     fct_i_remote_port_t *irp, fct_i_cmd_t *icmd);
 424 fct_cmd_t *fct_create_solct(fct_local_port_t *port, fct_remote_port_t *rp,
 425     uint16_t ctop, fct_icmd_cb_t icmdcb);
 426 fct_cmd_t *fct_create_solels(fct_local_port_t *port, fct_remote_port_t *rp,
 427     int implicit, uchar_t elsop, uint32_t wkdid, fct_icmd_cb_t icmdcb);
 428 void fct_handle_solct(fct_cmd_t *cmd);
 429 void fct_post_to_solcmd_queue(fct_local_port_t *port, fct_cmd_t *cmd);
 430 void fct_logo_cb(fct_i_cmd_t *icmd);
 431 void fct_link_init_cb(fct_i_cmd_t *icmd);
 432 void fct_gsnn_cb(fct_i_cmd_t *icmd);
 433 void fct_gcs_cb(fct_i_cmd_t *icmd);
 434 void fct_gft_cb(fct_i_cmd_t *icmd);
 435 void fct_gspn_cb(fct_i_cmd_t *icmd);
 436 void fct_rls_cb(fct_i_cmd_t *icmd);
 437 disc_action_t fct_process_link_init(fct_i_local_port_t *iport);

 438 
 439 #ifdef  __cplusplus
 440 }
 441 #endif
 442 
 443 #endif /* _FCT_IMPL_H */


   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 #ifndef _FCT_IMPL_H
  27 #define _FCT_IMPL_H
  28 
  29 #ifdef  __cplusplus
  30 extern "C" {
  31 #endif
  32 
  33 #define RSCN_OPTION_VERIFY      0x0001
  34 
  35 typedef enum fct_li_state {
  36         LI_STATE_DO_FLOGI = 0,  /* FLOGI handled by FCA */
  37         LI_STATE_FINI_TOPOLOGY, /* Finalize topology */
  38         LI_STATE_N2N_PLOGI,     /* In case of a N2N connection */
  39 
  40         LI_STATE_DO_FCLOGIN,    /* Login into 0xFFFFFD */
  41         LI_STATE_DO_SCR,        /* State change registration */
  42 
  43         LI_STATE_DO_NSLOGIN,    /* Login into 0xFFFFFC */


  56 #define LI_STATE_FLAG_CMD_RETCHECK      0x40
  57 /* Link init cmd is still outstanding */
  58 #define LI_STATE_FLAG_CMD_WAITING       0x80
  59 /* Flag to indicate that link info is not available yet */
  60 #define LI_STATE_FLAG_NO_LI_YET         0x100
  61 
  62 #define FCT_MAX_CACHED_CMDS     256
  63 #define USEC_ELS_TIMEOUT        (10 * 1000 * 1000)
  64 #define USEC_SOL_TIMEOUT        (10 * 1000 * 1000)
  65 #define USEC_DEREG_RP_TIMEOUT   (25 * 1000 * 1000)
  66 #define USEC_DEREG_RP_INTERVAL  (2 * 1000 * 1000)
  67 
  68 struct fct_i_cmd;
  69 typedef void (* fct_icmd_cb_t)(struct fct_i_cmd *icmd);
  70 typedef struct fct_i_cmd {
  71         fct_cmd_t               *icmd_cmd;
  72         uint32_t                 icmd_alloc_size;
  73         fct_struct_id_t          icmd_struct_id;
  74         uint32_t                 icmd_flags;
  75         clock_t                  icmd_start_time;
  76         list_node_t             icmd_node;      /* iport_abort_queue and irp */
  77         struct fct_i_cmd        *icmd_solcmd_next;      /* iport_solcmd_queue */
  78         fct_icmd_cb_t            icmd_cb;
  79         void                    *icmd_cb_private;
  80 } fct_i_cmd_t;
  81 
  82 /*
  83  * icmd_flags
  84  */
  85 #define ICMD_SESSION_AFFECTING          0x0002
  86 #define ICMD_IN_IRP_QUEUE               0x0004
  87 #define ICMD_BEING_ABORTED              0x0008
  88 #define ICMD_KNOWN_TO_FCA               0x0020
  89 #define ICMD_FCA_ABORT_CALLED           0x0040
  90 #define ICMD_CMD_COMPLETE               0x0080
  91 #define ICMD_IN_TRANSITION              0x0100
  92 #define ICMD_ABTS_RECEIVED              0x0200
  93 #define ICMD_IMPLICIT                   0x0400
  94 #define ICMD_IMPLICIT_CMD_HAS_RESOURCE  0x0800
  95 /* High order are debug flags */
  96 #define ICMD_ELS_PROCESSING_STARTED     0x80000000


 100  * 1) it's new. We need send it to FCA. ICMD_SOLCMD_NEW is set
 101  * 2) it's running. We are waiting for completion.
 102  * 3) it's completed. We need free it. ICMD_CMD_COMPLETE is set
 103  * ICMD_SOLCMD_NEW and ICMD_CMD_COMPLETE should not be set in the same time
 104  */
 105 #define ICMD_IN_SOLCMD_QUEUE            0x010000
 106 #define ICMD_SOLCMD_NEW                 0x020000
 107 
 108 typedef struct fct_i_remote_port {
 109         fct_remote_port_t               *irp_rp;
 110         uint32_t                        irp_alloc_size;
 111         fct_struct_id_t                 irp_struct_id;
 112         krwlock_t                       irp_lock;
 113 
 114         /* For queueing to local port */
 115         struct fct_i_remote_port        *irp_next;
 116 
 117         /* For queueing to handle elses */
 118         struct fct_i_remote_port        *irp_discovery_next;
 119 
 120         list_t                          irp_els_list;
 121 
 122         /*
 123          * sa stands for session affecting, nsa is non session affecting.
 124          * The els counts only represent elses under progress not the ones
 125          * that are terminated. active_xchg_count covers everything including
 126          * the ones waiting to be terminated.
 127          */
 128         uint16_t                        irp_sa_elses_count;
 129         uint16_t                        irp_nsa_elses_count;
 130         uint16_t                        irp_fcp_xchg_count;
 131         uint16_t                        irp_nonfcp_xchg_count;
 132 
 133         uint32_t                        irp_flags;
 134         clock_t                         irp_deregister_timer;
 135         uint32_t                        irp_dereg_count;
 136 
 137         uint32_t                        irp_portid;
 138         uint8_t                         irp_id[24];
 139         uint32_t                        irp_rcvd_prli_params;
 140         uint32_t                        irp_sent_prli_params;
 141 
 142         /*
 143          * Most HBAs will only register symbolic node name instead of port name,
 144          * so we use SNN as session alias.
 145          */
 146         stmf_scsi_session_t             *irp_session;
 147         char                            *irp_snn;
 148         uint16_t                        irp_snn_len;
 149 
 150         /* items will be filled in ns cmd */
 151         uint8_t                         irp_fc4types[32]; /* FC-4 types */
 152         char                            *irp_spn;       /* port symbolic name */
 153         uint16_t                        irp_spn_len;
 154         uint32_t                        irp_cos;        /* class of service */
 155 
 156         uint32_t                        irp_rscn_counter;
 157 } fct_i_remote_port_t;
 158 
 159 /*
 160  * structure used for fct_rls_cb() callback private data
 161  */
 162 typedef struct fct_rls_cb_data {
 163         struct fct_port_link_status     *fct_link_status;
 164         fct_status_t                    fct_els_res;
 165 } fct_rls_cb_data_t;
 166 
 167 /*
 168  * irp flags
 169  */
 170 #define IRP_PLOGI_DONE                  0x0001
 171 #define IRP_PRLI_DONE                   0x0002
 172 #define IRP_IN_DISCOVERY_QUEUE          0x0004
 173 #define IRP_FCP_CLEANUP                 0x0008


 218         uint64_t                iport_last_change;
 219 
 220         /*
 221          * These variables are used to manage fct_cmd_t cache for SCSI traffic
 222          */
 223         /*
 224          * Total # of cmds allocated by the driver. Some of which are free
 225          * and sitting on iport_cached_cmdlist. And some are executing.
 226          */
 227         uint32_t                iport_total_alloced_ncmds;
 228 
 229         /*
 230          * Max active cmds in last interval (10 or 30 seconds)
 231          */
 232         uint32_t                iport_max_active_ncmds;
 233 
 234         /*
 235          * # of free cmds sitting on the iport_cached_cmdlist
 236          */
 237         uint32_t                iport_cached_ncmds;
 238         list_t                  iport_cached_cmdlist;
 239         kmutex_t                iport_cached_cmd_lock;
 240 
 241         /*
 242          * To release free cmds periodically
 243          */
 244         clock_t                 iport_cmdcheck_clock;
 245 
 246         uint16_t                iport_task_green_limit;
 247         uint16_t                iport_task_yellow_limit;
 248         uint16_t                iport_task_red_limit;
 249         /* cmd slots */
 250         uint16_t                iport_nslots_free;
 251 
 252         /* upper 16 bits is just a counter to avoid ABA issues */
 253         uint32_t                iport_next_free_slot;
 254 
 255         uint8_t                 iport_login_retry; /* for flogi and N2N plogi */
 256         uint8_t                 iport_link_old_topology;
 257         uint8_t                 iport_link_cleanup_retry;
 258         clock_t                 iport_li_cmd_timeout; /* for li state m/c */
 259         fct_cmd_slot_t          *iport_cmd_slots;
 260 
 261         /* worker thread data */
 262         ddi_taskq_t             *iport_worker_taskq;
 263         kmutex_t                iport_worker_lock;
 264         kcondvar_t              iport_worker_cv;
 265         struct fct_i_event      *iport_event_head;
 266         struct fct_i_event      *iport_event_tail;
 267         list_t                  iport_abort_queue;
 268         struct fct_i_cmd        *iport_ppicmd_term;
 269 
 270         /* link initialization */
 271         fct_status_t            iport_li_comp_status;
 272         enum fct_li_state       iport_li_state;
 273 
 274         /* solicited cmd link */
 275         struct fct_i_cmd        *iport_solcmd_queue;
 276 
 277         /* rpwe = remote port with pending els(es) */
 278         fct_i_remote_port_t     *iport_rpwe_head;
 279         fct_i_remote_port_t     *iport_rpwe_tail;
 280         kstat_t                 *iport_kstat_portstat;
 281         ksema_t                 iport_rls_sema;
 282         fct_rls_cb_data_t       iport_rls_cb_data;
 283 } fct_i_local_port_t;
 284 
 285 #define IPORT_FLOGI_DONE(iport) PORT_FLOGI_DONE(&(iport)->iport_link_info)
 286 
 287 /*
 288  * iport flags


 421     void *arg, uint8_t *buf, uint32_t *bufsizep);
 422 void fct_event_handler(stmf_local_port_t *lport, int eventid,
 423     void *arg, uint32_t flags);
 424 uint16_t fct_alloc_cmd_slot(fct_i_local_port_t *iport, fct_cmd_t *cmd);
 425 void fct_post_to_discovery_queue(fct_i_local_port_t *iport,
 426     fct_i_remote_port_t *irp, fct_i_cmd_t *icmd);
 427 fct_cmd_t *fct_create_solct(fct_local_port_t *port, fct_remote_port_t *rp,
 428     uint16_t ctop, fct_icmd_cb_t icmdcb);
 429 fct_cmd_t *fct_create_solels(fct_local_port_t *port, fct_remote_port_t *rp,
 430     int implicit, uchar_t elsop, uint32_t wkdid, fct_icmd_cb_t icmdcb);
 431 void fct_handle_solct(fct_cmd_t *cmd);
 432 void fct_post_to_solcmd_queue(fct_local_port_t *port, fct_cmd_t *cmd);
 433 void fct_logo_cb(fct_i_cmd_t *icmd);
 434 void fct_link_init_cb(fct_i_cmd_t *icmd);
 435 void fct_gsnn_cb(fct_i_cmd_t *icmd);
 436 void fct_gcs_cb(fct_i_cmd_t *icmd);
 437 void fct_gft_cb(fct_i_cmd_t *icmd);
 438 void fct_gspn_cb(fct_i_cmd_t *icmd);
 439 void fct_rls_cb(fct_i_cmd_t *icmd);
 440 disc_action_t fct_process_link_init(fct_i_local_port_t *iport);
 441 void fct_cmd_unlink_els(fct_i_remote_port_t *irp, fct_i_cmd_t *icmd);
 442 
 443 #ifdef  __cplusplus
 444 }
 445 #endif
 446 
 447 #endif /* _FCT_IMPL_H */