--- old/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c	Mon Aug  5 17:31:21 2019
+++ new/usr/src/uts/common/io/scsi/adapters/mpt_sas/mptsas.c	Mon Aug  5 17:31:21 2019
@@ -21,7 +21,7 @@
 
 /*
  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
+ * Copyright 2019 Nexenta Systems, Inc.
  * Copyright (c) 2017, Joyent, Inc.
  * Copyright 2014 OmniTI Computer Consulting, Inc. All rights reserved.
  * Copyright (c) 2014, Tegile Systems Inc. All rights reserved.
@@ -79,6 +79,8 @@
 #include <sys/sata/sata_hba.h>
 #include <sys/scsi/generic/sas.h>
 #include <sys/scsi/impl/scsi_sas.h>
+#include <sys/sdt.h>
+#include <sys/mdi_impldefs.h>
 
 #pragma pack(1)
 #include <sys/scsi/adapters/mpt_sas/mpi/mpi2_type.h>
@@ -102,8 +104,6 @@
 #include <sys/scsi/adapters/mpt_sas/mptsas_hash.h>
 #include <sys/raidioctl.h>
 
-#include <sys/fs/dv_node.h>	/* devfs_clean */
-
 /*
  * FMA header files
  */
@@ -357,13 +357,14 @@
     uint16_t *handle, mptsas_target_t **pptgt);
 static void mptsas_update_phymask(mptsas_t *mpt);
 
-static int mptsas_send_sep(mptsas_t *mpt, mptsas_target_t *ptgt,
+static int mptsas_flush_led_status(mptsas_t *mpt, mptsas_enclosure_t *mep,
+    uint16_t idx);
+static int mptsas_send_sep(mptsas_t *mpt, mptsas_enclosure_t *mep, uint16_t idx,
     uint32_t *status, uint8_t cmd);
 static dev_info_t *mptsas_get_dip_from_dev(dev_t dev,
     mptsas_phymask_t *phymask);
 static mptsas_target_t *mptsas_addr_to_ptgt(mptsas_t *mpt, char *addr,
     mptsas_phymask_t phymask);
-static int mptsas_flush_led_status(mptsas_t *mpt, mptsas_target_t *ptgt);
 
 
 /*
@@ -396,13 +397,11 @@
 
 static void mptsas_offline_missed_luns(dev_info_t *pdip,
     uint16_t *repluns, int lun_cnt, mptsas_target_t *ptgt);
-static int mptsas_offline_lun(dev_info_t *pdip, dev_info_t *rdip,
-    mdi_pathinfo_t *rpip, uint_t flags);
+static int mptsas_offline_lun(dev_info_t *rdip, mdi_pathinfo_t *rpip);
 
 static int mptsas_config_smp(dev_info_t *pdip, uint64_t sas_wwn,
     dev_info_t **smp_dip);
-static int mptsas_offline_smp(dev_info_t *pdip, mptsas_smp_t *smp_node,
-    uint_t flags);
+static int mptsas_offline_smp(dev_info_t *pdip, mptsas_smp_t *smp_node);
 
 static int mptsas_event_query(mptsas_t *mpt, mptsas_event_query_t *data,
     int mode, int *rval);
@@ -469,6 +468,13 @@
 int mptsas_inq83_retry_timeout = 30;
 
 /*
+ * Tunable for default SCSI pkt timeout. Defaults to 5 seconds, which should
+ * be plenty for INQUIRY and REPORT_LUNS, which are the only commands currently
+ * issued by mptsas directly.
+ */
+int mptsas_scsi_pkt_time = 5;
+
+/*
  * This is used to allocate memory for message frame storage, not for
  * data I/O DMA. All message frames must be stored in the first 4G of
  * physical memory.
@@ -732,18 +738,6 @@
 }
 
 static int
-mptsas_target_eval_slot(const void *op, void *arg)
-{
-	mptsas_led_control_t *lcp = arg;
-	const mptsas_target_t *tp = op;
-
-	if (tp->m_enclosure != lcp->Enclosure)
-		return ((int)tp->m_enclosure - (int)lcp->Enclosure);
-
-	return ((int)tp->m_slot_num - (int)lcp->Slot);
-}
-
-static int
 mptsas_target_eval_nowwn(const void *op, void *arg)
 {
 	uint8_t phy = *(uint8_t *)arg;
@@ -1204,8 +1198,7 @@
 	 * Allocate softc information.
 	 */
 	if (ddi_soft_state_zalloc(mptsas_state, instance) != DDI_SUCCESS) {
-		mptsas_log(NULL, CE_WARN,
-		    "mptsas%d: cannot allocate soft state", instance);
+		mptsas_log(NULL, CE_WARN, "cannot allocate soft state");
 		goto fail;
 	}
 
@@ -1212,8 +1205,7 @@
 	mpt = ddi_get_soft_state(mptsas_state, instance);
 
 	if (mpt == NULL) {
-		mptsas_log(NULL, CE_WARN,
-		    "mptsas%d: cannot get soft state", instance);
+		mptsas_log(NULL, CE_WARN, "cannot get soft state");
 		goto fail;
 	}
 
@@ -1298,7 +1290,19 @@
 	    0, "mptsas_doneq_length_threshold_prop", 8);
 	mpt->m_doneq_thread_n = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
 	    0, "mptsas_doneq_thread_n_prop", 8);
+	mpt->m_max_tune_throttle = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
+	    0, "mptsas_max_throttle", MAX_THROTTLE);
 
+	/*
+	 *  Error check to make sure value is withing range. If nothing
+	 *  is set default to original design value.
+	 */
+	if (mpt->m_max_tune_throttle < THROTTLE_LO) {
+		mpt->m_max_tune_throttle = MAX_THROTTLE;
+	} else if (mpt->m_max_tune_throttle > THROTTLE_HI) {
+		mpt->m_max_tune_throttle = THROTTLE_HI;
+	}
+
 	if (mpt->m_doneq_thread_n) {
 		cv_init(&mpt->m_doneq_thread_cv, NULL, CV_DRIVER, NULL);
 		mutex_init(&mpt->m_doneq_mutex, NULL, MUTEX_DRIVER, NULL);
@@ -1356,9 +1360,12 @@
 	cv_init(&mpt->m_fw_cv, NULL, CV_DRIVER, NULL);
 	cv_init(&mpt->m_config_cv, NULL, CV_DRIVER, NULL);
 	cv_init(&mpt->m_fw_diag_cv, NULL, CV_DRIVER, NULL);
-	cv_init(&mpt->m_extreq_sense_refcount_cv, NULL, CV_DRIVER, NULL);
 	mutex_init_done++;
 
+#ifdef MPTSAS_FAULTINJECTION
+	TAILQ_INIT(&mpt->m_fminj_cmdq);
+#endif
+
 	mutex_enter(&mpt->m_mutex);
 	/*
 	 * Initialize power management component
@@ -1449,7 +1456,7 @@
 	if (mpt->m_scsi_reset_delay == 0) {
 		mptsas_log(mpt, CE_NOTE,
 		    "scsi_reset_delay of 0 is not recommended,"
-		    " resetting to SCSI_DEFAULT_RESET_DELAY\n");
+		    " resetting to SCSI_DEFAULT_RESET_DELAY");
 		mpt->m_scsi_reset_delay = SCSI_DEFAULT_RESET_DELAY;
 	}
 
@@ -1649,7 +1656,6 @@
 			cv_destroy(&mpt->m_fw_cv);
 			cv_destroy(&mpt->m_config_cv);
 			cv_destroy(&mpt->m_fw_diag_cv);
-			cv_destroy(&mpt->m_extreq_sense_refcount_cv);
 		}
 
 		if (map_setup) {
@@ -1897,7 +1903,8 @@
 			 */
 			ndi_devi_enter(scsi_vhci_dip, &circ1);
 			ndi_devi_enter(dip, &circ);
-			while (pip = mdi_get_next_client_path(dip, NULL)) {
+			while ((pip = mdi_get_next_client_path(dip, NULL)) !=
+			    NULL) {
 				if (mdi_pi_free(pip, 0) == MDI_SUCCESS) {
 					continue;
 				}
@@ -1924,8 +1931,7 @@
 			if (pm_raise_power(dip, 0, PM_LEVEL_D0) !=
 			    DDI_SUCCESS) {
 				mptsas_log(mpt, CE_WARN,
-				    "mptsas%d: Raise power request failed.",
-				    mpt->m_instance);
+				    "raise power request failed");
 				(void) pm_idle_component(dip, 0);
 				return (DDI_FAILURE);
 			}
@@ -2051,9 +2057,8 @@
 	if (mpt->m_options & MPTSAS_OPT_PM) {
 		if (pm_lower_power(dip, 0, PM_LEVEL_D3) != DDI_SUCCESS)
 			mptsas_log(mpt, CE_WARN,
-			    "!mptsas%d: Lower power request failed "
-			    "during detach, ignoring.",
-			    mpt->m_instance);
+			    "lower power request failed during detach, "
+			    "ignoring");
 	}
 
 	mutex_destroy(&mpt->m_tx_waitq_mutex);
@@ -2067,8 +2072,11 @@
 	cv_destroy(&mpt->m_fw_cv);
 	cv_destroy(&mpt->m_config_cv);
 	cv_destroy(&mpt->m_fw_diag_cv);
-	cv_destroy(&mpt->m_extreq_sense_refcount_cv);
 
+#ifdef MPTSAS_FAULTINJECTION
+	ASSERT(TAILQ_EMPTY(&mpt->m_fminj_cmdq));
+#endif
+
 	mptsas_smp_teardown(mpt);
 	mptsas_enc_teardown(mpt);
 	mptsas_hba_teardown(mpt);
@@ -2329,12 +2337,24 @@
 }
 
 static void
+mptsas_enc_free(mptsas_enclosure_t *mep)
+{
+	if (mep == NULL)
+		return;
+	if (mep->me_slotleds != NULL) {
+		VERIFY3U(mep->me_nslots, >, 0);
+		kmem_free(mep->me_slotleds, sizeof (uint8_t) * mep->me_nslots);
+	}
+	kmem_free(mep, sizeof (mptsas_enclosure_t));
+}
+
+static void
 mptsas_enc_teardown(mptsas_t *mpt)
 {
 	mptsas_enclosure_t *mep;
 
 	while ((mep = list_remove_head(&mpt->m_enclosures)) != NULL) {
-		kmem_free(mep, sizeof (mptsas_enclosure_t));
+		mptsas_enc_free(mep);
 	}
 	list_destroy(&mpt->m_enclosures);
 }
@@ -2471,8 +2491,7 @@
 		MPTSAS_POWER_OFF(mpt);
 		break;
 	default:
-		mptsas_log(mpt, CE_WARN, "mptsas%d: unknown power level <%x>.",
-		    mpt->m_instance, level);
+		mptsas_log(mpt, CE_WARN, "unknown power level <%x>", level);
 		rval = DDI_FAILURE;
 		break;
 	}
@@ -2701,8 +2720,6 @@
 	size_t			mem_size;
 	int			num_extrqsense_bufs;
 
-	ASSERT(mpt->m_extreq_sense_refcount == 0);
-
 	/*
 	 * re-alloc when it has already alloced
 	 */
@@ -3045,7 +3062,7 @@
 
 	ASSERT(reallen < len);
 	if (reallen >= len) {
-		mptsas_log(0, CE_WARN, "!mptsas_get_name: name parameter "
+		mptsas_log(0, CE_WARN, "mptsas_get_name: name parameter "
 		    "length too small, it needs to be %d bytes", reallen + 1);
 	}
 	return (DDI_SUCCESS);
@@ -3092,11 +3109,18 @@
 		ddi_set_name_addr(tgt_dip, NULL);
 		return (DDI_FAILURE);
 	}
+
 	/*
-	 * phymask is 0 means the virtual port for RAID
+	 * The phymask exists if the port is active, otherwise
+	 * nothing to do.
 	 */
+	if (ddi_prop_exists(DDI_DEV_T_ANY, hba_dip,
+	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "phymask") == 0)
+		return (DDI_FAILURE);
+
 	phymask = (mptsas_phymask_t)ddi_prop_get_int(DDI_DEV_T_ANY, hba_dip, 0,
 	    "phymask", 0);
+
 	if (mdi_component_is_client(tgt_dip, NULL) == MDI_SUCCESS) {
 		if ((pip = (void *)(sd->sd_private)) == NULL) {
 			/*
@@ -3108,7 +3132,7 @@
 
 		if (mdi_prop_lookup_int(pip, LUN_PROP, &lun) !=
 		    DDI_PROP_SUCCESS) {
-			mptsas_log(mpt, CE_WARN, "Get lun property failed\n");
+			mptsas_log(mpt, CE_WARN, "Get lun property failed");
 			return (DDI_FAILURE);
 		}
 
@@ -3141,7 +3165,7 @@
 	ptgt = refhash_lookup(mpt->m_targets, &addr);
 	mutex_exit(&mpt->m_mutex);
 	if (ptgt == NULL) {
-		mptsas_log(mpt, CE_WARN, "!tgt_init: target doesn't exist or "
+		mptsas_log(mpt, CE_WARN, "tgt_init: target doesn't exist or "
 		    "gone already! phymask:%x, saswwn %"PRIx64, phymask,
 		    sas_wwn);
 		return (DDI_FAILURE);
@@ -3186,7 +3210,7 @@
 				kmem_free(inq89, inq89_len);
 			}
 
-			mptsas_log(mpt, CE_WARN, "!mptsas request inquiry page "
+			mptsas_log(mpt, CE_WARN, "mptsas request inquiry page "
 			    "0x89 for SATA target:%x failed!", ptgt->m_devhdl);
 			return (DDI_SUCCESS);
 		}
@@ -3439,7 +3463,118 @@
 	return (rval);
 }
 
+#ifdef MPTSAS_FAULTINJECTION
+static void
+mptsas_fminj_move_cmd_to_doneq(mptsas_t *mpt, mptsas_cmd_t *cmd,
+    uchar_t reason, uint_t stat)
+{
+	struct scsi_pkt *pkt = cmd->cmd_pkt;
+
+	TAILQ_REMOVE(&mpt->m_fminj_cmdq, cmd, cmd_active_link);
+
+	/* Setup reason/statistics. */
+	pkt->pkt_reason = reason;
+	pkt->pkt_statistics = stat;
+
+	cmd->cmd_active_expiration = 0;
+
+	/* Move command to doneque. */
+	cmd->cmd_linkp = NULL;
+	cmd->cmd_flags |= CFLAG_FINISHED;
+	cmd->cmd_flags &= ~CFLAG_IN_TRANSPORT;
+
+	*mpt->m_donetail = cmd;
+	mpt->m_donetail = &cmd->cmd_linkp;
+	mpt->m_doneq_len++;
+}
+
+static void
+mptsas_fminj_move_tgt_to_doneq(mptsas_t *mpt, ushort_t target,
+    uchar_t reason, uint_t stat)
+{
+	mptsas_cmd_t *cmd;
+
+	ASSERT(mutex_owned(&mpt->m_mutex));
+
+	if (!TAILQ_EMPTY(&mpt->m_fminj_cmdq)) {
+		cmd = TAILQ_FIRST(&mpt->m_fminj_cmdq);
+		ASSERT(cmd != NULL);
+
+		while (cmd != NULL) {
+			mptsas_cmd_t *next = TAILQ_NEXT(cmd, cmd_active_link);
+
+			if (Tgt(cmd) == target) {
+				mptsas_fminj_move_cmd_to_doneq(mpt, cmd,
+				    reason, stat);
+			}
+			cmd = next;
+		}
+	}
+}
+
+static void
+mptsas_fminj_watchsubr(mptsas_t *mpt,
+    struct mptsas_active_cmdq *expired)
+{
+	mptsas_cmd_t *cmd;
+
+	ASSERT(mutex_owned(&mpt->m_mutex));
+
+	if (!TAILQ_EMPTY(&mpt->m_fminj_cmdq)) {
+		hrtime_t timestamp = gethrtime();
+
+		cmd = TAILQ_FIRST(&mpt->m_fminj_cmdq);
+		ASSERT(cmd != NULL);
+
+		while (cmd != NULL) {
+			mptsas_cmd_t *next = TAILQ_NEXT(cmd, cmd_active_link);
+
+			if (cmd->cmd_active_expiration <= timestamp) {
+				struct scsi_pkt *pkt = cmd->cmd_pkt;
+
+				DTRACE_PROBE1(mptsas__command__timeout,
+				    struct scsi_pkt *, pkt);
+
+				/* Setup proper flags. */
+				pkt->pkt_reason = CMD_TIMEOUT;
+				pkt->pkt_statistics = (STAT_TIMEOUT |
+				    STAT_DEV_RESET);
+				cmd->cmd_active_expiration = 0;
+
+				TAILQ_REMOVE(&mpt->m_fminj_cmdq, cmd,
+				    cmd_active_link);
+				TAILQ_INSERT_TAIL(expired, cmd,
+				    cmd_active_link);
+			}
+			cmd = next;
+		}
+	}
+}
+
 static int
+mptsas_fminject(mptsas_t *mpt, mptsas_cmd_t *cmd)
+{
+	struct scsi_pkt *pkt = cmd->cmd_pkt;
+
+	ASSERT(mutex_owned(&mpt->m_mutex));
+
+	if (pkt->pkt_flags & FLAG_PKT_TIMEOUT) {
+		if (((pkt->pkt_flags & FLAG_NOINTR) == 0) &&
+		    (pkt->pkt_comp != NULL)) {
+			pkt->pkt_state = (STATE_GOT_BUS|STATE_GOT_TARGET|
+			    STATE_SENT_CMD);
+			cmd->cmd_active_expiration =
+			    gethrtime() + (hrtime_t)pkt->pkt_time * NANOSEC;
+			TAILQ_INSERT_TAIL(&mpt->m_fminj_cmdq,
+			    cmd, cmd_active_link);
+			return (0);
+		}
+	}
+	return (-1);
+}
+#endif /* MPTSAS_FAULTINJECTION */
+
+static int
 mptsas_accept_pkt(mptsas_t *mpt, mptsas_cmd_t *cmd)
 {
 	int		rval = TRAN_ACCEPT;
@@ -3468,13 +3603,14 @@
 	}
 
 	/*
-	 * If HBA is being reset, the DevHandles are being re-initialized,
-	 * which means that they could be invalid even if the target is still
-	 * attached.  Check if being reset and if DevHandle is being
-	 * re-initialized.  If this is the case, return BUSY so the I/O can be
-	 * retried later.
+	 * If HBA is being reset, the device handles will be invalidated.
+	 * This is temporary and, if target is still attached, the device
+	 * handles will be re-assigned when firmware reset completes.
+	 * Then, if command was already waiting, complete the command
+	 * otherwise return BUSY and expect transport retry.
 	 */
 	if ((ptgt->m_devhdl == MPTSAS_INVALID_DEVHDL) && mpt->m_in_reset) {
+		NDBG20(("retry command, invalid devhdl, during FW reset."));
 		mptsas_set_pkt_reason(mpt, cmd, CMD_RESET, STAT_BUS_RESET);
 		if (cmd->cmd_flags & CFLAG_TXQ) {
 			mptsas_doneq_add(mpt, cmd);
@@ -3486,15 +3622,14 @@
 	}
 
 	/*
-	 * If device handle has already been invalidated, just
-	 * fail the command. In theory, command from scsi_vhci
-	 * client is impossible send down command with invalid
-	 * devhdl since devhdl is set after path offline, target
-	 * driver is not suppose to select a offlined path.
+	 * If the device handle has been invalidated, set the response
+	 * reason to indicate the device is gone. Then add the
+	 * command to the done queue and run the completion routine
+	 * so the initiator of the command can clean up.
 	 */
 	if (ptgt->m_devhdl == MPTSAS_INVALID_DEVHDL) {
-		NDBG3(("rejecting command, it might because invalid devhdl "
-		    "request."));
+		NDBG20(("rejecting command, invalid devhdl because "
+		    "device gone."));
 		mptsas_set_pkt_reason(mpt, cmd, CMD_DEV_GONE, STAT_TERMINATED);
 		if (cmd->cmd_flags & CFLAG_TXQ) {
 			mptsas_doneq_add(mpt, cmd);
@@ -3504,7 +3639,18 @@
 			return (TRAN_FATAL_ERROR);
 		}
 	}
+
 	/*
+	 * Do fault injecttion before transmitting command.
+	 * FLAG_NOINTR commands are skipped.
+	 */
+#ifdef MPTSAS_FAULTINJECTION
+	if (!mptsas_fminject(mpt, cmd)) {
+		return (TRAN_ACCEPT);
+	}
+#endif
+
+	/*
 	 * The first case is the normal case.  mpt gets a command from the
 	 * target driver and starts it.
 	 * Since SMID 0 is reserved and the TM slot is reserved, the actual max
@@ -3617,6 +3763,13 @@
 
 	NDBG1(("mptsas_prepare_pkt: cmd=0x%p", (void *)cmd));
 
+#ifdef MPTSAS_FAULTINJECTION
+	/* Check for fault flags prior to perform actual initialization. */
+	if (pkt->pkt_flags & FLAG_PKT_BUSY) {
+		return (TRAN_BUSY);
+	}
+#endif
+
 	/*
 	 * Reinitialize some fields that need it; the packet may
 	 * have been resubmitted
@@ -3729,56 +3882,13 @@
 		if ((cmdlen > sizeof (cmd->cmd_cdb)) ||
 		    (tgtlen > PKT_PRIV_LEN) ||
 		    (statuslen > EXTCMDS_STATUS_SIZE)) {
-			int failure;
-
-			/*
-			 * We are going to allocate external packet space which
-			 * might include the sense data buffer for DMA so we
-			 * need to increase the reference counter here.  In a
-			 * case the HBA is in reset we just simply free the
-			 * allocated packet and bail out.
-			 */
-			mutex_enter(&mpt->m_mutex);
-			if (mpt->m_in_reset) {
-				mutex_exit(&mpt->m_mutex);
-
-				cmd->cmd_flags = CFLAG_FREE;
-				kmem_cache_free(mpt->m_kmem_cache, cmd);
-				return (NULL);
-			}
-			mpt->m_extreq_sense_refcount++;
-			ASSERT(mpt->m_extreq_sense_refcount > 0);
-			mutex_exit(&mpt->m_mutex);
-
-			/*
-			 * if extern alloc fails, all will be
-			 * deallocated, including cmd
-			 */
-			failure = mptsas_pkt_alloc_extern(mpt, cmd,
-			    cmdlen, tgtlen, statuslen, kf);
-
-			if (failure != 0 || cmd->cmd_extrqslen == 0) {
+			if (mptsas_pkt_alloc_extern(mpt, cmd,
+			    cmdlen, tgtlen, statuslen, kf)) {
 				/*
-				 * If the external packet space allocation
-				 * failed, or we didn't allocate the sense
-				 * data buffer for DMA we need to decrease the
-				 * reference counter.
+				 * if extern allocation fails, it will
+				 * deallocate the new pkt as well
 				 */
-				mutex_enter(&mpt->m_mutex);
-				ASSERT(mpt->m_extreq_sense_refcount > 0);
-				mpt->m_extreq_sense_refcount--;
-				if (mpt->m_extreq_sense_refcount == 0)
-					cv_broadcast(
-					    &mpt->m_extreq_sense_refcount_cv);
-				mutex_exit(&mpt->m_mutex);
-
-				if (failure != 0) {
-					/*
-					 * if extern allocation fails, it will
-					 * deallocate the new pkt as well
-					 */
-					return (NULL);
-				}
+				return (NULL);
 			}
 		}
 		new_cmd = cmd;
@@ -3785,6 +3895,8 @@
 
 	} else {
 		cmd = PKT2CMD(pkt);
+		pkt->pkt_start = 0;
+		pkt->pkt_stop = 0;
 		new_cmd = NULL;
 	}
 
@@ -3903,7 +4015,7 @@
 		ASSERT(cmd->cmd_cookiec > 0);
 
 		if (cmd->cmd_cookiec > MPTSAS_MAX_CMD_SEGS) {
-			mptsas_log(mpt, CE_NOTE, "large cookiec received %d\n",
+			mptsas_log(mpt, CE_NOTE, "large cookiec received %d",
 			    cmd->cmd_cookiec);
 			bioerror(bp, EINVAL);
 			if (new_cmd) {
@@ -4065,22 +4177,7 @@
 		cmd->cmd_flags = CFLAG_FREE;
 		kmem_cache_free(mpt->m_kmem_cache, (void *)cmd);
 	} else {
-		boolean_t extrqslen = cmd->cmd_extrqslen != 0;
-
 		mptsas_pkt_destroy_extern(mpt, cmd);
-
-		/*
-		 * If the packet had the sense data buffer for DMA allocated we
-		 * need to decrease the reference counter.
-		 */
-		if (extrqslen) {
-			mutex_enter(&mpt->m_mutex);
-			ASSERT(mpt->m_extreq_sense_refcount > 0);
-			mpt->m_extreq_sense_refcount--;
-			if (mpt->m_extreq_sense_refcount == 0)
-				cv_broadcast(&mpt->m_extreq_sense_refcount_cv);
-			mutex_exit(&mpt->m_mutex);
-		}
 	}
 }
 
@@ -5240,8 +5337,7 @@
 	 * would not come into this reply handler.
 	 */
 	if ((SMID == 0) || (SMID > slots->m_n_normal)) {
-		mptsas_log(mpt, CE_WARN, "?Received invalid SMID of %d\n",
-		    SMID);
+		mptsas_log(mpt, CE_WARN, "received invalid SMID of %d", SMID);
 		ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 		return;
 	}
@@ -5252,12 +5348,14 @@
 	 * print warning and return if the slot is empty
 	 */
 	if (cmd == NULL) {
-		mptsas_log(mpt, CE_WARN, "?NULL command for successful SCSI IO "
+		mptsas_log(mpt, CE_WARN, "NULL command for successful SCSI IO "
 		    "in slot %d", SMID);
 		return;
 	}
 
 	pkt = CMD2PKT(cmd);
+	ASSERT(pkt->pkt_start != 0);
+	pkt->pkt_stop = gethrtime();
 	pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
 	    STATE_GOT_STATUS);
 	if (cmd->cmd_flags & CFLAG_DMAVALID) {
@@ -5321,8 +5419,8 @@
 	    (mpt->m_reply_frame_size * mpt->m_max_replies))) ||
 	    ((reply_addr - reply_frame_dma_baseaddr) %
 	    mpt->m_reply_frame_size != 0)) {
-		mptsas_log(mpt, CE_WARN, "?Received invalid reply frame "
-		    "address 0x%x\n", reply_addr);
+		mptsas_log(mpt, CE_WARN, "received invalid reply frame "
+		    "address 0x%x", reply_addr);
 		ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 		return;
 	}
@@ -5347,8 +5445,8 @@
 		 * so allow for that.
 		 */
 		if ((SMID == 0) || (SMID > (slots->m_n_normal + 1))) {
-			mptsas_log(mpt, CE_WARN, "?Received invalid SMID of "
-			    "%d\n", SMID);
+			mptsas_log(mpt, CE_WARN, "received invalid SMID of "
+			    "%d", SMID);
 			ddi_fm_service_impact(mpt->m_dip,
 			    DDI_SERVICE_UNAFFECTED);
 			return;
@@ -5360,7 +5458,7 @@
 		 * print warning and return if the slot is empty
 		 */
 		if (cmd == NULL) {
-			mptsas_log(mpt, CE_WARN, "?NULL command for address "
+			mptsas_log(mpt, CE_WARN, "NULL command for address "
 			    "reply in slot %d", SMID);
 			return;
 		}
@@ -5473,7 +5571,7 @@
 			 * print warning and return if the slot is empty
 			 */
 			if (cmd == NULL) {
-				mptsas_log(mpt, CE_WARN, "?NULL command for "
+				mptsas_log(mpt, CE_WARN, "NULL command for "
 				    "address reply in slot %d", SMID);
 				return;
 			}
@@ -5558,8 +5656,8 @@
 		loginfo = ddi_get32(mpt->m_acc_reply_frame_hdl,
 		    &reply->IOCLogInfo);
 		mptsas_log(mpt, CE_NOTE,
-		    "?Log info 0x%x received for target %d %s.\n"
-		    "\tscsi_status=0x%x, ioc_status=0x%x, scsi_state=0x%x",
+		    "log info 0x%x received for target %d %s, "
+		    "scsi_status=0x%x, ioc_status=0x%x, scsi_state=0x%x",
 		    loginfo, Tgt(cmd), wwn_str, scsi_status, ioc_status,
 		    scsi_state);
 	}
@@ -5568,6 +5666,8 @@
 	    scsi_status, ioc_status, scsi_state));
 
 	pkt = CMD2PKT(cmd);
+	ASSERT(pkt->pkt_start != 0);
+	pkt->pkt_stop = gethrtime();
 	*(pkt->pkt_scbp) = scsi_status;
 
 	if (loginfo == 0x31170000) {
@@ -5595,7 +5695,7 @@
 	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
 		responsedata &= 0x000000FF;
 		if (responsedata & MPTSAS_SCSI_RESPONSE_CODE_TLR_OFF) {
-			mptsas_log(mpt, CE_NOTE, "Do not support the TLR\n");
+			mptsas_log(mpt, CE_NOTE, "TLR not supported");
 			pkt->pkt_reason = CMD_TLR_OFF;
 			return;
 		}
@@ -5671,7 +5771,7 @@
 			if (topo_node == NULL) {
 				mptsas_log(mpt, CE_NOTE, "No memory"
 				    "resource for handle SAS dynamic"
-				    "reconfigure.\n");
+				    "reconfigure");
 				break;
 			}
 			topo_node->mpt = mpt;
@@ -5689,7 +5789,7 @@
 				    sizeof (mptsas_topo_change_list_t));
 				mptsas_log(mpt, CE_NOTE, "mptsas start taskq"
 				    "for handle SAS dynamic reconfigure"
-				    "failed. \n");
+				    "failed");
 			}
 		}
 		break;
@@ -5769,7 +5869,7 @@
 			break;
 		default:
 			mptsas_log(mpt, CE_WARN,
-			    "unknown ioc_status = %x\n", ioc_status);
+			    "unknown ioc_status = %x", ioc_status);
 			mptsas_log(mpt, CE_CONT, "scsi_state = %x, transfer "
 			    "count = %x, scsi_status = %x", scsi_state,
 			    xferred, scsi_status);
@@ -5786,10 +5886,10 @@
 		NDBG31(("scsi_status reservation conflict received"));
 		break;
 	default:
-		mptsas_log(mpt, CE_WARN, "scsi_status=%x, ioc_status=%x\n",
+		mptsas_log(mpt, CE_WARN, "scsi_status=%x, ioc_status=%x",
 		    scsi_status, ioc_status);
 		mptsas_log(mpt, CE_WARN,
-		    "mptsas_process_intr: invalid scsi status\n");
+		    "mptsas_process_intr: invalid scsi status");
 		break;
 	}
 }
@@ -5811,7 +5911,7 @@
 
 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
 		mptsas_log(mpt, CE_WARN, "mptsas_check_task_mgt: Task 0x%x "
-		    "failed. IOCStatus=0x%x IOCLogInfo=0x%x target=%d\n",
+		    "failed. IOCStatus=0x%x IOCLogInfo=0x%x target=%d",
 		    task_type, ioc_status, log_info, dev_handle);
 		pkt->pkt_reason = CMD_INCOMPLETE;
 		return;
@@ -5833,7 +5933,7 @@
 		 * sends bad command.  DevHandle of 0 could cause problems.
 		 */
 		if (dev_handle == 0) {
-			mptsas_log(mpt, CE_WARN, "!Can't flush target with"
+			mptsas_log(mpt, CE_WARN, "Can't flush target with"
 			    " DevHandle of 0.");
 		} else {
 			mptsas_flush_target(mpt, dev_handle, Lun(cmd),
@@ -6020,7 +6120,7 @@
 	} else if (reply_type == MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
 		mptsas_handle_address_reply(mpt, reply_desc_union);
 	} else {
-		mptsas_log(mpt, CE_WARN, "?Bad reply type %x", reply_type);
+		mptsas_log(mpt, CE_WARN, "bad reply type %x", reply_type);
 		ddi_fm_service_impact(mpt->m_dip, DDI_SERVICE_UNAFFECTED);
 	}
 
@@ -6182,7 +6282,7 @@
 	uint32_t		page_address;
 	uint16_t		bay_num, enclosure, io_flags;
 	uint32_t		dev_info;
-	char 			uabuf[SCSI_WWN_BUFLEN];
+	char			uabuf[SCSI_WWN_BUFLEN];
 	dev_info_t		*dip;
 	mdi_pathinfo_t		*pip;
 
@@ -6464,10 +6564,11 @@
 		 * If HBA is being reset, don't perform operations depending
 		 * on the IOC. We must free the topo list, however.
 		 */
-		if (!mpt->m_in_reset)
+		if (!mpt->m_in_reset) {
 			mptsas_handle_topo_change(topo_node, parent);
-		else
+		} else {
 			NDBG20(("skipping topo change received during reset"));
+		}
 		save_node = topo_node;
 		topo_node = topo_node->next;
 		ASSERT(save_node);
@@ -6553,16 +6654,16 @@
 			if (rval == DEV_INFO_WRONG_DEVICE_TYPE) {
 				mptsas_log(mpt, CE_NOTE,
 				    "mptsas_handle_topo_change: target %d is "
-				    "not a SAS/SATA device. \n",
+				    "not a SAS/SATA device",
 				    topo_node->devhdl);
 			} else if (rval == DEV_INFO_FAIL_ALLOC) {
 				mptsas_log(mpt, CE_NOTE,
 				    "mptsas_handle_topo_change: could not "
-				    "allocate memory. \n");
+				    "allocate memory");
 			} else if (rval == DEV_INFO_FAIL_GUID) {
 				mptsas_log(mpt, CE_NOTE,
 				    "mptsas_handle_topo_change: could not "
-				    "get SATA GUID for target %d. \n",
+				    "get SATA GUID for target %d",
 				    topo_node->devhdl);
 			}
 			/*
@@ -6789,8 +6890,6 @@
 		}
 
 		mutex_enter(&mpt->m_mutex);
-		ptgt->m_led_status = 0;
-		(void) mptsas_flush_led_status(mpt, ptgt);
 		if (rval == DDI_SUCCESS) {
 			refhash_remove(mpt->m_targets, ptgt);
 			ptgt = NULL;
@@ -6890,7 +6989,7 @@
 		mutex_exit(&mpt->m_mutex);
 
 		ndi_devi_enter(parent, &circ1);
-		rval = mptsas_offline_smp(parent, psmp, NDI_DEVI_REMOVE);
+		rval = mptsas_offline_smp(parent, psmp);
 		ndi_devi_exit(parent, circ1);
 
 		dev_info = psmp->m_deviceinfo;
@@ -6903,6 +7002,7 @@
 				    MPTSAS_VIRTUAL_PORT);
 				mptsas_log(mpt, CE_WARN, "mptsas virtual port "
 				    "prop update failed");
+				mutex_enter(&mpt->m_mutex);
 				return;
 			}
 			/*
@@ -6915,6 +7015,7 @@
 				    MPTSAS_NUM_PHYS);
 				mptsas_log(mpt, CE_WARN, "mptsas num phys"
 				    "prop update failed");
+				mutex_enter(&mpt->m_mutex);
 				return;
 			}
 			/*
@@ -6928,6 +7029,7 @@
 				    SCSI_ADDR_PROP_ATTACHED_PORT);
 				mptsas_log(mpt, CE_WARN, "mptsas attached port "
 				    "prop update failed");
+				mutex_enter(&mpt->m_mutex);
 				return;
 			}
 		}
@@ -7062,11 +7164,11 @@
 	    (mpt->m_reply_frame_dma_addr & 0xffffffffu)));
 	event = ddi_get16(mpt->m_acc_reply_frame_hdl, &eventreply->Event);
 
-	if (iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
-	    &eventreply->IOCStatus)) {
+	if ((iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
+	    &eventreply->IOCStatus)) != 0) {
 		if (iocstatus == MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
 			mptsas_log(mpt, CE_WARN,
-			    "!mptsas_handle_event_sync: event 0x%x, "
+			    "mptsas_handle_event_sync: event 0x%x, "
 			    "IOCStatus=0x%x, "
 			    "IOCLogInfo=0x%x", event, iocstatus,
 			    ddi_get32(mpt->m_acc_reply_frame_hdl,
@@ -7538,7 +7640,7 @@
 					    sizeof (mptsas_topo_change_list_t));
 				}
 				mptsas_log(mpt, CE_NOTE, "mptsas start taskq "
-				    "for handle SAS DR event failed. \n");
+				    "for handle SAS DR event failed");
 			}
 		}
 		break;
@@ -7722,7 +7824,7 @@
 					    sizeof (mptsas_topo_change_list_t));
 				}
 				mptsas_log(mpt, CE_NOTE, "mptsas start taskq "
-				    "for handle SAS DR event failed. \n");
+				    "for handle SAS DR event failed");
 			}
 		}
 		break;
@@ -7767,11 +7869,11 @@
 	    (mpt->m_reply_frame_dma_addr & 0xffffffffu)));
 	event = ddi_get16(mpt->m_acc_reply_frame_hdl, &eventreply->Event);
 
-	if (iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
-	    &eventreply->IOCStatus)) {
+	if ((iocstatus = ddi_get16(mpt->m_acc_reply_frame_hdl,
+	    &eventreply->IOCStatus)) != 0) {
 		if (iocstatus == MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
 			mptsas_log(mpt, CE_WARN,
-			    "!mptsas_handle_event: IOCStatus=0x%x, "
+			    "mptsas_handle_event: IOCStatus=0x%x, "
 			    "IOCLogInfo=0x%x", iocstatus,
 			    ddi_get32(mpt->m_acc_reply_frame_hdl,
 			    &eventreply->IOCLogInfo));
@@ -7882,7 +7984,8 @@
 			mep = mptsas_enc_lookup(mpt, enchdl);
 			if (mep != NULL) {
 				list_remove(&mpt->m_enclosures, mep);
-				kmem_free(mep, sizeof (*mep));
+				mptsas_enc_free(mep);
+				mep = NULL;
 			}
 			(void) sprintf(string, ", not responding");
 			break;
@@ -8149,7 +8252,7 @@
 			mptsas_log(mpt, CE_NOTE, " Volume %d settings changed"
 			    ", auto-config of hot-swap drives is %s"
 			    ", write caching is %s"
-			    ", hot-spare pool mask is %02x\n",
+			    ", hot-spare pool mask is %02x",
 			    vol, state &
 			    MPI2_RAIDVOL0_SETTING_AUTO_CONFIG_HSWAP_DISABLE
 			    ? "disabled" : "enabled",
@@ -8169,7 +8272,7 @@
 			    (uint8_t)state;
 
 			mptsas_log(mpt, CE_NOTE,
-			    "Volume %d is now %s\n", vol,
+			    "Volume %d is now %s", vol,
 			    state == MPI2_RAID_VOL_STATE_OPTIMAL
 			    ? "optimal" :
 			    state == MPI2_RAID_VOL_STATE_DEGRADED
@@ -8191,7 +8294,7 @@
 			    m_statusflags = state;
 
 			mptsas_log(mpt, CE_NOTE,
-			    " Volume %d is now %s%s%s%s%s%s%s%s%s\n",
+			    " Volume %d is now %s%s%s%s%s%s%s%s%s",
 			    vol,
 			    state & MPI2_RAIDVOL0_STATUS_FLAG_ENABLED
 			    ? ", enabled" : ", disabled",
@@ -8261,7 +8364,7 @@
 			mptsas_log(mpt, CE_NOTE,
 			    " PhysDiskNum %d with DevHandle 0x%x in slot %d "
 			    "for enclosure with handle 0x%x is now "
-			    "%s%s%s%s%s\n", physdisknum, devhandle, slot,
+			    "%s%s%s%s%s", physdisknum, devhandle, slot,
 			    enchandle,
 			    status & MPI2_PHYSDISK0_STATUS_FLAG_INACTIVE_VOLUME
 			    ? ", inactive" : ", active",
@@ -8279,7 +8382,7 @@
 		case MPI2_EVENT_IR_PHYSDISK_RC_STATE_CHANGED:
 			mptsas_log(mpt, CE_NOTE,
 			    " PhysDiskNum %d with DevHandle 0x%x in slot %d "
-			    "for enclosure with handle 0x%x is now %s\n",
+			    "for enclosure with handle 0x%x is now %s",
 			    physdisknum, devhandle, slot, enchandle,
 			    state == MPI2_RAID_PD_STATE_OPTIMAL
 			    ? "optimal" :
@@ -8302,6 +8405,60 @@
 		}
 		break;
 	}
+	case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
+	{
+		pMpi26EventDataActiveCableExcept_t	actcable;
+		uint32_t power;
+		uint8_t reason, id;
+
+		actcable = (pMpi26EventDataActiveCableExcept_t)
+		    eventreply->EventData;
+		power = ddi_get32(mpt->m_acc_reply_frame_hdl,
+		    &actcable->ActiveCablePowerRequirement);
+		reason = ddi_get8(mpt->m_acc_reply_frame_hdl,
+		    &actcable->ReasonCode);
+		id = ddi_get8(mpt->m_acc_reply_frame_hdl,
+		    &actcable->ReceptacleID);
+
+		/*
+		 * It'd be nice if this weren't just logging to the system but
+		 * were telling FMA about the active cable problem and FMA was
+		 * aware of the cable topology and state.
+		 */
+		switch (reason) {
+		case MPI26_EVENT_ACTIVE_CABLE_PRESENT:
+			/* Don't log anything if it's fine */
+			break;
+		case MPI26_EVENT_ACTIVE_CABLE_INSUFFICIENT_POWER:
+			mptsas_log(mpt, CE_WARN, "An active cable (id %u) does "
+			    "not have sufficient power to be enabled. "
+			    "Devices connected to this cable will not be "
+			    "visible to the system.", id);
+			if (power == UINT32_MAX) {
+				mptsas_log(mpt, CE_CONT, "The cable's power "
+				    "requirements are unknown.\n");
+			} else {
+				mptsas_log(mpt, CE_CONT, "The cable requires "
+				    "%u mW of power to function.\n", power);
+			}
+			break;
+		case MPI26_EVENT_ACTIVE_CABLE_DEGRADED:
+			mptsas_log(mpt, CE_WARN, "An active cable (id %u) is "
+			    "degraded and not running at its full speed. "
+			    "Some devices might not appear.", id);
+			break;
+		default:
+			break;
+		}
+		break;
+	}
+	case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
+	case MPI2_EVENT_PCIE_ENUMERATION:
+	case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
+	case MPI2_EVENT_PCIE_LINK_COUNTER:
+		mptsas_log(mpt, CE_NOTE, "Unhandled mpt_sas PCIe device "
+		    "event received (0x%x)", event);
+		break;
 	default:
 		NDBG20(("mptsas%d: unknown event %x received",
 		    mpt->m_instance, event));
@@ -8549,7 +8706,7 @@
 		mutex_exit(&mpt->m_tx_waitq_mutex);
 		if (mptsas_accept_pkt(mpt, cmd) != TRAN_ACCEPT)
 			cmn_err(CE_WARN, "mpt: mptsas_accept_tx_waitq: failed "
-			    "to accept cmd on queue\n");
+			    "to accept cmd on queue");
 		mutex_enter(&mpt->m_tx_waitq_mutex);
 	}
 }
@@ -8635,7 +8792,7 @@
 			control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
 			break;
 		default:
-			mptsas_log(mpt, CE_WARN, "mpt: Invalid tag type\n");
+			mptsas_log(mpt, CE_WARN, "invalid tag type");
 			break;
 		}
 	} else {
@@ -8716,6 +8873,7 @@
 	(void) ddi_dma_sync(dma_hdl, 0, 0, DDI_DMA_SYNC_FORDEV);
 	(void) ddi_dma_sync(mpt->m_dma_req_sense_hdl, 0, 0,
 	    DDI_DMA_SYNC_FORDEV);
+	pkt->pkt_start = gethrtime();
 
 	/*
 	 * Build request descriptor and write it to the request desc post reg.
@@ -8727,8 +8885,9 @@
 	/*
 	 * Start timeout.
 	 */
-	cmd->cmd_active_expiration =
-	    gethrtime() + (hrtime_t)pkt->pkt_time * NANOSEC;
+	cmd->cmd_active_expiration = pkt->pkt_start +
+	    (hrtime_t)pkt->pkt_time * (hrtime_t)NANOSEC;
+
 #ifdef MPTSAS_TEST
 	/*
 	 * Force timeouts to happen immediately.
@@ -9250,9 +9409,12 @@
 	}
 
 	if (what == HOLD_THROTTLE) {
-		ptgt->m_t_throttle = HOLD_THROTTLE;
-	} else if (ptgt->m_reset_delay == 0) {
 		ptgt->m_t_throttle = what;
+	} else if (ptgt->m_reset_delay == 0) {
+		if (what == MAX_THROTTLE)
+			ptgt->m_t_throttle = mpt->m_max_tune_throttle;
+		else
+			ptgt->m_t_throttle = what;
 	}
 }
 
@@ -9402,6 +9564,10 @@
 		    tasktype);
 		break;
 	}
+
+#ifdef MPTSAS_FAULTINJECTION
+	mptsas_fminj_move_tgt_to_doneq(mpt, target, reason, stat);
+#endif
 }
 
 /*
@@ -9693,6 +9859,17 @@
 		/* abort the specified packet */
 		sp = PKT2CMD(pkt);
 
+#ifdef MPTSAS_FAULTINJECTION
+	/* Command already on the list. */
+	if (((pkt->pkt_flags & FLAG_PKT_TIMEOUT) != 0) &&
+	    (sp->cmd_active_expiration != 0)) {
+		mptsas_fminj_move_cmd_to_doneq(mpt, sp, CMD_ABORTED,
+		    STAT_ABORTED);
+		rval = TRUE;
+		goto done;
+	}
+#endif
+
 		if (sp->cmd_queued) {
 			NDBG23(("mptsas_do_scsi_abort: queued sp=0x%p aborted",
 			    (void *)sp));
@@ -9983,10 +10160,10 @@
 	(void) vsprintf(mptsas_log_buf, fmt, ap);
 	va_end(ap);
 
-	if (level == CE_CONT) {
-		scsi_log(dev, mptsas_label, level, "%s\n", mptsas_log_buf);
+	if (level == CE_CONT || level == CE_NOTE) {
+		scsi_log(dev, mptsas_label, level, "!%s\n", mptsas_log_buf);
 	} else {
-		scsi_log(dev, mptsas_label, level, "%s", mptsas_log_buf);
+		scsi_log(dev, mptsas_label, level, "!%s", mptsas_log_buf);
 	}
 
 	mutex_exit(&mptsas_log_mutex);
@@ -10054,6 +10231,12 @@
 	mptsas_t	*mpt;
 	uint32_t	doorbell;
 
+#ifdef MPTSAS_FAULTINJECTION
+	struct mptsas_active_cmdq finj_cmds;
+
+	TAILQ_INIT(&finj_cmds);
+#endif
+
 	NDBG30(("mptsas_watch"));
 
 	rw_enter(&mptsas_global_rwlock, RW_READER);
@@ -10097,6 +10280,10 @@
 			(void) pm_idle_component(mpt->m_dip, 0);
 		}
 
+#ifdef MPTSAS_FAULTINJECTION
+		mptsas_fminj_watchsubr(mpt, &finj_cmds);
+#endif
+
 		mutex_exit(&mpt->m_mutex);
 	}
 	rw_exit(&mptsas_global_rwlock);
@@ -10105,6 +10292,22 @@
 	if (mptsas_timeouts_enabled)
 		mptsas_timeout_id = timeout(mptsas_watch, NULL, mptsas_tick);
 	mutex_exit(&mptsas_global_mutex);
+
+#ifdef MPTSAS_FAULTINJECTION
+	/* Complete all completed commands. */
+	if (!TAILQ_EMPTY(&finj_cmds)) {
+		mptsas_cmd_t *cmd;
+
+		while ((cmd = TAILQ_FIRST(&finj_cmds)) != NULL) {
+			TAILQ_REMOVE(&finj_cmds, cmd, cmd_active_link);
+			struct scsi_pkt *pkt = cmd->cmd_pkt;
+
+			if (pkt->pkt_comp != NULL) {
+				(*pkt->pkt_comp)(pkt);
+			}
+		}
+	}
+#endif
 }
 
 static void
@@ -10671,15 +10874,17 @@
 
 	pt->sgl_offset = offsetof(MPI2_FW_DOWNLOAD_REQUEST, SGL) +
 	    sizeof (*tcsge);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_fw_download(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    (int)pt->request_size, (int)pt->sgl_offset,
 		    (int)pt->dataout_size));
-	if (pt->data_size < sizeof (MPI2_FW_DOWNLOAD_REPLY))
+	}
+	if (pt->data_size < sizeof (MPI2_FW_DOWNLOAD_REPLY)) {
 		NDBG15(("mpi_pre_fw_download(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_FW_DOWNLOAD_REPLY)));
+	}
 }
 
 /*
@@ -10709,15 +10914,17 @@
 	req25->ImageSize = tcsge->ImageSize;
 
 	pt->sgl_offset = offsetof(MPI25_FW_DOWNLOAD_REQUEST, SGL);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_fw_25_download(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    pt->request_size, pt->sgl_offset,
 		    pt->dataout_size));
-	if (pt->data_size < sizeof (MPI2_FW_DOWNLOAD_REPLY))
+	}
+	if (pt->data_size < sizeof (MPI2_FW_DOWNLOAD_REPLY)) {
 		NDBG15(("mpi_pre_fw_25_download(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_FW_UPLOAD_REPLY)));
+	}
 }
 
 /*
@@ -10753,15 +10960,17 @@
 
 	pt->sgl_offset = offsetof(MPI2_FW_UPLOAD_REQUEST, SGL) +
 	    sizeof (*tcsge);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_fw_upload(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    pt->request_size, pt->sgl_offset,
 		    pt->dataout_size));
-	if (pt->data_size < sizeof (MPI2_FW_UPLOAD_REPLY))
+	}
+	if (pt->data_size < sizeof (MPI2_FW_UPLOAD_REPLY)) {
 		NDBG15(("mpi_pre_fw_upload(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_FW_UPLOAD_REPLY)));
+	}
 }
 
 /*
@@ -10791,15 +11000,17 @@
 	req25->ImageSize = tcsge->ImageSize;
 
 	pt->sgl_offset = offsetof(MPI25_FW_UPLOAD_REQUEST, SGL);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_fw_25_upload(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    pt->request_size, pt->sgl_offset,
 		    pt->dataout_size));
-	if (pt->data_size < sizeof (MPI2_FW_UPLOAD_REPLY))
+	}
+	if (pt->data_size < sizeof (MPI2_FW_UPLOAD_REPLY)) {
 		NDBG15(("mpi_pre_fw_25_upload(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_FW_UPLOAD_REPLY)));
+	}
 }
 
 /*
@@ -10811,16 +11022,18 @@
 #ifndef __lock_lint
 	_NOTE(ARGUNUSED(mpt))
 #endif
-	if (pt->request_size != sizeof (MPI2_IOC_FACTS_REQUEST))
+	if (pt->request_size != sizeof (MPI2_IOC_FACTS_REQUEST)) {
 		NDBG15(("mpi_pre_ioc_facts(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    pt->request_size,
 		    (int)sizeof (MPI2_IOC_FACTS_REQUEST),
 		    pt->dataout_size));
-	if (pt->data_size != sizeof (MPI2_IOC_FACTS_REPLY))
+	}
+	if (pt->data_size != sizeof (MPI2_IOC_FACTS_REPLY)) {
 		NDBG15(("mpi_pre_ioc_facts(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_IOC_FACTS_REPLY)));
+	}
 	pt->sgl_offset = (uint16_t)pt->request_size;
 }
 
@@ -10833,16 +11046,18 @@
 #ifndef __lock_lint
 	_NOTE(ARGUNUSED(mpt))
 #endif
-	if (pt->request_size != sizeof (MPI2_PORT_FACTS_REQUEST))
+	if (pt->request_size != sizeof (MPI2_PORT_FACTS_REQUEST)) {
 		NDBG15(("mpi_pre_port_facts(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    pt->request_size,
 		    (int)sizeof (MPI2_PORT_FACTS_REQUEST),
 		    pt->dataout_size));
-	if (pt->data_size != sizeof (MPI2_PORT_FACTS_REPLY))
+	}
+	if (pt->data_size != sizeof (MPI2_PORT_FACTS_REPLY)) {
 		NDBG15(("mpi_pre_port_facts(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_PORT_FACTS_REPLY)));
+	}
 	pt->sgl_offset = (uint16_t)pt->request_size;
 }
 
@@ -10856,15 +11071,17 @@
 	_NOTE(ARGUNUSED(mpt))
 #endif
 	pt->sgl_offset = offsetof(MPI2_SATA_PASSTHROUGH_REQUEST, SGL);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_sata_passthrough(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    pt->request_size, pt->sgl_offset,
 		    pt->dataout_size));
-	if (pt->data_size != sizeof (MPI2_SATA_PASSTHROUGH_REPLY))
+	}
+	if (pt->data_size != sizeof (MPI2_SATA_PASSTHROUGH_REPLY)) {
 		NDBG15(("mpi_pre_sata_passthrough(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_SATA_PASSTHROUGH_REPLY)));
+	}
 }
 
 static void
@@ -10874,15 +11091,17 @@
 	_NOTE(ARGUNUSED(mpt))
 #endif
 	pt->sgl_offset = offsetof(MPI2_SMP_PASSTHROUGH_REQUEST, SGL);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_smp_passthrough(): Incorrect req size, "
 		    "0x%x, should be 0x%x, dataoutsz 0x%x",
 		    pt->request_size, pt->sgl_offset,
 		    pt->dataout_size));
-	if (pt->data_size != sizeof (MPI2_SMP_PASSTHROUGH_REPLY))
+	}
+	if (pt->data_size != sizeof (MPI2_SMP_PASSTHROUGH_REPLY)) {
 		NDBG15(("mpi_pre_smp_passthrough(): Incorrect rep size, "
 		    "0x%x, should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_SMP_PASSTHROUGH_REPLY)));
+	}
 }
 
 /*
@@ -10895,14 +11114,16 @@
 	_NOTE(ARGUNUSED(mpt))
 #endif
 	pt->sgl_offset = offsetof(MPI2_CONFIG_REQUEST, PageBufferSGE);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_config(): Incorrect req size, 0x%x, "
 		    "should be 0x%x, dataoutsz 0x%x", pt->request_size,
 		    pt->sgl_offset, pt->dataout_size));
-	if (pt->data_size != sizeof (MPI2_CONFIG_REPLY))
+	}
+	if (pt->data_size != sizeof (MPI2_CONFIG_REPLY)) {
 		NDBG15(("mpi_pre_config(): Incorrect rep size, 0x%x, "
 		    "should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_CONFIG_REPLY)));
+	}
 	pt->simple = 1;
 }
 
@@ -10916,15 +11137,17 @@
 	_NOTE(ARGUNUSED(mpt))
 #endif
 	pt->sgl_offset = offsetof(MPI2_SCSI_IO_REQUEST, SGL);
-	if (pt->request_size != pt->sgl_offset)
+	if (pt->request_size != pt->sgl_offset) {
 		NDBG15(("mpi_pre_config(): Incorrect req size, 0x%x, "
 		    "should be 0x%x, dataoutsz 0x%x", pt->request_size,
 		    pt->sgl_offset,
 		    pt->dataout_size));
-	if (pt->data_size != sizeof (MPI2_SCSI_IO_REPLY))
+	}
+	if (pt->data_size != sizeof (MPI2_SCSI_IO_REPLY)) {
 		NDBG15(("mpi_pre_config(): Incorrect rep size, 0x%x, "
 		    "should be 0x%x", pt->data_size,
 		    (int)sizeof (MPI2_SCSI_IO_REPLY)));
+	}
 }
 
 /*
@@ -11129,6 +11352,7 @@
 	pkt->pkt_ha_private	= (opaque_t)&pt;
 	pkt->pkt_flags		= FLAG_HEAD;
 	pkt->pkt_time		= timeout;
+	pkt->pkt_start		= gethrtime();
 	cmd->cmd_pkt		= pkt;
 	cmd->cmd_flags		= CFLAG_CMDIOC | CFLAG_PASSTHRU;
 
@@ -12581,7 +12805,8 @@
 {
 	int ret = 0;
 	mptsas_led_control_t lc;
-	mptsas_target_t *ptgt;
+	mptsas_enclosure_t *mep;
+	uint16_t slotidx;
 
 	if (ddi_copyin((void *)data, &lc, sizeof (lc), mode) != 0) {
 		return (EFAULT);
@@ -12600,29 +12825,42 @@
 	    (lc.Command == MPTSAS_LEDCTL_FLAG_GET && (mode & FREAD) == 0))
 		return (EACCES);
 
-	/* Locate the target we're interrogating... */
+	/* Locate the required enclosure */
 	mutex_enter(&mpt->m_mutex);
-	ptgt = refhash_linear_search(mpt->m_targets,
-	    mptsas_target_eval_slot, &lc);
-	if (ptgt == NULL) {
-		/* We could not find a target for that enclosure/slot. */
+	mep = mptsas_enc_lookup(mpt, lc.Enclosure);
+	if (mep == NULL) {
 		mutex_exit(&mpt->m_mutex);
 		return (ENOENT);
 	}
 
+	if (lc.Slot < mep->me_fslot) {
+		mutex_exit(&mpt->m_mutex);
+		return (ENOENT);
+	}
+
+	/*
+	 * Slots on the enclosure are maintained in array where me_fslot is
+	 * entry zero. We normalize the requested slot.
+	 */
+	slotidx = lc.Slot - mep->me_fslot;
+	if (slotidx >= mep->me_nslots) {
+		mutex_exit(&mpt->m_mutex);
+		return (ENOENT);
+	}
+
 	if (lc.Command == MPTSAS_LEDCTL_FLAG_SET) {
 		/* Update our internal LED state. */
-		ptgt->m_led_status &= ~(1 << (lc.Led - 1));
-		ptgt->m_led_status |= lc.LedStatus << (lc.Led - 1);
+		mep->me_slotleds[slotidx] &= ~(1 << (lc.Led - 1));
+		mep->me_slotleds[slotidx] |= lc.LedStatus << (lc.Led - 1);
 
 		/* Flush it to the controller. */
-		ret = mptsas_flush_led_status(mpt, ptgt);
+		ret = mptsas_flush_led_status(mpt, mep, slotidx);
 		mutex_exit(&mpt->m_mutex);
 		return (ret);
 	}
 
 	/* Return our internal LED state. */
-	lc.LedStatus = (ptgt->m_led_status >> (lc.Led - 1)) & 1;
+	lc.LedStatus = (mep->me_slotleds[slotidx] >> (lc.Led - 1)) & 1;
 	mutex_exit(&mpt->m_mutex);
 
 	if (ddi_copyout(&lc, (void *)data, sizeof (lc), mode) != 0) {
@@ -12764,8 +13002,7 @@
 			if (pm_raise_power(mpt->m_dip, 0, PM_LEVEL_D0) !=
 			    DDI_SUCCESS) {
 				mptsas_log(mpt, CE_WARN,
-				    "mptsas%d: mptsas_ioctl: Raise power "
-				    "request failed.", mpt->m_instance);
+				    "raise power request failed");
 				(void) pm_idle_component(mpt->m_dip, 0);
 				return (ENXIO);
 			}
@@ -12799,21 +13036,6 @@
 				ndi_dc_freehdl(dcp);
 				goto out;
 			}
-			mutex_enter(&mpt->m_mutex);
-			if (cmd == DEVCTL_DEVICE_ONLINE) {
-				ptgt->m_tgt_unconfigured = 0;
-			} else if (cmd == DEVCTL_DEVICE_OFFLINE) {
-				ptgt->m_tgt_unconfigured = 1;
-			}
-			if (cmd == DEVCTL_DEVICE_OFFLINE) {
-				ptgt->m_led_status |=
-				    (1 << (MPTSAS_LEDCTL_LED_OK2RM - 1));
-			} else {
-				ptgt->m_led_status &=
-				    ~(1 << (MPTSAS_LEDCTL_LED_OK2RM - 1));
-			}
-			(void) mptsas_flush_led_status(mpt, ptgt);
-			mutex_exit(&mpt->m_mutex);
 			ndi_dc_freehdl(dcp);
 		}
 		goto out;
@@ -13000,12 +13222,6 @@
 	mpt->m_in_reset = TRUE;
 
 	/*
-	 * Wait until all the allocated sense data buffers for DMA are freed.
-	 */
-	while (mpt->m_extreq_sense_refcount > 0)
-		cv_wait(&mpt->m_extreq_sense_refcount_cv, &mpt->m_mutex);
-
-	/*
 	 * Set all throttles to HOLD
 	 */
 	for (ptgt = refhash_first(mpt->m_targets); ptgt != NULL;
@@ -13114,34 +13330,41 @@
 		goto fail;
 	}
 
-	if (mptsas_alloc_active_slots(mpt, KM_SLEEP)) {
-		goto fail;
+	if (first_time) {
+		if (mptsas_alloc_active_slots(mpt, KM_SLEEP)) {
+			goto fail;
+		}
+		/*
+		 * Allocate request message frames, reply free queue, reply
+		 * descriptor post queue, and reply message frames using
+		 * latest IOC facts.
+		 */
+		if (mptsas_alloc_request_frames(mpt) == DDI_FAILURE) {
+			mptsas_log(mpt, CE_WARN,
+			    "mptsas_alloc_request_frames failed");
+			goto fail;
+		}
+		if (mptsas_alloc_sense_bufs(mpt) == DDI_FAILURE) {
+			mptsas_log(mpt, CE_WARN,
+			    "mptsas_alloc_sense_bufs failed");
+			goto fail;
+		}
+		if (mptsas_alloc_free_queue(mpt) == DDI_FAILURE) {
+			mptsas_log(mpt, CE_WARN,
+			    "mptsas_alloc_free_queue failed!");
+			goto fail;
+		}
+		if (mptsas_alloc_post_queue(mpt) == DDI_FAILURE) {
+			mptsas_log(mpt, CE_WARN,
+			    "mptsas_alloc_post_queue failed!");
+			goto fail;
+		}
+		if (mptsas_alloc_reply_frames(mpt) == DDI_FAILURE) {
+			mptsas_log(mpt, CE_WARN,
+			    "mptsas_alloc_reply_frames failed!");
+			goto fail;
+		}
 	}
-	/*
-	 * Allocate request message frames, reply free queue, reply descriptor
-	 * post queue, and reply message frames using latest IOC facts.
-	 */
-	if (mptsas_alloc_request_frames(mpt) == DDI_FAILURE) {
-		mptsas_log(mpt, CE_WARN, "mptsas_alloc_request_frames failed");
-		goto fail;
-	}
-	if (mptsas_alloc_sense_bufs(mpt) == DDI_FAILURE) {
-		mptsas_log(mpt, CE_WARN, "mptsas_alloc_sense_bufs failed");
-		goto fail;
-	}
-	if (mptsas_alloc_free_queue(mpt) == DDI_FAILURE) {
-		mptsas_log(mpt, CE_WARN, "mptsas_alloc_free_queue failed!");
-		goto fail;
-	}
-	if (mptsas_alloc_post_queue(mpt) == DDI_FAILURE) {
-		mptsas_log(mpt, CE_WARN, "mptsas_alloc_post_queue failed!");
-		goto fail;
-	}
-	if (mptsas_alloc_reply_frames(mpt) == DDI_FAILURE) {
-		mptsas_log(mpt, CE_WARN, "mptsas_alloc_reply_frames failed!");
-		goto fail;
-	}
-
 mur:
 	/*
 	 * Re-Initialize ioc to operational state
@@ -13285,12 +13508,12 @@
 		 */
 		if (++cap_count > 48) {
 			mptsas_log(mpt, CE_WARN,
-			    "too many device capabilities.\n");
+			    "too many device capabilities");
 			break;
 		}
 		if (caps_ptr < 64) {
 			mptsas_log(mpt, CE_WARN,
-			    "capabilities pointer 0x%x out of range.\n",
+			    "capabilities pointer 0x%x out of range",
 			    caps_ptr);
 			break;
 		}
@@ -13303,8 +13526,7 @@
 		switch (cap) {
 			case PCI_CAP_ID_PM:
 				mptsas_log(mpt, CE_NOTE,
-				    "?mptsas%d supports power management.\n",
-				    mpt->m_instance);
+				    "power management supported");
 				mpt->m_options |= MPTSAS_OPT_PM;
 
 				/* Save PMCSR offset */
@@ -13322,8 +13544,7 @@
 				break;
 			default:
 				mptsas_log(mpt, CE_NOTE,
-				    "?mptsas%d unrecognized capability "
-				    "0x%x.\n", mpt->m_instance, cap);
+				    "unrecognized capability 0x%x", cap);
 				break;
 		}
 
@@ -13361,14 +13582,13 @@
 	 * If power management is supported by this chip, create
 	 * pm-components property for the power management framework
 	 */
-	(void) sprintf(pmc_name, "NAME=mptsas%d", mpt->m_instance);
+	(void) sprintf(pmc_name, "NAME=mpt_sas%d", mpt->m_instance);
 	pmc[0] = pmc_name;
 	if (ddi_prop_update_string_array(DDI_DEV_T_NONE, mpt->m_dip,
 	    "pm-components", pmc, 3) != DDI_PROP_SUCCESS) {
 		mpt->m_options &= ~MPTSAS_OPT_PM;
 		mptsas_log(mpt, CE_WARN,
-		    "mptsas%d: pm-component property creation failed.",
-		    mpt->m_instance);
+		    "pm-component property creation failed");
 		return (DDI_FAILURE);
 	}
 
@@ -13379,8 +13599,7 @@
 	pmcsr_stat = pci_config_get16(mpt->m_config_handle,
 	    mpt->m_pmcsr_offset);
 	if ((pmcsr_stat & PCI_PMCSR_STATE_MASK) != PCI_PMCSR_D0) {
-		mptsas_log(mpt, CE_WARN, "mptsas%d: Power up the device",
-		    mpt->m_instance);
+		mptsas_log(mpt, CE_WARN, "power up the device");
 		pci_config_put16(mpt->m_config_handle, mpt->m_pmcsr_offset,
 		    PCI_PMCSR_D0);
 	}
@@ -13409,7 +13628,7 @@
 	/* Get supported interrupt types */
 	if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
 		mptsas_log(mpt, CE_WARN, "ddi_intr_get_supported_types "
-		    "failed\n");
+		    "failed");
 		return (FALSE);
 	}
 
@@ -13463,7 +13682,7 @@
 	ret = ddi_intr_get_nintrs(dip, intr_type, &count);
 	if ((ret != DDI_SUCCESS) || (count <= 0)) {
 		mptsas_log(mpt, CE_WARN, "ddi_intr_get_nintrs() failed, "
-		    "ret %d count %d\n", ret, count);
+		    "ret %d count %d", ret, count);
 
 		return (DDI_FAILURE);
 	}
@@ -13472,7 +13691,7 @@
 	ret = ddi_intr_get_navail(dip, intr_type, &avail);
 	if ((ret != DDI_SUCCESS) || (avail == 0)) {
 		mptsas_log(mpt, CE_WARN, "ddi_intr_get_navail() failed, "
-		    "ret %d avail %d\n", ret, avail);
+		    "ret %d avail %d", ret, avail);
 
 		return (DDI_FAILURE);
 	}
@@ -13498,7 +13717,7 @@
 	    count, &actual, flag);
 
 	if ((ret != DDI_SUCCESS) || (actual == 0)) {
-		mptsas_log(mpt, CE_WARN, "ddi_intr_alloc() failed, ret %d\n",
+		mptsas_log(mpt, CE_WARN, "ddi_intr_alloc() failed, ret %d",
 		    ret);
 		kmem_free(mpt->m_htable, mpt->m_intr_size);
 		return (DDI_FAILURE);
@@ -13506,7 +13725,7 @@
 
 	/* use interrupt count returned or abort? */
 	if (actual < count) {
-		mptsas_log(mpt, CE_NOTE, "Requested: %d, Received: %d\n",
+		mptsas_log(mpt, CE_NOTE, "Requested: %d, Received: %d",
 		    count, actual);
 	}
 
@@ -13517,7 +13736,7 @@
 	 */
 	if ((ret = ddi_intr_get_pri(mpt->m_htable[0],
 	    &mpt->m_intr_pri)) != DDI_SUCCESS) {
-		mptsas_log(mpt, CE_WARN, "ddi_intr_get_pri() failed %d\n", ret);
+		mptsas_log(mpt, CE_WARN, "ddi_intr_get_pri() failed %d", ret);
 
 		/* Free already allocated intr */
 		for (i = 0; i < actual; i++) {
@@ -13531,7 +13750,7 @@
 	/* Test for high level mutex */
 	if (mpt->m_intr_pri >= ddi_intr_get_hilevel_pri()) {
 		mptsas_log(mpt, CE_WARN, "mptsas_add_intrs: "
-		    "Hi level interrupt not supported\n");
+		    "Hi level interrupt not supported");
 
 		/* Free already allocated intr */
 		for (i = 0; i < actual; i++) {
@@ -13547,7 +13766,7 @@
 		if ((ret = ddi_intr_add_handler(mpt->m_htable[i], mptsas_intr,
 		    (caddr_t)mpt, (caddr_t)(uintptr_t)i)) != DDI_SUCCESS) {
 			mptsas_log(mpt, CE_WARN, "ddi_intr_add_handler() "
-			    "failed %d\n", ret);
+			    "failed %d", ret);
 
 			/* Free already allocated intr */
 			for (i = 0; i < actual; i++) {
@@ -13561,7 +13780,7 @@
 
 	if ((ret = ddi_intr_get_cap(mpt->m_htable[0], &mpt->m_intr_cap))
 	    != DDI_SUCCESS) {
-		mptsas_log(mpt, CE_WARN, "ddi_intr_get_cap() failed %d\n", ret);
+		mptsas_log(mpt, CE_WARN, "ddi_intr_get_cap() failed %d", ret);
 
 		/* Free already allocated intr */
 		for (i = 0; i < actual; i++) {
@@ -13862,7 +14081,7 @@
 	rval = mptsas_inquiry(mpt, ptgt, lun, 0x83, inq83,
 	    inq83_len, NULL, 1);
 	if (rval != DDI_SUCCESS) {
-		mptsas_log(mpt, CE_WARN, "!mptsas request inquiry page "
+		mptsas_log(mpt, CE_WARN, "mptsas request inquiry page "
 		    "0x83 for target:%x, lun:%x failed!", target, lun);
 		sata_guid = -1;
 		goto out;
@@ -13870,7 +14089,7 @@
 	/* According to SAT2, the first descriptor is logic unit name */
 	dblk = &inq83[4];
 	if ((dblk[1] & 0x30) != 0) {
-		mptsas_log(mpt, CE_WARN, "!Descriptor is not lun associated.");
+		mptsas_log(mpt, CE_WARN, "Descriptor is not lun associated.");
 		goto out;
 	}
 	pwwn = (uint64_t *)(void *)(&dblk[4]);
@@ -13978,6 +14197,7 @@
 	}
 	bcopy(cdb, pktp->pkt_cdbp, cdblen);
 	pktp->pkt_flags = FLAG_NOPARITY;
+	pktp->pkt_time = mptsas_scsi_pkt_time;
 	if (scsi_poll(pktp) < 0) {
 		goto out;
 	}
@@ -14188,6 +14408,9 @@
 		mptsas_config_all(pdip);
 		ret = NDI_SUCCESS;
 		break;
+	default:
+		ret = NDI_FAILURE;
+		break;
 	}
 
 	if ((ret == NDI_SUCCESS) && bconfig) {
@@ -14235,6 +14458,14 @@
 	mptsas_target_t	*ptgt = NULL;
 
 	/*
+	 * The phymask exists if the port is active, otherwise
+	 * nothing to do.
+	 */
+	if (ddi_prop_exists(DDI_DEV_T_ANY, pdip,
+	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "phymask") == 0)
+		return (DDI_FAILURE);
+
+	/*
 	 * Get the physical port associated to the iport
 	 */
 	phymask = ddi_prop_get_int(DDI_DEV_T_ANY, pdip, 0,
@@ -14274,7 +14505,10 @@
 		return (DDI_SUCCESS);
 	}
 
-	if (phymask == 0) {
+	/*
+	 * If this is a RAID, configure the volumes
+	 */
+	if (mpt->m_num_raid_configs > 0) {
 		/*
 		 * Configure IR volume
 		 */
@@ -14296,6 +14530,13 @@
 	mptsas_target_t	*ptgt = NULL;
 
 	/*
+	 * The phymask exists if the port is active, otherwise
+	 * nothing to do.
+	 */
+	if (ddi_prop_exists(DDI_DEV_T_ANY, pdip,
+	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "phymask") == 0)
+		return (DDI_FAILURE);
+	/*
 	 * Get the physical port associated to the iport
 	 */
 	phymask = (mptsas_phymask_t)ddi_prop_get_int(DDI_DEV_T_ANY, pdip, 0,
@@ -14475,11 +14716,13 @@
 			continue;
 		}
 		saved_repluns[lun_cnt] = lun_num;
-		if (cdip = mptsas_find_child_addr(pdip, sas_wwn, lun_num))
+		if ((cdip = mptsas_find_child_addr(pdip, sas_wwn, lun_num)) !=
+		    NULL) {
 			ret = DDI_SUCCESS;
-		else
+		} else {
 			ret = mptsas_probe_lun(pdip, lun_num, &cdip,
 			    ptgt);
+		}
 		if ((ret == DDI_SUCCESS) && (cdip != NULL)) {
 			(void) ndi_prop_remove(DDI_DEV_T_NONE, cdip,
 			    MPTSAS_DEV_GONE);
@@ -14610,8 +14853,7 @@
 			/*
 			 * The lun has not been there already
 			 */
-			(void) mptsas_offline_lun(pdip, savechild, NULL,
-			    NDI_DEVI_REMOVE);
+			(void) mptsas_offline_lun(savechild, NULL);
 		}
 	}
 
@@ -14647,8 +14889,7 @@
 			/*
 			 * The lun has not been there already
 			 */
-			(void) mptsas_offline_lun(pdip, NULL, savepip,
-			    NDI_DEVI_REMOVE);
+			(void) mptsas_offline_lun(NULL, savepip);
 		}
 	}
 }
@@ -14665,7 +14906,39 @@
 	ASSERT(MUTEX_HELD(&mpt->m_mutex));
 	m = mptsas_enc_lookup(mpt, mep->me_enchdl);
 	if (m != NULL) {
+		uint8_t *ledp;
 		m->me_flags = mep->me_flags;
+
+
+		/*
+		 * If the number of slots and the first slot entry in the
+		 * enclosure has not changed, then we don't need to do anything
+		 * here. Otherwise, we need to allocate a new array for the LED
+		 * status of the slot.
+		 */
+		if (m->me_fslot == mep->me_fslot &&
+		    m->me_nslots == mep->me_nslots)
+			return;
+
+		/*
+		 * If the number of slots or the first slot has changed, it's
+		 * not clear that we're really in a place that we can continue
+		 * to honor the existing flags.
+		 */
+		if (mep->me_nslots > 0) {
+			ledp = kmem_zalloc(sizeof (uint8_t) * mep->me_nslots,
+			    KM_SLEEP);
+		} else {
+			ledp = NULL;
+		}
+
+		if (m->me_slotleds != NULL) {
+			kmem_free(m->me_slotleds, sizeof (uint8_t) *
+			    m->me_nslots);
+		}
+		m->me_slotleds = ledp;
+		m->me_fslot = mep->me_fslot;
+		m->me_nslots = mep->me_nslots;
 		return;
 	}
 
@@ -14672,6 +14945,19 @@
 	m = kmem_zalloc(sizeof (*m), KM_SLEEP);
 	m->me_enchdl = mep->me_enchdl;
 	m->me_flags = mep->me_flags;
+	m->me_nslots = mep->me_nslots;
+	m->me_fslot = mep->me_fslot;
+	if (m->me_nslots > 0) {
+		m->me_slotleds = kmem_zalloc(sizeof (uint8_t) * mep->me_nslots,
+		    KM_SLEEP);
+		/*
+		 * It may make sense to optionally flush all of the slots and/or
+		 * read the slot status flag here to synchronize between
+		 * ourselves and the card. So far, that hasn't been needed
+		 * annecdotally when enumerating something new. If we do, we
+		 * should kick that off in a taskq potentially.
+		 */
+	}
 	list_insert_tail(&mpt->m_enclosures, m);
 }
 
@@ -14802,15 +15088,20 @@
 	mptsas_smp_t	*psmp;
 
 	/*
-	 * Get the phymask associated to the iport
+	 * The phymask exists if the port is active, otherwise
+	 * nothing to do.
 	 */
+	if (ddi_prop_exists(DDI_DEV_T_ANY, pdip,
+	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "phymask") == 0)
+		return;
+
 	phymask = ddi_prop_get_int(DDI_DEV_T_ANY, pdip, 0,
 	    "phymask", 0);
 
 	/*
-	 * Enumerate RAID volumes here (phymask == 0).
+	 * If this is a RAID, enumerate the volumes
 	 */
-	if (phymask == 0) {
+	if (mpt->m_num_raid_configs > 0) {
 		mptsas_config_all_viport(pdip);
 		return;
 	}
@@ -14897,8 +15188,7 @@
 			continue;
 		}
 
-		tmp_rval = mptsas_offline_lun(pdip, prechild, NULL,
-		    NDI_DEVI_REMOVE);
+		tmp_rval = mptsas_offline_lun(prechild, NULL);
 		if (tmp_rval != DDI_SUCCESS) {
 			rval = DDI_FAILURE;
 			if (ndi_prop_create_boolean(DDI_DEV_T_NONE,
@@ -14930,8 +15220,7 @@
 			continue;
 		}
 
-		(void) mptsas_offline_lun(pdip, NULL, savepip,
-		    NDI_DEVI_REMOVE);
+		(void) mptsas_offline_lun(NULL, savepip);
 		/*
 		 * driver will not invoke mdi_pi_free, so path will not
 		 * be freed forever, return DDI_FAILURE.
@@ -14942,48 +15231,19 @@
 }
 
 static int
-mptsas_offline_lun(dev_info_t *pdip, dev_info_t *rdip,
-    mdi_pathinfo_t *rpip, uint_t flags)
+mptsas_offline_lun(dev_info_t *rdip, mdi_pathinfo_t *rpip)
 {
 	int		rval = DDI_FAILURE;
-	char		*devname;
-	dev_info_t	*cdip, *parent;
 
 	if (rpip != NULL) {
-		parent = scsi_vhci_dip;
-		cdip = mdi_pi_get_client(rpip);
-	} else if (rdip != NULL) {
-		parent = pdip;
-		cdip = rdip;
-	} else {
-		return (DDI_FAILURE);
-	}
-
-	/*
-	 * Make sure node is attached otherwise
-	 * it won't have related cache nodes to
-	 * clean up.  i_ddi_devi_attached is
-	 * similiar to i_ddi_node_state(cdip) >=
-	 * DS_ATTACHED.
-	 */
-	if (i_ddi_devi_attached(cdip)) {
-
-		/* Get full devname */
-		devname = kmem_alloc(MAXNAMELEN + 1, KM_SLEEP);
-		(void) ddi_deviname(cdip, devname);
-		/* Clean cache */
-		(void) devfs_clean(parent, devname + 1,
-		    DV_CLEAN_FORCE);
-		kmem_free(devname, MAXNAMELEN + 1);
-	}
-	if (rpip != NULL) {
 		if (MDI_PI_IS_OFFLINE(rpip)) {
 			rval = DDI_SUCCESS;
 		} else {
 			rval = mdi_pi_offline(rpip, 0);
 		}
-	} else {
-		rval = ndi_devi_offline(cdip, flags);
+	} else if (rdip != NULL) {
+		rval = ndi_devi_offline(rdip,
+		    NDI_DEVFS_CLEAN | NDI_DEVI_REMOVE | NDI_DEVI_GONE);
 	}
 
 	return (rval);
@@ -15015,10 +15275,9 @@
 }
 
 static int
-mptsas_offline_smp(dev_info_t *pdip, mptsas_smp_t *smp_node, uint_t flags)
+mptsas_offline_smp(dev_info_t *pdip, mptsas_smp_t *smp_node)
 {
 	int		rval = DDI_FAILURE;
-	char		*devname;
 	char		wwn_str[MPTSAS_WWN_STRLEN];
 	dev_info_t	*cdip;
 
@@ -15025,30 +15284,11 @@
 	(void) sprintf(wwn_str, "%"PRIx64, smp_node->m_addr.mta_wwn);
 
 	cdip = mptsas_find_smp_child(pdip, wwn_str);
-
 	if (cdip == NULL)
 		return (DDI_SUCCESS);
 
-	/*
-	 * Make sure node is attached otherwise
-	 * it won't have related cache nodes to
-	 * clean up.  i_ddi_devi_attached is
-	 * similiar to i_ddi_node_state(cdip) >=
-	 * DS_ATTACHED.
-	 */
-	if (i_ddi_devi_attached(cdip)) {
+	rval = ndi_devi_offline(cdip, NDI_DEVFS_CLEAN | NDI_DEVI_REMOVE);
 
-		/* Get full devname */
-		devname = kmem_alloc(MAXNAMELEN + 1, KM_SLEEP);
-		(void) ddi_deviname(cdip, devname);
-		/* Clean cache */
-		(void) devfs_clean(pdip, devname + 1,
-		    DV_CLEAN_FORCE);
-		kmem_free(devname, MAXNAMELEN + 1);
-	}
-
-	rval = ndi_devi_offline(cdip, flags);
-
 	return (rval);
 }
 
@@ -15183,7 +15423,7 @@
 		rval = mptsas_inquiry(mpt, ptgt, lun, 0x83, inq83,
 		    inq83_len1, &inq83_len, 1);
 		if (rval != 0) {
-			mptsas_log(mpt, CE_WARN, "!mptsas request inquiry page "
+			mptsas_log(mpt, CE_WARN, "mptsas request inquiry page "
 			    "0x83 for target:%x, lun:%x failed!", target, lun);
 			if (mptsas_physical_bind_failed_page_83 != B_FALSE)
 				goto create_lun;
@@ -15210,7 +15450,7 @@
 				ddi_devid_free_guid(guid);
 				guid = NULL;
 				if (mpt->m_mpxio_enable == TRUE) {
-					mptsas_log(mpt, CE_NOTE, "!Target:%x, "
+					mptsas_log(mpt, CE_NOTE, "Target:%x, "
 					    "lun:%x doesn't have a valid GUID, "
 					    "multipathing for this drive is "
 					    "not enabled", target, lun);
@@ -15232,7 +15472,7 @@
 			delay(1 * drv_usectohz(1000000));
 			continue;
 		} else {
-			mptsas_log(mpt, CE_WARN, "!Encode devid failed for "
+			mptsas_log(mpt, CE_WARN, "Encode devid failed for "
 			    "path target:%x, lun:%x", target, lun);
 			rval = DDI_FAILURE;
 			goto create_lun;
@@ -15240,7 +15480,7 @@
 	}
 
 	if (i == mptsas_inq83_retry_timeout) {
-		mptsas_log(mpt, CE_WARN, "!Repeated page83 requests timeout "
+		mptsas_log(mpt, CE_WARN, "Repeated page83 requests timeout "
 		    "for path target:%x, lun:%x", target, lun);
 	}
 
@@ -15328,11 +15568,6 @@
 				    (!MDI_PI_IS_STANDBY(*pip)) &&
 				    (ptgt->m_tgt_unconfigured == 0)) {
 					rval = mdi_pi_online(*pip, 0);
-					mutex_enter(&mpt->m_mutex);
-					ptgt->m_led_status = 0;
-					(void) mptsas_flush_led_status(mpt,
-					    ptgt);
-					mutex_exit(&mpt->m_mutex);
 				} else {
 					rval = DDI_SUCCESS;
 				}
@@ -15366,7 +15601,8 @@
 						return (DDI_FAILURE);
 					}
 				}
-				if (mdi_pi_free(*pip, 0) != MDI_SUCCESS) {
+				if (mdi_pi_free(*pip,
+				    MDI_CLIENT_FLAGS_NO_EVENT) != MDI_SUCCESS) {
 					mptsas_log(mpt, CE_WARN, "path:target:"
 					    "%x, lun:%x free failed!", target,
 					    lun);
@@ -15607,18 +15843,12 @@
 		}
 		NDBG20(("new path:%s onlining,", MDI_PI(*pip)->pi_addr));
 		mdi_rtn = mdi_pi_online(*pip, 0);
-		if (mdi_rtn == MDI_SUCCESS) {
-			mutex_enter(&mpt->m_mutex);
-			ptgt->m_led_status = 0;
-			(void) mptsas_flush_led_status(mpt, ptgt);
-			mutex_exit(&mpt->m_mutex);
-		}
 		if (mdi_rtn == MDI_NOT_SUPPORTED) {
 			mdi_rtn = MDI_FAILURE;
 		}
 virt_create_done:
 		if (*pip && mdi_rtn != MDI_SUCCESS) {
-			(void) mdi_pi_free(*pip, 0);
+			(void) mdi_pi_free(*pip, MDI_CLIENT_FLAGS_NO_EVENT);
 			*pip = NULL;
 			*lun_dip = NULL;
 		}
@@ -15985,12 +16215,6 @@
 			 */
 			ndi_rtn = ndi_devi_online(*lun_dip, NDI_ONLINE_ATTACH);
 		}
-		if (ndi_rtn == NDI_SUCCESS) {
-			mutex_enter(&mpt->m_mutex);
-			ptgt->m_led_status = 0;
-			(void) mptsas_flush_led_status(mpt, ptgt);
-			mutex_exit(&mpt->m_mutex);
-		}
 
 		/*
 		 * If success set rtn flag, else unwire alloc'd lun
@@ -16042,9 +16266,13 @@
 	int		phymask;
 
 	/*
-	 * Get the physical port associated to the iport
-	 * PHYMASK TODO
+	 * The phymask exists if the port is active, otherwise
+	 * nothing to do.
 	 */
+	if (ddi_prop_exists(DDI_DEV_T_ANY, pdip,
+	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "phymask") == 0)
+		return (DDI_FAILURE);
+
 	phymask = ddi_prop_get_int(DDI_DEV_T_ANY, pdip, 0,
 	    "phymask", 0);
 	/*
@@ -16677,11 +16905,24 @@
 {
 	dev_info_t	*dip;
 	int		prop;
+
 	dip = e_ddi_hold_devi_by_dev(dev, 0);
 	if (dip == NULL)
 		return (dip);
+
+	/*
+	 * The phymask exists if the port is active, otherwise
+	 * nothing to do.
+	 */
+	if (ddi_prop_exists(DDI_DEV_T_ANY, dip,
+	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "phymask") == 0) {
+		ddi_release_devi(dip);
+		return ((dev_info_t *)NULL);
+	}
+
 	prop = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 0,
 	    "phymask", 0);
+
 	*phymask = (mptsas_phymask_t)prop;
 	ddi_release_devi(dip);
 	return (dip);
@@ -16706,22 +16947,24 @@
 }
 
 static int
-mptsas_flush_led_status(mptsas_t *mpt, mptsas_target_t *ptgt)
+mptsas_flush_led_status(mptsas_t *mpt, mptsas_enclosure_t *mep, uint16_t idx)
 {
 	uint32_t slotstatus = 0;
 
+	ASSERT3U(idx, <, mep->me_nslots);
+
 	/* Build an MPI2 Slot Status based on our view of the world */
-	if (ptgt->m_led_status & (1 << (MPTSAS_LEDCTL_LED_IDENT - 1)))
+	if (mep->me_slotleds[idx] & (1 << (MPTSAS_LEDCTL_LED_IDENT - 1)))
 		slotstatus |= MPI2_SEP_REQ_SLOTSTATUS_IDENTIFY_REQUEST;
-	if (ptgt->m_led_status & (1 << (MPTSAS_LEDCTL_LED_FAIL - 1)))
+	if (mep->me_slotleds[idx] & (1 << (MPTSAS_LEDCTL_LED_FAIL - 1)))
 		slotstatus |= MPI2_SEP_REQ_SLOTSTATUS_PREDICTED_FAULT;
-	if (ptgt->m_led_status & (1 << (MPTSAS_LEDCTL_LED_OK2RM - 1)))
+	if (mep->me_slotleds[idx] & (1 << (MPTSAS_LEDCTL_LED_OK2RM - 1)))
 		slotstatus |= MPI2_SEP_REQ_SLOTSTATUS_REQUEST_REMOVE;
 
 	/* Write it to the controller */
 	NDBG14(("mptsas_ioctl: set LED status %x for slot %x",
-	    slotstatus, ptgt->m_slot_num));
-	return (mptsas_send_sep(mpt, ptgt, &slotstatus,
+	    slotstatus, idx + mep->me_fslot));
+	return (mptsas_send_sep(mpt, mep, idx, &slotstatus,
 	    MPI2_SEP_REQ_ACTION_WRITE_STATUS));
 }
 
@@ -16729,43 +16972,22 @@
  *  send sep request, use enclosure/slot addressing
  */
 static int
-mptsas_send_sep(mptsas_t *mpt, mptsas_target_t *ptgt,
+mptsas_send_sep(mptsas_t *mpt, mptsas_enclosure_t *mep, uint16_t idx,
     uint32_t *status, uint8_t act)
 {
 	Mpi2SepRequest_t	req;
 	Mpi2SepReply_t		rep;
 	int			ret;
-	mptsas_enclosure_t	*mep;
 	uint16_t 		enctype;
+	uint16_t		slot;
 
 	ASSERT(mutex_owned(&mpt->m_mutex));
 
 	/*
-	 * We only support SEP control of directly-attached targets, in which
-	 * case the "SEP" we're talking to is a virtual one contained within
-	 * the HBA itself.  This is necessary because DA targets typically have
-	 * no other mechanism for LED control.  Targets for which a separate
-	 * enclosure service processor exists should be controlled via ses(7d)
-	 * or sgen(7d).  Furthermore, since such requests can time out, they
-	 * should be made in user context rather than in response to
-	 * asynchronous fabric changes.
-	 *
-	 * In addition, we do not support this operation for RAID volumes,
-	 * since there is no slot associated with them.
-	 */
-	if (!(ptgt->m_deviceinfo & DEVINFO_DIRECT_ATTACHED) ||
-	    ptgt->m_addr.mta_phymask == 0) {
-		return (ENOTTY);
-	}
-
-	/*
 	 * Look through the enclosures and make sure that this enclosure is
 	 * something that is directly attached device. If we didn't find an
 	 * enclosure for this device, don't send the ioctl.
 	 */
-	mep = mptsas_enc_lookup(mpt, ptgt->m_enclosure);
-	if (mep == NULL)
-		return (ENOTTY);
 	enctype = mep->me_flags & MPI2_SAS_ENCLS0_FLAGS_MNG_MASK;
 	if (enctype != MPI2_SAS_ENCLS0_FLAGS_MNG_IOC_SES &&
 	    enctype != MPI2_SAS_ENCLS0_FLAGS_MNG_IOC_SGPIO &&
@@ -16772,6 +16994,7 @@
 	    enctype != MPI2_SAS_ENCLS0_FLAGS_MNG_IOC_GPIO) {
 		return (ENOTTY);
 	}
+	slot = idx + mep->me_fslot;
 
 	bzero(&req, sizeof (req));
 	bzero(&rep, sizeof (rep));
@@ -16779,8 +17002,8 @@
 	req.Function = MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR;
 	req.Action = act;
 	req.Flags = MPI2_SEP_REQ_FLAGS_ENCLOSURE_SLOT_ADDRESS;
-	req.EnclosureHandle = LE_16(ptgt->m_enclosure);
-	req.Slot = LE_16(ptgt->m_slot_num);
+	req.EnclosureHandle = LE_16(mep->me_enchdl);
+	req.Slot = LE_16(slot);
 	if (act == MPI2_SEP_REQ_ACTION_WRITE_STATUS) {
 		req.SlotStatus = LE_32(*status);
 	}
