--- old/usr/src/uts/common/io/comstar/lu/stmf_sbd/sbd_scsi.c	Mon Aug  5 17:27:18 2019
+++ new/usr/src/uts/common/io/comstar/lu/stmf_sbd/sbd_scsi.c	Mon Aug  5 17:27:18 2019
@@ -18,10 +18,11 @@
  *
  * CDDL HEADER END
  */
+
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
  */
 
 #include <sys/conf.h>
@@ -37,6 +38,7 @@
 #include <sys/atomic.h>
 #include <sys/sdt.h>
 #include <sys/dkio.h>
+#include <sys/dkioc_free_util.h>
 
 #include <sys/stmf.h>
 #include <sys/lpif.h>
@@ -86,6 +88,21 @@
 	    (((cdb[4]) & 0xF0) == 0) && (((cdb[4]) & 0x01) == 0))))
 /* End of SCSI2_CONFLICT_FREE_CMDS */
 
+uint8_t HardwareAcceleratedInit = 1;
+uint8_t sbd_unmap_enable = 1;		/* allow unmap by default */
+
+/*
+ * An /etc/system tunable which specifies the maximum number of LBAs supported
+ * in a single UNMAP operation. Default is 0x002000 blocks or 4MB in size.
+ */
+int stmf_sbd_unmap_max_nblks  = 0x002000;
+
+/*
+ * An /etc/system tunable which indicates if READ ops can run on the standby
+ * path or return an error.
+ */
+int stmf_standby_fail_reads = 0;
+
 stmf_status_t sbd_lu_reset_state(stmf_lu_t *lu);
 static void sbd_handle_sync_cache(struct scsi_task *task,
     struct stmf_data_buf *initial_dbuf);
@@ -93,8 +110,6 @@
     sbd_cmd_t *scmd, struct stmf_data_buf *dbuf);
 void sbd_handle_short_write_xfer_completion(scsi_task_t *task,
     stmf_data_buf_t *dbuf);
-void sbd_handle_short_write_transfers(scsi_task_t *task,
-    stmf_data_buf_t *dbuf, uint32_t cdb_xfer_size);
 void sbd_handle_mode_select_xfer(scsi_task_t *task, uint8_t *buf,
     uint32_t buflen);
 void sbd_handle_mode_select(scsi_task_t *task, stmf_data_buf_t *dbuf);
@@ -105,7 +120,7 @@
 static void sbd_handle_unmap(scsi_task_t *task, stmf_data_buf_t *dbuf);
 
 extern void sbd_pgr_initialize_it(scsi_task_t *, sbd_it_data_t *);
-extern int sbd_pgr_reservation_conflict(scsi_task_t *);
+extern int sbd_pgr_reservation_conflict(scsi_task_t *, struct sbd_lu *sl);
 extern void sbd_pgr_reset(sbd_lu_t *);
 extern void sbd_pgr_remove_it_handle(sbd_lu_t *, sbd_it_data_t *);
 extern void sbd_handle_pgr_in_cmd(scsi_task_t *, stmf_data_buf_t *);
@@ -119,6 +134,7 @@
     struct stmf_data_buf *dbuf, uint8_t dbuf_reusable);
 static void sbd_handle_write_same_xfer_completion(struct scsi_task *task,
     sbd_cmd_t *scmd, struct stmf_data_buf *dbuf, uint8_t dbuf_reusable);
+
 /*
  * IMPORTANT NOTE:
  * =================
@@ -130,7 +146,7 @@
 
 void
 sbd_do_read_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
-					struct stmf_data_buf *dbuf)
+    struct stmf_data_buf *dbuf)
 {
 	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
 	uint64_t laddr;
@@ -143,7 +159,8 @@
 	    (task->task_cmd_xfer_length < (32 * 1024))) ? 2 :
 	    task->task_max_nbufs;
 
-	len = scmd->len > dbuf->db_buf_size ? dbuf->db_buf_size : scmd->len;
+	len = ATOMIC32_GET(scmd->len) > dbuf->db_buf_size ?
+	    dbuf->db_buf_size : ATOMIC32_GET(scmd->len);
 	laddr = scmd->addr + scmd->current_ro;
 
 	for (buflen = 0, ndx = 0; (buflen < len) &&
@@ -167,12 +184,14 @@
 	dbuf->db_data_size = buflen;
 	dbuf->db_flags = DB_DIRECTION_TO_RPORT;
 	(void) stmf_xfer_data(task, dbuf, 0);
-	scmd->len -= buflen;
+	atomic_add_32(&scmd->len, -buflen);
 	scmd->current_ro += buflen;
-	if (scmd->len && (scmd->nbufs < bufs_to_take)) {
+	if (ATOMIC32_GET(scmd->len) &&
+	    (ATOMIC8_GET(scmd->nbufs) < bufs_to_take)) {
 		uint32_t maxsize, minsize, old_minsize;
 
-		maxsize = (scmd->len > (128*1024)) ? 128*1024 : scmd->len;
+		maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ? 128*1024 :
+		    ATOMIC32_GET(scmd->len);
 		minsize = maxsize >> 2;
 		do {
 			/*
@@ -187,7 +206,7 @@
 		if (dbuf == NULL) {
 			return;
 		}
-		scmd->nbufs++;
+		atomic_inc_8(&scmd->nbufs);
 		sbd_do_read_xfer(task, scmd, dbuf);
 	}
 }
@@ -218,6 +237,7 @@
 	uint_t nblks;
 	uint64_t blksize = sl->sl_blksize;
 	size_t db_private_sz;
+	hrtime_t xfer_start;
 	uintptr_t pad;
 
 	ASSERT(rw_read_held(&sl->sl_access_state_lock));
@@ -259,14 +279,15 @@
 		first_len = 0;
 	}
 
-	while (scmd->len && scmd->nbufs < task->task_max_nbufs) {
+	while (ATOMIC32_GET(scmd->len) &&
+	    ATOMIC8_GET(scmd->nbufs) < task->task_max_nbufs) {
 
-		xfer_len = MIN(max_len, scmd->len);
+		xfer_len = MIN(max_len, ATOMIC32_GET(scmd->len));
 		if (first_len) {
 			xfer_len = MIN(xfer_len, first_len);
 			first_len = 0;
 		}
-		if (scmd->len == xfer_len) {
+		if (ATOMIC32_GET(scmd->len) == xfer_len) {
 			final_xfer = 1;
 		} else {
 			/*
@@ -333,6 +354,7 @@
 		 * Accounting for start of read.
 		 * Note there is no buffer address for the probe yet.
 		 */
+		xfer_start = gethrtime();
 		DTRACE_PROBE5(backing__store__read__start, sbd_lu_t *, sl,
 		    uint8_t *, NULL, uint64_t, xfer_len,
 		    uint64_t, offset, scsi_task_t *, task);
@@ -339,6 +361,8 @@
 
 		ret = sbd_zvol_alloc_read_bufs(sl, dbuf);
 
+		stmf_lu_xfer_done(task, B_TRUE /* read */,
+		    (gethrtime() - xfer_start));
 		DTRACE_PROBE6(backing__store__read__end, sbd_lu_t *, sl,
 		    uint8_t *, NULL, uint64_t, xfer_len,
 		    uint64_t, offset, int, ret, scsi_task_t *, task);
@@ -348,9 +372,10 @@
 			 * Read failure from the backend.
 			 */
 			stmf_free(dbuf);
-			if (scmd->nbufs == 0) {
+			if (ATOMIC8_GET(scmd->nbufs) == 0) {
 				/* nothing queued, just finish */
 				scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+				sbd_ats_remove_by_task(task);
 				stmf_scsilib_send_status(task, STATUS_CHECK,
 				    STMF_SAA_READ_ERROR);
 				rw_exit(&sl->sl_access_state_lock);
@@ -361,7 +386,6 @@
 			return;
 		}
 
-
 		/*
 		 * Allow PP to do setup
 		 */
@@ -375,7 +399,7 @@
 			 */
 			sbd_zvol_rele_read_bufs(sl, dbuf);
 			stmf_free(dbuf);
-			if (scmd->nbufs > 0) {
+			if (ATOMIC8_GET(scmd->nbufs) > 0) {
 				/* completion of previous dbuf will retry */
 				return;
 			}
@@ -383,6 +407,7 @@
 			 * Done with this command.
 			 */
 			scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+			sbd_ats_remove_by_task(task);
 			if (first_xfer)
 				stmf_scsilib_send_status(task, STATUS_QFULL, 0);
 			else
@@ -394,7 +419,7 @@
 		/*
 		 * dbuf is now queued on task
 		 */
-		scmd->nbufs++;
+		atomic_inc_8(&scmd->nbufs);
 
 		/* XXX leave this in for FW? */
 		DTRACE_PROBE4(sbd__xfer, struct scsi_task *, task,
@@ -416,8 +441,8 @@
 			sbd_zvol_rele_read_bufs(sl, dbuf);
 			stmf_teardown_dbuf(task, dbuf);
 			stmf_free(dbuf);
-			scmd->nbufs--;
-			if (scmd->nbufs > 0) {
+			atomic_dec_8(&scmd->nbufs);
+			if (ATOMIC8_GET(scmd->nbufs) > 0) {
 				/* completion of previous dbuf will retry */
 				return;
 			}
@@ -426,6 +451,7 @@
 			 */
 			rw_exit(&sl->sl_access_state_lock);
 			scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+			sbd_ats_remove_by_task(task);
 			if (first_xfer)
 				stmf_scsilib_send_status(task, STATUS_QFULL, 0);
 			else
@@ -437,6 +463,7 @@
 			 * Completion from task_done will cleanup
 			 */
 			scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+			sbd_ats_remove_by_task(task);
 			return;
 		}
 		/*
@@ -443,7 +470,7 @@
 		 * Update the xfer progress.
 		 */
 		ASSERT(scmd->len >= xfer_len);
-		scmd->len -= xfer_len;
+		atomic_add_32(&scmd->len, -xfer_len);
 		scmd->current_ro += xfer_len;
 	}
 }
@@ -450,7 +477,7 @@
 
 void
 sbd_handle_read_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
-				struct stmf_data_buf *dbuf)
+    struct stmf_data_buf *dbuf)
 {
 	if (dbuf->db_xfer_status != STMF_SUCCESS) {
 		stmf_abort(STMF_QUEUE_TASK_ABORT, task,
@@ -458,12 +485,14 @@
 		return;
 	}
 	task->task_nbytes_transferred += dbuf->db_data_size;
-	if (scmd->len == 0 || scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
+	if (ATOMIC32_GET(scmd->len) == 0 ||
+	    scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
 		stmf_free_dbuf(task, dbuf);
-		scmd->nbufs--;
-		if (scmd->nbufs)
+		atomic_dec_8(&scmd->nbufs);
+		if (ATOMIC8_GET(scmd->nbufs))
 			return;	/* wait for all buffers to complete */
 		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+		sbd_ats_remove_by_task(task);
 		if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL)
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_READ_ERROR);
@@ -476,7 +505,8 @@
 		uint32_t maxsize, minsize, old_minsize;
 		stmf_free_dbuf(task, dbuf);
 
-		maxsize = (scmd->len > (128*1024)) ? 128*1024 : scmd->len;
+		maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ?
+		    128*1024 : ATOMIC32_GET(scmd->len);
 		minsize = maxsize >> 2;
 		do {
 			old_minsize = minsize;
@@ -484,8 +514,8 @@
 		} while ((dbuf == NULL) && (old_minsize > minsize) &&
 		    (minsize >= 512));
 		if (dbuf == NULL) {
-			scmd->nbufs --;
-			if (scmd->nbufs == 0) {
+			atomic_dec_8(&scmd->nbufs);
+			if (ATOMIC8_GET(scmd->nbufs) == 0) {
 				stmf_abort(STMF_QUEUE_TASK_ABORT, task,
 				    STMF_ALLOC_FAILURE, NULL);
 			}
@@ -503,7 +533,7 @@
  */
 void
 sbd_handle_sgl_read_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
-				struct stmf_data_buf *dbuf)
+    struct stmf_data_buf *dbuf)
 {
 	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
 	stmf_status_t xfer_status;
@@ -513,7 +543,7 @@
 	ASSERT(dbuf->db_lu_private);
 	ASSERT(scmd->cmd_type == SBD_CMD_SCSI_READ);
 
-	scmd->nbufs--;	/* account for this dbuf */
+	atomic_dec_8(&scmd->nbufs);	/* account for this dbuf */
 	/*
 	 * Release the DMU resources.
 	 */
@@ -534,7 +564,8 @@
 	scmd_err = (((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
 	    (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) ||
 	    (xfer_status != STMF_SUCCESS));
-	if (scmd->nbufs == 0 && (scmd->len == 0 || scmd_err)) {
+	if ((ATOMIC8_GET(scmd->nbufs) == 0) &&
+	    (ATOMIC32_GET(scmd->len) == 0 || scmd_err)) {
 		/* all DMU state has been released */
 		rw_exit(&sl->sl_access_state_lock);
 	}
@@ -548,7 +579,8 @@
 		 * This chunk completed successfully
 		 */
 		task->task_nbytes_transferred += data_size;
-		if (scmd->nbufs == 0 && scmd->len == 0) {
+		if (ATOMIC8_GET(scmd->nbufs) == 0 &&
+		    ATOMIC32_GET(scmd->len) == 0) {
 			/*
 			 * This command completed successfully
 			 *
@@ -556,6 +588,7 @@
 			 * completion will occur. Tell stmf we are done.
 			 */
 			scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+			sbd_ats_remove_by_task(task);
 			stmf_task_lu_done(task);
 			return;
 		}
@@ -574,8 +607,9 @@
 		 * and wait for the last completion to send the status check.
 		 */
 		if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
-			if (scmd->nbufs == 0) {
+			if (ATOMIC8_GET(scmd->nbufs) == 0) {
 				scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+				sbd_ats_remove_by_task(task);
 				stmf_scsilib_send_status(task, STATUS_CHECK,
 				    STMF_SAA_READ_ERROR);
 			}
@@ -585,7 +619,14 @@
 		 * Must have been a failure on current dbuf
 		 */
 		ASSERT(xfer_status != STMF_SUCCESS);
+
+		/*
+		 * Actually this is a bug. stmf abort should have reset the
+		 * active flag but since its been there for some time.
+		 * I wont change it.
+		 */
 		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+		sbd_ats_remove_by_task(task);
 		stmf_abort(STMF_QUEUE_TASK_ABORT, task, xfer_status, NULL);
 	}
 }
@@ -592,7 +633,7 @@
 
 void
 sbd_handle_sgl_write_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
-				struct stmf_data_buf *dbuf)
+    struct stmf_data_buf *dbuf)
 {
 	sbd_zvol_io_t *zvio = dbuf->db_lu_private;
 	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
@@ -600,6 +641,7 @@
 	int scmd_err, scmd_xfer_done;
 	stmf_status_t xfer_status = dbuf->db_xfer_status;
 	uint32_t data_size = dbuf->db_data_size;
+	hrtime_t xfer_start;
 
 	ASSERT(zvio);
 
@@ -609,16 +651,18 @@
 	 */
 	stmf_teardown_dbuf(task, dbuf);
 
-	scmd->nbufs--;	/* account for this dbuf */
+	atomic_dec_8(&scmd->nbufs);	/* account for this dbuf */
 	/*
 	 * All data was queued and this is the last completion,
 	 * but there could still be an error.
 	 */
-	scmd_xfer_done = (scmd->len == 0 && scmd->nbufs == 0);
+	scmd_xfer_done = (ATOMIC32_GET(scmd->len) == 0 &&
+	    (ATOMIC8_GET(scmd->nbufs) == 0));
 	scmd_err = (((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
 	    (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) ||
 	    (xfer_status != STMF_SUCCESS));
 
+	xfer_start = gethrtime();
 	DTRACE_PROBE5(backing__store__write__start, sbd_lu_t *, sl,
 	    uint8_t *, NULL, uint64_t, data_size,
 	    uint64_t, zvio->zvio_offset, scsi_task_t *, task);
@@ -636,6 +680,8 @@
 		ret = sbd_zvol_rele_write_bufs(sl, dbuf);
 	}
 
+	stmf_lu_xfer_done(task, B_FALSE /* write */,
+	    (gethrtime() - xfer_start));
 	DTRACE_PROBE6(backing__store__write__end, sbd_lu_t *, sl,
 	    uint8_t *, NULL, uint64_t, data_size,
 	    uint64_t, zvio->zvio_offset, int, ret,  scsi_task_t *, task);
@@ -655,7 +701,8 @@
 	 * transferred or an error encountered, then no more dbufs
 	 * will be queued.
 	 */
-	if (scmd->nbufs == 0 && (scmd->len == 0 || scmd_err)) {
+	if ((ATOMIC8_GET(scmd->nbufs) == 0) &&
+	    (ATOMIC32_GET(scmd->len) == 0 || scmd_err)) {
 		/* all DMU state has been released */
 		rw_exit(&sl->sl_access_state_lock);
 	}
@@ -669,6 +716,7 @@
 		if (scmd_xfer_done) {
 			/* This command completed successfully */
 			scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+			sbd_ats_remove_by_task(task);
 			if ((scmd->flags & SBD_SCSI_CMD_SYNC_WRITE) &&
 			    (sbd_flush_data_cache(sl, 0) != SBD_SUCCESS)) {
 				stmf_scsilib_send_status(task, STATUS_CHECK,
@@ -689,8 +737,9 @@
 	 */
 	if (scmd->flags & SBD_SCSI_CMD_ACTIVE) {
 		if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
-			if (scmd->nbufs == 0) {
+			if (ATOMIC8_GET(scmd->nbufs) == 0) {
 				scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+				sbd_ats_remove_by_task(task);
 				stmf_scsilib_send_status(task, STATUS_CHECK,
 				    STMF_SAA_WRITE_ERROR);
 			}
@@ -700,6 +749,7 @@
 			return;
 		}
 		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+		sbd_ats_remove_by_task(task);
 		ASSERT(xfer_status != STMF_SUCCESS);
 		stmf_abort(STMF_QUEUE_TASK_ABORT, task, xfer_status, NULL);
 	}
@@ -725,6 +775,7 @@
 	struct iovec		*iov, *tiov, iov1[8];
 	uint32_t		len, resid;
 	int			ret, i, iovcnt, flags;
+	hrtime_t		xfer_start;
 	boolean_t		is_read;
 
 	ASSERT(cmd == SBD_CMD_SCSI_READ || cmd == SBD_CMD_SCSI_WRITE);
@@ -762,6 +813,7 @@
 	uio.uio_resid = (uint64_t)len;
 	uio.uio_llimit = RLIM64_INFINITY;
 
+	xfer_start = gethrtime();
 	if (is_read == B_TRUE) {
 		uio.uio_fmode = FREAD;
 		uio.uio_extflg = UIO_COPY_CACHED;
@@ -790,6 +842,8 @@
 		    uint8_t *, NULL, uint64_t, len, uint64_t, laddr, int, ret,
 		    scsi_task_t *, task);
 	}
+	/* finalize accounting */
+	stmf_lu_xfer_done(task, is_read, (gethrtime() - xfer_start));
 
 	if (iov != &iov1[0])
 		kmem_free(iov, iovcnt * sizeof (*iov));
@@ -804,6 +858,7 @@
 sbd_handle_read(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
 {
 	uint64_t lba, laddr;
+	uint64_t blkcount;
 	uint32_t len;
 	uint8_t op = task->task_cdb[0];
 	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
@@ -810,7 +865,19 @@
 	sbd_cmd_t *scmd;
 	stmf_data_buf_t *dbuf;
 	int fast_path;
+	boolean_t fua_bit = B_FALSE;
 
+	/*
+	 * Check to see if the command is READ(10), READ(12), or READ(16).
+	 * If it is then check for bit 3 being set to indicate if Forced
+	 * Unit Access is being requested. If so, we'll bypass the use of
+	 * DMA buffers to simplify support of this feature.
+	 */
+	if (((op == SCMD_READ_G1) || (op == SCMD_READ_G4) ||
+	    (op == SCMD_READ_G5)) &&
+	    (task->task_cdb[1] & BIT_3)) {
+		fua_bit = B_TRUE;
+	}
 	if (op == SCMD_READ) {
 		lba = READ_SCSI21(&task->task_cdb[1], uint64_t);
 		len = (uint32_t)task->task_cdb[4];
@@ -834,6 +901,7 @@
 	}
 
 	laddr = lba << sl->sl_data_blocksize_shift;
+	blkcount = len;
 	len <<= sl->sl_data_blocksize_shift;
 
 	if ((laddr + (uint64_t)len) > sl->sl_lu_size) {
@@ -860,6 +928,13 @@
 		return;
 	}
 
+	if (sbd_ats_handling_before_io(task, sl, lba, blkcount) !=
+	    SBD_SUCCESS) {
+		if (stmf_task_poll_lu(task, 10) != STMF_SUCCESS) {
+			stmf_scsilib_send_status(task, STATUS_BUSY, 0);
+		}
+		return;
+	}
 	/*
 	 * Determine if this read can directly use DMU buffers.
 	 */
@@ -867,8 +942,8 @@
 	    initial_dbuf == NULL &&		/* No PP buffer passed in */
 	    sl->sl_flags & SL_CALL_ZVOL &&	/* zvol backing store */
 	    (task->task_additional_flags &
-	    TASK_AF_ACCEPT_LU_DBUF))		/* PP allows it */
-	{
+	    TASK_AF_ACCEPT_LU_DBUF) &&		/* PP allows it */
+	    !fua_bit) {
 		/*
 		 * Reduced copy path
 		 */
@@ -883,6 +958,7 @@
 		rw_enter(&sl->sl_access_state_lock, RW_READER);
 		if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
 			rw_exit(&sl->sl_access_state_lock);
+			sbd_ats_remove_by_task(task);
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_READ_ERROR);
 			return;
@@ -903,6 +979,7 @@
 			    SBD_CMD_SCSI_READ, 0);
 			/* done with the backend */
 			rw_exit(&sl->sl_access_state_lock);
+			sbd_ats_remove_by_task(task);
 			if (ret != 0) {
 				/* backend error */
 				stmf_scsilib_send_status(task, STATUS_CHECK,
@@ -935,13 +1012,12 @@
 		/*
 		 * Setup scmd to track read progress.
 		 */
-		scmd->flags = SBD_SCSI_CMD_ACTIVE;
+		scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED;
 		scmd->cmd_type = SBD_CMD_SCSI_READ;
 		scmd->nbufs = 0;
 		scmd->addr = laddr;
 		scmd->len = len;
 		scmd->current_ro = 0;
-
 		/*
 		 * Kick-off the read.
 		 */
@@ -961,6 +1037,7 @@
 		} while ((initial_dbuf == NULL) && (old_minsize > minsize) &&
 		    (minsize >= 512));
 		if (initial_dbuf == NULL) {
+			sbd_ats_remove_by_task(task);
 			stmf_scsilib_send_status(task, STATUS_QFULL, 0);
 			return;
 		}
@@ -984,6 +1061,7 @@
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_READ_ERROR);
 		}
+		sbd_ats_remove_by_task(task);
 		return;
 	}
 
@@ -993,7 +1071,7 @@
 		scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
 		task->task_lu_private = scmd;
 	}
-	scmd->flags = SBD_SCSI_CMD_ACTIVE;
+	scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED;
 	scmd->cmd_type = SBD_CMD_SCSI_READ;
 	scmd->nbufs = 1;
 	scmd->addr = laddr;
@@ -1010,7 +1088,7 @@
 	uint32_t len;
 	int bufs_to_take;
 
-	if (scmd->len == 0) {
+	if (ATOMIC32_GET(scmd->len) == 0) {
 		goto DO_WRITE_XFER_DONE;
 	}
 
@@ -1025,14 +1103,14 @@
 		stmf_free_dbuf(task, dbuf);
 		dbuf = NULL;
 	}
-	if (scmd->nbufs >= bufs_to_take) {
+	if (ATOMIC8_GET(scmd->nbufs) >= bufs_to_take) {
 		goto DO_WRITE_XFER_DONE;
 	}
 	if (dbuf == NULL) {
 		uint32_t maxsize, minsize, old_minsize;
 
-		maxsize = (scmd->len > (128*1024)) ? 128*1024 :
-		    scmd->len;
+		maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ? 128*1024 :
+		    ATOMIC32_GET(scmd->len);
 		minsize = maxsize >> 2;
 		do {
 			old_minsize = minsize;
@@ -1040,7 +1118,7 @@
 		} while ((dbuf == NULL) && (old_minsize > minsize) &&
 		    (minsize >= 512));
 		if (dbuf == NULL) {
-			if (scmd->nbufs == 0) {
+			if (ATOMIC8_GET(scmd->nbufs) == 0) {
 				stmf_abort(STMF_QUEUE_TASK_ABORT, task,
 				    STMF_ALLOC_FAILURE, NULL);
 			}
@@ -1048,18 +1126,20 @@
 		}
 	}
 
-	len = scmd->len > dbuf->db_buf_size ? dbuf->db_buf_size :
-	    scmd->len;
+	len = ATOMIC32_GET(scmd->len) > dbuf->db_buf_size ? dbuf->db_buf_size :
+	    ATOMIC32_GET(scmd->len);
 
 	dbuf->db_relative_offset = scmd->current_ro;
 	dbuf->db_data_size = len;
 	dbuf->db_flags = DB_DIRECTION_FROM_RPORT;
 	(void) stmf_xfer_data(task, dbuf, 0);
-	scmd->nbufs++; /* outstanding port xfers and bufs used */
-	scmd->len -= len;
+	/* outstanding port xfers and bufs used */
+	atomic_inc_8(&scmd->nbufs);
+	atomic_add_32(&scmd->len, -len);
 	scmd->current_ro += len;
 
-	if ((scmd->len != 0) && (scmd->nbufs < bufs_to_take)) {
+	if ((ATOMIC32_GET(scmd->len) != 0) &&
+	    (ATOMIC8_GET(scmd->nbufs) < bufs_to_take)) {
 		sbd_do_write_xfer(task, scmd, NULL, 0);
 	}
 	return;
@@ -1125,14 +1205,14 @@
 	}
 
 
-	while (scmd->len && scmd->nbufs < task->task_max_nbufs) {
-
-		xfer_len = MIN(max_len, scmd->len);
+	while (ATOMIC32_GET(scmd->len) &&
+	    ATOMIC8_GET(scmd->nbufs) < task->task_max_nbufs) {
+		xfer_len = MIN(max_len, ATOMIC32_GET(scmd->len));
 		if (first_len) {
 			xfer_len = MIN(xfer_len, first_len);
 			first_len = 0;
 		}
-		if (xfer_len < scmd->len) {
+		if (xfer_len < ATOMIC32_GET(scmd->len)) {
 			/*
 			 * Attempt to end xfer on a block boundary.
 			 * The only way this does not happen is if the
@@ -1196,10 +1276,11 @@
 			 */
 			stmf_free(dbuf);
 			scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
-			if (scmd->nbufs == 0) {
+			if (ATOMIC8_GET(scmd->nbufs) == 0) {
 				/*
 				 * Nothing queued, so no completions coming
 				 */
+				sbd_ats_remove_by_task(task);
 				stmf_scsilib_send_status(task, STATUS_CHECK,
 				    STMF_SAA_WRITE_ERROR);
 				rw_exit(&sl->sl_access_state_lock);
@@ -1223,7 +1304,7 @@
 			 */
 			sbd_zvol_rele_write_bufs_abort(sl, dbuf);
 			stmf_free(dbuf);
-			if (scmd->nbufs > 0) {
+			if (ATOMIC8_GET(scmd->nbufs) > 0) {
 				/* completion of previous dbuf will retry */
 				return;
 			}
@@ -1231,6 +1312,7 @@
 			 * Done with this command.
 			 */
 			scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+			sbd_ats_remove_by_task(task);
 			if (first_xfer)
 				stmf_scsilib_send_status(task, STATUS_QFULL, 0);
 			else
@@ -1243,7 +1325,7 @@
 		/*
 		 * dbuf is now queued on task
 		 */
-		scmd->nbufs++;
+		atomic_inc_8(&scmd->nbufs);
 
 		xstat = stmf_xfer_data(task, dbuf, 0);
 		switch (xstat) {
@@ -1257,8 +1339,8 @@
 			sbd_zvol_rele_write_bufs_abort(sl, dbuf);
 			stmf_teardown_dbuf(task, dbuf);
 			stmf_free(dbuf);
-			scmd->nbufs--;
-			if (scmd->nbufs > 0) {
+			atomic_dec_8(&scmd->nbufs);
+			if (ATOMIC8_GET(scmd->nbufs) > 0) {
 				/* completion of previous dbuf will retry */
 				return;
 			}
@@ -1266,6 +1348,7 @@
 			 * Done with this command.
 			 */
 			scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+			sbd_ats_remove_by_task(task);
 			if (first_xfer)
 				stmf_scsilib_send_status(task, STATUS_QFULL, 0);
 			else
@@ -1283,7 +1366,7 @@
 		/*
 		 * Update the xfer progress.
 		 */
-		scmd->len -= xfer_len;
+		atomic_add_32(&scmd->len, -xfer_len);
 		scmd->current_ro += xfer_len;
 	}
 }
@@ -1296,17 +1379,20 @@
 	uint64_t laddr;
 	uint32_t buflen, iolen;
 	int ndx;
+	uint8_t op = task->task_cdb[0];
+	boolean_t fua_bit = B_FALSE;
 
-	if (scmd->nbufs > 0) {
+	if (ATOMIC8_GET(scmd->nbufs) > 0) {
 		/*
 		 * Decrement the count to indicate the port xfer
 		 * into the dbuf has completed even though the buf is
 		 * still in use here in the LU provider.
 		 */
-		scmd->nbufs--;
+		atomic_dec_8(&scmd->nbufs);
 	}
 
 	if (dbuf->db_xfer_status != STMF_SUCCESS) {
+		sbd_ats_remove_by_task(task);
 		stmf_abort(STMF_QUEUE_TASK_ABORT, task,
 		    dbuf->db_xfer_status, NULL);
 		return;
@@ -1316,7 +1402,7 @@
 		goto WRITE_XFER_DONE;
 	}
 
-	if (scmd->len != 0) {
+	if (ATOMIC32_GET(scmd->len) != 0) {
 		/*
 		 * Initiate the next port xfer to occur in parallel
 		 * with writing this buf.
@@ -1324,6 +1410,16 @@
 		sbd_do_write_xfer(task, scmd, NULL, 0);
 	}
 
+	/*
+	 * Check to see if the command is WRITE(10), WRITE(12), or WRITE(16).
+	 * If it is then check for bit 3 being set to indicate if Forced
+	 * Unit Access is being requested. If so, we'll bypass the direct
+	 * call and handle it in sbd_data_write().
+	 */
+	if (((op == SCMD_WRITE_G1) || (op == SCMD_WRITE_G4) ||
+	    (op == SCMD_WRITE_G5)) && (task->task_cdb[1] & BIT_3)) {
+		fua_bit = B_TRUE;
+	}
 	laddr = scmd->addr + dbuf->db_relative_offset;
 
 	/*
@@ -1333,13 +1429,17 @@
 	 */
 	if (sl->sl_flags & SL_CALL_ZVOL &&
 	    (task->task_additional_flags & TASK_AF_ACCEPT_LU_DBUF) &&
-	    (sbd_zcopy & (4|1))) {
+	    (sbd_zcopy & (4|1)) && !fua_bit) {
 		int commit;
 
-		commit = (scmd->len == 0 && scmd->nbufs == 0);
-		if (sbd_copy_rdwr(task, laddr, dbuf, SBD_CMD_SCSI_WRITE,
+		commit = (ATOMIC32_GET(scmd->len) == 0 &&
+		    ATOMIC8_GET(scmd->nbufs) == 0);
+		rw_enter(&sl->sl_access_state_lock, RW_READER);
+		if ((sl->sl_flags & SL_MEDIA_LOADED) == 0 ||
+		    sbd_copy_rdwr(task, laddr, dbuf, SBD_CMD_SCSI_WRITE,
 		    commit) != STMF_SUCCESS)
 			scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
+		rw_exit(&sl->sl_access_state_lock);
 		buflen = dbuf->db_data_size;
 	} else {
 		for (buflen = 0, ndx = 0; (buflen < dbuf->db_data_size) &&
@@ -1359,11 +1459,13 @@
 	}
 	task->task_nbytes_transferred += buflen;
 WRITE_XFER_DONE:
-	if (scmd->len == 0 || scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
+	if (ATOMIC32_GET(scmd->len) == 0 ||
+	    scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
 		stmf_free_dbuf(task, dbuf);
-		if (scmd->nbufs)
+		if (ATOMIC8_GET(scmd->nbufs))
 			return;	/* wait for all buffers to complete */
 		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+		sbd_ats_remove_by_task(task);
 		if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_WRITE_ERROR);
@@ -1429,7 +1531,9 @@
 	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
 	sbd_cmd_t *scmd;
 	stmf_data_buf_t *dbuf;
+	uint64_t blkcount;
 	uint8_t	sync_wr_flag = 0;
+	boolean_t fua_bit = B_FALSE;
 
 	if (sl->sl_flags & SL_WRITE_PROTECTED) {
 		stmf_scsilib_send_status(task, STATUS_CHECK,
@@ -1436,6 +1540,16 @@
 		    STMF_SAA_WRITE_PROTECTED);
 		return;
 	}
+	/*
+	 * Check to see if the command is WRITE(10), WRITE(12), or WRITE(16).
+	 * If it is then check for bit 3 being set to indicate if Forced
+	 * Unit Access is being requested. If so, we'll bypass the fast path
+	 * code to simplify support of this feature.
+	 */
+	if (((op == SCMD_WRITE_G1) || (op == SCMD_WRITE_G4) ||
+	    (op == SCMD_WRITE_G5)) && (task->task_cdb[1] & BIT_3)) {
+		fua_bit = B_TRUE;
+	}
 	if (op == SCMD_WRITE) {
 		lba = READ_SCSI21(&task->task_cdb[1], uint64_t);
 		len = (uint32_t)task->task_cdb[4];
@@ -1471,6 +1585,7 @@
 	}
 
 	laddr = lba << sl->sl_data_blocksize_shift;
+	blkcount = len;
 	len <<= sl->sl_data_blocksize_shift;
 
 	if ((laddr + (uint64_t)len) > sl->sl_lu_size) {
@@ -1492,12 +1607,21 @@
 		return;
 	}
 
+	if (sbd_ats_handling_before_io(task, sl, lba, blkcount) !=
+	    SBD_SUCCESS) {
+		if (stmf_task_poll_lu(task, 10) != STMF_SUCCESS) {
+			stmf_scsilib_send_status(task, STATUS_BUSY, 0);
+		}
+		return;
+	}
+
 	if (sbd_zcopy & (4|1) &&		/* Debug switch */
 	    initial_dbuf == NULL &&		/* No PP buf passed in */
 	    sl->sl_flags & SL_CALL_ZVOL &&	/* zvol backing store */
 	    (task->task_additional_flags &
 	    TASK_AF_ACCEPT_LU_DBUF) &&		/* PP allows it */
-	    sbd_zcopy_write_useful(task, laddr, len, sl->sl_blksize)) {
+	    sbd_zcopy_write_useful(task, laddr, len, sl->sl_blksize) &&
+	    !fua_bit) {
 
 		/*
 		 * XXX Note that disallowing initial_dbuf will eliminate
@@ -1509,6 +1633,7 @@
 		rw_enter(&sl->sl_access_state_lock, RW_READER);
 		if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
 			rw_exit(&sl->sl_access_state_lock);
+			sbd_ats_remove_by_task(task);
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_READ_ERROR);
 			return;
@@ -1523,7 +1648,8 @@
 			    KM_SLEEP);
 			task->task_lu_private = scmd;
 		}
-		scmd->flags = SBD_SCSI_CMD_ACTIVE | sync_wr_flag;
+		scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED |
+		    sync_wr_flag;
 		scmd->cmd_type = SBD_CMD_SCSI_WRITE;
 		scmd->nbufs = 0;
 		scmd->addr = laddr;
@@ -1554,7 +1680,8 @@
 		scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
 		task->task_lu_private = scmd;
 	}
-	scmd->flags = SBD_SCSI_CMD_ACTIVE | sync_wr_flag;
+	scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED |
+	    sync_wr_flag;
 	scmd->cmd_type = SBD_CMD_SCSI_WRITE;
 	scmd->nbufs = 0;
 	scmd->addr = laddr;
@@ -1566,7 +1693,7 @@
 		 * Account for data passed in this write command
 		 */
 		(void) stmf_xfer_data(task, dbuf, STMF_IOF_STATS_ONLY);
-		scmd->len -= dbuf->db_data_size;
+		atomic_add_32(&scmd->len, -dbuf->db_data_size);
 		scmd->current_ro += dbuf->db_data_size;
 		dbuf->db_xfer_status = STMF_SUCCESS;
 		sbd_handle_write_xfer_completion(task, scmd, dbuf, 0);
@@ -1641,7 +1768,7 @@
 
 void
 sbd_handle_short_read_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
-				struct stmf_data_buf *dbuf)
+    struct stmf_data_buf *dbuf)
 {
 	if (dbuf->db_xfer_status != STMF_SUCCESS) {
 		stmf_abort(STMF_QUEUE_TASK_ABORT, task,
@@ -1749,6 +1876,9 @@
 		sbd_handle_unmap_xfer(task,
 		    dbuf->db_sglist[0].seg_addr, dbuf->db_data_size);
 		break;
+	case SCMD_EXTENDED_COPY:
+		sbd_handle_xcopy_xfer(task, dbuf->db_sglist[0].seg_addr);
+		break;
 	case SCMD_PERSISTENT_RESERVE_OUT:
 		if (sl->sl_access_state == SBD_LU_STANDBY) {
 			st_ret = stmf_proxy_scsi_cmd(task, dbuf);
@@ -1843,7 +1973,7 @@
 	uint8_t *cdb;
 	uint32_t ncyl;
 	uint8_t nsectors, nheads;
-	uint8_t page, ctrl, header_size, pc_valid;
+	uint8_t page, ctrl, header_size;
 	uint16_t nbytes;
 	uint8_t *p;
 	uint64_t s = sl->sl_lu_size;
@@ -1854,25 +1984,21 @@
 	cdb = &task->task_cdb[0];
 	page = cdb[2] & 0x3F;
 	ctrl = (cdb[2] >> 6) & 3;
-	cmd_size = (cdb[0] == SCMD_MODE_SENSE) ? cdb[4] :
-	    READ_SCSI16(&cdb[7], uint32_t);
 
 	if (cdb[0] == SCMD_MODE_SENSE) {
+		cmd_size = cdb[4];
 		header_size = 4;
 		dev_spec_param_offset = 2;
 	} else {
+		cmd_size = READ_SCSI16(&cdb[7], uint32_t);
 		header_size = 8;
 		dev_spec_param_offset = 3;
 	}
 
 	/* Now validate the command */
-	if ((cdb[2] == 0) || (page == MODEPAGE_ALLPAGES) || (page == 0x08) ||
-	    (page == 0x0A) || (page == 0x03) || (page == 0x04)) {
-		pc_valid = 1;
-	} else {
-		pc_valid = 0;
-	}
-	if ((cmd_size < header_size) || (pc_valid == 0)) {
+	if ((cdb[2] != 0) && (page != MODEPAGE_ALLPAGES) &&
+	    (page != MODEPAGE_CACHING) && (page != MODEPAGE_CTRL_MODE) &&
+	    (page != MODEPAGE_FORMAT) && (page != MODEPAGE_GEOMETRY)) {
 		stmf_scsilib_send_status(task, STATUS_CHECK,
 		    STMF_SAA_INVALID_FIELD_IN_CDB);
 		return;
@@ -1890,7 +2016,7 @@
 	nbytes = ((uint16_t)1) << sl->sl_data_blocksize_shift;
 	sbd_calc_geometry(s, nbytes, &nsectors, &nheads, &ncyl);
 
-	if ((page == 0x03) || (page == MODEPAGE_ALLPAGES)) {
+	if ((page == MODEPAGE_FORMAT) || (page == MODEPAGE_ALLPAGES)) {
 		p[n] = 0x03;
 		p[n+1] = 0x16;
 		if (ctrl != 1) {
@@ -1901,7 +2027,7 @@
 		}
 		n += 24;
 	}
-	if ((page == 0x04) || (page == MODEPAGE_ALLPAGES)) {
+	if ((page == MODEPAGE_GEOMETRY) || (page == MODEPAGE_ALLPAGES)) {
 		p[n] = 0x04;
 		p[n + 1] = 0x16;
 		if (ctrl != 1) {
@@ -1976,11 +2102,11 @@
 		 * of bytes in the length field, so adjust the count.
 		 * Byte count minus header length field size.
 		 */
-		buf[0] = (n - 1) & 0xff;
+		buf[0] = (n - header_size) & 0xff;
 	} else {
 		/* Byte count minus header length field size. */
-		buf[1] = (n - 2) & 0xff;
-		buf[0] = ((n - 2) >> 8) & 0xff;
+		buf[1] = (n - header_size) & 0xff;
+		buf[0] = ((n - header_size) >> 8) & 0xff;
 	}
 
 	sbd_handle_short_read_transfers(task, initial_dbuf, buf,
@@ -2189,7 +2315,8 @@
  *       and returns the length of the URL
  */
 uint16_t
-sbd_parse_mgmt_url(char **url_addr) {
+sbd_parse_mgmt_url(char **url_addr)
+{
 	uint16_t url_length = 0;
 	char *url;
 	url = *url_addr;
@@ -2300,6 +2427,21 @@
 }
 
 static void
+sbd_write_same_release_resources(struct scsi_task *task)
+{
+	sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
+
+	if (scmd->nbufs == 0XFF)
+		cmn_err(CE_WARN, "%s invalid buffer count %x",
+		    __func__, scmd->nbufs);
+	if ((scmd->trans_data_len != 0) && (scmd->trans_data != NULL))
+		kmem_free(scmd->trans_data, scmd->trans_data_len);
+	scmd->trans_data = NULL;
+	scmd->trans_data_len = 0;
+	scmd->flags &= ~SBD_SCSI_CMD_TRANS_DATA;
+}
+
+static void
 sbd_handle_write_same_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
     struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
 {
@@ -2307,7 +2449,13 @@
 	uint32_t buflen, iolen;
 	int ndx, ret;
 
+	if (ATOMIC8_GET(scmd->nbufs) > 0) {
+		atomic_dec_8(&scmd->nbufs);
+	}
+
 	if (dbuf->db_xfer_status != STMF_SUCCESS) {
+		sbd_write_same_release_resources(task);
+		sbd_ats_remove_by_task(task);
 		stmf_abort(STMF_QUEUE_TASK_ABORT, task,
 		    dbuf->db_xfer_status, NULL);
 		return;
@@ -2317,7 +2465,16 @@
 		goto write_same_xfer_done;
 	}
 
-	if (scmd->len != 0) {
+	/* if this is a unnessary callback just return */
+	if (((scmd->flags & SBD_SCSI_CMD_TRANS_DATA) == 0) ||
+	    ((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
+	    (scmd->trans_data == NULL)) {
+		sbd_ats_remove_by_task(task);
+		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
+		return;
+	}
+
+	if (ATOMIC32_GET(scmd->len) != 0) {
 		/*
 		 * Initiate the next port xfer to occur in parallel
 		 * with writing this buf.
@@ -2341,14 +2498,21 @@
 	task->task_nbytes_transferred += buflen;
 
 write_same_xfer_done:
-	if (scmd->len == 0 || scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
+	if (ATOMIC32_GET(scmd->len) == 0 ||
+	    scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
 		stmf_free_dbuf(task, dbuf);
+		if (ATOMIC8_GET(scmd->nbufs) > 0)
+			return;
 		scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
 		if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
+			sbd_ats_remove_by_task(task);
+			sbd_write_same_release_resources(task);
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_WRITE_ERROR);
 		} else {
 			ret = sbd_write_same_data(task, scmd);
+			sbd_ats_remove_by_task(task);
+			sbd_write_same_release_resources(task);
 			if (ret != SBD_SUCCESS) {
 				stmf_scsilib_send_status(task, STATUS_CHECK,
 				    STMF_SAA_WRITE_ERROR);
@@ -2356,14 +2520,6 @@
 				stmf_scsilib_send_status(task, STATUS_GOOD, 0);
 			}
 		}
-		/*
-		 * Only way we should get here is via handle_write_same(),
-		 * and that should make the following assertion always pass.
-		 */
-		ASSERT((scmd->flags & SBD_SCSI_CMD_TRANS_DATA) &&
-		    scmd->trans_data != NULL);
-		kmem_free(scmd->trans_data, scmd->trans_data_len);
-		scmd->flags &= ~SBD_SCSI_CMD_TRANS_DATA;
 		return;
 	}
 	sbd_do_write_same_xfer(task, scmd, dbuf, dbuf_reusable);
@@ -2375,7 +2531,7 @@
 {
 	uint32_t len;
 
-	if (scmd->len == 0) {
+	if (ATOMIC32_GET(scmd->len) == 0) {
 		if (dbuf != NULL)
 			stmf_free_dbuf(task, dbuf);
 		return;
@@ -2390,8 +2546,8 @@
 	if (dbuf == NULL) {
 		uint32_t maxsize, minsize, old_minsize;
 
-		maxsize = (scmd->len > (128*1024)) ? 128*1024 :
-		    scmd->len;
+		maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ? 128*1024 :
+		    ATOMIC32_GET(scmd->len);
 		minsize = maxsize >> 2;
 		do {
 			old_minsize = minsize;
@@ -2399,7 +2555,9 @@
 		} while ((dbuf == NULL) && (old_minsize > minsize) &&
 		    (minsize >= 512));
 		if (dbuf == NULL) {
-			if (scmd->nbufs == 0) {
+			sbd_ats_remove_by_task(task);
+			sbd_write_same_release_resources(task);
+			if (ATOMIC8_GET(scmd->nbufs) == 0) {
 				stmf_abort(STMF_QUEUE_TASK_ABORT, task,
 				    STMF_ALLOC_FAILURE, NULL);
 			}
@@ -2407,15 +2565,16 @@
 		}
 	}
 
-	len = scmd->len > dbuf->db_buf_size ? dbuf->db_buf_size :
-	    scmd->len;
+	len = ATOMIC32_GET(scmd->len) > dbuf->db_buf_size ? dbuf->db_buf_size :
+	    ATOMIC32_GET(scmd->len);
 
 	dbuf->db_relative_offset = scmd->current_ro;
 	dbuf->db_data_size = len;
 	dbuf->db_flags = DB_DIRECTION_FROM_RPORT;
 	(void) stmf_xfer_data(task, dbuf, 0);
-	scmd->nbufs++; /* outstanding port xfers and bufs used */
-	scmd->len -= len;
+	/* outstanding port xfers and bufs used */
+	atomic_inc_8(&scmd->nbufs);
+	atomic_add_32(&scmd->len, -len);
 	scmd->current_ro += len;
 }
 
@@ -2429,6 +2588,12 @@
 	uint8_t unmap;
 	uint8_t do_immediate_data = 0;
 
+	if (HardwareAcceleratedInit == 0) {
+		stmf_scsilib_send_status(task, STATUS_CHECK,
+		    STMF_SAA_INVALID_OPCODE);
+		return;
+	}
+
 	task->task_cmd_xfer_length = 0;
 	if (task->task_additional_flags &
 	    TASK_AF_NO_EXPECTED_XFER_LENGTH) {
@@ -2445,11 +2610,13 @@
 		return;
 	}
 	unmap = task->task_cdb[1] & 0x08;
+
 	if (unmap && ((sl->sl_flags & SL_UNMAP_ENABLED) == 0)) {
 		stmf_scsilib_send_status(task, STATUS_CHECK,
 		    STMF_SAA_INVALID_FIELD_IN_CDB);
 		return;
 	}
+
 	if (task->task_cdb[0] == SCMD_WRITE_SAME_G1) {
 		addr = READ_SCSI32(&task->task_cdb[2], uint64_t);
 		len = READ_SCSI16(&task->task_cdb[7], uint64_t);
@@ -2457,22 +2624,37 @@
 		addr = READ_SCSI64(&task->task_cdb[2], uint64_t);
 		len = READ_SCSI32(&task->task_cdb[10], uint64_t);
 	}
+
 	if (len == 0) {
 		stmf_scsilib_send_status(task, STATUS_CHECK,
 		    STMF_SAA_INVALID_FIELD_IN_CDB);
 		return;
 	}
+
+	if (sbd_ats_handling_before_io(task, sl, addr, len) !=
+	    SBD_SUCCESS) {
+		if (stmf_task_poll_lu(task, 10) != STMF_SUCCESS)
+			stmf_scsilib_send_status(task, STATUS_BUSY, 0);
+		return;
+	}
+
 	addr <<= sl->sl_data_blocksize_shift;
 	len <<= sl->sl_data_blocksize_shift;
 
 	/* Check if the command is for the unmap function */
 	if (unmap) {
-		if (sbd_unmap(sl, addr, len) != 0) {
+		dkioc_free_list_t *dfl = kmem_zalloc(DFL_SZ(1), KM_SLEEP);
+
+		dfl->dfl_num_exts = 1;
+		dfl->dfl_exts[0].dfle_start = addr;
+		dfl->dfl_exts[0].dfle_length = len;
+		if (sbd_unmap(sl, dfl) != 0) {
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_LBA_OUT_OF_RANGE);
 		} else {
 			stmf_scsilib_send_status(task, STATUS_GOOD, 0);
 		}
+		dfl_free(dfl);
 		return;
 	}
 
@@ -2484,6 +2666,7 @@
 		task->task_expected_xfer_length = task->task_cmd_xfer_length;
 	}
 	if ((addr + len) > sl->sl_lu_size) {
+		sbd_ats_remove_by_task(task);
 		stmf_scsilib_send_status(task, STATUS_CHECK,
 		    STMF_SAA_LBA_OUT_OF_RANGE);
 		return;
@@ -2494,6 +2677,7 @@
 
 	/* Some basic checks */
 	if ((len == 0) || (len != task->task_expected_xfer_length)) {
+		sbd_ats_remove_by_task(task);
 		stmf_scsilib_send_status(task, STATUS_CHECK,
 		    STMF_SAA_INVALID_FIELD_IN_CDB);
 		return;
@@ -2505,6 +2689,7 @@
 			if (initial_dbuf->db_data_size >
 			    task->task_expected_xfer_length) {
 				/* protocol error */
+				sbd_ats_remove_by_task(task);
 				stmf_abort(STMF_QUEUE_TASK_ABORT, task,
 				    STMF_INVALID_ARG, NULL);
 				return;
@@ -2521,7 +2706,8 @@
 		scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
 		task->task_lu_private = scmd;
 	}
-	scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_TRANS_DATA;
+	scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_TRANS_DATA |
+	    SBD_SCSI_CMD_ATS_RELATED;
 	scmd->cmd_type = SBD_CMD_SCSI_WRITE;
 	scmd->nbufs = 0;
 	scmd->len = (uint32_t)len;
@@ -2534,7 +2720,7 @@
 		 * Account for data passed in this write command
 		 */
 		(void) stmf_xfer_data(task, dbuf, STMF_IOF_STATS_ONLY);
-		scmd->len -= dbuf->db_data_size;
+		atomic_add_32(&scmd->len, -dbuf->db_data_size);
 		scmd->current_ro += dbuf->db_data_size;
 		dbuf->db_xfer_status = STMF_SUCCESS;
 		sbd_handle_write_same_xfer_completion(task, scmd, dbuf, 0);
@@ -2546,8 +2732,20 @@
 static void
 sbd_handle_unmap(scsi_task_t *task, stmf_data_buf_t *dbuf)
 {
+	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
 	uint32_t cmd_xfer_len;
 
+	if (sbd_unmap_enable == 0) {
+		stmf_scsilib_send_status(task, STATUS_CHECK,
+		    STMF_SAA_INVALID_OPCODE);
+		return;
+	}
+
+	if (sl->sl_flags & SL_WRITE_PROTECTED) {
+		stmf_scsilib_send_status(task, STATUS_CHECK,
+		    STMF_SAA_WRITE_PROTECTED);
+		return;
+	}
 	cmd_xfer_len = READ_SCSI16(&task->task_cdb[7], uint32_t);
 
 	if (task->task_cdb[1] & 1) {
@@ -2576,7 +2774,9 @@
 	uint32_t ulen, dlen, num_desc;
 	uint64_t addr, len;
 	uint8_t *p;
+	dkioc_free_list_t *dfl;
 	int ret;
+	int i;
 
 	if (buflen < 24) {
 		stmf_scsilib_send_status(task, STATUS_CHECK,
@@ -2593,20 +2793,40 @@
 		return;
 	}
 
-	for (p = buf + 8; num_desc; num_desc--, p += 16) {
+	dfl = kmem_zalloc(DFL_SZ(num_desc), KM_SLEEP);
+	dfl->dfl_num_exts = num_desc;
+	/*
+	 * This should use ATS locking but that was disabled by the
+	 * changes to ZFS top take advantage of TRIM in SSDs.
+	 *
+	 * Since the entire list is passed to ZFS in one list ATS
+	 * locking is not done.  This may be detectable, and if it is
+	 * then the entire list needs to be locked and then after the
+	 * unmap completes the entire list must be unlocked
+	 */
+	for (p = buf + 8, i = 0; num_desc; num_desc--, p += 16, i++) {
 		addr = READ_SCSI64(p, uint64_t);
-		addr <<= sl->sl_data_blocksize_shift;
 		len = READ_SCSI32(p+8, uint64_t);
+		addr <<= sl->sl_data_blocksize_shift;
 		len <<= sl->sl_data_blocksize_shift;
-		ret = sbd_unmap(sl, addr, len);
-		if (ret != 0) {
-			stmf_scsilib_send_status(task, STATUS_CHECK,
-			    STMF_SAA_LBA_OUT_OF_RANGE);
-			return;
-		}
+
+		/* Prepare a list of extents to unmap */
+		dfl->dfl_exts[i].dfle_start = addr;
+		dfl->dfl_exts[i].dfle_length = len;
+
+		/* release the overlap */
 	}
+	ASSERT(i == dfl->dfl_num_exts);
 
-unmap_done:
+	/* Finally execute the unmap operations in a single step */
+	ret = sbd_unmap(sl, dfl);
+	dfl_free(dfl);
+	if (ret != 0) {
+		stmf_scsilib_send_status(task, STATUS_CHECK,
+		    STMF_SAA_LBA_OUT_OF_RANGE);
+		return;
+	}
+
 	stmf_scsilib_send_status(task, STATUS_GOOD, 0);
 }
 
@@ -2677,6 +2897,7 @@
 
 		inq->inq_tpgs = TPGS_FAILOVER_IMPLICIT;
 		inq->inq_cmdque = 1;
+		inq->inq_3pc = 1;
 
 		if (sl->sl_flags & SL_VID_VALID) {
 			bcopy(sl->sl_vendor_id, inq->inq_vid, 8);
@@ -2775,13 +2996,15 @@
 
 	switch (cdbp[2]) {
 	case 0x00:
-		page_length = 4 + (mgmt_url_size ? 1 : 0);
+		page_length = 5 + (mgmt_url_size ? 1 : 0);
+
 		if (sl->sl_flags & SL_UNMAP_ENABLED)
-			page_length += 2;
+			page_length += 1;
 
 		p[0] = byte0;
 		p[3] = page_length;
 		/* Supported VPD pages in ascending order */
+		/* CSTYLED */
 		{
 			uint8_t i = 5;
 
@@ -2790,8 +3013,8 @@
 			if (mgmt_url_size != 0)
 				p[i++] = 0x85;
 			p[i++] = 0x86;
+			p[i++] = 0xb0;
 			if (sl->sl_flags & SL_UNMAP_ENABLED) {
-				p[i++] = 0xb0;
 				p[i++] = 0xb2;
 			}
 		}
@@ -2824,7 +3047,7 @@
 			stmf_scsilib_send_status(task, STATUS_CHECK,
 			    STMF_SAA_INVALID_FIELD_IN_CDB);
 			goto err_done;
-		}
+		} /* CSTYLED */
 		{
 			uint16_t idx, newidx, sz, url_size;
 			char *url;
@@ -2885,17 +3108,23 @@
 		break;
 
 	case 0xb0:
-		if ((sl->sl_flags & SL_UNMAP_ENABLED) == 0) {
-			stmf_scsilib_send_status(task, STATUS_CHECK,
-			    STMF_SAA_INVALID_FIELD_IN_CDB);
-			goto err_done;
-		}
 		page_length = 0x3c;
 		p[0] = byte0;
 		p[1] = 0xb0;
 		p[3] = page_length;
-		p[20] = p[21] = p[22] = p[23] = 0xFF;
-		p[24] = p[25] = p[26] = p[27] = 0xFF;
+		p[4] = 1;
+		p[5] = sbd_ats_max_nblks();
+		if (sl->sl_flags & SL_UNMAP_ENABLED && sbd_unmap_enable) {
+			p[20] = (stmf_sbd_unmap_max_nblks >> 24) & 0xff;
+			p[21] = (stmf_sbd_unmap_max_nblks >> 16) & 0xff;
+			p[22] = (stmf_sbd_unmap_max_nblks >> 8) & 0xff;
+			p[23] = stmf_sbd_unmap_max_nblks & 0xff;
+
+			p[24] = 0;
+			p[25] = 0;
+			p[26] = 0;
+			p[27] = 0xFF;
+		}
 		xfer_size = page_length + 4;
 		break;
 
@@ -2917,7 +3146,7 @@
 			exp++;
 		}
 		p[4] = exp;
-		p[5] = 0xc0;
+		p[5] = 0xc0;	/* Logical provisioning UNMAP and WRITE SAME */
 		xfer_size = page_length + 4;
 		break;
 
@@ -2938,7 +3167,7 @@
 sbd_task_alloc(struct scsi_task *task)
 {
 	if ((task->task_lu_private =
-	    kmem_alloc(sizeof (sbd_cmd_t), KM_NOSLEEP)) != NULL) {
+	    kmem_zalloc(sizeof (sbd_cmd_t), KM_NOSLEEP)) != NULL) {
 		sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
 		scmd->flags = 0;
 		return (STMF_SUCCESS);
@@ -3004,8 +3233,40 @@
 	mutex_exit(&sl->sl_lock);
 }
 
+/*
+ * Given a LU and a task, check if the task is causing reservation
+ * conflict. Returns 1 in case of conflict, 0 otherwise.
+ * Note that the LU might not be the same LU as in the task but the
+ * caller makes sure that the LU can be accessed.
+ */
+int
+sbd_check_reservation_conflict(struct sbd_lu *sl, struct scsi_task *task)
+{
+	sbd_it_data_t *it;
 
+	it = task->task_lu_itl_handle;
+	ASSERT(it);
+	if (sl->sl_access_state == SBD_LU_ACTIVE) {
+		if (SBD_PGR_RSVD(sl->sl_pgr)) {
+			if (sbd_pgr_reservation_conflict(task, sl)) {
+				return (1);
+			}
+		} else if ((sl->sl_flags & SL_LU_HAS_SCSI2_RESERVATION) &&
+		    ((it->sbd_it_flags & SBD_IT_HAS_SCSI2_RESERVATION) == 0)) {
+			if (!(SCSI2_CONFLICT_FREE_CMDS(task->task_cdb))) {
+				return (1);
+			}
+		}
+	}
 
+	return (0);
+}
+
+/*
+ * Keep in mind that sbd_new_task can be called multiple times for the same
+ * task because of us calling stmf_task_poll_lu resulting in a call to
+ * sbd_task_poll().
+ */
 void
 sbd_new_task(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
 {
@@ -3069,12 +3330,74 @@
 	if (sl->sl_access_state == SBD_LU_TRANSITION_TO_STANDBY ||
 	    sl->sl_access_state == SBD_LU_TRANSITION_TO_ACTIVE) {
 		stmf_scsilib_send_status(task, STATUS_CHECK,
-		    STMF_SAA_LU_NO_ACCESS_UNAVAIL);
+		    STMF_SAA_LU_NO_ACCESS_TRANSITION);
 		return;
 	}
 
-	/* Checking ua conditions as per SAM3R14 5.3.2 specified order */
-	if ((it->sbd_it_ua_conditions) && (task->task_cdb[0] != SCMD_INQUIRY)) {
+	cdb0 = task->task_cdb[0];
+	cdb1 = task->task_cdb[1];
+	/*
+	 * Special case for different versions of Windows.
+	 * 1) Windows 2012 and VMWare will fail to discover LU's if a READ
+	 *    operation sent down the standby path returns an error. By default
+	 *    standby_fail_reads will be set to 0.
+	 * 2) Windows 2008 R2 has a severe performace problem if READ ops
+	 *    aren't rejected on the standby path. 2008 sends commands
+	 *    down the standby path which then must be proxied over to the
+	 *    active node and back.
+	 */
+	if ((sl->sl_access_state == SBD_LU_STANDBY) &&
+	    stmf_standby_fail_reads &&
+	    (cdb0 == SCMD_READ || cdb0 == SCMD_READ_G1 ||
+	    cdb0 == SCMD_READ_G4 || cdb0 == SCMD_READ_G5)) {
+		stmf_scsilib_send_status(task, STATUS_CHECK,
+		    STMF_SAA_LU_NO_ACCESS_STANDBY);
+		return;
+	}
+
+	/*
+	 * Don't go further if cmd is unsupported in standby mode
+	 */
+	if (sl->sl_access_state == SBD_LU_STANDBY) {
+		if (cdb0 != SCMD_INQUIRY &&
+		    cdb0 != SCMD_MODE_SENSE &&
+		    cdb0 != SCMD_MODE_SENSE_G1 &&
+		    cdb0 != SCMD_MODE_SELECT &&
+		    cdb0 != SCMD_MODE_SELECT_G1 &&
+		    cdb0 != SCMD_RESERVE &&
+		    cdb0 != SCMD_RELEASE &&
+		    cdb0 != SCMD_PERSISTENT_RESERVE_OUT &&
+		    cdb0 != SCMD_PERSISTENT_RESERVE_IN &&
+		    cdb0 != SCMD_REQUEST_SENSE &&
+		    cdb0 != SCMD_READ_CAPACITY &&
+		    cdb0 != SCMD_TEST_UNIT_READY &&
+		    cdb0 != SCMD_START_STOP &&
+		    cdb0 != SCMD_READ &&
+		    cdb0 != SCMD_READ_G1 &&
+		    cdb0 != SCMD_READ_G4 &&
+		    cdb0 != SCMD_READ_G5 &&
+		    !(cdb0 == SCMD_SVC_ACTION_IN_G4 &&
+		    cdb1 == SSVC_ACTION_READ_CAPACITY_G4) &&
+		    !(cdb0 == SCMD_MAINTENANCE_IN &&
+		    (cdb1 & 0x1F) == 0x05) &&
+		    !(cdb0 == SCMD_MAINTENANCE_IN &&
+		    (cdb1 & 0x1F) == 0x0A)) {
+			stmf_scsilib_send_status(task, STATUS_CHECK,
+			    STMF_SAA_LU_NO_ACCESS_STANDBY);
+			return;
+		}
+	}
+
+	/*
+	 * Checking ua conditions as per SAM3R14 5.3.2 specified order. During
+	 * MPIO/ALUA failover, cmds come in through local ports and proxy port
+	 * port provider (i.e. pppt), we want to report unit attention to
+	 * only local cmds since initiators (Windows MPIO/DSM) would continue
+	 * sending I/O to the target that reported unit attention.
+	 */
+	if ((it->sbd_it_ua_conditions) &&
+	    !(task->task_additional_flags & TASK_AF_PPPT_TASK) &&
+	    (task->task_cdb[0] != SCMD_INQUIRY)) {
 		uint32_t saa = 0;
 
 		mutex_enter(&sl->sl_lock);
@@ -3081,6 +3404,11 @@
 		if (it->sbd_it_ua_conditions & SBD_UA_POR) {
 			it->sbd_it_ua_conditions &= ~SBD_UA_POR;
 			saa = STMF_SAA_POR;
+		} else if (it->sbd_it_ua_conditions &
+		    SBD_UA_ASYMMETRIC_ACCESS_CHANGED) {
+			it->sbd_it_ua_conditions &=
+			    ~SBD_UA_ASYMMETRIC_ACCESS_CHANGED;
+			saa = STMF_SAA_ASYMMETRIC_ACCESS_CHANGED;
 		}
 		mutex_exit(&sl->sl_lock);
 		if (saa) {
@@ -3090,21 +3418,10 @@
 	}
 
 	/* Reservation conflict checks */
-	if (sl->sl_access_state == SBD_LU_ACTIVE) {
-		if (SBD_PGR_RSVD(sl->sl_pgr)) {
-			if (sbd_pgr_reservation_conflict(task)) {
-				stmf_scsilib_send_status(task,
-				    STATUS_RESERVATION_CONFLICT, 0);
-				return;
-			}
-		} else if ((sl->sl_flags & SL_LU_HAS_SCSI2_RESERVATION) &&
-		    ((it->sbd_it_flags & SBD_IT_HAS_SCSI2_RESERVATION) == 0)) {
-			if (!(SCSI2_CONFLICT_FREE_CMDS(task->task_cdb))) {
-				stmf_scsilib_send_status(task,
-				    STATUS_RESERVATION_CONFLICT, 0);
-				return;
-			}
-		}
+	if (sbd_check_reservation_conflict(sl, task)) {
+		stmf_scsilib_send_status(task,
+		    STATUS_RESERVATION_CONFLICT, 0);
+		return;
 	}
 
 	/* Rest of the ua conndition checks */
@@ -3129,9 +3446,9 @@
 			saa = STMF_SAA_MODE_PARAMETERS_CHANGED;
 		} else if (it->sbd_it_ua_conditions &
 		    SBD_UA_ASYMMETRIC_ACCESS_CHANGED) {
-			it->sbd_it_ua_conditions &=
-			    ~SBD_UA_ASYMMETRIC_ACCESS_CHANGED;
-			saa = STMF_SAA_ASYMMETRIC_ACCESS_CHANGED;
+			saa = 0;
+		} else if (it->sbd_it_ua_conditions & SBD_UA_POR) {
+			saa = 0;
 		} else if (it->sbd_it_ua_conditions &
 		    SBD_UA_ACCESS_STATE_TRANSITION) {
 			it->sbd_it_ua_conditions &=
@@ -3148,38 +3465,7 @@
 		}
 	}
 
-	cdb0 = task->task_cdb[0];
-	cdb1 = task->task_cdb[1];
-
 	if (sl->sl_access_state == SBD_LU_STANDBY) {
-		if (cdb0 != SCMD_INQUIRY &&
-		    cdb0 != SCMD_MODE_SENSE &&
-		    cdb0 != SCMD_MODE_SENSE_G1 &&
-		    cdb0 != SCMD_MODE_SELECT &&
-		    cdb0 != SCMD_MODE_SELECT_G1 &&
-		    cdb0 != SCMD_RESERVE &&
-		    cdb0 != SCMD_RELEASE &&
-		    cdb0 != SCMD_PERSISTENT_RESERVE_OUT &&
-		    cdb0 != SCMD_PERSISTENT_RESERVE_IN &&
-		    cdb0 != SCMD_REQUEST_SENSE &&
-		    cdb0 != SCMD_READ_CAPACITY &&
-		    cdb0 != SCMD_TEST_UNIT_READY &&
-		    cdb0 != SCMD_START_STOP &&
-		    cdb0 != SCMD_READ &&
-		    cdb0 != SCMD_READ_G1 &&
-		    cdb0 != SCMD_READ_G4 &&
-		    cdb0 != SCMD_READ_G5 &&
-		    !(cdb0 == SCMD_SVC_ACTION_IN_G4 &&
-		    cdb1 == SSVC_ACTION_READ_CAPACITY_G4) &&
-		    !(cdb0 == SCMD_MAINTENANCE_IN &&
-		    (cdb1 & 0x1F) == 0x05) &&
-		    !(cdb0 == SCMD_MAINTENANCE_IN &&
-		    (cdb1 & 0x1F) == 0x0A)) {
-			stmf_scsilib_send_status(task, STATUS_CHECK,
-			    STMF_SAA_LU_NO_ACCESS_STANDBY);
-			return;
-		}
-
 		/*
 		 * is this a short write?
 		 * if so, we'll need to wait until we have the buffer
@@ -3357,6 +3643,21 @@
 		return;
 	}
 
+	if (cdb0 == SCMD_COMPARE_AND_WRITE) {
+		sbd_handle_ats(task, initial_dbuf);
+		return;
+	}
+
+	if (cdb0 == SCMD_EXTENDED_COPY) {
+		sbd_handle_xcopy(task, initial_dbuf);
+		return;
+	}
+
+	if (cdb0 == SCMD_RECV_COPY_RESULTS) {
+		sbd_handle_recv_copy_results(task, initial_dbuf);
+		return;
+	}
+
 	if (cdb0 == SCMD_TEST_UNIT_READY) {	/* Test unit ready */
 		task->task_cmd_xfer_length = 0;
 		stmf_scsilib_send_status(task, STATUS_GOOD, 0);
@@ -3463,12 +3764,18 @@
 		break;
 
 	case (SBD_CMD_SCSI_WRITE):
-		if ((task->task_cdb[0] == SCMD_WRITE_SAME_G1) ||
-		    (task->task_cdb[0] == SCMD_WRITE_SAME_G4)) {
+		switch (task->task_cdb[0]) {
+		case SCMD_WRITE_SAME_G1:
+		case SCMD_WRITE_SAME_G4:
 			sbd_handle_write_same_xfer_completion(task, scmd, dbuf,
 			    1);
-		} else {
+			break;
+		case SCMD_COMPARE_AND_WRITE:
+			sbd_handle_ats_xfer_completion(task, scmd, dbuf, 1);
+			break;
+		default:
 			sbd_handle_write_xfer_completion(task, scmd, dbuf, 1);
+			/* FALLTHRU */
 		}
 		break;
 
@@ -3534,6 +3841,7 @@
 
 	ASSERT(abort_cmd == STMF_LU_ABORT_TASK);
 	task = (scsi_task_t *)arg;
+	sbd_ats_remove_by_task(task);
 	if (task->task_lu_private) {
 		sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
 
@@ -3551,6 +3859,15 @@
 	return (STMF_NOT_FOUND);
 }
 
+void
+sbd_task_poll(struct scsi_task *task)
+{
+	stmf_data_buf_t *initial_dbuf;
+
+	initial_dbuf = stmf_handle_to_buf(task, 0);
+	sbd_new_task(task, initial_dbuf);
+}
+
 /*
  * This function is called during task clean-up if the
  * DB_LU_FLAG is set on the dbuf. This should only be called for
@@ -3563,7 +3880,7 @@
 	sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
 
 	ASSERT(dbuf->db_lu_private);
-	ASSERT(scmd && scmd->nbufs > 0);
+	ASSERT(scmd && ATOMIC8_GET(scmd->nbufs) > 0);
 	ASSERT((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0);
 	ASSERT(dbuf->db_flags & DB_LU_DATA_BUF);
 	ASSERT(task->task_additional_flags & TASK_AF_ACCEPT_LU_DBUF);
@@ -3577,7 +3894,7 @@
 		cmn_err(CE_PANIC, "Unknown cmd type %d, task = %p",
 		    scmd->cmd_type, (void *)task);
 	}
-	if (--scmd->nbufs == 0)
+	if (atomic_dec_8_nv(&scmd->nbufs) == 0)
 		rw_exit(&sl->sl_access_state_lock);
 	stmf_teardown_dbuf(task, dbuf);
 	stmf_free(dbuf);
@@ -3674,13 +3991,20 @@
 sbd_flush_data_cache(sbd_lu_t *sl, int fsync_done)
 {
 	int r = 0;
-	int ret;
+	sbd_status_t ret;
 
+	rw_enter(&sl->sl_access_state_lock, RW_READER);
+	if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
+		ret = SBD_FILEIO_FAILURE;
+		goto flush_fail;
+	}
 	if (fsync_done)
 		goto over_fsync;
 	if ((sl->sl_data_vtype == VREG) || (sl->sl_data_vtype == VBLK)) {
-		if (VOP_FSYNC(sl->sl_data_vp, FSYNC, kcred, NULL))
-			return (SBD_FAILURE);
+		if (VOP_FSYNC(sl->sl_data_vp, FSYNC, kcred, NULL)) {
+			ret = SBD_FAILURE;
+			goto flush_fail;
+		}
 	}
 over_fsync:
 	if (((sl->sl_data_vtype == VCHR) || (sl->sl_data_vtype == VBLK)) &&
@@ -3691,12 +4015,14 @@
 			mutex_enter(&sl->sl_lock);
 			sl->sl_flags |= SL_NO_DATA_DKIOFLUSH;
 			mutex_exit(&sl->sl_lock);
-		} else if (ret != 0) {
-			return (SBD_FAILURE);
+		} else {
+			ret = (ret != 0) ? SBD_FAILURE : SBD_SUCCESS;
 		}
 	}
+flush_fail:
+	rw_exit(&sl->sl_access_state_lock);
 
-	return (SBD_SUCCESS);
+	return (ret);
 }
 
 /* ARGSUSED */
