1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
24 * Copyright (c) 2013 by Delphix. All rights reserved.
25 */
26
27 #include <sys/conf.h>
28 #include <sys/file.h>
29 #include <sys/ddi.h>
30 #include <sys/sunddi.h>
31 #include <sys/modctl.h>
32 #include <sys/scsi/scsi.h>
33 #include <sys/scsi/impl/scsi_reset_notify.h>
34 #include <sys/scsi/generic/mode.h>
35 #include <sys/disp.h>
36 #include <sys/byteorder.h>
37 #include <sys/atomic.h>
38 #include <sys/sdt.h>
39 #include <sys/dkio.h>
40
41 #include <sys/stmf.h>
42 #include <sys/lpif.h>
43 #include <sys/portif.h>
44 #include <sys/stmf_ioctl.h>
45 #include <sys/stmf_sbd_ioctl.h>
46
47 #include "stmf_sbd.h"
48 #include "sbd_impl.h"
49
50 #define SCSI2_CONFLICT_FREE_CMDS(cdb) ( \
51 /* ----------------------- */ \
52 /* Refer Both */ \
53 /* SPC-2 (rev 20) Table 10 */ \
54 /* SPC-3 (rev 23) Table 31 */ \
55 /* ----------------------- */ \
56 ((cdb[0]) == SCMD_INQUIRY) || \
57 ((cdb[0]) == SCMD_LOG_SENSE_G1) || \
58 ((cdb[0]) == SCMD_RELEASE) || \
59 ((cdb[0]) == SCMD_RELEASE_G1) || \
69 /* REPORT TARGET PORT GROUPS (0x0A) REPORT TIMESTAMP (0x0F) */ \
70 (((cdb[0]) == SCMD_MAINTENANCE_IN) && ( \
71 (((cdb[1]) & 0x1F) == 0x0B) || \
72 (((cdb[1]) & 0x1F) == 0x05) || \
73 (((cdb[1]) & 0x1F) == 0x0E) || \
74 (((cdb[1]) & 0x1F) == 0x0A) || \
75 (((cdb[1]) & 0x1F) == 0x0F))) || \
76 /* ----------------------- */ \
77 /* SBC-3 (rev 17) Table 3 */ \
78 /* ----------------------- */ \
79 /* READ CAPACITY(10) */ \
80 ((cdb[0]) == SCMD_READ_CAPACITY) || \
81 /* READ CAPACITY(16) */ \
82 (((cdb[0]) == SCMD_SVC_ACTION_IN_G4) && ( \
83 ((cdb[1]) & 0x1F) == 0x10)) || \
84 /* START STOP UNIT with START bit 0 and POWER CONDITION 0 */ \
85 (((cdb[0]) == SCMD_START_STOP) && ( \
86 (((cdb[4]) & 0xF0) == 0) && (((cdb[4]) & 0x01) == 0))))
87 /* End of SCSI2_CONFLICT_FREE_CMDS */
88
89 stmf_status_t sbd_lu_reset_state(stmf_lu_t *lu);
90 static void sbd_handle_sync_cache(struct scsi_task *task,
91 struct stmf_data_buf *initial_dbuf);
92 void sbd_handle_read_xfer_completion(struct scsi_task *task,
93 sbd_cmd_t *scmd, struct stmf_data_buf *dbuf);
94 void sbd_handle_short_write_xfer_completion(scsi_task_t *task,
95 stmf_data_buf_t *dbuf);
96 void sbd_handle_short_write_transfers(scsi_task_t *task,
97 stmf_data_buf_t *dbuf, uint32_t cdb_xfer_size);
98 void sbd_handle_mode_select_xfer(scsi_task_t *task, uint8_t *buf,
99 uint32_t buflen);
100 void sbd_handle_mode_select(scsi_task_t *task, stmf_data_buf_t *dbuf);
101 void sbd_handle_identifying_info(scsi_task_t *task, stmf_data_buf_t *dbuf);
102
103 static void sbd_handle_unmap_xfer(scsi_task_t *task, uint8_t *buf,
104 uint32_t buflen);
105 static void sbd_handle_unmap(scsi_task_t *task, stmf_data_buf_t *dbuf);
106
107 extern void sbd_pgr_initialize_it(scsi_task_t *, sbd_it_data_t *);
108 extern int sbd_pgr_reservation_conflict(scsi_task_t *);
109 extern void sbd_pgr_reset(sbd_lu_t *);
110 extern void sbd_pgr_remove_it_handle(sbd_lu_t *, sbd_it_data_t *);
111 extern void sbd_handle_pgr_in_cmd(scsi_task_t *, stmf_data_buf_t *);
112 extern void sbd_handle_pgr_out_cmd(scsi_task_t *, stmf_data_buf_t *);
113 extern void sbd_handle_pgr_out_data(scsi_task_t *, stmf_data_buf_t *);
114 void sbd_do_sgl_write_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
115 int first_xfer);
116 static void sbd_handle_write_same(scsi_task_t *task,
117 struct stmf_data_buf *initial_dbuf);
118 static void sbd_do_write_same_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
119 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable);
120 static void sbd_handle_write_same_xfer_completion(struct scsi_task *task,
121 sbd_cmd_t *scmd, struct stmf_data_buf *dbuf, uint8_t dbuf_reusable);
122 /*
123 * IMPORTANT NOTE:
124 * =================
125 * The whole world here is based on the assumption that everything within
126 * a scsi task executes in a single threaded manner, even the aborts.
127 * Dont ever change that. There wont be any performance gain but there
128 * will be tons of race conditions.
129 */
130
131 void
132 sbd_do_read_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
133 struct stmf_data_buf *dbuf)
134 {
135 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
136 uint64_t laddr;
137 uint32_t len, buflen, iolen;
138 int ndx;
139 int bufs_to_take;
140
141 /* Lets try not to hog all the buffers the port has. */
142 bufs_to_take = ((task->task_max_nbufs > 2) &&
143 (task->task_cmd_xfer_length < (32 * 1024))) ? 2 :
144 task->task_max_nbufs;
145
146 len = scmd->len > dbuf->db_buf_size ? dbuf->db_buf_size : scmd->len;
147 laddr = scmd->addr + scmd->current_ro;
148
149 for (buflen = 0, ndx = 0; (buflen < len) &&
150 (ndx < dbuf->db_sglist_length); ndx++) {
151 iolen = min(len - buflen, dbuf->db_sglist[ndx].seg_length);
152 if (iolen == 0)
153 break;
154 if (sbd_data_read(sl, task, laddr, (uint64_t)iolen,
155 dbuf->db_sglist[ndx].seg_addr) != STMF_SUCCESS) {
156 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
157 /* Do not need to do xfer anymore, just complete it */
158 dbuf->db_data_size = 0;
159 dbuf->db_xfer_status = STMF_SUCCESS;
160 sbd_handle_read_xfer_completion(task, scmd, dbuf);
161 return;
162 }
163 buflen += iolen;
164 laddr += (uint64_t)iolen;
165 }
166 dbuf->db_relative_offset = scmd->current_ro;
167 dbuf->db_data_size = buflen;
168 dbuf->db_flags = DB_DIRECTION_TO_RPORT;
169 (void) stmf_xfer_data(task, dbuf, 0);
170 scmd->len -= buflen;
171 scmd->current_ro += buflen;
172 if (scmd->len && (scmd->nbufs < bufs_to_take)) {
173 uint32_t maxsize, minsize, old_minsize;
174
175 maxsize = (scmd->len > (128*1024)) ? 128*1024 : scmd->len;
176 minsize = maxsize >> 2;
177 do {
178 /*
179 * A bad port implementation can keep on failing the
180 * the request but keep on sending us a false
181 * minsize.
182 */
183 old_minsize = minsize;
184 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
185 } while ((dbuf == NULL) && (old_minsize > minsize) &&
186 (minsize >= 512));
187 if (dbuf == NULL) {
188 return;
189 }
190 scmd->nbufs++;
191 sbd_do_read_xfer(task, scmd, dbuf);
192 }
193 }
194
195 /*
196 * sbd_zcopy: Bail-out switch for reduced copy path.
197 *
198 * 0 - read & write off
199 * 1 - read & write on
200 * 2 - only read on
201 * 4 - only write on
202 */
203 int sbd_zcopy = 1; /* enable zcopy read & write path */
204 uint32_t sbd_max_xfer_len = 0; /* Valid if non-zero */
205 uint32_t sbd_1st_xfer_len = 0; /* Valid if non-zero */
206 uint32_t sbd_copy_threshold = 0; /* Valid if non-zero */
207
208 static void
209 sbd_do_sgl_read_xfer(struct scsi_task *task, sbd_cmd_t *scmd, int first_xfer)
210 {
211 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
212 sbd_zvol_io_t *zvio;
213 int ret, final_xfer;
214 uint64_t offset;
215 uint32_t xfer_len, max_len, first_len;
216 stmf_status_t xstat;
217 stmf_data_buf_t *dbuf;
218 uint_t nblks;
219 uint64_t blksize = sl->sl_blksize;
220 size_t db_private_sz;
221 uintptr_t pad;
222
223 ASSERT(rw_read_held(&sl->sl_access_state_lock));
224 ASSERT((sl->sl_flags & SL_MEDIA_LOADED) != 0);
225
226 /*
227 * Calculate the limits on xfer_len to the minimum of :
228 * - task limit
229 * - lun limit
230 * - sbd global limit if set
231 * - first xfer limit if set
232 *
233 * First, protect against silly over-ride value
234 */
235 if (sbd_max_xfer_len && ((sbd_max_xfer_len % DEV_BSIZE) != 0)) {
236 cmn_err(CE_WARN, "sbd_max_xfer_len invalid %d, resetting\n",
237 sbd_max_xfer_len);
238 sbd_max_xfer_len = 0;
239 }
240 if (sbd_1st_xfer_len && ((sbd_1st_xfer_len % DEV_BSIZE) != 0)) {
242 sbd_1st_xfer_len);
243 sbd_1st_xfer_len = 0;
244 }
245
246 max_len = MIN(task->task_max_xfer_len, sl->sl_max_xfer_len);
247 if (sbd_max_xfer_len)
248 max_len = MIN(max_len, sbd_max_xfer_len);
249 /*
250 * Special case the first xfer if hints are set.
251 */
252 if (first_xfer && (sbd_1st_xfer_len || task->task_1st_xfer_len)) {
253 /* global over-ride has precedence */
254 if (sbd_1st_xfer_len)
255 first_len = sbd_1st_xfer_len;
256 else
257 first_len = task->task_1st_xfer_len;
258 } else {
259 first_len = 0;
260 }
261
262 while (scmd->len && scmd->nbufs < task->task_max_nbufs) {
263
264 xfer_len = MIN(max_len, scmd->len);
265 if (first_len) {
266 xfer_len = MIN(xfer_len, first_len);
267 first_len = 0;
268 }
269 if (scmd->len == xfer_len) {
270 final_xfer = 1;
271 } else {
272 /*
273 * Attempt to end xfer on a block boundary.
274 * The only way this does not happen is if the
275 * xfer_len is small enough to stay contained
276 * within the same block.
277 */
278 uint64_t xfer_offset, xfer_aligned_end;
279
280 final_xfer = 0;
281 xfer_offset = scmd->addr + scmd->current_ro;
282 xfer_aligned_end =
283 P2ALIGN(xfer_offset+xfer_len, blksize);
284 if (xfer_aligned_end > xfer_offset)
285 xfer_len = xfer_aligned_end - xfer_offset;
286 }
287 /*
288 * Allocate object to track the read and reserve
289 * enough space for scatter/gather list.
316 dbuf->db_data_size = xfer_len;
317 dbuf->db_relative_offset = scmd->current_ro;
318 dbuf->db_sglist_length = (uint16_t)nblks;
319 dbuf->db_xfer_status = 0;
320 dbuf->db_handle = 0;
321
322 dbuf->db_flags = (DB_DONT_CACHE | DB_DONT_REUSE |
323 DB_DIRECTION_TO_RPORT | DB_LU_DATA_BUF);
324 if (final_xfer)
325 dbuf->db_flags |= DB_SEND_STATUS_GOOD;
326
327 zvio = dbuf->db_lu_private;
328 /* Need absolute offset for zvol access */
329 zvio->zvio_offset = offset;
330 zvio->zvio_flags = ZVIO_SYNC;
331
332 /*
333 * Accounting for start of read.
334 * Note there is no buffer address for the probe yet.
335 */
336 DTRACE_PROBE5(backing__store__read__start, sbd_lu_t *, sl,
337 uint8_t *, NULL, uint64_t, xfer_len,
338 uint64_t, offset, scsi_task_t *, task);
339
340 ret = sbd_zvol_alloc_read_bufs(sl, dbuf);
341
342 DTRACE_PROBE6(backing__store__read__end, sbd_lu_t *, sl,
343 uint8_t *, NULL, uint64_t, xfer_len,
344 uint64_t, offset, int, ret, scsi_task_t *, task);
345
346 if (ret != 0) {
347 /*
348 * Read failure from the backend.
349 */
350 stmf_free(dbuf);
351 if (scmd->nbufs == 0) {
352 /* nothing queued, just finish */
353 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
354 stmf_scsilib_send_status(task, STATUS_CHECK,
355 STMF_SAA_READ_ERROR);
356 rw_exit(&sl->sl_access_state_lock);
357 } else {
358 /* process failure when other dbufs finish */
359 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
360 }
361 return;
362 }
363
364
365 /*
366 * Allow PP to do setup
367 */
368 xstat = stmf_setup_dbuf(task, dbuf, 0);
369 if (xstat != STMF_SUCCESS) {
370 /*
371 * This could happen if the driver cannot get the
372 * DDI resources it needs for this request.
373 * If other dbufs are queued, try again when the next
374 * one completes, otherwise give up.
375 */
376 sbd_zvol_rele_read_bufs(sl, dbuf);
377 stmf_free(dbuf);
378 if (scmd->nbufs > 0) {
379 /* completion of previous dbuf will retry */
380 return;
381 }
382 /*
383 * Done with this command.
384 */
385 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
386 if (first_xfer)
387 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
388 else
389 stmf_scsilib_send_status(task, STATUS_CHECK,
390 STMF_SAA_READ_ERROR);
391 rw_exit(&sl->sl_access_state_lock);
392 return;
393 }
394 /*
395 * dbuf is now queued on task
396 */
397 scmd->nbufs++;
398
399 /* XXX leave this in for FW? */
400 DTRACE_PROBE4(sbd__xfer, struct scsi_task *, task,
401 struct stmf_data_buf *, dbuf, uint64_t, offset,
402 uint32_t, xfer_len);
403 /*
404 * Do not pass STMF_IOF_LU_DONE so that the zvol
405 * state can be released in the completion callback.
406 */
407 xstat = stmf_xfer_data(task, dbuf, 0);
408 switch (xstat) {
409 case STMF_SUCCESS:
410 break;
411 case STMF_BUSY:
412 /*
413 * The dbuf is queued on the task, but unknown
414 * to the PP, thus no completion will occur.
415 */
416 sbd_zvol_rele_read_bufs(sl, dbuf);
417 stmf_teardown_dbuf(task, dbuf);
418 stmf_free(dbuf);
419 scmd->nbufs--;
420 if (scmd->nbufs > 0) {
421 /* completion of previous dbuf will retry */
422 return;
423 }
424 /*
425 * Done with this command.
426 */
427 rw_exit(&sl->sl_access_state_lock);
428 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
429 if (first_xfer)
430 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
431 else
432 stmf_scsilib_send_status(task, STATUS_CHECK,
433 STMF_SAA_READ_ERROR);
434 return;
435 case STMF_ABORTED:
436 /*
437 * Completion from task_done will cleanup
438 */
439 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
440 return;
441 }
442 /*
443 * Update the xfer progress.
444 */
445 ASSERT(scmd->len >= xfer_len);
446 scmd->len -= xfer_len;
447 scmd->current_ro += xfer_len;
448 }
449 }
450
451 void
452 sbd_handle_read_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
453 struct stmf_data_buf *dbuf)
454 {
455 if (dbuf->db_xfer_status != STMF_SUCCESS) {
456 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
457 dbuf->db_xfer_status, NULL);
458 return;
459 }
460 task->task_nbytes_transferred += dbuf->db_data_size;
461 if (scmd->len == 0 || scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
462 stmf_free_dbuf(task, dbuf);
463 scmd->nbufs--;
464 if (scmd->nbufs)
465 return; /* wait for all buffers to complete */
466 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
467 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL)
468 stmf_scsilib_send_status(task, STATUS_CHECK,
469 STMF_SAA_READ_ERROR);
470 else
471 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
472 return;
473 }
474 if (dbuf->db_flags & DB_DONT_REUSE) {
475 /* allocate new dbuf */
476 uint32_t maxsize, minsize, old_minsize;
477 stmf_free_dbuf(task, dbuf);
478
479 maxsize = (scmd->len > (128*1024)) ? 128*1024 : scmd->len;
480 minsize = maxsize >> 2;
481 do {
482 old_minsize = minsize;
483 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
484 } while ((dbuf == NULL) && (old_minsize > minsize) &&
485 (minsize >= 512));
486 if (dbuf == NULL) {
487 scmd->nbufs --;
488 if (scmd->nbufs == 0) {
489 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
490 STMF_ALLOC_FAILURE, NULL);
491 }
492 return;
493 }
494 }
495 sbd_do_read_xfer(task, scmd, dbuf);
496 }
497
498 /*
499 * This routine must release the DMU resources and free the dbuf
500 * in all cases. If this is the final dbuf of the task, then drop
501 * the reader lock on the LU state. If there are no errors and more
502 * work to do, then queue more xfer operations.
503 */
504 void
505 sbd_handle_sgl_read_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
506 struct stmf_data_buf *dbuf)
507 {
508 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
509 stmf_status_t xfer_status;
510 uint32_t data_size;
511 int scmd_err;
512
513 ASSERT(dbuf->db_lu_private);
514 ASSERT(scmd->cmd_type == SBD_CMD_SCSI_READ);
515
516 scmd->nbufs--; /* account for this dbuf */
517 /*
518 * Release the DMU resources.
519 */
520 sbd_zvol_rele_read_bufs(sl, dbuf);
521 /*
522 * Release the dbuf after retrieving needed fields.
523 */
524 xfer_status = dbuf->db_xfer_status;
525 data_size = dbuf->db_data_size;
526 stmf_teardown_dbuf(task, dbuf);
527 stmf_free(dbuf);
528 /*
529 * Release the state lock if this is the last completion.
530 * If this is the last dbuf on task and all data has been
531 * transferred or an error encountered, then no more dbufs
532 * will be queued.
533 */
534 scmd_err = (((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
535 (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) ||
536 (xfer_status != STMF_SUCCESS));
537 if (scmd->nbufs == 0 && (scmd->len == 0 || scmd_err)) {
538 /* all DMU state has been released */
539 rw_exit(&sl->sl_access_state_lock);
540 }
541
542 /*
543 * If there have been no errors, either complete the task
544 * or issue more data xfer operations.
545 */
546 if (!scmd_err) {
547 /*
548 * This chunk completed successfully
549 */
550 task->task_nbytes_transferred += data_size;
551 if (scmd->nbufs == 0 && scmd->len == 0) {
552 /*
553 * This command completed successfully
554 *
555 * Status was sent along with data, so no status
556 * completion will occur. Tell stmf we are done.
557 */
558 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
559 stmf_task_lu_done(task);
560 return;
561 }
562 /*
563 * Start more xfers
564 */
565 sbd_do_sgl_read_xfer(task, scmd, 0);
566 return;
567 }
568 /*
569 * Sort out the failure
570 */
571 if (scmd->flags & SBD_SCSI_CMD_ACTIVE) {
572 /*
573 * If a previous error occurred, leave the command active
574 * and wait for the last completion to send the status check.
575 */
576 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
577 if (scmd->nbufs == 0) {
578 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
579 stmf_scsilib_send_status(task, STATUS_CHECK,
580 STMF_SAA_READ_ERROR);
581 }
582 return;
583 }
584 /*
585 * Must have been a failure on current dbuf
586 */
587 ASSERT(xfer_status != STMF_SUCCESS);
588 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
589 stmf_abort(STMF_QUEUE_TASK_ABORT, task, xfer_status, NULL);
590 }
591 }
592
593 void
594 sbd_handle_sgl_write_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
595 struct stmf_data_buf *dbuf)
596 {
597 sbd_zvol_io_t *zvio = dbuf->db_lu_private;
598 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
599 int ret;
600 int scmd_err, scmd_xfer_done;
601 stmf_status_t xfer_status = dbuf->db_xfer_status;
602 uint32_t data_size = dbuf->db_data_size;
603
604 ASSERT(zvio);
605
606 /*
607 * Allow PP to free up resources before releasing the write bufs
608 * as writing to the backend could take some time.
609 */
610 stmf_teardown_dbuf(task, dbuf);
611
612 scmd->nbufs--; /* account for this dbuf */
613 /*
614 * All data was queued and this is the last completion,
615 * but there could still be an error.
616 */
617 scmd_xfer_done = (scmd->len == 0 && scmd->nbufs == 0);
618 scmd_err = (((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
619 (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) ||
620 (xfer_status != STMF_SUCCESS));
621
622 DTRACE_PROBE5(backing__store__write__start, sbd_lu_t *, sl,
623 uint8_t *, NULL, uint64_t, data_size,
624 uint64_t, zvio->zvio_offset, scsi_task_t *, task);
625
626 if (scmd_err) {
627 /* just return the write buffers */
628 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
629 ret = 0;
630 } else {
631 if (scmd_xfer_done)
632 zvio->zvio_flags = ZVIO_COMMIT;
633 else
634 zvio->zvio_flags = 0;
635 /* write the data */
636 ret = sbd_zvol_rele_write_bufs(sl, dbuf);
637 }
638
639 DTRACE_PROBE6(backing__store__write__end, sbd_lu_t *, sl,
640 uint8_t *, NULL, uint64_t, data_size,
641 uint64_t, zvio->zvio_offset, int, ret, scsi_task_t *, task);
642
643 if (ret != 0) {
644 /* update the error flag */
645 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
646 scmd_err = 1;
647 }
648
649 /* Release the dbuf */
650 stmf_free(dbuf);
651
652 /*
653 * Release the state lock if this is the last completion.
654 * If this is the last dbuf on task and all data has been
655 * transferred or an error encountered, then no more dbufs
656 * will be queued.
657 */
658 if (scmd->nbufs == 0 && (scmd->len == 0 || scmd_err)) {
659 /* all DMU state has been released */
660 rw_exit(&sl->sl_access_state_lock);
661 }
662 /*
663 * If there have been no errors, either complete the task
664 * or issue more data xfer operations.
665 */
666 if (!scmd_err) {
667 /* This chunk completed successfully */
668 task->task_nbytes_transferred += data_size;
669 if (scmd_xfer_done) {
670 /* This command completed successfully */
671 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
672 if ((scmd->flags & SBD_SCSI_CMD_SYNC_WRITE) &&
673 (sbd_flush_data_cache(sl, 0) != SBD_SUCCESS)) {
674 stmf_scsilib_send_status(task, STATUS_CHECK,
675 STMF_SAA_WRITE_ERROR);
676 } else {
677 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
678 }
679 return;
680 }
681 /*
682 * Start more xfers
683 */
684 sbd_do_sgl_write_xfer(task, scmd, 0);
685 return;
686 }
687 /*
688 * Sort out the failure
689 */
690 if (scmd->flags & SBD_SCSI_CMD_ACTIVE) {
691 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
692 if (scmd->nbufs == 0) {
693 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
694 stmf_scsilib_send_status(task, STATUS_CHECK,
695 STMF_SAA_WRITE_ERROR);
696 }
697 /*
698 * Leave the command active until last dbuf completes.
699 */
700 return;
701 }
702 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
703 ASSERT(xfer_status != STMF_SUCCESS);
704 stmf_abort(STMF_QUEUE_TASK_ABORT, task, xfer_status, NULL);
705 }
706 }
707
708 /*
709 * Handle a copy operation using the zvol interface.
710 *
711 * Similar to the sbd_data_read/write path, except it goes directly through
712 * the zvol interfaces. It can pass a port provider sglist in the
713 * form of uio which is lost through the vn_rdwr path.
714 *
715 * Returns:
716 * STMF_SUCCESS - request handled
717 * STMF_FAILURE - request not handled, caller must deal with error
718 */
719 static stmf_status_t
720 sbd_copy_rdwr(scsi_task_t *task, uint64_t laddr, stmf_data_buf_t *dbuf,
721 int cmd, int commit)
722 {
723 sbd_lu_t *sl = task->task_lu->lu_provider_private;
724 struct uio uio;
725 struct iovec *iov, *tiov, iov1[8];
726 uint32_t len, resid;
727 int ret, i, iovcnt, flags;
728 boolean_t is_read;
729
730 ASSERT(cmd == SBD_CMD_SCSI_READ || cmd == SBD_CMD_SCSI_WRITE);
731
732 is_read = (cmd == SBD_CMD_SCSI_READ) ? B_TRUE : B_FALSE;
733 iovcnt = dbuf->db_sglist_length;
734 /* use the stack for small iovecs */
735 if (iovcnt > 8) {
736 iov = kmem_alloc(iovcnt * sizeof (*iov), KM_SLEEP);
737 } else {
738 iov = &iov1[0];
739 }
740
741 /* Convert dbuf sglist to iovec format */
742 len = dbuf->db_data_size;
743 resid = len;
744 tiov = iov;
745 for (i = 0; i < iovcnt; i++) {
746 tiov->iov_base = (caddr_t)dbuf->db_sglist[i].seg_addr;
747 tiov->iov_len = MIN(resid, dbuf->db_sglist[i].seg_length);
748 resid -= tiov->iov_len;
749 tiov++;
750 }
751 if (resid != 0) {
752 cmn_err(CE_WARN, "inconsistant sglist rem %d", resid);
753 if (iov != &iov1[0])
754 kmem_free(iov, iovcnt * sizeof (*iov));
755 return (STMF_FAILURE);
756 }
757 /* Setup the uio struct */
758 uio.uio_iov = iov;
759 uio.uio_iovcnt = iovcnt;
760 uio.uio_loffset = laddr;
761 uio.uio_segflg = (short)UIO_SYSSPACE;
762 uio.uio_resid = (uint64_t)len;
763 uio.uio_llimit = RLIM64_INFINITY;
764
765 if (is_read == B_TRUE) {
766 uio.uio_fmode = FREAD;
767 uio.uio_extflg = UIO_COPY_CACHED;
768 DTRACE_PROBE5(backing__store__read__start, sbd_lu_t *, sl,
769 uint8_t *, NULL, uint64_t, len, uint64_t, laddr,
770 scsi_task_t *, task);
771
772 /* Fetch the data */
773 ret = sbd_zvol_copy_read(sl, &uio);
774
775 DTRACE_PROBE6(backing__store__read__end, sbd_lu_t *, sl,
776 uint8_t *, NULL, uint64_t, len, uint64_t, laddr, int, ret,
777 scsi_task_t *, task);
778 } else {
779 uio.uio_fmode = FWRITE;
780 uio.uio_extflg = UIO_COPY_DEFAULT;
781 DTRACE_PROBE5(backing__store__write__start, sbd_lu_t *, sl,
782 uint8_t *, NULL, uint64_t, len, uint64_t, laddr,
783 scsi_task_t *, task);
784
785 flags = (commit) ? ZVIO_COMMIT : 0;
786 /* Write the data */
787 ret = sbd_zvol_copy_write(sl, &uio, flags);
788
789 DTRACE_PROBE6(backing__store__write__end, sbd_lu_t *, sl,
790 uint8_t *, NULL, uint64_t, len, uint64_t, laddr, int, ret,
791 scsi_task_t *, task);
792 }
793
794 if (iov != &iov1[0])
795 kmem_free(iov, iovcnt * sizeof (*iov));
796 if (ret != 0) {
797 /* Backend I/O error */
798 return (STMF_FAILURE);
799 }
800 return (STMF_SUCCESS);
801 }
802
803 void
804 sbd_handle_read(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
805 {
806 uint64_t lba, laddr;
807 uint32_t len;
808 uint8_t op = task->task_cdb[0];
809 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
810 sbd_cmd_t *scmd;
811 stmf_data_buf_t *dbuf;
812 int fast_path;
813
814 if (op == SCMD_READ) {
815 lba = READ_SCSI21(&task->task_cdb[1], uint64_t);
816 len = (uint32_t)task->task_cdb[4];
817
818 if (len == 0) {
819 len = 256;
820 }
821 } else if (op == SCMD_READ_G1) {
822 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
823 len = READ_SCSI16(&task->task_cdb[7], uint32_t);
824 } else if (op == SCMD_READ_G5) {
825 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
826 len = READ_SCSI32(&task->task_cdb[6], uint32_t);
827 } else if (op == SCMD_READ_G4) {
828 lba = READ_SCSI64(&task->task_cdb[2], uint64_t);
829 len = READ_SCSI32(&task->task_cdb[10], uint32_t);
830 } else {
831 stmf_scsilib_send_status(task, STATUS_CHECK,
832 STMF_SAA_INVALID_OPCODE);
833 return;
834 }
835
836 laddr = lba << sl->sl_data_blocksize_shift;
837 len <<= sl->sl_data_blocksize_shift;
838
839 if ((laddr + (uint64_t)len) > sl->sl_lu_size) {
840 stmf_scsilib_send_status(task, STATUS_CHECK,
841 STMF_SAA_LBA_OUT_OF_RANGE);
842 return;
843 }
844
845 task->task_cmd_xfer_length = len;
846 if (task->task_additional_flags & TASK_AF_NO_EXPECTED_XFER_LENGTH) {
847 task->task_expected_xfer_length = len;
848 }
849
850 if (len != task->task_expected_xfer_length) {
851 fast_path = 0;
852 len = (len > task->task_expected_xfer_length) ?
853 task->task_expected_xfer_length : len;
854 } else {
855 fast_path = 1;
856 }
857
858 if (len == 0) {
859 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
860 return;
861 }
862
863 /*
864 * Determine if this read can directly use DMU buffers.
865 */
866 if (sbd_zcopy & (2|1) && /* Debug switch */
867 initial_dbuf == NULL && /* No PP buffer passed in */
868 sl->sl_flags & SL_CALL_ZVOL && /* zvol backing store */
869 (task->task_additional_flags &
870 TASK_AF_ACCEPT_LU_DBUF)) /* PP allows it */
871 {
872 /*
873 * Reduced copy path
874 */
875 uint32_t copy_threshold, minsize;
876 int ret;
877
878 /*
879 * The sl_access_state_lock will be held shared
880 * for the entire request and released when all
881 * dbufs have completed.
882 */
883 rw_enter(&sl->sl_access_state_lock, RW_READER);
884 if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
885 rw_exit(&sl->sl_access_state_lock);
886 stmf_scsilib_send_status(task, STATUS_CHECK,
887 STMF_SAA_READ_ERROR);
888 return;
889 }
890
891 /*
892 * Check if setup is more expensive than copying the data.
893 *
894 * Use the global over-ride sbd_zcopy_threshold if set.
895 */
896 copy_threshold = (sbd_copy_threshold > 0) ?
897 sbd_copy_threshold : task->task_copy_threshold;
898 minsize = len;
899 if (len < copy_threshold &&
900 (dbuf = stmf_alloc_dbuf(task, len, &minsize, 0)) != 0) {
901
902 ret = sbd_copy_rdwr(task, laddr, dbuf,
903 SBD_CMD_SCSI_READ, 0);
904 /* done with the backend */
905 rw_exit(&sl->sl_access_state_lock);
906 if (ret != 0) {
907 /* backend error */
908 stmf_scsilib_send_status(task, STATUS_CHECK,
909 STMF_SAA_READ_ERROR);
910 } else {
911 /* send along good data */
912 dbuf->db_relative_offset = 0;
913 dbuf->db_data_size = len;
914 dbuf->db_flags = DB_SEND_STATUS_GOOD |
915 DB_DIRECTION_TO_RPORT;
916 /* XXX keep for FW? */
917 DTRACE_PROBE4(sbd__xfer,
918 struct scsi_task *, task,
919 struct stmf_data_buf *, dbuf,
920 uint64_t, laddr, uint32_t, len);
921 (void) stmf_xfer_data(task, dbuf,
922 STMF_IOF_LU_DONE);
923 }
924 return;
925 }
926
927 /* committed to reduced copy */
928 if (task->task_lu_private) {
929 scmd = (sbd_cmd_t *)task->task_lu_private;
930 } else {
931 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t),
932 KM_SLEEP);
933 task->task_lu_private = scmd;
934 }
935 /*
936 * Setup scmd to track read progress.
937 */
938 scmd->flags = SBD_SCSI_CMD_ACTIVE;
939 scmd->cmd_type = SBD_CMD_SCSI_READ;
940 scmd->nbufs = 0;
941 scmd->addr = laddr;
942 scmd->len = len;
943 scmd->current_ro = 0;
944
945 /*
946 * Kick-off the read.
947 */
948 sbd_do_sgl_read_xfer(task, scmd, 1);
949 return;
950 }
951
952 if (initial_dbuf == NULL) {
953 uint32_t maxsize, minsize, old_minsize;
954
955 maxsize = (len > (128*1024)) ? 128*1024 : len;
956 minsize = maxsize >> 2;
957 do {
958 old_minsize = minsize;
959 initial_dbuf = stmf_alloc_dbuf(task, maxsize,
960 &minsize, 0);
961 } while ((initial_dbuf == NULL) && (old_minsize > minsize) &&
962 (minsize >= 512));
963 if (initial_dbuf == NULL) {
964 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
965 return;
966 }
967 }
968 dbuf = initial_dbuf;
969
970 if ((dbuf->db_buf_size >= len) && fast_path &&
971 (dbuf->db_sglist_length == 1)) {
972 if (sbd_data_read(sl, task, laddr, (uint64_t)len,
973 dbuf->db_sglist[0].seg_addr) == STMF_SUCCESS) {
974 dbuf->db_relative_offset = 0;
975 dbuf->db_data_size = len;
976 dbuf->db_flags = DB_SEND_STATUS_GOOD |
977 DB_DIRECTION_TO_RPORT;
978 /* XXX keep for FW? */
979 DTRACE_PROBE4(sbd__xfer, struct scsi_task *, task,
980 struct stmf_data_buf *, dbuf,
981 uint64_t, laddr, uint32_t, len);
982 (void) stmf_xfer_data(task, dbuf, STMF_IOF_LU_DONE);
983 } else {
984 stmf_scsilib_send_status(task, STATUS_CHECK,
985 STMF_SAA_READ_ERROR);
986 }
987 return;
988 }
989
990 if (task->task_lu_private) {
991 scmd = (sbd_cmd_t *)task->task_lu_private;
992 } else {
993 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
994 task->task_lu_private = scmd;
995 }
996 scmd->flags = SBD_SCSI_CMD_ACTIVE;
997 scmd->cmd_type = SBD_CMD_SCSI_READ;
998 scmd->nbufs = 1;
999 scmd->addr = laddr;
1000 scmd->len = len;
1001 scmd->current_ro = 0;
1002
1003 sbd_do_read_xfer(task, scmd, dbuf);
1004 }
1005
1006 void
1007 sbd_do_write_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
1008 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
1009 {
1010 uint32_t len;
1011 int bufs_to_take;
1012
1013 if (scmd->len == 0) {
1014 goto DO_WRITE_XFER_DONE;
1015 }
1016
1017 /* Lets try not to hog all the buffers the port has. */
1018 bufs_to_take = ((task->task_max_nbufs > 2) &&
1019 (task->task_cmd_xfer_length < (32 * 1024))) ? 2 :
1020 task->task_max_nbufs;
1021
1022 if ((dbuf != NULL) &&
1023 ((dbuf->db_flags & DB_DONT_REUSE) || (dbuf_reusable == 0))) {
1024 /* free current dbuf and allocate a new one */
1025 stmf_free_dbuf(task, dbuf);
1026 dbuf = NULL;
1027 }
1028 if (scmd->nbufs >= bufs_to_take) {
1029 goto DO_WRITE_XFER_DONE;
1030 }
1031 if (dbuf == NULL) {
1032 uint32_t maxsize, minsize, old_minsize;
1033
1034 maxsize = (scmd->len > (128*1024)) ? 128*1024 :
1035 scmd->len;
1036 minsize = maxsize >> 2;
1037 do {
1038 old_minsize = minsize;
1039 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
1040 } while ((dbuf == NULL) && (old_minsize > minsize) &&
1041 (minsize >= 512));
1042 if (dbuf == NULL) {
1043 if (scmd->nbufs == 0) {
1044 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1045 STMF_ALLOC_FAILURE, NULL);
1046 }
1047 return;
1048 }
1049 }
1050
1051 len = scmd->len > dbuf->db_buf_size ? dbuf->db_buf_size :
1052 scmd->len;
1053
1054 dbuf->db_relative_offset = scmd->current_ro;
1055 dbuf->db_data_size = len;
1056 dbuf->db_flags = DB_DIRECTION_FROM_RPORT;
1057 (void) stmf_xfer_data(task, dbuf, 0);
1058 scmd->nbufs++; /* outstanding port xfers and bufs used */
1059 scmd->len -= len;
1060 scmd->current_ro += len;
1061
1062 if ((scmd->len != 0) && (scmd->nbufs < bufs_to_take)) {
1063 sbd_do_write_xfer(task, scmd, NULL, 0);
1064 }
1065 return;
1066
1067 DO_WRITE_XFER_DONE:
1068 if (dbuf != NULL) {
1069 stmf_free_dbuf(task, dbuf);
1070 }
1071 }
1072
1073 void
1074 sbd_do_sgl_write_xfer(struct scsi_task *task, sbd_cmd_t *scmd, int first_xfer)
1075 {
1076 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1077 sbd_zvol_io_t *zvio;
1078 int ret;
1079 uint32_t xfer_len, max_len, first_len;
1080 stmf_status_t xstat;
1081 stmf_data_buf_t *dbuf;
1082 uint_t nblks;
1108 sbd_1st_xfer_len = 0;
1109 }
1110
1111 max_len = MIN(task->task_max_xfer_len, sl->sl_max_xfer_len);
1112 if (sbd_max_xfer_len)
1113 max_len = MIN(max_len, sbd_max_xfer_len);
1114 /*
1115 * Special case the first xfer if hints are set.
1116 */
1117 if (first_xfer && (sbd_1st_xfer_len || task->task_1st_xfer_len)) {
1118 /* global over-ride has precedence */
1119 if (sbd_1st_xfer_len)
1120 first_len = sbd_1st_xfer_len;
1121 else
1122 first_len = task->task_1st_xfer_len;
1123 } else {
1124 first_len = 0;
1125 }
1126
1127
1128 while (scmd->len && scmd->nbufs < task->task_max_nbufs) {
1129
1130 xfer_len = MIN(max_len, scmd->len);
1131 if (first_len) {
1132 xfer_len = MIN(xfer_len, first_len);
1133 first_len = 0;
1134 }
1135 if (xfer_len < scmd->len) {
1136 /*
1137 * Attempt to end xfer on a block boundary.
1138 * The only way this does not happen is if the
1139 * xfer_len is small enough to stay contained
1140 * within the same block.
1141 */
1142 uint64_t xfer_offset, xfer_aligned_end;
1143
1144 xfer_offset = scmd->addr + scmd->current_ro;
1145 xfer_aligned_end =
1146 P2ALIGN(xfer_offset+xfer_len, blksize);
1147 if (xfer_aligned_end > xfer_offset)
1148 xfer_len = xfer_aligned_end - xfer_offset;
1149 }
1150 /*
1151 * Allocate object to track the write and reserve
1152 * enough space for scatter/gather list.
1153 */
1154 offset = scmd->addr + scmd->current_ro;
1155 nblks = sbd_zvol_numsegs(sl, offset, xfer_len);
1179 dbuf->db_data_size = xfer_len;
1180 dbuf->db_relative_offset = scmd->current_ro;
1181 dbuf->db_sglist_length = (uint16_t)nblks;
1182 dbuf->db_xfer_status = 0;
1183 dbuf->db_handle = 0;
1184 dbuf->db_flags = (DB_DONT_CACHE | DB_DONT_REUSE |
1185 DB_DIRECTION_FROM_RPORT | DB_LU_DATA_BUF);
1186
1187 zvio = dbuf->db_lu_private;
1188 zvio->zvio_offset = offset;
1189
1190 /* get the buffers */
1191 ret = sbd_zvol_alloc_write_bufs(sl, dbuf);
1192 if (ret != 0) {
1193 /*
1194 * Could not allocate buffers from the backend;
1195 * treat it like an IO error.
1196 */
1197 stmf_free(dbuf);
1198 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1199 if (scmd->nbufs == 0) {
1200 /*
1201 * Nothing queued, so no completions coming
1202 */
1203 stmf_scsilib_send_status(task, STATUS_CHECK,
1204 STMF_SAA_WRITE_ERROR);
1205 rw_exit(&sl->sl_access_state_lock);
1206 }
1207 /*
1208 * Completions of previous buffers will cleanup.
1209 */
1210 return;
1211 }
1212
1213 /*
1214 * Allow PP to do setup
1215 */
1216 xstat = stmf_setup_dbuf(task, dbuf, 0);
1217 if (xstat != STMF_SUCCESS) {
1218 /*
1219 * This could happen if the driver cannot get the
1220 * DDI resources it needs for this request.
1221 * If other dbufs are queued, try again when the next
1222 * one completes, otherwise give up.
1223 */
1224 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
1225 stmf_free(dbuf);
1226 if (scmd->nbufs > 0) {
1227 /* completion of previous dbuf will retry */
1228 return;
1229 }
1230 /*
1231 * Done with this command.
1232 */
1233 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
1234 if (first_xfer)
1235 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
1236 else
1237 stmf_scsilib_send_status(task, STATUS_CHECK,
1238 STMF_SAA_WRITE_ERROR);
1239 rw_exit(&sl->sl_access_state_lock);
1240 return;
1241 }
1242
1243 /*
1244 * dbuf is now queued on task
1245 */
1246 scmd->nbufs++;
1247
1248 xstat = stmf_xfer_data(task, dbuf, 0);
1249 switch (xstat) {
1250 case STMF_SUCCESS:
1251 break;
1252 case STMF_BUSY:
1253 /*
1254 * The dbuf is queued on the task, but unknown
1255 * to the PP, thus no completion will occur.
1256 */
1257 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
1258 stmf_teardown_dbuf(task, dbuf);
1259 stmf_free(dbuf);
1260 scmd->nbufs--;
1261 if (scmd->nbufs > 0) {
1262 /* completion of previous dbuf will retry */
1263 return;
1264 }
1265 /*
1266 * Done with this command.
1267 */
1268 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
1269 if (first_xfer)
1270 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
1271 else
1272 stmf_scsilib_send_status(task, STATUS_CHECK,
1273 STMF_SAA_WRITE_ERROR);
1274 rw_exit(&sl->sl_access_state_lock);
1275 return;
1276 case STMF_ABORTED:
1277 /*
1278 * Completion code will cleanup.
1279 */
1280 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1281 return;
1282 }
1283 /*
1284 * Update the xfer progress.
1285 */
1286 scmd->len -= xfer_len;
1287 scmd->current_ro += xfer_len;
1288 }
1289 }
1290
1291 void
1292 sbd_handle_write_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
1293 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
1294 {
1295 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1296 uint64_t laddr;
1297 uint32_t buflen, iolen;
1298 int ndx;
1299
1300 if (scmd->nbufs > 0) {
1301 /*
1302 * Decrement the count to indicate the port xfer
1303 * into the dbuf has completed even though the buf is
1304 * still in use here in the LU provider.
1305 */
1306 scmd->nbufs--;
1307 }
1308
1309 if (dbuf->db_xfer_status != STMF_SUCCESS) {
1310 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1311 dbuf->db_xfer_status, NULL);
1312 return;
1313 }
1314
1315 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
1316 goto WRITE_XFER_DONE;
1317 }
1318
1319 if (scmd->len != 0) {
1320 /*
1321 * Initiate the next port xfer to occur in parallel
1322 * with writing this buf.
1323 */
1324 sbd_do_write_xfer(task, scmd, NULL, 0);
1325 }
1326
1327 laddr = scmd->addr + dbuf->db_relative_offset;
1328
1329 /*
1330 * If this is going to a zvol, use the direct call to
1331 * sbd_zvol_copy_{read,write}. The direct call interface is
1332 * restricted to PPs that accept sglists, but that is not required.
1333 */
1334 if (sl->sl_flags & SL_CALL_ZVOL &&
1335 (task->task_additional_flags & TASK_AF_ACCEPT_LU_DBUF) &&
1336 (sbd_zcopy & (4|1))) {
1337 int commit;
1338
1339 commit = (scmd->len == 0 && scmd->nbufs == 0);
1340 if (sbd_copy_rdwr(task, laddr, dbuf, SBD_CMD_SCSI_WRITE,
1341 commit) != STMF_SUCCESS)
1342 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1343 buflen = dbuf->db_data_size;
1344 } else {
1345 for (buflen = 0, ndx = 0; (buflen < dbuf->db_data_size) &&
1346 (ndx < dbuf->db_sglist_length); ndx++) {
1347 iolen = min(dbuf->db_data_size - buflen,
1348 dbuf->db_sglist[ndx].seg_length);
1349 if (iolen == 0)
1350 break;
1351 if (sbd_data_write(sl, task, laddr, (uint64_t)iolen,
1352 dbuf->db_sglist[ndx].seg_addr) != STMF_SUCCESS) {
1353 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1354 break;
1355 }
1356 buflen += iolen;
1357 laddr += (uint64_t)iolen;
1358 }
1359 }
1360 task->task_nbytes_transferred += buflen;
1361 WRITE_XFER_DONE:
1362 if (scmd->len == 0 || scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
1363 stmf_free_dbuf(task, dbuf);
1364 if (scmd->nbufs)
1365 return; /* wait for all buffers to complete */
1366 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
1367 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
1368 stmf_scsilib_send_status(task, STATUS_CHECK,
1369 STMF_SAA_WRITE_ERROR);
1370 } else {
1371 /*
1372 * If SYNC_WRITE flag is on then we need to flush
1373 * cache before sending status.
1374 * Note: this may be a no-op because of how
1375 * SL_WRITEBACK_CACHE_DISABLE and
1376 * SL_FLUSH_ON_DISABLED_WRITECACHE are set, but not
1377 * worth code complexity of checking those in this code
1378 * path, SBD_SCSI_CMD_SYNC_WRITE is rarely set.
1379 */
1380 if ((scmd->flags & SBD_SCSI_CMD_SYNC_WRITE) &&
1381 (sbd_flush_data_cache(sl, 0) != SBD_SUCCESS)) {
1382 stmf_scsilib_send_status(task, STATUS_CHECK,
1383 STMF_SAA_WRITE_ERROR);
1384 } else {
1385 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
1386 }
1412 return (0);
1413 /*
1414 * Calculate amount of data that will avoid the copy path.
1415 * The calculation is only valid if len >= blksize.
1416 */
1417 no_copy_span = P2ALIGN(laddr+len, blksize) -
1418 P2ROUNDUP(laddr, blksize);
1419 return (no_copy_span >= len/2);
1420 }
1421 }
1422
1423 void
1424 sbd_handle_write(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
1425 {
1426 uint64_t lba, laddr;
1427 uint32_t len;
1428 uint8_t op = task->task_cdb[0], do_immediate_data = 0;
1429 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1430 sbd_cmd_t *scmd;
1431 stmf_data_buf_t *dbuf;
1432 uint8_t sync_wr_flag = 0;
1433
1434 if (sl->sl_flags & SL_WRITE_PROTECTED) {
1435 stmf_scsilib_send_status(task, STATUS_CHECK,
1436 STMF_SAA_WRITE_PROTECTED);
1437 return;
1438 }
1439 if (op == SCMD_WRITE) {
1440 lba = READ_SCSI21(&task->task_cdb[1], uint64_t);
1441 len = (uint32_t)task->task_cdb[4];
1442
1443 if (len == 0) {
1444 len = 256;
1445 }
1446 } else if (op == SCMD_WRITE_G1) {
1447 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1448 len = READ_SCSI16(&task->task_cdb[7], uint32_t);
1449 } else if (op == SCMD_WRITE_G5) {
1450 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1451 len = READ_SCSI32(&task->task_cdb[6], uint32_t);
1452 } else if (op == SCMD_WRITE_G4) {
1453 lba = READ_SCSI64(&task->task_cdb[2], uint64_t);
1454 len = READ_SCSI32(&task->task_cdb[10], uint32_t);
1455 } else if (op == SCMD_WRITE_VERIFY) {
1456 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1457 len = READ_SCSI16(&task->task_cdb[7], uint32_t);
1458 sync_wr_flag = SBD_SCSI_CMD_SYNC_WRITE;
1459 } else if (op == SCMD_WRITE_VERIFY_G5) {
1460 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1461 len = READ_SCSI32(&task->task_cdb[6], uint32_t);
1462 sync_wr_flag = SBD_SCSI_CMD_SYNC_WRITE;
1463 } else if (op == SCMD_WRITE_VERIFY_G4) {
1464 lba = READ_SCSI64(&task->task_cdb[2], uint64_t);
1465 len = READ_SCSI32(&task->task_cdb[10], uint32_t);
1466 sync_wr_flag = SBD_SCSI_CMD_SYNC_WRITE;
1467 } else {
1468 stmf_scsilib_send_status(task, STATUS_CHECK,
1469 STMF_SAA_INVALID_OPCODE);
1470 return;
1471 }
1472
1473 laddr = lba << sl->sl_data_blocksize_shift;
1474 len <<= sl->sl_data_blocksize_shift;
1475
1476 if ((laddr + (uint64_t)len) > sl->sl_lu_size) {
1477 stmf_scsilib_send_status(task, STATUS_CHECK,
1478 STMF_SAA_LBA_OUT_OF_RANGE);
1479 return;
1480 }
1481
1482 task->task_cmd_xfer_length = len;
1483 if (task->task_additional_flags & TASK_AF_NO_EXPECTED_XFER_LENGTH) {
1484 task->task_expected_xfer_length = len;
1485 }
1486
1487 len = (len > task->task_expected_xfer_length) ?
1488 task->task_expected_xfer_length : len;
1489
1490 if (len == 0) {
1491 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
1492 return;
1493 }
1494
1495 if (sbd_zcopy & (4|1) && /* Debug switch */
1496 initial_dbuf == NULL && /* No PP buf passed in */
1497 sl->sl_flags & SL_CALL_ZVOL && /* zvol backing store */
1498 (task->task_additional_flags &
1499 TASK_AF_ACCEPT_LU_DBUF) && /* PP allows it */
1500 sbd_zcopy_write_useful(task, laddr, len, sl->sl_blksize)) {
1501
1502 /*
1503 * XXX Note that disallowing initial_dbuf will eliminate
1504 * iSCSI from participating. For small writes, that is
1505 * probably ok. For large writes, it may be best to just
1506 * copy the data from the initial dbuf and use zcopy for
1507 * the rest.
1508 */
1509 rw_enter(&sl->sl_access_state_lock, RW_READER);
1510 if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
1511 rw_exit(&sl->sl_access_state_lock);
1512 stmf_scsilib_send_status(task, STATUS_CHECK,
1513 STMF_SAA_READ_ERROR);
1514 return;
1515 }
1516 /*
1517 * Setup scmd to track the write progress.
1518 */
1519 if (task->task_lu_private) {
1520 scmd = (sbd_cmd_t *)task->task_lu_private;
1521 } else {
1522 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t),
1523 KM_SLEEP);
1524 task->task_lu_private = scmd;
1525 }
1526 scmd->flags = SBD_SCSI_CMD_ACTIVE | sync_wr_flag;
1527 scmd->cmd_type = SBD_CMD_SCSI_WRITE;
1528 scmd->nbufs = 0;
1529 scmd->addr = laddr;
1530 scmd->len = len;
1531 scmd->current_ro = 0;
1532 sbd_do_sgl_write_xfer(task, scmd, 1);
1533 return;
1534 }
1535
1536 if ((initial_dbuf != NULL) && (task->task_flags & TF_INITIAL_BURST)) {
1537 if (initial_dbuf->db_data_size > len) {
1538 if (initial_dbuf->db_data_size >
1539 task->task_expected_xfer_length) {
1540 /* protocol error */
1541 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1542 STMF_INVALID_ARG, NULL);
1543 return;
1544 }
1545 initial_dbuf->db_data_size = len;
1546 }
1547 do_immediate_data = 1;
1548 }
1549 dbuf = initial_dbuf;
1550
1551 if (task->task_lu_private) {
1552 scmd = (sbd_cmd_t *)task->task_lu_private;
1553 } else {
1554 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
1555 task->task_lu_private = scmd;
1556 }
1557 scmd->flags = SBD_SCSI_CMD_ACTIVE | sync_wr_flag;
1558 scmd->cmd_type = SBD_CMD_SCSI_WRITE;
1559 scmd->nbufs = 0;
1560 scmd->addr = laddr;
1561 scmd->len = len;
1562 scmd->current_ro = 0;
1563
1564 if (do_immediate_data) {
1565 /*
1566 * Account for data passed in this write command
1567 */
1568 (void) stmf_xfer_data(task, dbuf, STMF_IOF_STATS_ONLY);
1569 scmd->len -= dbuf->db_data_size;
1570 scmd->current_ro += dbuf->db_data_size;
1571 dbuf->db_xfer_status = STMF_SUCCESS;
1572 sbd_handle_write_xfer_completion(task, scmd, dbuf, 0);
1573 } else {
1574 sbd_do_write_xfer(task, scmd, dbuf, 0);
1575 }
1576 }
1577
1578 /*
1579 * Utility routine to handle small non performance data transfers to the
1580 * initiators. dbuf is an initial data buf (if any), 'p' points to a data
1581 * buffer which is source of data for transfer, cdb_xfer_size is the
1582 * transfer size based on CDB, cmd_xfer_size is the actual amount of data
1583 * which this command would transfer (the size of data pointed to by 'p').
1584 */
1585 void
1586 sbd_handle_short_read_transfers(scsi_task_t *task, stmf_data_buf_t *dbuf,
1587 uint8_t *p, uint32_t cdb_xfer_size, uint32_t cmd_xfer_size)
1588 {
1589 uint32_t bufsize, ndx;
1732
1733 /* Lets find out who to call */
1734 switch (task->task_cdb[0]) {
1735 case SCMD_MODE_SELECT:
1736 case SCMD_MODE_SELECT_G1:
1737 if (sl->sl_access_state == SBD_LU_STANDBY) {
1738 st_ret = stmf_proxy_scsi_cmd(task, dbuf);
1739 if (st_ret != STMF_SUCCESS) {
1740 stmf_scsilib_send_status(task, STATUS_CHECK,
1741 STMF_SAA_LU_NO_ACCESS_UNAVAIL);
1742 }
1743 } else {
1744 sbd_handle_mode_select_xfer(task,
1745 dbuf->db_sglist[0].seg_addr, dbuf->db_data_size);
1746 }
1747 break;
1748 case SCMD_UNMAP:
1749 sbd_handle_unmap_xfer(task,
1750 dbuf->db_sglist[0].seg_addr, dbuf->db_data_size);
1751 break;
1752 case SCMD_PERSISTENT_RESERVE_OUT:
1753 if (sl->sl_access_state == SBD_LU_STANDBY) {
1754 st_ret = stmf_proxy_scsi_cmd(task, dbuf);
1755 if (st_ret != STMF_SUCCESS) {
1756 stmf_scsilib_send_status(task, STATUS_CHECK,
1757 STMF_SAA_LU_NO_ACCESS_UNAVAIL);
1758 }
1759 } else {
1760 sbd_handle_pgr_out_data(task, dbuf);
1761 }
1762 break;
1763 default:
1764 /* This should never happen */
1765 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1766 STMF_ABORTED, NULL);
1767 }
1768 }
1769
1770 void
1771 sbd_handle_read_capacity(struct scsi_task *task,
1826 if (s < (4ull * 1024ull * 1024ull * 1024ull)) {
1827 *nsectors = 32;
1828 *nheads = 8;
1829 } else {
1830 *nsectors = 254;
1831 *nheads = 254;
1832 }
1833 *ncyl = s / ((uint64_t)blksize * (uint64_t)(*nsectors) *
1834 (uint64_t)(*nheads));
1835 }
1836
1837 void
1838 sbd_handle_mode_sense(struct scsi_task *task,
1839 struct stmf_data_buf *initial_dbuf, uint8_t *buf)
1840 {
1841 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1842 uint32_t cmd_size, n;
1843 uint8_t *cdb;
1844 uint32_t ncyl;
1845 uint8_t nsectors, nheads;
1846 uint8_t page, ctrl, header_size, pc_valid;
1847 uint16_t nbytes;
1848 uint8_t *p;
1849 uint64_t s = sl->sl_lu_size;
1850 uint32_t dev_spec_param_offset;
1851
1852 p = buf; /* buf is assumed to be zeroed out and large enough */
1853 n = 0;
1854 cdb = &task->task_cdb[0];
1855 page = cdb[2] & 0x3F;
1856 ctrl = (cdb[2] >> 6) & 3;
1857 cmd_size = (cdb[0] == SCMD_MODE_SENSE) ? cdb[4] :
1858 READ_SCSI16(&cdb[7], uint32_t);
1859
1860 if (cdb[0] == SCMD_MODE_SENSE) {
1861 header_size = 4;
1862 dev_spec_param_offset = 2;
1863 } else {
1864 header_size = 8;
1865 dev_spec_param_offset = 3;
1866 }
1867
1868 /* Now validate the command */
1869 if ((cdb[2] == 0) || (page == MODEPAGE_ALLPAGES) || (page == 0x08) ||
1870 (page == 0x0A) || (page == 0x03) || (page == 0x04)) {
1871 pc_valid = 1;
1872 } else {
1873 pc_valid = 0;
1874 }
1875 if ((cmd_size < header_size) || (pc_valid == 0)) {
1876 stmf_scsilib_send_status(task, STATUS_CHECK,
1877 STMF_SAA_INVALID_FIELD_IN_CDB);
1878 return;
1879 }
1880
1881 /* We will update the length in the mode header at the end */
1882
1883 /* Block dev device specific param in mode param header has wp bit */
1884 if (sl->sl_flags & SL_WRITE_PROTECTED) {
1885 p[n + dev_spec_param_offset] = BIT_7;
1886 }
1887 n += header_size;
1888 /* We are not going to return any block descriptor */
1889
1890 nbytes = ((uint16_t)1) << sl->sl_data_blocksize_shift;
1891 sbd_calc_geometry(s, nbytes, &nsectors, &nheads, &ncyl);
1892
1893 if ((page == 0x03) || (page == MODEPAGE_ALLPAGES)) {
1894 p[n] = 0x03;
1895 p[n+1] = 0x16;
1896 if (ctrl != 1) {
1897 p[n + 11] = nsectors;
1898 p[n + 12] = nbytes >> 8;
1899 p[n + 13] = nbytes & 0xff;
1900 p[n + 20] = 0x80;
1901 }
1902 n += 24;
1903 }
1904 if ((page == 0x04) || (page == MODEPAGE_ALLPAGES)) {
1905 p[n] = 0x04;
1906 p[n + 1] = 0x16;
1907 if (ctrl != 1) {
1908 p[n + 2] = ncyl >> 16;
1909 p[n + 3] = ncyl >> 8;
1910 p[n + 4] = ncyl & 0xff;
1911 p[n + 5] = nheads;
1912 p[n + 20] = 0x15;
1913 p[n + 21] = 0x18;
1914 }
1915 n += 24;
1916 }
1917 if ((page == MODEPAGE_CACHING) || (page == MODEPAGE_ALLPAGES)) {
1918 struct mode_caching *mode_caching_page;
1919
1920 mode_caching_page = (struct mode_caching *)&p[n];
1921
1922 mode_caching_page->mode_page.code = MODEPAGE_CACHING;
1923 mode_caching_page->mode_page.ps = 1; /* A saveable page */
1924 mode_caching_page->mode_page.length = 0x12;
1959 PAGELENGTH_MODE_CONTROL_SCSI3;
1960 if (ctrl != 1) {
1961 /* If not looking for changeable values, report this. */
1962 mode_control_page->que_mod = CTRL_QMOD_UNRESTRICT;
1963 }
1964 n += (sizeof (struct mode_page) +
1965 mode_control_page->mode_page.length);
1966 }
1967
1968 if (cdb[0] == SCMD_MODE_SENSE) {
1969 if (n > 255) {
1970 stmf_scsilib_send_status(task, STATUS_CHECK,
1971 STMF_SAA_INVALID_FIELD_IN_CDB);
1972 return;
1973 }
1974 /*
1975 * Mode parameter header length doesn't include the number
1976 * of bytes in the length field, so adjust the count.
1977 * Byte count minus header length field size.
1978 */
1979 buf[0] = (n - 1) & 0xff;
1980 } else {
1981 /* Byte count minus header length field size. */
1982 buf[1] = (n - 2) & 0xff;
1983 buf[0] = ((n - 2) >> 8) & 0xff;
1984 }
1985
1986 sbd_handle_short_read_transfers(task, initial_dbuf, buf,
1987 cmd_size, n);
1988 }
1989
1990 void
1991 sbd_handle_mode_select(scsi_task_t *task, stmf_data_buf_t *dbuf)
1992 {
1993 uint32_t cmd_xfer_len;
1994
1995 if (task->task_cdb[0] == SCMD_MODE_SELECT) {
1996 cmd_xfer_len = (uint32_t)task->task_cdb[4];
1997 } else {
1998 cmd_xfer_len = READ_SCSI16(&task->task_cdb[7], uint32_t);
1999 }
2000
2001 if ((task->task_cdb[1] & 0xFE) != 0x10) {
2002 stmf_scsilib_send_status(task, STATUS_CHECK,
2003 STMF_SAA_INVALID_FIELD_IN_CDB);
2172 kmem_free(buf, 260);
2173 return;
2174 }
2175 sbd_handle_short_read_transfers(task, initial_dbuf, buf,
2176 cmd_size, xfer_size);
2177 kmem_free(buf, 260);
2178 }
2179
2180 /*
2181 * This function parse through a string, passed to it as a pointer to a string,
2182 * by adjusting the pointer to the first non-space character and returns
2183 * the count/length of the first bunch of non-space characters. Multiple
2184 * Management URLs are stored as a space delimited string in sl_mgmt_url
2185 * field of sbd_lu_t. This function is used to retrieve one url at a time.
2186 *
2187 * i/p : pointer to pointer to a url string
2188 * o/p : Adjust the pointer to the url to the first non white character
2189 * and returns the length of the URL
2190 */
2191 uint16_t
2192 sbd_parse_mgmt_url(char **url_addr) {
2193 uint16_t url_length = 0;
2194 char *url;
2195 url = *url_addr;
2196
2197 while (*url != '\0') {
2198 if (*url == ' ' || *url == '\t' || *url == '\n') {
2199 (*url_addr)++;
2200 url = *url_addr;
2201 } else {
2202 break;
2203 }
2204 }
2205
2206 while (*url != '\0') {
2207 if (*url == ' ' || *url == '\t' ||
2208 *url == '\n' || *url == '\0') {
2209 break;
2210 }
2211 url++;
2212 url_length++;
2283
2284 /* Do the actual I/O. Recycle xfer_size now to be write size. */
2285 DTRACE_PROBE1(write__same__io__begin, uint64_t, len);
2286 for (sz_done = 0; sz_done < len; sz_done += (uint64_t)xfer_size) {
2287 xfer_size = ((big_buf_size + sz_done) <= len) ? big_buf_size :
2288 len - sz_done;
2289 ret = sbd_data_write(sl, task, addr + sz_done,
2290 (uint64_t)xfer_size, big_buf);
2291 if (ret != SBD_SUCCESS)
2292 break;
2293 }
2294 DTRACE_PROBE2(write__same__io__end, uint64_t, len, uint64_t, sz_done);
2295
2296 if (big_buf != scmd->trans_data)
2297 kmem_free(big_buf, big_buf_size);
2298
2299 return (ret);
2300 }
2301
2302 static void
2303 sbd_handle_write_same_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
2304 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
2305 {
2306 uint64_t laddr;
2307 uint32_t buflen, iolen;
2308 int ndx, ret;
2309
2310 if (dbuf->db_xfer_status != STMF_SUCCESS) {
2311 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
2312 dbuf->db_xfer_status, NULL);
2313 return;
2314 }
2315
2316 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
2317 goto write_same_xfer_done;
2318 }
2319
2320 if (scmd->len != 0) {
2321 /*
2322 * Initiate the next port xfer to occur in parallel
2323 * with writing this buf.
2324 */
2325 sbd_do_write_same_xfer(task, scmd, NULL, 0);
2326 }
2327
2328 laddr = dbuf->db_relative_offset;
2329
2330 for (buflen = 0, ndx = 0; (buflen < dbuf->db_data_size) &&
2331 (ndx < dbuf->db_sglist_length); ndx++) {
2332 iolen = min(dbuf->db_data_size - buflen,
2333 dbuf->db_sglist[ndx].seg_length);
2334 if (iolen == 0)
2335 break;
2336 bcopy(dbuf->db_sglist[ndx].seg_addr, &scmd->trans_data[laddr],
2337 iolen);
2338 buflen += iolen;
2339 laddr += (uint64_t)iolen;
2340 }
2341 task->task_nbytes_transferred += buflen;
2342
2343 write_same_xfer_done:
2344 if (scmd->len == 0 || scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
2345 stmf_free_dbuf(task, dbuf);
2346 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
2347 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
2348 stmf_scsilib_send_status(task, STATUS_CHECK,
2349 STMF_SAA_WRITE_ERROR);
2350 } else {
2351 ret = sbd_write_same_data(task, scmd);
2352 if (ret != SBD_SUCCESS) {
2353 stmf_scsilib_send_status(task, STATUS_CHECK,
2354 STMF_SAA_WRITE_ERROR);
2355 } else {
2356 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2357 }
2358 }
2359 /*
2360 * Only way we should get here is via handle_write_same(),
2361 * and that should make the following assertion always pass.
2362 */
2363 ASSERT((scmd->flags & SBD_SCSI_CMD_TRANS_DATA) &&
2364 scmd->trans_data != NULL);
2365 kmem_free(scmd->trans_data, scmd->trans_data_len);
2366 scmd->flags &= ~SBD_SCSI_CMD_TRANS_DATA;
2367 return;
2368 }
2369 sbd_do_write_same_xfer(task, scmd, dbuf, dbuf_reusable);
2370 }
2371
2372 static void
2373 sbd_do_write_same_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
2374 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
2375 {
2376 uint32_t len;
2377
2378 if (scmd->len == 0) {
2379 if (dbuf != NULL)
2380 stmf_free_dbuf(task, dbuf);
2381 return;
2382 }
2383
2384 if ((dbuf != NULL) &&
2385 ((dbuf->db_flags & DB_DONT_REUSE) || (dbuf_reusable == 0))) {
2386 /* free current dbuf and allocate a new one */
2387 stmf_free_dbuf(task, dbuf);
2388 dbuf = NULL;
2389 }
2390 if (dbuf == NULL) {
2391 uint32_t maxsize, minsize, old_minsize;
2392
2393 maxsize = (scmd->len > (128*1024)) ? 128*1024 :
2394 scmd->len;
2395 minsize = maxsize >> 2;
2396 do {
2397 old_minsize = minsize;
2398 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
2399 } while ((dbuf == NULL) && (old_minsize > minsize) &&
2400 (minsize >= 512));
2401 if (dbuf == NULL) {
2402 if (scmd->nbufs == 0) {
2403 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
2404 STMF_ALLOC_FAILURE, NULL);
2405 }
2406 return;
2407 }
2408 }
2409
2410 len = scmd->len > dbuf->db_buf_size ? dbuf->db_buf_size :
2411 scmd->len;
2412
2413 dbuf->db_relative_offset = scmd->current_ro;
2414 dbuf->db_data_size = len;
2415 dbuf->db_flags = DB_DIRECTION_FROM_RPORT;
2416 (void) stmf_xfer_data(task, dbuf, 0);
2417 scmd->nbufs++; /* outstanding port xfers and bufs used */
2418 scmd->len -= len;
2419 scmd->current_ro += len;
2420 }
2421
2422 static void
2423 sbd_handle_write_same(scsi_task_t *task, struct stmf_data_buf *initial_dbuf)
2424 {
2425 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
2426 uint64_t addr, len;
2427 sbd_cmd_t *scmd;
2428 stmf_data_buf_t *dbuf;
2429 uint8_t unmap;
2430 uint8_t do_immediate_data = 0;
2431
2432 task->task_cmd_xfer_length = 0;
2433 if (task->task_additional_flags &
2434 TASK_AF_NO_EXPECTED_XFER_LENGTH) {
2435 task->task_expected_xfer_length = 0;
2436 }
2437 if (sl->sl_flags & SL_WRITE_PROTECTED) {
2438 stmf_scsilib_send_status(task, STATUS_CHECK,
2439 STMF_SAA_WRITE_PROTECTED);
2440 return;
2441 }
2442 if (task->task_cdb[1] & 0xF7) {
2443 stmf_scsilib_send_status(task, STATUS_CHECK,
2444 STMF_SAA_INVALID_FIELD_IN_CDB);
2445 return;
2446 }
2447 unmap = task->task_cdb[1] & 0x08;
2448 if (unmap && ((sl->sl_flags & SL_UNMAP_ENABLED) == 0)) {
2449 stmf_scsilib_send_status(task, STATUS_CHECK,
2450 STMF_SAA_INVALID_FIELD_IN_CDB);
2451 return;
2452 }
2453 if (task->task_cdb[0] == SCMD_WRITE_SAME_G1) {
2454 addr = READ_SCSI32(&task->task_cdb[2], uint64_t);
2455 len = READ_SCSI16(&task->task_cdb[7], uint64_t);
2456 } else {
2457 addr = READ_SCSI64(&task->task_cdb[2], uint64_t);
2458 len = READ_SCSI32(&task->task_cdb[10], uint64_t);
2459 }
2460 if (len == 0) {
2461 stmf_scsilib_send_status(task, STATUS_CHECK,
2462 STMF_SAA_INVALID_FIELD_IN_CDB);
2463 return;
2464 }
2465 addr <<= sl->sl_data_blocksize_shift;
2466 len <<= sl->sl_data_blocksize_shift;
2467
2468 /* Check if the command is for the unmap function */
2469 if (unmap) {
2470 if (sbd_unmap(sl, addr, len) != 0) {
2471 stmf_scsilib_send_status(task, STATUS_CHECK,
2472 STMF_SAA_LBA_OUT_OF_RANGE);
2473 } else {
2474 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2475 }
2476 return;
2477 }
2478
2479 /* Write same function */
2480
2481 task->task_cmd_xfer_length = 1 << sl->sl_data_blocksize_shift;
2482 if (task->task_additional_flags &
2483 TASK_AF_NO_EXPECTED_XFER_LENGTH) {
2484 task->task_expected_xfer_length = task->task_cmd_xfer_length;
2485 }
2486 if ((addr + len) > sl->sl_lu_size) {
2487 stmf_scsilib_send_status(task, STATUS_CHECK,
2488 STMF_SAA_LBA_OUT_OF_RANGE);
2489 return;
2490 }
2491
2492 /* For rest of this I/O the transfer length is 1 block */
2493 len = ((uint64_t)1) << sl->sl_data_blocksize_shift;
2494
2495 /* Some basic checks */
2496 if ((len == 0) || (len != task->task_expected_xfer_length)) {
2497 stmf_scsilib_send_status(task, STATUS_CHECK,
2498 STMF_SAA_INVALID_FIELD_IN_CDB);
2499 return;
2500 }
2501
2502
2503 if ((initial_dbuf != NULL) && (task->task_flags & TF_INITIAL_BURST)) {
2504 if (initial_dbuf->db_data_size > len) {
2505 if (initial_dbuf->db_data_size >
2506 task->task_expected_xfer_length) {
2507 /* protocol error */
2508 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
2509 STMF_INVALID_ARG, NULL);
2510 return;
2511 }
2512 initial_dbuf->db_data_size = (uint32_t)len;
2513 }
2514 do_immediate_data = 1;
2515 }
2516 dbuf = initial_dbuf;
2517
2518 if (task->task_lu_private) {
2519 scmd = (sbd_cmd_t *)task->task_lu_private;
2520 } else {
2521 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
2522 task->task_lu_private = scmd;
2523 }
2524 scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_TRANS_DATA;
2525 scmd->cmd_type = SBD_CMD_SCSI_WRITE;
2526 scmd->nbufs = 0;
2527 scmd->len = (uint32_t)len;
2528 scmd->trans_data_len = (uint32_t)len;
2529 scmd->trans_data = kmem_alloc((size_t)len, KM_SLEEP);
2530 scmd->current_ro = 0;
2531
2532 if (do_immediate_data) {
2533 /*
2534 * Account for data passed in this write command
2535 */
2536 (void) stmf_xfer_data(task, dbuf, STMF_IOF_STATS_ONLY);
2537 scmd->len -= dbuf->db_data_size;
2538 scmd->current_ro += dbuf->db_data_size;
2539 dbuf->db_xfer_status = STMF_SUCCESS;
2540 sbd_handle_write_same_xfer_completion(task, scmd, dbuf, 0);
2541 } else {
2542 sbd_do_write_same_xfer(task, scmd, dbuf, 0);
2543 }
2544 }
2545
2546 static void
2547 sbd_handle_unmap(scsi_task_t *task, stmf_data_buf_t *dbuf)
2548 {
2549 uint32_t cmd_xfer_len;
2550
2551 cmd_xfer_len = READ_SCSI16(&task->task_cdb[7], uint32_t);
2552
2553 if (task->task_cdb[1] & 1) {
2554 stmf_scsilib_send_status(task, STATUS_CHECK,
2555 STMF_SAA_INVALID_FIELD_IN_CDB);
2556 return;
2557 }
2558
2559 if (cmd_xfer_len == 0) {
2560 task->task_cmd_xfer_length = 0;
2561 if (task->task_additional_flags &
2562 TASK_AF_NO_EXPECTED_XFER_LENGTH) {
2563 task->task_expected_xfer_length = 0;
2564 }
2565 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2566 return;
2567 }
2568
2569 sbd_handle_short_write_transfers(task, dbuf, cmd_xfer_len);
2570 }
2571
2572 static void
2573 sbd_handle_unmap_xfer(scsi_task_t *task, uint8_t *buf, uint32_t buflen)
2574 {
2575 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
2576 uint32_t ulen, dlen, num_desc;
2577 uint64_t addr, len;
2578 uint8_t *p;
2579 int ret;
2580
2581 if (buflen < 24) {
2582 stmf_scsilib_send_status(task, STATUS_CHECK,
2583 STMF_SAA_INVALID_FIELD_IN_CDB);
2584 return;
2585 }
2586 ulen = READ_SCSI16(buf, uint32_t);
2587 dlen = READ_SCSI16(buf + 2, uint32_t);
2588 num_desc = dlen >> 4;
2589 if (((ulen + 2) != buflen) || ((dlen + 8) != buflen) || (dlen & 0xf) ||
2590 (num_desc == 0)) {
2591 stmf_scsilib_send_status(task, STATUS_CHECK,
2592 STMF_SAA_INVALID_FIELD_IN_CDB);
2593 return;
2594 }
2595
2596 for (p = buf + 8; num_desc; num_desc--, p += 16) {
2597 addr = READ_SCSI64(p, uint64_t);
2598 addr <<= sl->sl_data_blocksize_shift;
2599 len = READ_SCSI32(p+8, uint64_t);
2600 len <<= sl->sl_data_blocksize_shift;
2601 ret = sbd_unmap(sl, addr, len);
2602 if (ret != 0) {
2603 stmf_scsilib_send_status(task, STATUS_CHECK,
2604 STMF_SAA_LBA_OUT_OF_RANGE);
2605 return;
2606 }
2607 }
2608
2609 unmap_done:
2610 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2611 }
2612
2613 void
2614 sbd_handle_inquiry(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
2615 {
2616 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
2617 uint8_t *cdbp = (uint8_t *)&task->task_cdb[0];
2618 uint8_t *p;
2619 uint8_t byte0;
2620 uint8_t page_length;
2621 uint16_t bsize = 512;
2622 uint16_t cmd_size;
2623 uint32_t xfer_size = 4;
2624 uint32_t mgmt_url_size = 0;
2625 uint8_t exp;
2626 uint64_t s;
2627 char *mgmt_url = NULL;
2628
2629
2660 */
2661
2662 if ((cdbp[1] & 1) == 0) {
2663 int i;
2664 struct scsi_inquiry *inq;
2665
2666 p = (uint8_t *)kmem_zalloc(bsize, KM_SLEEP);
2667 inq = (struct scsi_inquiry *)p;
2668
2669 page_length = 69;
2670 xfer_size = page_length + 5;
2671
2672 inq->inq_dtype = DTYPE_DIRECT;
2673 inq->inq_ansi = 5; /* SPC-3 */
2674 inq->inq_hisup = 1;
2675 inq->inq_rdf = 2; /* Response data format for SPC-3 */
2676 inq->inq_len = page_length;
2677
2678 inq->inq_tpgs = TPGS_FAILOVER_IMPLICIT;
2679 inq->inq_cmdque = 1;
2680
2681 if (sl->sl_flags & SL_VID_VALID) {
2682 bcopy(sl->sl_vendor_id, inq->inq_vid, 8);
2683 } else {
2684 bcopy(sbd_vendor_id, inq->inq_vid, 8);
2685 }
2686
2687 if (sl->sl_flags & SL_PID_VALID) {
2688 bcopy(sl->sl_product_id, inq->inq_pid, 16);
2689 } else {
2690 bcopy(sbd_product_id, inq->inq_pid, 16);
2691 }
2692
2693 if (sl->sl_flags & SL_REV_VALID) {
2694 bcopy(sl->sl_revision, inq->inq_revision, 4);
2695 } else {
2696 bcopy(sbd_revision, inq->inq_revision, 4);
2697 }
2698
2699 /* Adding Version Descriptors */
2758 mgmt_url_size = strlen(sl->sl_mgmt_url);
2759 mgmt_url = sl->sl_mgmt_url;
2760 } else if (sbd_mgmt_url) {
2761 mgmt_url_size = strlen(sbd_mgmt_url);
2762 mgmt_url = sbd_mgmt_url;
2763 }
2764
2765 /*
2766 * EVPD handling
2767 */
2768
2769 /* Default 512 bytes may not be enough, increase bsize if necessary */
2770 if (cdbp[2] == 0x83 || cdbp[2] == 0x85) {
2771 if (bsize < cmd_size)
2772 bsize = cmd_size;
2773 }
2774 p = (uint8_t *)kmem_zalloc(bsize, KM_SLEEP);
2775
2776 switch (cdbp[2]) {
2777 case 0x00:
2778 page_length = 4 + (mgmt_url_size ? 1 : 0);
2779 if (sl->sl_flags & SL_UNMAP_ENABLED)
2780 page_length += 2;
2781
2782 p[0] = byte0;
2783 p[3] = page_length;
2784 /* Supported VPD pages in ascending order */
2785 {
2786 uint8_t i = 5;
2787
2788 p[i++] = 0x80;
2789 p[i++] = 0x83;
2790 if (mgmt_url_size != 0)
2791 p[i++] = 0x85;
2792 p[i++] = 0x86;
2793 if (sl->sl_flags & SL_UNMAP_ENABLED) {
2794 p[i++] = 0xb0;
2795 p[i++] = 0xb2;
2796 }
2797 }
2798 xfer_size = page_length + 4;
2799 break;
2800
2801 case 0x80:
2802 if (sl->sl_serial_no_size) {
2803 page_length = sl->sl_serial_no_size;
2804 bcopy(sl->sl_serial_no, p + 4, sl->sl_serial_no_size);
2805 } else {
2806 /* if no serial num is specified set 4 spaces */
2807 page_length = 4;
2808 bcopy(" ", p + 4, 4);
2809 }
2810 p[0] = byte0;
2811 p[1] = 0x80;
2812 p[3] = page_length;
2813 xfer_size = page_length + 4;
2814 break;
2815
2816 case 0x83:
2817 xfer_size = stmf_scsilib_prepare_vpd_page83(task, p,
2818 bsize, byte0, STMF_VPD_LU_ID|STMF_VPD_TARGET_ID|
2819 STMF_VPD_TP_GROUP|STMF_VPD_RELATIVE_TP_ID);
2820 break;
2821
2822 case 0x85:
2823 if (mgmt_url_size == 0) {
2824 stmf_scsilib_send_status(task, STATUS_CHECK,
2825 STMF_SAA_INVALID_FIELD_IN_CDB);
2826 goto err_done;
2827 }
2828 {
2829 uint16_t idx, newidx, sz, url_size;
2830 char *url;
2831
2832 p[0] = byte0;
2833 p[1] = 0x85;
2834
2835 idx = 4;
2836 url = mgmt_url;
2837 url_size = sbd_parse_mgmt_url(&url);
2838 /* Creating Network Service Descriptors */
2839 while (url_size != 0) {
2840 /* Null terminated and 4 Byte aligned */
2841 sz = url_size + 1;
2842 sz += (sz % 4) ? 4 - (sz % 4) : 0;
2843 newidx = idx + sz + 4;
2844
2845 if (newidx < bsize) {
2846 /*
2847 * SPC-3r23 : Table 320 (Sec 7.6.5)
2868 }
2869
2870 case 0x86:
2871 page_length = 0x3c;
2872
2873 p[0] = byte0;
2874 p[1] = 0x86; /* Page 86 response */
2875 p[3] = page_length;
2876
2877 /*
2878 * Bits 0, 1, and 2 will need to be updated
2879 * to reflect the queue tag handling if/when
2880 * that is implemented. For now, we're going
2881 * to claim support only for Simple TA.
2882 */
2883 p[5] = 1;
2884 xfer_size = page_length + 4;
2885 break;
2886
2887 case 0xb0:
2888 if ((sl->sl_flags & SL_UNMAP_ENABLED) == 0) {
2889 stmf_scsilib_send_status(task, STATUS_CHECK,
2890 STMF_SAA_INVALID_FIELD_IN_CDB);
2891 goto err_done;
2892 }
2893 page_length = 0x3c;
2894 p[0] = byte0;
2895 p[1] = 0xb0;
2896 p[3] = page_length;
2897 p[20] = p[21] = p[22] = p[23] = 0xFF;
2898 p[24] = p[25] = p[26] = p[27] = 0xFF;
2899 xfer_size = page_length + 4;
2900 break;
2901
2902 case 0xb2:
2903 if ((sl->sl_flags & SL_UNMAP_ENABLED) == 0) {
2904 stmf_scsilib_send_status(task, STATUS_CHECK,
2905 STMF_SAA_INVALID_FIELD_IN_CDB);
2906 goto err_done;
2907 }
2908 page_length = 4;
2909 p[0] = byte0;
2910 p[1] = 0xb2;
2911 p[3] = page_length;
2912
2913 exp = (uint8_t)sl->sl_data_blocksize_shift;
2914 s = sl->sl_lu_size >> sl->sl_data_blocksize_shift;
2915 while (s & ((uint64_t)0xFFFFFFFF80000000ull)) {
2916 s >>= 1;
2917 exp++;
2918 }
2919 p[4] = exp;
2920 p[5] = 0xc0;
2921 xfer_size = page_length + 4;
2922 break;
2923
2924 default:
2925 stmf_scsilib_send_status(task, STATUS_CHECK,
2926 STMF_SAA_INVALID_FIELD_IN_CDB);
2927 goto err_done;
2928 }
2929
2930 sbd_handle_short_read_transfers(task, initial_dbuf, p, cmd_size,
2931 min(cmd_size, xfer_size));
2932 err_done:
2933 kmem_free(p, bsize);
2934 rw_exit(&sbd_global_prop_lock);
2935 }
2936
2937 stmf_status_t
2938 sbd_task_alloc(struct scsi_task *task)
2939 {
2940 if ((task->task_lu_private =
2941 kmem_alloc(sizeof (sbd_cmd_t), KM_NOSLEEP)) != NULL) {
2942 sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
2943 scmd->flags = 0;
2944 return (STMF_SUCCESS);
2945 }
2946 return (STMF_ALLOC_FAILURE);
2947 }
2948
2949 void
2950 sbd_remove_it_handle(sbd_lu_t *sl, sbd_it_data_t *it)
2951 {
2952 sbd_it_data_t **ppit;
2953
2954 sbd_pgr_remove_it_handle(sl, it);
2955 mutex_enter(&sl->sl_lock);
2956 for (ppit = &sl->sl_it_list; *ppit != NULL;
2957 ppit = &((*ppit)->sbd_it_next)) {
2958 if ((*ppit) == it) {
2959 *ppit = it->sbd_it_next;
2960 break;
2961 }
2987 break;
2988 }
2989 }
2990 ASSERT(it != NULL);
2991 } else {
2992 /*
2993 * We were passed an I_T nexus. If this nexus does not hold
2994 * the reservation, do nothing. This is why this function is
2995 * called "check_and_clear".
2996 */
2997 if ((it->sbd_it_flags & SBD_IT_HAS_SCSI2_RESERVATION) == 0) {
2998 mutex_exit(&sl->sl_lock);
2999 return;
3000 }
3001 }
3002 it->sbd_it_flags &= ~SBD_IT_HAS_SCSI2_RESERVATION;
3003 sl->sl_flags &= ~SL_LU_HAS_SCSI2_RESERVATION;
3004 mutex_exit(&sl->sl_lock);
3005 }
3006
3007
3008
3009 void
3010 sbd_new_task(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
3011 {
3012 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
3013 sbd_it_data_t *it;
3014 uint8_t cdb0, cdb1;
3015 stmf_status_t st_ret;
3016
3017 if ((it = task->task_lu_itl_handle) == NULL) {
3018 mutex_enter(&sl->sl_lock);
3019 for (it = sl->sl_it_list; it != NULL; it = it->sbd_it_next) {
3020 if (it->sbd_it_session_id ==
3021 task->task_session->ss_session_id) {
3022 mutex_exit(&sl->sl_lock);
3023 stmf_scsilib_send_status(task, STATUS_BUSY, 0);
3024 return;
3025 }
3026 }
3027 it = (sbd_it_data_t *)kmem_zalloc(sizeof (*it), KM_NOSLEEP);
3028 if (it == NULL) {
3052 }
3053 } else if (it->sbd_it_flags & SBD_IT_PGR_CHECK_FLAG) {
3054 mutex_enter(&sl->sl_lock);
3055 it->sbd_it_flags &= ~SBD_IT_PGR_CHECK_FLAG;
3056 mutex_exit(&sl->sl_lock);
3057 sbd_pgr_initialize_it(task, it);
3058 }
3059
3060 if (task->task_mgmt_function) {
3061 stmf_scsilib_handle_task_mgmt(task);
3062 return;
3063 }
3064
3065 /*
3066 * if we're transitioning between access
3067 * states, return NOT READY
3068 */
3069 if (sl->sl_access_state == SBD_LU_TRANSITION_TO_STANDBY ||
3070 sl->sl_access_state == SBD_LU_TRANSITION_TO_ACTIVE) {
3071 stmf_scsilib_send_status(task, STATUS_CHECK,
3072 STMF_SAA_LU_NO_ACCESS_UNAVAIL);
3073 return;
3074 }
3075
3076 /* Checking ua conditions as per SAM3R14 5.3.2 specified order */
3077 if ((it->sbd_it_ua_conditions) && (task->task_cdb[0] != SCMD_INQUIRY)) {
3078 uint32_t saa = 0;
3079
3080 mutex_enter(&sl->sl_lock);
3081 if (it->sbd_it_ua_conditions & SBD_UA_POR) {
3082 it->sbd_it_ua_conditions &= ~SBD_UA_POR;
3083 saa = STMF_SAA_POR;
3084 }
3085 mutex_exit(&sl->sl_lock);
3086 if (saa) {
3087 stmf_scsilib_send_status(task, STATUS_CHECK, saa);
3088 return;
3089 }
3090 }
3091
3092 /* Reservation conflict checks */
3093 if (sl->sl_access_state == SBD_LU_ACTIVE) {
3094 if (SBD_PGR_RSVD(sl->sl_pgr)) {
3095 if (sbd_pgr_reservation_conflict(task)) {
3096 stmf_scsilib_send_status(task,
3097 STATUS_RESERVATION_CONFLICT, 0);
3098 return;
3099 }
3100 } else if ((sl->sl_flags & SL_LU_HAS_SCSI2_RESERVATION) &&
3101 ((it->sbd_it_flags & SBD_IT_HAS_SCSI2_RESERVATION) == 0)) {
3102 if (!(SCSI2_CONFLICT_FREE_CMDS(task->task_cdb))) {
3103 stmf_scsilib_send_status(task,
3104 STATUS_RESERVATION_CONFLICT, 0);
3105 return;
3106 }
3107 }
3108 }
3109
3110 /* Rest of the ua conndition checks */
3111 if ((it->sbd_it_ua_conditions) && (task->task_cdb[0] != SCMD_INQUIRY)) {
3112 uint32_t saa = 0;
3113
3114 mutex_enter(&sl->sl_lock);
3115 if (it->sbd_it_ua_conditions & SBD_UA_CAPACITY_CHANGED) {
3116 it->sbd_it_ua_conditions &= ~SBD_UA_CAPACITY_CHANGED;
3117 if ((task->task_cdb[0] == SCMD_READ_CAPACITY) ||
3118 ((task->task_cdb[0] == SCMD_SVC_ACTION_IN_G4) &&
3119 (task->task_cdb[1] ==
3120 SSVC_ACTION_READ_CAPACITY_G4))) {
3121 saa = 0;
3122 } else {
3123 saa = STMF_SAA_CAPACITY_DATA_HAS_CHANGED;
3124 }
3125 } else if (it->sbd_it_ua_conditions &
3126 SBD_UA_MODE_PARAMETERS_CHANGED) {
3127 it->sbd_it_ua_conditions &=
3128 ~SBD_UA_MODE_PARAMETERS_CHANGED;
3129 saa = STMF_SAA_MODE_PARAMETERS_CHANGED;
3130 } else if (it->sbd_it_ua_conditions &
3131 SBD_UA_ASYMMETRIC_ACCESS_CHANGED) {
3132 it->sbd_it_ua_conditions &=
3133 ~SBD_UA_ASYMMETRIC_ACCESS_CHANGED;
3134 saa = STMF_SAA_ASYMMETRIC_ACCESS_CHANGED;
3135 } else if (it->sbd_it_ua_conditions &
3136 SBD_UA_ACCESS_STATE_TRANSITION) {
3137 it->sbd_it_ua_conditions &=
3138 ~SBD_UA_ACCESS_STATE_TRANSITION;
3139 saa = STMF_SAA_LU_NO_ACCESS_TRANSITION;
3140 } else {
3141 it->sbd_it_ua_conditions = 0;
3142 saa = 0;
3143 }
3144 mutex_exit(&sl->sl_lock);
3145 if (saa) {
3146 stmf_scsilib_send_status(task, STATUS_CHECK, saa);
3147 return;
3148 }
3149 }
3150
3151 cdb0 = task->task_cdb[0];
3152 cdb1 = task->task_cdb[1];
3153
3154 if (sl->sl_access_state == SBD_LU_STANDBY) {
3155 if (cdb0 != SCMD_INQUIRY &&
3156 cdb0 != SCMD_MODE_SENSE &&
3157 cdb0 != SCMD_MODE_SENSE_G1 &&
3158 cdb0 != SCMD_MODE_SELECT &&
3159 cdb0 != SCMD_MODE_SELECT_G1 &&
3160 cdb0 != SCMD_RESERVE &&
3161 cdb0 != SCMD_RELEASE &&
3162 cdb0 != SCMD_PERSISTENT_RESERVE_OUT &&
3163 cdb0 != SCMD_PERSISTENT_RESERVE_IN &&
3164 cdb0 != SCMD_REQUEST_SENSE &&
3165 cdb0 != SCMD_READ_CAPACITY &&
3166 cdb0 != SCMD_TEST_UNIT_READY &&
3167 cdb0 != SCMD_START_STOP &&
3168 cdb0 != SCMD_READ &&
3169 cdb0 != SCMD_READ_G1 &&
3170 cdb0 != SCMD_READ_G4 &&
3171 cdb0 != SCMD_READ_G5 &&
3172 !(cdb0 == SCMD_SVC_ACTION_IN_G4 &&
3173 cdb1 == SSVC_ACTION_READ_CAPACITY_G4) &&
3174 !(cdb0 == SCMD_MAINTENANCE_IN &&
3175 (cdb1 & 0x1F) == 0x05) &&
3176 !(cdb0 == SCMD_MAINTENANCE_IN &&
3177 (cdb1 & 0x1F) == 0x0A)) {
3178 stmf_scsilib_send_status(task, STATUS_CHECK,
3179 STMF_SAA_LU_NO_ACCESS_STANDBY);
3180 return;
3181 }
3182
3183 /*
3184 * is this a short write?
3185 * if so, we'll need to wait until we have the buffer
3186 * before proxying the command
3187 */
3188 switch (cdb0) {
3189 case SCMD_MODE_SELECT:
3190 case SCMD_MODE_SELECT_G1:
3191 case SCMD_PERSISTENT_RESERVE_OUT:
3192 break;
3193 default:
3194 st_ret = stmf_proxy_scsi_cmd(task,
3195 initial_dbuf);
3196 if (st_ret != STMF_SUCCESS) {
3197 stmf_scsilib_send_status(task,
3198 STATUS_CHECK,
3199 STMF_SAA_LU_NO_ACCESS_UNAVAIL);
3200 }
3201 return;
3202 }
3340 sbd_handle_mode_sense(task, initial_dbuf, p);
3341 kmem_free(p, 512);
3342 return;
3343 }
3344
3345 if ((cdb0 == SCMD_MODE_SELECT) || (cdb0 == SCMD_MODE_SELECT_G1)) {
3346 sbd_handle_mode_select(task, initial_dbuf);
3347 return;
3348 }
3349
3350 if ((cdb0 == SCMD_UNMAP) && (sl->sl_flags & SL_UNMAP_ENABLED)) {
3351 sbd_handle_unmap(task, initial_dbuf);
3352 return;
3353 }
3354
3355 if ((cdb0 == SCMD_WRITE_SAME_G4) || (cdb0 == SCMD_WRITE_SAME_G1)) {
3356 sbd_handle_write_same(task, initial_dbuf);
3357 return;
3358 }
3359
3360 if (cdb0 == SCMD_TEST_UNIT_READY) { /* Test unit ready */
3361 task->task_cmd_xfer_length = 0;
3362 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
3363 return;
3364 }
3365
3366 if (cdb0 == SCMD_READ_CAPACITY) { /* Read Capacity */
3367 sbd_handle_read_capacity(task, initial_dbuf);
3368 return;
3369 }
3370
3371 if (cdb0 == SCMD_SVC_ACTION_IN_G4) { /* Read Capacity or read long */
3372 if (cdb1 == SSVC_ACTION_READ_CAPACITY_G4) {
3373 sbd_handle_read_capacity(task, initial_dbuf);
3374 return;
3375 /*
3376 * } else if (cdb1 == SSVC_ACTION_READ_LONG_G4) {
3377 * sbd_handle_read(task, initial_dbuf);
3378 * return;
3379 */
3446 case (SBD_CMD_SCSI_WRITE):
3447 sbd_handle_sgl_write_xfer_completion(task, scmd, dbuf);
3448 break;
3449 default:
3450 cmn_err(CE_PANIC, "Unknown cmd type, task = %p",
3451 (void *)task);
3452 break;
3453 }
3454 return;
3455 }
3456
3457 if ((scmd == NULL) || ((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0))
3458 return;
3459
3460 switch (scmd->cmd_type) {
3461 case (SBD_CMD_SCSI_READ):
3462 sbd_handle_read_xfer_completion(task, scmd, dbuf);
3463 break;
3464
3465 case (SBD_CMD_SCSI_WRITE):
3466 if ((task->task_cdb[0] == SCMD_WRITE_SAME_G1) ||
3467 (task->task_cdb[0] == SCMD_WRITE_SAME_G4)) {
3468 sbd_handle_write_same_xfer_completion(task, scmd, dbuf,
3469 1);
3470 } else {
3471 sbd_handle_write_xfer_completion(task, scmd, dbuf, 1);
3472 }
3473 break;
3474
3475 case (SBD_CMD_SMALL_READ):
3476 sbd_handle_short_read_xfer_completion(task, scmd, dbuf);
3477 break;
3478
3479 case (SBD_CMD_SMALL_WRITE):
3480 sbd_handle_short_write_xfer_completion(task, dbuf);
3481 break;
3482
3483 default:
3484 cmn_err(CE_PANIC, "Unknown cmd type, task = %p", (void *)task);
3485 break;
3486 }
3487 }
3488
3489 /* ARGSUSED */
3490 void
3491 sbd_send_status_done(struct scsi_task *task)
3517 */
3518 /* ARGSUSED */
3519 stmf_status_t
3520 sbd_abort(struct stmf_lu *lu, int abort_cmd, void *arg, uint32_t flags)
3521 {
3522 sbd_lu_t *sl = (sbd_lu_t *)lu->lu_provider_private;
3523 scsi_task_t *task;
3524
3525 if (abort_cmd == STMF_LU_RESET_STATE) {
3526 return (sbd_lu_reset_state(lu));
3527 }
3528
3529 if (abort_cmd == STMF_LU_ITL_HANDLE_REMOVED) {
3530 sbd_check_and_clear_scsi2_reservation(sl, (sbd_it_data_t *)arg);
3531 sbd_remove_it_handle(sl, (sbd_it_data_t *)arg);
3532 return (STMF_SUCCESS);
3533 }
3534
3535 ASSERT(abort_cmd == STMF_LU_ABORT_TASK);
3536 task = (scsi_task_t *)arg;
3537 if (task->task_lu_private) {
3538 sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
3539
3540 if (scmd->flags & SBD_SCSI_CMD_ACTIVE) {
3541 if (scmd->flags & SBD_SCSI_CMD_TRANS_DATA) {
3542 kmem_free(scmd->trans_data,
3543 scmd->trans_data_len);
3544 scmd->flags &= ~SBD_SCSI_CMD_TRANS_DATA;
3545 }
3546 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
3547 return (STMF_ABORT_SUCCESS);
3548 }
3549 }
3550
3551 return (STMF_NOT_FOUND);
3552 }
3553
3554 /*
3555 * This function is called during task clean-up if the
3556 * DB_LU_FLAG is set on the dbuf. This should only be called for
3557 * abort processing after sbd_abort has been called for the task.
3558 */
3559 void
3560 sbd_dbuf_free(struct scsi_task *task, struct stmf_data_buf *dbuf)
3561 {
3562 sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
3563 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
3564
3565 ASSERT(dbuf->db_lu_private);
3566 ASSERT(scmd && scmd->nbufs > 0);
3567 ASSERT((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0);
3568 ASSERT(dbuf->db_flags & DB_LU_DATA_BUF);
3569 ASSERT(task->task_additional_flags & TASK_AF_ACCEPT_LU_DBUF);
3570 ASSERT((curthread->t_flag & T_INTR_THREAD) == 0);
3571
3572 if (scmd->cmd_type == SBD_CMD_SCSI_READ) {
3573 sbd_zvol_rele_read_bufs(sl, dbuf);
3574 } else if (scmd->cmd_type == SBD_CMD_SCSI_WRITE) {
3575 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
3576 } else {
3577 cmn_err(CE_PANIC, "Unknown cmd type %d, task = %p",
3578 scmd->cmd_type, (void *)task);
3579 }
3580 if (--scmd->nbufs == 0)
3581 rw_exit(&sl->sl_access_state_lock);
3582 stmf_teardown_dbuf(task, dbuf);
3583 stmf_free(dbuf);
3584 }
3585
3586 /* ARGSUSED */
3587 void
3588 sbd_ctl(struct stmf_lu *lu, int cmd, void *arg)
3589 {
3590 sbd_lu_t *sl = (sbd_lu_t *)lu->lu_provider_private;
3591 stmf_change_status_t st;
3592
3593 ASSERT((cmd == STMF_CMD_LU_ONLINE) ||
3594 (cmd == STMF_CMD_LU_OFFLINE) ||
3595 (cmd == STMF_ACK_LU_ONLINE_COMPLETE) ||
3596 (cmd == STMF_ACK_LU_OFFLINE_COMPLETE));
3597
3598 st.st_completion_status = STMF_SUCCESS;
3599 st.st_additional_info = NULL;
3600
3657 }
3658 } else {
3659 sl->sl_flags &= ~SL_WRITEBACK_CACHE_DISABLE;
3660 mutex_exit(&sl->sl_lock);
3661 if (sl->sl_access_state == SBD_LU_ACTIVE) {
3662 (void) sbd_wcd_set(0, sl);
3663 }
3664 }
3665 sbd_pgr_reset(sl);
3666 sbd_check_and_clear_scsi2_reservation(sl, NULL);
3667 if (stmf_deregister_all_lu_itl_handles(lu) != STMF_SUCCESS) {
3668 return (STMF_FAILURE);
3669 }
3670 return (STMF_SUCCESS);
3671 }
3672
3673 sbd_status_t
3674 sbd_flush_data_cache(sbd_lu_t *sl, int fsync_done)
3675 {
3676 int r = 0;
3677 int ret;
3678
3679 if (fsync_done)
3680 goto over_fsync;
3681 if ((sl->sl_data_vtype == VREG) || (sl->sl_data_vtype == VBLK)) {
3682 if (VOP_FSYNC(sl->sl_data_vp, FSYNC, kcred, NULL))
3683 return (SBD_FAILURE);
3684 }
3685 over_fsync:
3686 if (((sl->sl_data_vtype == VCHR) || (sl->sl_data_vtype == VBLK)) &&
3687 ((sl->sl_flags & SL_NO_DATA_DKIOFLUSH) == 0)) {
3688 ret = VOP_IOCTL(sl->sl_data_vp, DKIOCFLUSHWRITECACHE, NULL,
3689 FKIOCTL, kcred, &r, NULL);
3690 if ((ret == ENOTTY) || (ret == ENOTSUP)) {
3691 mutex_enter(&sl->sl_lock);
3692 sl->sl_flags |= SL_NO_DATA_DKIOFLUSH;
3693 mutex_exit(&sl->sl_lock);
3694 } else if (ret != 0) {
3695 return (SBD_FAILURE);
3696 }
3697 }
3698
3699 return (SBD_SUCCESS);
3700 }
3701
3702 /* ARGSUSED */
3703 static void
3704 sbd_handle_sync_cache(struct scsi_task *task,
3705 struct stmf_data_buf *initial_dbuf)
3706 {
3707 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
3708 uint64_t lba, laddr;
3709 sbd_status_t sret;
3710 uint32_t len;
3711 int is_g4 = 0;
3712 int immed;
3713
3714 task->task_cmd_xfer_length = 0;
3715 /*
3716 * Determine if this is a 10 or 16 byte CDB
3717 */
3718
3719 if (task->task_cdb[0] == SCMD_SYNCHRONIZE_CACHE_G4)
|
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2013 by Delphix. All rights reserved.
25 * Copyright 2016 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 #include <sys/conf.h>
29 #include <sys/file.h>
30 #include <sys/ddi.h>
31 #include <sys/sunddi.h>
32 #include <sys/modctl.h>
33 #include <sys/scsi/scsi.h>
34 #include <sys/scsi/impl/scsi_reset_notify.h>
35 #include <sys/scsi/generic/mode.h>
36 #include <sys/disp.h>
37 #include <sys/byteorder.h>
38 #include <sys/atomic.h>
39 #include <sys/sdt.h>
40 #include <sys/dkio.h>
41 #include <sys/dkioc_free_util.h>
42
43 #include <sys/stmf.h>
44 #include <sys/lpif.h>
45 #include <sys/portif.h>
46 #include <sys/stmf_ioctl.h>
47 #include <sys/stmf_sbd_ioctl.h>
48
49 #include "stmf_sbd.h"
50 #include "sbd_impl.h"
51
52 #define SCSI2_CONFLICT_FREE_CMDS(cdb) ( \
53 /* ----------------------- */ \
54 /* Refer Both */ \
55 /* SPC-2 (rev 20) Table 10 */ \
56 /* SPC-3 (rev 23) Table 31 */ \
57 /* ----------------------- */ \
58 ((cdb[0]) == SCMD_INQUIRY) || \
59 ((cdb[0]) == SCMD_LOG_SENSE_G1) || \
60 ((cdb[0]) == SCMD_RELEASE) || \
61 ((cdb[0]) == SCMD_RELEASE_G1) || \
71 /* REPORT TARGET PORT GROUPS (0x0A) REPORT TIMESTAMP (0x0F) */ \
72 (((cdb[0]) == SCMD_MAINTENANCE_IN) && ( \
73 (((cdb[1]) & 0x1F) == 0x0B) || \
74 (((cdb[1]) & 0x1F) == 0x05) || \
75 (((cdb[1]) & 0x1F) == 0x0E) || \
76 (((cdb[1]) & 0x1F) == 0x0A) || \
77 (((cdb[1]) & 0x1F) == 0x0F))) || \
78 /* ----------------------- */ \
79 /* SBC-3 (rev 17) Table 3 */ \
80 /* ----------------------- */ \
81 /* READ CAPACITY(10) */ \
82 ((cdb[0]) == SCMD_READ_CAPACITY) || \
83 /* READ CAPACITY(16) */ \
84 (((cdb[0]) == SCMD_SVC_ACTION_IN_G4) && ( \
85 ((cdb[1]) & 0x1F) == 0x10)) || \
86 /* START STOP UNIT with START bit 0 and POWER CONDITION 0 */ \
87 (((cdb[0]) == SCMD_START_STOP) && ( \
88 (((cdb[4]) & 0xF0) == 0) && (((cdb[4]) & 0x01) == 0))))
89 /* End of SCSI2_CONFLICT_FREE_CMDS */
90
91 uint8_t HardwareAcceleratedInit = 1;
92 uint8_t sbd_unmap_enable = 1; /* allow unmap by default */
93
94 /*
95 * An /etc/system tunable which specifies the maximum number of LBAs supported
96 * in a single UNMAP operation. Default is 0x002000 blocks or 4MB in size.
97 */
98 int stmf_sbd_unmap_max_nblks = 0x002000;
99
100 /*
101 * An /etc/system tunable which indicates if READ ops can run on the standby
102 * path or return an error.
103 */
104 int stmf_standby_fail_reads = 0;
105
106 stmf_status_t sbd_lu_reset_state(stmf_lu_t *lu);
107 static void sbd_handle_sync_cache(struct scsi_task *task,
108 struct stmf_data_buf *initial_dbuf);
109 void sbd_handle_read_xfer_completion(struct scsi_task *task,
110 sbd_cmd_t *scmd, struct stmf_data_buf *dbuf);
111 void sbd_handle_short_write_xfer_completion(scsi_task_t *task,
112 stmf_data_buf_t *dbuf);
113 void sbd_handle_mode_select_xfer(scsi_task_t *task, uint8_t *buf,
114 uint32_t buflen);
115 void sbd_handle_mode_select(scsi_task_t *task, stmf_data_buf_t *dbuf);
116 void sbd_handle_identifying_info(scsi_task_t *task, stmf_data_buf_t *dbuf);
117
118 static void sbd_handle_unmap_xfer(scsi_task_t *task, uint8_t *buf,
119 uint32_t buflen);
120 static void sbd_handle_unmap(scsi_task_t *task, stmf_data_buf_t *dbuf);
121
122 extern void sbd_pgr_initialize_it(scsi_task_t *, sbd_it_data_t *);
123 extern int sbd_pgr_reservation_conflict(scsi_task_t *, struct sbd_lu *sl);
124 extern void sbd_pgr_reset(sbd_lu_t *);
125 extern void sbd_pgr_remove_it_handle(sbd_lu_t *, sbd_it_data_t *);
126 extern void sbd_handle_pgr_in_cmd(scsi_task_t *, stmf_data_buf_t *);
127 extern void sbd_handle_pgr_out_cmd(scsi_task_t *, stmf_data_buf_t *);
128 extern void sbd_handle_pgr_out_data(scsi_task_t *, stmf_data_buf_t *);
129 void sbd_do_sgl_write_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
130 int first_xfer);
131 static void sbd_handle_write_same(scsi_task_t *task,
132 struct stmf_data_buf *initial_dbuf);
133 static void sbd_do_write_same_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
134 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable);
135 static void sbd_handle_write_same_xfer_completion(struct scsi_task *task,
136 sbd_cmd_t *scmd, struct stmf_data_buf *dbuf, uint8_t dbuf_reusable);
137
138 /*
139 * IMPORTANT NOTE:
140 * =================
141 * The whole world here is based on the assumption that everything within
142 * a scsi task executes in a single threaded manner, even the aborts.
143 * Dont ever change that. There wont be any performance gain but there
144 * will be tons of race conditions.
145 */
146
147 void
148 sbd_do_read_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
149 struct stmf_data_buf *dbuf)
150 {
151 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
152 uint64_t laddr;
153 uint32_t len, buflen, iolen;
154 int ndx;
155 int bufs_to_take;
156
157 /* Lets try not to hog all the buffers the port has. */
158 bufs_to_take = ((task->task_max_nbufs > 2) &&
159 (task->task_cmd_xfer_length < (32 * 1024))) ? 2 :
160 task->task_max_nbufs;
161
162 len = ATOMIC32_GET(scmd->len) > dbuf->db_buf_size ?
163 dbuf->db_buf_size : ATOMIC32_GET(scmd->len);
164 laddr = scmd->addr + scmd->current_ro;
165
166 for (buflen = 0, ndx = 0; (buflen < len) &&
167 (ndx < dbuf->db_sglist_length); ndx++) {
168 iolen = min(len - buflen, dbuf->db_sglist[ndx].seg_length);
169 if (iolen == 0)
170 break;
171 if (sbd_data_read(sl, task, laddr, (uint64_t)iolen,
172 dbuf->db_sglist[ndx].seg_addr) != STMF_SUCCESS) {
173 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
174 /* Do not need to do xfer anymore, just complete it */
175 dbuf->db_data_size = 0;
176 dbuf->db_xfer_status = STMF_SUCCESS;
177 sbd_handle_read_xfer_completion(task, scmd, dbuf);
178 return;
179 }
180 buflen += iolen;
181 laddr += (uint64_t)iolen;
182 }
183 dbuf->db_relative_offset = scmd->current_ro;
184 dbuf->db_data_size = buflen;
185 dbuf->db_flags = DB_DIRECTION_TO_RPORT;
186 (void) stmf_xfer_data(task, dbuf, 0);
187 atomic_add_32(&scmd->len, -buflen);
188 scmd->current_ro += buflen;
189 if (ATOMIC32_GET(scmd->len) &&
190 (ATOMIC8_GET(scmd->nbufs) < bufs_to_take)) {
191 uint32_t maxsize, minsize, old_minsize;
192
193 maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ? 128*1024 :
194 ATOMIC32_GET(scmd->len);
195 minsize = maxsize >> 2;
196 do {
197 /*
198 * A bad port implementation can keep on failing the
199 * the request but keep on sending us a false
200 * minsize.
201 */
202 old_minsize = minsize;
203 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
204 } while ((dbuf == NULL) && (old_minsize > minsize) &&
205 (minsize >= 512));
206 if (dbuf == NULL) {
207 return;
208 }
209 atomic_inc_8(&scmd->nbufs);
210 sbd_do_read_xfer(task, scmd, dbuf);
211 }
212 }
213
214 /*
215 * sbd_zcopy: Bail-out switch for reduced copy path.
216 *
217 * 0 - read & write off
218 * 1 - read & write on
219 * 2 - only read on
220 * 4 - only write on
221 */
222 int sbd_zcopy = 1; /* enable zcopy read & write path */
223 uint32_t sbd_max_xfer_len = 0; /* Valid if non-zero */
224 uint32_t sbd_1st_xfer_len = 0; /* Valid if non-zero */
225 uint32_t sbd_copy_threshold = 0; /* Valid if non-zero */
226
227 static void
228 sbd_do_sgl_read_xfer(struct scsi_task *task, sbd_cmd_t *scmd, int first_xfer)
229 {
230 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
231 sbd_zvol_io_t *zvio;
232 int ret, final_xfer;
233 uint64_t offset;
234 uint32_t xfer_len, max_len, first_len;
235 stmf_status_t xstat;
236 stmf_data_buf_t *dbuf;
237 uint_t nblks;
238 uint64_t blksize = sl->sl_blksize;
239 size_t db_private_sz;
240 hrtime_t xfer_start;
241 uintptr_t pad;
242
243 ASSERT(rw_read_held(&sl->sl_access_state_lock));
244 ASSERT((sl->sl_flags & SL_MEDIA_LOADED) != 0);
245
246 /*
247 * Calculate the limits on xfer_len to the minimum of :
248 * - task limit
249 * - lun limit
250 * - sbd global limit if set
251 * - first xfer limit if set
252 *
253 * First, protect against silly over-ride value
254 */
255 if (sbd_max_xfer_len && ((sbd_max_xfer_len % DEV_BSIZE) != 0)) {
256 cmn_err(CE_WARN, "sbd_max_xfer_len invalid %d, resetting\n",
257 sbd_max_xfer_len);
258 sbd_max_xfer_len = 0;
259 }
260 if (sbd_1st_xfer_len && ((sbd_1st_xfer_len % DEV_BSIZE) != 0)) {
262 sbd_1st_xfer_len);
263 sbd_1st_xfer_len = 0;
264 }
265
266 max_len = MIN(task->task_max_xfer_len, sl->sl_max_xfer_len);
267 if (sbd_max_xfer_len)
268 max_len = MIN(max_len, sbd_max_xfer_len);
269 /*
270 * Special case the first xfer if hints are set.
271 */
272 if (first_xfer && (sbd_1st_xfer_len || task->task_1st_xfer_len)) {
273 /* global over-ride has precedence */
274 if (sbd_1st_xfer_len)
275 first_len = sbd_1st_xfer_len;
276 else
277 first_len = task->task_1st_xfer_len;
278 } else {
279 first_len = 0;
280 }
281
282 while (ATOMIC32_GET(scmd->len) &&
283 ATOMIC8_GET(scmd->nbufs) < task->task_max_nbufs) {
284
285 xfer_len = MIN(max_len, ATOMIC32_GET(scmd->len));
286 if (first_len) {
287 xfer_len = MIN(xfer_len, first_len);
288 first_len = 0;
289 }
290 if (ATOMIC32_GET(scmd->len) == xfer_len) {
291 final_xfer = 1;
292 } else {
293 /*
294 * Attempt to end xfer on a block boundary.
295 * The only way this does not happen is if the
296 * xfer_len is small enough to stay contained
297 * within the same block.
298 */
299 uint64_t xfer_offset, xfer_aligned_end;
300
301 final_xfer = 0;
302 xfer_offset = scmd->addr + scmd->current_ro;
303 xfer_aligned_end =
304 P2ALIGN(xfer_offset+xfer_len, blksize);
305 if (xfer_aligned_end > xfer_offset)
306 xfer_len = xfer_aligned_end - xfer_offset;
307 }
308 /*
309 * Allocate object to track the read and reserve
310 * enough space for scatter/gather list.
337 dbuf->db_data_size = xfer_len;
338 dbuf->db_relative_offset = scmd->current_ro;
339 dbuf->db_sglist_length = (uint16_t)nblks;
340 dbuf->db_xfer_status = 0;
341 dbuf->db_handle = 0;
342
343 dbuf->db_flags = (DB_DONT_CACHE | DB_DONT_REUSE |
344 DB_DIRECTION_TO_RPORT | DB_LU_DATA_BUF);
345 if (final_xfer)
346 dbuf->db_flags |= DB_SEND_STATUS_GOOD;
347
348 zvio = dbuf->db_lu_private;
349 /* Need absolute offset for zvol access */
350 zvio->zvio_offset = offset;
351 zvio->zvio_flags = ZVIO_SYNC;
352
353 /*
354 * Accounting for start of read.
355 * Note there is no buffer address for the probe yet.
356 */
357 xfer_start = gethrtime();
358 DTRACE_PROBE5(backing__store__read__start, sbd_lu_t *, sl,
359 uint8_t *, NULL, uint64_t, xfer_len,
360 uint64_t, offset, scsi_task_t *, task);
361
362 ret = sbd_zvol_alloc_read_bufs(sl, dbuf);
363
364 stmf_lu_xfer_done(task, B_TRUE /* read */,
365 (gethrtime() - xfer_start));
366 DTRACE_PROBE6(backing__store__read__end, sbd_lu_t *, sl,
367 uint8_t *, NULL, uint64_t, xfer_len,
368 uint64_t, offset, int, ret, scsi_task_t *, task);
369
370 if (ret != 0) {
371 /*
372 * Read failure from the backend.
373 */
374 stmf_free(dbuf);
375 if (ATOMIC8_GET(scmd->nbufs) == 0) {
376 /* nothing queued, just finish */
377 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
378 sbd_ats_remove_by_task(task);
379 stmf_scsilib_send_status(task, STATUS_CHECK,
380 STMF_SAA_READ_ERROR);
381 rw_exit(&sl->sl_access_state_lock);
382 } else {
383 /* process failure when other dbufs finish */
384 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
385 }
386 return;
387 }
388
389 /*
390 * Allow PP to do setup
391 */
392 xstat = stmf_setup_dbuf(task, dbuf, 0);
393 if (xstat != STMF_SUCCESS) {
394 /*
395 * This could happen if the driver cannot get the
396 * DDI resources it needs for this request.
397 * If other dbufs are queued, try again when the next
398 * one completes, otherwise give up.
399 */
400 sbd_zvol_rele_read_bufs(sl, dbuf);
401 stmf_free(dbuf);
402 if (ATOMIC8_GET(scmd->nbufs) > 0) {
403 /* completion of previous dbuf will retry */
404 return;
405 }
406 /*
407 * Done with this command.
408 */
409 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
410 sbd_ats_remove_by_task(task);
411 if (first_xfer)
412 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
413 else
414 stmf_scsilib_send_status(task, STATUS_CHECK,
415 STMF_SAA_READ_ERROR);
416 rw_exit(&sl->sl_access_state_lock);
417 return;
418 }
419 /*
420 * dbuf is now queued on task
421 */
422 atomic_inc_8(&scmd->nbufs);
423
424 /* XXX leave this in for FW? */
425 DTRACE_PROBE4(sbd__xfer, struct scsi_task *, task,
426 struct stmf_data_buf *, dbuf, uint64_t, offset,
427 uint32_t, xfer_len);
428 /*
429 * Do not pass STMF_IOF_LU_DONE so that the zvol
430 * state can be released in the completion callback.
431 */
432 xstat = stmf_xfer_data(task, dbuf, 0);
433 switch (xstat) {
434 case STMF_SUCCESS:
435 break;
436 case STMF_BUSY:
437 /*
438 * The dbuf is queued on the task, but unknown
439 * to the PP, thus no completion will occur.
440 */
441 sbd_zvol_rele_read_bufs(sl, dbuf);
442 stmf_teardown_dbuf(task, dbuf);
443 stmf_free(dbuf);
444 atomic_dec_8(&scmd->nbufs);
445 if (ATOMIC8_GET(scmd->nbufs) > 0) {
446 /* completion of previous dbuf will retry */
447 return;
448 }
449 /*
450 * Done with this command.
451 */
452 rw_exit(&sl->sl_access_state_lock);
453 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
454 sbd_ats_remove_by_task(task);
455 if (first_xfer)
456 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
457 else
458 stmf_scsilib_send_status(task, STATUS_CHECK,
459 STMF_SAA_READ_ERROR);
460 return;
461 case STMF_ABORTED:
462 /*
463 * Completion from task_done will cleanup
464 */
465 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
466 sbd_ats_remove_by_task(task);
467 return;
468 }
469 /*
470 * Update the xfer progress.
471 */
472 ASSERT(scmd->len >= xfer_len);
473 atomic_add_32(&scmd->len, -xfer_len);
474 scmd->current_ro += xfer_len;
475 }
476 }
477
478 void
479 sbd_handle_read_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
480 struct stmf_data_buf *dbuf)
481 {
482 if (dbuf->db_xfer_status != STMF_SUCCESS) {
483 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
484 dbuf->db_xfer_status, NULL);
485 return;
486 }
487 task->task_nbytes_transferred += dbuf->db_data_size;
488 if (ATOMIC32_GET(scmd->len) == 0 ||
489 scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
490 stmf_free_dbuf(task, dbuf);
491 atomic_dec_8(&scmd->nbufs);
492 if (ATOMIC8_GET(scmd->nbufs))
493 return; /* wait for all buffers to complete */
494 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
495 sbd_ats_remove_by_task(task);
496 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL)
497 stmf_scsilib_send_status(task, STATUS_CHECK,
498 STMF_SAA_READ_ERROR);
499 else
500 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
501 return;
502 }
503 if (dbuf->db_flags & DB_DONT_REUSE) {
504 /* allocate new dbuf */
505 uint32_t maxsize, minsize, old_minsize;
506 stmf_free_dbuf(task, dbuf);
507
508 maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ?
509 128*1024 : ATOMIC32_GET(scmd->len);
510 minsize = maxsize >> 2;
511 do {
512 old_minsize = minsize;
513 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
514 } while ((dbuf == NULL) && (old_minsize > minsize) &&
515 (minsize >= 512));
516 if (dbuf == NULL) {
517 atomic_dec_8(&scmd->nbufs);
518 if (ATOMIC8_GET(scmd->nbufs) == 0) {
519 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
520 STMF_ALLOC_FAILURE, NULL);
521 }
522 return;
523 }
524 }
525 sbd_do_read_xfer(task, scmd, dbuf);
526 }
527
528 /*
529 * This routine must release the DMU resources and free the dbuf
530 * in all cases. If this is the final dbuf of the task, then drop
531 * the reader lock on the LU state. If there are no errors and more
532 * work to do, then queue more xfer operations.
533 */
534 void
535 sbd_handle_sgl_read_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
536 struct stmf_data_buf *dbuf)
537 {
538 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
539 stmf_status_t xfer_status;
540 uint32_t data_size;
541 int scmd_err;
542
543 ASSERT(dbuf->db_lu_private);
544 ASSERT(scmd->cmd_type == SBD_CMD_SCSI_READ);
545
546 atomic_dec_8(&scmd->nbufs); /* account for this dbuf */
547 /*
548 * Release the DMU resources.
549 */
550 sbd_zvol_rele_read_bufs(sl, dbuf);
551 /*
552 * Release the dbuf after retrieving needed fields.
553 */
554 xfer_status = dbuf->db_xfer_status;
555 data_size = dbuf->db_data_size;
556 stmf_teardown_dbuf(task, dbuf);
557 stmf_free(dbuf);
558 /*
559 * Release the state lock if this is the last completion.
560 * If this is the last dbuf on task and all data has been
561 * transferred or an error encountered, then no more dbufs
562 * will be queued.
563 */
564 scmd_err = (((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
565 (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) ||
566 (xfer_status != STMF_SUCCESS));
567 if ((ATOMIC8_GET(scmd->nbufs) == 0) &&
568 (ATOMIC32_GET(scmd->len) == 0 || scmd_err)) {
569 /* all DMU state has been released */
570 rw_exit(&sl->sl_access_state_lock);
571 }
572
573 /*
574 * If there have been no errors, either complete the task
575 * or issue more data xfer operations.
576 */
577 if (!scmd_err) {
578 /*
579 * This chunk completed successfully
580 */
581 task->task_nbytes_transferred += data_size;
582 if (ATOMIC8_GET(scmd->nbufs) == 0 &&
583 ATOMIC32_GET(scmd->len) == 0) {
584 /*
585 * This command completed successfully
586 *
587 * Status was sent along with data, so no status
588 * completion will occur. Tell stmf we are done.
589 */
590 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
591 sbd_ats_remove_by_task(task);
592 stmf_task_lu_done(task);
593 return;
594 }
595 /*
596 * Start more xfers
597 */
598 sbd_do_sgl_read_xfer(task, scmd, 0);
599 return;
600 }
601 /*
602 * Sort out the failure
603 */
604 if (scmd->flags & SBD_SCSI_CMD_ACTIVE) {
605 /*
606 * If a previous error occurred, leave the command active
607 * and wait for the last completion to send the status check.
608 */
609 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
610 if (ATOMIC8_GET(scmd->nbufs) == 0) {
611 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
612 sbd_ats_remove_by_task(task);
613 stmf_scsilib_send_status(task, STATUS_CHECK,
614 STMF_SAA_READ_ERROR);
615 }
616 return;
617 }
618 /*
619 * Must have been a failure on current dbuf
620 */
621 ASSERT(xfer_status != STMF_SUCCESS);
622
623 /*
624 * Actually this is a bug. stmf abort should have reset the
625 * active flag but since its been there for some time.
626 * I wont change it.
627 */
628 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
629 sbd_ats_remove_by_task(task);
630 stmf_abort(STMF_QUEUE_TASK_ABORT, task, xfer_status, NULL);
631 }
632 }
633
634 void
635 sbd_handle_sgl_write_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
636 struct stmf_data_buf *dbuf)
637 {
638 sbd_zvol_io_t *zvio = dbuf->db_lu_private;
639 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
640 int ret;
641 int scmd_err, scmd_xfer_done;
642 stmf_status_t xfer_status = dbuf->db_xfer_status;
643 uint32_t data_size = dbuf->db_data_size;
644 hrtime_t xfer_start;
645
646 ASSERT(zvio);
647
648 /*
649 * Allow PP to free up resources before releasing the write bufs
650 * as writing to the backend could take some time.
651 */
652 stmf_teardown_dbuf(task, dbuf);
653
654 atomic_dec_8(&scmd->nbufs); /* account for this dbuf */
655 /*
656 * All data was queued and this is the last completion,
657 * but there could still be an error.
658 */
659 scmd_xfer_done = (ATOMIC32_GET(scmd->len) == 0 &&
660 (ATOMIC8_GET(scmd->nbufs) == 0));
661 scmd_err = (((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
662 (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) ||
663 (xfer_status != STMF_SUCCESS));
664
665 xfer_start = gethrtime();
666 DTRACE_PROBE5(backing__store__write__start, sbd_lu_t *, sl,
667 uint8_t *, NULL, uint64_t, data_size,
668 uint64_t, zvio->zvio_offset, scsi_task_t *, task);
669
670 if (scmd_err) {
671 /* just return the write buffers */
672 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
673 ret = 0;
674 } else {
675 if (scmd_xfer_done)
676 zvio->zvio_flags = ZVIO_COMMIT;
677 else
678 zvio->zvio_flags = 0;
679 /* write the data */
680 ret = sbd_zvol_rele_write_bufs(sl, dbuf);
681 }
682
683 stmf_lu_xfer_done(task, B_FALSE /* write */,
684 (gethrtime() - xfer_start));
685 DTRACE_PROBE6(backing__store__write__end, sbd_lu_t *, sl,
686 uint8_t *, NULL, uint64_t, data_size,
687 uint64_t, zvio->zvio_offset, int, ret, scsi_task_t *, task);
688
689 if (ret != 0) {
690 /* update the error flag */
691 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
692 scmd_err = 1;
693 }
694
695 /* Release the dbuf */
696 stmf_free(dbuf);
697
698 /*
699 * Release the state lock if this is the last completion.
700 * If this is the last dbuf on task and all data has been
701 * transferred or an error encountered, then no more dbufs
702 * will be queued.
703 */
704 if ((ATOMIC8_GET(scmd->nbufs) == 0) &&
705 (ATOMIC32_GET(scmd->len) == 0 || scmd_err)) {
706 /* all DMU state has been released */
707 rw_exit(&sl->sl_access_state_lock);
708 }
709 /*
710 * If there have been no errors, either complete the task
711 * or issue more data xfer operations.
712 */
713 if (!scmd_err) {
714 /* This chunk completed successfully */
715 task->task_nbytes_transferred += data_size;
716 if (scmd_xfer_done) {
717 /* This command completed successfully */
718 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
719 sbd_ats_remove_by_task(task);
720 if ((scmd->flags & SBD_SCSI_CMD_SYNC_WRITE) &&
721 (sbd_flush_data_cache(sl, 0) != SBD_SUCCESS)) {
722 stmf_scsilib_send_status(task, STATUS_CHECK,
723 STMF_SAA_WRITE_ERROR);
724 } else {
725 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
726 }
727 return;
728 }
729 /*
730 * Start more xfers
731 */
732 sbd_do_sgl_write_xfer(task, scmd, 0);
733 return;
734 }
735 /*
736 * Sort out the failure
737 */
738 if (scmd->flags & SBD_SCSI_CMD_ACTIVE) {
739 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
740 if (ATOMIC8_GET(scmd->nbufs) == 0) {
741 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
742 sbd_ats_remove_by_task(task);
743 stmf_scsilib_send_status(task, STATUS_CHECK,
744 STMF_SAA_WRITE_ERROR);
745 }
746 /*
747 * Leave the command active until last dbuf completes.
748 */
749 return;
750 }
751 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
752 sbd_ats_remove_by_task(task);
753 ASSERT(xfer_status != STMF_SUCCESS);
754 stmf_abort(STMF_QUEUE_TASK_ABORT, task, xfer_status, NULL);
755 }
756 }
757
758 /*
759 * Handle a copy operation using the zvol interface.
760 *
761 * Similar to the sbd_data_read/write path, except it goes directly through
762 * the zvol interfaces. It can pass a port provider sglist in the
763 * form of uio which is lost through the vn_rdwr path.
764 *
765 * Returns:
766 * STMF_SUCCESS - request handled
767 * STMF_FAILURE - request not handled, caller must deal with error
768 */
769 static stmf_status_t
770 sbd_copy_rdwr(scsi_task_t *task, uint64_t laddr, stmf_data_buf_t *dbuf,
771 int cmd, int commit)
772 {
773 sbd_lu_t *sl = task->task_lu->lu_provider_private;
774 struct uio uio;
775 struct iovec *iov, *tiov, iov1[8];
776 uint32_t len, resid;
777 int ret, i, iovcnt, flags;
778 hrtime_t xfer_start;
779 boolean_t is_read;
780
781 ASSERT(cmd == SBD_CMD_SCSI_READ || cmd == SBD_CMD_SCSI_WRITE);
782
783 is_read = (cmd == SBD_CMD_SCSI_READ) ? B_TRUE : B_FALSE;
784 iovcnt = dbuf->db_sglist_length;
785 /* use the stack for small iovecs */
786 if (iovcnt > 8) {
787 iov = kmem_alloc(iovcnt * sizeof (*iov), KM_SLEEP);
788 } else {
789 iov = &iov1[0];
790 }
791
792 /* Convert dbuf sglist to iovec format */
793 len = dbuf->db_data_size;
794 resid = len;
795 tiov = iov;
796 for (i = 0; i < iovcnt; i++) {
797 tiov->iov_base = (caddr_t)dbuf->db_sglist[i].seg_addr;
798 tiov->iov_len = MIN(resid, dbuf->db_sglist[i].seg_length);
799 resid -= tiov->iov_len;
800 tiov++;
801 }
802 if (resid != 0) {
803 cmn_err(CE_WARN, "inconsistant sglist rem %d", resid);
804 if (iov != &iov1[0])
805 kmem_free(iov, iovcnt * sizeof (*iov));
806 return (STMF_FAILURE);
807 }
808 /* Setup the uio struct */
809 uio.uio_iov = iov;
810 uio.uio_iovcnt = iovcnt;
811 uio.uio_loffset = laddr;
812 uio.uio_segflg = (short)UIO_SYSSPACE;
813 uio.uio_resid = (uint64_t)len;
814 uio.uio_llimit = RLIM64_INFINITY;
815
816 xfer_start = gethrtime();
817 if (is_read == B_TRUE) {
818 uio.uio_fmode = FREAD;
819 uio.uio_extflg = UIO_COPY_CACHED;
820 DTRACE_PROBE5(backing__store__read__start, sbd_lu_t *, sl,
821 uint8_t *, NULL, uint64_t, len, uint64_t, laddr,
822 scsi_task_t *, task);
823
824 /* Fetch the data */
825 ret = sbd_zvol_copy_read(sl, &uio);
826
827 DTRACE_PROBE6(backing__store__read__end, sbd_lu_t *, sl,
828 uint8_t *, NULL, uint64_t, len, uint64_t, laddr, int, ret,
829 scsi_task_t *, task);
830 } else {
831 uio.uio_fmode = FWRITE;
832 uio.uio_extflg = UIO_COPY_DEFAULT;
833 DTRACE_PROBE5(backing__store__write__start, sbd_lu_t *, sl,
834 uint8_t *, NULL, uint64_t, len, uint64_t, laddr,
835 scsi_task_t *, task);
836
837 flags = (commit) ? ZVIO_COMMIT : 0;
838 /* Write the data */
839 ret = sbd_zvol_copy_write(sl, &uio, flags);
840
841 DTRACE_PROBE6(backing__store__write__end, sbd_lu_t *, sl,
842 uint8_t *, NULL, uint64_t, len, uint64_t, laddr, int, ret,
843 scsi_task_t *, task);
844 }
845 /* finalize accounting */
846 stmf_lu_xfer_done(task, is_read, (gethrtime() - xfer_start));
847
848 if (iov != &iov1[0])
849 kmem_free(iov, iovcnt * sizeof (*iov));
850 if (ret != 0) {
851 /* Backend I/O error */
852 return (STMF_FAILURE);
853 }
854 return (STMF_SUCCESS);
855 }
856
857 void
858 sbd_handle_read(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
859 {
860 uint64_t lba, laddr;
861 uint64_t blkcount;
862 uint32_t len;
863 uint8_t op = task->task_cdb[0];
864 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
865 sbd_cmd_t *scmd;
866 stmf_data_buf_t *dbuf;
867 int fast_path;
868 boolean_t fua_bit = B_FALSE;
869
870 /*
871 * Check to see if the command is READ(10), READ(12), or READ(16).
872 * If it is then check for bit 3 being set to indicate if Forced
873 * Unit Access is being requested. If so, we'll bypass the use of
874 * DMA buffers to simplify support of this feature.
875 */
876 if (((op == SCMD_READ_G1) || (op == SCMD_READ_G4) ||
877 (op == SCMD_READ_G5)) &&
878 (task->task_cdb[1] & BIT_3)) {
879 fua_bit = B_TRUE;
880 }
881 if (op == SCMD_READ) {
882 lba = READ_SCSI21(&task->task_cdb[1], uint64_t);
883 len = (uint32_t)task->task_cdb[4];
884
885 if (len == 0) {
886 len = 256;
887 }
888 } else if (op == SCMD_READ_G1) {
889 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
890 len = READ_SCSI16(&task->task_cdb[7], uint32_t);
891 } else if (op == SCMD_READ_G5) {
892 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
893 len = READ_SCSI32(&task->task_cdb[6], uint32_t);
894 } else if (op == SCMD_READ_G4) {
895 lba = READ_SCSI64(&task->task_cdb[2], uint64_t);
896 len = READ_SCSI32(&task->task_cdb[10], uint32_t);
897 } else {
898 stmf_scsilib_send_status(task, STATUS_CHECK,
899 STMF_SAA_INVALID_OPCODE);
900 return;
901 }
902
903 laddr = lba << sl->sl_data_blocksize_shift;
904 blkcount = len;
905 len <<= sl->sl_data_blocksize_shift;
906
907 if ((laddr + (uint64_t)len) > sl->sl_lu_size) {
908 stmf_scsilib_send_status(task, STATUS_CHECK,
909 STMF_SAA_LBA_OUT_OF_RANGE);
910 return;
911 }
912
913 task->task_cmd_xfer_length = len;
914 if (task->task_additional_flags & TASK_AF_NO_EXPECTED_XFER_LENGTH) {
915 task->task_expected_xfer_length = len;
916 }
917
918 if (len != task->task_expected_xfer_length) {
919 fast_path = 0;
920 len = (len > task->task_expected_xfer_length) ?
921 task->task_expected_xfer_length : len;
922 } else {
923 fast_path = 1;
924 }
925
926 if (len == 0) {
927 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
928 return;
929 }
930
931 if (sbd_ats_handling_before_io(task, sl, lba, blkcount) !=
932 SBD_SUCCESS) {
933 if (stmf_task_poll_lu(task, 10) != STMF_SUCCESS) {
934 stmf_scsilib_send_status(task, STATUS_BUSY, 0);
935 }
936 return;
937 }
938 /*
939 * Determine if this read can directly use DMU buffers.
940 */
941 if (sbd_zcopy & (2|1) && /* Debug switch */
942 initial_dbuf == NULL && /* No PP buffer passed in */
943 sl->sl_flags & SL_CALL_ZVOL && /* zvol backing store */
944 (task->task_additional_flags &
945 TASK_AF_ACCEPT_LU_DBUF) && /* PP allows it */
946 !fua_bit) {
947 /*
948 * Reduced copy path
949 */
950 uint32_t copy_threshold, minsize;
951 int ret;
952
953 /*
954 * The sl_access_state_lock will be held shared
955 * for the entire request and released when all
956 * dbufs have completed.
957 */
958 rw_enter(&sl->sl_access_state_lock, RW_READER);
959 if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
960 rw_exit(&sl->sl_access_state_lock);
961 sbd_ats_remove_by_task(task);
962 stmf_scsilib_send_status(task, STATUS_CHECK,
963 STMF_SAA_READ_ERROR);
964 return;
965 }
966
967 /*
968 * Check if setup is more expensive than copying the data.
969 *
970 * Use the global over-ride sbd_zcopy_threshold if set.
971 */
972 copy_threshold = (sbd_copy_threshold > 0) ?
973 sbd_copy_threshold : task->task_copy_threshold;
974 minsize = len;
975 if (len < copy_threshold &&
976 (dbuf = stmf_alloc_dbuf(task, len, &minsize, 0)) != 0) {
977
978 ret = sbd_copy_rdwr(task, laddr, dbuf,
979 SBD_CMD_SCSI_READ, 0);
980 /* done with the backend */
981 rw_exit(&sl->sl_access_state_lock);
982 sbd_ats_remove_by_task(task);
983 if (ret != 0) {
984 /* backend error */
985 stmf_scsilib_send_status(task, STATUS_CHECK,
986 STMF_SAA_READ_ERROR);
987 } else {
988 /* send along good data */
989 dbuf->db_relative_offset = 0;
990 dbuf->db_data_size = len;
991 dbuf->db_flags = DB_SEND_STATUS_GOOD |
992 DB_DIRECTION_TO_RPORT;
993 /* XXX keep for FW? */
994 DTRACE_PROBE4(sbd__xfer,
995 struct scsi_task *, task,
996 struct stmf_data_buf *, dbuf,
997 uint64_t, laddr, uint32_t, len);
998 (void) stmf_xfer_data(task, dbuf,
999 STMF_IOF_LU_DONE);
1000 }
1001 return;
1002 }
1003
1004 /* committed to reduced copy */
1005 if (task->task_lu_private) {
1006 scmd = (sbd_cmd_t *)task->task_lu_private;
1007 } else {
1008 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t),
1009 KM_SLEEP);
1010 task->task_lu_private = scmd;
1011 }
1012 /*
1013 * Setup scmd to track read progress.
1014 */
1015 scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED;
1016 scmd->cmd_type = SBD_CMD_SCSI_READ;
1017 scmd->nbufs = 0;
1018 scmd->addr = laddr;
1019 scmd->len = len;
1020 scmd->current_ro = 0;
1021 /*
1022 * Kick-off the read.
1023 */
1024 sbd_do_sgl_read_xfer(task, scmd, 1);
1025 return;
1026 }
1027
1028 if (initial_dbuf == NULL) {
1029 uint32_t maxsize, minsize, old_minsize;
1030
1031 maxsize = (len > (128*1024)) ? 128*1024 : len;
1032 minsize = maxsize >> 2;
1033 do {
1034 old_minsize = minsize;
1035 initial_dbuf = stmf_alloc_dbuf(task, maxsize,
1036 &minsize, 0);
1037 } while ((initial_dbuf == NULL) && (old_minsize > minsize) &&
1038 (minsize >= 512));
1039 if (initial_dbuf == NULL) {
1040 sbd_ats_remove_by_task(task);
1041 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
1042 return;
1043 }
1044 }
1045 dbuf = initial_dbuf;
1046
1047 if ((dbuf->db_buf_size >= len) && fast_path &&
1048 (dbuf->db_sglist_length == 1)) {
1049 if (sbd_data_read(sl, task, laddr, (uint64_t)len,
1050 dbuf->db_sglist[0].seg_addr) == STMF_SUCCESS) {
1051 dbuf->db_relative_offset = 0;
1052 dbuf->db_data_size = len;
1053 dbuf->db_flags = DB_SEND_STATUS_GOOD |
1054 DB_DIRECTION_TO_RPORT;
1055 /* XXX keep for FW? */
1056 DTRACE_PROBE4(sbd__xfer, struct scsi_task *, task,
1057 struct stmf_data_buf *, dbuf,
1058 uint64_t, laddr, uint32_t, len);
1059 (void) stmf_xfer_data(task, dbuf, STMF_IOF_LU_DONE);
1060 } else {
1061 stmf_scsilib_send_status(task, STATUS_CHECK,
1062 STMF_SAA_READ_ERROR);
1063 }
1064 sbd_ats_remove_by_task(task);
1065 return;
1066 }
1067
1068 if (task->task_lu_private) {
1069 scmd = (sbd_cmd_t *)task->task_lu_private;
1070 } else {
1071 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
1072 task->task_lu_private = scmd;
1073 }
1074 scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED;
1075 scmd->cmd_type = SBD_CMD_SCSI_READ;
1076 scmd->nbufs = 1;
1077 scmd->addr = laddr;
1078 scmd->len = len;
1079 scmd->current_ro = 0;
1080
1081 sbd_do_read_xfer(task, scmd, dbuf);
1082 }
1083
1084 void
1085 sbd_do_write_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
1086 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
1087 {
1088 uint32_t len;
1089 int bufs_to_take;
1090
1091 if (ATOMIC32_GET(scmd->len) == 0) {
1092 goto DO_WRITE_XFER_DONE;
1093 }
1094
1095 /* Lets try not to hog all the buffers the port has. */
1096 bufs_to_take = ((task->task_max_nbufs > 2) &&
1097 (task->task_cmd_xfer_length < (32 * 1024))) ? 2 :
1098 task->task_max_nbufs;
1099
1100 if ((dbuf != NULL) &&
1101 ((dbuf->db_flags & DB_DONT_REUSE) || (dbuf_reusable == 0))) {
1102 /* free current dbuf and allocate a new one */
1103 stmf_free_dbuf(task, dbuf);
1104 dbuf = NULL;
1105 }
1106 if (ATOMIC8_GET(scmd->nbufs) >= bufs_to_take) {
1107 goto DO_WRITE_XFER_DONE;
1108 }
1109 if (dbuf == NULL) {
1110 uint32_t maxsize, minsize, old_minsize;
1111
1112 maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ? 128*1024 :
1113 ATOMIC32_GET(scmd->len);
1114 minsize = maxsize >> 2;
1115 do {
1116 old_minsize = minsize;
1117 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
1118 } while ((dbuf == NULL) && (old_minsize > minsize) &&
1119 (minsize >= 512));
1120 if (dbuf == NULL) {
1121 if (ATOMIC8_GET(scmd->nbufs) == 0) {
1122 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1123 STMF_ALLOC_FAILURE, NULL);
1124 }
1125 return;
1126 }
1127 }
1128
1129 len = ATOMIC32_GET(scmd->len) > dbuf->db_buf_size ? dbuf->db_buf_size :
1130 ATOMIC32_GET(scmd->len);
1131
1132 dbuf->db_relative_offset = scmd->current_ro;
1133 dbuf->db_data_size = len;
1134 dbuf->db_flags = DB_DIRECTION_FROM_RPORT;
1135 (void) stmf_xfer_data(task, dbuf, 0);
1136 /* outstanding port xfers and bufs used */
1137 atomic_inc_8(&scmd->nbufs);
1138 atomic_add_32(&scmd->len, -len);
1139 scmd->current_ro += len;
1140
1141 if ((ATOMIC32_GET(scmd->len) != 0) &&
1142 (ATOMIC8_GET(scmd->nbufs) < bufs_to_take)) {
1143 sbd_do_write_xfer(task, scmd, NULL, 0);
1144 }
1145 return;
1146
1147 DO_WRITE_XFER_DONE:
1148 if (dbuf != NULL) {
1149 stmf_free_dbuf(task, dbuf);
1150 }
1151 }
1152
1153 void
1154 sbd_do_sgl_write_xfer(struct scsi_task *task, sbd_cmd_t *scmd, int first_xfer)
1155 {
1156 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1157 sbd_zvol_io_t *zvio;
1158 int ret;
1159 uint32_t xfer_len, max_len, first_len;
1160 stmf_status_t xstat;
1161 stmf_data_buf_t *dbuf;
1162 uint_t nblks;
1188 sbd_1st_xfer_len = 0;
1189 }
1190
1191 max_len = MIN(task->task_max_xfer_len, sl->sl_max_xfer_len);
1192 if (sbd_max_xfer_len)
1193 max_len = MIN(max_len, sbd_max_xfer_len);
1194 /*
1195 * Special case the first xfer if hints are set.
1196 */
1197 if (first_xfer && (sbd_1st_xfer_len || task->task_1st_xfer_len)) {
1198 /* global over-ride has precedence */
1199 if (sbd_1st_xfer_len)
1200 first_len = sbd_1st_xfer_len;
1201 else
1202 first_len = task->task_1st_xfer_len;
1203 } else {
1204 first_len = 0;
1205 }
1206
1207
1208 while (ATOMIC32_GET(scmd->len) &&
1209 ATOMIC8_GET(scmd->nbufs) < task->task_max_nbufs) {
1210 xfer_len = MIN(max_len, ATOMIC32_GET(scmd->len));
1211 if (first_len) {
1212 xfer_len = MIN(xfer_len, first_len);
1213 first_len = 0;
1214 }
1215 if (xfer_len < ATOMIC32_GET(scmd->len)) {
1216 /*
1217 * Attempt to end xfer on a block boundary.
1218 * The only way this does not happen is if the
1219 * xfer_len is small enough to stay contained
1220 * within the same block.
1221 */
1222 uint64_t xfer_offset, xfer_aligned_end;
1223
1224 xfer_offset = scmd->addr + scmd->current_ro;
1225 xfer_aligned_end =
1226 P2ALIGN(xfer_offset+xfer_len, blksize);
1227 if (xfer_aligned_end > xfer_offset)
1228 xfer_len = xfer_aligned_end - xfer_offset;
1229 }
1230 /*
1231 * Allocate object to track the write and reserve
1232 * enough space for scatter/gather list.
1233 */
1234 offset = scmd->addr + scmd->current_ro;
1235 nblks = sbd_zvol_numsegs(sl, offset, xfer_len);
1259 dbuf->db_data_size = xfer_len;
1260 dbuf->db_relative_offset = scmd->current_ro;
1261 dbuf->db_sglist_length = (uint16_t)nblks;
1262 dbuf->db_xfer_status = 0;
1263 dbuf->db_handle = 0;
1264 dbuf->db_flags = (DB_DONT_CACHE | DB_DONT_REUSE |
1265 DB_DIRECTION_FROM_RPORT | DB_LU_DATA_BUF);
1266
1267 zvio = dbuf->db_lu_private;
1268 zvio->zvio_offset = offset;
1269
1270 /* get the buffers */
1271 ret = sbd_zvol_alloc_write_bufs(sl, dbuf);
1272 if (ret != 0) {
1273 /*
1274 * Could not allocate buffers from the backend;
1275 * treat it like an IO error.
1276 */
1277 stmf_free(dbuf);
1278 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1279 if (ATOMIC8_GET(scmd->nbufs) == 0) {
1280 /*
1281 * Nothing queued, so no completions coming
1282 */
1283 sbd_ats_remove_by_task(task);
1284 stmf_scsilib_send_status(task, STATUS_CHECK,
1285 STMF_SAA_WRITE_ERROR);
1286 rw_exit(&sl->sl_access_state_lock);
1287 }
1288 /*
1289 * Completions of previous buffers will cleanup.
1290 */
1291 return;
1292 }
1293
1294 /*
1295 * Allow PP to do setup
1296 */
1297 xstat = stmf_setup_dbuf(task, dbuf, 0);
1298 if (xstat != STMF_SUCCESS) {
1299 /*
1300 * This could happen if the driver cannot get the
1301 * DDI resources it needs for this request.
1302 * If other dbufs are queued, try again when the next
1303 * one completes, otherwise give up.
1304 */
1305 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
1306 stmf_free(dbuf);
1307 if (ATOMIC8_GET(scmd->nbufs) > 0) {
1308 /* completion of previous dbuf will retry */
1309 return;
1310 }
1311 /*
1312 * Done with this command.
1313 */
1314 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
1315 sbd_ats_remove_by_task(task);
1316 if (first_xfer)
1317 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
1318 else
1319 stmf_scsilib_send_status(task, STATUS_CHECK,
1320 STMF_SAA_WRITE_ERROR);
1321 rw_exit(&sl->sl_access_state_lock);
1322 return;
1323 }
1324
1325 /*
1326 * dbuf is now queued on task
1327 */
1328 atomic_inc_8(&scmd->nbufs);
1329
1330 xstat = stmf_xfer_data(task, dbuf, 0);
1331 switch (xstat) {
1332 case STMF_SUCCESS:
1333 break;
1334 case STMF_BUSY:
1335 /*
1336 * The dbuf is queued on the task, but unknown
1337 * to the PP, thus no completion will occur.
1338 */
1339 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
1340 stmf_teardown_dbuf(task, dbuf);
1341 stmf_free(dbuf);
1342 atomic_dec_8(&scmd->nbufs);
1343 if (ATOMIC8_GET(scmd->nbufs) > 0) {
1344 /* completion of previous dbuf will retry */
1345 return;
1346 }
1347 /*
1348 * Done with this command.
1349 */
1350 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
1351 sbd_ats_remove_by_task(task);
1352 if (first_xfer)
1353 stmf_scsilib_send_status(task, STATUS_QFULL, 0);
1354 else
1355 stmf_scsilib_send_status(task, STATUS_CHECK,
1356 STMF_SAA_WRITE_ERROR);
1357 rw_exit(&sl->sl_access_state_lock);
1358 return;
1359 case STMF_ABORTED:
1360 /*
1361 * Completion code will cleanup.
1362 */
1363 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1364 return;
1365 }
1366 /*
1367 * Update the xfer progress.
1368 */
1369 atomic_add_32(&scmd->len, -xfer_len);
1370 scmd->current_ro += xfer_len;
1371 }
1372 }
1373
1374 void
1375 sbd_handle_write_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
1376 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
1377 {
1378 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1379 uint64_t laddr;
1380 uint32_t buflen, iolen;
1381 int ndx;
1382 uint8_t op = task->task_cdb[0];
1383 boolean_t fua_bit = B_FALSE;
1384
1385 if (ATOMIC8_GET(scmd->nbufs) > 0) {
1386 /*
1387 * Decrement the count to indicate the port xfer
1388 * into the dbuf has completed even though the buf is
1389 * still in use here in the LU provider.
1390 */
1391 atomic_dec_8(&scmd->nbufs);
1392 }
1393
1394 if (dbuf->db_xfer_status != STMF_SUCCESS) {
1395 sbd_ats_remove_by_task(task);
1396 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1397 dbuf->db_xfer_status, NULL);
1398 return;
1399 }
1400
1401 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
1402 goto WRITE_XFER_DONE;
1403 }
1404
1405 if (ATOMIC32_GET(scmd->len) != 0) {
1406 /*
1407 * Initiate the next port xfer to occur in parallel
1408 * with writing this buf.
1409 */
1410 sbd_do_write_xfer(task, scmd, NULL, 0);
1411 }
1412
1413 /*
1414 * Check to see if the command is WRITE(10), WRITE(12), or WRITE(16).
1415 * If it is then check for bit 3 being set to indicate if Forced
1416 * Unit Access is being requested. If so, we'll bypass the direct
1417 * call and handle it in sbd_data_write().
1418 */
1419 if (((op == SCMD_WRITE_G1) || (op == SCMD_WRITE_G4) ||
1420 (op == SCMD_WRITE_G5)) && (task->task_cdb[1] & BIT_3)) {
1421 fua_bit = B_TRUE;
1422 }
1423 laddr = scmd->addr + dbuf->db_relative_offset;
1424
1425 /*
1426 * If this is going to a zvol, use the direct call to
1427 * sbd_zvol_copy_{read,write}. The direct call interface is
1428 * restricted to PPs that accept sglists, but that is not required.
1429 */
1430 if (sl->sl_flags & SL_CALL_ZVOL &&
1431 (task->task_additional_flags & TASK_AF_ACCEPT_LU_DBUF) &&
1432 (sbd_zcopy & (4|1)) && !fua_bit) {
1433 int commit;
1434
1435 commit = (ATOMIC32_GET(scmd->len) == 0 &&
1436 ATOMIC8_GET(scmd->nbufs) == 0);
1437 rw_enter(&sl->sl_access_state_lock, RW_READER);
1438 if ((sl->sl_flags & SL_MEDIA_LOADED) == 0 ||
1439 sbd_copy_rdwr(task, laddr, dbuf, SBD_CMD_SCSI_WRITE,
1440 commit) != STMF_SUCCESS)
1441 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1442 rw_exit(&sl->sl_access_state_lock);
1443 buflen = dbuf->db_data_size;
1444 } else {
1445 for (buflen = 0, ndx = 0; (buflen < dbuf->db_data_size) &&
1446 (ndx < dbuf->db_sglist_length); ndx++) {
1447 iolen = min(dbuf->db_data_size - buflen,
1448 dbuf->db_sglist[ndx].seg_length);
1449 if (iolen == 0)
1450 break;
1451 if (sbd_data_write(sl, task, laddr, (uint64_t)iolen,
1452 dbuf->db_sglist[ndx].seg_addr) != STMF_SUCCESS) {
1453 scmd->flags |= SBD_SCSI_CMD_XFER_FAIL;
1454 break;
1455 }
1456 buflen += iolen;
1457 laddr += (uint64_t)iolen;
1458 }
1459 }
1460 task->task_nbytes_transferred += buflen;
1461 WRITE_XFER_DONE:
1462 if (ATOMIC32_GET(scmd->len) == 0 ||
1463 scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
1464 stmf_free_dbuf(task, dbuf);
1465 if (ATOMIC8_GET(scmd->nbufs))
1466 return; /* wait for all buffers to complete */
1467 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
1468 sbd_ats_remove_by_task(task);
1469 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
1470 stmf_scsilib_send_status(task, STATUS_CHECK,
1471 STMF_SAA_WRITE_ERROR);
1472 } else {
1473 /*
1474 * If SYNC_WRITE flag is on then we need to flush
1475 * cache before sending status.
1476 * Note: this may be a no-op because of how
1477 * SL_WRITEBACK_CACHE_DISABLE and
1478 * SL_FLUSH_ON_DISABLED_WRITECACHE are set, but not
1479 * worth code complexity of checking those in this code
1480 * path, SBD_SCSI_CMD_SYNC_WRITE is rarely set.
1481 */
1482 if ((scmd->flags & SBD_SCSI_CMD_SYNC_WRITE) &&
1483 (sbd_flush_data_cache(sl, 0) != SBD_SUCCESS)) {
1484 stmf_scsilib_send_status(task, STATUS_CHECK,
1485 STMF_SAA_WRITE_ERROR);
1486 } else {
1487 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
1488 }
1514 return (0);
1515 /*
1516 * Calculate amount of data that will avoid the copy path.
1517 * The calculation is only valid if len >= blksize.
1518 */
1519 no_copy_span = P2ALIGN(laddr+len, blksize) -
1520 P2ROUNDUP(laddr, blksize);
1521 return (no_copy_span >= len/2);
1522 }
1523 }
1524
1525 void
1526 sbd_handle_write(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
1527 {
1528 uint64_t lba, laddr;
1529 uint32_t len;
1530 uint8_t op = task->task_cdb[0], do_immediate_data = 0;
1531 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1532 sbd_cmd_t *scmd;
1533 stmf_data_buf_t *dbuf;
1534 uint64_t blkcount;
1535 uint8_t sync_wr_flag = 0;
1536 boolean_t fua_bit = B_FALSE;
1537
1538 if (sl->sl_flags & SL_WRITE_PROTECTED) {
1539 stmf_scsilib_send_status(task, STATUS_CHECK,
1540 STMF_SAA_WRITE_PROTECTED);
1541 return;
1542 }
1543 /*
1544 * Check to see if the command is WRITE(10), WRITE(12), or WRITE(16).
1545 * If it is then check for bit 3 being set to indicate if Forced
1546 * Unit Access is being requested. If so, we'll bypass the fast path
1547 * code to simplify support of this feature.
1548 */
1549 if (((op == SCMD_WRITE_G1) || (op == SCMD_WRITE_G4) ||
1550 (op == SCMD_WRITE_G5)) && (task->task_cdb[1] & BIT_3)) {
1551 fua_bit = B_TRUE;
1552 }
1553 if (op == SCMD_WRITE) {
1554 lba = READ_SCSI21(&task->task_cdb[1], uint64_t);
1555 len = (uint32_t)task->task_cdb[4];
1556
1557 if (len == 0) {
1558 len = 256;
1559 }
1560 } else if (op == SCMD_WRITE_G1) {
1561 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1562 len = READ_SCSI16(&task->task_cdb[7], uint32_t);
1563 } else if (op == SCMD_WRITE_G5) {
1564 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1565 len = READ_SCSI32(&task->task_cdb[6], uint32_t);
1566 } else if (op == SCMD_WRITE_G4) {
1567 lba = READ_SCSI64(&task->task_cdb[2], uint64_t);
1568 len = READ_SCSI32(&task->task_cdb[10], uint32_t);
1569 } else if (op == SCMD_WRITE_VERIFY) {
1570 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1571 len = READ_SCSI16(&task->task_cdb[7], uint32_t);
1572 sync_wr_flag = SBD_SCSI_CMD_SYNC_WRITE;
1573 } else if (op == SCMD_WRITE_VERIFY_G5) {
1574 lba = READ_SCSI32(&task->task_cdb[2], uint64_t);
1575 len = READ_SCSI32(&task->task_cdb[6], uint32_t);
1576 sync_wr_flag = SBD_SCSI_CMD_SYNC_WRITE;
1577 } else if (op == SCMD_WRITE_VERIFY_G4) {
1578 lba = READ_SCSI64(&task->task_cdb[2], uint64_t);
1579 len = READ_SCSI32(&task->task_cdb[10], uint32_t);
1580 sync_wr_flag = SBD_SCSI_CMD_SYNC_WRITE;
1581 } else {
1582 stmf_scsilib_send_status(task, STATUS_CHECK,
1583 STMF_SAA_INVALID_OPCODE);
1584 return;
1585 }
1586
1587 laddr = lba << sl->sl_data_blocksize_shift;
1588 blkcount = len;
1589 len <<= sl->sl_data_blocksize_shift;
1590
1591 if ((laddr + (uint64_t)len) > sl->sl_lu_size) {
1592 stmf_scsilib_send_status(task, STATUS_CHECK,
1593 STMF_SAA_LBA_OUT_OF_RANGE);
1594 return;
1595 }
1596
1597 task->task_cmd_xfer_length = len;
1598 if (task->task_additional_flags & TASK_AF_NO_EXPECTED_XFER_LENGTH) {
1599 task->task_expected_xfer_length = len;
1600 }
1601
1602 len = (len > task->task_expected_xfer_length) ?
1603 task->task_expected_xfer_length : len;
1604
1605 if (len == 0) {
1606 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
1607 return;
1608 }
1609
1610 if (sbd_ats_handling_before_io(task, sl, lba, blkcount) !=
1611 SBD_SUCCESS) {
1612 if (stmf_task_poll_lu(task, 10) != STMF_SUCCESS) {
1613 stmf_scsilib_send_status(task, STATUS_BUSY, 0);
1614 }
1615 return;
1616 }
1617
1618 if (sbd_zcopy & (4|1) && /* Debug switch */
1619 initial_dbuf == NULL && /* No PP buf passed in */
1620 sl->sl_flags & SL_CALL_ZVOL && /* zvol backing store */
1621 (task->task_additional_flags &
1622 TASK_AF_ACCEPT_LU_DBUF) && /* PP allows it */
1623 sbd_zcopy_write_useful(task, laddr, len, sl->sl_blksize) &&
1624 !fua_bit) {
1625
1626 /*
1627 * XXX Note that disallowing initial_dbuf will eliminate
1628 * iSCSI from participating. For small writes, that is
1629 * probably ok. For large writes, it may be best to just
1630 * copy the data from the initial dbuf and use zcopy for
1631 * the rest.
1632 */
1633 rw_enter(&sl->sl_access_state_lock, RW_READER);
1634 if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
1635 rw_exit(&sl->sl_access_state_lock);
1636 sbd_ats_remove_by_task(task);
1637 stmf_scsilib_send_status(task, STATUS_CHECK,
1638 STMF_SAA_READ_ERROR);
1639 return;
1640 }
1641 /*
1642 * Setup scmd to track the write progress.
1643 */
1644 if (task->task_lu_private) {
1645 scmd = (sbd_cmd_t *)task->task_lu_private;
1646 } else {
1647 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t),
1648 KM_SLEEP);
1649 task->task_lu_private = scmd;
1650 }
1651 scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED |
1652 sync_wr_flag;
1653 scmd->cmd_type = SBD_CMD_SCSI_WRITE;
1654 scmd->nbufs = 0;
1655 scmd->addr = laddr;
1656 scmd->len = len;
1657 scmd->current_ro = 0;
1658 sbd_do_sgl_write_xfer(task, scmd, 1);
1659 return;
1660 }
1661
1662 if ((initial_dbuf != NULL) && (task->task_flags & TF_INITIAL_BURST)) {
1663 if (initial_dbuf->db_data_size > len) {
1664 if (initial_dbuf->db_data_size >
1665 task->task_expected_xfer_length) {
1666 /* protocol error */
1667 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1668 STMF_INVALID_ARG, NULL);
1669 return;
1670 }
1671 initial_dbuf->db_data_size = len;
1672 }
1673 do_immediate_data = 1;
1674 }
1675 dbuf = initial_dbuf;
1676
1677 if (task->task_lu_private) {
1678 scmd = (sbd_cmd_t *)task->task_lu_private;
1679 } else {
1680 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
1681 task->task_lu_private = scmd;
1682 }
1683 scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_ATS_RELATED |
1684 sync_wr_flag;
1685 scmd->cmd_type = SBD_CMD_SCSI_WRITE;
1686 scmd->nbufs = 0;
1687 scmd->addr = laddr;
1688 scmd->len = len;
1689 scmd->current_ro = 0;
1690
1691 if (do_immediate_data) {
1692 /*
1693 * Account for data passed in this write command
1694 */
1695 (void) stmf_xfer_data(task, dbuf, STMF_IOF_STATS_ONLY);
1696 atomic_add_32(&scmd->len, -dbuf->db_data_size);
1697 scmd->current_ro += dbuf->db_data_size;
1698 dbuf->db_xfer_status = STMF_SUCCESS;
1699 sbd_handle_write_xfer_completion(task, scmd, dbuf, 0);
1700 } else {
1701 sbd_do_write_xfer(task, scmd, dbuf, 0);
1702 }
1703 }
1704
1705 /*
1706 * Utility routine to handle small non performance data transfers to the
1707 * initiators. dbuf is an initial data buf (if any), 'p' points to a data
1708 * buffer which is source of data for transfer, cdb_xfer_size is the
1709 * transfer size based on CDB, cmd_xfer_size is the actual amount of data
1710 * which this command would transfer (the size of data pointed to by 'p').
1711 */
1712 void
1713 sbd_handle_short_read_transfers(scsi_task_t *task, stmf_data_buf_t *dbuf,
1714 uint8_t *p, uint32_t cdb_xfer_size, uint32_t cmd_xfer_size)
1715 {
1716 uint32_t bufsize, ndx;
1859
1860 /* Lets find out who to call */
1861 switch (task->task_cdb[0]) {
1862 case SCMD_MODE_SELECT:
1863 case SCMD_MODE_SELECT_G1:
1864 if (sl->sl_access_state == SBD_LU_STANDBY) {
1865 st_ret = stmf_proxy_scsi_cmd(task, dbuf);
1866 if (st_ret != STMF_SUCCESS) {
1867 stmf_scsilib_send_status(task, STATUS_CHECK,
1868 STMF_SAA_LU_NO_ACCESS_UNAVAIL);
1869 }
1870 } else {
1871 sbd_handle_mode_select_xfer(task,
1872 dbuf->db_sglist[0].seg_addr, dbuf->db_data_size);
1873 }
1874 break;
1875 case SCMD_UNMAP:
1876 sbd_handle_unmap_xfer(task,
1877 dbuf->db_sglist[0].seg_addr, dbuf->db_data_size);
1878 break;
1879 case SCMD_EXTENDED_COPY:
1880 sbd_handle_xcopy_xfer(task, dbuf->db_sglist[0].seg_addr);
1881 break;
1882 case SCMD_PERSISTENT_RESERVE_OUT:
1883 if (sl->sl_access_state == SBD_LU_STANDBY) {
1884 st_ret = stmf_proxy_scsi_cmd(task, dbuf);
1885 if (st_ret != STMF_SUCCESS) {
1886 stmf_scsilib_send_status(task, STATUS_CHECK,
1887 STMF_SAA_LU_NO_ACCESS_UNAVAIL);
1888 }
1889 } else {
1890 sbd_handle_pgr_out_data(task, dbuf);
1891 }
1892 break;
1893 default:
1894 /* This should never happen */
1895 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
1896 STMF_ABORTED, NULL);
1897 }
1898 }
1899
1900 void
1901 sbd_handle_read_capacity(struct scsi_task *task,
1956 if (s < (4ull * 1024ull * 1024ull * 1024ull)) {
1957 *nsectors = 32;
1958 *nheads = 8;
1959 } else {
1960 *nsectors = 254;
1961 *nheads = 254;
1962 }
1963 *ncyl = s / ((uint64_t)blksize * (uint64_t)(*nsectors) *
1964 (uint64_t)(*nheads));
1965 }
1966
1967 void
1968 sbd_handle_mode_sense(struct scsi_task *task,
1969 struct stmf_data_buf *initial_dbuf, uint8_t *buf)
1970 {
1971 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
1972 uint32_t cmd_size, n;
1973 uint8_t *cdb;
1974 uint32_t ncyl;
1975 uint8_t nsectors, nheads;
1976 uint8_t page, ctrl, header_size;
1977 uint16_t nbytes;
1978 uint8_t *p;
1979 uint64_t s = sl->sl_lu_size;
1980 uint32_t dev_spec_param_offset;
1981
1982 p = buf; /* buf is assumed to be zeroed out and large enough */
1983 n = 0;
1984 cdb = &task->task_cdb[0];
1985 page = cdb[2] & 0x3F;
1986 ctrl = (cdb[2] >> 6) & 3;
1987
1988 if (cdb[0] == SCMD_MODE_SENSE) {
1989 cmd_size = cdb[4];
1990 header_size = 4;
1991 dev_spec_param_offset = 2;
1992 } else {
1993 cmd_size = READ_SCSI16(&cdb[7], uint32_t);
1994 header_size = 8;
1995 dev_spec_param_offset = 3;
1996 }
1997
1998 /* Now validate the command */
1999 if ((cdb[2] != 0) && (page != MODEPAGE_ALLPAGES) &&
2000 (page != MODEPAGE_CACHING) && (page != MODEPAGE_CTRL_MODE) &&
2001 (page != MODEPAGE_FORMAT) && (page != MODEPAGE_GEOMETRY)) {
2002 stmf_scsilib_send_status(task, STATUS_CHECK,
2003 STMF_SAA_INVALID_FIELD_IN_CDB);
2004 return;
2005 }
2006
2007 /* We will update the length in the mode header at the end */
2008
2009 /* Block dev device specific param in mode param header has wp bit */
2010 if (sl->sl_flags & SL_WRITE_PROTECTED) {
2011 p[n + dev_spec_param_offset] = BIT_7;
2012 }
2013 n += header_size;
2014 /* We are not going to return any block descriptor */
2015
2016 nbytes = ((uint16_t)1) << sl->sl_data_blocksize_shift;
2017 sbd_calc_geometry(s, nbytes, &nsectors, &nheads, &ncyl);
2018
2019 if ((page == MODEPAGE_FORMAT) || (page == MODEPAGE_ALLPAGES)) {
2020 p[n] = 0x03;
2021 p[n+1] = 0x16;
2022 if (ctrl != 1) {
2023 p[n + 11] = nsectors;
2024 p[n + 12] = nbytes >> 8;
2025 p[n + 13] = nbytes & 0xff;
2026 p[n + 20] = 0x80;
2027 }
2028 n += 24;
2029 }
2030 if ((page == MODEPAGE_GEOMETRY) || (page == MODEPAGE_ALLPAGES)) {
2031 p[n] = 0x04;
2032 p[n + 1] = 0x16;
2033 if (ctrl != 1) {
2034 p[n + 2] = ncyl >> 16;
2035 p[n + 3] = ncyl >> 8;
2036 p[n + 4] = ncyl & 0xff;
2037 p[n + 5] = nheads;
2038 p[n + 20] = 0x15;
2039 p[n + 21] = 0x18;
2040 }
2041 n += 24;
2042 }
2043 if ((page == MODEPAGE_CACHING) || (page == MODEPAGE_ALLPAGES)) {
2044 struct mode_caching *mode_caching_page;
2045
2046 mode_caching_page = (struct mode_caching *)&p[n];
2047
2048 mode_caching_page->mode_page.code = MODEPAGE_CACHING;
2049 mode_caching_page->mode_page.ps = 1; /* A saveable page */
2050 mode_caching_page->mode_page.length = 0x12;
2085 PAGELENGTH_MODE_CONTROL_SCSI3;
2086 if (ctrl != 1) {
2087 /* If not looking for changeable values, report this. */
2088 mode_control_page->que_mod = CTRL_QMOD_UNRESTRICT;
2089 }
2090 n += (sizeof (struct mode_page) +
2091 mode_control_page->mode_page.length);
2092 }
2093
2094 if (cdb[0] == SCMD_MODE_SENSE) {
2095 if (n > 255) {
2096 stmf_scsilib_send_status(task, STATUS_CHECK,
2097 STMF_SAA_INVALID_FIELD_IN_CDB);
2098 return;
2099 }
2100 /*
2101 * Mode parameter header length doesn't include the number
2102 * of bytes in the length field, so adjust the count.
2103 * Byte count minus header length field size.
2104 */
2105 buf[0] = (n - header_size) & 0xff;
2106 } else {
2107 /* Byte count minus header length field size. */
2108 buf[1] = (n - header_size) & 0xff;
2109 buf[0] = ((n - header_size) >> 8) & 0xff;
2110 }
2111
2112 sbd_handle_short_read_transfers(task, initial_dbuf, buf,
2113 cmd_size, n);
2114 }
2115
2116 void
2117 sbd_handle_mode_select(scsi_task_t *task, stmf_data_buf_t *dbuf)
2118 {
2119 uint32_t cmd_xfer_len;
2120
2121 if (task->task_cdb[0] == SCMD_MODE_SELECT) {
2122 cmd_xfer_len = (uint32_t)task->task_cdb[4];
2123 } else {
2124 cmd_xfer_len = READ_SCSI16(&task->task_cdb[7], uint32_t);
2125 }
2126
2127 if ((task->task_cdb[1] & 0xFE) != 0x10) {
2128 stmf_scsilib_send_status(task, STATUS_CHECK,
2129 STMF_SAA_INVALID_FIELD_IN_CDB);
2298 kmem_free(buf, 260);
2299 return;
2300 }
2301 sbd_handle_short_read_transfers(task, initial_dbuf, buf,
2302 cmd_size, xfer_size);
2303 kmem_free(buf, 260);
2304 }
2305
2306 /*
2307 * This function parse through a string, passed to it as a pointer to a string,
2308 * by adjusting the pointer to the first non-space character and returns
2309 * the count/length of the first bunch of non-space characters. Multiple
2310 * Management URLs are stored as a space delimited string in sl_mgmt_url
2311 * field of sbd_lu_t. This function is used to retrieve one url at a time.
2312 *
2313 * i/p : pointer to pointer to a url string
2314 * o/p : Adjust the pointer to the url to the first non white character
2315 * and returns the length of the URL
2316 */
2317 uint16_t
2318 sbd_parse_mgmt_url(char **url_addr)
2319 {
2320 uint16_t url_length = 0;
2321 char *url;
2322 url = *url_addr;
2323
2324 while (*url != '\0') {
2325 if (*url == ' ' || *url == '\t' || *url == '\n') {
2326 (*url_addr)++;
2327 url = *url_addr;
2328 } else {
2329 break;
2330 }
2331 }
2332
2333 while (*url != '\0') {
2334 if (*url == ' ' || *url == '\t' ||
2335 *url == '\n' || *url == '\0') {
2336 break;
2337 }
2338 url++;
2339 url_length++;
2410
2411 /* Do the actual I/O. Recycle xfer_size now to be write size. */
2412 DTRACE_PROBE1(write__same__io__begin, uint64_t, len);
2413 for (sz_done = 0; sz_done < len; sz_done += (uint64_t)xfer_size) {
2414 xfer_size = ((big_buf_size + sz_done) <= len) ? big_buf_size :
2415 len - sz_done;
2416 ret = sbd_data_write(sl, task, addr + sz_done,
2417 (uint64_t)xfer_size, big_buf);
2418 if (ret != SBD_SUCCESS)
2419 break;
2420 }
2421 DTRACE_PROBE2(write__same__io__end, uint64_t, len, uint64_t, sz_done);
2422
2423 if (big_buf != scmd->trans_data)
2424 kmem_free(big_buf, big_buf_size);
2425
2426 return (ret);
2427 }
2428
2429 static void
2430 sbd_write_same_release_resources(struct scsi_task *task)
2431 {
2432 sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
2433
2434 if (scmd->nbufs == 0XFF)
2435 cmn_err(CE_WARN, "%s invalid buffer count %x",
2436 __func__, scmd->nbufs);
2437 if ((scmd->trans_data_len != 0) && (scmd->trans_data != NULL))
2438 kmem_free(scmd->trans_data, scmd->trans_data_len);
2439 scmd->trans_data = NULL;
2440 scmd->trans_data_len = 0;
2441 scmd->flags &= ~SBD_SCSI_CMD_TRANS_DATA;
2442 }
2443
2444 static void
2445 sbd_handle_write_same_xfer_completion(struct scsi_task *task, sbd_cmd_t *scmd,
2446 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
2447 {
2448 uint64_t laddr;
2449 uint32_t buflen, iolen;
2450 int ndx, ret;
2451
2452 if (ATOMIC8_GET(scmd->nbufs) > 0) {
2453 atomic_dec_8(&scmd->nbufs);
2454 }
2455
2456 if (dbuf->db_xfer_status != STMF_SUCCESS) {
2457 sbd_write_same_release_resources(task);
2458 sbd_ats_remove_by_task(task);
2459 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
2460 dbuf->db_xfer_status, NULL);
2461 return;
2462 }
2463
2464 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
2465 goto write_same_xfer_done;
2466 }
2467
2468 /* if this is a unnessary callback just return */
2469 if (((scmd->flags & SBD_SCSI_CMD_TRANS_DATA) == 0) ||
2470 ((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0) ||
2471 (scmd->trans_data == NULL)) {
2472 sbd_ats_remove_by_task(task);
2473 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
2474 return;
2475 }
2476
2477 if (ATOMIC32_GET(scmd->len) != 0) {
2478 /*
2479 * Initiate the next port xfer to occur in parallel
2480 * with writing this buf.
2481 */
2482 sbd_do_write_same_xfer(task, scmd, NULL, 0);
2483 }
2484
2485 laddr = dbuf->db_relative_offset;
2486
2487 for (buflen = 0, ndx = 0; (buflen < dbuf->db_data_size) &&
2488 (ndx < dbuf->db_sglist_length); ndx++) {
2489 iolen = min(dbuf->db_data_size - buflen,
2490 dbuf->db_sglist[ndx].seg_length);
2491 if (iolen == 0)
2492 break;
2493 bcopy(dbuf->db_sglist[ndx].seg_addr, &scmd->trans_data[laddr],
2494 iolen);
2495 buflen += iolen;
2496 laddr += (uint64_t)iolen;
2497 }
2498 task->task_nbytes_transferred += buflen;
2499
2500 write_same_xfer_done:
2501 if (ATOMIC32_GET(scmd->len) == 0 ||
2502 scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
2503 stmf_free_dbuf(task, dbuf);
2504 if (ATOMIC8_GET(scmd->nbufs) > 0)
2505 return;
2506 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
2507 if (scmd->flags & SBD_SCSI_CMD_XFER_FAIL) {
2508 sbd_ats_remove_by_task(task);
2509 sbd_write_same_release_resources(task);
2510 stmf_scsilib_send_status(task, STATUS_CHECK,
2511 STMF_SAA_WRITE_ERROR);
2512 } else {
2513 ret = sbd_write_same_data(task, scmd);
2514 sbd_ats_remove_by_task(task);
2515 sbd_write_same_release_resources(task);
2516 if (ret != SBD_SUCCESS) {
2517 stmf_scsilib_send_status(task, STATUS_CHECK,
2518 STMF_SAA_WRITE_ERROR);
2519 } else {
2520 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2521 }
2522 }
2523 return;
2524 }
2525 sbd_do_write_same_xfer(task, scmd, dbuf, dbuf_reusable);
2526 }
2527
2528 static void
2529 sbd_do_write_same_xfer(struct scsi_task *task, sbd_cmd_t *scmd,
2530 struct stmf_data_buf *dbuf, uint8_t dbuf_reusable)
2531 {
2532 uint32_t len;
2533
2534 if (ATOMIC32_GET(scmd->len) == 0) {
2535 if (dbuf != NULL)
2536 stmf_free_dbuf(task, dbuf);
2537 return;
2538 }
2539
2540 if ((dbuf != NULL) &&
2541 ((dbuf->db_flags & DB_DONT_REUSE) || (dbuf_reusable == 0))) {
2542 /* free current dbuf and allocate a new one */
2543 stmf_free_dbuf(task, dbuf);
2544 dbuf = NULL;
2545 }
2546 if (dbuf == NULL) {
2547 uint32_t maxsize, minsize, old_minsize;
2548
2549 maxsize = (ATOMIC32_GET(scmd->len) > (128*1024)) ? 128*1024 :
2550 ATOMIC32_GET(scmd->len);
2551 minsize = maxsize >> 2;
2552 do {
2553 old_minsize = minsize;
2554 dbuf = stmf_alloc_dbuf(task, maxsize, &minsize, 0);
2555 } while ((dbuf == NULL) && (old_minsize > minsize) &&
2556 (minsize >= 512));
2557 if (dbuf == NULL) {
2558 sbd_ats_remove_by_task(task);
2559 sbd_write_same_release_resources(task);
2560 if (ATOMIC8_GET(scmd->nbufs) == 0) {
2561 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
2562 STMF_ALLOC_FAILURE, NULL);
2563 }
2564 return;
2565 }
2566 }
2567
2568 len = ATOMIC32_GET(scmd->len) > dbuf->db_buf_size ? dbuf->db_buf_size :
2569 ATOMIC32_GET(scmd->len);
2570
2571 dbuf->db_relative_offset = scmd->current_ro;
2572 dbuf->db_data_size = len;
2573 dbuf->db_flags = DB_DIRECTION_FROM_RPORT;
2574 (void) stmf_xfer_data(task, dbuf, 0);
2575 /* outstanding port xfers and bufs used */
2576 atomic_inc_8(&scmd->nbufs);
2577 atomic_add_32(&scmd->len, -len);
2578 scmd->current_ro += len;
2579 }
2580
2581 static void
2582 sbd_handle_write_same(scsi_task_t *task, struct stmf_data_buf *initial_dbuf)
2583 {
2584 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
2585 uint64_t addr, len;
2586 sbd_cmd_t *scmd;
2587 stmf_data_buf_t *dbuf;
2588 uint8_t unmap;
2589 uint8_t do_immediate_data = 0;
2590
2591 if (HardwareAcceleratedInit == 0) {
2592 stmf_scsilib_send_status(task, STATUS_CHECK,
2593 STMF_SAA_INVALID_OPCODE);
2594 return;
2595 }
2596
2597 task->task_cmd_xfer_length = 0;
2598 if (task->task_additional_flags &
2599 TASK_AF_NO_EXPECTED_XFER_LENGTH) {
2600 task->task_expected_xfer_length = 0;
2601 }
2602 if (sl->sl_flags & SL_WRITE_PROTECTED) {
2603 stmf_scsilib_send_status(task, STATUS_CHECK,
2604 STMF_SAA_WRITE_PROTECTED);
2605 return;
2606 }
2607 if (task->task_cdb[1] & 0xF7) {
2608 stmf_scsilib_send_status(task, STATUS_CHECK,
2609 STMF_SAA_INVALID_FIELD_IN_CDB);
2610 return;
2611 }
2612 unmap = task->task_cdb[1] & 0x08;
2613
2614 if (unmap && ((sl->sl_flags & SL_UNMAP_ENABLED) == 0)) {
2615 stmf_scsilib_send_status(task, STATUS_CHECK,
2616 STMF_SAA_INVALID_FIELD_IN_CDB);
2617 return;
2618 }
2619
2620 if (task->task_cdb[0] == SCMD_WRITE_SAME_G1) {
2621 addr = READ_SCSI32(&task->task_cdb[2], uint64_t);
2622 len = READ_SCSI16(&task->task_cdb[7], uint64_t);
2623 } else {
2624 addr = READ_SCSI64(&task->task_cdb[2], uint64_t);
2625 len = READ_SCSI32(&task->task_cdb[10], uint64_t);
2626 }
2627
2628 if (len == 0) {
2629 stmf_scsilib_send_status(task, STATUS_CHECK,
2630 STMF_SAA_INVALID_FIELD_IN_CDB);
2631 return;
2632 }
2633
2634 if (sbd_ats_handling_before_io(task, sl, addr, len) !=
2635 SBD_SUCCESS) {
2636 if (stmf_task_poll_lu(task, 10) != STMF_SUCCESS)
2637 stmf_scsilib_send_status(task, STATUS_BUSY, 0);
2638 return;
2639 }
2640
2641 addr <<= sl->sl_data_blocksize_shift;
2642 len <<= sl->sl_data_blocksize_shift;
2643
2644 /* Check if the command is for the unmap function */
2645 if (unmap) {
2646 dkioc_free_list_t *dfl = kmem_zalloc(DFL_SZ(1), KM_SLEEP);
2647
2648 dfl->dfl_num_exts = 1;
2649 dfl->dfl_exts[0].dfle_start = addr;
2650 dfl->dfl_exts[0].dfle_length = len;
2651 if (sbd_unmap(sl, dfl) != 0) {
2652 stmf_scsilib_send_status(task, STATUS_CHECK,
2653 STMF_SAA_LBA_OUT_OF_RANGE);
2654 } else {
2655 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2656 }
2657 dfl_free(dfl);
2658 return;
2659 }
2660
2661 /* Write same function */
2662
2663 task->task_cmd_xfer_length = 1 << sl->sl_data_blocksize_shift;
2664 if (task->task_additional_flags &
2665 TASK_AF_NO_EXPECTED_XFER_LENGTH) {
2666 task->task_expected_xfer_length = task->task_cmd_xfer_length;
2667 }
2668 if ((addr + len) > sl->sl_lu_size) {
2669 sbd_ats_remove_by_task(task);
2670 stmf_scsilib_send_status(task, STATUS_CHECK,
2671 STMF_SAA_LBA_OUT_OF_RANGE);
2672 return;
2673 }
2674
2675 /* For rest of this I/O the transfer length is 1 block */
2676 len = ((uint64_t)1) << sl->sl_data_blocksize_shift;
2677
2678 /* Some basic checks */
2679 if ((len == 0) || (len != task->task_expected_xfer_length)) {
2680 sbd_ats_remove_by_task(task);
2681 stmf_scsilib_send_status(task, STATUS_CHECK,
2682 STMF_SAA_INVALID_FIELD_IN_CDB);
2683 return;
2684 }
2685
2686
2687 if ((initial_dbuf != NULL) && (task->task_flags & TF_INITIAL_BURST)) {
2688 if (initial_dbuf->db_data_size > len) {
2689 if (initial_dbuf->db_data_size >
2690 task->task_expected_xfer_length) {
2691 /* protocol error */
2692 sbd_ats_remove_by_task(task);
2693 stmf_abort(STMF_QUEUE_TASK_ABORT, task,
2694 STMF_INVALID_ARG, NULL);
2695 return;
2696 }
2697 initial_dbuf->db_data_size = (uint32_t)len;
2698 }
2699 do_immediate_data = 1;
2700 }
2701 dbuf = initial_dbuf;
2702
2703 if (task->task_lu_private) {
2704 scmd = (sbd_cmd_t *)task->task_lu_private;
2705 } else {
2706 scmd = (sbd_cmd_t *)kmem_alloc(sizeof (sbd_cmd_t), KM_SLEEP);
2707 task->task_lu_private = scmd;
2708 }
2709 scmd->flags = SBD_SCSI_CMD_ACTIVE | SBD_SCSI_CMD_TRANS_DATA |
2710 SBD_SCSI_CMD_ATS_RELATED;
2711 scmd->cmd_type = SBD_CMD_SCSI_WRITE;
2712 scmd->nbufs = 0;
2713 scmd->len = (uint32_t)len;
2714 scmd->trans_data_len = (uint32_t)len;
2715 scmd->trans_data = kmem_alloc((size_t)len, KM_SLEEP);
2716 scmd->current_ro = 0;
2717
2718 if (do_immediate_data) {
2719 /*
2720 * Account for data passed in this write command
2721 */
2722 (void) stmf_xfer_data(task, dbuf, STMF_IOF_STATS_ONLY);
2723 atomic_add_32(&scmd->len, -dbuf->db_data_size);
2724 scmd->current_ro += dbuf->db_data_size;
2725 dbuf->db_xfer_status = STMF_SUCCESS;
2726 sbd_handle_write_same_xfer_completion(task, scmd, dbuf, 0);
2727 } else {
2728 sbd_do_write_same_xfer(task, scmd, dbuf, 0);
2729 }
2730 }
2731
2732 static void
2733 sbd_handle_unmap(scsi_task_t *task, stmf_data_buf_t *dbuf)
2734 {
2735 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
2736 uint32_t cmd_xfer_len;
2737
2738 if (sbd_unmap_enable == 0) {
2739 stmf_scsilib_send_status(task, STATUS_CHECK,
2740 STMF_SAA_INVALID_OPCODE);
2741 return;
2742 }
2743
2744 if (sl->sl_flags & SL_WRITE_PROTECTED) {
2745 stmf_scsilib_send_status(task, STATUS_CHECK,
2746 STMF_SAA_WRITE_PROTECTED);
2747 return;
2748 }
2749 cmd_xfer_len = READ_SCSI16(&task->task_cdb[7], uint32_t);
2750
2751 if (task->task_cdb[1] & 1) {
2752 stmf_scsilib_send_status(task, STATUS_CHECK,
2753 STMF_SAA_INVALID_FIELD_IN_CDB);
2754 return;
2755 }
2756
2757 if (cmd_xfer_len == 0) {
2758 task->task_cmd_xfer_length = 0;
2759 if (task->task_additional_flags &
2760 TASK_AF_NO_EXPECTED_XFER_LENGTH) {
2761 task->task_expected_xfer_length = 0;
2762 }
2763 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2764 return;
2765 }
2766
2767 sbd_handle_short_write_transfers(task, dbuf, cmd_xfer_len);
2768 }
2769
2770 static void
2771 sbd_handle_unmap_xfer(scsi_task_t *task, uint8_t *buf, uint32_t buflen)
2772 {
2773 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
2774 uint32_t ulen, dlen, num_desc;
2775 uint64_t addr, len;
2776 uint8_t *p;
2777 dkioc_free_list_t *dfl;
2778 int ret;
2779 int i;
2780
2781 if (buflen < 24) {
2782 stmf_scsilib_send_status(task, STATUS_CHECK,
2783 STMF_SAA_INVALID_FIELD_IN_CDB);
2784 return;
2785 }
2786 ulen = READ_SCSI16(buf, uint32_t);
2787 dlen = READ_SCSI16(buf + 2, uint32_t);
2788 num_desc = dlen >> 4;
2789 if (((ulen + 2) != buflen) || ((dlen + 8) != buflen) || (dlen & 0xf) ||
2790 (num_desc == 0)) {
2791 stmf_scsilib_send_status(task, STATUS_CHECK,
2792 STMF_SAA_INVALID_FIELD_IN_CDB);
2793 return;
2794 }
2795
2796 dfl = kmem_zalloc(DFL_SZ(num_desc), KM_SLEEP);
2797 dfl->dfl_num_exts = num_desc;
2798 /*
2799 * This should use ATS locking but that was disabled by the
2800 * changes to ZFS top take advantage of TRIM in SSDs.
2801 *
2802 * Since the entire list is passed to ZFS in one list ATS
2803 * locking is not done. This may be detectable, and if it is
2804 * then the entire list needs to be locked and then after the
2805 * unmap completes the entire list must be unlocked
2806 */
2807 for (p = buf + 8, i = 0; num_desc; num_desc--, p += 16, i++) {
2808 addr = READ_SCSI64(p, uint64_t);
2809 len = READ_SCSI32(p+8, uint64_t);
2810 addr <<= sl->sl_data_blocksize_shift;
2811 len <<= sl->sl_data_blocksize_shift;
2812
2813 /* Prepare a list of extents to unmap */
2814 dfl->dfl_exts[i].dfle_start = addr;
2815 dfl->dfl_exts[i].dfle_length = len;
2816
2817 /* release the overlap */
2818 }
2819 ASSERT(i == dfl->dfl_num_exts);
2820
2821 /* Finally execute the unmap operations in a single step */
2822 ret = sbd_unmap(sl, dfl);
2823 dfl_free(dfl);
2824 if (ret != 0) {
2825 stmf_scsilib_send_status(task, STATUS_CHECK,
2826 STMF_SAA_LBA_OUT_OF_RANGE);
2827 return;
2828 }
2829
2830 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
2831 }
2832
2833 void
2834 sbd_handle_inquiry(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
2835 {
2836 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
2837 uint8_t *cdbp = (uint8_t *)&task->task_cdb[0];
2838 uint8_t *p;
2839 uint8_t byte0;
2840 uint8_t page_length;
2841 uint16_t bsize = 512;
2842 uint16_t cmd_size;
2843 uint32_t xfer_size = 4;
2844 uint32_t mgmt_url_size = 0;
2845 uint8_t exp;
2846 uint64_t s;
2847 char *mgmt_url = NULL;
2848
2849
2880 */
2881
2882 if ((cdbp[1] & 1) == 0) {
2883 int i;
2884 struct scsi_inquiry *inq;
2885
2886 p = (uint8_t *)kmem_zalloc(bsize, KM_SLEEP);
2887 inq = (struct scsi_inquiry *)p;
2888
2889 page_length = 69;
2890 xfer_size = page_length + 5;
2891
2892 inq->inq_dtype = DTYPE_DIRECT;
2893 inq->inq_ansi = 5; /* SPC-3 */
2894 inq->inq_hisup = 1;
2895 inq->inq_rdf = 2; /* Response data format for SPC-3 */
2896 inq->inq_len = page_length;
2897
2898 inq->inq_tpgs = TPGS_FAILOVER_IMPLICIT;
2899 inq->inq_cmdque = 1;
2900 inq->inq_3pc = 1;
2901
2902 if (sl->sl_flags & SL_VID_VALID) {
2903 bcopy(sl->sl_vendor_id, inq->inq_vid, 8);
2904 } else {
2905 bcopy(sbd_vendor_id, inq->inq_vid, 8);
2906 }
2907
2908 if (sl->sl_flags & SL_PID_VALID) {
2909 bcopy(sl->sl_product_id, inq->inq_pid, 16);
2910 } else {
2911 bcopy(sbd_product_id, inq->inq_pid, 16);
2912 }
2913
2914 if (sl->sl_flags & SL_REV_VALID) {
2915 bcopy(sl->sl_revision, inq->inq_revision, 4);
2916 } else {
2917 bcopy(sbd_revision, inq->inq_revision, 4);
2918 }
2919
2920 /* Adding Version Descriptors */
2979 mgmt_url_size = strlen(sl->sl_mgmt_url);
2980 mgmt_url = sl->sl_mgmt_url;
2981 } else if (sbd_mgmt_url) {
2982 mgmt_url_size = strlen(sbd_mgmt_url);
2983 mgmt_url = sbd_mgmt_url;
2984 }
2985
2986 /*
2987 * EVPD handling
2988 */
2989
2990 /* Default 512 bytes may not be enough, increase bsize if necessary */
2991 if (cdbp[2] == 0x83 || cdbp[2] == 0x85) {
2992 if (bsize < cmd_size)
2993 bsize = cmd_size;
2994 }
2995 p = (uint8_t *)kmem_zalloc(bsize, KM_SLEEP);
2996
2997 switch (cdbp[2]) {
2998 case 0x00:
2999 page_length = 5 + (mgmt_url_size ? 1 : 0);
3000
3001 if (sl->sl_flags & SL_UNMAP_ENABLED)
3002 page_length += 1;
3003
3004 p[0] = byte0;
3005 p[3] = page_length;
3006 /* Supported VPD pages in ascending order */
3007 /* CSTYLED */
3008 {
3009 uint8_t i = 5;
3010
3011 p[i++] = 0x80;
3012 p[i++] = 0x83;
3013 if (mgmt_url_size != 0)
3014 p[i++] = 0x85;
3015 p[i++] = 0x86;
3016 p[i++] = 0xb0;
3017 if (sl->sl_flags & SL_UNMAP_ENABLED) {
3018 p[i++] = 0xb2;
3019 }
3020 }
3021 xfer_size = page_length + 4;
3022 break;
3023
3024 case 0x80:
3025 if (sl->sl_serial_no_size) {
3026 page_length = sl->sl_serial_no_size;
3027 bcopy(sl->sl_serial_no, p + 4, sl->sl_serial_no_size);
3028 } else {
3029 /* if no serial num is specified set 4 spaces */
3030 page_length = 4;
3031 bcopy(" ", p + 4, 4);
3032 }
3033 p[0] = byte0;
3034 p[1] = 0x80;
3035 p[3] = page_length;
3036 xfer_size = page_length + 4;
3037 break;
3038
3039 case 0x83:
3040 xfer_size = stmf_scsilib_prepare_vpd_page83(task, p,
3041 bsize, byte0, STMF_VPD_LU_ID|STMF_VPD_TARGET_ID|
3042 STMF_VPD_TP_GROUP|STMF_VPD_RELATIVE_TP_ID);
3043 break;
3044
3045 case 0x85:
3046 if (mgmt_url_size == 0) {
3047 stmf_scsilib_send_status(task, STATUS_CHECK,
3048 STMF_SAA_INVALID_FIELD_IN_CDB);
3049 goto err_done;
3050 } /* CSTYLED */
3051 {
3052 uint16_t idx, newidx, sz, url_size;
3053 char *url;
3054
3055 p[0] = byte0;
3056 p[1] = 0x85;
3057
3058 idx = 4;
3059 url = mgmt_url;
3060 url_size = sbd_parse_mgmt_url(&url);
3061 /* Creating Network Service Descriptors */
3062 while (url_size != 0) {
3063 /* Null terminated and 4 Byte aligned */
3064 sz = url_size + 1;
3065 sz += (sz % 4) ? 4 - (sz % 4) : 0;
3066 newidx = idx + sz + 4;
3067
3068 if (newidx < bsize) {
3069 /*
3070 * SPC-3r23 : Table 320 (Sec 7.6.5)
3091 }
3092
3093 case 0x86:
3094 page_length = 0x3c;
3095
3096 p[0] = byte0;
3097 p[1] = 0x86; /* Page 86 response */
3098 p[3] = page_length;
3099
3100 /*
3101 * Bits 0, 1, and 2 will need to be updated
3102 * to reflect the queue tag handling if/when
3103 * that is implemented. For now, we're going
3104 * to claim support only for Simple TA.
3105 */
3106 p[5] = 1;
3107 xfer_size = page_length + 4;
3108 break;
3109
3110 case 0xb0:
3111 page_length = 0x3c;
3112 p[0] = byte0;
3113 p[1] = 0xb0;
3114 p[3] = page_length;
3115 p[4] = 1;
3116 p[5] = sbd_ats_max_nblks();
3117 if (sl->sl_flags & SL_UNMAP_ENABLED && sbd_unmap_enable) {
3118 p[20] = (stmf_sbd_unmap_max_nblks >> 24) & 0xff;
3119 p[21] = (stmf_sbd_unmap_max_nblks >> 16) & 0xff;
3120 p[22] = (stmf_sbd_unmap_max_nblks >> 8) & 0xff;
3121 p[23] = stmf_sbd_unmap_max_nblks & 0xff;
3122
3123 p[24] = 0;
3124 p[25] = 0;
3125 p[26] = 0;
3126 p[27] = 0xFF;
3127 }
3128 xfer_size = page_length + 4;
3129 break;
3130
3131 case 0xb2:
3132 if ((sl->sl_flags & SL_UNMAP_ENABLED) == 0) {
3133 stmf_scsilib_send_status(task, STATUS_CHECK,
3134 STMF_SAA_INVALID_FIELD_IN_CDB);
3135 goto err_done;
3136 }
3137 page_length = 4;
3138 p[0] = byte0;
3139 p[1] = 0xb2;
3140 p[3] = page_length;
3141
3142 exp = (uint8_t)sl->sl_data_blocksize_shift;
3143 s = sl->sl_lu_size >> sl->sl_data_blocksize_shift;
3144 while (s & ((uint64_t)0xFFFFFFFF80000000ull)) {
3145 s >>= 1;
3146 exp++;
3147 }
3148 p[4] = exp;
3149 p[5] = 0xc0; /* Logical provisioning UNMAP and WRITE SAME */
3150 xfer_size = page_length + 4;
3151 break;
3152
3153 default:
3154 stmf_scsilib_send_status(task, STATUS_CHECK,
3155 STMF_SAA_INVALID_FIELD_IN_CDB);
3156 goto err_done;
3157 }
3158
3159 sbd_handle_short_read_transfers(task, initial_dbuf, p, cmd_size,
3160 min(cmd_size, xfer_size));
3161 err_done:
3162 kmem_free(p, bsize);
3163 rw_exit(&sbd_global_prop_lock);
3164 }
3165
3166 stmf_status_t
3167 sbd_task_alloc(struct scsi_task *task)
3168 {
3169 if ((task->task_lu_private =
3170 kmem_zalloc(sizeof (sbd_cmd_t), KM_NOSLEEP)) != NULL) {
3171 sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
3172 scmd->flags = 0;
3173 return (STMF_SUCCESS);
3174 }
3175 return (STMF_ALLOC_FAILURE);
3176 }
3177
3178 void
3179 sbd_remove_it_handle(sbd_lu_t *sl, sbd_it_data_t *it)
3180 {
3181 sbd_it_data_t **ppit;
3182
3183 sbd_pgr_remove_it_handle(sl, it);
3184 mutex_enter(&sl->sl_lock);
3185 for (ppit = &sl->sl_it_list; *ppit != NULL;
3186 ppit = &((*ppit)->sbd_it_next)) {
3187 if ((*ppit) == it) {
3188 *ppit = it->sbd_it_next;
3189 break;
3190 }
3216 break;
3217 }
3218 }
3219 ASSERT(it != NULL);
3220 } else {
3221 /*
3222 * We were passed an I_T nexus. If this nexus does not hold
3223 * the reservation, do nothing. This is why this function is
3224 * called "check_and_clear".
3225 */
3226 if ((it->sbd_it_flags & SBD_IT_HAS_SCSI2_RESERVATION) == 0) {
3227 mutex_exit(&sl->sl_lock);
3228 return;
3229 }
3230 }
3231 it->sbd_it_flags &= ~SBD_IT_HAS_SCSI2_RESERVATION;
3232 sl->sl_flags &= ~SL_LU_HAS_SCSI2_RESERVATION;
3233 mutex_exit(&sl->sl_lock);
3234 }
3235
3236 /*
3237 * Given a LU and a task, check if the task is causing reservation
3238 * conflict. Returns 1 in case of conflict, 0 otherwise.
3239 * Note that the LU might not be the same LU as in the task but the
3240 * caller makes sure that the LU can be accessed.
3241 */
3242 int
3243 sbd_check_reservation_conflict(struct sbd_lu *sl, struct scsi_task *task)
3244 {
3245 sbd_it_data_t *it;
3246
3247 it = task->task_lu_itl_handle;
3248 ASSERT(it);
3249 if (sl->sl_access_state == SBD_LU_ACTIVE) {
3250 if (SBD_PGR_RSVD(sl->sl_pgr)) {
3251 if (sbd_pgr_reservation_conflict(task, sl)) {
3252 return (1);
3253 }
3254 } else if ((sl->sl_flags & SL_LU_HAS_SCSI2_RESERVATION) &&
3255 ((it->sbd_it_flags & SBD_IT_HAS_SCSI2_RESERVATION) == 0)) {
3256 if (!(SCSI2_CONFLICT_FREE_CMDS(task->task_cdb))) {
3257 return (1);
3258 }
3259 }
3260 }
3261
3262 return (0);
3263 }
3264
3265 /*
3266 * Keep in mind that sbd_new_task can be called multiple times for the same
3267 * task because of us calling stmf_task_poll_lu resulting in a call to
3268 * sbd_task_poll().
3269 */
3270 void
3271 sbd_new_task(struct scsi_task *task, struct stmf_data_buf *initial_dbuf)
3272 {
3273 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
3274 sbd_it_data_t *it;
3275 uint8_t cdb0, cdb1;
3276 stmf_status_t st_ret;
3277
3278 if ((it = task->task_lu_itl_handle) == NULL) {
3279 mutex_enter(&sl->sl_lock);
3280 for (it = sl->sl_it_list; it != NULL; it = it->sbd_it_next) {
3281 if (it->sbd_it_session_id ==
3282 task->task_session->ss_session_id) {
3283 mutex_exit(&sl->sl_lock);
3284 stmf_scsilib_send_status(task, STATUS_BUSY, 0);
3285 return;
3286 }
3287 }
3288 it = (sbd_it_data_t *)kmem_zalloc(sizeof (*it), KM_NOSLEEP);
3289 if (it == NULL) {
3313 }
3314 } else if (it->sbd_it_flags & SBD_IT_PGR_CHECK_FLAG) {
3315 mutex_enter(&sl->sl_lock);
3316 it->sbd_it_flags &= ~SBD_IT_PGR_CHECK_FLAG;
3317 mutex_exit(&sl->sl_lock);
3318 sbd_pgr_initialize_it(task, it);
3319 }
3320
3321 if (task->task_mgmt_function) {
3322 stmf_scsilib_handle_task_mgmt(task);
3323 return;
3324 }
3325
3326 /*
3327 * if we're transitioning between access
3328 * states, return NOT READY
3329 */
3330 if (sl->sl_access_state == SBD_LU_TRANSITION_TO_STANDBY ||
3331 sl->sl_access_state == SBD_LU_TRANSITION_TO_ACTIVE) {
3332 stmf_scsilib_send_status(task, STATUS_CHECK,
3333 STMF_SAA_LU_NO_ACCESS_TRANSITION);
3334 return;
3335 }
3336
3337 cdb0 = task->task_cdb[0];
3338 cdb1 = task->task_cdb[1];
3339 /*
3340 * Special case for different versions of Windows.
3341 * 1) Windows 2012 and VMWare will fail to discover LU's if a READ
3342 * operation sent down the standby path returns an error. By default
3343 * standby_fail_reads will be set to 0.
3344 * 2) Windows 2008 R2 has a severe performace problem if READ ops
3345 * aren't rejected on the standby path. 2008 sends commands
3346 * down the standby path which then must be proxied over to the
3347 * active node and back.
3348 */
3349 if ((sl->sl_access_state == SBD_LU_STANDBY) &&
3350 stmf_standby_fail_reads &&
3351 (cdb0 == SCMD_READ || cdb0 == SCMD_READ_G1 ||
3352 cdb0 == SCMD_READ_G4 || cdb0 == SCMD_READ_G5)) {
3353 stmf_scsilib_send_status(task, STATUS_CHECK,
3354 STMF_SAA_LU_NO_ACCESS_STANDBY);
3355 return;
3356 }
3357
3358 /*
3359 * Don't go further if cmd is unsupported in standby mode
3360 */
3361 if (sl->sl_access_state == SBD_LU_STANDBY) {
3362 if (cdb0 != SCMD_INQUIRY &&
3363 cdb0 != SCMD_MODE_SENSE &&
3364 cdb0 != SCMD_MODE_SENSE_G1 &&
3365 cdb0 != SCMD_MODE_SELECT &&
3366 cdb0 != SCMD_MODE_SELECT_G1 &&
3367 cdb0 != SCMD_RESERVE &&
3368 cdb0 != SCMD_RELEASE &&
3369 cdb0 != SCMD_PERSISTENT_RESERVE_OUT &&
3370 cdb0 != SCMD_PERSISTENT_RESERVE_IN &&
3371 cdb0 != SCMD_REQUEST_SENSE &&
3372 cdb0 != SCMD_READ_CAPACITY &&
3373 cdb0 != SCMD_TEST_UNIT_READY &&
3374 cdb0 != SCMD_START_STOP &&
3375 cdb0 != SCMD_READ &&
3376 cdb0 != SCMD_READ_G1 &&
3377 cdb0 != SCMD_READ_G4 &&
3378 cdb0 != SCMD_READ_G5 &&
3379 !(cdb0 == SCMD_SVC_ACTION_IN_G4 &&
3380 cdb1 == SSVC_ACTION_READ_CAPACITY_G4) &&
3381 !(cdb0 == SCMD_MAINTENANCE_IN &&
3382 (cdb1 & 0x1F) == 0x05) &&
3383 !(cdb0 == SCMD_MAINTENANCE_IN &&
3384 (cdb1 & 0x1F) == 0x0A)) {
3385 stmf_scsilib_send_status(task, STATUS_CHECK,
3386 STMF_SAA_LU_NO_ACCESS_STANDBY);
3387 return;
3388 }
3389 }
3390
3391 /*
3392 * Checking ua conditions as per SAM3R14 5.3.2 specified order. During
3393 * MPIO/ALUA failover, cmds come in through local ports and proxy port
3394 * port provider (i.e. pppt), we want to report unit attention to
3395 * only local cmds since initiators (Windows MPIO/DSM) would continue
3396 * sending I/O to the target that reported unit attention.
3397 */
3398 if ((it->sbd_it_ua_conditions) &&
3399 !(task->task_additional_flags & TASK_AF_PPPT_TASK) &&
3400 (task->task_cdb[0] != SCMD_INQUIRY)) {
3401 uint32_t saa = 0;
3402
3403 mutex_enter(&sl->sl_lock);
3404 if (it->sbd_it_ua_conditions & SBD_UA_POR) {
3405 it->sbd_it_ua_conditions &= ~SBD_UA_POR;
3406 saa = STMF_SAA_POR;
3407 } else if (it->sbd_it_ua_conditions &
3408 SBD_UA_ASYMMETRIC_ACCESS_CHANGED) {
3409 it->sbd_it_ua_conditions &=
3410 ~SBD_UA_ASYMMETRIC_ACCESS_CHANGED;
3411 saa = STMF_SAA_ASYMMETRIC_ACCESS_CHANGED;
3412 }
3413 mutex_exit(&sl->sl_lock);
3414 if (saa) {
3415 stmf_scsilib_send_status(task, STATUS_CHECK, saa);
3416 return;
3417 }
3418 }
3419
3420 /* Reservation conflict checks */
3421 if (sbd_check_reservation_conflict(sl, task)) {
3422 stmf_scsilib_send_status(task,
3423 STATUS_RESERVATION_CONFLICT, 0);
3424 return;
3425 }
3426
3427 /* Rest of the ua conndition checks */
3428 if ((it->sbd_it_ua_conditions) && (task->task_cdb[0] != SCMD_INQUIRY)) {
3429 uint32_t saa = 0;
3430
3431 mutex_enter(&sl->sl_lock);
3432 if (it->sbd_it_ua_conditions & SBD_UA_CAPACITY_CHANGED) {
3433 it->sbd_it_ua_conditions &= ~SBD_UA_CAPACITY_CHANGED;
3434 if ((task->task_cdb[0] == SCMD_READ_CAPACITY) ||
3435 ((task->task_cdb[0] == SCMD_SVC_ACTION_IN_G4) &&
3436 (task->task_cdb[1] ==
3437 SSVC_ACTION_READ_CAPACITY_G4))) {
3438 saa = 0;
3439 } else {
3440 saa = STMF_SAA_CAPACITY_DATA_HAS_CHANGED;
3441 }
3442 } else if (it->sbd_it_ua_conditions &
3443 SBD_UA_MODE_PARAMETERS_CHANGED) {
3444 it->sbd_it_ua_conditions &=
3445 ~SBD_UA_MODE_PARAMETERS_CHANGED;
3446 saa = STMF_SAA_MODE_PARAMETERS_CHANGED;
3447 } else if (it->sbd_it_ua_conditions &
3448 SBD_UA_ASYMMETRIC_ACCESS_CHANGED) {
3449 saa = 0;
3450 } else if (it->sbd_it_ua_conditions & SBD_UA_POR) {
3451 saa = 0;
3452 } else if (it->sbd_it_ua_conditions &
3453 SBD_UA_ACCESS_STATE_TRANSITION) {
3454 it->sbd_it_ua_conditions &=
3455 ~SBD_UA_ACCESS_STATE_TRANSITION;
3456 saa = STMF_SAA_LU_NO_ACCESS_TRANSITION;
3457 } else {
3458 it->sbd_it_ua_conditions = 0;
3459 saa = 0;
3460 }
3461 mutex_exit(&sl->sl_lock);
3462 if (saa) {
3463 stmf_scsilib_send_status(task, STATUS_CHECK, saa);
3464 return;
3465 }
3466 }
3467
3468 if (sl->sl_access_state == SBD_LU_STANDBY) {
3469 /*
3470 * is this a short write?
3471 * if so, we'll need to wait until we have the buffer
3472 * before proxying the command
3473 */
3474 switch (cdb0) {
3475 case SCMD_MODE_SELECT:
3476 case SCMD_MODE_SELECT_G1:
3477 case SCMD_PERSISTENT_RESERVE_OUT:
3478 break;
3479 default:
3480 st_ret = stmf_proxy_scsi_cmd(task,
3481 initial_dbuf);
3482 if (st_ret != STMF_SUCCESS) {
3483 stmf_scsilib_send_status(task,
3484 STATUS_CHECK,
3485 STMF_SAA_LU_NO_ACCESS_UNAVAIL);
3486 }
3487 return;
3488 }
3626 sbd_handle_mode_sense(task, initial_dbuf, p);
3627 kmem_free(p, 512);
3628 return;
3629 }
3630
3631 if ((cdb0 == SCMD_MODE_SELECT) || (cdb0 == SCMD_MODE_SELECT_G1)) {
3632 sbd_handle_mode_select(task, initial_dbuf);
3633 return;
3634 }
3635
3636 if ((cdb0 == SCMD_UNMAP) && (sl->sl_flags & SL_UNMAP_ENABLED)) {
3637 sbd_handle_unmap(task, initial_dbuf);
3638 return;
3639 }
3640
3641 if ((cdb0 == SCMD_WRITE_SAME_G4) || (cdb0 == SCMD_WRITE_SAME_G1)) {
3642 sbd_handle_write_same(task, initial_dbuf);
3643 return;
3644 }
3645
3646 if (cdb0 == SCMD_COMPARE_AND_WRITE) {
3647 sbd_handle_ats(task, initial_dbuf);
3648 return;
3649 }
3650
3651 if (cdb0 == SCMD_EXTENDED_COPY) {
3652 sbd_handle_xcopy(task, initial_dbuf);
3653 return;
3654 }
3655
3656 if (cdb0 == SCMD_RECV_COPY_RESULTS) {
3657 sbd_handle_recv_copy_results(task, initial_dbuf);
3658 return;
3659 }
3660
3661 if (cdb0 == SCMD_TEST_UNIT_READY) { /* Test unit ready */
3662 task->task_cmd_xfer_length = 0;
3663 stmf_scsilib_send_status(task, STATUS_GOOD, 0);
3664 return;
3665 }
3666
3667 if (cdb0 == SCMD_READ_CAPACITY) { /* Read Capacity */
3668 sbd_handle_read_capacity(task, initial_dbuf);
3669 return;
3670 }
3671
3672 if (cdb0 == SCMD_SVC_ACTION_IN_G4) { /* Read Capacity or read long */
3673 if (cdb1 == SSVC_ACTION_READ_CAPACITY_G4) {
3674 sbd_handle_read_capacity(task, initial_dbuf);
3675 return;
3676 /*
3677 * } else if (cdb1 == SSVC_ACTION_READ_LONG_G4) {
3678 * sbd_handle_read(task, initial_dbuf);
3679 * return;
3680 */
3747 case (SBD_CMD_SCSI_WRITE):
3748 sbd_handle_sgl_write_xfer_completion(task, scmd, dbuf);
3749 break;
3750 default:
3751 cmn_err(CE_PANIC, "Unknown cmd type, task = %p",
3752 (void *)task);
3753 break;
3754 }
3755 return;
3756 }
3757
3758 if ((scmd == NULL) || ((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0))
3759 return;
3760
3761 switch (scmd->cmd_type) {
3762 case (SBD_CMD_SCSI_READ):
3763 sbd_handle_read_xfer_completion(task, scmd, dbuf);
3764 break;
3765
3766 case (SBD_CMD_SCSI_WRITE):
3767 switch (task->task_cdb[0]) {
3768 case SCMD_WRITE_SAME_G1:
3769 case SCMD_WRITE_SAME_G4:
3770 sbd_handle_write_same_xfer_completion(task, scmd, dbuf,
3771 1);
3772 break;
3773 case SCMD_COMPARE_AND_WRITE:
3774 sbd_handle_ats_xfer_completion(task, scmd, dbuf, 1);
3775 break;
3776 default:
3777 sbd_handle_write_xfer_completion(task, scmd, dbuf, 1);
3778 /* FALLTHRU */
3779 }
3780 break;
3781
3782 case (SBD_CMD_SMALL_READ):
3783 sbd_handle_short_read_xfer_completion(task, scmd, dbuf);
3784 break;
3785
3786 case (SBD_CMD_SMALL_WRITE):
3787 sbd_handle_short_write_xfer_completion(task, dbuf);
3788 break;
3789
3790 default:
3791 cmn_err(CE_PANIC, "Unknown cmd type, task = %p", (void *)task);
3792 break;
3793 }
3794 }
3795
3796 /* ARGSUSED */
3797 void
3798 sbd_send_status_done(struct scsi_task *task)
3824 */
3825 /* ARGSUSED */
3826 stmf_status_t
3827 sbd_abort(struct stmf_lu *lu, int abort_cmd, void *arg, uint32_t flags)
3828 {
3829 sbd_lu_t *sl = (sbd_lu_t *)lu->lu_provider_private;
3830 scsi_task_t *task;
3831
3832 if (abort_cmd == STMF_LU_RESET_STATE) {
3833 return (sbd_lu_reset_state(lu));
3834 }
3835
3836 if (abort_cmd == STMF_LU_ITL_HANDLE_REMOVED) {
3837 sbd_check_and_clear_scsi2_reservation(sl, (sbd_it_data_t *)arg);
3838 sbd_remove_it_handle(sl, (sbd_it_data_t *)arg);
3839 return (STMF_SUCCESS);
3840 }
3841
3842 ASSERT(abort_cmd == STMF_LU_ABORT_TASK);
3843 task = (scsi_task_t *)arg;
3844 sbd_ats_remove_by_task(task);
3845 if (task->task_lu_private) {
3846 sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
3847
3848 if (scmd->flags & SBD_SCSI_CMD_ACTIVE) {
3849 if (scmd->flags & SBD_SCSI_CMD_TRANS_DATA) {
3850 kmem_free(scmd->trans_data,
3851 scmd->trans_data_len);
3852 scmd->flags &= ~SBD_SCSI_CMD_TRANS_DATA;
3853 }
3854 scmd->flags &= ~SBD_SCSI_CMD_ACTIVE;
3855 return (STMF_ABORT_SUCCESS);
3856 }
3857 }
3858
3859 return (STMF_NOT_FOUND);
3860 }
3861
3862 void
3863 sbd_task_poll(struct scsi_task *task)
3864 {
3865 stmf_data_buf_t *initial_dbuf;
3866
3867 initial_dbuf = stmf_handle_to_buf(task, 0);
3868 sbd_new_task(task, initial_dbuf);
3869 }
3870
3871 /*
3872 * This function is called during task clean-up if the
3873 * DB_LU_FLAG is set on the dbuf. This should only be called for
3874 * abort processing after sbd_abort has been called for the task.
3875 */
3876 void
3877 sbd_dbuf_free(struct scsi_task *task, struct stmf_data_buf *dbuf)
3878 {
3879 sbd_cmd_t *scmd = (sbd_cmd_t *)task->task_lu_private;
3880 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
3881
3882 ASSERT(dbuf->db_lu_private);
3883 ASSERT(scmd && ATOMIC8_GET(scmd->nbufs) > 0);
3884 ASSERT((scmd->flags & SBD_SCSI_CMD_ACTIVE) == 0);
3885 ASSERT(dbuf->db_flags & DB_LU_DATA_BUF);
3886 ASSERT(task->task_additional_flags & TASK_AF_ACCEPT_LU_DBUF);
3887 ASSERT((curthread->t_flag & T_INTR_THREAD) == 0);
3888
3889 if (scmd->cmd_type == SBD_CMD_SCSI_READ) {
3890 sbd_zvol_rele_read_bufs(sl, dbuf);
3891 } else if (scmd->cmd_type == SBD_CMD_SCSI_WRITE) {
3892 sbd_zvol_rele_write_bufs_abort(sl, dbuf);
3893 } else {
3894 cmn_err(CE_PANIC, "Unknown cmd type %d, task = %p",
3895 scmd->cmd_type, (void *)task);
3896 }
3897 if (atomic_dec_8_nv(&scmd->nbufs) == 0)
3898 rw_exit(&sl->sl_access_state_lock);
3899 stmf_teardown_dbuf(task, dbuf);
3900 stmf_free(dbuf);
3901 }
3902
3903 /* ARGSUSED */
3904 void
3905 sbd_ctl(struct stmf_lu *lu, int cmd, void *arg)
3906 {
3907 sbd_lu_t *sl = (sbd_lu_t *)lu->lu_provider_private;
3908 stmf_change_status_t st;
3909
3910 ASSERT((cmd == STMF_CMD_LU_ONLINE) ||
3911 (cmd == STMF_CMD_LU_OFFLINE) ||
3912 (cmd == STMF_ACK_LU_ONLINE_COMPLETE) ||
3913 (cmd == STMF_ACK_LU_OFFLINE_COMPLETE));
3914
3915 st.st_completion_status = STMF_SUCCESS;
3916 st.st_additional_info = NULL;
3917
3974 }
3975 } else {
3976 sl->sl_flags &= ~SL_WRITEBACK_CACHE_DISABLE;
3977 mutex_exit(&sl->sl_lock);
3978 if (sl->sl_access_state == SBD_LU_ACTIVE) {
3979 (void) sbd_wcd_set(0, sl);
3980 }
3981 }
3982 sbd_pgr_reset(sl);
3983 sbd_check_and_clear_scsi2_reservation(sl, NULL);
3984 if (stmf_deregister_all_lu_itl_handles(lu) != STMF_SUCCESS) {
3985 return (STMF_FAILURE);
3986 }
3987 return (STMF_SUCCESS);
3988 }
3989
3990 sbd_status_t
3991 sbd_flush_data_cache(sbd_lu_t *sl, int fsync_done)
3992 {
3993 int r = 0;
3994 sbd_status_t ret;
3995
3996 rw_enter(&sl->sl_access_state_lock, RW_READER);
3997 if ((sl->sl_flags & SL_MEDIA_LOADED) == 0) {
3998 ret = SBD_FILEIO_FAILURE;
3999 goto flush_fail;
4000 }
4001 if (fsync_done)
4002 goto over_fsync;
4003 if ((sl->sl_data_vtype == VREG) || (sl->sl_data_vtype == VBLK)) {
4004 if (VOP_FSYNC(sl->sl_data_vp, FSYNC, kcred, NULL)) {
4005 ret = SBD_FAILURE;
4006 goto flush_fail;
4007 }
4008 }
4009 over_fsync:
4010 if (((sl->sl_data_vtype == VCHR) || (sl->sl_data_vtype == VBLK)) &&
4011 ((sl->sl_flags & SL_NO_DATA_DKIOFLUSH) == 0)) {
4012 ret = VOP_IOCTL(sl->sl_data_vp, DKIOCFLUSHWRITECACHE, NULL,
4013 FKIOCTL, kcred, &r, NULL);
4014 if ((ret == ENOTTY) || (ret == ENOTSUP)) {
4015 mutex_enter(&sl->sl_lock);
4016 sl->sl_flags |= SL_NO_DATA_DKIOFLUSH;
4017 mutex_exit(&sl->sl_lock);
4018 } else {
4019 ret = (ret != 0) ? SBD_FAILURE : SBD_SUCCESS;
4020 }
4021 }
4022 flush_fail:
4023 rw_exit(&sl->sl_access_state_lock);
4024
4025 return (ret);
4026 }
4027
4028 /* ARGSUSED */
4029 static void
4030 sbd_handle_sync_cache(struct scsi_task *task,
4031 struct stmf_data_buf *initial_dbuf)
4032 {
4033 sbd_lu_t *sl = (sbd_lu_t *)task->task_lu->lu_provider_private;
4034 uint64_t lba, laddr;
4035 sbd_status_t sret;
4036 uint32_t len;
4037 int is_g4 = 0;
4038 int immed;
4039
4040 task->task_cmd_xfer_length = 0;
4041 /*
4042 * Determine if this is a 10 or 16 byte CDB
4043 */
4044
4045 if (task->task_cdb[0] == SCMD_SYNCHRONIZE_CACHE_G4)
|