1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2018 Nexenta Systems, Inc.
14 */
15
16 #include <sys/atomic.h>
17 #include <sys/cmn_err.h>
18 #include <sys/conf.h>
19 #include <sys/cpuvar.h>
20 #include <sys/ddi.h>
21 #include <sys/errno.h>
22 #include <sys/kmem.h>
23 #include <sys/kmem_impl.h>
24 #include <sys/list.h>
25 #include <sys/modctl.h>
26 #include <sys/pci.h>
27 #include <sys/scsi/scsi.h>
28 #include <sys/sunddi.h>
29 #include <sys/sysmacros.h>
30 #include <sys/time.h>
31 #include <sys/types.h>
32
33 #include "pvscsi.h"
34 #include "pvscsi_var.h"
35
36 int pvscsi_enable_msi = 1;
37 int pvscsi_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
38 int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
39
40 static int pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt);
41
42 static void *pvscsi_sstate;
43
44 /* HBA DMA attributes */
45 static ddi_dma_attr_t pvscsi_hba_dma_attr = {
46 .dma_attr_version = DMA_ATTR_V0,
47 .dma_attr_addr_lo = 0x0000000000000000ull,
48 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
49 .dma_attr_count_max = 0x000000007FFFFFFFull,
50 .dma_attr_align = 0x0000000000000001ull,
51 .dma_attr_burstsizes = 0x7ff,
52 .dma_attr_minxfer = 0x00000001u,
53 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
54 .dma_attr_seg = 0x00000000FFFFFFFFull,
55 .dma_attr_sgllen = 1,
56 .dma_attr_granular = 0x00000200u,
57 .dma_attr_flags = 0
58 };
59
60 /* DMA attributes for req/comp rings */
61 static ddi_dma_attr_t pvscsi_ring_dma_attr = {
62 .dma_attr_version = DMA_ATTR_V0,
63 .dma_attr_addr_lo = 0x0000000000000000ull,
64 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
65 .dma_attr_count_max = 0x000000007FFFFFFFull,
66 .dma_attr_align = 0x0000000000000001ull,
67 .dma_attr_burstsizes = 0x7ff,
68 .dma_attr_minxfer = 0x00000001u,
69 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
70 .dma_attr_seg = 0x00000000FFFFFFFFull,
71 .dma_attr_sgllen = 1,
72 .dma_attr_granular = 0x00000001u,
73 .dma_attr_flags = 0
74 };
75
76 /* DMA attributes for buffer I/O */
77 static ddi_dma_attr_t pvscsi_io_dma_attr = {
78 .dma_attr_version = DMA_ATTR_V0,
79 .dma_attr_addr_lo = 0x0000000000000000ull,
80 .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFFull,
81 .dma_attr_count_max = 0x000000007FFFFFFFull,
82 .dma_attr_align = 0x0000000000000001ull,
83 .dma_attr_burstsizes = 0x7ff,
84 .dma_attr_minxfer = 0x00000001u,
85 .dma_attr_maxxfer = 0x00000000FFFFFFFFull,
86 .dma_attr_seg = 0x00000000FFFFFFFFull,
87 .dma_attr_sgllen = PVSCSI_MAX_SG_SIZE,
88 .dma_attr_granular = 0x00000200u,
89 .dma_attr_flags = 0
90 };
91
92 static ddi_device_acc_attr_t pvscsi_mmio_attr = {
93 DDI_DEVICE_ATTR_V1,
94 DDI_STRUCTURE_LE_ACC,
95 DDI_STRICTORDER_ACC,
96 DDI_DEFAULT_ACC
97 };
98
99 static ddi_device_acc_attr_t pvscsi_dma_attrs = {
100 DDI_DEVICE_ATTR_V0,
101 DDI_STRUCTURE_LE_ACC,
102 DDI_STRICTORDER_ACC,
103 DDI_DEFAULT_ACC,
104 };
105
106 static void
107 pvscsi_add_to_queue(pvscsi_cmd_t *cmd)
108 {
109 pvscsi_softc_t *pvs = cmd->cmd_pvs;
110
111 ASSERT(pvs != NULL);
112 ASSERT(mutex_owned(&pvs->mutex));
113 ASSERT(!list_link_active(&(cmd)->cmd_queue_node));
114
115 list_insert_tail(&pvs->cmd_queue, cmd);
116 pvs->cmd_queue_len++;
117 }
118
119 static void
120 pvscsi_remove_from_queue(pvscsi_cmd_t *cmd)
121 {
122 pvscsi_softc_t *pvs = cmd->cmd_pvs;
123
124 ASSERT(pvs != NULL);
125 ASSERT(mutex_owned(&pvs->mutex));
126
127 if (list_link_active(&cmd->cmd_queue_node)) {
128 ASSERT(pvs->cmd_queue_len > 0);
129 list_remove(&pvs->cmd_queue, cmd);
130 pvs->cmd_queue_len--;
131 }
132 }
133
134 static uint64_t
135 pvscsi_map_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_ctx_t *io_ctx)
136 {
137 return (io_ctx - pvs->cmd_ctx + 1);
138 }
139
140 static pvscsi_cmd_ctx_t *
141 pvscsi_lookup_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
142 {
143 pvscsi_cmd_ctx_t *ctx, *end;
144
145 end = &pvs->cmd_ctx[pvs->req_depth];
146 for (ctx = pvs->cmd_ctx; ctx < end; ctx++) {
147 if (ctx->cmd == cmd)
148 return (ctx);
149 }
150
151 return (NULL);
152 }
153
154 static pvscsi_cmd_ctx_t *
155 pvscsi_resolve_ctx(pvscsi_softc_t *pvs, uint64_t ctx)
156 {
157 if (ctx > 0 && ctx <= pvs->req_depth)
158 return (&pvs->cmd_ctx[ctx - 1]);
159 else
160 return (NULL);
161 }
162
163 static boolean_t
164 pvscsi_acquire_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
165 {
166 pvscsi_cmd_ctx_t *ctx;
167
168 if (list_is_empty(&pvs->cmd_ctx_pool))
169 return (B_FALSE);
170
171 ctx = (pvscsi_cmd_ctx_t *)list_remove_head(&pvs->cmd_ctx_pool);
172 ASSERT(ctx != NULL);
173
174 ctx->cmd = cmd;
175 cmd->ctx = ctx;
176
177 return (B_TRUE);
178 }
179
180 static void
181 pvscsi_release_ctx(pvscsi_cmd_t *cmd)
182 {
183 pvscsi_softc_t *pvs = cmd->cmd_pvs;
184
185 ASSERT(mutex_owned(&pvs->mutex));
186
187 cmd->ctx->cmd = NULL;
188 list_insert_tail(&pvs->cmd_ctx_pool, cmd->ctx);
189 cmd->ctx = NULL;
190 }
191
192 static uint32_t
193 pvscsi_reg_read(pvscsi_softc_t *pvs, uint32_t offset)
194 {
195 uint32_t ret;
196
197 ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
198
199 ret = ddi_get32(pvs->mmio_handle,
200 (uint32_t *)(pvs->mmio_base + offset));
201
202 return (ret);
203 }
204
205 static void
206 pvscsi_reg_write(pvscsi_softc_t *pvs, uint32_t offset, uint32_t value)
207 {
208 ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
209
210 ddi_put32(pvs->mmio_handle, (uint32_t *)(pvs->mmio_base + offset),
211 value);
212 }
213
214 static void
215 pvscsi_write_cmd_desc(pvscsi_softc_t *pvs, uint32_t cmd, void *desc, size_t len)
216 {
217 len /= sizeof (uint32_t);
218 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_COMMAND, cmd);
219 ddi_rep_put32(pvs->mmio_handle, (uint32_t *)desc,
220 (uint32_t *)(pvs->mmio_base + PVSCSI_REG_OFFSET_COMMAND_DATA),
221 len, DDI_DEV_NO_AUTOINCR);
222 }
223
224 static uint32_t
225 pvscsi_read_intr_status(pvscsi_softc_t *pvs)
226 {
227 return (pvscsi_reg_read(pvs, PVSCSI_REG_OFFSET_INTR_STATUS));
228 }
229
230 static void
231 pvscsi_write_intr_status(pvscsi_softc_t *pvs, uint32_t val)
232 {
233 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_STATUS, val);
234 }
235
236 static void
237 pvscsi_mask_intr(pvscsi_softc_t *pvs)
238 {
239 mutex_enter(&pvs->intr_mutex);
240
241 VERIFY(pvs->intr_lock_counter >= 0);
242
243 if (++pvs->intr_lock_counter == 1)
244 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
245
246 mutex_exit(&pvs->intr_mutex);
247 }
248
249 static void
250 pvscsi_unmask_intr(pvscsi_softc_t *pvs)
251 {
252 mutex_enter(&pvs->intr_mutex);
253
254 VERIFY(pvs->intr_lock_counter > 0);
255
256 if (--pvs->intr_lock_counter == 0) {
257 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
258 PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
259 }
260
261 mutex_exit(&pvs->intr_mutex);
262 }
263
264 static void
265 pvscsi_reset_hba(pvscsi_softc_t *pvs)
266 {
267 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
268 }
269
270 static void
271 pvscsi_reset_bus(pvscsi_softc_t *pvs)
272 {
273 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_BUS, NULL, 0);
274 }
275
276 static void
277 pvscsi_submit_nonrw_io(pvscsi_softc_t *pvs)
278 {
279 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
280 }
281
282 static void
283 pvscsi_submit_rw_io(pvscsi_softc_t *pvs)
284 {
285 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
286 }
287
288
289 static int
290 pvscsi_inquiry_target(pvscsi_softc_t *pvs, int target, struct scsi_inquiry *inq)
291 {
292 int len = sizeof (struct scsi_inquiry);
293 int ret = -1;
294 struct buf *b;
295 struct scsi_address ap;
296 struct scsi_pkt *pkt;
297 uint8_t cdb[CDB_GROUP0];
298
299 ap.a_hba_tran = pvs->tran;
300 ap.a_target = (ushort_t)target;
301 ap.a_lun = (uchar_t)0;
302
303 if ((b = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL, len,
304 B_READ, NULL_FUNC, NULL)) == NULL)
305 return (-1);
306
307 if ((pkt = scsi_init_pkt(&ap, (struct scsi_pkt *)NULL, b,
308 CDB_GROUP0, sizeof (struct scsi_arq_status), 0, 0,
309 NULL_FUNC, NULL)) == NULL)
310 goto free_buf;
311
312 cdb[0] = SCMD_INQUIRY;
313 cdb[1] = 0;
314 cdb[2] = 0;
315 cdb[3] = (len & 0xff00) >> 8;
316 cdb[4] = (len & 0x00ff);
317 cdb[5] = 0;
318
319 if (inq != NULL)
320 bzero(inq, sizeof (*inq));
321 bcopy(cdb, pkt->pkt_cdbp, CDB_GROUP0);
322 bzero((struct scsi_inquiry *)b->b_un.b_addr, sizeof (*inq));
323
324 if ((ret = scsi_poll(pkt)) == 0 && inq != NULL)
325 bcopy(b->b_un.b_addr, inq, sizeof (*inq));
326
327 scsi_destroy_pkt(pkt);
328
329 free_buf:
330 scsi_free_consistent_buf(b);
331
332 return (ret);
333 }
334
335 static int
336 pvscsi_config_one(dev_info_t *pdip, pvscsi_softc_t *pvs, int target,
337 dev_info_t **childp)
338 {
339 char **compatible = NULL;
340 char *nodename = NULL;
341 dev_info_t *dip;
342 int inqrc;
343 int ncompatible = 0;
344 pvscsi_device_t *devnode;
345 struct scsi_inquiry inq;
346
347 ASSERT(DEVI_BUSY_OWNED(pdip));
348
349 /* Inquiry target */
350 inqrc = pvscsi_inquiry_target(pvs, target, &inq);
351
352 /* Find devnode */
353 for (devnode = list_head(&pvs->devnodes); devnode != NULL;
354 devnode = list_next(&pvs->devnodes, devnode)) {
355 if (devnode->target == target)
356 break;
357 }
358
359 if (devnode != NULL) {
360 if (inqrc != 0) {
361 (void) ndi_devi_offline(devnode->pdip,
362 NDI_DEVFS_CLEAN | NDI_DEVI_REMOVE);
363 list_remove(&pvs->devnodes, devnode);
364 kmem_free(devnode, sizeof (*devnode));
365 } else if (childp != NULL) {
366 /* Target exists */
367 *childp = devnode->pdip;
368 }
369 return (NDI_SUCCESS);
370 } else if (inqrc != 0) {
371 /* Target doesn't exist */
372 return (NDI_FAILURE);
373 }
374
375 scsi_hba_nodename_compatible_get(&inq, NULL, inq.inq_dtype, NULL,
376 &nodename, &compatible, &ncompatible);
377 if (nodename == NULL)
378 goto free_nodename;
379
380 if (ndi_devi_alloc(pdip, nodename, DEVI_SID_NODEID,
381 &dip) != NDI_SUCCESS) {
382 dev_err(pvs->dip, CE_WARN, "!failed to alloc device instance");
383 goto free_nodename;
384 }
385
386 if (ndi_prop_update_string(DDI_DEV_T_NONE, dip,
387 "device-type", "scsi") != DDI_PROP_SUCCESS ||
388 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
389 "target", target) != DDI_PROP_SUCCESS ||
390 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
391 "lun", 0) != DDI_PROP_SUCCESS ||
392 ndi_prop_update_int(DDI_DEV_T_NONE, dip,
393 "pm-capable", 1) != DDI_PROP_SUCCESS ||
394 ndi_prop_update_string_array(DDI_DEV_T_NONE, dip,
395 "compatible", compatible, ncompatible) != DDI_PROP_SUCCESS) {
396 dev_err(pvs->dip, CE_WARN,
397 "!failed to update props for target %d", target);
398 goto free_devi;
399 }
400
401 if ((devnode = kmem_zalloc(sizeof (*devnode), KM_NOSLEEP)) == NULL)
402 goto free_devi;
403
404 if (ndi_devi_online(dip, NDI_ONLINE_ATTACH) != NDI_SUCCESS) {
405 dev_err(pvs->dip, CE_WARN, "!failed to online target %d",
406 target);
407 kmem_free(devnode, sizeof (*devnode));
408 goto free_devi;
409 }
410
411 devnode->target = target;
412 devnode->pdip = dip;
413 devnode->parent = pdip;
414 list_insert_tail(&pvs->devnodes, devnode);
415
416 if (childp != NULL)
417 *childp = dip;
418
419 scsi_hba_nodename_compatible_free(nodename, compatible);
420
421 return (NDI_SUCCESS);
422
423 free_devi:
424 ndi_prop_remove_all(dip);
425 (void) ndi_devi_free(dip);
426 free_nodename:
427 scsi_hba_nodename_compatible_free(nodename, compatible);
428
429 return (NDI_FAILURE);
430 }
431
432 static int
433 pvscsi_config_all(dev_info_t *pdip, pvscsi_softc_t *pvs)
434 {
435 int target;
436
437 for (target = 0; target < PVSCSI_MAXTGTS; target++) {
438 /* ndi_devi_enter is done in pvscsi_bus_config */
439 (void) pvscsi_config_one(pdip, pvs, target, NULL);
440 }
441
442 return (NDI_SUCCESS);
443 }
444
445 static pvscsi_cmd_t *
446 pvscsi_process_comp_ring(pvscsi_softc_t *pvs)
447 {
448 pvscsi_cmd_t **pnext_cmd = NULL;
449 pvscsi_cmd_t *cmd;
450 pvscsi_cmd_t *head = NULL;
451 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
452 uint32_t cmp_ne = sdesc->cmpNumEntriesLog2;
453
454 ASSERT(mutex_owned(&pvs->rx_mutex));
455
456 while (sdesc->cmpConsIdx != sdesc->cmpProdIdx) {
457 pvscsi_cmd_ctx_t *ctx;
458 struct PVSCSIRingCmpDesc *cdesc;
459
460 cdesc = CMP_RING(pvs) + (sdesc->cmpConsIdx & MASK(cmp_ne));
461 membar_consumer();
462
463 ctx = pvscsi_resolve_ctx(pvs, cdesc->context);
464 ASSERT(ctx != NULL);
465
466 if ((cmd = ctx->cmd) != NULL) {
467 cmd->next_cmd = NULL;
468
469 /* Save command status for further processing */
470 cmd->cmp_stat.host_status = cdesc->hostStatus;
471 cmd->cmp_stat.scsi_status = cdesc->scsiStatus;
472 cmd->cmp_stat.data_len = cdesc->dataLen;
473
474 /* Mark this command as arrived from hardware */
475 cmd->flags |= PVSCSI_FLAG_HW_STATUS;
476
477 if (head == NULL) {
478 head = cmd;
479 head->tail_cmd = cmd;
480 } else {
481 head->tail_cmd = cmd;
482 }
483
484 if (pnext_cmd == NULL) {
485 pnext_cmd = &cmd->next_cmd;
486 } else {
487 *pnext_cmd = cmd;
488 pnext_cmd = &cmd->next_cmd;
489 }
490 }
491
492 membar_consumer();
493 sdesc->cmpConsIdx++;
494 }
495
496 return (head);
497 }
498
499 static pvscsi_msg_t *
500 pvscsi_process_msg_ring(pvscsi_softc_t *pvs)
501 {
502 pvscsi_msg_t *msg;
503 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
504 struct PVSCSIRingMsgDesc *mdesc;
505 struct PVSCSIMsgDescDevStatusChanged *desc;
506 uint32_t msg_ne = sdesc->msgNumEntriesLog2;
507
508 ASSERT(mutex_owned(&pvs->rx_mutex));
509
510 if (sdesc->msgProdIdx == sdesc->msgConsIdx)
511 return (NULL);
512
513 mdesc = MSG_RING(pvs) + (sdesc->msgConsIdx & MASK(msg_ne));
514 membar_consumer();
515
516 switch (mdesc->type) {
517 case PVSCSI_MSG_DEV_ADDED:
518 case PVSCSI_MSG_DEV_REMOVED:
519 desc = (struct PVSCSIMsgDescDevStatusChanged *)mdesc;
520 msg = kmem_alloc(sizeof (pvscsi_msg_t), KM_NOSLEEP);
521 if (msg == NULL)
522 return (NULL);
523 msg->msg_pvs = pvs;
524 msg->type = mdesc->type;
525 msg->target = desc->target;
526 break;
527 default:
528 dev_err(pvs->dip, CE_WARN, "!unknown msg type: %d",
529 mdesc->type);
530 return (NULL);
531 }
532
533 membar_consumer();
534 sdesc->msgConsIdx++;
535
536 return (msg);
537 }
538
539 static void
540 pvscsi_handle_msg(void *arg)
541 {
542 pvscsi_msg_t *msg = (pvscsi_msg_t *)arg;
543 dev_info_t *dip = msg->msg_pvs->dip;
544 int circ;
545
546 ndi_devi_enter(dip, &circ);
547 (void) pvscsi_config_one(dip, msg->msg_pvs, msg->target, NULL);
548 ndi_devi_exit(dip, circ);
549
550 kmem_free(msg, sizeof (pvscsi_msg_t));
551 }
552
553 static int
554 pvscsi_abort_cmd(pvscsi_cmd_t *cmd, pvscsi_cmd_t **pending)
555 {
556 pvscsi_softc_t *pvs = cmd->cmd_pvs;
557 pvscsi_cmd_t *c;
558 pvscsi_cmd_t *done;
559 struct PVSCSICmdDescAbortCmd acmd;
560
561 dev_err(pvs->dip, CE_WARN, "!aborting command %p", (void *)cmd);
562
563 ASSERT(mutex_owned(&pvs->rx_mutex));
564 ASSERT(mutex_owned(&pvs->tx_mutex));
565
566 /* Check if the cmd was already completed by the HBA */
567 *pending = done = pvscsi_process_comp_ring(pvs);
568 for (c = done; c != NULL; c = c->next_cmd) {
569 if (c == cmd)
570 return (CMD_CMPLT);
571 }
572
573 /* Check if cmd was really scheduled by the HBA */
574 if (pvscsi_lookup_ctx(pvs, cmd) == NULL)
575 return (CMD_CMPLT);
576
577 /* Abort cmd in the HBA */
578 bzero(&acmd, sizeof (acmd));
579 acmd.target = cmd->cmd_target;
580 acmd.context = pvscsi_map_ctx(pvs, cmd->ctx);
581 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ABORT_CMD, &acmd, sizeof (acmd));
582
583 /* Check if cmd was completed by the HBA before it could be aborted */
584 if ((done = pvscsi_process_comp_ring(pvs)) != NULL) {
585 done->tail_cmd->next_cmd = *pending;
586 *pending = done;
587 for (c = done; c != NULL; c = c->next_cmd) {
588 if (c == cmd)
589 return (CMD_CMPLT);
590 }
591 }
592
593 /* Release I/O ctx */
594 mutex_enter(&pvs->mutex);
595 if (cmd->ctx != NULL)
596 pvscsi_release_ctx(cmd);
597 /* Remove cmd from the queue */
598 pvscsi_remove_from_queue(cmd);
599 mutex_exit(&pvs->mutex);
600
601 /* Insert cmd at the beginning of the list */
602 cmd->next_cmd = *pending;
603 *pending = cmd;
604
605 dev_err(pvs->dip, CE_WARN, "!command %p aborted", (void *)cmd);
606
607 return (CMD_ABORTED);
608 }
609
610 static void
611 pvscsi_map_buffers(pvscsi_cmd_t *cmd, struct PVSCSIRingReqDesc *rdesc)
612 {
613 int i;
614
615 ASSERT(cmd->ctx);
616 ASSERT(cmd->cmd_dmaccount > 0 && cmd->cmd_dmaccount <=
617 PVSCSI_MAX_SG_SIZE);
618
619 rdesc->dataLen = cmd->cmd_dma_count;
620 rdesc->dataAddr = 0;
621
622 if (cmd->cmd_dma_count == 0)
623 return;
624
625 if (cmd->cmd_dmaccount > 1) {
626 struct PVSCSISGElement *sgl = CMD_CTX_SGLIST_VA(cmd->ctx);
627
628 for (i = 0; i < cmd->cmd_dmaccount; i++) {
629 sgl[i].addr = cmd->cached_cookies[i].dmac_laddress;
630 sgl[i].length = cmd->cached_cookies[i].dmac_size;
631 sgl[i].flags = 0;
632 }
633 rdesc->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
634 rdesc->dataAddr = (uint64_t)CMD_CTX_SGLIST_PA(cmd->ctx);
635 } else {
636 rdesc->dataAddr = cmd->cached_cookies[0].dmac_laddress;
637 }
638 }
639
640 static void
641 pvscsi_comp_cmd(pvscsi_cmd_t *cmd, uint8_t status)
642 {
643 struct scsi_pkt *pkt = CMD2PKT(cmd);
644
645 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
646 STATE_GOT_STATUS);
647 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0)
648 pkt->pkt_state |= STATE_XFERRED_DATA;
649 pkt->pkt_reason = CMD_CMPLT;
650 pkt->pkt_resid = 0;
651 *(pkt->pkt_scbp) = status;
652 }
653
654 static void
655 pvscsi_set_status(pvscsi_cmd_t *cmd)
656 {
657 pvscsi_softc_t *pvs = cmd->cmd_pvs;
658 struct scsi_pkt *pkt = CMD2PKT(cmd);
659 uchar_t scsi_status = cmd->cmp_stat.scsi_status;
660 uint32_t host_status = cmd->cmp_stat.host_status;
661
662 if (scsi_status != STATUS_GOOD &&
663 (host_status == BTSTAT_SUCCESS ||
664 (host_status == BTSTAT_LINKED_COMMAND_COMPLETED) ||
665 (host_status == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG))) {
666 if (scsi_status == STATUS_CHECK) {
667 struct scsi_arq_status *astat = (void*)(pkt->pkt_scbp);
668 uint8_t *sensedata;
669 int arq_size;
670
671 *pkt->pkt_scbp = scsi_status;
672 pkt->pkt_state |= STATE_ARQ_DONE;
673
674 if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
675 arq_size = (cmd->cmd_rqslen >=
676 SENSE_BUFFER_SIZE) ? SENSE_BUFFER_SIZE :
677 cmd->cmd_rqslen;
678
679 astat->sts_rqpkt_resid = SENSE_BUFFER_SIZE -
680 arq_size;
681 sensedata = (uint8_t *)&astat->sts_sensedata;
682 bcopy(cmd->arqbuf->b_un.b_addr, sensedata,
683 arq_size);
684
685 pkt->pkt_state |= STATE_XARQ_DONE;
686 } else {
687 astat->sts_rqpkt_resid = 0;
688 }
689
690 astat->sts_rqpkt_statistics = 0;
691 astat->sts_rqpkt_reason = CMD_CMPLT;
692 (*(uint8_t *)&astat->sts_rqpkt_status) = STATUS_GOOD;
693 astat->sts_rqpkt_state = STATE_GOT_BUS |
694 STATE_GOT_TARGET | STATE_SENT_CMD |
695 STATE_XFERRED_DATA | STATE_GOT_STATUS;
696 }
697 pvscsi_comp_cmd(cmd, scsi_status);
698
699 return;
700 }
701
702 switch (host_status) {
703 case BTSTAT_SUCCESS:
704 case BTSTAT_LINKED_COMMAND_COMPLETED:
705 case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
706 pvscsi_comp_cmd(cmd, STATUS_GOOD);
707 break;
708 case BTSTAT_DATARUN:
709 pkt->pkt_reason = CMD_DATA_OVR;
710 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
711 STATE_SENT_CMD | STATE_GOT_STATUS |
712 STATE_XFERRED_DATA);
713 pkt->pkt_resid = 0;
714 break;
715 case BTSTAT_DATA_UNDERRUN:
716 pkt->pkt_reason = pkt->pkt_state |= (STATE_GOT_BUS |
717 STATE_GOT_TARGET | STATE_SENT_CMD | STATE_GOT_STATUS);
718 pkt->pkt_resid = cmd->dma_count - cmd->cmp_stat.data_len;
719 if (pkt->pkt_resid != cmd->dma_count)
720 pkt->pkt_state |= STATE_XFERRED_DATA;
721 break;
722 case BTSTAT_SELTIMEO:
723 pkt->pkt_reason = CMD_DEV_GONE;
724 pkt->pkt_state |= STATE_GOT_BUS;
725 break;
726 case BTSTAT_TAGREJECT:
727 pkt->pkt_reason = CMD_TAG_REJECT;
728 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
729 STATE_SENT_CMD | STATE_GOT_STATUS);
730 break;
731 case BTSTAT_BADMSG:
732 pkt->pkt_reason = CMD_BADMSG;
733 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
734 STATE_SENT_CMD | STATE_GOT_STATUS);
735 break;
736 case BTSTAT_SENTRST:
737 case BTSTAT_RECVRST:
738 case BTSTAT_BUSRESET:
739 pkt->pkt_reason = CMD_RESET;
740 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
741 STATE_SENT_CMD | STATE_GOT_STATUS);
742 break;
743 case BTSTAT_ABORTQUEUE:
744 pkt->pkt_reason = CMD_ABORTED;
745 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
746 STATE_SENT_CMD | STATE_GOT_STATUS);
747 break;
748 case BTSTAT_HAHARDWARE:
749 case BTSTAT_INVPHASE:
750 case BTSTAT_HATIMEOUT:
751 case BTSTAT_NORESPONSE:
752 case BTSTAT_DISCONNECT:
753 case BTSTAT_HASOFTWARE:
754 case BTSTAT_BUSFREE:
755 case BTSTAT_SENSFAILED:
756 pkt->pkt_reason = CMD_TRAN_ERR;
757 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
758 STATE_SENT_CMD | STATE_GOT_STATUS);
759 break;
760 default:
761 dev_err(pvs->dip, CE_WARN,
762 "!unknown host status code: %d", host_status);
763 pkt->pkt_reason = CMD_TRAN_ERR;
764 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
765 STATE_SENT_CMD | STATE_GOT_STATUS);
766 break;
767 }
768 }
769
770 static void
771 pvscsi_complete_chained(void *arg)
772 {
773 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)arg;
774 pvscsi_cmd_t *c;
775 struct scsi_pkt *pkt;
776
777 while (cmd != NULL) {
778 pvscsi_softc_t *pvs = cmd->cmd_pvs;
779
780 c = cmd->next_cmd;
781 cmd->next_cmd = NULL;
782
783 pkt = CMD2PKT(cmd);
784 if (pkt == NULL)
785 return;
786
787 if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0 &&
788 (cmd->flags & PVSCSI_FLAG_IO_READ) != 0) {
789 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
790 DDI_DMA_SYNC_FORCPU);
791 }
792
793 mutex_enter(&pvs->mutex);
794 /* Release I/O ctx */
795 if (cmd->ctx != NULL)
796 pvscsi_release_ctx(cmd);
797 /* Remove command from queue */
798 pvscsi_remove_from_queue(cmd);
799 mutex_exit(&pvs->mutex);
800
801 if ((cmd->flags & PVSCSI_FLAG_HW_STATUS) != 0) {
802 pvscsi_set_status(cmd);
803 } else {
804 ASSERT((cmd->flags & PVSCSI_FLAGS_NON_HW_COMPLETION) !=
805 0);
806
807 if ((cmd->flags & PVSCSI_FLAG_TIMED_OUT) != 0) {
808 cmd->pkt->pkt_reason = CMD_TIMEOUT;
809 cmd->pkt->pkt_statistics |=
810 (STAT_TIMEOUT | STAT_ABORTED);
811 } else if ((cmd->flags & PVSCSI_FLAG_ABORTED) != 0) {
812 cmd->pkt->pkt_reason = CMD_ABORTED;
813 cmd->pkt->pkt_statistics |=
814 (STAT_TIMEOUT | STAT_ABORTED);
815 } else if ((cmd->flags & PVSCSI_FLAGS_RESET) != 0) {
816 cmd->pkt->pkt_reason = CMD_RESET;
817 if ((cmd->flags & PVSCSI_FLAG_RESET_BUS) != 0) {
818 cmd->pkt->pkt_statistics |=
819 STAT_BUS_RESET;
820 } else {
821 cmd->pkt->pkt_statistics |=
822 STAT_DEV_RESET;
823 }
824 }
825 }
826
827 cmd->flags |= PVSCSI_FLAG_DONE;
828 cmd->flags &= ~PVSCSI_FLAG_TRANSPORT;
829
830 if ((pkt->pkt_flags & FLAG_NOINTR) == 0 &&
831 pkt->pkt_comp != NULL)
832 (*pkt->pkt_comp)(pkt);
833
834 cmd = c;
835 }
836 }
837
838 static void
839 pvscsi_dev_reset(pvscsi_softc_t *pvs, int target)
840 {
841 struct PVSCSICmdDescResetDevice cmd = { 0 };
842
843 cmd.target = target;
844 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof (cmd));
845 }
846
847 static int
848 pvscsi_poll_cmd(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
849 {
850 boolean_t seen_intr;
851 int cycles = (cmd->pkt->pkt_time * 1000000) / USECS_TO_WAIT;
852 int i;
853 pvscsi_cmd_t *dcmd;
854 struct scsi_pkt *pkt = CMD2PKT(cmd);
855
856 /*
857 * Make sure we're not missing any commands completed
858 * concurrently before we have actually disabled interrupts.
859 */
860 mutex_enter(&pvs->rx_mutex);
861 dcmd = pvscsi_process_comp_ring(pvs);
862 mutex_exit(&pvs->rx_mutex);
863
864 pvscsi_complete_chained(dcmd);
865
866 while ((cmd->flags & PVSCSI_FLAG_DONE) == 0) {
867 seen_intr = B_FALSE;
868
869 /* Disable interrupts from H/W */
870 pvscsi_mask_intr(pvs);
871
872 /* Wait for interrupt to arrive */
873 for (i = 0; i < cycles; i++) {
874 uint32_t status;
875
876 mutex_enter(&pvs->rx_mutex);
877 mutex_enter(&pvs->intr_mutex);
878 status = pvscsi_read_intr_status(pvs);
879 if ((status & PVSCSI_INTR_ALL_SUPPORTED) != 0) {
880 /* Check completion ring */
881 mutex_exit(&pvs->intr_mutex);
882 dcmd = pvscsi_process_comp_ring(pvs);
883 mutex_exit(&pvs->rx_mutex);
884 seen_intr = B_TRUE;
885 break;
886 } else {
887 mutex_exit(&pvs->intr_mutex);
888 mutex_exit(&pvs->rx_mutex);
889 drv_usecwait(USECS_TO_WAIT);
890 }
891 }
892
893 /* Enable interrupts from H/W */
894 pvscsi_unmask_intr(pvs);
895
896 if (!seen_intr) {
897 /* No interrupts seen from device during the timeout */
898 mutex_enter(&pvs->tx_mutex);
899 mutex_enter(&pvs->rx_mutex);
900 if ((cmd->flags & PVSCSI_FLAGS_COMPLETION) != 0) {
901 /* Command was cancelled asynchronously */
902 dcmd = NULL;
903 } else if ((pvscsi_abort_cmd(cmd,
904 &dcmd)) == CMD_ABORTED) {
905 /* Command was cancelled in hardware */
906 pkt->pkt_state |= (STAT_TIMEOUT | STAT_ABORTED);
907 pkt->pkt_statistics |= (STAT_TIMEOUT |
908 STAT_ABORTED);
909 pkt->pkt_reason = CMD_TIMEOUT;
910 }
911 mutex_exit(&pvs->rx_mutex);
912 mutex_exit(&pvs->tx_mutex);
913
914 /*
915 * Complete commands that might be on completion list.
916 * Target command can also be on the list in case it was
917 * completed before it could be actually cancelled.
918 */
919 break;
920 }
921
922 pvscsi_complete_chained(dcmd);
923
924 if (!seen_intr)
925 break;
926 }
927
928 return (TRAN_ACCEPT);
929 }
930
931 static void
932 pvscsi_abort_all(struct scsi_address *ap, pvscsi_softc_t *pvs,
933 pvscsi_cmd_t **pending, int marker_flag)
934 {
935 int qlen = pvs->cmd_queue_len;
936 pvscsi_cmd_t *cmd, *pcmd, *phead = NULL;
937
938 ASSERT(mutex_owned(&pvs->rx_mutex));
939 ASSERT(mutex_owned(&pvs->tx_mutex));
940
941 /*
942 * Try to abort all queued commands, merging commands waiting
943 * for completion into a single list to complete them at one
944 * time when mutex is released.
945 */
946 while (qlen > 0) {
947 mutex_enter(&pvs->mutex);
948 cmd = list_remove_head(&pvs->cmd_queue);
949 ASSERT(cmd != NULL);
950
951 qlen--;
952
953 if (ap == NULL || ap->a_target == cmd->cmd_target) {
954 int c = --pvs->cmd_queue_len;
955
956 mutex_exit(&pvs->mutex);
957
958 if (pvscsi_abort_cmd(cmd, &pcmd) == CMD_ABORTED) {
959 /*
960 * Assume command is completely cancelled now,
961 * so mark it as requested.
962 */
963 cmd->flags |= marker_flag;
964 }
965
966 qlen -= (c - pvs->cmd_queue_len);
967
968 /*
969 * Now merge current pending commands with
970 * previous ones.
971 */
972 if (phead == NULL) {
973 phead = pcmd;
974 } else if (pcmd != NULL) {
975 phead->tail_cmd->next_cmd = pcmd;
976 phead->tail_cmd = pcmd->tail_cmd;
977 }
978 } else {
979 list_insert_tail(&pvs->cmd_queue, cmd);
980 mutex_exit(&pvs->mutex);
981 }
982 }
983
984 *pending = phead;
985 }
986
987 static void
988 pvscsi_quiesce_notify(pvscsi_softc_t *pvs)
989 {
990 mutex_enter(&pvs->mutex);
991 if (pvs->cmd_queue_len == 0 &&
992 (pvs->flags & PVSCSI_HBA_QUIESCE_PENDING) != 0) {
993 pvs->flags &= ~PVSCSI_HBA_QUIESCE_PENDING;
994 cv_broadcast(&pvs->quiescevar);
995 }
996 mutex_exit(&pvs->mutex);
997 }
998
999 static int
1000 pvscsi_transport_command(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
1001 {
1002 struct PVSCSIRingReqDesc *rdesc;
1003 struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
1004 struct scsi_pkt *pkt = CMD2PKT(cmd);
1005 uint32_t req_ne = sdesc->reqNumEntriesLog2;
1006
1007 mutex_enter(&pvs->tx_mutex);
1008 mutex_enter(&pvs->mutex);
1009 if (!pvscsi_acquire_ctx(pvs, cmd)) {
1010 mutex_exit(&pvs->mutex);
1011 mutex_exit(&pvs->tx_mutex);
1012 dev_err(pvs->dip, CE_WARN, "!no free ctx available");
1013 return (TRAN_BUSY);
1014 }
1015
1016 if ((sdesc->reqProdIdx - sdesc->cmpConsIdx) >= (1 << req_ne)) {
1017 pvscsi_release_ctx(cmd);
1018 mutex_exit(&pvs->mutex);
1019 mutex_exit(&pvs->tx_mutex);
1020 dev_err(pvs->dip, CE_WARN, "!no free I/O slots available");
1021 return (TRAN_BUSY);
1022 }
1023 mutex_exit(&pvs->mutex);
1024
1025 cmd->flags |= PVSCSI_FLAG_TRANSPORT;
1026
1027 rdesc = REQ_RING(pvs) + (sdesc->reqProdIdx & MASK(req_ne));
1028
1029 bzero(&rdesc->lun, sizeof (rdesc->lun));
1030
1031 rdesc->bus = 0;
1032 rdesc->target = cmd->cmd_target;
1033
1034 if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
1035 bzero((void*)cmd->arqbuf->b_un.b_addr, SENSE_BUFFER_SIZE);
1036 rdesc->senseLen = SENSE_BUFFER_SIZE;
1037 rdesc->senseAddr = cmd->arqc.dmac_laddress;
1038 } else {
1039 rdesc->senseLen = 0;
1040 rdesc->senseAddr = 0;
1041 }
1042
1043 rdesc->vcpuHint = CPU->cpu_id;
1044 rdesc->cdbLen = cmd->cmdlen;
1045 bcopy(cmd->cmd_cdb, rdesc->cdb, cmd->cmdlen);
1046
1047 /* Setup tag info */
1048 if ((cmd->flags & PVSCSI_FLAG_TAG) != 0)
1049 rdesc->tag = cmd->tag;
1050 else
1051 rdesc->tag = MSG_SIMPLE_QTAG;
1052
1053 /* Setup I/O direction and map data buffers */
1054 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1055 if ((cmd->flags & PVSCSI_FLAG_IO_READ) != 0)
1056 rdesc->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
1057 else
1058 rdesc->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
1059 pvscsi_map_buffers(cmd, rdesc);
1060 } else {
1061 rdesc->flags = 0;
1062 }
1063
1064 rdesc->context = pvscsi_map_ctx(pvs, cmd->ctx);
1065 membar_producer();
1066
1067 sdesc->reqProdIdx++;
1068 membar_producer();
1069
1070 mutex_enter(&pvs->mutex);
1071 cmd->timeout_lbolt = ddi_get_lbolt() + SEC_TO_TICK(pkt->pkt_time);
1072 pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD);
1073 pvscsi_add_to_queue(cmd);
1074
1075 switch (cmd->pkt->pkt_cdbp[0]) {
1076 case SCMD_READ:
1077 case SCMD_WRITE:
1078 case SCMD_READ_G1:
1079 case SCMD_WRITE_G1:
1080 case SCMD_READ_G4:
1081 case SCMD_WRITE_G4:
1082 case SCMD_READ_G5:
1083 case SCMD_WRITE_G5:
1084 ASSERT((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0);
1085 pvscsi_submit_rw_io(pvs);
1086 break;
1087 default:
1088 pvscsi_submit_nonrw_io(pvs);
1089 break;
1090 }
1091 mutex_exit(&pvs->mutex);
1092 mutex_exit(&pvs->tx_mutex);
1093
1094 return (TRAN_ACCEPT);
1095 }
1096
1097 static int
1098 pvscsi_reset_generic(pvscsi_softc_t *pvs, struct scsi_address *ap)
1099 {
1100 boolean_t bus_reset = (ap == NULL);
1101 int flags;
1102 pvscsi_cmd_t *done, *aborted;
1103
1104 flags = bus_reset ? PVSCSI_FLAG_RESET_BUS : PVSCSI_FLAG_RESET_DEV;
1105
1106 mutex_enter(&pvs->tx_mutex);
1107 mutex_enter(&pvs->rx_mutex);
1108 /* Try to process pending requests */
1109 done = pvscsi_process_comp_ring(pvs);
1110
1111 /* Abort all pending requests */
1112 pvscsi_abort_all(ap, pvs, &aborted, flags);
1113
1114 /* Reset at hardware level */
1115 if (bus_reset) {
1116 pvscsi_reset_bus(pvs);
1117 /* Should never happen after bus reset */
1118 ASSERT(pvscsi_process_comp_ring(pvs) == NULL);
1119 } else {
1120 pvscsi_dev_reset(pvs, ap->a_target);
1121 }
1122 mutex_exit(&pvs->rx_mutex);
1123 mutex_exit(&pvs->tx_mutex);
1124
1125 pvscsi_complete_chained(done);
1126 pvscsi_complete_chained(aborted);
1127
1128 return (1);
1129 }
1130
1131 static void
1132 pvscsi_cmd_ext_free(pvscsi_cmd_t *cmd)
1133 {
1134 struct scsi_pkt *pkt = CMD2PKT(cmd);
1135
1136 if ((cmd->flags & PVSCSI_FLAG_CDB_EXT) != 0) {
1137 kmem_free(pkt->pkt_cdbp, cmd->cmdlen);
1138 cmd->flags &= ~PVSCSI_FLAG_CDB_EXT;
1139 }
1140 if ((cmd->flags & PVSCSI_FLAG_SCB_EXT) != 0) {
1141 kmem_free(pkt->pkt_scbp, cmd->statuslen);
1142 cmd->flags &= ~PVSCSI_FLAG_SCB_EXT;
1143 }
1144 if ((cmd->flags & PVSCSI_FLAG_PRIV_EXT) != 0) {
1145 kmem_free(pkt->pkt_private, cmd->tgtlen);
1146 cmd->flags &= ~PVSCSI_FLAG_PRIV_EXT;
1147 }
1148 }
1149
1150 /* ARGSUSED pvs */
1151 static int
1152 pvscsi_cmd_ext_alloc(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd, int kf)
1153 {
1154 struct scsi_pkt *pkt = CMD2PKT(cmd);
1155 void *buf;
1156
1157 if (cmd->cmdlen > sizeof (cmd->cmd_cdb)) {
1158 if ((buf = kmem_zalloc(cmd->cmdlen, kf)) == NULL)
1159 return (DDI_FAILURE);
1160 pkt->pkt_cdbp = buf;
1161 cmd->flags |= PVSCSI_FLAG_CDB_EXT;
1162 }
1163
1164 if (cmd->statuslen > sizeof (cmd->cmd_scb)) {
1165 if ((buf = kmem_zalloc(cmd->statuslen, kf)) == NULL)
1166 goto out;
1167 pkt->pkt_scbp = buf;
1168 cmd->flags |= PVSCSI_FLAG_SCB_EXT;
1169 cmd->cmd_rqslen = (cmd->statuslen - sizeof (cmd->cmd_scb));
1170 }
1171
1172 if (cmd->tgtlen > sizeof (cmd->tgt_priv)) {
1173 if ((buf = kmem_zalloc(cmd->tgtlen, kf)) == NULL)
1174 goto out;
1175 pkt->pkt_private = buf;
1176 cmd->flags |= PVSCSI_FLAG_PRIV_EXT;
1177 }
1178
1179 return (DDI_SUCCESS);
1180
1181 out:
1182 pvscsi_cmd_ext_free(cmd);
1183
1184 return (DDI_FAILURE);
1185 }
1186
1187 static int
1188 pvscsi_setup_dma_buffer(pvscsi_softc_t *pvs, size_t length,
1189 pvscsi_dma_buf_t *buf)
1190 {
1191 ddi_dma_cookie_t cookie;
1192 uint_t ccount;
1193
1194 if ((ddi_dma_alloc_handle(pvs->dip, &pvscsi_ring_dma_attr,
1195 DDI_DMA_SLEEP, NULL, &buf->dma_handle)) != DDI_SUCCESS) {
1196 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1197 return (DDI_FAILURE);
1198 }
1199
1200 if ((ddi_dma_mem_alloc(buf->dma_handle, length, &pvscsi_dma_attrs,
1201 DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &buf->addr,
1202 &buf->real_length, &buf->acc_handle)) != DDI_SUCCESS) {
1203 dev_err(pvs->dip, CE_WARN,
1204 "!failed to allocate %ld bytes for DMA buffer", length);
1205 ddi_dma_free_handle(&buf->dma_handle);
1206 return (DDI_FAILURE);
1207 }
1208
1209 if ((ddi_dma_addr_bind_handle(buf->dma_handle, NULL, buf->addr,
1210 buf->real_length, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, DDI_DMA_SLEEP,
1211 NULL, &cookie, &ccount)) != DDI_SUCCESS) {
1212 dev_err(pvs->dip, CE_WARN, "!failed to bind DMA buffer");
1213 ddi_dma_free_handle(&buf->dma_handle);
1214 ddi_dma_mem_free(&buf->acc_handle);
1215 return (DDI_FAILURE);
1216 }
1217
1218 /* TODO Support multipart SG regions */
1219 ASSERT(ccount == 1);
1220
1221 buf->pa = cookie.dmac_laddress;
1222
1223 return (DDI_SUCCESS);
1224 }
1225
1226 static void
1227 pvscsi_free_dma_buffer(pvscsi_dma_buf_t *buf)
1228 {
1229 ddi_dma_free_handle(&buf->dma_handle);
1230 ddi_dma_mem_free(&buf->acc_handle);
1231 }
1232
1233 static int
1234 pvscsi_setup_sg(pvscsi_softc_t *pvs)
1235 {
1236 int i;
1237 pvscsi_cmd_ctx_t *ctx;
1238 size_t size = pvs->req_depth * sizeof (pvscsi_cmd_ctx_t);
1239
1240 ctx = pvs->cmd_ctx = kmem_zalloc(size, KM_SLEEP);
1241
1242 for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1243 list_insert_tail(&pvs->cmd_ctx_pool, ctx);
1244 if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1245 &ctx->dma_buf) != DDI_SUCCESS)
1246 goto cleanup;
1247 }
1248
1249 return (DDI_SUCCESS);
1250
1251 cleanup:
1252 for (; i >= 0; --i, --ctx) {
1253 list_remove(&pvs->cmd_ctx_pool, ctx);
1254 pvscsi_free_dma_buffer(&ctx->dma_buf);
1255 }
1256 kmem_free(pvs->cmd_ctx, size);
1257
1258 return (DDI_FAILURE);
1259 }
1260
1261 static void
1262 pvscsi_free_sg(pvscsi_softc_t *pvs)
1263 {
1264 int i;
1265 pvscsi_cmd_ctx_t *ctx = pvs->cmd_ctx;
1266
1267 for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1268 list_remove(&pvs->cmd_ctx_pool, ctx);
1269 pvscsi_free_dma_buffer(&ctx->dma_buf);
1270 }
1271
1272 kmem_free(pvs->cmd_ctx, pvs->req_pages << PAGE_SHIFT);
1273 }
1274
1275 static int
1276 pvscsi_allocate_rings(pvscsi_softc_t *pvs)
1277 {
1278 /* Allocate DMA buffer for rings state */
1279 if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1280 &pvs->rings_state_buf) != DDI_SUCCESS)
1281 return (DDI_FAILURE);
1282
1283 /* Allocate DMA buffer for request ring */
1284 pvs->req_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_REQ_RING);
1285 pvs->req_depth = pvs->req_pages * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
1286 if (pvscsi_setup_dma_buffer(pvs, pvs->req_pages * PAGE_SIZE,
1287 &pvs->req_ring_buf) != DDI_SUCCESS)
1288 goto free_rings_state;
1289
1290 /* Allocate completion ring */
1291 pvs->cmp_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_CMP_RING);
1292 if (pvscsi_setup_dma_buffer(pvs, pvs->cmp_pages * PAGE_SIZE,
1293 &pvs->cmp_ring_buf) != DDI_SUCCESS)
1294 goto free_req_buf;
1295
1296 /* Allocate message ring */
1297 pvs->msg_pages = MIN(pvscsi_msg_ring_pages,
1298 PVSCSI_MAX_NUM_PAGES_MSG_RING);
1299 if (pvscsi_setup_dma_buffer(pvs, pvs->msg_pages * PAGE_SIZE,
1300 &pvs->msg_ring_buf) != DDI_SUCCESS)
1301 goto free_cmp_buf;
1302
1303 return (DDI_SUCCESS);
1304
1305 free_cmp_buf:
1306 pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1307 free_req_buf:
1308 pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1309 free_rings_state:
1310 pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1311
1312 return (DDI_FAILURE);
1313 }
1314
1315 static void
1316 pvscsi_free_rings(pvscsi_softc_t *pvs)
1317 {
1318 pvscsi_free_dma_buffer(&pvs->msg_ring_buf);
1319 pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1320 pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1321 pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1322 }
1323
1324 static void
1325 pvscsi_setup_rings(pvscsi_softc_t *pvs)
1326 {
1327 int i;
1328 struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
1329 struct PVSCSICmdDescSetupRings cmd = { 0 };
1330 uint64_t base;
1331
1332 cmd.ringsStatePPN = pvs->rings_state_buf.pa >> PAGE_SHIFT;
1333 cmd.reqRingNumPages = pvs->req_pages;
1334 cmd.cmpRingNumPages = pvs->cmp_pages;
1335
1336 /* Setup request ring */
1337 base = pvs->req_ring_buf.pa;
1338 for (i = 0; i < pvs->req_pages; i++) {
1339 cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
1340 base += PAGE_SIZE;
1341 }
1342
1343 /* Setup completion ring */
1344 base = pvs->cmp_ring_buf.pa;
1345 for (i = 0; i < pvs->cmp_pages; i++) {
1346 cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
1347 base += PAGE_SIZE;
1348 }
1349
1350 bzero(RINGS_STATE(pvs), PAGE_SIZE);
1351 bzero(REQ_RING(pvs), pvs->req_pages * PAGE_SIZE);
1352 bzero(CMP_RING(pvs), pvs->cmp_pages * PAGE_SIZE);
1353
1354 /* Issue SETUP command */
1355 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof (cmd));
1356
1357 /* Setup message ring */
1358 cmd_msg.numPages = pvs->msg_pages;
1359 base = pvs->msg_ring_buf.pa;
1360
1361 for (i = 0; i < pvs->msg_pages; i++) {
1362 cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
1363 base += PAGE_SIZE;
1364 }
1365 bzero(MSG_RING(pvs), pvs->msg_pages * PAGE_SIZE);
1366
1367 pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_MSG_RING, &cmd_msg,
1368 sizeof (cmd_msg));
1369 }
1370
1371 static int
1372 pvscsi_setup_io(pvscsi_softc_t *pvs)
1373 {
1374 int offset, rcount, rn, type;
1375 int ret = DDI_FAILURE;
1376 off_t regsize;
1377 pci_regspec_t *regs;
1378 uint_t regs_length;
1379
1380 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, pvs->dip,
1381 DDI_PROP_DONTPASS, "reg", (int **)®s,
1382 ®s_length) != DDI_PROP_SUCCESS) {
1383 dev_err(pvs->dip, CE_WARN, "!failed to lookup 'reg' property");
1384 return (DDI_FAILURE);
1385 }
1386
1387 rcount = regs_length * sizeof (int) / sizeof (pci_regspec_t);
1388
1389 for (offset = PCI_CONF_BASE0; offset <= PCI_CONF_BASE5; offset += 4) {
1390 for (rn = 0; rn < rcount; ++rn) {
1391 if (PCI_REG_REG_G(regs[rn].pci_phys_hi) == offset) {
1392 type = regs[rn].pci_phys_hi & PCI_ADDR_MASK;
1393 break;
1394 }
1395 }
1396
1397 if (rn >= rcount)
1398 continue;
1399
1400 if (type != PCI_ADDR_IO) {
1401 if (ddi_dev_regsize(pvs->dip, rn,
1402 ®size) != DDI_SUCCESS) {
1403 dev_err(pvs->dip, CE_WARN,
1404 "!failed to get size of reg %d", rn);
1405 goto out;
1406 }
1407 if (regsize == PVSCSI_MEM_SPACE_SIZE) {
1408 if (ddi_regs_map_setup(pvs->dip, rn,
1409 &pvs->mmio_base, 0, 0,
1410 &pvscsi_mmio_attr,
1411 &pvs->mmio_handle) != DDI_SUCCESS) {
1412 dev_err(pvs->dip, CE_WARN,
1413 "!failed to map MMIO BAR");
1414 goto out;
1415 }
1416 ret = DDI_SUCCESS;
1417 break;
1418 }
1419 }
1420 }
1421
1422 out:
1423 ddi_prop_free(regs);
1424
1425 return (ret);
1426 }
1427
1428 static void
1429 pvscsi_free_io(pvscsi_softc_t *pvs)
1430 {
1431 ddi_regs_map_free(&pvs->mmio_handle);
1432 }
1433
1434 static int
1435 pvscsi_enable_intrs(pvscsi_softc_t *pvs)
1436 {
1437 int i, rc, intr_caps;
1438
1439 if ((rc = ddi_intr_get_cap(pvs->intr_htable[0], &intr_caps)) !=
1440 DDI_SUCCESS) {
1441 dev_err(pvs->dip, CE_WARN, "!failed to get interrupt caps");
1442 return (DDI_FAILURE);
1443 }
1444
1445 if ((intr_caps & DDI_INTR_FLAG_BLOCK) != 0) {
1446 if ((rc = ddi_intr_block_enable(pvs->intr_htable,
1447 pvs->intr_cnt)) != DDI_SUCCESS) {
1448 dev_err(pvs->dip, CE_WARN,
1449 "!failed to enable interrupt block");
1450 }
1451 } else {
1452 for (i = 0; i < pvs->intr_cnt; i++) {
1453 if ((rc = ddi_intr_enable(pvs->intr_htable[i])) ==
1454 DDI_SUCCESS)
1455 continue;
1456 dev_err(pvs->dip, CE_WARN,
1457 "!failed to enable interrupt");
1458 while (--i >= 0)
1459 (void) ddi_intr_disable(pvs->intr_htable[i]);
1460 break;
1461 }
1462 }
1463
1464 /* Unmask interrupts */
1465 if (rc == DDI_SUCCESS) {
1466 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
1467 PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
1468 }
1469
1470 return (rc);
1471 }
1472
1473 /* ARGSUSED arg2 */
1474 static uint32_t
1475 pvscsi_intr_handler(caddr_t arg1, caddr_t arg2)
1476 {
1477 boolean_t handled;
1478 pvscsi_softc_t *pvs = (pvscsi_softc_t *)arg1;
1479 uint32_t status;
1480
1481 mutex_enter(&pvs->intr_mutex);
1482 if (pvs->num_pollers > 0) {
1483 mutex_exit(&pvs->intr_mutex);
1484 return (DDI_INTR_CLAIMED);
1485 }
1486
1487 if (pvscsi_enable_msi) {
1488 handled = B_TRUE;
1489 } else {
1490 status = pvscsi_read_intr_status(pvs);
1491 handled = (status & PVSCSI_INTR_ALL_SUPPORTED) != 0;
1492 if (handled)
1493 pvscsi_write_intr_status(pvs, status);
1494 }
1495 mutex_exit(&pvs->intr_mutex);
1496
1497 if (handled) {
1498 boolean_t qnotify;
1499 pvscsi_cmd_t *pending;
1500 pvscsi_msg_t *msg;
1501
1502 mutex_enter(&pvs->rx_mutex);
1503 pending = pvscsi_process_comp_ring(pvs);
1504 msg = pvscsi_process_msg_ring(pvs);
1505 mutex_exit(&pvs->rx_mutex);
1506
1507 mutex_enter(&pvs->mutex);
1508 qnotify = HBA_QUIESCE_PENDING(pvs);
1509 mutex_exit(&pvs->mutex);
1510
1511 if (pending != NULL && ddi_taskq_dispatch(pvs->comp_tq,
1512 pvscsi_complete_chained, pending,
1513 DDI_NOSLEEP) == DDI_FAILURE)
1514 pvscsi_complete_chained(pending);
1515
1516 if (msg != NULL && ddi_taskq_dispatch(pvs->msg_tq,
1517 pvscsi_handle_msg, msg, DDI_NOSLEEP) == DDI_FAILURE) {
1518 dev_err(pvs->dip, CE_WARN,
1519 "!failed to process msg type %d for target %d",
1520 msg->type, msg->target);
1521 kmem_free(msg, sizeof (pvscsi_msg_t));
1522 }
1523
1524 if (qnotify)
1525 pvscsi_quiesce_notify(pvs);
1526 }
1527
1528 return (handled ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
1529 }
1530
1531 static int
1532 pvscsi_register_isr(pvscsi_softc_t *pvs, int type)
1533 {
1534 int navail, nactual;
1535 int i;
1536
1537 if (ddi_intr_get_navail(pvs->dip, type, &navail) != DDI_SUCCESS ||
1538 navail == 0) {
1539 dev_err(pvs->dip, CE_WARN,
1540 "!failed to get number of available interrupts of type %d",
1541 type);
1542 return (DDI_FAILURE);
1543 }
1544 navail = MIN(navail, PVSCSI_MAX_INTRS);
1545
1546 pvs->intr_size = navail * sizeof (ddi_intr_handle_t);
1547 if ((pvs->intr_htable = kmem_alloc(pvs->intr_size, KM_SLEEP)) == NULL) {
1548 dev_err(pvs->dip, CE_WARN,
1549 "!failed to allocate %d bytes for interrupt hashtable",
1550 pvs->intr_size);
1551 return (DDI_FAILURE);
1552 }
1553
1554 if (ddi_intr_alloc(pvs->dip, pvs->intr_htable, type, 0, navail,
1555 &nactual, DDI_INTR_ALLOC_NORMAL) != DDI_SUCCESS || nactual == 0) {
1556 dev_err(pvs->dip, CE_WARN, "!failed to allocate %d interrupts",
1557 navail);
1558 goto free_htable;
1559 }
1560
1561 pvs->intr_cnt = nactual;
1562
1563 if (ddi_intr_get_pri(pvs->intr_htable[0],
1564 (uint_t *)&pvs->intr_pri) != DDI_SUCCESS) {
1565 dev_err(pvs->dip, CE_WARN, "!failed to get interrupt priority");
1566 goto free_intrs;
1567 }
1568
1569 for (i = 0; i < nactual; i++) {
1570 if (ddi_intr_add_handler(pvs->intr_htable[i],
1571 pvscsi_intr_handler, (caddr_t)pvs, NULL) != DDI_SUCCESS) {
1572 dev_err(pvs->dip, CE_WARN,
1573 "!failed to add interrupt handler");
1574 goto free_intrs;
1575 }
1576 }
1577
1578 return (DDI_SUCCESS);
1579
1580 free_intrs:
1581 for (i = 0; i < nactual; i++)
1582 (void) ddi_intr_free(pvs->intr_htable[i]);
1583 free_htable:
1584 kmem_free(pvs->intr_htable, pvs->intr_size);
1585
1586 return (DDI_FAILURE);
1587 }
1588
1589 static void
1590 pvscsi_free_intr_resources(pvscsi_softc_t *pvs)
1591 {
1592 int i;
1593
1594 for (i = 0; i < pvs->intr_cnt; i++) {
1595 (void) ddi_intr_disable(pvs->intr_htable[i]);
1596 (void) ddi_intr_remove_handler(pvs->intr_htable[i]);
1597 (void) ddi_intr_free(pvs->intr_htable[i]);
1598 }
1599 kmem_free(pvs->intr_htable, pvs->intr_size);
1600 }
1601
1602 static int
1603 pvscsi_setup_isr(pvscsi_softc_t *pvs)
1604 {
1605 int intr_types;
1606
1607 if (ddi_intr_get_supported_types(pvs->dip,
1608 &intr_types) != DDI_SUCCESS) {
1609 dev_err(pvs->dip, CE_WARN,
1610 "!failed to get supported interrupt types");
1611 return (DDI_FAILURE);
1612 }
1613
1614 if ((intr_types & DDI_INTR_TYPE_MSIX) != 0 && pvscsi_enable_msi) {
1615 if (pvscsi_register_isr(pvs,
1616 DDI_INTR_TYPE_MSIX) == DDI_SUCCESS) {
1617 pvs->intr_type = DDI_INTR_TYPE_MSIX;
1618 } else {
1619 dev_err(pvs->dip, CE_WARN,
1620 "!failed to install MSI-X interrupt handler");
1621 }
1622 } else if ((intr_types & DDI_INTR_TYPE_MSI) != 0 && pvscsi_enable_msi) {
1623 if (pvscsi_register_isr(pvs,
1624 DDI_INTR_TYPE_MSI) == DDI_SUCCESS) {
1625 pvs->intr_type = DDI_INTR_TYPE_MSI;
1626 } else {
1627 dev_err(pvs->dip, CE_WARN,
1628 "!failed to install MSI interrupt handler");
1629 }
1630 } else if ((intr_types & DDI_INTR_TYPE_FIXED) != 0) {
1631 if (pvscsi_register_isr(pvs,
1632 DDI_INTR_TYPE_FIXED) == DDI_SUCCESS) {
1633 pvs->intr_type = DDI_INTR_TYPE_FIXED;
1634 } else {
1635 dev_err(pvs->dip, CE_WARN,
1636 "!failed to install FIXED interrupt handler");
1637 }
1638 }
1639
1640 return (pvs->intr_type == 0 ? DDI_FAILURE : DDI_SUCCESS);
1641 }
1642
1643 static void
1644 pvscsi_wd_thread(pvscsi_softc_t *pvs)
1645 {
1646 clock_t now;
1647 pvscsi_cmd_t *expired, *c, *cn, **pnext;
1648
1649 mutex_enter(&pvs->mutex);
1650 for (;;) {
1651 expired = NULL;
1652 pnext = NULL;
1653 now = ddi_get_lbolt();
1654
1655 for (c = list_head(&pvs->cmd_queue); c != NULL; ) {
1656 cn = list_next(&pvs->cmd_queue, c);
1657
1658 /*
1659 * Commands with 'FLAG_NOINTR' are watched using their
1660 * own timeouts, so we should not touch them.
1661 */
1662 if ((c->pkt->pkt_flags & FLAG_NOINTR) == 0 &&
1663 now > c->timeout_lbolt) {
1664 dev_err(pvs->dip, CE_WARN,
1665 "!expired command: %p (%ld > %ld)",
1666 (void *)c, now, c->timeout_lbolt);
1667 pvscsi_remove_from_queue(c);
1668 if (expired == NULL)
1669 expired = c;
1670 if (pnext == NULL) {
1671 pnext = &c->next_cmd;
1672 } else {
1673 *pnext = c;
1674 pnext = &c->next_cmd;
1675 }
1676 }
1677 c = cn;
1678 }
1679 mutex_exit(&pvs->mutex);
1680
1681 /* Now cancel all expired commands */
1682 if (expired != NULL) {
1683 struct scsi_address sa = {0};
1684 /* Build a fake SCSI address */
1685 sa.a_hba_tran = pvs->tran;
1686 while (expired != NULL) {
1687 c = expired->next_cmd;
1688 sa.a_target = expired->cmd_target;
1689 sa.a_lun = 0;
1690 (void) pvscsi_abort(&sa, CMD2PKT(expired));
1691 expired = c;
1692 }
1693 }
1694
1695 mutex_enter(&pvs->mutex);
1696 if ((pvs->flags & PVSCSI_DRIVER_SHUTDOWN) != 0) {
1697 /* Finish job */
1698 break;
1699 }
1700 if (cv_reltimedwait(&pvs->wd_condvar, &pvs->mutex,
1701 SEC_TO_TICK(1), TR_CLOCK_TICK) > 0) {
1702 /* Explicitly woken up, finish job */
1703 break;
1704 }
1705 }
1706
1707 /* Confirm thread termination */
1708 cv_signal(&pvs->syncvar);
1709 mutex_exit(&pvs->mutex);
1710 }
1711
1712 static int
1713 pvscsi_ccache_constructor(void *buf, void *cdrarg, int kmflags)
1714 {
1715 int (*callback)(caddr_t);
1716 uint_t cookiec;
1717 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)buf;
1718 pvscsi_softc_t *pvs = cdrarg;
1719 struct scsi_address ap;
1720
1721 callback = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
1722 ap.a_hba_tran = pvs->tran;
1723 ap.a_target = 0;
1724 ap.a_lun = 0;
1725
1726 /* Allocate a DMA handle for data transfers */
1727 if ((ddi_dma_alloc_handle(pvs->dip, &pvs->io_dma_attr, callback,
1728 NULL, &cmd->cmd_dmahdl)) != DDI_SUCCESS) {
1729 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1730 return (-1);
1731 }
1732
1733 /* Setup ARQ buffer */
1734 if ((cmd->arqbuf = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL,
1735 SENSE_BUFFER_SIZE, B_READ, callback, NULL)) == NULL) {
1736 dev_err(pvs->dip, CE_WARN, "!failed to allocate ARQ buffer");
1737 goto free_handle;
1738 }
1739
1740 if (ddi_dma_alloc_handle(pvs->dip, &pvs->hba_dma_attr,
1741 callback, NULL, &cmd->arqhdl) != DDI_SUCCESS) {
1742 dev_err(pvs->dip, CE_WARN,
1743 "!failed to allocate DMA handle for ARQ buffer");
1744 goto free_arqbuf;
1745 }
1746
1747 if (ddi_dma_buf_bind_handle(cmd->arqhdl, cmd->arqbuf,
1748 (DDI_DMA_READ | DDI_DMA_CONSISTENT), callback, NULL,
1749 &cmd->arqc, &cookiec) != DDI_SUCCESS) {
1750 dev_err(pvs->dip, CE_WARN, "!failed to bind ARQ buffer");
1751 goto free_arqhdl;
1752 }
1753
1754 return (0);
1755
1756 free_arqhdl:
1757 ddi_dma_free_handle(&cmd->arqhdl);
1758 free_arqbuf:
1759 scsi_free_consistent_buf(cmd->arqbuf);
1760 free_handle:
1761 ddi_dma_free_handle(&cmd->cmd_dmahdl);
1762
1763 return (-1);
1764 }
1765
1766 /* ARGSUSED cdrarg */
1767 static void
1768 pvscsi_ccache_destructor(void *buf, void *cdrarg)
1769 {
1770 pvscsi_cmd_t *cmd = (pvscsi_cmd_t *)buf;
1771
1772 if (cmd->cmd_dmahdl != NULL) {
1773 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1774 ddi_dma_free_handle(&cmd->cmd_dmahdl);
1775 cmd->cmd_dmahdl = NULL;
1776 }
1777
1778 if (cmd->arqhdl != NULL) {
1779 (void) ddi_dma_unbind_handle(cmd->arqhdl);
1780 ddi_dma_free_handle(&cmd->arqhdl);
1781 cmd->arqhdl = NULL;
1782 }
1783
1784 if (cmd->arqbuf != NULL) {
1785 scsi_free_consistent_buf(cmd->arqbuf);
1786 cmd->arqbuf = NULL;
1787 }
1788 }
1789
1790 /* tran_* entry points and setup */
1791 /* ARGSUSED hba_dip tgt_dip hba_tran */
1792 static int
1793 pvscsi_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
1794 scsi_hba_tran_t *hba_tran, struct scsi_device *sd)
1795 {
1796 pvscsi_softc_t *pvs = SDEV2PRIV(sd);
1797
1798 ASSERT(pvs != NULL);
1799
1800 if (sd->sd_address.a_target >= PVSCSI_MAXTGTS)
1801 return (DDI_FAILURE);
1802
1803 return (DDI_SUCCESS);
1804 }
1805
1806 static int
1807 pvscsi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
1808 {
1809 boolean_t poll = ((pkt->pkt_flags & FLAG_NOINTR) != 0);
1810 int rc;
1811 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1812 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1813
1814 ASSERT(cmd->pkt == pkt);
1815 ASSERT(cmd->cmd_pvs == pvs);
1816
1817 /*
1818 * Reinitialize some fields because the packet may
1819 * have been resubmitted.
1820 */
1821 pkt->pkt_reason = CMD_CMPLT;
1822 pkt->pkt_state = 0;
1823 pkt->pkt_statistics = 0;
1824
1825 /* Zero status byte */
1826 *(pkt->pkt_scbp) = 0;
1827
1828 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1829 ASSERT(cmd->cmd_dma_count != 0);
1830 pkt->pkt_resid = cmd->cmd_dma_count;
1831
1832 /*
1833 * Consistent packets need to be synced first
1834 * (only for data going out).
1835 */
1836 if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0) {
1837 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
1838 DDI_DMA_SYNC_FORDEV);
1839 }
1840 }
1841
1842 cmd->cmd_target = ap->a_target;
1843
1844 mutex_enter(&pvs->mutex);
1845 if (HBA_IS_QUIESCED(pvs) && !poll) {
1846 mutex_exit(&pvs->mutex);
1847 return (TRAN_BUSY);
1848 }
1849 mutex_exit(&pvs->mutex);
1850
1851 rc = pvscsi_transport_command(pvs, cmd);
1852
1853 if (poll) {
1854 pvscsi_cmd_t *dcmd;
1855 boolean_t qnotify;
1856
1857 if (rc == TRAN_ACCEPT)
1858 rc = pvscsi_poll_cmd(pvs, cmd);
1859
1860 mutex_enter(&pvs->rx_mutex);
1861 dcmd = pvscsi_process_comp_ring(pvs);
1862 mutex_exit(&pvs->rx_mutex);
1863
1864 mutex_enter(&pvs->mutex);
1865 qnotify = HBA_QUIESCE_PENDING(pvs);
1866 mutex_exit(&pvs->mutex);
1867
1868 pvscsi_complete_chained(dcmd);
1869
1870 if (qnotify)
1871 pvscsi_quiesce_notify(pvs);
1872 }
1873
1874 return (rc);
1875 }
1876
1877 static int
1878 pvscsi_reset(struct scsi_address *ap, int level)
1879 {
1880 pvscsi_softc_t *pvs = AP2PRIV(ap);
1881
1882 switch (level) {
1883 case RESET_ALL:
1884 return (pvscsi_reset_generic(pvs, NULL));
1885 case RESET_TARGET:
1886 ASSERT(ap != NULL);
1887 return (pvscsi_reset_generic(pvs, ap));
1888 default:
1889 return (0);
1890 }
1891 }
1892
1893 static int
1894 pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
1895 {
1896 boolean_t qnotify = B_FALSE;
1897 pvscsi_cmd_t *pending;
1898 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1899
1900 mutex_enter(&pvs->tx_mutex);
1901 mutex_enter(&pvs->rx_mutex);
1902 if (pkt != NULL) {
1903 /* Abort single command */
1904 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1905
1906 if (pvscsi_abort_cmd(cmd, &pending) == CMD_ABORTED) {
1907 /* Assume command is completely cancelled now */
1908 cmd->flags |= PVSCSI_FLAG_ABORTED;
1909 }
1910 } else {
1911 /* Abort all commands on the bus */
1912 pvscsi_abort_all(ap, pvs, &pending, PVSCSI_FLAG_ABORTED);
1913 }
1914 qnotify = HBA_QUIESCE_PENDING(pvs);
1915 mutex_exit(&pvs->rx_mutex);
1916 mutex_exit(&pvs->tx_mutex);
1917
1918 pvscsi_complete_chained(pending);
1919
1920 if (qnotify)
1921 pvscsi_quiesce_notify(pvs);
1922
1923 return (1);
1924 }
1925
1926 /* ARGSUSED tgtonly */
1927 static int
1928 pvscsi_getcap(struct scsi_address *ap, char *cap, int tgtonly)
1929 {
1930 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1931
1932 if (cap == NULL)
1933 return (-1);
1934
1935 switch (scsi_hba_lookup_capstr(cap)) {
1936 case SCSI_CAP_ARQ:
1937 return ((pvs->flags & PVSCSI_HBA_AUTO_REQUEST_SENSE) != 0);
1938 case SCSI_CAP_UNTAGGED_QING:
1939 return (1);
1940 default:
1941 return (-1);
1942 }
1943 }
1944
1945 /* ARGSUSED tgtonly */
1946 static int
1947 pvscsi_setcap(struct scsi_address *ap, char *cap, int value, int tgtonly)
1948 {
1949 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1950
1951 if (cap == NULL)
1952 return (-1);
1953
1954 switch (scsi_hba_lookup_capstr(cap)) {
1955 case SCSI_CAP_ARQ:
1956 mutex_enter(&pvs->mutex);
1957 if (value == 0)
1958 pvs->flags &= ~PVSCSI_HBA_AUTO_REQUEST_SENSE;
1959 else
1960 pvs->flags |= PVSCSI_HBA_AUTO_REQUEST_SENSE;
1961 mutex_exit(&pvs->mutex);
1962 return (1);
1963 default:
1964 return (0);
1965 }
1966 }
1967
1968 static void
1969 pvscsi_destroy_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
1970 {
1971 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1972 pvscsi_softc_t *pvs = ap->a_hba_tran->tran_hba_private;
1973
1974 ASSERT(cmd->cmd_pvs == pvs);
1975
1976 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1977 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
1978 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1979 }
1980
1981 if (cmd->ctx != NULL) {
1982 mutex_enter(&pvs->mutex);
1983 pvscsi_release_ctx(cmd);
1984 mutex_exit(&pvs->mutex);
1985 }
1986
1987 if ((cmd->flags & PVSCSI_FLAGS_EXT) != 0)
1988 pvscsi_cmd_ext_free(cmd);
1989
1990 kmem_cache_free(pvs->cmd_cache, cmd);
1991 }
1992
1993 static struct scsi_pkt *
1994 pvscsi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt, struct buf *bp,
1995 int cmdlen, int statuslen, int tgtlen, int flags, int (*callback)(),
1996 caddr_t arg)
1997 {
1998 boolean_t is_new;
1999 int kf = (callback == SLEEP_FUNC) ? KM_SLEEP: KM_NOSLEEP;
2000 int rc, i;
2001 pvscsi_cmd_t *cmd;
2002 pvscsi_softc_t *pvs;
2003
2004 pvs = ap->a_hba_tran->tran_hba_private;
2005 ASSERT(pvs != NULL);
2006
2007 /* Allocate a new SCSI packet */
2008 if (pkt == NULL) {
2009 ddi_dma_handle_t saved_dmahdl, saved_arqhdl;
2010 struct buf *saved_arqbuf;
2011 ddi_dma_cookie_t saved_arqc;
2012
2013 is_new = B_TRUE;
2014
2015 if ((cmd = kmem_cache_alloc(pvs->cmd_cache, kf)) == NULL)
2016 return (NULL);
2017
2018 saved_dmahdl = cmd->cmd_dmahdl;
2019 saved_arqhdl = cmd->arqhdl;
2020 saved_arqbuf = cmd->arqbuf;
2021 saved_arqc = cmd->arqc;
2022
2023 bzero(cmd, sizeof (pvscsi_cmd_t) -
2024 sizeof (cmd->cached_cookies));
2025
2026 cmd->cmd_pvs = pvs;
2027 cmd->cmd_dmahdl = saved_dmahdl;
2028 cmd->arqhdl = saved_arqhdl;
2029 cmd->arqbuf = saved_arqbuf;
2030 cmd->arqc = saved_arqc;
2031
2032 pkt = &cmd->cached_pkt;
2033 pkt->pkt_ha_private = (opaque_t)cmd;
2034 pkt->pkt_address = *ap;
2035 pkt->pkt_scbp = (uint8_t *)&cmd->cmd_scb;
2036 pkt->pkt_cdbp = (uint8_t *)&cmd->cmd_cdb;
2037 pkt->pkt_private = (opaque_t)&cmd->tgt_priv;
2038
2039 cmd->tgtlen = tgtlen;
2040 cmd->statuslen = statuslen;
2041 cmd->cmdlen = cmdlen;
2042 cmd->pkt = pkt;
2043 cmd->ctx = NULL;
2044
2045 /* Allocate extended buffers */
2046 if ((cmdlen > sizeof (cmd->cmd_cdb)) ||
2047 (statuslen > sizeof (cmd->cmd_scb)) ||
2048 (tgtlen > sizeof (cmd->tgt_priv))) {
2049 if (pvscsi_cmd_ext_alloc(pvs, cmd, kf) != DDI_SUCCESS) {
2050 dev_err(pvs->dip, CE_WARN,
2051 "!extent allocation failed");
2052 goto out;
2053 }
2054 }
2055 } else {
2056 is_new = B_FALSE;
2057
2058 cmd = PKT2CMD(pkt);
2059 cmd->flags &= PVSCSI_FLAGS_PERSISTENT;
2060 }
2061
2062 ASSERT((cmd->flags & PVSCSI_FLAG_TRANSPORT) == 0);
2063
2064 if ((flags & PKT_XARQ) != 0)
2065 cmd->flags |= PVSCSI_FLAG_XARQ;
2066
2067 /* Handle partial DMA transfers */
2068 if (cmd->cmd_nwin > 0) {
2069 if (++cmd->cmd_winindex >= cmd->cmd_nwin)
2070 return (NULL);
2071 if (ddi_dma_getwin(cmd->cmd_dmahdl, cmd->cmd_winindex,
2072 &cmd->cmd_dma_offset, &cmd->cmd_dma_len,
2073 &cmd->cmd_dmac, &cmd->cmd_dmaccount) == DDI_FAILURE)
2074 return (NULL);
2075 goto handle_dma_cookies;
2076 }
2077
2078 /* Setup data buffer */
2079 if (bp != NULL && bp->b_bcount > 0 &&
2080 (cmd->flags & PVSCSI_FLAG_DMA_VALID) == 0) {
2081 int dma_flags;
2082
2083 ASSERT(cmd->cmd_dmahdl != NULL);
2084
2085 if ((bp->b_flags & B_READ) != 0) {
2086 cmd->flags |= PVSCSI_FLAG_IO_READ;
2087 dma_flags = DDI_DMA_READ;
2088 } else {
2089 cmd->flags &= ~PVSCSI_FLAG_IO_READ;
2090 dma_flags = DDI_DMA_WRITE;
2091 }
2092 if ((flags & PKT_CONSISTENT) != 0) {
2093 cmd->flags |= PVSCSI_FLAG_IO_IOPB;
2094 dma_flags |= DDI_DMA_CONSISTENT;
2095 }
2096 if ((flags & PKT_DMA_PARTIAL) != 0)
2097 dma_flags |= DDI_DMA_PARTIAL;
2098
2099 rc = ddi_dma_buf_bind_handle(cmd->cmd_dmahdl, bp,
2100 dma_flags, callback, arg, &cmd->cmd_dmac,
2101 &cmd->cmd_dmaccount);
2102 if (rc == DDI_DMA_PARTIAL_MAP) {
2103 (void) ddi_dma_numwin(cmd->cmd_dmahdl,
2104 &cmd->cmd_nwin);
2105 cmd->cmd_winindex = 0;
2106 (void) ddi_dma_getwin(cmd->cmd_dmahdl,
2107 cmd->cmd_winindex, &cmd->cmd_dma_offset,
2108 &cmd->cmd_dma_len, &cmd->cmd_dmac,
2109 &cmd->cmd_dmaccount);
2110 } else if (rc != 0 && rc != DDI_DMA_MAPPED) {
2111 switch (rc) {
2112 case DDI_DMA_NORESOURCES:
2113 bioerror(bp, 0);
2114 break;
2115 case DDI_DMA_BADATTR:
2116 case DDI_DMA_NOMAPPING:
2117 bioerror(bp, EFAULT);
2118 break;
2119 case DDI_DMA_TOOBIG:
2120 default:
2121 bioerror(bp, EINVAL);
2122 break;
2123 }
2124 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2125 goto out;
2126 }
2127
2128 handle_dma_cookies:
2129 ASSERT(cmd->cmd_dmaccount > 0);
2130 if (cmd->cmd_dmaccount > PVSCSI_MAX_SG_SIZE) {
2131 dev_err(pvs->dip, CE_WARN,
2132 "!invalid cookie count: %d (max %d)",
2133 cmd->cmd_dmaccount, PVSCSI_MAX_SG_SIZE);
2134 bioerror(bp, EINVAL);
2135 goto out;
2136 }
2137
2138 cmd->flags |= PVSCSI_FLAG_DMA_VALID;
2139 cmd->cmd_dma_count = cmd->cmd_dmac.dmac_size;
2140 cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2141
2142 cmd->cached_cookies[0] = cmd->cmd_dmac;
2143
2144 /*
2145 * Calculate total amount of bytes for this I/O and
2146 * store cookies for further processing.
2147 */
2148 for (i = 1; i < cmd->cmd_dmaccount; i++) {
2149 ddi_dma_nextcookie(cmd->cmd_dmahdl, &cmd->cmd_dmac);
2150 cmd->cached_cookies[i] = cmd->cmd_dmac;
2151 cmd->cmd_dma_count += cmd->cmd_dmac.dmac_size;
2152 cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2153 }
2154
2155 pkt->pkt_resid = (bp->b_bcount - cmd->cmd_total_dma_count);
2156 }
2157
2158 return (pkt);
2159
2160 out:
2161 if (is_new)
2162 pvscsi_destroy_pkt(ap, pkt);
2163
2164 return (NULL);
2165 }
2166
2167 /* ARGSUSED ap */
2168 static void
2169 pvscsi_dmafree(struct scsi_address *ap, struct scsi_pkt *pkt)
2170 {
2171 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
2172
2173 if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
2174 (void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
2175 cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2176 }
2177 }
2178
2179 /* ARGSUSED ap */
2180 static void
2181 pvscsi_sync_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
2182 {
2183 pvscsi_cmd_t *cmd = PKT2CMD(pkt);
2184
2185 if (cmd->cmd_dmahdl != NULL) {
2186 (void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
2187 (cmd->flags & PVSCSI_FLAG_IO_READ) ?
2188 DDI_DMA_SYNC_FORCPU : DDI_DMA_SYNC_FORDEV);
2189 }
2190
2191 }
2192
2193 /* ARGSUSED ap flag callback arg */
2194 static int
2195 pvscsi_reset_notify(struct scsi_address *ap, int flag,
2196 void (*callback)(caddr_t), caddr_t arg)
2197 {
2198 return (DDI_FAILURE);
2199 }
2200
2201 static int
2202 pvscsi_quiesce_hba(dev_info_t *dip)
2203 {
2204 pvscsi_softc_t *pvs;
2205 scsi_hba_tran_t *tran;
2206
2207 if ((tran = ddi_get_driver_private(dip)) == NULL ||
2208 (pvs = TRAN2PRIV(tran)) == NULL)
2209 return (-1);
2210
2211 mutex_enter(&pvs->mutex);
2212 if (!HBA_IS_QUIESCED(pvs))
2213 pvs->flags |= PVSCSI_HBA_QUIESCED;
2214
2215 if (pvs->cmd_queue_len != 0) {
2216 /* Outstanding commands present, wait */
2217 pvs->flags |= PVSCSI_HBA_QUIESCE_PENDING;
2218 cv_wait(&pvs->quiescevar, &pvs->mutex);
2219 ASSERT(pvs->cmd_queue_len == 0);
2220 }
2221 mutex_exit(&pvs->mutex);
2222
2223 /* Suspend taskq delivery and complete all scheduled tasks */
2224 ddi_taskq_suspend(pvs->msg_tq);
2225 ddi_taskq_wait(pvs->msg_tq);
2226 ddi_taskq_suspend(pvs->comp_tq);
2227 ddi_taskq_wait(pvs->comp_tq);
2228
2229 return (0);
2230 }
2231
2232 static int
2233 pvscsi_unquiesce_hba(dev_info_t *dip)
2234 {
2235 pvscsi_softc_t *pvs;
2236 scsi_hba_tran_t *tran;
2237
2238 if ((tran = ddi_get_driver_private(dip)) == NULL ||
2239 (pvs = TRAN2PRIV(tran)) == NULL)
2240 return (-1);
2241
2242 mutex_enter(&pvs->mutex);
2243 if (!HBA_IS_QUIESCED(pvs)) {
2244 mutex_exit(&pvs->mutex);
2245 return (0);
2246 }
2247 ASSERT(pvs->cmd_queue_len == 0);
2248 pvs->flags &= ~PVSCSI_HBA_QUIESCED;
2249 mutex_exit(&pvs->mutex);
2250
2251 /* Resume taskq delivery */
2252 ddi_taskq_resume(pvs->msg_tq);
2253 ddi_taskq_resume(pvs->comp_tq);
2254
2255 return (0);
2256 }
2257
2258 static int
2259 pvscsi_bus_config(dev_info_t *pdip, uint_t flags, ddi_bus_config_op_t op,
2260 void *arg, dev_info_t **childp)
2261 {
2262 char *p;
2263 int circ;
2264 int ret = NDI_FAILURE;
2265 long target = 0;
2266 pvscsi_softc_t *pvs;
2267 scsi_hba_tran_t *tran;
2268
2269 tran = ddi_get_driver_private(pdip);
2270 pvs = tran->tran_hba_private;
2271
2272 ndi_devi_enter(pdip, &circ);
2273 switch (op) {
2274 case BUS_CONFIG_ONE:
2275 if ((p = strrchr((char *)arg, '@')) != NULL &&
2276 ddi_strtol(p + 1, NULL, 16, &target) == 0)
2277 ret = pvscsi_config_one(pdip, pvs, (int)target, childp);
2278 break;
2279 case BUS_CONFIG_DRIVER:
2280 case BUS_CONFIG_ALL:
2281 ret = pvscsi_config_all(pdip, pvs);
2282 break;
2283 default:
2284 break;
2285 }
2286
2287 if (ret == NDI_SUCCESS)
2288 ret = ndi_busop_bus_config(pdip, flags, op, arg, childp, 0);
2289 ndi_devi_exit(pdip, circ);
2290
2291 return (ret);
2292 }
2293
2294 static int
2295 pvscsi_hba_setup(pvscsi_softc_t *pvs)
2296 {
2297 scsi_hba_tran_t *hba_tran;
2298
2299 hba_tran = pvs->tran = scsi_hba_tran_alloc(pvs->dip,
2300 SCSI_HBA_CANSLEEP);
2301 ASSERT(pvs->tran != NULL);
2302
2303 hba_tran->tran_hba_private = pvs;
2304 hba_tran->tran_tgt_private = NULL;
2305
2306 hba_tran->tran_tgt_init = pvscsi_tgt_init;
2307 hba_tran->tran_tgt_free = NULL;
2308 hba_tran->tran_tgt_probe = scsi_hba_probe;
2309
2310 hba_tran->tran_start = pvscsi_start;
2311 hba_tran->tran_reset = pvscsi_reset;
2312 hba_tran->tran_abort = pvscsi_abort;
2313 hba_tran->tran_getcap = pvscsi_getcap;
2314 hba_tran->tran_setcap = pvscsi_setcap;
2315 hba_tran->tran_init_pkt = pvscsi_init_pkt;
2316 hba_tran->tran_destroy_pkt = pvscsi_destroy_pkt;
2317
2318 hba_tran->tran_dmafree = pvscsi_dmafree;
2319 hba_tran->tran_sync_pkt = pvscsi_sync_pkt;
2320 hba_tran->tran_reset_notify = pvscsi_reset_notify;
2321
2322 hba_tran->tran_quiesce = pvscsi_quiesce_hba;
2323 hba_tran->tran_unquiesce = pvscsi_unquiesce_hba;
2324 hba_tran->tran_bus_reset = NULL;
2325
2326 hba_tran->tran_add_eventcall = NULL;
2327 hba_tran->tran_get_eventcookie = NULL;
2328 hba_tran->tran_post_event = NULL;
2329 hba_tran->tran_remove_eventcall = NULL;
2330
2331 hba_tran->tran_bus_config = pvscsi_bus_config;
2332
2333 hba_tran->tran_interconnect_type = INTERCONNECT_SAS;
2334
2335 if (scsi_hba_attach_setup(pvs->dip, &pvs->hba_dma_attr, hba_tran,
2336 SCSI_HBA_TRAN_CDB | SCSI_HBA_TRAN_SCB | SCSI_HBA_TRAN_CLONE) !=
2337 DDI_SUCCESS) {
2338 dev_err(pvs->dip, CE_WARN, "!failed to attach HBA");
2339 scsi_hba_tran_free(hba_tran);
2340 pvs->tran = NULL;
2341 return (-1);
2342 }
2343
2344 return (0);
2345 }
2346
2347 static int
2348 pvscsi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2349 {
2350 int instance;
2351 pvscsi_softc_t *pvs;
2352 char buf[32];
2353
2354 ASSERT(scsi_hba_iport_unit_address(dip) == NULL);
2355
2356 switch (cmd) {
2357 case DDI_ATTACH:
2358 case DDI_RESUME:
2359 break;
2360 default:
2361 return (DDI_FAILURE);
2362 }
2363
2364 instance = ddi_get_instance(dip);
2365
2366 /* Allocate softstate information */
2367 if (ddi_soft_state_zalloc(pvscsi_sstate, instance) != DDI_SUCCESS) {
2368 cmn_err(CE_WARN,
2369 "!ddi_soft_state_zalloc() failed for instance %d",
2370 instance);
2371 return (DDI_FAILURE);
2372 }
2373
2374 if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2375 cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2376 instance);
2377 goto fail;
2378 }
2379
2380 /*
2381 * Indicate that we are 'sizeof (scsi_*(9S))' clean, we use
2382 * scsi_pkt_size() instead.
2383 */
2384 scsi_size_clean(dip);
2385
2386 /* Setup HBA instance */
2387 pvs->instance = instance;
2388 pvs->dip = dip;
2389 pvs->hba_dma_attr = pvscsi_hba_dma_attr;
2390 pvs->ring_dma_attr = pvscsi_ring_dma_attr;
2391 pvs->io_dma_attr = pvscsi_io_dma_attr;
2392 mutex_init(&pvs->mutex, "pvscsi instance mutex", MUTEX_DRIVER, NULL);
2393 mutex_init(&pvs->intr_mutex, "pvscsi instance interrupt mutex",
2394 MUTEX_DRIVER, NULL);
2395 mutex_init(&pvs->rx_mutex, "pvscsi rx ring mutex", MUTEX_DRIVER, NULL);
2396 mutex_init(&pvs->tx_mutex, "pvscsi tx ring mutex", MUTEX_DRIVER, NULL);
2397 list_create(&pvs->cmd_ctx_pool, sizeof (pvscsi_cmd_ctx_t),
2398 offsetof(pvscsi_cmd_ctx_t, list));
2399 list_create(&pvs->devnodes, sizeof (pvscsi_device_t),
2400 offsetof(pvscsi_device_t, list));
2401 list_create(&pvs->cmd_queue, sizeof (pvscsi_cmd_t),
2402 offsetof(pvscsi_cmd_t, cmd_queue_node));
2403 cv_init(&pvs->syncvar, "pvscsi synchronization cv", CV_DRIVER, NULL);
2404 cv_init(&pvs->wd_condvar, "pvscsi watchdog cv", CV_DRIVER, NULL);
2405 cv_init(&pvs->quiescevar, "pvscsi quiesce cv", CV_DRIVER, NULL);
2406
2407 (void) sprintf(buf, "pvscsi%d_cache", instance);
2408 pvs->cmd_cache = kmem_cache_create(buf, sizeof (pvscsi_cmd_t), 0,
2409 pvscsi_ccache_constructor, pvscsi_ccache_destructor, NULL,
2410 (void *)pvs, NULL, 0);
2411 if (pvs->cmd_cache == NULL) {
2412 dev_err(pvs->dip, CE_WARN,
2413 "!failed to create a cache for SCSI commands");
2414 goto fail;
2415 }
2416
2417 if ((pvscsi_setup_io(pvs)) != DDI_SUCCESS) {
2418 dev_err(pvs->dip, CE_WARN, "!failed to setup I/O region");
2419 goto free_cache;
2420 }
2421
2422 pvscsi_reset_hba(pvs);
2423
2424 if ((pvscsi_allocate_rings(pvs)) != DDI_SUCCESS) {
2425 dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA rings");
2426 goto free_io;
2427 }
2428
2429 pvscsi_setup_rings(pvs);
2430
2431 if (pvscsi_setup_isr(pvs) != DDI_SUCCESS) {
2432 dev_err(pvs->dip, CE_WARN, "!failed to setup ISR");
2433 goto free_rings;
2434 }
2435
2436 if (pvscsi_setup_sg(pvs) != DDI_SUCCESS) {
2437 dev_err(pvs->dip, CE_WARN, "!failed to setup S/G");
2438 goto free_intr;
2439 }
2440
2441 if (pvscsi_hba_setup(pvs) != 0) {
2442 dev_err(pvs->dip, CE_WARN, "!failed to setup HBA");
2443 goto free_sg;
2444 }
2445
2446 if ((pvs->comp_tq = ddi_taskq_create(pvs->dip, "comp_tq",
2447 MIN(UINT16_MAX, ncpus), TASKQ_DEFAULTPRI, 0)) == NULL) {
2448 dev_err(pvs->dip, CE_WARN,
2449 "!failed to create completion taskq");
2450 goto free_sg;
2451 }
2452
2453 if ((pvs->msg_tq = ddi_taskq_create(pvs->dip, "msg_tq",
2454 1, TASKQ_DEFAULTPRI, 0)) == NULL) {
2455 dev_err(pvs->dip, CE_WARN,
2456 "!failed to create message taskq");
2457 goto free_comp_tq;
2458 }
2459
2460 if (pvscsi_enable_intrs(pvs) != DDI_SUCCESS) {
2461 dev_err(pvs->dip, CE_WARN, "!failed to enable interrupts");
2462 goto free_msg_tq;
2463 }
2464
2465 /* Launch watchdog thread */
2466 pvs->wd_thread = thread_create(NULL, 0, pvscsi_wd_thread, pvs, 0, &p0,
2467 TS_RUN, minclsyspri);
2468
2469 return (DDI_SUCCESS);
2470
2471 free_msg_tq:
2472 ddi_taskq_destroy(pvs->msg_tq);
2473 free_comp_tq:
2474 ddi_taskq_destroy(pvs->comp_tq);
2475 free_sg:
2476 pvscsi_free_sg(pvs);
2477 free_intr:
2478 pvscsi_free_intr_resources(pvs);
2479 free_rings:
2480 pvscsi_reset_hba(pvs);
2481 pvscsi_free_rings(pvs);
2482 free_io:
2483 pvscsi_free_io(pvs);
2484 free_cache:
2485 kmem_cache_destroy(pvs->cmd_cache);
2486 fail:
2487 ddi_soft_state_free(pvscsi_sstate, instance);
2488
2489 return (DDI_FAILURE);
2490 }
2491
2492 static int
2493 pvscsi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2494 {
2495 int instance;
2496 pvscsi_softc_t *pvs;
2497
2498 switch (cmd) {
2499 case DDI_DETACH:
2500 break;
2501 default:
2502 return (DDI_FAILURE);
2503 }
2504
2505 instance = ddi_get_instance(dip);
2506 if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2507 cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2508 instance);
2509 return (DDI_FAILURE);
2510 }
2511
2512 pvscsi_reset_hba(pvs);
2513 pvscsi_free_intr_resources(pvs);
2514
2515 /* Shutdown message taskq */
2516 ddi_taskq_wait(pvs->msg_tq);
2517 ddi_taskq_destroy(pvs->msg_tq);
2518
2519 /* Shutdown completion taskq */
2520 ddi_taskq_wait(pvs->comp_tq);
2521 ddi_taskq_destroy(pvs->comp_tq);
2522
2523 /* Shutdown watchdog thread */
2524 mutex_enter(&pvs->mutex);
2525 pvs->flags |= PVSCSI_DRIVER_SHUTDOWN;
2526 cv_signal(&pvs->wd_condvar);
2527 cv_wait(&pvs->syncvar, &pvs->mutex);
2528 mutex_exit(&pvs->mutex);
2529
2530 pvscsi_free_sg(pvs);
2531 pvscsi_free_rings(pvs);
2532 pvscsi_free_io(pvs);
2533
2534 kmem_cache_destroy(pvs->cmd_cache);
2535
2536 mutex_destroy(&pvs->mutex);
2537 mutex_destroy(&pvs->intr_mutex);
2538 mutex_destroy(&pvs->rx_mutex);
2539
2540 cv_destroy(&pvs->syncvar);
2541 cv_destroy(&pvs->wd_condvar);
2542 cv_destroy(&pvs->quiescevar);
2543
2544 ddi_soft_state_free(pvscsi_sstate, instance);
2545 ddi_prop_remove_all(dip);
2546
2547 return (DDI_SUCCESS);
2548 }
2549
2550 static int
2551 pvscsi_ioctl(dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp,
2552 int *rval)
2553 {
2554 int ret;
2555
2556 if (ddi_get_soft_state(pvscsi_sstate, getminor(dev)) == NULL) {
2557 cmn_err(CE_WARN, "!invalid device instance: %d", getminor(dev));
2558 return (ENXIO);
2559 }
2560
2561 /* Try to handle command in a common way */
2562 if ((ret = scsi_hba_ioctl(dev, cmd, data, mode, credp, rval)) != ENOTTY)
2563 return (ret);
2564
2565 cmn_err(CE_WARN, "!unsupported IOCTL command: 0x%X", cmd);
2566
2567 return (ENXIO);
2568 }
2569
2570 static int
2571 pvscsi_quiesce(dev_info_t *devi)
2572 {
2573 scsi_hba_tran_t *tran;
2574 pvscsi_softc_t *pvs;
2575
2576 if ((tran = ddi_get_driver_private(devi)) == NULL)
2577 return (DDI_SUCCESS);
2578
2579 if ((pvs = tran->tran_hba_private) == NULL)
2580 return (DDI_SUCCESS);
2581
2582 /* Mask all interrupts from device */
2583 pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
2584
2585 /* Reset the HBA */
2586 pvscsi_reset_hba(pvs);
2587
2588 return (DDI_SUCCESS);
2589 }
2590
2591 /* module */
2592
2593 static struct cb_ops pvscsi_cb_ops = {
2594 .cb_open = scsi_hba_open,
2595 .cb_close = scsi_hba_close,
2596 .cb_strategy = nodev,
2597 .cb_print = nodev,
2598 .cb_dump = nodev,
2599 .cb_read = nodev,
2600 .cb_write = nodev,
2601 .cb_ioctl = pvscsi_ioctl,
2602 .cb_devmap = nodev,
2603 .cb_mmap = nodev,
2604 .cb_segmap = nodev,
2605 .cb_chpoll = nochpoll,
2606 .cb_prop_op = ddi_prop_op,
2607 .cb_str = NULL,
2608 .cb_flag = D_MP,
2609 .cb_rev = CB_REV,
2610 .cb_aread = nodev,
2611 .cb_awrite = nodev
2612 };
2613
2614 static struct dev_ops pvscsi_ops = {
2615 .devo_rev = DEVO_REV,
2616 .devo_refcnt = 0,
2617 .devo_getinfo = ddi_no_info,
2618 .devo_identify = nulldev,
2619 .devo_probe = nulldev,
2620 .devo_attach = pvscsi_attach,
2621 .devo_detach = pvscsi_detach,
2622 .devo_reset = nodev,
2623 .devo_cb_ops = &pvscsi_cb_ops,
2624 .devo_bus_ops = NULL,
2625 .devo_power = NULL,
2626 .devo_quiesce = pvscsi_quiesce
2627 };
2628
2629 #define PVSCSI_IDENT "VMware PVSCSI"
2630
2631 static struct modldrv modldrv = {
2632 &mod_driverops,
2633 PVSCSI_IDENT,
2634 &pvscsi_ops,
2635 };
2636
2637 static struct modlinkage modlinkage = {
2638 MODREV_1,
2639 &modldrv,
2640 NULL
2641 };
2642
2643 int
2644 _init(void)
2645 {
2646 int ret;
2647
2648 if ((ret = ddi_soft_state_init(&pvscsi_sstate,
2649 sizeof (struct pvscsi_softc), PVSCSI_INITIAL_SSTATE_ITEMS)) != 0) {
2650 cmn_err(CE_WARN, "!ddi_soft_state_init() failed");
2651 return (ret);
2652 }
2653
2654 if ((ret = scsi_hba_init(&modlinkage)) != 0) {
2655 cmn_err(CE_WARN, "!scsi_hba_init() failed");
2656 ddi_soft_state_fini(&pvscsi_sstate);
2657 return (ret);
2658 }
2659
2660 if ((ret = mod_install(&modlinkage)) != 0) {
2661 cmn_err(CE_WARN, "!mod_install() failed");
2662 ddi_soft_state_fini(&pvscsi_sstate);
2663 scsi_hba_fini(&modlinkage);
2664 }
2665
2666 return (ret);
2667 }
2668
2669 int
2670 _info(struct modinfo *modinfop)
2671 {
2672 return (mod_info(&modlinkage, modinfop));
2673 }
2674
2675 int
2676 _fini(void)
2677 {
2678 int ret;
2679
2680 if ((ret = mod_remove(&modlinkage)) == 0) {
2681 ddi_soft_state_fini(&pvscsi_sstate);
2682 scsi_hba_fini(&modlinkage);
2683 }
2684
2685 return (ret);
2686 }