1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2018 Nexenta Systems, Inc.
14 */
15
16 /*
17 * This file contains code necessary to send SCSI commands to HBA.
18 */
19 #include <smartpqi.h>
20
21 /*
22 * []------------------------------------------------------------------[]
23 * | Forward declarations for support/utility functions |
24 * []------------------------------------------------------------------[]
25 */
26 static void aio_io_complete(pqi_io_request_t *io, void *context);
27 static void raid_io_complete(pqi_io_request_t *io, void *context);
28 static void build_aio_sg_list(pqi_state_t s,
29 pqi_aio_path_request_t *rqst, pqi_cmd_t cmd, pqi_io_request_t *);
30 static void build_raid_sg_list(pqi_state_t s,
31 pqi_raid_path_request_t *rqst, pqi_cmd_t cmd, pqi_io_request_t *);
32 static pqi_io_request_t *setup_aio_request(pqi_state_t s, pqi_cmd_t cmd);
33 static pqi_io_request_t *setup_raid_request(pqi_state_t s, pqi_cmd_t cmd);
34 static uint32_t read_heartbeat_counter(pqi_state_t s);
35 static void take_ctlr_offline(pqi_state_t s);
36 static uint32_t free_elem_count(pqi_index_t pi, pqi_index_t ci,
37 uint32_t per_iq);
38 static void ack_event(pqi_state_t s, pqi_event_t e);
39 static boolean_t is_aio_enabled(pqi_device_t d);
40 static void lun_reset_worker(void *v);
41 static void lun_reset_complete(pqi_io_request_t *io, void *ctx);
42
43 #define DIV_UP(n, d) ((n + (d - 1)) / d)
44
45 /*
46 * []------------------------------------------------------------------[]
47 * | Main entry points in file. |
48 * []------------------------------------------------------------------[]
49 */
50
51 /*
52 * pqi_watchdog -- interrupt count and/or heartbeat must increase over time.
53 */
54 void
55 pqi_watchdog(void *v)
56 {
57 pqi_state_t s = v;
58 uint32_t hb;
59
60 if (pqi_is_offline(s))
61 return;
62
63 hb = read_heartbeat_counter(s);
64 if ((s->s_last_intr_count == s->s_intr_count) &&
65 (s->s_last_heartbeat_count == hb)) {
66 dev_err(s->s_dip, CE_NOTE, "No heartbeat");
67 pqi_show_dev_state(s);
68 take_ctlr_offline(s);
69 } else {
70 s->s_last_intr_count = s->s_intr_count;
71 s->s_last_heartbeat_count = hb;
72 s->s_watchdog = timeout(pqi_watchdog, s,
73 drv_usectohz(WATCHDOG));
74 }
75 if (pqi_do_scan && s->s_instance == pqi_do_ctrl) {
76 pqi_do_scan = 0;
77 s->s_rescan = timeout(pqi_do_rescan, (void *)s,
78 drv_usectohz(MICROSEC));
79 }
80 }
81
82 /*
83 * pqi_start_io -- queues command to HBA.
84 *
85 * This method can be called either from the upper layer with a non-zero
86 * io argument or called during an interrupt to load the outgoing queue
87 * with more commands.
88 */
89 void
90 pqi_start_io(pqi_state_t s, pqi_queue_group_t *qg, pqi_path_t path,
91 pqi_io_request_t *io)
92 {
93 pqi_iu_header_t *rqst;
94 size_t iu_len;
95 size_t copy_to_end;
96 pqi_index_t iq_pi;
97 pqi_index_t iq_ci;
98 uint32_t elem_needed;
99 uint32_t elem_to_end;
100 caddr_t next_elem;
101 int sending = 0;
102
103 mutex_enter(&qg->submit_lock[path]);
104 if (io != NULL)
105 list_insert_tail(&qg->request_list[path], io);
106
107
108 iq_pi = qg->iq_pi_copy[path];
109 while ((io = list_head(&qg->request_list[path])) != NULL) {
110
111 /* ---- Primary cause for !active is controller failure ---- */
112 if (qg->qg_active == B_FALSE && io->io_cmd) {
113 list_remove(&qg->request_list[path], io);
114 mutex_enter(&io->io_cmd->pc_device->pd_mutex);
115 pqi_fail_cmd(io->io_cmd, CMD_DEV_GONE, STAT_TERMINATED);
116 mutex_exit(&io->io_cmd->pc_device->pd_mutex);
117 continue;
118 }
119
120 rqst = io->io_iu;
121 iu_len = rqst->iu_length + PQI_REQUEST_HEADER_LENGTH;
122 elem_needed = DIV_UP(iu_len, PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
123 (void) ddi_dma_sync(s->s_queue_dma->handle,
124 (uintptr_t)qg->iq_ci[path] -
125 (uintptr_t)s->s_queue_dma->alloc_memory, sizeof (iq_ci),
126 DDI_DMA_SYNC_FORCPU);
127 iq_ci = *qg->iq_ci[path];
128
129 if (elem_needed > free_elem_count(iq_pi, iq_ci,
130 s->s_num_elements_per_iq))
131 break;
132
133 io->io_pi = iq_pi;
134 rqst->iu_id = qg->oq_id;
135 next_elem = qg->iq_element_array[path] +
136 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
137 elem_to_end = s->s_num_elements_per_iq - iq_pi;
138 if (elem_needed <= elem_to_end) {
139 (void) memcpy(next_elem, rqst, iu_len);
140 (void) ddi_dma_sync(s->s_queue_dma->handle,
141 (uintptr_t)next_elem -
142 (uintptr_t)s->s_queue_dma->alloc_memory, iu_len,
143 DDI_DMA_SYNC_FORDEV);
144 } else {
145 copy_to_end = elem_to_end *
146 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
147 (void) memcpy(next_elem, rqst, copy_to_end);
148 (void) ddi_dma_sync(s->s_queue_dma->handle,
149 (uintptr_t)next_elem -
150 (uintptr_t)s->s_queue_dma->alloc_memory,
151 copy_to_end, DDI_DMA_SYNC_FORDEV);
152 (void) memcpy(qg->iq_element_array[path],
153 (caddr_t)rqst + copy_to_end,
154 iu_len - copy_to_end);
155 (void) ddi_dma_sync(s->s_queue_dma->handle,
156 0, iu_len - copy_to_end, DDI_DMA_SYNC_FORDEV);
157 }
158 sending += elem_needed;
159 if (io->io_cmd != NULL)
160 pqi_cmd_sm(io->io_cmd, PQI_CMD_STARTED, B_TRUE);
161 else if ((rqst->iu_type == PQI_REQUEST_IU_RAID_PATH_IO) &&
162 (s->s_debug_level & (DBG_LVL_CDB | DBG_LVL_RQST)))
163 pqi_dump_io(io);
164
165 iq_pi = (iq_pi + elem_needed) % s->s_num_elements_per_iq;
166 list_remove(&qg->request_list[path], io);
167 }
168
169 qg->submit_count += sending;
170 if (iq_pi != qg->iq_pi_copy[path]) {
171 qg->iq_pi_copy[path] = iq_pi;
172 ddi_put32(s->s_datap, qg->iq_pi[path], iq_pi);
173 } else {
174 ASSERT0(sending);
175 }
176 mutex_exit(&qg->submit_lock[path]);
177 }
178
179 int
180 pqi_transport_command(pqi_state_t s, pqi_cmd_t cmd)
181 {
182 pqi_device_t devp = cmd->pc_device;
183 int path;
184 pqi_io_request_t *io;
185
186 if (is_aio_enabled(devp) == B_TRUE) {
187 path = AIO_PATH;
188 io = setup_aio_request(s, cmd);
189 } else {
190 path = RAID_PATH;
191 io = setup_raid_request(s, cmd);
192 }
193
194 if (io == NULL)
195 return (TRAN_BUSY);
196
197 cmd->pc_io_rqst = io;
198
199 pqi_start_io(s, &s->s_queue_groups[PQI_DEFAULT_QUEUE_GROUP],
200 path, io);
201
202 return (TRAN_ACCEPT);
203 }
204
205 void
206 pqi_do_rescan(void *v)
207 {
208 pqi_state_t s = v;
209 int circ = 0;
210 int circ1 = 0;
211
212 ndi_devi_enter(scsi_vhci_dip, &circ1);
213 ndi_devi_enter(s->s_dip, &circ);
214 pqi_rescan_devices(s);
215 (void) pqi_config_all(s->s_dip, s);
216 ndi_devi_exit(s->s_dip, circ);
217 ndi_devi_exit(scsi_vhci_dip, circ1);
218 }
219
220 void
221 pqi_event_worker(void *v)
222 {
223 pqi_state_t s = v;
224 int i;
225 pqi_event_t e;
226 boolean_t non_heartbeat = B_FALSE;
227
228 if (pqi_is_offline(s))
229 return;
230
231 e = s->s_events;
232 for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
233 if (e->ev_pending == B_TRUE) {
234 e->ev_pending = B_FALSE;
235 ack_event(s, e);
236 if (pqi_map_event(PQI_EVENT_TYPE_HEARTBEAT) != i)
237 non_heartbeat = B_TRUE;
238 }
239 e++;
240 }
241
242 if (non_heartbeat == B_TRUE)
243 pqi_do_rescan(s);
244 }
245
246 /*
247 * pqi_fail_cmd -- given a reason and stats the command is failed.
248 *
249 * NOTE: pqi_device->pd_mutex must be held. Also note that during the
250 * call to pqi_cmd_sm() the lock will be dropped and reacquired.
251 */
252 void
253 pqi_fail_cmd(pqi_cmd_t cmd, uchar_t reason, uint_t stats)
254 {
255 struct scsi_pkt *pkt = CMD2PKT(cmd);
256
257 ASSERT(MUTEX_HELD(&cmd->pc_device->pd_mutex));
258
259 pkt->pkt_reason = reason;
260 pkt->pkt_statistics = stats;
261
262 pqi_cmd_sm(cmd, PQI_CMD_FATAL, B_FALSE);
263 }
264
265 void
266 pqi_fail_drive_cmds(pqi_device_t devp)
267 {
268 pqi_cmd_t cmd;
269
270 restart:
271 mutex_enter(&devp->pd_mutex);
272 while ((cmd = list_head(&devp->pd_cmd_list)) != NULL) {
273
274 if (cmd->pc_flags & PQI_FLAG_FINISHING) {
275 /*
276 * This will be a very short wait since
277 * raid_io_complete is a quick function that will
278 * call pqi_cmd_sm() which removes the command
279 * from pd_cmd_list.
280 */
281 mutex_exit(&devp->pd_mutex);
282 drv_usecwait(100);
283 goto restart;
284 }
285 pqi_fail_cmd(cmd, CMD_DEV_GONE, STAT_TERMINATED);
286 }
287
288 mutex_exit(&devp->pd_mutex);
289 }
290
291 uint32_t
292 pqi_disable_intr(pqi_state_t s)
293 {
294 uint32_t db;
295 uint32_t rval;
296
297 rval = db = G32(s, sis_host_to_ctrl_doorbell);
298 db &= ~(SIS_ENABLE_MSIX | SIS_ENABLE_INTX);
299 S32(s, sis_host_to_ctrl_doorbell, db);
300 return (rval);
301 }
302
303 void
304 pqi_enable_intr(pqi_state_t s, uint32_t old_state)
305 {
306 S32(s, sis_host_to_ctrl_doorbell, old_state);
307 }
308
309 typedef struct reset_closure {
310 pqi_state_t rc_s;
311 pqi_device_t rc_d;
312 } *reset_closure_t;
313
314 /*
315 * pqi_lun_reset -- set up callback to reset the device
316 *
317 * Dispatch queue is used here because the call tree can come from the interrupt
318 * routine. (pqi_process_io_intr -> aio_io_complete -> SCSA -> tran_reset ->
319 * pqi_lun_reset). If pqi_lun_reset were to actually do the reset work it would
320 * then wait for an interrupt which would never arrive since the current thread
321 * would be the interrupt thread. So, start a task to reset the device and
322 * wait for completion.
323 */
324 boolean_t
325 pqi_lun_reset(pqi_state_t s, pqi_device_t d)
326 {
327 reset_closure_t r = kmem_alloc(sizeof (struct reset_closure), KM_SLEEP);
328
329 r->rc_s = s;
330 r->rc_d = d;
331 (void) ddi_taskq_dispatch(s->s_events_taskq, lun_reset_worker, r, 0);
332 return (B_TRUE);
333 }
334
335 /*
336 * []------------------------------------------------------------------[]
337 * | Support/utility functions for main entry points |
338 * []------------------------------------------------------------------[]
339 */
340
341 static void
342 lun_reset_worker(void *v)
343 {
344 reset_closure_t r = v;
345 pqi_state_t s;
346 pqi_device_t d;
347 pqi_io_request_t *io;
348 ksema_t sema;
349 pqi_task_management_rqst_t *rqst;
350
351 s = r->rc_s;
352 d = r->rc_d;
353 kmem_free(r, sizeof (*r));
354 sema_p(&s->s_sync_rqst);
355 s->s_sync_expire = gethrtime() + (SYNC_CMDS_TIMEOUT_SECS * NANOSEC);
356
357 sema_init(&sema, 0, NULL, SEMA_DRIVER, NULL);
358
359 io = pqi_alloc_io(s);
360 io->io_cb = lun_reset_complete;
361 io->io_context = &sema;
362
363 rqst = io->io_iu;
364 (void) memset(rqst, 0, sizeof (*rqst));
365
366 rqst->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
367 rqst->header.iu_length = sizeof (*rqst) - PQI_REQUEST_HEADER_LENGTH;
368 rqst->request_id = io->io_index;
369 (void) memcpy(rqst->lun_number, d->pd_scsi3addr,
370 sizeof (rqst->lun_number));
371 rqst->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
372
373 s->s_sync_io = io;
374 pqi_start_io(s, &s->s_queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
375 io);
376
377 sema_p(&sema);
378 pqi_free_io(io);
379 s->s_sync_io = NULL;
380 s->s_sync_expire = 0;
381
382 sema_v(&s->s_sync_rqst);
383 }
384
385 /*ARGSUSED*/
386 static void
387 lun_reset_complete(pqi_io_request_t *io, void *ctx)
388 {
389 sema_v((ksema_t *)ctx);
390 }
391
392 static void
393 send_event_ack(pqi_state_t s, pqi_event_acknowledge_request_t *rqst)
394 {
395 pqi_queue_group_t *qg;
396 caddr_t next_element;
397 pqi_index_t iq_ci;
398 pqi_index_t iq_pi;
399 int ms_timeo = 1000 * 10;
400
401 qg = &s->s_queue_groups[PQI_DEFAULT_QUEUE_GROUP];
402 rqst->header.iu_id = qg->oq_id;
403
404 for (;;) {
405 mutex_enter(&qg->submit_lock[RAID_PATH]);
406 iq_pi = qg->iq_pi_copy[RAID_PATH];
407 iq_ci = ddi_get32(s->s_queue_dma->acc, qg->iq_ci[RAID_PATH]);
408
409 if (free_elem_count(iq_pi, iq_ci, s->s_num_elements_per_iq))
410 break;
411
412 mutex_exit(&qg->submit_lock[RAID_PATH]);
413 if (pqi_is_offline(s))
414 return;
415 }
416 next_element = qg->iq_element_array[RAID_PATH] +
417 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
418
419 (void) memcpy(next_element, rqst, sizeof (*rqst));
420 (void) ddi_dma_sync(s->s_queue_dma->handle, 0, 0, DDI_DMA_SYNC_FORDEV);
421
422 iq_pi = (iq_pi + 1) % s->s_num_elements_per_iq;
423 qg->iq_pi_copy[RAID_PATH] = iq_pi;
424
425 ddi_put32(s->s_datap, qg->iq_pi[RAID_PATH], iq_pi);
426
427 /*
428 * Special case processing for events required. The driver must
429 * wait until the acknowledgement is processed before proceeding.
430 * Unfortunately, the HBA doesn't provide an interrupt which means
431 * the code must busy wait.
432 * Code will wait up to 10 seconds.
433 */
434 while (ms_timeo--) {
435 drv_usecwait(1000);
436 iq_ci = ddi_get32(s->s_queue_dma->acc, qg->iq_ci[RAID_PATH]);
437 if (iq_pi == iq_ci)
438 break;
439 }
440
441 mutex_exit(&qg->submit_lock[RAID_PATH]);
442 }
443
444 static void
445 ack_event(pqi_state_t s, pqi_event_t e)
446 {
447 pqi_event_acknowledge_request_t rqst;
448
449 (void) memset(&rqst, 0, sizeof (rqst));
450 rqst.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
451 rqst.header.iu_length = sizeof (rqst) - PQI_REQUEST_HEADER_LENGTH;
452 rqst.event_type = e->ev_type;
453 rqst.event_id = e->ev_id;
454 rqst.additional_event_id = e->ev_additional;
455
456 send_event_ack(s, &rqst);
457 }
458
459 static pqi_io_request_t *
460 setup_aio_request(pqi_state_t s, pqi_cmd_t cmd)
461 {
462 pqi_io_request_t *io;
463 pqi_aio_path_request_t *rqst;
464 pqi_device_t devp = cmd->pc_device;
465
466 /* ---- Most likely received a signal during a cv_wait ---- */
467 if ((io = pqi_alloc_io(s)) == NULL)
468 return (NULL);
469
470 io->io_cb = aio_io_complete;
471 io->io_cmd = cmd;
472 io->io_raid_bypass = 0;
473
474 rqst = io->io_iu;
475 (void) memset(rqst, 0, sizeof (*rqst));
476
477 rqst->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
478 rqst->nexus_id = devp->pd_aio_handle;
479 rqst->buffer_length = cmd->pc_dma_count;
480 rqst->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
481 rqst->request_id = io->io_index;
482 rqst->error_index = rqst->request_id;
483 rqst->cdb_length = cmd->pc_cmdlen;
484 (void) memcpy(rqst->cdb, cmd->pc_cdb, cmd->pc_cmdlen);
485 (void) memcpy(rqst->lun_number, devp->pd_scsi3addr,
486 sizeof (rqst->lun_number));
487
488 if (cmd->pc_flags & PQI_FLAG_DMA_VALID) {
489 if (cmd->pc_flags & PQI_FLAG_IO_READ)
490 rqst->data_direction = SOP_READ_FLAG;
491 else
492 rqst->data_direction = SOP_WRITE_FLAG;
493 } else {
494 rqst->data_direction = SOP_NO_DIRECTION_FLAG;
495 }
496
497 build_aio_sg_list(s, rqst, cmd, io);
498 return (io);
499 }
500
501 static pqi_io_request_t *
502 setup_raid_request(pqi_state_t s, pqi_cmd_t cmd)
503 {
504 pqi_io_request_t *io;
505 pqi_raid_path_request_t *rqst;
506 pqi_device_t devp = cmd->pc_device;
507
508 /* ---- Most likely received a signal during a cv_wait ---- */
509 if ((io = pqi_alloc_io(s)) == NULL)
510 return (NULL);
511
512 io->io_cb = raid_io_complete;
513 io->io_cmd = cmd;
514 io->io_raid_bypass = 0;
515
516 rqst = io->io_iu;
517 (void) memset(rqst, 0, sizeof (*rqst));
518 rqst->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
519 rqst->rp_data_len = cmd->pc_dma_count;
520 rqst->rp_task_attr = SOP_TASK_ATTRIBUTE_SIMPLE;
521 rqst->rp_id = io->io_index;
522 rqst->rp_error_index = rqst->rp_id;
523 (void) memcpy(rqst->rp_lun, devp->pd_scsi3addr, sizeof (rqst->rp_lun));
524 (void) memcpy(rqst->rp_cdb, cmd->pc_cdb, cmd->pc_cmdlen);
525
526 ASSERT(cmd->pc_cmdlen <= 16);
527 rqst->rp_additional_cdb = SOP_ADDITIONAL_CDB_BYTES_0;
528
529 if (cmd->pc_flags & PQI_FLAG_DMA_VALID) {
530 if (cmd->pc_flags & PQI_FLAG_IO_READ)
531 rqst->rp_data_dir = SOP_READ_FLAG;
532 else
533 rqst->rp_data_dir = SOP_WRITE_FLAG;
534 } else {
535 rqst->rp_data_dir = SOP_NO_DIRECTION_FLAG;
536 }
537
538 build_raid_sg_list(s, rqst, cmd, io);
539 return (io);
540 }
541
542 /*ARGSUSED*/
543 pqi_cmd_t
544 pqi_process_comp_ring(pqi_state_t s)
545 {
546 return (NULL);
547 }
548
549 static void
550 raid_io_complete(pqi_io_request_t *io, void *context)
551 {
552 /*
553 * ---- XXX Not sure if this complete function will be the same
554 * or different in the end. If it's the same this will be removed
555 * and aio_io_complete will have it's named changed to something
556 * more generic.
557 */
558 aio_io_complete(io, context);
559 }
560
561 /*
562 * special_error_check -- See if sense buffer matches "offline" status.
563 *
564 * spc3r23 section 4.5.6 -- Sense key and sense code definitions.
565 * Sense key == 5 (KEY_ILLEGAL_REQUEST) indicates one of several conditions
566 * a) Command addressed to incorrect logical unit.
567 * b) Command had an invalid task attribute.
568 * ...
569 * Table 28 also shows that ASC 0x26 and ASCQ of 0x00 is an INVALID FIELD
570 * IN PARAMETER LIST.
571 * At no other time does this combination of KEY/ASC/ASCQ occur except when
572 * a device or cable is pulled from the system along with a Hotplug event.
573 * Without documentation it's only a guess, but it's the best that's available.
574 * So, if the conditions are true the command packet pkt_reason will be changed
575 * to CMD_DEV_GONE which causes MPxIO to switch to the other path and the
576 * Hotplug event will cause a scan to occur which removes other inactive
577 * devices in case of a cable pull.
578 */
579 boolean_t
580 special_error_check(pqi_cmd_t cmd)
581 {
582 struct scsi_arq_status *arq;
583
584 /* LINTED E_BAD_PTR_CAST_ALIGN */
585 arq = (struct scsi_arq_status *)cmd->pc_pkt->pkt_scbp;
586
587 if (((*cmd->pc_pkt->pkt_scbp & STATUS_MASK) == STATUS_CHECK) &&
588 (arq->sts_sensedata.es_key == KEY_ILLEGAL_REQUEST) &&
589 (arq->sts_sensedata.es_add_code == 0x26) &&
590 (arq->sts_sensedata.es_qual_code == 0)) {
591 return (B_TRUE);
592 } else {
593 return (B_FALSE);
594 }
595 }
596
597 /*ARGSUSED*/
598 static void
599 aio_io_complete(pqi_io_request_t *io, void *context)
600 {
601 pqi_cmd_t cmd = io->io_cmd;
602 struct scsi_pkt *pkt = CMD2PKT(cmd);
603
604 if (cmd->pc_flags & (PQI_FLAG_IO_READ | PQI_FLAG_IO_IOPB))
605 (void) ddi_dma_sync(cmd->pc_dmahdl, 0, 0, DDI_DMA_SYNC_FORCPU);
606
607 switch (io->io_status) {
608 case PQI_DATA_IN_OUT_UNDERFLOW:
609 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
610 STATE_SENT_CMD | STATE_GOT_STATUS;
611 if (pkt->pkt_resid == cmd->pc_dma_count) {
612 pkt->pkt_reason = CMD_INCOMPLETE;
613 } else {
614 pkt->pkt_state |= STATE_XFERRED_DATA;
615 pkt->pkt_reason = CMD_CMPLT;
616 }
617 break;
618
619 case PQI_DATA_IN_OUT_GOOD:
620 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
621 STATE_SENT_CMD | STATE_GOT_STATUS;
622 if (cmd->pc_flags & PQI_FLAG_DMA_VALID)
623 pkt->pkt_state |= STATE_XFERRED_DATA;
624 pkt->pkt_reason = CMD_CMPLT;
625 pkt->pkt_resid = 0;
626 pkt->pkt_statistics = 0;
627 break;
628
629 case PQI_DATA_IN_OUT_ERROR:
630 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
631 STATE_SENT_CMD;
632 if (pkt->pkt_resid != cmd->pc_dma_count) {
633 pkt->pkt_state |= STATE_XFERRED_DATA;
634 pkt->pkt_reason = CMD_CMPLT;
635 } else {
636 pkt->pkt_reason = CMD_CMPLT;
637 }
638 break;
639
640 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
641 pkt->pkt_reason = CMD_TERMINATED;
642 pkt->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET;
643 break;
644
645 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
646 pkt->pkt_reason = CMD_CMPLT;
647 pkt->pkt_state |= STATE_GOT_BUS;
648 break;
649
650 default:
651 pkt->pkt_reason = CMD_INCOMPLETE;
652 break;
653 }
654
655 if (special_error_check(cmd) == B_TRUE) {
656 pkt->pkt_reason = CMD_DEV_GONE;
657 pkt->pkt_statistics = STAT_TERMINATED;
658
659 pqi_cmd_sm(cmd, PQI_CMD_FATAL, B_TRUE);
660 } else {
661 pqi_cmd_sm(cmd, PQI_CMD_CMPLT, B_TRUE);
662 }
663 }
664
665 static void
666 fail_outstanding_cmds(pqi_state_t s)
667 {
668 pqi_device_t devp;
669 int i;
670 pqi_queue_group_t *qg;
671
672 ASSERT(MUTEX_HELD(&s->s_mutex));
673 if (s->s_sync_io != NULL) {
674 s->s_sync_io->io_status = PQI_DATA_IN_OUT_UNSOLICITED_ABORT;
675 (s->s_sync_io->io_cb)(s->s_sync_io,
676 s->s_sync_io->io_context);
677 }
678
679 for (i = 0; i < s->s_num_queue_groups; i++) {
680 qg = &s->s_queue_groups[i];
681 mutex_enter(&qg->submit_lock[RAID_PATH]);
682 mutex_enter(&qg->submit_lock[AIO_PATH]);
683 qg->qg_active = B_FALSE;
684 mutex_exit(&qg->submit_lock[AIO_PATH]);
685 mutex_exit(&qg->submit_lock[RAID_PATH]);
686 }
687
688 for (devp = list_head(&s->s_devnodes); devp != NULL;
689 devp = list_next(&s->s_devnodes, devp)) {
690 pqi_fail_drive_cmds(devp);
691 }
692 }
693
694 static void
695 set_sg_descriptor(pqi_sg_entry_t *sg, ddi_dma_cookie_t *cookie)
696 {
697 sg->sg_addr = cookie->dmac_laddress;
698 sg->sg_len = cookie->dmac_size;
699 sg->sg_flags = 0;
700 }
701
702 static void
703 build_aio_sg_list(pqi_state_t s, pqi_aio_path_request_t *rqst,
704 pqi_cmd_t cmd, pqi_io_request_t *io)
705 {
706 int i;
707 int max_sg_per_iu;
708 uint16_t iu_length;
709 uint8_t chained;
710 uint8_t num_sg_in_iu = 0;
711 ddi_dma_cookie_t *cookies;
712 pqi_sg_entry_t *sg;
713
714 iu_length = offsetof(struct pqi_aio_path_request, ap_sglist) -
715 PQI_REQUEST_HEADER_LENGTH;
716
717 if (cmd->pc_dmaccount == 0)
718 goto out;
719 sg = rqst->ap_sglist;
720 cookies = cmd->pc_cached_cookies;
721 max_sg_per_iu = s->s_max_sg_per_iu - 1;
722 i = 0;
723 chained = 0;
724
725 for (;;) {
726 set_sg_descriptor(sg, cookies);
727 if (!chained)
728 num_sg_in_iu++;
729 i++;
730 if (i == cmd->pc_dmaccount)
731 break;
732 sg++;
733 cookies++;
734 if (i == max_sg_per_iu) {
735 sg->sg_addr = io->io_sg_chain_dma->dma_addr;
736 sg->sg_len = (cmd->pc_dmaccount - num_sg_in_iu) *
737 sizeof (*sg);
738 sg->sg_flags = CISS_SG_CHAIN;
739 chained = 1;
740 num_sg_in_iu++;
741 sg = (pqi_sg_entry_t *)
742 io->io_sg_chain_dma->alloc_memory;
743 }
744 }
745 sg->sg_flags = CISS_SG_LAST;
746 rqst->partial = chained;
747 if (chained) {
748 (void) ddi_dma_sync(io->io_sg_chain_dma->handle, 0, 0,
749 DDI_DMA_SYNC_FORDEV);
750 }
751 iu_length += num_sg_in_iu * sizeof (*sg);
752
753 out:
754 rqst->header.iu_length = iu_length;
755 rqst->num_sg_descriptors = num_sg_in_iu;
756 }
757
758 static void
759 build_raid_sg_list(pqi_state_t s, pqi_raid_path_request_t *rqst,
760 pqi_cmd_t cmd, pqi_io_request_t *io)
761 {
762 int i = 0;
763 int max_sg_per_iu;
764 int num_sg_in_iu = 0;
765 uint16_t iu_length;
766 uint8_t chained = 0;
767 ddi_dma_cookie_t *cookies;
768 pqi_sg_entry_t *sg;
769
770 iu_length = offsetof(struct pqi_raid_path_request, rp_sglist) -
771 PQI_REQUEST_HEADER_LENGTH;
772
773 if (cmd->pc_dmaccount == 0)
774 goto out;
775
776 sg = rqst->rp_sglist;
777 cookies = cmd->pc_cached_cookies;
778 max_sg_per_iu = s->s_max_sg_per_iu - 1;
779
780 for (;;) {
781 set_sg_descriptor(sg, cookies);
782 if (!chained)
783 num_sg_in_iu++;
784 i++;
785 if (i == cmd->pc_dmaccount)
786 break;
787 sg++;
788 cookies++;
789 if (i == max_sg_per_iu) {
790 ASSERT(io->io_sg_chain_dma != NULL);
791 sg->sg_addr = io->io_sg_chain_dma->dma_addr;
792 sg->sg_len = (cmd->pc_dmaccount - num_sg_in_iu) *
793 sizeof (*sg);
794 sg->sg_flags = CISS_SG_CHAIN;
795 chained = 1;
796 num_sg_in_iu++;
797 sg = (pqi_sg_entry_t *)
798 io->io_sg_chain_dma->alloc_memory;
799 }
800 }
801 sg->sg_flags = CISS_SG_LAST;
802 rqst->rp_partial = chained;
803 if (chained) {
804 (void) ddi_dma_sync(io->io_sg_chain_dma->handle, 0, 0,
805 DDI_DMA_SYNC_FORDEV);
806 }
807 iu_length += num_sg_in_iu * sizeof (*sg);
808
809 out:
810 rqst->header.iu_length = iu_length;
811 }
812
813 static uint32_t
814 read_heartbeat_counter(pqi_state_t s)
815 {
816 return (ddi_get32(s->s_datap, s->s_heartbeat_counter));
817 }
818
819 static void
820 take_ctlr_offline(pqi_state_t s)
821 {
822 int circ = 0;
823 int circ1 = 0;
824
825 mutex_enter(&s->s_mutex);
826 s->s_offline = 1;
827 s->s_watchdog = 0;
828 fail_outstanding_cmds(s);
829 mutex_exit(&s->s_mutex);
830
831 /*
832 * This will have the effect of releasing the device's dip
833 * structure from the NDI layer do to s_offline == 1.
834 */
835 ndi_devi_enter(scsi_vhci_dip, &circ1);
836 ndi_devi_enter(s->s_dip, &circ);
837 (void) pqi_config_all(s->s_dip, s);
838 ndi_devi_exit(s->s_dip, circ);
839 ndi_devi_exit(scsi_vhci_dip, circ1);
840 }
841
842 static uint32_t
843 free_elem_count(pqi_index_t pi, pqi_index_t ci, uint32_t per_iq)
844 {
845 pqi_index_t used;
846 if (pi >= ci) {
847 used = pi - ci;
848 } else {
849 used = per_iq - ci + pi;
850 }
851 return (per_iq - used - 1);
852 }
853
854 static boolean_t
855 is_aio_enabled(pqi_device_t d)
856 {
857 return (d->pd_aio_enabled ? B_TRUE : B_FALSE);
858 }