1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2018 Nexenta Systems, Inc.
14 */
15
16 /*
17 * This file sets up the interrupts with the system and also processes
18 * interrupts from the hardware.
19 */
20
21 /* ---- Driver specific header ---- */
22 #include <smartpqi.h>
23
24 /* ---- Forward declarations of private methods ---- */
25 static int add_intrs(pqi_state_t s, int type);
26 static uint_t intr_handler(caddr_t arg1, caddr_t arg2);
27 static void sync_error(pqi_state_t s, pqi_io_request_t *io,
28 pqi_io_response_t *rsp);
29 static void process_raid_io_error(pqi_io_request_t *io);
30 static void process_aio_io_error(pqi_io_request_t *io);
31 static void disable_aio_path(pqi_io_request_t *io);
32
33 /*
34 * smartpqi_register_intrs -- Figure out which type of interrupts and register
35 * them with the framework.
36 */
37 int
38 smartpqi_register_intrs(pqi_state_t s)
39 {
40 int intr_types;
41
42 /* ---- Get supported interrupt types ---- */
43 if (ddi_intr_get_supported_types(s->s_dip, &intr_types) !=
44 DDI_SUCCESS) {
45 dev_err(s->s_dip, CE_NOTE,
46 "failed to get supported intr types");
47 return (FALSE);
48 }
49
50 if (intr_types & DDI_INTR_TYPE_MSIX) {
51 if (add_intrs(s, DDI_INTR_TYPE_MSIX) == TRUE) {
52 s->s_intr_type = DDI_INTR_TYPE_MSIX;
53 return (TRUE);
54 }
55 } else if (intr_types & DDI_INTR_TYPE_MSI) {
56 if (add_intrs(s, DDI_INTR_TYPE_MSI) == TRUE) {
57 s->s_intr_type = DDI_INTR_TYPE_MSI;
58 return (TRUE);
59 }
60 } else if (intr_types & DDI_INTR_TYPE_FIXED) {
61 if (add_intrs(s, DDI_INTR_TYPE_FIXED) == TRUE) {
62 s->s_intr_type = DDI_INTR_TYPE_FIXED;
63 return (TRUE);
64 }
65 } else {
66 /* ---- Warning since it's a DDI framework error ---- */
67 dev_err(s->s_dip, CE_WARN,
68 "ddi_intr_get_supported_types returned bogus type of 0x%x",
69 intr_types);
70 }
71
72 return (FALSE);
73 }
74
75 /*
76 * smartqpi_unregister_intrs -- Disable and remove interrupt handlers
77 */
78 void
79 smartpqi_unregister_intrs(pqi_state_t s)
80 {
81 int i;
82
83 /* --- First disable the interrupts ---- */
84 if (s->s_intr_cap & DDI_INTR_FLAG_BLOCK) {
85 (void) ddi_intr_block_disable(s->s_itable, s->s_intr_cnt);
86 } else {
87 for (i = 0; i < s->s_intr_cnt; i++) {
88 (void) ddi_intr_disable(s->s_itable[i]);
89 }
90 }
91
92 /* ---- Next remove the interrupt handlers ---- */
93 for (i = 0; i < s->s_intr_cnt; i++) {
94 (void) ddi_intr_remove_handler(s->s_itable[i]);
95 (void) ddi_intr_free(s->s_itable[i]);
96 }
97
98 kmem_free(s->s_itable, s->s_intr_size);
99 /* ---- Just in case ---- */
100 s->s_itable = NULL;
101 s->s_intr_size = 0;
102 }
103
104 void
105 pqi_process_io_intr(pqi_state_t s, pqi_queue_group_t *qg)
106 {
107 pqi_index_t oq_pi;
108 pqi_index_t oq_ci;
109 pqi_io_request_t *io;
110 pqi_io_response_t *rsp;
111 uint16_t rqst_id;
112 int response_cnt = 0;
113 int qnotify;
114
115 oq_ci = qg->oq_ci_copy;
116 atomic_inc_32(&s->s_intr_count);
117
118 mutex_enter(&s->s_intr_mutex);
119 for (;;) {
120 (void) ddi_dma_sync(s->s_queue_dma->handle,
121 (uintptr_t)qg->oq_pi -
122 (uintptr_t)s->s_queue_dma->alloc_memory,
123 sizeof (oq_pi), DDI_DMA_SYNC_FORCPU);
124
125 oq_pi = *qg->oq_pi;
126 if (oq_pi == oq_ci)
127 break;
128
129 rsp = (pqi_io_response_t *)(qg->oq_element_array +
130 (oq_ci * PQI_OPERATIONAL_OQ_ELEMENT_LENGTH));
131 (void) ddi_dma_sync(s->s_queue_dma->handle,
132 (uintptr_t)rsp - (uintptr_t)s->s_queue_dma->alloc_memory,
133 sizeof (*rsp), DDI_DMA_SYNC_FORCPU);
134 rqst_id = rsp->request_id;
135 ASSERT(rqst_id < s->s_max_io_slots);
136 io = &s->s_io_rqst_pool[rqst_id];
137
138 ASSERT(io->io_refcount == 1);
139
140 if (io->io_cmd != NULL) {
141 pqi_cmd_t cmd = io->io_cmd;
142
143 mutex_enter(&cmd->pc_device->pd_mutex);
144 if (cmd->pc_flags & PQI_FLAG_ABORTED) {
145 mutex_exit(&cmd->pc_device->pd_mutex);
146 response_cnt++;
147 oq_ci = (oq_ci + 1) % s->s_num_elements_per_oq;
148 continue;
149 }
150 cmd->pc_flags |= PQI_FLAG_FINISHING;
151 mutex_exit(&cmd->pc_device->pd_mutex);
152 }
153
154 io->io_iu_type = rsp->header.iu_type;
155 switch (rsp->header.iu_type) {
156 case PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS:
157 case PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS:
158 case PQI_RESPONSE_IU_GENERAL_MANAGEMENT:
159 io->io_status = PQI_DATA_IN_OUT_GOOD;
160 break;
161 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
162 io->io_status = PQI_DATA_IN_OUT_ERROR;
163 sync_error(s, io, rsp);
164 process_raid_io_error(io);
165 break;
166 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
167 io->io_status = PQI_DATA_IN_OUT_ERROR;
168 sync_error(s, io, rsp);
169 process_aio_io_error(io);
170 break;
171 case PQI_RESPONSE_IU_AIO_PATH_DISABLED:
172 io->io_status = PQI_DATA_IN_OUT_PROTOCOL_ERROR;
173 disable_aio_path(io);
174 break;
175
176 default:
177 ASSERT(0);
178 break;
179 }
180 io->io_cb(io, io->io_context);
181 response_cnt++;
182 oq_ci = (oq_ci + 1) % s->s_num_elements_per_oq;
183 }
184
185 if (response_cnt) {
186 qg->cmplt_count += response_cnt;
187 qg->oq_ci_copy = oq_ci;
188 ddi_put32(s->s_datap, qg->oq_ci, oq_ci);
189 }
190 mutex_exit(&s->s_intr_mutex);
191
192 mutex_enter(&s->s_mutex);
193 qnotify = HBA_QUIESCED_PENDING(s);
194 mutex_exit(&s->s_mutex);
195
196 if (qnotify)
197 pqi_quiesced_notify(s);
198
199 }
200
201 /*
202 * add_intrs --
203 */
204 static int
205 add_intrs(pqi_state_t s, int type)
206 {
207 dev_info_t *dip = s->s_dip;
208 int avail;
209 int actual;
210 int count = 0;
211 int i;
212 int ret;
213
214 /* ---- Get number of interrupts ---- */
215 ret = ddi_intr_get_nintrs(dip, type, &count);
216 if (ret != DDI_SUCCESS || count <= 0) {
217 dev_err(s->s_dip, CE_NOTE, "ddi_intr_get_nintrs failed, "
218 "ret=%d, count=%d", ret, count);
219 return (FALSE);
220 }
221
222 /* ---- Get number of available interrupts ---- */
223 ret = ddi_intr_get_navail(dip, type, &avail);
224 if (ret != DDI_SUCCESS || avail == 0) {
225 dev_err(s->s_dip, CE_NOTE, "ddi_intr_get_navail failed, "
226 "ret=%d, avail=%d", ret, avail);
227 return (FALSE);
228 }
229
230 if (type != DDI_INTR_TYPE_FIXED)
231 count = 1;
232
233 s->s_intr_size = count * sizeof (ddi_intr_handle_t);
234 s->s_itable = kmem_zalloc(s->s_intr_size, KM_SLEEP);
235 ret = ddi_intr_alloc(dip, s->s_itable, type, 0, count, &actual,
236 DDI_INTR_ALLOC_NORMAL);
237 if (ret != DDI_SUCCESS || actual == 0) {
238 dev_err(s->s_dip, CE_NOTE, "ddi_intr_alloc failed, ret=%d",
239 ret);
240 return (FALSE);
241 }
242
243 /* ---- Use count return or abort? Make note of at least---- */
244 if (actual < count) {
245 dev_err(s->s_dip, CE_NOTE,
246 "interrupts: requested=%d, received=%d",
247 count, actual);
248 }
249 s->s_intr_cnt = actual;
250
251 /* ---- Get priority for first intr, assume rest are the same ---- */
252 if ((ret = ddi_intr_get_pri(s->s_itable[0], &s->s_intr_pri)) !=
253 DDI_SUCCESS) {
254 dev_err(s->s_dip, CE_NOTE, "ddi_intr_get_pri failed, ret=%d",
255 ret);
256 goto failure;
257 }
258
259 /* ---- Test for high level mutex ---- */
260 if (s->s_intr_pri >= ddi_intr_get_hilevel_pri()) {
261 dev_err(s->s_dip, CE_NOTE, "Hi level interrupts not supported");
262 goto failure;
263 }
264
265 /* ---- Install interrupt handler ---- */
266 for (i = 0; i < actual; i++) {
267 if ((ret = ddi_intr_add_handler(s->s_itable[i], intr_handler,
268 (caddr_t)s, (caddr_t)(uintptr_t)i)) != DDI_SUCCESS) {
269 dev_err(s->s_dip, CE_NOTE,
270 "ddi_intr_add_handler failed, index=%d, ret=%d",
271 i, ret);
272 goto failure;
273 }
274 }
275
276 if ((ret = ddi_intr_get_cap(s->s_itable[0], &s->s_intr_cap))
277 != DDI_SUCCESS) {
278 dev_err(s->s_dip, CE_NOTE, "ddi_intr_get_cap failed, ret=%d",
279 ret);
280 goto failure;
281 }
282
283 /* ---- Enable interrupts ---- */
284 if (s->s_intr_cap & DDI_INTR_FLAG_BLOCK) {
285 (void) ddi_intr_block_enable(s->s_itable, s->s_intr_cnt);
286 } else {
287 /* --- Enable interrupts for either MSI or FIXED ---- */
288 for (i = 0; i < actual; i++)
289 (void) ddi_intr_enable(s->s_itable[i]);
290 }
291
292 return (TRUE);
293
294 failure:
295 /* ---- Free allocated interrupts pointers ---- */
296 for (i = 0; i < actual; i++)
297 (void) ddi_intr_free(s->s_itable[i]);
298 kmem_free(s->s_itable, s->s_intr_size);
299 s->s_itable = NULL;
300 s->s_intr_size = 0;
301 return (FALSE);
302 }
303
304 static void
305 disable_aio_path(pqi_io_request_t *io)
306 {
307 pqi_device_t devp;
308
309 devp = io->io_cmd->pc_device;
310 devp->pd_aio_enabled = 0;
311 }
312
313 static void
314 process_raid_io_error(pqi_io_request_t *io)
315 {
316 pqi_raid_error_info_t ei;
317 pqi_cmd_t cmd;
318 int sense_len;
319 int statusbuf_len;
320 int sense_len_to_copy;
321 struct scsi_arq_status *arq;
322 struct scsi_pkt *pkt;
323
324 if ((ei = io->io_error_info) != NULL) {
325 io->io_status = ei->data_out_result;
326 if ((cmd = io->io_cmd) == NULL)
327 return;
328
329 pkt = cmd->pc_pkt;
330 pkt->pkt_resid -= ei->data_out_transferred;
331 /* LINTED E_BAD_PTR_CAST_ALIGN */
332 arq = (struct scsi_arq_status *)pkt->pkt_scbp;
333 *((uchar_t *)&arq->sts_status) = ei->status;
334 *((uchar_t *)&arq->sts_rqpkt_status) = STATUS_GOOD;
335 arq->sts_rqpkt_state = STATE_GOT_BUS | STATE_GOT_TARGET |
336 STATE_SENT_CMD | STATE_XFERRED_DATA | STATE_GOT_STATUS |
337 STATE_ARQ_DONE;
338
339 sense_len = ei->sense_data_length;
340 if (sense_len == 0)
341 sense_len = ei->response_data_length;
342
343 if (sense_len == 0) {
344 /* ---- auto request sense failed ---- */
345 arq->sts_rqpkt_status.sts_chk = 1;
346 arq->sts_rqpkt_resid = cmd->pc_statuslen;
347 return;
348 } else if (sense_len < cmd->pc_statuslen) {
349 /* ---- auto request sense short ---- */
350 arq->sts_rqpkt_resid = cmd->pc_statuslen -
351 sense_len;
352 } else {
353 /* ---- auto request sense complete ---- */
354 arq->sts_rqpkt_resid = 0;
355 }
356 arq->sts_rqpkt_statistics = 0;
357 pkt->pkt_state |= STATE_ARQ_DONE;
358 if (cmd->pc_statuslen > PQI_ARQ_STATUS_NOSENSE_LEN) {
359 statusbuf_len = cmd->pc_statuslen -
360 PQI_ARQ_STATUS_NOSENSE_LEN;
361 } else {
362 statusbuf_len = 0;
363 }
364
365 if (sense_len > sizeof (ei->data))
366 sense_len = sizeof (ei->data);
367 sense_len_to_copy = min(sense_len, statusbuf_len);
368
369 if (sense_len_to_copy) {
370 (void) memcpy(&arq->sts_sensedata, ei->data,
371 sense_len_to_copy);
372 }
373 } else {
374 /*
375 * sync_error is called before this and sets io_error_info
376 * which means the value must be non-zero
377 */
378 ASSERT(0);
379 }
380 }
381
382 /*ARGSUSED*/
383 static void
384 process_aio_io_error(pqi_io_request_t *io)
385 {
386 }
387
388 static void
389 sync_error(pqi_state_t s, pqi_io_request_t *io, pqi_io_response_t *rsp)
390 {
391 (void) ddi_dma_sync(s->s_error_dma->handle,
392 rsp->error_index * PQI_ERROR_BUFFER_ELEMENT_LENGTH,
393 PQI_ERROR_BUFFER_ELEMENT_LENGTH, DDI_DMA_SYNC_FORCPU);
394
395 io->io_error_info = s->s_error_dma->alloc_memory +
396 (rsp->error_index * PQI_ERROR_BUFFER_ELEMENT_LENGTH);
397 }
398
399 static void
400 process_event_intr(pqi_state_t s)
401 {
402 pqi_event_queue_t *q = &s->s_event_queue;
403 pqi_event_response_t *rsp;
404 int idx;
405 int num_events = 0;
406 pqi_event_t e;
407 pqi_index_t oq_ci;
408 pqi_index_t oq_pi;
409
410 oq_ci = q->oq_ci_copy;
411
412 mutex_enter(&s->s_intr_mutex);
413 for (;;) {
414 (void) ddi_dma_sync(s->s_queue_dma->handle,
415 (uintptr_t)q->oq_pi -
416 (uintptr_t)s->s_queue_dma->alloc_memory,
417 sizeof (oq_pi), DDI_DMA_SYNC_FORCPU);
418 oq_pi = *q->oq_pi;
419
420 if (oq_pi == oq_ci)
421 break;
422
423 num_events++;
424 (void) ddi_dma_sync(s->s_queue_dma->handle,
425 (uintptr_t)q->oq_element_array +
426 (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH) -
427 (uintptr_t)s->s_queue_dma->alloc_memory,
428 sizeof (*rsp),
429 DDI_DMA_SYNC_FORCPU);
430 rsp = (pqi_event_response_t *)((uintptr_t)q->oq_element_array +
431 (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH));
432 idx = pqi_map_event(rsp->event_type);
433
434 if (idx != -1 && rsp->request_acknowlege) {
435 e = &s->s_events[idx];
436 e->ev_pending = B_TRUE;
437 e->ev_type = rsp->event_type;
438 e->ev_id = rsp->event_id;
439 e->ev_additional = rsp->additional_event_id;
440 }
441 oq_ci = (oq_ci + 1) % PQI_NUM_EVENT_QUEUE_ELEMENTS;
442 }
443
444 if (num_events != 0) {
445 q->oq_ci_copy = oq_ci;
446 ddi_put32(s->s_datap, q->oq_ci, oq_ci);
447 (void) ddi_taskq_dispatch(s->s_events_taskq, pqi_event_worker,
448 s, 0);
449 }
450 mutex_exit(&s->s_intr_mutex);
451 }
452
453 static uint_t
454 intr_handler(caddr_t arg1, caddr_t arg2)
455 {
456 /* LINTED E_BAD_PTR_CAST_ALIGN */
457 pqi_state_t s = (pqi_state_t)arg1;
458 int queue_group_idx = (int)(intptr_t)arg2;
459 pqi_queue_group_t *qg;
460
461 if (s->s_intr_ready == 0)
462 return (DDI_INTR_CLAIMED);
463
464 qg = &s->s_queue_groups[queue_group_idx];
465 pqi_process_io_intr(s, qg);
466 if (queue_group_idx == s->s_event_queue.int_msg_num)
467 process_event_intr(s);
468
469 pqi_start_io(s, qg, RAID_PATH, NULL);
470 pqi_start_io(s, qg, AIO_PATH, NULL);
471
472 return (DDI_INTR_CLAIMED);
473 }