1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
24 * Copyright 2012 Alexey Zaytsev <alexey.zaytsev@gmail.com>
25 * Copyright (c) 2016 by Delphix. All rights reserved.
26 */
27
28 /* Based on the NetBSD virtio driver by Minoura Makoto. */
29 /*
30 * Copyright (c) 2010 Minoura Makoto.
31 * All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 *
42 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
43 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
44 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
45 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
46 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
47 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
48 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
49 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
50 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
51 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52 *
53 */
54
55 #include <sys/conf.h>
56 #include <sys/kmem.h>
57 #include <sys/debug.h>
58 #include <sys/modctl.h>
59 #include <sys/autoconf.h>
60 #include <sys/ddi_impldefs.h>
61 #include <sys/ddi.h>
62 #include <sys/sunddi.h>
63 #include <sys/sunndi.h>
64 #include <sys/avintr.h>
65 #include <sys/spl.h>
66 #include <sys/promif.h>
67 #include <sys/list.h>
68 #include <sys/bootconf.h>
69 #include <sys/bootsvcs.h>
70 #include <sys/sysmacros.h>
71 #include <sys/pci.h>
72
73 #include "virtiovar.h"
74 #include "virtioreg.h"
75
76 #define NDEVNAMES (sizeof (virtio_device_name) / sizeof (char *))
77 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */
78 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1)) & \
79 ~(VIRTIO_PAGE_SIZE-1))
80
81 void
82 virtio_set_status(struct virtio_softc *sc, unsigned int status)
83 {
84 int old = 0;
85
86 if (status != 0) {
87 old = ddi_get8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
88 VIRTIO_CONFIG_DEVICE_STATUS));
89 }
90
91 ddi_put8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
92 VIRTIO_CONFIG_DEVICE_STATUS), status | old);
93 }
94
95 /*
96 * Negotiate features, save the result in sc->sc_features
97 */
98 uint32_t
99 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features)
100 {
101 uint32_t host_features;
102 uint32_t features;
103
104 host_features = ddi_get32(sc->sc_ioh,
105 /* LINTED E_BAD_PTR_CAST_ALIGN */
106 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_DEVICE_FEATURES));
107
108 dev_debug(sc->sc_dev, CE_NOTE, "host features: %x, guest features: %x",
109 host_features, guest_features);
110
111 features = host_features & guest_features;
112 ddi_put32(sc->sc_ioh,
113 /* LINTED E_BAD_PTR_CAST_ALIGN */
114 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_GUEST_FEATURES),
115 features);
116
117 sc->sc_features = features;
118
119 return (host_features);
120 }
121
122 size_t
123 virtio_show_features(uint32_t features, char *buf, size_t len)
124 {
125 char *orig_buf = buf;
126 char *bufend = buf + len;
127
128 /* LINTED E_PTRDIFF_OVERFLOW */
129 buf += snprintf(buf, bufend - buf, "Generic ( ");
130 if (features & VIRTIO_F_RING_INDIRECT_DESC)
131 /* LINTED E_PTRDIFF_OVERFLOW */
132 buf += snprintf(buf, bufend - buf, "INDIRECT_DESC ");
133
134 /* LINTED E_PTRDIFF_OVERFLOW */
135 buf += snprintf(buf, bufend - buf, ") ");
136
137 /* LINTED E_PTRDIFF_OVERFLOW */
138 return (buf - orig_buf);
139 }
140
141 boolean_t
142 virtio_has_feature(struct virtio_softc *sc, uint32_t feature)
143 {
144 return (sc->sc_features & feature);
145 }
146
147 /*
148 * Device configuration registers.
149 */
150 uint8_t
151 virtio_read_device_config_1(struct virtio_softc *sc, unsigned int index)
152 {
153 ASSERT(sc->sc_config_offset);
154 return ddi_get8(sc->sc_ioh,
155 (uint8_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
156 }
157
158 uint16_t
159 virtio_read_device_config_2(struct virtio_softc *sc, unsigned int index)
160 {
161 ASSERT(sc->sc_config_offset);
162 return ddi_get16(sc->sc_ioh,
163 /* LINTED E_BAD_PTR_CAST_ALIGN */
164 (uint16_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
165 }
166
167 uint32_t
168 virtio_read_device_config_4(struct virtio_softc *sc, unsigned int index)
169 {
170 ASSERT(sc->sc_config_offset);
171 return ddi_get32(sc->sc_ioh,
172 /* LINTED E_BAD_PTR_CAST_ALIGN */
173 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
174 }
175
176 uint64_t
177 virtio_read_device_config_8(struct virtio_softc *sc, unsigned int index)
178 {
179 uint64_t r;
180
181 ASSERT(sc->sc_config_offset);
182 r = ddi_get32(sc->sc_ioh,
183 /* LINTED E_BAD_PTR_CAST_ALIGN */
184 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset +
185 index + sizeof (uint32_t)));
186
187 r <<= 32;
188
189 r += ddi_get32(sc->sc_ioh,
190 /* LINTED E_BAD_PTR_CAST_ALIGN */
191 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index));
192 return (r);
193 }
194
195 void
196 virtio_write_device_config_1(struct virtio_softc *sc, unsigned int index,
197 uint8_t value)
198 {
199 ASSERT(sc->sc_config_offset);
200 ddi_put8(sc->sc_ioh,
201 (uint8_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
202 }
203
204 void
205 virtio_write_device_config_2(struct virtio_softc *sc, unsigned int index,
206 uint16_t value)
207 {
208 ASSERT(sc->sc_config_offset);
209 ddi_put16(sc->sc_ioh,
210 /* LINTED E_BAD_PTR_CAST_ALIGN */
211 (uint16_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
212 }
213
214 void
215 virtio_write_device_config_4(struct virtio_softc *sc, unsigned int index,
216 uint32_t value)
217 {
218 ASSERT(sc->sc_config_offset);
219 ddi_put32(sc->sc_ioh,
220 /* LINTED E_BAD_PTR_CAST_ALIGN */
221 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index), value);
222 }
223
224 void
225 virtio_write_device_config_8(struct virtio_softc *sc, unsigned int index,
226 uint64_t value)
227 {
228 ASSERT(sc->sc_config_offset);
229 ddi_put32(sc->sc_ioh,
230 /* LINTED E_BAD_PTR_CAST_ALIGN */
231 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset + index),
232 value & 0xFFFFFFFF);
233 ddi_put32(sc->sc_ioh,
234 /* LINTED E_BAD_PTR_CAST_ALIGN */
235 (uint32_t *)(sc->sc_io_addr + sc->sc_config_offset +
236 index + sizeof (uint32_t)), value >> 32);
237 }
238
239 /*
240 * Start/stop vq interrupt. No guarantee.
241 */
242 void
243 virtio_stop_vq_intr(struct virtqueue *vq)
244 {
245 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
246 }
247
248 void
249 virtio_start_vq_intr(struct virtqueue *vq)
250 {
251 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
252 }
253
254 static ddi_dma_attr_t virtio_vq_dma_attr = {
255 DMA_ATTR_V0, /* Version number */
256 0, /* low address */
257 0x00000FFFFFFFFFFF, /* high address. Has to fit into 32 bits */
258 /* after page-shifting */
259 0xFFFFFFFF, /* counter register max */
260 VIRTIO_PAGE_SIZE, /* page alignment required */
261 0x3F, /* burst sizes: 1 - 32 */
262 0x1, /* minimum transfer size */
263 0xFFFFFFFF, /* max transfer size */
264 0xFFFFFFFF, /* address register max */
265 1, /* no scatter-gather */
266 1, /* device operates on bytes */
267 0, /* attr flag: set to 0 */
268 };
269
270 static ddi_dma_attr_t virtio_vq_indirect_dma_attr = {
271 DMA_ATTR_V0, /* Version number */
272 0, /* low address */
273 0xFFFFFFFFFFFFFFFF, /* high address */
274 0xFFFFFFFF, /* counter register max */
275 1, /* No specific alignment */
276 0x3F, /* burst sizes: 1 - 32 */
277 0x1, /* minimum transfer size */
278 0xFFFFFFFF, /* max transfer size */
279 0xFFFFFFFF, /* address register max */
280 1, /* no scatter-gather */
281 1, /* device operates on bytes */
282 0, /* attr flag: set to 0 */
283 };
284
285 /* Same for direct and indirect descriptors. */
286 static ddi_device_acc_attr_t virtio_vq_devattr = {
287 DDI_DEVICE_ATTR_V0,
288 DDI_NEVERSWAP_ACC,
289 DDI_STORECACHING_OK_ACC,
290 DDI_DEFAULT_ACC
291 };
292
293 static void
294 virtio_free_indirect(struct vq_entry *entry)
295 {
296
297 (void) ddi_dma_unbind_handle(entry->qe_indirect_dma_handle);
298 ddi_dma_mem_free(&entry->qe_indirect_dma_acch);
299 ddi_dma_free_handle(&entry->qe_indirect_dma_handle);
300
301 entry->qe_indirect_descs = NULL;
302 }
303
304
305 static int
306 virtio_alloc_indirect(struct virtio_softc *sc, struct vq_entry *entry)
307 {
308 int allocsize, num;
309 size_t len;
310 unsigned int ncookies;
311 int ret;
312
313 num = entry->qe_queue->vq_indirect_num;
314 ASSERT(num > 1);
315
316 allocsize = sizeof (struct vring_desc) * num;
317
318 ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_indirect_dma_attr,
319 DDI_DMA_SLEEP, NULL, &entry->qe_indirect_dma_handle);
320 if (ret != DDI_SUCCESS) {
321 dev_err(sc->sc_dev, CE_WARN,
322 "Failed to allocate dma handle for indirect descriptors, "
323 "entry %d, vq %d", entry->qe_index,
324 entry->qe_queue->vq_index);
325 goto out_alloc_handle;
326 }
327
328 ret = ddi_dma_mem_alloc(entry->qe_indirect_dma_handle, allocsize,
329 &virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
330 (caddr_t *)&entry->qe_indirect_descs, &len,
331 &entry->qe_indirect_dma_acch);
332 if (ret != DDI_SUCCESS) {
333 dev_err(sc->sc_dev, CE_WARN,
334 "Failed to allocate dma memory for indirect descriptors, "
335 "entry %d, vq %d,", entry->qe_index,
336 entry->qe_queue->vq_index);
337 goto out_alloc;
338 }
339
340 (void) memset(entry->qe_indirect_descs, 0xff, allocsize);
341
342 ret = ddi_dma_addr_bind_handle(entry->qe_indirect_dma_handle, NULL,
343 (caddr_t)entry->qe_indirect_descs, len,
344 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
345 &entry->qe_indirect_dma_cookie, &ncookies);
346 if (ret != DDI_DMA_MAPPED) {
347 dev_err(sc->sc_dev, CE_WARN,
348 "Failed to bind dma memory for indirect descriptors, "
349 "entry %d, vq %d", entry->qe_index,
350 entry->qe_queue->vq_index);
351 goto out_bind;
352 }
353
354 /* We asked for a single segment */
355 ASSERT(ncookies == 1);
356
357 return (0);
358
359 out_bind:
360 ddi_dma_mem_free(&entry->qe_indirect_dma_acch);
361 out_alloc:
362 ddi_dma_free_handle(&entry->qe_indirect_dma_handle);
363 out_alloc_handle:
364
365 return (ret);
366 }
367
368 /*
369 * Initialize the vq structure.
370 */
371 static int
372 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq)
373 {
374 int ret;
375 uint16_t i;
376 int vq_size = vq->vq_num;
377 int indirect_num = vq->vq_indirect_num;
378
379 /* free slot management */
380 list_create(&vq->vq_freelist, sizeof (struct vq_entry),
381 offsetof(struct vq_entry, qe_list));
382
383 for (i = 0; i < vq_size; i++) {
384 struct vq_entry *entry = &vq->vq_entries[i];
385 list_insert_tail(&vq->vq_freelist, entry);
386 entry->qe_index = i;
387 entry->qe_desc = &vq->vq_descs[i];
388 entry->qe_queue = vq;
389
390 if (indirect_num) {
391 ret = virtio_alloc_indirect(sc, entry);
392 if (ret)
393 goto out_indirect;
394 }
395 }
396
397 mutex_init(&vq->vq_freelist_lock, "virtio-freelist", MUTEX_DRIVER,
398 DDI_INTR_PRI(sc->sc_intr_prio));
399 mutex_init(&vq->vq_avail_lock, "virtio-avail", MUTEX_DRIVER,
400 DDI_INTR_PRI(sc->sc_intr_prio));
401 mutex_init(&vq->vq_used_lock, "virtio-used", MUTEX_DRIVER,
402 DDI_INTR_PRI(sc->sc_intr_prio));
403
404 return (0);
405
406 out_indirect:
407 for (i = 0; i < vq_size; i++) {
408 struct vq_entry *entry = &vq->vq_entries[i];
409 if (entry->qe_indirect_descs)
410 virtio_free_indirect(entry);
411 }
412
413 return (ret);
414 }
415
416 /*
417 * Allocate/free a vq.
418 */
419 struct virtqueue *
420 virtio_alloc_vq(struct virtio_softc *sc, unsigned int index, unsigned int size,
421 unsigned int indirect_num, const char *name)
422 {
423 int vq_size, allocsize1, allocsize2, allocsize = 0;
424 int ret;
425 unsigned int ncookies;
426 size_t len;
427 struct virtqueue *vq;
428
429 ddi_put16(sc->sc_ioh,
430 /* LINTED E_BAD_PTR_CAST_ALIGN */
431 (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT), index);
432 vq_size = ddi_get16(sc->sc_ioh,
433 /* LINTED E_BAD_PTR_CAST_ALIGN */
434 (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SIZE));
435 if (vq_size == 0) {
436 dev_err(sc->sc_dev, CE_WARN,
437 "virtqueue dest not exist, index %d for %s\n", index, name);
438 goto out;
439 }
440
441 vq = kmem_zalloc(sizeof (struct virtqueue), KM_SLEEP);
442
443 /* size 0 => use native vq size, good for receive queues. */
444 if (size)
445 vq_size = MIN(vq_size, size);
446
447 /* allocsize1: descriptor table + avail ring + pad */
448 allocsize1 = VIRTQUEUE_ALIGN(sizeof (struct vring_desc) * vq_size +
449 sizeof (struct vring_avail) + sizeof (uint16_t) * vq_size);
450 /* allocsize2: used ring + pad */
451 allocsize2 = VIRTQUEUE_ALIGN(sizeof (struct vring_used) +
452 sizeof (struct vring_used_elem) * vq_size);
453
454 allocsize = allocsize1 + allocsize2;
455
456 ret = ddi_dma_alloc_handle(sc->sc_dev, &virtio_vq_dma_attr,
457 DDI_DMA_SLEEP, NULL, &vq->vq_dma_handle);
458 if (ret != DDI_SUCCESS) {
459 dev_err(sc->sc_dev, CE_WARN,
460 "Failed to allocate dma handle for vq %d", index);
461 goto out_alloc_handle;
462 }
463
464 ret = ddi_dma_mem_alloc(vq->vq_dma_handle, allocsize,
465 &virtio_vq_devattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
466 (caddr_t *)&vq->vq_vaddr, &len, &vq->vq_dma_acch);
467 if (ret != DDI_SUCCESS) {
468 dev_err(sc->sc_dev, CE_WARN,
469 "Failed to allocate dma memory for vq %d", index);
470 goto out_alloc;
471 }
472
473 ret = ddi_dma_addr_bind_handle(vq->vq_dma_handle, NULL,
474 (caddr_t)vq->vq_vaddr, len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
475 DDI_DMA_SLEEP, NULL, &vq->vq_dma_cookie, &ncookies);
476 if (ret != DDI_DMA_MAPPED) {
477 dev_err(sc->sc_dev, CE_WARN,
478 "Failed to bind dma memory for vq %d", index);
479 goto out_bind;
480 }
481
482 /* We asked for a single segment */
483 ASSERT(ncookies == 1);
484 /* and page-ligned buffers. */
485 ASSERT(vq->vq_dma_cookie.dmac_laddress % VIRTIO_PAGE_SIZE == 0);
486
487 (void) memset(vq->vq_vaddr, 0, allocsize);
488
489 /* Make sure all zeros hit the buffer before we point the host to it */
490 membar_producer();
491
492 /* set the vq address */
493 ddi_put32(sc->sc_ioh,
494 /* LINTED E_BAD_PTR_CAST_ALIGN */
495 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS),
496 (vq->vq_dma_cookie.dmac_laddress / VIRTIO_PAGE_SIZE));
497
498 /* remember addresses and offsets for later use */
499 vq->vq_owner = sc;
500 vq->vq_num = vq_size;
501 vq->vq_index = index;
502 vq->vq_descs = vq->vq_vaddr;
503 vq->vq_availoffset = sizeof (struct vring_desc)*vq_size;
504 vq->vq_avail = (void *)(((char *)vq->vq_descs) + vq->vq_availoffset);
505 vq->vq_usedoffset = allocsize1;
506 vq->vq_used = (void *)(((char *)vq->vq_descs) + vq->vq_usedoffset);
507
508 ASSERT(indirect_num == 0 ||
509 virtio_has_feature(sc, VIRTIO_F_RING_INDIRECT_DESC));
510 vq->vq_indirect_num = indirect_num;
511
512 /* free slot management */
513 vq->vq_entries = kmem_zalloc(sizeof (struct vq_entry) * vq_size,
514 KM_SLEEP);
515
516 ret = virtio_init_vq(sc, vq);
517 if (ret)
518 goto out_init;
519
520 dev_debug(sc->sc_dev, CE_NOTE,
521 "Allocated %d entries for vq %d:%s (%d indirect descs)",
522 vq_size, index, name, indirect_num * vq_size);
523
524 return (vq);
525
526 out_init:
527 kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq_size);
528 (void) ddi_dma_unbind_handle(vq->vq_dma_handle);
529 out_bind:
530 ddi_dma_mem_free(&vq->vq_dma_acch);
531 out_alloc:
532 ddi_dma_free_handle(&vq->vq_dma_handle);
533 out_alloc_handle:
534 kmem_free(vq, sizeof (struct virtqueue));
535 out:
536 return (NULL);
537 }
538
539 void
540 virtio_free_vq(struct virtqueue *vq)
541 {
542 struct virtio_softc *sc = vq->vq_owner;
543 int i;
544
545 /* tell device that there's no virtqueue any longer */
546 ddi_put16(sc->sc_ioh,
547 /* LINTED E_BAD_PTR_CAST_ALIGN */
548 (uint16_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_SELECT),
549 vq->vq_index);
550 ddi_put32(sc->sc_ioh,
551 /* LINTED E_BAD_PTR_CAST_ALIGN */
552 (uint32_t *)(sc->sc_io_addr + VIRTIO_CONFIG_QUEUE_ADDRESS), 0);
553
554 /* Free the indirect descriptors, if any. */
555 for (i = 0; i < vq->vq_num; i++) {
556 struct vq_entry *entry = &vq->vq_entries[i];
557 if (entry->qe_indirect_descs)
558 virtio_free_indirect(entry);
559 }
560
561 kmem_free(vq->vq_entries, sizeof (struct vq_entry) * vq->vq_num);
562
563 (void) ddi_dma_unbind_handle(vq->vq_dma_handle);
564 ddi_dma_mem_free(&vq->vq_dma_acch);
565 ddi_dma_free_handle(&vq->vq_dma_handle);
566
567 mutex_destroy(&vq->vq_used_lock);
568 mutex_destroy(&vq->vq_avail_lock);
569 mutex_destroy(&vq->vq_freelist_lock);
570
571 kmem_free(vq, sizeof (struct virtqueue));
572 }
573
574 /*
575 * Free descriptor management.
576 */
577 struct vq_entry *
578 vq_alloc_entry(struct virtqueue *vq)
579 {
580 struct vq_entry *qe;
581
582 mutex_enter(&vq->vq_freelist_lock);
583 if (list_is_empty(&vq->vq_freelist)) {
584 mutex_exit(&vq->vq_freelist_lock);
585 return (NULL);
586 }
587 qe = list_remove_head(&vq->vq_freelist);
588
589 ASSERT(vq->vq_used_entries >= 0);
590 vq->vq_used_entries++;
591
592 mutex_exit(&vq->vq_freelist_lock);
593
594 qe->qe_next = NULL;
595 qe->qe_indirect_next = 0;
596 (void) memset(qe->qe_desc, 0, sizeof (struct vring_desc));
597
598 return (qe);
599 }
600
601 void
602 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe)
603 {
604 mutex_enter(&vq->vq_freelist_lock);
605
606 list_insert_head(&vq->vq_freelist, qe);
607 vq->vq_used_entries--;
608 ASSERT(vq->vq_used_entries >= 0);
609 mutex_exit(&vq->vq_freelist_lock);
610 }
611
612 /*
613 * We (intentionally) don't have a global vq mutex, so you are
614 * responsible for external locking to avoid allocting/freeing any
615 * entries before using the returned value. Have fun.
616 */
617 uint_t
618 vq_num_used(struct virtqueue *vq)
619 {
620 /* vq->vq_freelist_lock would not help here. */
621 return (vq->vq_used_entries);
622 }
623
624 static inline void
625 virtio_ve_set_desc(struct vring_desc *desc, uint64_t paddr, uint32_t len,
626 boolean_t write)
627 {
628 desc->addr = paddr;
629 desc->len = len;
630 desc->next = 0;
631 desc->flags = 0;
632
633 /* 'write' - from the driver's point of view */
634 if (!write)
635 desc->flags = VRING_DESC_F_WRITE;
636 }
637
638 void
639 virtio_ve_set(struct vq_entry *qe, uint64_t paddr, uint32_t len,
640 boolean_t write)
641 {
642 virtio_ve_set_desc(qe->qe_desc, paddr, len, write);
643 }
644
645 unsigned int
646 virtio_ve_indirect_available(struct vq_entry *qe)
647 {
648 return (qe->qe_queue->vq_indirect_num - qe->qe_indirect_next);
649 }
650
651 void
652 virtio_ve_add_indirect_buf(struct vq_entry *qe, uint64_t paddr, uint32_t len,
653 boolean_t write)
654 {
655 struct vring_desc *indirect_desc;
656
657 ASSERT(qe->qe_queue->vq_indirect_num);
658 ASSERT(qe->qe_indirect_next < qe->qe_queue->vq_indirect_num);
659
660 indirect_desc = &qe->qe_indirect_descs[qe->qe_indirect_next];
661 virtio_ve_set_desc(indirect_desc, paddr, len, write);
662 qe->qe_indirect_next++;
663 }
664
665 void
666 virtio_ve_add_cookie(struct vq_entry *qe, ddi_dma_handle_t dma_handle,
667 ddi_dma_cookie_t dma_cookie, unsigned int ncookies, boolean_t write)
668 {
669 int i;
670
671 for (i = 0; i < ncookies; i++) {
672 virtio_ve_add_indirect_buf(qe, dma_cookie.dmac_laddress,
673 dma_cookie.dmac_size, write);
674 ddi_dma_nextcookie(dma_handle, &dma_cookie);
675 }
676 }
677
678 void
679 virtio_sync_vq(struct virtqueue *vq)
680 {
681 struct virtio_softc *vsc = vq->vq_owner;
682
683 /* Make sure the avail ring update hit the buffer */
684 membar_producer();
685
686 vq->vq_avail->idx = vq->vq_avail_idx;
687
688 /* Make sure the avail idx update hits the buffer */
689 membar_producer();
690
691 /* Make sure we see the flags update */
692 membar_consumer();
693
694 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) {
695 ddi_put16(vsc->sc_ioh,
696 /* LINTED E_BAD_PTR_CAST_ALIGN */
697 (uint16_t *)(vsc->sc_io_addr +
698 VIRTIO_CONFIG_QUEUE_NOTIFY),
699 vq->vq_index);
700 }
701 }
702
703 void
704 virtio_push_chain(struct vq_entry *qe, boolean_t sync)
705 {
706 struct virtqueue *vq = qe->qe_queue;
707 struct vq_entry *head = qe;
708 struct vring_desc *desc;
709 int idx;
710
711 ASSERT(qe);
712
713 /*
714 * Bind the descs together, paddr and len should be already
715 * set with virtio_ve_set
716 */
717 do {
718 /* Bind the indirect descriptors */
719 if (qe->qe_indirect_next > 1) {
720 uint16_t i = 0;
721
722 /*
723 * Set the pointer/flags to the
724 * first indirect descriptor
725 */
726 virtio_ve_set_desc(qe->qe_desc,
727 qe->qe_indirect_dma_cookie.dmac_laddress,
728 sizeof (struct vring_desc) * qe->qe_indirect_next,
729 B_FALSE);
730 qe->qe_desc->flags |= VRING_DESC_F_INDIRECT;
731
732 /* For all but the last one, add the next index/flag */
733 do {
734 desc = &qe->qe_indirect_descs[i];
735 i++;
736
737 desc->flags |= VRING_DESC_F_NEXT;
738 desc->next = i;
739 } while (i < qe->qe_indirect_next - 1);
740
741 }
742
743 if (qe->qe_next) {
744 qe->qe_desc->flags |= VRING_DESC_F_NEXT;
745 qe->qe_desc->next = qe->qe_next->qe_index;
746 }
747
748 qe = qe->qe_next;
749 } while (qe);
750
751 mutex_enter(&vq->vq_avail_lock);
752 idx = vq->vq_avail_idx;
753 vq->vq_avail_idx++;
754
755 /* Make sure the bits hit the descriptor(s) */
756 membar_producer();
757 vq->vq_avail->ring[idx % vq->vq_num] = head->qe_index;
758
759 /* Notify the device, if needed. */
760 if (sync)
761 virtio_sync_vq(vq);
762
763 mutex_exit(&vq->vq_avail_lock);
764 }
765
766 /*
767 * Get a chain of descriptors from the used ring, if one is available.
768 */
769 struct vq_entry *
770 virtio_pull_chain(struct virtqueue *vq, uint32_t *len)
771 {
772 struct vq_entry *head;
773 int slot;
774 int usedidx;
775
776 mutex_enter(&vq->vq_used_lock);
777
778 /* No used entries? Bye. */
779 if (vq->vq_used_idx == vq->vq_used->idx) {
780 mutex_exit(&vq->vq_used_lock);
781 return (NULL);
782 }
783
784 usedidx = vq->vq_used_idx;
785 vq->vq_used_idx++;
786 mutex_exit(&vq->vq_used_lock);
787
788 usedidx %= vq->vq_num;
789
790 /* Make sure we do the next step _after_ checking the idx. */
791 membar_consumer();
792
793 slot = vq->vq_used->ring[usedidx].id;
794 *len = vq->vq_used->ring[usedidx].len;
795
796 head = &vq->vq_entries[slot];
797
798 return (head);
799 }
800
801 void
802 virtio_free_chain(struct vq_entry *qe)
803 {
804 struct vq_entry *tmp;
805 struct virtqueue *vq = qe->qe_queue;
806
807 ASSERT(qe);
808
809 do {
810 ASSERT(qe->qe_queue == vq);
811 tmp = qe->qe_next;
812 vq_free_entry(vq, qe);
813 qe = tmp;
814 } while (tmp != NULL);
815 }
816
817 void
818 virtio_ventry_stick(struct vq_entry *first, struct vq_entry *second)
819 {
820 first->qe_next = second;
821 }
822
823 static int
824 virtio_register_msi(struct virtio_softc *sc,
825 struct virtio_int_handler *config_handler,
826 struct virtio_int_handler vq_handlers[], int intr_types)
827 {
828 int count, actual;
829 int int_type;
830 int i;
831 int handler_count;
832 int ret;
833
834 /* If both MSI and MSI-x are reported, prefer MSI-x. */
835 int_type = DDI_INTR_TYPE_MSI;
836 if (intr_types & DDI_INTR_TYPE_MSIX)
837 int_type = DDI_INTR_TYPE_MSIX;
838
839 /* Walk the handler table to get the number of handlers. */
840 for (handler_count = 0;
841 vq_handlers && vq_handlers[handler_count].vh_func;
842 handler_count++)
843 ;
844
845 /* +1 if there is a config change handler. */
846 if (config_handler != NULL)
847 handler_count++;
848
849 /* Number of MSIs supported by the device. */
850 ret = ddi_intr_get_nintrs(sc->sc_dev, int_type, &count);
851 if (ret != DDI_SUCCESS) {
852 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_nintrs failed");
853 return (ret);
854 }
855
856 /*
857 * Those who try to register more handlers then the device
858 * supports shall suffer.
859 */
860 ASSERT(handler_count <= count);
861
862 sc->sc_intr_htable = kmem_zalloc(sizeof (ddi_intr_handle_t) *
863 handler_count, KM_SLEEP);
864
865 ret = ddi_intr_alloc(sc->sc_dev, sc->sc_intr_htable, int_type, 0,
866 handler_count, &actual, DDI_INTR_ALLOC_NORMAL);
867 if (ret != DDI_SUCCESS) {
868 dev_err(sc->sc_dev, CE_WARN, "Failed to allocate MSI: %d", ret);
869 goto out_msi_alloc;
870 }
871
872 if (actual != handler_count) {
873 dev_err(sc->sc_dev, CE_WARN,
874 "Not enough MSI available: need %d, available %d",
875 handler_count, actual);
876 goto out_msi_available;
877 }
878
879 sc->sc_intr_num = handler_count;
880 sc->sc_intr_config = B_FALSE;
881 if (config_handler != NULL) {
882 sc->sc_intr_config = B_TRUE;
883 }
884
885 /* Assume they are all same priority */
886 ret = ddi_intr_get_pri(sc->sc_intr_htable[0], &sc->sc_intr_prio);
887 if (ret != DDI_SUCCESS) {
888 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_pri failed");
889 goto out_msi_prio;
890 }
891
892 /* Add the vq handlers */
893 for (i = 0; vq_handlers[i].vh_func; i++) {
894 ret = ddi_intr_add_handler(sc->sc_intr_htable[i],
895 vq_handlers[i].vh_func, sc, vq_handlers[i].vh_priv);
896 if (ret != DDI_SUCCESS) {
897 dev_err(sc->sc_dev, CE_WARN,
898 "ddi_intr_add_handler failed");
899 /* Remove the handlers that succeeded. */
900 while (--i >= 0) {
901 (void) ddi_intr_remove_handler(
902 sc->sc_intr_htable[i]);
903 }
904 goto out_add_handlers;
905 }
906 }
907
908 /* Don't forget the config handler */
909 if (config_handler != NULL) {
910 ret = ddi_intr_add_handler(sc->sc_intr_htable[i],
911 config_handler->vh_func, sc, config_handler->vh_priv);
912 if (ret != DDI_SUCCESS) {
913 dev_err(sc->sc_dev, CE_WARN,
914 "ddi_intr_add_handler failed");
915 /* Remove the handlers that succeeded. */
916 while (--i >= 0) {
917 (void) ddi_intr_remove_handler(
918 sc->sc_intr_htable[i]);
919 }
920 goto out_add_handlers;
921 }
922 }
923
924 ret = ddi_intr_get_cap(sc->sc_intr_htable[0], &sc->sc_intr_cap);
925 if (ret == DDI_SUCCESS) {
926 sc->sc_int_type = int_type;
927 return (DDI_SUCCESS);
928 }
929
930 out_add_handlers:
931 out_msi_prio:
932 out_msi_available:
933 for (i = 0; i < actual; i++)
934 (void) ddi_intr_free(sc->sc_intr_htable[i]);
935 out_msi_alloc:
936 kmem_free(sc->sc_intr_htable,
937 sizeof (ddi_intr_handle_t) * handler_count);
938
939 return (ret);
940 }
941
942 struct virtio_handler_container {
943 int nhandlers;
944 struct virtio_int_handler config_handler;
945 struct virtio_int_handler vq_handlers[];
946 };
947
948 uint_t
949 virtio_intx_dispatch(caddr_t arg1, caddr_t arg2)
950 {
951 struct virtio_softc *sc = (void *)arg1;
952 struct virtio_handler_container *vhc = (void *)arg2;
953 uint8_t isr_status;
954 int i;
955
956 isr_status = ddi_get8(sc->sc_ioh, (uint8_t *)(sc->sc_io_addr +
957 VIRTIO_CONFIG_ISR_STATUS));
958
959 if (!isr_status)
960 return (DDI_INTR_UNCLAIMED);
961
962 if ((isr_status & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
963 vhc->config_handler.vh_func) {
964 vhc->config_handler.vh_func((void *)sc,
965 vhc->config_handler.vh_priv);
966 }
967
968 /* Notify all handlers */
969 for (i = 0; i < vhc->nhandlers; i++) {
970 vhc->vq_handlers[i].vh_func((void *)sc,
971 vhc->vq_handlers[i].vh_priv);
972 }
973
974 return (DDI_INTR_CLAIMED);
975 }
976
977 /*
978 * config_handler and vq_handlers may be allocated on stack.
979 * Take precautions not to loose them.
980 */
981 static int
982 virtio_register_intx(struct virtio_softc *sc,
983 struct virtio_int_handler *config_handler,
984 struct virtio_int_handler vq_handlers[])
985 {
986 int vq_handler_count;
987 int config_handler_count = 0;
988 int actual;
989 struct virtio_handler_container *vhc;
990 int ret = DDI_FAILURE;
991
992 /* Walk the handler table to get the number of handlers. */
993 for (vq_handler_count = 0;
994 vq_handlers && vq_handlers[vq_handler_count].vh_func;
995 vq_handler_count++)
996 ;
997
998 if (config_handler != NULL)
999 config_handler_count = 1;
1000
1001 vhc = kmem_zalloc(sizeof (struct virtio_handler_container) +
1002 sizeof (struct virtio_int_handler) * vq_handler_count, KM_SLEEP);
1003
1004 vhc->nhandlers = vq_handler_count;
1005 (void) memcpy(vhc->vq_handlers, vq_handlers,
1006 sizeof (struct virtio_int_handler) * vq_handler_count);
1007
1008 if (config_handler != NULL) {
1009 (void) memcpy(&vhc->config_handler, config_handler,
1010 sizeof (struct virtio_int_handler));
1011 }
1012
1013 /* Just a single entry for a single interrupt. */
1014 sc->sc_intr_htable = kmem_zalloc(sizeof (ddi_intr_handle_t), KM_SLEEP);
1015
1016 ret = ddi_intr_alloc(sc->sc_dev, sc->sc_intr_htable,
1017 DDI_INTR_TYPE_FIXED, 0, 1, &actual, DDI_INTR_ALLOC_NORMAL);
1018 if (ret != DDI_SUCCESS) {
1019 dev_err(sc->sc_dev, CE_WARN,
1020 "Failed to allocate a fixed interrupt: %d", ret);
1021 goto out_int_alloc;
1022 }
1023
1024 ASSERT(actual == 1);
1025 sc->sc_intr_num = 1;
1026
1027 ret = ddi_intr_get_pri(sc->sc_intr_htable[0], &sc->sc_intr_prio);
1028 if (ret != DDI_SUCCESS) {
1029 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_get_pri failed");
1030 goto out_prio;
1031 }
1032
1033 ret = ddi_intr_add_handler(sc->sc_intr_htable[0],
1034 virtio_intx_dispatch, sc, vhc);
1035 if (ret != DDI_SUCCESS) {
1036 dev_err(sc->sc_dev, CE_WARN, "ddi_intr_add_handler failed");
1037 goto out_add_handlers;
1038 }
1039
1040 sc->sc_int_type = DDI_INTR_TYPE_FIXED;
1041
1042 return (DDI_SUCCESS);
1043
1044 out_add_handlers:
1045 out_prio:
1046 (void) ddi_intr_free(sc->sc_intr_htable[0]);
1047 out_int_alloc:
1048 kmem_free(sc->sc_intr_htable, sizeof (ddi_intr_handle_t));
1049 kmem_free(vhc, sizeof (struct virtio_int_handler) *
1050 (vq_handler_count + config_handler_count));
1051 return (ret);
1052 }
1053
1054 /*
1055 * We find out if we support MSI during this, and the register layout
1056 * depends on the MSI (doh). Don't acces the device specific bits in
1057 * BAR 0 before calling it!
1058 */
1059 int
1060 virtio_register_ints(struct virtio_softc *sc,
1061 struct virtio_int_handler *config_handler,
1062 struct virtio_int_handler vq_handlers[])
1063 {
1064 int ret;
1065 int intr_types;
1066
1067 /* Default offset until MSI-X is enabled, if ever. */
1068 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX;
1069
1070 /* Determine which types of interrupts are supported */
1071 ret = ddi_intr_get_supported_types(sc->sc_dev, &intr_types);
1072 if (ret != DDI_SUCCESS) {
1073 dev_err(sc->sc_dev, CE_WARN, "Can't get supported int types");
1074 goto out_inttype;
1075 }
1076
1077 /* If we have msi, let's use them. */
1078 if (intr_types & (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) {
1079 ret = virtio_register_msi(sc, config_handler,
1080 vq_handlers, intr_types);
1081 if (!ret)
1082 return (0);
1083 }
1084
1085 /* Fall back to old-fashioned interrupts. */
1086 if (intr_types & DDI_INTR_TYPE_FIXED) {
1087 dev_debug(sc->sc_dev, CE_WARN,
1088 "Using legacy interrupts");
1089
1090 return (virtio_register_intx(sc, config_handler, vq_handlers));
1091 }
1092
1093 dev_err(sc->sc_dev, CE_WARN,
1094 "MSI failed and fixed interrupts not supported. Giving up.");
1095 ret = DDI_FAILURE;
1096
1097 out_inttype:
1098 return (ret);
1099 }
1100
1101 static int
1102 virtio_enable_msi(struct virtio_softc *sc)
1103 {
1104 int ret, i;
1105 int vq_handler_count = sc->sc_intr_num;
1106
1107 /* Number of handlers, not counting the counfig. */
1108 if (sc->sc_intr_config)
1109 vq_handler_count--;
1110
1111 /* Enable the interrupts. Either the whole block, or one by one. */
1112 if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
1113 ret = ddi_intr_block_enable(sc->sc_intr_htable,
1114 sc->sc_intr_num);
1115 if (ret != DDI_SUCCESS) {
1116 dev_err(sc->sc_dev, CE_WARN,
1117 "Failed to enable MSI, falling back to INTx");
1118 goto out_enable;
1119 }
1120 } else {
1121 for (i = 0; i < sc->sc_intr_num; i++) {
1122 ret = ddi_intr_enable(sc->sc_intr_htable[i]);
1123 if (ret != DDI_SUCCESS) {
1124 dev_err(sc->sc_dev, CE_WARN,
1125 "Failed to enable MSI %d, "
1126 "falling back to INTx", i);
1127
1128 while (--i >= 0) {
1129 (void) ddi_intr_disable(
1130 sc->sc_intr_htable[i]);
1131 }
1132 goto out_enable;
1133 }
1134 }
1135 }
1136
1137 /* Bind the allocated MSI to the queues and config */
1138 for (i = 0; i < vq_handler_count; i++) {
1139 int check;
1140
1141 ddi_put16(sc->sc_ioh,
1142 /* LINTED E_BAD_PTR_CAST_ALIGN */
1143 (uint16_t *)(sc->sc_io_addr +
1144 VIRTIO_CONFIG_QUEUE_SELECT), i);
1145
1146 ddi_put16(sc->sc_ioh,
1147 /* LINTED E_BAD_PTR_CAST_ALIGN */
1148 (uint16_t *)(sc->sc_io_addr +
1149 VIRTIO_CONFIG_QUEUE_VECTOR), i);
1150
1151 check = ddi_get16(sc->sc_ioh,
1152 /* LINTED E_BAD_PTR_CAST_ALIGN */
1153 (uint16_t *)(sc->sc_io_addr +
1154 VIRTIO_CONFIG_QUEUE_VECTOR));
1155 if (check != i) {
1156 dev_err(sc->sc_dev, CE_WARN, "Failed to bind handler "
1157 "for VQ %d, MSI %d. Check = %x", i, i, check);
1158 ret = ENODEV;
1159 goto out_bind;
1160 }
1161 }
1162
1163 if (sc->sc_intr_config) {
1164 int check;
1165
1166 ddi_put16(sc->sc_ioh,
1167 /* LINTED E_BAD_PTR_CAST_ALIGN */
1168 (uint16_t *)(sc->sc_io_addr +
1169 VIRTIO_CONFIG_CONFIG_VECTOR), i);
1170
1171 check = ddi_get16(sc->sc_ioh,
1172 /* LINTED E_BAD_PTR_CAST_ALIGN */
1173 (uint16_t *)(sc->sc_io_addr +
1174 VIRTIO_CONFIG_CONFIG_VECTOR));
1175 if (check != i) {
1176 dev_err(sc->sc_dev, CE_WARN, "Failed to bind handler "
1177 "for Config updates, MSI %d", i);
1178 ret = ENODEV;
1179 goto out_bind;
1180 }
1181 }
1182
1183 /* Configuration offset depends on whether MSI-X is used. */
1184 if (sc->sc_int_type == DDI_INTR_TYPE_MSIX)
1185 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSIX;
1186 else
1187 ASSERT(sc->sc_int_type == DDI_INTR_TYPE_MSI);
1188
1189 return (DDI_SUCCESS);
1190
1191 out_bind:
1192 /* Unbind the vqs */
1193 for (i = 0; i < vq_handler_count - 1; i++) {
1194 ddi_put16(sc->sc_ioh,
1195 /* LINTED E_BAD_PTR_CAST_ALIGN */
1196 (uint16_t *)(sc->sc_io_addr +
1197 VIRTIO_CONFIG_QUEUE_SELECT), i);
1198
1199 ddi_put16(sc->sc_ioh,
1200 /* LINTED E_BAD_PTR_CAST_ALIGN */
1201 (uint16_t *)(sc->sc_io_addr +
1202 VIRTIO_CONFIG_QUEUE_VECTOR),
1203 VIRTIO_MSI_NO_VECTOR);
1204 }
1205 /* And the config */
1206 /* LINTED E_BAD_PTR_CAST_ALIGN */
1207 ddi_put16(sc->sc_ioh, (uint16_t *)(sc->sc_io_addr +
1208 VIRTIO_CONFIG_CONFIG_VECTOR), VIRTIO_MSI_NO_VECTOR);
1209
1210 /* Disable the interrupts. Either the whole block, or one by one. */
1211 if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
1212 ret = ddi_intr_block_disable(sc->sc_intr_htable,
1213 sc->sc_intr_num);
1214 if (ret != DDI_SUCCESS) {
1215 dev_err(sc->sc_dev, CE_WARN,
1216 "Failed to disable MSIs, won't be able to "
1217 "reuse next time");
1218 }
1219 } else {
1220 for (i = 0; i < sc->sc_intr_num; i++) {
1221 ret = ddi_intr_disable(sc->sc_intr_htable[i]);
1222 if (ret != DDI_SUCCESS) {
1223 dev_err(sc->sc_dev, CE_WARN,
1224 "Failed to disable interrupt %d, "
1225 "won't be able to reuse", i);
1226 }
1227 }
1228 }
1229
1230 ret = DDI_FAILURE;
1231
1232 out_enable:
1233 return (ret);
1234 }
1235
1236 static int
1237 virtio_enable_intx(struct virtio_softc *sc)
1238 {
1239 int ret;
1240
1241 ret = ddi_intr_enable(sc->sc_intr_htable[0]);
1242 if (ret != DDI_SUCCESS) {
1243 dev_err(sc->sc_dev, CE_WARN,
1244 "Failed to enable interrupt: %d", ret);
1245 }
1246
1247 return (ret);
1248 }
1249
1250 /*
1251 * We can't enable/disable individual handlers in the INTx case so do
1252 * the whole bunch even in the msi case.
1253 */
1254 int
1255 virtio_enable_ints(struct virtio_softc *sc)
1256 {
1257
1258 ASSERT(sc->sc_config_offset == VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX);
1259
1260 /* See if we are using MSI. */
1261 if (sc->sc_int_type == DDI_INTR_TYPE_MSIX ||
1262 sc->sc_int_type == DDI_INTR_TYPE_MSI)
1263 return (virtio_enable_msi(sc));
1264
1265 ASSERT(sc->sc_int_type == DDI_INTR_TYPE_FIXED);
1266 return (virtio_enable_intx(sc));
1267 }
1268
1269 void
1270 virtio_release_ints(struct virtio_softc *sc)
1271 {
1272 int i;
1273 int ret;
1274
1275 /* We were running with MSI, unbind them. */
1276 if (sc->sc_int_type == DDI_INTR_TYPE_MSIX ||
1277 sc->sc_int_type == DDI_INTR_TYPE_MSI) {
1278 /* Unbind all vqs */
1279 for (i = 0; i < sc->sc_nvqs; i++) {
1280 ddi_put16(sc->sc_ioh,
1281 /* LINTED E_BAD_PTR_CAST_ALIGN */
1282 (uint16_t *)(sc->sc_io_addr +
1283 VIRTIO_CONFIG_QUEUE_SELECT), i);
1284
1285 ddi_put16(sc->sc_ioh,
1286 /* LINTED E_BAD_PTR_CAST_ALIGN */
1287 (uint16_t *)(sc->sc_io_addr +
1288 VIRTIO_CONFIG_QUEUE_VECTOR),
1289 VIRTIO_MSI_NO_VECTOR);
1290 }
1291 /* And the config */
1292 /* LINTED E_BAD_PTR_CAST_ALIGN */
1293 ddi_put16(sc->sc_ioh, (uint16_t *)(sc->sc_io_addr +
1294 VIRTIO_CONFIG_CONFIG_VECTOR),
1295 VIRTIO_MSI_NO_VECTOR);
1296
1297 }
1298
1299 /* Disable the interrupts. Either the whole block, or one by one. */
1300 if (sc->sc_intr_cap & DDI_INTR_FLAG_BLOCK) {
1301 ret = ddi_intr_block_disable(sc->sc_intr_htable,
1302 sc->sc_intr_num);
1303 if (ret != DDI_SUCCESS) {
1304 dev_err(sc->sc_dev, CE_WARN,
1305 "Failed to disable MSIs, won't be able to "
1306 "reuse next time");
1307 }
1308 } else {
1309 for (i = 0; i < sc->sc_intr_num; i++) {
1310 ret = ddi_intr_disable(sc->sc_intr_htable[i]);
1311 if (ret != DDI_SUCCESS) {
1312 dev_err(sc->sc_dev, CE_WARN,
1313 "Failed to disable interrupt %d, "
1314 "won't be able to reuse", i);
1315 }
1316 }
1317 }
1318
1319
1320 for (i = 0; i < sc->sc_intr_num; i++) {
1321 (void) ddi_intr_remove_handler(sc->sc_intr_htable[i]);
1322 }
1323
1324 for (i = 0; i < sc->sc_intr_num; i++)
1325 (void) ddi_intr_free(sc->sc_intr_htable[i]);
1326
1327 kmem_free(sc->sc_intr_htable, sizeof (ddi_intr_handle_t) *
1328 sc->sc_intr_num);
1329
1330 /* After disabling interrupts, the config offset is non-MSI-X. */
1331 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSIX;
1332 }
1333
1334 /*
1335 * Module linkage information for the kernel.
1336 */
1337 static struct modlmisc modlmisc = {
1338 &mod_miscops, /* Type of module */
1339 "VirtIO common library module",
1340 };
1341
1342 static struct modlinkage modlinkage = {
1343 MODREV_1,
1344 {
1345 (void *)&modlmisc,
1346 NULL
1347 }
1348 };
1349
1350 int
1351 _init(void)
1352 {
1353 return (mod_install(&modlinkage));
1354 }
1355
1356 int
1357 _fini(void)
1358 {
1359 return (mod_remove(&modlinkage));
1360 }
1361
1362 int
1363 _info(struct modinfo *modinfop)
1364 {
1365 return (mod_info(&modlinkage, modinfop));
1366 }