1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
24 * Copyright (c) 2012, Alexey Zaytsev <alexey.zaytsev@gmail.com>
25 */
26
27
28 #include <sys/modctl.h>
29 #include <sys/blkdev.h>
30 #include <sys/types.h>
31 #include <sys/errno.h>
32 #include <sys/param.h>
33 #include <sys/stropts.h>
34 #include <sys/stream.h>
35 #include <sys/strsubr.h>
36 #include <sys/kmem.h>
37 #include <sys/conf.h>
38 #include <sys/devops.h>
39 #include <sys/ksynch.h>
40 #include <sys/stat.h>
41 #include <sys/modctl.h>
42 #include <sys/debug.h>
43 #include <sys/pci.h>
44 #include <sys/sysmacros.h>
45 #include "virtiovar.h"
46 #include "virtioreg.h"
47
48 /* Feature bits */
49 #define VIRTIO_BLK_F_BARRIER (1<<0)
50 #define VIRTIO_BLK_F_SIZE_MAX (1<<1)
51 #define VIRTIO_BLK_F_SEG_MAX (1<<2)
52 #define VIRTIO_BLK_F_GEOMETRY (1<<4)
53 #define VIRTIO_BLK_F_RO (1<<5)
54 #define VIRTIO_BLK_F_BLK_SIZE (1<<6)
55 #define VIRTIO_BLK_F_SCSI (1<<7)
56 #define VIRTIO_BLK_F_FLUSH (1<<9)
57 #define VIRTIO_BLK_F_TOPOLOGY (1<<10)
58
59 /* Configuration registers */
60 #define VIRTIO_BLK_CONFIG_CAPACITY 0 /* 64bit */
61 #define VIRTIO_BLK_CONFIG_SIZE_MAX 8 /* 32bit */
62 #define VIRTIO_BLK_CONFIG_SEG_MAX 12 /* 32bit */
63 #define VIRTIO_BLK_CONFIG_GEOMETRY_C 16 /* 16bit */
64 #define VIRTIO_BLK_CONFIG_GEOMETRY_H 18 /* 8bit */
65 #define VIRTIO_BLK_CONFIG_GEOMETRY_S 19 /* 8bit */
66 #define VIRTIO_BLK_CONFIG_BLK_SIZE 20 /* 32bit */
67 #define VIRTIO_BLK_CONFIG_TOPO_PBEXP 24 /* 8bit */
68 #define VIRTIO_BLK_CONFIG_TOPO_ALIGN 25 /* 8bit */
69 #define VIRTIO_BLK_CONFIG_TOPO_MIN_SZ 26 /* 16bit */
70 #define VIRTIO_BLK_CONFIG_TOPO_OPT_SZ 28 /* 32bit */
71
72 /* Command */
73 #define VIRTIO_BLK_T_IN 0
74 #define VIRTIO_BLK_T_OUT 1
75 #define VIRTIO_BLK_T_SCSI_CMD 2
76 #define VIRTIO_BLK_T_SCSI_CMD_OUT 3
77 #define VIRTIO_BLK_T_FLUSH 4
78 #define VIRTIO_BLK_T_FLUSH_OUT 5
79 #define VIRTIO_BLK_T_GET_ID 8
80 #define VIRTIO_BLK_T_BARRIER 0x80000000
81
82 #define VIRTIO_BLK_ID_BYTES 20 /* devid */
83
84 /* Statuses */
85 #define VIRTIO_BLK_S_OK 0
86 #define VIRTIO_BLK_S_IOERR 1
87 #define VIRTIO_BLK_S_UNSUPP 2
88
89 #define DEF_MAXINDIRECT (128)
90 #define DEF_MAXSECTOR (4096)
91
92 #define VIOBLK_POISON 0xdead0001dead0001
93
94 /*
95 * Static Variables.
96 */
97 static char vioblk_ident[] = "VirtIO block driver";
98
99 /* Request header structure */
100 struct vioblk_req_hdr {
101 uint32_t type; /* VIRTIO_BLK_T_* */
102 uint32_t ioprio;
103 uint64_t sector;
104 };
105
106 struct vioblk_req {
107 struct vioblk_req_hdr hdr;
108 uint8_t status;
109 uint8_t unused[3];
110 unsigned int ndmac;
111 ddi_dma_handle_t dmah;
112 ddi_dma_handle_t bd_dmah;
113 ddi_dma_cookie_t dmac;
114 bd_xfer_t *xfer;
115 };
116
117 struct vioblk_stats {
118 struct kstat_named sts_rw_outofmemory;
119 struct kstat_named sts_rw_badoffset;
120 struct kstat_named sts_rw_queuemax;
121 struct kstat_named sts_rw_cookiesmax;
122 struct kstat_named sts_rw_cacheflush;
123 struct kstat_named sts_intr_queuemax;
124 struct kstat_named sts_intr_total;
125 struct kstat_named sts_io_errors;
126 struct kstat_named sts_unsupp_errors;
127 struct kstat_named sts_nxio_errors;
128 };
129
130 struct vioblk_lstats {
131 uint64_t rw_cacheflush;
132 uint64_t intr_total;
133 unsigned int rw_cookiesmax;
134 unsigned int intr_queuemax;
135 unsigned int io_errors;
136 unsigned int unsupp_errors;
137 unsigned int nxio_errors;
138 };
139
140 struct vioblk_softc {
141 dev_info_t *sc_dev; /* mirrors virtio_softc->sc_dev */
142 struct virtio_softc sc_virtio;
143 struct virtqueue *sc_vq;
144 bd_handle_t bd_h;
145 struct vioblk_req *sc_reqs;
146 struct vioblk_stats *ks_data;
147 kstat_t *sc_intrstat;
148 uint64_t sc_capacity;
149 uint64_t sc_nblks;
150 struct vioblk_lstats sc_stats;
151 short sc_blkflags;
152 boolean_t sc_in_poll_mode;
153 boolean_t sc_readonly;
154 int sc_blk_size;
155 int sc_pblk_size;
156 int sc_seg_max;
157 int sc_seg_size_max;
158 kmutex_t lock_devid;
159 kcondvar_t cv_devid;
160 char devid[VIRTIO_BLK_ID_BYTES + 1];
161 };
162
163 static int vioblk_get_id(struct vioblk_softc *sc);
164
165 static int vioblk_read(void *arg, bd_xfer_t *xfer);
166 static int vioblk_write(void *arg, bd_xfer_t *xfer);
167 static int vioblk_flush(void *arg, bd_xfer_t *xfer);
168 static void vioblk_driveinfo(void *arg, bd_drive_t *drive);
169 static int vioblk_mediainfo(void *arg, bd_media_t *media);
170 static int vioblk_devid_init(void *, dev_info_t *, ddi_devid_t *);
171 uint_t vioblk_int_handler(caddr_t arg1, caddr_t arg2);
172
173 static bd_ops_t vioblk_ops = {
174 BD_OPS_VERSION_0,
175 vioblk_driveinfo,
176 vioblk_mediainfo,
177 vioblk_devid_init,
178 vioblk_flush,
179 vioblk_read,
180 vioblk_write,
181 };
182
183 static int vioblk_quiesce(dev_info_t *);
184 static int vioblk_attach(dev_info_t *, ddi_attach_cmd_t);
185 static int vioblk_detach(dev_info_t *, ddi_detach_cmd_t);
186
187 static struct dev_ops vioblk_dev_ops = {
188 DEVO_REV,
189 0,
190 ddi_no_info,
191 nulldev, /* identify */
192 nulldev, /* probe */
193 vioblk_attach, /* attach */
194 vioblk_detach, /* detach */
195 nodev, /* reset */
196 NULL, /* cb_ops */
197 NULL, /* bus_ops */
198 NULL, /* power */
199 vioblk_quiesce /* quiesce */
200 };
201
202
203
204 /* Standard Module linkage initialization for a Streams driver */
205 extern struct mod_ops mod_driverops;
206
207 static struct modldrv modldrv = {
208 &mod_driverops, /* Type of module. This one is a driver */
209 vioblk_ident, /* short description */
210 &vioblk_dev_ops /* driver specific ops */
211 };
212
213 static struct modlinkage modlinkage = {
214 MODREV_1,
215 {
216 (void *)&modldrv,
217 NULL,
218 },
219 };
220
221 ddi_device_acc_attr_t vioblk_attr = {
222 DDI_DEVICE_ATTR_V0,
223 DDI_NEVERSWAP_ACC, /* virtio is always native byte order */
224 DDI_STORECACHING_OK_ACC,
225 DDI_DEFAULT_ACC
226 };
227
228 /* DMA attr for the header/status blocks. */
229 static ddi_dma_attr_t vioblk_req_dma_attr = {
230 DMA_ATTR_V0, /* dma_attr version */
231 0, /* dma_attr_addr_lo */
232 0xFFFFFFFFFFFFFFFFull, /* dma_attr_addr_hi */
233 0x00000000FFFFFFFFull, /* dma_attr_count_max */
234 1, /* dma_attr_align */
235 1, /* dma_attr_burstsizes */
236 1, /* dma_attr_minxfer */
237 0xFFFFFFFFull, /* dma_attr_maxxfer */
238 0xFFFFFFFFFFFFFFFFull, /* dma_attr_seg */
239 1, /* dma_attr_sgllen */
240 1, /* dma_attr_granular */
241 0, /* dma_attr_flags */
242 };
243
244 /* DMA attr for the data blocks. */
245 static ddi_dma_attr_t vioblk_bd_dma_attr = {
246 DMA_ATTR_V0, /* dma_attr version */
247 0, /* dma_attr_addr_lo */
248 0xFFFFFFFFFFFFFFFFull, /* dma_attr_addr_hi */
249 0x00000000FFFFFFFFull, /* dma_attr_count_max */
250 1, /* dma_attr_align */
251 1, /* dma_attr_burstsizes */
252 1, /* dma_attr_minxfer */
253 0, /* dma_attr_maxxfer, set in attach */
254 0xFFFFFFFFFFFFFFFFull, /* dma_attr_seg */
255 0, /* dma_attr_sgllen, set in attach */
256 1, /* dma_attr_granular */
257 0, /* dma_attr_flags */
258 };
259
260 static int
261 vioblk_rw(struct vioblk_softc *sc, bd_xfer_t *xfer, int type,
262 uint32_t len)
263 {
264 struct vioblk_req *req;
265 struct vq_entry *ve_hdr;
266 int total_cookies, write;
267
268 write = (type == VIRTIO_BLK_T_OUT ||
269 type == VIRTIO_BLK_T_FLUSH_OUT) ? 1 : 0;
270 total_cookies = 2;
271
272 if ((xfer->x_blkno + xfer->x_nblks) > sc->sc_nblks) {
273 sc->ks_data->sts_rw_badoffset.value.ui64++;
274 return (EINVAL);
275 }
276
277 /* allocate top entry */
278 ve_hdr = vq_alloc_entry(sc->sc_vq);
279 if (!ve_hdr) {
280 sc->ks_data->sts_rw_outofmemory.value.ui64++;
281 return (ENOMEM);
282 }
283
284 /* getting request */
285 req = &sc->sc_reqs[ve_hdr->qe_index];
286 req->hdr.type = type;
287 req->hdr.ioprio = 0;
288 req->hdr.sector = xfer->x_blkno;
289 req->xfer = xfer;
290
291 /* Header */
292 virtio_ve_add_indirect_buf(ve_hdr, req->dmac.dmac_laddress,
293 sizeof (struct vioblk_req_hdr), B_TRUE);
294
295 /* Payload */
296 if (len > 0) {
297 virtio_ve_add_cookie(ve_hdr, xfer->x_dmah, xfer->x_dmac,
298 xfer->x_ndmac, write ? B_TRUE : B_FALSE);
299 total_cookies += xfer->x_ndmac;
300 }
301
302 /* Status */
303 virtio_ve_add_indirect_buf(ve_hdr,
304 req->dmac.dmac_laddress + sizeof (struct vioblk_req_hdr),
305 sizeof (uint8_t), B_FALSE);
306
307 /* sending the whole chain to the device */
308 virtio_push_chain(ve_hdr, B_TRUE);
309
310 if (sc->sc_stats.rw_cookiesmax < total_cookies)
311 sc->sc_stats.rw_cookiesmax = total_cookies;
312
313 return (DDI_SUCCESS);
314 }
315
316 /*
317 * Now in polling mode. Interrupts are off, so we
318 * 1) poll for the already queued requests to complete.
319 * 2) push our request.
320 * 3) wait for our request to complete.
321 */
322 static int
323 vioblk_rw_poll(struct vioblk_softc *sc, bd_xfer_t *xfer,
324 int type, uint32_t len)
325 {
326 clock_t tmout;
327 int ret;
328
329 ASSERT(xfer->x_flags & BD_XFER_POLL);
330
331 /* Prevent a hard hang. */
332 tmout = drv_usectohz(30000000);
333
334 /* Poll for an empty queue */
335 while (vq_num_used(sc->sc_vq)) {
336 /* Check if any pending requests completed. */
337 ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL);
338 if (ret != DDI_INTR_CLAIMED) {
339 drv_usecwait(10);
340 tmout -= 10;
341 return (ETIMEDOUT);
342 }
343 }
344
345 ret = vioblk_rw(sc, xfer, type, len);
346 if (ret)
347 return (ret);
348
349 tmout = drv_usectohz(30000000);
350 /* Poll for an empty queue again. */
351 while (vq_num_used(sc->sc_vq)) {
352 /* Check if any pending requests completed. */
353 ret = vioblk_int_handler((caddr_t)&sc->sc_virtio, NULL);
354 if (ret != DDI_INTR_CLAIMED) {
355 drv_usecwait(10);
356 tmout -= 10;
357 return (ETIMEDOUT);
358 }
359 }
360
361 return (DDI_SUCCESS);
362 }
363
364 static int
365 vioblk_read(void *arg, bd_xfer_t *xfer)
366 {
367 int ret;
368 struct vioblk_softc *sc = (void *)arg;
369
370 if (xfer->x_flags & BD_XFER_POLL) {
371 if (!sc->sc_in_poll_mode) {
372 virtio_stop_vq_intr(sc->sc_vq);
373 sc->sc_in_poll_mode = 1;
374 }
375
376 ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_IN,
377 xfer->x_nblks * DEV_BSIZE);
378 } else {
379 if (sc->sc_in_poll_mode) {
380 virtio_start_vq_intr(sc->sc_vq);
381 sc->sc_in_poll_mode = 0;
382 }
383
384 ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_IN,
385 xfer->x_nblks * DEV_BSIZE);
386 }
387
388 return (ret);
389 }
390
391 static int
392 vioblk_write(void *arg, bd_xfer_t *xfer)
393 {
394 int ret;
395 struct vioblk_softc *sc = (void *)arg;
396
397 if (xfer->x_flags & BD_XFER_POLL) {
398 if (!sc->sc_in_poll_mode) {
399 virtio_stop_vq_intr(sc->sc_vq);
400 sc->sc_in_poll_mode = 1;
401 }
402
403 ret = vioblk_rw_poll(sc, xfer, VIRTIO_BLK_T_OUT,
404 xfer->x_nblks * DEV_BSIZE);
405 } else {
406 if (sc->sc_in_poll_mode) {
407 virtio_start_vq_intr(sc->sc_vq);
408 sc->sc_in_poll_mode = 0;
409 }
410
411 ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_OUT,
412 xfer->x_nblks * DEV_BSIZE);
413 }
414 return (ret);
415 }
416
417 static int
418 vioblk_flush(void *arg, bd_xfer_t *xfer)
419 {
420 int ret;
421 struct vioblk_softc *sc = (void *)arg;
422
423 ASSERT((xfer->x_flags & BD_XFER_POLL) == 0);
424
425 ret = vioblk_rw(sc, xfer, VIRTIO_BLK_T_FLUSH_OUT,
426 xfer->x_nblks * DEV_BSIZE);
427
428 if (!ret)
429 sc->sc_stats.rw_cacheflush++;
430
431 return (ret);
432 }
433
434
435 static void
436 vioblk_driveinfo(void *arg, bd_drive_t *drive)
437 {
438 struct vioblk_softc *sc = (void *)arg;
439
440 drive->d_qsize = sc->sc_vq->vq_num;
441 drive->d_removable = B_FALSE;
442 drive->d_hotpluggable = B_TRUE;
443 drive->d_target = 0;
444 drive->d_lun = 0;
445
446 drive->d_vendor = "Virtio";
447 drive->d_vendor_len = strlen(drive->d_vendor);
448
449 drive->d_product = "Block Device";
450 drive->d_product_len = strlen(drive->d_product);
451
452 (void) vioblk_get_id(sc);
453 drive->d_serial = sc->devid;
454 drive->d_serial_len = strlen(drive->d_serial);
455
456 drive->d_revision = "0000";
457 drive->d_revision_len = strlen(drive->d_revision);
458 }
459
460 static int
461 vioblk_mediainfo(void *arg, bd_media_t *media)
462 {
463 struct vioblk_softc *sc = (void *)arg;
464
465 media->m_nblks = sc->sc_nblks;
466 media->m_blksize = sc->sc_blk_size;
467 media->m_readonly = sc->sc_readonly;
468 media->m_pblksize = sc->sc_pblk_size;
469 return (0);
470 }
471
472 static int
473 vioblk_get_id(struct vioblk_softc *sc)
474 {
475 clock_t deadline;
476 int ret;
477 bd_xfer_t xfer;
478
479 deadline = ddi_get_lbolt() + (clock_t)drv_usectohz(3 * 1000000);
480 (void) memset(&xfer, 0, sizeof (bd_xfer_t));
481 xfer.x_nblks = 1;
482
483 ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_bd_dma_attr,
484 DDI_DMA_SLEEP, NULL, &xfer.x_dmah);
485 if (ret != DDI_SUCCESS)
486 goto out_alloc;
487
488 ret = ddi_dma_addr_bind_handle(xfer.x_dmah, NULL, (caddr_t)&sc->devid,
489 VIRTIO_BLK_ID_BYTES, DDI_DMA_READ | DDI_DMA_CONSISTENT,
490 DDI_DMA_SLEEP, NULL, &xfer.x_dmac, &xfer.x_ndmac);
491 if (ret != DDI_DMA_MAPPED) {
492 ret = DDI_FAILURE;
493 goto out_map;
494 }
495
496 mutex_enter(&sc->lock_devid);
497
498 ret = vioblk_rw(sc, &xfer, VIRTIO_BLK_T_GET_ID,
499 VIRTIO_BLK_ID_BYTES);
500 if (ret) {
501 mutex_exit(&sc->lock_devid);
502 goto out_rw;
503 }
504
505 /* wait for reply */
506 ret = cv_timedwait(&sc->cv_devid, &sc->lock_devid, deadline);
507 mutex_exit(&sc->lock_devid);
508
509 (void) ddi_dma_unbind_handle(xfer.x_dmah);
510 ddi_dma_free_handle(&xfer.x_dmah);
511
512 /* timeout */
513 if (ret < 0) {
514 dev_err(sc->sc_dev, CE_WARN,
515 "Cannot get devid from the device");
516 return (DDI_FAILURE);
517 }
518
519 return (0);
520
521 out_rw:
522 (void) ddi_dma_unbind_handle(xfer.x_dmah);
523 out_map:
524 ddi_dma_free_handle(&xfer.x_dmah);
525 out_alloc:
526 return (ret);
527 }
528
529 static int
530 vioblk_devid_init(void *arg, dev_info_t *devinfo, ddi_devid_t *devid)
531 {
532 struct vioblk_softc *sc = (void *)arg;
533 int ret;
534
535 ret = vioblk_get_id(sc);
536 if (ret != DDI_SUCCESS)
537 return (ret);
538
539 ret = ddi_devid_init(devinfo, DEVID_ATA_SERIAL,
540 VIRTIO_BLK_ID_BYTES, sc->devid, devid);
541 if (ret != DDI_SUCCESS) {
542 dev_err(devinfo, CE_WARN, "Cannot build devid from the device");
543 return (ret);
544 }
545
546 dev_debug(sc->sc_dev, CE_NOTE,
547 "devid %x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x%x",
548 sc->devid[0], sc->devid[1], sc->devid[2], sc->devid[3],
549 sc->devid[4], sc->devid[5], sc->devid[6], sc->devid[7],
550 sc->devid[8], sc->devid[9], sc->devid[10], sc->devid[11],
551 sc->devid[12], sc->devid[13], sc->devid[14], sc->devid[15],
552 sc->devid[16], sc->devid[17], sc->devid[18], sc->devid[19]);
553
554 return (0);
555 }
556
557 static void
558 vioblk_show_features(struct vioblk_softc *sc, const char *prefix,
559 uint32_t features)
560 {
561 char buf[512];
562 char *bufp = buf;
563 char *bufend = buf + sizeof (buf);
564
565 /* LINTED E_PTRDIFF_OVERFLOW */
566 bufp += snprintf(bufp, bufend - bufp, prefix);
567
568 /* LINTED E_PTRDIFF_OVERFLOW */
569 bufp += virtio_show_features(features, bufp, bufend - bufp);
570
571
572 /* LINTED E_PTRDIFF_OVERFLOW */
573 bufp += snprintf(bufp, bufend - bufp, "Vioblk ( ");
574
575 if (features & VIRTIO_BLK_F_BARRIER)
576 /* LINTED E_PTRDIFF_OVERFLOW */
577 bufp += snprintf(bufp, bufend - bufp, "BARRIER ");
578 if (features & VIRTIO_BLK_F_SIZE_MAX)
579 /* LINTED E_PTRDIFF_OVERFLOW */
580 bufp += snprintf(bufp, bufend - bufp, "SIZE_MAX ");
581 if (features & VIRTIO_BLK_F_SEG_MAX)
582 /* LINTED E_PTRDIFF_OVERFLOW */
583 bufp += snprintf(bufp, bufend - bufp, "SEG_MAX ");
584 if (features & VIRTIO_BLK_F_GEOMETRY)
585 /* LINTED E_PTRDIFF_OVERFLOW */
586 bufp += snprintf(bufp, bufend - bufp, "GEOMETRY ");
587 if (features & VIRTIO_BLK_F_RO)
588 /* LINTED E_PTRDIFF_OVERFLOW */
589 bufp += snprintf(bufp, bufend - bufp, "RO ");
590 if (features & VIRTIO_BLK_F_BLK_SIZE)
591 /* LINTED E_PTRDIFF_OVERFLOW */
592 bufp += snprintf(bufp, bufend - bufp, "BLK_SIZE ");
593 if (features & VIRTIO_BLK_F_SCSI)
594 /* LINTED E_PTRDIFF_OVERFLOW */
595 bufp += snprintf(bufp, bufend - bufp, "SCSI ");
596 if (features & VIRTIO_BLK_F_FLUSH)
597 /* LINTED E_PTRDIFF_OVERFLOW */
598 bufp += snprintf(bufp, bufend - bufp, "FLUSH ");
599 if (features & VIRTIO_BLK_F_TOPOLOGY)
600 /* LINTED E_PTRDIFF_OVERFLOW */
601 bufp += snprintf(bufp, bufend - bufp, "TOPOLOGY ");
602
603 /* LINTED E_PTRDIFF_OVERFLOW */
604 bufp += snprintf(bufp, bufend - bufp, ")");
605 *bufp = '\0';
606
607 dev_debug(sc->sc_dev, CE_NOTE, "%s", buf);
608 }
609
610 static int
611 vioblk_dev_features(struct vioblk_softc *sc)
612 {
613 uint32_t host_features;
614
615 host_features = virtio_negotiate_features(&sc->sc_virtio,
616 VIRTIO_BLK_F_RO |
617 VIRTIO_BLK_F_GEOMETRY |
618 VIRTIO_BLK_F_BLK_SIZE |
619 VIRTIO_BLK_F_FLUSH |
620 VIRTIO_BLK_F_TOPOLOGY |
621 VIRTIO_BLK_F_SEG_MAX |
622 VIRTIO_BLK_F_SIZE_MAX |
623 VIRTIO_F_RING_INDIRECT_DESC);
624
625 vioblk_show_features(sc, "Host features: ", host_features);
626 vioblk_show_features(sc, "Negotiated features: ",
627 sc->sc_virtio.sc_features);
628
629 if (!(sc->sc_virtio.sc_features & VIRTIO_F_RING_INDIRECT_DESC)) {
630 dev_err(sc->sc_dev, CE_NOTE,
631 "Host does not support RING_INDIRECT_DESC, bye.");
632 return (DDI_FAILURE);
633 }
634
635 return (DDI_SUCCESS);
636 }
637
638 /* ARGSUSED */
639 uint_t
640 vioblk_int_handler(caddr_t arg1, caddr_t arg2)
641 {
642 struct virtio_softc *vsc = (void *)arg1;
643 struct vioblk_softc *sc = container_of(vsc,
644 struct vioblk_softc, sc_virtio);
645 struct vq_entry *ve;
646 uint32_t len;
647 int i = 0, error;
648
649 while ((ve = virtio_pull_chain(sc->sc_vq, &len))) {
650 struct vioblk_req *req = &sc->sc_reqs[ve->qe_index];
651 bd_xfer_t *xfer = req->xfer;
652 uint8_t status = req->status;
653 uint32_t type = req->hdr.type;
654
655 if (req->xfer == (void *)VIOBLK_POISON) {
656 dev_err(sc->sc_dev, CE_WARN, "Poisoned descriptor!");
657 virtio_free_chain(ve);
658 return (DDI_INTR_CLAIMED);
659 }
660
661 req->xfer = (void *) VIOBLK_POISON;
662
663 /* Note: blkdev tears down the payload mapping for us. */
664 virtio_free_chain(ve);
665
666 /* returning payload back to blkdev */
667 switch (status) {
668 case VIRTIO_BLK_S_OK:
669 error = 0;
670 break;
671 case VIRTIO_BLK_S_IOERR:
672 error = EIO;
673 sc->sc_stats.io_errors++;
674 break;
675 case VIRTIO_BLK_S_UNSUPP:
676 sc->sc_stats.unsupp_errors++;
677 error = ENOTTY;
678 break;
679 default:
680 sc->sc_stats.nxio_errors++;
681 error = ENXIO;
682 break;
683 }
684
685 if (type == VIRTIO_BLK_T_GET_ID) {
686 /* notify devid_init */
687 mutex_enter(&sc->lock_devid);
688 cv_broadcast(&sc->cv_devid);
689 mutex_exit(&sc->lock_devid);
690 } else
691 bd_xfer_done(xfer, error);
692
693 i++;
694 }
695
696 /* update stats */
697 if (sc->sc_stats.intr_queuemax < i)
698 sc->sc_stats.intr_queuemax = i;
699 sc->sc_stats.intr_total++;
700
701 return (DDI_INTR_CLAIMED);
702 }
703
704 /* ARGSUSED */
705 uint_t
706 vioblk_config_handler(caddr_t arg1, caddr_t arg2)
707 {
708 return (DDI_INTR_CLAIMED);
709 }
710
711 static int
712 vioblk_register_ints(struct vioblk_softc *sc)
713 {
714 int ret;
715
716 struct virtio_int_handler vioblk_conf_h = {
717 vioblk_config_handler
718 };
719
720 struct virtio_int_handler vioblk_vq_h[] = {
721 { vioblk_int_handler },
722 { NULL },
723 };
724
725 ret = virtio_register_ints(&sc->sc_virtio,
726 &vioblk_conf_h, vioblk_vq_h);
727
728 return (ret);
729 }
730
731 static void
732 vioblk_free_reqs(struct vioblk_softc *sc)
733 {
734 int i, qsize;
735
736 qsize = sc->sc_vq->vq_num;
737
738 for (i = 0; i < qsize; i++) {
739 struct vioblk_req *req = &sc->sc_reqs[i];
740
741 if (req->ndmac)
742 (void) ddi_dma_unbind_handle(req->dmah);
743
744 if (req->dmah)
745 ddi_dma_free_handle(&req->dmah);
746 }
747
748 kmem_free(sc->sc_reqs, sizeof (struct vioblk_req) * qsize);
749 }
750
751 static int
752 vioblk_alloc_reqs(struct vioblk_softc *sc)
753 {
754 int i, qsize;
755 int ret;
756
757 qsize = sc->sc_vq->vq_num;
758
759 sc->sc_reqs = kmem_zalloc(sizeof (struct vioblk_req) * qsize, KM_SLEEP);
760
761 for (i = 0; i < qsize; i++) {
762 struct vioblk_req *req = &sc->sc_reqs[i];
763
764 ret = ddi_dma_alloc_handle(sc->sc_dev, &vioblk_req_dma_attr,
765 DDI_DMA_SLEEP, NULL, &req->dmah);
766 if (ret != DDI_SUCCESS) {
767
768 dev_err(sc->sc_dev, CE_WARN,
769 "Can't allocate dma handle for req "
770 "buffer %d", i);
771 goto exit;
772 }
773
774 ret = ddi_dma_addr_bind_handle(req->dmah, NULL,
775 (caddr_t)&req->hdr,
776 sizeof (struct vioblk_req_hdr) + sizeof (uint8_t),
777 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
778 NULL, &req->dmac, &req->ndmac);
779 if (ret != DDI_DMA_MAPPED) {
780 dev_err(sc->sc_dev, CE_WARN,
781 "Can't bind req buffer %d", i);
782 goto exit;
783 }
784 }
785
786 return (0);
787
788 exit:
789 vioblk_free_reqs(sc);
790 return (ENOMEM);
791 }
792
793
794 static int
795 vioblk_ksupdate(kstat_t *ksp, int rw)
796 {
797 struct vioblk_softc *sc = ksp->ks_private;
798
799 if (rw == KSTAT_WRITE)
800 return (EACCES);
801
802 sc->ks_data->sts_rw_cookiesmax.value.ui32 = sc->sc_stats.rw_cookiesmax;
803 sc->ks_data->sts_intr_queuemax.value.ui32 = sc->sc_stats.intr_queuemax;
804 sc->ks_data->sts_unsupp_errors.value.ui32 = sc->sc_stats.unsupp_errors;
805 sc->ks_data->sts_nxio_errors.value.ui32 = sc->sc_stats.nxio_errors;
806 sc->ks_data->sts_io_errors.value.ui32 = sc->sc_stats.io_errors;
807 sc->ks_data->sts_rw_cacheflush.value.ui64 = sc->sc_stats.rw_cacheflush;
808 sc->ks_data->sts_intr_total.value.ui64 = sc->sc_stats.intr_total;
809
810
811 return (0);
812 }
813
814 static int
815 vioblk_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
816 {
817 int ret = DDI_SUCCESS;
818 int instance;
819 struct vioblk_softc *sc;
820 struct virtio_softc *vsc;
821 struct vioblk_stats *ks_data;
822
823 instance = ddi_get_instance(devinfo);
824
825 switch (cmd) {
826 case DDI_ATTACH:
827 break;
828
829 case DDI_RESUME:
830 case DDI_PM_RESUME:
831 dev_err(devinfo, CE_WARN, "resume not supported yet");
832 ret = DDI_FAILURE;
833 goto exit;
834
835 default:
836 dev_err(devinfo, CE_WARN, "cmd 0x%x not recognized", cmd);
837 ret = DDI_FAILURE;
838 goto exit;
839 }
840
841 sc = kmem_zalloc(sizeof (struct vioblk_softc), KM_SLEEP);
842 ddi_set_driver_private(devinfo, sc);
843
844 vsc = &sc->sc_virtio;
845
846 /* Duplicate for faster access / less typing */
847 sc->sc_dev = devinfo;
848 vsc->sc_dev = devinfo;
849
850 cv_init(&sc->cv_devid, NULL, CV_DRIVER, NULL);
851 mutex_init(&sc->lock_devid, NULL, MUTEX_DRIVER, NULL);
852
853 /*
854 * Initialize interrupt kstat. This should not normally fail, since
855 * we don't use a persistent stat. We do it this way to avoid having
856 * to test for it at run time on the hot path.
857 */
858 sc->sc_intrstat = kstat_create("vioblk", instance,
859 "intrs", "controller", KSTAT_TYPE_NAMED,
860 sizeof (struct vioblk_stats) / sizeof (kstat_named_t),
861 KSTAT_FLAG_PERSISTENT);
862 if (sc->sc_intrstat == NULL) {
863 dev_err(devinfo, CE_WARN, "kstat_create failed");
864 goto exit_intrstat;
865 }
866 ks_data = (struct vioblk_stats *)sc->sc_intrstat->ks_data;
867 kstat_named_init(&ks_data->sts_rw_outofmemory,
868 "total_rw_outofmemory", KSTAT_DATA_UINT64);
869 kstat_named_init(&ks_data->sts_rw_badoffset,
870 "total_rw_badoffset", KSTAT_DATA_UINT64);
871 kstat_named_init(&ks_data->sts_intr_total,
872 "total_intr", KSTAT_DATA_UINT64);
873 kstat_named_init(&ks_data->sts_io_errors,
874 "total_io_errors", KSTAT_DATA_UINT32);
875 kstat_named_init(&ks_data->sts_unsupp_errors,
876 "total_unsupp_errors", KSTAT_DATA_UINT32);
877 kstat_named_init(&ks_data->sts_nxio_errors,
878 "total_nxio_errors", KSTAT_DATA_UINT32);
879 kstat_named_init(&ks_data->sts_rw_cacheflush,
880 "total_rw_cacheflush", KSTAT_DATA_UINT64);
881 kstat_named_init(&ks_data->sts_rw_cookiesmax,
882 "max_rw_cookies", KSTAT_DATA_UINT32);
883 kstat_named_init(&ks_data->sts_intr_queuemax,
884 "max_intr_queue", KSTAT_DATA_UINT32);
885 sc->ks_data = ks_data;
886 sc->sc_intrstat->ks_private = sc;
887 sc->sc_intrstat->ks_update = vioblk_ksupdate;
888 kstat_install(sc->sc_intrstat);
889
890 /* map BAR0 */
891 ret = ddi_regs_map_setup(devinfo, 1,
892 (caddr_t *)&sc->sc_virtio.sc_io_addr,
893 0, 0, &vioblk_attr, &sc->sc_virtio.sc_ioh);
894 if (ret != DDI_SUCCESS) {
895 dev_err(devinfo, CE_WARN, "unable to map bar0: [%d]", ret);
896 goto exit_map;
897 }
898
899 virtio_device_reset(&sc->sc_virtio);
900 virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
901 virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
902
903 if (vioblk_register_ints(sc)) {
904 dev_err(devinfo, CE_WARN, "Unable to add interrupt");
905 goto exit_int;
906 }
907
908 ret = vioblk_dev_features(sc);
909 if (ret)
910 goto exit_features;
911
912 if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_RO)
913 sc->sc_readonly = B_TRUE;
914 else
915 sc->sc_readonly = B_FALSE;
916
917 sc->sc_capacity = virtio_read_device_config_8(&sc->sc_virtio,
918 VIRTIO_BLK_CONFIG_CAPACITY);
919 sc->sc_nblks = sc->sc_capacity;
920
921 sc->sc_blk_size = DEV_BSIZE;
922 if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_BLK_SIZE) {
923 sc->sc_blk_size = virtio_read_device_config_4(&sc->sc_virtio,
924 VIRTIO_BLK_CONFIG_BLK_SIZE);
925 }
926
927 sc->sc_pblk_size = sc->sc_blk_size;
928 if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_TOPOLOGY) {
929 sc->sc_pblk_size <<= virtio_read_device_config_1(&sc->sc_virtio,
930 VIRTIO_BLK_CONFIG_TOPO_PBEXP);
931 }
932
933 /* Flushing is not supported. */
934 if (!(sc->sc_virtio.sc_features & VIRTIO_BLK_F_FLUSH)) {
935 vioblk_ops.o_sync_cache = NULL;
936 }
937
938 sc->sc_seg_max = DEF_MAXINDIRECT;
939 /* The max number of segments (cookies) in a request */
940 if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SEG_MAX) {
941 sc->sc_seg_max = virtio_read_device_config_4(&sc->sc_virtio,
942 VIRTIO_BLK_CONFIG_SEG_MAX);
943
944 /* That's what Linux does. */
945 if (!sc->sc_seg_max)
946 sc->sc_seg_max = 1;
947
948 /*
949 * SEG_MAX corresponds to the number of _data_
950 * blocks in a request
951 */
952 sc->sc_seg_max += 2;
953 }
954 /* 2 descriptors taken for header/status */
955 vioblk_bd_dma_attr.dma_attr_sgllen = sc->sc_seg_max - 2;
956
957
958 /* The maximum size for a cookie in a request. */
959 sc->sc_seg_size_max = DEF_MAXSECTOR;
960 if (sc->sc_virtio.sc_features & VIRTIO_BLK_F_SIZE_MAX) {
961 sc->sc_seg_size_max = virtio_read_device_config_4(
962 &sc->sc_virtio, VIRTIO_BLK_CONFIG_SIZE_MAX);
963 }
964
965 /* The maximum request size */
966 vioblk_bd_dma_attr.dma_attr_maxxfer =
967 vioblk_bd_dma_attr.dma_attr_sgllen * sc->sc_seg_size_max;
968
969 dev_debug(devinfo, CE_NOTE,
970 "nblks=%" PRIu64 " blksize=%d (%d) num_seg=%d, "
971 "seg_size=%d, maxxfer=%" PRIu64,
972 sc->sc_nblks, sc->sc_blk_size, sc->sc_pblk_size,
973 vioblk_bd_dma_attr.dma_attr_sgllen,
974 sc->sc_seg_size_max,
975 vioblk_bd_dma_attr.dma_attr_maxxfer);
976
977
978 sc->sc_vq = virtio_alloc_vq(&sc->sc_virtio, 0, 0,
979 sc->sc_seg_max, "I/O request");
980 if (sc->sc_vq == NULL) {
981 goto exit_alloc1;
982 }
983
984 ret = vioblk_alloc_reqs(sc);
985 if (ret) {
986 goto exit_alloc2;
987 }
988
989 sc->bd_h = bd_alloc_handle(sc, &vioblk_ops, &vioblk_bd_dma_attr,
990 KM_SLEEP);
991
992
993 virtio_set_status(&sc->sc_virtio,
994 VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
995 virtio_start_vq_intr(sc->sc_vq);
996
997 ret = virtio_enable_ints(&sc->sc_virtio);
998 if (ret)
999 goto exit_enable_ints;
1000
1001 ret = bd_attach_handle(devinfo, sc->bd_h);
1002 if (ret != DDI_SUCCESS) {
1003 dev_err(devinfo, CE_WARN, "Failed to attach blkdev");
1004 goto exit_attach_bd;
1005 }
1006
1007 return (DDI_SUCCESS);
1008
1009 exit_attach_bd:
1010 /*
1011 * There is no virtio_disable_ints(), it's done in virtio_release_ints.
1012 * If they ever get split, don't forget to add a call here.
1013 */
1014 exit_enable_ints:
1015 virtio_stop_vq_intr(sc->sc_vq);
1016 bd_free_handle(sc->bd_h);
1017 vioblk_free_reqs(sc);
1018 exit_alloc2:
1019 virtio_free_vq(sc->sc_vq);
1020 exit_alloc1:
1021 exit_features:
1022 virtio_release_ints(&sc->sc_virtio);
1023 exit_int:
1024 virtio_set_status(&sc->sc_virtio, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
1025 ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
1026 exit_map:
1027 kstat_delete(sc->sc_intrstat);
1028 exit_intrstat:
1029 mutex_destroy(&sc->lock_devid);
1030 cv_destroy(&sc->cv_devid);
1031 kmem_free(sc, sizeof (struct vioblk_softc));
1032 exit:
1033 return (ret);
1034 }
1035
1036 static int
1037 vioblk_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
1038 {
1039 struct vioblk_softc *sc = ddi_get_driver_private(devinfo);
1040
1041 switch (cmd) {
1042 case DDI_DETACH:
1043 break;
1044
1045 case DDI_PM_SUSPEND:
1046 cmn_err(CE_WARN, "suspend not supported yet");
1047 return (DDI_FAILURE);
1048
1049 default:
1050 cmn_err(CE_WARN, "cmd 0x%x unrecognized", cmd);
1051 return (DDI_FAILURE);
1052 }
1053
1054 (void) bd_detach_handle(sc->bd_h);
1055 virtio_stop_vq_intr(sc->sc_vq);
1056 virtio_release_ints(&sc->sc_virtio);
1057 vioblk_free_reqs(sc);
1058 virtio_free_vq(sc->sc_vq);
1059 virtio_device_reset(&sc->sc_virtio);
1060 ddi_regs_map_free(&sc->sc_virtio.sc_ioh);
1061 kstat_delete(sc->sc_intrstat);
1062 kmem_free(sc, sizeof (struct vioblk_softc));
1063
1064 return (DDI_SUCCESS);
1065 }
1066
1067 static int
1068 vioblk_quiesce(dev_info_t *devinfo)
1069 {
1070 struct vioblk_softc *sc = ddi_get_driver_private(devinfo);
1071
1072 virtio_stop_vq_intr(sc->sc_vq);
1073 virtio_device_reset(&sc->sc_virtio);
1074
1075 return (DDI_SUCCESS);
1076 }
1077
1078 int
1079 _init(void)
1080 {
1081 int rv;
1082
1083 bd_mod_init(&vioblk_dev_ops);
1084
1085 if ((rv = mod_install(&modlinkage)) != 0) {
1086 bd_mod_fini(&vioblk_dev_ops);
1087 }
1088
1089 return (rv);
1090 }
1091
1092 int
1093 _fini(void)
1094 {
1095 int rv;
1096
1097 if ((rv = mod_remove(&modlinkage)) == 0) {
1098 bd_mod_fini(&vioblk_dev_ops);
1099 }
1100
1101 return (rv);
1102 }
1103
1104 int
1105 _info(struct modinfo *modinfop)
1106 {
1107 return (mod_info(&modlinkage, modinfop));
1108 }