1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2012 Garrett D'Amore <garrett@damore.org>
25 * Copyright 2014 Pluribus Networks, Inc.
26 * Copyright 2016 Nexenta Systems, Inc.
27 */
28
29 /*
30 * PC specific DDI implementation
31 */
32 #include <sys/types.h>
33 #include <sys/autoconf.h>
34 #include <sys/avintr.h>
35 #include <sys/bootconf.h>
36 #include <sys/conf.h>
37 #include <sys/cpuvar.h>
38 #include <sys/ddi_impldefs.h>
39 #include <sys/ddi_subrdefs.h>
40 #include <sys/ethernet.h>
41 #include <sys/fp.h>
42 #include <sys/instance.h>
43 #include <sys/kmem.h>
44 #include <sys/machsystm.h>
45 #include <sys/modctl.h>
46 #include <sys/promif.h>
47 #include <sys/prom_plat.h>
48 #include <sys/sunndi.h>
49 #include <sys/ndi_impldefs.h>
50 #include <sys/ddi_impldefs.h>
51 #include <sys/sysmacros.h>
52 #include <sys/systeminfo.h>
53 #include <sys/utsname.h>
54 #include <sys/atomic.h>
55 #include <sys/spl.h>
56 #include <sys/archsystm.h>
57 #include <vm/seg_kmem.h>
58 #include <sys/ontrap.h>
59 #include <sys/fm/protocol.h>
60 #include <sys/ramdisk.h>
61 #include <sys/sunndi.h>
62 #include <sys/vmem.h>
63 #include <sys/pci_impl.h>
64 #if defined(__xpv)
65 #include <sys/hypervisor.h>
66 #endif
67 #include <sys/mach_intr.h>
68 #include <vm/hat_i86.h>
69 #include <sys/x86_archext.h>
70 #include <sys/avl.h>
71
72 /*
73 * DDI Boot Configuration
74 */
75
76 /*
77 * Platform drivers on this platform
78 */
79 char *platform_module_list[] = {
80 "acpippm",
81 "ppm",
82 (char *)0
83 };
84
85 /* pci bus resource maps */
86 struct pci_bus_resource *pci_bus_res;
87
88 size_t dma_max_copybuf_size = 0x101000; /* 1M + 4K */
89
90 uint64_t ramdisk_start, ramdisk_end;
91
92 int pseudo_isa = 0;
93
94 /*
95 * Forward declarations
96 */
97 static int getlongprop_buf();
98 static void get_boot_properties(void);
99 static void impl_bus_initialprobe(void);
100 static void impl_bus_reprobe(void);
101
102 static int poke_mem(peekpoke_ctlops_t *in_args);
103 static int peek_mem(peekpoke_ctlops_t *in_args);
104
105 static int kmem_override_cache_attrs(caddr_t, size_t, uint_t);
106
107 #if defined(__amd64) && !defined(__xpv)
108 extern void immu_init(void);
109 #endif
110
111 /*
112 * We use an AVL tree to store contiguous address allocations made with the
113 * kalloca() routine, so that we can return the size to free with kfreea().
114 * Note that in the future it would be vastly faster if we could eliminate
115 * this lookup by insisting that all callers keep track of their own sizes,
116 * just as for kmem_alloc().
117 */
118 struct ctgas {
119 avl_node_t ctg_link;
120 void *ctg_addr;
121 size_t ctg_size;
122 };
123
124 static avl_tree_t ctgtree;
125
126 static kmutex_t ctgmutex;
127 #define CTGLOCK() mutex_enter(&ctgmutex)
128 #define CTGUNLOCK() mutex_exit(&ctgmutex)
129
130 /*
131 * Minimum pfn value of page_t's put on the free list. This is to simplify
132 * support of ddi dma memory requests which specify small, non-zero addr_lo
133 * values.
134 *
135 * The default value of 2, which corresponds to the only known non-zero addr_lo
136 * value used, means a single page will be sacrificed (pfn typically starts
137 * at 1). ddiphysmin can be set to 0 to disable. It cannot be set above 0x100
138 * otherwise mp startup panics.
139 */
140 pfn_t ddiphysmin = 2;
141
142 static void
143 check_driver_disable(void)
144 {
145 int proplen = 128;
146 char *prop_name;
147 char *drv_name, *propval;
148 major_t major;
149
150 prop_name = kmem_alloc(proplen, KM_SLEEP);
151 for (major = 0; major < devcnt; major++) {
152 drv_name = ddi_major_to_name(major);
153 if (drv_name == NULL)
154 continue;
155 (void) snprintf(prop_name, proplen, "disable-%s", drv_name);
156 if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
157 DDI_PROP_DONTPASS, prop_name, &propval) == DDI_SUCCESS) {
158 if (strcmp(propval, "true") == 0) {
159 devnamesp[major].dn_flags |= DN_DRIVER_REMOVED;
160 cmn_err(CE_NOTE, "driver %s disabled",
161 drv_name);
162 }
163 ddi_prop_free(propval);
164 }
165 }
166 kmem_free(prop_name, proplen);
167 }
168
169
170 /*
171 * Configure the hardware on the system.
172 * Called before the rootfs is mounted
173 */
174 void
175 configure(void)
176 {
177 extern void i_ddi_init_root();
178
179 #if defined(__i386)
180 extern int fpu_pentium_fdivbug;
181 #endif /* __i386 */
182 extern int fpu_ignored;
183
184 /*
185 * Determine if an FPU is attached
186 */
187
188 fpu_probe();
189
190 #if defined(__i386)
191 if (fpu_pentium_fdivbug) {
192 printf("\
193 FP hardware exhibits Pentium floating point divide problem\n");
194 }
195 #endif /* __i386 */
196
197 if (fpu_ignored) {
198 printf("FP hardware will not be used\n");
199 } else if (!fpu_exists) {
200 printf("No FPU in configuration\n");
201 }
202
203 /*
204 * Initialize devices on the machine.
205 * Uses configuration tree built by the PROMs to determine what
206 * is present, and builds a tree of prototype dev_info nodes
207 * corresponding to the hardware which identified itself.
208 */
209
210 /*
211 * Initialize root node.
212 */
213 i_ddi_init_root();
214
215 /* reprogram devices not set up by firmware (BIOS) */
216 impl_bus_reprobe();
217
218 #if defined(__amd64) && !defined(__xpv)
219 /*
220 * Setup but don't startup the IOMMU
221 * Startup happens later via a direct call
222 * to IOMMU code by boot code.
223 * At this point, all PCI bus renumbering
224 * is done, so safe to init the IMMU
225 * AKA Intel IOMMU.
226 */
227 immu_init();
228 #endif
229
230 /*
231 * attach the isa nexus to get ACPI resource usage
232 * isa is "kind of" a pseudo node
233 */
234 #if defined(__xpv)
235 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
236 if (pseudo_isa)
237 (void) i_ddi_attach_pseudo_node("isa");
238 else
239 (void) i_ddi_attach_hw_nodes("isa");
240 }
241 #else
242 if (pseudo_isa)
243 (void) i_ddi_attach_pseudo_node("isa");
244 else
245 (void) i_ddi_attach_hw_nodes("isa");
246 #endif
247 }
248
249 /*
250 * The "status" property indicates the operational status of a device.
251 * If this property is present, the value is a string indicating the
252 * status of the device as follows:
253 *
254 * "okay" operational.
255 * "disabled" not operational, but might become operational.
256 * "fail" not operational because a fault has been detected,
257 * and it is unlikely that the device will become
258 * operational without repair. no additional details
259 * are available.
260 * "fail-xxx" not operational because a fault has been detected,
261 * and it is unlikely that the device will become
262 * operational without repair. "xxx" is additional
263 * human-readable information about the particular
264 * fault condition that was detected.
265 *
266 * The absence of this property means that the operational status is
267 * unknown or okay.
268 *
269 * This routine checks the status property of the specified device node
270 * and returns 0 if the operational status indicates failure, and 1 otherwise.
271 *
272 * The property may exist on plug-in cards the existed before IEEE 1275-1994.
273 * And, in that case, the property may not even be a string. So we carefully
274 * check for the value "fail", in the beginning of the string, noting
275 * the property length.
276 */
277 int
278 status_okay(int id, char *buf, int buflen)
279 {
280 char status_buf[OBP_MAXPROPNAME];
281 char *bufp = buf;
282 int len = buflen;
283 int proplen;
284 static const char *status = "status";
285 static const char *fail = "fail";
286 int fail_len = (int)strlen(fail);
287
288 /*
289 * Get the proplen ... if it's smaller than "fail",
290 * or doesn't exist ... then we don't care, since
291 * the value can't begin with the char string "fail".
292 *
293 * NB: proplen, if it's a string, includes the NULL in the
294 * the size of the property, and fail_len does not.
295 */
296 proplen = prom_getproplen((pnode_t)id, (caddr_t)status);
297 if (proplen <= fail_len) /* nonexistant or uninteresting len */
298 return (1);
299
300 /*
301 * if a buffer was provided, use it
302 */
303 if ((buf == (char *)NULL) || (buflen <= 0)) {
304 bufp = status_buf;
305 len = sizeof (status_buf);
306 }
307 *bufp = (char)0;
308
309 /*
310 * Get the property into the buffer, to the extent of the buffer,
311 * and in case the buffer is smaller than the property size,
312 * NULL terminate the buffer. (This handles the case where
313 * a buffer was passed in and the caller wants to print the
314 * value, but the buffer was too small).
315 */
316 (void) prom_bounded_getprop((pnode_t)id, (caddr_t)status,
317 (caddr_t)bufp, len);
318 *(bufp + len - 1) = (char)0;
319
320 /*
321 * If the value begins with the char string "fail",
322 * then it means the node is failed. We don't care
323 * about any other values. We assume the node is ok
324 * although it might be 'disabled'.
325 */
326 if (strncmp(bufp, fail, fail_len) == 0)
327 return (0);
328
329 return (1);
330 }
331
332 /*
333 * Check the status of the device node passed as an argument.
334 *
335 * if ((status is OKAY) || (status is DISABLED))
336 * return DDI_SUCCESS
337 * else
338 * print a warning and return DDI_FAILURE
339 */
340 /*ARGSUSED1*/
341 int
342 check_status(int id, char *name, dev_info_t *parent)
343 {
344 char status_buf[64];
345 char devtype_buf[OBP_MAXPROPNAME];
346 int retval = DDI_FAILURE;
347
348 /*
349 * is the status okay?
350 */
351 if (status_okay(id, status_buf, sizeof (status_buf)))
352 return (DDI_SUCCESS);
353
354 /*
355 * a status property indicating bad memory will be associated
356 * with a node which has a "device_type" property with a value of
357 * "memory-controller". in this situation, return DDI_SUCCESS
358 */
359 if (getlongprop_buf(id, OBP_DEVICETYPE, devtype_buf,
360 sizeof (devtype_buf)) > 0) {
361 if (strcmp(devtype_buf, "memory-controller") == 0)
362 retval = DDI_SUCCESS;
363 }
364
365 /*
366 * print the status property information
367 */
368 cmn_err(CE_WARN, "status '%s' for '%s'", status_buf, name);
369 return (retval);
370 }
371
372 /*ARGSUSED*/
373 uint_t
374 softlevel1(caddr_t arg1, caddr_t arg2)
375 {
376 softint();
377 return (1);
378 }
379
380 /*
381 * Allow for implementation specific correction of PROM property values.
382 */
383
384 /*ARGSUSED*/
385 void
386 impl_fix_props(dev_info_t *dip, dev_info_t *ch_dip, char *name, int len,
387 caddr_t buffer)
388 {
389 /*
390 * There are no adjustments needed in this implementation.
391 */
392 }
393
394 static int
395 getlongprop_buf(int id, char *name, char *buf, int maxlen)
396 {
397 int size;
398
399 size = prom_getproplen((pnode_t)id, name);
400 if (size <= 0 || (size > maxlen - 1))
401 return (-1);
402
403 if (-1 == prom_getprop((pnode_t)id, name, buf))
404 return (-1);
405
406 if (strcmp("name", name) == 0) {
407 if (buf[size - 1] != '\0') {
408 buf[size] = '\0';
409 size += 1;
410 }
411 }
412
413 return (size);
414 }
415
416 static int
417 get_prop_int_array(dev_info_t *di, char *pname, int **pval, uint_t *plen)
418 {
419 int ret;
420
421 if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, di,
422 DDI_PROP_DONTPASS, pname, pval, plen))
423 == DDI_PROP_SUCCESS) {
424 *plen = (*plen) * (sizeof (int));
425 }
426 return (ret);
427 }
428
429
430 /*
431 * Node Configuration
432 */
433
434 struct prop_ispec {
435 uint_t pri, vec;
436 };
437
438 /*
439 * For the x86, we're prepared to claim that the interrupt string
440 * is in the form of a list of <ipl,vec> specifications.
441 */
442
443 #define VEC_MIN 1
444 #define VEC_MAX 255
445
446 static int
447 impl_xlate_intrs(dev_info_t *child, int *in,
448 struct ddi_parent_private_data *pdptr)
449 {
450 size_t size;
451 int n;
452 struct intrspec *new;
453 caddr_t got_prop;
454 int *inpri;
455 int got_len;
456 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
457
458 static char bad_intr_fmt[] =
459 "bad interrupt spec from %s%d - ipl %d, irq %d\n";
460
461 /*
462 * determine if the driver is expecting the new style "interrupts"
463 * property which just contains the IRQ, or the old style which
464 * contains pairs of <IPL,IRQ>. if it is the new style, we always
465 * assign IPL 5 unless an "interrupt-priorities" property exists.
466 * in that case, the "interrupt-priorities" property contains the
467 * IPL values that match, one for one, the IRQ values in the
468 * "interrupts" property.
469 */
470 inpri = NULL;
471 if ((ddi_getprop(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
472 "ignore-hardware-nodes", -1) != -1) || ignore_hardware_nodes) {
473 /* the old style "interrupts" property... */
474
475 /*
476 * The list consists of <ipl,vec> elements
477 */
478 if ((n = (*in++ >> 1)) < 1)
479 return (DDI_FAILURE);
480
481 pdptr->par_nintr = n;
482 size = n * sizeof (struct intrspec);
483 new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP);
484
485 while (n--) {
486 int level = *in++;
487 int vec = *in++;
488
489 if (level < 1 || level > MAXIPL ||
490 vec < VEC_MIN || vec > VEC_MAX) {
491 cmn_err(CE_CONT, bad_intr_fmt,
492 DEVI(child)->devi_name,
493 DEVI(child)->devi_instance, level, vec);
494 goto broken;
495 }
496 new->intrspec_pri = level;
497 if (vec != 2)
498 new->intrspec_vec = vec;
499 else
500 /*
501 * irq 2 on the PC bus is tied to irq 9
502 * on ISA, EISA and MicroChannel
503 */
504 new->intrspec_vec = 9;
505 new++;
506 }
507
508 return (DDI_SUCCESS);
509 } else {
510 /* the new style "interrupts" property... */
511
512 /*
513 * The list consists of <vec> elements
514 */
515 if ((n = (*in++)) < 1)
516 return (DDI_FAILURE);
517
518 pdptr->par_nintr = n;
519 size = n * sizeof (struct intrspec);
520 new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP);
521
522 /* XXX check for "interrupt-priorities" property... */
523 if (ddi_getlongprop(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
524 "interrupt-priorities", (caddr_t)&got_prop, &got_len)
525 == DDI_PROP_SUCCESS) {
526 if (n != (got_len / sizeof (int))) {
527 cmn_err(CE_CONT,
528 "bad interrupt-priorities length"
529 " from %s%d: expected %d, got %d\n",
530 DEVI(child)->devi_name,
531 DEVI(child)->devi_instance, n,
532 (int)(got_len / sizeof (int)));
533 goto broken;
534 }
535 inpri = (int *)got_prop;
536 }
537
538 while (n--) {
539 int level;
540 int vec = *in++;
541
542 if (inpri == NULL)
543 level = 5;
544 else
545 level = *inpri++;
546
547 if (level < 1 || level > MAXIPL ||
548 vec < VEC_MIN || vec > VEC_MAX) {
549 cmn_err(CE_CONT, bad_intr_fmt,
550 DEVI(child)->devi_name,
551 DEVI(child)->devi_instance, level, vec);
552 goto broken;
553 }
554 new->intrspec_pri = level;
555 if (vec != 2)
556 new->intrspec_vec = vec;
557 else
558 /*
559 * irq 2 on the PC bus is tied to irq 9
560 * on ISA, EISA and MicroChannel
561 */
562 new->intrspec_vec = 9;
563 new++;
564 }
565
566 if (inpri != NULL)
567 kmem_free(got_prop, got_len);
568 return (DDI_SUCCESS);
569 }
570
571 broken:
572 kmem_free(pdptr->par_intr, size);
573 pdptr->par_intr = NULL;
574 pdptr->par_nintr = 0;
575 if (inpri != NULL)
576 kmem_free(got_prop, got_len);
577
578 return (DDI_FAILURE);
579 }
580
581 /*
582 * Create a ddi_parent_private_data structure from the ddi properties of
583 * the dev_info node.
584 *
585 * The "reg" and either an "intr" or "interrupts" properties are required
586 * if the driver wishes to create mappings or field interrupts on behalf
587 * of the device.
588 *
589 * The "reg" property is assumed to be a list of at least one triple
590 *
591 * <bustype, address, size>*1
592 *
593 * The "intr" property is assumed to be a list of at least one duple
594 *
595 * <SPARC ipl, vector#>*1
596 *
597 * The "interrupts" property is assumed to be a list of at least one
598 * n-tuples that describes the interrupt capabilities of the bus the device
599 * is connected to. For SBus, this looks like
600 *
601 * <SBus-level>*1
602 *
603 * (This property obsoletes the 'intr' property).
604 *
605 * The "ranges" property is optional.
606 */
607 void
608 make_ddi_ppd(dev_info_t *child, struct ddi_parent_private_data **ppd)
609 {
610 struct ddi_parent_private_data *pdptr;
611 int n;
612 int *reg_prop, *rng_prop, *intr_prop, *irupts_prop;
613 uint_t reg_len, rng_len, intr_len, irupts_len;
614
615 *ppd = pdptr = kmem_zalloc(sizeof (*pdptr), KM_SLEEP);
616
617 /*
618 * Handle the 'reg' property.
619 */
620 if ((get_prop_int_array(child, "reg", ®_prop, ®_len) ==
621 DDI_PROP_SUCCESS) && (reg_len != 0)) {
622 pdptr->par_nreg = reg_len / (int)sizeof (struct regspec);
623 pdptr->par_reg = (struct regspec *)reg_prop;
624 }
625
626 /*
627 * See if I have a range (adding one where needed - this
628 * means to add one for sbus node in sun4c, when romvec > 0,
629 * if no range is already defined in the PROM node.
630 * (Currently no sun4c PROMS define range properties,
631 * but they should and may in the future.) For the SBus
632 * node, the range is defined by the SBus reg property.
633 */
634 if (get_prop_int_array(child, "ranges", &rng_prop, &rng_len)
635 == DDI_PROP_SUCCESS) {
636 pdptr->par_nrng = rng_len / (int)(sizeof (struct rangespec));
637 pdptr->par_rng = (struct rangespec *)rng_prop;
638 }
639
640 /*
641 * Handle the 'intr' and 'interrupts' properties
642 */
643
644 /*
645 * For backwards compatibility
646 * we first look for the 'intr' property for the device.
647 */
648 if (get_prop_int_array(child, "intr", &intr_prop, &intr_len)
649 != DDI_PROP_SUCCESS) {
650 intr_len = 0;
651 }
652
653 /*
654 * If we're to support bus adapters and future platforms cleanly,
655 * we need to support the generalized 'interrupts' property.
656 */
657 if (get_prop_int_array(child, "interrupts", &irupts_prop,
658 &irupts_len) != DDI_PROP_SUCCESS) {
659 irupts_len = 0;
660 } else if (intr_len != 0) {
661 /*
662 * If both 'intr' and 'interrupts' are defined,
663 * then 'interrupts' wins and we toss the 'intr' away.
664 */
665 ddi_prop_free((void *)intr_prop);
666 intr_len = 0;
667 }
668
669 if (intr_len != 0) {
670
671 /*
672 * Translate the 'intr' property into an array
673 * an array of struct intrspec's. There's not really
674 * very much to do here except copy what's out there.
675 */
676
677 struct intrspec *new;
678 struct prop_ispec *l;
679
680 n = pdptr->par_nintr = intr_len / sizeof (struct prop_ispec);
681 l = (struct prop_ispec *)intr_prop;
682 pdptr->par_intr =
683 new = kmem_zalloc(n * sizeof (struct intrspec), KM_SLEEP);
684 while (n--) {
685 new->intrspec_pri = l->pri;
686 new->intrspec_vec = l->vec;
687 new++;
688 l++;
689 }
690 ddi_prop_free((void *)intr_prop);
691
692 } else if ((n = irupts_len) != 0) {
693 size_t size;
694 int *out;
695
696 /*
697 * Translate the 'interrupts' property into an array
698 * of intrspecs for the rest of the DDI framework to
699 * toy with. Only our ancestors really know how to
700 * do this, so ask 'em. We massage the 'interrupts'
701 * property so that it is pre-pended by a count of
702 * the number of integers in the argument.
703 */
704 size = sizeof (int) + n;
705 out = kmem_alloc(size, KM_SLEEP);
706 *out = n / sizeof (int);
707 bcopy(irupts_prop, out + 1, (size_t)n);
708 ddi_prop_free((void *)irupts_prop);
709 if (impl_xlate_intrs(child, out, pdptr) != DDI_SUCCESS) {
710 cmn_err(CE_CONT,
711 "Unable to translate 'interrupts' for %s%d\n",
712 DEVI(child)->devi_binding_name,
713 DEVI(child)->devi_instance);
714 }
715 kmem_free(out, size);
716 }
717 }
718
719 /*
720 * Name a child
721 */
722 static int
723 impl_sunbus_name_child(dev_info_t *child, char *name, int namelen)
724 {
725 /*
726 * Fill in parent-private data and this function returns to us
727 * an indication if it used "registers" to fill in the data.
728 */
729 if (ddi_get_parent_data(child) == NULL) {
730 struct ddi_parent_private_data *pdptr;
731 make_ddi_ppd(child, &pdptr);
732 ddi_set_parent_data(child, pdptr);
733 }
734
735 name[0] = '\0';
736 if (sparc_pd_getnreg(child) > 0) {
737 (void) snprintf(name, namelen, "%x,%x",
738 (uint_t)sparc_pd_getreg(child, 0)->regspec_bustype,
739 (uint_t)sparc_pd_getreg(child, 0)->regspec_addr);
740 }
741
742 return (DDI_SUCCESS);
743 }
744
745 /*
746 * Called from the bus_ctl op of sunbus (sbus, obio, etc) nexus drivers
747 * to implement the DDI_CTLOPS_INITCHILD operation. That is, it names
748 * the children of sun busses based on the reg spec.
749 *
750 * Handles the following properties (in make_ddi_ppd):
751 * Property value
752 * Name type
753 * reg register spec
754 * intr old-form interrupt spec
755 * interrupts new (bus-oriented) interrupt spec
756 * ranges range spec
757 */
758 int
759 impl_ddi_sunbus_initchild(dev_info_t *child)
760 {
761 char name[MAXNAMELEN];
762 void impl_ddi_sunbus_removechild(dev_info_t *);
763
764 /*
765 * Name the child, also makes parent private data
766 */
767 (void) impl_sunbus_name_child(child, name, MAXNAMELEN);
768 ddi_set_name_addr(child, name);
769
770 /*
771 * Attempt to merge a .conf node; if successful, remove the
772 * .conf node.
773 */
774 if ((ndi_dev_is_persistent_node(child) == 0) &&
775 (ndi_merge_node(child, impl_sunbus_name_child) == DDI_SUCCESS)) {
776 /*
777 * Return failure to remove node
778 */
779 impl_ddi_sunbus_removechild(child);
780 return (DDI_FAILURE);
781 }
782 return (DDI_SUCCESS);
783 }
784
785 void
786 impl_free_ddi_ppd(dev_info_t *dip)
787 {
788 struct ddi_parent_private_data *pdptr;
789 size_t n;
790
791 if ((pdptr = ddi_get_parent_data(dip)) == NULL)
792 return;
793
794 if ((n = (size_t)pdptr->par_nintr) != 0)
795 /*
796 * Note that kmem_free is used here (instead of
797 * ddi_prop_free) because the contents of the
798 * property were placed into a separate buffer and
799 * mucked with a bit before being stored in par_intr.
800 * The actual return value from the prop lookup
801 * was freed with ddi_prop_free previously.
802 */
803 kmem_free(pdptr->par_intr, n * sizeof (struct intrspec));
804
805 if ((n = (size_t)pdptr->par_nrng) != 0)
806 ddi_prop_free((void *)pdptr->par_rng);
807
808 if ((n = pdptr->par_nreg) != 0)
809 ddi_prop_free((void *)pdptr->par_reg);
810
811 kmem_free(pdptr, sizeof (*pdptr));
812 ddi_set_parent_data(dip, NULL);
813 }
814
815 void
816 impl_ddi_sunbus_removechild(dev_info_t *dip)
817 {
818 impl_free_ddi_ppd(dip);
819 ddi_set_name_addr(dip, NULL);
820 /*
821 * Strip the node to properly convert it back to prototype form
822 */
823 impl_rem_dev_props(dip);
824 }
825
826 /*
827 * DDI Interrupt
828 */
829
830 /*
831 * turn this on to force isa, eisa, and mca device to ignore the new
832 * hardware nodes in the device tree (normally turned on only for
833 * drivers that need it by setting the property "ignore-hardware-nodes"
834 * in their driver.conf file).
835 *
836 * 7/31/96 -- Turned off globally. Leaving variable in for the moment
837 * as safety valve.
838 */
839 int ignore_hardware_nodes = 0;
840
841 /*
842 * Local data
843 */
844 static struct impl_bus_promops *impl_busp;
845
846
847 /*
848 * New DDI interrupt framework
849 */
850
851 /*
852 * i_ddi_intr_ops:
853 *
854 * This is the interrupt operator function wrapper for the bus function
855 * bus_intr_op.
856 */
857 int
858 i_ddi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t op,
859 ddi_intr_handle_impl_t *hdlp, void * result)
860 {
861 dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent;
862 int ret = DDI_FAILURE;
863
864 /* request parent to process this interrupt op */
865 if (NEXUS_HAS_INTR_OP(pdip))
866 ret = (*(DEVI(pdip)->devi_ops->devo_bus_ops->bus_intr_op))(
867 pdip, rdip, op, hdlp, result);
868 else
869 cmn_err(CE_WARN, "Failed to process interrupt "
870 "for %s%d due to down-rev nexus driver %s%d",
871 ddi_get_name(rdip), ddi_get_instance(rdip),
872 ddi_get_name(pdip), ddi_get_instance(pdip));
873 return (ret);
874 }
875
876 /*
877 * i_ddi_add_softint - allocate and add a soft interrupt to the system
878 */
879 int
880 i_ddi_add_softint(ddi_softint_hdl_impl_t *hdlp)
881 {
882 int ret;
883
884 /* add soft interrupt handler */
885 ret = add_avsoftintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func,
886 DEVI(hdlp->ih_dip)->devi_name, hdlp->ih_cb_arg1, hdlp->ih_cb_arg2);
887 return (ret ? DDI_SUCCESS : DDI_FAILURE);
888 }
889
890
891 void
892 i_ddi_remove_softint(ddi_softint_hdl_impl_t *hdlp)
893 {
894 (void) rem_avsoftintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func);
895 }
896
897
898 extern void (*setsoftint)(int, struct av_softinfo *);
899 extern boolean_t av_check_softint_pending(struct av_softinfo *, boolean_t);
900
901 int
902 i_ddi_trigger_softint(ddi_softint_hdl_impl_t *hdlp, void *arg2)
903 {
904 if (av_check_softint_pending(hdlp->ih_pending, B_FALSE))
905 return (DDI_EPENDING);
906
907 update_avsoftintr_args((void *)hdlp, hdlp->ih_pri, arg2);
908
909 (*setsoftint)(hdlp->ih_pri, hdlp->ih_pending);
910 return (DDI_SUCCESS);
911 }
912
913 /*
914 * i_ddi_set_softint_pri:
915 *
916 * The way this works is that it first tries to add a softint vector
917 * at the new priority in hdlp. If that succeeds; then it removes the
918 * existing softint vector at the old priority.
919 */
920 int
921 i_ddi_set_softint_pri(ddi_softint_hdl_impl_t *hdlp, uint_t old_pri)
922 {
923 int ret;
924
925 /*
926 * If a softint is pending at the old priority then fail the request.
927 */
928 if (av_check_softint_pending(hdlp->ih_pending, B_TRUE))
929 return (DDI_FAILURE);
930
931 ret = av_softint_movepri((void *)hdlp, old_pri);
932 return (ret ? DDI_SUCCESS : DDI_FAILURE);
933 }
934
935 void
936 i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t *hdlp)
937 {
938 hdlp->ih_private = (void *)kmem_zalloc(sizeof (ihdl_plat_t), KM_SLEEP);
939 }
940
941 void
942 i_ddi_free_intr_phdl(ddi_intr_handle_impl_t *hdlp)
943 {
944 kmem_free(hdlp->ih_private, sizeof (ihdl_plat_t));
945 hdlp->ih_private = NULL;
946 }
947
948 int
949 i_ddi_get_intx_nintrs(dev_info_t *dip)
950 {
951 struct ddi_parent_private_data *pdp;
952
953 if ((pdp = ddi_get_parent_data(dip)) == NULL)
954 return (0);
955
956 return (pdp->par_nintr);
957 }
958
959 /*
960 * DDI Memory/DMA
961 */
962
963 /*
964 * Support for allocating DMAable memory to implement
965 * ddi_dma_mem_alloc(9F) interface.
966 */
967
968 #define KA_ALIGN_SHIFT 7
969 #define KA_ALIGN (1 << KA_ALIGN_SHIFT)
970 #define KA_NCACHE (PAGESHIFT + 1 - KA_ALIGN_SHIFT)
971
972 /*
973 * Dummy DMA attribute template for kmem_io[].kmem_io_attr. We only
974 * care about addr_lo, addr_hi, and align. addr_hi will be dynamically set.
975 */
976
977 static ddi_dma_attr_t kmem_io_attr = {
978 DMA_ATTR_V0,
979 0x0000000000000000ULL, /* dma_attr_addr_lo */
980 0x0000000000000000ULL, /* dma_attr_addr_hi */
981 0x00ffffff,
982 0x1000, /* dma_attr_align */
983 1, 1, 0xffffffffULL, 0xffffffffULL, 0x1, 1, 0
984 };
985
986 /* kmem io memory ranges and indices */
987 enum {
988 IO_4P, IO_64G, IO_4G, IO_2G, IO_1G, IO_512M,
989 IO_256M, IO_128M, IO_64M, IO_32M, IO_16M, MAX_MEM_RANGES
990 };
991
992 static struct {
993 vmem_t *kmem_io_arena;
994 kmem_cache_t *kmem_io_cache[KA_NCACHE];
995 ddi_dma_attr_t kmem_io_attr;
996 } kmem_io[MAX_MEM_RANGES];
997
998 static int kmem_io_idx; /* index of first populated kmem_io[] */
999
1000 static page_t *
1001 page_create_io_wrapper(void *addr, size_t len, int vmflag, void *arg)
1002 {
1003 extern page_t *page_create_io(vnode_t *, u_offset_t, uint_t,
1004 uint_t, struct as *, caddr_t, ddi_dma_attr_t *);
1005
1006 return (page_create_io(&kvp, (u_offset_t)(uintptr_t)addr, len,
1007 PG_EXCL | ((vmflag & VM_NOSLEEP) ? 0 : PG_WAIT), &kas, addr, arg));
1008 }
1009
1010 #ifdef __xpv
1011 static void
1012 segkmem_free_io(vmem_t *vmp, void * ptr, size_t size)
1013 {
1014 extern void page_destroy_io(page_t *);
1015 segkmem_xfree(vmp, ptr, size, page_destroy_io);
1016 }
1017 #endif
1018
1019 static void *
1020 segkmem_alloc_io_4P(vmem_t *vmp, size_t size, int vmflag)
1021 {
1022 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1023 page_create_io_wrapper, &kmem_io[IO_4P].kmem_io_attr));
1024 }
1025
1026 static void *
1027 segkmem_alloc_io_64G(vmem_t *vmp, size_t size, int vmflag)
1028 {
1029 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1030 page_create_io_wrapper, &kmem_io[IO_64G].kmem_io_attr));
1031 }
1032
1033 static void *
1034 segkmem_alloc_io_4G(vmem_t *vmp, size_t size, int vmflag)
1035 {
1036 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1037 page_create_io_wrapper, &kmem_io[IO_4G].kmem_io_attr));
1038 }
1039
1040 static void *
1041 segkmem_alloc_io_2G(vmem_t *vmp, size_t size, int vmflag)
1042 {
1043 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1044 page_create_io_wrapper, &kmem_io[IO_2G].kmem_io_attr));
1045 }
1046
1047 static void *
1048 segkmem_alloc_io_1G(vmem_t *vmp, size_t size, int vmflag)
1049 {
1050 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1051 page_create_io_wrapper, &kmem_io[IO_1G].kmem_io_attr));
1052 }
1053
1054 static void *
1055 segkmem_alloc_io_512M(vmem_t *vmp, size_t size, int vmflag)
1056 {
1057 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1058 page_create_io_wrapper, &kmem_io[IO_512M].kmem_io_attr));
1059 }
1060
1061 static void *
1062 segkmem_alloc_io_256M(vmem_t *vmp, size_t size, int vmflag)
1063 {
1064 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1065 page_create_io_wrapper, &kmem_io[IO_256M].kmem_io_attr));
1066 }
1067
1068 static void *
1069 segkmem_alloc_io_128M(vmem_t *vmp, size_t size, int vmflag)
1070 {
1071 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1072 page_create_io_wrapper, &kmem_io[IO_128M].kmem_io_attr));
1073 }
1074
1075 static void *
1076 segkmem_alloc_io_64M(vmem_t *vmp, size_t size, int vmflag)
1077 {
1078 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1079 page_create_io_wrapper, &kmem_io[IO_64M].kmem_io_attr));
1080 }
1081
1082 static void *
1083 segkmem_alloc_io_32M(vmem_t *vmp, size_t size, int vmflag)
1084 {
1085 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1086 page_create_io_wrapper, &kmem_io[IO_32M].kmem_io_attr));
1087 }
1088
1089 static void *
1090 segkmem_alloc_io_16M(vmem_t *vmp, size_t size, int vmflag)
1091 {
1092 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1093 page_create_io_wrapper, &kmem_io[IO_16M].kmem_io_attr));
1094 }
1095
1096 struct {
1097 uint64_t io_limit;
1098 char *io_name;
1099 void *(*io_alloc)(vmem_t *, size_t, int);
1100 int io_initial; /* kmem_io_init during startup */
1101 } io_arena_params[MAX_MEM_RANGES] = {
1102 {0x000fffffffffffffULL, "kmem_io_4P", segkmem_alloc_io_4P, 1},
1103 {0x0000000fffffffffULL, "kmem_io_64G", segkmem_alloc_io_64G, 0},
1104 {0x00000000ffffffffULL, "kmem_io_4G", segkmem_alloc_io_4G, 1},
1105 {0x000000007fffffffULL, "kmem_io_2G", segkmem_alloc_io_2G, 1},
1106 {0x000000003fffffffULL, "kmem_io_1G", segkmem_alloc_io_1G, 0},
1107 {0x000000001fffffffULL, "kmem_io_512M", segkmem_alloc_io_512M, 0},
1108 {0x000000000fffffffULL, "kmem_io_256M", segkmem_alloc_io_256M, 0},
1109 {0x0000000007ffffffULL, "kmem_io_128M", segkmem_alloc_io_128M, 0},
1110 {0x0000000003ffffffULL, "kmem_io_64M", segkmem_alloc_io_64M, 0},
1111 {0x0000000001ffffffULL, "kmem_io_32M", segkmem_alloc_io_32M, 0},
1112 {0x0000000000ffffffULL, "kmem_io_16M", segkmem_alloc_io_16M, 1}
1113 };
1114
1115 void
1116 kmem_io_init(int a)
1117 {
1118 int c;
1119 char name[40];
1120
1121 kmem_io[a].kmem_io_arena = vmem_create(io_arena_params[a].io_name,
1122 NULL, 0, PAGESIZE, io_arena_params[a].io_alloc,
1123 #ifdef __xpv
1124 segkmem_free_io,
1125 #else
1126 segkmem_free,
1127 #endif
1128 heap_arena, 0, VM_SLEEP);
1129
1130 for (c = 0; c < KA_NCACHE; c++) {
1131 size_t size = KA_ALIGN << c;
1132 (void) sprintf(name, "%s_%lu",
1133 io_arena_params[a].io_name, size);
1134 kmem_io[a].kmem_io_cache[c] = kmem_cache_create(name,
1135 size, size, NULL, NULL, NULL, NULL,
1136 kmem_io[a].kmem_io_arena, 0);
1137 }
1138 }
1139
1140 /*
1141 * Return the index of the highest memory range for addr.
1142 */
1143 static int
1144 kmem_io_index(uint64_t addr)
1145 {
1146 int n;
1147
1148 for (n = kmem_io_idx; n < MAX_MEM_RANGES; n++) {
1149 if (kmem_io[n].kmem_io_attr.dma_attr_addr_hi <= addr) {
1150 if (kmem_io[n].kmem_io_arena == NULL)
1151 kmem_io_init(n);
1152 return (n);
1153 }
1154 }
1155 panic("kmem_io_index: invalid addr - must be at least 16m");
1156
1157 /*NOTREACHED*/
1158 }
1159
1160 /*
1161 * Return the index of the next kmem_io populated memory range
1162 * after curindex.
1163 */
1164 static int
1165 kmem_io_index_next(int curindex)
1166 {
1167 int n;
1168
1169 for (n = curindex + 1; n < MAX_MEM_RANGES; n++) {
1170 if (kmem_io[n].kmem_io_arena)
1171 return (n);
1172 }
1173 return (-1);
1174 }
1175
1176 /*
1177 * allow kmem to be mapped in with different PTE cache attribute settings.
1178 * Used by i_ddi_mem_alloc()
1179 */
1180 int
1181 kmem_override_cache_attrs(caddr_t kva, size_t size, uint_t order)
1182 {
1183 uint_t hat_flags;
1184 caddr_t kva_end;
1185 uint_t hat_attr;
1186 pfn_t pfn;
1187
1188 if (hat_getattr(kas.a_hat, kva, &hat_attr) == -1) {
1189 return (-1);
1190 }
1191
1192 hat_attr &= ~HAT_ORDER_MASK;
1193 hat_attr |= order | HAT_NOSYNC;
1194 hat_flags = HAT_LOAD_LOCK;
1195
1196 kva_end = (caddr_t)(((uintptr_t)kva + size + PAGEOFFSET) &
1197 (uintptr_t)PAGEMASK);
1198 kva = (caddr_t)((uintptr_t)kva & (uintptr_t)PAGEMASK);
1199
1200 while (kva < kva_end) {
1201 pfn = hat_getpfnum(kas.a_hat, kva);
1202 hat_unload(kas.a_hat, kva, PAGESIZE, HAT_UNLOAD_UNLOCK);
1203 hat_devload(kas.a_hat, kva, PAGESIZE, pfn, hat_attr, hat_flags);
1204 kva += MMU_PAGESIZE;
1205 }
1206
1207 return (0);
1208 }
1209
1210 static int
1211 ctgcompare(const void *a1, const void *a2)
1212 {
1213 /* we just want to compare virtual addresses */
1214 a1 = ((struct ctgas *)a1)->ctg_addr;
1215 a2 = ((struct ctgas *)a2)->ctg_addr;
1216 return (a1 == a2 ? 0 : (a1 < a2 ? -1 : 1));
1217 }
1218
1219 void
1220 ka_init(void)
1221 {
1222 int a;
1223 paddr_t maxphysaddr;
1224 #if !defined(__xpv)
1225 extern pfn_t physmax;
1226
1227 maxphysaddr = mmu_ptob((paddr_t)physmax) + MMU_PAGEOFFSET;
1228 #else
1229 maxphysaddr = mmu_ptob((paddr_t)HYPERVISOR_memory_op(
1230 XENMEM_maximum_ram_page, NULL)) + MMU_PAGEOFFSET;
1231 #endif
1232
1233 ASSERT(maxphysaddr <= io_arena_params[0].io_limit);
1234
1235 for (a = 0; a < MAX_MEM_RANGES; a++) {
1236 if (maxphysaddr >= io_arena_params[a + 1].io_limit) {
1237 if (maxphysaddr > io_arena_params[a + 1].io_limit)
1238 io_arena_params[a].io_limit = maxphysaddr;
1239 else
1240 a++;
1241 break;
1242 }
1243 }
1244 kmem_io_idx = a;
1245
1246 for (; a < MAX_MEM_RANGES; a++) {
1247 kmem_io[a].kmem_io_attr = kmem_io_attr;
1248 kmem_io[a].kmem_io_attr.dma_attr_addr_hi =
1249 io_arena_params[a].io_limit;
1250 /*
1251 * initialize kmem_io[] arena/cache corresponding to
1252 * maxphysaddr and to the "common" io memory ranges that
1253 * have io_initial set to a non-zero value.
1254 */
1255 if (io_arena_params[a].io_initial || a == kmem_io_idx)
1256 kmem_io_init(a);
1257 }
1258
1259 /* initialize ctgtree */
1260 avl_create(&ctgtree, ctgcompare, sizeof (struct ctgas),
1261 offsetof(struct ctgas, ctg_link));
1262 }
1263
1264 /*
1265 * put contig address/size
1266 */
1267 static void *
1268 putctgas(void *addr, size_t size)
1269 {
1270 struct ctgas *ctgp;
1271 if ((ctgp = kmem_zalloc(sizeof (*ctgp), KM_NOSLEEP)) != NULL) {
1272 ctgp->ctg_addr = addr;
1273 ctgp->ctg_size = size;
1274 CTGLOCK();
1275 avl_add(&ctgtree, ctgp);
1276 CTGUNLOCK();
1277 }
1278 return (ctgp);
1279 }
1280
1281 /*
1282 * get contig size by addr
1283 */
1284 static size_t
1285 getctgsz(void *addr)
1286 {
1287 struct ctgas *ctgp;
1288 struct ctgas find;
1289 size_t sz = 0;
1290
1291 find.ctg_addr = addr;
1292 CTGLOCK();
1293 if ((ctgp = avl_find(&ctgtree, &find, NULL)) != NULL) {
1294 avl_remove(&ctgtree, ctgp);
1295 }
1296 CTGUNLOCK();
1297
1298 if (ctgp != NULL) {
1299 sz = ctgp->ctg_size;
1300 kmem_free(ctgp, sizeof (*ctgp));
1301 }
1302
1303 return (sz);
1304 }
1305
1306 /*
1307 * contig_alloc:
1308 *
1309 * allocates contiguous memory to satisfy the 'size' and dma attributes
1310 * specified in 'attr'.
1311 *
1312 * Not all of memory need to be physically contiguous if the
1313 * scatter-gather list length is greater than 1.
1314 */
1315
1316 /*ARGSUSED*/
1317 void *
1318 contig_alloc(size_t size, ddi_dma_attr_t *attr, uintptr_t align, int cansleep)
1319 {
1320 pgcnt_t pgcnt = btopr(size);
1321 size_t asize = pgcnt * PAGESIZE;
1322 page_t *ppl;
1323 int pflag;
1324 void *addr;
1325
1326 extern page_t *page_create_io(vnode_t *, u_offset_t, uint_t,
1327 uint_t, struct as *, caddr_t, ddi_dma_attr_t *);
1328
1329 /* segkmem_xalloc */
1330
1331 if (align <= PAGESIZE)
1332 addr = vmem_alloc(heap_arena, asize,
1333 (cansleep) ? VM_SLEEP : VM_NOSLEEP);
1334 else
1335 addr = vmem_xalloc(heap_arena, asize, align, 0, 0, NULL, NULL,
1336 (cansleep) ? VM_SLEEP : VM_NOSLEEP);
1337 if (addr) {
1338 ASSERT(!((uintptr_t)addr & (align - 1)));
1339
1340 if (page_resv(pgcnt, (cansleep) ? KM_SLEEP : KM_NOSLEEP) == 0) {
1341 vmem_free(heap_arena, addr, asize);
1342 return (NULL);
1343 }
1344 pflag = PG_EXCL;
1345
1346 if (cansleep)
1347 pflag |= PG_WAIT;
1348
1349 /* 4k req gets from freelists rather than pfn search */
1350 if (pgcnt > 1 || align > PAGESIZE)
1351 pflag |= PG_PHYSCONTIG;
1352
1353 ppl = page_create_io(&kvp, (u_offset_t)(uintptr_t)addr,
1354 asize, pflag, &kas, (caddr_t)addr, attr);
1355
1356 if (!ppl) {
1357 vmem_free(heap_arena, addr, asize);
1358 page_unresv(pgcnt);
1359 return (NULL);
1360 }
1361
1362 while (ppl != NULL) {
1363 page_t *pp = ppl;
1364 page_sub(&ppl, pp);
1365 ASSERT(page_iolock_assert(pp));
1366 page_io_unlock(pp);
1367 page_downgrade(pp);
1368 hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset,
1369 pp, (PROT_ALL & ~PROT_USER) |
1370 HAT_NOSYNC, HAT_LOAD_LOCK);
1371 }
1372 }
1373 return (addr);
1374 }
1375
1376 void
1377 contig_free(void *addr, size_t size)
1378 {
1379 pgcnt_t pgcnt = btopr(size);
1380 size_t asize = pgcnt * PAGESIZE;
1381 caddr_t a, ea;
1382 page_t *pp;
1383
1384 hat_unload(kas.a_hat, addr, asize, HAT_UNLOAD_UNLOCK);
1385
1386 for (a = addr, ea = a + asize; a < ea; a += PAGESIZE) {
1387 pp = page_find(&kvp, (u_offset_t)(uintptr_t)a);
1388 if (!pp)
1389 panic("contig_free: contig pp not found");
1390
1391 if (!page_tryupgrade(pp)) {
1392 page_unlock(pp);
1393 pp = page_lookup(&kvp,
1394 (u_offset_t)(uintptr_t)a, SE_EXCL);
1395 if (pp == NULL)
1396 panic("contig_free: page freed");
1397 }
1398 page_destroy(pp, 0);
1399 }
1400
1401 page_unresv(pgcnt);
1402 vmem_free(heap_arena, addr, asize);
1403 }
1404
1405 /*
1406 * Allocate from the system, aligned on a specific boundary.
1407 * The alignment, if non-zero, must be a power of 2.
1408 */
1409 static void *
1410 kalloca(size_t size, size_t align, int cansleep, int physcontig,
1411 ddi_dma_attr_t *attr)
1412 {
1413 size_t *addr, *raddr, rsize;
1414 size_t hdrsize = 4 * sizeof (size_t); /* must be power of 2 */
1415 int a, i, c;
1416 vmem_t *vmp;
1417 kmem_cache_t *cp = NULL;
1418
1419 if (attr->dma_attr_addr_lo > mmu_ptob((uint64_t)ddiphysmin))
1420 return (NULL);
1421
1422 align = MAX(align, hdrsize);
1423 ASSERT((align & (align - 1)) == 0);
1424
1425 /*
1426 * All of our allocators guarantee 16-byte alignment, so we don't
1427 * need to reserve additional space for the header.
1428 * To simplify picking the correct kmem_io_cache, we round up to
1429 * a multiple of KA_ALIGN.
1430 */
1431 rsize = P2ROUNDUP_TYPED(size + align, KA_ALIGN, size_t);
1432
1433 if (physcontig && rsize > PAGESIZE) {
1434 if (addr = contig_alloc(size, attr, align, cansleep)) {
1435 if (!putctgas(addr, size))
1436 contig_free(addr, size);
1437 else
1438 return (addr);
1439 }
1440 return (NULL);
1441 }
1442
1443 a = kmem_io_index(attr->dma_attr_addr_hi);
1444
1445 if (rsize > PAGESIZE) {
1446 vmp = kmem_io[a].kmem_io_arena;
1447 raddr = vmem_alloc(vmp, rsize,
1448 (cansleep) ? VM_SLEEP : VM_NOSLEEP);
1449 } else {
1450 c = highbit((rsize >> KA_ALIGN_SHIFT) - 1);
1451 cp = kmem_io[a].kmem_io_cache[c];
1452 raddr = kmem_cache_alloc(cp, (cansleep) ? KM_SLEEP :
1453 KM_NOSLEEP);
1454 }
1455
1456 if (raddr == NULL) {
1457 int na;
1458
1459 ASSERT(cansleep == 0);
1460 if (rsize > PAGESIZE)
1461 return (NULL);
1462 /*
1463 * System does not have memory in the requested range.
1464 * Try smaller kmem io ranges and larger cache sizes
1465 * to see if there might be memory available in
1466 * these other caches.
1467 */
1468
1469 for (na = kmem_io_index_next(a); na >= 0;
1470 na = kmem_io_index_next(na)) {
1471 ASSERT(kmem_io[na].kmem_io_arena);
1472 cp = kmem_io[na].kmem_io_cache[c];
1473 raddr = kmem_cache_alloc(cp, KM_NOSLEEP);
1474 if (raddr)
1475 goto kallocdone;
1476 }
1477 /* now try the larger kmem io cache sizes */
1478 for (na = a; na >= 0; na = kmem_io_index_next(na)) {
1479 for (i = c + 1; i < KA_NCACHE; i++) {
1480 cp = kmem_io[na].kmem_io_cache[i];
1481 raddr = kmem_cache_alloc(cp, KM_NOSLEEP);
1482 if (raddr)
1483 goto kallocdone;
1484 }
1485 }
1486 return (NULL);
1487 }
1488
1489 kallocdone:
1490 ASSERT(!P2BOUNDARY((uintptr_t)raddr, rsize, PAGESIZE) ||
1491 rsize > PAGESIZE);
1492
1493 addr = (size_t *)P2ROUNDUP((uintptr_t)raddr + hdrsize, align);
1494 ASSERT((uintptr_t)addr + size - (uintptr_t)raddr <= rsize);
1495
1496 addr[-4] = (size_t)cp;
1497 addr[-3] = (size_t)vmp;
1498 addr[-2] = (size_t)raddr;
1499 addr[-1] = rsize;
1500
1501 return (addr);
1502 }
1503
1504 static void
1505 kfreea(void *addr)
1506 {
1507 size_t size;
1508
1509 if (!((uintptr_t)addr & PAGEOFFSET) && (size = getctgsz(addr))) {
1510 contig_free(addr, size);
1511 } else {
1512 size_t *saddr = addr;
1513 if (saddr[-4] == 0)
1514 vmem_free((vmem_t *)saddr[-3], (void *)saddr[-2],
1515 saddr[-1]);
1516 else
1517 kmem_cache_free((kmem_cache_t *)saddr[-4],
1518 (void *)saddr[-2]);
1519 }
1520 }
1521
1522 /*ARGSUSED*/
1523 void
1524 i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t *devaccp, uint_t *hataccp)
1525 {
1526 }
1527
1528 /*
1529 * Check if the specified cache attribute is supported on the platform.
1530 * This function must be called before i_ddi_cacheattr_to_hatacc().
1531 */
1532 boolean_t
1533 i_ddi_check_cache_attr(uint_t flags)
1534 {
1535 /*
1536 * The cache attributes are mutually exclusive. Any combination of
1537 * the attributes leads to a failure.
1538 */
1539 uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1540 if ((cache_attr != 0) && !ISP2(cache_attr))
1541 return (B_FALSE);
1542
1543 /* All cache attributes are supported on X86/X64 */
1544 if (cache_attr & (IOMEM_DATA_UNCACHED | IOMEM_DATA_CACHED |
1545 IOMEM_DATA_UC_WR_COMBINE))
1546 return (B_TRUE);
1547
1548 /* undefined attributes */
1549 return (B_FALSE);
1550 }
1551
1552 /* set HAT cache attributes from the cache attributes */
1553 void
1554 i_ddi_cacheattr_to_hatacc(uint_t flags, uint_t *hataccp)
1555 {
1556 uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1557 static char *fname = "i_ddi_cacheattr_to_hatacc";
1558
1559 /*
1560 * If write-combining is not supported, then it falls back
1561 * to uncacheable.
1562 */
1563 if (cache_attr == IOMEM_DATA_UC_WR_COMBINE &&
1564 !is_x86_feature(x86_featureset, X86FSET_PAT))
1565 cache_attr = IOMEM_DATA_UNCACHED;
1566
1567 /*
1568 * set HAT attrs according to the cache attrs.
1569 */
1570 switch (cache_attr) {
1571 case IOMEM_DATA_UNCACHED:
1572 *hataccp &= ~HAT_ORDER_MASK;
1573 *hataccp |= (HAT_STRICTORDER | HAT_PLAT_NOCACHE);
1574 break;
1575 case IOMEM_DATA_UC_WR_COMBINE:
1576 *hataccp &= ~HAT_ORDER_MASK;
1577 *hataccp |= (HAT_MERGING_OK | HAT_PLAT_NOCACHE);
1578 break;
1579 case IOMEM_DATA_CACHED:
1580 *hataccp &= ~HAT_ORDER_MASK;
1581 *hataccp |= HAT_UNORDERED_OK;
1582 break;
1583 /*
1584 * This case must not occur because the cache attribute is scrutinized
1585 * before this function is called.
1586 */
1587 default:
1588 /*
1589 * set cacheable to hat attrs.
1590 */
1591 *hataccp &= ~HAT_ORDER_MASK;
1592 *hataccp |= HAT_UNORDERED_OK;
1593 cmn_err(CE_WARN, "%s: cache_attr=0x%x is ignored.",
1594 fname, cache_attr);
1595 }
1596 }
1597
1598 /*
1599 * This should actually be called i_ddi_dma_mem_alloc. There should
1600 * also be an i_ddi_pio_mem_alloc. i_ddi_dma_mem_alloc should call
1601 * through the device tree with the DDI_CTLOPS_DMA_ALIGN ctl ops to
1602 * get alignment requirements for DMA memory. i_ddi_pio_mem_alloc
1603 * should use DDI_CTLOPS_PIO_ALIGN. Since we only have i_ddi_mem_alloc
1604 * so far which is used for both, DMA and PIO, we have to use the DMA
1605 * ctl ops to make everybody happy.
1606 */
1607 /*ARGSUSED*/
1608 int
1609 i_ddi_mem_alloc(dev_info_t *dip, ddi_dma_attr_t *attr,
1610 size_t length, int cansleep, int flags,
1611 ddi_device_acc_attr_t *accattrp, caddr_t *kaddrp,
1612 size_t *real_length, ddi_acc_hdl_t *ap)
1613 {
1614 caddr_t a;
1615 int iomin;
1616 ddi_acc_impl_t *iap;
1617 int physcontig = 0;
1618 pgcnt_t npages;
1619 pgcnt_t minctg;
1620 uint_t order;
1621 int e;
1622
1623 /*
1624 * Check legality of arguments
1625 */
1626 if (length == 0 || kaddrp == NULL || attr == NULL) {
1627 return (DDI_FAILURE);
1628 }
1629
1630 if (attr->dma_attr_minxfer == 0 || attr->dma_attr_align == 0 ||
1631 !ISP2(attr->dma_attr_align) || !ISP2(attr->dma_attr_minxfer)) {
1632 return (DDI_FAILURE);
1633 }
1634
1635 /*
1636 * figure out most restrictive alignment requirement
1637 */
1638 iomin = attr->dma_attr_minxfer;
1639 iomin = maxbit(iomin, attr->dma_attr_align);
1640 if (iomin == 0)
1641 return (DDI_FAILURE);
1642
1643 ASSERT((iomin & (iomin - 1)) == 0);
1644
1645 /*
1646 * if we allocate memory with IOMEM_DATA_UNCACHED or
1647 * IOMEM_DATA_UC_WR_COMBINE, make sure we allocate a page aligned
1648 * memory that ends on a page boundry.
1649 * Don't want to have to different cache mappings to the same
1650 * physical page.
1651 */
1652 if (OVERRIDE_CACHE_ATTR(flags)) {
1653 iomin = (iomin + MMU_PAGEOFFSET) & MMU_PAGEMASK;
1654 length = (length + MMU_PAGEOFFSET) & (size_t)MMU_PAGEMASK;
1655 }
1656
1657 /*
1658 * Determine if we need to satisfy the request for physically
1659 * contiguous memory or alignments larger than pagesize.
1660 */
1661 npages = btopr(length + attr->dma_attr_align);
1662 minctg = howmany(npages, attr->dma_attr_sgllen);
1663
1664 if (minctg > 1) {
1665 uint64_t pfnseg = attr->dma_attr_seg >> PAGESHIFT;
1666 /*
1667 * verify that the minimum contig requirement for the
1668 * actual length does not cross segment boundary.
1669 */
1670 length = P2ROUNDUP_TYPED(length, attr->dma_attr_minxfer,
1671 size_t);
1672 npages = btopr(length);
1673 minctg = howmany(npages, attr->dma_attr_sgllen);
1674 if (minctg > pfnseg + 1)
1675 return (DDI_FAILURE);
1676 physcontig = 1;
1677 } else {
1678 length = P2ROUNDUP_TYPED(length, iomin, size_t);
1679 }
1680
1681 /*
1682 * Allocate the requested amount from the system.
1683 */
1684 a = kalloca(length, iomin, cansleep, physcontig, attr);
1685
1686 if ((*kaddrp = a) == NULL)
1687 return (DDI_FAILURE);
1688
1689 /*
1690 * if we to modify the cache attributes, go back and muck with the
1691 * mappings.
1692 */
1693 if (OVERRIDE_CACHE_ATTR(flags)) {
1694 order = 0;
1695 i_ddi_cacheattr_to_hatacc(flags, &order);
1696 e = kmem_override_cache_attrs(a, length, order);
1697 if (e != 0) {
1698 kfreea(a);
1699 return (DDI_FAILURE);
1700 }
1701 }
1702
1703 if (real_length) {
1704 *real_length = length;
1705 }
1706 if (ap) {
1707 /*
1708 * initialize access handle
1709 */
1710 iap = (ddi_acc_impl_t *)ap->ah_platform_private;
1711 iap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1712 impl_acc_hdl_init(ap);
1713 }
1714
1715 return (DDI_SUCCESS);
1716 }
1717
1718 /* ARGSUSED */
1719 void
1720 i_ddi_mem_free(caddr_t kaddr, ddi_acc_hdl_t *ap)
1721 {
1722 if (ap != NULL) {
1723 /*
1724 * if we modified the cache attributes on alloc, go back and
1725 * fix them since this memory could be returned to the
1726 * general pool.
1727 */
1728 if (OVERRIDE_CACHE_ATTR(ap->ah_xfermodes)) {
1729 uint_t order = 0;
1730 int e;
1731 i_ddi_cacheattr_to_hatacc(IOMEM_DATA_CACHED, &order);
1732 e = kmem_override_cache_attrs(kaddr, ap->ah_len, order);
1733 if (e != 0) {
1734 cmn_err(CE_WARN, "i_ddi_mem_free() failed to "
1735 "override cache attrs, memory leaked\n");
1736 return;
1737 }
1738 }
1739 }
1740 kfreea(kaddr);
1741 }
1742
1743 /*
1744 * Access Barriers
1745 *
1746 */
1747 /*ARGSUSED*/
1748 int
1749 i_ddi_ontrap(ddi_acc_handle_t hp)
1750 {
1751 return (DDI_FAILURE);
1752 }
1753
1754 /*ARGSUSED*/
1755 void
1756 i_ddi_notrap(ddi_acc_handle_t hp)
1757 {
1758 }
1759
1760
1761 /*
1762 * Misc Functions
1763 */
1764
1765 /*
1766 * Implementation instance override functions
1767 *
1768 * No override on i86pc
1769 */
1770 /*ARGSUSED*/
1771 uint_t
1772 impl_assign_instance(dev_info_t *dip)
1773 {
1774 return ((uint_t)-1);
1775 }
1776
1777 /*ARGSUSED*/
1778 int
1779 impl_keep_instance(dev_info_t *dip)
1780 {
1781
1782 #if defined(__xpv)
1783 /*
1784 * Do not persist instance numbers assigned to devices in dom0
1785 */
1786 dev_info_t *pdip;
1787 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1788 if (((pdip = ddi_get_parent(dip)) != NULL) &&
1789 (strcmp(ddi_get_name(pdip), "xpvd") == 0))
1790 return (DDI_SUCCESS);
1791 }
1792 #endif
1793 return (DDI_FAILURE);
1794 }
1795
1796 /*ARGSUSED*/
1797 int
1798 impl_free_instance(dev_info_t *dip)
1799 {
1800 return (DDI_FAILURE);
1801 }
1802
1803 /*ARGSUSED*/
1804 int
1805 impl_check_cpu(dev_info_t *devi)
1806 {
1807 return (DDI_SUCCESS);
1808 }
1809
1810 /*
1811 * Referenced in common/cpr_driver.c: Power off machine.
1812 * Don't know how to power off i86pc.
1813 */
1814 void
1815 arch_power_down()
1816 {}
1817
1818 /*
1819 * Copy name to property_name, since name
1820 * is in the low address range below kernelbase.
1821 */
1822 static void
1823 copy_boot_str(const char *boot_str, char *kern_str, int len)
1824 {
1825 int i = 0;
1826
1827 while (i < len - 1 && boot_str[i] != '\0') {
1828 kern_str[i] = boot_str[i];
1829 i++;
1830 }
1831
1832 kern_str[i] = 0; /* null terminate */
1833 if (boot_str[i] != '\0')
1834 cmn_err(CE_WARN,
1835 "boot property string is truncated to %s", kern_str);
1836 }
1837
1838 static void
1839 get_boot_properties(void)
1840 {
1841 extern char hw_provider[];
1842 dev_info_t *devi;
1843 char *name;
1844 int length;
1845 char property_name[50], property_val[50];
1846 void *bop_staging_area;
1847
1848 bop_staging_area = kmem_zalloc(MMU_PAGESIZE, KM_NOSLEEP);
1849
1850 /*
1851 * Import "root" properties from the boot.
1852 *
1853 * We do this by invoking BOP_NEXTPROP until the list
1854 * is completely copied in.
1855 */
1856
1857 devi = ddi_root_node();
1858 for (name = BOP_NEXTPROP(bootops, ""); /* get first */
1859 name; /* NULL => DONE */
1860 name = BOP_NEXTPROP(bootops, name)) { /* get next */
1861
1862 /* copy string to memory above kernelbase */
1863 copy_boot_str(name, property_name, 50);
1864
1865 /*
1866 * Skip vga properties. They will be picked up later
1867 * by get_vga_properties.
1868 */
1869 if (strcmp(property_name, "display-edif-block") == 0 ||
1870 strcmp(property_name, "display-edif-id") == 0) {
1871 continue;
1872 }
1873
1874 length = BOP_GETPROPLEN(bootops, property_name);
1875 if (length == 0)
1876 continue;
1877 if (length > MMU_PAGESIZE) {
1878 cmn_err(CE_NOTE,
1879 "boot property %s longer than 0x%x, ignored\n",
1880 property_name, MMU_PAGESIZE);
1881 continue;
1882 }
1883 BOP_GETPROP(bootops, property_name, bop_staging_area);
1884
1885 /*
1886 * special properties:
1887 * si-machine, si-hw-provider
1888 * goes to kernel data structures.
1889 * bios-boot-device and stdout
1890 * goes to hardware property list so it may show up
1891 * in the prtconf -vp output. This is needed by
1892 * Install/Upgrade. Once we fix install upgrade,
1893 * this can be taken out.
1894 */
1895 if (strcmp(name, "si-machine") == 0) {
1896 (void) strncpy(utsname.machine, bop_staging_area,
1897 SYS_NMLN);
1898 utsname.machine[SYS_NMLN - 1] = (char)NULL;
1899 } else if (strcmp(name, "si-hw-provider") == 0) {
1900 (void) strncpy(hw_provider, bop_staging_area, SYS_NMLN);
1901 hw_provider[SYS_NMLN - 1] = (char)NULL;
1902 } else if (strcmp(name, "bios-boot-device") == 0) {
1903 copy_boot_str(bop_staging_area, property_val, 50);
1904 (void) ndi_prop_update_string(DDI_DEV_T_NONE, devi,
1905 property_name, property_val);
1906 } else if (strcmp(name, "acpi-root-tab") == 0) {
1907 (void) ndi_prop_update_int64(DDI_DEV_T_NONE, devi,
1908 property_name, *((int64_t *)bop_staging_area));
1909 } else if (strcmp(name, "smbios-address") == 0) {
1910 (void) ndi_prop_update_int64(DDI_DEV_T_NONE, devi,
1911 property_name, *((int64_t *)bop_staging_area));
1912 } else if (strcmp(name, "efi-systab") == 0) {
1913 (void) ndi_prop_update_int64(DDI_DEV_T_NONE, devi,
1914 property_name, *((int64_t *)bop_staging_area));
1915 } else if (strcmp(name, "efi-systype") == 0) {
1916 copy_boot_str(bop_staging_area, property_val, 50);
1917 (void) ndi_prop_update_string(DDI_DEV_T_NONE, devi,
1918 property_name, property_val);
1919 } else if (strcmp(name, "stdout") == 0) {
1920 (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi,
1921 property_name, *((int *)bop_staging_area));
1922 } else if (strcmp(name, "boot-args") == 0) {
1923 copy_boot_str(bop_staging_area, property_val, 50);
1924 (void) e_ddi_prop_update_string(DDI_DEV_T_NONE, devi,
1925 property_name, property_val);
1926 } else if (strcmp(name, "bootargs") == 0) {
1927 copy_boot_str(bop_staging_area, property_val, 50);
1928 (void) e_ddi_prop_update_string(DDI_DEV_T_NONE, devi,
1929 property_name, property_val);
1930 } else if (strcmp(name, "bootp-response") == 0) {
1931 (void) e_ddi_prop_update_byte_array(DDI_DEV_T_NONE,
1932 devi, property_name, bop_staging_area, length);
1933 } else if (strcmp(name, "ramdisk_start") == 0) {
1934 (void) e_ddi_prop_update_int64(DDI_DEV_T_NONE, devi,
1935 property_name, *((int64_t *)bop_staging_area));
1936 } else if (strcmp(name, "ramdisk_end") == 0) {
1937 (void) e_ddi_prop_update_int64(DDI_DEV_T_NONE, devi,
1938 property_name, *((int64_t *)bop_staging_area));
1939 } else if (strncmp(name, "module-addr-", 12) == 0) {
1940 (void) e_ddi_prop_update_int64(DDI_DEV_T_NONE, devi,
1941 property_name, *((int64_t *)bop_staging_area));
1942 } else if (strncmp(name, "module-size-", 12) == 0) {
1943 (void) e_ddi_prop_update_int64(DDI_DEV_T_NONE, devi,
1944 property_name, *((int64_t *)bop_staging_area));
1945 } else {
1946 /* Property type unknown, use old prop interface */
1947 (void) e_ddi_prop_create(DDI_DEV_T_NONE, devi,
1948 DDI_PROP_CANSLEEP, property_name, bop_staging_area,
1949 length);
1950 }
1951 }
1952
1953 kmem_free(bop_staging_area, MMU_PAGESIZE);
1954 }
1955
1956 static void
1957 get_vga_properties(void)
1958 {
1959 dev_info_t *devi;
1960 major_t major;
1961 char *name;
1962 int length;
1963 char property_val[50];
1964 void *bop_staging_area;
1965
1966 /*
1967 * XXXX Hack Allert!
1968 * There really needs to be a better way for identifying various
1969 * console framebuffers and their related issues. Till then,
1970 * check for this one as a replacement to vgatext.
1971 */
1972 major = ddi_name_to_major("ragexl");
1973 if (major == (major_t)-1) {
1974 major = ddi_name_to_major("vgatext");
1975 if (major == (major_t)-1)
1976 return;
1977 }
1978 devi = devnamesp[major].dn_head;
1979 if (devi == NULL)
1980 return;
1981
1982 bop_staging_area = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
1983
1984 /*
1985 * Import "vga" properties from the boot.
1986 */
1987 name = "display-edif-block";
1988 length = BOP_GETPROPLEN(bootops, name);
1989 if (length > 0 && length < MMU_PAGESIZE) {
1990 BOP_GETPROP(bootops, name, bop_staging_area);
1991 (void) ndi_prop_update_byte_array(DDI_DEV_T_NONE,
1992 devi, name, bop_staging_area, length);
1993 }
1994
1995 /*
1996 * kdmconfig is also looking for display-type and
1997 * video-adapter-type. We default to color and svga.
1998 *
1999 * Could it be "monochrome", "vga"?
2000 * Nah, you've got to come to the 21st century...
2001 * And you can set monitor type manually in kdmconfig
2002 * if you are really an old junky.
2003 */
2004 (void) ndi_prop_update_string(DDI_DEV_T_NONE,
2005 devi, "display-type", "color");
2006 (void) ndi_prop_update_string(DDI_DEV_T_NONE,
2007 devi, "video-adapter-type", "svga");
2008
2009 name = "display-edif-id";
2010 length = BOP_GETPROPLEN(bootops, name);
2011 if (length > 0 && length < MMU_PAGESIZE) {
2012 BOP_GETPROP(bootops, name, bop_staging_area);
2013 copy_boot_str(bop_staging_area, property_val, length);
2014 (void) ndi_prop_update_string(DDI_DEV_T_NONE,
2015 devi, name, property_val);
2016 }
2017
2018 kmem_free(bop_staging_area, MMU_PAGESIZE);
2019 }
2020
2021
2022 /*
2023 * This is temporary, but absolutely necessary. If we are being
2024 * booted with a device tree created by the DevConf project's bootconf
2025 * program, then we have device information nodes that reflect
2026 * reality. At this point in time in the Solaris release schedule, the
2027 * kernel drivers aren't prepared for reality. They still depend on their
2028 * own ad-hoc interpretations of the properties created when their .conf
2029 * files were interpreted. These drivers use an "ignore-hardware-nodes"
2030 * property to prevent them from using the nodes passed up from the bootconf
2031 * device tree.
2032 *
2033 * Trying to assemble root file system drivers as we are booting from
2034 * devconf will fail if the kernel driver is basing its name_addr's on the
2035 * psuedo-node device info while the bootpath passed up from bootconf is using
2036 * reality-based name_addrs. We help the boot along in this case by
2037 * looking at the pre-bootconf bootpath and determining if we would have
2038 * successfully matched if that had been the bootpath we had chosen.
2039 *
2040 * Note that we only even perform this extra check if we've booted
2041 * using bootconf's 1275 compliant bootpath, this is the boot device, and
2042 * we're trying to match the name_addr specified in the 1275 bootpath.
2043 */
2044
2045 #define MAXCOMPONENTLEN 32
2046
2047 int
2048 x86_old_bootpath_name_addr_match(dev_info_t *cdip, char *caddr, char *naddr)
2049 {
2050 /*
2051 * There are multiple criteria to be met before we can even
2052 * consider allowing a name_addr match here.
2053 *
2054 * 1) We must have been booted such that the bootconf program
2055 * created device tree nodes and properties. This can be
2056 * determined by examining the 'bootpath' property. This
2057 * property will be a non-null string iff bootconf was
2058 * involved in the boot.
2059 *
2060 * 2) The module that we want to match must be the boot device.
2061 *
2062 * 3) The instance of the module we are thinking of letting be
2063 * our match must be ignoring hardware nodes.
2064 *
2065 * 4) The name_addr we want to match must be the name_addr
2066 * specified in the 1275 bootpath.
2067 */
2068 static char bootdev_module[MAXCOMPONENTLEN];
2069 static char bootdev_oldmod[MAXCOMPONENTLEN];
2070 static char bootdev_newaddr[MAXCOMPONENTLEN];
2071 static char bootdev_oldaddr[MAXCOMPONENTLEN];
2072 static int quickexit;
2073
2074 char *daddr;
2075 int dlen;
2076
2077 char *lkupname;
2078 int rv = DDI_FAILURE;
2079
2080 if ((ddi_getlongprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
2081 "devconf-addr", (caddr_t)&daddr, &dlen) == DDI_PROP_SUCCESS) &&
2082 (ddi_getprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
2083 "ignore-hardware-nodes", -1) != -1)) {
2084 if (strcmp(daddr, caddr) == 0) {
2085 return (DDI_SUCCESS);
2086 }
2087 }
2088
2089 if (quickexit)
2090 return (rv);
2091
2092 if (bootdev_module[0] == '\0') {
2093 char *addrp, *eoaddrp;
2094 char *busp, *modp, *atp;
2095 char *bp1275, *bp;
2096 int bp1275len, bplen;
2097
2098 bp1275 = bp = addrp = eoaddrp = busp = modp = atp = NULL;
2099
2100 if (ddi_getlongprop(DDI_DEV_T_ANY,
2101 ddi_root_node(), 0, "bootpath",
2102 (caddr_t)&bp1275, &bp1275len) != DDI_PROP_SUCCESS ||
2103 bp1275len <= 1) {
2104 /*
2105 * We didn't boot from bootconf so we never need to
2106 * do any special matches.
2107 */
2108 quickexit = 1;
2109 if (bp1275)
2110 kmem_free(bp1275, bp1275len);
2111 return (rv);
2112 }
2113
2114 if (ddi_getlongprop(DDI_DEV_T_ANY,
2115 ddi_root_node(), 0, "boot-path",
2116 (caddr_t)&bp, &bplen) != DDI_PROP_SUCCESS || bplen <= 1) {
2117 /*
2118 * No fallback position for matching. This is
2119 * certainly unexpected, but we'll handle it
2120 * just in case.
2121 */
2122 quickexit = 1;
2123 kmem_free(bp1275, bp1275len);
2124 if (bp)
2125 kmem_free(bp, bplen);
2126 return (rv);
2127 }
2128
2129 /*
2130 * Determine boot device module and 1275 name_addr
2131 *
2132 * bootpath assumed to be of the form /bus/module@name_addr
2133 */
2134 if (busp = strchr(bp1275, '/')) {
2135 if (modp = strchr(busp + 1, '/')) {
2136 if (atp = strchr(modp + 1, '@')) {
2137 *atp = '\0';
2138 addrp = atp + 1;
2139 if (eoaddrp = strchr(addrp, '/'))
2140 *eoaddrp = '\0';
2141 }
2142 }
2143 }
2144
2145 if (modp && addrp) {
2146 (void) strncpy(bootdev_module, modp + 1,
2147 MAXCOMPONENTLEN);
2148 bootdev_module[MAXCOMPONENTLEN - 1] = '\0';
2149
2150 (void) strncpy(bootdev_newaddr, addrp, MAXCOMPONENTLEN);
2151 bootdev_newaddr[MAXCOMPONENTLEN - 1] = '\0';
2152 } else {
2153 quickexit = 1;
2154 kmem_free(bp1275, bp1275len);
2155 kmem_free(bp, bplen);
2156 return (rv);
2157 }
2158
2159 /*
2160 * Determine fallback name_addr
2161 *
2162 * 10/3/96 - Also save fallback module name because it
2163 * might actually be different than the current module
2164 * name. E.G., ISA pnp drivers have new names.
2165 *
2166 * bootpath assumed to be of the form /bus/module@name_addr
2167 */
2168 addrp = NULL;
2169 if (busp = strchr(bp, '/')) {
2170 if (modp = strchr(busp + 1, '/')) {
2171 if (atp = strchr(modp + 1, '@')) {
2172 *atp = '\0';
2173 addrp = atp + 1;
2174 if (eoaddrp = strchr(addrp, '/'))
2175 *eoaddrp = '\0';
2176 }
2177 }
2178 }
2179
2180 if (modp && addrp) {
2181 (void) strncpy(bootdev_oldmod, modp + 1,
2182 MAXCOMPONENTLEN);
2183 bootdev_module[MAXCOMPONENTLEN - 1] = '\0';
2184
2185 (void) strncpy(bootdev_oldaddr, addrp, MAXCOMPONENTLEN);
2186 bootdev_oldaddr[MAXCOMPONENTLEN - 1] = '\0';
2187 }
2188
2189 /* Free up the bootpath storage now that we're done with it. */
2190 kmem_free(bp1275, bp1275len);
2191 kmem_free(bp, bplen);
2192
2193 if (bootdev_oldaddr[0] == '\0') {
2194 quickexit = 1;
2195 return (rv);
2196 }
2197 }
2198
2199 if (((lkupname = ddi_get_name(cdip)) != NULL) &&
2200 (strcmp(bootdev_module, lkupname) == 0 ||
2201 strcmp(bootdev_oldmod, lkupname) == 0) &&
2202 ((ddi_getprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
2203 "ignore-hardware-nodes", -1) != -1) ||
2204 ignore_hardware_nodes) &&
2205 strcmp(bootdev_newaddr, caddr) == 0 &&
2206 strcmp(bootdev_oldaddr, naddr) == 0) {
2207 rv = DDI_SUCCESS;
2208 }
2209
2210 return (rv);
2211 }
2212
2213 /*
2214 * Perform a copy from a memory mapped device (whose devinfo pointer is devi)
2215 * separately mapped at devaddr in the kernel to a kernel buffer at kaddr.
2216 */
2217 /*ARGSUSED*/
2218 int
2219 e_ddi_copyfromdev(dev_info_t *devi,
2220 off_t off, const void *devaddr, void *kaddr, size_t len)
2221 {
2222 bcopy(devaddr, kaddr, len);
2223 return (0);
2224 }
2225
2226 /*
2227 * Perform a copy to a memory mapped device (whose devinfo pointer is devi)
2228 * separately mapped at devaddr in the kernel from a kernel buffer at kaddr.
2229 */
2230 /*ARGSUSED*/
2231 int
2232 e_ddi_copytodev(dev_info_t *devi,
2233 off_t off, const void *kaddr, void *devaddr, size_t len)
2234 {
2235 bcopy(kaddr, devaddr, len);
2236 return (0);
2237 }
2238
2239
2240 static int
2241 poke_mem(peekpoke_ctlops_t *in_args)
2242 {
2243 int err = DDI_SUCCESS;
2244 on_trap_data_t otd;
2245
2246 /* Set up protected environment. */
2247 if (!on_trap(&otd, OT_DATA_ACCESS)) {
2248 switch (in_args->size) {
2249 case sizeof (uint8_t):
2250 *(uint8_t *)(in_args->dev_addr) =
2251 *(uint8_t *)in_args->host_addr;
2252 break;
2253
2254 case sizeof (uint16_t):
2255 *(uint16_t *)(in_args->dev_addr) =
2256 *(uint16_t *)in_args->host_addr;
2257 break;
2258
2259 case sizeof (uint32_t):
2260 *(uint32_t *)(in_args->dev_addr) =
2261 *(uint32_t *)in_args->host_addr;
2262 break;
2263
2264 case sizeof (uint64_t):
2265 *(uint64_t *)(in_args->dev_addr) =
2266 *(uint64_t *)in_args->host_addr;
2267 break;
2268
2269 default:
2270 err = DDI_FAILURE;
2271 break;
2272 }
2273 } else
2274 err = DDI_FAILURE;
2275
2276 /* Take down protected environment. */
2277 no_trap();
2278
2279 return (err);
2280 }
2281
2282
2283 static int
2284 peek_mem(peekpoke_ctlops_t *in_args)
2285 {
2286 int err = DDI_SUCCESS;
2287 on_trap_data_t otd;
2288
2289 if (!on_trap(&otd, OT_DATA_ACCESS)) {
2290 switch (in_args->size) {
2291 case sizeof (uint8_t):
2292 *(uint8_t *)in_args->host_addr =
2293 *(uint8_t *)in_args->dev_addr;
2294 break;
2295
2296 case sizeof (uint16_t):
2297 *(uint16_t *)in_args->host_addr =
2298 *(uint16_t *)in_args->dev_addr;
2299 break;
2300
2301 case sizeof (uint32_t):
2302 *(uint32_t *)in_args->host_addr =
2303 *(uint32_t *)in_args->dev_addr;
2304 break;
2305
2306 case sizeof (uint64_t):
2307 *(uint64_t *)in_args->host_addr =
2308 *(uint64_t *)in_args->dev_addr;
2309 break;
2310
2311 default:
2312 err = DDI_FAILURE;
2313 break;
2314 }
2315 } else
2316 err = DDI_FAILURE;
2317
2318 no_trap();
2319 return (err);
2320 }
2321
2322
2323 /*
2324 * This is called only to process peek/poke when the DIP is NULL.
2325 * Assume that this is for memory, as nexi take care of device safe accesses.
2326 */
2327 int
2328 peekpoke_mem(ddi_ctl_enum_t cmd, peekpoke_ctlops_t *in_args)
2329 {
2330 return (cmd == DDI_CTLOPS_PEEK ? peek_mem(in_args) : poke_mem(in_args));
2331 }
2332
2333 /*
2334 * we've just done a cautious put/get. Check if it was successful by
2335 * calling pci_ereport_post() on all puts and for any gets that return -1
2336 */
2337 static int
2338 pci_peekpoke_check_fma(dev_info_t *dip, void *arg, ddi_ctl_enum_t ctlop,
2339 void (*scan)(dev_info_t *, ddi_fm_error_t *))
2340 {
2341 int rval = DDI_SUCCESS;
2342 peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg;
2343 ddi_fm_error_t de;
2344 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
2345 ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle;
2346 int check_err = 0;
2347 int repcount = in_args->repcount;
2348
2349 if (ctlop == DDI_CTLOPS_POKE &&
2350 hdlp->ah_acc.devacc_attr_access != DDI_CAUTIOUS_ACC)
2351 return (DDI_SUCCESS);
2352
2353 if (ctlop == DDI_CTLOPS_PEEK &&
2354 hdlp->ah_acc.devacc_attr_access != DDI_CAUTIOUS_ACC) {
2355 for (; repcount; repcount--) {
2356 switch (in_args->size) {
2357 case sizeof (uint8_t):
2358 if (*(uint8_t *)in_args->host_addr == 0xff)
2359 check_err = 1;
2360 break;
2361 case sizeof (uint16_t):
2362 if (*(uint16_t *)in_args->host_addr == 0xffff)
2363 check_err = 1;
2364 break;
2365 case sizeof (uint32_t):
2366 if (*(uint32_t *)in_args->host_addr ==
2367 0xffffffff)
2368 check_err = 1;
2369 break;
2370 case sizeof (uint64_t):
2371 if (*(uint64_t *)in_args->host_addr ==
2372 0xffffffffffffffff)
2373 check_err = 1;
2374 break;
2375 }
2376 }
2377 if (check_err == 0)
2378 return (DDI_SUCCESS);
2379 }
2380 /*
2381 * for a cautious put or get or a non-cautious get that returned -1 call
2382 * io framework to see if there really was an error
2383 */
2384 bzero(&de, sizeof (ddi_fm_error_t));
2385 de.fme_version = DDI_FME_VERSION;
2386 de.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
2387 if (hdlp->ah_acc.devacc_attr_access == DDI_CAUTIOUS_ACC) {
2388 de.fme_flag = DDI_FM_ERR_EXPECTED;
2389 de.fme_acc_handle = in_args->handle;
2390 } else if (hdlp->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
2391 /*
2392 * We only get here with DDI_DEFAULT_ACC for config space gets.
2393 * Non-hardened drivers may be probing the hardware and
2394 * expecting -1 returned. So need to treat errors on
2395 * DDI_DEFAULT_ACC as DDI_FM_ERR_EXPECTED.
2396 */
2397 de.fme_flag = DDI_FM_ERR_EXPECTED;
2398 de.fme_acc_handle = in_args->handle;
2399 } else {
2400 /*
2401 * Hardened driver doing protected accesses shouldn't
2402 * get errors unless there's a hardware problem. Treat
2403 * as nonfatal if there's an error, but set UNEXPECTED
2404 * so we raise ereports on any errors and potentially
2405 * fault the device
2406 */
2407 de.fme_flag = DDI_FM_ERR_UNEXPECTED;
2408 }
2409 (void) scan(dip, &de);
2410 if (hdlp->ah_acc.devacc_attr_access != DDI_DEFAULT_ACC &&
2411 de.fme_status != DDI_FM_OK) {
2412 ndi_err_t *errp = (ndi_err_t *)hp->ahi_err;
2413 rval = DDI_FAILURE;
2414 errp->err_ena = de.fme_ena;
2415 errp->err_expected = de.fme_flag;
2416 errp->err_status = DDI_FM_NONFATAL;
2417 }
2418 return (rval);
2419 }
2420
2421 /*
2422 * pci_peekpoke_check_nofma() is for when an error occurs on a register access
2423 * during pci_ereport_post(). We can't call pci_ereport_post() again or we'd
2424 * recurse, so assume all puts are OK and gets have failed if they return -1
2425 */
2426 static int
2427 pci_peekpoke_check_nofma(void *arg, ddi_ctl_enum_t ctlop)
2428 {
2429 int rval = DDI_SUCCESS;
2430 peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg;
2431 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
2432 ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle;
2433 int repcount = in_args->repcount;
2434
2435 if (ctlop == DDI_CTLOPS_POKE)
2436 return (rval);
2437
2438 for (; repcount; repcount--) {
2439 switch (in_args->size) {
2440 case sizeof (uint8_t):
2441 if (*(uint8_t *)in_args->host_addr == 0xff)
2442 rval = DDI_FAILURE;
2443 break;
2444 case sizeof (uint16_t):
2445 if (*(uint16_t *)in_args->host_addr == 0xffff)
2446 rval = DDI_FAILURE;
2447 break;
2448 case sizeof (uint32_t):
2449 if (*(uint32_t *)in_args->host_addr == 0xffffffff)
2450 rval = DDI_FAILURE;
2451 break;
2452 case sizeof (uint64_t):
2453 if (*(uint64_t *)in_args->host_addr ==
2454 0xffffffffffffffff)
2455 rval = DDI_FAILURE;
2456 break;
2457 }
2458 }
2459 if (hdlp->ah_acc.devacc_attr_access != DDI_DEFAULT_ACC &&
2460 rval == DDI_FAILURE) {
2461 ndi_err_t *errp = (ndi_err_t *)hp->ahi_err;
2462 errp->err_ena = fm_ena_generate(0, FM_ENA_FMT1);
2463 errp->err_expected = DDI_FM_ERR_UNEXPECTED;
2464 errp->err_status = DDI_FM_NONFATAL;
2465 }
2466 return (rval);
2467 }
2468
2469 int
2470 pci_peekpoke_check(dev_info_t *dip, dev_info_t *rdip,
2471 ddi_ctl_enum_t ctlop, void *arg, void *result,
2472 int (*handler)(dev_info_t *, dev_info_t *, ddi_ctl_enum_t, void *,
2473 void *), kmutex_t *err_mutexp, kmutex_t *peek_poke_mutexp,
2474 void (*scan)(dev_info_t *, ddi_fm_error_t *))
2475 {
2476 int rval;
2477 peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg;
2478 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
2479
2480 /*
2481 * this function only supports cautious accesses, not peeks/pokes
2482 * which don't have a handle
2483 */
2484 if (hp == NULL)
2485 return (DDI_FAILURE);
2486
2487 if (hp->ahi_acc_attr & DDI_ACCATTR_CONFIG_SPACE) {
2488 if (!mutex_tryenter(err_mutexp)) {
2489 /*
2490 * As this may be a recursive call from within
2491 * pci_ereport_post() we can't wait for the mutexes.
2492 * Fortunately we know someone is already calling
2493 * pci_ereport_post() which will handle the error bits
2494 * for us, and as this is a config space access we can
2495 * just do the access and check return value for -1
2496 * using pci_peekpoke_check_nofma().
2497 */
2498 rval = handler(dip, rdip, ctlop, arg, result);
2499 if (rval == DDI_SUCCESS)
2500 rval = pci_peekpoke_check_nofma(arg, ctlop);
2501 return (rval);
2502 }
2503 /*
2504 * This can't be a recursive call. Drop the err_mutex and get
2505 * both mutexes in the right order. If an error hasn't already
2506 * been detected by the ontrap code, use pci_peekpoke_check_fma
2507 * which will call pci_ereport_post() to check error status.
2508 */
2509 mutex_exit(err_mutexp);
2510 }
2511 mutex_enter(peek_poke_mutexp);
2512 rval = handler(dip, rdip, ctlop, arg, result);
2513 if (rval == DDI_SUCCESS) {
2514 mutex_enter(err_mutexp);
2515 rval = pci_peekpoke_check_fma(dip, arg, ctlop, scan);
2516 mutex_exit(err_mutexp);
2517 }
2518 mutex_exit(peek_poke_mutexp);
2519 return (rval);
2520 }
2521
2522 void
2523 impl_setup_ddi(void)
2524 {
2525 #if !defined(__xpv)
2526 extern void startup_bios_disk(void);
2527 extern int post_fastreboot;
2528 #endif
2529 dev_info_t *xdip, *isa_dip;
2530 rd_existing_t rd_mem_prop;
2531 int err;
2532
2533 ndi_devi_alloc_sleep(ddi_root_node(), "ramdisk",
2534 (pnode_t)DEVI_SID_NODEID, &xdip);
2535
2536 (void) BOP_GETPROP(bootops,
2537 "ramdisk_start", (void *)&ramdisk_start);
2538 (void) BOP_GETPROP(bootops,
2539 "ramdisk_end", (void *)&ramdisk_end);
2540
2541 #ifdef __xpv
2542 ramdisk_start -= ONE_GIG;
2543 ramdisk_end -= ONE_GIG;
2544 #endif
2545 rd_mem_prop.phys = ramdisk_start;
2546 rd_mem_prop.size = ramdisk_end - ramdisk_start + 1;
2547
2548 (void) ndi_prop_update_byte_array(DDI_DEV_T_NONE, xdip,
2549 RD_EXISTING_PROP_NAME, (uchar_t *)&rd_mem_prop,
2550 sizeof (rd_mem_prop));
2551 err = ndi_devi_bind_driver(xdip, 0);
2552 ASSERT(err == 0);
2553
2554 /* isa node */
2555 if (pseudo_isa) {
2556 ndi_devi_alloc_sleep(ddi_root_node(), "isa",
2557 (pnode_t)DEVI_SID_NODEID, &isa_dip);
2558 (void) ndi_prop_update_string(DDI_DEV_T_NONE, isa_dip,
2559 "device_type", "isa");
2560 (void) ndi_prop_update_string(DDI_DEV_T_NONE, isa_dip,
2561 "bus-type", "isa");
2562 (void) ndi_devi_bind_driver(isa_dip, 0);
2563 }
2564
2565 /*
2566 * Read in the properties from the boot.
2567 */
2568 get_boot_properties();
2569
2570 /* not framebuffer should be enumerated, if present */
2571 get_vga_properties();
2572
2573 /*
2574 * Check for administratively disabled drivers.
2575 */
2576 check_driver_disable();
2577
2578 #if !defined(__xpv)
2579 if (!post_fastreboot && BOP_GETPROPLEN(bootops, "efi-systab") < 0)
2580 startup_bios_disk();
2581 #endif
2582 /* do bus dependent probes. */
2583 impl_bus_initialprobe();
2584 }
2585
2586 dev_t
2587 getrootdev(void)
2588 {
2589 /*
2590 * Usually rootfs.bo_name is initialized by the
2591 * the bootpath property from bootenv.rc, but
2592 * defaults to "/ramdisk:a" otherwise.
2593 */
2594 return (ddi_pathname_to_dev_t(rootfs.bo_name));
2595 }
2596
2597 static struct bus_probe {
2598 struct bus_probe *next;
2599 void (*probe)(int);
2600 } *bus_probes;
2601
2602 void
2603 impl_bus_add_probe(void (*func)(int))
2604 {
2605 struct bus_probe *probe;
2606 struct bus_probe *lastprobe = NULL;
2607
2608 probe = kmem_alloc(sizeof (*probe), KM_SLEEP);
2609 probe->probe = func;
2610 probe->next = NULL;
2611
2612 if (!bus_probes) {
2613 bus_probes = probe;
2614 return;
2615 }
2616
2617 lastprobe = bus_probes;
2618 while (lastprobe->next)
2619 lastprobe = lastprobe->next;
2620 lastprobe->next = probe;
2621 }
2622
2623 /*ARGSUSED*/
2624 void
2625 impl_bus_delete_probe(void (*func)(int))
2626 {
2627 struct bus_probe *prev = NULL;
2628 struct bus_probe *probe = bus_probes;
2629
2630 while (probe) {
2631 if (probe->probe == func)
2632 break;
2633 prev = probe;
2634 probe = probe->next;
2635 }
2636
2637 if (probe == NULL)
2638 return;
2639
2640 if (prev)
2641 prev->next = probe->next;
2642 else
2643 bus_probes = probe->next;
2644
2645 kmem_free(probe, sizeof (struct bus_probe));
2646 }
2647
2648 /*
2649 * impl_bus_initialprobe
2650 * Modload the prom simulator, then let it probe to verify existence
2651 * and type of PCI support.
2652 */
2653 static void
2654 impl_bus_initialprobe(void)
2655 {
2656 struct bus_probe *probe;
2657
2658 /* load modules to install bus probes */
2659 #if defined(__xpv)
2660 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
2661 if (modload("misc", "pci_autoconfig") < 0) {
2662 panic("failed to load misc/pci_autoconfig");
2663 }
2664
2665 if (modload("drv", "isa") < 0)
2666 panic("failed to load drv/isa");
2667 }
2668
2669 (void) modload("misc", "xpv_autoconfig");
2670 #else
2671 if (modload("misc", "pci_autoconfig") < 0) {
2672 panic("failed to load misc/pci_autoconfig");
2673 }
2674
2675 (void) modload("misc", "acpidev");
2676
2677 if (modload("drv", "isa") < 0)
2678 panic("failed to load drv/isa");
2679 #endif
2680
2681 probe = bus_probes;
2682 while (probe) {
2683 /* run the probe functions */
2684 (*probe->probe)(0);
2685 probe = probe->next;
2686 }
2687 }
2688
2689 /*
2690 * impl_bus_reprobe
2691 * Reprogram devices not set up by firmware.
2692 */
2693 static void
2694 impl_bus_reprobe(void)
2695 {
2696 struct bus_probe *probe;
2697
2698 probe = bus_probes;
2699 while (probe) {
2700 /* run the probe function */
2701 (*probe->probe)(1);
2702 probe = probe->next;
2703 }
2704 }
2705
2706
2707 /*
2708 * The following functions ready a cautious request to go up to the nexus
2709 * driver. It is up to the nexus driver to decide how to process the request.
2710 * It may choose to call i_ddi_do_caut_get/put in this file, or do it
2711 * differently.
2712 */
2713
2714 static void
2715 i_ddi_caut_getput_ctlops(ddi_acc_impl_t *hp, uint64_t host_addr,
2716 uint64_t dev_addr, size_t size, size_t repcount, uint_t flags,
2717 ddi_ctl_enum_t cmd)
2718 {
2719 peekpoke_ctlops_t cautacc_ctlops_arg;
2720
2721 cautacc_ctlops_arg.size = size;
2722 cautacc_ctlops_arg.dev_addr = dev_addr;
2723 cautacc_ctlops_arg.host_addr = host_addr;
2724 cautacc_ctlops_arg.handle = (ddi_acc_handle_t)hp;
2725 cautacc_ctlops_arg.repcount = repcount;
2726 cautacc_ctlops_arg.flags = flags;
2727
2728 (void) ddi_ctlops(hp->ahi_common.ah_dip, hp->ahi_common.ah_dip, cmd,
2729 &cautacc_ctlops_arg, NULL);
2730 }
2731
2732 uint8_t
2733 i_ddi_caut_get8(ddi_acc_impl_t *hp, uint8_t *addr)
2734 {
2735 uint8_t value;
2736 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2737 sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK);
2738
2739 return (value);
2740 }
2741
2742 uint16_t
2743 i_ddi_caut_get16(ddi_acc_impl_t *hp, uint16_t *addr)
2744 {
2745 uint16_t value;
2746 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2747 sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK);
2748
2749 return (value);
2750 }
2751
2752 uint32_t
2753 i_ddi_caut_get32(ddi_acc_impl_t *hp, uint32_t *addr)
2754 {
2755 uint32_t value;
2756 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2757 sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK);
2758
2759 return (value);
2760 }
2761
2762 uint64_t
2763 i_ddi_caut_get64(ddi_acc_impl_t *hp, uint64_t *addr)
2764 {
2765 uint64_t value;
2766 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2767 sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK);
2768
2769 return (value);
2770 }
2771
2772 void
2773 i_ddi_caut_put8(ddi_acc_impl_t *hp, uint8_t *addr, uint8_t value)
2774 {
2775 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2776 sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE);
2777 }
2778
2779 void
2780 i_ddi_caut_put16(ddi_acc_impl_t *hp, uint16_t *addr, uint16_t value)
2781 {
2782 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2783 sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE);
2784 }
2785
2786 void
2787 i_ddi_caut_put32(ddi_acc_impl_t *hp, uint32_t *addr, uint32_t value)
2788 {
2789 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2790 sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE);
2791 }
2792
2793 void
2794 i_ddi_caut_put64(ddi_acc_impl_t *hp, uint64_t *addr, uint64_t value)
2795 {
2796 i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2797 sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE);
2798 }
2799
2800 void
2801 i_ddi_caut_rep_get8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
2802 size_t repcount, uint_t flags)
2803 {
2804 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2805 sizeof (uint8_t), repcount, flags, DDI_CTLOPS_PEEK);
2806 }
2807
2808 void
2809 i_ddi_caut_rep_get16(ddi_acc_impl_t *hp, uint16_t *host_addr,
2810 uint16_t *dev_addr, size_t repcount, uint_t flags)
2811 {
2812 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2813 sizeof (uint16_t), repcount, flags, DDI_CTLOPS_PEEK);
2814 }
2815
2816 void
2817 i_ddi_caut_rep_get32(ddi_acc_impl_t *hp, uint32_t *host_addr,
2818 uint32_t *dev_addr, size_t repcount, uint_t flags)
2819 {
2820 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2821 sizeof (uint32_t), repcount, flags, DDI_CTLOPS_PEEK);
2822 }
2823
2824 void
2825 i_ddi_caut_rep_get64(ddi_acc_impl_t *hp, uint64_t *host_addr,
2826 uint64_t *dev_addr, size_t repcount, uint_t flags)
2827 {
2828 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2829 sizeof (uint64_t), repcount, flags, DDI_CTLOPS_PEEK);
2830 }
2831
2832 void
2833 i_ddi_caut_rep_put8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
2834 size_t repcount, uint_t flags)
2835 {
2836 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2837 sizeof (uint8_t), repcount, flags, DDI_CTLOPS_POKE);
2838 }
2839
2840 void
2841 i_ddi_caut_rep_put16(ddi_acc_impl_t *hp, uint16_t *host_addr,
2842 uint16_t *dev_addr, size_t repcount, uint_t flags)
2843 {
2844 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2845 sizeof (uint16_t), repcount, flags, DDI_CTLOPS_POKE);
2846 }
2847
2848 void
2849 i_ddi_caut_rep_put32(ddi_acc_impl_t *hp, uint32_t *host_addr,
2850 uint32_t *dev_addr, size_t repcount, uint_t flags)
2851 {
2852 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2853 sizeof (uint32_t), repcount, flags, DDI_CTLOPS_POKE);
2854 }
2855
2856 void
2857 i_ddi_caut_rep_put64(ddi_acc_impl_t *hp, uint64_t *host_addr,
2858 uint64_t *dev_addr, size_t repcount, uint_t flags)
2859 {
2860 i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2861 sizeof (uint64_t), repcount, flags, DDI_CTLOPS_POKE);
2862 }
2863
2864 boolean_t
2865 i_ddi_copybuf_required(ddi_dma_attr_t *attrp)
2866 {
2867 uint64_t hi_pa;
2868
2869 hi_pa = ((uint64_t)physmax + 1ull) << PAGESHIFT;
2870 if (attrp->dma_attr_addr_hi < hi_pa) {
2871 return (B_TRUE);
2872 }
2873
2874 return (B_FALSE);
2875 }
2876
2877 size_t
2878 i_ddi_copybuf_size()
2879 {
2880 return (dma_max_copybuf_size);
2881 }
2882
2883 /*
2884 * i_ddi_dma_max()
2885 * returns the maximum DMA size which can be performed in a single DMA
2886 * window taking into account the devices DMA contraints (attrp), the
2887 * maximum copy buffer size (if applicable), and the worse case buffer
2888 * fragmentation.
2889 */
2890 /*ARGSUSED*/
2891 uint32_t
2892 i_ddi_dma_max(dev_info_t *dip, ddi_dma_attr_t *attrp)
2893 {
2894 uint64_t maxxfer;
2895
2896
2897 /*
2898 * take the min of maxxfer and the the worse case fragementation
2899 * (e.g. every cookie <= 1 page)
2900 */
2901 maxxfer = MIN(attrp->dma_attr_maxxfer,
2902 ((uint64_t)(attrp->dma_attr_sgllen - 1) << PAGESHIFT));
2903
2904 /*
2905 * If the DMA engine can't reach all off memory, we also need to take
2906 * the max size of the copybuf into consideration.
2907 */
2908 if (i_ddi_copybuf_required(attrp)) {
2909 maxxfer = MIN(i_ddi_copybuf_size(), maxxfer);
2910 }
2911
2912 /*
2913 * we only return a 32-bit value. Make sure it's not -1. Round to a
2914 * page so it won't be mistaken for an error value during debug.
2915 */
2916 if (maxxfer >= 0xFFFFFFFF) {
2917 maxxfer = 0xFFFFF000;
2918 }
2919
2920 /*
2921 * make sure the value we return is a whole multiple of the
2922 * granlarity.
2923 */
2924 if (attrp->dma_attr_granular > 1) {
2925 maxxfer = maxxfer - (maxxfer % attrp->dma_attr_granular);
2926 }
2927
2928 return ((uint32_t)maxxfer);
2929 }
2930
2931 /*ARGSUSED*/
2932 void
2933 translate_devid(dev_info_t *dip)
2934 {
2935 }
2936
2937 pfn_t
2938 i_ddi_paddr_to_pfn(paddr_t paddr)
2939 {
2940 pfn_t pfn;
2941
2942 #ifdef __xpv
2943 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
2944 pfn = xen_assign_pfn(mmu_btop(paddr));
2945 } else {
2946 pfn = mmu_btop(paddr);
2947 }
2948 #else
2949 pfn = mmu_btop(paddr);
2950 #endif
2951
2952 return (pfn);
2953 }