1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24 /*
25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved.
27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
28 * Copyright 2017 Joyent, Inc.
29 */
30
31 /*
32 * x86 root nexus driver
33 */
34
35 #include <sys/sysmacros.h>
36 #include <sys/conf.h>
37 #include <sys/autoconf.h>
38 #include <sys/sysmacros.h>
39 #include <sys/debug.h>
40 #include <sys/psw.h>
41 #include <sys/ddidmareq.h>
42 #include <sys/promif.h>
43 #include <sys/devops.h>
44 #include <sys/kmem.h>
45 #include <sys/cmn_err.h>
46 #include <vm/seg.h>
47 #include <vm/seg_kmem.h>
48 #include <vm/seg_dev.h>
49 #include <sys/vmem.h>
50 #include <sys/mman.h>
51 #include <vm/hat.h>
52 #include <vm/as.h>
53 #include <vm/page.h>
54 #include <sys/avintr.h>
55 #include <sys/errno.h>
56 #include <sys/modctl.h>
57 #include <sys/ddi_impldefs.h>
58 #include <sys/sunddi.h>
59 #include <sys/sunndi.h>
60 #include <sys/mach_intr.h>
61 #include <sys/psm.h>
62 #include <sys/ontrap.h>
63 #include <sys/atomic.h>
64 #include <sys/sdt.h>
65 #include <sys/rootnex.h>
66 #include <vm/hat_i86.h>
67 #include <sys/ddifm.h>
68 #include <sys/ddi_isa.h>
69 #include <sys/apic.h>
70
71 #ifdef __xpv
72 #include <sys/bootinfo.h>
73 #include <sys/hypervisor.h>
74 #include <sys/bootconf.h>
75 #include <vm/kboot_mmu.h>
76 #endif
77
78 #if defined(__amd64) && !defined(__xpv)
79 #include <sys/immu.h>
80 #endif
81
82
83 /*
84 * enable/disable extra checking of function parameters. Useful for debugging
85 * drivers.
86 */
87 #ifdef DEBUG
88 int rootnex_alloc_check_parms = 1;
89 int rootnex_bind_check_parms = 1;
90 int rootnex_bind_check_inuse = 1;
91 int rootnex_unbind_verify_buffer = 0;
92 int rootnex_sync_check_parms = 1;
93 #else
94 int rootnex_alloc_check_parms = 0;
95 int rootnex_bind_check_parms = 0;
96 int rootnex_bind_check_inuse = 0;
97 int rootnex_unbind_verify_buffer = 0;
98 int rootnex_sync_check_parms = 0;
99 #endif
100
101 boolean_t rootnex_dmar_not_setup;
102
103 /* Master Abort and Target Abort panic flag */
104 int rootnex_fm_ma_ta_panic_flag = 0;
105
106 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
107 int rootnex_bind_fail = 1;
108 int rootnex_bind_warn = 1;
109 uint8_t *rootnex_warn_list;
110 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
111 #define ROOTNEX_BIND_WARNING (0x1 << 0)
112
113 /*
114 * revert back to old broken behavior of always sync'ing entire copy buffer.
115 * This is useful if be have a buggy driver which doesn't correctly pass in
116 * the offset and size into ddi_dma_sync().
117 */
118 int rootnex_sync_ignore_params = 0;
119
120 /*
121 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
122 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
123 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
124 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
125 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
126 * (< 8K). We will still need to allocate the copy buffer during bind though
127 * (if we need one). These can only be modified in /etc/system before rootnex
128 * attach.
129 */
130 #if defined(__amd64)
131 int rootnex_prealloc_cookies = 65;
132 int rootnex_prealloc_windows = 4;
133 int rootnex_prealloc_copybuf = 2;
134 #else
135 int rootnex_prealloc_cookies = 33;
136 int rootnex_prealloc_windows = 4;
137 int rootnex_prealloc_copybuf = 2;
138 #endif
139
140 /* driver global state */
141 static rootnex_state_t *rootnex_state;
142
143 #ifdef DEBUG
144 /* shortcut to rootnex counters */
145 static uint64_t *rootnex_cnt;
146 #endif
147
148 /*
149 * XXX - does x86 even need these or are they left over from the SPARC days?
150 */
151 /* statically defined integer/boolean properties for the root node */
152 static rootnex_intprop_t rootnex_intprp[] = {
153 { "PAGESIZE", PAGESIZE },
154 { "MMU_PAGESIZE", MMU_PAGESIZE },
155 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET },
156 { DDI_RELATIVE_ADDRESSING, 1 },
157 };
158 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
159
160 /*
161 * If we're dom0, we're using a real device so we need to load
162 * the cookies with MFNs instead of PFNs.
163 */
164 #ifdef __xpv
165 typedef maddr_t rootnex_addr_t;
166 #define ROOTNEX_PADDR_TO_RBASE(pa) \
167 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
168 #else
169 typedef paddr_t rootnex_addr_t;
170 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa)
171 #endif
172
173 static struct cb_ops rootnex_cb_ops = {
174 nodev, /* open */
175 nodev, /* close */
176 nodev, /* strategy */
177 nodev, /* print */
178 nodev, /* dump */
179 nodev, /* read */
180 nodev, /* write */
181 nodev, /* ioctl */
182 nodev, /* devmap */
183 nodev, /* mmap */
184 nodev, /* segmap */
185 nochpoll, /* chpoll */
186 ddi_prop_op, /* cb_prop_op */
187 NULL, /* struct streamtab */
188 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
189 CB_REV, /* Rev */
190 nodev, /* cb_aread */
191 nodev /* cb_awrite */
192 };
193
194 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
195 off_t offset, off_t len, caddr_t *vaddrp);
196 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
197 struct hat *hat, struct seg *seg, caddr_t addr,
198 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
199 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
200 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
201 ddi_dma_handle_t *handlep);
202 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
203 ddi_dma_handle_t handle);
204 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
205 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
206 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
207 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
208 ddi_dma_handle_t handle);
209 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
210 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
211 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
212 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
213 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
214 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
215 ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
216 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
217 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
218 ddi_ctl_enum_t ctlop, void *arg, void *result);
219 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
220 ddi_iblock_cookie_t *ibc);
221 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
222 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
223 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
224 void *);
225 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
226
227 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
228 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
229 ddi_dma_handle_t *handlep);
230 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
231 ddi_dma_handle_t handle);
232 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
233 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
234 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
235 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
236 ddi_dma_handle_t handle);
237 #if defined(__amd64) && !defined(__xpv)
238 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
239 ddi_dma_handle_t handle);
240 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
241 ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
242 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
243 ddi_dma_cookie_t *cookiep, uint_t ccount);
244 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
245 ddi_dma_handle_t handle);
246 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
247 #endif
248 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
249 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
250 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
251 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
252 ddi_dma_cookie_t *cookiep, uint_t *ccountp);
253
254 #if defined(__amd64) && !defined(__xpv)
255 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
256 ddi_dma_handle_t handle, void *v);
257 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
258 ddi_dma_handle_t handle);
259 #endif
260
261
262 static struct bus_ops rootnex_bus_ops = {
263 BUSO_REV,
264 rootnex_map,
265 NULL,
266 NULL,
267 NULL,
268 rootnex_map_fault,
269 0,
270 rootnex_dma_allochdl,
271 rootnex_dma_freehdl,
272 rootnex_dma_bindhdl,
273 rootnex_dma_unbindhdl,
274 rootnex_dma_sync,
275 rootnex_dma_win,
276 rootnex_dma_mctl,
277 rootnex_ctlops,
278 ddi_bus_prop_op,
279 i_ddi_rootnex_get_eventcookie,
280 i_ddi_rootnex_add_eventcall,
281 i_ddi_rootnex_remove_eventcall,
282 i_ddi_rootnex_post_event,
283 0, /* bus_intr_ctl */
284 0, /* bus_config */
285 0, /* bus_unconfig */
286 rootnex_fm_init, /* bus_fm_init */
287 NULL, /* bus_fm_fini */
288 NULL, /* bus_fm_access_enter */
289 NULL, /* bus_fm_access_exit */
290 NULL, /* bus_powr */
291 rootnex_intr_ops /* bus_intr_op */
292 };
293
294 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
295 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
296 static int rootnex_quiesce(dev_info_t *dip);
297
298 static struct dev_ops rootnex_ops = {
299 DEVO_REV,
300 0,
301 ddi_no_info,
302 nulldev,
303 nulldev,
304 rootnex_attach,
305 rootnex_detach,
306 nulldev,
307 &rootnex_cb_ops,
308 &rootnex_bus_ops,
309 NULL,
310 rootnex_quiesce, /* quiesce */
311 };
312
313 static struct modldrv rootnex_modldrv = {
314 &mod_driverops,
315 "i86pc root nexus",
316 &rootnex_ops
317 };
318
319 static struct modlinkage rootnex_modlinkage = {
320 MODREV_1,
321 (void *)&rootnex_modldrv,
322 NULL
323 };
324
325 #if defined(__amd64) && !defined(__xpv)
326 static iommulib_nexops_t iommulib_nexops = {
327 IOMMU_NEXOPS_VERSION,
328 "Rootnex IOMMU ops Vers 1.1",
329 NULL,
330 rootnex_coredma_allochdl,
331 rootnex_coredma_freehdl,
332 rootnex_coredma_bindhdl,
333 rootnex_coredma_unbindhdl,
334 rootnex_coredma_reset_cookies,
335 rootnex_coredma_get_cookies,
336 rootnex_coredma_set_cookies,
337 rootnex_coredma_clear_cookies,
338 rootnex_coredma_get_sleep_flags,
339 rootnex_coredma_sync,
340 rootnex_coredma_win,
341 rootnex_coredma_hdl_setprivate,
342 rootnex_coredma_hdl_getprivate
343 };
344 #endif
345
346 /*
347 * extern hacks
348 */
349 extern struct seg_ops segdev_ops;
350 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */
351 #ifdef DDI_MAP_DEBUG
352 extern int ddi_map_debug_flag;
353 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf
354 #endif
355 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
356 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
357 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
358 psm_intr_op_t, int *);
359 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
360 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
361
362 /*
363 * Use device arena to use for device control register mappings.
364 * Various kernel memory walkers (debugger, dtrace) need to know
365 * to avoid this address range to prevent undesired device activity.
366 */
367 extern void *device_arena_alloc(size_t size, int vm_flag);
368 extern void device_arena_free(void * vaddr, size_t size);
369
370
371 /*
372 * Internal functions
373 */
374 static int rootnex_dma_init();
375 static void rootnex_add_props(dev_info_t *);
376 static int rootnex_ctl_reportdev(dev_info_t *dip);
377 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
378 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
379 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
380 static int rootnex_map_handle(ddi_map_req_t *mp);
381 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
382 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
383 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
384 ddi_dma_attr_t *attr);
385 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
386 rootnex_sglinfo_t *sglinfo);
387 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
388 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
389 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
390 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
391 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
392 rootnex_dma_t *dma, ddi_dma_attr_t *attr);
393 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
394 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
395 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
396 static void rootnex_teardown_windows(rootnex_dma_t *dma);
397 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
398 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
399 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
400 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
401 size_t *copybuf_used, page_t **cur_pp);
402 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
403 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
404 ddi_dma_attr_t *attr, off_t cur_offset);
405 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
406 rootnex_dma_t *dma, rootnex_window_t **windowp,
407 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
408 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
409 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
410 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
411 off_t offset, size_t size, uint_t cache_flags);
412 static int rootnex_verify_buffer(rootnex_dma_t *dma);
413 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
414 const void *comp_addr, const void *not_used);
415 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
416 rootnex_sglinfo_t *sglinfo);
417 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
418
419 /*
420 * _init()
421 *
422 */
423 int
424 _init(void)
425 {
426
427 rootnex_state = NULL;
428 return (mod_install(&rootnex_modlinkage));
429 }
430
431
432 /*
433 * _info()
434 *
435 */
436 int
437 _info(struct modinfo *modinfop)
438 {
439 return (mod_info(&rootnex_modlinkage, modinfop));
440 }
441
442
443 /*
444 * _fini()
445 *
446 */
447 int
448 _fini(void)
449 {
450 return (EBUSY);
451 }
452
453
454 /*
455 * rootnex_attach()
456 *
457 */
458 static int
459 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
460 {
461 int fmcap;
462 int e;
463
464 switch (cmd) {
465 case DDI_ATTACH:
466 break;
467 case DDI_RESUME:
468 #if defined(__amd64) && !defined(__xpv)
469 return (immu_unquiesce());
470 #else
471 return (DDI_SUCCESS);
472 #endif
473 default:
474 return (DDI_FAILURE);
475 }
476
477 /*
478 * We should only have one instance of rootnex. Save it away since we
479 * don't have an easy way to get it back later.
480 */
481 ASSERT(rootnex_state == NULL);
482 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
483
484 rootnex_state->r_dip = dip;
485 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
486 rootnex_state->r_reserved_msg_printed = B_FALSE;
487 #ifdef DEBUG
488 rootnex_cnt = &rootnex_state->r_counters[0];
489 #endif
490
491 /*
492 * Set minimum fm capability level for i86pc platforms and then
493 * initialize error handling. Since we're the rootnex, we don't
494 * care what's returned in the fmcap field.
495 */
496 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
497 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
498 fmcap = ddi_system_fmcap;
499 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
500
501 /* initialize DMA related state */
502 e = rootnex_dma_init();
503 if (e != DDI_SUCCESS) {
504 kmem_free(rootnex_state, sizeof (rootnex_state_t));
505 return (DDI_FAILURE);
506 }
507
508 /* Add static root node properties */
509 rootnex_add_props(dip);
510
511 /* since we can't call ddi_report_dev() */
512 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
513
514 /* Initialize rootnex event handle */
515 i_ddi_rootnex_init_events(dip);
516
517 #if defined(__amd64) && !defined(__xpv)
518 e = iommulib_nexus_register(dip, &iommulib_nexops,
519 &rootnex_state->r_iommulib_handle);
520
521 ASSERT(e == DDI_SUCCESS);
522 #endif
523
524 return (DDI_SUCCESS);
525 }
526
527
528 /*
529 * rootnex_detach()
530 *
531 */
532 /*ARGSUSED*/
533 static int
534 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
535 {
536 switch (cmd) {
537 case DDI_SUSPEND:
538 #if defined(__amd64) && !defined(__xpv)
539 return (immu_quiesce());
540 #else
541 return (DDI_SUCCESS);
542 #endif
543 default:
544 return (DDI_FAILURE);
545 }
546 /*NOTREACHED*/
547
548 }
549
550
551 /*
552 * rootnex_dma_init()
553 *
554 */
555 /*ARGSUSED*/
556 static int
557 rootnex_dma_init()
558 {
559 size_t bufsize;
560
561
562 /*
563 * size of our cookie/window/copybuf state needed in dma bind that we
564 * pre-alloc in dma_alloc_handle
565 */
566 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
567 rootnex_state->r_prealloc_size =
568 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
569 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
570 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
571
572 /*
573 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
574 * allocate 16 extra bytes for struct pointer alignment
575 * (p->dmai_private & dma->dp_prealloc_buffer)
576 */
577 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
578 rootnex_state->r_prealloc_size + 0x10;
579 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
580 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
581 if (rootnex_state->r_dmahdl_cache == NULL) {
582 return (DDI_FAILURE);
583 }
584
585 /*
586 * allocate array to track which major numbers we have printed warnings
587 * for.
588 */
589 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
590 KM_SLEEP);
591
592 return (DDI_SUCCESS);
593 }
594
595
596 /*
597 * rootnex_add_props()
598 *
599 */
600 static void
601 rootnex_add_props(dev_info_t *dip)
602 {
603 rootnex_intprop_t *rpp;
604 int i;
605
606 /* Add static integer/boolean properties to the root node */
607 rpp = rootnex_intprp;
608 for (i = 0; i < NROOT_INTPROPS; i++) {
609 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
610 rpp[i].prop_name, rpp[i].prop_value);
611 }
612 }
613
614
615
616 /*
617 * *************************
618 * ctlops related routines
619 * *************************
620 */
621
622 /*
623 * rootnex_ctlops()
624 *
625 */
626 /*ARGSUSED*/
627 static int
628 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
629 void *arg, void *result)
630 {
631 int n, *ptr;
632 struct ddi_parent_private_data *pdp;
633
634 switch (ctlop) {
635 case DDI_CTLOPS_DMAPMAPC:
636 /*
637 * Return 'partial' to indicate that dma mapping
638 * has to be done in the main MMU.
639 */
640 return (DDI_DMA_PARTIAL);
641
642 case DDI_CTLOPS_BTOP:
643 /*
644 * Convert byte count input to physical page units.
645 * (byte counts that are not a page-size multiple
646 * are rounded down)
647 */
648 *(ulong_t *)result = btop(*(ulong_t *)arg);
649 return (DDI_SUCCESS);
650
651 case DDI_CTLOPS_PTOB:
652 /*
653 * Convert size in physical pages to bytes
654 */
655 *(ulong_t *)result = ptob(*(ulong_t *)arg);
656 return (DDI_SUCCESS);
657
658 case DDI_CTLOPS_BTOPR:
659 /*
660 * Convert byte count input to physical page units
661 * (byte counts that are not a page-size multiple
662 * are rounded up)
663 */
664 *(ulong_t *)result = btopr(*(ulong_t *)arg);
665 return (DDI_SUCCESS);
666
667 case DDI_CTLOPS_INITCHILD:
668 return (impl_ddi_sunbus_initchild(arg));
669
670 case DDI_CTLOPS_UNINITCHILD:
671 impl_ddi_sunbus_removechild(arg);
672 return (DDI_SUCCESS);
673
674 case DDI_CTLOPS_REPORTDEV:
675 return (rootnex_ctl_reportdev(rdip));
676
677 case DDI_CTLOPS_IOMIN:
678 /*
679 * Nothing to do here but reflect back..
680 */
681 return (DDI_SUCCESS);
682
683 case DDI_CTLOPS_REGSIZE:
684 case DDI_CTLOPS_NREGS:
685 break;
686
687 case DDI_CTLOPS_SIDDEV:
688 if (ndi_dev_is_prom_node(rdip))
689 return (DDI_SUCCESS);
690 if (ndi_dev_is_persistent_node(rdip))
691 return (DDI_SUCCESS);
692 return (DDI_FAILURE);
693
694 case DDI_CTLOPS_POWER:
695 return ((*pm_platform_power)((power_req_t *)arg));
696
697 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
698 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
699 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
700 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
701 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
702 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
703 if (!rootnex_state->r_reserved_msg_printed) {
704 rootnex_state->r_reserved_msg_printed = B_TRUE;
705 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
706 "1 or more reserved/obsolete operations.");
707 }
708 return (DDI_FAILURE);
709
710 default:
711 return (DDI_FAILURE);
712 }
713 /*
714 * The rest are for "hardware" properties
715 */
716 if ((pdp = ddi_get_parent_data(rdip)) == NULL)
717 return (DDI_FAILURE);
718
719 if (ctlop == DDI_CTLOPS_NREGS) {
720 ptr = (int *)result;
721 *ptr = pdp->par_nreg;
722 } else {
723 off_t *size = (off_t *)result;
724
725 ptr = (int *)arg;
726 n = *ptr;
727 if (n >= pdp->par_nreg) {
728 return (DDI_FAILURE);
729 }
730 *size = (off_t)pdp->par_reg[n].regspec_size;
731 }
732 return (DDI_SUCCESS);
733 }
734
735
736 /*
737 * rootnex_ctl_reportdev()
738 *
739 */
740 static int
741 rootnex_ctl_reportdev(dev_info_t *dev)
742 {
743 int i, n, len, f_len = 0;
744 char *buf;
745
746 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
747 f_len += snprintf(buf, REPORTDEV_BUFSIZE,
748 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
749 len = strlen(buf);
750
751 for (i = 0; i < sparc_pd_getnreg(dev); i++) {
752
753 struct regspec *rp = sparc_pd_getreg(dev, i);
754
755 if (i == 0)
756 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
757 ": ");
758 else
759 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
760 " and ");
761 len = strlen(buf);
762
763 switch (rp->regspec_bustype) {
764
765 case BTEISA:
766 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
767 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
768 break;
769
770 case BTISA:
771 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
772 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
773 break;
774
775 default:
776 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
777 "space %x offset %x",
778 rp->regspec_bustype, rp->regspec_addr);
779 break;
780 }
781 len = strlen(buf);
782 }
783 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
784 int pri;
785
786 if (i != 0) {
787 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
788 ",");
789 len = strlen(buf);
790 }
791 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
792 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
793 " sparc ipl %d", pri);
794 len = strlen(buf);
795 }
796 #ifdef DEBUG
797 if (f_len + 1 >= REPORTDEV_BUFSIZE) {
798 cmn_err(CE_NOTE, "next message is truncated: "
799 "printed length 1024, real length %d", f_len);
800 }
801 #endif /* DEBUG */
802 cmn_err(CE_CONT, "?%s\n", buf);
803 kmem_free(buf, REPORTDEV_BUFSIZE);
804 return (DDI_SUCCESS);
805 }
806
807
808 /*
809 * ******************
810 * map related code
811 * ******************
812 */
813
814 /*
815 * rootnex_map()
816 *
817 */
818 static int
819 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
820 off_t len, caddr_t *vaddrp)
821 {
822 struct regspec *orp = NULL;
823 struct regspec64 rp = { 0 };
824 ddi_map_req_t mr = *mp; /* Get private copy of request */
825
826 mp = &mr;
827
828 switch (mp->map_op) {
829 case DDI_MO_MAP_LOCKED:
830 case DDI_MO_UNMAP:
831 case DDI_MO_MAP_HANDLE:
832 break;
833 default:
834 #ifdef DDI_MAP_DEBUG
835 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
836 mp->map_op);
837 #endif /* DDI_MAP_DEBUG */
838 return (DDI_ME_UNIMPLEMENTED);
839 }
840
841 if (mp->map_flags & DDI_MF_USER_MAPPING) {
842 #ifdef DDI_MAP_DEBUG
843 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
844 #endif /* DDI_MAP_DEBUG */
845 return (DDI_ME_UNIMPLEMENTED);
846 }
847
848 /*
849 * First, we need to get the original regspec out before we convert it
850 * to the extended format. If we have a register number, then we need to
851 * convert that to a regspec.
852 */
853 if (mp->map_type == DDI_MT_RNUMBER) {
854
855 int rnumber = mp->map_obj.rnumber;
856 #ifdef DDI_MAP_DEBUG
857 static char *out_of_range =
858 "rootnex_map: Out of range rnumber <%d>, device <%s>";
859 #endif /* DDI_MAP_DEBUG */
860
861 orp = i_ddi_rnumber_to_regspec(rdip, rnumber);
862 if (orp == NULL) {
863 #ifdef DDI_MAP_DEBUG
864 cmn_err(CE_WARN, out_of_range, rnumber,
865 ddi_get_name(rdip));
866 #endif /* DDI_MAP_DEBUG */
867 return (DDI_ME_RNUMBER_RANGE);
868 }
869 } else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) {
870 orp = mp->map_obj.rp;
871 }
872
873 /*
874 * Ensure that we are always using a 64-bit extended regspec regardless
875 * of what was passed into us. If the child driver is using a 64-bit
876 * regspec, then we need to make sure that we copy this to the local
877 * regspec64, rp.
878 */
879 if (orp != NULL) {
880 rp.regspec_bustype = orp->regspec_bustype;
881 rp.regspec_addr = orp->regspec_addr;
882 rp.regspec_size = orp->regspec_size;
883 } else {
884 struct regspec64 *rp64;
885 rp64 = (struct regspec64 *)mp->map_obj.rp;
886 rp = *rp64;
887 }
888
889 mp->map_type = DDI_MT_REGSPEC;
890 mp->map_flags |= DDI_MF_EXT_REGSPEC;
891 mp->map_obj.rp = (struct regspec *)&rp;
892
893 /*
894 * Adjust offset and length correspnding to called values...
895 * XXX: A non-zero length means override the one in the regspec
896 * XXX: (regardless of what's in the parent's range?)
897 */
898
899 #ifdef DDI_MAP_DEBUG
900 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
901 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
902 rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset,
903 len, mp->map_handlep);
904 #endif /* DDI_MAP_DEBUG */
905
906 /*
907 * I/O or memory mapping:
908 *
909 * <bustype=0, addr=x, len=x>: memory
910 * <bustype=1, addr=x, len=x>: i/o
911 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
912 */
913
914 if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) {
915 cmn_err(CE_WARN, "<%s,%s> invalid register spec"
916 " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">",
917 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
918 rp.regspec_addr, rp.regspec_size);
919 return (DDI_ME_INVAL);
920 }
921
922 if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) {
923 /*
924 * compatibility i/o mapping
925 */
926 rp.regspec_bustype += offset;
927 } else {
928 /*
929 * Normal memory or i/o mapping
930 */
931 rp.regspec_addr += offset;
932 }
933
934 if (len != 0)
935 rp.regspec_size = len;
936
937 #ifdef DDI_MAP_DEBUG
938 cmn_err(CE_CONT, " <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64
939 ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n",
940 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
941 rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep);
942 #endif /* DDI_MAP_DEBUG */
943
944
945 /*
946 * The x86 root nexus does not have any notion of valid ranges of
947 * addresses. Its children have valid ranges, but because there are none
948 * for the nexus, we don't need to call i_ddi_apply_range(). Verify
949 * that is the case.
950 */
951 ASSERT0(sparc_pd_getnrng(dip));
952
953 switch (mp->map_op) {
954 case DDI_MO_MAP_LOCKED:
955
956 /*
957 * Set up the locked down kernel mapping to the regspec...
958 */
959
960 return (rootnex_map_regspec(mp, vaddrp));
961
962 case DDI_MO_UNMAP:
963
964 /*
965 * Release mapping...
966 */
967
968 return (rootnex_unmap_regspec(mp, vaddrp));
969
970 case DDI_MO_MAP_HANDLE:
971
972 return (rootnex_map_handle(mp));
973
974 default:
975 return (DDI_ME_UNIMPLEMENTED);
976 }
977 }
978
979
980 /*
981 * rootnex_map_fault()
982 *
983 * fault in mappings for requestors
984 */
985 /*ARGSUSED*/
986 static int
987 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
988 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
989 uint_t lock)
990 {
991
992 #ifdef DDI_MAP_DEBUG
993 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
994 ddi_map_debug(" Seg <%s>\n",
995 seg->s_ops == &segdev_ops ? "segdev" :
996 seg == &kvseg ? "segkmem" : "NONE!");
997 #endif /* DDI_MAP_DEBUG */
998
999 /*
1000 * This is all terribly broken, but it is a start
1001 *
1002 * XXX Note that this test means that segdev_ops
1003 * must be exported from seg_dev.c.
1004 * XXX What about devices with their own segment drivers?
1005 */
1006 if (seg->s_ops == &segdev_ops) {
1007 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1008
1009 if (hat == NULL) {
1010 /*
1011 * This is one plausible interpretation of
1012 * a null hat i.e. use the first hat on the
1013 * address space hat list which by convention is
1014 * the hat of the system MMU. At alternative
1015 * would be to panic .. this might well be better ..
1016 */
1017 ASSERT(AS_READ_HELD(seg->s_as));
1018 hat = seg->s_as->a_hat;
1019 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1020 }
1021 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1022 (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1023 } else if (seg == &kvseg && dp == NULL) {
1024 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1025 HAT_LOAD_LOCK);
1026 } else
1027 return (DDI_FAILURE);
1028 return (DDI_SUCCESS);
1029 }
1030
1031
1032 static int
1033 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1034 {
1035 rootnex_addr_t rbase;
1036 void *cvaddr;
1037 uint64_t npages, pgoffset;
1038 struct regspec64 *rp;
1039 ddi_acc_hdl_t *hp;
1040 ddi_acc_impl_t *ap;
1041 uint_t hat_acc_flags;
1042 paddr_t pbase;
1043
1044 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1045 rp = (struct regspec64 *)mp->map_obj.rp;
1046 hp = mp->map_handlep;
1047
1048 #ifdef DDI_MAP_DEBUG
1049 ddi_map_debug(
1050 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1051 rp->regspec_bustype, rp->regspec_addr,
1052 rp->regspec_size, mp->map_handlep);
1053 #endif /* DDI_MAP_DEBUG */
1054
1055 /*
1056 * I/O or memory mapping
1057 *
1058 * <bustype=0, addr=x, len=x>: memory
1059 * <bustype=1, addr=x, len=x>: i/o
1060 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1061 */
1062
1063 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1064 cmn_err(CE_WARN, "rootnex: invalid register spec"
1065 " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">",
1066 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size);
1067 return (DDI_FAILURE);
1068 }
1069
1070 if (rp->regspec_bustype != 0) {
1071 /*
1072 * I/O space - needs a handle.
1073 */
1074 if (hp == NULL) {
1075 return (DDI_FAILURE);
1076 }
1077 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1078 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1079 impl_acc_hdl_init(hp);
1080
1081 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1082 #ifdef DDI_MAP_DEBUG
1083 ddi_map_debug("rootnex_map_regspec: mmap() "
1084 "to I/O space is not supported.\n");
1085 #endif /* DDI_MAP_DEBUG */
1086 return (DDI_ME_INVAL);
1087 } else {
1088 /*
1089 * 1275-compliant vs. compatibility i/o mapping
1090 */
1091 *vaddrp =
1092 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1093 ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1094 ((caddr_t)(uintptr_t)rp->regspec_addr);
1095 #ifdef __xpv
1096 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1097 hp->ah_pfn = xen_assign_pfn(
1098 mmu_btop((ulong_t)rp->regspec_addr &
1099 MMU_PAGEMASK));
1100 } else {
1101 hp->ah_pfn = mmu_btop(
1102 (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1103 }
1104 #else
1105 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1106 MMU_PAGEMASK);
1107 #endif
1108 hp->ah_pnum = mmu_btopr(rp->regspec_size +
1109 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1110 }
1111
1112 #ifdef DDI_MAP_DEBUG
1113 ddi_map_debug(
1114 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1115 rp->regspec_size, *vaddrp);
1116 #endif /* DDI_MAP_DEBUG */
1117 return (DDI_SUCCESS);
1118 }
1119
1120 /*
1121 * Memory space
1122 */
1123
1124 if (hp != NULL) {
1125 /*
1126 * hat layer ignores
1127 * hp->ah_acc.devacc_attr_endian_flags.
1128 */
1129 switch (hp->ah_acc.devacc_attr_dataorder) {
1130 case DDI_STRICTORDER_ACC:
1131 hat_acc_flags = HAT_STRICTORDER;
1132 break;
1133 case DDI_UNORDERED_OK_ACC:
1134 hat_acc_flags = HAT_UNORDERED_OK;
1135 break;
1136 case DDI_MERGING_OK_ACC:
1137 hat_acc_flags = HAT_MERGING_OK;
1138 break;
1139 case DDI_LOADCACHING_OK_ACC:
1140 hat_acc_flags = HAT_LOADCACHING_OK;
1141 break;
1142 case DDI_STORECACHING_OK_ACC:
1143 hat_acc_flags = HAT_STORECACHING_OK;
1144 break;
1145 }
1146 ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1147 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1148 impl_acc_hdl_init(hp);
1149 hp->ah_hat_flags = hat_acc_flags;
1150 } else {
1151 hat_acc_flags = HAT_STRICTORDER;
1152 }
1153
1154 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1155 #ifdef __xpv
1156 /*
1157 * If we're dom0, we're using a real device so we need to translate
1158 * the MA to a PA.
1159 */
1160 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1161 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1162 } else {
1163 pbase = rbase;
1164 }
1165 #else
1166 pbase = rbase;
1167 #endif
1168 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1169
1170 if (rp->regspec_size == 0) {
1171 #ifdef DDI_MAP_DEBUG
1172 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1173 #endif /* DDI_MAP_DEBUG */
1174 return (DDI_ME_INVAL);
1175 }
1176
1177 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1178 /* extra cast to make gcc happy */
1179 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1180 } else {
1181 npages = mmu_btopr(rp->regspec_size + pgoffset);
1182
1183 #ifdef DDI_MAP_DEBUG
1184 ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1185 "physical %llx", npages, pbase);
1186 #endif /* DDI_MAP_DEBUG */
1187
1188 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1189 if (cvaddr == NULL)
1190 return (DDI_ME_NORESOURCES);
1191
1192 /*
1193 * Now map in the pages we've allocated...
1194 */
1195 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1196 mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1197 HAT_LOAD_LOCK);
1198 *vaddrp = (caddr_t)cvaddr + pgoffset;
1199
1200 /* save away pfn and npages for FMA */
1201 hp = mp->map_handlep;
1202 if (hp) {
1203 hp->ah_pfn = mmu_btop(pbase);
1204 hp->ah_pnum = npages;
1205 }
1206 }
1207
1208 #ifdef DDI_MAP_DEBUG
1209 ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1210 #endif /* DDI_MAP_DEBUG */
1211 return (DDI_SUCCESS);
1212 }
1213
1214
1215 static int
1216 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1217 {
1218 caddr_t addr = (caddr_t)*vaddrp;
1219 uint64_t npages, pgoffset;
1220 struct regspec64 *rp;
1221
1222 if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1223 return (0);
1224
1225 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1226 rp = (struct regspec64 *)mp->map_obj.rp;
1227
1228 if (rp->regspec_size == 0) {
1229 #ifdef DDI_MAP_DEBUG
1230 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1231 #endif /* DDI_MAP_DEBUG */
1232 return (DDI_ME_INVAL);
1233 }
1234
1235 /*
1236 * I/O or memory mapping:
1237 *
1238 * <bustype=0, addr=x, len=x>: memory
1239 * <bustype=1, addr=x, len=x>: i/o
1240 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1241 */
1242 if (rp->regspec_bustype != 0) {
1243 /*
1244 * This is I/O space, which requires no particular
1245 * processing on unmap since it isn't mapped in the
1246 * first place.
1247 */
1248 return (DDI_SUCCESS);
1249 }
1250
1251 /*
1252 * Memory space
1253 */
1254 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1255 npages = mmu_btopr(rp->regspec_size + pgoffset);
1256 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1257 device_arena_free(addr - pgoffset, ptob(npages));
1258
1259 /*
1260 * Destroy the pointer - the mapping has logically gone
1261 */
1262 *vaddrp = NULL;
1263
1264 return (DDI_SUCCESS);
1265 }
1266
1267 static int
1268 rootnex_map_handle(ddi_map_req_t *mp)
1269 {
1270 rootnex_addr_t rbase;
1271 ddi_acc_hdl_t *hp;
1272 uint64_t pgoffset;
1273 struct regspec64 *rp;
1274 paddr_t pbase;
1275
1276 rp = (struct regspec64 *)mp->map_obj.rp;
1277
1278 #ifdef DDI_MAP_DEBUG
1279 ddi_map_debug(
1280 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1281 rp->regspec_bustype, rp->regspec_addr,
1282 rp->regspec_size, mp->map_handlep);
1283 #endif /* DDI_MAP_DEBUG */
1284
1285 /*
1286 * I/O or memory mapping:
1287 *
1288 * <bustype=0, addr=x, len=x>: memory
1289 * <bustype=1, addr=x, len=x>: i/o
1290 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
1291 */
1292 if (rp->regspec_bustype != 0) {
1293 /*
1294 * This refers to I/O space, and we don't support "mapping"
1295 * I/O space to a user.
1296 */
1297 return (DDI_FAILURE);
1298 }
1299
1300 /*
1301 * Set up the hat_flags for the mapping.
1302 */
1303 hp = mp->map_handlep;
1304
1305 switch (hp->ah_acc.devacc_attr_endian_flags) {
1306 case DDI_NEVERSWAP_ACC:
1307 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1308 break;
1309 case DDI_STRUCTURE_LE_ACC:
1310 hp->ah_hat_flags = HAT_STRUCTURE_LE;
1311 break;
1312 case DDI_STRUCTURE_BE_ACC:
1313 return (DDI_FAILURE);
1314 default:
1315 return (DDI_REGS_ACC_CONFLICT);
1316 }
1317
1318 switch (hp->ah_acc.devacc_attr_dataorder) {
1319 case DDI_STRICTORDER_ACC:
1320 break;
1321 case DDI_UNORDERED_OK_ACC:
1322 hp->ah_hat_flags |= HAT_UNORDERED_OK;
1323 break;
1324 case DDI_MERGING_OK_ACC:
1325 hp->ah_hat_flags |= HAT_MERGING_OK;
1326 break;
1327 case DDI_LOADCACHING_OK_ACC:
1328 hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1329 break;
1330 case DDI_STORECACHING_OK_ACC:
1331 hp->ah_hat_flags |= HAT_STORECACHING_OK;
1332 break;
1333 default:
1334 return (DDI_FAILURE);
1335 }
1336
1337 rbase = (rootnex_addr_t)rp->regspec_addr &
1338 (~(rootnex_addr_t)MMU_PAGEOFFSET);
1339 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1340
1341 if (rp->regspec_size == 0)
1342 return (DDI_ME_INVAL);
1343
1344 #ifdef __xpv
1345 /*
1346 * If we're dom0, we're using a real device so we need to translate
1347 * the MA to a PA.
1348 */
1349 if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1350 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1351 (rbase & MMU_PAGEOFFSET);
1352 } else {
1353 pbase = rbase;
1354 }
1355 #else
1356 pbase = rbase;
1357 #endif
1358
1359 hp->ah_pfn = mmu_btop(pbase);
1360 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1361
1362 return (DDI_SUCCESS);
1363 }
1364
1365
1366
1367 /*
1368 * ************************
1369 * interrupt related code
1370 * ************************
1371 */
1372
1373 /*
1374 * rootnex_intr_ops()
1375 * bus_intr_op() function for interrupt support
1376 */
1377 /* ARGSUSED */
1378 static int
1379 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1380 ddi_intr_handle_impl_t *hdlp, void *result)
1381 {
1382 struct intrspec *ispec;
1383
1384 DDI_INTR_NEXDBG((CE_CONT,
1385 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1386 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1387
1388 /* Process the interrupt operation */
1389 switch (intr_op) {
1390 case DDI_INTROP_GETCAP:
1391 /* First check with pcplusmp */
1392 if (psm_intr_ops == NULL)
1393 return (DDI_FAILURE);
1394
1395 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1396 *(int *)result = 0;
1397 return (DDI_FAILURE);
1398 }
1399 break;
1400 case DDI_INTROP_SETCAP:
1401 if (psm_intr_ops == NULL)
1402 return (DDI_FAILURE);
1403
1404 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1405 return (DDI_FAILURE);
1406 break;
1407 case DDI_INTROP_ALLOC:
1408 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1409 return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1410 case DDI_INTROP_FREE:
1411 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1412 return (rootnex_free_intr_fixed(rdip, hdlp));
1413 case DDI_INTROP_GETPRI:
1414 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1415 return (DDI_FAILURE);
1416 *(int *)result = ispec->intrspec_pri;
1417 break;
1418 case DDI_INTROP_SETPRI:
1419 /* Validate the interrupt priority passed to us */
1420 if (*(int *)result > LOCK_LEVEL)
1421 return (DDI_FAILURE);
1422
1423 /* Ensure that PSM is all initialized and ispec is ok */
1424 if ((psm_intr_ops == NULL) ||
1425 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1426 return (DDI_FAILURE);
1427
1428 /* Change the priority */
1429 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1430 PSM_FAILURE)
1431 return (DDI_FAILURE);
1432
1433 /* update the ispec with the new priority */
1434 ispec->intrspec_pri = *(int *)result;
1435 break;
1436 case DDI_INTROP_ADDISR:
1437 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1438 return (DDI_FAILURE);
1439 ispec->intrspec_func = hdlp->ih_cb_func;
1440 break;
1441 case DDI_INTROP_REMISR:
1442 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1443 return (DDI_FAILURE);
1444 ispec->intrspec_func = (uint_t (*)()) 0;
1445 break;
1446 case DDI_INTROP_ENABLE:
1447 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1448 return (DDI_FAILURE);
1449
1450 /* Call psmi to translate irq with the dip */
1451 if (psm_intr_ops == NULL)
1452 return (DDI_FAILURE);
1453
1454 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1455 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1456 (int *)&hdlp->ih_vector) == PSM_FAILURE)
1457 return (DDI_FAILURE);
1458
1459 /* Add the interrupt handler */
1460 if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1461 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1462 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1463 return (DDI_FAILURE);
1464 break;
1465 case DDI_INTROP_DISABLE:
1466 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1467 return (DDI_FAILURE);
1468
1469 /* Call psm_ops() to translate irq with the dip */
1470 if (psm_intr_ops == NULL)
1471 return (DDI_FAILURE);
1472
1473 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1474 (void) (*psm_intr_ops)(rdip, hdlp,
1475 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1476
1477 /* Remove the interrupt handler */
1478 rem_avintr((void *)hdlp, ispec->intrspec_pri,
1479 hdlp->ih_cb_func, hdlp->ih_vector);
1480 break;
1481 case DDI_INTROP_SETMASK:
1482 if (psm_intr_ops == NULL)
1483 return (DDI_FAILURE);
1484
1485 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1486 return (DDI_FAILURE);
1487 break;
1488 case DDI_INTROP_CLRMASK:
1489 if (psm_intr_ops == NULL)
1490 return (DDI_FAILURE);
1491
1492 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1493 return (DDI_FAILURE);
1494 break;
1495 case DDI_INTROP_GETPENDING:
1496 if (psm_intr_ops == NULL)
1497 return (DDI_FAILURE);
1498
1499 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1500 result)) {
1501 *(int *)result = 0;
1502 return (DDI_FAILURE);
1503 }
1504 break;
1505 case DDI_INTROP_NAVAIL:
1506 case DDI_INTROP_NINTRS:
1507 *(int *)result = i_ddi_get_intx_nintrs(rdip);
1508 if (*(int *)result == 0) {
1509 /*
1510 * Special case for 'pcic' driver' only. This driver
1511 * driver is a child of 'isa' and 'rootnex' drivers.
1512 *
1513 * See detailed comments on this in the function
1514 * rootnex_get_ispec().
1515 *
1516 * Children of 'pcic' send 'NINITR' request all the
1517 * way to rootnex driver. But, the 'pdp->par_nintr'
1518 * field may not initialized. So, we fake it here
1519 * to return 1 (a la what PCMCIA nexus does).
1520 */
1521 if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1522 *(int *)result = 1;
1523 else
1524 return (DDI_FAILURE);
1525 }
1526 break;
1527 case DDI_INTROP_SUPPORTED_TYPES:
1528 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */
1529 break;
1530 default:
1531 return (DDI_FAILURE);
1532 }
1533
1534 return (DDI_SUCCESS);
1535 }
1536
1537
1538 /*
1539 * rootnex_get_ispec()
1540 * convert an interrupt number to an interrupt specification.
1541 * The interrupt number determines which interrupt spec will be
1542 * returned if more than one exists.
1543 *
1544 * Look into the parent private data area of the 'rdip' to find out
1545 * the interrupt specification. First check to make sure there is
1546 * one that matchs "inumber" and then return a pointer to it.
1547 *
1548 * Return NULL if one could not be found.
1549 *
1550 * NOTE: This is needed for rootnex_intr_ops()
1551 */
1552 static struct intrspec *
1553 rootnex_get_ispec(dev_info_t *rdip, int inum)
1554 {
1555 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1556
1557 /*
1558 * Special case handling for drivers that provide their own
1559 * intrspec structures instead of relying on the DDI framework.
1560 *
1561 * A broken hardware driver in ON could potentially provide its
1562 * own intrspec structure, instead of relying on the hardware.
1563 * If these drivers are children of 'rootnex' then we need to
1564 * continue to provide backward compatibility to them here.
1565 *
1566 * Following check is a special case for 'pcic' driver which
1567 * was found to have broken hardwre andby provides its own intrspec.
1568 *
1569 * Verbatim comments from this driver are shown here:
1570 * "Don't use the ddi_add_intr since we don't have a
1571 * default intrspec in all cases."
1572 *
1573 * Since an 'ispec' may not be always created for it,
1574 * check for that and create one if so.
1575 *
1576 * NOTE: Currently 'pcic' is the only driver found to do this.
1577 */
1578 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1579 pdp->par_nintr = 1;
1580 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1581 pdp->par_nintr, KM_SLEEP);
1582 }
1583
1584 /* Validate the interrupt number */
1585 if (inum >= pdp->par_nintr)
1586 return (NULL);
1587
1588 /* Get the interrupt structure pointer and return that */
1589 return ((struct intrspec *)&pdp->par_intr[inum]);
1590 }
1591
1592 /*
1593 * Allocate interrupt vector for FIXED (legacy) type.
1594 */
1595 static int
1596 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1597 void *result)
1598 {
1599 struct intrspec *ispec;
1600 ddi_intr_handle_impl_t info_hdl;
1601 int ret;
1602 int free_phdl = 0;
1603 apic_get_type_t type_info;
1604
1605 if (psm_intr_ops == NULL)
1606 return (DDI_FAILURE);
1607
1608 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1609 return (DDI_FAILURE);
1610
1611 /*
1612 * If the PSM module is "APIX" then pass the request for it
1613 * to allocate the vector now.
1614 */
1615 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1616 info_hdl.ih_private = &type_info;
1617 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1618 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1619 if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1620 free_phdl = 1;
1621 i_ddi_alloc_intr_phdl(hdlp);
1622 }
1623 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1624 ret = (*psm_intr_ops)(rdip, hdlp,
1625 PSM_INTR_OP_ALLOC_VECTORS, result);
1626 if (free_phdl) { /* free up the phdl structure */
1627 free_phdl = 0;
1628 i_ddi_free_intr_phdl(hdlp);
1629 hdlp->ih_private = NULL;
1630 }
1631 } else {
1632 /*
1633 * No APIX module; fall back to the old scheme where the
1634 * interrupt vector is allocated during ddi_enable_intr() call.
1635 */
1636 hdlp->ih_pri = ispec->intrspec_pri;
1637 *(int *)result = hdlp->ih_scratch1;
1638 ret = DDI_SUCCESS;
1639 }
1640
1641 return (ret);
1642 }
1643
1644 /*
1645 * Free up interrupt vector for FIXED (legacy) type.
1646 */
1647 static int
1648 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1649 {
1650 struct intrspec *ispec;
1651 struct ddi_parent_private_data *pdp;
1652 ddi_intr_handle_impl_t info_hdl;
1653 int ret;
1654 apic_get_type_t type_info;
1655
1656 if (psm_intr_ops == NULL)
1657 return (DDI_FAILURE);
1658
1659 /*
1660 * If the PSM module is "APIX" then pass the request for it
1661 * to free up the vector now.
1662 */
1663 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1664 info_hdl.ih_private = &type_info;
1665 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1666 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1667 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1668 return (DDI_FAILURE);
1669 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1670 ret = (*psm_intr_ops)(rdip, hdlp,
1671 PSM_INTR_OP_FREE_VECTORS, NULL);
1672 } else {
1673 /*
1674 * No APIX module; fall back to the old scheme where
1675 * the interrupt vector was already freed during
1676 * ddi_disable_intr() call.
1677 */
1678 ret = DDI_SUCCESS;
1679 }
1680
1681 pdp = ddi_get_parent_data(rdip);
1682
1683 /*
1684 * Special case for 'pcic' driver' only.
1685 * If an intrspec was created for it, clean it up here
1686 * See detailed comments on this in the function
1687 * rootnex_get_ispec().
1688 */
1689 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1690 kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1691 pdp->par_nintr);
1692 /*
1693 * Set it to zero; so that
1694 * DDI framework doesn't free it again
1695 */
1696 pdp->par_intr = NULL;
1697 pdp->par_nintr = 0;
1698 }
1699
1700 return (ret);
1701 }
1702
1703
1704 /*
1705 * ******************
1706 * dma related code
1707 * ******************
1708 */
1709
1710 /*ARGSUSED*/
1711 static int
1712 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1713 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1714 ddi_dma_handle_t *handlep)
1715 {
1716 uint64_t maxsegmentsize_ll;
1717 uint_t maxsegmentsize;
1718 ddi_dma_impl_t *hp;
1719 rootnex_dma_t *dma;
1720 uint64_t count_max;
1721 uint64_t seg;
1722 int kmflag;
1723 int e;
1724
1725
1726 /* convert our sleep flags */
1727 if (waitfp == DDI_DMA_SLEEP) {
1728 kmflag = KM_SLEEP;
1729 } else {
1730 kmflag = KM_NOSLEEP;
1731 }
1732
1733 /*
1734 * We try to do only one memory allocation here. We'll do a little
1735 * pointer manipulation later. If the bind ends up taking more than
1736 * our prealloc's space, we'll have to allocate more memory in the
1737 * bind operation. Not great, but much better than before and the
1738 * best we can do with the current bind interfaces.
1739 */
1740 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1741 if (hp == NULL)
1742 return (DDI_DMA_NORESOURCES);
1743
1744 /* Do our pointer manipulation now, align the structures */
1745 hp->dmai_private = (void *)(((uintptr_t)hp +
1746 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1747 dma = (rootnex_dma_t *)hp->dmai_private;
1748 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1749 sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1750
1751 /* setup the handle */
1752 rootnex_clean_dmahdl(hp);
1753 hp->dmai_error.err_fep = NULL;
1754 hp->dmai_error.err_cf = NULL;
1755 dma->dp_dip = rdip;
1756 dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1757 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1758
1759 /*
1760 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1761 * is being used. Set the upper limit to the seg value.
1762 * There will be enough DVMA space to always get addresses
1763 * that will match the constraints.
1764 */
1765 if (IOMMU_USED(rdip) &&
1766 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1767 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1768 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1769 } else
1770 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1771
1772 hp->dmai_minxfer = attr->dma_attr_minxfer;
1773 hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1774 hp->dmai_rdip = rdip;
1775 hp->dmai_attr = *attr;
1776
1777 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1778 dma->dp_sglinfo.si_cancross = B_FALSE;
1779 else
1780 dma->dp_sglinfo.si_cancross = B_TRUE;
1781
1782 /* we don't need to worry about the SPL since we do a tryenter */
1783 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1784
1785 /*
1786 * Figure out our maximum segment size. If the segment size is greater
1787 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1788 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1789 * dma_attr_count_max are size-1 type values.
1790 *
1791 * Maximum segment size is the largest physically contiguous chunk of
1792 * memory that we can return from a bind (i.e. the maximum size of a
1793 * single cookie).
1794 */
1795
1796 /* handle the rollover cases */
1797 seg = attr->dma_attr_seg + 1;
1798 if (seg < attr->dma_attr_seg) {
1799 seg = attr->dma_attr_seg;
1800 }
1801 count_max = attr->dma_attr_count_max + 1;
1802 if (count_max < attr->dma_attr_count_max) {
1803 count_max = attr->dma_attr_count_max;
1804 }
1805
1806 /*
1807 * granularity may or may not be a power of two. If it isn't, we can't
1808 * use a simple mask.
1809 */
1810 if (!ISP2(attr->dma_attr_granular)) {
1811 dma->dp_granularity_power_2 = B_FALSE;
1812 } else {
1813 dma->dp_granularity_power_2 = B_TRUE;
1814 }
1815
1816 /*
1817 * maxxfer should be a whole multiple of granularity. If we're going to
1818 * break up a window because we're greater than maxxfer, we might as
1819 * well make sure it's maxxfer is a whole multiple so we don't have to
1820 * worry about triming the window later on for this case.
1821 */
1822 if (attr->dma_attr_granular > 1) {
1823 if (dma->dp_granularity_power_2) {
1824 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1825 (attr->dma_attr_maxxfer &
1826 (attr->dma_attr_granular - 1));
1827 } else {
1828 dma->dp_maxxfer = attr->dma_attr_maxxfer -
1829 (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1830 }
1831 } else {
1832 dma->dp_maxxfer = attr->dma_attr_maxxfer;
1833 }
1834
1835 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1836 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1837 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1838 maxsegmentsize = 0xFFFFFFFF;
1839 } else {
1840 maxsegmentsize = maxsegmentsize_ll;
1841 }
1842 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1843 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1844
1845 /* check the ddi_dma_attr arg to make sure it makes a little sense */
1846 if (rootnex_alloc_check_parms) {
1847 e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1848 if (e != DDI_SUCCESS) {
1849 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1850 (void) rootnex_dma_freehdl(dip, rdip,
1851 (ddi_dma_handle_t)hp);
1852 return (e);
1853 }
1854 }
1855
1856 *handlep = (ddi_dma_handle_t)hp;
1857
1858 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1859 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1860 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1861
1862 return (DDI_SUCCESS);
1863 }
1864
1865
1866 /*
1867 * rootnex_dma_allochdl()
1868 * called from ddi_dma_alloc_handle().
1869 */
1870 static int
1871 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1872 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1873 {
1874 int retval = DDI_SUCCESS;
1875 #if defined(__amd64) && !defined(__xpv)
1876
1877 if (IOMMU_UNITIALIZED(rdip)) {
1878 retval = iommulib_nex_open(dip, rdip);
1879
1880 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1881 return (retval);
1882 }
1883
1884 if (IOMMU_UNUSED(rdip)) {
1885 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1886 handlep);
1887 } else {
1888 retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1889 waitfp, arg, handlep);
1890 }
1891 #else
1892 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1893 handlep);
1894 #endif
1895 switch (retval) {
1896 case DDI_DMA_NORESOURCES:
1897 if (waitfp != DDI_DMA_DONTWAIT) {
1898 ddi_set_callback(waitfp, arg,
1899 &rootnex_state->r_dvma_call_list_id);
1900 }
1901 break;
1902 case DDI_SUCCESS:
1903 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1904 break;
1905 default:
1906 break;
1907 }
1908 return (retval);
1909 }
1910
1911 /*ARGSUSED*/
1912 static int
1913 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1914 ddi_dma_handle_t handle)
1915 {
1916 ddi_dma_impl_t *hp;
1917 rootnex_dma_t *dma;
1918
1919
1920 hp = (ddi_dma_impl_t *)handle;
1921 dma = (rootnex_dma_t *)hp->dmai_private;
1922
1923 /* unbind should have been called first */
1924 ASSERT(!dma->dp_inuse);
1925
1926 mutex_destroy(&dma->dp_mutex);
1927 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1928
1929 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1930 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1931 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1932
1933 return (DDI_SUCCESS);
1934 }
1935
1936 /*
1937 * rootnex_dma_freehdl()
1938 * called from ddi_dma_free_handle().
1939 */
1940 static int
1941 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1942 {
1943 int ret;
1944
1945 ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1946 #if defined(__amd64) && !defined(__xpv)
1947 if (IOMMU_USED(rdip))
1948 ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1949 else
1950 #endif
1951 ret = rootnex_coredma_freehdl(dip, rdip, handle);
1952
1953 if (rootnex_state->r_dvma_call_list_id)
1954 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1955
1956 return (ret);
1957 }
1958
1959 /*ARGSUSED*/
1960 static int
1961 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1962 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1963 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1964 {
1965 rootnex_sglinfo_t *sinfo;
1966 ddi_dma_obj_t *dmao;
1967 #if defined(__amd64) && !defined(__xpv)
1968 struct dvmaseg *dvs;
1969 ddi_dma_cookie_t *cookie;
1970 #endif
1971 ddi_dma_attr_t *attr;
1972 ddi_dma_impl_t *hp;
1973 rootnex_dma_t *dma;
1974 int kmflag;
1975 int e;
1976 uint_t ncookies;
1977
1978 hp = (ddi_dma_impl_t *)handle;
1979 dma = (rootnex_dma_t *)hp->dmai_private;
1980 dmao = &dma->dp_dma;
1981 sinfo = &dma->dp_sglinfo;
1982 attr = &hp->dmai_attr;
1983
1984 /* convert the sleep flags */
1985 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1986 dma->dp_sleep_flags = kmflag = KM_SLEEP;
1987 } else {
1988 dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1989 }
1990
1991 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1992
1993 /*
1994 * This is useful for debugging a driver. Not as useful in a production
1995 * system. The only time this will fail is if you have a driver bug.
1996 */
1997 if (rootnex_bind_check_inuse) {
1998 /*
1999 * No one else should ever have this lock unless someone else
2000 * is trying to use this handle. So contention on the lock
2001 * is the same as inuse being set.
2002 */
2003 e = mutex_tryenter(&dma->dp_mutex);
2004 if (e == 0) {
2005 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2006 return (DDI_DMA_INUSE);
2007 }
2008 if (dma->dp_inuse) {
2009 mutex_exit(&dma->dp_mutex);
2010 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2011 return (DDI_DMA_INUSE);
2012 }
2013 dma->dp_inuse = B_TRUE;
2014 mutex_exit(&dma->dp_mutex);
2015 }
2016
2017 /* check the ddi_dma_attr arg to make sure it makes a little sense */
2018 if (rootnex_bind_check_parms) {
2019 e = rootnex_valid_bind_parms(dmareq, attr);
2020 if (e != DDI_SUCCESS) {
2021 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2022 rootnex_clean_dmahdl(hp);
2023 return (e);
2024 }
2025 }
2026
2027 /* save away the original bind info */
2028 dma->dp_dma = dmareq->dmar_object;
2029
2030 #if defined(__amd64) && !defined(__xpv)
2031 if (IOMMU_USED(rdip)) {
2032 dmao = &dma->dp_dvma;
2033 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2034 switch (e) {
2035 case DDI_SUCCESS:
2036 if (sinfo->si_cancross ||
2037 dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2038 dmao->dmao_size > sinfo->si_max_cookie_size) {
2039 dma->dp_dvma_used = B_TRUE;
2040 break;
2041 }
2042 sinfo->si_sgl_size = 1;
2043 hp->dmai_rflags |= DMP_NOSYNC;
2044
2045 dma->dp_dvma_used = B_TRUE;
2046 dma->dp_need_to_free_cookie = B_FALSE;
2047
2048 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2049 cookie = hp->dmai_cookie = dma->dp_cookies =
2050 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2051 cookie->dmac_laddress = dvs->dvs_start +
2052 dmao->dmao_obj.dvma_obj.dv_off;
2053 cookie->dmac_size = dvs->dvs_len;
2054 cookie->dmac_type = 0;
2055
2056 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2057 rdip);
2058 goto fast;
2059 case DDI_ENOTSUP:
2060 break;
2061 default:
2062 rootnex_clean_dmahdl(hp);
2063 return (e);
2064 }
2065 }
2066 #endif
2067
2068 /*
2069 * Figure out a rough estimate of what maximum number of pages
2070 * this buffer could use (a high estimate of course).
2071 */
2072 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2073
2074 if (dma->dp_dvma_used) {
2075 /*
2076 * The number of physical pages is the worst case.
2077 *
2078 * For DVMA, the worst case is the length divided
2079 * by the maximum cookie length, plus 1. Add to that
2080 * the number of segment boundaries potentially crossed, and
2081 * the additional number of DVMA segments that was returned.
2082 *
2083 * In the normal case, for modern devices, si_cancross will
2084 * be false, and dv_nseg will be 1, and the fast path will
2085 * have been taken above.
2086 */
2087 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2088 + 1;
2089 if (sinfo->si_cancross)
2090 ncookies +=
2091 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2092 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2093
2094 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2095 }
2096
2097 /*
2098 * We'll use the pre-allocated cookies for any bind that will *always*
2099 * fit (more important to be consistent, we don't want to create
2100 * additional degenerate cases).
2101 */
2102 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2103 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2104 dma->dp_need_to_free_cookie = B_FALSE;
2105 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2106 uint_t, sinfo->si_max_pages);
2107
2108 /*
2109 * For anything larger than that, we'll go ahead and allocate the
2110 * maximum number of pages we expect to see. Hopefuly, we won't be
2111 * seeing this path in the fast path for high performance devices very
2112 * frequently.
2113 *
2114 * a ddi bind interface that allowed the driver to provide storage to
2115 * the bind interface would speed this case up.
2116 */
2117 } else {
2118 /*
2119 * Save away how much memory we allocated. If we're doing a
2120 * nosleep, the alloc could fail...
2121 */
2122 dma->dp_cookie_size = sinfo->si_max_pages *
2123 sizeof (ddi_dma_cookie_t);
2124 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2125 if (dma->dp_cookies == NULL) {
2126 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2127 rootnex_clean_dmahdl(hp);
2128 return (DDI_DMA_NORESOURCES);
2129 }
2130 dma->dp_need_to_free_cookie = B_TRUE;
2131 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2132 uint_t, sinfo->si_max_pages);
2133 }
2134 hp->dmai_cookie = dma->dp_cookies;
2135
2136 /*
2137 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2138 * looking at the constraints in the dma structure. It will then put
2139 * some additional state about the sgl in the dma struct (i.e. is
2140 * the sgl clean, or do we need to do some munging; how many pages
2141 * need to be copied, etc.)
2142 */
2143 if (dma->dp_dvma_used)
2144 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2145 else
2146 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2147
2148 out:
2149 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2150 /* if we don't need a copy buffer, we don't need to sync */
2151 if (sinfo->si_copybuf_req == 0) {
2152 hp->dmai_rflags |= DMP_NOSYNC;
2153 }
2154
2155 /*
2156 * if we don't need the copybuf and we don't need to do a partial, we
2157 * hit the fast path. All the high performance devices should be trying
2158 * to hit this path. To hit this path, a device should be able to reach
2159 * all of memory, shouldn't try to bind more than it can transfer, and
2160 * the buffer shouldn't require more cookies than the driver/device can
2161 * handle [sgllen]).
2162 *
2163 * Note that negative values of dma_attr_sgllen are supposed
2164 * to mean unlimited, but we just cast them to mean a
2165 * "ridiculous large limit". This saves some extra checks on
2166 * hot paths.
2167 */
2168 if ((sinfo->si_copybuf_req == 0) &&
2169 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2170 (dmao->dmao_size <= dma->dp_maxxfer)) {
2171 fast:
2172 /*
2173 * If the driver supports FMA, insert the handle in the FMA DMA
2174 * handle cache.
2175 */
2176 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2177 hp->dmai_error.err_cf = rootnex_dma_check;
2178
2179 /*
2180 * copy out the first cookie and ccountp, set the cookie
2181 * pointer to the second cookie. The first cookie is passed
2182 * back on the stack. Additional cookies are accessed via
2183 * ddi_dma_nextcookie()
2184 */
2185 *cookiep = dma->dp_cookies[0];
2186 *ccountp = sinfo->si_sgl_size;
2187 hp->dmai_cookie++;
2188 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2189 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2190 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2191 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2192 uint_t, dmao->dmao_size, uint_t, *ccountp);
2193
2194
2195 return (DDI_DMA_MAPPED);
2196 }
2197
2198 /*
2199 * go to the slow path, we may need to alloc more memory, create
2200 * multiple windows, and munge up a sgl to make the device happy.
2201 */
2202
2203 /*
2204 * With the IOMMU mapobject method used, we should never hit
2205 * the slow path. If we do, something is seriously wrong.
2206 * Clean up and return an error.
2207 */
2208
2209 #if defined(__amd64) && !defined(__xpv)
2210
2211 if (dma->dp_dvma_used) {
2212 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2213 &dma->dp_dvma);
2214 e = DDI_DMA_NOMAPPING;
2215 } else {
2216 #endif
2217 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2218 kmflag);
2219 #if defined(__amd64) && !defined(__xpv)
2220 }
2221 #endif
2222 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2223 if (dma->dp_need_to_free_cookie) {
2224 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2225 }
2226 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2227 rootnex_clean_dmahdl(hp); /* must be after free cookie */
2228 return (e);
2229 }
2230
2231 /*
2232 * If the driver supports FMA, insert the handle in the FMA DMA handle
2233 * cache.
2234 */
2235 if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2236 hp->dmai_error.err_cf = rootnex_dma_check;
2237
2238 /* if the first window uses the copy buffer, sync it for the device */
2239 if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2240 (hp->dmai_rflags & DDI_DMA_WRITE)) {
2241 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2242 DDI_DMA_SYNC_FORDEV);
2243 }
2244
2245 /*
2246 * copy out the first cookie and ccountp, set the cookie pointer to the
2247 * second cookie. Make sure the partial flag is set/cleared correctly.
2248 * If we have a partial map (i.e. multiple windows), the number of
2249 * cookies we return is the number of cookies in the first window.
2250 */
2251 if (e == DDI_DMA_MAPPED) {
2252 hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2253 *ccountp = sinfo->si_sgl_size;
2254 hp->dmai_nwin = 1;
2255 } else {
2256 hp->dmai_rflags |= DDI_DMA_PARTIAL;
2257 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2258 ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2259 }
2260 *cookiep = dma->dp_cookies[0];
2261 hp->dmai_cookie++;
2262
2263 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2264 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2265 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2266 dmao->dmao_size, uint_t, *ccountp);
2267 return (e);
2268 }
2269
2270 /*
2271 * rootnex_dma_bindhdl()
2272 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2273 */
2274 static int
2275 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2276 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2277 ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2278 {
2279 int ret;
2280 #if defined(__amd64) && !defined(__xpv)
2281 if (IOMMU_USED(rdip))
2282 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2283 cookiep, ccountp);
2284 else
2285 #endif
2286 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2287 cookiep, ccountp);
2288
2289 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2290 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2291 &rootnex_state->r_dvma_call_list_id);
2292 }
2293
2294 return (ret);
2295 }
2296
2297
2298
2299 /*ARGSUSED*/
2300 static int
2301 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2302 ddi_dma_handle_t handle)
2303 {
2304 ddi_dma_impl_t *hp;
2305 rootnex_dma_t *dma;
2306 int e;
2307
2308 hp = (ddi_dma_impl_t *)handle;
2309 dma = (rootnex_dma_t *)hp->dmai_private;
2310
2311 /* make sure the buffer wasn't free'd before calling unbind */
2312 if (rootnex_unbind_verify_buffer) {
2313 e = rootnex_verify_buffer(dma);
2314 if (e != DDI_SUCCESS) {
2315 ASSERT(0);
2316 return (DDI_FAILURE);
2317 }
2318 }
2319
2320 /* sync the current window before unbinding the buffer */
2321 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2322 (hp->dmai_rflags & DDI_DMA_READ)) {
2323 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2324 DDI_DMA_SYNC_FORCPU);
2325 }
2326
2327 /*
2328 * cleanup and copy buffer or window state. if we didn't use the copy
2329 * buffer or windows, there won't be much to do :-)
2330 */
2331 rootnex_teardown_copybuf(dma);
2332 rootnex_teardown_windows(dma);
2333
2334 #if defined(__amd64) && !defined(__xpv)
2335 if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2336 (void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2337 &dma->dp_dvma);
2338 #endif
2339
2340 /*
2341 * If we had to allocate space to for the worse case sgl (it didn't
2342 * fit into our pre-allocate buffer), free that up now
2343 */
2344 if (dma->dp_need_to_free_cookie) {
2345 kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2346 }
2347
2348 /*
2349 * clean up the handle so it's ready for the next bind (i.e. if the
2350 * handle is reused).
2351 */
2352 rootnex_clean_dmahdl(hp);
2353 hp->dmai_error.err_cf = NULL;
2354
2355 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2356 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2357 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2358
2359 return (DDI_SUCCESS);
2360 }
2361
2362 /*
2363 * rootnex_dma_unbindhdl()
2364 * called from ddi_dma_unbind_handle()
2365 */
2366 /*ARGSUSED*/
2367 static int
2368 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2369 ddi_dma_handle_t handle)
2370 {
2371 int ret;
2372
2373 #if defined(__amd64) && !defined(__xpv)
2374 if (IOMMU_USED(rdip))
2375 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2376 else
2377 #endif
2378 ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2379
2380 if (rootnex_state->r_dvma_call_list_id)
2381 ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2382
2383 return (ret);
2384 }
2385
2386 #if defined(__amd64) && !defined(__xpv)
2387
2388 static int
2389 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2390 {
2391 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2392 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2393
2394 if (dma->dp_sleep_flags != KM_SLEEP &&
2395 dma->dp_sleep_flags != KM_NOSLEEP)
2396 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2397 return (dma->dp_sleep_flags);
2398 }
2399 /*ARGSUSED*/
2400 static void
2401 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2402 {
2403 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2404 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2405 rootnex_window_t *window;
2406
2407 if (dma->dp_window) {
2408 window = &dma->dp_window[dma->dp_current_win];
2409 hp->dmai_cookie = window->wd_first_cookie;
2410 } else {
2411 hp->dmai_cookie = dma->dp_cookies;
2412 }
2413 hp->dmai_cookie++;
2414 }
2415
2416 /*ARGSUSED*/
2417 static int
2418 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2419 ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2420 {
2421 int i;
2422 int km_flags;
2423 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2424 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2425 rootnex_window_t *window;
2426 ddi_dma_cookie_t *cp;
2427 ddi_dma_cookie_t *cookie;
2428
2429 ASSERT(*cookiepp == NULL);
2430 ASSERT(*ccountp == 0);
2431
2432 if (dma->dp_window) {
2433 window = &dma->dp_window[dma->dp_current_win];
2434 cp = window->wd_first_cookie;
2435 *ccountp = window->wd_cookie_cnt;
2436 } else {
2437 cp = dma->dp_cookies;
2438 *ccountp = dma->dp_sglinfo.si_sgl_size;
2439 }
2440
2441 km_flags = rootnex_coredma_get_sleep_flags(handle);
2442 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2443 if (cookie == NULL) {
2444 return (DDI_DMA_NORESOURCES);
2445 }
2446
2447 for (i = 0; i < *ccountp; i++) {
2448 cookie[i].dmac_notused = cp[i].dmac_notused;
2449 cookie[i].dmac_type = cp[i].dmac_type;
2450 cookie[i].dmac_address = cp[i].dmac_address;
2451 cookie[i].dmac_size = cp[i].dmac_size;
2452 }
2453
2454 *cookiepp = cookie;
2455
2456 return (DDI_SUCCESS);
2457 }
2458
2459 /*ARGSUSED*/
2460 static int
2461 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2462 ddi_dma_cookie_t *cookiep, uint_t ccount)
2463 {
2464 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2465 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2466 rootnex_window_t *window;
2467 ddi_dma_cookie_t *cur_cookiep;
2468
2469 ASSERT(cookiep);
2470 ASSERT(ccount != 0);
2471 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2472
2473 if (dma->dp_window) {
2474 window = &dma->dp_window[dma->dp_current_win];
2475 dma->dp_saved_cookies = window->wd_first_cookie;
2476 window->wd_first_cookie = cookiep;
2477 ASSERT(ccount == window->wd_cookie_cnt);
2478 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2479 + window->wd_first_cookie;
2480 } else {
2481 dma->dp_saved_cookies = dma->dp_cookies;
2482 dma->dp_cookies = cookiep;
2483 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2484 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2485 + dma->dp_cookies;
2486 }
2487
2488 dma->dp_need_to_switch_cookies = B_TRUE;
2489 hp->dmai_cookie = cur_cookiep;
2490
2491 return (DDI_SUCCESS);
2492 }
2493
2494 /*ARGSUSED*/
2495 static int
2496 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2497 {
2498 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2499 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2500 rootnex_window_t *window;
2501 ddi_dma_cookie_t *cur_cookiep;
2502 ddi_dma_cookie_t *cookie_array;
2503 uint_t ccount;
2504
2505 /* check if cookies have not been switched */
2506 if (dma->dp_need_to_switch_cookies == B_FALSE)
2507 return (DDI_SUCCESS);
2508
2509 ASSERT(dma->dp_saved_cookies);
2510
2511 if (dma->dp_window) {
2512 window = &dma->dp_window[dma->dp_current_win];
2513 cookie_array = window->wd_first_cookie;
2514 window->wd_first_cookie = dma->dp_saved_cookies;
2515 dma->dp_saved_cookies = NULL;
2516 ccount = window->wd_cookie_cnt;
2517 cur_cookiep = (hp->dmai_cookie - cookie_array)
2518 + window->wd_first_cookie;
2519 } else {
2520 cookie_array = dma->dp_cookies;
2521 dma->dp_cookies = dma->dp_saved_cookies;
2522 dma->dp_saved_cookies = NULL;
2523 ccount = dma->dp_sglinfo.si_sgl_size;
2524 cur_cookiep = (hp->dmai_cookie - cookie_array)
2525 + dma->dp_cookies;
2526 }
2527
2528 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2529
2530 hp->dmai_cookie = cur_cookiep;
2531
2532 dma->dp_need_to_switch_cookies = B_FALSE;
2533
2534 return (DDI_SUCCESS);
2535 }
2536
2537 #endif
2538
2539 static struct as *
2540 rootnex_get_as(ddi_dma_obj_t *dmao)
2541 {
2542 struct as *asp;
2543
2544 switch (dmao->dmao_type) {
2545 case DMA_OTYP_VADDR:
2546 case DMA_OTYP_BUFVADDR:
2547 asp = dmao->dmao_obj.virt_obj.v_as;
2548 if (asp == NULL)
2549 asp = &kas;
2550 break;
2551 default:
2552 asp = NULL;
2553 break;
2554 }
2555 return (asp);
2556 }
2557
2558 /*
2559 * rootnex_verify_buffer()
2560 * verify buffer wasn't free'd
2561 */
2562 static int
2563 rootnex_verify_buffer(rootnex_dma_t *dma)
2564 {
2565 page_t **pplist;
2566 caddr_t vaddr;
2567 uint_t pcnt;
2568 uint_t poff;
2569 page_t *pp;
2570 char b;
2571 int i;
2572
2573 /* Figure out how many pages this buffer occupies */
2574 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2575 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2576 } else {
2577 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2578 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2579 }
2580 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2581
2582 switch (dma->dp_dma.dmao_type) {
2583 case DMA_OTYP_PAGES:
2584 /*
2585 * for a linked list of pp's walk through them to make sure
2586 * they're locked and not free.
2587 */
2588 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2589 for (i = 0; i < pcnt; i++) {
2590 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2591 return (DDI_FAILURE);
2592 }
2593 pp = pp->p_next;
2594 }
2595 break;
2596
2597 case DMA_OTYP_VADDR:
2598 case DMA_OTYP_BUFVADDR:
2599 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2600 /*
2601 * for an array of pp's walk through them to make sure they're
2602 * not free. It's possible that they may not be locked.
2603 */
2604 if (pplist) {
2605 for (i = 0; i < pcnt; i++) {
2606 if (PP_ISFREE(pplist[i])) {
2607 return (DDI_FAILURE);
2608 }
2609 }
2610
2611 /* For a virtual address, try to peek at each page */
2612 } else {
2613 if (rootnex_get_as(&dma->dp_dma) == &kas) {
2614 for (i = 0; i < pcnt; i++) {
2615 if (ddi_peek8(NULL, vaddr, &b) ==
2616 DDI_FAILURE)
2617 return (DDI_FAILURE);
2618 vaddr += MMU_PAGESIZE;
2619 }
2620 }
2621 }
2622 break;
2623
2624 default:
2625 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2626 break;
2627 }
2628
2629 return (DDI_SUCCESS);
2630 }
2631
2632
2633 /*
2634 * rootnex_clean_dmahdl()
2635 * Clean the dma handle. This should be called on a handle alloc and an
2636 * unbind handle. Set the handle state to the default settings.
2637 */
2638 static void
2639 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2640 {
2641 rootnex_dma_t *dma;
2642
2643
2644 dma = (rootnex_dma_t *)hp->dmai_private;
2645
2646 hp->dmai_nwin = 0;
2647 dma->dp_current_cookie = 0;
2648 dma->dp_copybuf_size = 0;
2649 dma->dp_window = NULL;
2650 dma->dp_cbaddr = NULL;
2651 dma->dp_inuse = B_FALSE;
2652 dma->dp_dvma_used = B_FALSE;
2653 dma->dp_need_to_free_cookie = B_FALSE;
2654 dma->dp_need_to_switch_cookies = B_FALSE;
2655 dma->dp_saved_cookies = NULL;
2656 dma->dp_sleep_flags = KM_PANIC;
2657 dma->dp_need_to_free_window = B_FALSE;
2658 dma->dp_partial_required = B_FALSE;
2659 dma->dp_trim_required = B_FALSE;
2660 dma->dp_sglinfo.si_copybuf_req = 0;
2661 #if !defined(__amd64)
2662 dma->dp_cb_remaping = B_FALSE;
2663 dma->dp_kva = NULL;
2664 #endif
2665
2666 /* FMA related initialization */
2667 hp->dmai_fault = 0;
2668 hp->dmai_fault_check = NULL;
2669 hp->dmai_fault_notify = NULL;
2670 hp->dmai_error.err_ena = 0;
2671 hp->dmai_error.err_status = DDI_FM_OK;
2672 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2673 hp->dmai_error.err_ontrap = NULL;
2674 }
2675
2676
2677 /*
2678 * rootnex_valid_alloc_parms()
2679 * Called in ddi_dma_alloc_handle path to validate its parameters.
2680 */
2681 static int
2682 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2683 {
2684 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2685 (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2686 (attr->dma_attr_granular > MMU_PAGESIZE) ||
2687 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2688 return (DDI_DMA_BADATTR);
2689 }
2690
2691 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2692 return (DDI_DMA_BADATTR);
2693 }
2694
2695 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2696 MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2697 attr->dma_attr_sgllen == 0) {
2698 return (DDI_DMA_BADATTR);
2699 }
2700
2701 /* We should be able to DMA into every byte offset in a page */
2702 if (maxsegmentsize < MMU_PAGESIZE) {
2703 return (DDI_DMA_BADATTR);
2704 }
2705
2706 /* if we're bouncing on seg, seg must be <= addr_hi */
2707 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2708 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2709 return (DDI_DMA_BADATTR);
2710 }
2711 return (DDI_SUCCESS);
2712 }
2713
2714 /*
2715 * rootnex_valid_bind_parms()
2716 * Called in ddi_dma_*_bind_handle path to validate its parameters.
2717 */
2718 /* ARGSUSED */
2719 static int
2720 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2721 {
2722 #if !defined(__amd64)
2723 /*
2724 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2725 * we can track the offset for the obsoleted interfaces.
2726 */
2727 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2728 return (DDI_DMA_TOOBIG);
2729 }
2730 #endif
2731
2732 return (DDI_SUCCESS);
2733 }
2734
2735
2736 /*
2737 * rootnex_need_bounce_seg()
2738 * check to see if the buffer lives on both side of the seg.
2739 */
2740 static boolean_t
2741 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2742 {
2743 ddi_dma_atyp_t buftype;
2744 rootnex_addr_t raddr;
2745 boolean_t lower_addr;
2746 boolean_t upper_addr;
2747 uint64_t offset;
2748 page_t **pplist;
2749 uint64_t paddr;
2750 uint32_t psize;
2751 uint32_t size;
2752 caddr_t vaddr;
2753 uint_t pcnt;
2754 page_t *pp;
2755
2756
2757 /* shortcuts */
2758 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2759 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2760 buftype = dmar_object->dmao_type;
2761 size = dmar_object->dmao_size;
2762
2763 lower_addr = B_FALSE;
2764 upper_addr = B_FALSE;
2765 pcnt = 0;
2766
2767 /*
2768 * Process the first page to handle the initial offset of the buffer.
2769 * We'll use the base address we get later when we loop through all
2770 * the pages.
2771 */
2772 if (buftype == DMA_OTYP_PAGES) {
2773 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2774 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2775 MMU_PAGEOFFSET;
2776 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2777 psize = MIN(size, (MMU_PAGESIZE - offset));
2778 pp = pp->p_next;
2779 sglinfo->si_asp = NULL;
2780 } else if (pplist != NULL) {
2781 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2782 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2783 if (sglinfo->si_asp == NULL) {
2784 sglinfo->si_asp = &kas;
2785 }
2786 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2787 paddr += offset;
2788 psize = MIN(size, (MMU_PAGESIZE - offset));
2789 pcnt++;
2790 } else {
2791 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2792 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2793 if (sglinfo->si_asp == NULL) {
2794 sglinfo->si_asp = &kas;
2795 }
2796 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2797 paddr += offset;
2798 psize = MIN(size, (MMU_PAGESIZE - offset));
2799 vaddr += psize;
2800 }
2801
2802 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2803
2804 if ((raddr + psize) > sglinfo->si_segmask) {
2805 upper_addr = B_TRUE;
2806 } else {
2807 lower_addr = B_TRUE;
2808 }
2809 size -= psize;
2810
2811 /*
2812 * Walk through the rest of the pages in the buffer. Track to see
2813 * if we have pages on both sides of the segment boundary.
2814 */
2815 while (size > 0) {
2816 /* partial or full page */
2817 psize = MIN(size, MMU_PAGESIZE);
2818
2819 if (buftype == DMA_OTYP_PAGES) {
2820 /* get the paddr from the page_t */
2821 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2822 paddr = pfn_to_pa(pp->p_pagenum);
2823 pp = pp->p_next;
2824 } else if (pplist != NULL) {
2825 /* index into the array of page_t's to get the paddr */
2826 ASSERT(!PP_ISFREE(pplist[pcnt]));
2827 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2828 pcnt++;
2829 } else {
2830 /* call into the VM to get the paddr */
2831 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2832 vaddr));
2833 vaddr += psize;
2834 }
2835
2836 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2837
2838 if ((raddr + psize) > sglinfo->si_segmask) {
2839 upper_addr = B_TRUE;
2840 } else {
2841 lower_addr = B_TRUE;
2842 }
2843 /*
2844 * if the buffer lives both above and below the segment
2845 * boundary, or the current page is the page immediately
2846 * after the segment, we will use a copy/bounce buffer for
2847 * all pages > seg.
2848 */
2849 if ((lower_addr && upper_addr) ||
2850 (raddr == (sglinfo->si_segmask + 1))) {
2851 return (B_TRUE);
2852 }
2853
2854 size -= psize;
2855 }
2856
2857 return (B_FALSE);
2858 }
2859
2860 /*
2861 * rootnex_get_sgl()
2862 * Called in bind fastpath to get the sgl. Most of this will be replaced
2863 * with a call to the vm layer when vm2.0 comes around...
2864 */
2865 static void
2866 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2867 rootnex_sglinfo_t *sglinfo)
2868 {
2869 ddi_dma_atyp_t buftype;
2870 rootnex_addr_t raddr;
2871 uint64_t last_page;
2872 uint64_t offset;
2873 uint64_t addrhi;
2874 uint64_t addrlo;
2875 uint64_t maxseg;
2876 page_t **pplist;
2877 uint64_t paddr;
2878 uint32_t psize;
2879 uint32_t size;
2880 caddr_t vaddr;
2881 uint_t pcnt;
2882 page_t *pp;
2883 uint_t cnt;
2884
2885
2886 /* shortcuts */
2887 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2888 vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2889 maxseg = sglinfo->si_max_cookie_size;
2890 buftype = dmar_object->dmao_type;
2891 addrhi = sglinfo->si_max_addr;
2892 addrlo = sglinfo->si_min_addr;
2893 size = dmar_object->dmao_size;
2894
2895 pcnt = 0;
2896 cnt = 0;
2897
2898
2899 /*
2900 * check to see if we need to use the copy buffer for pages over
2901 * the segment attr.
2902 */
2903 sglinfo->si_bounce_on_seg = B_FALSE;
2904 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2905 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2906 dmar_object, sglinfo);
2907 }
2908
2909 /*
2910 * if we were passed down a linked list of pages, i.e. pointer to
2911 * page_t, use this to get our physical address and buf offset.
2912 */
2913 if (buftype == DMA_OTYP_PAGES) {
2914 pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2915 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2916 offset = dmar_object->dmao_obj.pp_obj.pp_offset &
2917 MMU_PAGEOFFSET;
2918 paddr = pfn_to_pa(pp->p_pagenum) + offset;
2919 psize = MIN(size, (MMU_PAGESIZE - offset));
2920 pp = pp->p_next;
2921 sglinfo->si_asp = NULL;
2922
2923 /*
2924 * We weren't passed down a linked list of pages, but if we were passed
2925 * down an array of pages, use this to get our physical address and buf
2926 * offset.
2927 */
2928 } else if (pplist != NULL) {
2929 ASSERT((buftype == DMA_OTYP_VADDR) ||
2930 (buftype == DMA_OTYP_BUFVADDR));
2931
2932 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2933 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2934 if (sglinfo->si_asp == NULL) {
2935 sglinfo->si_asp = &kas;
2936 }
2937
2938 ASSERT(!PP_ISFREE(pplist[pcnt]));
2939 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2940 paddr += offset;
2941 psize = MIN(size, (MMU_PAGESIZE - offset));
2942 pcnt++;
2943
2944 /*
2945 * All we have is a virtual address, we'll need to call into the VM
2946 * to get the physical address.
2947 */
2948 } else {
2949 ASSERT((buftype == DMA_OTYP_VADDR) ||
2950 (buftype == DMA_OTYP_BUFVADDR));
2951
2952 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2953 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2954 if (sglinfo->si_asp == NULL) {
2955 sglinfo->si_asp = &kas;
2956 }
2957
2958 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2959 paddr += offset;
2960 psize = MIN(size, (MMU_PAGESIZE - offset));
2961 vaddr += psize;
2962 }
2963
2964 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2965
2966 /*
2967 * Setup the first cookie with the physical address of the page and the
2968 * size of the page (which takes into account the initial offset into
2969 * the page.
2970 */
2971 sgl[cnt].dmac_laddress = raddr;
2972 sgl[cnt].dmac_size = psize;
2973 sgl[cnt].dmac_type = 0;
2974
2975 /*
2976 * Save away the buffer offset into the page. We'll need this later in
2977 * the copy buffer code to help figure out the page index within the
2978 * buffer and the offset into the current page.
2979 */
2980 sglinfo->si_buf_offset = offset;
2981
2982 /*
2983 * If we are using the copy buffer for anything over the segment
2984 * boundary, and this page is over the segment boundary.
2985 * OR
2986 * if the DMA engine can't reach the physical address.
2987 */
2988 if (((sglinfo->si_bounce_on_seg) &&
2989 ((raddr + psize) > sglinfo->si_segmask)) ||
2990 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2991 /*
2992 * Increase how much copy buffer we use. We always increase by
2993 * pagesize so we don't have to worry about converting offsets.
2994 * Set a flag in the cookies dmac_type to indicate that it uses
2995 * the copy buffer. If this isn't the last cookie, go to the
2996 * next cookie (since we separate each page which uses the copy
2997 * buffer in case the copy buffer is not physically contiguous.
2998 */
2999 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3000 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3001 if ((cnt + 1) < sglinfo->si_max_pages) {
3002 cnt++;
3003 sgl[cnt].dmac_laddress = 0;
3004 sgl[cnt].dmac_size = 0;
3005 sgl[cnt].dmac_type = 0;
3006 }
3007 }
3008
3009 /*
3010 * save this page's physical address so we can figure out if the next
3011 * page is physically contiguous. Keep decrementing size until we are
3012 * done with the buffer.
3013 */
3014 last_page = raddr & MMU_PAGEMASK;
3015 size -= psize;
3016
3017 while (size > 0) {
3018 /* Get the size for this page (i.e. partial or full page) */
3019 psize = MIN(size, MMU_PAGESIZE);
3020
3021 if (buftype == DMA_OTYP_PAGES) {
3022 /* get the paddr from the page_t */
3023 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3024 paddr = pfn_to_pa(pp->p_pagenum);
3025 pp = pp->p_next;
3026 } else if (pplist != NULL) {
3027 /* index into the array of page_t's to get the paddr */
3028 ASSERT(!PP_ISFREE(pplist[pcnt]));
3029 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3030 pcnt++;
3031 } else {
3032 /* call into the VM to get the paddr */
3033 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3034 vaddr));
3035 vaddr += psize;
3036 }
3037
3038 raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3039
3040 /*
3041 * If we are using the copy buffer for anything over the
3042 * segment boundary, and this page is over the segment
3043 * boundary.
3044 * OR
3045 * if the DMA engine can't reach the physical address.
3046 */
3047 if (((sglinfo->si_bounce_on_seg) &&
3048 ((raddr + psize) > sglinfo->si_segmask)) ||
3049 ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3050
3051 sglinfo->si_copybuf_req += MMU_PAGESIZE;
3052
3053 /*
3054 * if there is something in the current cookie, go to
3055 * the next one. We only want one page in a cookie which
3056 * uses the copybuf since the copybuf doesn't have to
3057 * be physically contiguous.
3058 */
3059 if (sgl[cnt].dmac_size != 0) {
3060 cnt++;
3061 }
3062 sgl[cnt].dmac_laddress = raddr;
3063 sgl[cnt].dmac_size = psize;
3064 #if defined(__amd64)
3065 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3066 #else
3067 /*
3068 * save the buf offset for 32-bit kernel. used in the
3069 * obsoleted interfaces.
3070 */
3071 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3072 (dmar_object->dmao_size - size);
3073 #endif
3074 /* if this isn't the last cookie, go to the next one */
3075 if ((cnt + 1) < sglinfo->si_max_pages) {
3076 cnt++;
3077 sgl[cnt].dmac_laddress = 0;
3078 sgl[cnt].dmac_size = 0;
3079 sgl[cnt].dmac_type = 0;
3080 }
3081
3082 /*
3083 * this page didn't need the copy buffer, if it's not physically
3084 * contiguous, or it would put us over a segment boundary, or it
3085 * puts us over the max cookie size, or the current sgl doesn't
3086 * have anything in it.
3087 */
3088 } else if (((last_page + MMU_PAGESIZE) != raddr) ||
3089 !(raddr & sglinfo->si_segmask) ||
3090 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3091 (sgl[cnt].dmac_size == 0)) {
3092 /*
3093 * if we're not already in a new cookie, go to the next
3094 * cookie.
3095 */
3096 if (sgl[cnt].dmac_size != 0) {
3097 cnt++;
3098 }
3099
3100 /* save the cookie information */
3101 sgl[cnt].dmac_laddress = raddr;
3102 sgl[cnt].dmac_size = psize;
3103 #if defined(__amd64)
3104 sgl[cnt].dmac_type = 0;
3105 #else
3106 /*
3107 * save the buf offset for 32-bit kernel. used in the
3108 * obsoleted interfaces.
3109 */
3110 sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3111 #endif
3112
3113 /*
3114 * this page didn't need the copy buffer, it is physically
3115 * contiguous with the last page, and it's <= the max cookie
3116 * size.
3117 */
3118 } else {
3119 sgl[cnt].dmac_size += psize;
3120
3121 /*
3122 * if this exactly == the maximum cookie size, and
3123 * it isn't the last cookie, go to the next cookie.
3124 */
3125 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3126 ((cnt + 1) < sglinfo->si_max_pages)) {
3127 cnt++;
3128 sgl[cnt].dmac_laddress = 0;
3129 sgl[cnt].dmac_size = 0;
3130 sgl[cnt].dmac_type = 0;
3131 }
3132 }
3133
3134 /*
3135 * save this page's physical address so we can figure out if the
3136 * next page is physically contiguous. Keep decrementing size
3137 * until we are done with the buffer.
3138 */
3139 last_page = raddr;
3140 size -= psize;
3141 }
3142
3143 /* we're done, save away how many cookies the sgl has */
3144 if (sgl[cnt].dmac_size == 0) {
3145 ASSERT(cnt < sglinfo->si_max_pages);
3146 sglinfo->si_sgl_size = cnt;
3147 } else {
3148 sglinfo->si_sgl_size = cnt + 1;
3149 }
3150 }
3151
3152 static void
3153 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3154 rootnex_sglinfo_t *sglinfo)
3155 {
3156 uint64_t offset;
3157 uint64_t maxseg;
3158 uint64_t dvaddr;
3159 struct dvmaseg *dvs;
3160 uint64_t paddr;
3161 uint32_t psize, ssize;
3162 uint32_t size;
3163 uint_t cnt;
3164 int physcontig;
3165
3166 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3167
3168 /* shortcuts */
3169 maxseg = sglinfo->si_max_cookie_size;
3170 size = dmar_object->dmao_size;
3171
3172 cnt = 0;
3173 sglinfo->si_bounce_on_seg = B_FALSE;
3174
3175 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3176 offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3177 ssize = dvs->dvs_len;
3178 paddr = dvs->dvs_start;
3179 paddr += offset;
3180 psize = MIN(ssize, (maxseg - offset));
3181 dvaddr = paddr + psize;
3182 ssize -= psize;
3183
3184 sgl[cnt].dmac_laddress = paddr;
3185 sgl[cnt].dmac_size = psize;
3186 sgl[cnt].dmac_type = 0;
3187
3188 size -= psize;
3189 while (size > 0) {
3190 if (ssize == 0) {
3191 dvs++;
3192 ssize = dvs->dvs_len;
3193 dvaddr = dvs->dvs_start;
3194 physcontig = 0;
3195 } else
3196 physcontig = 1;
3197
3198 paddr = dvaddr;
3199 psize = MIN(ssize, maxseg);
3200 dvaddr += psize;
3201 ssize -= psize;
3202
3203 if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3204 ((sgl[cnt].dmac_size + psize) > maxseg) ||
3205 (sgl[cnt].dmac_size == 0)) {
3206 /*
3207 * if we're not already in a new cookie, go to the next
3208 * cookie.
3209 */
3210 if (sgl[cnt].dmac_size != 0) {
3211 cnt++;
3212 }
3213
3214 /* save the cookie information */
3215 sgl[cnt].dmac_laddress = paddr;
3216 sgl[cnt].dmac_size = psize;
3217 sgl[cnt].dmac_type = 0;
3218 } else {
3219 sgl[cnt].dmac_size += psize;
3220
3221 /*
3222 * if this exactly == the maximum cookie size, and
3223 * it isn't the last cookie, go to the next cookie.
3224 */
3225 if (((sgl[cnt].dmac_size + psize) == maxseg) &&
3226 ((cnt + 1) < sglinfo->si_max_pages)) {
3227 cnt++;
3228 sgl[cnt].dmac_laddress = 0;
3229 sgl[cnt].dmac_size = 0;
3230 sgl[cnt].dmac_type = 0;
3231 }
3232 }
3233 size -= psize;
3234 }
3235
3236 /* we're done, save away how many cookies the sgl has */
3237 if (sgl[cnt].dmac_size == 0) {
3238 sglinfo->si_sgl_size = cnt;
3239 } else {
3240 sglinfo->si_sgl_size = cnt + 1;
3241 }
3242 }
3243
3244 /*
3245 * rootnex_bind_slowpath()
3246 * Call in the bind path if the calling driver can't use the sgl without
3247 * modifying it. We either need to use the copy buffer and/or we will end up
3248 * with a partial bind.
3249 */
3250 static int
3251 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3252 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3253 {
3254 rootnex_sglinfo_t *sinfo;
3255 rootnex_window_t *window;
3256 ddi_dma_cookie_t *cookie;
3257 size_t copybuf_used;
3258 size_t dmac_size;
3259 boolean_t partial;
3260 off_t cur_offset;
3261 page_t *cur_pp;
3262 major_t mnum;
3263 int e;
3264 int i;
3265
3266
3267 sinfo = &dma->dp_sglinfo;
3268 copybuf_used = 0;
3269 partial = B_FALSE;
3270
3271 /*
3272 * If we're using the copybuf, set the copybuf state in dma struct.
3273 * Needs to be first since it sets the copy buffer size.
3274 */
3275 if (sinfo->si_copybuf_req != 0) {
3276 e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3277 if (e != DDI_SUCCESS) {
3278 return (e);
3279 }
3280 } else {
3281 dma->dp_copybuf_size = 0;
3282 }
3283
3284 /*
3285 * Figure out if we need to do a partial mapping. If so, figure out
3286 * if we need to trim the buffers when we munge the sgl.
3287 */
3288 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3289 (dmao->dmao_size > dma->dp_maxxfer) ||
3290 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3291 dma->dp_partial_required = B_TRUE;
3292 if (attr->dma_attr_granular != 1) {
3293 dma->dp_trim_required = B_TRUE;
3294 }
3295 } else {
3296 dma->dp_partial_required = B_FALSE;
3297 dma->dp_trim_required = B_FALSE;
3298 }
3299
3300 /* If we need to do a partial bind, make sure the driver supports it */
3301 if (dma->dp_partial_required &&
3302 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3303
3304 mnum = ddi_driver_major(dma->dp_dip);
3305 /*
3306 * patchable which allows us to print one warning per major
3307 * number.
3308 */
3309 if ((rootnex_bind_warn) &&
3310 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3311 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3312 cmn_err(CE_WARN, "!%s: coding error detected, the "
3313 "driver is using ddi_dma_attr(9S) incorrectly. "
3314 "There is a small risk of data corruption in "
3315 "particular with large I/Os. The driver should be "
3316 "replaced with a corrected version for proper "
3317 "system operation. To disable this warning, add "
3318 "'set rootnex:rootnex_bind_warn=0' to "
3319 "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3320 }
3321 return (DDI_DMA_TOOBIG);
3322 }
3323
3324 /*
3325 * we might need multiple windows, setup state to handle them. In this
3326 * code path, we will have at least one window.
3327 */
3328 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3329 if (e != DDI_SUCCESS) {
3330 rootnex_teardown_copybuf(dma);
3331 return (e);
3332 }
3333
3334 window = &dma->dp_window[0];
3335 cookie = &dma->dp_cookies[0];
3336 cur_offset = 0;
3337 rootnex_init_win(hp, dma, window, cookie, cur_offset);
3338 if (dmao->dmao_type == DMA_OTYP_PAGES) {
3339 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3340 }
3341
3342 /* loop though all the cookies we got back from get_sgl() */
3343 for (i = 0; i < sinfo->si_sgl_size; i++) {
3344 /*
3345 * If we're using the copy buffer, check this cookie and setup
3346 * its associated copy buffer state. If this cookie uses the
3347 * copy buffer, make sure we sync this window during dma_sync.
3348 */
3349 if (dma->dp_copybuf_size > 0) {
3350 rootnex_setup_cookie(dmao, dma, cookie,
3351 cur_offset, ©buf_used, &cur_pp);
3352 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3353 window->wd_dosync = B_TRUE;
3354 }
3355 }
3356
3357 /*
3358 * save away the cookie size, since it could be modified in
3359 * the windowing code.
3360 */
3361 dmac_size = cookie->dmac_size;
3362
3363 /* if we went over max copybuf size */
3364 if (dma->dp_copybuf_size &&
3365 (copybuf_used > dma->dp_copybuf_size)) {
3366 partial = B_TRUE;
3367 e = rootnex_copybuf_window_boundary(hp, dma, &window,
3368 cookie, cur_offset, ©buf_used);
3369 if (e != DDI_SUCCESS) {
3370 rootnex_teardown_copybuf(dma);
3371 rootnex_teardown_windows(dma);
3372 return (e);
3373 }
3374
3375 /*
3376 * if the coookie uses the copy buffer, make sure the
3377 * new window we just moved to is set to sync.
3378 */
3379 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3380 window->wd_dosync = B_TRUE;
3381 }
3382 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3383 dma->dp_dip);
3384
3385 /* if the cookie cnt == max sgllen, move to the next window */
3386 } else if (window->wd_cookie_cnt >=
3387 (unsigned)attr->dma_attr_sgllen) {
3388 partial = B_TRUE;
3389 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3390 e = rootnex_sgllen_window_boundary(hp, dma, &window,
3391 cookie, attr, cur_offset);
3392 if (e != DDI_SUCCESS) {
3393 rootnex_teardown_copybuf(dma);
3394 rootnex_teardown_windows(dma);
3395 return (e);
3396 }
3397
3398 /*
3399 * if the coookie uses the copy buffer, make sure the
3400 * new window we just moved to is set to sync.
3401 */
3402 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3403 window->wd_dosync = B_TRUE;
3404 }
3405 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3406 dma->dp_dip);
3407
3408 /* else if we will be over maxxfer */
3409 } else if ((window->wd_size + dmac_size) >
3410 dma->dp_maxxfer) {
3411 partial = B_TRUE;
3412 e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3413 cookie);
3414 if (e != DDI_SUCCESS) {
3415 rootnex_teardown_copybuf(dma);
3416 rootnex_teardown_windows(dma);
3417 return (e);
3418 }
3419
3420 /*
3421 * if the coookie uses the copy buffer, make sure the
3422 * new window we just moved to is set to sync.
3423 */
3424 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3425 window->wd_dosync = B_TRUE;
3426 }
3427 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3428 dma->dp_dip);
3429
3430 /* else this cookie fits in the current window */
3431 } else {
3432 window->wd_cookie_cnt++;
3433 window->wd_size += dmac_size;
3434 }
3435
3436 /* track our offset into the buffer, go to the next cookie */
3437 ASSERT(dmac_size <= dmao->dmao_size);
3438 ASSERT(cookie->dmac_size <= dmac_size);
3439 cur_offset += dmac_size;
3440 cookie++;
3441 }
3442
3443 /* if we ended up with a zero sized window in the end, clean it up */
3444 if (window->wd_size == 0) {
3445 hp->dmai_nwin--;
3446 window--;
3447 }
3448
3449 ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3450
3451 if (!partial) {
3452 return (DDI_DMA_MAPPED);
3453 }
3454
3455 ASSERT(dma->dp_partial_required);
3456 return (DDI_DMA_PARTIAL_MAP);
3457 }
3458
3459 /*
3460 * rootnex_setup_copybuf()
3461 * Called in bind slowpath. Figures out if we're going to use the copy
3462 * buffer, and if we do, sets up the basic state to handle it.
3463 */
3464 static int
3465 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3466 rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3467 {
3468 rootnex_sglinfo_t *sinfo;
3469 ddi_dma_attr_t lattr;
3470 size_t max_copybuf;
3471 int cansleep;
3472 int e;
3473 #if !defined(__amd64)
3474 int vmflag;
3475 #endif
3476
3477 ASSERT(!dma->dp_dvma_used);
3478
3479 sinfo = &dma->dp_sglinfo;
3480
3481 /* read this first so it's consistent through the routine */
3482 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3483
3484 /* We need to call into the rootnex on ddi_dma_sync() */
3485 hp->dmai_rflags &= ~DMP_NOSYNC;
3486
3487 /* make sure the copybuf size <= the max size */
3488 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3489 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3490
3491 #if !defined(__amd64)
3492 /*
3493 * if we don't have kva space to copy to/from, allocate the KVA space
3494 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3495 * the 64-bit kernel.
3496 */
3497 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3498 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3499
3500 /* convert the sleep flags */
3501 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3502 vmflag = VM_SLEEP;
3503 } else {
3504 vmflag = VM_NOSLEEP;
3505 }
3506
3507 /* allocate Kernel VA space that we can bcopy to/from */
3508 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3509 vmflag);
3510 if (dma->dp_kva == NULL) {
3511 return (DDI_DMA_NORESOURCES);
3512 }
3513 }
3514 #endif
3515
3516 /* convert the sleep flags */
3517 if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3518 cansleep = 1;
3519 } else {
3520 cansleep = 0;
3521 }
3522
3523 /*
3524 * Allocate the actual copy buffer. This needs to fit within the DMA
3525 * engine limits, so we can't use kmem_alloc... We don't need
3526 * contiguous memory (sgllen) since we will be forcing windows on
3527 * sgllen anyway.
3528 */
3529 lattr = *attr;
3530 lattr.dma_attr_align = MMU_PAGESIZE;
3531 lattr.dma_attr_sgllen = -1; /* no limit */
3532 /*
3533 * if we're using the copy buffer because of seg, use that for our
3534 * upper address limit.
3535 */
3536 if (sinfo->si_bounce_on_seg) {
3537 lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3538 }
3539 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3540 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3541 if (e != DDI_SUCCESS) {
3542 #if !defined(__amd64)
3543 if (dma->dp_kva != NULL) {
3544 vmem_free(heap_arena, dma->dp_kva,
3545 dma->dp_copybuf_size);
3546 }
3547 #endif
3548 return (DDI_DMA_NORESOURCES);
3549 }
3550
3551 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3552 size_t, dma->dp_copybuf_size);
3553
3554 return (DDI_SUCCESS);
3555 }
3556
3557
3558 /*
3559 * rootnex_setup_windows()
3560 * Called in bind slowpath to setup the window state. We always have windows
3561 * in the slowpath. Even if the window count = 1.
3562 */
3563 static int
3564 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3565 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3566 {
3567 rootnex_window_t *windowp;
3568 rootnex_sglinfo_t *sinfo;
3569 size_t copy_state_size;
3570 size_t win_state_size;
3571 size_t state_available;
3572 size_t space_needed;
3573 uint_t copybuf_win;
3574 uint_t maxxfer_win;
3575 size_t space_used;
3576 uint_t sglwin;
3577
3578
3579 sinfo = &dma->dp_sglinfo;
3580
3581 dma->dp_current_win = 0;
3582 hp->dmai_nwin = 0;
3583
3584 /* If we don't need to do a partial, we only have one window */
3585 if (!dma->dp_partial_required) {
3586 dma->dp_max_win = 1;
3587
3588 /*
3589 * we need multiple windows, need to figure out the worse case number
3590 * of windows.
3591 */
3592 } else {
3593 /*
3594 * if we need windows because we need more copy buffer that
3595 * we allow, the worse case number of windows we could need
3596 * here would be (copybuf space required / copybuf space that
3597 * we have) plus one for remainder, and plus 2 to handle the
3598 * extra pages on the trim for the first and last pages of the
3599 * buffer (a page is the minimum window size so under the right
3600 * attr settings, you could have a window for each page).
3601 * The last page will only be hit here if the size is not a
3602 * multiple of the granularity (which theoretically shouldn't
3603 * be the case but never has been enforced, so we could have
3604 * broken things without it).
3605 */
3606 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3607 ASSERT(dma->dp_copybuf_size > 0);
3608 copybuf_win = (sinfo->si_copybuf_req /
3609 dma->dp_copybuf_size) + 1 + 2;
3610 } else {
3611 copybuf_win = 0;
3612 }
3613
3614 /*
3615 * if we need windows because we have more cookies than the H/W
3616 * can handle, the number of windows we would need here would
3617 * be (cookie count / cookies count H/W supports minus 1[for
3618 * trim]) plus one for remainder.
3619 */
3620 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3621 sglwin = (sinfo->si_sgl_size /
3622 (attr->dma_attr_sgllen - 1)) + 1;
3623 } else {
3624 sglwin = 0;
3625 }
3626
3627 /*
3628 * if we need windows because we're binding more memory than the
3629 * H/W can transfer at once, the number of windows we would need
3630 * here would be (xfer count / max xfer H/W supports) plus one
3631 * for remainder, and plus 2 to handle the extra pages on the
3632 * trim (see above comment about trim)
3633 */
3634 if (dmao->dmao_size > dma->dp_maxxfer) {
3635 maxxfer_win = (dmao->dmao_size /
3636 dma->dp_maxxfer) + 1 + 2;
3637 } else {
3638 maxxfer_win = 0;
3639 }
3640 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win;
3641 ASSERT(dma->dp_max_win > 0);
3642 }
3643 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3644
3645 /*
3646 * Get space for window and potential copy buffer state. Before we
3647 * go and allocate memory, see if we can get away with using what's
3648 * left in the pre-allocted state or the dynamically allocated sgl.
3649 */
3650 space_used = (uintptr_t)(sinfo->si_sgl_size *
3651 sizeof (ddi_dma_cookie_t));
3652
3653 /* if we dynamically allocated space for the cookies */
3654 if (dma->dp_need_to_free_cookie) {
3655 /* if we have more space in the pre-allocted buffer, use it */
3656 ASSERT(space_used <= dma->dp_cookie_size);
3657 if ((dma->dp_cookie_size - space_used) <=
3658 rootnex_state->r_prealloc_size) {
3659 state_available = rootnex_state->r_prealloc_size;
3660 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3661
3662 /*
3663 * else, we have more free space in the dynamically allocated
3664 * buffer, i.e. the buffer wasn't worse case fragmented so we
3665 * didn't need a lot of cookies.
3666 */
3667 } else {
3668 state_available = dma->dp_cookie_size - space_used;
3669 windowp = (rootnex_window_t *)
3670 &dma->dp_cookies[sinfo->si_sgl_size];
3671 }
3672
3673 /* we used the pre-alloced buffer */
3674 } else {
3675 ASSERT(space_used <= rootnex_state->r_prealloc_size);
3676 state_available = rootnex_state->r_prealloc_size - space_used;
3677 windowp = (rootnex_window_t *)
3678 &dma->dp_cookies[sinfo->si_sgl_size];
3679 }
3680
3681 /*
3682 * figure out how much state we need to track the copy buffer. Add an
3683 * addition 8 bytes for pointer alignemnt later.
3684 */
3685 if (dma->dp_copybuf_size > 0) {
3686 copy_state_size = sinfo->si_max_pages *
3687 sizeof (rootnex_pgmap_t);
3688 } else {
3689 copy_state_size = 0;
3690 }
3691 /* add an additional 8 bytes for pointer alignment */
3692 space_needed = win_state_size + copy_state_size + 0x8;
3693
3694 /* if we have enough space already, use it */
3695 if (state_available >= space_needed) {
3696 dma->dp_window = windowp;
3697 dma->dp_need_to_free_window = B_FALSE;
3698
3699 /* not enough space, need to allocate more. */
3700 } else {
3701 dma->dp_window = kmem_alloc(space_needed, kmflag);
3702 if (dma->dp_window == NULL) {
3703 return (DDI_DMA_NORESOURCES);
3704 }
3705 dma->dp_need_to_free_window = B_TRUE;
3706 dma->dp_window_size = space_needed;
3707 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3708 dma->dp_dip, size_t, space_needed);
3709 }
3710
3711 /*
3712 * we allocate copy buffer state and window state at the same time.
3713 * setup our copy buffer state pointers. Make sure it's aligned.
3714 */
3715 if (dma->dp_copybuf_size > 0) {
3716 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3717 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3718
3719 #if !defined(__amd64)
3720 /*
3721 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3722 * false/NULL. Should be quicker to bzero vs loop and set.
3723 */
3724 bzero(dma->dp_pgmap, copy_state_size);
3725 #endif
3726 } else {
3727 dma->dp_pgmap = NULL;
3728 }
3729
3730 return (DDI_SUCCESS);
3731 }
3732
3733
3734 /*
3735 * rootnex_teardown_copybuf()
3736 * cleans up after rootnex_setup_copybuf()
3737 */
3738 static void
3739 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3740 {
3741 #if !defined(__amd64)
3742 int i;
3743
3744 /*
3745 * if we allocated kernel heap VMEM space, go through all the pages and
3746 * map out any of the ones that we're mapped into the kernel heap VMEM
3747 * arena. Then free the VMEM space.
3748 */
3749 if (dma->dp_kva != NULL) {
3750 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3751 if (dma->dp_pgmap[i].pm_mapped) {
3752 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3753 MMU_PAGESIZE, HAT_UNLOAD);
3754 dma->dp_pgmap[i].pm_mapped = B_FALSE;
3755 }
3756 }
3757
3758 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3759 }
3760
3761 #endif
3762
3763 /* if we allocated a copy buffer, free it */
3764 if (dma->dp_cbaddr != NULL) {
3765 i_ddi_mem_free(dma->dp_cbaddr, NULL);
3766 }
3767 }
3768
3769
3770 /*
3771 * rootnex_teardown_windows()
3772 * cleans up after rootnex_setup_windows()
3773 */
3774 static void
3775 rootnex_teardown_windows(rootnex_dma_t *dma)
3776 {
3777 /*
3778 * if we had to allocate window state on the last bind (because we
3779 * didn't have enough pre-allocated space in the handle), free it.
3780 */
3781 if (dma->dp_need_to_free_window) {
3782 kmem_free(dma->dp_window, dma->dp_window_size);
3783 }
3784 }
3785
3786
3787 /*
3788 * rootnex_init_win()
3789 * Called in bind slow path during creation of a new window. Initializes
3790 * window state to default values.
3791 */
3792 /*ARGSUSED*/
3793 static void
3794 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3795 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3796 {
3797 hp->dmai_nwin++;
3798 window->wd_dosync = B_FALSE;
3799 window->wd_offset = cur_offset;
3800 window->wd_size = 0;
3801 window->wd_first_cookie = cookie;
3802 window->wd_cookie_cnt = 0;
3803 window->wd_trim.tr_trim_first = B_FALSE;
3804 window->wd_trim.tr_trim_last = B_FALSE;
3805 window->wd_trim.tr_first_copybuf_win = B_FALSE;
3806 window->wd_trim.tr_last_copybuf_win = B_FALSE;
3807 #if !defined(__amd64)
3808 window->wd_remap_copybuf = dma->dp_cb_remaping;
3809 #endif
3810 }
3811
3812
3813 /*
3814 * rootnex_setup_cookie()
3815 * Called in the bind slow path when the sgl uses the copy buffer. If any of
3816 * the sgl uses the copy buffer, we need to go through each cookie, figure
3817 * out if it uses the copy buffer, and if it does, save away everything we'll
3818 * need during sync.
3819 */
3820 static void
3821 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3822 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3823 page_t **cur_pp)
3824 {
3825 boolean_t copybuf_sz_power_2;
3826 rootnex_sglinfo_t *sinfo;
3827 paddr_t paddr;
3828 uint_t pidx;
3829 uint_t pcnt;
3830 off_t poff;
3831 #if defined(__amd64)
3832 pfn_t pfn;
3833 #else
3834 page_t **pplist;
3835 #endif
3836
3837 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3838
3839 sinfo = &dma->dp_sglinfo;
3840
3841 /*
3842 * Calculate the page index relative to the start of the buffer. The
3843 * index to the current page for our buffer is the offset into the
3844 * first page of the buffer plus our current offset into the buffer
3845 * itself, shifted of course...
3846 */
3847 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3848 ASSERT(pidx < sinfo->si_max_pages);
3849
3850 /* if this cookie uses the copy buffer */
3851 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3852 /*
3853 * NOTE: we know that since this cookie uses the copy buffer, it
3854 * is <= MMU_PAGESIZE.
3855 */
3856
3857 /*
3858 * get the offset into the page. For the 64-bit kernel, get the
3859 * pfn which we'll use with seg kpm.
3860 */
3861 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3862 #if defined(__amd64)
3863 /* mfn_to_pfn() is a NOP on i86pc */
3864 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3865 #endif /* __amd64 */
3866
3867 /* figure out if the copybuf size is a power of 2 */
3868 if (!ISP2(dma->dp_copybuf_size)) {
3869 copybuf_sz_power_2 = B_FALSE;
3870 } else {
3871 copybuf_sz_power_2 = B_TRUE;
3872 }
3873
3874 /* This page uses the copy buffer */
3875 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3876
3877 /*
3878 * save the copy buffer KVA that we'll use with this page.
3879 * if we still fit within the copybuf, it's a simple add.
3880 * otherwise, we need to wrap over using & or % accordingly.
3881 */
3882 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3883 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3884 *copybuf_used;
3885 } else {
3886 if (copybuf_sz_power_2) {
3887 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3888 (uintptr_t)dma->dp_cbaddr +
3889 (*copybuf_used &
3890 (dma->dp_copybuf_size - 1)));
3891 } else {
3892 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3893 (uintptr_t)dma->dp_cbaddr +
3894 (*copybuf_used % dma->dp_copybuf_size));
3895 }
3896 }
3897
3898 /*
3899 * over write the cookie physical address with the address of
3900 * the physical address of the copy buffer page that we will
3901 * use.
3902 */
3903 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3904 dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3905
3906 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3907
3908 /* if we have a kernel VA, it's easy, just save that address */
3909 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3910 (sinfo->si_asp == &kas)) {
3911 /*
3912 * save away the page aligned virtual address of the
3913 * driver buffer. Offsets are handled in the sync code.
3914 */
3915 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3916 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3917 & MMU_PAGEMASK);
3918 #if !defined(__amd64)
3919 /*
3920 * we didn't need to, and will never need to map this
3921 * page.
3922 */
3923 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3924 #endif
3925
3926 /* we don't have a kernel VA. We need one for the bcopy. */
3927 } else {
3928 #if defined(__amd64)
3929 /*
3930 * for the 64-bit kernel, it's easy. We use seg kpm to
3931 * get a Kernel VA for the corresponding pfn.
3932 */
3933 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3934 #else
3935 /*
3936 * for the 32-bit kernel, this is a pain. First we'll
3937 * save away the page_t or user VA for this page. This
3938 * is needed in rootnex_dma_win() when we switch to a
3939 * new window which requires us to re-map the copy
3940 * buffer.
3941 */
3942 pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3943 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3944 dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3945 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3946 } else if (pplist != NULL) {
3947 dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3948 dma->dp_pgmap[pidx].pm_vaddr = NULL;
3949 } else {
3950 dma->dp_pgmap[pidx].pm_pp = NULL;
3951 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3952 (((uintptr_t)
3953 dmar_object->dmao_obj.virt_obj.v_addr +
3954 cur_offset) & MMU_PAGEMASK);
3955 }
3956
3957 /*
3958 * save away the page aligned virtual address which was
3959 * allocated from the kernel heap arena (taking into
3960 * account if we need more copy buffer than we alloced
3961 * and use multiple windows to handle this, i.e. &,%).
3962 * NOTE: there isn't and physical memory backing up this
3963 * virtual address space currently.
3964 */
3965 if ((*copybuf_used + MMU_PAGESIZE) <=
3966 dma->dp_copybuf_size) {
3967 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3968 (((uintptr_t)dma->dp_kva + *copybuf_used) &
3969 MMU_PAGEMASK);
3970 } else {
3971 if (copybuf_sz_power_2) {
3972 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3973 (((uintptr_t)dma->dp_kva +
3974 (*copybuf_used &
3975 (dma->dp_copybuf_size - 1))) &
3976 MMU_PAGEMASK);
3977 } else {
3978 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3979 (((uintptr_t)dma->dp_kva +
3980 (*copybuf_used %
3981 dma->dp_copybuf_size)) &
3982 MMU_PAGEMASK);
3983 }
3984 }
3985
3986 /*
3987 * if we haven't used up the available copy buffer yet,
3988 * map the kva to the physical page.
3989 */
3990 if (!dma->dp_cb_remaping && ((*copybuf_used +
3991 MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3992 dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3993 if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3994 i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3995 dma->dp_pgmap[pidx].pm_kaddr);
3996 } else {
3997 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
3998 sinfo->si_asp,
3999 dma->dp_pgmap[pidx].pm_kaddr);
4000 }
4001
4002 /*
4003 * we've used up the available copy buffer, this page
4004 * will have to be mapped during rootnex_dma_win() when
4005 * we switch to a new window which requires a re-map
4006 * the copy buffer. (32-bit kernel only)
4007 */
4008 } else {
4009 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4010 }
4011 #endif
4012 /* go to the next page_t */
4013 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4014 *cur_pp = (*cur_pp)->p_next;
4015 }
4016 }
4017
4018 /* add to the copy buffer count */
4019 *copybuf_used += MMU_PAGESIZE;
4020
4021 /*
4022 * This cookie doesn't use the copy buffer. Walk through the pages this
4023 * cookie occupies to reflect this.
4024 */
4025 } else {
4026 /*
4027 * figure out how many pages the cookie occupies. We need to
4028 * use the original page offset of the buffer and the cookies
4029 * offset in the buffer to do this.
4030 */
4031 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4032 pcnt = mmu_btopr(cookie->dmac_size + poff);
4033
4034 while (pcnt > 0) {
4035 #if !defined(__amd64)
4036 /*
4037 * the 32-bit kernel doesn't have seg kpm, so we need
4038 * to map in the driver buffer (if it didn't come down
4039 * with a kernel VA) on the fly. Since this page doesn't
4040 * use the copy buffer, it's not, or will it ever, have
4041 * to be mapped in.
4042 */
4043 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4044 #endif
4045 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4046
4047 /*
4048 * we need to update pidx and cur_pp or we'll loose
4049 * track of where we are.
4050 */
4051 if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4052 *cur_pp = (*cur_pp)->p_next;
4053 }
4054 pidx++;
4055 pcnt--;
4056 }
4057 }
4058 }
4059
4060
4061 /*
4062 * rootnex_sgllen_window_boundary()
4063 * Called in the bind slow path when the next cookie causes us to exceed (in
4064 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4065 * length supported by the DMA H/W.
4066 */
4067 static int
4068 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4069 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4070 off_t cur_offset)
4071 {
4072 off_t new_offset;
4073 size_t trim_sz;
4074 off_t coffset;
4075
4076
4077 /*
4078 * if we know we'll never have to trim, it's pretty easy. Just move to
4079 * the next window and init it. We're done.
4080 */
4081 if (!dma->dp_trim_required) {
4082 (*windowp)++;
4083 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4084 (*windowp)->wd_cookie_cnt++;
4085 (*windowp)->wd_size = cookie->dmac_size;
4086 return (DDI_SUCCESS);
4087 }
4088
4089 /* figure out how much we need to trim from the window */
4090 ASSERT(attr->dma_attr_granular != 0);
4091 if (dma->dp_granularity_power_2) {
4092 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4093 } else {
4094 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4095 }
4096
4097 /* The window's a whole multiple of granularity. We're done */
4098 if (trim_sz == 0) {
4099 (*windowp)++;
4100 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4101 (*windowp)->wd_cookie_cnt++;
4102 (*windowp)->wd_size = cookie->dmac_size;
4103 return (DDI_SUCCESS);
4104 }
4105
4106 /*
4107 * The window's not a whole multiple of granularity, since we know this
4108 * is due to the sgllen, we need to go back to the last cookie and trim
4109 * that one, add the left over part of the old cookie into the new
4110 * window, and then add in the new cookie into the new window.
4111 */
4112
4113 /*
4114 * make sure the driver isn't making us do something bad... Trimming and
4115 * sgllen == 1 don't go together.
4116 */
4117 if (attr->dma_attr_sgllen == 1) {
4118 return (DDI_DMA_NOMAPPING);
4119 }
4120
4121 /*
4122 * first, setup the current window to account for the trim. Need to go
4123 * back to the last cookie for this.
4124 */
4125 cookie--;
4126 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4127 (*windowp)->wd_trim.tr_last_cookie = cookie;
4128 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4129 ASSERT(cookie->dmac_size > trim_sz);
4130 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4131 (*windowp)->wd_size -= trim_sz;
4132
4133 /* save the buffer offsets for the next window */
4134 coffset = cookie->dmac_size - trim_sz;
4135 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4136
4137 /*
4138 * set this now in case this is the first window. all other cases are
4139 * set in dma_win()
4140 */
4141 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4142
4143 /*
4144 * initialize the next window using what's left over in the previous
4145 * cookie.
4146 */
4147 (*windowp)++;
4148 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4149 (*windowp)->wd_cookie_cnt++;
4150 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4151 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4152 (*windowp)->wd_trim.tr_first_size = trim_sz;
4153 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4154 (*windowp)->wd_dosync = B_TRUE;
4155 }
4156
4157 /*
4158 * now go back to the current cookie and add it to the new window. set
4159 * the new window size to the what was left over from the previous
4160 * cookie and what's in the current cookie.
4161 */
4162 cookie++;
4163 (*windowp)->wd_cookie_cnt++;
4164 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4165
4166 /*
4167 * trim plus the next cookie could put us over maxxfer (a cookie can be
4168 * a max size of maxxfer). Handle that case.
4169 */
4170 if ((*windowp)->wd_size > dma->dp_maxxfer) {
4171 /*
4172 * maxxfer is already a whole multiple of granularity, and this
4173 * trim will be <= the previous trim (since a cookie can't be
4174 * larger than maxxfer). Make things simple here.
4175 */
4176 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4177 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4178 (*windowp)->wd_trim.tr_last_cookie = cookie;
4179 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4180 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4181 (*windowp)->wd_size -= trim_sz;
4182 ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4183
4184 /* save the buffer offsets for the next window */
4185 coffset = cookie->dmac_size - trim_sz;
4186 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4187
4188 /* setup the next window */
4189 (*windowp)++;
4190 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4191 (*windowp)->wd_cookie_cnt++;
4192 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4193 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4194 coffset;
4195 (*windowp)->wd_trim.tr_first_size = trim_sz;
4196 }
4197
4198 return (DDI_SUCCESS);
4199 }
4200
4201
4202 /*
4203 * rootnex_copybuf_window_boundary()
4204 * Called in bind slowpath when we get to a window boundary because we used
4205 * up all the copy buffer that we have.
4206 */
4207 static int
4208 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4209 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4210 size_t *copybuf_used)
4211 {
4212 rootnex_sglinfo_t *sinfo;
4213 off_t new_offset;
4214 size_t trim_sz;
4215 paddr_t paddr;
4216 off_t coffset;
4217 uint_t pidx;
4218 off_t poff;
4219
4220
4221 sinfo = &dma->dp_sglinfo;
4222
4223 /*
4224 * the copy buffer should be a whole multiple of page size. We know that
4225 * this cookie is <= MMU_PAGESIZE.
4226 */
4227 ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4228
4229 /*
4230 * from now on, all new windows in this bind need to be re-mapped during
4231 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4232 * space...
4233 */
4234 #if !defined(__amd64)
4235 dma->dp_cb_remaping = B_TRUE;
4236 #endif
4237
4238 /* reset copybuf used */
4239 *copybuf_used = 0;
4240
4241 /*
4242 * if we don't have to trim (since granularity is set to 1), go to the
4243 * next window and add the current cookie to it. We know the current
4244 * cookie uses the copy buffer since we're in this code path.
4245 */
4246 if (!dma->dp_trim_required) {
4247 (*windowp)++;
4248 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4249
4250 /* Add this cookie to the new window */
4251 (*windowp)->wd_cookie_cnt++;
4252 (*windowp)->wd_size += cookie->dmac_size;
4253 *copybuf_used += MMU_PAGESIZE;
4254 return (DDI_SUCCESS);
4255 }
4256
4257 /*
4258 * *** may need to trim, figure it out.
4259 */
4260
4261 /* figure out how much we need to trim from the window */
4262 if (dma->dp_granularity_power_2) {
4263 trim_sz = (*windowp)->wd_size &
4264 (hp->dmai_attr.dma_attr_granular - 1);
4265 } else {
4266 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4267 }
4268
4269 /*
4270 * if the window's a whole multiple of granularity, go to the next
4271 * window, init it, then add in the current cookie. We know the current
4272 * cookie uses the copy buffer since we're in this code path.
4273 */
4274 if (trim_sz == 0) {
4275 (*windowp)++;
4276 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4277
4278 /* Add this cookie to the new window */
4279 (*windowp)->wd_cookie_cnt++;
4280 (*windowp)->wd_size += cookie->dmac_size;
4281 *copybuf_used += MMU_PAGESIZE;
4282 return (DDI_SUCCESS);
4283 }
4284
4285 /*
4286 * *** We figured it out, we definitly need to trim
4287 */
4288
4289 /*
4290 * make sure the driver isn't making us do something bad...
4291 * Trimming and sgllen == 1 don't go together.
4292 */
4293 if (hp->dmai_attr.dma_attr_sgllen == 1) {
4294 return (DDI_DMA_NOMAPPING);
4295 }
4296
4297 /*
4298 * first, setup the current window to account for the trim. Need to go
4299 * back to the last cookie for this. Some of the last cookie will be in
4300 * the current window, and some of the last cookie will be in the new
4301 * window. All of the current cookie will be in the new window.
4302 */
4303 cookie--;
4304 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4305 (*windowp)->wd_trim.tr_last_cookie = cookie;
4306 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4307 ASSERT(cookie->dmac_size > trim_sz);
4308 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4309 (*windowp)->wd_size -= trim_sz;
4310
4311 /*
4312 * we're trimming the last cookie (not the current cookie). So that
4313 * last cookie may have or may not have been using the copy buffer (
4314 * we know the cookie passed in uses the copy buffer since we're in
4315 * this code path).
4316 *
4317 * If the last cookie doesn't use the copy buffer, nothing special to
4318 * do. However, if it does uses the copy buffer, it will be both the
4319 * last page in the current window and the first page in the next
4320 * window. Since we are reusing the copy buffer (and KVA space on the
4321 * 32-bit kernel), this page will use the end of the copy buffer in the
4322 * current window, and the start of the copy buffer in the next window.
4323 * Track that info... The cookie physical address was already set to
4324 * the copy buffer physical address in setup_cookie..
4325 */
4326 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4327 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4328 (*windowp)->wd_size) >> MMU_PAGESHIFT;
4329 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4330 (*windowp)->wd_trim.tr_last_pidx = pidx;
4331 (*windowp)->wd_trim.tr_last_cbaddr =
4332 dma->dp_pgmap[pidx].pm_cbaddr;
4333 #if !defined(__amd64)
4334 (*windowp)->wd_trim.tr_last_kaddr =
4335 dma->dp_pgmap[pidx].pm_kaddr;
4336 #endif
4337 }
4338
4339 /* save the buffer offsets for the next window */
4340 coffset = cookie->dmac_size - trim_sz;
4341 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4342
4343 /*
4344 * set this now in case this is the first window. all other cases are
4345 * set in dma_win()
4346 */
4347 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4348
4349 /*
4350 * initialize the next window using what's left over in the previous
4351 * cookie.
4352 */
4353 (*windowp)++;
4354 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4355 (*windowp)->wd_cookie_cnt++;
4356 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4357 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4358 (*windowp)->wd_trim.tr_first_size = trim_sz;
4359
4360 /*
4361 * again, we're tracking if the last cookie uses the copy buffer.
4362 * read the comment above for more info on why we need to track
4363 * additional state.
4364 *
4365 * For the first cookie in the new window, we need reset the physical
4366 * address to DMA into to the start of the copy buffer plus any
4367 * initial page offset which may be present.
4368 */
4369 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4370 (*windowp)->wd_dosync = B_TRUE;
4371 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4372 (*windowp)->wd_trim.tr_first_pidx = pidx;
4373 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4374 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4375
4376 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4377 poff;
4378 (*windowp)->wd_trim.tr_first_paddr =
4379 ROOTNEX_PADDR_TO_RBASE(paddr);
4380
4381 #if !defined(__amd64)
4382 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4383 #endif
4384 /* account for the cookie copybuf usage in the new window */
4385 *copybuf_used += MMU_PAGESIZE;
4386
4387 /*
4388 * every piece of code has to have a hack, and here is this
4389 * ones :-)
4390 *
4391 * There is a complex interaction between setup_cookie and the
4392 * copybuf window boundary. The complexity had to be in either
4393 * the maxxfer window, or the copybuf window, and I chose the
4394 * copybuf code.
4395 *
4396 * So in this code path, we have taken the last cookie,
4397 * virtually broken it in half due to the trim, and it happens
4398 * to use the copybuf which further complicates life. At the
4399 * same time, we have already setup the current cookie, which
4400 * is now wrong. More background info: the current cookie uses
4401 * the copybuf, so it is only a page long max. So we need to
4402 * fix the current cookies copy buffer address, physical
4403 * address, and kva for the 32-bit kernel. We due this by
4404 * bumping them by page size (of course, we can't due this on
4405 * the physical address since the copy buffer may not be
4406 * physically contiguous).
4407 */
4408 cookie++;
4409 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4410 poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4411
4412 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4413 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4414 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4415
4416 #if !defined(__amd64)
4417 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4418 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4419 #endif
4420 } else {
4421 /* go back to the current cookie */
4422 cookie++;
4423 }
4424
4425 /*
4426 * add the current cookie to the new window. set the new window size to
4427 * the what was left over from the previous cookie and what's in the
4428 * current cookie.
4429 */
4430 (*windowp)->wd_cookie_cnt++;
4431 (*windowp)->wd_size = trim_sz + cookie->dmac_size;
4432 ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4433
4434 /*
4435 * we know that the cookie passed in always uses the copy buffer. We
4436 * wouldn't be here if it didn't.
4437 */
4438 *copybuf_used += MMU_PAGESIZE;
4439
4440 return (DDI_SUCCESS);
4441 }
4442
4443
4444 /*
4445 * rootnex_maxxfer_window_boundary()
4446 * Called in bind slowpath when we get to a window boundary because we will
4447 * go over maxxfer.
4448 */
4449 static int
4450 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4451 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4452 {
4453 size_t dmac_size;
4454 off_t new_offset;
4455 size_t trim_sz;
4456 off_t coffset;
4457
4458
4459 /*
4460 * calculate how much we have to trim off of the current cookie to equal
4461 * maxxfer. We don't have to account for granularity here since our
4462 * maxxfer already takes that into account.
4463 */
4464 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4465 ASSERT(trim_sz <= cookie->dmac_size);
4466 ASSERT(trim_sz <= dma->dp_maxxfer);
4467
4468 /* save cookie size since we need it later and we might change it */
4469 dmac_size = cookie->dmac_size;
4470
4471 /*
4472 * if we're not trimming the entire cookie, setup the current window to
4473 * account for the trim.
4474 */
4475 if (trim_sz < cookie->dmac_size) {
4476 (*windowp)->wd_cookie_cnt++;
4477 (*windowp)->wd_trim.tr_trim_last = B_TRUE;
4478 (*windowp)->wd_trim.tr_last_cookie = cookie;
4479 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4480 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4481 (*windowp)->wd_size = dma->dp_maxxfer;
4482
4483 /*
4484 * set the adjusted cookie size now in case this is the first
4485 * window. All other windows are taken care of in get win
4486 */
4487 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4488 }
4489
4490 /*
4491 * coffset is the current offset within the cookie, new_offset is the
4492 * current offset with the entire buffer.
4493 */
4494 coffset = dmac_size - trim_sz;
4495 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4496
4497 /* initialize the next window */
4498 (*windowp)++;
4499 rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4500 (*windowp)->wd_cookie_cnt++;
4501 (*windowp)->wd_size = trim_sz;
4502 if (trim_sz < dmac_size) {
4503 (*windowp)->wd_trim.tr_trim_first = B_TRUE;
4504 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4505 coffset;
4506 (*windowp)->wd_trim.tr_first_size = trim_sz;
4507 }
4508
4509 return (DDI_SUCCESS);
4510 }
4511
4512
4513 /*ARGSUSED*/
4514 static int
4515 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4516 off_t off, size_t len, uint_t cache_flags)
4517 {
4518 rootnex_sglinfo_t *sinfo;
4519 rootnex_pgmap_t *cbpage;
4520 rootnex_window_t *win;
4521 ddi_dma_impl_t *hp;
4522 rootnex_dma_t *dma;
4523 caddr_t fromaddr;
4524 caddr_t toaddr;
4525 uint_t psize;
4526 off_t offset;
4527 uint_t pidx;
4528 size_t size;
4529 off_t poff;
4530 int e;
4531
4532
4533 hp = (ddi_dma_impl_t *)handle;
4534 dma = (rootnex_dma_t *)hp->dmai_private;
4535 sinfo = &dma->dp_sglinfo;
4536
4537 /*
4538 * if we don't have any windows, we don't need to sync. A copybuf
4539 * will cause us to have at least one window.
4540 */
4541 if (dma->dp_window == NULL) {
4542 return (DDI_SUCCESS);
4543 }
4544
4545 /* This window may not need to be sync'd */
4546 win = &dma->dp_window[dma->dp_current_win];
4547 if (!win->wd_dosync) {
4548 return (DDI_SUCCESS);
4549 }
4550
4551 /* handle off and len special cases */
4552 if ((off == 0) || (rootnex_sync_ignore_params)) {
4553 offset = win->wd_offset;
4554 } else {
4555 offset = off;
4556 }
4557 if ((len == 0) || (rootnex_sync_ignore_params)) {
4558 size = win->wd_size;
4559 } else {
4560 size = len;
4561 }
4562
4563 /* check the sync args to make sure they make a little sense */
4564 if (rootnex_sync_check_parms) {
4565 e = rootnex_valid_sync_parms(hp, win, offset, size,
4566 cache_flags);
4567 if (e != DDI_SUCCESS) {
4568 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4569 return (DDI_FAILURE);
4570 }
4571 }
4572
4573 /*
4574 * special case the first page to handle the offset into the page. The
4575 * offset to the current page for our buffer is the offset into the
4576 * first page of the buffer plus our current offset into the buffer
4577 * itself, masked of course.
4578 */
4579 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4580 psize = MIN((MMU_PAGESIZE - poff), size);
4581
4582 /* go through all the pages that we want to sync */
4583 while (size > 0) {
4584 /*
4585 * Calculate the page index relative to the start of the buffer.
4586 * The index to the current page for our buffer is the offset
4587 * into the first page of the buffer plus our current offset
4588 * into the buffer itself, shifted of course...
4589 */
4590 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4591 ASSERT(pidx < sinfo->si_max_pages);
4592
4593 /*
4594 * if this page uses the copy buffer, we need to sync it,
4595 * otherwise, go on to the next page.
4596 */
4597 cbpage = &dma->dp_pgmap[pidx];
4598 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4599 (cbpage->pm_uses_copybuf == B_FALSE));
4600 if (cbpage->pm_uses_copybuf) {
4601 /* cbaddr and kaddr should be page aligned */
4602 ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4603 MMU_PAGEOFFSET) == 0);
4604 ASSERT(((uintptr_t)cbpage->pm_kaddr &
4605 MMU_PAGEOFFSET) == 0);
4606
4607 /*
4608 * if we're copying for the device, we are going to
4609 * copy from the drivers buffer and to the rootnex
4610 * allocated copy buffer.
4611 */
4612 if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4613 fromaddr = cbpage->pm_kaddr + poff;
4614 toaddr = cbpage->pm_cbaddr + poff;
4615 ROOTNEX_DPROBE2(rootnex__sync__dev,
4616 dev_info_t *, dma->dp_dip, size_t, psize);
4617
4618 /*
4619 * if we're copying for the cpu/kernel, we are going to
4620 * copy from the rootnex allocated copy buffer to the
4621 * drivers buffer.
4622 */
4623 } else {
4624 fromaddr = cbpage->pm_cbaddr + poff;
4625 toaddr = cbpage->pm_kaddr + poff;
4626 ROOTNEX_DPROBE2(rootnex__sync__cpu,
4627 dev_info_t *, dma->dp_dip, size_t, psize);
4628 }
4629
4630 bcopy(fromaddr, toaddr, psize);
4631 }
4632
4633 /*
4634 * decrement size until we're done, update our offset into the
4635 * buffer, and get the next page size.
4636 */
4637 size -= psize;
4638 offset += psize;
4639 psize = MIN(MMU_PAGESIZE, size);
4640
4641 /* page offset is zero for the rest of this loop */
4642 poff = 0;
4643 }
4644
4645 return (DDI_SUCCESS);
4646 }
4647
4648 /*
4649 * rootnex_dma_sync()
4650 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4651 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4652 * is set, ddi_dma_sync() returns immediately passing back success.
4653 */
4654 /*ARGSUSED*/
4655 static int
4656 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4657 off_t off, size_t len, uint_t cache_flags)
4658 {
4659 #if defined(__amd64) && !defined(__xpv)
4660 if (IOMMU_USED(rdip)) {
4661 return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4662 cache_flags));
4663 }
4664 #endif
4665 return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4666 cache_flags));
4667 }
4668
4669 /*
4670 * rootnex_valid_sync_parms()
4671 * checks the parameters passed to sync to verify they are correct.
4672 */
4673 static int
4674 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4675 off_t offset, size_t size, uint_t cache_flags)
4676 {
4677 off_t woffset;
4678
4679
4680 /*
4681 * the first part of the test to make sure the offset passed in is
4682 * within the window.
4683 */
4684 if (offset < win->wd_offset) {
4685 return (DDI_FAILURE);
4686 }
4687
4688 /*
4689 * second and last part of the test to make sure the offset and length
4690 * passed in is within the window.
4691 */
4692 woffset = offset - win->wd_offset;
4693 if ((woffset + size) > win->wd_size) {
4694 return (DDI_FAILURE);
4695 }
4696
4697 /*
4698 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4699 * be set too.
4700 */
4701 if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4702 (hp->dmai_rflags & DDI_DMA_WRITE)) {
4703 return (DDI_SUCCESS);
4704 }
4705
4706 /*
4707 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4708 * should be set. Also DDI_DMA_READ should be set in the flags.
4709 */
4710 if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4711 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4712 (hp->dmai_rflags & DDI_DMA_READ)) {
4713 return (DDI_SUCCESS);
4714 }
4715
4716 return (DDI_FAILURE);
4717 }
4718
4719
4720 /*ARGSUSED*/
4721 static int
4722 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4723 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4724 uint_t *ccountp)
4725 {
4726 rootnex_window_t *window;
4727 rootnex_trim_t *trim;
4728 ddi_dma_impl_t *hp;
4729 rootnex_dma_t *dma;
4730 ddi_dma_obj_t *dmao;
4731 #if !defined(__amd64)
4732 rootnex_sglinfo_t *sinfo;
4733 rootnex_pgmap_t *pmap;
4734 uint_t pidx;
4735 uint_t pcnt;
4736 off_t poff;
4737 int i;
4738 #endif
4739
4740
4741 hp = (ddi_dma_impl_t *)handle;
4742 dma = (rootnex_dma_t *)hp->dmai_private;
4743 #if !defined(__amd64)
4744 sinfo = &dma->dp_sglinfo;
4745 #endif
4746
4747 /* If we try and get a window which doesn't exist, return failure */
4748 if (win >= hp->dmai_nwin) {
4749 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4750 return (DDI_FAILURE);
4751 }
4752
4753 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4754
4755 /*
4756 * if we don't have any windows, and they're asking for the first
4757 * window, setup the cookie pointer to the first cookie in the bind.
4758 * setup our return values, then increment the cookie since we return
4759 * the first cookie on the stack.
4760 */
4761 if (dma->dp_window == NULL) {
4762 if (win != 0) {
4763 ROOTNEX_DPROF_INC(
4764 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4765 return (DDI_FAILURE);
4766 }
4767 hp->dmai_cookie = dma->dp_cookies;
4768 *offp = 0;
4769 *lenp = dmao->dmao_size;
4770 *ccountp = dma->dp_sglinfo.si_sgl_size;
4771 *cookiep = hp->dmai_cookie[0];
4772 hp->dmai_cookie++;
4773 return (DDI_SUCCESS);
4774 }
4775
4776 /* sync the old window before moving on to the new one */
4777 window = &dma->dp_window[dma->dp_current_win];
4778 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4779 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4780 DDI_DMA_SYNC_FORCPU);
4781 }
4782
4783 #if !defined(__amd64)
4784 /*
4785 * before we move to the next window, if we need to re-map, unmap all
4786 * the pages in this window.
4787 */
4788 if (dma->dp_cb_remaping) {
4789 /*
4790 * If we switch to this window again, we'll need to map in
4791 * on the fly next time.
4792 */
4793 window->wd_remap_copybuf = B_TRUE;
4794
4795 /*
4796 * calculate the page index into the buffer where this window
4797 * starts, and the number of pages this window takes up.
4798 */
4799 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4800 MMU_PAGESHIFT;
4801 poff = (sinfo->si_buf_offset + window->wd_offset) &
4802 MMU_PAGEOFFSET;
4803 pcnt = mmu_btopr(window->wd_size + poff);
4804 ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4805
4806 /* unmap pages which are currently mapped in this window */
4807 for (i = 0; i < pcnt; i++) {
4808 if (dma->dp_pgmap[pidx].pm_mapped) {
4809 hat_unload(kas.a_hat,
4810 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4811 HAT_UNLOAD);
4812 dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4813 }
4814 pidx++;
4815 }
4816 }
4817 #endif
4818
4819 /*
4820 * Move to the new window.
4821 * NOTE: current_win must be set for sync to work right
4822 */
4823 dma->dp_current_win = win;
4824 window = &dma->dp_window[win];
4825
4826 /* if needed, adjust the first and/or last cookies for trim */
4827 trim = &window->wd_trim;
4828 if (trim->tr_trim_first) {
4829 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4830 window->wd_first_cookie->dmac_size = trim->tr_first_size;
4831 #if !defined(__amd64)
4832 window->wd_first_cookie->dmac_type =
4833 (window->wd_first_cookie->dmac_type &
4834 ROOTNEX_USES_COPYBUF) + window->wd_offset;
4835 #endif
4836 if (trim->tr_first_copybuf_win) {
4837 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4838 trim->tr_first_cbaddr;
4839 #if !defined(__amd64)
4840 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4841 trim->tr_first_kaddr;
4842 #endif
4843 }
4844 }
4845 if (trim->tr_trim_last) {
4846 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4847 trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4848 if (trim->tr_last_copybuf_win) {
4849 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4850 trim->tr_last_cbaddr;
4851 #if !defined(__amd64)
4852 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4853 trim->tr_last_kaddr;
4854 #endif
4855 }
4856 }
4857
4858 /*
4859 * setup the cookie pointer to the first cookie in the window. setup
4860 * our return values, then increment the cookie since we return the
4861 * first cookie on the stack.
4862 */
4863 hp->dmai_cookie = window->wd_first_cookie;
4864 *offp = window->wd_offset;
4865 *lenp = window->wd_size;
4866 *ccountp = window->wd_cookie_cnt;
4867 *cookiep = hp->dmai_cookie[0];
4868 hp->dmai_cookie++;
4869
4870 #if !defined(__amd64)
4871 /* re-map copybuf if required for this window */
4872 if (dma->dp_cb_remaping) {
4873 /*
4874 * calculate the page index into the buffer where this
4875 * window starts.
4876 */
4877 pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4878 MMU_PAGESHIFT;
4879 ASSERT(pidx < sinfo->si_max_pages);
4880
4881 /*
4882 * the first page can get unmapped if it's shared with the
4883 * previous window. Even if the rest of this window is already
4884 * mapped in, we need to still check this one.
4885 */
4886 pmap = &dma->dp_pgmap[pidx];
4887 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4888 if (pmap->pm_pp != NULL) {
4889 pmap->pm_mapped = B_TRUE;
4890 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4891 } else if (pmap->pm_vaddr != NULL) {
4892 pmap->pm_mapped = B_TRUE;
4893 i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4894 pmap->pm_kaddr);
4895 }
4896 }
4897 pidx++;
4898
4899 /* map in the rest of the pages if required */
4900 if (window->wd_remap_copybuf) {
4901 window->wd_remap_copybuf = B_FALSE;
4902
4903 /* figure out many pages this window takes up */
4904 poff = (sinfo->si_buf_offset + window->wd_offset) &
4905 MMU_PAGEOFFSET;
4906 pcnt = mmu_btopr(window->wd_size + poff);
4907 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4908
4909 /* map pages which require it */
4910 for (i = 1; i < pcnt; i++) {
4911 pmap = &dma->dp_pgmap[pidx];
4912 if (pmap->pm_uses_copybuf) {
4913 ASSERT(pmap->pm_mapped == B_FALSE);
4914 if (pmap->pm_pp != NULL) {
4915 pmap->pm_mapped = B_TRUE;
4916 i86_pp_map(pmap->pm_pp,
4917 pmap->pm_kaddr);
4918 } else if (pmap->pm_vaddr != NULL) {
4919 pmap->pm_mapped = B_TRUE;
4920 i86_va_map(pmap->pm_vaddr,
4921 sinfo->si_asp,
4922 pmap->pm_kaddr);
4923 }
4924 }
4925 pidx++;
4926 }
4927 }
4928 }
4929 #endif
4930
4931 /* if the new window uses the copy buffer, sync it for the device */
4932 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4933 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4934 DDI_DMA_SYNC_FORDEV);
4935 }
4936
4937 return (DDI_SUCCESS);
4938 }
4939
4940 /*
4941 * rootnex_dma_win()
4942 * called from ddi_dma_getwin()
4943 */
4944 /*ARGSUSED*/
4945 static int
4946 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4947 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4948 uint_t *ccountp)
4949 {
4950 #if defined(__amd64) && !defined(__xpv)
4951 if (IOMMU_USED(rdip)) {
4952 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4953 cookiep, ccountp));
4954 }
4955 #endif
4956
4957 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4958 cookiep, ccountp));
4959 }
4960
4961 #if defined(__amd64) && !defined(__xpv)
4962 /*ARGSUSED*/
4963 static int
4964 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4965 ddi_dma_handle_t handle, void *v)
4966 {
4967 ddi_dma_impl_t *hp;
4968 rootnex_dma_t *dma;
4969
4970 hp = (ddi_dma_impl_t *)handle;
4971 dma = (rootnex_dma_t *)hp->dmai_private;
4972 dma->dp_iommu_private = v;
4973
4974 return (DDI_SUCCESS);
4975 }
4976
4977 /*ARGSUSED*/
4978 static void *
4979 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4980 ddi_dma_handle_t handle)
4981 {
4982 ddi_dma_impl_t *hp;
4983 rootnex_dma_t *dma;
4984
4985 hp = (ddi_dma_impl_t *)handle;
4986 dma = (rootnex_dma_t *)hp->dmai_private;
4987
4988 return (dma->dp_iommu_private);
4989 }
4990 #endif
4991
4992 /*
4993 * ************************
4994 * obsoleted dma routines
4995 * ************************
4996 */
4997
4998 /*
4999 * rootnex_dma_mctl()
5000 *
5001 * We don't support this legacy interface any more on x86.
5002 */
5003 /* ARGSUSED */
5004 static int
5005 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5006 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5007 uint_t cache_flags)
5008 {
5009 /*
5010 * The only thing dma_mctl is usef for anymore is legacy SPARC
5011 * dvma and sbus-specific routines.
5012 */
5013 return (DDI_FAILURE);
5014 }
5015
5016 /*
5017 * *********
5018 * FMA Code
5019 * *********
5020 */
5021
5022 /*
5023 * rootnex_fm_init()
5024 * FMA init busop
5025 */
5026 /* ARGSUSED */
5027 static int
5028 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5029 ddi_iblock_cookie_t *ibc)
5030 {
5031 *ibc = rootnex_state->r_err_ibc;
5032
5033 return (ddi_system_fmcap);
5034 }
5035
5036 /*
5037 * rootnex_dma_check()
5038 * Function called after a dma fault occurred to find out whether the
5039 * fault address is associated with a driver that is able to handle faults
5040 * and recover from faults.
5041 */
5042 /* ARGSUSED */
5043 static int
5044 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5045 const void *not_used)
5046 {
5047 rootnex_window_t *window;
5048 uint64_t start_addr;
5049 uint64_t fault_addr;
5050 ddi_dma_impl_t *hp;
5051 rootnex_dma_t *dma;
5052 uint64_t end_addr;
5053 size_t csize;
5054 int i;
5055 int j;
5056
5057
5058 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5059 hp = (ddi_dma_impl_t *)handle;
5060 ASSERT(hp);
5061
5062 dma = (rootnex_dma_t *)hp->dmai_private;
5063
5064 /* Get the address that we need to search for */
5065 fault_addr = *(uint64_t *)addr;
5066
5067 /*
5068 * if we don't have any windows, we can just walk through all the
5069 * cookies.
5070 */
5071 if (dma->dp_window == NULL) {
5072 /* for each cookie */
5073 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5074 /*
5075 * if the faulted address is within the physical address
5076 * range of the cookie, return DDI_FM_NONFATAL.
5077 */
5078 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5079 (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5080 dma->dp_cookies[i].dmac_size))) {
5081 return (DDI_FM_NONFATAL);
5082 }
5083 }
5084
5085 /* fault_addr not within this DMA handle */
5086 return (DDI_FM_UNKNOWN);
5087 }
5088
5089 /* we have mutiple windows, walk through each window */
5090 for (i = 0; i < hp->dmai_nwin; i++) {
5091 window = &dma->dp_window[i];
5092
5093 /* Go through all the cookies in the window */
5094 for (j = 0; j < window->wd_cookie_cnt; j++) {
5095
5096 start_addr = window->wd_first_cookie[j].dmac_laddress;
5097 csize = window->wd_first_cookie[j].dmac_size;
5098
5099 /*
5100 * if we are trimming the first cookie in the window,
5101 * and this is the first cookie, adjust the start
5102 * address and size of the cookie to account for the
5103 * trim.
5104 */
5105 if (window->wd_trim.tr_trim_first && (j == 0)) {
5106 start_addr = window->wd_trim.tr_first_paddr;
5107 csize = window->wd_trim.tr_first_size;
5108 }
5109
5110 /*
5111 * if we are trimming the last cookie in the window,
5112 * and this is the last cookie, adjust the start
5113 * address and size of the cookie to account for the
5114 * trim.
5115 */
5116 if (window->wd_trim.tr_trim_last &&
5117 (j == (window->wd_cookie_cnt - 1))) {
5118 start_addr = window->wd_trim.tr_last_paddr;
5119 csize = window->wd_trim.tr_last_size;
5120 }
5121
5122 end_addr = start_addr + csize;
5123
5124 /*
5125 * if the faulted address is within the physical
5126 * address of the cookie, return DDI_FM_NONFATAL.
5127 */
5128 if ((fault_addr >= start_addr) &&
5129 (fault_addr <= end_addr)) {
5130 return (DDI_FM_NONFATAL);
5131 }
5132 }
5133 }
5134
5135 /* fault_addr not within this DMA handle */
5136 return (DDI_FM_UNKNOWN);
5137 }
5138
5139 /*ARGSUSED*/
5140 static int
5141 rootnex_quiesce(dev_info_t *dip)
5142 {
5143 #if defined(__amd64) && !defined(__xpv)
5144 return (immu_quiesce());
5145 #else
5146 return (DDI_SUCCESS);
5147 #endif
5148 }
5149
5150 #if defined(__xpv)
5151 void
5152 immu_init(void)
5153 {
5154 ;
5155 }
5156
5157 void
5158 immu_startup(void)
5159 {
5160 ;
5161 }
5162 /*ARGSUSED*/
5163 void
5164 immu_physmem_update(uint64_t addr, uint64_t size)
5165 {
5166 ;
5167 }
5168 #endif