1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright (c) 2018, Joyent, Inc.
14 */
15
16 /*
17 * illumos USB framework endpoints and functions for xHCI.
18 *
19 * Please see the big theory statement in xhci.c for more information.
20 */
21
22 #include <sys/usb/hcd/xhci/xhci.h>
23 #include <sys/sysmacros.h>
24 #include <sys/strsun.h>
25 #include <sys/strsubr.h>
26
27 static xhci_t *
28 xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud)
29 {
30 dev_info_t *dip = ud->usb_root_hub_dip;
31 xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state,
32 ddi_get_instance(dip));
33 VERIFY(xhcip != NULL);
34 return (xhcip);
35 }
36
37 static xhci_t *
38 xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph)
39 {
40 return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device));
41 }
42
43 /*
44 * While the xHCI hardware is capable of supporting power management, we don't
45 * in the driver right now. Note, USBA doesn't seem to end up calling this entry
46 * point.
47 */
48 /* ARGSUSED */
49 static int
50 xhci_hcdi_pm_support(dev_info_t *dip)
51 {
52 return (USB_FAILURE);
53 }
54
55 static int
56 xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
57 {
58 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
59 xhci_pipe_t *pipe;
60 xhci_endpoint_t *xep;
61 xhci_device_t *xd;
62 int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP;
63 int ret;
64 uint_t epid;
65
66 mutex_enter(&xhcip->xhci_lock);
67 if (xhcip->xhci_state & XHCI_S_ERROR) {
68 mutex_exit(&xhcip->xhci_lock);
69 return (USB_HC_HARDWARE_ERROR);
70 }
71 mutex_exit(&xhcip->xhci_lock);
72
73 /*
74 * If we're here, something must be trying to open an already-opened
75 * pipe which is bad news.
76 */
77 if (ph->p_hcd_private != NULL) {
78 return (USB_FAILURE);
79 }
80
81 pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags);
82 if (pipe == NULL) {
83 return (USB_NO_RESOURCES);
84 }
85 pipe->xp_opentime = gethrtime();
86 pipe->xp_pipe = ph;
87
88 /*
89 * If this is the root hub, there's nothing special to do on open. Just
90 * go ahead and allow it to be opened. All we have to do is add this to
91 * the list of our tracking structures for open pipes.
92 */
93 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
94 xep = NULL;
95 goto add;
96 }
97
98 /*
99 * Now that we're here, we're being asked to open up an endpoint of some
100 * kind. Because we've already handled the case of the root hub,
101 * everything should have a device.
102 */
103 epid = xhci_endpoint_pipe_to_epid(ph);
104 xd = usba_hcdi_get_device_private(ph->p_usba_device);
105 if (xd == NULL) {
106 xhci_error(xhcip, "!encountered endpoint (%d) without device "
107 "during pipe open", epid);
108 kmem_free(pipe, sizeof (xhci_pipe_t));
109 return (USB_FAILURE);
110 }
111
112 /*
113 * See if this endpoint exists or not, in general endpoints should not
114 * exist except for the default control endpoint, which we don't tear
115 * down until the device itself is cleaned up. Otherwise, a given pipe
116 * can only be open once.
117 */
118 mutex_enter(&xhcip->xhci_lock);
119 if (epid == XHCI_DEFAULT_ENDPOINT) {
120 xep = xd->xd_endpoints[epid];
121 VERIFY(xep != NULL);
122 VERIFY(xep->xep_pipe == NULL);
123 xep->xep_pipe = ph;
124 mutex_exit(&xhcip->xhci_lock);
125 ret = xhci_endpoint_update_default(xhcip, xd, xep);
126 if (ret != USB_SUCCESS) {
127 kmem_free(pipe, sizeof (xhci_pipe_t));
128 return (ret);
129 }
130 goto add;
131 }
132
133 if (xd->xd_endpoints[epid] != NULL) {
134 mutex_exit(&xhcip->xhci_lock);
135 kmem_free(pipe, sizeof (xhci_pipe_t));
136 xhci_log(xhcip, "!asked to open endpoint %d on slot %d and "
137 "port %d, but endpoint already exists", epid, xd->xd_slot,
138 xd->xd_port);
139 return (USB_FAILURE);
140 }
141
142 /*
143 * If we're opening an endpoint other than the default control endpoint,
144 * then the device should have had a USB address assigned by the
145 * controller. Sanity check that before continuing.
146 */
147 if (epid != XHCI_DEFAULT_ENDPOINT) {
148 VERIFY(xd->xd_addressed == B_TRUE);
149 }
150
151 /*
152 * Okay, at this point we need to go create and set up an endpoint.
153 * Once we're done, we'll try to install it and make sure that it
154 * doesn't conflict with something else going on.
155 */
156 ret = xhci_endpoint_init(xhcip, xd, ph);
157 if (ret != 0) {
158 mutex_exit(&xhcip->xhci_lock);
159 kmem_free(pipe, sizeof (xhci_pipe_t));
160 if (ret == EIO) {
161 xhci_error(xhcip, "failed to initialize endpoint %d "
162 "on device slot %d and port %d: encountered fatal "
163 "FM error, resetting device", epid, xd->xd_slot,
164 xd->xd_port);
165 xhci_fm_runtime_reset(xhcip);
166 }
167 return (USB_HC_HARDWARE_ERROR);
168 }
169 xep = xd->xd_endpoints[epid];
170
171 mutex_enter(&xd->xd_imtx);
172 mutex_exit(&xhcip->xhci_lock);
173
174 /*
175 * Update the slot and input context for this endpoint. We make sure to
176 * always set the slot as having changed in the context field as the
177 * specification suggests we should and some hardware requires it.
178 */
179 xd->xd_input->xic_drop_flags = LE_32(0);
180 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
181 XHCI_INCTX_MASK_DCI(epid + 1));
182
183 if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) {
184 uint32_t info;
185
186 info = xd->xd_slotin->xsc_info;
187 info &= ~XHCI_SCTX_DCI_MASK;
188 info |= XHCI_SCTX_SET_DCI(epid + 1);
189 xd->xd_slotin->xsc_info = info;
190 }
191
192 XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
193 if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
194 mutex_exit(&xd->xd_imtx);
195 xhci_endpoint_fini(xd, epid);
196 kmem_free(pipe, sizeof (xhci_pipe_t));
197 xhci_error(xhcip, "failed to open pipe on endpoint %d of "
198 "device with slot %d and port %d: encountered fatal FM "
199 "error syncing device input context, resetting device",
200 epid, xd->xd_slot, xd->xd_port);
201 xhci_fm_runtime_reset(xhcip);
202 return (USB_HC_HARDWARE_ERROR);
203 }
204
205 if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) {
206 mutex_exit(&xd->xd_imtx);
207 xhci_endpoint_fini(xd, epid);
208 kmem_free(pipe, sizeof (xhci_pipe_t));
209 return (ret);
210 }
211
212 mutex_exit(&xd->xd_imtx);
213 add:
214 pipe->xp_ep = xep;
215 ph->p_hcd_private = (usb_opaque_t)pipe;
216 mutex_enter(&xhcip->xhci_lock);
217 list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe);
218 mutex_exit(&xhcip->xhci_lock);
219
220 return (USB_SUCCESS);
221 }
222
223 static void
224 xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp)
225 {
226 int i;
227 xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
228
229 if (xpp->xpp_tsize == 0)
230 return;
231
232 for (i = 0; i < xpp->xpp_ntransfers; i++) {
233 if (xpp->xpp_transfers[i] == NULL)
234 continue;
235 xhci_transfer_free(xhcip, xpp->xpp_transfers[i]);
236 xpp->xpp_transfers[i] = NULL;
237 }
238
239 xpp->xpp_ntransfers = 0;
240 xpp->xpp_tsize = 0;
241 }
242
243 /*
244 * Iterate over all transfers and free everything on the pipe. Once done, update
245 * the ring to basically 'consume' everything. For periodic IN endpoints, we
246 * need to handle this somewhat differently and actually close the original
247 * request and not deallocate the related pieces as those exist for the lifetime
248 * of the endpoint and are constantly reused.
249 */
250 static void
251 xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code)
252 {
253 xhci_transfer_t *xt;
254
255 ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
256
257 while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) {
258 if (xhci_endpoint_is_periodic_in(xep) == B_FALSE) {
259 usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req,
260 USB_CR_FLUSHED);
261 xhci_transfer_free(xhcip, xt);
262 }
263 }
264
265 if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) {
266 xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
267 xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
268
269 if (xpp->xpp_usb_req != NULL) {
270 usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req,
271 intr_code);
272 xpp->xpp_usb_req = NULL;
273 }
274 }
275 }
276
277 /*
278 * We've been asked to terminate some set of regular I/O on an interrupt pipe.
279 * If this is for the root device, e.g. the xhci driver itself, then we remove
280 * our interrupt callback. Otherwise we stop the device for interrupt polling as
281 * follows:
282 *
283 * 1. Issue a stop endpoint command
284 * 2. Check to make sure that the endpoint stopped and reset it if needed.
285 * 3. Any thing that gets resolved can callback in the interim.
286 * 4. Ensure that nothing is scheduled on the ring
287 * 5. Skip the contents of the ring and set the TR dequeue pointer.
288 * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the
289 * callback in the process.
290 */
291 static int
292 xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close)
293 {
294 int ret;
295 uint_t epid;
296 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
297 xhci_device_t *xd;
298 xhci_endpoint_t *xep;
299 xhci_pipe_t *xp;
300 xhci_periodic_pipe_t *xpp;
301 usb_opaque_t urp;
302
303 mutex_enter(&xhcip->xhci_lock);
304 if (xhcip->xhci_state & XHCI_S_ERROR) {
305 mutex_exit(&xhcip->xhci_lock);
306 return (USB_HC_HARDWARE_ERROR);
307 }
308
309 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
310 xhci_root_hub_intr_root_disable(xhcip);
311 ret = USB_SUCCESS;
312 mutex_exit(&xhcip->xhci_lock);
313 return (ret);
314 }
315
316 xd = usba_hcdi_get_device_private(ph->p_usba_device);
317 epid = xhci_endpoint_pipe_to_epid(ph);
318 if (xd->xd_endpoints[epid] == NULL) {
319 mutex_exit(&xhcip->xhci_lock);
320 xhci_error(xhcip, "asked to stop intr polling on slot %d, "
321 "port %d, endpoint: %d, but no endpoint structure",
322 xd->xd_slot, xd->xd_port, epid);
323 return (USB_FAILURE);
324 }
325 xep = xd->xd_endpoints[epid];
326 xp = (xhci_pipe_t *)ph->p_hcd_private;
327 if (xp == NULL) {
328 mutex_exit(&xhcip->xhci_lock);
329 xhci_error(xhcip, "asked to do finish polling on slot %d, "
330 "port %d, endpoint: %d, but no pipe structure",
331 xd->xd_slot, xd->xd_port, epid);
332 return (USB_FAILURE);
333 }
334 xpp = &xp->xp_periodic;
335
336 /*
337 * Ensure that no other resets or time outs are going on right now.
338 */
339 while ((xep->xep_state & (XHCI_ENDPOINT_SERIALIZE)) != 0) {
340 cv_wait(&xep->xep_state_cv, &xhcip->xhci_lock);
341 }
342
343 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) {
344 mutex_exit(&xhcip->xhci_lock);
345 return (USB_SUCCESS);
346 }
347
348 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) {
349 mutex_exit(&xhcip->xhci_lock);
350 return (USB_FAILURE);
351 }
352
353 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING;
354 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
355 ret = xhci_endpoint_quiesce(xhcip, xd, xep);
356 if (ret != USB_SUCCESS) {
357 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
358 "port %d, endpoint: %d, failed with %d.",
359 xd->xd_slot, xd->xd_port, epid, ret);
360 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
361 cv_broadcast(&xep->xep_state_cv);
362 mutex_exit(&xhcip->xhci_lock);
363 return (ret);
364 }
365
366 /*
367 * Okay, we've stopped this ring time to wrap it all up. Remove all the
368 * transfers, note they aren't freed like a pipe reset.
369 */
370 while (list_is_empty(&xep->xep_transfers) == 0)
371 (void) list_remove_head(&xep->xep_transfers);
372 xhci_ring_skip(&xep->xep_ring);
373 mutex_exit(&xhcip->xhci_lock);
374
375 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
376 USB_SUCCESS) {
377 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
378 "port %d, endpoint: %d, failed with %d.",
379 xd->xd_slot, xd->xd_port, epid, ret);
380 mutex_enter(&xhcip->xhci_lock);
381 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
382 cv_broadcast(&xep->xep_state_cv);
383 mutex_exit(&xhcip->xhci_lock);
384 return (ret);
385 }
386
387 mutex_enter(&xhcip->xhci_lock);
388 urp = xpp->xpp_usb_req;
389 xpp->xpp_usb_req = NULL;
390 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE;
391 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
392 mutex_exit(&xhcip->xhci_lock);
393
394 /*
395 * It's possible that with a persistent pipe, we may not actually have
396 * anything left to call back on, because we already had.
397 */
398 if (urp != NULL) {
399 usba_hcdi_cb(ph, urp, is_close == B_TRUE ?
400 USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING);
401 }
402
403 /*
404 * Notify anything waiting for us that we're done quiescing this device.
405 */
406 mutex_enter(&xhcip->xhci_lock);
407 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
408 cv_broadcast(&xep->xep_state_cv);
409 mutex_exit(&xhcip->xhci_lock);
410
411 return (USB_SUCCESS);
412
413 }
414
415 /*
416 * Tear down everything that we did in open. After this, the consumer of this
417 * USB device is done.
418 */
419 /* ARGSUSED */
420 static int
421 xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
422 {
423 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
424 xhci_pipe_t *xp;
425 xhci_device_t *xd;
426 xhci_endpoint_t *xep;
427 uint32_t info;
428 int ret, i;
429 uint_t epid;
430
431 if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR &&
432 xhcip->xhci_usba.xa_intr_cb_ph != NULL) {
433 if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) !=
434 USB_SUCCESS) {
435 return (ret);
436 }
437 }
438
439 mutex_enter(&xhcip->xhci_lock);
440
441 xp = (xhci_pipe_t *)ph->p_hcd_private;
442 VERIFY(xp != NULL);
443
444 /*
445 * The default endpoint is special. It is created and destroyed with the
446 * device. So like with open, closing it is just state tracking. The
447 * same is true for the root hub.
448 */
449 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR)
450 goto remove;
451
452 xd = usba_hcdi_get_device_private(ph->p_usba_device);
453 epid = xhci_endpoint_pipe_to_epid(ph);
454 if (xd->xd_endpoints[epid] == NULL) {
455 mutex_exit(&xhcip->xhci_lock);
456 xhci_error(xhcip, "asked to do close pipe on slot %d, "
457 "port %d, endpoint: %d, but no endpoint structure",
458 xd->xd_slot, xd->xd_port, epid);
459 return (USB_FAILURE);
460 }
461 xep = xd->xd_endpoints[epid];
462
463 if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) {
464 xep->xep_pipe = NULL;
465 goto remove;
466 }
467
468 /*
469 * We need to clean up the endpoint. So the first thing we need to do is
470 * stop it with a configure endpoint command. Once it's stopped, we can
471 * free all associated resources.
472 */
473 mutex_enter(&xd->xd_imtx);
474
475 /*
476 * Potentially update the slot input context about the current max
477 * endpoint. Make sure to set that the slot context is being updated
478 * here as it may be changing and some hardware requires it.
479 */
480 xd->xd_input->xic_drop_flags = LE_32(XHCI_INCTX_MASK_DCI(epid + 1));
481 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
482 for (i = XHCI_NUM_ENDPOINTS - 1; i >= 0; i--) {
483 if (xd->xd_endpoints[i] != NULL &&
484 xd->xd_endpoints[i] != xep)
485 break;
486 }
487 info = xd->xd_slotin->xsc_info;
488 info &= ~XHCI_SCTX_DCI_MASK;
489 info |= XHCI_SCTX_SET_DCI(i + 1);
490 xd->xd_slotin->xsc_info = info;
491
492 /*
493 * Also zero out our context for this endpoint. Note that we don't
494 * bother with syncing DMA memory here as it's not required to be synced
495 * for this operation.
496 */
497 bzero(xd->xd_endin[xep->xep_num], sizeof (xhci_endpoint_context_t));
498
499 /*
500 * Stop the device and kill our timeout. Note, it is safe to hold the
501 * device's input mutex across the untimeout, this lock should never be
502 * referenced by the timeout code.
503 */
504 xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
505 mutex_exit(&xhcip->xhci_lock);
506 (void) untimeout(xep->xep_timeout);
507
508 ret = xhci_command_configure_endpoint(xhcip, xd);
509 mutex_exit(&xd->xd_imtx);
510 if (ret != USB_SUCCESS)
511 return (ret);
512 mutex_enter(&xhcip->xhci_lock);
513
514 /*
515 * Now that we've unconfigured the endpoint. See if we need to flush any
516 * transfers.
517 */
518 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING);
519 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
520 xhci_hcdi_periodic_free(xhcip, xp);
521 }
522
523 xhci_endpoint_fini(xd, epid);
524
525 remove:
526 ph->p_hcd_private = NULL;
527 list_remove(&xhcip->xhci_usba.xa_pipes, xp);
528 kmem_free(xp, sizeof (xhci_pipe_t));
529
530 mutex_exit(&xhcip->xhci_lock);
531
532 return (USB_SUCCESS);
533 }
534
535 /*
536 * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an
537 * arbitrary state, it may be running or it may be halted. In this case, we go
538 * through and check whether or not we know it's been halted or not. If it has
539 * not, then we stop the endpoint.
540 *
541 * Once the endpoint has been stopped, walk all transfers and go ahead and
542 * basically return them as being flushed. Then finally set the dequeue point
543 * for this endpoint.
544 */
545 /* ARGSUSED */
546 static int
547 xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
548 {
549 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
550 xhci_device_t *xd;
551 xhci_endpoint_t *xep;
552 uint_t epid;
553 int ret;
554
555 mutex_enter(&xhcip->xhci_lock);
556 if (xhcip->xhci_state & XHCI_S_ERROR) {
557 mutex_exit(&xhcip->xhci_lock);
558 return (USB_HC_HARDWARE_ERROR);
559 }
560
561 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
562 mutex_exit(&xhcip->xhci_lock);
563 return (USB_NOT_SUPPORTED);
564 }
565
566 xd = usba_hcdi_get_device_private(ph->p_usba_device);
567 epid = xhci_endpoint_pipe_to_epid(ph);
568 if (xd->xd_endpoints[epid] == NULL) {
569 mutex_exit(&xhcip->xhci_lock);
570 xhci_error(xhcip, "asked to do reset pipe on slot %d, "
571 "port %d, endpoint: %d, but no endpoint structure",
572 xd->xd_slot, xd->xd_port, epid);
573 return (USB_FAILURE);
574 }
575
576 xep = xd->xd_endpoints[epid];
577
578 /*
579 * Ensure that no other resets or time outs are going on right now.
580 */
581 while ((xep->xep_state & (XHCI_ENDPOINT_SERIALIZE)) != 0) {
582 cv_wait(&xep->xep_state_cv, &xhcip->xhci_lock);
583 }
584
585 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
586 ret = xhci_endpoint_quiesce(xhcip, xd, xep);
587 if (ret != USB_SUCCESS) {
588 /*
589 * We failed to quiesce for some reason, remove the flag and let
590 * someone else give it a shot.
591 */
592 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
593 "port %d, endpoint: %d, failed with %d.",
594 xd->xd_slot, xd->xd_port, epid, ret);
595 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
596 cv_broadcast(&xep->xep_state_cv);
597 mutex_exit(&xhcip->xhci_lock);
598 return (ret);
599 }
600
601 xhci_ring_skip(&xep->xep_ring);
602
603 mutex_exit(&xhcip->xhci_lock);
604 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
605 USB_SUCCESS) {
606 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
607 "port %d, endpoint: %d, failed setting ring dequeue with "
608 "%d.", xd->xd_slot, xd->xd_port, epid, ret);
609 mutex_enter(&xhcip->xhci_lock);
610 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
611 cv_broadcast(&xep->xep_state_cv);
612 mutex_exit(&xhcip->xhci_lock);
613 return (ret);
614 }
615
616 mutex_enter(&xhcip->xhci_lock);
617 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET);
618
619 /*
620 * We need to remove the periodic flag as part of resetting, as if this
621 * was used for periodic activity, it no longer is and therefore can now
622 * be used for such purposes.
623 *
624 * Notify anything waiting for us that we're done quiescing this device.
625 */
626 xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC);
627 cv_broadcast(&xep->xep_state_cv);
628 mutex_exit(&xhcip->xhci_lock);
629
630 return (USB_SUCCESS);
631 }
632
633 /*
634 * We're asked to reset or change the data toggle, which is used in a few cases.
635 * However, there doesn't seem to be a good way to do this in xHCI as the data
636 * toggle isn't exposed. It seems that dropping a reset endpoint would
637 * theoretically do this; however, that can only be used when in the HALTED
638 * state. As such, for now we just return.
639 */
640 /* ARGSUSED */
641 void
642 xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle)
643 {
644 }
645
646 /*
647 * We need to convert the USB request to an 8-byte little endian value. If we
648 * didn't have to think about big endian systems, this would be fine.
649 * Unfortunately, with them, this is a bit confusing. The problem is that if you
650 * think of this as a struct layout, the order that we or things together
651 * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP
652 * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if
653 * ends up at byte 7, where as it needs to be at one. Hence why we do a final
654 * LE_64 at the end of this, to convert this into the byte order that it's
655 * expected to be in.
656 */
657 static uint64_t
658 xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp)
659 {
660 uint64_t ret = ucrp->ctrl_bmRequestType |
661 (ucrp->ctrl_bRequest << 8) |
662 ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) |
663 ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) |
664 ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48);
665 return (LE_64(ret));
666 }
667
668 /*
669 * USBA calls us in order to make a specific control type request to a device,
670 * potentially even the root hub. If the request is for the root hub, then we
671 * need to intercept this and cons up the requested data.
672 */
673 static int
674 xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp,
675 usb_flags_t usb_flags)
676 {
677 int ret, statusdir, trt;
678 uint_t ep;
679 xhci_device_t *xd;
680 xhci_endpoint_t *xep;
681 xhci_transfer_t *xt;
682 boolean_t datain;
683
684 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
685
686 mutex_enter(&xhcip->xhci_lock);
687 if (xhcip->xhci_state & XHCI_S_ERROR) {
688 mutex_exit(&xhcip->xhci_lock);
689 return (USB_HC_HARDWARE_ERROR);
690 }
691
692 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
693 ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp);
694 mutex_exit(&xhcip->xhci_lock);
695 return (ret);
696 }
697
698 /*
699 * Determine the device and endpoint.
700 */
701 xd = usba_hcdi_get_device_private(ph->p_usba_device);
702 ep = xhci_endpoint_pipe_to_epid(ph);
703 if (xd->xd_endpoints[ep] == NULL) {
704 mutex_exit(&xhcip->xhci_lock);
705 xhci_error(xhcip, "asked to do control transfer on slot %d, "
706 "port %d, endpoint: %d, but no endpoint structure",
707 xd->xd_slot, xd->xd_port, ep);
708 return (USB_FAILURE);
709 }
710 xep = xd->xd_endpoints[ep];
711
712 /*
713 * There are several types of requests that we have to handle in special
714 * ways in xHCI. If we have one of those requests, then we don't
715 * necessarily go through the normal path. These special cases are all
716 * documented in xHCI 1.1 / 4.5.4.
717 *
718 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF
719 * special cased here. This action is a little confusing by default. The
720 * xHCI specification requires that we may need to issue a configure
721 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2
722 * states that we don't actually need to if nothing in the endpoint
723 * configuration context has changed. Because nothing in it should have
724 * changed as part of this, we don't need to do anything and instead
725 * just can issue the request normally. We're also assuming in the
726 * USB_REQ_SET_IF case that if something's changing the interface, the
727 * non-default endpoint will have yet to be opened.
728 */
729 if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV &&
730 ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) {
731 /*
732 * As we've defined an explicit set-address endpoint, we should
733 * never call this function. If we get here, always fail.
734 */
735 mutex_exit(&xhcip->xhci_lock);
736 usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED);
737 return (USB_SUCCESS);
738 }
739
740 mutex_exit(&xhcip->xhci_lock);
741
742 /*
743 * Allocate the transfer memory, etc.
744 */
745 xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags);
746 if (xt == NULL) {
747 return (USB_NO_RESOURCES);
748 }
749 xt->xt_usba_req = (usb_opaque_t)ucrp;
750 xt->xt_timeout = ucrp->ctrl_timeout;
751 if (xt->xt_timeout == 0) {
752 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
753 }
754
755 if (ucrp->ctrl_wLength > 0) {
756 if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) {
757 trt = XHCI_TRB_TRT_IN;
758 datain = B_TRUE;
759 statusdir = 0;
760 } else {
761 trt = XHCI_TRB_TRT_OUT;
762 datain = B_FALSE;
763 statusdir = XHCI_TRB_DIR_IN;
764
765 xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr,
766 ucrp->ctrl_wLength, B_FALSE);
767 if (xhci_transfer_sync(xhcip, xt,
768 DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
769 xhci_transfer_free(xhcip, xt);
770 xhci_error(xhcip, "failed to synchronize ctrl "
771 "transfer DMA memory on endpoint %u of "
772 "device on slot %d and port %d: resetting "
773 "device", xep->xep_num, xd->xd_slot,
774 xd->xd_port);
775 xhci_fm_runtime_reset(xhcip);
776 return (USB_HC_HARDWARE_ERROR);
777 }
778 }
779 } else {
780 trt = 0;
781 datain = B_FALSE;
782 statusdir = XHCI_TRB_DIR_IN;
783 }
784
785 /*
786 * We always fill in the required setup and status TRBs ourselves;
787 * however, to minimize our knowledge about how the data has been split
788 * across multiple DMA cookies in an SGL, we leave that to the transfer
789 * logic to fill in.
790 */
791 xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp);
792 xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0));
793 xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT |
794 XHCI_TRB_TYPE_SETUP);
795
796 if (ucrp->ctrl_wLength > 0)
797 xhci_transfer_trb_fill_data(xep, xt, 1, datain);
798
799 xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0;
800 xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0));
801 xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS |
802 XHCI_TRB_IOC | statusdir);
803
804
805 mutex_enter(&xhcip->xhci_lock);
806
807 /*
808 * Schedule the transfer, allocating resources in the process.
809 */
810 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
811 xhci_transfer_free(xhcip, xt);
812 mutex_exit(&xhcip->xhci_lock);
813 return (USB_NO_RESOURCES);
814 }
815
816 mutex_exit(&xhcip->xhci_lock);
817
818 return (USB_SUCCESS);
819 }
820
821 /*
822 * This request is trying to get the upper bound on the amount of data we're
823 * willing transfer in one go. Note that this amount can be broken down into
824 * multiple SGL entries, this interface doesn't particularly care about that.
825 */
826 /* ARGSUSED */
827 static int
828 xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep)
829 {
830 if (sizep != NULL)
831 *sizep = XHCI_MAX_TRANSFER;
832 return (USB_SUCCESS);
833 }
834
835 /*
836 * Perform a bulk transfer. This is a pretty straightforward action. We
837 * basically just allocate the appropriate transfer and try to schedule it,
838 * hoping there is enough space.
839 */
840 static int
841 xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp,
842 usb_flags_t usb_flags)
843 {
844 uint_t epid;
845 xhci_device_t *xd;
846 xhci_endpoint_t *xep;
847 xhci_transfer_t *xt;
848 boolean_t datain;
849
850 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
851
852 mutex_enter(&xhcip->xhci_lock);
853 if (xhcip->xhci_state & XHCI_S_ERROR) {
854 mutex_exit(&xhcip->xhci_lock);
855 return (USB_HC_HARDWARE_ERROR);
856 }
857
858 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
859 mutex_exit(&xhcip->xhci_lock);
860 return (USB_NOT_SUPPORTED);
861 }
862
863 xd = usba_hcdi_get_device_private(ph->p_usba_device);
864 epid = xhci_endpoint_pipe_to_epid(ph);
865 if (xd->xd_endpoints[epid] == NULL) {
866 mutex_exit(&xhcip->xhci_lock);
867 xhci_error(xhcip, "asked to do control transfer on slot %d, "
868 "port %d, endpoint: %d, but no endpoint structure",
869 xd->xd_slot, xd->xd_port, epid);
870 return (USB_FAILURE);
871 }
872 xep = xd->xd_endpoints[epid];
873 mutex_exit(&xhcip->xhci_lock);
874
875 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
876 datain = B_TRUE;
877 } else {
878 datain = B_FALSE;
879 }
880
881 xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags);
882 if (xt == NULL) {
883 return (USB_NO_RESOURCES);
884 }
885 xt->xt_usba_req = (usb_opaque_t)ubrp;
886 xt->xt_timeout = ubrp->bulk_timeout;
887 if (xt->xt_timeout == 0) {
888 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
889 }
890
891 if (ubrp->bulk_len > 0 && datain == B_FALSE) {
892 xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len,
893 B_FALSE);
894 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
895 DDI_FM_OK) {
896 xhci_transfer_free(xhcip, xt);
897 xhci_error(xhcip, "failed to synchronize bulk "
898 "transfer DMA memory on endpoint %u of "
899 "device on slot %d and port %d: resetting "
900 "device", xep->xep_num, xd->xd_slot,
901 xd->xd_port);
902 xhci_fm_runtime_reset(xhcip);
903 return (USB_HC_HARDWARE_ERROR);
904 }
905 }
906
907 xhci_transfer_trb_fill_data(xep, xt, 0, datain);
908 mutex_enter(&xhcip->xhci_lock);
909 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
910 xhci_transfer_free(xhcip, xt);
911 mutex_exit(&xhcip->xhci_lock);
912 return (USB_NO_RESOURCES);
913 }
914 mutex_exit(&xhcip->xhci_lock);
915
916 return (USB_SUCCESS);
917 }
918
919 static void
920 xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep,
921 xhci_transfer_t *xt, usb_isoc_req_t *usrp)
922 {
923 int i;
924 uintptr_t buf;
925
926 buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress;
927 for (i = 0; i < usrp->isoc_pkts_count; i++) {
928 int flags;
929 uint_t tbc, tlbpc;
930
931 ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length;
932 xhci_trb_t *trb = &xt->xt_trbs[i];
933
934 trb->trb_addr = LE_64(buf);
935
936 /*
937 * Because we know that a single frame can have all of its data
938 * in a single instance, we know that we don't need to do
939 * anything special here.
940 */
941 trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) |
942 XHCI_TRB_INTR(0));
943
944 /*
945 * Always enable SIA to start the frame ASAP. We also always
946 * enable an interrupt on a short packet. If this is the last
947 * trb, then we will set IOC. Each TRB created here is really
948 * its own TD. However, we only set an interrupt on the last
949 * entry to better deal with scheduling.
950 */
951 flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0);
952 flags |= XHCI_TRB_TYPE_ISOCH;
953
954 if (i + 1 == usrp->isoc_pkts_count)
955 flags |= XHCI_TRB_IOC;
956
957 /*
958 * Now we need to calculate the TBC and the TLBPC.
959 */
960 xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc);
961 flags |= XHCI_TRB_SET_TBC(tbc);
962 flags |= XHCI_TRB_SET_TLBPC(tlbpc);
963
964 trb->trb_flags = LE_32(flags);
965 buf += len;
966
967 /*
968 * Go through and copy the required data to our local copy of
969 * the isoc descriptor. By default, we assume that all data will
970 * be copied and the status set to OK. This mirrors the fact
971 * that we won't get a notification unless there's been an
972 * error or short packet transfer.
973 */
974 xt->xt_isoc[i].isoc_pkt_length = len;
975 xt->xt_isoc[i].isoc_pkt_actual_length = len;
976 xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK;
977 }
978 }
979
980 /*
981 * Initialize periodic IN requests (both interrupt and isochronous)
982 */
983 static int
984 xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
985 usb_opaque_t usb_req, size_t len, int usb_flags)
986 {
987 int i, ret;
988 uint_t epid;
989 xhci_device_t *xd;
990 xhci_endpoint_t *xep;
991 xhci_pipe_t *xp;
992 xhci_periodic_pipe_t *xpp;
993
994 mutex_enter(&xhcip->xhci_lock);
995 if (xhcip->xhci_state & XHCI_S_ERROR) {
996 mutex_exit(&xhcip->xhci_lock);
997 return (USB_HC_HARDWARE_ERROR);
998 }
999
1000 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1001 epid = xhci_endpoint_pipe_to_epid(ph);
1002 if (xd->xd_endpoints[epid] == NULL) {
1003 xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1004 "port %d, endpoint: %d, but no endpoint structure",
1005 xd->xd_slot, xd->xd_port, epid);
1006 mutex_exit(&xhcip->xhci_lock);
1007 return (USB_FAILURE);
1008 }
1009 xep = xd->xd_endpoints[epid];
1010 xp = (xhci_pipe_t *)ph->p_hcd_private;
1011 if (xp == NULL) {
1012 xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1013 "port %d, endpoint: %d, but no pipe structure",
1014 xd->xd_slot, xd->xd_port, epid);
1015 mutex_exit(&xhcip->xhci_lock);
1016 return (USB_FAILURE);
1017 }
1018 xpp = &xp->xp_periodic;
1019
1020 /*
1021 * Only allow a single polling request at any given time.
1022 */
1023 if (xpp->xpp_usb_req != NULL) {
1024 mutex_exit(&xhcip->xhci_lock);
1025 return (USB_BUSY);
1026 }
1027
1028 /*
1029 * We keep allocations around in case we restart polling, which most
1030 * devices do (not really caring about a lost event). However, we don't
1031 * support a driver changing that size on us, which it probably won't.
1032 * If we stumble across driver that does, then this will need to become
1033 * a lot more complicated.
1034 */
1035 if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) {
1036 mutex_exit(&xhcip->xhci_lock);
1037 return (USB_INVALID_REQUEST);
1038 }
1039
1040 if (xpp->xpp_tsize == 0) {
1041 int ntrbs;
1042 int ntransfers;
1043
1044 /*
1045 * What we allocate varies based on whether or not this is an
1046 * isochronous or interrupt IN periodic.
1047 */
1048 if (xep->xep_type == USB_EP_ATTR_INTR) {
1049 ntrbs = 0;
1050 ntransfers = XHCI_INTR_IN_NTRANSFERS;
1051 } else {
1052 usb_isoc_req_t *usrp;
1053 ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH);
1054
1055 usrp = (usb_isoc_req_t *)usb_req;
1056 ntrbs = usrp->isoc_pkts_count;
1057 ntransfers = XHCI_ISOC_IN_NTRANSFERS;
1058 }
1059
1060 xpp->xpp_tsize = len;
1061 xpp->xpp_ntransfers = ntransfers;
1062
1063 for (i = 0; i < xpp->xpp_ntransfers; i++) {
1064 xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep,
1065 len, ntrbs, usb_flags);
1066 if (xt == NULL) {
1067 xhci_hcdi_periodic_free(xhcip, xp);
1068 mutex_exit(&xhcip->xhci_lock);
1069 return (USB_NO_RESOURCES);
1070 }
1071
1072 if (xep->xep_type == USB_EP_ATTR_INTR) {
1073 xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE);
1074 } else {
1075 usb_isoc_req_t *usrp;
1076 usrp = (usb_isoc_req_t *)usb_req;
1077 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1078 xt->xt_data_tohost = B_TRUE;
1079 }
1080 xpp->xpp_transfers[i] = xt;
1081 }
1082 }
1083
1084 /*
1085 * Mark the endpoint as periodic so we don't have timeouts at play.
1086 */
1087 xep->xep_state |= XHCI_ENDPOINT_PERIODIC;
1088
1089 /*
1090 * Now that we've allocated everything, go ahead and schedule them and
1091 * kick off the ring.
1092 */
1093 for (i = 0; i < xpp->xpp_ntransfers; i++) {
1094 int ret;
1095 ret = xhci_endpoint_schedule(xhcip, xd, xep,
1096 xpp->xpp_transfers[i], B_FALSE);
1097 if (ret != 0) {
1098 (void) xhci_ring_reset(xhcip, &xep->xep_ring);
1099 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
1100 mutex_exit(&xhcip->xhci_lock);
1101 return (ret);
1102 }
1103 }
1104
1105 /*
1106 * Don't worry about freeing memory, it'll be done when the endpoint
1107 * closes and the whole system is reset.
1108 */
1109 xpp->xpp_usb_req = usb_req;
1110 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE;
1111
1112 ret = xhci_endpoint_ring(xhcip, xd, xep);
1113 mutex_exit(&xhcip->xhci_lock);
1114 return (ret);
1115 }
1116
1117 static int
1118 xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1119 usb_intr_req_t *uirp, usb_flags_t usb_flags)
1120 {
1121 uint_t epid;
1122 xhci_device_t *xd;
1123 xhci_endpoint_t *xep;
1124 xhci_transfer_t *xt;
1125 boolean_t datain;
1126 mblk_t *mp = NULL;
1127
1128 mutex_enter(&xhcip->xhci_lock);
1129 if (xhcip->xhci_state & XHCI_S_ERROR) {
1130 mutex_exit(&xhcip->xhci_lock);
1131 return (USB_HC_HARDWARE_ERROR);
1132 }
1133
1134 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1135 epid = xhci_endpoint_pipe_to_epid(ph);
1136 if (xd->xd_endpoints[epid] == NULL) {
1137 xhci_error(xhcip, "asked to do interrupt transfer on slot %d, "
1138 "port %d, endpoint: %d, but no endpoint structure",
1139 xd->xd_slot, xd->xd_port, epid);
1140 mutex_exit(&xhcip->xhci_lock);
1141 return (USB_FAILURE);
1142 }
1143 xep = xd->xd_endpoints[epid];
1144
1145 mutex_exit(&xhcip->xhci_lock);
1146
1147 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1148 datain = B_TRUE;
1149 } else {
1150 datain = B_FALSE;
1151 }
1152
1153 xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags);
1154 if (xt == NULL) {
1155 return (USB_NO_RESOURCES);
1156 }
1157
1158 xt->xt_usba_req = (usb_opaque_t)uirp;
1159 xt->xt_timeout = uirp->intr_timeout;
1160 if (xt->xt_timeout == 0) {
1161 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1162 }
1163
1164 /*
1165 * Unlike other request types, USB Interrupt-IN requests aren't required
1166 * to have allocated the message block for data. If they haven't, we
1167 * take care of that now.
1168 */
1169 if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) {
1170 if (usb_flags & USB_FLAGS_SLEEP) {
1171 mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG,
1172 NULL);
1173 } else {
1174 mp = allocb(uirp->intr_len, 0);
1175 }
1176 if (mp == NULL) {
1177 xhci_transfer_free(xhcip, xt);
1178 mutex_exit(&xhcip->xhci_lock);
1179 return (USB_NO_RESOURCES);
1180 }
1181 uirp->intr_data = mp;
1182 }
1183
1184 if (uirp->intr_len > 0 && datain == B_FALSE) {
1185 xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len,
1186 B_FALSE);
1187 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
1188 DDI_FM_OK) {
1189 xhci_transfer_free(xhcip, xt);
1190 xhci_error(xhcip, "failed to synchronize interrupt "
1191 "transfer DMA memory on endpoint %u of "
1192 "device on slot %d and port %d: resetting "
1193 "device", xep->xep_num, xd->xd_slot,
1194 xd->xd_port);
1195 xhci_fm_runtime_reset(xhcip);
1196 return (USB_HC_HARDWARE_ERROR);
1197 }
1198 }
1199
1200 xhci_transfer_trb_fill_data(xep, xt, 0, datain);
1201 mutex_enter(&xhcip->xhci_lock);
1202 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
1203 if (mp != NULL) {
1204 uirp->intr_data = NULL;
1205 freemsg(mp);
1206 }
1207 xhci_transfer_free(xhcip, xt);
1208 mutex_exit(&xhcip->xhci_lock);
1209 return (USB_NO_RESOURCES);
1210 }
1211 mutex_exit(&xhcip->xhci_lock);
1212
1213 return (USB_SUCCESS);
1214 }
1215
1216 /*
1217 * We've been asked to perform an interrupt transfer. When this is an interrupt
1218 * IN endpoint, that means that the hcd is being asked to start polling on the
1219 * endpoint. When the endpoint is the root hub, it effectively becomes synthetic
1220 * polling.
1221 *
1222 * When we have an interrupt out endpoint, then this is just a single simple
1223 * interrupt request that we send out and there isn't much special to do beyond
1224 * the normal activity.
1225 */
1226 static int
1227 xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp,
1228 usb_flags_t usb_flags)
1229 {
1230 int ret;
1231 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
1232
1233 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1234 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1235 ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp);
1236 } else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) {
1237 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp,
1238 usb_flags);
1239 } else {
1240 ret = xhci_hcdi_periodic_init(xhcip, ph,
1241 (usb_opaque_t)uirp, uirp->intr_len, usb_flags);
1242 }
1243 } else {
1244 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1245 return (USB_NOT_SUPPORTED);
1246 }
1247 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags);
1248 }
1249
1250 return (ret);
1251 }
1252
1253 /* ARGSUSED */
1254 static int
1255 xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph,
1256 usb_flags_t usb_flags)
1257 {
1258 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1259 }
1260
1261 static int
1262 xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1263 usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1264 {
1265 int i;
1266 size_t count;
1267
1268 count = 0;
1269 for (i = 0; i < usrp->isoc_pkts_count; i++) {
1270 count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1271 }
1272
1273 return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count,
1274 usb_flags));
1275 }
1276
1277 /*
1278 * This is used to create an isochronous request to send data out to the device.
1279 * This is a single one shot request, it is not something that we'll have to
1280 * repeat over and over.
1281 */
1282 static int
1283 xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1284 usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1285 {
1286 int i, ret;
1287 uint_t epid;
1288 size_t count, mblen;
1289 xhci_device_t *xd;
1290 xhci_endpoint_t *xep;
1291 xhci_transfer_t *xt;
1292
1293 count = 0;
1294 for (i = 0; i < usrp->isoc_pkts_count; i++) {
1295 count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1296 }
1297 mblen = MBLKL(usrp->isoc_data);
1298
1299 if (count != mblen) {
1300 return (USB_INVALID_ARGS);
1301 }
1302
1303 mutex_enter(&xhcip->xhci_lock);
1304 if (xhcip->xhci_state & XHCI_S_ERROR) {
1305 mutex_exit(&xhcip->xhci_lock);
1306 return (USB_HC_HARDWARE_ERROR);
1307 }
1308
1309 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1310 epid = xhci_endpoint_pipe_to_epid(ph);
1311 if (xd->xd_endpoints[epid] == NULL) {
1312 xhci_error(xhcip, "asked to do isochronous transfer on slot "
1313 "%d, port %d, endpoint: %d, but no endpoint structure",
1314 xd->xd_slot, xd->xd_port, epid);
1315 mutex_exit(&xhcip->xhci_lock);
1316 return (USB_FAILURE);
1317 }
1318 xep = xd->xd_endpoints[epid];
1319 mutex_exit(&xhcip->xhci_lock);
1320
1321 xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count,
1322 usb_flags);
1323 if (xt == NULL) {
1324 return (USB_NO_RESOURCES);
1325 }
1326 xt->xt_usba_req = (usb_opaque_t)usrp;
1327
1328 /*
1329 * USBA doesn't provide any real way for a timeout to be defined for an
1330 * isochronous event. However, since we technically aren't a periodic
1331 * endpoint, go ahead and always set the default timeout. It's better
1332 * than nothing.
1333 */
1334 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1335
1336 xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE);
1337 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
1338 xhci_transfer_free(xhcip, xt);
1339 xhci_error(xhcip, "failed to synchronize isochronous "
1340 "transfer DMA memory on endpoint %u of "
1341 "device on slot %d and port %d: resetting "
1342 "device", xep->xep_num, xd->xd_slot,
1343 xd->xd_port);
1344 xhci_fm_runtime_reset(xhcip);
1345 return (USB_HC_HARDWARE_ERROR);
1346 }
1347
1348 /*
1349 * Fill in the ISOC data. Note, that we always use ASAP scheduling and
1350 * we don't support specifying the frame at this time, for better or
1351 * worse.
1352 */
1353 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1354
1355 mutex_enter(&xhcip->xhci_lock);
1356 ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE);
1357 mutex_exit(&xhcip->xhci_lock);
1358
1359 return (ret);
1360 }
1361
1362 static int
1363 xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp,
1364 usb_flags_t usb_flags)
1365 {
1366 int ret;
1367 xhci_t *xhcip;
1368
1369 xhcip = xhci_hcdi_get_xhcip(ph);
1370
1371 /*
1372 * We don't support isochronous transactions on the root hub at all.
1373 * Always fail them if for some reason we end up here.
1374 */
1375 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1376 return (USB_NOT_SUPPORTED);
1377 }
1378
1379 /*
1380 * We do not support being asked to set the frame ID at this time. We
1381 * require that everything specify the attribute
1382 * USB_ATTRS_ISOC_XFER_ASAP.
1383 */
1384 if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) {
1385 return (USB_NOT_SUPPORTED);
1386 }
1387
1388 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1389 /*
1390 * Note, there is no such thing as a non-periodic isochronous
1391 * incoming transfer.
1392 */
1393 ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags);
1394 } else {
1395 ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags);
1396 }
1397
1398 return (ret);
1399 }
1400
1401 /* ARGSUSED */
1402 static int
1403 xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph,
1404 usb_flags_t usb_flags)
1405 {
1406 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1407 }
1408
1409 /*
1410 * This is asking us for the current frame number. The USBA expects this to
1411 * actually be a bit of a fiction, as it tries to maintain a frame number well
1412 * beyond what the hardware actually contains in its registers. Hardware
1413 * basically has a 14-bit counter, whereas we need to have a constant amount of
1414 * milliseconds.
1415 *
1416 * Today, no client drivers actually use this API and everyone specifies the
1417 * attribute to say that we should schedule things ASAP. So until we have some
1418 * real device that want this functionality, we're going to fail.
1419 */
1420 /* ARGSUSED */
1421 static int
1422 xhci_hcdi_get_current_frame_number(usba_device_t *usba_device,
1423 usb_frame_number_t *frame_number)
1424 {
1425 return (USB_FAILURE);
1426 }
1427
1428 /*
1429 * See the comments around the XHCI_ISOC_MAX_TRB macro for more information.
1430 */
1431 /* ARGSUSED */
1432 static int
1433 xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device,
1434 uint_t *max_isoc_pkts_per_request)
1435 {
1436 *max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB;
1437 return (USB_SUCCESS);
1438 }
1439
1440 /*
1441 * The next series of routines is used for both the OBP console and general USB
1442 * console polled I/O. In general, we opt not to support any of that at this
1443 * time in xHCI. As we have the need of that, we can start plumbing that
1444 * through.
1445 */
1446 /* ARGSUSED */
1447 static int
1448 xhci_hcdi_console_input_init(usba_pipe_handle_data_t *pipe_handle,
1449 uchar_t **obp_buf, usb_console_info_impl_t *console_input_info)
1450 {
1451 return (USB_NOT_SUPPORTED);
1452 }
1453
1454 /* ARGSUSED */
1455 static int
1456 xhci_hcdi_console_input_fini(usb_console_info_impl_t *console_input_info)
1457 {
1458 return (USB_NOT_SUPPORTED);
1459 }
1460
1461 /* ARGSUSED */
1462 static int
1463 xhci_hcdi_console_input_enter(usb_console_info_impl_t *console_input_info)
1464 {
1465 return (USB_NOT_SUPPORTED);
1466 }
1467
1468 /* ARGSUSED */
1469 static int
1470 xhci_hcdi_console_read(usb_console_info_impl_t *console_input_info,
1471 uint_t *num_characters)
1472 {
1473 return (USB_NOT_SUPPORTED);
1474 }
1475
1476 /* ARGSUSED */
1477 static int
1478 xhci_hcdi_console_input_exit(usb_console_info_impl_t *console_input_info)
1479 {
1480 return (USB_NOT_SUPPORTED);
1481 }
1482
1483 /* ARGSUSED */
1484 static int
1485 xhci_hcdi_console_output_init(usba_pipe_handle_data_t *pipe_handle,
1486 usb_console_info_impl_t *console_output_info)
1487 {
1488 return (USB_NOT_SUPPORTED);
1489 }
1490
1491 /* ARGSUSED */
1492 static int
1493 xhci_hcdi_console_output_fini(usb_console_info_impl_t *console_output_info)
1494 {
1495 return (USB_NOT_SUPPORTED);
1496 }
1497
1498 /* ARGSUSED */
1499 static int
1500 xhci_hcdi_console_output_enter(usb_console_info_impl_t *console_output_info)
1501 {
1502 return (USB_NOT_SUPPORTED);
1503 }
1504
1505 /* ARGSUSED */
1506 static int
1507 xhci_hcdi_console_write(usb_console_info_impl_t *console_output_info,
1508 uchar_t *buf, uint_t num_characters, uint_t *num_characters_written)
1509 {
1510 return (USB_NOT_SUPPORTED);
1511 }
1512
1513 /* ARGSUSED */
1514 static int
1515 xhci_hcdi_console_output_exit(usb_console_info_impl_t *console_output_info)
1516 {
1517 return (USB_NOT_SUPPORTED);
1518 }
1519
1520 /*
1521 * VERSION 2 ops and helpers
1522 */
1523
1524 static void
1525 xhci_hcdi_device_free(xhci_device_t *xd)
1526 {
1527 xhci_dma_free(&xd->xd_ictx);
1528 xhci_dma_free(&xd->xd_octx);
1529 mutex_destroy(&xd->xd_imtx);
1530 kmem_free(xd, sizeof (xhci_device_t));
1531 }
1532
1533 /*
1534 * Calculate the device's route string. In USB 3.0 the route string is a 20-bit
1535 * number. Each four bits represent a port number attached to a deeper hub.
1536 * Particularly it represents the port on that current hub that you need to go
1537 * down to reach the next device. Bits 0-3 represent the first *external* hub.
1538 * So a device connected to a root hub has a route string of zero. Imagine the
1539 * following set of devices:
1540 *
1541 * . port 2 . port 5
1542 * . .
1543 * +----------+ . +--------+ . +-------+
1544 * | root hub |-*->| hub 1 |-*->| hub 2 |
1545 * +----------+ +--------+ +-------+
1546 * * . port 12 * . port 8 * . port 1
1547 * v v v
1548 * +-------+ +-------+ +-------+
1549 * | dev a | | dev b | | dev c |
1550 * +-------+ +-------+ +-------+
1551 *
1552 * So, based on the above diagram, device a should have a route string of 0,
1553 * because it's directly connected to the root port. Device b would simply have
1554 * a route string of 8. This is because it travels through one non-root hub, hub
1555 * 1, and it does so on port 8. The root ports basically don't matter. Device c
1556 * would then have a route string of 0x15, as it's first traversing through hub
1557 * 1 on port 2 and then hub 2 on port 5.
1558 *
1559 * Finally, it's worth mentioning that because it's a four bit field, if for
1560 * some reason a device has more than 15 ports, we just treat the value as 15.
1561 *
1562 * Note, as part of this, we also grab what port on the root hub this whole
1563 * chain is on, as we're required to store that information in the slot context.
1564 */
1565 static void
1566 xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port)
1567 {
1568 uint32_t route = 0;
1569 usba_device_t *hub = ud->usb_parent_hub;
1570 usba_device_t *port_dev = ud;
1571
1572 ASSERT(hub != NULL);
1573
1574 /*
1575 * Iterate over every hub, updating the route as we go. When we
1576 * encounter a hub without a parent, then we're at the root hub. At
1577 * which point, the port we want is on port_dev (the child of hub).
1578 */
1579 while (hub->usb_parent_hub != NULL) {
1580 uint32_t p;
1581
1582 p = port_dev->usb_port;
1583 if (p > 15)
1584 p = 15;
1585 route <<= 4;
1586 route |= p & 0xf;
1587 port_dev = hub;
1588 hub = hub->usb_parent_hub;
1589 }
1590
1591 ASSERT(port_dev->usb_parent_hub == hub);
1592 *root_port = port_dev->usb_port;
1593 *routep = XHCI_ROUTE_MASK(route);
1594 }
1595
1596 /*
1597 * If a low or full speed device is behind a high-speed device that is not a
1598 * root hub, then we must include the port and slot of that device. USBA already
1599 * stores this device in the usb_hs_hub_usba_dev member.
1600 */
1601 static uint32_t
1602 xhci_hcdi_device_tt(usba_device_t *ud)
1603 {
1604 uint32_t ret;
1605 xhci_device_t *xd;
1606
1607 if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV)
1608 return (0);
1609
1610 if (ud->usb_hs_hub_usba_dev == NULL)
1611 return (0);
1612
1613 ASSERT(ud->usb_hs_hub_usba_dev != NULL);
1614 ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL);
1615 xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev);
1616 ASSERT(xd != NULL);
1617
1618 ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot);
1619 ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port);
1620
1621 return (ret);
1622 }
1623
1624 /*
1625 * Initialize a new device. This allocates a device slot from the controller,
1626 * which tranfers it to our control.
1627 */
1628 static int
1629 xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp)
1630 {
1631 int ret, i;
1632 xhci_device_t *xd;
1633 ddi_device_acc_attr_t acc;
1634 ddi_dma_attr_t attr;
1635 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1636 size_t isize, osize, incr;
1637 uint32_t route, rp, info, info2, tt;
1638
1639 xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP);
1640 xd->xd_port = port;
1641 xd->xd_usbdev = ud;
1642 mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER,
1643 (void *)(uintptr_t)xhcip->xhci_intr_pri);
1644
1645 /*
1646 * The size of the context structures is based upon the presence of the
1647 * context flag which determines whether we have a 32-byte or 64-byte
1648 * context. Note that the input context always has to account for the
1649 * entire size of the xhci_input_contex_t, which is 32-bytes by default.
1650 */
1651 if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) {
1652 incr = 64;
1653 osize = XHCI_DEVICE_CONTEXT_64;
1654 isize = XHCI_DEVICE_CONTEXT_64 + incr;
1655 } else {
1656 incr = 32;
1657 osize = XHCI_DEVICE_CONTEXT_32;
1658 isize = XHCI_DEVICE_CONTEXT_32 + incr;
1659 }
1660
1661 xhci_dma_acc_attr(xhcip, &acc);
1662 xhci_dma_dma_attr(xhcip, &attr);
1663 if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE,
1664 isize, B_FALSE) == B_FALSE) {
1665 xhci_hcdi_device_free(xd);
1666 return (USB_NO_RESOURCES);
1667 }
1668
1669 xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va;
1670 xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr);
1671 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1672 xd->xd_endin[i] =
1673 (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va +
1674 (i + 2) * incr);
1675 }
1676
1677 if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE,
1678 osize, B_FALSE) == B_FALSE) {
1679 xhci_hcdi_device_free(xd);
1680 return (USB_NO_RESOURCES);
1681 }
1682 xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va;
1683 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1684 xd->xd_endout[i] =
1685 (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va +
1686 (i + 1) * incr);
1687 }
1688
1689 ret = xhci_command_enable_slot(xhcip, &xd->xd_slot);
1690 if (ret != USB_SUCCESS) {
1691 xhci_hcdi_device_free(xd);
1692 return (ret);
1693 }
1694
1695 /*
1696 * These are the default slot context and the endpoint zero context that
1697 * we're enabling. See 4.3.3.
1698 */
1699 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
1700 XHCI_INCTX_MASK_DCI(1));
1701
1702 /*
1703 * Note, we never need to set the MTT bit as illumos never enables the
1704 * alternate MTT interface.
1705 */
1706 xhci_hcdi_device_route(ud, &route, &rp);
1707 info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1);
1708 switch (ud->usb_port_status) {
1709 case USBA_LOW_SPEED_DEV:
1710 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW);
1711 break;
1712 case USBA_HIGH_SPEED_DEV:
1713 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH);
1714 break;
1715 case USBA_FULL_SPEED_DEV:
1716 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL);
1717 break;
1718 case USBA_SUPER_SPEED_DEV:
1719 default:
1720 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER);
1721 break;
1722 }
1723 info2 = XHCI_SCTX_SET_RHPORT(rp);
1724 tt = XHCI_SCTX_SET_IRQ_TARGET(0);
1725 tt |= xhci_hcdi_device_tt(ud);
1726
1727 xd->xd_slotin->xsc_info = LE_32(info);
1728 xd->xd_slotin->xsc_info2 = LE_32(info2);
1729 xd->xd_slotin->xsc_tt = LE_32(tt);
1730
1731 if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) {
1732 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1733 xhci_hcdi_device_free(xd);
1734 return (USB_HC_HARDWARE_ERROR);
1735 }
1736
1737 if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) {
1738 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1739 xhci_endpoint_fini(xd, 0);
1740 xhci_hcdi_device_free(xd);
1741 return (USB_HC_HARDWARE_ERROR);
1742 }
1743
1744 if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) {
1745 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1746 xhci_context_slot_output_fini(xhcip, xd);
1747 xhci_endpoint_fini(xd, 0);
1748 xhci_hcdi_device_free(xd);
1749 return (ret);
1750 }
1751
1752 mutex_enter(&xhcip->xhci_lock);
1753 list_insert_tail(&xhcip->xhci_usba.xa_devices, xd);
1754 mutex_exit(&xhcip->xhci_lock);
1755
1756 *hcdpp = xd;
1757 return (ret);
1758 }
1759
1760 /*
1761 * We're tearing down a device now. That means that the only endpoint context
1762 * that's still valid would be endpoint zero.
1763 */
1764 static void
1765 xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp)
1766 {
1767 int ret;
1768 xhci_endpoint_t *xep;
1769 xhci_device_t *xd;
1770 xhci_t *xhcip;
1771
1772 /*
1773 * Right now, it's theoretically possible that USBA may try and call
1774 * us here even if we hadn't successfully finished the device_init()
1775 * endpoint. We should probably modify the USBA to make sure that this
1776 * can't happen.
1777 */
1778 if (hcdp == NULL)
1779 return;
1780
1781 xd = hcdp;
1782 xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1783
1784 /*
1785 * Make sure we have no timeout running on the default endpoint still.
1786 */
1787 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1788 mutex_enter(&xhcip->xhci_lock);
1789 xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
1790 mutex_exit(&xhcip->xhci_lock);
1791 (void) untimeout(xep->xep_timeout);
1792
1793 /*
1794 * Go ahead and disable the slot. There's no reason to do anything
1795 * special about the default endpoint as it will be disabled as a part
1796 * of the slot disabling. However, if this all fails, we'll leave this
1797 * sitting here in a failed state, eating up a device slot. It is
1798 * unlikely this will occur.
1799 */
1800 ret = xhci_command_disable_slot(xhcip, xd->xd_slot);
1801 if (ret != USB_SUCCESS) {
1802 xhci_error(xhcip, "failed to disable slot %d: %d",
1803 xd->xd_slot, ret);
1804 return;
1805 }
1806
1807 xhci_context_slot_output_fini(xhcip, xd);
1808 xhci_endpoint_fini(xd, XHCI_DEFAULT_ENDPOINT);
1809
1810 mutex_enter(&xhcip->xhci_lock);
1811 list_remove(&xhcip->xhci_usba.xa_devices, xd);
1812 mutex_exit(&xhcip->xhci_lock);
1813
1814 xhci_hcdi_device_free(xd);
1815 }
1816
1817 /*
1818 * Synchronously attempt to set the device address. For xHCI this involves it
1819 * deciding what address to use.
1820 */
1821 static int
1822 xhci_hcdi_device_address(usba_device_t *ud)
1823 {
1824 int ret;
1825 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1826 xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1827 xhci_endpoint_t *xep;
1828
1829 mutex_enter(&xhcip->xhci_lock);
1830
1831 /*
1832 * This device may already be addressed from the perspective of the xhci
1833 * controller. For example, the device this represents may have been
1834 * unconfigured, which does not actually remove the slot or other
1835 * information, merely tears down all the active use of it and the child
1836 * driver. In such cases, if we're already addressed, just return
1837 * success. The actual USB address is a fiction for USBA anyways.
1838 */
1839 if (xd->xd_addressed == B_TRUE) {
1840 mutex_exit(&xhcip->xhci_lock);
1841 return (USB_SUCCESS);
1842 }
1843
1844 ASSERT(xd->xd_addressed == B_FALSE);
1845 xd->xd_addressed = B_TRUE;
1846 VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL);
1847 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1848 mutex_exit(&xhcip->xhci_lock);
1849
1850 if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) {
1851 ASSERT(ret == EIO);
1852 return (USB_HC_HARDWARE_ERROR);
1853 }
1854
1855 ret = xhci_command_set_address(xhcip, xd, B_FALSE);
1856
1857 if (ret != USB_SUCCESS) {
1858 mutex_enter(&xhcip->xhci_lock);
1859 xd->xd_addressed = B_FALSE;
1860 mutex_exit(&xhcip->xhci_lock);
1861 }
1862
1863 return (ret);
1864 }
1865
1866 /*
1867 * This is called relatively early on in a hub's life time. At this point, it's
1868 * descriptors have all been pulled and the default control pipe is still open.
1869 * What we need to do is go through and update the slot context to indicate that
1870 * this is a hub, otherwise, the controller will never let us speak to
1871 * downstream ports.
1872 */
1873 static int
1874 xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt)
1875 {
1876 int ret;
1877 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1878 xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1879
1880 if (xd == NULL)
1881 return (USB_FAILURE);
1882
1883 if (ud->usb_hubdi == NULL) {
1884 return (USB_FAILURE);
1885 }
1886
1887 mutex_enter(&xd->xd_imtx);
1888
1889 /*
1890 * Note, that usba never sets the interface of a hub to Multi TT. Hence
1891 * why we're never setting the MTT bit in xsc_info.
1892 */
1893 xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1));
1894 xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports));
1895 if (ud->usb_port_status == USBA_HIGH_SPEED_DEV)
1896 xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt));
1897
1898 /*
1899 * We're only updating the slot context, no endpoint contexts should be
1900 * touched.
1901 */
1902 xd->xd_input->xic_drop_flags = LE_32(0);
1903 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
1904
1905 ret = xhci_command_evaluate_context(xhcip, xd);
1906 mutex_exit(&xd->xd_imtx);
1907 return (ret);
1908 }
1909
1910 void
1911 xhci_hcd_fini(xhci_t *xhcip)
1912 {
1913 usba_hcdi_unregister(xhcip->xhci_dip);
1914 usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops);
1915 list_destroy(&xhcip->xhci_usba.xa_pipes);
1916 list_destroy(&xhcip->xhci_usba.xa_devices);
1917 }
1918
1919 int
1920 xhci_hcd_init(xhci_t *xhcip)
1921 {
1922 usba_hcdi_register_args_t hreg;
1923 usba_hcdi_ops_t *ops;
1924
1925 ops = usba_alloc_hcdi_ops();
1926 VERIFY(ops != NULL);
1927
1928 ops->usba_hcdi_ops_version = HCDI_OPS_VERSION;
1929 ops->usba_hcdi_dip = xhcip->xhci_dip;
1930
1931 ops->usba_hcdi_pm_support = xhci_hcdi_pm_support;
1932 ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open;
1933 ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close;
1934 ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset;
1935 ops->usba_hcdi_pipe_reset_data_toggle =
1936 xhci_hcdi_pipe_reset_data_toggle;
1937 ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer;
1938 ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size;
1939 ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer;
1940 ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer;
1941 ops->usba_hcdi_pipe_stop_intr_polling =
1942 xhci_hcdi_pipe_stop_intr_polling;
1943 ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer;
1944 ops->usba_hcdi_pipe_stop_isoc_polling =
1945 xhci_hcdi_pipe_stop_isoc_polling;
1946 ops->usba_hcdi_get_current_frame_number =
1947 xhci_hcdi_get_current_frame_number;
1948 ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts;
1949 ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init;
1950 ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini;
1951 ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter;
1952 ops->usba_hcdi_console_read = xhci_hcdi_console_read;
1953 ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit;
1954
1955 ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init;
1956 ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini;
1957 ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter;
1958 ops->usba_hcdi_console_write = xhci_hcdi_console_write;
1959 ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit;
1960
1961 ops->usba_hcdi_device_init = xhci_hcdi_device_init;
1962 ops->usba_hcdi_device_fini = xhci_hcdi_device_fini;
1963 ops->usba_hcdi_device_address = xhci_hcdi_device_address;
1964 ops->usba_hcdi_hub_update = xhci_hcdi_hub_update;
1965
1966 hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION;
1967 hreg.usba_hcdi_register_dip = xhcip->xhci_dip;
1968 hreg.usba_hcdi_register_ops = ops;
1969
1970 /*
1971 * We're required to give xhci a set of DMA attributes that it may loan
1972 * out to other devices. Therefore we'll be conservative with what we
1973 * end up giving it.
1974 */
1975 xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr);
1976 hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr;
1977
1978 hreg.usba_hcdi_register_iblock_cookie =
1979 (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri;
1980
1981 if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) {
1982 usba_free_hcdi_ops(ops);
1983 return (DDI_FAILURE);
1984 }
1985
1986 xhcip->xhci_usba.xa_ops = ops;
1987
1988 list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t),
1989 offsetof(xhci_device_t, xd_link));
1990 list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t),
1991 offsetof(xhci_pipe_t, xp_link));
1992
1993
1994 return (DDI_SUCCESS);
1995 }