1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2016 Joyent, Inc.
14 */
15
16 /*
17 * illumos USB framework endpoints and functions for xHCI.
18 *
19 * Please see the big theory statement in xhci.c for more information.
20 */
21
22 #include <sys/usb/hcd/xhci/xhci.h>
23 #include <sys/sysmacros.h>
24 #include <sys/strsun.h>
25 #include <sys/strsubr.h>
26
27 static xhci_t *
28 xhci_hcdi_get_xhcip_from_dev(usba_device_t *ud)
29 {
30 dev_info_t *dip = ud->usb_root_hub_dip;
31 xhci_t *xhcip = ddi_get_soft_state(xhci_soft_state,
32 ddi_get_instance(dip));
33 VERIFY(xhcip != NULL);
34 return (xhcip);
35 }
36
37 static xhci_t *
38 xhci_hcdi_get_xhcip(usba_pipe_handle_data_t *ph)
39 {
40 return (xhci_hcdi_get_xhcip_from_dev(ph->p_usba_device));
41 }
42
43 /*
44 * While the xHCI hardware is capable of supporting power management, we don't
45 * in the driver right now. Note, USBA doesn't seem to end up calling this entry
46 * point.
47 */
48 /* ARGSUSED */
49 static int
50 xhci_hcdi_pm_support(dev_info_t *dip)
51 {
52 return (USB_FAILURE);
53 }
54
55 static int
56 xhci_hcdi_pipe_open(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
57 {
58 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
59 xhci_pipe_t *pipe;
60 xhci_endpoint_t *xep;
61 xhci_device_t *xd;
62 int kmflags = usb_flags & USB_FLAGS_SLEEP ? KM_SLEEP : KM_NOSLEEP;
63 int ret;
64 uint_t epid;
65
66 mutex_enter(&xhcip->xhci_lock);
67 if (xhcip->xhci_state & XHCI_S_ERROR) {
68 mutex_exit(&xhcip->xhci_lock);
69 return (USB_HC_HARDWARE_ERROR);
70 }
71 mutex_exit(&xhcip->xhci_lock);
72
73 /*
74 * If we're here, something must be trying to open an already-opened
75 * pipe which is bad news.
76 */
77 if (ph->p_hcd_private != NULL) {
78 return (USB_FAILURE);
79 }
80
81 pipe = kmem_zalloc(sizeof (xhci_pipe_t), kmflags);
82 if (pipe == NULL) {
83 return (USB_NO_RESOURCES);
84 }
85 pipe->xp_opentime = gethrtime();
86 pipe->xp_pipe = ph;
87
88 /*
89 * If this is the root hub, there's nothing special to do on open. Just
90 * go ahead and allow it to be opened. All we have to do is add this to
91 * the list of our tracking structures for open pipes.
92 */
93 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
94 xep = NULL;
95 goto add;
96 }
97
98 /*
99 * Now that we're here, we're being asked to open up an endpoint of some
100 * kind. Because we've already handled the case of the root hub,
101 * everything should have a device.
102 */
103 epid = xhci_endpoint_pipe_to_epid(ph);
104 xd = usba_hcdi_get_device_private(ph->p_usba_device);
105 if (xd == NULL) {
106 xhci_error(xhcip, "!encountered endpoint (%d) without device "
107 "during pipe open", epid);
108 kmem_free(pipe, sizeof (xhci_pipe_t));
109 return (USB_FAILURE);
110 }
111
112 /*
113 * See if this endpoint exists or not, in general endpoints should not
114 * exist except for the default control endpoint, which we don't tear
115 * down until the device itself is cleaned up. Otherwise, a given pipe
116 * can only be open once.
117 */
118 mutex_enter(&xhcip->xhci_lock);
119 if (epid == XHCI_DEFAULT_ENDPOINT) {
120 xep = xd->xd_endpoints[epid];
121 VERIFY(xep != NULL);
122 VERIFY(xep->xep_pipe == NULL);
123 xep->xep_pipe = ph;
124 mutex_exit(&xhcip->xhci_lock);
125 ret = xhci_endpoint_update_default(xhcip, xd, xep);
126 if (ret != USB_SUCCESS) {
127 kmem_free(pipe, sizeof (xhci_pipe_t));
128 return (ret);
129 }
130 goto add;
131 }
132
133 if (xd->xd_endpoints[epid] != NULL) {
134 mutex_exit(&xhcip->xhci_lock);
135 kmem_free(pipe, sizeof (xhci_pipe_t));
136 xhci_log(xhcip, "!asked to open endpoint %d on slot %d and "
137 "port %d, but endpoint already exists", epid, xd->xd_slot,
138 xd->xd_port);
139 return (USB_FAILURE);
140 }
141
142 /*
143 * If we're opening an endpoint other than the default control endpoint,
144 * then the device should have had a USB address assigned by the
145 * controller. Sanity check that before continuing.
146 */
147 if (epid != XHCI_DEFAULT_ENDPOINT) {
148 VERIFY(xd->xd_addressed == B_TRUE);
149 }
150
151 /*
152 * Okay, at this point we need to go create and set up an endpoint.
153 * Once we're done, we'll try to install it and make sure that it
154 * doesn't conflict with something else going on.
155 */
156 ret = xhci_endpoint_init(xhcip, xd, ph);
157 if (ret != 0) {
158 mutex_exit(&xhcip->xhci_lock);
159 kmem_free(pipe, sizeof (xhci_pipe_t));
160 if (ret == EIO) {
161 xhci_error(xhcip, "failed to initialize endpoint %d "
162 "on device slot %d and port %d: encountered fatal "
163 "FM error, resetting device", epid, xd->xd_slot,
164 xd->xd_port);
165 xhci_fm_runtime_reset(xhcip);
166 }
167 return (USB_HC_HARDWARE_ERROR);
168 }
169 xep = xd->xd_endpoints[epid];
170
171 mutex_enter(&xd->xd_imtx);
172 mutex_exit(&xhcip->xhci_lock);
173
174 /*
175 * Update the slot and input context for this endpoint.
176 */
177 xd->xd_input->xic_drop_flags = LE_32(0);
178 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(epid + 1));
179
180 if (epid + 1 > XHCI_SCTX_GET_DCI(LE_32(xd->xd_slotin->xsc_info))) {
181 uint32_t info;
182
183 info = xd->xd_slotin->xsc_info;
184 info &= ~XHCI_SCTX_DCI_MASK;
185 info |= XHCI_SCTX_SET_DCI(epid + 1);
186 xd->xd_slotin->xsc_info = info;
187 }
188
189 XHCI_DMA_SYNC(xd->xd_ictx, DDI_DMA_SYNC_FORDEV);
190 if (xhci_check_dma_handle(xhcip, &xd->xd_ictx) != DDI_FM_OK) {
191 mutex_exit(&xd->xd_imtx);
192 xhci_endpoint_fini(xd, epid);
193 kmem_free(pipe, sizeof (xhci_pipe_t));
194 xhci_error(xhcip, "failed to open pipe on endpoint %d of "
195 "device with slot %d and port %d: encountered fatal FM "
196 "error syncing device input context, resetting device",
197 epid, xd->xd_slot, xd->xd_port);
198 xhci_fm_runtime_reset(xhcip);
199 return (USB_HC_HARDWARE_ERROR);
200 }
201
202 if ((ret = xhci_command_configure_endpoint(xhcip, xd)) != USB_SUCCESS) {
203 mutex_exit(&xd->xd_imtx);
204 xhci_endpoint_fini(xd, epid);
205 kmem_free(pipe, sizeof (xhci_pipe_t));
206 return (ret);
207 }
208
209 mutex_exit(&xd->xd_imtx);
210 add:
211 pipe->xp_ep = xep;
212 ph->p_hcd_private = (usb_opaque_t)pipe;
213 mutex_enter(&xhcip->xhci_lock);
214 list_insert_tail(&xhcip->xhci_usba.xa_pipes, pipe);
215 mutex_exit(&xhcip->xhci_lock);
216
217 return (USB_SUCCESS);
218 }
219
220 static void
221 xhci_hcdi_periodic_free(xhci_t *xhcip, xhci_pipe_t *xp)
222 {
223 int i;
224 xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
225
226 if (xpp->xpp_tsize == 0)
227 return;
228
229 for (i = 0; i < xpp->xpp_ntransfers; i++) {
230 if (xpp->xpp_transfers[i] == NULL)
231 continue;
232 xhci_transfer_free(xhcip, xpp->xpp_transfers[i]);
233 xpp->xpp_transfers[i] = NULL;
234 }
235
236 xpp->xpp_ntransfers = 0;
237 xpp->xpp_tsize = 0;
238 }
239
240 /*
241 * Iterate over all transfers and free everything on the pipe. Once done, update
242 * the ring to basically 'consume' everything. For periodic IN endpoints, we
243 * need to handle this somewhat differently and actually close the original
244 * request and not deallocate the related pieces as those exist for the lifetime
245 * of the endpoint and are constantly reused.
246 */
247 static void
248 xhci_hcdi_pipe_flush(xhci_t *xhcip, xhci_endpoint_t *xep, int intr_code)
249 {
250 xhci_transfer_t *xt;
251
252 ASSERT(MUTEX_HELD(&xhcip->xhci_lock));
253
254 while ((xt = list_remove_head(&xep->xep_transfers)) != NULL) {
255 if (xhci_endpoint_is_periodic_in(xep) == B_FALSE) {
256 usba_hcdi_cb(xep->xep_pipe, xt->xt_usba_req,
257 USB_CR_FLUSHED);
258 xhci_transfer_free(xhcip, xt);
259 }
260 }
261
262 if (xhci_endpoint_is_periodic_in(xep) == B_TRUE) {
263 xhci_pipe_t *xp = (xhci_pipe_t *)xep->xep_pipe->p_hcd_private;
264 xhci_periodic_pipe_t *xpp = &xp->xp_periodic;
265
266 if (xpp->xpp_usb_req != NULL) {
267 usba_hcdi_cb(xep->xep_pipe, xpp->xpp_usb_req,
268 intr_code);
269 xpp->xpp_usb_req = NULL;
270 }
271 }
272 }
273
274 /*
275 * We've been asked to terminate some set of regular I/O on an interrupt pipe.
276 * If this is for the root device, e.g. the xhci driver itself, then we remove
277 * our interrupt callback. Otherwise we stop the device for interrupt polling as
278 * follows:
279 *
280 * 1. Issue a stop endpoint command
281 * 2. Check to make sure that the endpoint stopped and reset it if needed.
282 * 3. Any thing that gets resolved can callback in the interim.
283 * 4. Ensure that nothing is scheduled on the ring
284 * 5. Skip the contents of the ring and set the TR dequeue pointer.
285 * 6. Return the original callback with a USB_CR_STOPPED_POLLING, NULL out the
286 * callback in the process.
287 */
288 static int
289 xhci_hcdi_pipe_poll_fini(usba_pipe_handle_data_t *ph, boolean_t is_close)
290 {
291 int ret;
292 uint_t epid;
293 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
294 xhci_device_t *xd;
295 xhci_endpoint_t *xep;
296 xhci_pipe_t *xp;
297 xhci_periodic_pipe_t *xpp;
298 usb_opaque_t urp;
299
300 mutex_enter(&xhcip->xhci_lock);
301 if (xhcip->xhci_state & XHCI_S_ERROR) {
302 mutex_exit(&xhcip->xhci_lock);
303 return (USB_HC_HARDWARE_ERROR);
304 }
305
306 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
307 xhci_root_hub_intr_root_disable(xhcip);
308 ret = USB_SUCCESS;
309 mutex_exit(&xhcip->xhci_lock);
310 return (ret);
311 }
312
313 xd = usba_hcdi_get_device_private(ph->p_usba_device);
314 epid = xhci_endpoint_pipe_to_epid(ph);
315 if (xd->xd_endpoints[epid] == NULL) {
316 mutex_exit(&xhcip->xhci_lock);
317 xhci_error(xhcip, "asked to stop intr polling on slot %d, "
318 "port %d, endpoint: %d, but no endpoint structure",
319 xd->xd_slot, xd->xd_port, epid);
320 return (USB_FAILURE);
321 }
322 xep = xd->xd_endpoints[epid];
323 xp = (xhci_pipe_t *)ph->p_hcd_private;
324 if (xp == NULL) {
325 mutex_exit(&xhcip->xhci_lock);
326 xhci_error(xhcip, "asked to do finish polling on slot %d, "
327 "port %d, endpoint: %d, but no pipe structure",
328 xd->xd_slot, xd->xd_port, epid);
329 return (USB_FAILURE);
330 }
331 xpp = &xp->xp_periodic;
332
333 /*
334 * Ensure that no other resets or time outs are going on right now.
335 */
336 while ((xep->xep_state & (XHCI_ENDPOINT_SERIALIZE)) != 0) {
337 cv_wait(&xep->xep_state_cv, &xhcip->xhci_lock);
338 }
339
340 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_IDLE) {
341 mutex_exit(&xhcip->xhci_lock);
342 return (USB_SUCCESS);
343 }
344
345 if (xpp->xpp_poll_state == XHCI_PERIODIC_POLL_STOPPING) {
346 mutex_exit(&xhcip->xhci_lock);
347 return (USB_FAILURE);
348 }
349
350 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_STOPPING;
351 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
352 ret = xhci_endpoint_quiesce(xhcip, xd, xep);
353 if (ret != USB_SUCCESS) {
354 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
355 "port %d, endpoint: %d, failed with %d.",
356 xd->xd_slot, xd->xd_port, epid, ret);
357 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
358 cv_broadcast(&xep->xep_state_cv);
359 mutex_exit(&xhcip->xhci_lock);
360 return (ret);
361 }
362
363 /*
364 * Okay, we've stopped this ring time to wrap it all up. Remove all the
365 * transfers, note they aren't freed like a pipe reset.
366 */
367 while (list_is_empty(&xep->xep_transfers) == 0)
368 (void) list_remove_head(&xep->xep_transfers);
369 xhci_ring_skip(&xep->xep_ring);
370 mutex_exit(&xhcip->xhci_lock);
371
372 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
373 USB_SUCCESS) {
374 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
375 "port %d, endpoint: %d, failed with %d.",
376 xd->xd_slot, xd->xd_port, epid, ret);
377 mutex_enter(&xhcip->xhci_lock);
378 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
379 cv_broadcast(&xep->xep_state_cv);
380 mutex_exit(&xhcip->xhci_lock);
381 return (ret);
382 }
383
384 mutex_enter(&xhcip->xhci_lock);
385 urp = xpp->xpp_usb_req;
386 xpp->xpp_usb_req = NULL;
387 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_IDLE;
388 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
389 mutex_exit(&xhcip->xhci_lock);
390
391 /*
392 * It's possible that with a persistent pipe, we may not actually have
393 * anything left to call back on, because we already had.
394 */
395 if (urp != NULL) {
396 usba_hcdi_cb(ph, urp, is_close == B_TRUE ?
397 USB_CR_PIPE_CLOSING : USB_CR_STOPPED_POLLING);
398 }
399
400 /*
401 * Notify anything waiting for us that we're done quiescing this device.
402 */
403 mutex_enter(&xhcip->xhci_lock);
404 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
405 cv_broadcast(&xep->xep_state_cv);
406 mutex_exit(&xhcip->xhci_lock);
407
408 return (USB_SUCCESS);
409
410 }
411
412 /*
413 * Tear down everything that we did in open. After this, the consumer of this
414 * USB device is done.
415 */
416 /* ARGSUSED */
417 static int
418 xhci_hcdi_pipe_close(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
419 {
420 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
421 xhci_pipe_t *xp;
422 xhci_device_t *xd;
423 xhci_endpoint_t *xep;
424 uint32_t info;
425 int ret, i;
426 uint_t epid;
427
428 if ((ph->p_ep.bmAttributes & USB_EP_ATTR_MASK) == USB_EP_ATTR_INTR &&
429 xhcip->xhci_usba.xa_intr_cb_ph != NULL) {
430 if ((ret = xhci_hcdi_pipe_poll_fini(ph, B_TRUE)) !=
431 USB_SUCCESS) {
432 return (ret);
433 }
434 }
435
436 mutex_enter(&xhcip->xhci_lock);
437
438 xp = (xhci_pipe_t *)ph->p_hcd_private;
439 VERIFY(xp != NULL);
440
441 /*
442 * The default endpoint is special. It is created and destroyed with the
443 * device. So like with open, closing it is just state tracking. The
444 * same is true for the root hub.
445 */
446 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR)
447 goto remove;
448
449 xd = usba_hcdi_get_device_private(ph->p_usba_device);
450 epid = xhci_endpoint_pipe_to_epid(ph);
451 if (xd->xd_endpoints[epid] == NULL) {
452 mutex_exit(&xhcip->xhci_lock);
453 xhci_error(xhcip, "asked to do close pipe on slot %d, "
454 "port %d, endpoint: %d, but no endpoint structure",
455 xd->xd_slot, xd->xd_port, epid);
456 return (USB_FAILURE);
457 }
458 xep = xd->xd_endpoints[epid];
459
460 if (xp->xp_ep != NULL && xp->xp_ep->xep_num == XHCI_DEFAULT_ENDPOINT) {
461 xep->xep_pipe = NULL;
462 goto remove;
463 }
464
465 /*
466 * We need to clean up the endpoint. So the first thing we need to do is
467 * stop it with a configure endpoint command. Once it's stopped, we can
468 * free all associated resources.
469 */
470 mutex_enter(&xd->xd_imtx);
471
472 /*
473 * Potentially update the slot input context about the current max
474 * endpoint. While we don't update the slot context with this,
475 * surrounding code expects it to be updated to be consistent.
476 */
477 xd->xd_input->xic_drop_flags = LE_32(XHCI_INCTX_MASK_DCI(epid + 1));
478 xd->xd_input->xic_add_flags = LE_32(0);
479 for (i = XHCI_NUM_ENDPOINTS - 1; i >= 0; i--) {
480 if (xd->xd_endpoints[i] != NULL &&
481 xd->xd_endpoints[i] != xep)
482 break;
483 }
484 info = xd->xd_slotin->xsc_info;
485 info &= ~XHCI_SCTX_DCI_MASK;
486 info |= XHCI_SCTX_SET_DCI(i + 1);
487 xd->xd_slotin->xsc_info = info;
488
489 /*
490 * Also zero out our context for this endpoint. Note that we don't
491 * bother with syncing DMA memory here as it's not required to be synced
492 * for this operation.
493 */
494 bzero(xd->xd_endin[xep->xep_num], sizeof (xhci_endpoint_context_t));
495
496 /*
497 * Stop the device and kill our timeout. Note, it is safe to hold the
498 * device's input mutex across the untimeout, this lock should never be
499 * referenced by the timeout code.
500 */
501 xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
502 mutex_exit(&xhcip->xhci_lock);
503 (void) untimeout(xep->xep_timeout);
504
505 ret = xhci_command_configure_endpoint(xhcip, xd);
506 mutex_exit(&xd->xd_imtx);
507 if (ret != USB_SUCCESS)
508 return (ret);
509 mutex_enter(&xhcip->xhci_lock);
510
511 /*
512 * Now that we've unconfigured the endpoint. See if we need to flush any
513 * transfers.
514 */
515 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_CLOSING);
516 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
517 xhci_hcdi_periodic_free(xhcip, xp);
518 }
519
520 xhci_endpoint_fini(xd, epid);
521
522 remove:
523 ph->p_hcd_private = NULL;
524 list_remove(&xhcip->xhci_usba.xa_pipes, xp);
525 kmem_free(xp, sizeof (xhci_pipe_t));
526
527 mutex_exit(&xhcip->xhci_lock);
528
529 return (USB_SUCCESS);
530 }
531
532 /*
533 * We've been asked to reset a pipe aka an endpoint. This endpoint may be in an
534 * arbitrary state, it may be running or it may be halted. In this case, we go
535 * through and check whether or not we know it's been halted or not. If it has
536 * not, then we stop the endpoint.
537 *
538 * Once the endpoint has been stopped, walk all transfers and go ahead and
539 * basically return them as being flushed. Then finally set the dequeue point
540 * for this endpoint.
541 */
542 /* ARGSUSED */
543 static int
544 xhci_hcdi_pipe_reset(usba_pipe_handle_data_t *ph, usb_flags_t usb_flags)
545 {
546 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
547 xhci_device_t *xd;
548 xhci_endpoint_t *xep;
549 uint_t epid;
550 int ret;
551
552 mutex_enter(&xhcip->xhci_lock);
553 if (xhcip->xhci_state & XHCI_S_ERROR) {
554 mutex_exit(&xhcip->xhci_lock);
555 return (USB_HC_HARDWARE_ERROR);
556 }
557
558 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
559 mutex_exit(&xhcip->xhci_lock);
560 return (USB_NOT_SUPPORTED);
561 }
562
563 xd = usba_hcdi_get_device_private(ph->p_usba_device);
564 epid = xhci_endpoint_pipe_to_epid(ph);
565 if (xd->xd_endpoints[epid] == NULL) {
566 mutex_exit(&xhcip->xhci_lock);
567 xhci_error(xhcip, "asked to do reset pipe on slot %d, "
568 "port %d, endpoint: %d, but no endpoint structure",
569 xd->xd_slot, xd->xd_port, epid);
570 return (USB_FAILURE);
571 }
572
573 xep = xd->xd_endpoints[epid];
574
575 /*
576 * Ensure that no other resets or time outs are going on right now.
577 */
578 while ((xep->xep_state & (XHCI_ENDPOINT_SERIALIZE)) != 0) {
579 cv_wait(&xep->xep_state_cv, &xhcip->xhci_lock);
580 }
581
582 xep->xep_state |= XHCI_ENDPOINT_QUIESCE;
583 ret = xhci_endpoint_quiesce(xhcip, xd, xep);
584 if (ret != USB_SUCCESS) {
585 /*
586 * We failed to quiesce for some reason, remove the flag and let
587 * someone else give it a shot.
588 */
589 xhci_error(xhcip, "!failed to quiesce endpoint on slot %d, "
590 "port %d, endpoint: %d, failed with %d.",
591 xd->xd_slot, xd->xd_port, epid, ret);
592 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
593 cv_broadcast(&xep->xep_state_cv);
594 mutex_exit(&xhcip->xhci_lock);
595 return (ret);
596 }
597
598 xhci_ring_skip(&xep->xep_ring);
599
600 mutex_exit(&xhcip->xhci_lock);
601 if ((ret = xhci_command_set_tr_dequeue(xhcip, xd, xep)) !=
602 USB_SUCCESS) {
603 xhci_error(xhcip, "!failed to reset endpoint ring on slot %d, "
604 "port %d, endpoint: %d, failed setting ring dequeue with "
605 "%d.", xd->xd_slot, xd->xd_port, epid, ret);
606 mutex_enter(&xhcip->xhci_lock);
607 xep->xep_state &= ~XHCI_ENDPOINT_QUIESCE;
608 cv_broadcast(&xep->xep_state_cv);
609 mutex_exit(&xhcip->xhci_lock);
610 return (ret);
611 }
612
613 mutex_enter(&xhcip->xhci_lock);
614 xhci_hcdi_pipe_flush(xhcip, xep, USB_CR_PIPE_RESET);
615
616 /*
617 * We need to remove the periodic flag as part of resetting, as if this
618 * was used for periodic activity, it no longer is and therefore can now
619 * be used for such purposes.
620 *
621 * Notify anything waiting for us that we're done quiescing this device.
622 */
623 xep->xep_state &= ~(XHCI_ENDPOINT_QUIESCE | XHCI_ENDPOINT_PERIODIC);
624 cv_broadcast(&xep->xep_state_cv);
625 mutex_exit(&xhcip->xhci_lock);
626
627 return (USB_SUCCESS);
628 }
629
630 /*
631 * We're asked to reset or change the data toggle, which is used in a few cases.
632 * However, there doesn't seem to be a good way to do this in xHCI as the data
633 * toggle isn't exposed. It seems that dropping a reset endpoint would
634 * theoretically do this; however, that can only be used when in the HALTED
635 * state. As such, for now we just return.
636 */
637 /* ARGSUSED */
638 void
639 xhci_hcdi_pipe_reset_data_toggle(usba_pipe_handle_data_t *pipe_handle)
640 {
641 }
642
643 /*
644 * We need to convert the USB request to an 8-byte little endian value. If we
645 * didn't have to think about big endian systems, this would be fine.
646 * Unfortunately, with them, this is a bit confusing. The problem is that if you
647 * think of this as a struct layout, the order that we or things together
648 * represents their byte layout. e.g. ctrl_bRequest is at offset 1 in the SETUP
649 * STAGE trb. However, when it becomes a part of a 64-bit big endian number, if
650 * ends up at byte 7, where as it needs to be at one. Hence why we do a final
651 * LE_64 at the end of this, to convert this into the byte order that it's
652 * expected to be in.
653 */
654 static uint64_t
655 xhci_hcdi_ctrl_req_to_trb(usb_ctrl_req_t *ucrp)
656 {
657 uint64_t ret = ucrp->ctrl_bmRequestType |
658 (ucrp->ctrl_bRequest << 8) |
659 ((uint64_t)LE_16(ucrp->ctrl_wValue) << 16) |
660 ((uint64_t)LE_16(ucrp->ctrl_wIndex) << 32) |
661 ((uint64_t)LE_16(ucrp->ctrl_wLength) << 48);
662 return (LE_64(ret));
663 }
664
665 /*
666 * USBA calls us in order to make a specific control type request to a device,
667 * potentially even the root hub. If the request is for the root hub, then we
668 * need to intercept this and cons up the requested data.
669 */
670 static int
671 xhci_hcdi_pipe_ctrl_xfer(usba_pipe_handle_data_t *ph, usb_ctrl_req_t *ucrp,
672 usb_flags_t usb_flags)
673 {
674 int ret, statusdir, trt;
675 uint_t ep;
676 xhci_device_t *xd;
677 xhci_endpoint_t *xep;
678 xhci_transfer_t *xt;
679 boolean_t datain;
680
681 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
682
683 mutex_enter(&xhcip->xhci_lock);
684 if (xhcip->xhci_state & XHCI_S_ERROR) {
685 mutex_exit(&xhcip->xhci_lock);
686 return (USB_HC_HARDWARE_ERROR);
687 }
688
689 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
690 ret = xhci_root_hub_ctrl_req(xhcip, ph, ucrp);
691 mutex_exit(&xhcip->xhci_lock);
692 return (ret);
693 }
694
695 /*
696 * Determine the device and endpoint.
697 */
698 xd = usba_hcdi_get_device_private(ph->p_usba_device);
699 ep = xhci_endpoint_pipe_to_epid(ph);
700 if (xd->xd_endpoints[ep] == NULL) {
701 mutex_exit(&xhcip->xhci_lock);
702 xhci_error(xhcip, "asked to do control transfer on slot %d, "
703 "port %d, endpoint: %d, but no endpoint structure",
704 xd->xd_slot, xd->xd_port, ep);
705 return (USB_FAILURE);
706 }
707 xep = xd->xd_endpoints[ep];
708
709 /*
710 * There are several types of requests that we have to handle in special
711 * ways in xHCI. If we have one of those requests, then we don't
712 * necessarily go through the normal path. These special cases are all
713 * documented in xHCI 1.1 / 4.5.4.
714 *
715 * Looking at that, you may ask why aren't SET_CONFIGURATION and SET_IF
716 * special cased here. This action is a little confusing by default. The
717 * xHCI specification requires that we may need to issue a configure
718 * endpoint command as part of this. However, the xHCI 1.1 / 4.5.4.2
719 * states that we don't actually need to if nothing in the endpoint
720 * configuration context has changed. Because nothing in it should have
721 * changed as part of this, we don't need to do anything and instead
722 * just can issue the request normally. We're also assuming in the
723 * USB_REQ_SET_IF case that if something's changing the interface, the
724 * non-default endpoint will have yet to be opened.
725 */
726 if (ucrp->ctrl_bmRequestType == USB_DEV_REQ_HOST_TO_DEV &&
727 ucrp->ctrl_bRequest == USB_REQ_SET_ADDRESS) {
728 /*
729 * As we've defined an explicit set-address endpoint, we should
730 * never call this function. If we get here, always fail.
731 */
732 mutex_exit(&xhcip->xhci_lock);
733 usba_hcdi_cb(ph, (usb_opaque_t)ucrp, USB_CR_NOT_SUPPORTED);
734 return (USB_SUCCESS);
735 }
736
737 mutex_exit(&xhcip->xhci_lock);
738
739 /*
740 * Allocate the transfer memory, etc.
741 */
742 xt = xhci_transfer_alloc(xhcip, xep, ucrp->ctrl_wLength, 2, usb_flags);
743 if (xt == NULL) {
744 return (USB_NO_RESOURCES);
745 }
746 xt->xt_usba_req = (usb_opaque_t)ucrp;
747 xt->xt_timeout = ucrp->ctrl_timeout;
748 if (xt->xt_timeout == 0) {
749 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
750 }
751
752 if (ucrp->ctrl_wLength > 0) {
753 if ((ucrp->ctrl_bmRequestType & USB_DEV_REQ_DEV_TO_HOST) != 0) {
754 trt = XHCI_TRB_TRT_IN;
755 datain = B_TRUE;
756 statusdir = 0;
757 } else {
758 trt = XHCI_TRB_TRT_OUT;
759 datain = B_FALSE;
760 statusdir = XHCI_TRB_DIR_IN;
761
762 xhci_transfer_copy(xt, ucrp->ctrl_data->b_rptr,
763 ucrp->ctrl_wLength, B_FALSE);
764 if (xhci_transfer_sync(xhcip, xt,
765 DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
766 xhci_transfer_free(xhcip, xt);
767 xhci_error(xhcip, "failed to synchronize ctrl "
768 "transfer DMA memory on endpoint %u of "
769 "device on slot %d and port %d: resetting "
770 "device", xep->xep_num, xd->xd_slot,
771 xd->xd_port);
772 xhci_fm_runtime_reset(xhcip);
773 return (USB_HC_HARDWARE_ERROR);
774 }
775 }
776 } else {
777 trt = 0;
778 datain = B_FALSE;
779 statusdir = XHCI_TRB_DIR_IN;
780 }
781
782 /*
783 * We always fill in the required setup and status TRBs ourselves;
784 * however, to minimize our knowledge about how the data has been split
785 * across multiple DMA cookies in an SGL, we leave that to the transfer
786 * logic to fill in.
787 */
788 xt->xt_trbs[0].trb_addr = xhci_hcdi_ctrl_req_to_trb(ucrp);
789 xt->xt_trbs[0].trb_status = LE_32(XHCI_TRB_LEN(8) | XHCI_TRB_INTR(0));
790 xt->xt_trbs[0].trb_flags = LE_32(trt | XHCI_TRB_IDT |
791 XHCI_TRB_TYPE_SETUP);
792
793 if (ucrp->ctrl_wLength > 0)
794 xhci_transfer_trb_fill_data(xep, xt, 1, datain);
795
796 xt->xt_trbs[xt->xt_ntrbs - 1].trb_addr = 0;
797 xt->xt_trbs[xt->xt_ntrbs - 1].trb_status = LE_32(XHCI_TRB_INTR(0));
798 xt->xt_trbs[xt->xt_ntrbs - 1].trb_flags = LE_32(XHCI_TRB_TYPE_STATUS |
799 XHCI_TRB_IOC | statusdir);
800
801 mutex_enter(&xhcip->xhci_lock);
802
803 /*
804 * Schedule the transfer, allocating resources in the process.
805 */
806 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
807 xhci_transfer_free(xhcip, xt);
808 mutex_exit(&xhcip->xhci_lock);
809 return (USB_NO_RESOURCES);
810 }
811
812 mutex_exit(&xhcip->xhci_lock);
813
814 return (USB_SUCCESS);
815 }
816
817 /*
818 * This request is trying to get the upper bound on the amount of data we're
819 * willing transfer in one go. Note that this amount can be broken down into
820 * multiple SGL entries, this interface doesn't particularly care about that.
821 */
822 /* ARGSUSED */
823 static int
824 xhci_hcdi_bulk_transfer_size(usba_device_t *ud, size_t *sizep)
825 {
826 if (sizep != NULL)
827 *sizep = XHCI_MAX_TRANSFER;
828 return (USB_SUCCESS);
829 }
830
831 /*
832 * Perform a bulk transfer. This is a pretty straightforward action. We
833 * basically just allocate the appropriate transfer and try to schedule it,
834 * hoping there is enough space.
835 */
836 static int
837 xhci_hcdi_pipe_bulk_xfer(usba_pipe_handle_data_t *ph, usb_bulk_req_t *ubrp,
838 usb_flags_t usb_flags)
839 {
840 uint_t epid;
841 xhci_device_t *xd;
842 xhci_endpoint_t *xep;
843 xhci_transfer_t *xt;
844 boolean_t datain;
845
846 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
847
848 mutex_enter(&xhcip->xhci_lock);
849 if (xhcip->xhci_state & XHCI_S_ERROR) {
850 mutex_exit(&xhcip->xhci_lock);
851 return (USB_HC_HARDWARE_ERROR);
852 }
853
854 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
855 mutex_exit(&xhcip->xhci_lock);
856 return (USB_NOT_SUPPORTED);
857 }
858
859 xd = usba_hcdi_get_device_private(ph->p_usba_device);
860 epid = xhci_endpoint_pipe_to_epid(ph);
861 if (xd->xd_endpoints[epid] == NULL) {
862 mutex_exit(&xhcip->xhci_lock);
863 xhci_error(xhcip, "asked to do control transfer on slot %d, "
864 "port %d, endpoint: %d, but no endpoint structure",
865 xd->xd_slot, xd->xd_port, epid);
866 return (USB_FAILURE);
867 }
868 xep = xd->xd_endpoints[epid];
869 mutex_exit(&xhcip->xhci_lock);
870
871 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
872 datain = B_TRUE;
873 } else {
874 datain = B_FALSE;
875 }
876
877 xt = xhci_transfer_alloc(xhcip, xep, ubrp->bulk_len, 0, usb_flags);
878 if (xt == NULL) {
879 return (USB_NO_RESOURCES);
880 }
881 xt->xt_usba_req = (usb_opaque_t)ubrp;
882 xt->xt_timeout = ubrp->bulk_timeout;
883 if (xt->xt_timeout == 0) {
884 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
885 }
886
887 if (ubrp->bulk_len > 0 && datain == B_FALSE) {
888 xhci_transfer_copy(xt, ubrp->bulk_data->b_rptr, ubrp->bulk_len,
889 B_FALSE);
890 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
891 DDI_FM_OK) {
892 xhci_transfer_free(xhcip, xt);
893 xhci_error(xhcip, "failed to synchronize bulk "
894 "transfer DMA memory on endpoint %u of "
895 "device on slot %d and port %d: resetting "
896 "device", xep->xep_num, xd->xd_slot,
897 xd->xd_port);
898 xhci_fm_runtime_reset(xhcip);
899 return (USB_HC_HARDWARE_ERROR);
900 }
901 }
902
903 xhci_transfer_trb_fill_data(xep, xt, 0, datain);
904 mutex_enter(&xhcip->xhci_lock);
905 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
906 xhci_transfer_free(xhcip, xt);
907 mutex_exit(&xhcip->xhci_lock);
908 return (USB_NO_RESOURCES);
909 }
910 mutex_exit(&xhcip->xhci_lock);
911
912 return (USB_SUCCESS);
913 }
914
915 static void
916 xhci_hcdi_isoc_transfer_fill(xhci_device_t *xd, xhci_endpoint_t *xep,
917 xhci_transfer_t *xt, usb_isoc_req_t *usrp)
918 {
919 int i;
920 uintptr_t buf;
921
922 buf = xt->xt_buffer.xdb_cookies[0].dmac_laddress;
923 for (i = 0; i < usrp->isoc_pkts_count; i++) {
924 int flags;
925 uint_t tbc, tlbpc;
926
927 ushort_t len = usrp->isoc_pkt_descr[i].isoc_pkt_length;
928 xhci_trb_t *trb = &xt->xt_trbs[i];
929
930 trb->trb_addr = LE_64(buf);
931
932 /*
933 * Beacuse we know that a single frame can have all of its data
934 * in a single instance, we know that we don't neeed to do
935 * anything special here.
936 */
937 trb->trb_status = LE_32(XHCI_TRB_LEN(len) | XHCI_TRB_TDREM(0) |
938 XHCI_TRB_INTR(0));
939
940 /*
941 * Always enable SIA to start the frame ASAP. We also always
942 * enable an interrupt on a short packet. If this is the last
943 * trb, then we will set IOC.
944 */
945 flags = XHCI_TRB_SIA | XHCI_TRB_ISP | XHCI_TRB_SET_FRAME(0);
946 flags |= XHCI_TRB_TYPE_ISOCH;
947
948 if (i + 1 == usrp->isoc_pkts_count)
949 flags |= XHCI_TRB_IOC;
950
951 /*
952 * Now we need to calculate the TBC and the TLBPC.
953 */
954 xhci_transfer_calculate_isoc(xd, xep, len, &tbc, &tlbpc);
955 flags |= XHCI_TRB_SET_TBC(tbc);
956 flags |= XHCI_TRB_SET_TLBPC(tlbpc);
957
958 trb->trb_flags = LE_32(flags);
959 buf += len;
960
961 /*
962 * Go through and copy the required data to our local copy of
963 * the isoc descriptor. By default, we assume that all data will
964 * be copied and the status set to OK. This mirrors the fact
965 * that we won't get a notification unless there's been an
966 * error or short packet transfer.
967 */
968 xt->xt_isoc[i].isoc_pkt_length = len;
969 xt->xt_isoc[i].isoc_pkt_actual_length = len;
970 xt->xt_isoc[i].isoc_pkt_status = USB_CR_OK;
971 }
972 }
973
974 /*
975 * Initialize periodic IN requests (both interrupt and isochronous)
976 */
977 static int
978 xhci_hcdi_periodic_init(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
979 usb_opaque_t usb_req, size_t len, int usb_flags)
980 {
981 int i, ret;
982 uint_t epid;
983 xhci_device_t *xd;
984 xhci_endpoint_t *xep;
985 xhci_pipe_t *xp;
986 xhci_periodic_pipe_t *xpp;
987
988 mutex_enter(&xhcip->xhci_lock);
989 if (xhcip->xhci_state & XHCI_S_ERROR) {
990 mutex_exit(&xhcip->xhci_lock);
991 return (USB_HC_HARDWARE_ERROR);
992 }
993
994 xd = usba_hcdi_get_device_private(ph->p_usba_device);
995 epid = xhci_endpoint_pipe_to_epid(ph);
996 if (xd->xd_endpoints[epid] == NULL) {
997 xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
998 "port %d, endpoint: %d, but no endpoint structure",
999 xd->xd_slot, xd->xd_port, epid);
1000 mutex_exit(&xhcip->xhci_lock);
1001 return (USB_FAILURE);
1002 }
1003 xep = xd->xd_endpoints[epid];
1004 xp = (xhci_pipe_t *)ph->p_hcd_private;
1005 if (xp == NULL) {
1006 xhci_error(xhcip, "asked to do periodic transfer on slot %d, "
1007 "port %d, endpoint: %d, but no pipe structure",
1008 xd->xd_slot, xd->xd_port, epid);
1009 mutex_exit(&xhcip->xhci_lock);
1010 return (USB_FAILURE);
1011 }
1012 xpp = &xp->xp_periodic;
1013
1014 /*
1015 * Only allow a single polling request at any given time.
1016 */
1017 if (xpp->xpp_usb_req != NULL) {
1018 mutex_exit(&xhcip->xhci_lock);
1019 return (USB_BUSY);
1020 }
1021
1022 /*
1023 * We keep allocations around in case we restart polling, which most
1024 * devices do (not really caring about a lost event). However, we don't
1025 * support a driver changing that size on us, which it probably won't.
1026 * If we stumble across driver that does, then this will need to become
1027 * a lot more complicated.
1028 */
1029 if (xpp->xpp_tsize > 0 && xpp->xpp_tsize < len) {
1030 mutex_exit(&xhcip->xhci_lock);
1031 return (USB_INVALID_REQUEST);
1032 }
1033
1034 if (xpp->xpp_tsize == 0) {
1035 int ntrbs;
1036 int ntransfers;
1037
1038 /*
1039 * What we allocate varies based on whether or not this is an
1040 * isochronous or interrupt IN periodic.
1041 */
1042 if (xep->xep_type == USB_EP_ATTR_INTR) {
1043 ntrbs = 0;
1044 ntransfers = XHCI_INTR_IN_NTRANSFERS;
1045 } else {
1046 usb_isoc_req_t *usrp;
1047 ASSERT(xep->xep_type == USB_EP_ATTR_ISOCH);
1048
1049 usrp = (usb_isoc_req_t *)usb_req;
1050 ntrbs = usrp->isoc_pkts_count;
1051 ntransfers = XHCI_ISOC_IN_NTRANSFERS;
1052 }
1053
1054 xpp->xpp_tsize = len;
1055 xpp->xpp_ntransfers = ntransfers;
1056
1057 for (i = 0; i < xpp->xpp_ntransfers; i++) {
1058 xhci_transfer_t *xt = xhci_transfer_alloc(xhcip, xep,
1059 len, ntrbs, usb_flags);
1060 if (xt == NULL) {
1061 xhci_hcdi_periodic_free(xhcip, xp);
1062 mutex_exit(&xhcip->xhci_lock);
1063 return (USB_NO_RESOURCES);
1064 }
1065
1066 if (xep->xep_type == USB_EP_ATTR_INTR) {
1067 xhci_transfer_trb_fill_data(xep, xt, 0, B_TRUE);
1068 } else {
1069 usb_isoc_req_t *usrp;
1070 usrp = (usb_isoc_req_t *)usb_req;
1071 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1072 xt->xt_data_tohost = B_TRUE;
1073 }
1074 xpp->xpp_transfers[i] = xt;
1075 }
1076 }
1077
1078 /*
1079 * Mark the endpoint as periodic so we don't have timeouts at play.
1080 */
1081 xep->xep_state |= XHCI_ENDPOINT_PERIODIC;
1082
1083 /*
1084 * Now that we've allocated everything, go ahead and schedule them and
1085 * kick off the ring.
1086 */
1087 for (i = 0; i < xpp->xpp_ntransfers; i++) {
1088 int ret;
1089 ret = xhci_endpoint_schedule(xhcip, xd, xep,
1090 xpp->xpp_transfers[i], B_FALSE);
1091 if (ret != 0) {
1092 (void) xhci_ring_reset(xhcip, &xep->xep_ring);
1093 xep->xep_state &= ~XHCI_ENDPOINT_PERIODIC;
1094 mutex_exit(&xhcip->xhci_lock);
1095 return (ret);
1096 }
1097 }
1098
1099 /*
1100 * Don't worry about freeing memory, it'll be done when the endpoint
1101 * closes and the whole system is reset.
1102 */
1103 xpp->xpp_usb_req = usb_req;
1104 xpp->xpp_poll_state = XHCI_PERIODIC_POLL_ACTIVE;
1105
1106 ret = xhci_endpoint_ring(xhcip, xd, xep);
1107 mutex_exit(&xhcip->xhci_lock);
1108 return (ret);
1109 }
1110
1111 static int
1112 xhci_hcdi_intr_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1113 usb_intr_req_t *uirp, usb_flags_t usb_flags)
1114 {
1115 uint_t epid;
1116 xhci_device_t *xd;
1117 xhci_endpoint_t *xep;
1118 xhci_transfer_t *xt;
1119 boolean_t datain;
1120 mblk_t *mp = NULL;
1121
1122 mutex_enter(&xhcip->xhci_lock);
1123 if (xhcip->xhci_state & XHCI_S_ERROR) {
1124 mutex_exit(&xhcip->xhci_lock);
1125 return (USB_HC_HARDWARE_ERROR);
1126 }
1127
1128 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1129 epid = xhci_endpoint_pipe_to_epid(ph);
1130 if (xd->xd_endpoints[epid] == NULL) {
1131 xhci_error(xhcip, "asked to do interrupt transfer on slot %d, "
1132 "port %d, endpoint: %d, but no endpoint structure",
1133 xd->xd_slot, xd->xd_port, epid);
1134 mutex_exit(&xhcip->xhci_lock);
1135 return (USB_FAILURE);
1136 }
1137 xep = xd->xd_endpoints[epid];
1138
1139 mutex_exit(&xhcip->xhci_lock);
1140
1141 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1142 datain = B_TRUE;
1143 } else {
1144 datain = B_FALSE;
1145 }
1146
1147 xt = xhci_transfer_alloc(xhcip, xep, uirp->intr_len, 0, usb_flags);
1148 if (xt == NULL) {
1149 return (USB_NO_RESOURCES);
1150 }
1151
1152 xt->xt_usba_req = (usb_opaque_t)uirp;
1153 xt->xt_timeout = uirp->intr_timeout;
1154 if (xt->xt_timeout == 0) {
1155 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1156 }
1157
1158 /*
1159 * Unlike other request types, USB Interrupt-IN requests aren't required
1160 * to have allocated the message block for data. If they haven't, we
1161 * take care of that now.
1162 */
1163 if (uirp->intr_len > 0 && datain == B_TRUE && uirp->intr_data == NULL) {
1164 if (usb_flags & USB_FLAGS_SLEEP) {
1165 mp = allocb_wait(uirp->intr_len, BPRI_LO, STR_NOSIG,
1166 NULL);
1167 } else {
1168 mp = allocb(uirp->intr_len, 0);
1169 }
1170 if (mp == NULL) {
1171 xhci_transfer_free(xhcip, xt);
1172 mutex_exit(&xhcip->xhci_lock);
1173 return (USB_NO_RESOURCES);
1174 }
1175 uirp->intr_data = mp;
1176 }
1177
1178 if (uirp->intr_len > 0 && datain == B_FALSE) {
1179 xhci_transfer_copy(xt, uirp->intr_data->b_rptr, uirp->intr_len,
1180 B_FALSE);
1181 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) !=
1182 DDI_FM_OK) {
1183 xhci_transfer_free(xhcip, xt);
1184 xhci_error(xhcip, "failed to synchronize interrupt "
1185 "transfer DMA memory on endpoint %u of "
1186 "device on slot %d and port %d: resetting "
1187 "device", xep->xep_num, xd->xd_slot,
1188 xd->xd_port);
1189 xhci_fm_runtime_reset(xhcip);
1190 return (USB_HC_HARDWARE_ERROR);
1191 }
1192 }
1193
1194 xhci_transfer_trb_fill_data(xep, xt, 0, datain);
1195 mutex_enter(&xhcip->xhci_lock);
1196 if (xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE) != 0) {
1197 if (mp != NULL) {
1198 uirp->intr_data = NULL;
1199 freemsg(mp);
1200 }
1201 xhci_transfer_free(xhcip, xt);
1202 mutex_exit(&xhcip->xhci_lock);
1203 return (USB_NO_RESOURCES);
1204 }
1205 mutex_exit(&xhcip->xhci_lock);
1206
1207 return (USB_SUCCESS);
1208 }
1209
1210 /*
1211 * We've been asked to perform an interrupt transfer. When this is an interrupt
1212 * IN endpoint, that means that the hcd is being asked to start polling on the
1213 * endpoint. When the endpoint is the root hub, it effectively becomes synthetic
1214 * polling.
1215 *
1216 * When we have an interrupt out endpoint, then this is just a single simple
1217 * interrupt request that we send out and there isn't much special to do beyond
1218 * the normal activity.
1219 */
1220 static int
1221 xhci_hcdi_pipe_intr_xfer(usba_pipe_handle_data_t *ph, usb_intr_req_t *uirp,
1222 usb_flags_t usb_flags)
1223 {
1224 int ret;
1225 xhci_t *xhcip = xhci_hcdi_get_xhcip(ph);
1226
1227 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1228 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1229 ret = xhci_root_hub_intr_root_enable(xhcip, ph, uirp);
1230 } else if (uirp->intr_attributes & USB_ATTRS_ONE_XFER) {
1231 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp,
1232 usb_flags);
1233 } else {
1234 ret = xhci_hcdi_periodic_init(xhcip, ph,
1235 (usb_opaque_t)uirp, uirp->intr_len, usb_flags);
1236 }
1237 } else {
1238 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1239 return (USB_NOT_SUPPORTED);
1240 }
1241 ret = xhci_hcdi_intr_oneshot(xhcip, ph, uirp, usb_flags);
1242 }
1243
1244 return (ret);
1245 }
1246
1247 /* ARGSUSED */
1248 static int
1249 xhci_hcdi_pipe_stop_intr_polling(usba_pipe_handle_data_t *ph,
1250 usb_flags_t usb_flags)
1251 {
1252 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1253 }
1254
1255 static int
1256 xhci_hcdi_isoc_periodic(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1257 usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1258 {
1259 int i;
1260 size_t count;
1261
1262 count = 0;
1263 for (i = 0; i < usrp->isoc_pkts_count; i++) {
1264 count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1265 }
1266
1267 return (xhci_hcdi_periodic_init(xhcip, ph, (usb_opaque_t)usrp, count,
1268 usb_flags));
1269 }
1270
1271 /*
1272 * This is used to create an isochronous request to send data out to the device.
1273 * This is a single one shot request, it is not something that we'll have to
1274 * repeat over and over.
1275 */
1276 static int
1277 xhci_hcdi_isoc_oneshot(xhci_t *xhcip, usba_pipe_handle_data_t *ph,
1278 usb_isoc_req_t *usrp, usb_flags_t usb_flags)
1279 {
1280 int i, ret;
1281 uint_t epid;
1282 size_t count, mblen;
1283 xhci_device_t *xd;
1284 xhci_endpoint_t *xep;
1285 xhci_transfer_t *xt;
1286
1287 count = 0;
1288 for (i = 0; i < usrp->isoc_pkts_count; i++) {
1289 count += usrp->isoc_pkt_descr[i].isoc_pkt_length;
1290 }
1291 mblen = MBLKL(usrp->isoc_data);
1292
1293 if (count != mblen) {
1294 return (USB_INVALID_ARGS);
1295 }
1296
1297 mutex_enter(&xhcip->xhci_lock);
1298 if (xhcip->xhci_state & XHCI_S_ERROR) {
1299 mutex_exit(&xhcip->xhci_lock);
1300 return (USB_HC_HARDWARE_ERROR);
1301 }
1302
1303 xd = usba_hcdi_get_device_private(ph->p_usba_device);
1304 epid = xhci_endpoint_pipe_to_epid(ph);
1305 if (xd->xd_endpoints[epid] == NULL) {
1306 xhci_error(xhcip, "asked to do isochronous transfer on slot "
1307 "%d, port %d, endpoint: %d, but no endpoint structure",
1308 xd->xd_slot, xd->xd_port, epid);
1309 mutex_exit(&xhcip->xhci_lock);
1310 return (USB_FAILURE);
1311 }
1312 xep = xd->xd_endpoints[epid];
1313 mutex_exit(&xhcip->xhci_lock);
1314
1315 xt = xhci_transfer_alloc(xhcip, xep, mblen, usrp->isoc_pkts_count,
1316 usb_flags);
1317 if (xt == NULL) {
1318 return (USB_NO_RESOURCES);
1319 }
1320 xt->xt_usba_req = (usb_opaque_t)usrp;
1321
1322 /*
1323 * USBA doesn't provide any real way for a timeout to be defined for an
1324 * isochronous event. However, since we technically aren't a periodic
1325 * endpoint, go ahead and always set the default timeout. It's better
1326 * than nothing.
1327 */
1328 xt->xt_timeout = HCDI_DEFAULT_TIMEOUT;
1329
1330 xhci_transfer_copy(xt, usrp->isoc_data->b_rptr, mblen, B_FALSE);
1331 if (xhci_transfer_sync(xhcip, xt, DDI_DMA_SYNC_FORDEV) != DDI_FM_OK) {
1332 xhci_transfer_free(xhcip, xt);
1333 xhci_error(xhcip, "failed to synchronize isochronous "
1334 "transfer DMA memory on endpoint %u of "
1335 "device on slot %d and port %d: resetting "
1336 "device", xep->xep_num, xd->xd_slot,
1337 xd->xd_port);
1338 xhci_fm_runtime_reset(xhcip);
1339 return (USB_HC_HARDWARE_ERROR);
1340 }
1341
1342 /*
1343 * Fill in the ISOC data. Note, that we always use ASAP scheduling and
1344 * we don't support specifying the frame at this time, for better or
1345 * worse.
1346 */
1347 xhci_hcdi_isoc_transfer_fill(xd, xep, xt, usrp);
1348
1349 mutex_enter(&xhcip->xhci_lock);
1350 ret = xhci_endpoint_schedule(xhcip, xd, xep, xt, B_TRUE);
1351 mutex_exit(&xhcip->xhci_lock);
1352
1353 return (ret);
1354 }
1355
1356 static int
1357 xhci_hcdi_pipe_isoc_xfer(usba_pipe_handle_data_t *ph, usb_isoc_req_t *usrp,
1358 usb_flags_t usb_flags)
1359 {
1360 int ret;
1361 xhci_t *xhcip;
1362
1363 xhcip = xhci_hcdi_get_xhcip(ph);
1364
1365 /*
1366 * We don't support isochronous transactions on the root hub at all.
1367 * Always fail them if for some reason we end up here.
1368 */
1369 if (ph->p_usba_device->usb_addr == ROOT_HUB_ADDR) {
1370 return (USB_NOT_SUPPORTED);
1371 }
1372
1373 /*
1374 * We do not support being asked to set the frame ID at this time. We
1375 * require that everything specify the attribute
1376 * USB_ATTRS_ISOC_XFER_ASAP.
1377 */
1378 if (!(usrp->isoc_attributes & USB_ATTRS_ISOC_XFER_ASAP)) {
1379 return (USB_NOT_SUPPORTED);
1380 }
1381
1382 if ((ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK) == USB_EP_DIR_IN) {
1383 /*
1384 * Note, there is no such thing as a non-periodic isochronous
1385 * incoming transfer.
1386 */
1387 ret = xhci_hcdi_isoc_periodic(xhcip, ph, usrp, usb_flags);
1388 } else {
1389 ret = xhci_hcdi_isoc_oneshot(xhcip, ph, usrp, usb_flags);
1390 }
1391
1392 return (ret);
1393 }
1394
1395 /* ARGSUSED */
1396 static int
1397 xhci_hcdi_pipe_stop_isoc_polling(usba_pipe_handle_data_t *ph,
1398 usb_flags_t usb_flags)
1399 {
1400 return (xhci_hcdi_pipe_poll_fini(ph, B_FALSE));
1401 }
1402
1403 /*
1404 * This is asking us for the current frame number. The USBA expects this to
1405 * actually be a bit of a fiction, as it tries to maintain a frame number well
1406 * beyond what the hardware actually contains in its registers. Hardware
1407 * basically has a 14-bit counter, whereas we need to have a constant amount of
1408 * milliseconds.
1409 *
1410 * Today, no client drivers actually use this API and everyone specifies the
1411 * attribute to say that we should schedule things ASAP. So until we have some
1412 * real device that want this functionality, we're going to fail.
1413 */
1414 /* ARGSUSED */
1415 static int
1416 xhci_hcdi_get_current_frame_number(usba_device_t *usba_device,
1417 usb_frame_number_t *frame_number)
1418 {
1419 return (USB_FAILURE);
1420 }
1421
1422 /*
1423 * See the comments around the XHCI_ISOC_MAX_TRB macro for more information.
1424 */
1425 /* ARGSUSED */
1426 static int
1427 xhci_hcdi_get_max_isoc_pkts(usba_device_t *usba_device,
1428 uint_t *max_isoc_pkts_per_request)
1429 {
1430 *max_isoc_pkts_per_request = XHCI_ISOC_MAX_TRB;
1431 return (USB_SUCCESS);
1432 }
1433
1434 /*
1435 * The next series of routines is used for both the OBP console and general USB
1436 * console polled I/O. In general, we opt not to support any of that at this
1437 * time in xHCI. As we have the need of that, we can start plumbing that
1438 * through.
1439 */
1440 /* ARGSUSED */
1441 static int
1442 xhci_hcdi_console_input_init(usba_pipe_handle_data_t *pipe_handle,
1443 uchar_t **obp_buf, usb_console_info_impl_t *console_input_info)
1444 {
1445 return (USB_NOT_SUPPORTED);
1446 }
1447
1448 /* ARGSUSED */
1449 static int
1450 xhci_hcdi_console_input_fini(usb_console_info_impl_t *console_input_info)
1451 {
1452 return (USB_NOT_SUPPORTED);
1453 }
1454
1455 /* ARGSUSED */
1456 static int
1457 xhci_hcdi_console_input_enter(usb_console_info_impl_t *console_input_info)
1458 {
1459 return (USB_NOT_SUPPORTED);
1460 }
1461
1462 /* ARGSUSED */
1463 static int
1464 xhci_hcdi_console_read(usb_console_info_impl_t *console_input_info,
1465 uint_t *num_characters)
1466 {
1467 return (USB_NOT_SUPPORTED);
1468 }
1469
1470 /* ARGSUSED */
1471 static int
1472 xhci_hcdi_console_input_exit(usb_console_info_impl_t *console_input_info)
1473 {
1474 return (USB_NOT_SUPPORTED);
1475 }
1476
1477 /* ARGSUSED */
1478 static int
1479 xhci_hcdi_console_output_init(usba_pipe_handle_data_t *pipe_handle,
1480 usb_console_info_impl_t *console_output_info)
1481 {
1482 return (USB_NOT_SUPPORTED);
1483 }
1484
1485 /* ARGSUSED */
1486 static int
1487 xhci_hcdi_console_output_fini(usb_console_info_impl_t *console_output_info)
1488 {
1489 return (USB_NOT_SUPPORTED);
1490 }
1491
1492 /* ARGSUSED */
1493 static int
1494 xhci_hcdi_console_output_enter(usb_console_info_impl_t *console_output_info)
1495 {
1496 return (USB_NOT_SUPPORTED);
1497 }
1498
1499 /* ARGSUSED */
1500 static int
1501 xhci_hcdi_console_write(usb_console_info_impl_t *console_output_info,
1502 uchar_t *buf, uint_t num_characters, uint_t *num_characters_written)
1503 {
1504 return (USB_NOT_SUPPORTED);
1505 }
1506
1507 /* ARGSUSED */
1508 static int
1509 xhci_hcdi_console_output_exit(usb_console_info_impl_t *console_output_info)
1510 {
1511 return (USB_NOT_SUPPORTED);
1512 }
1513
1514 /*
1515 * VERSION 2 ops and helpers
1516 */
1517
1518 static void
1519 xhci_hcdi_device_free(xhci_device_t *xd)
1520 {
1521 xhci_dma_free(&xd->xd_ictx);
1522 xhci_dma_free(&xd->xd_octx);
1523 mutex_destroy(&xd->xd_imtx);
1524 kmem_free(xd, sizeof (xhci_device_t));
1525 }
1526
1527 /*
1528 * Calculate the device's route string. In USB 3.0 the route string is a 20-bit
1529 * number. Each four bits represent a port number attached to a deeper hub.
1530 * Particularly it represents the port on that current hub that you need to go
1531 * down to reach the next device. Bits 0-3 represent the first *external* hub.
1532 * So a device connected to a root hub has a route string of zero. Imagine the
1533 * following set of devices:
1534 *
1535 * . port 2 . port 5
1536 * . .
1537 * +----------+ . +--------+ . +-------+
1538 * | root hub |-*->| hub 1 |-*->| hub 2 |
1539 * +----------+ +--------+ +-------+
1540 * * . port 12 * . port 8 * . port 1
1541 * v v v
1542 * +-------+ +-------+ +-------+
1543 * | dev a | | dev b | | dev c |
1544 * +-------+ +-------+ +-------+
1545 *
1546 * So, based on the above diagram, device a should have a route string of 0,
1547 * because it's directly connected to the root port. Device b would simply have
1548 * a route string of 8. This is because it travels through one non-root hub, hub
1549 * 1, and it does so on port 8. The root ports basically don't matter. Device c
1550 * would then have a route string of 0x15, as it's first traversing through hub
1551 * 1 on port 2 and then hub 2 on port 5.
1552 *
1553 * Finally, it's worth mentioning that because it's a four bit field, if for
1554 * some reason a device has more than 15 ports, we just treat the value as 15.
1555 *
1556 * Note, as part of this, we also grab what port on the root hub this whole
1557 * chain is on, as we're required to store that information in the slot context.
1558 */
1559 static void
1560 xhci_hcdi_device_route(usba_device_t *ud, uint32_t *routep, uint32_t *root_port)
1561 {
1562 uint32_t route = 0;
1563 usba_device_t *hub = ud->usb_parent_hub;
1564 usba_device_t *port_dev = ud;
1565
1566 ASSERT(hub != NULL);
1567
1568 /*
1569 * Iterate over every hub, updating the route as we go. When we
1570 * encounter a hub without a parent, then we're at the root hub. At
1571 * which point, the port we want is on port_dev (the child of hub).
1572 */
1573 while (hub->usb_parent_hub != NULL) {
1574 uint32_t p;
1575
1576 p = port_dev->usb_port;
1577 if (p > 15)
1578 p = 15;
1579 route <<= 4;
1580 route |= p & 0xf;
1581 port_dev = hub;
1582 hub = hub->usb_parent_hub;
1583 }
1584
1585 ASSERT(port_dev->usb_parent_hub == hub);
1586 *root_port = port_dev->usb_port;
1587 *routep = XHCI_ROUTE_MASK(route);
1588 }
1589
1590 /*
1591 * If a low or full speed device is behind a high-speed device that is not a
1592 * root hub, then we must include the port and slot of that device. USBA already
1593 * stores this device in the usb_hs_hub_usba_dev member.
1594 */
1595 static uint32_t
1596 xhci_hcdi_device_tt(usba_device_t *ud)
1597 {
1598 uint32_t ret;
1599 xhci_device_t *xd;
1600
1601 if (ud->usb_port_status >= USBA_HIGH_SPEED_DEV)
1602 return (0);
1603
1604 if (ud->usb_hs_hub_usba_dev == NULL)
1605 return (0);
1606
1607 ASSERT(ud->usb_hs_hub_usba_dev != NULL);
1608 ASSERT(ud->usb_hs_hub_usba_dev->usb_parent_hub != NULL);
1609 xd = usba_hcdi_get_device_private(ud->usb_hs_hub_usba_dev);
1610 ASSERT(xd != NULL);
1611
1612 ret = XHCI_SCTX_SET_TT_HUB_SID(xd->xd_slot);
1613 ret |= XHCI_SCTX_SET_TT_PORT_NUM(ud->usb_hs_hub_usba_dev->usb_port);
1614
1615 return (ret);
1616 }
1617
1618 /*
1619 * Initialize a new device. This allocates a device slot from the controller,
1620 * which tranfers it to our control.
1621 */
1622 static int
1623 xhci_hcdi_device_init(usba_device_t *ud, usb_port_t port, void **hcdpp)
1624 {
1625 int ret, i;
1626 xhci_device_t *xd;
1627 ddi_device_acc_attr_t acc;
1628 ddi_dma_attr_t attr;
1629 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1630 size_t isize, osize, incr;
1631 uint32_t route, rp, info, info2, tt;
1632
1633 xd = kmem_zalloc(sizeof (xhci_device_t), KM_SLEEP);
1634 xd->xd_port = port;
1635 xd->xd_usbdev = ud;
1636 mutex_init(&xd->xd_imtx, NULL, MUTEX_DRIVER,
1637 (void *)(uintptr_t)xhcip->xhci_intr_pri);
1638
1639 /*
1640 * The size of the context structures is based upon the presence of the
1641 * context flag which determines whether we have a 32-byte or 64-byte
1642 * context. Note that the input context always has to account for the
1643 * entire size of the xhci_input_contex_t, which is 32-bytes by default.
1644 */
1645 if (xhcip->xhci_caps.xcap_flags & XCAP_CSZ) {
1646 incr = 64;
1647 osize = XHCI_DEVICE_CONTEXT_64;
1648 isize = XHCI_DEVICE_CONTEXT_64 + incr;
1649 } else {
1650 incr = 32;
1651 osize = XHCI_DEVICE_CONTEXT_32;
1652 isize = XHCI_DEVICE_CONTEXT_32 + incr;
1653 }
1654
1655 xhci_dma_acc_attr(xhcip, &acc);
1656 xhci_dma_dma_attr(xhcip, &attr);
1657 if (xhci_dma_alloc(xhcip, &xd->xd_ictx, &attr, &acc, B_TRUE,
1658 isize, B_FALSE) == B_FALSE) {
1659 xhci_hcdi_device_free(xd);
1660 return (USB_NO_RESOURCES);
1661 }
1662
1663 xd->xd_input = (xhci_input_context_t *)xd->xd_ictx.xdb_va;
1664 xd->xd_slotin = (xhci_slot_context_t *)(xd->xd_ictx.xdb_va + incr);
1665 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1666 xd->xd_endin[i] =
1667 (xhci_endpoint_context_t *)(xd->xd_ictx.xdb_va +
1668 (i + 2) * incr);
1669 }
1670
1671 if (xhci_dma_alloc(xhcip, &xd->xd_octx, &attr, &acc, B_TRUE,
1672 osize, B_FALSE) == B_FALSE) {
1673 xhci_hcdi_device_free(xd);
1674 return (USB_NO_RESOURCES);
1675 }
1676 xd->xd_slotout = (xhci_slot_context_t *)xd->xd_octx.xdb_va;
1677 for (i = 0; i < XHCI_NUM_ENDPOINTS; i++) {
1678 xd->xd_endout[i] =
1679 (xhci_endpoint_context_t *)(xd->xd_octx.xdb_va +
1680 (i + 1) * incr);
1681 }
1682
1683 ret = xhci_command_enable_slot(xhcip, &xd->xd_slot);
1684 if (ret != USB_SUCCESS) {
1685 xhci_hcdi_device_free(xd);
1686 return (ret);
1687 }
1688
1689 /*
1690 * These are the default slot context and the endpoint zero context that
1691 * we're enabling. See 4.3.3.
1692 */
1693 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0) |
1694 XHCI_INCTX_MASK_DCI(1));
1695
1696 /*
1697 * Note, we never need to set the MTT bit as illumos never enables the
1698 * alternate MTT interface.
1699 */
1700 xhci_hcdi_device_route(ud, &route, &rp);
1701 info = XHCI_SCTX_SET_ROUTE(route) | XHCI_SCTX_SET_DCI(1);
1702 switch (ud->usb_port_status) {
1703 case USBA_LOW_SPEED_DEV:
1704 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_LOW);
1705 break;
1706 case USBA_HIGH_SPEED_DEV:
1707 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_HIGH);
1708 break;
1709 case USBA_FULL_SPEED_DEV:
1710 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_FULL);
1711 break;
1712 case USBA_SUPER_SPEED_DEV:
1713 default:
1714 info |= XHCI_SCTX_SET_SPEED(XHCI_SPEED_SUPER);
1715 break;
1716 }
1717 info2 = XHCI_SCTX_SET_RHPORT(rp);
1718 tt = XHCI_SCTX_SET_IRQ_TARGET(0);
1719 tt |= xhci_hcdi_device_tt(ud);
1720
1721 xd->xd_slotin->xsc_info = LE_32(info);
1722 xd->xd_slotin->xsc_info2 = LE_32(info2);
1723 xd->xd_slotin->xsc_tt = LE_32(tt);
1724
1725 if ((ret = xhci_endpoint_init(xhcip, xd, NULL)) != 0) {
1726 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1727 xhci_hcdi_device_free(xd);
1728 return (USB_HC_HARDWARE_ERROR);
1729 }
1730
1731 if (xhci_context_slot_output_init(xhcip, xd) != B_TRUE) {
1732 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1733 xhci_endpoint_fini(xd, 0);
1734 xhci_hcdi_device_free(xd);
1735 return (USB_HC_HARDWARE_ERROR);
1736 }
1737
1738 if ((ret = xhci_command_set_address(xhcip, xd, B_TRUE)) != 0) {
1739 (void) xhci_command_disable_slot(xhcip, xd->xd_slot);
1740 xhci_context_slot_output_fini(xhcip, xd);
1741 xhci_endpoint_fini(xd, 0);
1742 xhci_hcdi_device_free(xd);
1743 return (ret);
1744 }
1745
1746 mutex_enter(&xhcip->xhci_lock);
1747 list_insert_tail(&xhcip->xhci_usba.xa_devices, xd);
1748 mutex_exit(&xhcip->xhci_lock);
1749
1750 *hcdpp = xd;
1751 return (ret);
1752 }
1753
1754 /*
1755 * We're tearing down a device now. That means that the only endpoint context
1756 * that's still valid would be endpoint zero.
1757 */
1758 static void
1759 xhci_hcdi_device_fini(usba_device_t *ud, void *hcdp)
1760 {
1761 int ret;
1762 xhci_endpoint_t *xep;
1763 xhci_device_t *xd;
1764 xhci_t *xhcip;
1765
1766 /*
1767 * Right now, it's theoretically possible that USBA may try and call
1768 * us here even if we hadn't successfully finished the device_init()
1769 * endpoint. We should probably modify the USBA to make sure that this
1770 * can't happen.
1771 */
1772 if (hcdp == NULL)
1773 return;
1774
1775 xd = hcdp;
1776 xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1777
1778 /*
1779 * Make sure we have no timeout running on the default endpoint still.
1780 */
1781 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1782 mutex_enter(&xhcip->xhci_lock);
1783 xep->xep_state |= XHCI_ENDPOINT_TEARDOWN;
1784 mutex_exit(&xhcip->xhci_lock);
1785 (void) untimeout(xep->xep_timeout);
1786
1787 /*
1788 * Go ahead and disable the slot. There's no reason to do anything
1789 * special about the default endpoint as it will be disabled as a part
1790 * of the slot disabling. However, if this all fails, we'll leave this
1791 * sitting here in a failed state, eating up a device slot. It is
1792 * unlikely this will occur.
1793 */
1794 ret = xhci_command_disable_slot(xhcip, xd->xd_slot);
1795 if (ret != USB_SUCCESS) {
1796 xhci_error(xhcip, "failed to disable slot %d: %d",
1797 xd->xd_slot, ret);
1798 return;
1799 }
1800
1801 xhci_context_slot_output_fini(xhcip, xd);
1802 xhci_endpoint_fini(xd, XHCI_DEFAULT_ENDPOINT);
1803
1804 mutex_enter(&xhcip->xhci_lock);
1805 list_remove(&xhcip->xhci_usba.xa_devices, xd);
1806 mutex_exit(&xhcip->xhci_lock);
1807
1808 xhci_hcdi_device_free(xd);
1809 }
1810
1811 /*
1812 * Synchronously attempt to set the device address. For xHCI this involves it
1813 * deciding what address to use.
1814 */
1815 static int
1816 xhci_hcdi_device_address(usba_device_t *ud)
1817 {
1818 int ret;
1819 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1820 xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1821 xhci_endpoint_t *xep;
1822
1823 mutex_enter(&xhcip->xhci_lock);
1824
1825 /*
1826 * This device may already be addressed from the perspective of the xhci
1827 * controller. For example, the device this represents may have been
1828 * unconfigured, which does not actually remove the slot or other
1829 * information, merely tears down all the active use of it and the child
1830 * driver. In such cases, if we're already addressed, just return
1831 * success. The actual USB address is a fiction for USBA anyways.
1832 */
1833 if (xd->xd_addressed == B_TRUE) {
1834 mutex_exit(&xhcip->xhci_lock);
1835 return (USB_SUCCESS);
1836 }
1837
1838 ASSERT(xd->xd_addressed == B_FALSE);
1839 xd->xd_addressed = B_TRUE;
1840 VERIFY3P(xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT], !=, NULL);
1841 xep = xd->xd_endpoints[XHCI_DEFAULT_ENDPOINT];
1842 mutex_exit(&xhcip->xhci_lock);
1843
1844 if ((ret = xhci_endpoint_setup_default_context(xhcip, xd, xep)) != 0) {
1845 ASSERT(ret == EIO);
1846 return (USB_HC_HARDWARE_ERROR);
1847 }
1848
1849 ret = xhci_command_set_address(xhcip, xd, B_FALSE);
1850
1851 if (ret != USB_SUCCESS) {
1852 mutex_enter(&xhcip->xhci_lock);
1853 xd->xd_addressed = B_FALSE;
1854 mutex_exit(&xhcip->xhci_lock);
1855 }
1856
1857 return (ret);
1858 }
1859
1860 /*
1861 * This is called relatively early on in a hub's life time. At this point, it's
1862 * descriptors have all been pulled and the default control pipe is still open.
1863 * What we need to do is go through and update the slot context to indicate that
1864 * this is a hub, otherwise, the controller will never let us speak to
1865 * downstream ports.
1866 */
1867 static int
1868 xhci_hcdi_hub_update(usba_device_t *ud, uint8_t nports, uint8_t tt)
1869 {
1870 int ret;
1871 xhci_t *xhcip = xhci_hcdi_get_xhcip_from_dev(ud);
1872 xhci_device_t *xd = usba_hcdi_get_device_private(ud);
1873
1874 if (xd == NULL)
1875 return (USB_FAILURE);
1876
1877 if (ud->usb_hubdi == NULL) {
1878 return (USB_FAILURE);
1879 }
1880
1881 mutex_enter(&xd->xd_imtx);
1882
1883 /*
1884 * Note, that usba never sets the interface of a hub to Multi TT. Hence
1885 * why we're never setting the MTT bit in xsc_info.
1886 */
1887 xd->xd_slotin->xsc_info |= LE_32(XHCI_SCTX_SET_HUB(1));
1888 xd->xd_slotin->xsc_info2 |= LE_32(XHCI_SCTX_SET_NPORTS(nports));
1889 if (ud->usb_port_status == USBA_HIGH_SPEED_DEV)
1890 xd->xd_slotin->xsc_tt |= LE_32(XHCI_SCTX_SET_TT_THINK_TIME(tt));
1891
1892 /*
1893 * We're only updating the slot context, no endpoint contexts should be
1894 * touched.
1895 */
1896 xd->xd_input->xic_drop_flags = LE_32(0);
1897 xd->xd_input->xic_add_flags = LE_32(XHCI_INCTX_MASK_DCI(0));
1898
1899 ret = xhci_command_evaluate_context(xhcip, xd);
1900 mutex_exit(&xd->xd_imtx);
1901 return (ret);
1902 }
1903
1904 void
1905 xhci_hcd_fini(xhci_t *xhcip)
1906 {
1907 usba_hcdi_unregister(xhcip->xhci_dip);
1908 usba_free_hcdi_ops(xhcip->xhci_usba.xa_ops);
1909 list_destroy(&xhcip->xhci_usba.xa_pipes);
1910 list_destroy(&xhcip->xhci_usba.xa_devices);
1911 }
1912
1913 int
1914 xhci_hcd_init(xhci_t *xhcip)
1915 {
1916 usba_hcdi_register_args_t hreg;
1917 usba_hcdi_ops_t *ops;
1918
1919 ops = usba_alloc_hcdi_ops();
1920 VERIFY(ops != NULL);
1921
1922 ops->usba_hcdi_ops_version = HCDI_OPS_VERSION;
1923 ops->usba_hcdi_dip = xhcip->xhci_dip;
1924
1925 ops->usba_hcdi_pm_support = xhci_hcdi_pm_support;
1926 ops->usba_hcdi_pipe_open = xhci_hcdi_pipe_open;
1927 ops->usba_hcdi_pipe_close = xhci_hcdi_pipe_close;
1928 ops->usba_hcdi_pipe_reset = xhci_hcdi_pipe_reset;
1929 ops->usba_hcdi_pipe_reset_data_toggle =
1930 xhci_hcdi_pipe_reset_data_toggle;
1931 ops->usba_hcdi_pipe_ctrl_xfer = xhci_hcdi_pipe_ctrl_xfer;
1932 ops->usba_hcdi_bulk_transfer_size = xhci_hcdi_bulk_transfer_size;
1933 ops->usba_hcdi_pipe_bulk_xfer = xhci_hcdi_pipe_bulk_xfer;
1934 ops->usba_hcdi_pipe_intr_xfer = xhci_hcdi_pipe_intr_xfer;
1935 ops->usba_hcdi_pipe_stop_intr_polling =
1936 xhci_hcdi_pipe_stop_intr_polling;
1937 ops->usba_hcdi_pipe_isoc_xfer = xhci_hcdi_pipe_isoc_xfer;
1938 ops->usba_hcdi_pipe_stop_isoc_polling =
1939 xhci_hcdi_pipe_stop_isoc_polling;
1940 ops->usba_hcdi_get_current_frame_number =
1941 xhci_hcdi_get_current_frame_number;
1942 ops->usba_hcdi_get_max_isoc_pkts = xhci_hcdi_get_max_isoc_pkts;
1943 ops->usba_hcdi_console_input_init = xhci_hcdi_console_input_init;
1944 ops->usba_hcdi_console_input_fini = xhci_hcdi_console_input_fini;
1945 ops->usba_hcdi_console_input_enter = xhci_hcdi_console_input_enter;
1946 ops->usba_hcdi_console_read = xhci_hcdi_console_read;
1947 ops->usba_hcdi_console_input_exit = xhci_hcdi_console_input_exit;
1948
1949 ops->usba_hcdi_console_output_init = xhci_hcdi_console_output_init;
1950 ops->usba_hcdi_console_output_fini = xhci_hcdi_console_output_fini;
1951 ops->usba_hcdi_console_output_enter = xhci_hcdi_console_output_enter;
1952 ops->usba_hcdi_console_write = xhci_hcdi_console_write;
1953 ops->usba_hcdi_console_output_exit = xhci_hcdi_console_output_exit;
1954
1955 ops->usba_hcdi_device_init = xhci_hcdi_device_init;
1956 ops->usba_hcdi_device_fini = xhci_hcdi_device_fini;
1957 ops->usba_hcdi_device_address = xhci_hcdi_device_address;
1958 ops->usba_hcdi_hub_update = xhci_hcdi_hub_update;
1959
1960 hreg.usba_hcdi_register_version = HCDI_REGISTER_VERSION;
1961 hreg.usba_hcdi_register_dip = xhcip->xhci_dip;
1962 hreg.usba_hcdi_register_ops = ops;
1963
1964 /*
1965 * We're required to give xhci a set of DMA attributes that it may loan
1966 * out to other devices. Therefore we'll be conservative with what we
1967 * end up giving it.
1968 */
1969 xhci_dma_dma_attr(xhcip, &xhcip->xhci_usba.xa_dma_attr);
1970 hreg.usba_hcdi_register_dma_attr = &xhcip->xhci_usba.xa_dma_attr;
1971
1972 hreg.usba_hcdi_register_iblock_cookie =
1973 (ddi_iblock_cookie_t)(uintptr_t)xhcip->xhci_intr_pri;
1974
1975 if (usba_hcdi_register(&hreg, 0) != DDI_SUCCESS) {
1976 usba_free_hcdi_ops(ops);
1977 return (DDI_FAILURE);
1978 }
1979
1980 xhcip->xhci_usba.xa_ops = ops;
1981
1982 list_create(&xhcip->xhci_usba.xa_devices, sizeof (xhci_device_t),
1983 offsetof(xhci_device_t, xd_link));
1984 list_create(&xhcip->xhci_usba.xa_pipes, sizeof (xhci_pipe_t),
1985 offsetof(xhci_pipe_t, xp_link));
1986
1987
1988 return (DDI_SUCCESS);
1989 }