1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2016 Joyent, Inc.
14 */
15
16 /*
17 * xHCI DMA Management Routines
18 *
19 * Please see the big theory statement in xhci.c for more information.
20 */
21
22 #include <sys/usb/hcd/xhci/xhci.h>
23
24 int
25 xhci_check_dma_handle(xhci_t *xhcip, xhci_dma_buffer_t *xdb)
26 {
27 ddi_fm_error_t de;
28
29 if (!DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps))
30 return (0);
31
32 ddi_fm_dma_err_get(xdb->xdb_dma_handle, &de, DDI_FME_VERSION);
33 return (de.fme_status);
251 xdb->xdb_len = size;
252 xdb->xdb_ncookies = ncookies;
253 xdb->xdb_cookies[0] = cookie;
254 for (i = 1; i < ncookies; i++) {
255 ddi_dma_nextcookie(xdb->xdb_dma_handle, &xdb->xdb_cookies[i]);
256 }
257
258
259 return (B_TRUE);
260 }
261
262 void
263 xhci_transfer_free(xhci_t *xhcip, xhci_transfer_t *xt)
264 {
265 if (xt == NULL)
266 return;
267
268 VERIFY(xhcip != NULL);
269 xhci_dma_free(&xt->xt_buffer);
270 if (xt->xt_isoc != NULL) {
271 ASSERT(xt->xt_ntrbs > 0);
272 kmem_free(xt->xt_isoc, sizeof (usb_isoc_pkt_descr_t) *
273 xt->xt_ntrbs);
274 xt->xt_isoc = NULL;
275 }
276 if (xt->xt_trbs != NULL) {
277 ASSERT(xt->xt_ntrbs > 0);
278 kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * xt->xt_ntrbs);
279 xt->xt_trbs = NULL;
280 }
281 kmem_free(xt, sizeof (xhci_transfer_t));
282 }
283
284 xhci_transfer_t *
285 xhci_transfer_alloc(xhci_t *xhcip, xhci_endpoint_t *xep, size_t size, int trbs,
286 int usb_flags)
287 {
288 int kmflags;
289 boolean_t dmawait;
290 xhci_transfer_t *xt;
291 ddi_device_acc_attr_t acc;
292 ddi_dma_attr_t attr;
293
294 if (usb_flags & USB_FLAGS_SLEEP) {
295 kmflags = KM_SLEEP;
296 dmawait = B_TRUE;
297 } else {
298 kmflags = KM_NOSLEEP;
299 dmawait = B_FALSE;
300 }
301
302 xt = kmem_zalloc(sizeof (xhci_transfer_t), kmflags);
303 if (xt == NULL)
304 return (NULL);
305
306 if (size != 0) {
307 int sgl = XHCI_DEF_DMA_SGL;
308
309 /*
310 * For BULK transfers, we always increase the number of SGL
311 * entries that we support to make things easier for the kernel.
312 * However, for control transfers, we currently opt to keep
313 * things a bit simpler and use our default of one SGL. There's
314 * no good technical reason for this, rather it just keeps
315 * things a bit easier.
316 *
317 * To simplify things, we don't use additional SGL entries for
318 * ISOC transfers. While this isn't the best, it isn't too far
319 * off from what ehci and co. have done before. If this becomes
320 * a technical issue, it's certainly possible to increase the
321 * SGL entry count.
322 */
323 if (xep->xep_type == USB_EP_ATTR_BULK)
324 sgl = XHCI_TRANSFER_DMA_SGL;
325
326 xhci_dma_acc_attr(xhcip, &acc);
327 xhci_dma_transfer_attr(xhcip, &attr, sgl);
328 if (xhci_dma_alloc(xhcip, &xt->xt_buffer, &attr, &acc, B_FALSE,
329 size, dmawait) == B_FALSE) {
330 kmem_free(xt, sizeof (xhci_transfer_t));
331 return (NULL);
332 }
333
334 /*
335 * ISOC transfers are a bit special and don't need additional
336 * TRBs for data.
337 */
338 if (xep->xep_type != USB_EP_ATTR_ISOCH)
339 trbs += xt->xt_buffer.xdb_ncookies;
340 }
341
342 xt->xt_trbs = kmem_zalloc(sizeof (xhci_trb_t) * trbs, kmflags);
343 if (xt->xt_trbs == NULL) {
344 xhci_dma_free(&xt->xt_buffer);
345 kmem_free(xt, sizeof (xhci_transfer_t));
346 return (NULL);
347 }
348
349 /*
350 * For ISOCH transfers, we need to also allocate the results data.
351 */
352 if (xep->xep_type == USB_EP_ATTR_ISOCH) {
353 xt->xt_isoc = kmem_zalloc(sizeof (usb_isoc_pkt_descr_t) * trbs,
354 kmflags);
355 if (xt->xt_isoc == NULL) {
356 kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs);
357 xhci_dma_free(&xt->xt_buffer);
358 kmem_free(xt, sizeof (xhci_transfer_t));
359 return (NULL);
360 }
361 }
362
363 xt->xt_ntrbs = trbs;
364 xt->xt_cr = USB_CR_OK;
365
366 return (xt);
367 }
368
369 /*
370 * Abstract the notion of copying out to handle the case of multiple DMA
371 * cookies. If tobuf is true, we are copying to the kernel provided buffer,
372 * otherwise we're copying into the DMA memory.
373 */
374 void
375 xhci_transfer_copy(xhci_transfer_t *xt, void *buf, size_t len,
390 }
391
392 /*
393 * We're required to try and inform the xHCI controller about the number of data
394 * packets that are required. The algorithm to use is described in xHCI 1.1 /
395 * 4.11.2.4. While it might be tempting to just try and calculate the number of
396 * packets based on simple rounding of the remaining number of bytes, that
397 * misses a critical problem -- DMA boundaries may cause us to need additional
398 * packets that are missed initially. Consider a transfer made up of four
399 * different DMA buffers sized in bytes: 4096, 4096, 256, 256, with a 512 byte
400 * packet size.
401 *
402 * Remain 4608 512 256 0
403 * Bytes 4096 4096 256 256
404 * Naive TD 9 1 1 0
405 * Act TD 10 2 1 0
406 *
407 * This means that the only safe way forward here is to work backwards and see
408 * how many we need to work up to this point.
409 */
410 static int
411 xhci_transfer_get_tdsize(xhci_transfer_t *xt, uint_t off, uint_t mps)
412 {
413 int i;
414 uint_t npkt = 0;
415
416 /*
417 * There are always zero packets for the last TRB.
418 */
419 ASSERT(xt->xt_buffer.xdb_ncookies > 0);
420 for (i = xt->xt_buffer.xdb_ncookies - 1; i > off; i--) {
421 size_t len;
422
423 /*
424 * The maximum value we can return is 31 packets. So, in that
425 * case we short-circuit and return.
426 */
427 if (npkt >= 31)
428 return (31);
429
430 len = roundup(xt->xt_buffer.xdb_cookies[i].dmac_size, mps);
431 npkt += len / mps;
432 }
433
434 return (npkt);
435 }
436
437 void
438 xhci_transfer_trb_fill_data(xhci_endpoint_t *xep, xhci_transfer_t *xt, int off,
439 boolean_t in)
440 {
441 uint_t mps, tdsize, flags;
442 int i;
443
444 VERIFY(xt->xt_buffer.xdb_ncookies > 0);
445 VERIFY(xep->xep_pipe != NULL);
446 VERIFY(off + xt->xt_buffer.xdb_ncookies <= xt->xt_ntrbs);
447 mps = xep->xep_pipe->p_ep.wMaxPacketSize;
448
449 for (i = 0; i < xt->xt_buffer.xdb_ncookies; i++) {
450 uint64_t pa, dmasz;
451
452 pa = xt->xt_buffer.xdb_cookies[i].dmac_laddress;
453 dmasz = xt->xt_buffer.xdb_cookies[i].dmac_size;
454
455 tdsize = xhci_transfer_get_tdsize(xt, i, mps);
456
457 flags = XHCI_TRB_TYPE_NORMAL;
458 if (i == 0 && xep->xep_type == USB_EP_ATTR_CONTROL) {
459 flags = XHCI_TRB_TYPE_DATA;
460 if (in == B_TRUE)
461 flags |= XHCI_TRB_DIR_IN;
462 }
463
464 /*
465 * When reading data in (from the device), we may get shorter
466 * transfers than the buffer allowed for. To make sure we get
467 * notified about that and handle that, we need to set the ISP
468 * flag.
469 */
470 if (in == B_TRUE) {
471 flags |= XHCI_TRB_ISP;
472 xt->xt_data_tohost = B_TRUE;
473 }
474
475 /*
476 * When we have more than one cookie, we are technically
477 * chaining together things according to the controllers view,
478 * hence why we need to set the chain flag.
479 */
480 if (xt->xt_buffer.xdb_ncookies > 1 &&
481 i != (xt->xt_buffer.xdb_ncookies - 1)) {
482 flags |= XHCI_TRB_CHAIN;
483 }
484
485 /*
486 * If we have a non-control transfer, then we need to make sure
487 * that we set ourselves up to be interrupted, which we set for
488 * the last entry.
489 */
490 if (i + 1 == xt->xt_buffer.xdb_ncookies &&
491 xep->xep_type != USB_EP_ATTR_CONTROL) {
492 flags |= XHCI_TRB_IOC;
493 }
494
495 xt->xt_trbs[off + i].trb_addr = LE_64(pa);
496 xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_LEN(dmasz) |
497 XHCI_TRB_TDREM(tdsize) | XHCI_TRB_INTR(0));
498 xt->xt_trbs[off + i].trb_flags = LE_32(flags);
499 }
500 }
501
502 /*
503 * These are utility functions for isochronus transfers to help calculate the
504 * transfer burst count (TBC) and transfer last burst packet count (TLPBC)
505 * entries for an isochronus entry. See xHCI 1.1 / 4.11.2.3 for how to calculate
506 * them.
507 */
508 void
509 xhci_transfer_calculate_isoc(xhci_device_t *xd, xhci_endpoint_t *xep,
510 uint_t trb_len, uint_t *tbc, uint_t *tlbpc)
511 {
512 uint_t mps, tdpc, burst;
513
514 /*
515 * Even if we're asked to send no data, that actually requires the
516 * equivalent of sending one byte of data.
517 */
518 if (trb_len == 0)
519 trb_len = 1;
|
1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright (c) 2018, Joyent, Inc.
14 */
15
16 /*
17 * xHCI DMA Management Routines
18 *
19 * Please see the big theory statement in xhci.c for more information.
20 */
21
22 #include <sys/usb/hcd/xhci/xhci.h>
23
24 int
25 xhci_check_dma_handle(xhci_t *xhcip, xhci_dma_buffer_t *xdb)
26 {
27 ddi_fm_error_t de;
28
29 if (!DDI_FM_DMA_ERR_CAP(xhcip->xhci_fm_caps))
30 return (0);
31
32 ddi_fm_dma_err_get(xdb->xdb_dma_handle, &de, DDI_FME_VERSION);
33 return (de.fme_status);
251 xdb->xdb_len = size;
252 xdb->xdb_ncookies = ncookies;
253 xdb->xdb_cookies[0] = cookie;
254 for (i = 1; i < ncookies; i++) {
255 ddi_dma_nextcookie(xdb->xdb_dma_handle, &xdb->xdb_cookies[i]);
256 }
257
258
259 return (B_TRUE);
260 }
261
262 void
263 xhci_transfer_free(xhci_t *xhcip, xhci_transfer_t *xt)
264 {
265 if (xt == NULL)
266 return;
267
268 VERIFY(xhcip != NULL);
269 xhci_dma_free(&xt->xt_buffer);
270 if (xt->xt_isoc != NULL) {
271 ASSERT3U(xt->xt_ntrbs, >, 0);
272 kmem_free(xt->xt_isoc, sizeof (usb_isoc_pkt_descr_t) *
273 xt->xt_ntrbs);
274 xt->xt_isoc = NULL;
275 }
276 if (xt->xt_trbs != NULL) {
277 ASSERT3U(xt->xt_ntrbs, >, 0);
278 kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * xt->xt_ntrbs);
279 xt->xt_trbs = NULL;
280 }
281 if (xt->xt_trbs_pa != NULL) {
282 ASSERT3U(xt->xt_ntrbs, >, 0);
283 kmem_free(xt->xt_trbs_pa, sizeof (uint64_t) * xt->xt_ntrbs);
284 xt->xt_trbs_pa = NULL;
285 }
286 kmem_free(xt, sizeof (xhci_transfer_t));
287 }
288
289 xhci_transfer_t *
290 xhci_transfer_alloc(xhci_t *xhcip, xhci_endpoint_t *xep, size_t size,
291 uint_t trbs, int usb_flags)
292 {
293 int kmflags;
294 boolean_t dmawait;
295 xhci_transfer_t *xt;
296 ddi_device_acc_attr_t acc;
297 ddi_dma_attr_t attr;
298
299 if (usb_flags & USB_FLAGS_SLEEP) {
300 kmflags = KM_SLEEP;
301 dmawait = B_TRUE;
302 } else {
303 kmflags = KM_NOSLEEP;
304 dmawait = B_FALSE;
305 }
306
307 xt = kmem_zalloc(sizeof (xhci_transfer_t), kmflags);
308 if (xt == NULL)
309 return (NULL);
310
311 if (size != 0) {
312 int sgl = XHCI_DEF_DMA_SGL;
313
314 /*
315 * For BULK transfers, we always increase the number of SGL
316 * entries that we support to make things easier for the kernel.
317 * However, for control transfers, we currently opt to keep
318 * things a bit simpler and use our default of one SGL. There's
319 * no good technical reason for this, rather it just keeps
320 * things a bit easier.
321 *
322 * To simplify things, we don't use additional SGL entries for
323 * ISOC transfers. While this isn't the best, it isn't too far
324 * off from what ehci and co. have done before. If this becomes
325 * a technical issue, it's certainly possible to increase the
326 * SGL entry count.
327 *
328 * When we use the larger SGL count, we change our strategy for
329 * being notified. In such a case we will opt to use an event
330 * data packet. This helps deal with cases where some
331 * controllers don't properly generate events for the last entry
332 * in a TD with IOC when IOSP is set.
333 */
334 if (xep->xep_type == USB_EP_ATTR_BULK) {
335 sgl = XHCI_TRANSFER_DMA_SGL;
336 trbs++;
337 }
338
339 xhci_dma_acc_attr(xhcip, &acc);
340 xhci_dma_transfer_attr(xhcip, &attr, sgl);
341 if (xhci_dma_alloc(xhcip, &xt->xt_buffer, &attr, &acc, B_FALSE,
342 size, dmawait) == B_FALSE) {
343 kmem_free(xt, sizeof (xhci_transfer_t));
344 return (NULL);
345 }
346
347 /*
348 * ISOC transfers are a bit special and don't need additional
349 * TRBs for data.
350 */
351 if (xep->xep_type != USB_EP_ATTR_ISOCH)
352 trbs += xt->xt_buffer.xdb_ncookies;
353 }
354
355 xt->xt_trbs = kmem_zalloc(sizeof (xhci_trb_t) * trbs, kmflags);
356 if (xt->xt_trbs == NULL) {
357 xhci_dma_free(&xt->xt_buffer);
358 kmem_free(xt, sizeof (xhci_transfer_t));
359 return (NULL);
360 }
361
362 xt->xt_trbs_pa = kmem_zalloc(sizeof (uint64_t) * trbs, kmflags);
363 if (xt->xt_trbs_pa == NULL) {
364 kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs);
365 xhci_dma_free(&xt->xt_buffer);
366 kmem_free(xt, sizeof (xhci_transfer_t));
367 return (NULL);
368 }
369
370 /*
371 * For ISOCH transfers, we need to also allocate the results data.
372 */
373 if (xep->xep_type == USB_EP_ATTR_ISOCH) {
374 xt->xt_isoc = kmem_zalloc(sizeof (usb_isoc_pkt_descr_t) * trbs,
375 kmflags);
376 if (xt->xt_isoc == NULL) {
377 kmem_free(xt->xt_trbs_pa, sizeof (uint64_t) * trbs);
378 kmem_free(xt->xt_trbs, sizeof (xhci_trb_t) * trbs);
379 xhci_dma_free(&xt->xt_buffer);
380 kmem_free(xt, sizeof (xhci_transfer_t));
381 return (NULL);
382 }
383 }
384
385 xt->xt_ntrbs = trbs;
386 xt->xt_cr = USB_CR_OK;
387
388 return (xt);
389 }
390
391 /*
392 * Abstract the notion of copying out to handle the case of multiple DMA
393 * cookies. If tobuf is true, we are copying to the kernel provided buffer,
394 * otherwise we're copying into the DMA memory.
395 */
396 void
397 xhci_transfer_copy(xhci_transfer_t *xt, void *buf, size_t len,
412 }
413
414 /*
415 * We're required to try and inform the xHCI controller about the number of data
416 * packets that are required. The algorithm to use is described in xHCI 1.1 /
417 * 4.11.2.4. While it might be tempting to just try and calculate the number of
418 * packets based on simple rounding of the remaining number of bytes, that
419 * misses a critical problem -- DMA boundaries may cause us to need additional
420 * packets that are missed initially. Consider a transfer made up of four
421 * different DMA buffers sized in bytes: 4096, 4096, 256, 256, with a 512 byte
422 * packet size.
423 *
424 * Remain 4608 512 256 0
425 * Bytes 4096 4096 256 256
426 * Naive TD 9 1 1 0
427 * Act TD 10 2 1 0
428 *
429 * This means that the only safe way forward here is to work backwards and see
430 * how many we need to work up to this point.
431 */
432 static uint_t
433 xhci_transfer_get_tdsize(xhci_transfer_t *xt, uint_t off, uint_t mps)
434 {
435 int i;
436 uint_t npkt = 0;
437
438 /*
439 * There are always zero packets for the last TRB.
440 */
441 ASSERT(xt->xt_buffer.xdb_ncookies > 0);
442 for (i = xt->xt_buffer.xdb_ncookies - 1; i > off; i--) {
443 size_t len = roundup(xt->xt_buffer.xdb_cookies[i].dmac_size,
444 mps);
445 npkt += len / mps;
446 }
447
448 /*
449 * Make sure to clamp this value otherwise we risk truncation.
450 */
451 if (npkt >= XHCI_MAX_TDSIZE)
452 return (XHCI_MAX_TDSIZE);
453
454 return (npkt);
455 }
456
457 void
458 xhci_transfer_trb_fill_data(xhci_endpoint_t *xep, xhci_transfer_t *xt, int off,
459 boolean_t in)
460 {
461 uint_t mps, tdsize, flags;
462 int i;
463
464 VERIFY(xt->xt_buffer.xdb_ncookies > 0);
465 VERIFY(xep->xep_pipe != NULL);
466 VERIFY(off + xt->xt_buffer.xdb_ncookies <= xt->xt_ntrbs);
467 mps = xep->xep_pipe->p_ep.wMaxPacketSize;
468
469 if (in == B_TRUE) {
470 xt->xt_data_tohost = B_TRUE;
471 }
472
473 /*
474 * We assume that if we have a non-bulk endpoint, then we should only
475 * have a single cookie. This falls out from the default SGL length that
476 * we use for these other device types.
477 */
478 if (xep->xep_type != USB_EP_ATTR_BULK) {
479 VERIFY3U(xt->xt_buffer.xdb_ncookies, ==, 1);
480 }
481
482 for (i = 0; i < xt->xt_buffer.xdb_ncookies; i++) {
483 uint64_t pa, dmasz;
484
485 pa = xt->xt_buffer.xdb_cookies[i].dmac_laddress;
486 dmasz = xt->xt_buffer.xdb_cookies[i].dmac_size;
487
488 tdsize = xhci_transfer_get_tdsize(xt, i, mps);
489
490 flags = XHCI_TRB_TYPE_NORMAL;
491 if (i == 0 && xep->xep_type == USB_EP_ATTR_CONTROL) {
492 flags = XHCI_TRB_TYPE_DATA;
493 if (in == B_TRUE)
494 flags |= XHCI_TRB_DIR_IN;
495 }
496
497 /*
498 * If we have more than one cookie, then we need to set chaining
499 * on every TRB and the last TRB will turn into an event data
500 * TRB. If we only have a single TRB, then we just set interrupt
501 * on completion (IOC). There's no need to specifically set
502 * interrupt on short packet (IOSP) in that case, as we'll
503 * always get the event notification. We still need the chain
504 * bit set on the last packet, so we can chain into the event
505 * data. Even if all the data on a bulk endpoint (the only
506 * endpoint type that uses chaining today) has only one cookie,
507 * then we'll still schedule an event data block.
508 */
509 if (xep->xep_type == USB_EP_ATTR_BULK ||
510 xt->xt_buffer.xdb_ncookies > 1) {
511 flags |= XHCI_TRB_CHAIN;
512 }
513
514 /*
515 * What we set for the last TRB depends on the type of the
516 * endpoint. If it's a bulk endpoint, then we have to set
517 * evaluate next trb (ENT) so we successfully process the event
518 * data TRB we'll set up. Otherwise, we need to make sure that
519 * we set interrupt on completion, so we get the event. However,
520 * we don't set the event on control endpoints, as the status
521 * stage TD will be the one where we get the event. But, we do
522 * still need an interrupt on short packet, because technically
523 * the status stage is in its own TD.
524 */
525 if (i + 1 == xt->xt_buffer.xdb_ncookies) {
526 switch (xep->xep_type) {
527 case USB_EP_ATTR_BULK:
528 flags |= XHCI_TRB_ENT;
529 break;
530 case USB_EP_ATTR_CONTROL:
531 flags |= XHCI_TRB_ISP;
532 break;
533 default:
534 flags |= XHCI_TRB_IOC;
535 break;
536 }
537 }
538
539 xt->xt_trbs[off + i].trb_addr = LE_64(pa);
540 xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_LEN(dmasz) |
541 XHCI_TRB_TDREM(tdsize) | XHCI_TRB_INTR(0));
542 xt->xt_trbs[off + i].trb_flags = LE_32(flags);
543 }
544
545 /*
546 * The last TRB in any bulk transfer is the Event Data TRB.
547 */
548 if (xep->xep_type == USB_EP_ATTR_BULK) {
549 VERIFY(off + xt->xt_buffer.xdb_ncookies + 1 <= xt->xt_ntrbs);
550 xt->xt_trbs[off + i].trb_addr = LE_64((uintptr_t)xt);
551 xt->xt_trbs[off + i].trb_status = LE_32(XHCI_TRB_INTR(0));
552 xt->xt_trbs[off + i].trb_flags = LE_32(XHCI_TRB_TYPE_EVENT |
553 XHCI_TRB_IOC);
554 }
555 }
556
557 /*
558 * These are utility functions for isochronus transfers to help calculate the
559 * transfer burst count (TBC) and transfer last burst packet count (TLPBC)
560 * entries for an isochronus entry. See xHCI 1.1 / 4.11.2.3 for how to calculate
561 * them.
562 */
563 void
564 xhci_transfer_calculate_isoc(xhci_device_t *xd, xhci_endpoint_t *xep,
565 uint_t trb_len, uint_t *tbc, uint_t *tlbpc)
566 {
567 uint_t mps, tdpc, burst;
568
569 /*
570 * Even if we're asked to send no data, that actually requires the
571 * equivalent of sending one byte of data.
572 */
573 if (trb_len == 0)
574 trb_len = 1;
|