1 // Code for handling XHCI "Super speed" USB controllers.
3 // Copyright (C) 2013 Gerd Hoffmann <kraxel@redhat.com>
4 // Copyright (C) 2014 Kevin O'Connor <kevin@koconnor.net>
6 // This file may be distributed under the terms of the GNU LGPLv3 license.
8 #include "config.h" // CONFIG_*
9 #include "malloc.h" // memalign_low
10 #include "memmap.h" // PAGE_SIZE
11 #include "output.h" // dprintf
12 #include "pci.h" // pci_bdf_to_bus
13 #include "pci_ids.h" // PCI_CLASS_SERIAL_USB_XHCI
14 #include "pci_regs.h" // PCI_BASE_ADDRESS_0
15 #include "string.h" // memcpy
16 #include "usb.h" // struct usb_s
17 #include "usb-xhci.h" // struct ehci_qh
18 #include "util.h" // timer_calc
19 #include "x86.h" // readl
21 // --------------------------------------------------------------
24 #define XHCI_RING_ITEMS 16
25 #define XHCI_RING_SIZE (XHCI_RING_ITEMS*sizeof(struct xhci_trb))
28 * xhci_ring structs are allocated with XHCI_RING_SIZE alignment,
29 * then we can get it from a trb pointer (provided by evt ring).
31 #define XHCI_RING(_trb) \
32 ((struct xhci_ring*)((u32)(_trb) & ~(XHCI_RING_SIZE-1)))
34 // --------------------------------------------------------------
37 #define XHCI_CMD_RS (1<<0)
38 #define XHCI_CMD_HCRST (1<<1)
39 #define XHCI_CMD_INTE (1<<2)
40 #define XHCI_CMD_HSEE (1<<3)
41 #define XHCI_CMD_LHCRST (1<<7)
42 #define XHCI_CMD_CSS (1<<8)
43 #define XHCI_CMD_CRS (1<<9)
44 #define XHCI_CMD_EWE (1<<10)
45 #define XHCI_CMD_EU3S (1<<11)
47 #define XHCI_STS_HCH (1<<0)
48 #define XHCI_STS_HSE (1<<2)
49 #define XHCI_STS_EINT (1<<3)
50 #define XHCI_STS_PCD (1<<4)
51 #define XHCI_STS_SSS (1<<8)
52 #define XHCI_STS_RSS (1<<9)
53 #define XHCI_STS_SRE (1<<10)
54 #define XHCI_STS_CNR (1<<11)
55 #define XHCI_STS_HCE (1<<12)
57 #define XHCI_PORTSC_CCS (1<<0)
58 #define XHCI_PORTSC_PED (1<<1)
59 #define XHCI_PORTSC_OCA (1<<3)
60 #define XHCI_PORTSC_PR (1<<4)
61 #define XHCI_PORTSC_PLS_SHIFT 5
62 #define XHCI_PORTSC_PLS_MASK 0xf
63 #define XHCI_PORTSC_PP (1<<9)
64 #define XHCI_PORTSC_SPEED_SHIFT 10
65 #define XHCI_PORTSC_SPEED_MASK 0xf
66 #define XHCI_PORTSC_SPEED_FULL (1<<10)
67 #define XHCI_PORTSC_SPEED_LOW (2<<10)
68 #define XHCI_PORTSC_SPEED_HIGH (3<<10)
69 #define XHCI_PORTSC_SPEED_SUPER (4<<10)
70 #define XHCI_PORTSC_PIC_SHIFT 14
71 #define XHCI_PORTSC_PIC_MASK 0x3
72 #define XHCI_PORTSC_LWS (1<<16)
73 #define XHCI_PORTSC_CSC (1<<17)
74 #define XHCI_PORTSC_PEC (1<<18)
75 #define XHCI_PORTSC_WRC (1<<19)
76 #define XHCI_PORTSC_OCC (1<<20)
77 #define XHCI_PORTSC_PRC (1<<21)
78 #define XHCI_PORTSC_PLC (1<<22)
79 #define XHCI_PORTSC_CEC (1<<23)
80 #define XHCI_PORTSC_CAS (1<<24)
81 #define XHCI_PORTSC_WCE (1<<25)
82 #define XHCI_PORTSC_WDE (1<<26)
83 #define XHCI_PORTSC_WOE (1<<27)
84 #define XHCI_PORTSC_DR (1<<30)
85 #define XHCI_PORTSC_WPR (1<<31)
88 #define TRB_TYPE_SHIFT 10
89 #define TRB_TYPE_MASK 0x3f
90 #define TRB_TYPE(t) (((t) >> TRB_TYPE_SHIFT) & TRB_TYPE_MASK)
92 #define TRB_EV_ED (1<<2)
94 #define TRB_TR_ENT (1<<1)
95 #define TRB_TR_ISP (1<<2)
96 #define TRB_TR_NS (1<<3)
97 #define TRB_TR_CH (1<<4)
98 #define TRB_TR_IOC (1<<5)
99 #define TRB_TR_IDT (1<<6)
100 #define TRB_TR_TBC_SHIFT 7
101 #define TRB_TR_TBC_MASK 0x3
102 #define TRB_TR_BEI (1<<9)
103 #define TRB_TR_TLBPC_SHIFT 16
104 #define TRB_TR_TLBPC_MASK 0xf
105 #define TRB_TR_FRAMEID_SHIFT 20
106 #define TRB_TR_FRAMEID_MASK 0x7ff
107 #define TRB_TR_SIA (1<<31)
109 #define TRB_TR_DIR (1<<16)
111 #define TRB_CR_SLOTID_SHIFT 24
112 #define TRB_CR_SLOTID_MASK 0xff
113 #define TRB_CR_EPID_SHIFT 16
114 #define TRB_CR_EPID_MASK 0x1f
116 #define TRB_CR_BSR (1<<9)
117 #define TRB_CR_DC (1<<9)
119 #define TRB_LK_TC (1<<1)
121 #define TRB_INTR_SHIFT 22
122 #define TRB_INTR_MASK 0x3ff
123 #define TRB_INTR(t) (((t).status >> TRB_INTR_SHIFT) & TRB_INTR_MASK)
125 typedef enum TRBType {
138 CR_CONFIGURE_ENDPOINT,
146 CR_SET_LATENCY_TOLERANCE,
147 CR_GET_PORT_BANDWIDTH,
152 ER_PORT_STATUS_CHANGE,
153 ER_BANDWIDTH_REQUEST,
156 ER_DEVICE_NOTIFICATION,
160 typedef enum TRBCCode {
163 CC_DATA_BUFFER_ERROR,
165 CC_USB_TRANSACTION_ERROR,
171 CC_INVALID_STREAM_TYPE_ERROR,
172 CC_SLOT_NOT_ENABLED_ERROR,
173 CC_EP_NOT_ENABLED_ERROR,
179 CC_BANDWIDTH_OVERRUN,
180 CC_CONTEXT_STATE_ERROR,
181 CC_NO_PING_RESPONSE_ERROR,
182 CC_EVENT_RING_FULL_ERROR,
183 CC_INCOMPATIBLE_DEVICE_ERROR,
184 CC_MISSED_SERVICE_ERROR,
185 CC_COMMAND_RING_STOPPED,
188 CC_STOPPED_LENGTH_INVALID,
189 CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR = 29,
190 CC_ISOCH_BUFFER_OVERRUN = 31,
193 CC_INVALID_STREAM_ID_ERROR,
194 CC_SECONDARY_BANDWIDTH_ERROR,
195 CC_SPLIT_TRANSACTION_ERROR
209 PLS_COMPILANCE_MODE = 10,
214 #define xhci_get_field(data, field) \
215 (((data) >> field##_SHIFT) & field##_MASK)
217 // --------------------------------------------------------------
221 struct xhci_trb ring[XHCI_RING_ITEMS];
240 struct xhci_caps *caps;
246 /* xhci data structures */
247 struct xhci_devlist *devs;
248 struct xhci_ring *cmds;
249 struct xhci_ring *evts;
250 struct xhci_er_seg *eseg;
254 struct xhci_ring reqs;
256 struct usb_pipe pipe;
263 // --------------------------------------------------------------
266 static const char *speed_name[16] = {
274 static const int speed_from_xhci[16] = {
276 [ 1 ] = USB_FULLSPEED,
277 [ 2 ] = USB_LOWSPEED,
278 [ 3 ] = USB_HIGHSPEED,
279 [ 4 ] = USB_SUPERSPEED,
283 static const int speed_to_xhci[] = {
284 [ USB_FULLSPEED ] = 1,
285 [ USB_LOWSPEED ] = 2,
286 [ USB_HIGHSPEED ] = 3,
287 [ USB_SUPERSPEED ] = 4,
290 static const int eptype_to_xhci_in[] = {
291 [ USB_ENDPOINT_XFER_CONTROL] = 4,
292 [ USB_ENDPOINT_XFER_ISOC ] = 5,
293 [ USB_ENDPOINT_XFER_BULK ] = 6,
294 [ USB_ENDPOINT_XFER_INT ] = 7,
297 static const int eptype_to_xhci_out[] = {
298 [ USB_ENDPOINT_XFER_CONTROL] = 4,
299 [ USB_ENDPOINT_XFER_ISOC ] = 1,
300 [ USB_ENDPOINT_XFER_BULK ] = 2,
301 [ USB_ENDPOINT_XFER_INT ] = 3,
304 static int wait_bit(u32 *reg, u32 mask, int value, u32 timeout)
306 u32 end = timer_calc(timeout);
308 while ((readl(reg) & mask) != value) {
309 if (timer_check(end)) {
319 /****************************************************************
321 ****************************************************************/
323 #define XHCI_TIME_POSTPOWER 20
325 // Check if device attached to port
327 xhci_print_port_state(int loglevel, const char *prefix, u32 port, u32 portsc)
329 u32 pls = xhci_get_field(portsc, XHCI_PORTSC_PLS);
330 u32 speed = xhci_get_field(portsc, XHCI_PORTSC_SPEED);
332 dprintf(loglevel, "%s port #%d: 0x%08x,%s%s pls %d, speed %d [%s]\n",
333 prefix, port + 1, portsc,
334 (portsc & XHCI_PORTSC_PP) ? " powered," : "",
335 (portsc & XHCI_PORTSC_PED) ? " enabled," : "",
336 pls, speed, speed_name[speed]);
340 xhci_hub_detect(struct usbhub_s *hub, u32 port)
342 struct usb_xhci_s *xhci = container_of(hub->cntl, struct usb_xhci_s, usb);
343 u32 portsc = readl(&xhci->pr[port].portsc);
344 return (portsc & XHCI_PORTSC_CCS) ? 1 : 0;
347 // Reset device on port
349 xhci_hub_reset(struct usbhub_s *hub, u32 port)
351 struct usb_xhci_s *xhci = container_of(hub->cntl, struct usb_xhci_s, usb);
352 u32 portsc = readl(&xhci->pr[port].portsc);
353 if (!(portsc & XHCI_PORTSC_CCS))
354 // Device no longer connected?!
357 switch (xhci_get_field(portsc, XHCI_PORTSC_PLS)) {
359 // A USB3 port - controller automatically performs reset
362 // A USB2 port - perform device reset
363 xhci_print_port_state(3, __func__, port, portsc);
364 writel(&xhci->pr[port].portsc, portsc | XHCI_PORTSC_PR);
370 // Wait for device to complete reset and be enabled
371 u32 end = timer_calc(100);
373 portsc = readl(&xhci->pr[port].portsc);
374 if (!(portsc & XHCI_PORTSC_CCS))
375 // Device disconnected during reset
377 if (portsc & XHCI_PORTSC_PED)
380 if (timer_check(end)) {
387 int rc = speed_from_xhci[xhci_get_field(portsc, XHCI_PORTSC_SPEED)];
388 xhci_print_port_state(1, "XHCI", port, portsc);
393 xhci_hub_disconnect(struct usbhub_s *hub, u32 port)
395 // XXX - should turn the port power off.
398 static struct usbhub_op_s xhci_hub_ops = {
399 .detect = xhci_hub_detect,
400 .reset = xhci_hub_reset,
401 .disconnect = xhci_hub_disconnect,
404 // Find any devices connected to the root hub.
406 xhci_check_ports(struct usb_xhci_s *xhci)
408 // Wait for port power to stabilize.
409 msleep(XHCI_TIME_POSTPOWER);
412 memset(&hub, 0, sizeof(hub));
413 hub.cntl = &xhci->usb;
414 hub.portcount = xhci->ports;
415 hub.op = &xhci_hub_ops;
421 /****************************************************************
423 ****************************************************************/
426 xhci_free_pipes(struct usb_xhci_s *xhci)
428 // XXX - should walk list of pipes and free unused pipes.
432 configure_xhci(void *data)
434 struct usb_xhci_s *xhci = data;
437 xhci->devs = memalign_high(64, sizeof(*xhci->devs) * (xhci->slots + 1));
438 xhci->eseg = memalign_high(64, sizeof(*xhci->eseg));
439 xhci->cmds = memalign_high(XHCI_RING_SIZE, sizeof(*xhci->cmds));
440 xhci->evts = memalign_high(XHCI_RING_SIZE, sizeof(*xhci->evts));
441 if (!xhci->devs || !xhci->cmds || !xhci->evts || !xhci->eseg) {
445 memset(xhci->devs, 0, sizeof(*xhci->devs) * (xhci->slots + 1));
446 memset(xhci->cmds, 0, sizeof(*xhci->cmds));
447 memset(xhci->evts, 0, sizeof(*xhci->evts));
448 memset(xhci->eseg, 0, sizeof(*xhci->eseg));
450 reg = readl(&xhci->op->usbcmd);
451 if (reg & XHCI_CMD_RS) {
453 writel(&xhci->op->usbcmd, reg);
454 if (wait_bit(&xhci->op->usbsts, XHCI_STS_HCH, XHCI_STS_HCH, 32) != 0)
458 dprintf(3, "%s: resetting\n", __func__);
459 writel(&xhci->op->usbcmd, XHCI_CMD_HCRST);
460 if (wait_bit(&xhci->op->usbcmd, XHCI_CMD_HCRST, 0, 100) != 0)
462 if (wait_bit(&xhci->op->usbsts, XHCI_STS_CNR, 0, 100) != 0)
465 writel(&xhci->op->config, xhci->slots);
466 writel(&xhci->op->dcbaap_low, (u32)xhci->devs);
467 writel(&xhci->op->dcbaap_high, 0);
468 writel(&xhci->op->crcr_low, (u32)xhci->cmds | 1);
469 writel(&xhci->op->crcr_high, 0);
472 xhci->eseg->ptr_low = (u32)xhci->evts;
473 xhci->eseg->ptr_high = 0;
474 xhci->eseg->size = XHCI_RING_ITEMS;
475 writel(&xhci->ir->erstsz, 1);
476 writel(&xhci->ir->erdp_low, (u32)xhci->evts);
477 writel(&xhci->ir->erdp_high, 0);
478 writel(&xhci->ir->erstba_low, (u32)xhci->eseg);
479 writel(&xhci->ir->erstba_high, 0);
482 reg = readl(&xhci->caps->hcsparams2);
483 u32 spb = (reg >> 21 & 0x1f) << 5 | reg >> 27;
485 dprintf(3, "%s: setup %d scratch pad buffers\n", __func__, spb);
486 u64 *spba = memalign_high(64, sizeof(*spba) * spb);
487 void *pad = memalign_high(PAGE_SIZE, PAGE_SIZE * spb);
495 for (i = 0; i < spb; i++)
496 spba[i] = (u32)pad + (i * PAGE_SIZE);
497 xhci->devs[0].ptr_low = (u32)spba;
498 xhci->devs[0].ptr_high = 0;
501 reg = readl(&xhci->op->usbcmd);
503 writel(&xhci->op->usbcmd, reg);
506 int count = xhci_check_ports(xhci);
507 xhci_free_pipes(xhci);
512 // No devices found - shutdown and free controller.
513 dprintf(1, "XHCI no devices found\n");
514 reg = readl(&xhci->op->usbcmd);
516 writel(&xhci->op->usbcmd, reg);
517 wait_bit(&xhci->op->usbsts, XHCI_STS_HCH, XHCI_STS_HCH, 32);
528 xhci_controller_setup(struct pci_device *pci)
530 struct usb_xhci_s *xhci = malloc_high(sizeof(*xhci));
535 memset(xhci, 0, sizeof(*xhci));
537 wait_preempt(); // Avoid pci_config_readl when preempting
538 xhci->baseaddr = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0)
539 & PCI_BASE_ADDRESS_MEM_MASK;
540 xhci->caps = (void*)(xhci->baseaddr);
541 xhci->op = (void*)(xhci->baseaddr + readb(&xhci->caps->caplength));
542 xhci->pr = (void*)(xhci->baseaddr + readb(&xhci->caps->caplength) + 0x400);
543 xhci->db = (void*)(xhci->baseaddr + readl(&xhci->caps->dboff));
544 xhci->ir = (void*)(xhci->baseaddr + readl(&xhci->caps->rtsoff) + 0x20);
546 u32 hcs1 = readl(&xhci->caps->hcsparams1);
547 u32 hcc = readl(&xhci->caps->hccparams);
548 xhci->ports = (hcs1 >> 24) & 0xff;
549 xhci->slots = hcs1 & 0xff;
550 xhci->xcap = ((hcc >> 16) & 0xffff) << 2;
551 xhci->context64 = (hcc & 0x04) ? 1 : 0;
554 xhci->usb.type = USB_TYPE_XHCI;
556 dprintf(1, "XHCI init on dev %02x:%02x.%x: regs @ %p, %d ports, %d slots"
557 ", %d byte contexts\n"
558 , pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf)
559 , pci_bdf_to_fn(pci->bdf), xhci->caps
560 , xhci->ports, xhci->slots, xhci->context64 ? 64 : 32);
563 u32 off, addr = xhci->baseaddr + xhci->xcap;
565 struct xhci_xcap *xcap = (void*)addr;
566 u32 ports, name, cap = readl(&xcap->cap);
567 switch (cap & 0xff) {
569 name = readl(&xcap->data[0]);
570 ports = readl(&xcap->data[1]);
571 dprintf(1, "XHCI protocol %c%c%c%c %x.%02x"
572 ", %d ports (offset %d), def %x\n"
575 , (name >> 16) & 0xff
576 , (name >> 24) & 0xff
579 , (ports >> 8) & 0xff
580 , (ports >> 0) & 0xff
584 dprintf(1, "XHCI extcap 0x%x @ %x\n", cap & 0xff, addr);
587 off = (cap >> 8) & 0xff;
592 u32 pagesize = readl(&xhci->op->pagesize);
593 if (PAGE_SIZE != (pagesize<<12)) {
594 dprintf(1, "XHCI driver does not support page size code %d\n"
600 pci_config_maskw(pci->bdf, PCI_COMMAND, 0, PCI_COMMAND_MASTER);
602 run_thread(configure_xhci, xhci);
608 if (! CONFIG_USB_XHCI)
610 struct pci_device *pci;
612 if (pci_classprog(pci) == PCI_CLASS_SERIAL_USB_XHCI)
613 xhci_controller_setup(pci);
618 /****************************************************************
619 * End point communication
620 ****************************************************************/
622 static void xhci_doorbell(struct usb_xhci_s *xhci, u32 slotid, u32 value)
624 struct xhci_db *db = xhci->db;
625 void *addr = &db[slotid].doorbell;
629 static void xhci_process_events(struct usb_xhci_s *xhci)
631 struct xhci_ring *evts = xhci->evts;
634 /* check for event */
635 u32 nidx = evts->nidx;
637 struct xhci_trb *etrb = evts->ring + nidx;
638 u32 control = etrb->control;
639 if ((control & TRB_C) != (cs ? 1 : 0))
643 u32 evt_type = TRB_TYPE(control);
644 u32 evt_cc = (etrb->status >> 24) & 0xff;
647 case ER_COMMAND_COMPLETE:
649 struct xhci_trb *rtrb = (void*)etrb->ptr_low;
650 struct xhci_ring *ring = XHCI_RING(rtrb);
651 struct xhci_trb *evt = &ring->evt;
652 u32 eidx = rtrb - ring->ring + 1;
653 dprintf(5, "%s: ring %p [trb %p, evt %p, type %d, eidx %d, cc %d]\n",
654 __func__, ring, rtrb, evt, evt_type, eidx, evt_cc);
655 memcpy(evt, etrb, sizeof(*etrb));
659 case ER_PORT_STATUS_CHANGE:
661 u32 portid = (etrb->ptr_low >> 24) & 0xff;
662 dprintf(3, "%s: status change port #%d\n",
667 dprintf(1, "%s: unknown event, type %d, cc %d\n",
668 __func__, evt_type, evt_cc);
672 /* move ring index, notify xhci */
674 if (nidx == XHCI_RING_ITEMS) {
680 struct xhci_ir *ir = xhci->ir;
681 u32 erdp = (u32)(evts->ring + nidx);
682 writel(&ir->erdp_low, erdp);
683 writel(&ir->erdp_high, 0);
687 static int xhci_ring_busy(struct xhci_ring *ring)
689 u32 eidx = ring->eidx;
690 u32 nidx = ring->nidx;
691 return (eidx != nidx);
694 static int xhci_event_wait(struct usb_xhci_s *xhci,
695 struct xhci_ring *ring,
698 u32 end = timer_calc(timeout);
701 xhci_process_events(xhci);
702 if (!xhci_ring_busy(ring)) {
703 u32 status = ring->evt.status;
704 return (status >> 24) & 0xff;
706 if (timer_check(end)) {
714 static void xhci_trb_queue(struct xhci_ring *ring,
715 struct xhci_trb *trb)
717 u32 nidx = ring->nidx;
719 struct xhci_trb *dst;
722 if (nidx == XHCI_RING_ITEMS-1) {
723 dst = ring->ring + nidx;
724 control = (TR_LINK << 10); // trb type
725 control |= TRB_LK_TC;
726 control |= (cs ? TRB_C : 0);
727 dst->ptr_low = (u32)&ring[0];
730 dst->control = control;
736 dprintf(5, "%s: ring %p [linked]\n", __func__, ring);
739 dst = ring->ring + nidx;
740 control = trb->control | (cs ? TRB_C : 0);
742 dst->ptr_low = trb->ptr_low;
743 dst->ptr_high = trb->ptr_high;
744 dst->status = trb->status;
745 dst->control = control;
749 dprintf(5, "%s: ring %p [nidx %d, len %d]\n",
750 __func__, ring, nidx,
751 trb->status & 0xffff);
754 static int xhci_cmd_submit(struct usb_xhci_s *xhci,
755 struct xhci_trb *cmd)
759 mutex_lock(&xhci->cmds->lock);
760 xhci_trb_queue(xhci->cmds, cmd);
761 xhci_doorbell(xhci, 0, 0);
762 rc = xhci_event_wait(xhci, xhci->cmds, 1000);
763 mutex_unlock(&xhci->cmds->lock);
767 static int xhci_cmd_enable_slot(struct usb_xhci_s *xhci)
769 struct xhci_trb cmd = {
773 .control = (CR_ENABLE_SLOT << 10)
775 dprintf(3, "%s:\n", __func__);
776 int cc = xhci_cmd_submit(xhci, &cmd);
777 if (cc != CC_SUCCESS)
779 return (xhci->cmds->evt.control >> 24) & 0xff;
783 static int xhci_cmd_disable_slot(struct usb_xhci_s *xhci, u32 slotid)
785 struct xhci_trb cmd = {
789 .control = (slotid << 24) | (CR_DISABLE_SLOT << 10)
791 dprintf(3, "%s: slotid %d\n", __func__, slotid);
792 return xhci_cmd_submit(xhci, &cmd);
796 static int xhci_cmd_address_device(struct usb_xhci_s *xhci, u32 slotid
797 , struct xhci_inctx *inctx)
799 struct xhci_trb cmd = {
800 .ptr_low = (u32)inctx,
803 .control = (slotid << 24) | (CR_ADDRESS_DEVICE << 10)
805 dprintf(3, "%s: slotid %d\n", __func__, slotid);
806 return xhci_cmd_submit(xhci, &cmd);
809 static int xhci_cmd_configure_endpoint(struct usb_xhci_s *xhci, u32 slotid
810 , struct xhci_inctx *inctx)
812 struct xhci_trb cmd = {
813 .ptr_low = (u32)inctx,
816 .control = (slotid << 24) | (CR_CONFIGURE_ENDPOINT << 10)
818 dprintf(3, "%s: slotid %d, add 0x%x, del 0x%x\n", __func__,
819 slotid, inctx->add, inctx->del);
820 return xhci_cmd_submit(xhci, &cmd);
823 static int xhci_cmd_evaluate_context(struct usb_xhci_s *xhci, u32 slotid
824 , struct xhci_inctx *inctx)
826 struct xhci_trb cmd = {
827 .ptr_low = (u32)inctx,
830 .control = (slotid << 24) | (CR_EVALUATE_CONTEXT << 10)
832 dprintf(3, "%s: slotid %d, add 0x%x, del 0x%x\n", __func__,
833 slotid, inctx->add, inctx->del);
834 return xhci_cmd_submit(xhci, &cmd);
837 static struct xhci_inctx *
838 xhci_alloc_inctx(struct usbdevice_s *usbdev, int maxepid)
840 struct usb_xhci_s *xhci = container_of(
841 usbdev->hub->cntl, struct usb_xhci_s, usb);
842 int size = (sizeof(struct xhci_inctx) * 33) << xhci->context64;
843 struct xhci_inctx *in = memalign_tmphigh(2048 << xhci->context64, size);
850 struct xhci_slotctx *slot = (void*)&in[1 << xhci->context64];
851 slot->ctx[0] |= maxepid << 27; // context entries
852 slot->ctx[0] |= speed_to_xhci[usbdev->speed] << 20;
854 // Set high-speed hub flags.
855 struct usbdevice_s *hubdev = usbdev->hub->usbdev;
857 if (usbdev->speed == USB_LOWSPEED || usbdev->speed == USB_FULLSPEED) {
858 struct xhci_pipe *hpipe = container_of(
859 hubdev->defpipe, struct xhci_pipe, pipe);
860 if (hubdev->speed == USB_HIGHSPEED) {
861 slot->ctx[2] |= hpipe->slotid;
862 slot->ctx[2] |= (usbdev->port+1) << 8;
864 struct xhci_slotctx *hslot = (void*)xhci->devs[hpipe->slotid].ptr_low;
865 slot->ctx[2] = hslot->ctx[2];
869 while (usbdev->hub->usbdev) {
871 route |= (usbdev->port+1) & 0xf;
872 usbdev = usbdev->hub->usbdev;
874 slot->ctx[0] |= route;
877 slot->ctx[1] |= (usbdev->port+1) << 16;
882 static int xhci_config_hub(struct usbhub_s *hub)
884 struct usb_xhci_s *xhci = container_of(
885 hub->cntl, struct usb_xhci_s, usb);
886 struct xhci_pipe *pipe = container_of(
887 hub->usbdev->defpipe, struct xhci_pipe, pipe);
888 struct xhci_slotctx *hdslot = (void*)xhci->devs[pipe->slotid].ptr_low;
889 if ((hdslot->ctx[3] >> 27) == 3)
890 // Already configured
892 struct xhci_inctx *in = xhci_alloc_inctx(hub->usbdev, 1);
896 struct xhci_slotctx *slot = (void*)&in[1 << xhci->context64];
897 slot->ctx[0] |= 1 << 26;
898 slot->ctx[1] |= hub->portcount << 24;
900 int cc = xhci_cmd_configure_endpoint(xhci, pipe->slotid, in);
902 if (cc != CC_SUCCESS) {
903 dprintf(1, "%s: configure hub: failed (cc %d)\n", __func__, cc);
909 static struct usb_pipe *
910 xhci_alloc_pipe(struct usbdevice_s *usbdev
911 , struct usb_endpoint_descriptor *epdesc)
913 u8 eptype = epdesc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
914 struct usb_xhci_s *xhci = container_of(
915 usbdev->hub->cntl, struct usb_xhci_s, usb);
916 struct xhci_pipe *pipe;
919 if (epdesc->bEndpointAddress == 0) {
922 epid = (epdesc->bEndpointAddress & 0x0f) * 2;
923 epid += (epdesc->bEndpointAddress & USB_DIR_IN) ? 1 : 0;
926 if (eptype == USB_ENDPOINT_XFER_CONTROL)
927 pipe = memalign_high(XHCI_RING_SIZE, sizeof(*pipe));
929 pipe = memalign_low(XHCI_RING_SIZE, sizeof(*pipe));
934 memset(pipe, 0, sizeof(*pipe));
936 usb_desc2pipe(&pipe->pipe, usbdev, epdesc);
939 if (eptype == USB_ENDPOINT_XFER_INT) {
940 pipe->buf = malloc_high(pipe->pipe.maxpacket);
948 // Allocate input context and initialize endpoint info.
949 struct xhci_inctx *in = xhci_alloc_inctx(usbdev, epid);
952 in->add = 0x01 | (1 << epid);
953 struct xhci_epctx *ep = (void*)&in[(pipe->epid+1) << xhci->context64];
954 if (eptype == USB_ENDPOINT_XFER_INT)
955 ep->ctx[0] = (usb_get_period(usbdev, epdesc) + 3) << 16;
956 ep->ctx[1] |= eptype << 3;
957 if (epdesc->bEndpointAddress & USB_DIR_IN
958 || eptype == USB_ENDPOINT_XFER_CONTROL)
959 ep->ctx[1] |= 1 << 5;
960 ep->ctx[1] |= pipe->pipe.maxpacket << 16;
961 ep->deq_low = (u32)&pipe->reqs.ring[0];
962 ep->deq_low |= 1; // dcs
963 ep->length = pipe->pipe.maxpacket;
965 dprintf(3, "%s: usbdev %p, ring %p, slotid %d, epid %d\n", __func__,
966 usbdev, &pipe->reqs, pipe->slotid, pipe->epid);
967 if (pipe->epid == 1) {
968 if (usbdev->hub->usbdev) {
969 // Make sure parent hub is configured.
970 int ret = xhci_config_hub(usbdev->hub);
975 u32 size = (sizeof(struct xhci_slotctx) * 32) << xhci->context64;
976 struct xhci_slotctx *dev = memalign_high(1024 << xhci->context64, size);
981 int slotid = xhci_cmd_enable_slot(xhci);
983 dprintf(1, "%s: enable slot: failed\n", __func__);
987 dprintf(3, "%s: enable slot: got slotid %d\n", __func__, slotid);
988 memset(dev, 0, size);
989 pipe->slotid = usbdev->slotid = slotid;
990 xhci->devs[slotid].ptr_low = (u32)dev;
991 xhci->devs[slotid].ptr_high = 0;
993 // Send set_address command.
994 int cc = xhci_cmd_address_device(xhci, slotid, in);
995 if (cc != CC_SUCCESS) {
996 dprintf(1, "%s: address device: failed (cc %d)\n", __func__, cc);
1000 pipe->slotid = usbdev->slotid;
1001 // Send configure command.
1002 int cc = xhci_cmd_configure_endpoint(xhci, pipe->slotid, in);
1003 if (cc != CC_SUCCESS) {
1004 dprintf(1, "%s: configure endpoint: failed (cc %d)\n", __func__, cc);
1019 xhci_realloc_pipe(struct usbdevice_s *usbdev, struct usb_pipe *upipe
1020 , struct usb_endpoint_descriptor *epdesc)
1022 if (!CONFIG_USB_XHCI)
1025 usb_add_freelist(upipe);
1029 return xhci_alloc_pipe(usbdev, epdesc);
1030 u8 eptype = epdesc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
1031 int oldmaxpacket = upipe->maxpacket;
1032 usb_desc2pipe(upipe, usbdev, epdesc);
1033 struct xhci_pipe *pipe = container_of(upipe, struct xhci_pipe, pipe);
1034 struct usb_xhci_s *xhci = container_of(
1035 pipe->pipe.cntl, struct usb_xhci_s, usb);
1036 dprintf(3, "%s: usbdev %p, ring %p, slotid %d, epid %d\n", __func__,
1037 usbdev, &pipe->reqs, pipe->slotid, pipe->epid);
1038 if (eptype != USB_ENDPOINT_XFER_CONTROL || upipe->maxpacket == oldmaxpacket)
1041 // maxpacket has changed on control endpoint - update controller.
1042 dprintf(1, "%s: reconf ctl endpoint pkt size: %d -> %d\n",
1043 __func__, oldmaxpacket, pipe->pipe.maxpacket);
1044 struct xhci_inctx *in = xhci_alloc_inctx(usbdev, 1);
1048 struct xhci_epctx *ep = (void*)&in[2 << xhci->context64];
1049 ep->ctx[1] |= (pipe->pipe.maxpacket << 16);
1050 int cc = xhci_cmd_evaluate_context(xhci, pipe->slotid, in);
1051 if (cc != CC_SUCCESS) {
1052 dprintf(1, "%s: reconf ctl endpoint: failed (cc %d)\n",
1060 static void xhci_xfer_queue(struct xhci_pipe *pipe,
1061 void *data, int datalen, u32 flags)
1063 struct xhci_trb trb;
1064 memset(&trb, 0, sizeof(trb));
1065 if (flags & TRB_TR_IDT)
1066 memcpy(&trb.ptr_low, data, datalen);
1068 trb.ptr_low = (u32)data;
1069 trb.status = datalen;
1070 trb.control = flags;
1071 xhci_trb_queue(&pipe->reqs, &trb);
1074 static void xhci_xfer_kick(struct xhci_pipe *pipe)
1076 struct usb_xhci_s *xhci = container_of(
1077 pipe->pipe.cntl, struct usb_xhci_s, usb);
1078 u32 slotid = pipe->slotid;
1079 u32 epid = pipe->epid;
1081 dprintf(5, "%s: ring %p, slotid %d, epid %d\n",
1082 __func__, &pipe->reqs, slotid, epid);
1083 xhci_doorbell(xhci, slotid, epid);
1086 static void xhci_xfer_normal(struct xhci_pipe *pipe,
1087 void *data, int datalen)
1089 xhci_xfer_queue(pipe, data, datalen, (TR_NORMAL << 10) | TRB_TR_IOC);
1090 xhci_xfer_kick(pipe);
1094 xhci_send_pipe(struct usb_pipe *p, int dir, const void *cmd
1095 , void *data, int datalen)
1097 if (!CONFIG_USB_XHCI)
1099 struct xhci_pipe *pipe = container_of(p, struct xhci_pipe, pipe);
1100 struct usb_xhci_s *xhci = container_of(
1101 pipe->pipe.cntl, struct usb_xhci_s, usb);
1104 const struct usb_ctrlrequest *req = cmd;
1105 if (req->bRequest == USB_REQ_SET_ADDRESS)
1106 // Set address command sent during xhci_alloc_pipe.
1109 xhci_xfer_queue(pipe, (void*)req, USB_CONTROL_SETUP_SIZE
1110 , (TR_SETUP << 10) | TRB_TR_IDT
1111 | ((datalen ? (dir ? 3 : 2) : 0) << 16));
1113 xhci_xfer_queue(pipe, data, datalen, (TR_DATA << 10)
1114 | ((dir ? 1 : 0) << 16));
1115 xhci_xfer_queue(pipe, NULL, 0, (TR_STATUS << 10) | TRB_TR_IOC
1116 | ((dir ? 0 : 1) << 16));
1117 xhci_xfer_kick(pipe);
1119 xhci_xfer_normal(pipe, data, datalen);
1122 int cc = xhci_event_wait(xhci, &pipe->reqs, usb_xfer_time(p, datalen));
1123 if (cc != CC_SUCCESS) {
1124 dprintf(1, "%s: xfer failed (cc %d)\n", __func__, cc);
1132 xhci_poll_intr(struct usb_pipe *p, void *data)
1134 if (!CONFIG_USB_XHCI)
1137 struct xhci_pipe *pipe = container_of(p, struct xhci_pipe, pipe);
1138 struct usb_xhci_s *xhci = container_of(
1139 pipe->pipe.cntl, struct usb_xhci_s, usb);
1140 u32 len = pipe->pipe.maxpacket;
1141 void *buf = pipe->buf;
1142 int bufused = pipe->bufused;
1145 xhci_xfer_normal(pipe, buf, len);
1147 pipe->bufused = bufused;
1151 xhci_process_events(xhci);
1152 if (xhci_ring_busy(&pipe->reqs))
1154 dprintf(5, "%s: st %x ct %x [ %p <= %p / %d ]\n", __func__,
1155 pipe->reqs.evt.status,
1156 pipe->reqs.evt.control,
1158 memcpy(data, buf, len);
1159 xhci_xfer_normal(pipe, buf, len);