qemu/roms/SLOF/lib/libusb/usb-xhci.c

   1 /*****************************************************************************
   2  * Copyright (c) 2013 IBM Corporation
   3  * All rights reserved.
   4  * This program and the accompanying materials
   5  * are made available under the terms of the BSD License
   6  * which accompanies this distribution, and is available at
   7  * http://www.opensource.org/licenses/bsd-license.php
   8  *
   9  * Contributors:
  10  *     IBM Corporation - initial implementation
  11  *****************************************************************************/
  12
  13 #include <string.h>
  14 #include "usb.h"
  15 #include "usb-core.h"
  16 #include "usb-xhci.h"
  17 #include "tools.h"
  18 #include "paflof.h"
  19
  20 #undef XHCI_DEBUG
  21 //#define XHCI_DEBUG
  22 #ifdef XHCI_DEBUG
  23 #define dprintf(_x ...) do { printf("%s: ", __func__); printf(_x); } while (0)
  24 #else
  25 #define dprintf(_x ...)
  26 #endif
  27
  28 static void dump_xhci_regs(struct xhci_hcd *xhcd)
  29 {
  30 #ifdef XHCI_DEBUG
  31         struct xhci_cap_regs *cap;
  32         struct xhci_op_regs *op;
  33         struct xhci_run_regs *run;
  34
  35         cap = xhcd->cap_regs;
  36         op = xhcd->op_regs;
  37         run = xhcd->run_regs;
  38
  39         dprintf("\n");
  40         dprintf(" - CAPLENGTH           %02X\n", read_reg8 (&cap->caplength));
  41         dprintf(" - HCIVERSION          %04X\n", read_reg16(&cap->hciversion));
  42         dprintf(" - HCSPARAMS1          %08X\n", read_reg32(&cap->hcsparams1));
  43         dprintf(" - HCSPARAMS2          %08X\n", read_reg32(&cap->hcsparams2));
  44         dprintf(" - HCSPARAMS3          %08X\n", read_reg32(&cap->hcsparams3));
  45         dprintf(" - HCCPARAMS           %08X\n", read_reg32(&cap->hccparams));
  46         dprintf(" - DBOFF               %08X\n", read_reg32(&cap->dboff));
  47         dprintf(" - RTSOFF              %08X\n", read_reg32(&cap->rtsoff));
  48         dprintf("\n");
  49
  50         dprintf(" - USBCMD              %08X\n", read_reg32(&op->usbcmd));
  51         dprintf(" - USBSTS              %08X\n", read_reg32(&op->usbsts));
  52         dprintf(" - PAGESIZE            %08X\n", read_reg32(&op->pagesize));
  53         dprintf(" - DNCTRL              %08X\n", read_reg32(&op->dnctrl));
  54         dprintf(" - CRCR              %016llX\n", read_reg64(&op->crcr));
  55         dprintf(" - DCBAAP            %016llX\n", read_reg64(&op->dcbaap));
  56         dprintf(" - CONFIG              %08X\n", read_reg32(&op->config));
  57         dprintf("\n");
  58
  59         dprintf(" - MFINDEX             %08X\n", read_reg32(&run->mfindex));
  60         dprintf("\n");
  61 #endif
  62 }
  63
  64 static void print_port_status(struct xhci_port_regs *prs)
  65 {
  66 #ifdef XHCI_DEBUG
  67         uint32_t portsc;
  68         uint32_t CCS, PED, PP, PLS, i, PR = 0;
  69
  70         portsc = read_reg32(&prs->portsc);
  71         dprintf("portsc %08x portpmsc %08x portli %08x\n",
  72                 portsc,
  73                 read_reg32(&prs->portpmsc),
  74                 read_reg32(&prs->portli));
  75
  76         if (portsc & PORTSC_CCS) {
  77                 printf("CCS ");
  78                 CCS = 1;
  79         }
  80         if (portsc & PORTSC_PED) {
  81                 printf("PED ");
  82                 PED = 1;
  83         }
  84         if (portsc & PORTSC_OCA)
  85                 printf("OCA ");
  86         if (portsc & PORTSC_PR)
  87                 printf("OCA ");
  88         PLS = (portsc & PORTSC_PLS_MASK) >> 5;
  89         printf("PLS:%d ", PLS);
  90         if (portsc & PORTSC_PP) {
  91                 printf("PP ");
  92                 PP = 1;
  93         }
  94         printf("PS:%d ", (portsc & PORTSC_PS_MASK) >> 10);
  95         printf("PIC:%d ", (portsc & PORTSC_PIC_MASK) >> 14);
  96         if (portsc & PORTSC_LWS)
  97                 printf("LWS ");
  98         if (portsc & PORTSC_CSC)
  99                 printf("CSC ");
 100         if (portsc & PORTSC_PEC)
 101                 printf("PEC ");
 102         if (portsc & PORTSC_WRC)
 103                 printf("WRC ");
 104         if (portsc & PORTSC_OCC)
 105                 printf("OCC ");
 106         if (portsc & PORTSC_PRC)
 107                 printf("PRC ");
 108         if (portsc & PORTSC_PLC)
 109                 printf("PLC ");
 110         if (portsc & PORTSC_CEC)
 111                 printf("CEC ");
 112         if (portsc & PORTSC_CAS)
 113                 printf("CAS ");
 114         if (portsc & PORTSC_WCE)
 115                 printf("WCE ");
 116         if (portsc & PORTSC_WDE)
 117                 printf("WDE ");
 118         if (portsc & PORTSC_WOE)
 119                 printf("WOE ");
 120         if (portsc & PORTSC_DR)
 121                 printf("DR ");
 122         if (portsc & PORTSC_WPR)
 123                 printf("WPR ");
 124         printf("\n");
 125
 126         for (i = 0 ; i < (sizeof(ps_array_usb3)/sizeof(struct port_state)); i++) {
 127                 if (PP == ps_array_usb3[i].PP) {
 128                         if (CCS == ps_array_usb3[i].CCS) {
 129                                 if (PED == ps_array_usb3[i].PED) {
 130                                         if (PR == ps_array_usb3[i].PR) {
 131                                                 dprintf("%s - PLS %d\n", ps_array_usb3[i].state, PLS);
 132                                                 break;
 133                                         }
 134                                 }
 135                         }
 136                 }
 137         }
 138 #endif
 139
 140 }
 141
 142 static inline bool xhci_is_hc_ready(uint32_t *usbsts)
 143 {
 144         return !(read_reg32(usbsts) & XHCI_USBSTS_CNR);
 145 }
 146
 147 static inline bool xhci_wait_for_cnr(uint32_t *usbsts)
 148 {
 149         /* Standard:
 150          * Note: The xHC should halt within 16 ms. of software clearing the
 151          * R/S bit to ‘0’.
 152          * Give some more time... 32ms
 153          */
 154         int count = 320;
 155         dprintf("Waiting for Controller ready ..");
 156         while (!xhci_is_hc_ready(usbsts)) {
 157                 dprintf(".");
 158                 count--;
 159                 if (!count) {
 160                         dprintf("  failed %08X\n", read_reg32(usbsts));
 161                         return false;
 162                 }
 163                 SLOF_usleep(100);
 164         }
 165         dprintf("  done\n");
 166         return true;
 167 }
 168
 169 static bool xhci_hcd_set_runstop(struct xhci_op_regs *op, bool run_req)
 170 {
 171         uint32_t reg;
 172
 173         dprintf("Request %s\n", run_req ? "RUN" : "STOP");
 174         if (!xhci_is_hc_ready(&op->usbsts)) {
 175                 dprintf("Controller not ready\n");
 176                 return false;
 177         }
 178
 179         reg = read_reg32(&op->usbcmd);
 180         if (run_req)
 181                 reg |= run_req;
 182         else
 183                 reg &= (uint32_t)~1;
 184         dprintf("writing %08X\n", reg);
 185         write_reg32(&op->usbcmd, reg);
 186         mb();
 187         xhci_wait_for_cnr(&op->usbsts);
 188         return true;
 189 }
 190
 191 static bool xhci_hcd_reset(struct xhci_op_regs *op)
 192 {
 193         uint32_t reg;
 194
 195         /* Check if the controller is halted, else halt it */
 196         if (!(read_reg32(&op->usbsts) & XHCI_USBSTS_HCH)) {
 197                 dprintf("HCHalted not set\n");
 198                 if (!xhci_hcd_set_runstop(op, false))
 199                         return false;
 200         }
 201
 202         if (read_reg32(&op->usbsts) & XHCI_USBSTS_CNR) {
 203                 dprintf("Controller not ready\n");
 204                 return false;
 205         }
 206
 207         reg = read_reg32(&op->usbcmd) | XHCI_USBCMD_HCRST;
 208         /* Ready to Reset the controller now */
 209         write_reg32(&op->usbcmd, reg);
 210         xhci_wait_for_cnr(&op->usbsts);
 211         return true;
 212 }
 213
 214 static void xhci_handle_cmd_completion(struct xhci_hcd *xhcd,
 215                                 struct xhci_event_trb *event)
 216 {
 217         uint32_t flags, slot_id, status;
 218
 219         status = le32_to_cpu(event->status);
 220         flags = le32_to_cpu(event->flags);
 221         slot_id = TRB_SLOT_ID(flags);
 222         if (TRB_STATUS(status) == COMP_SUCCESS)
 223                 xhcd->slot_id = slot_id;
 224         else
 225                 xhcd->slot_id = 0;
 226 }
 227
 228 static uint64_t xhci_poll_event(struct xhci_hcd *xhcd,
 229                                 uint32_t event_type)
 230 {
 231         struct xhci_event_trb *event;
 232         uint64_t val, retval = 0;
 233         uint32_t flags, time;
 234         int index;
 235
 236         mb();
 237         event = (struct xhci_event_trb *)xhcd->ering.deq;
 238         flags = le32_to_cpu(event->flags);
 239
 240         dprintf("Reading from event ptr %p %08x\n", event, flags);
 241         time = SLOF_GetTimer() + USB_TIMEOUT;
 242
 243         while ((flags & TRB_CYCLE_STATE) != xhcd->ering.cycle_state) {
 244                 mb();
 245                 flags = le32_to_cpu(event->flags);
 246                 if (time < SLOF_GetTimer())
 247                         return 0;
 248         }
 249
 250         mb();
 251         flags = le32_to_cpu(event->flags);
 252         switch(TRB_TYPE(flags))
 253         {
 254         case TRB_CMD_COMPLETION:
 255                 dprintf("CMD Completion\n");
 256                 xhci_handle_cmd_completion(xhcd, event);
 257                 break;
 258         case TRB_PORT_STATUS:
 259                 dprintf("Port status event\n");
 260                 break;
 261         case TRB_TRANSFER_EVENT:
 262                 dprintf("XFER event addr %16lx, status %08x, flags %08x\n",
 263                         le64_to_cpu(event->addr),
 264                         le32_to_cpu(event->status),
 265                         le32_to_cpu(event->flags));
 266                 break;
 267         default:
 268                 printf("TRB_TYPE  %d\n", TRB_TYPE(flags));
 269                 dprintf("Event addr %16lx, status %08x, flags %08x state %d\n",
 270                         le64_to_cpu(event->addr),
 271                         le32_to_cpu(event->status),
 272                         flags, xhcd->ering.cycle_state);
 273                 break;
 274         }
 275         xhcd->ering.deq = (uint64_t) (event + 1);
 276         retval = le64_to_cpu(event->addr);
 277
 278         event->addr = 0;
 279         event->status = 0;
 280         event->flags = cpu_to_le32(xhcd->ering.cycle_state);
 281
 282         index = xhcd->ering.deq - (uint64_t)xhcd->ering.trbs;
 283         val = xhcd->ering.trbs_dma;
 284         val += (index % XHCI_EVENT_TRBS_SIZE);
 285         if (!(index % XHCI_EVENT_TRBS_SIZE)) {
 286                 xhcd->ering.deq = (uint64_t)xhcd->ering.trbs;
 287                 xhcd->ering.cycle_state = xhcd->ering.cycle_state ? 0 : 1;
 288                 dprintf("Rounding %d\n", xhcd->ering.cycle_state);
 289         }
 290         dprintf("Update start %x deq %x index %d\n",
 291                 xhcd->ering.trbs_dma, val, index/sizeof(*event));
 292         write_reg64(&xhcd->run_regs->irs[0].erdp, val);
 293
 294         if (retval == 0)
 295                 return (uint64_t)event;
 296         else
 297                 return retval;
 298 }
 299
 300 static void xhci_send_cmd(struct xhci_hcd *xhcd, uint32_t field1,
 301                         uint32_t field2, uint32_t field3, uint32_t field4)
 302 {
 303         struct xhci_db_regs *dbr;
 304         struct xhci_command_trb *cmd;
 305         uint32_t val, cycle_state;
 306
 307         dbr = xhcd->db_regs;
 308         cmd = (struct xhci_command_trb *)xhcd->crseg.enq;
 309
 310         cmd->field[0] = cpu_to_le32(field1);
 311         cmd->field[1] = cpu_to_le32(field2);
 312         cmd->field[2] = cpu_to_le32(field3);
 313
 314         val = le32_to_cpu(cmd->field[3]);
 315         cycle_state = (val & 0x1) ? 0 : 1;
 316         val = field4 | cycle_state;
 317         cmd->field[3] = cpu_to_le32(val);
 318
 319         dprintf("CMD %016lx val %08x cycle_state %d field1 %08x, field2  %08x, field3 %08x field4 %08x\n",
 320                 cmd, val, cycle_state,
 321                 le32_to_cpu(cmd->field[0]),
 322                 le32_to_cpu(cmd->field[1]),
 323                 le32_to_cpu(cmd->field[2]),
 324                 le32_to_cpu(cmd->field[3])
 325                 );
 326
 327         /* Ring the doorbell */
 328         write_reg32(&dbr->db[0], 0);
 329         xhci_poll_event(xhcd, 0);
 330         cmd++;
 331         xhcd->crseg.enq = (uint64_t)cmd;
 332         return;
 333 }
 334
 335 static void xhci_send_enable_slot(struct xhci_hcd *xhcd, uint32_t port)
 336 {
 337         uint32_t field1, field2, field3, field4;
 338
 339         field1 = 0;
 340         field2 = 0;
 341         field3 = 0;
 342         field4 = TRB_CMD_TYPE(TRB_ENABLE_SLOT);
 343         xhci_send_cmd(xhcd, field1, field2, field3, field4);
 344 }
 345
 346 static void xhci_send_addr_device(struct xhci_hcd *xhcd, uint32_t slot_id,
 347                         uint64_t dma_in_ctx)
 348 {
 349         uint32_t field1, field2, field3, field4;
 350
 351         dprintf("Address device %lx, low %x, high %x\n", dma_in_ctx,
 352                 TRB_ADDR_LOW(dma_in_ctx),
 353                 TRB_ADDR_HIGH(dma_in_ctx));
 354         field1 = TRB_ADDR_LOW(dma_in_ctx) & ~0xF;
 355         field2 = TRB_ADDR_HIGH(dma_in_ctx);
 356         field3 = 0;
 357         field4 = TRB_CMD_TYPE(TRB_ADDRESS_DEV) | TRB_CMD_SLOT_ID(slot_id);
 358         xhci_send_cmd(xhcd, field1, field2, field3, field4);
 359 }
 360
 361 static uint32_t xhci_get_epno(struct usb_pipe *pipe)
 362 {
 363         uint32_t x_epno;
 364         x_epno = pipe->dir | 2 * pipe->epno;
 365         dprintf("EPno %d:%d DIR %d\n", pipe->epno, x_epno, pipe->dir);
 366         return x_epno;
 367 }
 368
 369 static void xhci_configure_ep(struct xhci_hcd *xhcd, uint32_t slot_id,
 370                         uint64_t dma_in_ctx)
 371 {
 372         uint32_t field1, field2, field3, field4;
 373
 374         dprintf("Configure EP %lx, low %x, high %x\n", dma_in_ctx,
 375                 TRB_ADDR_LOW(dma_in_ctx),
 376                 TRB_ADDR_HIGH(dma_in_ctx));
 377         field1 = TRB_ADDR_LOW(dma_in_ctx) & ~0xF;
 378         field2 = TRB_ADDR_HIGH(dma_in_ctx);
 379         field3 = 0;
 380         field4 = TRB_CMD_TYPE(TRB_CONFIG_EP) | TRB_CMD_SLOT_ID(slot_id);
 381         xhci_send_cmd(xhcd, field1, field2, field3, field4);
 382 }
 383
 384 static void xhci_init_seg(struct xhci_seg *seg, uint32_t size, uint32_t type)
 385 {
 386         struct xhci_link_trb *link;
 387
 388         seg->size = size / XHCI_TRB_SIZE;
 389         seg->next = NULL;
 390         seg->type = type;
 391         seg->cycle_state = 1;
 392         seg->enq = (uint64_t)seg->trbs;
 393         seg->deq = (uint64_t)seg->trbs;
 394         memset((void *)seg->trbs, 0, size);
 395
 396         if (type != TYPE_EVENT) {
 397                 link =(struct xhci_link_trb *) (seg->trbs + seg->size - 1);
 398                 link->addr = cpu_to_le64(seg->trbs_dma);
 399                 link->field2 = 0;
 400                 link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
 401         }
 402         return;
 403 }
 404
 405 static bool xhci_alloc_seg(struct xhci_seg *seg, uint32_t size, uint32_t type)
 406 {
 407         seg->trbs = (union xhci_trb *)SLOF_dma_alloc(size);
 408         if (!seg->trbs) {
 409                 dprintf("Alloc failed\n");
 410                 return false;
 411         }
 412         xhci_init_seg(seg, size, type);
 413         seg->trbs_dma = SLOF_dma_map_in((void *)seg->trbs, size, false);
 414
 415         dprintf(" TRBs %016lX TRBS-DMA %016lX\n", seg->trbs, seg->trbs_dma);
 416         return true;
 417 }
 418
 419 static void xhci_free_seg(struct xhci_seg *seg, uint32_t size)
 420 {
 421         if (seg->trbs) {
 422                 dprintf(" TRBs %016lX TRBS-DMA %016lX size %x\n", seg->trbs, seg->trbs_dma, size);
 423                 SLOF_dma_map_out(seg->trbs_dma, (void *)seg->trbs, size);
 424                 SLOF_dma_free((void *)seg->trbs, size);
 425         }
 426         memset(seg, 0, sizeof(*seg));
 427 }
 428
 429 #define CTX_SIZE(x)  ( (x) ? 64 : 32 )
 430
 431 static bool xhci_alloc_ctx(struct xhci_ctx *ctx, uint32_t size, uint32_t type)
 432 {
 433         ctx->addr = (uint8_t *)SLOF_dma_alloc(size);
 434         if (!ctx->addr) {
 435                 dprintf("Alloc failed\n");
 436                 return false;
 437         }
 438         ctx->size = size;
 439         ctx->type = type;
 440         memset((void *)ctx->addr, 0, size);
 441         ctx->dma_addr = SLOF_dma_map_in((void *)ctx->addr, size, false);
 442         dprintf("ctx %llx, ctx_dma %llx\n", ctx->addr, ctx->dma_addr);
 443         return true;
 444 }
 445
 446 static struct xhci_control_ctx *xhci_get_control_ctx(struct xhci_ctx *ctx)
 447 {
 448         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 449                 return (struct xhci_control_ctx *) ctx->addr;
 450         return NULL;
 451 }
 452
 453 static struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_ctx *ctx, uint32_t ctx_size)
 454 {
 455         uint32_t offset = 0;
 456
 457         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 458                 offset += ctx_size;
 459         return (struct xhci_slot_ctx *)(ctx->addr + offset);
 460 }
 461
 462 static struct xhci_ep_ctx *xhci_get_ep0_ctx(struct xhci_ctx *ctx, uint32_t ctx_size)
 463 {
 464         uint32_t offset = 0;
 465
 466         offset = ctx_size;
 467         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 468                 offset += ctx_size;
 469         return (struct xhci_ep_ctx *)(ctx->addr + offset);
 470 }
 471
 472 static struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_ctx *ctx, uint32_t ctx_size,
 473                                         uint32_t epno)
 474 {
 475         uint32_t offset = 0;
 476
 477         offset = ctx_size * epno;
 478         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 479                 offset += ctx_size;
 480         return (struct xhci_ep_ctx *)(ctx->addr + offset);
 481 }
 482
 483 static void xhci_free_ctx(struct xhci_ctx *ctx, uint32_t size)
 484 {
 485         SLOF_dma_map_out(ctx->dma_addr, (void *)ctx->addr, size);
 486         SLOF_dma_free((void *)ctx->addr, size);
 487 }
 488
 489 static uint32_t usb_control_max_packet(uint32_t speed)
 490 {
 491         uint32_t max_packet = 0;
 492
 493         switch(speed)
 494         {
 495         case USB_LOW_SPEED:
 496                 max_packet = 8;
 497                 break;
 498         case USB_FULL_SPEED:
 499                 max_packet = 8;
 500                 break;
 501         case USB_HIGH_SPEED:
 502                 max_packet = 64;
 503                 break;
 504         case USB_SUPER_SPEED:
 505                 max_packet = 512;
 506                 break;
 507         default:
 508                 /* should not reach here */
 509                 dprintf("Unknown speed\n");
 510         }
 511         return max_packet;
 512 }
 513
 514 static bool xhci_alloc_dev(struct xhci_hcd *xhcd, uint32_t slot_id, uint32_t port)
 515 {
 516         struct usb_dev *dev;
 517         struct xhci_dev *xdev;
 518         struct xhci_slot_ctx *slot;
 519         struct xhci_control_ctx *ctrl;
 520         struct xhci_ep_ctx *ep0;
 521         uint32_t ctx_size, val;
 522         uint16_t max_packet;
 523         uint32_t newport;
 524
 525         ctx_size = CTX_SIZE(xhcd->hcc_csz_64);
 526         xdev = &xhcd->xdevs[slot_id];
 527         xdev->slot_id = slot_id;
 528         xdev->ctx_size = ctx_size;
 529
 530         /* 4.3.3 Device Slot initialization */
 531         /* Step 1 */
 532         if (!xhci_alloc_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE, XHCI_CTX_TYPE_INPUT)) {
 533                 dprintf("Failed allocating in_ctx\n");
 534                 return false;
 535         }
 536
 537         /* Step 2 */
 538         ctrl = xhci_get_control_ctx(&xdev->in_ctx);
 539         ctrl->a_flags = cpu_to_le32(0x3);          /* A0, A1 */
 540         ctrl->d_flags = 0;
 541
 542         /* Step 3 */
 543         slot = xhci_get_slot_ctx(&xdev->in_ctx, ctx_size);
 544         newport = port + 1;
 545         val = LAST_CONTEXT(1) | SLOT_SPEED_SS | (newport << 16); /* FIXME speed, read from PS */
 546         slot->field1 = cpu_to_le32(val);
 547         slot->field2 = cpu_to_le32(ROOT_HUB_PORT(newport)); /* FIXME how to get port no */
 548
 549         /* Step 4 */
 550         if (!xhci_alloc_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE, TYPE_CTRL)) {
 551                 dprintf("Failed allocating control\n");
 552                 goto fail_in_ctx;
 553         }
 554
 555         /* Step 5 */
 556         ep0 = xhci_get_ep0_ctx(&xdev->in_ctx, ctx_size);
 557         val = 0;
 558         max_packet = usb_control_max_packet(USB_SUPER_SPEED);
 559         max_packet = 64;
 560         val = EP_TYPE(EP_CTRL) | MAX_BURST(0) | ERROR_COUNT(3) |
 561                 MAX_PACKET_SIZE(max_packet);
 562         ep0->field2 = cpu_to_le32(val);;
 563         ep0->deq_addr = cpu_to_le64(xdev->control.trbs_dma | xdev->control.cycle_state);
 564         ep0->field4 = cpu_to_le32(8);
 565
 566         /* Step 6 */
 567         if (!xhci_alloc_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE, XHCI_CTX_TYPE_DEVICE)) {
 568                 dprintf("Failed allocating out_ctx\n");
 569                 goto fail_control_seg;
 570         }
 571
 572         /* Step 7 */
 573         xhcd->dcbaap[slot_id] = cpu_to_le64(xdev->out_ctx.dma_addr);
 574
 575         /* Step 8 */
 576         slot = xhci_get_slot_ctx(&xdev->out_ctx, ctx_size);
 577         ep0 = xhci_get_ep0_ctx(&xdev->out_ctx, ctx_size);
 578
 579         dprintf("Slot State %x \n", SLOT_STATE(le32_to_cpu(slot->field4)));
 580         xhci_send_addr_device(xhcd, slot_id, xdev->in_ctx.dma_addr);
 581         mb();
 582         dprintf("Slot State %x \n", SLOT_STATE(le32_to_cpu(slot->field4)));
 583
 584         dprintf("EP0 f0 %08X f1 %08X %016lX %08X\n",
 585                 le32_to_cpu(ep0->field1),
 586                 le32_to_cpu(ep0->field2),
 587                 le64_to_cpu(ep0->deq_addr),
 588                 le32_to_cpu(ep0->field4));
 589
 590         /* Step 9 - configure ep */
 591         ctrl->a_flags = cpu_to_le32(0x1);          /* A0 */
 592         ctrl->d_flags = 0;
 593         xhci_configure_ep(xhcd, slot_id, xdev->in_ctx.dma_addr);
 594         mb();
 595         dprintf("Slot State %x \n", SLOT_STATE(le32_to_cpu(slot->field4)));
 596         dprintf("USB Device address %d \n", USB_DEV_ADDRESS(le32_to_cpu(slot->field4)));
 597         dprintf("EP0 f0 %08X f1 %08X %016lX %08X\n",
 598                 le32_to_cpu(ep0->field1),
 599                 le32_to_cpu(ep0->field2),
 600                 le64_to_cpu(ep0->deq_addr),
 601                 le32_to_cpu(ep0->field4));
 602
 603         dev = usb_devpool_get();
 604         dprintf("allocated device %p\n", dev);
 605         dev->hcidev = xhcd->hcidev;
 606         dev->speed = USB_SUPER_SPEED;
 607         dev->addr = USB_DEV_ADDRESS(slot->field4);
 608         dev->port = newport;
 609         dev->priv = xdev;
 610         xdev->dev = dev;
 611         if (usb_setup_new_device(dev, newport)) {
 612                 usb_slof_populate_new_device(dev);
 613                 return true;
 614         }
 615
 616         xhci_free_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE);
 617 fail_control_seg:
 618         xhci_free_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE);
 619 fail_in_ctx:
 620         xhci_free_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE);
 621         return false;
 622 }
 623
 624 static void xhci_free_dev(struct xhci_dev *xdev)
 625 {
 626         xhci_free_seg(&xdev->bulk_in, XHCI_DATA_TRBS_SIZE);
 627         xhci_free_seg(&xdev->bulk_out, XHCI_DATA_TRBS_SIZE);
 628         xhci_free_seg(&xdev->intr, XHCI_INTR_TRBS_SIZE);
 629         xhci_free_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE);
 630         xhci_free_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE);
 631         xhci_free_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE);
 632 }
 633
 634 static bool usb3_dev_init(struct xhci_hcd *xhcd, uint32_t port)
 635 {
 636         /* Device enable slot */
 637         xhci_send_enable_slot(xhcd, port);
 638         if (!xhcd->slot_id) {
 639                 dprintf("Unable to get slot id\n");
 640                 return false;
 641         }
 642         dprintf("SLOT ID: %d\n", xhcd->slot_id);
 643         if (!xhci_alloc_dev(xhcd, xhcd->slot_id, port)) {
 644                 dprintf("Unable to allocate device\n");
 645                 return false;
 646         }
 647         return true;
 648 }
 649
 650 static int xhci_device_present(uint32_t portsc, uint32_t usb_ver)
 651 {
 652         if (usb_ver == USB_XHCI) {
 653                 /* Device present and enabled state */
 654                 if ((portsc & PORTSC_CCS) &&
 655                         (portsc & PORTSC_PP) &&
 656                         (portsc & PORTSC_PED)) {
 657                         return true;
 658                 }
 659         } else if (usb_ver == USB_EHCI) {
 660                 /* Device present and in disabled state */
 661                 if ((portsc & PORTSC_CCS) && (portsc & PORTSC_CSC))
 662                         return true;
 663         }
 664         return false;
 665 }
 666
 667 static int xhci_port_scan(struct xhci_hcd *xhcd,
 668                         uint32_t usb_ver)
 669 {
 670         uint32_t num_ports, portsc, i;
 671         struct xhci_op_regs *op;
 672         struct xhci_port_regs *prs;
 673         struct xhci_cap_regs *cap;
 674         uint32_t xecp_off;
 675         uint32_t *xecp_addr, *base;
 676         uint32_t port_off = 0, port_cnt;
 677
 678         dprintf("enter\n");
 679
 680         op = xhcd->op_regs;
 681         cap = xhcd->cap_regs;
 682         port_cnt = num_ports = read_reg32(&cap->hcsparams1) >> 24;
 683
 684         /* Read the xHCI extented capability to find usb3 ports and offset*/
 685         xecp_off = XHCI_HCCPARAMS_XECP(read_reg32(&cap->hccparams));
 686         base = (uint32_t *)cap;
 687         while (xecp_off > 0) {
 688                 xecp_addr = base + xecp_off;
 689                 dprintf("xecp_off %d %p %p \n", xecp_off, base, xecp_addr);
 690
 691                 if (XHCI_XECP_CAP_ID(read_reg32(xecp_addr)) == XHCI_XECP_CAP_SP &&
 692                     XHCI_XECP_CAP_SP_MJ(read_reg32(xecp_addr)) == usb_ver &&
 693                     XHCI_XECP_CAP_SP_MN(read_reg32(xecp_addr)) == 0) {
 694                         port_cnt = XHCI_XECP_CAP_SP_PC(read_reg32(xecp_addr + 2));
 695                         port_off = XHCI_XECP_CAP_SP_PO(read_reg32(xecp_addr + 2));
 696                         dprintf("PortCount %d Portoffset %d\n", port_cnt, port_off);
 697                 }
 698                 base = xecp_addr;
 699                 xecp_off = XHCI_XECP_NEXT_PTR(read_reg32(xecp_addr));
 700         }
 701         if (port_off == 0) /* port_off should always start from 1 */
 702                 return false;
 703         for (i = (port_off - 1); i < (port_off + port_cnt - 1); i++) {
 704                 prs = &op->prs[i];
 705                 portsc = read_reg32(&prs->portsc);
 706                 if (xhci_device_present(portsc, usb_ver)) {
 707                         /* Device present */
 708                         dprintf("Device present on port %d\n", i);
 709                         /* Reset the port */
 710                         portsc = read_reg32(&prs->portsc);
 711                         portsc = portsc | PORTSC_PR;
 712                         write_reg32(&prs->portsc, portsc);
 713                         /* FIXME poll for port event */
 714                         SLOF_msleep(20);
 715                         xhci_poll_event(xhcd, 0);
 716                         portsc = read_reg32(&prs->portsc);
 717                         if (portsc & ~PORTSC_PRC) {
 718                                 dprintf("Port reset complete %d\n", i);
 719                         }
 720                         print_port_status(prs);
 721                         if (!usb3_dev_init(xhcd, (i - (port_off - 1)))) {
 722                                 dprintf("USB device initialization failed\n");
 723                         }
 724                 }
 725         }
 726         dprintf("exit\n");
 727         return true;
 728 }
 729
 730 static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 731 {
 732         return xhci_port_scan(xhcd, USB_XHCI) | xhci_port_scan(xhcd, USB_EHCI);
 733 }
 734
 735 static bool xhci_hcd_init(struct xhci_hcd *xhcd)
 736 {
 737         struct xhci_op_regs *op;
 738         struct xhci_int_regs *irs;
 739         uint64_t val;
 740         uint32_t reg;
 741
 742         if (!xhcd) {
 743                 dprintf("NULL pointer\n");
 744                 goto fail;
 745         }
 746
 747         op = xhcd->op_regs;
 748         irs = &xhcd->run_regs->irs[0];
 749         if (!xhci_hcd_reset(op)) {
 750                 dprintf("Reset failed\n");
 751                 goto fail;
 752         }
 753
 754         write_reg32(&op->config, XHCI_CONFIG_MAX_SLOT);
 755         reg = read_reg32(&xhcd->cap_regs->hccparams);
 756         /* 64byte context !! */
 757         xhcd->hcc_csz_64 = (reg & XHCI_HCCPARAMS_CSZ) ? 1 : 0;
 758
 759         if (xhcd->hcc_csz_64) {
 760                 printf("usb-xhci: 64 Byte context not supported\n");
 761                 goto fail;
 762         }
 763         /*
 764          * 6.1 Device Context Base Address Array
 765          *
 766          * Allocate memory and initialize
 767          */
 768         xhcd->dcbaap = (uint64_t *)SLOF_dma_alloc(XHCI_DCBAAP_MAX_SIZE);
 769         if (!xhcd->dcbaap) {
 770                 dprintf("Alloc failed\n");
 771                 goto fail;
 772         }
 773         memset((void *)xhcd->dcbaap, 0, XHCI_DCBAAP_MAX_SIZE);
 774         xhcd->dcbaap_dma = SLOF_dma_map_in((void *)xhcd->dcbaap,
 775                                         XHCI_DCBAAP_MAX_SIZE, false);
 776         dprintf("dcbaap %llx, dcbaap_phys %llx\n", xhcd->dcbaap, xhcd->dcbaap_dma);
 777         write_reg64(&op->dcbaap, xhcd->dcbaap_dma);
 778
 779         /*
 780          * Command Ring Control - TRB
 781          * FIXME - better way to allocate it...
 782          */
 783         if (!xhci_alloc_seg(&xhcd->crseg, XHCI_CRCR_CRP_SIZE, TYPE_COMMAND))
 784                 goto fail_dcbaap;
 785
 786         val = read_reg64(&op->crcr) & ~XHCI_CRCR_CRP_MASK;
 787         val = val | (xhcd->crseg.trbs_dma & XHCI_CRCR_CRP_MASK);
 788         write_reg64(&op->crcr, val);
 789
 790         /*
 791          * Event Ring Control - TRB
 792          * Allocate event TRBS
 793          */
 794         if (!xhci_alloc_seg(&xhcd->ering, XHCI_EVENT_TRBS_SIZE, TYPE_EVENT))
 795                 goto fail_crseg;
 796
 797         /*
 798          * Populate event ring segment table.
 799          * Note: only using one segment.
 800          */
 801         xhcd->erst.entries = SLOF_dma_alloc(XHCI_EVENT_TRBS_SIZE);
 802         if (!xhcd->erst.entries)
 803                 goto fail_ering;
 804         xhcd->erst.dma = SLOF_dma_map_in((void *)xhcd->erst.entries,
 805                                         XHCI_EVENT_TRBS_SIZE, false);
 806         xhcd->erst.num_segs = XHCI_ERST_NUM_SEGS;
 807
 808         /* populate entries[0] */
 809         write_reg64(&xhcd->erst.entries->addr, xhcd->ering.trbs_dma);
 810         write_reg32(&xhcd->erst.entries->size, xhcd->ering.size);
 811         write_reg32(&xhcd->erst.entries->reserved, 0);
 812
 813         /* populate erdp */
 814         val = read_reg64(&irs->erdp) & ~XHCI_ERDP_MASK;
 815         val = val | (xhcd->ering.trbs_dma & XHCI_ERDP_MASK);
 816         write_reg64(&irs->erdp, val);
 817
 818         /* populate erstsz */
 819         val = read_reg32(&irs->erstsz) & ~XHCI_ERST_SIZE_MASK;
 820         val = val | xhcd->erst.num_segs;
 821         write_reg32(&irs->erstsz, val);
 822
 823         /* Now write the erstba */
 824         val = read_reg64(&irs->erstba) & ~XHCI_ERST_ADDR_MASK;
 825         val = val | (xhcd->erst.dma & XHCI_ERST_ADDR_MASK);
 826         write_reg64(&irs->erstba, val);
 827
 828         dprintf("ERDP %llx TRB-DMA %llx\n", read_reg64(&irs->erdp),
 829                 xhcd->ering.trbs_dma);
 830         dprintf("ERST %llx, ERST DMA %llx, size %d\n",
 831                 (uint64_t)xhcd->erst.entries, xhcd->erst.dma,
 832                 xhcd->erst.num_segs);
 833
 834         mb();
 835         if (!xhci_hcd_set_runstop(op, true))
 836                 goto fail_erst_entries;
 837
 838         if (!xhci_hub_check_ports(xhcd))
 839                 goto fail_erst_entries;
 840
 841         return true;
 842 fail_erst_entries:
 843         write_reg64(&irs->erstba, 0);
 844         mb();
 845         SLOF_dma_map_out(xhcd->erst.dma, (void *)xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 846         SLOF_dma_free((void *)xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 847 fail_ering:
 848         xhci_free_seg(&xhcd->ering, XHCI_EVENT_TRBS_SIZE);
 849 fail_crseg:
 850         val = read_reg64(&op->crcr) & ~XHCI_CRCR_CRP_MASK;
 851         write_reg64(&op->crcr, val);
 852         mb();
 853         xhci_free_seg(&xhcd->crseg, XHCI_CRCR_CRP_SIZE);
 854 fail_dcbaap:
 855         write_reg64(&op->dcbaap, 0);
 856         mb();
 857         SLOF_dma_map_out(xhcd->dcbaap_dma, (void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 858         SLOF_dma_free((void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 859 fail:
 860         return false;
 861 }
 862
 863 static bool xhci_hcd_exit(struct xhci_hcd *xhcd)
 864 {
 865         struct xhci_op_regs *op;
 866         struct xhci_int_regs *irs;
 867         uint64_t val;
 868         int i;
 869
 870         if (!xhcd) {
 871                 dprintf("NULL pointer\n");
 872                 return false;
 873         }
 874         op = xhcd->op_regs;
 875
 876         if (!xhci_hcd_set_runstop(op, false)) {
 877                 dprintf("NULL pointer\n");
 878         }
 879
 880         for (i = 1; i < XHCI_CONFIG_MAX_SLOT; i++) {
 881                 if (xhcd->xdevs[i].dev)
 882                         xhci_free_dev(&xhcd->xdevs[i]);
 883         }
 884
 885         irs = &xhcd->run_regs->irs[0];
 886         write_reg64(&irs->erstba, 0);
 887         mb();
 888         if (xhcd->erst.entries) {
 889                 SLOF_dma_map_out(xhcd->erst.dma, xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 890                 SLOF_dma_free(xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 891         }
 892         xhci_free_seg(&xhcd->ering, XHCI_EVENT_TRBS_SIZE);
 893
 894         val = read_reg64(&op->crcr) & ~XHCI_CRCR_CRP_MASK;
 895         write_reg64(&op->crcr, val);
 896         xhci_free_seg(&xhcd->crseg, XHCI_CRCR_CRP_SIZE);
 897         write_reg64(&op->dcbaap, 0);
 898         if (xhcd->dcbaap) {
 899                 SLOF_dma_map_out(xhcd->dcbaap_dma, (void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 900                 SLOF_dma_free((void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 901         }
 902
 903         /*
 904          * QEMU implementation of XHCI doesn't implement halt
 905          * properly. It basically says that it's halted immediately
 906          * but doesn't actually terminate ongoing activities and
 907          * DMAs. This needs to be fixed in QEMU.
 908          *
 909          * For now, wait for 50ms grace time till qemu stops using
 910          * this device.
 911          */
 912         SLOF_msleep(50);
 913
 914         return true;
 915 }
 916
 917 static void xhci_init(struct usb_hcd_dev *hcidev)
 918 {
 919         struct xhci_hcd *xhcd;
 920
 921         printf("  XHCI: Initializing\n");
 922         dprintf("device base address %p\n", hcidev->base);
 923
 924         hcidev->base = (void *)((uint64_t)hcidev->base & ~7);
 925         xhcd = SLOF_alloc_mem(sizeof(*xhcd));
 926         if (!xhcd) {
 927                 printf("usb-xhci: Unable to allocate memory\n");
 928                 return;
 929         }
 930         memset(xhcd, 0, sizeof(*xhcd));
 931
 932         hcidev->nextaddr = 1;
 933         hcidev->priv = xhcd;
 934         xhcd->hcidev = hcidev;
 935         xhcd->cap_regs = (struct xhci_cap_regs *)(hcidev->base);
 936         xhcd->op_regs = (struct xhci_op_regs *)(hcidev->base +
 937                                                 read_reg8(&xhcd->cap_regs->caplength));
 938         xhcd->run_regs = (struct xhci_run_regs *)(hcidev->base +
 939                                                 read_reg32(&xhcd->cap_regs->rtsoff));
 940         xhcd->db_regs = (struct xhci_db_regs *)(hcidev->base +
 941                                                 read_reg32(&xhcd->cap_regs->dboff));
 942         dump_xhci_regs(xhcd);
 943         if (!xhci_hcd_init(xhcd))
 944                 printf("usb-xhci: failed to initialize XHCI controller.\n");
 945         dump_xhci_regs(xhcd);
 946 }
 947
 948 static void xhci_exit(struct usb_hcd_dev *hcidev)
 949 {
 950         struct xhci_hcd *xhcd;
 951
 952         dprintf("%s: enter \n", __func__);
 953         if (!hcidev && !hcidev->priv) {
 954                 return;
 955         }
 956
 957         xhcd = hcidev->priv;
 958         xhci_hcd_exit(xhcd);
 959         SLOF_free_mem(xhcd, sizeof(*xhcd));
 960         hcidev->priv = NULL;
 961 }
 962
 963 static void fill_trb_buff(struct xhci_command_trb *cmd,  uint32_t field1,
 964                         uint32_t field2, uint32_t field3, uint32_t field4)
 965 {
 966         uint32_t val, cycle_state;
 967
 968         cmd->field[0] = cpu_to_le32(field1);
 969         cmd->field[1] = cpu_to_le32(field2);
 970         cmd->field[2] = cpu_to_le32(field3);
 971
 972         val = le32_to_cpu(cmd->field[3]);
 973         cycle_state = (val & 0x1) ? 0 : 1;
 974         val =  cycle_state | (field4 & ~0x1);
 975         cmd->field[3] = cpu_to_le32(val);
 976
 977         dprintf("CMD %016lx val %08x cycle_state %d field1 %08x, field2  %08x, field3 %08x field4 %08x\n",
 978                 cmd, val, cycle_state,
 979                 le32_to_cpu(cmd->field[0]),
 980                 le32_to_cpu(cmd->field[1]),
 981                 le32_to_cpu(cmd->field[2]),
 982                 le32_to_cpu(cmd->field[3])
 983                 );
 984
 985         return;
 986 }
 987
 988 static void fill_setup_trb(struct xhci_command_trb *cmd, struct usb_dev_req *req,
 989                         uint32_t size)
 990 {
 991         uint32_t field1, field2, field3, field4 = 0;
 992         uint64_t req_raw;
 993         uint32_t datalen = 0, pid = 0;
 994
 995         req_raw = *((uint64_t *)req);
 996         dprintf("%lx %lx \n", *((uint64_t *)req), req_raw);
 997         /* req_raw is already in right byte order... */
 998         field1 = cpu_to_le32(TRB_ADDR_HIGH(req_raw));
 999         field2 = cpu_to_le32(TRB_ADDR_LOW(req_raw));
1000         field3 = 8; /* ALWAYS 8 */
1001
1002         datalen = cpu_to_le16(req->wLength);
1003         if (datalen) {
1004                 pid = (req->bmRequestType & REQT_DIR_IN) ? 3 : 2;
1005                 field4 = TRB_TRT(pid);
1006         }
1007         field4 |= TRB_CMD_TYPE(TRB_SETUP_STAGE) | TRB_IDT;
1008         fill_trb_buff(cmd, field1, field2, field3, field4);
1009 }
1010
1011 static void fill_setup_data(struct xhci_command_trb *cmd, void *data,
1012                         uint32_t size, uint32_t dir)
1013 {
1014         uint32_t field1, field2, field3, field4;
1015
1016         field1 = TRB_ADDR_LOW(data);
1017         field2 = TRB_ADDR_HIGH(data);
1018         field3 = size;
1019         if (dir)
1020                 field4 = TRB_DIR_IN;
1021         field4 |= TRB_CMD_TYPE(TRB_DATA_STAGE);
1022         fill_trb_buff(cmd, field1, field2, field3, field4);
1023 }
1024
1025 static void fill_status_trb(struct xhci_command_trb *cmd, uint32_t dir)
1026 {
1027         uint32_t field1, field2, field3, field4;
1028
1029         field1 = 0;
1030         field2 = 0;
1031         field3 = 0;
1032         if (dir)
1033                 field4 = TRB_DIR_IN;
1034
1035         field4 |= TRB_CMD_TYPE(TRB_STATUS_STAGE) | TRB_IOC;
1036         fill_trb_buff(cmd, field1, field2, field3, field4);
1037 }
1038
1039 static void fill_normal_trb(struct xhci_transfer_trb *trb, void *data,
1040                         uint32_t size)
1041 {
1042         uint32_t field1, field2, field3, field4;
1043
1044         field1 = TRB_ADDR_LOW(data);
1045         field2 = TRB_ADDR_HIGH(data);
1046         field3 = size;
1047         field4 = TRB_CMD_TYPE(TRB_NORMAL) | TRB_IOC;
1048         fill_trb_buff((struct xhci_command_trb *)trb, field1, field2, field3, field4);
1049 }
1050
1051 static int xhci_send_ctrl(struct usb_pipe *pipe, struct usb_dev_req *req, void *data)
1052 {
1053         struct xhci_dev *xdev;
1054         struct xhci_seg *ctrl;
1055         struct xhci_hcd *xhcd;
1056         struct xhci_command_trb *cmd;
1057         struct xhci_db_regs *dbr;
1058         long req_phys = 0, data_phys = 0;
1059         int ret = true;
1060         uint32_t slot_id, pid = 0, datalen = 0;
1061
1062         if (!pipe->dev || !pipe->dev->hcidev) {
1063                 dprintf(" NULL pointer\n");
1064                 return false;
1065         }
1066
1067         xdev = pipe->dev->priv;
1068         slot_id = xdev->slot_id;
1069         ctrl = &xdev->control;
1070         xhcd = (struct xhci_hcd *)pipe->dev->hcidev->priv;
1071         dbr = xhcd->db_regs;
1072         if (!ctrl || !xdev || !xhcd) {
1073                 dprintf(" NULL pointer\n");
1074                 return false;
1075         }
1076
1077         cmd = (struct xhci_command_trb *)ctrl->enq;
1078         req_phys = SLOF_dma_map_in(req, sizeof(struct usb_dev_req), true);
1079         fill_setup_trb(cmd, req, sizeof(*req));
1080
1081         cmd++;
1082         datalen = cpu_to_le16(req->wLength);
1083         if (datalen)
1084                 pid = 1;
1085         if (datalen) {
1086                 data_phys = SLOF_dma_map_in(data, datalen, true);
1087                 fill_setup_data(cmd, (void *) data_phys, datalen, pid);
1088                 cmd++;
1089         }
1090
1091         fill_status_trb(cmd, pid);
1092         cmd++;
1093
1094         /* Ring the doorbell - ep0 */
1095         write_reg32(&dbr->db[slot_id], 1);
1096         if (!xhci_poll_event(xhcd, 0)) {
1097                 dprintf("Command failed\n");
1098                 ret = false;
1099         }
1100         ctrl->enq = (uint64_t) cmd;
1101         SLOF_dma_map_out(req_phys, req, sizeof(struct usb_dev_req));
1102         if (datalen)
1103                 SLOF_dma_map_out(data_phys, data, datalen);
1104         return ret;
1105 }
1106
1107 static inline struct xhci_pipe *xhci_pipe_get_xpipe(struct usb_pipe *pipe)
1108 {
1109         struct xhci_pipe *xpipe;
1110         xpipe = container_of(pipe, struct xhci_pipe, pipe);
1111         dprintf("%s: xpipe is %p\n", __func__, xpipe);
1112         return xpipe;
1113 }
1114
1115 static inline struct xhci_seg *xhci_pipe_get_seg(struct usb_pipe *pipe)
1116 {
1117         struct xhci_pipe *xpipe;
1118         xpipe = xhci_pipe_get_xpipe(pipe);
1119         return xpipe->seg;
1120 }
1121
1122 static inline void *xhci_get_trb(struct xhci_seg *seg)
1123 {
1124         uint64_t val, enq;
1125         int index;
1126         struct xhci_link_trb *link;
1127
1128         enq = val = seg->enq;
1129         val = val + XHCI_TRB_SIZE;
1130         index = (enq - (uint64_t)seg->trbs) / XHCI_TRB_SIZE + 1;
1131         dprintf("%s: enq %llx, val %llx %x\n", __func__, enq, val, index);
1132         /* TRBs being a cyclic buffer, here we cycle back to beginning. */
1133         if (index == (seg->size - 1)) {
1134                 dprintf("%s: rounding \n", __func__);
1135                 seg->enq = (uint64_t)seg->trbs;
1136                 seg->cycle_state ^= seg->cycle_state;
1137                 link = (struct xhci_link_trb *) (seg->trbs + seg->size - 1);
1138                 link->addr = cpu_to_le64(seg->trbs_dma);
1139                 link->field2 = 0;
1140                 link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
1141                 mb();
1142         }
1143         else {
1144                 seg->enq = seg->enq + XHCI_TRB_SIZE;
1145         }
1146
1147         return (void *)enq;
1148 }
1149
1150 static uint64_t xhci_get_trb_phys(struct xhci_seg *seg, uint64_t trb)
1151 {
1152         return seg->trbs_dma + (trb - (uint64_t)seg->trbs);
1153 }
1154
1155 static int usb_kb = false;
1156 static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
1157                         void *data, int datalen)
1158 {
1159         struct xhci_dev *xdev;
1160         struct xhci_seg *seg;
1161         struct xhci_hcd *xhcd;
1162         struct xhci_transfer_trb *trb;
1163         struct xhci_db_regs *dbr;
1164         int ret = true;
1165         uint32_t slot_id, epno, time;
1166         uint64_t trb_phys, event_phys;
1167
1168         if (!pipe->dev || !pipe->dev->hcidev) {
1169                 dprintf(" NULL pointer\n");
1170                 dprintf(" pipe dev %p hcidev %p\n", pipe->dev, pipe->dev->hcidev);
1171                 return false;
1172         }
1173
1174         xdev = pipe->dev->priv;
1175         slot_id = xdev->slot_id;
1176         seg = xhci_pipe_get_seg(pipe);
1177         xhcd = (struct xhci_hcd *)pipe->dev->hcidev->priv;
1178         dbr = xhcd->db_regs;
1179         if (!seg || !xdev || !xhcd) {
1180                 dprintf(" NULL pointer\n");
1181                 dprintf(" seg %p xdev %p xhcd %p\n", seg, xdev, xhcd);
1182                 return false;
1183         }
1184
1185         if (datalen > XHCI_MAX_BULK_SIZE) {
1186                 printf("usb-xhci: bulk transfer size too big\n");
1187                 return false;
1188         }
1189
1190         trb = xhci_get_trb(seg);
1191         trb_phys = xhci_get_trb_phys(seg, (uint64_t)trb);
1192         fill_normal_trb(trb, (void *)data, datalen);
1193
1194         epno = xhci_get_epno(pipe);
1195         write_reg32(&dbr->db[slot_id], epno);
1196
1197         time = SLOF_GetTimer() + USB_TIMEOUT;
1198         while (1) {
1199                 event_phys = xhci_poll_event(xhcd, 0);
1200                 if (event_phys == trb_phys) {
1201                         break;
1202                 } else if (event_phys == 0) { /* polling timed out */
1203                         ret = false;
1204                         break;
1205                 } else
1206                         usb_kb = true;
1207
1208                 /* transfer timed out */
1209                 if (time < SLOF_GetTimer())
1210                         return false;
1211         }
1212         trb->addr = 0;
1213         trb->len = 0;
1214         trb->flags = 0;
1215         mb();
1216
1217         return ret;
1218 }
1219
1220 static int xhci_alloc_pipe_pool(struct xhci_hcd *xhcd)
1221 {
1222         struct xhci_pipe *xpipe, *curr, *prev;
1223         unsigned int i, count;
1224         long xpipe_phys = 0;
1225
1226         count = XHCI_PIPE_POOL_SIZE/sizeof(*xpipe);
1227         xhcd->pool = xpipe = SLOF_dma_alloc(XHCI_PIPE_POOL_SIZE);
1228         if (!xpipe)
1229                 return -1;
1230         xhcd->pool_phys = xpipe_phys = SLOF_dma_map_in(xpipe, XHCI_PIPE_POOL_SIZE, true);
1231         dprintf("%s: xpipe %p, xpipe_phys %lx\n", __func__, xpipe, xpipe_phys);
1232
1233         /* Although an array, link them */
1234         for (i = 0, curr = xpipe, prev = NULL; i < count; i++, curr++) {
1235                 if (prev)
1236                         prev->pipe.next = &curr->pipe;
1237                 curr->pipe.next = NULL;
1238                 prev = curr;
1239         }
1240
1241         if (!xhcd->freelist)
1242                 xhcd->freelist = &xpipe->pipe;
1243         else
1244                 xhcd->end->next = &xpipe->pipe;
1245         xhcd->end = &prev->pipe;
1246
1247         return 0;
1248 }
1249
1250 static void xhci_init_bulk_ep(struct usb_dev *dev, struct usb_pipe *pipe)
1251 {
1252         struct xhci_hcd *xhcd;
1253         struct xhci_dev *xdev;
1254         struct xhci_seg *seg;
1255         struct xhci_pipe *xpipe;
1256         struct xhci_control_ctx *ctrl;
1257         struct xhci_ep_ctx *ep;
1258         uint32_t x_epno, val, type;
1259
1260         if (!pipe || !dev || !dev->priv)
1261                 return;
1262
1263         xdev = dev->priv;
1264         xhcd = dev->hcidev->priv;
1265         dprintf("dir %d\n", pipe->dir);
1266         seg = xhci_pipe_get_seg(pipe);
1267         xpipe = xhci_pipe_get_xpipe(pipe);
1268         if (pipe->dir) {
1269                 type = EP_BULK_IN;
1270                 seg = &xdev->bulk_in;
1271         }
1272         else {
1273                 type = EP_BULK_OUT;
1274                 seg = &xdev->bulk_out;
1275         }
1276
1277         if (!seg->trbs) {
1278                 if (!xhci_alloc_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK)) {
1279                         printf("usb-xhci: allocation failed for bulk endpoint\n");
1280                         return;
1281                 }
1282         } else {
1283                 xhci_init_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK);
1284         }
1285
1286         pipe->mps = XHCI_MAX_BULK_SIZE;
1287         ctrl = xhci_get_control_ctx(&xdev->in_ctx);
1288         x_epno = xhci_get_epno(pipe);
1289         ep = xhci_get_ep_ctx(&xdev->in_ctx, xdev->ctx_size, x_epno);
1290         val = EP_TYPE(type) | MAX_BURST(0) | ERROR_COUNT(3) |
1291                 MAX_PACKET_SIZE(pipe->mps);
1292         ep->field2 = cpu_to_le32(val);;
1293         ep->deq_addr = cpu_to_le64(seg->trbs_dma | seg->cycle_state);
1294         ep->field4 = cpu_to_le32(8);
1295         ctrl->a_flags = cpu_to_le32(BIT(x_epno) | 0x1);
1296         ctrl->d_flags = 0;
1297         xhci_configure_ep(xhcd, xdev->slot_id, xdev->in_ctx.dma_addr);
1298         xpipe->seg = seg;
1299 }
1300
1301 static int xhci_get_pipe_intr(struct usb_pipe *pipe,
1302                         struct xhci_hcd *xhcd,
1303                         char *buf, size_t len)
1304 {
1305         struct xhci_dev *xdev;
1306         struct xhci_seg *seg;
1307         struct xhci_pipe *xpipe;
1308         struct xhci_control_ctx *ctrl;
1309         struct xhci_ep_ctx *ep;
1310         uint32_t x_epno, val, type;
1311         struct usb_dev *dev;
1312         struct xhci_transfer_trb *trb;
1313
1314         dev = pipe->dev;
1315         if (dev->class != DEV_HID_KEYB)
1316                 return false;
1317
1318         xdev = dev->priv;
1319         pipe->mps = 8;
1320         seg = xhci_pipe_get_seg(pipe);
1321         xpipe = xhci_pipe_get_xpipe(pipe);
1322         type = EP_INT_IN;
1323         seg = &xdev->intr;
1324
1325         if (!seg->trbs) {
1326                 if (!xhci_alloc_seg(seg, XHCI_INTR_TRBS_SIZE, TYPE_BULK)) {
1327                         printf("usb-xhci: allocation failed for interrupt endpoint\n");
1328                         return false;
1329                 }
1330         } else {
1331                 xhci_init_seg(seg, XHCI_EVENT_TRBS_SIZE, TYPE_BULK);
1332         }
1333
1334         xpipe->buf = buf;
1335         xpipe->buf_phys = SLOF_dma_map_in(buf, len, false);
1336         xpipe->buflen = len;
1337
1338         ctrl = xhci_get_control_ctx(&xdev->in_ctx);
1339         x_epno = xhci_get_epno(pipe);
1340         ep = xhci_get_ep_ctx(&xdev->in_ctx, xdev->ctx_size, x_epno);
1341         val = EP_TYPE(type) | MAX_BURST(0) | ERROR_COUNT(3) |
1342                 MAX_PACKET_SIZE(pipe->mps);
1343         ep->field2 = cpu_to_le32(val);
1344         ep->deq_addr = cpu_to_le64(seg->trbs_dma | seg->cycle_state);
1345         ep->field4 = cpu_to_le32(8);
1346         ctrl->a_flags = cpu_to_le32(BIT(x_epno) | 0x1);
1347         ctrl->d_flags = 0;
1348         xhci_configure_ep(xhcd, xdev->slot_id, xdev->in_ctx.dma_addr);
1349         xpipe->seg = seg;
1350
1351         trb = xhci_get_trb(seg);
1352         fill_normal_trb(trb, (void *)xpipe->buf_phys, pipe->mps);
1353         return true;
1354 }
1355
1356 static struct usb_pipe* xhci_get_pipe(struct usb_dev *dev, struct usb_ep_descr *ep, char *buf, size_t len)
1357 {
1358         struct xhci_hcd *xhcd;
1359         struct usb_pipe *new = NULL;
1360
1361         if (!dev)
1362                 return NULL;
1363
1364         xhcd = (struct xhci_hcd *)dev->hcidev->priv;
1365         if (!xhcd->freelist) {
1366                 dprintf("usb-xhci: %s allocating pool\n", __func__);
1367                 if (xhci_alloc_pipe_pool(xhcd))
1368                         return NULL;
1369         }
1370
1371         new = xhcd->freelist;
1372         xhcd->freelist = xhcd->freelist->next;
1373         if (!xhcd->freelist)
1374                 xhcd->end = NULL;
1375
1376         memset(new, 0, sizeof(*new));
1377         new->dev = dev;
1378         new->next = NULL;
1379         new->type = ep->bmAttributes & USB_EP_TYPE_MASK;
1380         new->speed = dev->speed;
1381         new->mps = ep->wMaxPacketSize;
1382         new->dir = (ep->bEndpointAddress & 0x80) >> 7;
1383         new->epno = ep->bEndpointAddress & 0x0f;
1384
1385         if (new->type == USB_EP_TYPE_INTR) {
1386                 if (!xhci_get_pipe_intr(new, xhcd, buf, len)) {
1387                         printf("usb-xhci: %s alloc_intr failed  %p\n",
1388                                 __func__, new);
1389                 }
1390         }
1391         if (new->type == USB_EP_TYPE_BULK)
1392                 xhci_init_bulk_ep(dev, new);
1393
1394         return new;
1395 }
1396
1397 static void xhci_put_pipe(struct usb_pipe *pipe)
1398 {
1399         struct xhci_hcd *xhcd;
1400         struct xhci_pipe *xpipe;
1401
1402         dprintf("usb-xhci: %s enter - %p\n", __func__, pipe);
1403         if (!pipe || !pipe->dev)
1404                 return;
1405         xhcd = pipe->dev->hcidev->priv;
1406
1407         dprintf("dir %d\n", pipe->dir);
1408         if (pipe->type == USB_EP_TYPE_BULK) {
1409                 xpipe = xhci_pipe_get_xpipe(pipe);
1410                 xpipe->seg = NULL;
1411         } else if (pipe->type == USB_EP_TYPE_INTR) {
1412                 xpipe = xhci_pipe_get_xpipe(pipe);
1413                 SLOF_dma_map_out(xpipe->buf_phys, xpipe->buf, xpipe->buflen);
1414                 xpipe->seg = NULL;
1415         }
1416         if (xhcd->end)
1417                 xhcd->end->next = pipe;
1418         else
1419                 xhcd->freelist = pipe;
1420
1421         xhcd->end = pipe;
1422         pipe->next = NULL;
1423         pipe->dev = NULL;
1424         memset(pipe, 0, sizeof(*pipe));
1425
1426         dprintf("usb-xhci: %s exit\n", __func__);
1427 }
1428
1429 static int xhci_poll_intr(struct usb_pipe *pipe, uint8_t *data)
1430 {
1431         struct xhci_transfer_trb *trb;
1432         struct xhci_seg *seg;
1433         struct xhci_pipe *xpipe;
1434         struct xhci_dev *xdev;
1435         struct xhci_hcd *xhcd;
1436         struct xhci_db_regs *dbr;
1437         uint32_t x_epno;
1438         uint8_t *buf, ret = 1;
1439
1440         if (!pipe || !pipe->dev || !pipe->dev->hcidev)
1441                 return 0;
1442         xdev = pipe->dev->priv;
1443         xhcd = (struct xhci_hcd *)pipe->dev->hcidev->priv;
1444         x_epno = xhci_get_epno(pipe);
1445         seg = xhci_pipe_get_seg(pipe);
1446         xpipe = xhci_pipe_get_xpipe(pipe);
1447
1448         if (usb_kb == true) {
1449                 /* This event was consumed by bulk transfer */
1450                 usb_kb = false;
1451                 goto skip_poll;
1452         }
1453         buf = xpipe->buf;
1454         memset(buf, 0, 8);
1455
1456         mb();
1457         /* Ring the doorbell - x_epno */
1458         dbr = xhcd->db_regs;
1459         write_reg32(&dbr->db[xdev->slot_id], x_epno);
1460         if (!xhci_poll_event(xhcd, 0)) {
1461                 printf("poll intr failed\n");
1462                 return 0;
1463         }
1464         mb();
1465         memcpy(data, buf, 8);
1466
1467 skip_poll:
1468         trb = xhci_get_trb(seg);
1469         fill_normal_trb(trb, (void *)xpipe->buf_phys, pipe->mps);
1470         mb();
1471         return ret;
1472 }
1473
1474 struct usb_hcd_ops xhci_ops = {
1475         .name          = "xhci-hcd",
1476         .init          = xhci_init,
1477         .exit          = xhci_exit,
1478         .usb_type      = USB_XHCI,
1479         .get_pipe      = xhci_get_pipe,
1480         .put_pipe      = xhci_put_pipe,
1481         .poll_intr     = xhci_poll_intr,
1482         .send_ctrl     = xhci_send_ctrl,
1483         .transfer_bulk = xhci_transfer_bulk,
1484         .next          = NULL,
1485 };
1486
1487 void usb_xhci_register(void)
1488 {
1489         usb_hcd_register(&xhci_ops);
1490 }