qemu/roms/SLOF/lib/libusb/usb-xhci.c

   1 /*****************************************************************************
   2  * Copyright (c) 2013 IBM Corporation
   3  * All rights reserved.
   4  * This program and the accompanying materials
   5  * are made available under the terms of the BSD License
   6  * which accompanies this distribution, and is available at
   7  * http://www.opensource.org/licenses/bsd-license.php
   8  *
   9  * Contributors:
  10  *     IBM Corporation - initial implementation
  11  *****************************************************************************/
  12
  13 #include <string.h>
  14 #include "usb.h"
  15 #include "usb-core.h"
  16 #include "usb-xhci.h"
  17 #include "tools.h"
  18 #include "paflof.h"
  19
  20 #undef XHCI_DEBUG
  21 //#define XHCI_DEBUG
  22 #ifdef XHCI_DEBUG
  23 #define dprintf(_x ...) do { printf("%s: ", __func__); printf(_x); } while (0)
  24 #else
  25 #define dprintf(_x ...)
  26 #endif
  27
  28 static void dump_xhci_regs(struct xhci_hcd *xhcd)
  29 {
  30 #ifdef XHCI_DEBUG
  31         struct xhci_cap_regs *cap;
  32         struct xhci_op_regs *op;
  33         struct xhci_run_regs *run;
  34
  35         cap = xhcd->cap_regs;
  36         op = xhcd->op_regs;
  37         run = xhcd->run_regs;
  38
  39         dprintf("\n");
  40         dprintf(" - CAPLENGTH           %02X\n", read_reg8 (&cap->caplength));
  41         dprintf(" - HCIVERSION          %04X\n", read_reg16(&cap->hciversion));
  42         dprintf(" - HCSPARAMS1          %08X\n", read_reg32(&cap->hcsparams1));
  43         dprintf(" - HCSPARAMS2          %08X\n", read_reg32(&cap->hcsparams2));
  44         dprintf(" - HCSPARAMS3          %08X\n", read_reg32(&cap->hcsparams3));
  45         dprintf(" - HCCPARAMS           %08X\n", read_reg32(&cap->hccparams));
  46         dprintf(" - DBOFF               %08X\n", read_reg32(&cap->dboff));
  47         dprintf(" - RTSOFF              %08X\n", read_reg32(&cap->rtsoff));
  48         dprintf("\n");
  49
  50         dprintf(" - USBCMD              %08X\n", read_reg32(&op->usbcmd));
  51         dprintf(" - USBSTS              %08X\n", read_reg32(&op->usbsts));
  52         dprintf(" - PAGESIZE            %08X\n", read_reg32(&op->pagesize));
  53         dprintf(" - DNCTRL              %08X\n", read_reg32(&op->dnctrl));
  54         dprintf(" - CRCR              %016llX\n", read_reg64(&op->crcr));
  55         dprintf(" - DCBAAP            %016llX\n", read_reg64(&op->dcbaap));
  56         dprintf(" - CONFIG              %08X\n", read_reg32(&op->config));
  57         dprintf("\n");
  58
  59         dprintf(" - MFINDEX             %08X\n", read_reg32(&run->mfindex));
  60         dprintf("\n");
  61 #endif
  62 }
  63
  64 static void print_port_status(struct xhci_port_regs *prs)
  65 {
  66 #ifdef XHCI_DEBUG
  67         uint32_t portsc;
  68         uint32_t CCS, PED, PP, PLS, i, PR = 0;
  69
  70         portsc = read_reg32(&prs->portsc);
  71         dprintf("portsc %08x portpmsc %08x portli %08x\n",
  72                 portsc,
  73                 read_reg32(&prs->portpmsc),
  74                 read_reg32(&prs->portli));
  75
  76         if (portsc & PORTSC_CCS) {
  77                 printf("CCS ");
  78                 CCS = 1;
  79         }
  80         if (portsc & PORTSC_PED) {
  81                 printf("PED ");
  82                 PED = 1;
  83         }
  84         if (portsc & PORTSC_OCA)
  85                 printf("OCA ");
  86         if (portsc & PORTSC_PR)
  87                 printf("OCA ");
  88         PLS = (portsc & PORTSC_PLS_MASK) >> 5;
  89         printf("PLS:%d ", PLS);
  90         if (portsc & PORTSC_PP) {
  91                 printf("PP ");
  92                 PP = 1;
  93         }
  94         printf("PS:%d ", (portsc & PORTSC_PS_MASK) >> 10);
  95         printf("PIC:%d ", (portsc & PORTSC_PIC_MASK) >> 14);
  96         if (portsc & PORTSC_LWS)
  97                 printf("LWS ");
  98         if (portsc & PORTSC_CSC)
  99                 printf("CSC ");
 100         if (portsc & PORTSC_PEC)
 101                 printf("PEC ");
 102         if (portsc & PORTSC_WRC)
 103                 printf("WRC ");
 104         if (portsc & PORTSC_OCC)
 105                 printf("OCC ");
 106         if (portsc & PORTSC_PRC)
 107                 printf("PRC ");
 108         if (portsc & PORTSC_PLC)
 109                 printf("PLC ");
 110         if (portsc & PORTSC_CEC)
 111                 printf("CEC ");
 112         if (portsc & PORTSC_CAS)
 113                 printf("CAS ");
 114         if (portsc & PORTSC_WCE)
 115                 printf("WCE ");
 116         if (portsc & PORTSC_WDE)
 117                 printf("WDE ");
 118         if (portsc & PORTSC_WOE)
 119                 printf("WOE ");
 120         if (portsc & PORTSC_DR)
 121                 printf("DR ");
 122         if (portsc & PORTSC_WPR)
 123                 printf("WPR ");
 124         printf("\n");
 125
 126         for (i = 0 ; i < (sizeof(ps_array_usb3)/sizeof(struct port_state)); i++) {
 127                 if (PP == ps_array_usb3[i].PP) {
 128                         if (CCS == ps_array_usb3[i].CCS) {
 129                                 if (PED == ps_array_usb3[i].PED) {
 130                                         if (PR == ps_array_usb3[i].PR) {
 131                                                 dprintf("%s - PLS %d\n", ps_array_usb3[i].state, PLS);
 132                                                 break;
 133                                         }
 134                                 }
 135                         }
 136                 }
 137         }
 138 #endif
 139
 140 }
 141
 142 static inline bool xhci_is_hc_ready(uint32_t *usbsts)
 143 {
 144         return !(read_reg32(usbsts) & XHCI_USBSTS_CNR);
 145 }
 146
 147 static inline bool xhci_wait_for_cnr(uint32_t *usbsts)
 148 {
 149         /* Standard:
 150          * Note: The xHC should halt within 16 ms. of software clearing the
 151          * R/S bit to ‘0’.
 152          * Give some more time... 32ms
 153          */
 154         int count = 320;
 155         dprintf("Waiting for Controller ready ..");
 156         while (!xhci_is_hc_ready(usbsts)) {
 157                 dprintf(".");
 158                 count--;
 159                 if (!count) {
 160                         dprintf("  failed %08X\n", read_reg32(usbsts));
 161                         return false;
 162                 }
 163                 SLOF_usleep(100);
 164         }
 165         dprintf("  done\n");
 166         return true;
 167 }
 168
 169 static bool xhci_hcd_set_runstop(struct xhci_op_regs *op, bool run_req)
 170 {
 171         uint32_t reg;
 172
 173         dprintf("Request %s\n", run_req ? "RUN" : "STOP");
 174         if (!xhci_is_hc_ready(&op->usbsts)) {
 175                 dprintf("Controller not ready\n");
 176                 return false;
 177         }
 178
 179         reg = read_reg32(&op->usbcmd);
 180         if (run_req)
 181                 reg |= run_req;
 182         else
 183                 reg &= (uint32_t)~1;
 184         dprintf("writing %08X\n", reg);
 185         write_reg32(&op->usbcmd, reg);
 186         mb();
 187         xhci_wait_for_cnr(&op->usbsts);
 188         return true;
 189 }
 190
 191 static bool xhci_hcd_reset(struct xhci_op_regs *op)
 192 {
 193         uint32_t reg;
 194
 195         /* Check if the controller is halted, else halt it */
 196         if (!(read_reg32(&op->usbsts) & XHCI_USBSTS_HCH)) {
 197                 dprintf("HCHalted not set\n");
 198                 if (!xhci_hcd_set_runstop(op, false))
 199                         return false;
 200         }
 201
 202         if (read_reg32(&op->usbsts) & XHCI_USBSTS_CNR) {
 203                 dprintf("Controller not ready\n");
 204                 return false;
 205         }
 206
 207         reg = read_reg32(&op->usbcmd) | XHCI_USBCMD_HCRST;
 208         /* Ready to Reset the controller now */
 209         write_reg32(&op->usbcmd, reg);
 210         xhci_wait_for_cnr(&op->usbsts);
 211         return true;
 212 }
 213
 214 static void xhci_handle_cmd_completion(struct xhci_hcd *xhcd,
 215                                 struct xhci_event_trb *event)
 216 {
 217         uint32_t flags, slot_id, status;
 218
 219         status = le32_to_cpu(event->status);
 220         flags = le32_to_cpu(event->flags);
 221         slot_id = TRB_SLOT_ID(flags);
 222         if (TRB_STATUS(status) == COMP_SUCCESS)
 223                 xhcd->slot_id = slot_id;
 224         else
 225                 xhcd->slot_id = 0;
 226 }
 227
 228 static struct xhci_event_trb *xhci_poll_event(struct xhci_hcd *xhcd,
 229                                         uint32_t event_type)
 230 {
 231         struct xhci_event_trb *event;
 232         uint64_t val;
 233         uint32_t flags, time;
 234         int index;
 235
 236         mb();
 237         event = (struct xhci_event_trb *)xhcd->ering.deq;
 238         flags = le32_to_cpu(event->flags);
 239
 240         dprintf("Reading from event ptr %p %08x\n", event, flags);
 241         time = SLOF_GetTimer() + USB_TIMEOUT;
 242
 243         while ((flags & TRB_CYCLE_STATE) != xhcd->ering.cycle_state) {
 244                 mb();
 245                 flags = le32_to_cpu(event->flags);
 246                 if (time < SLOF_GetTimer())
 247                         return NULL;
 248         }
 249
 250         mb();
 251         flags = le32_to_cpu(event->flags);
 252         switch(TRB_TYPE(flags))
 253         {
 254         case TRB_CMD_COMPLETION:
 255                 dprintf("CMD Completion\n");
 256                 xhci_handle_cmd_completion(xhcd, event);
 257                 break;
 258         case TRB_PORT_STATUS:
 259                 dprintf("Port status event\n");
 260                 break;
 261         case TRB_TRANSFER_EVENT:
 262                 dprintf("XFER event addr %16lx, status %08x, flags %08x\n",
 263                         le64_to_cpu(event->addr),
 264                         le32_to_cpu(event->status),
 265                         le32_to_cpu(event->flags));
 266                 break;
 267         default:
 268                 printf("TRB_TYPE  %d\n", TRB_TYPE(flags));
 269                 dprintf("Event addr %16lx, status %08x, flags %08x state %d\n",
 270                         le64_to_cpu(event->addr),
 271                         le32_to_cpu(event->status),
 272                         flags, xhcd->ering.cycle_state);
 273                 break;
 274         }
 275         xhcd->ering.deq = (uint64_t) (event + 1);
 276
 277         event->addr = 0;
 278         event->status = 0;
 279         event->flags = cpu_to_le32(xhcd->ering.cycle_state);
 280
 281         index = xhcd->ering.deq - (uint64_t)xhcd->ering.trbs;
 282         val = xhcd->ering.trbs_dma;
 283         val += (index % XHCI_EVENT_TRBS_SIZE);
 284         if (!(index % XHCI_EVENT_TRBS_SIZE)) {
 285                 xhcd->ering.deq = (uint64_t)xhcd->ering.trbs;
 286                 xhcd->ering.cycle_state = xhcd->ering.cycle_state ? 0 : 1;
 287                 dprintf("Rounding %d\n", xhcd->ering.cycle_state);
 288         }
 289         dprintf("Update start %x deq %x index %d\n",
 290                 xhcd->ering.trbs_dma, val, index/sizeof(*event));
 291         write_reg64(&xhcd->run_regs->irs[0].erdp, val);
 292         return event;
 293 }
 294
 295 static void xhci_send_cmd(struct xhci_hcd *xhcd, uint32_t field1,
 296                         uint32_t field2, uint32_t field3, uint32_t field4)
 297 {
 298         struct xhci_db_regs *dbr;
 299         struct xhci_command_trb *cmd;
 300         uint32_t val, cycle_state;
 301
 302         dbr = xhcd->db_regs;
 303         cmd = (struct xhci_command_trb *)xhcd->crseg.enq;
 304
 305         cmd->field[0] = cpu_to_le32(field1);
 306         cmd->field[1] = cpu_to_le32(field2);
 307         cmd->field[2] = cpu_to_le32(field3);
 308
 309         val = le32_to_cpu(cmd->field[3]);
 310         cycle_state = (val & 0x1) ? 0 : 1;
 311         val = field4 | cycle_state;
 312         cmd->field[3] = cpu_to_le32(val);
 313
 314         dprintf("CMD %016lx val %08x cycle_state %d field1 %08x, field2  %08x, field3 %08x field4 %08x\n",
 315                 cmd, val, cycle_state,
 316                 le32_to_cpu(cmd->field[0]),
 317                 le32_to_cpu(cmd->field[1]),
 318                 le32_to_cpu(cmd->field[2]),
 319                 le32_to_cpu(cmd->field[3])
 320                 );
 321
 322         /* Ring the doorbell */
 323         write_reg32(&dbr->db[0], 0);
 324         xhci_poll_event(xhcd, 0);
 325         cmd++;
 326         xhcd->crseg.enq = (uint64_t)cmd;
 327         return;
 328 }
 329
 330 static void xhci_send_enable_slot(struct xhci_hcd *xhcd, uint32_t port)
 331 {
 332         uint32_t field1, field2, field3, field4;
 333
 334         field1 = 0;
 335         field2 = 0;
 336         field3 = 0;
 337         field4 = TRB_CMD_TYPE(TRB_ENABLE_SLOT);
 338         xhci_send_cmd(xhcd, field1, field2, field3, field4);
 339 }
 340
 341 static void xhci_send_addr_device(struct xhci_hcd *xhcd, uint32_t slot_id,
 342                         uint64_t dma_in_ctx)
 343 {
 344         uint32_t field1, field2, field3, field4;
 345
 346         dprintf("Address device %lx, low %x, high %x\n", dma_in_ctx,
 347                 TRB_ADDR_LOW(dma_in_ctx),
 348                 TRB_ADDR_HIGH(dma_in_ctx));
 349         field1 = TRB_ADDR_LOW(dma_in_ctx) & ~0xF;
 350         field2 = TRB_ADDR_HIGH(dma_in_ctx);
 351         field3 = 0;
 352         field4 = TRB_CMD_TYPE(TRB_ADDRESS_DEV) | TRB_CMD_SLOT_ID(slot_id);
 353         xhci_send_cmd(xhcd, field1, field2, field3, field4);
 354 }
 355
 356 static uint32_t xhci_get_epno(struct usb_pipe *pipe)
 357 {
 358         uint32_t x_epno;
 359         x_epno = pipe->dir | 2 * pipe->epno;
 360         dprintf("EPno %d:%d DIR %d\n", pipe->epno, x_epno, pipe->dir);
 361         return x_epno;
 362 }
 363
 364 static void xhci_configure_ep(struct xhci_hcd *xhcd, uint32_t slot_id,
 365                         uint64_t dma_in_ctx)
 366 {
 367         uint32_t field1, field2, field3, field4;
 368
 369         dprintf("Configure EP %lx, low %x, high %x\n", dma_in_ctx,
 370                 TRB_ADDR_LOW(dma_in_ctx),
 371                 TRB_ADDR_HIGH(dma_in_ctx));
 372         field1 = TRB_ADDR_LOW(dma_in_ctx) & ~0xF;
 373         field2 = TRB_ADDR_HIGH(dma_in_ctx);
 374         field3 = 0;
 375         field4 = TRB_CMD_TYPE(TRB_CONFIG_EP) | TRB_CMD_SLOT_ID(slot_id);
 376         xhci_send_cmd(xhcd, field1, field2, field3, field4);
 377 }
 378
 379 static void xhci_init_seg(struct xhci_seg *seg, uint32_t size, uint32_t type)
 380 {
 381         struct xhci_link_trb *link;
 382
 383         seg->size = size / XHCI_TRB_SIZE;
 384         seg->next = NULL;
 385         seg->type = type;
 386         seg->cycle_state = 1;
 387         seg->enq = (uint64_t)seg->trbs;
 388         seg->deq = (uint64_t)seg->trbs;
 389         memset((void *)seg->trbs, 0, size);
 390
 391         link =(struct xhci_link_trb *) (seg->trbs + seg->size - 1);
 392         link->addr = cpu_to_le64(seg->trbs_dma);
 393         link->field2 = 0;
 394         link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
 395         return;
 396 }
 397
 398 static bool xhci_alloc_seg(struct xhci_seg *seg, uint32_t size, uint32_t type)
 399 {
 400         seg->trbs = (union xhci_trb *)SLOF_dma_alloc(size);
 401         if (!seg->trbs) {
 402                 dprintf("Alloc failed\n");
 403                 return false;
 404         }
 405         xhci_init_seg(seg, size, type);
 406         seg->trbs_dma = SLOF_dma_map_in((void *)seg->trbs, size, false);
 407
 408         dprintf(" TRBs %016lX TRBS-DMA %016lX\n", seg->trbs, seg->trbs_dma);
 409         return true;
 410 }
 411
 412 static void xhci_free_seg(struct xhci_seg *seg, uint32_t size)
 413 {
 414         if (seg->trbs) {
 415                 dprintf(" TRBs %016lX TRBS-DMA %016lX size %x\n", seg->trbs, seg->trbs_dma, size);
 416                 SLOF_dma_map_out(seg->trbs_dma, (void *)seg->trbs, size);
 417                 SLOF_dma_free((void *)seg->trbs, size);
 418         }
 419         memset(seg, 0, sizeof(*seg));
 420 }
 421
 422 #define CTX_SIZE(x)  ( (x) ? 64 : 32 )
 423
 424 static bool xhci_alloc_ctx(struct xhci_ctx *ctx, uint32_t size, uint32_t type)
 425 {
 426         ctx->addr = (uint8_t *)SLOF_dma_alloc(size);
 427         if (!ctx->addr) {
 428                 dprintf("Alloc failed\n");
 429                 return false;
 430         }
 431         ctx->size = size;
 432         ctx->type = type;
 433         memset((void *)ctx->addr, 0, size);
 434         ctx->dma_addr = SLOF_dma_map_in((void *)ctx->addr, size, false);
 435         dprintf("ctx %llx, ctx_dma %llx\n", ctx->addr, ctx->dma_addr);
 436         return true;
 437 }
 438
 439 static struct xhci_control_ctx *xhci_get_control_ctx(struct xhci_ctx *ctx)
 440 {
 441         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 442                 return (struct xhci_control_ctx *) ctx->addr;
 443         return NULL;
 444 }
 445
 446 static struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_ctx *ctx, uint32_t ctx_size)
 447 {
 448         uint32_t offset = 0;
 449
 450         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 451                 offset += ctx_size;
 452         return (struct xhci_slot_ctx *)(ctx->addr + offset);
 453 }
 454
 455 static struct xhci_ep_ctx *xhci_get_ep0_ctx(struct xhci_ctx *ctx, uint32_t ctx_size)
 456 {
 457         uint32_t offset = 0;
 458
 459         offset = ctx_size;
 460         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 461                 offset += ctx_size;
 462         return (struct xhci_ep_ctx *)(ctx->addr + offset);
 463 }
 464
 465 static struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_ctx *ctx, uint32_t ctx_size,
 466                                         uint32_t epno)
 467 {
 468         uint32_t offset = 0;
 469
 470         offset = ctx_size * epno;
 471         if (ctx->type == XHCI_CTX_TYPE_INPUT)
 472                 offset += ctx_size;
 473         return (struct xhci_ep_ctx *)(ctx->addr + offset);
 474 }
 475
 476 static void xhci_free_ctx(struct xhci_ctx *ctx, uint32_t size)
 477 {
 478         SLOF_dma_map_out(ctx->dma_addr, (void *)ctx->addr, size);
 479         SLOF_dma_free((void *)ctx->addr, size);
 480 }
 481
 482 static uint32_t usb_control_max_packet(uint32_t speed)
 483 {
 484         uint32_t max_packet = 0;
 485
 486         switch(speed)
 487         {
 488         case USB_LOW_SPEED:
 489                 max_packet = 8;
 490                 break;
 491         case USB_FULL_SPEED:
 492                 max_packet = 8;
 493                 break;
 494         case USB_HIGH_SPEED:
 495                 max_packet = 64;
 496                 break;
 497         case USB_SUPER_SPEED:
 498                 max_packet = 512;
 499                 break;
 500         default:
 501                 /* should not reach here */
 502                 dprintf("Unknown speed\n");
 503         }
 504         return max_packet;
 505 }
 506
 507 static bool xhci_alloc_dev(struct xhci_hcd *xhcd, uint32_t slot_id, uint32_t port)
 508 {
 509         struct usb_dev *dev;
 510         struct xhci_dev *xdev;
 511         struct xhci_slot_ctx *slot;
 512         struct xhci_control_ctx *ctrl;
 513         struct xhci_ep_ctx *ep0;
 514         uint32_t ctx_size, val;
 515         uint16_t max_packet;
 516         uint32_t newport;
 517
 518         ctx_size = CTX_SIZE(xhcd->hcc_csz_64);
 519         xdev = &xhcd->xdevs[slot_id];
 520         xdev->slot_id = slot_id;
 521         xdev->ctx_size = ctx_size;
 522
 523         /* 4.3.3 Device Slot initialization */
 524         /* Step 1 */
 525         if (!xhci_alloc_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE, XHCI_CTX_TYPE_INPUT)) {
 526                 dprintf("Failed allocating in_ctx\n");
 527                 return false;
 528         }
 529
 530         /* Step 2 */
 531         ctrl = xhci_get_control_ctx(&xdev->in_ctx);
 532         ctrl->a_flags = cpu_to_le32(0x3);          /* A0, A1 */
 533         ctrl->d_flags = 0;
 534
 535         /* Step 3 */
 536         slot = xhci_get_slot_ctx(&xdev->in_ctx, ctx_size);
 537         newport = port + 1;
 538         val = LAST_CONTEXT(1) | SLOT_SPEED_SS | (newport << 16); /* FIXME speed, read from PS */
 539         slot->field1 = cpu_to_le32(val);
 540         slot->field2 = cpu_to_le32(ROOT_HUB_PORT(newport)); /* FIXME how to get port no */
 541
 542         /* Step 4 */
 543         if (!xhci_alloc_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE, TYPE_CTRL)) {
 544                 dprintf("Failed allocating control\n");
 545                 goto fail_in_ctx;
 546         }
 547
 548         /* Step 5 */
 549         ep0 = xhci_get_ep0_ctx(&xdev->in_ctx, ctx_size);
 550         val = 0;
 551         max_packet = usb_control_max_packet(USB_SUPER_SPEED);
 552         max_packet = 64;
 553         val = EP_TYPE(EP_CTRL) | MAX_BURST(0) | ERROR_COUNT(3) |
 554                 MAX_PACKET_SIZE(max_packet);
 555         ep0->field2 = cpu_to_le32(val);;
 556         ep0->deq_addr = cpu_to_le64(xdev->control.trbs_dma | xdev->control.cycle_state);
 557         ep0->field4 = cpu_to_le32(8);
 558
 559         /* Step 6 */
 560         if (!xhci_alloc_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE, XHCI_CTX_TYPE_DEVICE)) {
 561                 dprintf("Failed allocating out_ctx\n");
 562                 goto fail_control_seg;
 563         }
 564
 565         /* Step 7 */
 566         xhcd->dcbaap[slot_id] = cpu_to_le64(xdev->out_ctx.dma_addr);
 567
 568         /* Step 8 */
 569         slot = xhci_get_slot_ctx(&xdev->out_ctx, ctx_size);
 570         ep0 = xhci_get_ep0_ctx(&xdev->out_ctx, ctx_size);
 571
 572         dprintf("Slot State %x \n", SLOT_STATE(le32_to_cpu(slot->field4)));
 573         xhci_send_addr_device(xhcd, slot_id, xdev->in_ctx.dma_addr);
 574         mb();
 575         dprintf("Slot State %x \n", SLOT_STATE(le32_to_cpu(slot->field4)));
 576
 577         dprintf("EP0 f0 %08X f1 %08X %016lX %08X\n",
 578                 le32_to_cpu(ep0->field1),
 579                 le32_to_cpu(ep0->field2),
 580                 le64_to_cpu(ep0->deq_addr),
 581                 le32_to_cpu(ep0->field4));
 582
 583         /* Step 9 - configure ep */
 584         ctrl->a_flags = cpu_to_le32(0x1);          /* A0 */
 585         ctrl->d_flags = 0;
 586         xhci_configure_ep(xhcd, slot_id, xdev->in_ctx.dma_addr);
 587         mb();
 588         dprintf("Slot State %x \n", SLOT_STATE(le32_to_cpu(slot->field4)));
 589         dprintf("USB Device address %d \n", USB_DEV_ADDRESS(le32_to_cpu(slot->field4)));
 590         dprintf("EP0 f0 %08X f1 %08X %016lX %08X\n",
 591                 le32_to_cpu(ep0->field1),
 592                 le32_to_cpu(ep0->field2),
 593                 le64_to_cpu(ep0->deq_addr),
 594                 le32_to_cpu(ep0->field4));
 595
 596         dev = usb_devpool_get();
 597         dprintf("allocated device %p\n", dev);
 598         dev->hcidev = xhcd->hcidev;
 599         dev->speed = USB_SUPER_SPEED;
 600         dev->addr = USB_DEV_ADDRESS(slot->field4);
 601         dev->port = newport;
 602         dev->priv = xdev;
 603         xdev->dev = dev;
 604         if (setup_new_device(dev, newport))
 605                 return true;
 606
 607         xhci_free_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE);
 608 fail_control_seg:
 609         xhci_free_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE);
 610 fail_in_ctx:
 611         xhci_free_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE);
 612         return false;
 613 }
 614
 615 static void xhci_free_dev(struct xhci_dev *xdev)
 616 {
 617         xhci_free_seg(&xdev->bulk_in, XHCI_DATA_TRBS_SIZE);
 618         xhci_free_seg(&xdev->bulk_out, XHCI_DATA_TRBS_SIZE);
 619         xhci_free_seg(&xdev->control, XHCI_CONTROL_TRBS_SIZE);
 620         xhci_free_ctx(&xdev->in_ctx, XHCI_CTX_BUF_SIZE);
 621         xhci_free_ctx(&xdev->out_ctx, XHCI_CTX_BUF_SIZE);
 622 }
 623
 624 static bool usb3_dev_init(struct xhci_hcd *xhcd, uint32_t port)
 625 {
 626         /* Device enable slot */
 627         xhci_send_enable_slot(xhcd, port);
 628         if (!xhcd->slot_id) {
 629                 dprintf("Unable to get slot id\n");
 630                 return false;
 631         }
 632         dprintf("SLOT ID: %d\n", xhcd->slot_id);
 633         if (!xhci_alloc_dev(xhcd, xhcd->slot_id, port)) {
 634                 dprintf("Unable to allocate device\n");
 635                 return false;
 636         }
 637         return true;
 638 }
 639
 640 static int xhci_hub_check_ports(struct xhci_hcd *xhcd)
 641 {
 642         uint32_t num_ports, portsc, i;
 643         struct xhci_op_regs *op;
 644         struct xhci_port_regs *prs;
 645         struct xhci_cap_regs *cap;
 646         uint32_t xecp_off;
 647         uint32_t *xecp_addr, *base;
 648         uint32_t port_off = 1, port_cnt;
 649
 650         dprintf("enter\n");
 651
 652         op = xhcd->op_regs;
 653         cap = xhcd->cap_regs;
 654         port_cnt = num_ports = read_reg32(&cap->hcsparams1) >> 24;
 655
 656         /* Read the xHCI extented capability to find usb3 ports and offset*/
 657         xecp_off = XHCI_HCCPARAMS_XECP(read_reg32(&cap->hccparams));
 658         base = (uint32_t *)cap;
 659         while (xecp_off > 0) {
 660                 xecp_addr = base + xecp_off;
 661                 dprintf(stderr, "xecp_off %d %p %p \n", xecp_off, base, xecp_addr);
 662
 663                 if (XHCI_XECP_CAP_ID(read_reg32(xecp_addr)) == XHCI_XECP_CAP_SP &&
 664                     XHCI_XECP_CAP_SP_MJ(read_reg32(xecp_addr)) == 3 &&
 665                     XHCI_XECP_CAP_SP_MN(read_reg32(xecp_addr)) == 0) {
 666                         port_cnt = XHCI_XECP_CAP_SP_PC(read_reg32(xecp_addr + 2));
 667                         port_off = XHCI_XECP_CAP_SP_PO(read_reg32(xecp_addr + 2));
 668                         dprintf(stderr, "PortCount %d Portoffset %d\n", port_cnt, port_off);
 669                 }
 670                 base = xecp_addr;
 671                 xecp_off = XHCI_XECP_NEXT_PTR(read_reg32(xecp_addr));
 672         }
 673         if (port_off == 0) /* port_off should always start from 1 */
 674                 return false;
 675         for (i = (port_off - 1); i < (port_off + port_cnt - 1); i++) {
 676                 prs = &op->prs[i];
 677                 portsc = read_reg32(&prs->portsc);
 678                 if ((portsc & PORTSC_CCS) &&
 679                         (portsc & PORTSC_PP) &&
 680                         (portsc & PORTSC_PED)) {
 681                         /* Device present and enabled */
 682                         dprintf("Device present on port %d\n", i);
 683                         /* Reset the port */
 684                         portsc = read_reg32(&prs->portsc);
 685                         portsc = portsc | PORTSC_PR;
 686                         write_reg32(&prs->portsc, portsc);
 687                         /* FIXME poll for port event */
 688                         SLOF_msleep(20);
 689                         xhci_poll_event(xhcd, 0);
 690                         portsc = read_reg32(&prs->portsc);
 691                         if (portsc & ~PORTSC_PRC) {
 692                                 dprintf("Port reset complete %d\n", i);
 693                         }
 694                         print_port_status(prs);
 695                         if (!usb3_dev_init(xhcd, (i - (port_off - 1)))) {
 696                                 dprintf("USB device initialization failed\n");
 697                         }
 698                 }
 699         }
 700         dprintf("exit\n");
 701         return true;
 702 }
 703
 704 static bool xhci_hcd_init(struct xhci_hcd *xhcd)
 705 {
 706         struct xhci_op_regs *op;
 707         struct xhci_int_regs *irs;
 708         uint64_t val;
 709         uint32_t reg;
 710
 711         if (!xhcd) {
 712                 dprintf("NULL pointer\n");
 713                 goto fail;
 714         }
 715
 716         op = xhcd->op_regs;
 717         irs = &xhcd->run_regs->irs[0];
 718         if (!xhci_hcd_reset(op)) {
 719                 dprintf("Reset failed\n");
 720                 goto fail;
 721         }
 722
 723         write_reg32(&op->config, XHCI_CONFIG_MAX_SLOT);
 724         reg = read_reg32(&xhcd->cap_regs->hccparams);
 725         /* 64byte context !! */
 726         xhcd->hcc_csz_64 = (reg & XHCI_HCCPARAMS_CSZ) ? 1 : 0;
 727
 728         if (xhcd->hcc_csz_64) {
 729                 printf("usb-xhci: 64 Byte context not supported\n");
 730                 goto fail;
 731         }
 732         /*
 733          * 6.1 Device Context Base Address Array
 734          *
 735          * Allocate memory and initialize
 736          */
 737         xhcd->dcbaap = (uint64_t *)SLOF_dma_alloc(XHCI_DCBAAP_MAX_SIZE);
 738         if (!xhcd->dcbaap) {
 739                 dprintf("Alloc failed\n");
 740                 goto fail;
 741         }
 742         memset((void *)xhcd->dcbaap, 0, XHCI_DCBAAP_MAX_SIZE);
 743         xhcd->dcbaap_dma = SLOF_dma_map_in((void *)xhcd->dcbaap,
 744                                         XHCI_DCBAAP_MAX_SIZE, false);
 745         dprintf("dcbaap %llx, dcbaap_phys %llx\n", xhcd->dcbaap, xhcd->dcbaap_dma);
 746         write_reg64(&op->dcbaap, xhcd->dcbaap_dma);
 747
 748         /*
 749          * Command Ring Control - TRB
 750          * FIXME - better way to allocate it...
 751          */
 752         if (!xhci_alloc_seg(&xhcd->crseg, XHCI_CRCR_CRP_SIZE, TYPE_COMMAND))
 753                 goto fail_dcbaap;
 754
 755         val = read_reg64(&op->crcr) & ~XHCI_CRCR_CRP_MASK;
 756         val = val | (xhcd->crseg.trbs_dma & XHCI_CRCR_CRP_MASK);
 757         write_reg64(&op->crcr, val);
 758
 759         /*
 760          * Event Ring Control - TRB
 761          * Allocate event TRBS
 762          */
 763         if (!xhci_alloc_seg(&xhcd->ering, XHCI_EVENT_TRBS_SIZE, TYPE_EVENT))
 764                 goto fail_crseg;
 765
 766         /*
 767          * Populate event ring segment table.
 768          * Note: only using one segment.
 769          */
 770         xhcd->erst.entries = SLOF_dma_alloc(XHCI_EVENT_TRBS_SIZE);
 771         if (!xhcd->erst.entries)
 772                 goto fail_ering;
 773         xhcd->erst.dma = SLOF_dma_map_in((void *)xhcd->erst.entries,
 774                                         XHCI_EVENT_TRBS_SIZE, false);
 775         xhcd->erst.num_segs = XHCI_ERST_NUM_SEGS;
 776
 777         /* populate entries[0] */
 778         write_reg64(&xhcd->erst.entries->addr, xhcd->ering.trbs_dma);
 779         write_reg32(&xhcd->erst.entries->size, xhcd->ering.size);
 780         write_reg32(&xhcd->erst.entries->reserved, 0);
 781
 782         /* populate erdp */
 783         val = read_reg64(&irs->erdp) & ~XHCI_ERDP_MASK;
 784         val = val | (xhcd->ering.trbs_dma & XHCI_ERDP_MASK);
 785         write_reg64(&irs->erdp, val);
 786
 787         /* populate erstsz */
 788         val = read_reg32(&irs->erstsz) & ~XHCI_ERST_SIZE_MASK;
 789         val = val | xhcd->erst.num_segs;
 790         write_reg32(&irs->erstsz, val);
 791
 792         /* Now write the erstba */
 793         val = read_reg64(&irs->erstba) & ~XHCI_ERST_ADDR_MASK;
 794         val = val | (xhcd->erst.dma & XHCI_ERST_ADDR_MASK);
 795         write_reg64(&irs->erstba, val);
 796
 797         dprintf("ERDP %llx TRB-DMA %llx\n", read_reg64(&irs->erdp),
 798                 xhcd->ering.trbs_dma);
 799         dprintf("ERST %llx, ERST DMA %llx, size %d\n",
 800                 (uint64_t)xhcd->erst.entries, xhcd->erst.dma,
 801                 xhcd->erst.num_segs);
 802
 803         mb();
 804         if (!xhci_hcd_set_runstop(op, true))
 805                 goto fail_erst_entries;
 806
 807         if (!xhci_hub_check_ports(xhcd))
 808                 goto fail_erst_entries;
 809
 810         return true;
 811 fail_erst_entries:
 812         write_reg64(&irs->erstba, 0);
 813         mb();
 814         SLOF_dma_map_out(xhcd->erst.dma, (void *)xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 815         SLOF_dma_free((void *)xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 816 fail_ering:
 817         xhci_free_seg(&xhcd->ering, XHCI_EVENT_TRBS_SIZE);
 818 fail_crseg:
 819         val = read_reg64(&op->crcr) & ~XHCI_CRCR_CRP_MASK;
 820         write_reg64(&op->crcr, val);
 821         mb();
 822         xhci_free_seg(&xhcd->crseg, XHCI_CRCR_CRP_SIZE);
 823 fail_dcbaap:
 824         write_reg64(&op->dcbaap, 0);
 825         mb();
 826         SLOF_dma_map_out(xhcd->dcbaap_dma, (void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 827         SLOF_dma_free((void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 828 fail:
 829         return false;
 830 }
 831
 832 static bool xhci_hcd_exit(struct xhci_hcd *xhcd)
 833 {
 834         struct xhci_op_regs *op;
 835         struct xhci_int_regs *irs;
 836         uint64_t val;
 837         int i;
 838
 839         if (!xhcd) {
 840                 dprintf("NULL pointer\n");
 841                 return false;
 842         }
 843         op = xhcd->op_regs;
 844
 845         if (!xhci_hcd_set_runstop(op, false)) {
 846                 dprintf("NULL pointer\n");
 847         }
 848
 849         for (i = 1; i < XHCI_CONFIG_MAX_SLOT; i++) {
 850                 if (xhcd->xdevs[i].dev)
 851                         xhci_free_dev(&xhcd->xdevs[i]);
 852         }
 853
 854         irs = &xhcd->run_regs->irs[0];
 855         write_reg64(&irs->erstba, 0);
 856         mb();
 857         if (xhcd->erst.entries) {
 858                 SLOF_dma_map_out(xhcd->erst.dma, xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 859                 SLOF_dma_free(xhcd->erst.entries, XHCI_EVENT_TRBS_SIZE);
 860         }
 861         xhci_free_seg(&xhcd->ering, XHCI_EVENT_TRBS_SIZE);
 862
 863         val = read_reg64(&op->crcr) & ~XHCI_CRCR_CRP_MASK;
 864         write_reg64(&op->crcr, val);
 865         xhci_free_seg(&xhcd->crseg, XHCI_CRCR_CRP_SIZE);
 866         write_reg64(&op->dcbaap, 0);
 867         if (xhcd->dcbaap) {
 868                 SLOF_dma_map_out(xhcd->dcbaap_dma, (void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 869                 SLOF_dma_free((void *)xhcd->dcbaap, XHCI_DCBAAP_MAX_SIZE);
 870         }
 871         return true;
 872 }
 873
 874 static void xhci_init(struct usb_hcd_dev *hcidev)
 875 {
 876         struct xhci_hcd *xhcd;
 877
 878         printf("  XHCI: Initializing\n");
 879         dprintf("device base address %p\n", hcidev->base);
 880
 881         hcidev->base = (void *)((uint64_t)hcidev->base & ~7);
 882         xhcd = SLOF_alloc_mem(sizeof(*xhcd));
 883         if (!xhcd) {
 884                 printf("usb-xhci: Unable to allocate memory\n");
 885                 return;
 886         }
 887         memset(xhcd, 0, sizeof(*xhcd));
 888
 889         hcidev->nextaddr = 1;
 890         hcidev->priv = xhcd;
 891         xhcd->hcidev = hcidev;
 892         xhcd->cap_regs = (struct xhci_cap_regs *)(hcidev->base);
 893         xhcd->op_regs = (struct xhci_op_regs *)(hcidev->base +
 894                                                 read_reg8(&xhcd->cap_regs->caplength));
 895         xhcd->run_regs = (struct xhci_run_regs *)(hcidev->base +
 896                                                 read_reg32(&xhcd->cap_regs->rtsoff));
 897         xhcd->db_regs = (struct xhci_db_regs *)(hcidev->base +
 898                                                 read_reg32(&xhcd->cap_regs->dboff));
 899         dump_xhci_regs(xhcd);
 900         if (!xhci_hcd_init(xhcd))
 901                 printf("usb-xhci: failed to initialize XHCI controller.\n");
 902         dump_xhci_regs(xhcd);
 903 }
 904
 905 static void xhci_exit(struct usb_hcd_dev *hcidev)
 906 {
 907         struct xhci_hcd *xhcd;
 908
 909         dprintf("%s: enter \n", __func__);
 910         if (!hcidev && !hcidev->priv) {
 911                 return;
 912         }
 913
 914         xhcd = hcidev->priv;
 915         xhci_hcd_exit(xhcd);
 916         SLOF_free_mem(xhcd, sizeof(*xhcd));
 917         hcidev->priv = NULL;
 918 }
 919
 920 static void fill_trb_buff(struct xhci_command_trb *cmd,  uint32_t field1,
 921                         uint32_t field2, uint32_t field3, uint32_t field4)
 922 {
 923         uint32_t val, cycle_state;
 924
 925         cmd->field[0] = cpu_to_le32(field1);
 926         cmd->field[1] = cpu_to_le32(field2);
 927         cmd->field[2] = cpu_to_le32(field3);
 928
 929         val = le32_to_cpu(cmd->field[3]);
 930         cycle_state = (val & 0x1) ? 0 : 1;
 931         val =  cycle_state | (field4 & ~0x1);
 932         cmd->field[3] = cpu_to_le32(val);
 933
 934         dprintf("CMD %016lx val %08x cycle_state %d field1 %08x, field2  %08x, field3 %08x field4 %08x\n",
 935                 cmd, val, cycle_state,
 936                 le32_to_cpu(cmd->field[0]),
 937                 le32_to_cpu(cmd->field[1]),
 938                 le32_to_cpu(cmd->field[2]),
 939                 le32_to_cpu(cmd->field[3])
 940                 );
 941
 942         return;
 943 }
 944
 945 static void fill_setup_trb(struct xhci_command_trb *cmd, struct usb_dev_req *req,
 946                         uint32_t size)
 947 {
 948         uint32_t field1, field2, field3, field4 = 0;
 949         uint64_t req_raw;
 950         uint32_t datalen = 0, pid = 0;
 951
 952         req_raw = *((uint64_t *)req);
 953         dprintf("%lx %lx \n", *((uint64_t *)req), req_raw);
 954         /* req_raw is already in right byte order... */
 955         field1 = cpu_to_le32(TRB_ADDR_HIGH(req_raw));
 956         field2 = cpu_to_le32(TRB_ADDR_LOW(req_raw));
 957         field3 = 8; /* ALWAYS 8 */
 958
 959         datalen = cpu_to_le16(req->wLength);
 960         if (datalen) {
 961                 pid = (req->bmRequestType & REQT_DIR_IN) ? 3 : 2;
 962                 field4 = TRB_TRT(pid);
 963         }
 964         field4 |= TRB_CMD_TYPE(TRB_SETUP_STAGE) | TRB_IDT;
 965         fill_trb_buff(cmd, field1, field2, field3, field4);
 966 }
 967
 968 static void fill_setup_data(struct xhci_command_trb *cmd, void *data,
 969                         uint32_t size, uint32_t dir)
 970 {
 971         uint32_t field1, field2, field3, field4;
 972
 973         field1 = TRB_ADDR_LOW(data);
 974         field2 = TRB_ADDR_HIGH(data);
 975         field3 = size;
 976         if (dir)
 977                 field4 = TRB_DIR_IN;
 978         field4 |= TRB_CMD_TYPE(TRB_DATA_STAGE);
 979         fill_trb_buff(cmd, field1, field2, field3, field4);
 980 }
 981
 982 static void fill_status_trb(struct xhci_command_trb *cmd, uint32_t dir)
 983 {
 984         uint32_t field1, field2, field3, field4;
 985
 986         field1 = 0;
 987         field2 = 0;
 988         field3 = 0;
 989         if (dir)
 990                 field4 = TRB_DIR_IN;
 991
 992         field4 |= TRB_CMD_TYPE(TRB_STATUS_STAGE) | TRB_IOC;
 993         fill_trb_buff(cmd, field1, field2, field3, field4);
 994 }
 995
 996 static void fill_normal_trb(struct xhci_transfer_trb *trb, void *data,
 997                         uint32_t size)
 998 {
 999         uint32_t field1, field2, field3, field4;
1000
1001         field1 = TRB_ADDR_LOW(data);
1002         field2 = TRB_ADDR_HIGH(data);
1003         field3 = size;
1004         field4 = TRB_CMD_TYPE(TRB_NORMAL) | TRB_IOC;
1005         fill_trb_buff((struct xhci_command_trb *)trb, field1, field2, field3, field4);
1006 }
1007
1008 static int xhci_send_ctrl(struct usb_pipe *pipe, struct usb_dev_req *req, void *data)
1009 {
1010         struct xhci_dev *xdev;
1011         struct xhci_seg *ctrl;
1012         struct xhci_hcd *xhcd;
1013         struct xhci_command_trb *cmd;
1014         struct xhci_db_regs *dbr;
1015         long req_phys = 0, data_phys = 0;
1016         int ret = true;
1017         uint32_t slot_id, pid = 0, datalen = 0;
1018
1019         if (!pipe->dev || !pipe->dev->hcidev) {
1020                 dprintf(" NULL pointer\n");
1021                 return false;
1022         }
1023
1024         xdev = pipe->dev->priv;
1025         slot_id = xdev->slot_id;
1026         ctrl = &xdev->control;
1027         xhcd = (struct xhci_hcd *)pipe->dev->hcidev->priv;
1028         dbr = xhcd->db_regs;
1029         if (!ctrl || !xdev || !xhcd) {
1030                 dprintf(" NULL pointer\n");
1031                 return false;
1032         }
1033
1034         cmd = (struct xhci_command_trb *)ctrl->enq;
1035         req_phys = SLOF_dma_map_in(req, sizeof(struct usb_dev_req), true);
1036         fill_setup_trb(cmd, req, sizeof(*req));
1037
1038         cmd++;
1039         datalen = cpu_to_le16(req->wLength);
1040         if (datalen)
1041                 pid = 1;
1042         if (datalen) {
1043                 data_phys = SLOF_dma_map_in(data, datalen, true);
1044                 fill_setup_data(cmd, (void *) data_phys, datalen, pid);
1045                 cmd++;
1046         }
1047
1048         fill_status_trb(cmd, pid);
1049         cmd++;
1050
1051         /* Ring the doorbell - ep0 */
1052         write_reg32(&dbr->db[slot_id], 1);
1053         if (!xhci_poll_event(xhcd, 0)) {
1054                 dprintf("Command failed\n");
1055                 ret = false;
1056         }
1057         ctrl->enq = (uint64_t) cmd;
1058         SLOF_dma_map_out(req_phys, req, sizeof(struct usb_dev_req));
1059         if (datalen)
1060                 SLOF_dma_map_out(data_phys, data, datalen);
1061         return ret;
1062 }
1063
1064 static inline struct xhci_pipe *xhci_pipe_get_xpipe(struct usb_pipe *pipe)
1065 {
1066         struct xhci_pipe *xpipe;
1067         xpipe = container_of(pipe, struct xhci_pipe, pipe);
1068         dprintf("%s: xpipe is %p\n", __func__, xpipe);
1069         return xpipe;
1070 }
1071
1072 static inline struct xhci_seg *xhci_pipe_get_seg(struct usb_pipe *pipe)
1073 {
1074         struct xhci_pipe *xpipe;
1075         xpipe = xhci_pipe_get_xpipe(pipe);
1076         return xpipe->seg;
1077 }
1078
1079 static inline void *xhci_get_trb(struct xhci_seg *seg)
1080 {
1081         uint64_t val, enq;
1082         uint32_t size;
1083         struct xhci_link_trb *link;
1084
1085         enq = val = seg->enq;
1086         val = val + XHCI_TRB_SIZE;
1087         size = seg->size * XHCI_TRB_SIZE;
1088         /* TRBs being a cyclic buffer, here we cycle back to beginning. */
1089         if ((val % size) == 0) {
1090                 seg->enq = (uint64_t)seg->trbs;
1091                 enq = seg->enq;
1092                 seg->enq = seg->enq + XHCI_TRB_SIZE;
1093                 val = 0;
1094                 seg->cycle_state ^= seg->cycle_state;
1095                 link = (struct xhci_link_trb *) (seg->trbs + seg->size - 1);
1096                 link->addr = cpu_to_le64(seg->trbs_dma);
1097                 link->field2 = 0;
1098                 link->field3 = cpu_to_le32(0x1 | TRB_CMD_TYPE(TRB_LINK));
1099                 mb();
1100         }
1101         else {
1102                 seg->enq = seg->enq + XHCI_TRB_SIZE;
1103         }
1104
1105         return (void *)enq;
1106 }
1107
1108 static int xhci_transfer_bulk(struct usb_pipe *pipe, void *td, void *td_phys,
1109                         void *data, int datalen)
1110 {
1111         struct xhci_dev *xdev;
1112         struct xhci_seg *seg;
1113         struct xhci_hcd *xhcd;
1114         struct xhci_transfer_trb *trb;
1115         struct xhci_db_regs *dbr;
1116         int ret = true;
1117         uint32_t slot_id, epno;
1118
1119         if (!pipe->dev || !pipe->dev->hcidev) {
1120                 dprintf(" NULL pointer\n");
1121                 dprintf(" pipe dev %p hcidev %p\n", pipe->dev, pipe->dev->hcidev);
1122                 return false;
1123         }
1124
1125         xdev = pipe->dev->priv;
1126         slot_id = xdev->slot_id;
1127         seg = xhci_pipe_get_seg(pipe);
1128         xhcd = (struct xhci_hcd *)pipe->dev->hcidev->priv;
1129         dbr = xhcd->db_regs;
1130         if (!seg || !xdev || !xhcd) {
1131                 dprintf(" NULL pointer\n");
1132                 dprintf(" seg %p xdev %p xhcd %p\n", seg, xdev, xhcd);
1133                 return false;
1134         }
1135
1136         if (datalen > XHCI_MAX_BULK_SIZE) {
1137                 printf("usb-xhci: bulk transfer size too big\n");
1138                 return false;
1139         }
1140
1141         trb = xhci_get_trb(seg);
1142         fill_normal_trb(trb, (void *)data, datalen);
1143
1144         epno = xhci_get_epno(pipe);
1145         write_reg32(&dbr->db[slot_id], epno);
1146         if (!xhci_poll_event(xhcd, 0)) {
1147                 dprintf("Bulk failed\n");
1148                 ret = false;
1149         }
1150         trb->addr = 0;
1151         trb->len = 0;
1152         trb->flags = 0;
1153         mb();
1154
1155         return ret;
1156 }
1157
1158 static int xhci_alloc_pipe_pool(struct xhci_hcd *xhcd)
1159 {
1160         struct xhci_pipe *xpipe, *curr, *prev;
1161         unsigned int i, count;
1162         long xpipe_phys = 0;
1163
1164         count = XHCI_PIPE_POOL_SIZE/sizeof(*xpipe);
1165         xhcd->pool = xpipe = SLOF_dma_alloc(XHCI_PIPE_POOL_SIZE);
1166         if (!xpipe)
1167                 return -1;
1168         xhcd->pool_phys = xpipe_phys = SLOF_dma_map_in(xpipe, XHCI_PIPE_POOL_SIZE, true);
1169         dprintf("%s: xpipe %p, xpipe_phys %lx\n", __func__, xpipe, xpipe_phys);
1170
1171         /* Although an array, link them */
1172         for (i = 0, curr = xpipe, prev = NULL; i < count; i++, curr++) {
1173                 if (prev)
1174                         prev->pipe.next = &curr->pipe;
1175                 curr->pipe.next = NULL;
1176                 prev = curr;
1177         }
1178
1179         if (!xhcd->freelist)
1180                 xhcd->freelist = &xpipe->pipe;
1181         else
1182                 xhcd->end->next = &xpipe->pipe;
1183         xhcd->end = &prev->pipe;
1184
1185         return 0;
1186 }
1187
1188 static void xhci_init_bulk_ep(struct usb_dev *dev, struct usb_pipe *pipe)
1189 {
1190         struct xhci_hcd *xhcd;
1191         struct xhci_dev *xdev;
1192         struct xhci_seg *seg;
1193         struct xhci_pipe *xpipe;
1194         struct xhci_control_ctx *ctrl;
1195         struct xhci_ep_ctx *ep;
1196         uint32_t x_epno, val, type;
1197
1198         if (!pipe || !dev || !dev->priv)
1199                 return;
1200
1201         xdev = dev->priv;
1202         xhcd = dev->hcidev->priv;
1203         dprintf("dir %d\n", pipe->dir);
1204         seg = xhci_pipe_get_seg(pipe);
1205         xpipe = xhci_pipe_get_xpipe(pipe);
1206         if (pipe->dir) {
1207                 type = EP_BULK_IN;
1208                 seg = &xdev->bulk_in;
1209         }
1210         else {
1211                 type = EP_BULK_OUT;
1212                 seg = &xdev->bulk_out;
1213         }
1214
1215         if (!seg->trbs) {
1216                 if (!xhci_alloc_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK)) {
1217                         dprintf("Failed allocating seg\n");
1218                 }
1219         } else {
1220                 xhci_init_seg(seg, XHCI_DATA_TRBS_SIZE, TYPE_BULK);
1221         }
1222
1223         pipe->mps = XHCI_MAX_BULK_SIZE;
1224         ctrl = xhci_get_control_ctx(&xdev->in_ctx);
1225         x_epno = xhci_get_epno(pipe);
1226         ep = xhci_get_ep_ctx(&xdev->in_ctx, xdev->ctx_size, x_epno);
1227         val = EP_TYPE(type) | MAX_BURST(0) | ERROR_COUNT(3) |
1228                 MAX_PACKET_SIZE(pipe->mps);
1229         ep->field2 = cpu_to_le32(val);;
1230         ep->deq_addr = cpu_to_le64(seg->trbs_dma | seg->cycle_state);
1231         ep->field4 = cpu_to_le32(8);
1232         ctrl->a_flags = cpu_to_le32(BIT(x_epno) | 0x1);
1233         ctrl->d_flags = 0;
1234         xhci_configure_ep(xhcd, xdev->slot_id, xdev->in_ctx.dma_addr);
1235         xpipe->seg = seg;
1236 }
1237
1238 static struct usb_pipe* xhci_get_pipe(struct usb_dev *dev, struct usb_ep_descr *ep, char *buf, size_t len)
1239 {
1240         struct xhci_hcd *xhcd;
1241         struct usb_pipe *new = NULL;
1242
1243         if (!dev)
1244                 return NULL;
1245
1246         xhcd = (struct xhci_hcd *)dev->hcidev->priv;
1247         if (!xhcd->freelist) {
1248                 dprintf("usb-xhci: %s allocating pool\n", __func__);
1249                 if (xhci_alloc_pipe_pool(xhcd))
1250                         return NULL;
1251         }
1252
1253         new = xhcd->freelist;
1254         xhcd->freelist = xhcd->freelist->next;
1255         if (!xhcd->freelist)
1256                 xhcd->end = NULL;
1257
1258         memset(new, 0, sizeof(*new));
1259         new->dev = dev;
1260         new->next = NULL;
1261         new->type = ep->bmAttributes & USB_EP_TYPE_MASK;
1262         new->speed = dev->speed;
1263         new->mps = ep->wMaxPacketSize;
1264         new->dir = (ep->bEndpointAddress & 0x80) >> 7;
1265         new->epno = ep->bEndpointAddress & 0x0f;
1266
1267         if (new->type == USB_EP_TYPE_BULK)
1268                 xhci_init_bulk_ep(dev, new);
1269
1270         return new;
1271 }
1272
1273 static void xhci_put_pipe(struct usb_pipe *pipe)
1274 {
1275         struct xhci_hcd *xhcd;
1276         struct xhci_pipe *xpipe;
1277
1278         dprintf("usb-xhci: %s enter - %p\n", __func__, pipe);
1279         if (!pipe || !pipe->dev)
1280                 return;
1281         xhcd = pipe->dev->hcidev->priv;
1282
1283         dprintf("dir %d\n", pipe->dir);
1284         if (pipe->type == USB_EP_TYPE_BULK) {
1285                 xpipe = xhci_pipe_get_xpipe(pipe);
1286                 xpipe->seg = NULL;
1287         }
1288         if (xhcd->end)
1289                 xhcd->end->next = pipe;
1290         else
1291                 xhcd->freelist = pipe;
1292
1293         xhcd->end = pipe;
1294         pipe->next = NULL;
1295         pipe->dev = NULL;
1296         memset(pipe, 0, sizeof(*pipe));
1297
1298         dprintf("usb-xhci: %s exit\n", __func__);
1299 }
1300
1301 struct usb_hcd_ops xhci_ops = {
1302         .name          = "xhci-hcd",
1303         .init          = xhci_init,
1304         .exit          = xhci_exit,
1305         .usb_type      = USB_XHCI,
1306         .get_pipe      = xhci_get_pipe,
1307         .put_pipe      = xhci_put_pipe,
1308         .send_ctrl     = xhci_send_ctrl,
1309         .transfer_bulk = xhci_transfer_bulk,
1310         .next          = NULL,
1311 };
1312
1313 void usb_xhci_register(void)
1314 {
1315         usb_hcd_register(&xhci_ops);
1316 }