kernel/drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/prefetch.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/platform_device.h>
  53 #include <linux/usb.h>
  54
  55 #include <linux/time.h>
  56 #include <linux/delay.h>
  57
  58 #include <asm/octeon/cvmx.h>
  59 #include <asm/octeon/cvmx-iob-defs.h>
  60
  61 #include <linux/usb/hcd.h>
  62
  63 #include <linux/err.h>
  64
  65 #include <asm/octeon/octeon.h>
  66 #include <asm/octeon/cvmx-helper.h>
  67 #include <asm/octeon/cvmx-sysinfo.h>
  68 #include <asm/octeon/cvmx-helper-board.h>
  69
  70 #include "octeon-hcd.h"
  71
  72 /**
  73  * enum cvmx_usb_speed - the possible USB device speeds
  74  *
  75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  78  */
  79 enum cvmx_usb_speed {
  80         CVMX_USB_SPEED_HIGH = 0,
  81         CVMX_USB_SPEED_FULL = 1,
  82         CVMX_USB_SPEED_LOW = 2,
  83 };
  84
  85 /**
  86  * enum cvmx_usb_transfer - the possible USB transfer types
  87  *
  88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  89  *                                 transfers
  90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  91  *                                 priority periodic transfers
  92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  93  *                                 transfers
  94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  95  *                                 periodic transfers
  96  */
  97 enum cvmx_usb_transfer {
  98         CVMX_USB_TRANSFER_CONTROL = 0,
  99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
 100         CVMX_USB_TRANSFER_BULK = 2,
 101         CVMX_USB_TRANSFER_INTERRUPT = 3,
 102 };
 103
 104 /**
 105  * enum cvmx_usb_direction - the transfer directions
 106  *
 107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 109  */
 110 enum cvmx_usb_direction {
 111         CVMX_USB_DIRECTION_OUT,
 112         CVMX_USB_DIRECTION_IN,
 113 };
 114
 115 /**
 116  * enum cvmx_usb_complete - possible callback function status codes
 117  *
 118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 119  *                                any errors
 120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 122  *                                by a user call to cvmx_usb_cancel
 123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 124  *                                error status
 125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 126  *                                from the device
 127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 128  *                                device even after a number of retries
 129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 130  *                                error even after a number of retries
 131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 133  *                                even after a number of retries
 134  */
 135 enum cvmx_usb_complete {
 136         CVMX_USB_COMPLETE_SUCCESS,
 137         CVMX_USB_COMPLETE_SHORT,
 138         CVMX_USB_COMPLETE_CANCEL,
 139         CVMX_USB_COMPLETE_ERROR,
 140         CVMX_USB_COMPLETE_STALL,
 141         CVMX_USB_COMPLETE_XACTERR,
 142         CVMX_USB_COMPLETE_DATATGLERR,
 143         CVMX_USB_COMPLETE_BABBLEERR,
 144         CVMX_USB_COMPLETE_FRAMEERR,
 145 };
 146
 147 /**
 148  * struct cvmx_usb_port_status - the USB port status information
 149  *
 150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 151  * @port_over_current:  1 = Over current detected, 0 = Over current not
 152  *                      detected. Octeon doesn't support over current detection.
 153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 154  *                      power is off. Octeon doesn't support turning port power
 155  *                      off.
 156  * @port_speed:         Current port speed.
 157  * @connected:          1 = A device is connected to the port, 0 = No device is
 158  *                      connected.
 159  * @connect_change:     1 = Device connected state changed since the last set
 160  *                      status call.
 161  */
 162 struct cvmx_usb_port_status {
 163         uint32_t reserved               : 25;
 164         uint32_t port_enabled           : 1;
 165         uint32_t port_over_current      : 1;
 166         uint32_t port_powered           : 1;
 167         enum cvmx_usb_speed port_speed  : 2;
 168         uint32_t connected              : 1;
 169         uint32_t connect_change         : 1;
 170 };
 171
 172 /**
 173  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 174  *
 175  * @offset:     This is the offset in bytes into the main buffer where this data
 176  *              is stored.
 177  * @length:     This is the length in bytes of the data.
 178  * @status:     This is the status of this individual packet transfer.
 179  */
 180 struct cvmx_usb_iso_packet {
 181         int offset;
 182         int length;
 183         enum cvmx_usb_complete status;
 184 };
 185
 186 /**
 187  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 188  *
 189  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 190  *                                            as clock source at USB_XO and
 191  *                                            USB_XI.
 192  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 193  *                                            board clock source at USB_XO.
 194  *                                            USB_XI should be tied to GND.
 195  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 196  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 197  *                                            crystal
 198  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 199  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 200  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 201  *                                            data transfer use for the USB
 202  */
 203 enum cvmx_usb_initialize_flags {
 204         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 205         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 206         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 207         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 208         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 209         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 210         /* Bits 3-4 used to encode the clock frequency */
 211         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 212 };
 213
 214 /**
 215  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 216  *
 217  * @CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 218  *                                 actively using hardware.
 219  * @CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high speed
 220  *                                 pipe is in the ping state.
 221  */
 222 enum cvmx_usb_pipe_flags {
 223         CVMX_USB_PIPE_FLAGS_SCHEDULED   = 1 << 17,
 224         CVMX_USB_PIPE_FLAGS_NEED_PING   = 1 << 18,
 225 };
 226
 227 /* Maximum number of times to retry failed transactions */
 228 #define MAX_RETRIES             3
 229
 230 /* Maximum number of hardware channels supported by the USB block */
 231 #define MAX_CHANNELS            8
 232
 233 /*
 234  * The low level hardware can transfer a maximum of this number of bytes in each
 235  * transfer. The field is 19 bits wide
 236  */
 237 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 238
 239 /*
 240  * The low level hardware can transfer a maximum of this number of packets in
 241  * each transfer. The field is 10 bits wide
 242  */
 243 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 244
 245 /**
 246  * Logical transactions may take numerous low level
 247  * transactions, especially when splits are concerned. This
 248  * enum represents all of the possible stages a transaction can
 249  * be in. Note that split completes are always even. This is so
 250  * the NAK handler can backup to the previous low level
 251  * transaction with a simple clearing of bit 0.
 252  */
 253 enum cvmx_usb_stage {
 254         CVMX_USB_STAGE_NON_CONTROL,
 255         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 256         CVMX_USB_STAGE_SETUP,
 257         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 258         CVMX_USB_STAGE_DATA,
 259         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 260         CVMX_USB_STAGE_STATUS,
 261         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 262 };
 263
 264 /**
 265  * struct cvmx_usb_transaction - describes each pending USB transaction
 266  *                               regardless of type. These are linked together
 267  *                               to form a list of pending requests for a pipe.
 268  *
 269  * @node:               List node for transactions in the pipe.
 270  * @type:               Type of transaction, duplicated of the pipe.
 271  * @flags:              State flags for this transaction.
 272  * @buffer:             User's physical buffer address to read/write.
 273  * @buffer_length:      Size of the user's buffer in bytes.
 274  * @control_header:     For control transactions, physical address of the 8
 275  *                      byte standard header.
 276  * @iso_start_frame:    For ISO transactions, the starting frame number.
 277  * @iso_number_packets: For ISO transactions, the number of packets in the
 278  *                      request.
 279  * @iso_packets:        For ISO transactions, the sub packets in the request.
 280  * @actual_bytes:       Actual bytes transfer for this transaction.
 281  * @stage:              For control transactions, the current stage.
 282  * @urb:                URB.
 283  */
 284 struct cvmx_usb_transaction {
 285         struct list_head node;
 286         enum cvmx_usb_transfer type;
 287         uint64_t buffer;
 288         int buffer_length;
 289         uint64_t control_header;
 290         int iso_start_frame;
 291         int iso_number_packets;
 292         struct cvmx_usb_iso_packet *iso_packets;
 293         int xfersize;
 294         int pktcnt;
 295         int retries;
 296         int actual_bytes;
 297         enum cvmx_usb_stage stage;
 298         struct urb *urb;
 299 };
 300
 301 /**
 302  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 303  *                        and some USB device. It contains a list of pending
 304  *                        request to the device.
 305  *
 306  * @node:               List node for pipe list
 307  * @next:               Pipe after this one in the list
 308  * @transactions:       List of pending transactions
 309  * @interval:           For periodic pipes, the interval between packets in
 310  *                      frames
 311  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 312  * @flags:              State flags for this pipe
 313  * @device_speed:       Speed of device connected to this pipe
 314  * @transfer_type:      Type of transaction supported by this pipe
 315  * @transfer_dir:       IN or OUT. Ignored for Control
 316  * @multi_count:        Max packet in a row for the device
 317  * @max_packet:         The device's maximum packet size in bytes
 318  * @device_addr:        USB device address at other end of pipe
 319  * @endpoint_num:       USB endpoint number at other end of pipe
 320  * @hub_device_addr:    Hub address this device is connected to
 321  * @hub_port:           Hub port this device is connected to
 322  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 323  *                      the data pid needed
 324  * @channel:            Hardware DMA channel for this pipe
 325  * @split_sc_frame:     The low order bits of the frame number the split
 326  *                      complete should be sent on
 327  */
 328 struct cvmx_usb_pipe {
 329         struct list_head node;
 330         struct list_head transactions;
 331         uint64_t interval;
 332         uint64_t next_tx_frame;
 333         enum cvmx_usb_pipe_flags flags;
 334         enum cvmx_usb_speed device_speed;
 335         enum cvmx_usb_transfer transfer_type;
 336         enum cvmx_usb_direction transfer_dir;
 337         int multi_count;
 338         uint16_t max_packet;
 339         uint8_t device_addr;
 340         uint8_t endpoint_num;
 341         uint8_t hub_device_addr;
 342         uint8_t hub_port;
 343         uint8_t pid_toggle;
 344         uint8_t channel;
 345         int8_t split_sc_frame;
 346 };
 347
 348 struct cvmx_usb_tx_fifo {
 349         struct {
 350                 int channel;
 351                 int size;
 352                 uint64_t address;
 353         } entry[MAX_CHANNELS+1];
 354         int head;
 355         int tail;
 356 };
 357
 358 /**
 359  * struct cvmx_usb_state - the state of the USB block
 360  *
 361  * init_flags:             Flags passed to initialize.
 362  * index:                  Which USB block this is for.
 363  * idle_hardware_channels: Bit set for every idle hardware channel.
 364  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 365  *                         determine splits.
 366  * pipe_for_channel:       Map channels to pipes.
 367  * pipe:                   Storage for pipes.
 368  * indent:                 Used by debug output to indent functions.
 369  * port_status:            Last port status used for change notification.
 370  * idle_pipes:             List of open pipes that have no transactions.
 371  * active_pipes:           Active pipes indexed by transfer type.
 372  * frame_number:           Increments every SOF interrupt for time keeping.
 373  * active_split:           Points to the current active split, or NULL.
 374  */
 375 struct cvmx_usb_state {
 376         int init_flags;
 377         int index;
 378         int idle_hardware_channels;
 379         union cvmx_usbcx_hprt usbcx_hprt;
 380         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 381         int indent;
 382         struct cvmx_usb_port_status port_status;
 383         struct list_head idle_pipes;
 384         struct list_head active_pipes[4];
 385         uint64_t frame_number;
 386         struct cvmx_usb_transaction *active_split;
 387         struct cvmx_usb_tx_fifo periodic;
 388         struct cvmx_usb_tx_fifo nonperiodic;
 389 };
 390
 391 struct octeon_hcd {
 392         spinlock_t lock;
 393         struct cvmx_usb_state usb;
 394 };
 395
 396 /* This macro spins on a register waiting for it to reach a condition. */
 397 #define CVMX_WAIT_FOR_FIELD32(address, _union, cond, timeout_usec)          \
 398         ({int result;                                                       \
 399         do {                                                                \
 400                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 401                         octeon_get_clock_rate() / 1000000;                  \
 402                 union _union c;                                             \
 403                                                                             \
 404                 while (1) {                                                 \
 405                         c.u32 = cvmx_usb_read_csr32(usb, address);          \
 406                                                                             \
 407                         if (cond) {                                         \
 408                                 result = 0;                                 \
 409                                 break;                                      \
 410                         } else if (cvmx_get_cycle() > done) {               \
 411                                 result = -1;                                \
 412                                 break;                                      \
 413                         } else                                              \
 414                                 cvmx_wait(100);                             \
 415                 }                                                           \
 416         } while (0);                                                        \
 417         result; })
 418
 419 /*
 420  * This macro logically sets a single field in a CSR. It does the sequence
 421  * read, modify, and write
 422  */
 423 #define USB_SET_FIELD32(address, _union, field, value)          \
 424         do {                                                    \
 425                 union _union c;                                 \
 426                                                                 \
 427                 c.u32 = cvmx_usb_read_csr32(usb, address);      \
 428                 c.s.field = value;                              \
 429                 cvmx_usb_write_csr32(usb, address, c.u32);      \
 430         } while (0)
 431
 432 /* Returns the IO address to push/pop stuff data from the FIFOs */
 433 #define USB_FIFO_ADDRESS(channel, usb_index) \
 434         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 435
 436 /**
 437  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 438  * @orig_buffer: the original buffer passed by the USB stack
 439  * @data:        the newly allocated temporary buffer (excluding meta-data)
 440  *
 441  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 442  * the buffer is too short, we need to allocate a temporary one, and this struct
 443  * represents it.
 444  */
 445 struct octeon_temp_buffer {
 446         void *orig_buffer;
 447         u8 data[0];
 448 };
 449
 450 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
 451 {
 452         return container_of(p, struct octeon_hcd, usb);
 453 }
 454
 455 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
 456 {
 457         return container_of((void *)p, struct usb_hcd, hcd_priv);
 458 }
 459
 460 /**
 461  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 462  *                            (if needed)
 463  * @urb:        URB.
 464  * @mem_flags:  Memory allocation flags.
 465  *
 466  * This function allocates a temporary bounce buffer whenever it's needed
 467  * due to HW limitations.
 468  */
 469 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 470 {
 471         struct octeon_temp_buffer *temp;
 472
 473         if (urb->num_sgs || urb->sg ||
 474             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 475             !(urb->transfer_buffer_length % sizeof(u32)))
 476                 return 0;
 477
 478         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 479                        sizeof(*temp), mem_flags);
 480         if (!temp)
 481                 return -ENOMEM;
 482
 483         temp->orig_buffer = urb->transfer_buffer;
 484         if (usb_urb_dir_out(urb))
 485                 memcpy(temp->data, urb->transfer_buffer,
 486                        urb->transfer_buffer_length);
 487         urb->transfer_buffer = temp->data;
 488         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 489
 490         return 0;
 491 }
 492
 493 /**
 494  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 495  * @urb: URB.
 496  *
 497  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 498  */
 499 static void octeon_free_temp_buffer(struct urb *urb)
 500 {
 501         struct octeon_temp_buffer *temp;
 502
 503         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 504                 return;
 505
 506         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 507                             data);
 508         if (usb_urb_dir_in(urb))
 509                 memcpy(temp->orig_buffer, urb->transfer_buffer,
 510                        urb->actual_length);
 511         urb->transfer_buffer = temp->orig_buffer;
 512         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 513         kfree(temp);
 514 }
 515
 516 /**
 517  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 518  * @hcd:        USB HCD structure.
 519  * @urb:        URB.
 520  * @mem_flags:  Memory allocation flags.
 521  */
 522 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 523                                   gfp_t mem_flags)
 524 {
 525         int ret;
 526
 527         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 528         if (ret)
 529                 return ret;
 530
 531         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 532         if (ret)
 533                 octeon_free_temp_buffer(urb);
 534
 535         return ret;
 536 }
 537
 538 /**
 539  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 540  * @hcd:        USB HCD structure.
 541  * @urb:        URB.
 542  */
 543 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 544 {
 545         usb_hcd_unmap_urb_for_dma(hcd, urb);
 546         octeon_free_temp_buffer(urb);
 547 }
 548
 549 /**
 550  * Read a USB 32bit CSR. It performs the necessary address swizzle
 551  * for 32bit CSRs and logs the value in a readable format if
 552  * debugging is on.
 553  *
 554  * @usb:     USB block this access is for
 555  * @address: 64bit address to read
 556  *
 557  * Returns: Result of the read
 558  */
 559 static inline uint32_t cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 560                                            uint64_t address)
 561 {
 562         uint32_t result = cvmx_read64_uint32(address ^ 4);
 563         return result;
 564 }
 565
 566
 567 /**
 568  * Write a USB 32bit CSR. It performs the necessary address
 569  * swizzle for 32bit CSRs and logs the value in a readable format
 570  * if debugging is on.
 571  *
 572  * @usb:     USB block this access is for
 573  * @address: 64bit address to write
 574  * @value:   Value to write
 575  */
 576 static inline void cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 577                                         uint64_t address, uint32_t value)
 578 {
 579         cvmx_write64_uint32(address ^ 4, value);
 580         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 581 }
 582
 583 /**
 584  * Return non zero if this pipe connects to a non HIGH speed
 585  * device through a high speed hub.
 586  *
 587  * @usb:    USB block this access is for
 588  * @pipe:   Pipe to check
 589  *
 590  * Returns: Non zero if we need to do split transactions
 591  */
 592 static inline int cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 593                                             struct cvmx_usb_pipe *pipe)
 594 {
 595         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 596                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 597 }
 598
 599
 600 /**
 601  * Trivial utility function to return the correct PID for a pipe
 602  *
 603  * @pipe:   pipe to check
 604  *
 605  * Returns: PID for pipe
 606  */
 607 static inline int cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 608 {
 609         if (pipe->pid_toggle)
 610                 return 2; /* Data1 */
 611         return 0; /* Data0 */
 612 }
 613
 614 static void cvmx_fifo_setup(struct cvmx_usb_state *usb)
 615 {
 616         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 617         union cvmx_usbcx_gnptxfsiz npsiz;
 618         union cvmx_usbcx_hptxfsiz psiz;
 619
 620         usbcx_ghwcfg3.u32 = cvmx_usb_read_csr32(usb,
 621                                                 CVMX_USBCX_GHWCFG3(usb->index));
 622
 623         /*
 624          * Program the USBC_GRXFSIZ register to select the size of the receive
 625          * FIFO (25%).
 626          */
 627         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz,
 628                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 629
 630         /*
 631          * Program the USBC_GNPTXFSIZ register to select the size and the start
 632          * address of the non-periodic transmit FIFO for nonperiodic
 633          * transactions (50%).
 634          */
 635         npsiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
 636         npsiz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
 637         npsiz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
 638         cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), npsiz.u32);
 639
 640         /*
 641          * Program the USBC_HPTXFSIZ register to select the size and start
 642          * address of the periodic transmit FIFO for periodic transactions
 643          * (25%).
 644          */
 645         psiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
 646         psiz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
 647         psiz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
 648         cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), psiz.u32);
 649
 650         /* Flush all FIFOs */
 651         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 652                         cvmx_usbcx_grstctl, txfnum, 0x10);
 653         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 654                         cvmx_usbcx_grstctl, txfflsh, 1);
 655         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 656                               cvmx_usbcx_grstctl, c.s.txfflsh == 0, 100);
 657         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 658                         cvmx_usbcx_grstctl, rxfflsh, 1);
 659         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 660                               cvmx_usbcx_grstctl, c.s.rxfflsh == 0, 100);
 661 }
 662
 663 /**
 664  * Shutdown a USB port after a call to cvmx_usb_initialize().
 665  * The port should be disabled with all pipes closed when this
 666  * function is called.
 667  *
 668  * @usb: USB device state populated by cvmx_usb_initialize().
 669  *
 670  * Returns: 0 or a negative error code.
 671  */
 672 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 673 {
 674         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 675
 676         /* Make sure all pipes are closed */
 677         if (!list_empty(&usb->idle_pipes) ||
 678             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 679             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 680             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 681             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 682                 return -EBUSY;
 683
 684         /* Disable the clocks and put them in power on reset */
 685         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 686         usbn_clk_ctl.s.enable = 1;
 687         usbn_clk_ctl.s.por = 1;
 688         usbn_clk_ctl.s.hclk_rst = 1;
 689         usbn_clk_ctl.s.prst = 0;
 690         usbn_clk_ctl.s.hrst = 0;
 691         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 692         return 0;
 693 }
 694
 695 /**
 696  * Initialize a USB port for use. This must be called before any
 697  * other access to the Octeon USB port is made. The port starts
 698  * off in the disabled state.
 699  *
 700  * @dev:         Pointer to struct device for logging purposes.
 701  * @usb:         Pointer to struct cvmx_usb_state.
 702  *
 703  * Returns: 0 or a negative error code.
 704  */
 705 static int cvmx_usb_initialize(struct device *dev,
 706                                struct cvmx_usb_state *usb)
 707 {
 708         int channel;
 709         int divisor;
 710         int retries = 0;
 711         union cvmx_usbcx_hcfg usbcx_hcfg;
 712         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 713         union cvmx_usbcx_gintsts usbc_gintsts;
 714         union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 715         union cvmx_usbcx_gintmsk usbcx_gintmsk;
 716         union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 717         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 718
 719 retry:
 720         /*
 721          * Power On Reset and PHY Initialization
 722          *
 723          * 1. Wait for DCOK to assert (nothing to do)
 724          *
 725          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 726          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 727          */
 728         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 729         usbn_clk_ctl.s.por = 1;
 730         usbn_clk_ctl.s.hrst = 0;
 731         usbn_clk_ctl.s.prst = 0;
 732         usbn_clk_ctl.s.hclk_rst = 0;
 733         usbn_clk_ctl.s.enable = 0;
 734         /*
 735          * 2b. Select the USB reference clock/crystal parameters by writing
 736          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 737          */
 738         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 739                 /*
 740                  * The USB port uses 12/24/48MHz 2.5V board clock
 741                  * source at USB_XO. USB_XI should be tied to GND.
 742                  * Most Octeon evaluation boards require this setting
 743                  */
 744                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 745                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 746                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 747                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 748                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 749                 else
 750                         /* From CN52XX manual */
 751                         usbn_clk_ctl.s.p_rtype = 1;
 752
 753                 switch (usb->init_flags &
 754                         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 755                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 756                         usbn_clk_ctl.s.p_c_sel = 0;
 757                         break;
 758                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 759                         usbn_clk_ctl.s.p_c_sel = 1;
 760                         break;
 761                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 762                         usbn_clk_ctl.s.p_c_sel = 2;
 763                         break;
 764                 }
 765         } else {
 766                 /*
 767                  * The USB port uses a 12MHz crystal as clock source
 768                  * at USB_XO and USB_XI
 769                  */
 770                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 771                         /* From CN31XX,CN30XX manual */
 772                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 773                 else
 774                         /* From CN56XX,CN52XX,CN50XX manuals. */
 775                         usbn_clk_ctl.s.p_rtype = 0;
 776
 777                 usbn_clk_ctl.s.p_c_sel = 0;
 778         }
 779         /*
 780          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 781          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 782          *     such that USB is as close as possible to 125Mhz
 783          */
 784         divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
 785         /* Lower than 4 doesn't seem to work properly */
 786         if (divisor < 4)
 787                 divisor = 4;
 788         usbn_clk_ctl.s.divide = divisor;
 789         usbn_clk_ctl.s.divide2 = 0;
 790         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 791
 792         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 793         usbn_clk_ctl.s.hclk_rst = 1;
 794         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 795         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 796         cvmx_wait(64);
 797         /*
 798          * 3. Program the power-on reset field in the USBN clock-control
 799          *    register:
 800          *    USBN_CLK_CTL[POR] = 0
 801          */
 802         usbn_clk_ctl.s.por = 0;
 803         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 804         /* 4. Wait 1 ms for PHY clock to start */
 805         mdelay(1);
 806         /*
 807          * 5. Program the Reset input from automatic test equipment field in the
 808          *    USBP control and status register:
 809          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 810          */
 811         usbn_usbp_ctl_status.u64 =
 812                 cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 813         usbn_usbp_ctl_status.s.ate_reset = 1;
 814         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 815                             usbn_usbp_ctl_status.u64);
 816         /* 6. Wait 10 cycles */
 817         cvmx_wait(10);
 818         /*
 819          * 7. Clear ATE_RESET field in the USBN clock-control register:
 820          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 821          */
 822         usbn_usbp_ctl_status.s.ate_reset = 0;
 823         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 824                             usbn_usbp_ctl_status.u64);
 825         /*
 826          * 8. Program the PHY reset field in the USBN clock-control register:
 827          *    USBN_CLK_CTL[PRST] = 1
 828          */
 829         usbn_clk_ctl.s.prst = 1;
 830         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 831         /*
 832          * 9. Program the USBP control and status register to select host or
 833          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 834          *    device
 835          */
 836         usbn_usbp_ctl_status.s.hst_mode = 0;
 837         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 838                             usbn_usbp_ctl_status.u64);
 839         /* 10. Wait 1 us */
 840         udelay(1);
 841         /*
 842          * 11. Program the hreset_n field in the USBN clock-control register:
 843          *     USBN_CLK_CTL[HRST] = 1
 844          */
 845         usbn_clk_ctl.s.hrst = 1;
 846         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 847         /* 12. Proceed to USB core initialization */
 848         usbn_clk_ctl.s.enable = 1;
 849         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 850         udelay(1);
 851
 852         /*
 853          * USB Core Initialization
 854          *
 855          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 856          *    determine USB core configuration parameters.
 857          *
 858          *    Nothing needed
 859          *
 860          * 2. Program the following fields in the global AHB configuration
 861          *    register (USBC_GAHBCFG)
 862          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 863          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 864          *    Nonperiodic TxFIFO empty level (slave mode only),
 865          *    USBC_GAHBCFG[NPTXFEMPLVL]
 866          *    Periodic TxFIFO empty level (slave mode only),
 867          *    USBC_GAHBCFG[PTXFEMPLVL]
 868          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 869          */
 870         usbcx_gahbcfg.u32 = 0;
 871         usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 872                                   CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 873         usbcx_gahbcfg.s.hbstlen = 0;
 874         usbcx_gahbcfg.s.nptxfemplvl = 1;
 875         usbcx_gahbcfg.s.ptxfemplvl = 1;
 876         usbcx_gahbcfg.s.glblintrmsk = 1;
 877         cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 878                              usbcx_gahbcfg.u32);
 879
 880         /*
 881          * 3. Program the following fields in USBC_GUSBCFG register.
 882          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 883          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 884          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 885          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 886          */
 887         usbcx_gusbcfg.u32 = cvmx_usb_read_csr32(usb,
 888                                                 CVMX_USBCX_GUSBCFG(usb->index));
 889         usbcx_gusbcfg.s.toutcal = 0;
 890         usbcx_gusbcfg.s.ddrsel = 0;
 891         usbcx_gusbcfg.s.usbtrdtim = 0x5;
 892         usbcx_gusbcfg.s.phylpwrclksel = 0;
 893         cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 894                              usbcx_gusbcfg.u32);
 895
 896         /*
 897          * 4. The software must unmask the following bits in the USBC_GINTMSK
 898          *    register.
 899          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 900          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 901          */
 902         usbcx_gintmsk.u32 = cvmx_usb_read_csr32(usb,
 903                                                 CVMX_USBCX_GINTMSK(usb->index));
 904         usbcx_gintmsk.s.otgintmsk = 1;
 905         usbcx_gintmsk.s.modemismsk = 1;
 906         usbcx_gintmsk.s.hchintmsk = 1;
 907         usbcx_gintmsk.s.sofmsk = 0;
 908         /* We need RX FIFO interrupts if we don't have DMA */
 909         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 910                 usbcx_gintmsk.s.rxflvlmsk = 1;
 911         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 912                              usbcx_gintmsk.u32);
 913
 914         /*
 915          * Disable all channel interrupts. We'll enable them per channel later.
 916          */
 917         for (channel = 0; channel < 8; channel++)
 918                 cvmx_usb_write_csr32(usb,
 919                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
 920                                      0);
 921
 922         /*
 923          * Host Port Initialization
 924          *
 925          * 1. Program the host-port interrupt-mask field to unmask,
 926          *    USBC_GINTMSK[PRTINT] = 1
 927          */
 928         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 929                         cvmx_usbcx_gintmsk, prtintmsk, 1);
 930         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 931                         cvmx_usbcx_gintmsk, disconnintmsk, 1);
 932
 933         /*
 934          * 2. Program the USBC_HCFG register to select full-speed host
 935          *    or high-speed host.
 936          */
 937         usbcx_hcfg.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 938         usbcx_hcfg.s.fslssupp = 0;
 939         usbcx_hcfg.s.fslspclksel = 0;
 940         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 941
 942         cvmx_fifo_setup(usb);
 943
 944         /*
 945          * If the controller is getting port events right after the reset, it
 946          * means the initialization failed. Try resetting the controller again
 947          * in such case. This is seen to happen after cold boot on DSR-1000N.
 948          */
 949         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
 950                                                CVMX_USBCX_GINTSTS(usb->index));
 951         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
 952                              usbc_gintsts.u32);
 953         dev_dbg(dev, "gintsts after reset: 0x%x\n", (int)usbc_gintsts.u32);
 954         if (!usbc_gintsts.s.disconnint && !usbc_gintsts.s.prtint)
 955                 return 0;
 956         if (retries++ >= 5)
 957                 return -EAGAIN;
 958         dev_info(dev, "controller reset failed (gintsts=0x%x) - retrying\n",
 959                  (int)usbc_gintsts.u32);
 960         msleep(50);
 961         cvmx_usb_shutdown(usb);
 962         msleep(50);
 963         goto retry;
 964 }
 965
 966 /**
 967  * Reset a USB port. After this call succeeds, the USB port is
 968  * online and servicing requests.
 969  *
 970  * @usb: USB device state populated by cvmx_usb_initialize().
 971  */
 972 static void cvmx_usb_reset_port(struct cvmx_usb_state *usb)
 973 {
 974         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 975                                                   CVMX_USBCX_HPRT(usb->index));
 976
 977         /* Program the port reset bit to start the reset process */
 978         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 979                         prtrst, 1);
 980
 981         /*
 982          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 983          * process to complete.
 984          */
 985         mdelay(50);
 986
 987         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 988         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 989                         prtrst, 0);
 990
 991         /*
 992          * Read the port speed field to get the enumerated speed,
 993          * USBC_HPRT[PRTSPD].
 994          */
 995         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 996                                                   CVMX_USBCX_HPRT(usb->index));
 997 }
 998
 999
1000 /**
1001  * Disable a USB port. After this call the USB port will not
1002  * generate data transfers and will not generate events.
1003  * Transactions in process will fail and call their
1004  * associated callbacks.
1005  *
1006  * @usb: USB device state populated by cvmx_usb_initialize().
1007  *
1008  * Returns: 0 or a negative error code.
1009  */
1010 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1011 {
1012         /* Disable the port */
1013         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
1014                         prtena, 1);
1015         return 0;
1016 }
1017
1018
1019 /**
1020  * Get the current state of the USB port. Use this call to
1021  * determine if the usb port has anything connected, is enabled,
1022  * or has some sort of error condition. The return value of this
1023  * call has "changed" bits to signal of the value of some fields
1024  * have changed between calls.
1025  *
1026  * @usb: USB device state populated by cvmx_usb_initialize().
1027  *
1028  * Returns: Port status information
1029  */
1030 static struct cvmx_usb_port_status cvmx_usb_get_status(
1031                 struct cvmx_usb_state *usb)
1032 {
1033         union cvmx_usbcx_hprt usbc_hprt;
1034         struct cvmx_usb_port_status result;
1035
1036         memset(&result, 0, sizeof(result));
1037
1038         usbc_hprt.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1039         result.port_enabled = usbc_hprt.s.prtena;
1040         result.port_over_current = usbc_hprt.s.prtovrcurract;
1041         result.port_powered = usbc_hprt.s.prtpwr;
1042         result.port_speed = usbc_hprt.s.prtspd;
1043         result.connected = usbc_hprt.s.prtconnsts;
1044         result.connect_change =
1045                 (result.connected != usb->port_status.connected);
1046
1047         return result;
1048 }
1049
1050 /**
1051  * Open a virtual pipe between the host and a USB device. A pipe
1052  * must be opened before data can be transferred between a device
1053  * and Octeon.
1054  *
1055  * @usb:             USB device state populated by cvmx_usb_initialize().
1056  * @device_addr:
1057  *                   USB device address to open the pipe to
1058  *                   (0-127).
1059  * @endpoint_num:
1060  *                   USB endpoint number to open the pipe to
1061  *                   (0-15).
1062  * @device_speed:
1063  *                   The speed of the device the pipe is going
1064  *                   to. This must match the device's speed,
1065  *                   which may be different than the port speed.
1066  * @max_packet:      The maximum packet length the device can
1067  *                   transmit/receive (low speed=0-8, full
1068  *                   speed=0-1023, high speed=0-1024). This value
1069  *                   comes from the standard endpoint descriptor
1070  *                   field wMaxPacketSize bits <10:0>.
1071  * @transfer_type:
1072  *                   The type of transfer this pipe is for.
1073  * @transfer_dir:
1074  *                   The direction the pipe is in. This is not
1075  *                   used for control pipes.
1076  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1077  *                   this is how often the transfer is scheduled
1078  *                   for. All other transfers should specify
1079  *                   zero. The units are in frames (8000/sec at
1080  *                   high speed, 1000/sec for full speed).
1081  * @multi_count:
1082  *                   For high speed devices, this is the maximum
1083  *                   allowed number of packet per microframe.
1084  *                   Specify zero for non high speed devices. This
1085  *                   value comes from the standard endpoint descriptor
1086  *                   field wMaxPacketSize bits <12:11>.
1087  * @hub_device_addr:
1088  *                   Hub device address this device is connected
1089  *                   to. Devices connected directly to Octeon
1090  *                   use zero. This is only used when the device
1091  *                   is full/low speed behind a high speed hub.
1092  *                   The address will be of the high speed hub,
1093  *                   not and full speed hubs after it.
1094  * @hub_port:        Which port on the hub the device is
1095  *                   connected. Use zero for devices connected
1096  *                   directly to Octeon. Like hub_device_addr,
1097  *                   this is only used for full/low speed
1098  *                   devices behind a high speed hub.
1099  *
1100  * Returns: A non-NULL value is a pipe. NULL means an error.
1101  */
1102 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1103                                                 int device_addr,
1104                                                 int endpoint_num,
1105                                                 enum cvmx_usb_speed
1106                                                         device_speed,
1107                                                 int max_packet,
1108                                                 enum cvmx_usb_transfer
1109                                                         transfer_type,
1110                                                 enum cvmx_usb_direction
1111                                                         transfer_dir,
1112                                                 int interval, int multi_count,
1113                                                 int hub_device_addr,
1114                                                 int hub_port)
1115 {
1116         struct cvmx_usb_pipe *pipe;
1117
1118         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1119         if (!pipe)
1120                 return NULL;
1121         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1122                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1123                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1124                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
1125         pipe->device_addr = device_addr;
1126         pipe->endpoint_num = endpoint_num;
1127         pipe->device_speed = device_speed;
1128         pipe->max_packet = max_packet;
1129         pipe->transfer_type = transfer_type;
1130         pipe->transfer_dir = transfer_dir;
1131         INIT_LIST_HEAD(&pipe->transactions);
1132
1133         /*
1134          * All pipes use interval to rate limit NAK processing. Force an
1135          * interval if one wasn't supplied
1136          */
1137         if (!interval)
1138                 interval = 1;
1139         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1140                 pipe->interval = interval*8;
1141                 /* Force start splits to be schedule on uFrame 0 */
1142                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1143                                         pipe->interval;
1144         } else {
1145                 pipe->interval = interval;
1146                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1147         }
1148         pipe->multi_count = multi_count;
1149         pipe->hub_device_addr = hub_device_addr;
1150         pipe->hub_port = hub_port;
1151         pipe->pid_toggle = 0;
1152         pipe->split_sc_frame = -1;
1153         list_add_tail(&pipe->node, &usb->idle_pipes);
1154
1155         /*
1156          * We don't need to tell the hardware about this pipe yet since
1157          * it doesn't have any submitted requests
1158          */
1159
1160         return pipe;
1161 }
1162
1163
1164 /**
1165  * Poll the RX FIFOs and remove data as needed. This function is only used
1166  * in non DMA mode. It is very important that this function be called quickly
1167  * enough to prevent FIFO overflow.
1168  *
1169  * @usb:        USB device state populated by cvmx_usb_initialize().
1170  */
1171 static void cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1172 {
1173         union cvmx_usbcx_grxstsph rx_status;
1174         int channel;
1175         int bytes;
1176         uint64_t address;
1177         uint32_t *ptr;
1178
1179         rx_status.u32 = cvmx_usb_read_csr32(usb,
1180                                             CVMX_USBCX_GRXSTSPH(usb->index));
1181         /* Only read data if IN data is there */
1182         if (rx_status.s.pktsts != 2)
1183                 return;
1184         /* Check if no data is available */
1185         if (!rx_status.s.bcnt)
1186                 return;
1187
1188         channel = rx_status.s.chnum;
1189         bytes = rx_status.s.bcnt;
1190         if (!bytes)
1191                 return;
1192
1193         /* Get where the DMA engine would have written this data */
1194         address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
1195                                      channel * 8);
1196
1197         ptr = cvmx_phys_to_ptr(address);
1198         cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
1199                             address + bytes);
1200
1201         /* Loop writing the FIFO data for this packet into memory */
1202         while (bytes > 0) {
1203                 *ptr++ = cvmx_usb_read_csr32(usb,
1204                                         USB_FIFO_ADDRESS(channel, usb->index));
1205                 bytes -= 4;
1206         }
1207         CVMX_SYNCW;
1208 }
1209
1210
1211 /**
1212  * Fill the TX hardware fifo with data out of the software
1213  * fifos
1214  *
1215  * @usb:            USB device state populated by cvmx_usb_initialize().
1216  * @fifo:           Software fifo to use
1217  * @available:      Amount of space in the hardware fifo
1218  *
1219  * Returns: Non zero if the hardware fifo was too small and needs
1220  *          to be serviced again.
1221  */
1222 static int cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1223                                struct cvmx_usb_tx_fifo *fifo, int available)
1224 {
1225         /*
1226          * We're done either when there isn't anymore space or the software FIFO
1227          * is empty
1228          */
1229         while (available && (fifo->head != fifo->tail)) {
1230                 int i = fifo->tail;
1231                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1232                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1233                                                         usb->index) ^ 4;
1234                 int words = available;
1235
1236                 /* Limit the amount of data to waht the SW fifo has */
1237                 if (fifo->entry[i].size <= available) {
1238                         words = fifo->entry[i].size;
1239                         fifo->tail++;
1240                         if (fifo->tail > MAX_CHANNELS)
1241                                 fifo->tail = 0;
1242                 }
1243
1244                 /* Update the next locations and counts */
1245                 available -= words;
1246                 fifo->entry[i].address += words * 4;
1247                 fifo->entry[i].size -= words;
1248
1249                 /*
1250                  * Write the HW fifo data. The read every three writes is due
1251                  * to an errata on CN3XXX chips
1252                  */
1253                 while (words > 3) {
1254                         cvmx_write64_uint32(csr_address, *ptr++);
1255                         cvmx_write64_uint32(csr_address, *ptr++);
1256                         cvmx_write64_uint32(csr_address, *ptr++);
1257                         cvmx_read64_uint64(
1258                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1259                         words -= 3;
1260                 }
1261                 cvmx_write64_uint32(csr_address, *ptr++);
1262                 if (--words) {
1263                         cvmx_write64_uint32(csr_address, *ptr++);
1264                         if (--words)
1265                                 cvmx_write64_uint32(csr_address, *ptr++);
1266                 }
1267                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1268         }
1269         return fifo->head != fifo->tail;
1270 }
1271
1272
1273 /**
1274  * Check the hardware FIFOs and fill them as needed
1275  *
1276  * @usb:        USB device state populated by cvmx_usb_initialize().
1277  */
1278 static void cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1279 {
1280         if (usb->periodic.head != usb->periodic.tail) {
1281                 union cvmx_usbcx_hptxsts tx_status;
1282
1283                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1284                                         CVMX_USBCX_HPTXSTS(usb->index));
1285                 if (cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1286                                         tx_status.s.ptxfspcavail))
1287                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1288                                         cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1289                 else
1290                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1291                                         cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1292         }
1293
1294         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1295                 union cvmx_usbcx_gnptxsts tx_status;
1296
1297                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1298                                         CVMX_USBCX_GNPTXSTS(usb->index));
1299                 if (cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1300                                         tx_status.s.nptxfspcavail))
1301                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1302                                         cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1303                 else
1304                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1305                                         cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1306         }
1307 }
1308
1309
1310 /**
1311  * Fill the TX FIFO with an outgoing packet
1312  *
1313  * @usb:          USB device state populated by cvmx_usb_initialize().
1314  * @channel:      Channel number to get packet from
1315  */
1316 static void cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1317 {
1318         union cvmx_usbcx_hccharx hcchar;
1319         union cvmx_usbcx_hcspltx usbc_hcsplt;
1320         union cvmx_usbcx_hctsizx usbc_hctsiz;
1321         struct cvmx_usb_tx_fifo *fifo;
1322
1323         /* We only need to fill data on outbound channels */
1324         hcchar.u32 = cvmx_usb_read_csr32(usb,
1325                         CVMX_USBCX_HCCHARX(channel, usb->index));
1326         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1327                 return;
1328
1329         /* OUT Splits only have data on the start and not the complete */
1330         usbc_hcsplt.u32 = cvmx_usb_read_csr32(usb,
1331                                 CVMX_USBCX_HCSPLTX(channel, usb->index));
1332         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1333                 return;
1334
1335         /*
1336          * Find out how many bytes we need to fill and convert it into 32bit
1337          * words.
1338          */
1339         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1340                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1341         if (!usbc_hctsiz.s.xfersize)
1342                 return;
1343
1344         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1345                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1346                 fifo = &usb->periodic;
1347         else
1348                 fifo = &usb->nonperiodic;
1349
1350         fifo->entry[fifo->head].channel = channel;
1351         fifo->entry[fifo->head].address =
1352                 cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1353                                    channel * 8);
1354         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1355         fifo->head++;
1356         if (fifo->head > MAX_CHANNELS)
1357                 fifo->head = 0;
1358
1359         cvmx_usb_poll_tx_fifo(usb);
1360 }
1361
1362 /**
1363  * Perform channel specific setup for Control transactions. All
1364  * the generic stuff will already have been done in cvmx_usb_start_channel().
1365  *
1366  * @usb:          USB device state populated by cvmx_usb_initialize().
1367  * @channel:      Channel to setup
1368  * @pipe:         Pipe for control transaction
1369  */
1370 static void cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1371                                            int channel,
1372                                            struct cvmx_usb_pipe *pipe)
1373 {
1374         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1375         struct usb_hcd *hcd = octeon_to_hcd(priv);
1376         struct device *dev = hcd->self.controller;
1377         struct cvmx_usb_transaction *transaction =
1378                 list_first_entry(&pipe->transactions, typeof(*transaction),
1379                                  node);
1380         struct usb_ctrlrequest *header =
1381                 cvmx_phys_to_ptr(transaction->control_header);
1382         int bytes_to_transfer = transaction->buffer_length -
1383                 transaction->actual_bytes;
1384         int packets_to_transfer;
1385         union cvmx_usbcx_hctsizx usbc_hctsiz;
1386
1387         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1388                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1389
1390         switch (transaction->stage) {
1391         case CVMX_USB_STAGE_NON_CONTROL:
1392         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1393                 dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
1394                 break;
1395         case CVMX_USB_STAGE_SETUP:
1396                 usbc_hctsiz.s.pid = 3; /* Setup */
1397                 bytes_to_transfer = sizeof(*header);
1398                 /* All Control operations start with a setup going OUT */
1399                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1400                                 cvmx_usbcx_hccharx, epdir,
1401                                 CVMX_USB_DIRECTION_OUT);
1402                 /*
1403                  * Setup send the control header instead of the buffer data. The
1404                  * buffer data will be used in the next stage
1405                  */
1406                 cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1407                                         channel * 8,
1408                                     transaction->control_header);
1409                 break;
1410         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1411                 usbc_hctsiz.s.pid = 3; /* Setup */
1412                 bytes_to_transfer = 0;
1413                 /* All Control operations start with a setup going OUT */
1414                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1415                                 cvmx_usbcx_hccharx, epdir,
1416                                 CVMX_USB_DIRECTION_OUT);
1417
1418                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1419                                 cvmx_usbcx_hcspltx, compsplt, 1);
1420                 break;
1421         case CVMX_USB_STAGE_DATA:
1422                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1423                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1424                         if (header->bRequestType & USB_DIR_IN)
1425                                 bytes_to_transfer = 0;
1426                         else if (bytes_to_transfer > pipe->max_packet)
1427                                 bytes_to_transfer = pipe->max_packet;
1428                 }
1429                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1430                                 cvmx_usbcx_hccharx, epdir,
1431                                 ((header->bRequestType & USB_DIR_IN) ?
1432                                         CVMX_USB_DIRECTION_IN :
1433                                         CVMX_USB_DIRECTION_OUT));
1434                 break;
1435         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1436                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1437                 if (!(header->bRequestType & USB_DIR_IN))
1438                         bytes_to_transfer = 0;
1439                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1440                                 cvmx_usbcx_hccharx, epdir,
1441                                 ((header->bRequestType & USB_DIR_IN) ?
1442                                         CVMX_USB_DIRECTION_IN :
1443                                         CVMX_USB_DIRECTION_OUT));
1444                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1445                                 cvmx_usbcx_hcspltx, compsplt, 1);
1446                 break;
1447         case CVMX_USB_STAGE_STATUS:
1448                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1449                 bytes_to_transfer = 0;
1450                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1451                                 cvmx_usbcx_hccharx, epdir,
1452                                 ((header->bRequestType & USB_DIR_IN) ?
1453                                         CVMX_USB_DIRECTION_OUT :
1454                                         CVMX_USB_DIRECTION_IN));
1455                 break;
1456         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1457                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1458                 bytes_to_transfer = 0;
1459                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1460                                 cvmx_usbcx_hccharx, epdir,
1461                                 ((header->bRequestType & USB_DIR_IN) ?
1462                                         CVMX_USB_DIRECTION_OUT :
1463                                         CVMX_USB_DIRECTION_IN));
1464                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1465                                 cvmx_usbcx_hcspltx, compsplt, 1);
1466                 break;
1467         }
1468
1469         /*
1470          * Make sure the transfer never exceeds the byte limit of the hardware.
1471          * Further bytes will be sent as continued transactions
1472          */
1473         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1474                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1475                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1476                 bytes_to_transfer *= pipe->max_packet;
1477         }
1478
1479         /*
1480          * Calculate the number of packets to transfer. If the length is zero
1481          * we still need to transfer one packet
1482          */
1483         packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
1484                                            pipe->max_packet);
1485         if (packets_to_transfer == 0)
1486                 packets_to_transfer = 1;
1487         else if ((packets_to_transfer > 1) &&
1488                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1489                 /*
1490                  * Limit to one packet when not using DMA. Channels must be
1491                  * restarted between every packet for IN transactions, so there
1492                  * is no reason to do multiple packets in a row
1493                  */
1494                 packets_to_transfer = 1;
1495                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1496         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1497                 /*
1498                  * Limit the number of packet and data transferred to what the
1499                  * hardware can handle
1500                  */
1501                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1502                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1503         }
1504
1505         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1506         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1507
1508         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1509                              usbc_hctsiz.u32);
1510 }
1511
1512
1513 /**
1514  * Start a channel to perform the pipe's head transaction
1515  *
1516  * @usb:          USB device state populated by cvmx_usb_initialize().
1517  * @channel:      Channel to setup
1518  * @pipe:         Pipe to start
1519  */
1520 static void cvmx_usb_start_channel(struct cvmx_usb_state *usb, int channel,
1521                                    struct cvmx_usb_pipe *pipe)
1522 {
1523         struct cvmx_usb_transaction *transaction =
1524                 list_first_entry(&pipe->transactions, typeof(*transaction),
1525                                  node);
1526
1527         /* Make sure all writes to the DMA region get flushed */
1528         CVMX_SYNCW;
1529
1530         /* Attach the channel to the pipe */
1531         usb->pipe_for_channel[channel] = pipe;
1532         pipe->channel = channel;
1533         pipe->flags |= CVMX_USB_PIPE_FLAGS_SCHEDULED;
1534
1535         /* Mark this channel as in use */
1536         usb->idle_hardware_channels &= ~(1<<channel);
1537
1538         /* Enable the channel interrupt bits */
1539         {
1540                 union cvmx_usbcx_hcintx usbc_hcint;
1541                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1542                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1543
1544                 /* Clear all channel status bits */
1545                 usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
1546                                         CVMX_USBCX_HCINTX(channel, usb->index));
1547
1548                 cvmx_usb_write_csr32(usb,
1549                                      CVMX_USBCX_HCINTX(channel, usb->index),
1550                                      usbc_hcint.u32);
1551
1552                 usbc_hcintmsk.u32 = 0;
1553                 usbc_hcintmsk.s.chhltdmsk = 1;
1554                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1555                         /*
1556                          * Channels need these extra interrupts when we aren't
1557                          * in DMA mode.
1558                          */
1559                         usbc_hcintmsk.s.datatglerrmsk = 1;
1560                         usbc_hcintmsk.s.frmovrunmsk = 1;
1561                         usbc_hcintmsk.s.bblerrmsk = 1;
1562                         usbc_hcintmsk.s.xacterrmsk = 1;
1563                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1564                                 /*
1565                                  * Splits don't generate xfercompl, so we need
1566                                  * ACK and NYET.
1567                                  */
1568                                 usbc_hcintmsk.s.nyetmsk = 1;
1569                                 usbc_hcintmsk.s.ackmsk = 1;
1570                         }
1571                         usbc_hcintmsk.s.nakmsk = 1;
1572                         usbc_hcintmsk.s.stallmsk = 1;
1573                         usbc_hcintmsk.s.xfercomplmsk = 1;
1574                 }
1575                 cvmx_usb_write_csr32(usb,
1576                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
1577                                 usbc_hcintmsk.u32);
1578
1579                 /* Enable the channel interrupt to propagate */
1580                 usbc_haintmsk.u32 = cvmx_usb_read_csr32(usb,
1581                                         CVMX_USBCX_HAINTMSK(usb->index));
1582                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1583                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index),
1584                                      usbc_haintmsk.u32);
1585         }
1586
1587         /* Setup the location the DMA engine uses. */
1588         {
1589                 uint64_t reg;
1590                 uint64_t dma_address = transaction->buffer +
1591                                         transaction->actual_bytes;
1592
1593                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1594                         dma_address = transaction->buffer +
1595                                         transaction->iso_packets[0].offset +
1596                                         transaction->actual_bytes;
1597
1598                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT)
1599                         reg = CVMX_USBNX_DMA0_OUTB_CHN0(usb->index);
1600                 else
1601                         reg = CVMX_USBNX_DMA0_INB_CHN0(usb->index);
1602                 cvmx_write64_uint64(reg + channel * 8, dma_address);
1603         }
1604
1605         /* Setup both the size of the transfer and the SPLIT characteristics */
1606         {
1607                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1608                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1609                 int packets_to_transfer;
1610                 int bytes_to_transfer = transaction->buffer_length -
1611                         transaction->actual_bytes;
1612
1613                 /*
1614                  * ISOCHRONOUS transactions store each individual transfer size
1615                  * in the packet structure, not the global buffer_length
1616                  */
1617                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1618                         bytes_to_transfer =
1619                                 transaction->iso_packets[0].length -
1620                                 transaction->actual_bytes;
1621
1622                 /*
1623                  * We need to do split transactions when we are talking to non
1624                  * high speed devices that are behind a high speed hub
1625                  */
1626                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1627                         /*
1628                          * On the start split phase (stage is even) record the
1629                          * frame number we will need to send the split complete.
1630                          * We only store the lower two bits since the time ahead
1631                          * can only be two frames
1632                          */
1633                         if ((transaction->stage&1) == 0) {
1634                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1635                                         pipe->split_sc_frame =
1636                                                 (usb->frame_number + 1) & 0x7f;
1637                                 else
1638                                         pipe->split_sc_frame =
1639                                                 (usb->frame_number + 2) & 0x7f;
1640                         } else
1641                                 pipe->split_sc_frame = -1;
1642
1643                         usbc_hcsplt.s.spltena = 1;
1644                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1645                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1646                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1647                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1648
1649                         /*
1650                          * SPLIT transactions can only ever transmit one data
1651                          * packet so limit the transfer size to the max packet
1652                          * size
1653                          */
1654                         if (bytes_to_transfer > pipe->max_packet)
1655                                 bytes_to_transfer = pipe->max_packet;
1656
1657                         /*
1658                          * ISOCHRONOUS OUT splits are unique in that they limit
1659                          * data transfers to 188 byte chunks representing the
1660                          * begin/middle/end of the data or all
1661                          */
1662                         if (!usbc_hcsplt.s.compsplt &&
1663                                 (pipe->transfer_dir ==
1664                                  CVMX_USB_DIRECTION_OUT) &&
1665                                 (pipe->transfer_type ==
1666                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1667                                 /*
1668                                  * Clear the split complete frame number as
1669                                  * there isn't going to be a split complete
1670                                  */
1671                                 pipe->split_sc_frame = -1;
1672                                 /*
1673                                  * See if we've started this transfer and sent
1674                                  * data
1675                                  */
1676                                 if (transaction->actual_bytes == 0) {
1677                                         /*
1678                                          * Nothing sent yet, this is either a
1679                                          * begin or the entire payload
1680                                          */
1681                                         if (bytes_to_transfer <= 188)
1682                                                 /* Entire payload in one go */
1683                                                 usbc_hcsplt.s.xactpos = 3;
1684                                         else
1685                                                 /* First part of payload */
1686                                                 usbc_hcsplt.s.xactpos = 2;
1687                                 } else {
1688                                         /*
1689                                          * Continuing the previous data, we must
1690                                          * either be in the middle or at the end
1691                                          */
1692                                         if (bytes_to_transfer <= 188)
1693                                                 /* End of payload */
1694                                                 usbc_hcsplt.s.xactpos = 1;
1695                                         else
1696                                                 /* Middle of payload */
1697                                                 usbc_hcsplt.s.xactpos = 0;
1698                                 }
1699                                 /*
1700                                  * Again, the transfer size is limited to 188
1701                                  * bytes
1702                                  */
1703                                 if (bytes_to_transfer > 188)
1704                                         bytes_to_transfer = 188;
1705                         }
1706                 }
1707
1708                 /*
1709                  * Make sure the transfer never exceeds the byte limit of the
1710                  * hardware. Further bytes will be sent as continued
1711                  * transactions
1712                  */
1713                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1714                         /*
1715                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1716                          * size
1717                          */
1718                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1719                                 pipe->max_packet;
1720                         bytes_to_transfer *= pipe->max_packet;
1721                 }
1722
1723                 /*
1724                  * Calculate the number of packets to transfer. If the length is
1725                  * zero we still need to transfer one packet
1726                  */
1727                 packets_to_transfer =
1728                         DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
1729                 if (packets_to_transfer == 0)
1730                         packets_to_transfer = 1;
1731                 else if ((packets_to_transfer > 1) &&
1732                                 (usb->init_flags &
1733                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1734                         /*
1735                          * Limit to one packet when not using DMA. Channels must
1736                          * be restarted between every packet for IN
1737                          * transactions, so there is no reason to do multiple
1738                          * packets in a row
1739                          */
1740                         packets_to_transfer = 1;
1741                         bytes_to_transfer = packets_to_transfer *
1742                                 pipe->max_packet;
1743                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1744                         /*
1745                          * Limit the number of packet and data transferred to
1746                          * what the hardware can handle
1747                          */
1748                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1749                         bytes_to_transfer = packets_to_transfer *
1750                                 pipe->max_packet;
1751                 }
1752
1753                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1754                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1755
1756                 /* Update the DATA0/DATA1 toggle */
1757                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1758                 /*
1759                  * High speed pipes may need a hardware ping before they start
1760                  */
1761                 if (pipe->flags & CVMX_USB_PIPE_FLAGS_NEED_PING)
1762                         usbc_hctsiz.s.dopng = 1;
1763
1764                 cvmx_usb_write_csr32(usb,
1765                                      CVMX_USBCX_HCSPLTX(channel, usb->index),
1766                                      usbc_hcsplt.u32);
1767                 cvmx_usb_write_csr32(usb,
1768                                      CVMX_USBCX_HCTSIZX(channel, usb->index),
1769                                      usbc_hctsiz.u32);
1770         }
1771
1772         /* Setup the Host Channel Characteristics Register */
1773         {
1774                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1775
1776                 /*
1777                  * Set the startframe odd/even properly. This is only used for
1778                  * periodic
1779                  */
1780                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1781
1782                 /*
1783                  * Set the number of back to back packets allowed by this
1784                  * endpoint. Split transactions interpret "ec" as the number of
1785                  * immediate retries of failure. These retries happen too
1786                  * quickly, so we disable these entirely for splits
1787                  */
1788                 if (cvmx_usb_pipe_needs_split(usb, pipe))
1789                         usbc_hcchar.s.ec = 1;
1790                 else if (pipe->multi_count < 1)
1791                         usbc_hcchar.s.ec = 1;
1792                 else if (pipe->multi_count > 3)
1793                         usbc_hcchar.s.ec = 3;
1794                 else
1795                         usbc_hcchar.s.ec = pipe->multi_count;
1796
1797                 /* Set the rest of the endpoint specific settings */
1798                 usbc_hcchar.s.devaddr = pipe->device_addr;
1799                 usbc_hcchar.s.eptype = transaction->type;
1800                 usbc_hcchar.s.lspddev =
1801                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1802                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1803                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1804                 usbc_hcchar.s.mps = pipe->max_packet;
1805                 cvmx_usb_write_csr32(usb,
1806                                      CVMX_USBCX_HCCHARX(channel, usb->index),
1807                                      usbc_hcchar.u32);
1808         }
1809
1810         /* Do transaction type specific fixups as needed */
1811         switch (transaction->type) {
1812         case CVMX_USB_TRANSFER_CONTROL:
1813                 cvmx_usb_start_channel_control(usb, channel, pipe);
1814                 break;
1815         case CVMX_USB_TRANSFER_BULK:
1816         case CVMX_USB_TRANSFER_INTERRUPT:
1817                 break;
1818         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1819                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
1820                         /*
1821                          * ISO transactions require different PIDs depending on
1822                          * direction and how many packets are needed
1823                          */
1824                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1825                                 if (pipe->multi_count < 2) /* Need DATA0 */
1826                                         USB_SET_FIELD32(
1827                                                 CVMX_USBCX_HCTSIZX(channel,
1828                                                                    usb->index),
1829                                                 cvmx_usbcx_hctsizx, pid, 0);
1830                                 else /* Need MDATA */
1831                                         USB_SET_FIELD32(
1832                                                 CVMX_USBCX_HCTSIZX(channel,
1833                                                                    usb->index),
1834                                                 cvmx_usbcx_hctsizx, pid, 3);
1835                         }
1836                 }
1837                 break;
1838         }
1839         {
1840                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
1841                         cvmx_usb_read_csr32(usb,
1842                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
1843                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1844                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1845         }
1846         /* Remeber when we start a split transaction */
1847         if (cvmx_usb_pipe_needs_split(usb, pipe))
1848                 usb->active_split = transaction;
1849         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1850                         cvmx_usbcx_hccharx, chena, 1);
1851         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1852                 cvmx_usb_fill_tx_fifo(usb, channel);
1853 }
1854
1855
1856 /**
1857  * Find a pipe that is ready to be scheduled to hardware.
1858  * @usb:         USB device state populated by cvmx_usb_initialize().
1859  * @list:        Pipe list to search
1860  * @current_frame:
1861  *               Frame counter to use as a time reference.
1862  *
1863  * Returns: Pipe or NULL if none are ready
1864  */
1865 static struct cvmx_usb_pipe *cvmx_usb_find_ready_pipe(
1866                 struct cvmx_usb_state *usb,
1867                 struct list_head *list,
1868                 uint64_t current_frame)
1869 {
1870         struct cvmx_usb_pipe *pipe;
1871
1872         list_for_each_entry(pipe, list, node) {
1873                 struct cvmx_usb_transaction *t =
1874                         list_first_entry(&pipe->transactions, typeof(*t),
1875                                          node);
1876                 if (!(pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
1877                         (pipe->next_tx_frame <= current_frame) &&
1878                         ((pipe->split_sc_frame == -1) ||
1879                          ((((int)current_frame - (int)pipe->split_sc_frame)
1880                            & 0x7f) < 0x40)) &&
1881                         (!usb->active_split || (usb->active_split == t))) {
1882                         prefetch(t);
1883                         return pipe;
1884                 }
1885         }
1886         return NULL;
1887 }
1888
1889
1890 /**
1891  * Called whenever a pipe might need to be scheduled to the
1892  * hardware.
1893  *
1894  * @usb:         USB device state populated by cvmx_usb_initialize().
1895  * @is_sof:      True if this schedule was called on a SOF interrupt.
1896  */
1897 static void cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
1898 {
1899         int channel;
1900         struct cvmx_usb_pipe *pipe;
1901         int need_sof;
1902         enum cvmx_usb_transfer ttype;
1903
1904         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1905                 /*
1906                  * Without DMA we need to be careful to not schedule something
1907                  * at the end of a frame and cause an overrun.
1908                  */
1909                 union cvmx_usbcx_hfnum hfnum = {
1910                         .u32 = cvmx_usb_read_csr32(usb,
1911                                                 CVMX_USBCX_HFNUM(usb->index))
1912                 };
1913
1914                 union cvmx_usbcx_hfir hfir = {
1915                         .u32 = cvmx_usb_read_csr32(usb,
1916                                                 CVMX_USBCX_HFIR(usb->index))
1917                 };
1918
1919                 if (hfnum.s.frrem < hfir.s.frint/4)
1920                         goto done;
1921         }
1922
1923         while (usb->idle_hardware_channels) {
1924                 /* Find an idle channel */
1925                 channel = __fls(usb->idle_hardware_channels);
1926                 if (unlikely(channel > 7))
1927                         break;
1928
1929                 /* Find a pipe needing service */
1930                 pipe = NULL;
1931                 if (is_sof) {
1932                         /*
1933                          * Only process periodic pipes on SOF interrupts. This
1934                          * way we are sure that the periodic data is sent in the
1935                          * beginning of the frame
1936                          */
1937                         pipe = cvmx_usb_find_ready_pipe(usb,
1938                                         usb->active_pipes +
1939                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
1940                                         usb->frame_number);
1941                         if (likely(!pipe))
1942                                 pipe = cvmx_usb_find_ready_pipe(usb,
1943                                                 usb->active_pipes +
1944                                                 CVMX_USB_TRANSFER_INTERRUPT,
1945                                                 usb->frame_number);
1946                 }
1947                 if (likely(!pipe)) {
1948                         pipe = cvmx_usb_find_ready_pipe(usb,
1949                                         usb->active_pipes +
1950                                         CVMX_USB_TRANSFER_CONTROL,
1951                                         usb->frame_number);
1952                         if (likely(!pipe))
1953                                 pipe = cvmx_usb_find_ready_pipe(usb,
1954                                                 usb->active_pipes +
1955                                                 CVMX_USB_TRANSFER_BULK,
1956                                                 usb->frame_number);
1957                 }
1958                 if (!pipe)
1959                         break;
1960
1961                 cvmx_usb_start_channel(usb, channel, pipe);
1962         }
1963
1964 done:
1965         /*
1966          * Only enable SOF interrupts when we have transactions pending in the
1967          * future that might need to be scheduled
1968          */
1969         need_sof = 0;
1970         for (ttype = CVMX_USB_TRANSFER_CONTROL;
1971                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1972                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
1973                         if (pipe->next_tx_frame > usb->frame_number) {
1974                                 need_sof = 1;
1975                                 break;
1976                         }
1977                 }
1978         }
1979         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1980                         cvmx_usbcx_gintmsk, sofmsk, need_sof);
1981 }
1982
1983 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
1984                                              enum cvmx_usb_complete status,
1985                                              struct cvmx_usb_pipe *pipe,
1986                                              struct cvmx_usb_transaction
1987                                                 *transaction,
1988                                              int bytes_transferred,
1989                                              struct urb *urb)
1990 {
1991         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1992         struct usb_hcd *hcd = octeon_to_hcd(priv);
1993         struct device *dev = hcd->self.controller;
1994
1995         if (likely(status == CVMX_USB_COMPLETE_SUCCESS))
1996                 urb->actual_length = bytes_transferred;
1997         else
1998                 urb->actual_length = 0;
1999
2000         urb->hcpriv = NULL;
2001
2002         /* For Isochronous transactions we need to update the URB packet status
2003            list from data in our private copy */
2004         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2005                 int i;
2006                 /*
2007                  * The pointer to the private list is stored in the setup_packet
2008                  * field.
2009                  */
2010                 struct cvmx_usb_iso_packet *iso_packet =
2011                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2012                 /* Recalculate the transfer size by adding up each packet */
2013                 urb->actual_length = 0;
2014                 for (i = 0; i < urb->number_of_packets; i++) {
2015                         if (iso_packet[i].status ==
2016                                         CVMX_USB_COMPLETE_SUCCESS) {
2017                                 urb->iso_frame_desc[i].status = 0;
2018                                 urb->iso_frame_desc[i].actual_length =
2019                                         iso_packet[i].length;
2020                                 urb->actual_length +=
2021                                         urb->iso_frame_desc[i].actual_length;
2022                         } else {
2023                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2024                                         i, urb->number_of_packets,
2025                                         iso_packet[i].status, pipe,
2026                                         transaction, iso_packet[i].length);
2027                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2028                         }
2029                 }
2030                 /* Free the private list now that we don't need it anymore */
2031                 kfree(iso_packet);
2032                 urb->setup_packet = NULL;
2033         }
2034
2035         switch (status) {
2036         case CVMX_USB_COMPLETE_SUCCESS:
2037                 urb->status = 0;
2038                 break;
2039         case CVMX_USB_COMPLETE_CANCEL:
2040                 if (urb->status == 0)
2041                         urb->status = -ENOENT;
2042                 break;
2043         case CVMX_USB_COMPLETE_STALL:
2044                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2045                         pipe, transaction, bytes_transferred);
2046                 urb->status = -EPIPE;
2047                 break;
2048         case CVMX_USB_COMPLETE_BABBLEERR:
2049                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2050                         pipe, transaction, bytes_transferred);
2051                 urb->status = -EPIPE;
2052                 break;
2053         case CVMX_USB_COMPLETE_SHORT:
2054                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2055                         pipe, transaction, bytes_transferred);
2056                 urb->status = -EREMOTEIO;
2057                 break;
2058         case CVMX_USB_COMPLETE_ERROR:
2059         case CVMX_USB_COMPLETE_XACTERR:
2060         case CVMX_USB_COMPLETE_DATATGLERR:
2061         case CVMX_USB_COMPLETE_FRAMEERR:
2062                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2063                         status, pipe, transaction, bytes_transferred);
2064                 urb->status = -EPROTO;
2065                 break;
2066         }
2067         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(priv), urb);
2068         spin_unlock(&priv->lock);
2069         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2070         spin_lock(&priv->lock);
2071 }
2072
2073 /**
2074  * Signal the completion of a transaction and free it. The
2075  * transaction will be removed from the pipe transaction list.
2076  *
2077  * @usb:         USB device state populated by cvmx_usb_initialize().
2078  * @pipe:        Pipe the transaction is on
2079  * @transaction:
2080  *               Transaction that completed
2081  * @complete_code:
2082  *               Completion code
2083  */
2084 static void cvmx_usb_perform_complete(struct cvmx_usb_state *usb,
2085                                       struct cvmx_usb_pipe *pipe,
2086                                       struct cvmx_usb_transaction *transaction,
2087                                       enum cvmx_usb_complete complete_code)
2088 {
2089         /* If this was a split then clear our split in progress marker */
2090         if (usb->active_split == transaction)
2091                 usb->active_split = NULL;
2092
2093         /*
2094          * Isochronous transactions need extra processing as they might not be
2095          * done after a single data transfer
2096          */
2097         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2098                 /* Update the number of bytes transferred in this ISO packet */
2099                 transaction->iso_packets[0].length = transaction->actual_bytes;
2100                 transaction->iso_packets[0].status = complete_code;
2101
2102                 /*
2103                  * If there are more ISOs pending and we succeeded, schedule the
2104                  * next one
2105                  */
2106                 if ((transaction->iso_number_packets > 1) &&
2107                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2108                         /* No bytes transferred for this packet as of yet */
2109                         transaction->actual_bytes = 0;
2110                         /* One less ISO waiting to transfer */
2111                         transaction->iso_number_packets--;
2112                         /* Increment to the next location in our packet array */
2113                         transaction->iso_packets++;
2114                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2115                         return;
2116                 }
2117         }
2118
2119         /* Remove the transaction from the pipe list */
2120         list_del(&transaction->node);
2121         if (list_empty(&pipe->transactions))
2122                 list_move_tail(&pipe->node, &usb->idle_pipes);
2123         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2124                                          transaction,
2125                                          transaction->actual_bytes,
2126                                          transaction->urb);
2127         kfree(transaction);
2128 }
2129
2130
2131 /**
2132  * Submit a usb transaction to a pipe. Called for all types
2133  * of transactions.
2134  *
2135  * @usb:
2136  * @pipe:           Which pipe to submit to.
2137  * @type:           Transaction type
2138  * @buffer:         User buffer for the transaction
2139  * @buffer_length:
2140  *                  User buffer's length in bytes
2141  * @control_header:
2142  *                  For control transactions, the 8 byte standard header
2143  * @iso_start_frame:
2144  *                  For ISO transactions, the start frame
2145  * @iso_number_packets:
2146  *                  For ISO, the number of packet in the transaction.
2147  * @iso_packets:
2148  *                  A description of each ISO packet
2149  * @urb:            URB for the callback
2150  *
2151  * Returns: Transaction or NULL on failure.
2152  */
2153 static struct cvmx_usb_transaction *cvmx_usb_submit_transaction(
2154                                 struct cvmx_usb_state *usb,
2155                                 struct cvmx_usb_pipe *pipe,
2156                                 enum cvmx_usb_transfer type,
2157                                 uint64_t buffer,
2158                                 int buffer_length,
2159                                 uint64_t control_header,
2160                                 int iso_start_frame,
2161                                 int iso_number_packets,
2162                                 struct cvmx_usb_iso_packet *iso_packets,
2163                                 struct urb *urb)
2164 {
2165         struct cvmx_usb_transaction *transaction;
2166
2167         if (unlikely(pipe->transfer_type != type))
2168                 return NULL;
2169
2170         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2171         if (unlikely(!transaction))
2172                 return NULL;
2173
2174         transaction->type = type;
2175         transaction->buffer = buffer;
2176         transaction->buffer_length = buffer_length;
2177         transaction->control_header = control_header;
2178         /* FIXME: This is not used, implement it. */
2179         transaction->iso_start_frame = iso_start_frame;
2180         transaction->iso_number_packets = iso_number_packets;
2181         transaction->iso_packets = iso_packets;
2182         transaction->urb = urb;
2183         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2184                 transaction->stage = CVMX_USB_STAGE_SETUP;
2185         else
2186                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2187
2188         if (!list_empty(&pipe->transactions)) {
2189                 list_add_tail(&transaction->node, &pipe->transactions);
2190         } else {
2191                 list_add_tail(&transaction->node, &pipe->transactions);
2192                 list_move_tail(&pipe->node,
2193                                &usb->active_pipes[pipe->transfer_type]);
2194
2195                 /*
2196                  * We may need to schedule the pipe if this was the head of the
2197                  * pipe.
2198                  */
2199                 cvmx_usb_schedule(usb, 0);
2200         }
2201
2202         return transaction;
2203 }
2204
2205
2206 /**
2207  * Call to submit a USB Bulk transfer to a pipe.
2208  *
2209  * @usb:            USB device state populated by cvmx_usb_initialize().
2210  * @pipe:           Handle to the pipe for the transfer.
2211  * @urb:            URB.
2212  *
2213  * Returns: A submitted transaction or NULL on failure.
2214  */
2215 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2216                                                 struct cvmx_usb_state *usb,
2217                                                 struct cvmx_usb_pipe *pipe,
2218                                                 struct urb *urb)
2219 {
2220         return cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2221                                            urb->transfer_dma,
2222                                            urb->transfer_buffer_length,
2223                                            0, /* control_header */
2224                                            0, /* iso_start_frame */
2225                                            0, /* iso_number_packets */
2226                                            NULL, /* iso_packets */
2227                                            urb);
2228 }
2229
2230
2231 /**
2232  * Call to submit a USB Interrupt transfer to a pipe.
2233  *
2234  * @usb:            USB device state populated by cvmx_usb_initialize().
2235  * @pipe:           Handle to the pipe for the transfer.
2236  * @urb:            URB returned when the callback is called.
2237  *
2238  * Returns: A submitted transaction or NULL on failure.
2239  */
2240 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2241                                                 struct cvmx_usb_state *usb,
2242                                                 struct cvmx_usb_pipe *pipe,
2243                                                 struct urb *urb)
2244 {
2245         return cvmx_usb_submit_transaction(usb, pipe,
2246                                            CVMX_USB_TRANSFER_INTERRUPT,
2247                                            urb->transfer_dma,
2248                                            urb->transfer_buffer_length,
2249                                            0, /* control_header */
2250                                            0, /* iso_start_frame */
2251                                            0, /* iso_number_packets */
2252                                            NULL, /* iso_packets */
2253                                            urb);
2254 }
2255
2256
2257 /**
2258  * Call to submit a USB Control transfer to a pipe.
2259  *
2260  * @usb:            USB device state populated by cvmx_usb_initialize().
2261  * @pipe:           Handle to the pipe for the transfer.
2262  * @urb:            URB.
2263  *
2264  * Returns: A submitted transaction or NULL on failure.
2265  */
2266 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2267                                                 struct cvmx_usb_state *usb,
2268                                                 struct cvmx_usb_pipe *pipe,
2269                                                 struct urb *urb)
2270 {
2271         int buffer_length = urb->transfer_buffer_length;
2272         uint64_t control_header = urb->setup_dma;
2273         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2274
2275         if ((header->bRequestType & USB_DIR_IN) == 0)
2276                 buffer_length = le16_to_cpu(header->wLength);
2277
2278         return cvmx_usb_submit_transaction(usb, pipe,
2279                                            CVMX_USB_TRANSFER_CONTROL,
2280                                            urb->transfer_dma, buffer_length,
2281                                            control_header,
2282                                            0, /* iso_start_frame */
2283                                            0, /* iso_number_packets */
2284                                            NULL, /* iso_packets */
2285                                            urb);
2286 }
2287
2288
2289 /**
2290  * Call to submit a USB Isochronous transfer to a pipe.
2291  *
2292  * @usb:            USB device state populated by cvmx_usb_initialize().
2293  * @pipe:           Handle to the pipe for the transfer.
2294  * @urb:            URB returned when the callback is called.
2295  *
2296  * Returns: A submitted transaction or NULL on failure.
2297  */
2298 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2299                                                 struct cvmx_usb_state *usb,
2300                                                 struct cvmx_usb_pipe *pipe,
2301                                                 struct urb *urb)
2302 {
2303         struct cvmx_usb_iso_packet *packets;
2304
2305         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2306         return cvmx_usb_submit_transaction(usb, pipe,
2307                                            CVMX_USB_TRANSFER_ISOCHRONOUS,
2308                                            urb->transfer_dma,
2309                                            urb->transfer_buffer_length,
2310                                            0, /* control_header */
2311                                            urb->start_frame,
2312                                            urb->number_of_packets,
2313                                            packets, urb);
2314 }
2315
2316
2317 /**
2318  * Cancel one outstanding request in a pipe. Canceling a request
2319  * can fail if the transaction has already completed before cancel
2320  * is called. Even after a successful cancel call, it may take
2321  * a frame or two for the cvmx_usb_poll() function to call the
2322  * associated callback.
2323  *
2324  * @usb:         USB device state populated by cvmx_usb_initialize().
2325  * @pipe:        Pipe to cancel requests in.
2326  * @transaction: Transaction to cancel, returned by the submit function.
2327  *
2328  * Returns: 0 or a negative error code.
2329  */
2330 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2331                            struct cvmx_usb_pipe *pipe,
2332                            struct cvmx_usb_transaction *transaction)
2333 {
2334         /*
2335          * If the transaction is the HEAD of the queue and scheduled. We need to
2336          * treat it special
2337          */
2338         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2339             transaction && (pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2340                 union cvmx_usbcx_hccharx usbc_hcchar;
2341
2342                 usb->pipe_for_channel[pipe->channel] = NULL;
2343                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2344
2345                 CVMX_SYNCW;
2346
2347                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2348                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2349                 /*
2350                  * If the channel isn't enabled then the transaction already
2351                  * completed.
2352                  */
2353                 if (usbc_hcchar.s.chena) {
2354                         usbc_hcchar.s.chdis = 1;
2355                         cvmx_usb_write_csr32(usb,
2356                                         CVMX_USBCX_HCCHARX(pipe->channel,
2357                                                 usb->index),
2358                                         usbc_hcchar.u32);
2359                 }
2360         }
2361         cvmx_usb_perform_complete(usb, pipe, transaction,
2362                                   CVMX_USB_COMPLETE_CANCEL);
2363         return 0;
2364 }
2365
2366
2367 /**
2368  * Cancel all outstanding requests in a pipe. Logically all this
2369  * does is call cvmx_usb_cancel() in a loop.
2370  *
2371  * @usb:         USB device state populated by cvmx_usb_initialize().
2372  * @pipe:        Pipe to cancel requests in.
2373  *
2374  * Returns: 0 or a negative error code.
2375  */
2376 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2377                                struct cvmx_usb_pipe *pipe)
2378 {
2379         struct cvmx_usb_transaction *transaction, *next;
2380
2381         /* Simply loop through and attempt to cancel each transaction */
2382         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2383                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2384
2385                 if (unlikely(result != 0))
2386                         return result;
2387         }
2388         return 0;
2389 }
2390
2391
2392 /**
2393  * Close a pipe created with cvmx_usb_open_pipe().
2394  *
2395  * @usb:         USB device state populated by cvmx_usb_initialize().
2396  * @pipe:        Pipe to close.
2397  *
2398  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2399  *          outstanding transfers.
2400  */
2401 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2402                                struct cvmx_usb_pipe *pipe)
2403 {
2404         /* Fail if the pipe has pending transactions */
2405         if (!list_empty(&pipe->transactions))
2406                 return -EBUSY;
2407
2408         list_del(&pipe->node);
2409         kfree(pipe);
2410
2411         return 0;
2412 }
2413
2414 /**
2415  * Get the current USB protocol level frame number. The frame
2416  * number is always in the range of 0-0x7ff.
2417  *
2418  * @usb: USB device state populated by cvmx_usb_initialize().
2419  *
2420  * Returns: USB frame number
2421  */
2422 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2423 {
2424         int frame_number;
2425         union cvmx_usbcx_hfnum usbc_hfnum;
2426
2427         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2428         frame_number = usbc_hfnum.s.frnum;
2429
2430         return frame_number;
2431 }
2432
2433
2434 /**
2435  * Poll a channel for status
2436  *
2437  * @usb:     USB device
2438  * @channel: Channel to poll
2439  *
2440  * Returns: Zero on success
2441  */
2442 static int cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2443 {
2444         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2445         struct usb_hcd *hcd = octeon_to_hcd(priv);
2446         struct device *dev = hcd->self.controller;
2447         union cvmx_usbcx_hcintx usbc_hcint;
2448         union cvmx_usbcx_hctsizx usbc_hctsiz;
2449         union cvmx_usbcx_hccharx usbc_hcchar;
2450         struct cvmx_usb_pipe *pipe;
2451         struct cvmx_usb_transaction *transaction;
2452         int bytes_this_transfer;
2453         int bytes_in_last_packet;
2454         int packets_processed;
2455         int buffer_space_left;
2456
2457         /* Read the interrupt status bits for the channel */
2458         usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
2459                                 CVMX_USBCX_HCINTX(channel, usb->index));
2460
2461         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2462                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2463                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2464
2465                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2466                         /*
2467                          * There seems to be a bug in CN31XX which can cause
2468                          * interrupt IN transfers to get stuck until we do a
2469                          * write of HCCHARX without changing things
2470                          */
2471                         cvmx_usb_write_csr32(usb,
2472                                         CVMX_USBCX_HCCHARX(channel,
2473                                                            usb->index),
2474                                         usbc_hcchar.u32);
2475                         return 0;
2476                 }
2477
2478                 /*
2479                  * In non DMA mode the channels don't halt themselves. We need
2480                  * to manually disable channels that are left running
2481                  */
2482                 if (!usbc_hcint.s.chhltd) {
2483                         if (usbc_hcchar.s.chena) {
2484                                 union cvmx_usbcx_hcintmskx hcintmsk;
2485                                 /* Disable all interrupts except CHHLTD */
2486                                 hcintmsk.u32 = 0;
2487                                 hcintmsk.s.chhltdmsk = 1;
2488                                 cvmx_usb_write_csr32(usb,
2489                                                 CVMX_USBCX_HCINTMSKX(channel,
2490                                                         usb->index),
2491                                                 hcintmsk.u32);
2492                                 usbc_hcchar.s.chdis = 1;
2493                                 cvmx_usb_write_csr32(usb,
2494                                                 CVMX_USBCX_HCCHARX(channel,
2495                                                         usb->index),
2496                                                 usbc_hcchar.u32);
2497                                 return 0;
2498                         } else if (usbc_hcint.s.xfercompl) {
2499                                 /*
2500                                  * Successful IN/OUT with transfer complete.
2501                                  * Channel halt isn't needed.
2502                                  */
2503                         } else {
2504                                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
2505                                         usb->index, channel);
2506                                 return 0;
2507                         }
2508                 }
2509         } else {
2510                 /*
2511                  * There is are no interrupts that we need to process when the
2512                  * channel is still running
2513                  */
2514                 if (!usbc_hcint.s.chhltd)
2515                         return 0;
2516         }
2517
2518         /* Disable the channel interrupts now that it is done */
2519         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2520         usb->idle_hardware_channels |= (1<<channel);
2521
2522         /* Make sure this channel is tied to a valid pipe */
2523         pipe = usb->pipe_for_channel[channel];
2524         prefetch(pipe);
2525         if (!pipe)
2526                 return 0;
2527         transaction = list_first_entry(&pipe->transactions,
2528                                        typeof(*transaction),
2529                                        node);
2530         prefetch(transaction);
2531
2532         /*
2533          * Disconnect this pipe from the HW channel. Later the schedule
2534          * function will figure out which pipe needs to go
2535          */
2536         usb->pipe_for_channel[channel] = NULL;
2537         pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2538
2539         /*
2540          * Read the channel config info so we can figure out how much data
2541          * transferred
2542          */
2543         usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2544                         CVMX_USBCX_HCCHARX(channel, usb->index));
2545         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
2546                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2547
2548         /*
2549          * Calculating the number of bytes successfully transferred is dependent
2550          * on the transfer direction
2551          */
2552         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2553         if (usbc_hcchar.s.epdir) {
2554                 /*
2555                  * IN transactions are easy. For every byte received the
2556                  * hardware decrements xfersize. All we need to do is subtract
2557                  * the current value of xfersize from its starting value and we
2558                  * know how many bytes were written to the buffer
2559                  */
2560                 bytes_this_transfer = transaction->xfersize -
2561                         usbc_hctsiz.s.xfersize;
2562         } else {
2563                 /*
2564                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2565                  * decremented on every successful packet send. The hardware
2566                  * does this when it receives an ACK, or NYET. If it doesn't
2567                  * receive one of these responses pktcnt doesn't change
2568                  */
2569                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2570                 /*
2571                  * The last packet may not be a full transfer if we didn't have
2572                  * enough data
2573                  */
2574                 if (bytes_this_transfer > transaction->xfersize)
2575                         bytes_this_transfer = transaction->xfersize;
2576         }
2577         /* Figure out how many bytes were in the last packet of the transfer */
2578         if (packets_processed)
2579                 bytes_in_last_packet = bytes_this_transfer -
2580                         (packets_processed - 1) * usbc_hcchar.s.mps;
2581         else
2582                 bytes_in_last_packet = bytes_this_transfer;
2583
2584         /*
2585          * As a special case, setup transactions output the setup header, not
2586          * the user's data. For this reason we don't count setup data as bytes
2587          * transferred
2588          */
2589         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2590                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2591                 bytes_this_transfer = 0;
2592
2593         /*
2594          * Add the bytes transferred to the running total. It is important that
2595          * bytes_this_transfer doesn't count any data that needs to be
2596          * retransmitted
2597          */
2598         transaction->actual_bytes += bytes_this_transfer;
2599         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2600                 buffer_space_left = transaction->iso_packets[0].length -
2601                         transaction->actual_bytes;
2602         else
2603                 buffer_space_left = transaction->buffer_length -
2604                         transaction->actual_bytes;
2605
2606         /*
2607          * We need to remember the PID toggle state for the next transaction.
2608          * The hardware already updated it for the next transaction
2609          */
2610         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2611
2612         /*
2613          * For high speed bulk out, assume the next transaction will need to do
2614          * a ping before proceeding. If this isn't true the ACK processing below
2615          * will clear this flag
2616          */
2617         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2618                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2619                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2620                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2621
2622         if (unlikely(WARN_ON_ONCE(bytes_this_transfer < 0))) {
2623                 /*
2624                  * In some rare cases the DMA engine seems to get stuck and
2625                  * keeps substracting same byte count over and over again. In
2626                  * such case we just need to fail every transaction.
2627                  */
2628                 cvmx_usb_perform_complete(usb, pipe, transaction,
2629                                           CVMX_USB_COMPLETE_ERROR);
2630                 return 0;
2631         }
2632
2633         if (usbc_hcint.s.stall) {
2634                 /*
2635                  * STALL as a response means this transaction cannot be
2636                  * completed because the device can't process transactions. Tell
2637                  * the user. Any data that was transferred will be counted on
2638                  * the actual bytes transferred
2639                  */
2640                 pipe->pid_toggle = 0;
2641                 cvmx_usb_perform_complete(usb, pipe, transaction,
2642                                           CVMX_USB_COMPLETE_STALL);
2643         } else if (usbc_hcint.s.xacterr) {
2644                 /*
2645                  * XactErr as a response means the device signaled
2646                  * something wrong with the transfer. For example, PID
2647                  * toggle errors cause these.
2648                  */
2649                 cvmx_usb_perform_complete(usb, pipe, transaction,
2650                                           CVMX_USB_COMPLETE_XACTERR);
2651         } else if (usbc_hcint.s.bblerr) {
2652                 /* Babble Error (BblErr) */
2653                 cvmx_usb_perform_complete(usb, pipe, transaction,
2654                                           CVMX_USB_COMPLETE_BABBLEERR);
2655         } else if (usbc_hcint.s.datatglerr) {
2656                 /* Data toggle error */
2657                 cvmx_usb_perform_complete(usb, pipe, transaction,
2658                                           CVMX_USB_COMPLETE_DATATGLERR);
2659         } else if (usbc_hcint.s.nyet) {
2660                 /*
2661                  * NYET as a response is only allowed in three cases: as a
2662                  * response to a ping, as a response to a split transaction, and
2663                  * as a response to a bulk out. The ping case is handled by
2664                  * hardware, so we only have splits and bulk out
2665                  */
2666                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
2667                         transaction->retries = 0;
2668                         /*
2669                          * If there is more data to go then we need to try
2670                          * again. Otherwise this transaction is complete
2671                          */
2672                         if ((buffer_space_left == 0) ||
2673                                 (bytes_in_last_packet < pipe->max_packet))
2674                                 cvmx_usb_perform_complete(usb, pipe,
2675                                                 transaction,
2676                                                 CVMX_USB_COMPLETE_SUCCESS);
2677                 } else {
2678                         /*
2679                          * Split transactions retry the split complete 4 times
2680                          * then rewind to the start split and do the entire
2681                          * transactions again
2682                          */
2683                         transaction->retries++;
2684                         if ((transaction->retries & 0x3) == 0) {
2685                                 /*
2686                                  * Rewind to the beginning of the transaction by
2687                                  * anding off the split complete bit
2688                                  */
2689                                 transaction->stage &= ~1;
2690                                 pipe->split_sc_frame = -1;
2691                         }
2692                 }
2693         } else if (usbc_hcint.s.ack) {
2694                 transaction->retries = 0;
2695                 /*
2696                  * The ACK bit can only be checked after the other error bits.
2697                  * This is because a multi packet transfer may succeed in a
2698                  * number of packets and then get a different response on the
2699                  * last packet. In this case both ACK and the last response bit
2700                  * will be set. If none of the other response bits is set, then
2701                  * the last packet must have been an ACK
2702                  *
2703                  * Since we got an ACK, we know we don't need to do a ping on
2704                  * this pipe
2705                  */
2706                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_NEED_PING;
2707
2708                 switch (transaction->type) {
2709                 case CVMX_USB_TRANSFER_CONTROL:
2710                         switch (transaction->stage) {
2711                         case CVMX_USB_STAGE_NON_CONTROL:
2712                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2713                                 /* This should be impossible */
2714                                 cvmx_usb_perform_complete(usb, pipe,
2715                                         transaction, CVMX_USB_COMPLETE_ERROR);
2716                                 break;
2717                         case CVMX_USB_STAGE_SETUP:
2718                                 pipe->pid_toggle = 1;
2719                                 if (cvmx_usb_pipe_needs_split(usb, pipe))
2720                                         transaction->stage =
2721                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2722                                 else {
2723                                         struct usb_ctrlrequest *header =
2724                                                 cvmx_phys_to_ptr(transaction->control_header);
2725                                         if (header->wLength)
2726                                                 transaction->stage =
2727                                                         CVMX_USB_STAGE_DATA;
2728                                         else
2729                                                 transaction->stage =
2730                                                         CVMX_USB_STAGE_STATUS;
2731                                 }
2732                                 break;
2733                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2734                                 {
2735                                         struct usb_ctrlrequest *header =
2736                                                 cvmx_phys_to_ptr(transaction->control_header);
2737                                         if (header->wLength)
2738                                                 transaction->stage =
2739                                                         CVMX_USB_STAGE_DATA;
2740                                         else
2741                                                 transaction->stage =
2742                                                         CVMX_USB_STAGE_STATUS;
2743                                 }
2744                                 break;
2745                         case CVMX_USB_STAGE_DATA:
2746                                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2747                                         transaction->stage =
2748                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2749                                         /*
2750                                          * For setup OUT data that are splits,
2751                                          * the hardware doesn't appear to count
2752                                          * transferred data. Here we manually
2753                                          * update the data transferred
2754                                          */
2755                                         if (!usbc_hcchar.s.epdir) {
2756                                                 if (buffer_space_left < pipe->max_packet)
2757                                                         transaction->actual_bytes +=
2758                                                                 buffer_space_left;
2759                                                 else
2760                                                         transaction->actual_bytes +=
2761                                                                 pipe->max_packet;
2762                                         }
2763                                 } else if ((buffer_space_left == 0) ||
2764                                                 (bytes_in_last_packet <
2765                                                  pipe->max_packet)) {
2766                                         pipe->pid_toggle = 1;
2767                                         transaction->stage =
2768                                                 CVMX_USB_STAGE_STATUS;
2769                                 }
2770                                 break;
2771                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2772                                 if ((buffer_space_left == 0) ||
2773                                                 (bytes_in_last_packet <
2774                                                  pipe->max_packet)) {
2775                                         pipe->pid_toggle = 1;
2776                                         transaction->stage =
2777                                                 CVMX_USB_STAGE_STATUS;
2778                                 } else {
2779                                         transaction->stage =
2780                                                 CVMX_USB_STAGE_DATA;
2781                                 }
2782                                 break;
2783                         case CVMX_USB_STAGE_STATUS:
2784                                 if (cvmx_usb_pipe_needs_split(usb, pipe))
2785                                         transaction->stage =
2786                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2787                                 else
2788                                         cvmx_usb_perform_complete(usb, pipe,
2789                                                 transaction,
2790                                                 CVMX_USB_COMPLETE_SUCCESS);
2791                                 break;
2792                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2793                                 cvmx_usb_perform_complete(usb, pipe,
2794                                                 transaction,
2795                                                 CVMX_USB_COMPLETE_SUCCESS);
2796                                 break;
2797                         }
2798                         break;
2799                 case CVMX_USB_TRANSFER_BULK:
2800                 case CVMX_USB_TRANSFER_INTERRUPT:
2801                         /*
2802                          * The only time a bulk transfer isn't complete when it
2803                          * finishes with an ACK is during a split transaction.
2804                          * For splits we need to continue the transfer if more
2805                          * data is needed
2806                          */
2807                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2808                                 if (transaction->stage ==
2809                                                 CVMX_USB_STAGE_NON_CONTROL)
2810                                         transaction->stage =
2811                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2812                                 else {
2813                                         if (buffer_space_left &&
2814                                                 (bytes_in_last_packet ==
2815                                                  pipe->max_packet))
2816                                                 transaction->stage =
2817                                                         CVMX_USB_STAGE_NON_CONTROL;
2818                                         else {
2819                                                 if (transaction->type ==
2820                                                         CVMX_USB_TRANSFER_INTERRUPT)
2821                                                         pipe->next_tx_frame +=
2822                                                                 pipe->interval;
2823                                                         cvmx_usb_perform_complete(
2824                                                                 usb,
2825                                                                 pipe,
2826                                                                 transaction,
2827                                                                 CVMX_USB_COMPLETE_SUCCESS);
2828                                         }
2829                                 }
2830                         } else {
2831                                 if ((pipe->device_speed ==
2832                                         CVMX_USB_SPEED_HIGH) &&
2833                                     (pipe->transfer_type ==
2834                                      CVMX_USB_TRANSFER_BULK) &&
2835                                     (pipe->transfer_dir ==
2836                                      CVMX_USB_DIRECTION_OUT) &&
2837                                     (usbc_hcint.s.nak))
2838                                         pipe->flags |=
2839                                                 CVMX_USB_PIPE_FLAGS_NEED_PING;
2840                                 if (!buffer_space_left ||
2841                                         (bytes_in_last_packet <
2842                                          pipe->max_packet)) {
2843                                         if (transaction->type ==
2844                                                 CVMX_USB_TRANSFER_INTERRUPT)
2845                                                 pipe->next_tx_frame +=
2846                                                         pipe->interval;
2847                                         cvmx_usb_perform_complete(usb, pipe,
2848                                                 transaction,
2849                                                 CVMX_USB_COMPLETE_SUCCESS);
2850                                 }
2851                         }
2852                         break;
2853                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2854                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2855                                 /*
2856                                  * ISOCHRONOUS OUT splits don't require a
2857                                  * complete split stage. Instead they use a
2858                                  * sequence of begin OUT splits to transfer the
2859                                  * data 188 bytes at a time. Once the transfer
2860                                  * is complete, the pipe sleeps until the next
2861                                  * schedule interval
2862                                  */
2863                                 if (pipe->transfer_dir ==
2864                                         CVMX_USB_DIRECTION_OUT) {
2865                                         /*
2866                                          * If no space left or this wasn't a max
2867                                          * size packet then this transfer is
2868                                          * complete. Otherwise start it again to
2869                                          * send the next 188 bytes
2870                                          */
2871                                         if (!buffer_space_left ||
2872                                                 (bytes_this_transfer < 188)) {
2873                                                 pipe->next_tx_frame +=
2874                                                         pipe->interval;
2875                                                 cvmx_usb_perform_complete(usb,
2876                                                         pipe, transaction,
2877                                                         CVMX_USB_COMPLETE_SUCCESS);
2878                                         }
2879                                 } else {
2880                                         if (transaction->stage ==
2881                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2882                                                 /*
2883                                                  * We are in the incoming data
2884                                                  * phase. Keep getting data
2885                                                  * until we run out of space or
2886                                                  * get a small packet
2887                                                  */
2888                                                 if ((buffer_space_left == 0) ||
2889                                                         (bytes_in_last_packet <
2890                                                          pipe->max_packet)) {
2891                                                         pipe->next_tx_frame +=
2892                                                                 pipe->interval;
2893                                                         cvmx_usb_perform_complete(
2894                                                                 usb,
2895                                                                 pipe,
2896                                                                 transaction,
2897                                                                 CVMX_USB_COMPLETE_SUCCESS);
2898                                                 }
2899                                         } else
2900                                                 transaction->stage =
2901                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2902                                 }
2903                         } else {
2904                                 pipe->next_tx_frame += pipe->interval;
2905                                 cvmx_usb_perform_complete(usb, pipe,
2906                                                 transaction,
2907                                                 CVMX_USB_COMPLETE_SUCCESS);
2908                         }
2909                         break;
2910                 }
2911         } else if (usbc_hcint.s.nak) {
2912                 /*
2913                  * If this was a split then clear our split in progress marker.
2914                  */
2915                 if (usb->active_split == transaction)
2916                         usb->active_split = NULL;
2917                 /*
2918                  * NAK as a response means the device couldn't accept the
2919                  * transaction, but it should be retried in the future. Rewind
2920                  * to the beginning of the transaction by anding off the split
2921                  * complete bit. Retry in the next interval
2922                  */
2923                 transaction->retries = 0;
2924                 transaction->stage &= ~1;
2925                 pipe->next_tx_frame += pipe->interval;
2926                 if (pipe->next_tx_frame < usb->frame_number)
2927                         pipe->next_tx_frame = usb->frame_number +
2928                                 pipe->interval -
2929                                 (usb->frame_number - pipe->next_tx_frame) %
2930                                 pipe->interval;
2931         } else {
2932                 struct cvmx_usb_port_status port;
2933
2934                 port = cvmx_usb_get_status(usb);
2935                 if (port.port_enabled) {
2936                         /* We'll retry the exact same transaction again */
2937                         transaction->retries++;
2938                 } else {
2939                         /*
2940                          * We get channel halted interrupts with no result bits
2941                          * sets when the cable is unplugged
2942                          */
2943                         cvmx_usb_perform_complete(usb, pipe, transaction,
2944                                         CVMX_USB_COMPLETE_ERROR);
2945                 }
2946         }
2947         return 0;
2948 }
2949
2950 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
2951 {
2952         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2953
2954         spin_unlock(&priv->lock);
2955         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
2956         spin_lock(&priv->lock);
2957 }
2958
2959 /**
2960  * Poll the USB block for status and call all needed callback
2961  * handlers. This function is meant to be called in the interrupt
2962  * handler for the USB controller. It can also be called
2963  * periodically in a loop for non-interrupt based operation.
2964  *
2965  * @usb: USB device state populated by cvmx_usb_initialize().
2966  *
2967  * Returns: 0 or a negative error code.
2968  */
2969 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
2970 {
2971         union cvmx_usbcx_hfnum usbc_hfnum;
2972         union cvmx_usbcx_gintsts usbc_gintsts;
2973
2974         prefetch_range(usb, sizeof(*usb));
2975
2976         /* Update the frame counter */
2977         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2978         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
2979                 usb->frame_number += 0x4000;
2980         usb->frame_number &= ~0x3fffull;
2981         usb->frame_number |= usbc_hfnum.s.frnum;
2982
2983         /* Read the pending interrupts */
2984         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
2985                                                CVMX_USBCX_GINTSTS(usb->index));
2986
2987         /* Clear the interrupts now that we know about them */
2988         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
2989                              usbc_gintsts.u32);
2990
2991         if (usbc_gintsts.s.rxflvl) {
2992                 /*
2993                  * RxFIFO Non-Empty (RxFLvl)
2994                  * Indicates that there is at least one packet pending to be
2995                  * read from the RxFIFO.
2996                  *
2997                  * In DMA mode this is handled by hardware
2998                  */
2999                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3000                         cvmx_usb_poll_rx_fifo(usb);
3001         }
3002         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3003                 /* Fill the Tx FIFOs when not in DMA mode */
3004                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3005                         cvmx_usb_poll_tx_fifo(usb);
3006         }
3007         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3008                 union cvmx_usbcx_hprt usbc_hprt;
3009                 /*
3010                  * Disconnect Detected Interrupt (DisconnInt)
3011                  * Asserted when a device disconnect is detected.
3012                  *
3013                  * Host Port Interrupt (PrtInt)
3014                  * The core sets this bit to indicate a change in port status of
3015                  * one of the O2P USB core ports in Host mode. The application
3016                  * must read the Host Port Control and Status (HPRT) register to
3017                  * determine the exact event that caused this interrupt. The
3018                  * application must clear the appropriate status bit in the Host
3019                  * Port Control and Status register to clear this bit.
3020                  *
3021                  * Call the user's port callback
3022                  */
3023                 octeon_usb_port_callback(usb);
3024                 /* Clear the port change bits */
3025                 usbc_hprt.u32 = cvmx_usb_read_csr32(usb,
3026                                         CVMX_USBCX_HPRT(usb->index));
3027                 usbc_hprt.s.prtena = 0;
3028                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3029                                      usbc_hprt.u32);
3030         }
3031         if (usbc_gintsts.s.hchint) {
3032                 /*
3033                  * Host Channels Interrupt (HChInt)
3034                  * The core sets this bit to indicate that an interrupt is
3035                  * pending on one of the channels of the core (in Host mode).
3036                  * The application must read the Host All Channels Interrupt
3037                  * (HAINT) register to determine the exact number of the channel
3038                  * on which the interrupt occurred, and then read the
3039                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3040                  * determine the exact cause of the interrupt. The application
3041                  * must clear the appropriate status bit in the HCINTn register
3042                  * to clear this bit.
3043                  */
3044                 union cvmx_usbcx_haint usbc_haint;
3045
3046                 usbc_haint.u32 = cvmx_usb_read_csr32(usb,
3047                                         CVMX_USBCX_HAINT(usb->index));
3048                 while (usbc_haint.u32) {
3049                         int channel;
3050
3051                         channel = __fls(usbc_haint.u32);
3052                         cvmx_usb_poll_channel(usb, channel);
3053                         usbc_haint.u32 ^= 1<<channel;
3054                 }
3055         }
3056
3057         cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3058
3059         return 0;
3060 }
3061
3062 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3063 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3064 {
3065         return (struct octeon_hcd *)(hcd->hcd_priv);
3066 }
3067
3068 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3069 {
3070         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3071         unsigned long flags;
3072
3073         spin_lock_irqsave(&priv->lock, flags);
3074         cvmx_usb_poll(&priv->usb);
3075         spin_unlock_irqrestore(&priv->lock, flags);
3076         return IRQ_HANDLED;
3077 }
3078
3079 static int octeon_usb_start(struct usb_hcd *hcd)
3080 {
3081         hcd->state = HC_STATE_RUNNING;
3082         return 0;
3083 }
3084
3085 static void octeon_usb_stop(struct usb_hcd *hcd)
3086 {
3087         hcd->state = HC_STATE_HALT;
3088 }
3089
3090 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3091 {
3092         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3093
3094         return cvmx_usb_get_frame_number(&priv->usb);
3095 }
3096
3097 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3098                                   struct urb *urb,
3099                                   gfp_t mem_flags)
3100 {
3101         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3102         struct device *dev = hcd->self.controller;
3103         struct cvmx_usb_transaction *transaction = NULL;
3104         struct cvmx_usb_pipe *pipe;
3105         unsigned long flags;
3106         struct cvmx_usb_iso_packet *iso_packet;
3107         struct usb_host_endpoint *ep = urb->ep;
3108         int rc;
3109
3110         urb->status = 0;
3111         spin_lock_irqsave(&priv->lock, flags);
3112
3113         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3114         if (rc) {
3115                 spin_unlock_irqrestore(&priv->lock, flags);
3116                 return rc;
3117         }
3118
3119         if (!ep->hcpriv) {
3120                 enum cvmx_usb_transfer transfer_type;
3121                 enum cvmx_usb_speed speed;
3122                 int split_device = 0;
3123                 int split_port = 0;
3124
3125                 switch (usb_pipetype(urb->pipe)) {
3126                 case PIPE_ISOCHRONOUS:
3127                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3128                         break;
3129                 case PIPE_INTERRUPT:
3130                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3131                         break;
3132                 case PIPE_CONTROL:
3133                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3134                         break;
3135                 default:
3136                         transfer_type = CVMX_USB_TRANSFER_BULK;
3137                         break;
3138                 }
3139                 switch (urb->dev->speed) {
3140                 case USB_SPEED_LOW:
3141                         speed = CVMX_USB_SPEED_LOW;
3142                         break;
3143                 case USB_SPEED_FULL:
3144                         speed = CVMX_USB_SPEED_FULL;
3145                         break;
3146                 default:
3147                         speed = CVMX_USB_SPEED_HIGH;
3148                         break;
3149                 }
3150                 /*
3151                  * For slow devices on high speed ports we need to find the hub
3152                  * that does the speed translation so we know where to send the
3153                  * split transactions.
3154                  */
3155                 if (speed != CVMX_USB_SPEED_HIGH) {
3156                         /*
3157                          * Start at this device and work our way up the usb
3158                          * tree.
3159                          */
3160                         struct usb_device *dev = urb->dev;
3161
3162                         while (dev->parent) {
3163                                 /*
3164                                  * If our parent is high speed then he'll
3165                                  * receive the splits.
3166                                  */
3167                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3168                                         split_device = dev->parent->devnum;
3169                                         split_port = dev->portnum;
3170                                         break;
3171                                 }
3172                                 /*
3173                                  * Move up the tree one level. If we make it all
3174                                  * the way up the tree, then the port must not
3175                                  * be in high speed mode and we don't need a
3176                                  * split.
3177                                  */
3178                                 dev = dev->parent;
3179                         }
3180                 }
3181                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3182                                           usb_pipeendpoint(urb->pipe), speed,
3183                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3184                                           & 0x7ff,
3185                                           transfer_type,
3186                                           usb_pipein(urb->pipe) ?
3187                                                 CVMX_USB_DIRECTION_IN :
3188                                                 CVMX_USB_DIRECTION_OUT,
3189                                           urb->interval,
3190                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3191                                            >> 11) & 0x3,
3192                                           split_device, split_port);
3193                 if (!pipe) {
3194                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3195                         spin_unlock_irqrestore(&priv->lock, flags);
3196                         dev_dbg(dev, "Failed to create pipe\n");
3197                         return -ENOMEM;
3198                 }
3199                 ep->hcpriv = pipe;
3200         } else {
3201                 pipe = ep->hcpriv;
3202         }
3203
3204         switch (usb_pipetype(urb->pipe)) {
3205         case PIPE_ISOCHRONOUS:
3206                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3207                         usb_pipedevice(urb->pipe),
3208                         usb_pipeendpoint(urb->pipe));
3209                 /*
3210                  * Allocate a structure to use for our private list of
3211                  * isochronous packets.
3212                  */
3213                 iso_packet = kmalloc(urb->number_of_packets *
3214                                      sizeof(struct cvmx_usb_iso_packet),
3215                                      GFP_ATOMIC);
3216                 if (iso_packet) {
3217                         int i;
3218                         /* Fill the list with the data from the URB */
3219                         for (i = 0; i < urb->number_of_packets; i++) {
3220                                 iso_packet[i].offset =
3221                                         urb->iso_frame_desc[i].offset;
3222                                 iso_packet[i].length =
3223                                         urb->iso_frame_desc[i].length;
3224                                 iso_packet[i].status =
3225                                         CVMX_USB_COMPLETE_ERROR;
3226                         }
3227                         /*
3228                          * Store a pointer to the list in the URB setup_packet
3229                          * field. We know this currently isn't being used and
3230                          * this saves us a bunch of logic.
3231                          */
3232                         urb->setup_packet = (char *)iso_packet;
3233                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3234                                                                   pipe, urb);
3235                         /*
3236                          * If submit failed we need to free our private packet
3237                          * list.
3238                          */
3239                         if (!transaction) {
3240                                 urb->setup_packet = NULL;
3241                                 kfree(iso_packet);
3242                         }
3243                 }
3244                 break;
3245         case PIPE_INTERRUPT:
3246                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3247                         usb_pipedevice(urb->pipe),
3248                         usb_pipeendpoint(urb->pipe));
3249                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3250                 break;
3251         case PIPE_CONTROL:
3252                 dev_dbg(dev, "Submit control to %d.%d\n",
3253                         usb_pipedevice(urb->pipe),
3254                         usb_pipeendpoint(urb->pipe));
3255                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3256                 break;
3257         case PIPE_BULK:
3258                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3259                         usb_pipedevice(urb->pipe),
3260                         usb_pipeendpoint(urb->pipe));
3261                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3262                 break;
3263         }
3264         if (!transaction) {
3265                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3266                 spin_unlock_irqrestore(&priv->lock, flags);
3267                 dev_dbg(dev, "Failed to submit\n");
3268                 return -ENOMEM;
3269         }
3270         urb->hcpriv = transaction;
3271         spin_unlock_irqrestore(&priv->lock, flags);
3272         return 0;
3273 }
3274
3275 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3276                                   struct urb *urb,
3277                                   int status)
3278 {
3279         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3280         unsigned long flags;
3281         int rc;
3282
3283         if (!urb->dev)
3284                 return -EINVAL;
3285
3286         spin_lock_irqsave(&priv->lock, flags);
3287
3288         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3289         if (rc)
3290                 goto out;
3291
3292         urb->status = status;
3293         cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3294
3295 out:
3296         spin_unlock_irqrestore(&priv->lock, flags);
3297
3298         return rc;
3299 }
3300
3301 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3302                                         struct usb_host_endpoint *ep)
3303 {
3304         struct device *dev = hcd->self.controller;
3305
3306         if (ep->hcpriv) {
3307                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3308                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3309                 unsigned long flags;
3310
3311                 spin_lock_irqsave(&priv->lock, flags);
3312                 cvmx_usb_cancel_all(&priv->usb, pipe);
3313                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3314                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3315                 spin_unlock_irqrestore(&priv->lock, flags);
3316                 ep->hcpriv = NULL;
3317         }
3318 }
3319
3320 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3321 {
3322         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3323         struct cvmx_usb_port_status port_status;
3324         unsigned long flags;
3325
3326         spin_lock_irqsave(&priv->lock, flags);
3327         port_status = cvmx_usb_get_status(&priv->usb);
3328         spin_unlock_irqrestore(&priv->lock, flags);
3329         buf[0] = 0;
3330         buf[0] = port_status.connect_change << 1;
3331
3332         return buf[0] != 0;
3333 }
3334
3335 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3336                                 u16 wIndex, char *buf, u16 wLength)
3337 {
3338         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3339         struct device *dev = hcd->self.controller;
3340         struct cvmx_usb_port_status usb_port_status;
3341         struct cvmx_usb_state *usb = &priv->usb;
3342         int port_status;
3343         struct usb_hub_descriptor *desc;
3344         unsigned long flags;
3345
3346         switch (typeReq) {
3347         case ClearHubFeature:
3348                 dev_dbg(dev, "ClearHubFeature\n");
3349                 switch (wValue) {
3350                 case C_HUB_LOCAL_POWER:
3351                 case C_HUB_OVER_CURRENT:
3352                         /* Nothing required here */
3353                         break;
3354                 default:
3355                         return -EINVAL;
3356                 }
3357                 break;
3358         case ClearPortFeature:
3359                 dev_dbg(dev, "ClearPortFeature\n");
3360                 if (wIndex != 1) {
3361                         dev_dbg(dev, " INVALID\n");
3362                         return -EINVAL;
3363                 }
3364
3365                 switch (wValue) {
3366                 case USB_PORT_FEAT_ENABLE:
3367                         dev_dbg(dev, " ENABLE\n");
3368                         spin_lock_irqsave(&priv->lock, flags);
3369                         cvmx_usb_disable(&priv->usb);
3370                         spin_unlock_irqrestore(&priv->lock, flags);
3371                         break;
3372                 case USB_PORT_FEAT_SUSPEND:
3373                         dev_dbg(dev, " SUSPEND\n");
3374                         /* Not supported on Octeon */
3375                         break;
3376                 case USB_PORT_FEAT_POWER:
3377                         dev_dbg(dev, " POWER\n");
3378                         /* Not supported on Octeon */
3379                         break;
3380                 case USB_PORT_FEAT_INDICATOR:
3381                         dev_dbg(dev, " INDICATOR\n");
3382                         /* Port inidicator not supported */
3383                         break;
3384                 case USB_PORT_FEAT_C_CONNECTION:
3385                         dev_dbg(dev, " C_CONNECTION\n");
3386                         /* Clears drivers internal connect status change flag */
3387                         spin_lock_irqsave(&priv->lock, flags);
3388                         priv->usb.port_status =
3389                                 cvmx_usb_get_status(&priv->usb);
3390                         spin_unlock_irqrestore(&priv->lock, flags);
3391                         break;
3392                 case USB_PORT_FEAT_C_RESET:
3393                         dev_dbg(dev, " C_RESET\n");
3394                         /*
3395                          * Clears the driver's internal Port Reset Change flag.
3396                          */
3397                         spin_lock_irqsave(&priv->lock, flags);
3398                         priv->usb.port_status =
3399                                 cvmx_usb_get_status(&priv->usb);
3400                         spin_unlock_irqrestore(&priv->lock, flags);
3401                         break;
3402                 case USB_PORT_FEAT_C_ENABLE:
3403                         dev_dbg(dev, " C_ENABLE\n");
3404                         /*
3405                          * Clears the driver's internal Port Enable/Disable
3406                          * Change flag.
3407                          */
3408                         spin_lock_irqsave(&priv->lock, flags);
3409                         priv->usb.port_status =
3410                                 cvmx_usb_get_status(&priv->usb);
3411                         spin_unlock_irqrestore(&priv->lock, flags);
3412                         break;
3413                 case USB_PORT_FEAT_C_SUSPEND:
3414                         dev_dbg(dev, " C_SUSPEND\n");
3415                         /*
3416                          * Clears the driver's internal Port Suspend Change
3417                          * flag, which is set when resume signaling on the host
3418                          * port is complete.
3419                          */
3420                         break;
3421                 case USB_PORT_FEAT_C_OVER_CURRENT:
3422                         dev_dbg(dev, " C_OVER_CURRENT\n");
3423                         /* Clears the driver's overcurrent Change flag */
3424                         spin_lock_irqsave(&priv->lock, flags);
3425                         priv->usb.port_status =
3426                                 cvmx_usb_get_status(&priv->usb);
3427                         spin_unlock_irqrestore(&priv->lock, flags);
3428                         break;
3429                 default:
3430                         dev_dbg(dev, " UNKNOWN\n");
3431                         return -EINVAL;
3432                 }
3433                 break;
3434         case GetHubDescriptor:
3435                 dev_dbg(dev, "GetHubDescriptor\n");
3436                 desc = (struct usb_hub_descriptor *)buf;
3437                 desc->bDescLength = 9;
3438                 desc->bDescriptorType = 0x29;
3439                 desc->bNbrPorts = 1;
3440                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3441                 desc->bPwrOn2PwrGood = 1;
3442                 desc->bHubContrCurrent = 0;
3443                 desc->u.hs.DeviceRemovable[0] = 0;
3444                 desc->u.hs.DeviceRemovable[1] = 0xff;
3445                 break;
3446         case GetHubStatus:
3447                 dev_dbg(dev, "GetHubStatus\n");
3448                 *(__le32 *) buf = 0;
3449                 break;
3450         case GetPortStatus:
3451                 dev_dbg(dev, "GetPortStatus\n");
3452                 if (wIndex != 1) {
3453                         dev_dbg(dev, " INVALID\n");
3454                         return -EINVAL;
3455                 }
3456
3457                 spin_lock_irqsave(&priv->lock, flags);
3458                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3459                 spin_unlock_irqrestore(&priv->lock, flags);
3460                 port_status = 0;
3461
3462                 if (usb_port_status.connect_change) {
3463                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3464                         dev_dbg(dev, " C_CONNECTION\n");
3465                 }
3466
3467                 if (usb_port_status.port_enabled) {
3468                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3469                         dev_dbg(dev, " C_ENABLE\n");
3470                 }
3471
3472                 if (usb_port_status.connected) {
3473                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3474                         dev_dbg(dev, " CONNECTION\n");
3475                 }
3476
3477                 if (usb_port_status.port_enabled) {
3478                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3479                         dev_dbg(dev, " ENABLE\n");
3480                 }
3481
3482                 if (usb_port_status.port_over_current) {
3483                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3484                         dev_dbg(dev, " OVER_CURRENT\n");
3485                 }
3486
3487                 if (usb_port_status.port_powered) {
3488                         port_status |= (1 << USB_PORT_FEAT_POWER);
3489                         dev_dbg(dev, " POWER\n");
3490                 }
3491
3492                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3493                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3494                         dev_dbg(dev, " HIGHSPEED\n");
3495                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3496                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3497                         dev_dbg(dev, " LOWSPEED\n");
3498                 }
3499
3500                 *((__le32 *) buf) = cpu_to_le32(port_status);
3501                 break;
3502         case SetHubFeature:
3503                 dev_dbg(dev, "SetHubFeature\n");
3504                 /* No HUB features supported */
3505                 break;
3506         case SetPortFeature:
3507                 dev_dbg(dev, "SetPortFeature\n");
3508                 if (wIndex != 1) {
3509                         dev_dbg(dev, " INVALID\n");
3510                         return -EINVAL;
3511                 }
3512
3513                 switch (wValue) {
3514                 case USB_PORT_FEAT_SUSPEND:
3515                         dev_dbg(dev, " SUSPEND\n");
3516                         return -EINVAL;
3517                 case USB_PORT_FEAT_POWER:
3518                         dev_dbg(dev, " POWER\n");
3519                         /*
3520                          * Program the port power bit to drive VBUS on the USB.
3521                          */
3522                         spin_lock_irqsave(&priv->lock, flags);
3523                         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
3524                                         cvmx_usbcx_hprt, prtpwr, 1);
3525                         spin_unlock_irqrestore(&priv->lock, flags);
3526                         return 0;
3527                 case USB_PORT_FEAT_RESET:
3528                         dev_dbg(dev, " RESET\n");
3529                         spin_lock_irqsave(&priv->lock, flags);
3530                         cvmx_usb_reset_port(&priv->usb);
3531                         spin_unlock_irqrestore(&priv->lock, flags);
3532                         return 0;
3533                 case USB_PORT_FEAT_INDICATOR:
3534                         dev_dbg(dev, " INDICATOR\n");
3535                         /* Not supported */
3536                         break;
3537                 default:
3538                         dev_dbg(dev, " UNKNOWN\n");
3539                         return -EINVAL;
3540                 }
3541                 break;
3542         default:
3543                 dev_dbg(dev, "Unknown root hub request\n");
3544                 return -EINVAL;
3545         }
3546         return 0;
3547 }
3548
3549 static const struct hc_driver octeon_hc_driver = {
3550         .description            = "Octeon USB",
3551         .product_desc           = "Octeon Host Controller",
3552         .hcd_priv_size          = sizeof(struct octeon_hcd),
3553         .irq                    = octeon_usb_irq,
3554         .flags                  = HCD_MEMORY | HCD_USB2,
3555         .start                  = octeon_usb_start,
3556         .stop                   = octeon_usb_stop,
3557         .urb_enqueue            = octeon_usb_urb_enqueue,
3558         .urb_dequeue            = octeon_usb_urb_dequeue,
3559         .endpoint_disable       = octeon_usb_endpoint_disable,
3560         .get_frame_number       = octeon_usb_get_frame_number,
3561         .hub_status_data        = octeon_usb_hub_status_data,
3562         .hub_control            = octeon_usb_hub_control,
3563         .map_urb_for_dma        = octeon_map_urb_for_dma,
3564         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3565 };
3566
3567 static int octeon_usb_probe(struct platform_device *pdev)
3568 {
3569         int status;
3570         int initialize_flags;
3571         int usb_num;
3572         struct resource *res_mem;
3573         struct device_node *usbn_node;
3574         int irq = platform_get_irq(pdev, 0);
3575         struct device *dev = &pdev->dev;
3576         struct octeon_hcd *priv;
3577         struct usb_hcd *hcd;
3578         u32 clock_rate = 48000000;
3579         bool is_crystal_clock = false;
3580         const char *clock_type;
3581         int i;
3582
3583         if (dev->of_node == NULL) {
3584                 dev_err(dev, "Error: empty of_node\n");
3585                 return -ENXIO;
3586         }
3587         usbn_node = dev->of_node->parent;
3588
3589         i = of_property_read_u32(usbn_node,
3590                                  "refclk-frequency", &clock_rate);
3591         if (i) {
3592                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3593                 return -ENXIO;
3594         }
3595         switch (clock_rate) {
3596         case 12000000:
3597                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3598                 break;
3599         case 24000000:
3600                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3601                 break;
3602         case 48000000:
3603                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3604                 break;
3605         default:
3606                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3607                                 clock_rate);
3608                 return -ENXIO;
3609
3610         }
3611
3612         i = of_property_read_string(usbn_node,
3613                                     "refclk-type", &clock_type);
3614
3615         if (!i && strcmp("crystal", clock_type) == 0)
3616                 is_crystal_clock = true;
3617
3618         if (is_crystal_clock)
3619                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3620         else
3621                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3622
3623         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3624         if (res_mem == NULL) {
3625                 dev_err(dev, "found no memory resource\n");
3626                 return -ENXIO;
3627         }
3628         usb_num = (res_mem->start >> 44) & 1;
3629
3630         if (irq < 0) {
3631                 /* Defective device tree, but we know how to fix it. */
3632                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3633
3634                 irq = irq_create_mapping(NULL, hwirq);
3635         }
3636
3637         /*
3638          * Set the DMA mask to 64bits so we get buffers already translated for
3639          * DMA.
3640          */
3641         dev->coherent_dma_mask = ~0;
3642         dev->dma_mask = &dev->coherent_dma_mask;
3643
3644         /*
3645          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3646          * IOB priority registers.  Under heavy network load USB
3647          * hardware can be starved by the IOB causing a crash.  Give
3648          * it a priority boost if it has been waiting more than 400
3649          * cycles to avoid this situation.
3650          *
3651          * Testing indicates that a cnt_val of 8192 is not sufficient,
3652          * but no failures are seen with 4096.  We choose a value of
3653          * 400 to give a safety factor of 10.
3654          */
3655         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3656                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3657
3658                 pri_cnt.u64 = 0;
3659                 pri_cnt.s.cnt_enb = 1;
3660                 pri_cnt.s.cnt_val = 400;
3661                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3662         }
3663
3664         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3665         if (!hcd) {
3666                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3667                 return -1;
3668         }
3669         hcd->uses_new_polling = 1;
3670         priv = (struct octeon_hcd *)hcd->hcd_priv;
3671
3672         spin_lock_init(&priv->lock);
3673
3674         priv->usb.init_flags = initialize_flags;
3675
3676         /* Initialize the USB state structure */
3677         priv->usb.index = usb_num;
3678         INIT_LIST_HEAD(&priv->usb.idle_pipes);
3679         for (i = 0; i < ARRAY_SIZE(priv->usb.active_pipes); i++)
3680                 INIT_LIST_HEAD(&priv->usb.active_pipes[i]);
3681
3682         /* Due to an errata, CN31XX doesn't support DMA */
3683         if (OCTEON_IS_MODEL(OCTEON_CN31XX)) {
3684                 priv->usb.init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
3685                 /* Only use one channel with non DMA */
3686                 priv->usb.idle_hardware_channels = 0x1;
3687         } else if (OCTEON_IS_MODEL(OCTEON_CN5XXX)) {
3688                 /* CN5XXX have an errata with channel 3 */
3689                 priv->usb.idle_hardware_channels = 0xf7;
3690         } else {
3691                 priv->usb.idle_hardware_channels = 0xff;
3692         }
3693
3694         status = cvmx_usb_initialize(dev, &priv->usb);
3695         if (status) {
3696                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3697                 kfree(hcd);
3698                 return -1;
3699         }
3700
3701         status = usb_add_hcd(hcd, irq, 0);
3702         if (status) {
3703                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3704                 kfree(hcd);
3705                 return -1;
3706         }
3707         device_wakeup_enable(hcd->self.controller);
3708
3709         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3710
3711         return 0;
3712 }
3713
3714 static int octeon_usb_remove(struct platform_device *pdev)
3715 {
3716         int status;
3717         struct device *dev = &pdev->dev;
3718         struct usb_hcd *hcd = dev_get_drvdata(dev);
3719         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3720         unsigned long flags;
3721
3722         usb_remove_hcd(hcd);
3723         spin_lock_irqsave(&priv->lock, flags);
3724         status = cvmx_usb_shutdown(&priv->usb);
3725         spin_unlock_irqrestore(&priv->lock, flags);
3726         if (status)
3727                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3728
3729         kfree(hcd);
3730
3731         return 0;
3732 }
3733
3734 static const struct of_device_id octeon_usb_match[] = {
3735         {
3736                 .compatible = "cavium,octeon-5750-usbc",
3737         },
3738         {},
3739 };
3740
3741 static struct platform_driver octeon_usb_driver = {
3742         .driver = {
3743                 .name       = "OcteonUSB",
3744                 .of_match_table = octeon_usb_match,
3745         },
3746         .probe      = octeon_usb_probe,
3747         .remove     = octeon_usb_remove,
3748 };
3749
3750 static int __init octeon_usb_driver_init(void)
3751 {
3752         if (usb_disabled())
3753                 return 0;
3754
3755         return platform_driver_register(&octeon_usb_driver);
3756 }
3757 module_init(octeon_usb_driver_init);
3758
3759 static void __exit octeon_usb_driver_exit(void)
3760 {
3761         if (usb_disabled())
3762                 return;
3763
3764         platform_driver_unregister(&octeon_usb_driver);
3765 }
3766 module_exit(octeon_usb_driver_exit);
3767
3768 MODULE_LICENSE("GPL");
3769 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3770 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");