kernel/drivers/net/ethernet/cavium/liquidio/octeon_droq.c

   1 /**********************************************************************
   2 * Author: Cavium, Inc.
   3 *
   4 * Contact: support@cavium.com
   5 *          Please include "LiquidIO" in the subject.
   6 *
   7 * Copyright (c) 2003-2015 Cavium, Inc.
   8 *
   9 * This file is free software; you can redistribute it and/or modify
  10 * it under the terms of the GNU General Public License, Version 2, as
  11 * published by the Free Software Foundation.
  12 *
  13 * This file is distributed in the hope that it will be useful, but
  14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16 * NONINFRINGEMENT.  See the GNU General Public License for more
  17 * details.
  18 *
  19 * This file may also be available under a different license from Cavium.
  20 * Contact Cavium, Inc. for more information
  21 **********************************************************************/
  22 #include <linux/version.h>
  23 #include <linux/types.h>
  24 #include <linux/list.h>
  25 #include <linux/pci.h>
  26 #include <linux/kthread.h>
  27 #include <linux/netdevice.h>
  28 #include <linux/vmalloc.h>
  29 #include "octeon_config.h"
  30 #include "liquidio_common.h"
  31 #include "octeon_droq.h"
  32 #include "octeon_iq.h"
  33 #include "response_manager.h"
  34 #include "octeon_device.h"
  35 #include "octeon_nic.h"
  36 #include "octeon_main.h"
  37 #include "octeon_network.h"
  38 #include "cn66xx_regs.h"
  39 #include "cn66xx_device.h"
  40 #include "cn68xx_regs.h"
  41 #include "cn68xx_device.h"
  42 #include "liquidio_image.h"
  43 #include "octeon_mem_ops.h"
  44
  45 /* #define CAVIUM_ONLY_PERF_MODE */
  46
  47 #define     CVM_MIN(d1, d2)           (((d1) < (d2)) ? (d1) : (d2))
  48 #define     CVM_MAX(d1, d2)           (((d1) > (d2)) ? (d1) : (d2))
  49
  50 struct niclist {
  51         struct list_head list;
  52         void *ptr;
  53 };
  54
  55 struct __dispatch {
  56         struct list_head list;
  57         struct octeon_recv_info *rinfo;
  58         octeon_dispatch_fn_t disp_fn;
  59 };
  60
  61 /** Get the argument that the user set when registering dispatch
  62  *  function for a given opcode/subcode.
  63  *  @param  octeon_dev - the octeon device pointer.
  64  *  @param  opcode     - the opcode for which the dispatch argument
  65  *                       is to be checked.
  66  *  @param  subcode    - the subcode for which the dispatch argument
  67  *                       is to be checked.
  68  *  @return  Success: void * (argument to the dispatch function)
  69  *  @return  Failure: NULL
  70  *
  71  */
  72 static inline void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
  73                                             u16 opcode, u16 subcode)
  74 {
  75         int idx;
  76         struct list_head *dispatch;
  77         void *fn_arg = NULL;
  78         u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
  79
  80         idx = combined_opcode & OCTEON_OPCODE_MASK;
  81
  82         spin_lock_bh(&octeon_dev->dispatch.lock);
  83
  84         if (octeon_dev->dispatch.count == 0) {
  85                 spin_unlock_bh(&octeon_dev->dispatch.lock);
  86                 return NULL;
  87         }
  88
  89         if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
  90                 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
  91         } else {
  92                 list_for_each(dispatch,
  93                               &octeon_dev->dispatch.dlist[idx].list) {
  94                         if (((struct octeon_dispatch *)dispatch)->opcode ==
  95                             combined_opcode) {
  96                                 fn_arg = ((struct octeon_dispatch *)
  97                                           dispatch)->arg;
  98                                 break;
  99                         }
 100                 }
 101         }
 102
 103         spin_unlock_bh(&octeon_dev->dispatch.lock);
 104         return fn_arg;
 105 }
 106
 107 u32 octeon_droq_check_hw_for_pkts(struct octeon_device *oct,
 108                                   struct octeon_droq *droq)
 109 {
 110         u32 pkt_count = 0;
 111
 112         pkt_count = readl(droq->pkts_sent_reg);
 113         if (pkt_count) {
 114                 atomic_add(pkt_count, &droq->pkts_pending);
 115                 writel(pkt_count, droq->pkts_sent_reg);
 116         }
 117
 118         return pkt_count;
 119 }
 120
 121 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
 122 {
 123         u32 count = 0;
 124
 125         /* max_empty_descs is the max. no. of descs that can have no buffers.
 126          * If the empty desc count goes beyond this value, we cannot safely
 127          * read in a 64K packet sent by Octeon
 128          * (64K is max pkt size from Octeon)
 129          */
 130         droq->max_empty_descs = 0;
 131
 132         do {
 133                 droq->max_empty_descs++;
 134                 count += droq->buffer_size;
 135         } while (count < (64 * 1024));
 136
 137         droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
 138 }
 139
 140 static void octeon_droq_reset_indices(struct octeon_droq *droq)
 141 {
 142         droq->read_idx = 0;
 143         droq->write_idx = 0;
 144         droq->refill_idx = 0;
 145         droq->refill_count = 0;
 146         atomic_set(&droq->pkts_pending, 0);
 147 }
 148
 149 static void
 150 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
 151                                  struct octeon_droq *droq)
 152 {
 153         u32 i;
 154
 155         for (i = 0; i < droq->max_count; i++) {
 156                 if (droq->recv_buf_list[i].buffer) {
 157                         if (droq->desc_ring) {
 158                                 lio_unmap_ring_info(oct->pci_dev,
 159                                                     (u64)droq->
 160                                                     desc_ring[i].info_ptr,
 161                                                     OCT_DROQ_INFO_SIZE);
 162                                 lio_unmap_ring(oct->pci_dev,
 163                                                (u64)droq->desc_ring[i].
 164                                                buffer_ptr,
 165                                                droq->buffer_size);
 166                         }
 167                         recv_buffer_free(droq->recv_buf_list[i].buffer);
 168                         droq->recv_buf_list[i].buffer = NULL;
 169                 }
 170         }
 171
 172         octeon_droq_reset_indices(droq);
 173 }
 174
 175 static int
 176 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
 177                                struct octeon_droq *droq)
 178 {
 179         u32 i;
 180         void *buf;
 181         struct octeon_droq_desc *desc_ring = droq->desc_ring;
 182
 183         for (i = 0; i < droq->max_count; i++) {
 184                 buf = recv_buffer_alloc(oct, droq->q_no, droq->buffer_size);
 185
 186                 if (!buf) {
 187                         dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
 188                                 __func__);
 189                         return -ENOMEM;
 190                 }
 191
 192                 droq->recv_buf_list[i].buffer = buf;
 193                 droq->recv_buf_list[i].data = get_rbd(buf);
 194
 195                 droq->info_list[i].length = 0;
 196
 197                 /* map ring buffers into memory */
 198                 desc_ring[i].info_ptr = lio_map_ring_info(droq, i);
 199                 desc_ring[i].buffer_ptr =
 200                         lio_map_ring(oct->pci_dev,
 201                                      droq->recv_buf_list[i].buffer,
 202                                      droq->buffer_size);
 203         }
 204
 205         octeon_droq_reset_indices(droq);
 206
 207         octeon_droq_compute_max_packet_bufs(droq);
 208
 209         return 0;
 210 }
 211
 212 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
 213 {
 214         struct octeon_droq *droq = oct->droq[q_no];
 215
 216         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 217
 218         octeon_droq_destroy_ring_buffers(oct, droq);
 219         vfree(droq->recv_buf_list);
 220
 221         if (droq->info_base_addr)
 222                 cnnic_free_aligned_dma(oct->pci_dev, droq->info_list,
 223                                        droq->info_alloc_size,
 224                                        droq->info_base_addr,
 225                                        droq->info_list_dma);
 226
 227         if (droq->desc_ring)
 228                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 229                              droq->desc_ring, droq->desc_ring_dma);
 230
 231         memset(droq, 0, OCT_DROQ_SIZE);
 232
 233         return 0;
 234 }
 235
 236 int octeon_init_droq(struct octeon_device *oct,
 237                      u32 q_no,
 238                      u32 num_descs,
 239                      u32 desc_size,
 240                      void *app_ctx)
 241 {
 242         struct octeon_droq *droq;
 243         u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
 244         u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
 245
 246         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 247
 248         droq = oct->droq[q_no];
 249         memset(droq, 0, OCT_DROQ_SIZE);
 250
 251         droq->oct_dev = oct;
 252         droq->q_no = q_no;
 253         if (app_ctx)
 254                 droq->app_ctx = app_ctx;
 255         else
 256                 droq->app_ctx = (void *)(size_t)q_no;
 257
 258         c_num_descs = num_descs;
 259         c_buf_size = desc_size;
 260         if (OCTEON_CN6XXX(oct)) {
 261                 struct octeon_config *conf6x = CHIP_FIELD(oct, cn6xxx, conf);
 262
 263                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
 264                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
 265         }
 266
 267         droq->max_count = c_num_descs;
 268         droq->buffer_size = c_buf_size;
 269
 270         desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
 271         droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
 272                                         (dma_addr_t *)&droq->desc_ring_dma);
 273
 274         if (!droq->desc_ring) {
 275                 dev_err(&oct->pci_dev->dev,
 276                         "Output queue %d ring alloc failed\n", q_no);
 277                 return 1;
 278         }
 279
 280         dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
 281                 q_no, droq->desc_ring, droq->desc_ring_dma);
 282         dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
 283                 droq->max_count);
 284
 285         droq->info_list =
 286                 cnnic_alloc_aligned_dma(oct->pci_dev,
 287                                         (droq->max_count * OCT_DROQ_INFO_SIZE),
 288                                         &droq->info_alloc_size,
 289                                         &droq->info_base_addr,
 290                                         &droq->info_list_dma);
 291
 292         if (!droq->info_list) {
 293                 dev_err(&oct->pci_dev->dev, "Cannot allocate memory for info list.\n");
 294                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 295                              droq->desc_ring, droq->desc_ring_dma);
 296                 return 1;
 297         }
 298
 299         droq->recv_buf_list = (struct octeon_recv_buffer *)
 300                               vmalloc(droq->max_count *
 301                                                 OCT_DROQ_RECVBUF_SIZE);
 302         if (!droq->recv_buf_list) {
 303                 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
 304                 goto init_droq_fail;
 305         }
 306
 307         if (octeon_droq_setup_ring_buffers(oct, droq))
 308                 goto init_droq_fail;
 309
 310         droq->pkts_per_intr = c_pkts_per_intr;
 311         droq->refill_threshold = c_refill_threshold;
 312
 313         dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
 314                 droq->max_empty_descs);
 315
 316         spin_lock_init(&droq->lock);
 317
 318         INIT_LIST_HEAD(&droq->dispatch_list);
 319
 320         /* For 56xx Pass1, this function won't be called, so no checks. */
 321         oct->fn_list.setup_oq_regs(oct, q_no);
 322
 323         oct->io_qmask.oq |= (1 << q_no);
 324
 325         return 0;
 326
 327 init_droq_fail:
 328         octeon_delete_droq(oct, q_no);
 329         return 1;
 330 }
 331
 332 /* octeon_create_recv_info
 333  * Parameters:
 334  *  octeon_dev - pointer to the octeon device structure
 335  *  droq       - droq in which the packet arrived.
 336  *  buf_cnt    - no. of buffers used by the packet.
 337  *  idx        - index in the descriptor for the first buffer in the packet.
 338  * Description:
 339  *  Allocates a recv_info_t and copies the buffer addresses for packet data
 340  *  into the recv_pkt space which starts at an 8B offset from recv_info_t.
 341  *  Flags the descriptors for refill later. If available descriptors go
 342  *  below the threshold to receive a 64K pkt, new buffers are first allocated
 343  *  before the recv_pkt_t is created.
 344  *  This routine will be called in interrupt context.
 345  * Returns:
 346  *  Success: Pointer to recv_info_t
 347  *  Failure: NULL.
 348  * Locks:
 349  *  The droq->lock is held when this routine is called.
 350  */
 351 static inline struct octeon_recv_info *octeon_create_recv_info(
 352                 struct octeon_device *octeon_dev,
 353                 struct octeon_droq *droq,
 354                 u32 buf_cnt,
 355                 u32 idx)
 356 {
 357         struct octeon_droq_info *info;
 358         struct octeon_recv_pkt *recv_pkt;
 359         struct octeon_recv_info *recv_info;
 360         u32 i, bytes_left;
 361
 362         info = &droq->info_list[idx];
 363
 364         recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
 365         if (!recv_info)
 366                 return NULL;
 367
 368         recv_pkt = recv_info->recv_pkt;
 369         recv_pkt->rh = info->rh;
 370         recv_pkt->length = (u32)info->length;
 371         recv_pkt->buffer_count = (u16)buf_cnt;
 372         recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
 373
 374         i = 0;
 375         bytes_left = (u32)info->length;
 376
 377         while (buf_cnt) {
 378                 lio_unmap_ring(octeon_dev->pci_dev,
 379                                (u64)droq->desc_ring[idx].buffer_ptr,
 380                                droq->buffer_size);
 381
 382                 recv_pkt->buffer_size[i] =
 383                         (bytes_left >=
 384                          droq->buffer_size) ? droq->buffer_size : bytes_left;
 385
 386                 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
 387                 droq->recv_buf_list[idx].buffer = NULL;
 388
 389                 INCR_INDEX_BY1(idx, droq->max_count);
 390                 bytes_left -= droq->buffer_size;
 391                 i++;
 392                 buf_cnt--;
 393         }
 394
 395         return recv_info;
 396 }
 397
 398 /* If we were not able to refill all buffers, try to move around
 399  * the buffers that were not dispatched.
 400  */
 401 static inline u32
 402 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
 403                                 struct octeon_droq_desc *desc_ring)
 404 {
 405         u32 desc_refilled = 0;
 406
 407         u32 refill_index = droq->refill_idx;
 408
 409         while (refill_index != droq->read_idx) {
 410                 if (droq->recv_buf_list[refill_index].buffer) {
 411                         droq->recv_buf_list[droq->refill_idx].buffer =
 412                                 droq->recv_buf_list[refill_index].buffer;
 413                         droq->recv_buf_list[droq->refill_idx].data =
 414                                 droq->recv_buf_list[refill_index].data;
 415                         desc_ring[droq->refill_idx].buffer_ptr =
 416                                 desc_ring[refill_index].buffer_ptr;
 417                         droq->recv_buf_list[refill_index].buffer = NULL;
 418                         desc_ring[refill_index].buffer_ptr = 0;
 419                         do {
 420                                 INCR_INDEX_BY1(droq->refill_idx,
 421                                                droq->max_count);
 422                                 desc_refilled++;
 423                                 droq->refill_count--;
 424                         } while (droq->recv_buf_list[droq->refill_idx].
 425                                  buffer);
 426                 }
 427                 INCR_INDEX_BY1(refill_index, droq->max_count);
 428         }                       /* while */
 429         return desc_refilled;
 430 }
 431
 432 /* octeon_droq_refill
 433  * Parameters:
 434  *  droq       - droq in which descriptors require new buffers.
 435  * Description:
 436  *  Called during normal DROQ processing in interrupt mode or by the poll
 437  *  thread to refill the descriptors from which buffers were dispatched
 438  *  to upper layers. Attempts to allocate new buffers. If that fails, moves
 439  *  up buffers (that were not dispatched) to form a contiguous ring.
 440  * Returns:
 441  *  No of descriptors refilled.
 442  * Locks:
 443  *  This routine is called with droq->lock held.
 444  */
 445 static u32
 446 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
 447 {
 448         struct octeon_droq_desc *desc_ring;
 449         void *buf = NULL;
 450         u8 *data;
 451         u32 desc_refilled = 0;
 452
 453         desc_ring = droq->desc_ring;
 454
 455         while (droq->refill_count && (desc_refilled < droq->max_count)) {
 456                 /* If a valid buffer exists (happens if there is no dispatch),
 457                  * reuse
 458                  * the buffer, else allocate.
 459                  */
 460                 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
 461                         buf = recv_buffer_alloc(octeon_dev, droq->q_no,
 462                                                 droq->buffer_size);
 463                         /* If a buffer could not be allocated, no point in
 464                          * continuing
 465                          */
 466                         if (!buf)
 467                                 break;
 468                         droq->recv_buf_list[droq->refill_idx].buffer =
 469                                 buf;
 470                         data = get_rbd(buf);
 471                 } else {
 472                         data = get_rbd(droq->recv_buf_list
 473                                        [droq->refill_idx].buffer);
 474                 }
 475
 476                 droq->recv_buf_list[droq->refill_idx].data = data;
 477
 478                 desc_ring[droq->refill_idx].buffer_ptr =
 479                         lio_map_ring(octeon_dev->pci_dev,
 480                                      droq->recv_buf_list[droq->
 481                                      refill_idx].buffer,
 482                                      droq->buffer_size);
 483
 484                 /* Reset any previous values in the length field. */
 485                 droq->info_list[droq->refill_idx].length = 0;
 486
 487                 INCR_INDEX_BY1(droq->refill_idx, droq->max_count);
 488                 desc_refilled++;
 489                 droq->refill_count--;
 490         }
 491
 492         if (droq->refill_count)
 493                 desc_refilled +=
 494                         octeon_droq_refill_pullup_descs(droq, desc_ring);
 495
 496         /* if droq->refill_count
 497          * The refill count would not change in pass two. We only moved buffers
 498          * to close the gap in the ring, but we would still have the same no. of
 499          * buffers to refill.
 500          */
 501         return desc_refilled;
 502 }
 503
 504 static inline u32
 505 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
 506 {
 507         u32 buf_cnt = 0;
 508
 509         while (total_len > (buf_size * buf_cnt))
 510                 buf_cnt++;
 511         return buf_cnt;
 512 }
 513
 514 static int
 515 octeon_droq_dispatch_pkt(struct octeon_device *oct,
 516                          struct octeon_droq *droq,
 517                          union octeon_rh *rh,
 518                          struct octeon_droq_info *info)
 519 {
 520         u32 cnt;
 521         octeon_dispatch_fn_t disp_fn;
 522         struct octeon_recv_info *rinfo;
 523
 524         cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
 525
 526         disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
 527                                       (u16)rh->r.subcode);
 528         if (disp_fn) {
 529                 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
 530                 if (rinfo) {
 531                         struct __dispatch *rdisp = rinfo->rsvd;
 532
 533                         rdisp->rinfo = rinfo;
 534                         rdisp->disp_fn = disp_fn;
 535                         rinfo->recv_pkt->rh = *rh;
 536                         list_add_tail(&rdisp->list,
 537                                       &droq->dispatch_list);
 538                 } else {
 539                         droq->stats.dropped_nomem++;
 540                 }
 541         } else {
 542                 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function\n");
 543                 droq->stats.dropped_nodispatch++;
 544         }                       /* else (dispatch_fn ... */
 545
 546         return cnt;
 547 }
 548
 549 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
 550                                             struct octeon_droq *droq,
 551                                             u32 cnt)
 552 {
 553         u32 i = 0, buf_cnt;
 554         struct octeon_droq_info *info;
 555
 556         for (i = 0; i < cnt; i++) {
 557                 info = &droq->info_list[droq->read_idx];
 558                 octeon_swap_8B_data((u64 *)info, 2);
 559
 560                 if (info->length) {
 561                         info->length -= OCT_RH_SIZE;
 562                         droq->stats.bytes_received += info->length;
 563                         buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
 564                                                            (u32)info->length);
 565                 } else {
 566                         dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
 567                         buf_cnt = 1;
 568                 }
 569
 570                 INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
 571                 droq->refill_count += buf_cnt;
 572         }
 573 }
 574
 575 static u32
 576 octeon_droq_fast_process_packets(struct octeon_device *oct,
 577                                  struct octeon_droq *droq,
 578                                  u32 pkts_to_process)
 579 {
 580         struct octeon_droq_info *info;
 581         union octeon_rh *rh;
 582         u32 pkt, total_len = 0, pkt_count;
 583
 584         pkt_count = pkts_to_process;
 585
 586         for (pkt = 0; pkt < pkt_count; pkt++) {
 587                 u32 pkt_len = 0;
 588                 struct sk_buff *nicbuf = NULL;
 589
 590                 info = &droq->info_list[droq->read_idx];
 591                 octeon_swap_8B_data((u64 *)info, 2);
 592
 593                 if (!info->length) {
 594                         dev_err(&oct->pci_dev->dev,
 595                                 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
 596                                 droq->q_no, droq->read_idx, pkt_count);
 597                         print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
 598                                              (u8 *)info,
 599                                              OCT_DROQ_INFO_SIZE);
 600                         break;
 601                 }
 602
 603                 /* Len of resp hdr in included in the received data len. */
 604                 info->length -= OCT_RH_SIZE;
 605                 rh = &info->rh;
 606
 607                 total_len += (u32)info->length;
 608
 609                 if (OPCODE_SLOW_PATH(rh)) {
 610                         u32 buf_cnt;
 611
 612                         buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
 613                         INCR_INDEX(droq->read_idx, buf_cnt, droq->max_count);
 614                         droq->refill_count += buf_cnt;
 615                 } else {
 616                         if (info->length <= droq->buffer_size) {
 617                                 lio_unmap_ring(oct->pci_dev,
 618                                                (u64)droq->desc_ring[
 619                                                droq->read_idx].buffer_ptr,
 620                                                droq->buffer_size);
 621                                 pkt_len = (u32)info->length;
 622                                 nicbuf = droq->recv_buf_list[
 623                                         droq->read_idx].buffer;
 624                                 droq->recv_buf_list[droq->read_idx].buffer =
 625                                         NULL;
 626                                 INCR_INDEX_BY1(droq->read_idx, droq->max_count);
 627                                 skb_put(nicbuf, pkt_len);
 628                                 droq->refill_count++;
 629                         } else {
 630                                 nicbuf = octeon_fast_packet_alloc(oct, droq,
 631                                                                   droq->q_no,
 632                                                                   (u32)
 633                                                                   info->length);
 634                                 pkt_len = 0;
 635                                 /* nicbuf allocation can fail. We'll handle it
 636                                  * inside the loop.
 637                                  */
 638                                 while (pkt_len < info->length) {
 639                                         int cpy_len;
 640
 641                                         cpy_len = ((pkt_len +
 642                                                 droq->buffer_size) >
 643                                                 info->length) ?
 644                                                 ((u32)info->length - pkt_len) :
 645                                                 droq->buffer_size;
 646
 647                                         if (nicbuf) {
 648                                                 lio_unmap_ring(oct->pci_dev,
 649                                                                (u64)
 650                                                                droq->desc_ring
 651                                                                [droq->read_idx].
 652                                                                buffer_ptr,
 653                                                                droq->
 654                                                                buffer_size);
 655                                                 octeon_fast_packet_next(droq,
 656                                                                         nicbuf,
 657                                                                         cpy_len,
 658                                                                         droq->
 659                                                                         read_idx
 660                                                                         );
 661                                         }
 662
 663                                         pkt_len += cpy_len;
 664                                         INCR_INDEX_BY1(droq->read_idx,
 665                                                        droq->max_count);
 666                                         droq->refill_count++;
 667                                 }
 668                         }
 669
 670                         if (nicbuf) {
 671                                 if (droq->ops.fptr)
 672                                         droq->ops.fptr(oct->octeon_id,
 673                                         nicbuf, pkt_len,
 674                                         rh, &droq->napi);
 675                                 else
 676                                         recv_buffer_free(nicbuf);
 677                         }
 678                 }
 679
 680                 if (droq->refill_count >= droq->refill_threshold) {
 681                         int desc_refilled = octeon_droq_refill(oct, droq);
 682
 683                         /* Flush the droq descriptor data to memory to be sure
 684                         * that when we update the credits the data in memory
 685                         * is accurate.
 686                         */
 687                         wmb();
 688                         writel((desc_refilled), droq->pkts_credit_reg);
 689                         /* make sure mmio write completes */
 690                         mmiowb();
 691                 }
 692
 693         }                       /* for ( each packet )... */
 694
 695         /* Increment refill_count by the number of buffers processed. */
 696         droq->stats.pkts_received += pkt;
 697         droq->stats.bytes_received += total_len;
 698
 699         if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
 700                 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
 701
 702                 droq->stats.dropped_toomany += (pkts_to_process - pkt);
 703                 return pkts_to_process;
 704         }
 705
 706         return pkt;
 707 }
 708
 709 int
 710 octeon_droq_process_packets(struct octeon_device *oct,
 711                             struct octeon_droq *droq,
 712                             u32 budget)
 713 {
 714         u32 pkt_count = 0, pkts_processed = 0;
 715         struct list_head *tmp, *tmp2;
 716
 717         pkt_count = atomic_read(&droq->pkts_pending);
 718         if (!pkt_count)
 719                 return 0;
 720
 721         if (pkt_count > budget)
 722                 pkt_count = budget;
 723
 724         /* Grab the lock */
 725         spin_lock(&droq->lock);
 726
 727         pkts_processed = octeon_droq_fast_process_packets(oct, droq, pkt_count);
 728
 729         atomic_sub(pkts_processed, &droq->pkts_pending);
 730
 731         /* Release the spin lock */
 732         spin_unlock(&droq->lock);
 733
 734         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 735                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 736
 737                 list_del(tmp);
 738                 rdisp->disp_fn(rdisp->rinfo,
 739                                octeon_get_dispatch_arg
 740                                (oct,
 741                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 742                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 743         }
 744
 745         /* If there are packets pending. schedule tasklet again */
 746         if (atomic_read(&droq->pkts_pending))
 747                 return 1;
 748
 749         return 0;
 750 }
 751
 752 /**
 753  * Utility function to poll for packets. check_hw_for_packets must be
 754  * called before calling this routine.
 755  */
 756
 757 static int
 758 octeon_droq_process_poll_pkts(struct octeon_device *oct,
 759                               struct octeon_droq *droq, u32 budget)
 760 {
 761         struct list_head *tmp, *tmp2;
 762         u32 pkts_available = 0, pkts_processed = 0;
 763         u32 total_pkts_processed = 0;
 764
 765         if (budget > droq->max_count)
 766                 budget = droq->max_count;
 767
 768         spin_lock(&droq->lock);
 769
 770         while (total_pkts_processed < budget) {
 771                 pkts_available =
 772                         CVM_MIN((budget - total_pkts_processed),
 773                                 (u32)(atomic_read(&droq->pkts_pending)));
 774
 775                 if (pkts_available == 0)
 776                         break;
 777
 778                 pkts_processed =
 779                         octeon_droq_fast_process_packets(oct, droq,
 780                                                          pkts_available);
 781
 782                 atomic_sub(pkts_processed, &droq->pkts_pending);
 783
 784                 total_pkts_processed += pkts_processed;
 785
 786                 octeon_droq_check_hw_for_pkts(oct, droq);
 787         }
 788
 789         spin_unlock(&droq->lock);
 790
 791         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 792                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 793
 794                 list_del(tmp);
 795                 rdisp->disp_fn(rdisp->rinfo,
 796                                octeon_get_dispatch_arg
 797                                (oct,
 798                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 799                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 800         }
 801
 802         return total_pkts_processed;
 803 }
 804
 805 int
 806 octeon_process_droq_poll_cmd(struct octeon_device *oct, u32 q_no, int cmd,
 807                              u32 arg)
 808 {
 809         struct octeon_droq *droq;
 810         struct octeon_config *oct_cfg = NULL;
 811
 812         oct_cfg = octeon_get_conf(oct);
 813
 814         if (!oct_cfg)
 815                 return -EINVAL;
 816
 817         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 818                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 819                         __func__, q_no, (oct->num_oqs - 1));
 820                 return -EINVAL;
 821         }
 822
 823         droq = oct->droq[q_no];
 824
 825         if (cmd == POLL_EVENT_PROCESS_PKTS)
 826                 return octeon_droq_process_poll_pkts(oct, droq, arg);
 827
 828         if (cmd == POLL_EVENT_PENDING_PKTS) {
 829                 u32 pkt_cnt = atomic_read(&droq->pkts_pending);
 830
 831                 return  octeon_droq_process_packets(oct, droq, pkt_cnt);
 832         }
 833
 834         if (cmd == POLL_EVENT_ENABLE_INTR) {
 835                 u32 value;
 836                 unsigned long flags;
 837
 838                 /* Enable Pkt Interrupt */
 839                 switch (oct->chip_id) {
 840                 case OCTEON_CN66XX:
 841                 case OCTEON_CN68XX: {
 842                         struct octeon_cn6xxx *cn6xxx =
 843                                 (struct octeon_cn6xxx *)oct->chip;
 844                         spin_lock_irqsave
 845                                 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 846                         value =
 847                                 octeon_read_csr(oct,
 848                                                 CN6XXX_SLI_PKT_TIME_INT_ENB);
 849                         value |= (1 << q_no);
 850                         octeon_write_csr(oct,
 851                                          CN6XXX_SLI_PKT_TIME_INT_ENB,
 852                                          value);
 853                         value =
 854                                 octeon_read_csr(oct,
 855                                                 CN6XXX_SLI_PKT_CNT_INT_ENB);
 856                         value |= (1 << q_no);
 857                         octeon_write_csr(oct,
 858                                          CN6XXX_SLI_PKT_CNT_INT_ENB,
 859                                          value);
 860
 861                         /* don't bother flushing the enables */
 862
 863                         spin_unlock_irqrestore
 864                                 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 865                         return 0;
 866                 }
 867                 break;
 868                 }
 869
 870                 return 0;
 871         }
 872
 873         dev_err(&oct->pci_dev->dev, "%s Unknown command: %d\n", __func__, cmd);
 874         return -EINVAL;
 875 }
 876
 877 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
 878                              struct octeon_droq_ops *ops)
 879 {
 880         struct octeon_droq *droq;
 881         unsigned long flags;
 882         struct octeon_config *oct_cfg = NULL;
 883
 884         oct_cfg = octeon_get_conf(oct);
 885
 886         if (!oct_cfg)
 887                 return -EINVAL;
 888
 889         if (!(ops)) {
 890                 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
 891                         __func__);
 892                 return -EINVAL;
 893         }
 894
 895         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 896                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 897                         __func__, q_no, (oct->num_oqs - 1));
 898                 return -EINVAL;
 899         }
 900
 901         droq = oct->droq[q_no];
 902
 903         spin_lock_irqsave(&droq->lock, flags);
 904
 905         memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
 906
 907         spin_unlock_irqrestore(&droq->lock, flags);
 908
 909         return 0;
 910 }
 911
 912 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
 913 {
 914         unsigned long flags;
 915         struct octeon_droq *droq;
 916         struct octeon_config *oct_cfg = NULL;
 917
 918         oct_cfg = octeon_get_conf(oct);
 919
 920         if (!oct_cfg)
 921                 return -EINVAL;
 922
 923         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 924                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 925                         __func__, q_no, oct->num_oqs - 1);
 926                 return -EINVAL;
 927         }
 928
 929         droq = oct->droq[q_no];
 930
 931         if (!droq) {
 932                 dev_info(&oct->pci_dev->dev,
 933                          "Droq id (%d) not available.\n", q_no);
 934                 return 0;
 935         }
 936
 937         spin_lock_irqsave(&droq->lock, flags);
 938
 939         droq->ops.fptr = NULL;
 940         droq->ops.drop_on_max = 0;
 941
 942         spin_unlock_irqrestore(&droq->lock, flags);
 943
 944         return 0;
 945 }
 946
 947 int octeon_create_droq(struct octeon_device *oct,
 948                        u32 q_no, u32 num_descs,
 949                        u32 desc_size, void *app_ctx)
 950 {
 951         struct octeon_droq *droq;
 952
 953         if (oct->droq[q_no]) {
 954                 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
 955                         q_no);
 956                 return 1;
 957         }
 958
 959         /* Allocate the DS for the new droq. */
 960         droq = vmalloc(sizeof(*droq));
 961         if (!droq)
 962                 goto create_droq_fail;
 963         memset(droq, 0, sizeof(struct octeon_droq));
 964
 965         /*Disable the pkt o/p for this Q  */
 966         octeon_set_droq_pkt_op(oct, q_no, 0);
 967         oct->droq[q_no] = droq;
 968
 969         /* Initialize the Droq */
 970         octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx);
 971
 972         oct->num_oqs++;
 973
 974         dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
 975                 oct->num_oqs);
 976
 977         /* Global Droq register settings */
 978
 979         /* As of now not required, as setting are done for all 32 Droqs at
 980          * the same time.
 981          */
 982         return 0;
 983
 984 create_droq_fail:
 985         octeon_delete_droq(oct, q_no);
 986         return -1;
 987 }