kernel/drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42 #include <linux/slab.h>
  43
  44 #include "nes.h"
  45
  46 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  47 module_param(nes_lro_max_aggr, uint, 0444);
  48 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  49
  50 static int wide_ppm_offset;
  51 module_param(wide_ppm_offset, int, 0644);
  52 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  53
  54 static u32 crit_err_count;
  55 u32 int_mod_timer_init;
  56 u32 int_mod_cq_depth_256;
  57 u32 int_mod_cq_depth_128;
  58 u32 int_mod_cq_depth_32;
  59 u32 int_mod_cq_depth_24;
  60 u32 int_mod_cq_depth_16;
  61 u32 int_mod_cq_depth_4;
  62 u32 int_mod_cq_depth_1;
  63 static const u8 nes_max_critical_error_count = 100;
  64 #include "nes_cm.h"
  65
  66 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  67 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  68 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  69                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  70 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  71 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  72 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  73 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  74                                    struct nes_hw_aeqe *aeqe);
  75 static void process_critical_error(struct nes_device *nesdev);
  76 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  77 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  79
  80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  81 static unsigned char *nes_iwarp_state_str[] = {
  82         "Non-Existent",
  83         "Idle",
  84         "RTS",
  85         "Closing",
  86         "RSVD1",
  87         "Terminate",
  88         "Error",
  89         "RSVD2",
  90 };
  91
  92 static unsigned char *nes_tcp_state_str[] = {
  93         "Non-Existent",
  94         "Closed",
  95         "Listen",
  96         "SYN Sent",
  97         "SYN Rcvd",
  98         "Established",
  99         "Close Wait",
 100         "FIN Wait 1",
 101         "Closing",
 102         "Last Ack",
 103         "FIN Wait 2",
 104         "Time Wait",
 105         "RSVD1",
 106         "RSVD2",
 107         "RSVD3",
 108         "RSVD4",
 109 };
 110 #endif
 111
 112 static inline void print_ip(struct nes_cm_node *cm_node)
 113 {
 114         unsigned char *rem_addr;
 115         if (cm_node) {
 116                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 117                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 118         }
 119 }
 120
 121 /**
 122  * nes_nic_init_timer_defaults
 123  */
 124 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 125 {
 126         unsigned long flags;
 127         struct nes_adapter *nesadapter = nesdev->nesadapter;
 128         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 129
 130         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 131
 132         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 133         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 134         if (jumbomode) {
 135                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 136                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 137                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 138         } else {
 139                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 140                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 141                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 142         }
 143
 144         /* todo use netdev->mtu to set thresholds */
 145         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 146 }
 147
 148
 149 /**
 150  * nes_nic_init_timer
 151  */
 152 static void  nes_nic_init_timer(struct nes_device *nesdev)
 153 {
 154         unsigned long flags;
 155         struct nes_adapter *nesadapter = nesdev->nesadapter;
 156         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 157
 158         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 159
 160         if (shared_timer->timer_in_use_old == 0) {
 161                 nesdev->deepcq_count = 0;
 162                 shared_timer->timer_direction_upward = 0;
 163                 shared_timer->timer_direction_downward = 0;
 164                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 165                 shared_timer->timer_in_use_old = 0;
 166
 167         }
 168         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 169                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 170                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 171                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 172         }
 173         /* todo use netdev->mtu to set thresholds */
 174         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 175 }
 176
 177
 178 /**
 179  * nes_nic_tune_timer
 180  */
 181 static void nes_nic_tune_timer(struct nes_device *nesdev)
 182 {
 183         unsigned long flags;
 184         struct nes_adapter *nesadapter = nesdev->nesadapter;
 185         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 186         u16 cq_count = nesdev->currcq_count;
 187
 188         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 189
 190         if (shared_timer->cq_count_old <= cq_count)
 191                 shared_timer->cq_direction_downward = 0;
 192         else
 193                 shared_timer->cq_direction_downward++;
 194         shared_timer->cq_count_old = cq_count;
 195         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 196                 if (cq_count <= shared_timer->threshold_low &&
 197                     shared_timer->threshold_low > 4) {
 198                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 199                         shared_timer->cq_direction_downward=0;
 200                         nesdev->currcq_count = 0;
 201                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 202                         return;
 203                 }
 204         }
 205
 206         if (cq_count > 1) {
 207                 nesdev->deepcq_count += cq_count;
 208                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 209                         shared_timer->timer_direction_upward++;
 210                         shared_timer->timer_direction_downward = 0;
 211                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 212                         shared_timer->timer_direction_upward = 0;
 213                         shared_timer->timer_direction_downward = 0;
 214                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 215                         shared_timer->timer_direction_downward++;
 216                         shared_timer->timer_direction_upward = 0;
 217                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 218                         shared_timer->timer_in_use -= 2;
 219                         shared_timer->timer_direction_upward = 0;
 220                         shared_timer->timer_direction_downward++;
 221                 } else {
 222                         shared_timer->timer_in_use -= 4;
 223                         shared_timer->timer_direction_upward = 0;
 224                         shared_timer->timer_direction_downward++;
 225                 }
 226
 227                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 228                         shared_timer->timer_in_use += 3;
 229                         shared_timer->timer_direction_upward = 0;
 230                         shared_timer->timer_direction_downward = 0;
 231                 }
 232                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 233                         shared_timer->timer_in_use -= 4 ;
 234                         shared_timer->timer_direction_downward = 0;
 235                         shared_timer->timer_direction_upward = 0;
 236                 }
 237         }
 238
 239         /* boundary checking */
 240         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 241                 shared_timer->timer_in_use = shared_timer->threshold_high;
 242         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 243                 shared_timer->timer_in_use = shared_timer->threshold_low;
 244
 245         nesdev->currcq_count = 0;
 246
 247         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 248 }
 249
 250
 251 /**
 252  * nes_init_adapter - initialize adapter
 253  */
 254 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 255         struct nes_adapter *nesadapter = NULL;
 256         unsigned long num_pds;
 257         u32 u32temp;
 258         u32 port_count;
 259         u16 max_rq_wrs;
 260         u16 max_sq_wrs;
 261         u32 max_mr;
 262         u32 max_256pbl;
 263         u32 max_4kpbl;
 264         u32 max_qp;
 265         u32 max_irrq;
 266         u32 max_cq;
 267         u32 hte_index_mask;
 268         u32 adapter_size;
 269         u32 arp_table_size;
 270         u16 vendor_id;
 271         u16 device_id;
 272         u8  OneG_Mode;
 273         u8  func_index;
 274
 275         /* search the list of existing adapters */
 276         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 277                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 278                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 279                                 nesdev->pcidev->devfn,
 280                                 PCI_SLOT(nesadapter->devfn),
 281                                 nesadapter->bus_number,
 282                                 PCI_SLOT(nesdev->pcidev->devfn),
 283                                 nesdev->pcidev->bus->number );
 284                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 285                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 286                         nesadapter->ref_count++;
 287                         return nesadapter;
 288                 }
 289         }
 290
 291         /* no adapter found */
 292         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 293         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 294                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 295                                 hw_rev);
 296                 return NULL;
 297         }
 298
 299         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 300                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 301                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 302                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 303                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 304
 305         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 306
 307
 308         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 309                 return NULL;
 310
 311         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 312         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 313
 314         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 315         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 316                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 317                                 max_qp, u32temp);
 318                 max_qp = (u32)1 << (u32temp & 0x001f);
 319         }
 320
 321         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 322         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 323                         max_qp, hte_index_mask);
 324
 325         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 326
 327         max_irrq = 1 << (u32temp & 0x001f);
 328
 329         if (max_qp > max_irrq) {
 330                 max_qp = max_irrq;
 331                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 332                                 max_qp);
 333         }
 334
 335         /* there should be no reason to allocate more pds than qps */
 336         if (num_pds > max_qp)
 337                 num_pds = max_qp;
 338
 339         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 340         max_mr = (u32)8192 << (u32temp & 0x7);
 341
 342         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 343         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 344         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 345         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 346
 347         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 348         arp_table_size = 1 << u32temp;
 349
 350         adapter_size = (sizeof(struct nes_adapter) +
 351                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 352         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 353         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 354         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 355         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 356         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 357         adapter_size += sizeof(struct nes_qp **) * max_qp;
 358
 359         /* allocate a new adapter struct */
 360         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 361         if (nesadapter == NULL) {
 362                 return NULL;
 363         }
 364
 365         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 366                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 367
 368         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 369                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 370                 kfree(nesadapter);
 371                 return NULL;
 372         }
 373
 374         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 375                                 (nesadapter->mac_addr_low >> 24);
 376
 377         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 378                                  PCI_DEVICE_ID, &device_id);
 379         nesadapter->vendor_part_id = device_id;
 380
 381         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 382                                                         OneG_Mode)) {
 383                 kfree(nesadapter);
 384                 return NULL;
 385         }
 386         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 387
 388         memset(nesadapter->pft_mcast_map, 255,
 389                sizeof nesadapter->pft_mcast_map);
 390
 391         /* populate the new nesadapter */
 392         nesadapter->devfn = nesdev->pcidev->devfn;
 393         nesadapter->bus_number = nesdev->pcidev->bus->number;
 394         nesadapter->ref_count = 1;
 395         nesadapter->timer_int_req = 0xffff0000;
 396         nesadapter->OneG_Mode = OneG_Mode;
 397         nesadapter->doorbell_start = nesdev->doorbell_region;
 398
 399         /* nesadapter->tick_delta = clk_divisor; */
 400         nesadapter->hw_rev = hw_rev;
 401         nesadapter->port_count = port_count;
 402
 403         nesadapter->max_qp = max_qp;
 404         nesadapter->hte_index_mask = hte_index_mask;
 405         nesadapter->max_irrq = max_irrq;
 406         nesadapter->max_mr = max_mr;
 407         nesadapter->max_256pbl = max_256pbl - 1;
 408         nesadapter->max_4kpbl = max_4kpbl - 1;
 409         nesadapter->max_cq = max_cq;
 410         nesadapter->free_256pbl = max_256pbl - 1;
 411         nesadapter->free_4kpbl = max_4kpbl - 1;
 412         nesadapter->max_pd = num_pds;
 413         nesadapter->arp_table_size = arp_table_size;
 414
 415         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 416         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 417                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 418                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 419                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 420         } else {
 421                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 422                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 423                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 424                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 425         }
 426         /* Setup and enable the periodic timer */
 427         if (nesadapter->et_rx_coalesce_usecs_irq)
 428                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 429                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 430         else
 431                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 432
 433         nesadapter->base_pd = 1;
 434
 435         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 436                                        IB_DEVICE_MEM_WINDOW |
 437                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 438
 439         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 440                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 441         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 442         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 443         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 444         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 445         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 446
 447
 448         /* mark the usual suspect QPs, MR and CQs as in use */
 449         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 450                 set_bit(u32temp, nesadapter->allocated_qps);
 451                 set_bit(u32temp, nesadapter->allocated_cqs);
 452         }
 453         set_bit(0, nesadapter->allocated_mrs);
 454
 455         for (u32temp = 0; u32temp < 20; u32temp++)
 456                 set_bit(u32temp, nesadapter->allocated_pds);
 457         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 458
 459         max_rq_wrs = ((u32temp >> 8) & 3);
 460         switch (max_rq_wrs) {
 461                 case 0:
 462                         max_rq_wrs = 4;
 463                         break;
 464                 case 1:
 465                         max_rq_wrs = 16;
 466                         break;
 467                 case 2:
 468                         max_rq_wrs = 32;
 469                         break;
 470                 case 3:
 471                         max_rq_wrs = 512;
 472                         break;
 473         }
 474
 475         max_sq_wrs = (u32temp & 3);
 476         switch (max_sq_wrs) {
 477                 case 0:
 478                         max_sq_wrs = 4;
 479                         break;
 480                 case 1:
 481                         max_sq_wrs = 16;
 482                         break;
 483                 case 2:
 484                         max_sq_wrs = 32;
 485                         break;
 486                 case 3:
 487                         max_sq_wrs = 512;
 488                         break;
 489         }
 490         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 491         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 492
 493         nesadapter->max_sge = 4;
 494         nesadapter->max_cqe = 32766;
 495
 496         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 497                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 498                 kfree(nesadapter);
 499                 return NULL;
 500         }
 501
 502         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 503         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 504                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 505
 506         /* setup port configuration */
 507         if (nesadapter->port_count == 1) {
 508                 nesadapter->log_port = 0x00000000;
 509                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 510                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 511                 else
 512                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 513         } else {
 514                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 515                         nesadapter->log_port = 0x000000D8;
 516                 } else {
 517                         if (nesadapter->port_count == 2)
 518                                 nesadapter->log_port = 0x00000044;
 519                         else
 520                                 nesadapter->log_port = 0x000000e4;
 521                 }
 522                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 523         }
 524
 525         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 526                                                 nesadapter->log_port);
 527         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 528                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 529
 530         spin_lock_init(&nesadapter->resource_lock);
 531         spin_lock_init(&nesadapter->phy_lock);
 532         spin_lock_init(&nesadapter->pbl_lock);
 533         spin_lock_init(&nesadapter->periodic_timer_lock);
 534
 535         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 536         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 537         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 538         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 539
 540         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 541                 u32 pcs_control_status0, pcs_control_status1;
 542                 u32 reset_value;
 543                 u32 i = 0;
 544                 u32 int_cnt = 0;
 545                 u32 ext_cnt = 0;
 546                 unsigned long flags;
 547                 u32 j = 0;
 548
 549                 pcs_control_status0 = nes_read_indexed(nesdev,
 550                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 551                 pcs_control_status1 = nes_read_indexed(nesdev,
 552                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 553
 554                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 555                         pcs_control_status0 = nes_read_indexed(nesdev,
 556                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 557                         pcs_control_status1 = nes_read_indexed(nesdev,
 558                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 559                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 560                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 561                                 int_cnt++;
 562                         msleep(1);
 563                 }
 564                 if (int_cnt > 1) {
 565                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 566                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 567                         mh_detected++;
 568                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 569                         reset_value |= 0x0000003d;
 570                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 571
 572                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 573                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 574                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 575
 576                         pcs_control_status0 = nes_read_indexed(nesdev,
 577                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 578                         pcs_control_status1 = nes_read_indexed(nesdev,
 579                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 580
 581                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 582                                 pcs_control_status0 = nes_read_indexed(nesdev,
 583                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 584                                 pcs_control_status1 = nes_read_indexed(nesdev,
 585                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 586                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 587                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 588                                         if (++ext_cnt > int_cnt) {
 589                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 590                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 591                                                                 0x0000F088);
 592                                                 mh_detected++;
 593                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 594                                                 reset_value |= 0x0000003d;
 595                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 596
 597                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 598                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 599                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 600                                                 break;
 601                                         }
 602                                 }
 603                                 msleep(1);
 604                         }
 605                 }
 606         }
 607
 608         if (nesadapter->hw_rev == NE020_REV) {
 609                 init_timer(&nesadapter->mh_timer);
 610                 nesadapter->mh_timer.function = nes_mh_fix;
 611                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 612                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 613                 add_timer(&nesadapter->mh_timer);
 614         } else {
 615                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 616         }
 617
 618         init_timer(&nesadapter->lc_timer);
 619         nesadapter->lc_timer.function = nes_clc;
 620         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 621         nesadapter->lc_timer.data = (unsigned long)nesdev;
 622         add_timer(&nesadapter->lc_timer);
 623
 624         list_add_tail(&nesadapter->list, &nes_adapter_list);
 625
 626         for (func_index = 0; func_index < 8; func_index++) {
 627                 pci_bus_read_config_word(nesdev->pcidev->bus,
 628                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 629                                         func_index), 0, &vendor_id);
 630                 if (vendor_id == 0xffff)
 631                         break;
 632         }
 633         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 634                 func_index, pci_name(nesdev->pcidev));
 635         nesadapter->adapter_fcn_count = func_index;
 636
 637         return nesadapter;
 638 }
 639
 640
 641 /**
 642  * nes_reset_adapter_ne020
 643  */
 644 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 645 {
 646         u32 port_count;
 647         u32 u32temp;
 648         u32 i;
 649
 650         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 651         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 652         /* TODO: assuming that both SERDES are set the same for now */
 653         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 654         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 655                         u32temp, port_count);
 656         if (*OneG_Mode)
 657                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 658         u32temp &= 0xff00ffc0;
 659         switch (port_count) {
 660                 case 1:
 661                         u32temp |= 0x00ee0000;
 662                         break;
 663                 case 2:
 664                         u32temp |= 0x00cc0000;
 665                         break;
 666                 case 4:
 667                         u32temp |= 0x00000000;
 668                         break;
 669                 default:
 670                         return 0;
 671                         break;
 672         }
 673
 674         /* check and do full reset if needed */
 675         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 676                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 677                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 678
 679                 i = 0;
 680                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 681                         mdelay(1);
 682                 if (i > 10000) {
 683                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 684                         return 0;
 685                 }
 686
 687                 i = 0;
 688                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 689                         mdelay(1);
 690                 if (i > 10000) {
 691                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 692                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 693                         return 0;
 694                 }
 695         }
 696
 697         /* port reset */
 698         switch (port_count) {
 699                 case 1:
 700                         u32temp |= 0x00ee0010;
 701                         break;
 702                 case 2:
 703                         u32temp |= 0x00cc0030;
 704                         break;
 705                 case 4:
 706                         u32temp |= 0x00000030;
 707                         break;
 708         }
 709
 710         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 711         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 712
 713         i = 0;
 714         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 715                 mdelay(1);
 716         if (i > 10000) {
 717                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 718                 return 0;
 719         }
 720
 721         /* serdes 0 */
 722         i = 0;
 723         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 724                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 725                 mdelay(1);
 726         if (i > 5000) {
 727                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 728                 return 0;
 729         }
 730
 731         /* serdes 1 */
 732         if (port_count > 1) {
 733                 i = 0;
 734                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 735                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 736                         mdelay(1);
 737                 if (i > 5000) {
 738                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 739                         return 0;
 740                 }
 741         }
 742
 743         return port_count;
 744 }
 745
 746
 747 /**
 748  * nes_init_serdes
 749  */
 750 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 751                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 752 {
 753         int i;
 754         u32 u32temp;
 755         u32 sds;
 756
 757         if (hw_rev != NE020_REV) {
 758                 /* init serdes 0 */
 759                 switch (nesadapter->phy_type[0]) {
 760                 case NES_PHY_TYPE_CX4:
 761                         if (wide_ppm_offset)
 762                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 763                         else
 764                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 765                         break;
 766                 case NES_PHY_TYPE_KR:
 767                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 768                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 769                         break;
 770                 case NES_PHY_TYPE_PUMA_1G:
 771                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 772                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 773                         sds |= 0x00000100;
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 775                         break;
 776                 default:
 777                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 778                         break;
 779                 }
 780
 781                 if (!OneG_Mode)
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 783
 784                 if (port_count < 2)
 785                         return 0;
 786
 787                 /* init serdes 1 */
 788                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 789                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 790
 791                 switch (nesadapter->phy_type[1]) {
 792                 case NES_PHY_TYPE_ARGUS:
 793                 case NES_PHY_TYPE_SFP_D:
 794                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 795                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 796                         break;
 797                 case NES_PHY_TYPE_CX4:
 798                         if (wide_ppm_offset)
 799                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 800                         break;
 801                 case NES_PHY_TYPE_KR:
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 803                         break;
 804                 case NES_PHY_TYPE_PUMA_1G:
 805                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 806                         sds |= 0x000000100;
 807                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 808                 }
 809                 if (!OneG_Mode) {
 810                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 811                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 812                         sds &= 0xFFFFFFBF;
 813                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 814                 }
 815         } else {
 816                 /* init serdes 0 */
 817                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 818                 i = 0;
 819                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 820                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 821                         mdelay(1);
 822                 if (i > 5000) {
 823                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 824                         return 1;
 825                 }
 826                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 827                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 828                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 830                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 831                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 832                 if (OneG_Mode)
 833                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 834                 else
 835                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 836
 837                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 838                 if (port_count > 1) {
 839                         /* init serdes 1 */
 840                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 841                         i = 0;
 842                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 843                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 844                                 mdelay(1);
 845                         if (i > 5000) {
 846                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 847                                 /* return 1; */
 848                         }
 849                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 850                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 851                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 852                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 853                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 854                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 855                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 856                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 857                 }
 858         }
 859         return 0;
 860 }
 861
 862
 863 /**
 864  * nes_init_csr_ne020
 865  * Initialize registers for ne020 hardware
 866  */
 867 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 868 {
 869         u32 u32temp;
 870
 871         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 872
 873         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 874         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 875         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 876         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 877         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 878         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 879         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 880         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 881
 882         /* TODO: move these MAC register settings to NIC bringup */
 883         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 884         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 885         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 886         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 887         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 888         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 889         if (port_count > 1) {
 890                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 891                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 893                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 894                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 895                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 896                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 897         }
 898         if (port_count > 2) {
 899                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 900                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 902                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 903                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 904                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 905                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 906
 907                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 908                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 909                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 910                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 911                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 912                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 913                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 914         }
 915
 916         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 917         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 918         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 919                                                          0x00000001);
 920         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 921         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 922         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 923         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 924         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 925
 926         /* TODO: move this to code, get from EEPROM */
 927         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 928         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 929         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 930
 931         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 932         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 933
 934         if (hw_rev != NE020_REV) {
 935                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 936                 u32temp |= 0x80000000;
 937                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 938                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 939                 u32temp &= 0x7fffffff;
 940                 u32temp |= 0x7fff0010;
 941                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 942                 if (port_count > 1) {
 943                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 944                         u32temp &= 0x7fffffff;
 945                         u32temp |= 0x7fff0010;
 946                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 947                 }
 948         }
 949 }
 950
 951
 952 /**
 953  * nes_destroy_adapter - destroy the adapter structure
 954  */
 955 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 956 {
 957         struct nes_adapter *tmp_adapter;
 958
 959         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 960                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 961                                 tmp_adapter);
 962         }
 963
 964         nesadapter->ref_count--;
 965         if (!nesadapter->ref_count) {
 966                 if (nesadapter->hw_rev == NE020_REV) {
 967                         del_timer(&nesadapter->mh_timer);
 968                 }
 969                 del_timer(&nesadapter->lc_timer);
 970
 971                 list_del(&nesadapter->list);
 972                 kfree(nesadapter);
 973         }
 974 }
 975
 976
 977 /**
 978  * nes_init_cqp
 979  */
 980 int nes_init_cqp(struct nes_device *nesdev)
 981 {
 982         struct nes_adapter *nesadapter = nesdev->nesadapter;
 983         struct nes_hw_cqp_qp_context *cqp_qp_context;
 984         struct nes_hw_cqp_wqe *cqp_wqe;
 985         struct nes_hw_ceq *ceq;
 986         struct nes_hw_ceq *nic_ceq;
 987         struct nes_hw_aeq *aeq;
 988         void *vmem;
 989         dma_addr_t pmem;
 990         u32 count=0;
 991         u32 cqp_head;
 992         u64 u64temp;
 993         u32 u32temp;
 994
 995         /* allocate CQP memory */
 996         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 997         /* SQ is 512 byte aligned, others are 256 byte aligned */
 998         nesdev->cqp_mem_size = 512 +
 999                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
1000                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
1001                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
1002                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
1003                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
1004                         sizeof(struct nes_hw_cqp_qp_context);
1005
1006         nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev,
1007                                                   nesdev->cqp_mem_size,
1008                                                   &nesdev->cqp_pbase);
1009         if (!nesdev->cqp_vbase) {
1010                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1011                 return -ENOMEM;
1012         }
1013
1014         /* Allocate a twice the number of CQP requests as the SQ size */
1015         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1016                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1017         if (nesdev->nes_cqp_requests == NULL) {
1018                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1019                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1020                                 nesdev->cqp.sq_pbase);
1021                 return -ENOMEM;
1022         }
1023
1024         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1025                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1026
1027         spin_lock_init(&nesdev->cqp.lock);
1028         init_waitqueue_head(&nesdev->cqp.waitq);
1029
1030         /* Setup Various Structures */
1031         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1032                         ~(unsigned long)(512 - 1));
1033         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1034                         ~(unsigned long long)(512 - 1));
1035
1036         nesdev->cqp.sq_vbase = vmem;
1037         nesdev->cqp.sq_pbase = pmem;
1038         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1039         nesdev->cqp.sq_head = 0;
1040         nesdev->cqp.sq_tail = 0;
1041         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1042
1043         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1044         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1045
1046         nesdev->ccq.cq_vbase = vmem;
1047         nesdev->ccq.cq_pbase = pmem;
1048         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1049         nesdev->ccq.cq_head = 0;
1050         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1051         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1052
1053         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1054         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1055
1056         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1057         ceq = &nesadapter->ceq[nesdev->ceq_index];
1058         ceq->ceq_vbase = vmem;
1059         ceq->ceq_pbase = pmem;
1060         ceq->ceq_size = NES_CCEQ_SIZE;
1061         ceq->ceq_head = 0;
1062
1063         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1064         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1065
1066         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1067         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1068         nic_ceq->ceq_vbase = vmem;
1069         nic_ceq->ceq_pbase = pmem;
1070         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1071         nic_ceq->ceq_head = 0;
1072
1073         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1074         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1075
1076         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1077         aeq->aeq_vbase = vmem;
1078         aeq->aeq_pbase = pmem;
1079         aeq->aeq_size = nesadapter->max_qp;
1080         aeq->aeq_head = 0;
1081
1082         /* Setup QP Context */
1083         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1084         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1085
1086         cqp_qp_context = vmem;
1087         cqp_qp_context->context_words[0] =
1088                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1089         cqp_qp_context->context_words[1] = 0;
1090         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1091         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1092
1093
1094         /* Write the address to Create CQP */
1095         if ((sizeof(dma_addr_t) > 4)) {
1096                 nes_write_indexed(nesdev,
1097                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1098                                 ((u64)pmem) >> 32);
1099         } else {
1100                 nes_write_indexed(nesdev,
1101                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1102         }
1103         nes_write_indexed(nesdev,
1104                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1105                         (u32)pmem);
1106
1107         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1108         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1109
1110         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1111                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1112                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1113         }
1114
1115         /* Write Create CCQ WQE */
1116         cqp_head = nesdev->cqp.sq_head++;
1117         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1118         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1119         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1120                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1121                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1122         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1123                             (nesdev->ccq.cq_number |
1124                              ((u32)nesdev->ceq_index << 16)));
1125         u64temp = (u64)nesdev->ccq.cq_pbase;
1126         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1127         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1128         u64temp = (unsigned long)&nesdev->ccq;
1129         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1130                         cpu_to_le32((u32)(u64temp >> 1));
1131         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1132                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1133         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1134
1135         /* Write Create CEQ WQE */
1136         cqp_head = nesdev->cqp.sq_head++;
1137         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1138         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1139         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1140                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1141         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1142         u64temp = (u64)ceq->ceq_pbase;
1143         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1144
1145         /* Write Create AEQ WQE */
1146         cqp_head = nesdev->cqp.sq_head++;
1147         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1148         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1149         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1150                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1151         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1152         u64temp = (u64)aeq->aeq_pbase;
1153         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1154
1155         /* Write Create NIC CEQ WQE */
1156         cqp_head = nesdev->cqp.sq_head++;
1157         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1158         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1159         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1160                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1161         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1162         u64temp = (u64)nic_ceq->ceq_pbase;
1163         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1164
1165         /* Poll until CCQP done */
1166         count = 0;
1167         do {
1168                 if (count++ > 1000) {
1169                         printk(KERN_ERR PFX "Error creating CQP\n");
1170                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1171                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1172                         return -1;
1173                 }
1174                 udelay(10);
1175         } while (!(nes_read_indexed(nesdev,
1176                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1177
1178         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1179                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1180
1181         u32temp = 0x04800000;
1182         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1183
1184         /* wait for the CCQ, CEQ, and AEQ to get created */
1185         count = 0;
1186         do {
1187                 if (count++ > 1000) {
1188                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1189                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1190                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1191                         return -1;
1192                 }
1193                 udelay(10);
1194         } while (((nes_read_indexed(nesdev,
1195                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1196
1197         /* dump the QP status value */
1198         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1199                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1200
1201         nesdev->cqp.sq_tail++;
1202
1203         return 0;
1204 }
1205
1206
1207 /**
1208  * nes_destroy_cqp
1209  */
1210 int nes_destroy_cqp(struct nes_device *nesdev)
1211 {
1212         struct nes_hw_cqp_wqe *cqp_wqe;
1213         u32 count = 0;
1214         u32 cqp_head;
1215         unsigned long flags;
1216
1217         do {
1218                 if (count++ > 1000)
1219                         break;
1220                 udelay(10);
1221         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1222
1223         /* Reset CCQ */
1224         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1225                         nesdev->ccq.cq_number);
1226
1227         /* Disable device interrupts */
1228         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1229
1230         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1231
1232         /* Destroy the AEQ */
1233         cqp_head = nesdev->cqp.sq_head++;
1234         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1235         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1236         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1237                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1238         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1239
1240         /* Destroy the NIC CEQ */
1241         cqp_head = nesdev->cqp.sq_head++;
1242         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1243         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1244         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1245                         ((u32)nesdev->nic_ceq_index << 8));
1246
1247         /* Destroy the CEQ */
1248         cqp_head = nesdev->cqp.sq_head++;
1249         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1250         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1251         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1252                         (nesdev->ceq_index << 8));
1253
1254         /* Destroy the CCQ */
1255         cqp_head = nesdev->cqp.sq_head++;
1256         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1257         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1258         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1259         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1260                         ((u32)nesdev->ceq_index << 16));
1261
1262         /* Destroy CQP */
1263         cqp_head = nesdev->cqp.sq_head++;
1264         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1265         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1266         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1267                         NES_CQP_QP_TYPE_CQP);
1268         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1269
1270         barrier();
1271         /* Ring doorbell (5 WQEs) */
1272         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1273
1274         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1275
1276         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1277         count = 0;
1278         do {
1279                 if (count++ > 1000) {
1280                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1281                                         PCI_FUNC(nesdev->pcidev->devfn));
1282                         break;
1283                 }
1284                 udelay(10);
1285         } while (((nes_read_indexed(nesdev,
1286                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1287
1288         /* dump the QP status value */
1289         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1290                         PCI_FUNC(nesdev->pcidev->devfn),
1291                         nes_read_indexed(nesdev,
1292                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1293
1294         kfree(nesdev->nes_cqp_requests);
1295
1296         /* Free the control structures */
1297         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1298                         nesdev->cqp.sq_pbase);
1299
1300         return 0;
1301 }
1302
1303
1304 /**
1305  * nes_init_1g_phy
1306  */
1307 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1308 {
1309         u32 counter = 0;
1310         u16 phy_data;
1311         int ret = 0;
1312
1313         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1314         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1315
1316         /* Reset the PHY */
1317         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1318         udelay(100);
1319         counter = 0;
1320         do {
1321                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1322                 if (counter++ > 100) {
1323                         ret = -1;
1324                         break;
1325                 }
1326         } while (phy_data & 0x8000);
1327
1328         /* Setting no phy loopback */
1329         phy_data &= 0xbfff;
1330         phy_data |= 0x1140;
1331         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1332         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1333         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1334         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1335
1336         /* Setting the interrupt mask */
1337         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1338         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1339         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1340
1341         /* turning on flow control */
1342         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1343         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1344         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1345
1346         /* Clear Half duplex */
1347         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1348         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1349         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1350
1351         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1352         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1353
1354         return ret;
1355 }
1356
1357
1358 /**
1359  * nes_init_2025_phy
1360  */
1361 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1362 {
1363         u32 temp_phy_data = 0;
1364         u32 temp_phy_data2 = 0;
1365         u32 counter = 0;
1366         u32 sds;
1367         u32 mac_index = nesdev->mac_index;
1368         int ret = 0;
1369         unsigned int first_attempt = 1;
1370
1371         /* Check firmware heartbeat */
1372         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1373         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1374         udelay(1500);
1375         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1376         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1377
1378         if (temp_phy_data != temp_phy_data2) {
1379                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1380                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1381                 if ((temp_phy_data & 0xff) > 0x20)
1382                         return 0;
1383                 printk(PFX "Reinitialize external PHY\n");
1384         }
1385
1386         /* no heartbeat, configure the PHY */
1387         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1388         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1389         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1390         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1391
1392         switch (phy_type) {
1393         case NES_PHY_TYPE_ARGUS:
1394                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1399                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1400                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1401
1402                 /* setup LEDs */
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1406                 break;
1407
1408         case NES_PHY_TYPE_SFP_D:
1409                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1413                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1414                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1416
1417                 /* setup LEDs */
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1421                 break;
1422
1423         case NES_PHY_TYPE_KR:
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1431
1432                 /* setup LEDs */
1433                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1434                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1435                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1436
1437                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1438                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1439                 break;
1440         }
1441
1442         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1443
1444         /* Bring PHY out of reset */
1445         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1446
1447         /* Check for heartbeat */
1448         counter = 0;
1449         mdelay(690);
1450         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1451         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1452         do {
1453                 if (counter++ > 150) {
1454                         printk(PFX "No PHY heartbeat\n");
1455                         break;
1456                 }
1457                 mdelay(1);
1458                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1459                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1460         } while ((temp_phy_data2 == temp_phy_data));
1461
1462         /* wait for tracking */
1463         counter = 0;
1464         do {
1465                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1466                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1467                 if (counter++ > 300) {
1468                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1469                                 first_attempt = 0;
1470                                 counter = 0;
1471                                 /* reset AMCC PHY and try again */
1472                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1473                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1474                                 continue;
1475                         } else {
1476                                 ret = 1;
1477                                 break;
1478                         }
1479                 }
1480                 mdelay(10);
1481         } while ((temp_phy_data & 0xff) < 0x30);
1482
1483         /* setup signal integrity */
1484         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1485         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1486         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1487         if (phy_type == NES_PHY_TYPE_KR) {
1488                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1489         } else {
1490                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1491                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1492         }
1493
1494         /* reset serdes */
1495         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1496         sds |= 0x1;
1497         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1498         sds &= 0xfffffffe;
1499         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1500
1501         counter = 0;
1502         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1503                         && (counter++ < 5000))
1504                 ;
1505
1506         return ret;
1507 }
1508
1509
1510 /**
1511  * nes_init_phy
1512  */
1513 int nes_init_phy(struct nes_device *nesdev)
1514 {
1515         struct nes_adapter *nesadapter = nesdev->nesadapter;
1516         u32 mac_index = nesdev->mac_index;
1517         u32 tx_config = 0;
1518         unsigned long flags;
1519         u8  phy_type = nesadapter->phy_type[mac_index];
1520         u8  phy_index = nesadapter->phy_index[mac_index];
1521         int ret = 0;
1522
1523         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1524         if (phy_type == NES_PHY_TYPE_1G) {
1525                 /* setup 1G MDIO operation */
1526                 tx_config &= 0xFFFFFFE3;
1527                 tx_config |= 0x04;
1528         } else {
1529                 /* setup 10G MDIO operation */
1530                 tx_config &= 0xFFFFFFE3;
1531                 tx_config |= 0x1D;
1532         }
1533         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1534
1535         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1536
1537         switch (phy_type) {
1538         case NES_PHY_TYPE_1G:
1539                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1540                 break;
1541         case NES_PHY_TYPE_ARGUS:
1542         case NES_PHY_TYPE_SFP_D:
1543         case NES_PHY_TYPE_KR:
1544                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1545                 break;
1546         }
1547
1548         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1549
1550         return ret;
1551 }
1552
1553
1554 /**
1555  * nes_replenish_nic_rq
1556  */
1557 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1558 {
1559         unsigned long flags;
1560         dma_addr_t bus_address;
1561         struct sk_buff *skb;
1562         struct nes_hw_nic_rq_wqe *nic_rqe;
1563         struct nes_hw_nic *nesnic;
1564         struct nes_device *nesdev;
1565         struct nes_rskb_cb *cb;
1566         u32 rx_wqes_posted = 0;
1567
1568         nesnic = &nesvnic->nic;
1569         nesdev = nesvnic->nesdev;
1570         spin_lock_irqsave(&nesnic->rq_lock, flags);
1571         if (nesnic->replenishing_rq !=0) {
1572                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1573                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1574                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1575                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1576                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1577                         add_timer(&nesvnic->rq_wqes_timer);
1578                 } else
1579                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1580                 return;
1581         }
1582         nesnic->replenishing_rq = 1;
1583         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1584         do {
1585                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1586                 if (skb) {
1587                         skb->dev = nesvnic->netdev;
1588
1589                         bus_address = pci_map_single(nesdev->pcidev,
1590                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1591                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1592                         cb->busaddr = bus_address;
1593                         cb->maplen = nesvnic->max_frame_size;
1594
1595                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1596                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1597                                         cpu_to_le32(nesvnic->max_frame_size);
1598                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1599                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1600                                         cpu_to_le32((u32)bus_address);
1601                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1602                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1603                         nesnic->rx_skb[nesnic->rq_head] = skb;
1604                         nesnic->rq_head++;
1605                         nesnic->rq_head &= nesnic->rq_size - 1;
1606                         atomic_dec(&nesvnic->rx_skbs_needed);
1607                         barrier();
1608                         if (++rx_wqes_posted == 255) {
1609                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1610                                 rx_wqes_posted = 0;
1611                         }
1612                 } else {
1613                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1614                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1615                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1616                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1617                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1618                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1619                                 add_timer(&nesvnic->rq_wqes_timer);
1620                         } else
1621                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1622                         break;
1623                 }
1624         } while (atomic_read(&nesvnic->rx_skbs_needed));
1625         barrier();
1626         if (rx_wqes_posted)
1627                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1628         nesnic->replenishing_rq = 0;
1629 }
1630
1631
1632 /**
1633  * nes_rq_wqes_timeout
1634  */
1635 static void nes_rq_wqes_timeout(unsigned long parm)
1636 {
1637         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1638         printk("%s: Timer fired.\n", __func__);
1639         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1640         if (atomic_read(&nesvnic->rx_skbs_needed))
1641                 nes_replenish_nic_rq(nesvnic);
1642 }
1643
1644
1645 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1646                                void **tcph, u64 *hdr_flags, void *priv)
1647 {
1648         unsigned int ip_len;
1649         struct iphdr *iph;
1650         skb_reset_network_header(skb);
1651         iph = ip_hdr(skb);
1652         if (iph->protocol != IPPROTO_TCP)
1653                 return -1;
1654         ip_len = ip_hdrlen(skb);
1655         skb_set_transport_header(skb, ip_len);
1656         *tcph = tcp_hdr(skb);
1657
1658         *hdr_flags = LRO_IPV4 | LRO_TCP;
1659         *iphdr = iph;
1660         return 0;
1661 }
1662
1663
1664 /**
1665  * nes_init_nic_qp
1666  */
1667 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1668 {
1669         struct nes_hw_cqp_wqe *cqp_wqe;
1670         struct nes_hw_nic_sq_wqe *nic_sqe;
1671         struct nes_hw_nic_qp_context *nic_context;
1672         struct sk_buff *skb;
1673         struct nes_hw_nic_rq_wqe *nic_rqe;
1674         struct nes_vnic *nesvnic = netdev_priv(netdev);
1675         unsigned long flags;
1676         void *vmem;
1677         dma_addr_t pmem;
1678         u64 u64temp;
1679         int ret;
1680         u32 cqp_head;
1681         u32 counter;
1682         u32 wqe_count;
1683         struct nes_rskb_cb *cb;
1684         u8 jumbomode=0;
1685
1686         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1687         nesvnic->nic_mem_size = 256 +
1688                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1689                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1690                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1691                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1692                         sizeof(struct nes_hw_nic_qp_context);
1693
1694         nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev,
1695                                                    nesvnic->nic_mem_size,
1696                                                    &nesvnic->nic_pbase);
1697         if (!nesvnic->nic_vbase) {
1698                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1699                 return -ENOMEM;
1700         }
1701         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1702                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1703
1704         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1705                         ~(unsigned long)(256 - 1));
1706         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1707                         ~(unsigned long long)(256 - 1));
1708
1709         /* Setup the first Fragment buffers */
1710         nesvnic->nic.first_frag_vbase = vmem;
1711
1712         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1713                 nesvnic->nic.frag_paddr[counter] = pmem;
1714                 pmem += sizeof(struct nes_first_frag);
1715         }
1716
1717         /* setup the SQ */
1718         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1719
1720         nesvnic->nic.sq_vbase = (void *)vmem;
1721         nesvnic->nic.sq_pbase = pmem;
1722         nesvnic->nic.sq_head = 0;
1723         nesvnic->nic.sq_tail = 0;
1724         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1725         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1726                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1727                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1728                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1729                                 NES_NIC_SQ_WQE_COMPLETION);
1730                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1731                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1732                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1733                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1734                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1735                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1736         }
1737
1738         nesvnic->get_cqp_request = nes_get_cqp_request;
1739         nesvnic->post_cqp_request = nes_post_cqp_request;
1740         nesvnic->mcrq_mcast_filter = NULL;
1741
1742         spin_lock_init(&nesvnic->nic.rq_lock);
1743
1744         /* setup the RQ */
1745         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1746         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1747
1748
1749         nesvnic->nic.rq_vbase = vmem;
1750         nesvnic->nic.rq_pbase = pmem;
1751         nesvnic->nic.rq_head = 0;
1752         nesvnic->nic.rq_tail = 0;
1753         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1754
1755         /* setup the CQ */
1756         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1757         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1758
1759         if (nesdev->nesadapter->netdev_count > 2)
1760                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1761         else
1762                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1763
1764         nesvnic->nic_cq.cq_vbase = vmem;
1765         nesvnic->nic_cq.cq_pbase = pmem;
1766         nesvnic->nic_cq.cq_head = 0;
1767         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1768
1769         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1770
1771         /* Send CreateCQ request to CQP */
1772         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1773         cqp_head = nesdev->cqp.sq_head;
1774
1775         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1776         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1777
1778         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1779                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1780                         ((u32)nesvnic->nic_cq.cq_size << 16));
1781         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1782                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1783         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1784         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1785         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1786         u64temp = (unsigned long)&nesvnic->nic_cq;
1787         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1788         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1789                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1790         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1791         if (++cqp_head >= nesdev->cqp.sq_size)
1792                 cqp_head = 0;
1793         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1794         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1795
1796         /* Send CreateQP request to CQP */
1797         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1798         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1799                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1800                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1801         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1802                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1803                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1804         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1805                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1806         }
1807
1808         u64temp = (u64)nesvnic->nic.sq_pbase;
1809         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1810         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1811         u64temp = (u64)nesvnic->nic.rq_pbase;
1812         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1813         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1814
1815         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1816                         NES_CQP_QP_TYPE_NIC);
1817         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1818         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1819                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1820         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1821
1822         if (++cqp_head >= nesdev->cqp.sq_size)
1823                 cqp_head = 0;
1824         nesdev->cqp.sq_head = cqp_head;
1825
1826         barrier();
1827
1828         /* Ring doorbell (2 WQEs) */
1829         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1830
1831         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1832         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1833                         nesvnic->nic.qp_id);
1834
1835         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1836                         NES_EVENT_TIMEOUT);
1837         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1838                         nesvnic->nic.qp_id, ret);
1839         if (!ret) {
1840                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1841                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1842                                 nesvnic->nic_pbase);
1843                 return -EIO;
1844         }
1845
1846         /* Populate the RQ */
1847         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1848                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1849                 if (!skb) {
1850                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1851
1852                         nes_destroy_nic_qp(nesvnic);
1853                         return -ENOMEM;
1854                 }
1855
1856                 skb->dev = netdev;
1857
1858                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1859                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1860                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1861                 cb->busaddr = pmem;
1862                 cb->maplen = nesvnic->max_frame_size;
1863
1864                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1865                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1866                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1867                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1868                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1869                 nesvnic->nic.rx_skb[counter] = skb;
1870         }
1871
1872         wqe_count = NES_NIC_WQ_SIZE - 1;
1873         nesvnic->nic.rq_head = wqe_count;
1874         barrier();
1875         do {
1876                 counter = min(wqe_count, ((u32)255));
1877                 wqe_count -= counter;
1878                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1879         } while (wqe_count);
1880         init_timer(&nesvnic->rq_wqes_timer);
1881         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1882         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1883         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1884         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1885         {
1886                 nes_nic_init_timer(nesdev);
1887                 if (netdev->mtu > 1500)
1888                         jumbomode = 1;
1889                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1890         }
1891         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1892                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1893                         nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1894                         nes_destroy_nic_qp(nesvnic);
1895                 return -ENOMEM;
1896         }
1897
1898         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1899         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1900         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1901         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1902         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1903         nesvnic->lro_mgr.dev            = netdev;
1904         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1905         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1906         return 0;
1907 }
1908
1909
1910 /**
1911  * nes_destroy_nic_qp
1912  */
1913 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1914 {
1915         u64 u64temp;
1916         dma_addr_t bus_address;
1917         struct nes_device *nesdev = nesvnic->nesdev;
1918         struct nes_hw_cqp_wqe *cqp_wqe;
1919         struct nes_hw_nic_sq_wqe *nic_sqe;
1920         __le16 *wqe_fragment_length;
1921         u16  wqe_fragment_index;
1922         u32 cqp_head;
1923         u32 wqm_cfg0;
1924         unsigned long flags;
1925         struct sk_buff *rx_skb;
1926         struct nes_rskb_cb *cb;
1927         int ret;
1928
1929         if (nesdev->nesadapter->allow_unaligned_fpdus)
1930                 nes_destroy_mgt(nesvnic);
1931
1932         /* clear wqe stall before destroying NIC QP */
1933         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1934         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1935
1936         /* Free remaining NIC receive buffers */
1937         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1938                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1939                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1940                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1941                         PCI_DMA_FROMDEVICE);
1942
1943                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1944                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1945         }
1946
1947         /* Free remaining NIC transmit buffers */
1948         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1949                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1950                 wqe_fragment_index = 1;
1951                 wqe_fragment_length = (__le16 *)
1952                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1953                 /* bump past the vlan tag */
1954                 wqe_fragment_length++;
1955                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1956                         u64temp = (u64)le32_to_cpu(
1957                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1958                                 wqe_fragment_index*2]);
1959                         u64temp += ((u64)le32_to_cpu(
1960                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1961                                 + wqe_fragment_index*2]))<<32;
1962                         bus_address = (dma_addr_t)u64temp;
1963                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1964                                         nesvnic->nic.first_frag_overflow)) {
1965                                 pci_unmap_single(nesdev->pcidev,
1966                                                 bus_address,
1967                                                 le16_to_cpu(wqe_fragment_length[
1968                                                         wqe_fragment_index++]),
1969                                                 PCI_DMA_TODEVICE);
1970                         }
1971                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1972                                 if (wqe_fragment_length[wqe_fragment_index]) {
1973                                         u64temp = le32_to_cpu(
1974                                                 nic_sqe->wqe_words[
1975                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1976                                                 wqe_fragment_index*2]);
1977                                         u64temp += ((u64)le32_to_cpu(
1978                                                 nic_sqe->wqe_words[
1979                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1980                                                 wqe_fragment_index*2]))<<32;
1981                                         bus_address = (dma_addr_t)u64temp;
1982                                         pci_unmap_page(nesdev->pcidev,
1983                                                         bus_address,
1984                                                         le16_to_cpu(
1985                                                         wqe_fragment_length[
1986                                                         wqe_fragment_index]),
1987                                                         PCI_DMA_TODEVICE);
1988                                 } else
1989                                         break;
1990                         }
1991                 }
1992                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1993                         dev_kfree_skb(
1994                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1995
1996                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1997                                         & (nesvnic->nic.sq_size - 1);
1998         }
1999
2000         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2001
2002         /* Destroy NIC QP */
2003         cqp_head = nesdev->cqp.sq_head;
2004         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2005         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2006
2007         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2008                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
2009         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2010                 nesvnic->nic.qp_id);
2011
2012         if (++cqp_head >= nesdev->cqp.sq_size)
2013                 cqp_head = 0;
2014
2015         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2016
2017         /* Destroy NIC CQ */
2018         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2019         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2020                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
2021         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2022                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
2023
2024         if (++cqp_head >= nesdev->cqp.sq_size)
2025                 cqp_head = 0;
2026
2027         nesdev->cqp.sq_head = cqp_head;
2028         barrier();
2029
2030         /* Ring doorbell (2 WQEs) */
2031         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2032
2033         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2034         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2035                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2036                         cqp_head, nesdev->cqp.sq_head,
2037                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2038
2039         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2040                         NES_EVENT_TIMEOUT);
2041
2042         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2043                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2044                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2045         if (!ret) {
2046                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2047                                 nesvnic->nic.qp_id);
2048         }
2049
2050         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2051                         nesvnic->nic_pbase);
2052
2053         /* restore old wqm_cfg0 value */
2054         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2055 }
2056
2057 /**
2058  * nes_napi_isr
2059  */
2060 int nes_napi_isr(struct nes_device *nesdev)
2061 {
2062         struct nes_adapter *nesadapter = nesdev->nesadapter;
2063         u32 int_stat;
2064
2065         if (nesdev->napi_isr_ran) {
2066                 /* interrupt status has already been read in ISR */
2067                 int_stat = nesdev->int_stat;
2068         } else {
2069                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2070                 nesdev->int_stat = int_stat;
2071                 nesdev->napi_isr_ran = 1;
2072         }
2073
2074         int_stat &= nesdev->int_req;
2075         /* iff NIC, process here, else wait for DPC */
2076         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2077                 nesdev->napi_isr_ran = 0;
2078                 nes_write32(nesdev->regs + NES_INT_STAT,
2079                         (int_stat &
2080                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2081
2082                 /* Process the CEQs */
2083                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2084
2085                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2086                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2087                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2088                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2089                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2090                                 /* Enable Periodic timer interrupts */
2091                                 nesdev->int_req |= NES_INT_TIMER;
2092                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2093                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2094                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2095                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2096                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2097                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2098                         }
2099
2100                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2101                         {
2102                                 nes_nic_init_timer(nesdev);
2103                         }
2104                         /* Enable interrupts, except CEQs */
2105                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2106                 } else {
2107                         /* Enable interrupts, make sure timer is off */
2108                         nesdev->int_req &= ~NES_INT_TIMER;
2109                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2110                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2111                 }
2112                 nesdev->deepcq_count = 0;
2113                 return 1;
2114         } else {
2115                 return 0;
2116         }
2117 }
2118
2119 static void process_critical_error(struct nes_device *nesdev)
2120 {
2121         u32 debug_error;
2122         u32 nes_idx_debug_error_masks0 = 0;
2123         u16 error_module = 0;
2124
2125         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2126         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2127                         (u16)debug_error);
2128         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2129                         0x01010000 | (debug_error & 0x0000ffff));
2130         if (crit_err_count++ > 10)
2131                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2132         error_module = (u16) (debug_error & 0x1F00) >> 8;
2133         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2134                         nes_max_critical_error_count) {
2135                 printk(KERN_ERR PFX "Masking off critical error for module "
2136                         "0x%02X\n", (u16)error_module);
2137                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2138                         NES_IDX_DEBUG_ERROR_MASKS0);
2139                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2140                         nes_idx_debug_error_masks0 | (1 << error_module));
2141         }
2142 }
2143 /**
2144  * nes_dpc
2145  */
2146 void nes_dpc(unsigned long param)
2147 {
2148         struct nes_device *nesdev = (struct nes_device *)param;
2149         struct nes_adapter *nesadapter = nesdev->nesadapter;
2150         u32 counter;
2151         u32 loop_counter = 0;
2152         u32 int_status_bit;
2153         u32 int_stat;
2154         u32 timer_stat;
2155         u32 temp_int_stat;
2156         u32 intf_int_stat;
2157         u32 processed_intf_int = 0;
2158         u16 processed_timer_int = 0;
2159         u16 completion_ints = 0;
2160         u16 timer_ints = 0;
2161
2162         /* nes_debug(NES_DBG_ISR, "\n"); */
2163
2164         do {
2165                 timer_stat = 0;
2166                 if (nesdev->napi_isr_ran) {
2167                         nesdev->napi_isr_ran = 0;
2168                         int_stat = nesdev->int_stat;
2169                 } else
2170                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2171                 if (processed_intf_int != 0)
2172                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2173                 else
2174                         int_stat &= nesdev->int_req;
2175                 if (processed_timer_int == 0) {
2176                         processed_timer_int = 1;
2177                         if (int_stat & NES_INT_TIMER) {
2178                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2179                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2180                                         int_stat &= ~NES_INT_TIMER;
2181                                 }
2182                         }
2183                 } else {
2184                         int_stat &= ~NES_INT_TIMER;
2185                 }
2186
2187                 if (int_stat) {
2188                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2189                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2190                                 /* Ack the interrupts */
2191                                 nes_write32(nesdev->regs+NES_INT_STAT,
2192                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2193                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2194                         }
2195
2196                         temp_int_stat = int_stat;
2197                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2198                                 if (int_stat & int_status_bit) {
2199                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2200                                         temp_int_stat &= ~int_status_bit;
2201                                         completion_ints = 1;
2202                                 }
2203                                 if (!(temp_int_stat & 0x0000ffff))
2204                                         break;
2205                                 int_status_bit <<= 1;
2206                         }
2207
2208                         /* Process the AEQ for this pci function */
2209                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2210                         if (int_stat & int_status_bit) {
2211                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2212                         }
2213
2214                         /* Process the MAC interrupt for this pci function */
2215                         int_status_bit = 1 << (24 + nesdev->mac_index);
2216                         if (int_stat & int_status_bit) {
2217                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2218                         }
2219
2220                         if (int_stat & NES_INT_TIMER) {
2221                                 if (timer_stat & nesdev->timer_int_req) {
2222                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2223                                                         (timer_stat & nesdev->timer_int_req) |
2224                                                         ~(nesdev->nesadapter->timer_int_req));
2225                                         timer_ints = 1;
2226                                 }
2227                         }
2228
2229                         if (int_stat & NES_INT_INTF) {
2230                                 processed_intf_int = 1;
2231                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2232                                 intf_int_stat &= nesdev->intf_int_req;
2233                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2234                                         process_critical_error(nesdev);
2235                                 }
2236                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2237                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2238                                         BUG();
2239                                 }
2240                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2241                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2242                                         BUG();
2243                                 }
2244                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2245                         }
2246
2247                         if (int_stat & NES_INT_TSW) {
2248                         }
2249                 }
2250                 /* Don't use the interface interrupt bit stay in loop */
2251                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2252                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2253         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2254
2255         if (timer_ints == 1) {
2256                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2257                         if (completion_ints == 0) {
2258                                 nesdev->timer_only_int_count++;
2259                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2260                                         nesdev->timer_only_int_count = 0;
2261                                         nesdev->int_req &= ~NES_INT_TIMER;
2262                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2263                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2264                                 } else {
2265                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2266                                 }
2267                         } else {
2268                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2269                                 {
2270                                         nes_nic_init_timer(nesdev);
2271                                 }
2272                                 nesdev->timer_only_int_count = 0;
2273                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2274                         }
2275                 } else {
2276                         nesdev->timer_only_int_count = 0;
2277                         nesdev->int_req &= ~NES_INT_TIMER;
2278                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2279                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2280                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2281                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2282                 }
2283         } else {
2284                 if ( (completion_ints == 1) &&
2285                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2286                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2287                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2288                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2289                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2290                         nesdev->timer_only_int_count = 0;
2291                         nesdev->int_req |= NES_INT_TIMER;
2292                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2293                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2294                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2295                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2296                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2297                 } else {
2298                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2299                 }
2300         }
2301         nesdev->deepcq_count = 0;
2302 }
2303
2304
2305 /**
2306  * nes_process_ceq
2307  */
2308 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2309 {
2310         u64 u64temp;
2311         struct nes_hw_cq *cq;
2312         u32 head;
2313         u32 ceq_size;
2314
2315         /* nes_debug(NES_DBG_CQ, "\n"); */
2316         head = ceq->ceq_head;
2317         ceq_size = ceq->ceq_size;
2318
2319         do {
2320                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2321                                 NES_CEQE_VALID) {
2322                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2323                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2324                         u64temp <<= 1;
2325                         cq = *((struct nes_hw_cq **)&u64temp);
2326                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2327                         barrier();
2328                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2329
2330                         /* call the event handler */
2331                         cq->ce_handler(nesdev, cq);
2332
2333                         if (++head >= ceq_size)
2334                                 head = 0;
2335                 } else {
2336                         break;
2337                 }
2338
2339         } while (1);
2340
2341         ceq->ceq_head = head;
2342 }
2343
2344
2345 /**
2346  * nes_process_aeq
2347  */
2348 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2349 {
2350         /* u64 u64temp; */
2351         u32 head;
2352         u32 aeq_size;
2353         u32 aeqe_misc;
2354         u32 aeqe_cq_id;
2355         struct nes_hw_aeqe volatile *aeqe;
2356
2357         head = aeq->aeq_head;
2358         aeq_size = aeq->aeq_size;
2359
2360         do {
2361                 aeqe = &aeq->aeq_vbase[head];
2362                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2363                         break;
2364                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2365                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2366                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2367                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2368                                 /* dealing with an accelerated QP related AE */
2369                                 /*
2370                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2371                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2372                                  */
2373                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2374                         } else {
2375                                 /* TODO: dealing with a CQP related AE */
2376                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2377                                                 (u16)(aeqe_misc >> 16));
2378                         }
2379                 }
2380
2381                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2382
2383                 if (++head >= aeq_size)
2384                         head = 0;
2385
2386                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2387         }
2388         while (1);
2389         aeq->aeq_head = head;
2390 }
2391
2392 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2393 {
2394         struct nes_adapter *nesadapter = nesdev->nesadapter;
2395         u32 reset_value;
2396         u32 i=0;
2397         u32 u32temp;
2398
2399         if (nesadapter->hw_rev == NE020_REV) {
2400                 return;
2401         }
2402         mh_detected++;
2403
2404         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2405
2406         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2407                 reset_value |= 0x0000001d;
2408         else
2409                 reset_value |= 0x0000002d;
2410
2411         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2412                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2413                         nesadapter->link_interrupt_count[0] = 0;
2414                         nesadapter->link_interrupt_count[1] = 0;
2415                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2416                         if (0x00000040 & u32temp)
2417                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2418                         else
2419                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2420
2421                         reset_value |= 0x0000003d;
2422                 }
2423                 nesadapter->link_interrupt_count[mac_index] = 0;
2424         }
2425
2426         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2427
2428         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2429                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2430
2431         if (0x0000003d == (reset_value & 0x0000003d)) {
2432                 u32 pcs_control_status0, pcs_control_status1;
2433
2434                 for (i = 0; i < 10; i++) {
2435                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2436                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2437                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2438                              && (pcs_control_status0 & 0x00100000))
2439                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2440                                 && (pcs_control_status1 & 0x00100000)))
2441                                 continue;
2442                         else
2443                                 break;
2444                 }
2445                 if (10 == i) {
2446                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2447                         if (0x00000040 & u32temp)
2448                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2449                         else
2450                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2451
2452                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2453
2454                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2455                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2456                 }
2457         }
2458 }
2459
2460 /**
2461  * nes_process_mac_intr
2462  */
2463 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2464 {
2465         unsigned long flags;
2466         u32 pcs_control_status;
2467         struct nes_adapter *nesadapter = nesdev->nesadapter;
2468         struct nes_vnic *nesvnic;
2469         u32 mac_status;
2470         u32 mac_index = nesdev->mac_index;
2471         u32 u32temp;
2472         u16 phy_data;
2473         u16 temp_phy_data;
2474         u32 pcs_val  = 0x0f0f0000;
2475         u32 pcs_mask = 0x0f1f0000;
2476         u32 cdr_ctrl;
2477
2478         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2479         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2480                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2481                 return;
2482         }
2483         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2484
2485         /* ack the MAC interrupt */
2486         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2487         /* Clear the interrupt */
2488         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2489
2490         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2491
2492         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2493                 nesdev->link_status_interrupts++;
2494                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2495                         nes_reset_link(nesdev, mac_index);
2496
2497                 /* read the PHY interrupt status register */
2498                 if ((nesadapter->OneG_Mode) &&
2499                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2500                         do {
2501                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2502                                                 nesadapter->phy_index[mac_index], &phy_data);
2503                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2504                                                 nesadapter->phy_index[mac_index], phy_data);
2505                         } while (phy_data&0x8000);
2506
2507                         temp_phy_data = 0;
2508                         do {
2509                                 nes_read_1G_phy_reg(nesdev, 0x11,
2510                                                 nesadapter->phy_index[mac_index], &phy_data);
2511                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2512                                                 nesadapter->phy_index[mac_index], phy_data);
2513                                 if (temp_phy_data == phy_data)
2514                                         break;
2515                                 temp_phy_data = phy_data;
2516                         } while (1);
2517
2518                         nes_read_1G_phy_reg(nesdev, 0x1e,
2519                                         nesadapter->phy_index[mac_index], &phy_data);
2520                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2521                                         nesadapter->phy_index[mac_index], phy_data);
2522
2523                         nes_read_1G_phy_reg(nesdev, 1,
2524                                         nesadapter->phy_index[mac_index], &phy_data);
2525                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2526                                         nesadapter->phy_index[mac_index], phy_data);
2527
2528                         if (temp_phy_data & 0x1000) {
2529                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2530                                 phy_data = 4;
2531                         } else {
2532                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2533                         }
2534                 }
2535                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2536                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2537                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2538
2539                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2540                         switch (mac_index) {
2541                         case 1:
2542                         case 3:
2543                                 pcs_control_status = nes_read_indexed(nesdev,
2544                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2545                                 break;
2546                         default:
2547                                 pcs_control_status = nes_read_indexed(nesdev,
2548                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2549                                 break;
2550                         }
2551                 } else {
2552                         pcs_control_status = nes_read_indexed(nesdev,
2553                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2554                         pcs_control_status = nes_read_indexed(nesdev,
2555                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2556                 }
2557
2558                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2559                                 mac_index, pcs_control_status);
2560                 if ((nesadapter->OneG_Mode) &&
2561                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2562                         u32temp = 0x01010000;
2563                         if (nesadapter->port_count > 2) {
2564                                 u32temp |= 0x02020000;
2565                         }
2566                         if ((pcs_control_status & u32temp)!= u32temp) {
2567                                 phy_data = 0;
2568                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2569                         }
2570                 } else {
2571                         switch (nesadapter->phy_type[mac_index]) {
2572                         case NES_PHY_TYPE_ARGUS:
2573                         case NES_PHY_TYPE_SFP_D:
2574                         case NES_PHY_TYPE_KR:
2575                                 /* clear the alarms */
2576                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2577                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2578                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2579                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2580                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2581                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2582                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2583                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2584                                 /* check link status */
2585                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2586                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2587
2588                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2589                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2590                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2591                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2592
2593                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2594
2595                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2596                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2597                                 break;
2598
2599                         case NES_PHY_TYPE_PUMA_1G:
2600                                 if (mac_index < 2)
2601                                         pcs_val = pcs_mask = 0x01010000;
2602                                 else
2603                                         pcs_val = pcs_mask = 0x02020000;
2604                                 /* fall through */
2605                         default:
2606                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2607                                 break;
2608                         }
2609                 }
2610
2611                 if (phy_data & 0x0004) {
2612                         if (wide_ppm_offset &&
2613                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2614                             (nesadapter->hw_rev != NE020_REV)) {
2615                                 cdr_ctrl = nes_read_indexed(nesdev,
2616                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2617                                                             mac_index * 0x200);
2618                                 nes_write_indexed(nesdev,
2619                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2620                                                   mac_index * 0x200,
2621                                                   cdr_ctrl | 0x000F0000);
2622                         }
2623                         nesadapter->mac_link_down[mac_index] = 0;
2624                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2625                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2626                                                 nesvnic->linkup);
2627                                 if (nesvnic->linkup == 0) {
2628                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2629                                                         nesvnic->netdev->name, nesvnic->netdev);
2630                                         if (netif_queue_stopped(nesvnic->netdev))
2631                                                 netif_start_queue(nesvnic->netdev);
2632                                         nesvnic->linkup = 1;
2633                                         netif_carrier_on(nesvnic->netdev);
2634
2635                                         spin_lock(&nesvnic->port_ibevent_lock);
2636                                         if (nesvnic->of_device_registered) {
2637                                                 if (nesdev->iw_status == 0) {
2638                                                         nesdev->iw_status = 1;
2639                                                         nes_port_ibevent(nesvnic);
2640                                                 }
2641                                         }
2642                                         spin_unlock(&nesvnic->port_ibevent_lock);
2643                                 }
2644                         }
2645                 } else {
2646                         if (wide_ppm_offset &&
2647                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2648                             (nesadapter->hw_rev != NE020_REV)) {
2649                                 cdr_ctrl = nes_read_indexed(nesdev,
2650                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2651                                                             mac_index * 0x200);
2652                                 nes_write_indexed(nesdev,
2653                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2654                                                   mac_index * 0x200,
2655                                                   cdr_ctrl & 0xFFF0FFFF);
2656                         }
2657                         nesadapter->mac_link_down[mac_index] = 1;
2658                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2659                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2660                                                 nesvnic->linkup);
2661                                 if (nesvnic->linkup == 1) {
2662                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2663                                                         nesvnic->netdev->name, nesvnic->netdev);
2664                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2665                                                 netif_stop_queue(nesvnic->netdev);
2666                                         nesvnic->linkup = 0;
2667                                         netif_carrier_off(nesvnic->netdev);
2668
2669                                         spin_lock(&nesvnic->port_ibevent_lock);
2670                                         if (nesvnic->of_device_registered) {
2671                                                 if (nesdev->iw_status == 1) {
2672                                                         nesdev->iw_status = 0;
2673                                                         nes_port_ibevent(nesvnic);
2674                                                 }
2675                                         }
2676                                         spin_unlock(&nesvnic->port_ibevent_lock);
2677                                 }
2678                         }
2679                 }
2680                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2681                         nesdev->link_recheck = 1;
2682                         mod_delayed_work(system_wq, &nesdev->work,
2683                                          NES_LINK_RECHECK_DELAY);
2684                 }
2685         }
2686
2687         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2688
2689         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2690 }
2691
2692 void nes_recheck_link_status(struct work_struct *work)
2693 {
2694         unsigned long flags;
2695         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2696         struct nes_adapter *nesadapter = nesdev->nesadapter;
2697         struct nes_vnic *nesvnic;
2698         u32 mac_index = nesdev->mac_index;
2699         u16 phy_data;
2700         u16 temp_phy_data;
2701
2702         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2703
2704         /* check link status */
2705         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2706         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2707
2708         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2709         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2710         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2711         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2712
2713         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2714
2715         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2716                 __func__, phy_data,
2717                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2718
2719         if (phy_data & 0x0004) {
2720                 nesadapter->mac_link_down[mac_index] = 0;
2721                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2722                         if (nesvnic->linkup == 0) {
2723                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2724                                                 nesvnic->netdev->name, nesvnic->netdev);
2725                                 if (netif_queue_stopped(nesvnic->netdev))
2726                                         netif_start_queue(nesvnic->netdev);
2727                                 nesvnic->linkup = 1;
2728                                 netif_carrier_on(nesvnic->netdev);
2729
2730                                 spin_lock(&nesvnic->port_ibevent_lock);
2731                                 if (nesvnic->of_device_registered) {
2732                                         if (nesdev->iw_status == 0) {
2733                                                 nesdev->iw_status = 1;
2734                                                 nes_port_ibevent(nesvnic);
2735                                         }
2736                                 }
2737                                 spin_unlock(&nesvnic->port_ibevent_lock);
2738                         }
2739                 }
2740
2741         } else {
2742                 nesadapter->mac_link_down[mac_index] = 1;
2743                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2744                         if (nesvnic->linkup == 1) {
2745                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2746                                                 nesvnic->netdev->name, nesvnic->netdev);
2747                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2748                                         netif_stop_queue(nesvnic->netdev);
2749                                 nesvnic->linkup = 0;
2750                                 netif_carrier_off(nesvnic->netdev);
2751
2752                                 spin_lock(&nesvnic->port_ibevent_lock);
2753                                 if (nesvnic->of_device_registered) {
2754                                         if (nesdev->iw_status == 1) {
2755                                                 nesdev->iw_status = 0;
2756                                                 nes_port_ibevent(nesvnic);
2757                                         }
2758                                 }
2759                                 spin_unlock(&nesvnic->port_ibevent_lock);
2760                         }
2761                 }
2762         }
2763         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2764                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2765         else
2766                 nesdev->link_recheck = 0;
2767
2768         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2769 }
2770
2771
2772 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2773 {
2774         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2775
2776         napi_schedule(&nesvnic->napi);
2777 }
2778
2779
2780 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2781 * getting out of nic_ce_handler
2782 */
2783 #define MAX_RQES_TO_PROCESS     384
2784
2785 /**
2786  * nes_nic_ce_handler
2787  */
2788 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2789 {
2790         u64 u64temp;
2791         dma_addr_t bus_address;
2792         struct nes_hw_nic *nesnic;
2793         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2794         struct nes_adapter *nesadapter = nesdev->nesadapter;
2795         struct nes_hw_nic_rq_wqe *nic_rqe;
2796         struct nes_hw_nic_sq_wqe *nic_sqe;
2797         struct sk_buff *skb;
2798         struct sk_buff *rx_skb;
2799         struct nes_rskb_cb *cb;
2800         __le16 *wqe_fragment_length;
2801         u32 head;
2802         u32 cq_size;
2803         u32 rx_pkt_size;
2804         u32 cqe_count=0;
2805         u32 cqe_errv;
2806         u32 cqe_misc;
2807         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2808         u16 vlan_tag;
2809         u16 pkt_type;
2810         u16 rqes_processed = 0;
2811         u8 sq_cqes = 0;
2812         u8 nes_use_lro = 0;
2813
2814         head = cq->cq_head;
2815         cq_size = cq->cq_size;
2816         cq->cqes_pending = 1;
2817         if (nesvnic->netdev->features & NETIF_F_LRO)
2818                 nes_use_lro = 1;
2819         do {
2820                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2821                                 NES_NIC_CQE_VALID) {
2822                         nesnic = &nesvnic->nic;
2823                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2824                         if (cqe_misc & NES_NIC_CQE_SQ) {
2825                                 sq_cqes++;
2826                                 wqe_fragment_index = 1;
2827                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2828                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2829                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2830                                 /* bump past the vlan tag */
2831                                 wqe_fragment_length++;
2832                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2833                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2834                                                         wqe_fragment_index * 2]);
2835                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2836                                                         wqe_fragment_index * 2])) << 32;
2837                                         bus_address = (dma_addr_t)u64temp;
2838                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2839                                                 pci_unmap_single(nesdev->pcidev,
2840                                                                 bus_address,
2841                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2842                                                                 PCI_DMA_TODEVICE);
2843                                         }
2844                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2845                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2846                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2847                                                                                 wqe_fragment_index * 2]);
2848                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2849                                                                                 + wqe_fragment_index * 2])) <<32;
2850                                                         bus_address = (dma_addr_t)u64temp;
2851                                                         pci_unmap_page(nesdev->pcidev,
2852                                                                         bus_address,
2853                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2854                                                                         PCI_DMA_TODEVICE);
2855                                                 } else
2856                                                         break;
2857                                         }
2858                                 }
2859                                 if (skb)
2860                                         dev_kfree_skb_any(skb);
2861                                 nesnic->sq_tail++;
2862                                 nesnic->sq_tail &= nesnic->sq_size-1;
2863                                 if (sq_cqes > 128) {
2864                                         barrier();
2865                                         /* restart the queue if it had been stopped */
2866                                         if (netif_queue_stopped(nesvnic->netdev))
2867                                                 netif_wake_queue(nesvnic->netdev);
2868                                         sq_cqes = 0;
2869                                 }
2870                         } else {
2871                                 rqes_processed ++;
2872
2873                                 cq->rx_cqes_completed++;
2874                                 cq->rx_pkts_indicated++;
2875                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2876                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2877                                 /* Get the skb */
2878                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2879                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2880                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2881                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2882                                 pci_unmap_single(nesdev->pcidev, bus_address,
2883                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2884                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2885                                 cb->busaddr = 0;
2886                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2887                                 /* rx_skb->len = rx_pkt_size; */
2888                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2889                                 skb_put(rx_skb, rx_pkt_size);
2890                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2891                                 nesnic->rq_tail++;
2892                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2893
2894                                 atomic_inc(&nesvnic->rx_skbs_needed);
2895                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2896                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2897                                                         cq->cq_number | (cqe_count << 16));
2898                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2899                                         nesdev->currcq_count += cqe_count;
2900                                         cqe_count = 0;
2901                                         nes_replenish_nic_rq(nesvnic);
2902                                 }
2903                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2904                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2905                                 rx_skb->ip_summed = CHECKSUM_NONE;
2906
2907                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2908                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2909                                         if ((cqe_errv &
2910                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2911                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2912                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2913                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2914                                         } else
2915                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2916                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2917                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2918
2919                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2920                                         if ((cqe_errv &
2921                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2922                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2923                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2924                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2925                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2926                                                                   nesvnic->netdev->name); */
2927                                                 }
2928                                         } else
2929                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2930                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2931                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2932                                         }
2933                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2934                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2935
2936                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2937                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2938                                                 rx_skb = NULL;
2939                                 }
2940                                 if (rx_skb == NULL)
2941                                         goto skip_rx_indicate0;
2942
2943
2944                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2945                                         vlan_tag = (u16)(le32_to_cpu(
2946                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2947                                                         >> 16);
2948                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2949                                                         nesvnic->netdev->name, vlan_tag);
2950
2951                                         __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag);
2952                                 }
2953                                 if (nes_use_lro)
2954                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2955                                 else
2956                                         netif_receive_skb(rx_skb);
2957
2958 skip_rx_indicate0:
2959                                 ;
2960                                 /* nesvnic->netstats.rx_packets++; */
2961                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2962                         }
2963
2964                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2965                         /* Accounting... */
2966                         cqe_count++;
2967                         if (++head >= cq_size)
2968                                 head = 0;
2969                         if (cqe_count == 255) {
2970                                 /* Replenish Nic CQ */
2971                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2972                                                 cq->cq_number | (cqe_count << 16));
2973                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2974                                 nesdev->currcq_count += cqe_count;
2975                                 cqe_count = 0;
2976                         }
2977
2978                         if (cq->rx_cqes_completed >= nesvnic->budget)
2979                                 break;
2980                 } else {
2981                         cq->cqes_pending = 0;
2982                         break;
2983                 }
2984
2985         } while (1);
2986
2987         if (nes_use_lro)
2988                 lro_flush_all(&nesvnic->lro_mgr);
2989         if (sq_cqes) {
2990                 barrier();
2991                 /* restart the queue if it had been stopped */
2992                 if (netif_queue_stopped(nesvnic->netdev))
2993                         netif_wake_queue(nesvnic->netdev);
2994         }
2995         cq->cq_head = head;
2996         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2997                         cq->cq_number, cqe_count, cq->cq_head); */
2998         cq->cqe_allocs_pending = cqe_count;
2999         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
3000         {
3001                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
3002                 nesdev->currcq_count += cqe_count;
3003                 nes_nic_tune_timer(nesdev);
3004         }
3005         if (atomic_read(&nesvnic->rx_skbs_needed))
3006                 nes_replenish_nic_rq(nesvnic);
3007 }
3008
3009
3010
3011 /**
3012  * nes_cqp_ce_handler
3013  */
3014 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
3015 {
3016         u64 u64temp;
3017         unsigned long flags;
3018         struct nes_hw_cqp *cqp = NULL;
3019         struct nes_cqp_request *cqp_request;
3020         struct nes_hw_cqp_wqe *cqp_wqe;
3021         u32 head;
3022         u32 cq_size;
3023         u32 cqe_count=0;
3024         u32 error_code;
3025         u32 opcode;
3026         u32 ctx_index;
3027         /* u32 counter; */
3028
3029         head = cq->cq_head;
3030         cq_size = cq->cq_size;
3031
3032         do {
3033                 /* process the CQE */
3034                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
3035                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
3036
3037                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
3038                 if (opcode & NES_CQE_VALID) {
3039                         cqp = &nesdev->cqp;
3040
3041                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3042                         if (error_code) {
3043                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3044                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3045                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3046                                                 (u16)(error_code >> 16),
3047                                                 (u16)error_code);
3048                         }
3049
3050                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3051                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3052                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3053                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3054
3055                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3056                         if (cqp_request) {
3057                                 if (cqp_request->waiting) {
3058                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3059                                         cqp_request->major_code = (u16)(error_code >> 16);
3060                                         cqp_request->minor_code = (u16)error_code;
3061                                         barrier();
3062                                         cqp_request->request_done = 1;
3063                                         wake_up(&cqp_request->waitq);
3064                                         nes_put_cqp_request(nesdev, cqp_request);
3065                                 } else {
3066                                         if (cqp_request->callback)
3067                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3068                                         nes_free_cqp_request(nesdev, cqp_request);
3069                                 }
3070                         } else {
3071                                 wake_up(&nesdev->cqp.waitq);
3072                         }
3073
3074                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3075                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3076                         if (++cqp->sq_tail >= cqp->sq_size)
3077                                 cqp->sq_tail = 0;
3078
3079                         /* Accounting... */
3080                         cqe_count++;
3081                         if (++head >= cq_size)
3082                                 head = 0;
3083                 } else {
3084                         break;
3085                 }
3086         } while (1);
3087         cq->cq_head = head;
3088
3089         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3090         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3091                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3092                         (nesdev->cqp.sq_size - 1)) != 1)) {
3093                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3094                                 struct nes_cqp_request, list);
3095                 list_del_init(&cqp_request->list);
3096                 head = nesdev->cqp.sq_head++;
3097                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3098                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3099                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3100                 barrier();
3101
3102                 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3103                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3104                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3105                 else
3106                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3107                 cqp_wqe->wqe_words[ctx_index] =
3108                         cpu_to_le32((u32)((unsigned long)cqp_request));
3109                 cqp_wqe->wqe_words[ctx_index + 1] =
3110                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3111                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3112                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3113                 /* Ring doorbell (1 WQEs) */
3114                 barrier();
3115                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3116         }
3117         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3118
3119         /* Arm the CCQ */
3120         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3121                         cq->cq_number);
3122         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3123 }
3124
3125 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3126 {
3127         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3128                 /* skip over ethernet header */
3129                 pkt += ETH_HLEN;
3130
3131                 /* Skip over IP and TCP headers */
3132                 pkt += 4 * (pkt[0] & 0x0f);
3133                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3134         }
3135         return pkt;
3136 }
3137
3138 /* Determine if incoming error pkt is rdma layer */
3139 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3140 {
3141         u8 *pkt;
3142         u16 *mpa;
3143         u32 opcode = 0xffffffff;
3144
3145         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3146                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3147                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3148                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3149         }
3150
3151         return opcode;
3152 }
3153
3154 /* Build iWARP terminate header */
3155 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3156 {
3157         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3158         u16 ddp_seg_len;
3159         int copy_len = 0;
3160         u8 is_tagged = 0;
3161         u8 flush_code = 0;
3162         struct nes_terminate_hdr *termhdr;
3163
3164         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3165         memset(termhdr, 0, 64);
3166
3167         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3168
3169                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3170                 pkt = locate_mpa(pkt, aeq_info);
3171                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3172                 if (ddp_seg_len) {
3173                         copy_len = 2;
3174                         termhdr->hdrct = DDP_LEN_FLAG;
3175                         if (pkt[2] & 0x80) {
3176                                 is_tagged = 1;
3177                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3178                                         copy_len += TERM_DDP_LEN_TAGGED;
3179                                         termhdr->hdrct |= DDP_HDR_FLAG;
3180                                 }
3181                         } else {
3182                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3183                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3184                                         termhdr->hdrct |= DDP_HDR_FLAG;
3185                                 }
3186
3187                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3188                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3189                                                 copy_len += TERM_RDMA_LEN;
3190                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3191                                         }
3192                                 }
3193                         }
3194                 }
3195         }
3196
3197         switch (async_event_id) {
3198         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3199                 switch (iwarp_opcode(nesqp, aeq_info)) {
3200                 case IWARP_OPCODE_WRITE:
3201                         flush_code = IB_WC_LOC_PROT_ERR;
3202                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3203                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3204                         break;
3205                 default:
3206                         flush_code = IB_WC_REM_ACCESS_ERR;
3207                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3208                         termhdr->error_code = RDMAP_INV_STAG;
3209                 }
3210                 break;
3211         case NES_AEQE_AEID_AMP_INVALID_STAG:
3212                 flush_code = IB_WC_REM_ACCESS_ERR;
3213                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3214                 termhdr->error_code = RDMAP_INV_STAG;
3215                 break;
3216         case NES_AEQE_AEID_AMP_BAD_QP:
3217                 flush_code = IB_WC_LOC_QP_OP_ERR;
3218                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3219                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3220                 break;
3221         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3222         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3223                 switch (iwarp_opcode(nesqp, aeq_info)) {
3224                 case IWARP_OPCODE_SEND_INV:
3225                 case IWARP_OPCODE_SEND_SE_INV:
3226                         flush_code = IB_WC_REM_OP_ERR;
3227                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3228                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3229                         break;
3230                 default:
3231                         flush_code = IB_WC_REM_ACCESS_ERR;
3232                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3233                         termhdr->error_code = RDMAP_INV_STAG;
3234                 }
3235                 break;
3236         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3237                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3238                         flush_code = IB_WC_LOC_PROT_ERR;
3239                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3240                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3241                 } else {
3242                         flush_code = IB_WC_REM_ACCESS_ERR;
3243                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3244                         termhdr->error_code = RDMAP_INV_BOUNDS;
3245                 }
3246                 break;
3247         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3248         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3249         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3250                 flush_code = IB_WC_REM_ACCESS_ERR;
3251                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3252                 termhdr->error_code = RDMAP_ACCESS;
3253                 break;
3254         case NES_AEQE_AEID_AMP_TO_WRAP:
3255                 flush_code = IB_WC_REM_ACCESS_ERR;
3256                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3257                 termhdr->error_code = RDMAP_TO_WRAP;
3258                 break;
3259         case NES_AEQE_AEID_AMP_BAD_PD:
3260                 switch (iwarp_opcode(nesqp, aeq_info)) {
3261                 case IWARP_OPCODE_WRITE:
3262                         flush_code = IB_WC_LOC_PROT_ERR;
3263                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3264                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3265                         break;
3266                 case IWARP_OPCODE_SEND_INV:
3267                 case IWARP_OPCODE_SEND_SE_INV:
3268                         flush_code = IB_WC_REM_ACCESS_ERR;
3269                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3270                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3271                         break;
3272                 default:
3273                         flush_code = IB_WC_REM_ACCESS_ERR;
3274                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3275                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3276                 }
3277                 break;
3278         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3279                 flush_code = IB_WC_LOC_LEN_ERR;
3280                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3281                 termhdr->error_code = MPA_MARKER;
3282                 break;
3283         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3284                 flush_code = IB_WC_GENERAL_ERR;
3285                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3286                 termhdr->error_code = MPA_CRC;
3287                 break;
3288         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3289         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3290                 flush_code = IB_WC_LOC_LEN_ERR;
3291                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3292                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3293                 break;
3294         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3295         case NES_AEQE_AEID_DDP_NO_L_BIT:
3296                 flush_code = IB_WC_FATAL_ERR;
3297                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3298                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3299                 break;
3300         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3301         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3302                 flush_code = IB_WC_GENERAL_ERR;
3303                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3304                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3305                 break;
3306         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3307                 flush_code = IB_WC_LOC_LEN_ERR;
3308                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3309                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3310                 break;
3311         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3312                 flush_code = IB_WC_GENERAL_ERR;
3313                 if (is_tagged) {
3314                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3315                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3316                 } else {
3317                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3318                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3319                 }
3320                 break;
3321         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3322                 flush_code = IB_WC_GENERAL_ERR;
3323                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3324                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3325                 break;
3326         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3327                 flush_code = IB_WC_REM_OP_ERR;
3328                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3329                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3330                 break;
3331         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3332                 flush_code = IB_WC_GENERAL_ERR;
3333                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3334                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3335                 break;
3336         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3337                 flush_code = IB_WC_GENERAL_ERR;
3338                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3339                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3340                 break;
3341         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3342                 flush_code = IB_WC_LOC_QP_OP_ERR;
3343                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3344                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3345                 break;
3346         default:
3347                 flush_code = IB_WC_FATAL_ERR;
3348                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3349                 termhdr->error_code = RDMAP_UNSPECIFIED;
3350                 break;
3351         }
3352
3353         if (copy_len)
3354                 memcpy(termhdr + 1, pkt, copy_len);
3355
3356         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3357                 if (aeq_info & NES_AEQE_SQ)
3358                         nesqp->term_sq_flush_code = flush_code;
3359                 else
3360                         nesqp->term_rq_flush_code = flush_code;
3361         }
3362
3363         return sizeof(struct nes_terminate_hdr) + copy_len;
3364 }
3365
3366 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3367                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3368 {
3369         u64 context;
3370         unsigned long flags;
3371         u32 aeq_info;
3372         u16 async_event_id;
3373         u8 tcp_state;
3374         u8 iwarp_state;
3375         u32 termlen = 0;
3376         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3377                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3378         struct nes_adapter *nesadapter = nesdev->nesadapter;
3379
3380         if (nesqp->term_flags & NES_TERM_SENT)
3381                 return; /* Sanity check */
3382
3383         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3384         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3385         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3386         async_event_id = (u16)aeq_info;
3387
3388         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3389                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3390         if (!context) {
3391                 WARN_ON(!context);
3392                 return;
3393         }
3394
3395         nesqp = (struct nes_qp *)(unsigned long)context;
3396         spin_lock_irqsave(&nesqp->lock, flags);
3397         nesqp->hw_iwarp_state = iwarp_state;
3398         nesqp->hw_tcp_state = tcp_state;
3399         nesqp->last_aeq = async_event_id;
3400         nesqp->terminate_eventtype = eventtype;
3401         spin_unlock_irqrestore(&nesqp->lock, flags);
3402
3403         if (nesadapter->send_term_ok)
3404                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3405         else
3406                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3407
3408         if (!nesdev->iw_status)  {
3409                 nesqp->term_flags = NES_TERM_DONE;
3410                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3411                 nes_cm_disconn(nesqp);
3412         } else {
3413                 nes_terminate_start_timer(nesqp);
3414                 nesqp->term_flags |= NES_TERM_SENT;
3415                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3416         }
3417 }
3418
3419 static void nes_terminate_send_fin(struct nes_device *nesdev,
3420                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3421 {
3422         u32 aeq_info;
3423         u16 async_event_id;
3424         u8 tcp_state;
3425         u8 iwarp_state;
3426         unsigned long flags;
3427
3428         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3429         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3430         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3431         async_event_id = (u16)aeq_info;
3432
3433         spin_lock_irqsave(&nesqp->lock, flags);
3434         nesqp->hw_iwarp_state = iwarp_state;
3435         nesqp->hw_tcp_state = tcp_state;
3436         nesqp->last_aeq = async_event_id;
3437         spin_unlock_irqrestore(&nesqp->lock, flags);
3438
3439         /* Send the fin only */
3440         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3441                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3442 }
3443
3444 /* Cleanup after a terminate sent or received */
3445 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3446 {
3447         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3448         unsigned long flags;
3449         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3450         struct nes_device *nesdev = nesvnic->nesdev;
3451         u8 first_time = 0;
3452
3453         spin_lock_irqsave(&nesqp->lock, flags);
3454         if (nesqp->hte_added) {
3455                 nesqp->hte_added = 0;
3456                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3457         }
3458
3459         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3460         nesqp->term_flags |= NES_TERM_DONE;
3461         spin_unlock_irqrestore(&nesqp->lock, flags);
3462
3463         /* Make sure we go through this only once */
3464         if (first_time) {
3465                 if (timeout_occurred == 0)
3466                         del_timer(&nesqp->terminate_timer);
3467                 else
3468                         next_iwarp_state |= NES_CQP_QP_RESET;
3469
3470                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3471                 nes_cm_disconn(nesqp);
3472         }
3473 }
3474
3475 static void nes_terminate_received(struct nes_device *nesdev,
3476                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3477 {
3478         u32 aeq_info;
3479         u8 *pkt;
3480         u32 *mpa;
3481         u8 ddp_ctl;
3482         u8 rdma_ctl;
3483         u16 aeq_id = 0;
3484
3485         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3486         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3487                 /* Terminate is not a performance path so the silicon */
3488                 /* did not validate the frame - do it now */
3489                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3490                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3491                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3492                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3493                 if ((ddp_ctl & 0xc0) != 0x40)
3494                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3495                 else if ((ddp_ctl & 0x03) != 1)
3496                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3497                 else if (be32_to_cpu(mpa[2]) != 2)
3498                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3499                 else if (be32_to_cpu(mpa[3]) != 1)
3500                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3501                 else if (be32_to_cpu(mpa[4]) != 0)
3502                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3503                 else if ((rdma_ctl & 0xc0) != 0x40)
3504                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3505
3506                 if (aeq_id) {
3507                         /* Bad terminate recvd - send back a terminate */
3508                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3509                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3510                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3511                         return;
3512                 }
3513         }
3514
3515         nesqp->term_flags |= NES_TERM_RCVD;
3516         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3517         nes_terminate_start_timer(nesqp);
3518         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3519 }
3520
3521 /* Timeout routine in case terminate fails to complete */
3522 void nes_terminate_timeout(unsigned long context)
3523 {
3524         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3525
3526         nes_terminate_done(nesqp, 1);
3527 }
3528
3529 /* Set a timer in case hw cannot complete the terminate sequence */
3530 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3531 {
3532         mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3533 }
3534
3535 /**
3536  * nes_process_iwarp_aeqe
3537  */
3538 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3539                                    struct nes_hw_aeqe *aeqe)
3540 {
3541         u64 context;
3542         unsigned long flags;
3543         struct nes_qp *nesqp;
3544         struct nes_hw_cq *hw_cq;
3545         struct nes_cq *nescq;
3546         int resource_allocated;
3547         struct nes_adapter *nesadapter = nesdev->nesadapter;
3548         u32 aeq_info;
3549         u32 next_iwarp_state = 0;
3550         u32 aeqe_cq_id;
3551         u16 async_event_id;
3552         u8 tcp_state;
3553         u8 iwarp_state;
3554         struct ib_event ibevent;
3555
3556         nes_debug(NES_DBG_AEQ, "\n");
3557         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3558         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3559                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3560                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3561         } else {
3562                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3563                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3564                 BUG_ON(!context);
3565         }
3566
3567         /* context is nesqp unless async_event_id == CQ ERROR */
3568         nesqp = (struct nes_qp *)(unsigned long)context;
3569         async_event_id = (u16)aeq_info;
3570         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3571         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3572         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3573                         " Tcp state = %s, iWARP state = %s\n",
3574                         async_event_id,
3575                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3576                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3577
3578         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3579         if (aeq_info & NES_AEQE_QP) {
3580                 if (!nes_is_resource_allocated(nesadapter,
3581                                 nesadapter->allocated_qps,
3582                                 aeqe_cq_id))
3583                         return;
3584         }
3585
3586         switch (async_event_id) {
3587                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3588                         if (nesqp->term_flags)
3589                                 return; /* Ignore it, wait for close complete */
3590
3591                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3592                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3593                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3594                                         spin_lock_irqsave(&nesqp->lock, flags);
3595                                         nesqp->hw_iwarp_state = iwarp_state;
3596                                         nesqp->hw_tcp_state = tcp_state;
3597                                         nesqp->last_aeq = async_event_id;
3598                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3599                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3600                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3601                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3602                                         nes_cm_disconn(nesqp);
3603                                 }
3604                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3605                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3606                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3607                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3608                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3609                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3610                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3611                                                 async_event_id, nesqp->last_aeq, tcp_state);
3612                         }
3613                         break;
3614                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3615                         spin_lock_irqsave(&nesqp->lock, flags);
3616                         nesqp->hw_iwarp_state = iwarp_state;
3617                         nesqp->hw_tcp_state = tcp_state;
3618                         nesqp->last_aeq = async_event_id;
3619                         spin_unlock_irqrestore(&nesqp->lock, flags);
3620                         nes_cm_disconn(nesqp);
3621                         break;
3622
3623                 case NES_AEQE_AEID_RESET_SENT:
3624                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3625                         spin_lock_irqsave(&nesqp->lock, flags);
3626                         nesqp->hw_iwarp_state = iwarp_state;
3627                         nesqp->hw_tcp_state = tcp_state;
3628                         nesqp->last_aeq = async_event_id;
3629                         nesqp->hte_added = 0;
3630                         spin_unlock_irqrestore(&nesqp->lock, flags);
3631                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3632                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3633                         nes_cm_disconn(nesqp);
3634                         break;
3635
3636                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3637                         if (atomic_read(&nesqp->close_timer_started))
3638                                 return;
3639                         spin_lock_irqsave(&nesqp->lock, flags);
3640                         nesqp->hw_iwarp_state = iwarp_state;
3641                         nesqp->hw_tcp_state = tcp_state;
3642                         nesqp->last_aeq = async_event_id;
3643                         spin_unlock_irqrestore(&nesqp->lock, flags);
3644                         nes_cm_disconn(nesqp);
3645                         break;
3646
3647                 case NES_AEQE_AEID_TERMINATE_SENT:
3648                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3649                         break;
3650
3651                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3652                         nes_terminate_received(nesdev, nesqp, aeqe);
3653                         break;
3654
3655                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3656                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3657                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3658                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3659                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3660                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3661                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3662                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3663                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3664                 case NES_AEQE_AEID_AMP_TO_WRAP:
3665                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3666                                         nesqp->hwqp.qp_id, async_event_id);
3667                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3668                         break;
3669
3670                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3671                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3672                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3673                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3674                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3675                                 aeq_info &= 0xffff0000;
3676                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3677                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3678                         }
3679
3680                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3681                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3682                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3683                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3684                 case NES_AEQE_AEID_AMP_BAD_QP:
3685                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3686                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3687                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3688                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3689                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3690                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3691                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3692                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3693                 case NES_AEQE_AEID_AMP_BAD_PD:
3694                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3695                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3696                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3697                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3698                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3699                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3700                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3701                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3702                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3703                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3704                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3705                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3706                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3707                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3708                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3709                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3710                 case NES_AEQE_AEID_BAD_CLOSE:
3711                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3712                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3713                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3714                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3715                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3716                                         nesqp->hwqp.qp_id, async_event_id);
3717                         print_ip(nesqp->cm_node);
3718                         if (!atomic_read(&nesqp->close_timer_started))
3719                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3720                         break;
3721
3722                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3723                         context <<= 1;
3724                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3725                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3726                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3727                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3728                         if (resource_allocated) {
3729                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3730                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3731                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3732                                 if (hw_cq) {
3733                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3734                                         if (nescq->ibcq.event_handler) {
3735                                                 ibevent.device = nescq->ibcq.device;
3736                                                 ibevent.event = IB_EVENT_CQ_ERR;
3737                                                 ibevent.element.cq = &nescq->ibcq;
3738                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3739                                         }
3740                                 }
3741                         }
3742                         break;
3743
3744                 default:
3745                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3746                                         async_event_id);
3747                         break;
3748         }
3749
3750 }
3751
3752 /**
3753  * nes_iwarp_ce_handler
3754  */
3755 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3756 {
3757         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3758
3759         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3760                         nescq->hw_cq.cq_number); */
3761         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3762
3763         if (nescq->ibcq.comp_handler)
3764                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3765
3766         return;
3767 }
3768
3769
3770 /**
3771  * nes_manage_apbvt()
3772  */
3773 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3774                 u32 nic_index, u32 add_port)
3775 {
3776         struct nes_device *nesdev = nesvnic->nesdev;
3777         struct nes_hw_cqp_wqe *cqp_wqe;
3778         struct nes_cqp_request *cqp_request;
3779         int ret = 0;
3780         u16 major_code;
3781
3782         /* Send manage APBVT request to CQP */
3783         cqp_request = nes_get_cqp_request(nesdev);
3784         if (cqp_request == NULL) {
3785                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3786                 return -ENOMEM;
3787         }
3788         cqp_request->waiting = 1;
3789         cqp_wqe = &cqp_request->cqp_wqe;
3790
3791         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3792                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3793                         accel_local_port, accel_local_port, nic_index);
3794
3795         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3796         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3797                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3798         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3799                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3800
3801         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3802
3803         atomic_set(&cqp_request->refcount, 2);
3804         nes_post_cqp_request(nesdev, cqp_request);
3805
3806         if (add_port == NES_MANAGE_APBVT_ADD)
3807                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3808                                 NES_EVENT_TIMEOUT);
3809         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3810                         ret, cqp_request->major_code, cqp_request->minor_code);
3811         major_code = cqp_request->major_code;
3812
3813         nes_put_cqp_request(nesdev, cqp_request);
3814
3815         if (!ret)
3816                 return -ETIME;
3817         else if (major_code)
3818                 return -EIO;
3819         else
3820                 return 0;
3821 }
3822
3823
3824 /**
3825  * nes_manage_arp_cache
3826  */
3827 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3828                 u32 ip_addr, u32 action)
3829 {
3830         struct nes_hw_cqp_wqe *cqp_wqe;
3831         struct nes_vnic *nesvnic = netdev_priv(netdev);
3832         struct nes_device *nesdev;
3833         struct nes_cqp_request *cqp_request;
3834         int arp_index;
3835
3836         nesdev = nesvnic->nesdev;
3837         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3838         if (arp_index == -1) {
3839                 return;
3840         }
3841
3842         /* update the ARP entry */
3843         cqp_request = nes_get_cqp_request(nesdev);
3844         if (cqp_request == NULL) {
3845                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3846                 return;
3847         }
3848         cqp_request->waiting = 0;
3849         cqp_wqe = &cqp_request->cqp_wqe;
3850         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3851
3852         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3853                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3854         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3855                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3856         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3857
3858         if (action == NES_ARP_ADD) {
3859                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3860                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3861                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3862                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3863                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3864                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3865         } else {
3866                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3867                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3868         }
3869
3870         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3871                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3872
3873         atomic_set(&cqp_request->refcount, 1);
3874         nes_post_cqp_request(nesdev, cqp_request);
3875 }
3876
3877
3878 /**
3879  * flush_wqes
3880  */
3881 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3882                 u32 which_wq, u32 wait_completion)
3883 {
3884         struct nes_cqp_request *cqp_request;
3885         struct nes_hw_cqp_wqe *cqp_wqe;
3886         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3887         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3888         int ret;
3889
3890         cqp_request = nes_get_cqp_request(nesdev);
3891         if (cqp_request == NULL) {
3892                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3893                 return;
3894         }
3895         if (wait_completion) {
3896                 cqp_request->waiting = 1;
3897                 atomic_set(&cqp_request->refcount, 2);
3898         } else {
3899                 cqp_request->waiting = 0;
3900         }
3901         cqp_wqe = &cqp_request->cqp_wqe;
3902         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3903
3904         /* If wqe in error was identified, set code to be put into cqe */
3905         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3906                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3907                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3908                 nesqp->term_sq_flush_code = 0;
3909         }
3910
3911         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3912                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3913                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3914                 nesqp->term_rq_flush_code = 0;
3915         }
3916
3917         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3918                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3919                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3920         }
3921
3922         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3923                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3924         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3925
3926         nes_post_cqp_request(nesdev, cqp_request);
3927
3928         if (wait_completion) {
3929                 /* Wait for CQP */
3930                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3931                                 NES_EVENT_TIMEOUT);
3932                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3933                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3934                                 ret, cqp_request->major_code, cqp_request->minor_code);
3935                 nes_put_cqp_request(nesdev, cqp_request);
3936         }
3937 }