kernel/drivers/infiniband/hw/nes/nes_mgt.c

   1 /*
   2  * Copyright (c) 2006 - 2011 Intel-NE, Inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/skbuff.h>
  35 #include <linux/etherdevice.h>
  36 #include <linux/kthread.h>
  37 #include <linux/ip.h>
  38 #include <linux/tcp.h>
  39 #include <net/tcp.h>
  40 #include "nes.h"
  41 #include "nes_mgt.h"
  42
  43 atomic_t pau_qps_created;
  44 atomic_t pau_qps_destroyed;
  45
  46 static void nes_replenish_mgt_rq(struct nes_vnic_mgt *mgtvnic)
  47 {
  48         unsigned long flags;
  49         dma_addr_t bus_address;
  50         struct sk_buff *skb;
  51         struct nes_hw_nic_rq_wqe *nic_rqe;
  52         struct nes_hw_mgt *nesmgt;
  53         struct nes_device *nesdev;
  54         struct nes_rskb_cb *cb;
  55         u32 rx_wqes_posted = 0;
  56
  57         nesmgt = &mgtvnic->mgt;
  58         nesdev = mgtvnic->nesvnic->nesdev;
  59         spin_lock_irqsave(&nesmgt->rq_lock, flags);
  60         if (nesmgt->replenishing_rq != 0) {
  61                 if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) &&
  62                     (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) {
  63                         atomic_set(&mgtvnic->rx_skb_timer_running, 1);
  64                         spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
  65                         mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2);      /* 1/2 second */
  66                         add_timer(&mgtvnic->rq_wqes_timer);
  67                 } else {
  68                         spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
  69                 }
  70                 return;
  71         }
  72         nesmgt->replenishing_rq = 1;
  73         spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
  74         do {
  75                 skb = dev_alloc_skb(mgtvnic->nesvnic->max_frame_size);
  76                 if (skb) {
  77                         skb->dev = mgtvnic->nesvnic->netdev;
  78
  79                         bus_address = pci_map_single(nesdev->pcidev,
  80                                                      skb->data, mgtvnic->nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
  81                         cb = (struct nes_rskb_cb *)&skb->cb[0];
  82                         cb->busaddr = bus_address;
  83                         cb->maplen = mgtvnic->nesvnic->max_frame_size;
  84
  85                         nic_rqe = &nesmgt->rq_vbase[mgtvnic->mgt.rq_head];
  86                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
  87                                 cpu_to_le32(mgtvnic->nesvnic->max_frame_size);
  88                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
  89                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
  90                                 cpu_to_le32((u32)bus_address);
  91                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
  92                                 cpu_to_le32((u32)((u64)bus_address >> 32));
  93                         nesmgt->rx_skb[nesmgt->rq_head] = skb;
  94                         nesmgt->rq_head++;
  95                         nesmgt->rq_head &= nesmgt->rq_size - 1;
  96                         atomic_dec(&mgtvnic->rx_skbs_needed);
  97                         barrier();
  98                         if (++rx_wqes_posted == 255) {
  99                                 nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id);
 100                                 rx_wqes_posted = 0;
 101                         }
 102                 } else {
 103                         spin_lock_irqsave(&nesmgt->rq_lock, flags);
 104                         if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) &&
 105                             (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) {
 106                                 atomic_set(&mgtvnic->rx_skb_timer_running, 1);
 107                                 spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
 108                                 mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2);      /* 1/2 second */
 109                                 add_timer(&mgtvnic->rq_wqes_timer);
 110                         } else {
 111                                 spin_unlock_irqrestore(&nesmgt->rq_lock, flags);
 112                         }
 113                         break;
 114                 }
 115         } while (atomic_read(&mgtvnic->rx_skbs_needed));
 116         barrier();
 117         if (rx_wqes_posted)
 118                 nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id);
 119         nesmgt->replenishing_rq = 0;
 120 }
 121
 122 /**
 123  * nes_mgt_rq_wqes_timeout
 124  */
 125 static void nes_mgt_rq_wqes_timeout(unsigned long parm)
 126 {
 127         struct nes_vnic_mgt *mgtvnic = (struct nes_vnic_mgt *)parm;
 128
 129         atomic_set(&mgtvnic->rx_skb_timer_running, 0);
 130         if (atomic_read(&mgtvnic->rx_skbs_needed))
 131                 nes_replenish_mgt_rq(mgtvnic);
 132 }
 133
 134 /**
 135  * nes_mgt_free_skb - unmap and free skb
 136  */
 137 static void nes_mgt_free_skb(struct nes_device *nesdev, struct sk_buff *skb, u32 dir)
 138 {
 139         struct nes_rskb_cb *cb;
 140
 141         cb = (struct nes_rskb_cb *)&skb->cb[0];
 142         pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, dir);
 143         cb->busaddr = 0;
 144         dev_kfree_skb_any(skb);
 145 }
 146
 147 /**
 148  * nes_download_callback - handle download completions
 149  */
 150 static void nes_download_callback(struct nes_device *nesdev, struct nes_cqp_request *cqp_request)
 151 {
 152         struct pau_fpdu_info *fpdu_info = cqp_request->cqp_callback_pointer;
 153         struct nes_qp *nesqp = fpdu_info->nesqp;
 154         struct sk_buff *skb;
 155         int i;
 156
 157         for (i = 0; i < fpdu_info->frag_cnt; i++) {
 158                 skb = fpdu_info->frags[i].skb;
 159                 if (fpdu_info->frags[i].cmplt) {
 160                         nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE);
 161                         nes_rem_ref_cm_node(nesqp->cm_node);
 162                 }
 163         }
 164
 165         if (fpdu_info->hdr_vbase)
 166                 pci_free_consistent(nesdev->pcidev, fpdu_info->hdr_len,
 167                                     fpdu_info->hdr_vbase, fpdu_info->hdr_pbase);
 168         kfree(fpdu_info);
 169 }
 170
 171 /**
 172  * nes_get_seq - Get the seq, ack_seq and window from the packet
 173  */
 174 static u32 nes_get_seq(struct sk_buff *skb, u32 *ack, u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd)
 175 {
 176         struct nes_rskb_cb *cb = (struct nes_rskb_cb *)&skb->cb[0];
 177         struct iphdr *iph = (struct iphdr *)(cb->data_start + ETH_HLEN);
 178         struct tcphdr *tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
 179
 180         *ack = be32_to_cpu(tcph->ack_seq);
 181         *wnd = be16_to_cpu(tcph->window);
 182         *fin_rcvd = tcph->fin;
 183         *rst_rcvd = tcph->rst;
 184         return be32_to_cpu(tcph->seq);
 185 }
 186
 187 /**
 188  * nes_get_next_skb - Get the next skb based on where current skb is in the queue
 189  */
 190 static struct sk_buff *nes_get_next_skb(struct nes_device *nesdev, struct nes_qp *nesqp,
 191                                         struct sk_buff *skb, u32 nextseq, u32 *ack,
 192                                         u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd)
 193 {
 194         u32 seq;
 195         bool processacks;
 196         struct sk_buff *old_skb;
 197
 198         if (skb) {
 199                 /* Continue processing fpdu */
 200                 if (skb->next == (struct sk_buff *)&nesqp->pau_list)
 201                         goto out;
 202                 skb = skb->next;
 203                 processacks = false;
 204         } else {
 205                 /* Starting a new one */
 206                 if (skb_queue_empty(&nesqp->pau_list))
 207                         goto out;
 208                 skb = skb_peek(&nesqp->pau_list);
 209                 processacks = true;
 210         }
 211
 212         while (1) {
 213                 if (skb_queue_empty(&nesqp->pau_list))
 214                         goto out;
 215
 216                 seq = nes_get_seq(skb, ack, wnd, fin_rcvd, rst_rcvd);
 217                 if (seq == nextseq) {
 218                         if (skb->len || processacks)
 219                                 break;
 220                 } else if (after(seq, nextseq)) {
 221                         goto out;
 222                 }
 223
 224                 old_skb = skb;
 225                 skb = skb->next;
 226                 skb_unlink(old_skb, &nesqp->pau_list);
 227                 nes_mgt_free_skb(nesdev, old_skb, PCI_DMA_TODEVICE);
 228                 nes_rem_ref_cm_node(nesqp->cm_node);
 229                 if (skb == (struct sk_buff *)&nesqp->pau_list)
 230                         goto out;
 231         }
 232         return skb;
 233
 234 out:
 235         return NULL;
 236 }
 237
 238 /**
 239  * get_fpdu_info - Find the next complete fpdu and return its fragments.
 240  */
 241 static int get_fpdu_info(struct nes_device *nesdev, struct nes_qp *nesqp,
 242                          struct pau_fpdu_info **pau_fpdu_info)
 243 {
 244         struct sk_buff *skb;
 245         struct iphdr *iph;
 246         struct tcphdr *tcph;
 247         struct nes_rskb_cb *cb;
 248         struct pau_fpdu_info *fpdu_info = NULL;
 249         struct pau_fpdu_frag frags[MAX_FPDU_FRAGS];
 250         u32 fpdu_len = 0;
 251         u32 tmp_len;
 252         int frag_cnt = 0;
 253         u32 tot_len;
 254         u32 frag_tot;
 255         u32 ack;
 256         u32 fin_rcvd;
 257         u32 rst_rcvd;
 258         u16 wnd;
 259         int i;
 260         int rc = 0;
 261
 262         *pau_fpdu_info = NULL;
 263
 264         skb = nes_get_next_skb(nesdev, nesqp, NULL, nesqp->pau_rcv_nxt, &ack, &wnd, &fin_rcvd, &rst_rcvd);
 265         if (!skb)
 266                 goto out;
 267
 268         cb = (struct nes_rskb_cb *)&skb->cb[0];
 269         if (skb->len) {
 270                 fpdu_len = be16_to_cpu(*(__be16 *) skb->data) + MPA_FRAMING;
 271                 fpdu_len = (fpdu_len + 3) & 0xfffffffc;
 272                 tmp_len = fpdu_len;
 273
 274                 /* See if we have all of the fpdu */
 275                 frag_tot = 0;
 276                 memset(&frags, 0, sizeof frags);
 277                 for (i = 0; i < MAX_FPDU_FRAGS; i++) {
 278                         frags[i].physaddr = cb->busaddr;
 279                         frags[i].physaddr += skb->data - cb->data_start;
 280                         frags[i].frag_len = min(tmp_len, skb->len);
 281                         frags[i].skb = skb;
 282                         frags[i].cmplt = (skb->len == frags[i].frag_len);
 283                         frag_tot += frags[i].frag_len;
 284                         frag_cnt++;
 285
 286                         tmp_len -= frags[i].frag_len;
 287                         if (tmp_len == 0)
 288                                 break;
 289
 290                         skb = nes_get_next_skb(nesdev, nesqp, skb,
 291                                                nesqp->pau_rcv_nxt + frag_tot, &ack, &wnd, &fin_rcvd, &rst_rcvd);
 292                         if (!skb)
 293                                 goto out;
 294                         if (rst_rcvd) {
 295                                 /* rst received in the middle of fpdu */
 296                                 for (; i >= 0; i--) {
 297                                         skb_unlink(frags[i].skb, &nesqp->pau_list);
 298                                         nes_mgt_free_skb(nesdev, frags[i].skb, PCI_DMA_TODEVICE);
 299                                 }
 300                                 cb = (struct nes_rskb_cb *)&skb->cb[0];
 301                                 frags[0].physaddr = cb->busaddr;
 302                                 frags[0].physaddr += skb->data - cb->data_start;
 303                                 frags[0].frag_len = skb->len;
 304                                 frags[0].skb = skb;
 305                                 frags[0].cmplt = true;
 306                                 frag_cnt = 1;
 307                                 break;
 308                         }
 309
 310                         cb = (struct nes_rskb_cb *)&skb->cb[0];
 311                 }
 312         } else {
 313                 /* no data */
 314                 frags[0].physaddr = cb->busaddr;
 315                 frags[0].frag_len = 0;
 316                 frags[0].skb = skb;
 317                 frags[0].cmplt = true;
 318                 frag_cnt = 1;
 319         }
 320
 321         /* Found one */
 322         fpdu_info = kzalloc(sizeof(*fpdu_info), GFP_ATOMIC);
 323         if (fpdu_info == NULL) {
 324                 nes_debug(NES_DBG_PAU, "Failed to alloc a fpdu_info.\n");
 325                 rc = -ENOMEM;
 326                 goto out;
 327         }
 328
 329         fpdu_info->cqp_request = nes_get_cqp_request(nesdev);
 330         if (fpdu_info->cqp_request == NULL) {
 331                 nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n");
 332                 rc = -ENOMEM;
 333                 goto out;
 334         }
 335
 336         cb = (struct nes_rskb_cb *)&frags[0].skb->cb[0];
 337         iph = (struct iphdr *)(cb->data_start + ETH_HLEN);
 338         tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
 339         fpdu_info->hdr_len = (((unsigned char *)tcph) + 4 * (tcph->doff)) - cb->data_start;
 340         fpdu_info->data_len = fpdu_len;
 341         tot_len = fpdu_info->hdr_len + fpdu_len - ETH_HLEN;
 342
 343         if (frags[0].cmplt) {
 344                 fpdu_info->hdr_pbase = cb->busaddr;
 345                 fpdu_info->hdr_vbase = NULL;
 346         } else {
 347                 fpdu_info->hdr_vbase = pci_alloc_consistent(nesdev->pcidev,
 348                                                             fpdu_info->hdr_len, &fpdu_info->hdr_pbase);
 349                 if (!fpdu_info->hdr_vbase) {
 350                         nes_debug(NES_DBG_PAU, "Unable to allocate memory for pau first frag\n");
 351                         rc = -ENOMEM;
 352                         goto out;
 353                 }
 354
 355                 /* Copy hdrs, adjusting len and seqnum */
 356                 memcpy(fpdu_info->hdr_vbase, cb->data_start, fpdu_info->hdr_len);
 357                 iph = (struct iphdr *)(fpdu_info->hdr_vbase + ETH_HLEN);
 358                 tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
 359         }
 360
 361         iph->tot_len = cpu_to_be16(tot_len);
 362         iph->saddr = cpu_to_be32(0x7f000001);
 363
 364         tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt);
 365         tcph->ack_seq = cpu_to_be32(ack);
 366         tcph->window = cpu_to_be16(wnd);
 367
 368         nesqp->pau_rcv_nxt += fpdu_len + fin_rcvd;
 369
 370         memcpy(fpdu_info->frags, frags, sizeof(fpdu_info->frags));
 371         fpdu_info->frag_cnt = frag_cnt;
 372         fpdu_info->nesqp = nesqp;
 373         *pau_fpdu_info = fpdu_info;
 374
 375         /* Update skb's for next pass */
 376         for (i = 0; i < frag_cnt; i++) {
 377                 cb = (struct nes_rskb_cb *)&frags[i].skb->cb[0];
 378                 skb_pull(frags[i].skb, frags[i].frag_len);
 379
 380                 if (frags[i].skb->len == 0) {
 381                         /* Pull skb off the list - it will be freed in the callback */
 382                         if (!skb_queue_empty(&nesqp->pau_list))
 383                                 skb_unlink(frags[i].skb, &nesqp->pau_list);
 384                 } else {
 385                         /* Last skb still has data so update the seq */
 386                         iph = (struct iphdr *)(cb->data_start + ETH_HLEN);
 387                         tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
 388                         tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt);
 389                 }
 390         }
 391
 392 out:
 393         if (rc) {
 394                 if (fpdu_info) {
 395                         if (fpdu_info->cqp_request)
 396                                 nes_put_cqp_request(nesdev, fpdu_info->cqp_request);
 397                         kfree(fpdu_info);
 398                 }
 399         }
 400         return rc;
 401 }
 402
 403 /**
 404  * forward_fpdu - send complete fpdus, one at a time
 405  */
 406 static int forward_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp)
 407 {
 408         struct nes_device *nesdev = nesvnic->nesdev;
 409         struct pau_fpdu_info *fpdu_info;
 410         struct nes_hw_cqp_wqe *cqp_wqe;
 411         struct nes_cqp_request *cqp_request;
 412         unsigned long flags;
 413         u64 u64tmp;
 414         u32 u32tmp;
 415         int rc;
 416
 417         while (1) {
 418                 spin_lock_irqsave(&nesqp->pau_lock, flags);
 419                 rc = get_fpdu_info(nesdev, nesqp, &fpdu_info);
 420                 if (rc || (fpdu_info == NULL)) {
 421                         spin_unlock_irqrestore(&nesqp->pau_lock, flags);
 422                         return rc;
 423                 }
 424
 425                 cqp_request = fpdu_info->cqp_request;
 426                 cqp_wqe = &cqp_request->cqp_wqe;
 427                 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
 428                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_OPCODE_IDX,
 429                                     NES_CQP_DOWNLOAD_SEGMENT |
 430                                     (((u32)nesvnic->logical_port) << NES_CQP_OP_LOGICAL_PORT_SHIFT));
 431
 432                 u32tmp = fpdu_info->hdr_len << 16;
 433                 u32tmp |= fpdu_info->hdr_len + (u32)fpdu_info->data_len;
 434                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_LENGTH_0_TOTAL_IDX,
 435                                     u32tmp);
 436
 437                 u32tmp = (fpdu_info->frags[1].frag_len << 16) | fpdu_info->frags[0].frag_len;
 438                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_2_1_IDX,
 439                                     u32tmp);
 440
 441                 u32tmp = (fpdu_info->frags[3].frag_len << 16) | fpdu_info->frags[2].frag_len;
 442                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_4_3_IDX,
 443                                     u32tmp);
 444
 445                 u64tmp = (u64)fpdu_info->hdr_pbase;
 446                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX,
 447                                     lower_32_bits(u64tmp));
 448                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_HIGH_IDX,
 449                                     upper_32_bits(u64tmp));
 450
 451                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX,
 452                                     lower_32_bits(fpdu_info->frags[0].physaddr));
 453                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_HIGH_IDX,
 454                                     upper_32_bits(fpdu_info->frags[0].physaddr));
 455
 456                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_LOW_IDX,
 457                                     lower_32_bits(fpdu_info->frags[1].physaddr));
 458                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_HIGH_IDX,
 459                                     upper_32_bits(fpdu_info->frags[1].physaddr));
 460
 461                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_LOW_IDX,
 462                                     lower_32_bits(fpdu_info->frags[2].physaddr));
 463                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_HIGH_IDX,
 464                                     upper_32_bits(fpdu_info->frags[2].physaddr));
 465
 466                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_LOW_IDX,
 467                                     lower_32_bits(fpdu_info->frags[3].physaddr));
 468                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_HIGH_IDX,
 469                                     upper_32_bits(fpdu_info->frags[3].physaddr));
 470
 471                 cqp_request->cqp_callback_pointer = fpdu_info;
 472                 cqp_request->callback = 1;
 473                 cqp_request->cqp_callback = nes_download_callback;
 474
 475                 atomic_set(&cqp_request->refcount, 1);
 476                 nes_post_cqp_request(nesdev, cqp_request);
 477                 spin_unlock_irqrestore(&nesqp->pau_lock, flags);
 478         }
 479
 480         return 0;
 481 }
 482
 483 static void process_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp)
 484 {
 485         int again = 1;
 486         unsigned long flags;
 487
 488         do {
 489                 /* Ignore rc - if it failed, tcp retries will cause it to try again */
 490                 forward_fpdus(nesvnic, nesqp);
 491
 492                 spin_lock_irqsave(&nesqp->pau_lock, flags);
 493                 if (nesqp->pau_pending) {
 494                         nesqp->pau_pending = 0;
 495                 } else {
 496                         nesqp->pau_busy = 0;
 497                         again = 0;
 498                 }
 499
 500                 spin_unlock_irqrestore(&nesqp->pau_lock, flags);
 501         } while (again);
 502 }
 503
 504 /**
 505  * queue_fpdus - Handle fpdu's that hw passed up to sw
 506  */
 507 static void queue_fpdus(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp)
 508 {
 509         struct sk_buff *tmpskb;
 510         struct nes_rskb_cb *cb;
 511         struct iphdr *iph;
 512         struct tcphdr *tcph;
 513         unsigned char *tcph_end;
 514         u32 rcv_nxt;
 515         u32 rcv_wnd;
 516         u32 seqnum;
 517         u32 len;
 518         bool process_it = false;
 519         unsigned long flags;
 520
 521         /* Move data ptr to after tcp header */
 522         iph = (struct iphdr *)skb->data;
 523         tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl));
 524         seqnum = be32_to_cpu(tcph->seq);
 525         tcph_end = (((char *)tcph) + (4 * tcph->doff));
 526
 527         len = be16_to_cpu(iph->tot_len);
 528         if (skb->len > len)
 529                 skb_trim(skb, len);
 530         skb_pull(skb, tcph_end - skb->data);
 531
 532         /* Initialize tracking values */
 533         cb = (struct nes_rskb_cb *)&skb->cb[0];
 534         cb->seqnum = seqnum;
 535
 536         /* Make sure data is in the receive window */
 537         rcv_nxt = nesqp->pau_rcv_nxt;
 538         rcv_wnd = le32_to_cpu(nesqp->nesqp_context->rcv_wnd);
 539         if (!between(seqnum, rcv_nxt, (rcv_nxt + rcv_wnd))) {
 540                 nes_mgt_free_skb(nesvnic->nesdev, skb, PCI_DMA_TODEVICE);
 541                 nes_rem_ref_cm_node(nesqp->cm_node);
 542                 return;
 543         }
 544
 545         spin_lock_irqsave(&nesqp->pau_lock, flags);
 546
 547         if (nesqp->pau_busy)
 548                 nesqp->pau_pending = 1;
 549         else
 550                 nesqp->pau_busy = 1;
 551
 552         /* Queue skb by sequence number */
 553         if (skb_queue_len(&nesqp->pau_list) == 0) {
 554                 skb_queue_head(&nesqp->pau_list, skb);
 555         } else {
 556                 tmpskb = nesqp->pau_list.next;
 557                 while (tmpskb != (struct sk_buff *)&nesqp->pau_list) {
 558                         cb = (struct nes_rskb_cb *)&tmpskb->cb[0];
 559                         if (before(seqnum, cb->seqnum))
 560                                 break;
 561                         tmpskb = tmpskb->next;
 562                 }
 563                 skb_insert(tmpskb, skb, &nesqp->pau_list);
 564         }
 565         if (nesqp->pau_state == PAU_READY)
 566                 process_it = true;
 567         spin_unlock_irqrestore(&nesqp->pau_lock, flags);
 568
 569         if (process_it)
 570                 process_fpdus(nesvnic, nesqp);
 571
 572         return;
 573 }
 574
 575 /**
 576  * mgt_thread - Handle mgt skbs in a safe context
 577  */
 578 static int mgt_thread(void *context)
 579 {
 580         struct nes_vnic *nesvnic = context;
 581         struct sk_buff *skb;
 582         struct nes_rskb_cb *cb;
 583
 584         while (!kthread_should_stop()) {
 585                 wait_event_interruptible(nesvnic->mgt_wait_queue,
 586                                          skb_queue_len(&nesvnic->mgt_skb_list) || kthread_should_stop());
 587                 while ((skb_queue_len(&nesvnic->mgt_skb_list)) && !kthread_should_stop()) {
 588                         skb = skb_dequeue(&nesvnic->mgt_skb_list);
 589                         cb = (struct nes_rskb_cb *)&skb->cb[0];
 590                         cb->data_start = skb->data - ETH_HLEN;
 591                         cb->busaddr = pci_map_single(nesvnic->nesdev->pcidev, cb->data_start,
 592                                                      nesvnic->max_frame_size, PCI_DMA_TODEVICE);
 593                         queue_fpdus(skb, nesvnic, cb->nesqp);
 594                 }
 595         }
 596
 597         /* Closing down so delete any entries on the queue */
 598         while (skb_queue_len(&nesvnic->mgt_skb_list)) {
 599                 skb = skb_dequeue(&nesvnic->mgt_skb_list);
 600                 cb = (struct nes_rskb_cb *)&skb->cb[0];
 601                 nes_rem_ref_cm_node(cb->nesqp->cm_node);
 602                 dev_kfree_skb_any(skb);
 603         }
 604         return 0;
 605 }
 606
 607 /**
 608  * nes_queue_skbs - Queue skb so it can be handled in a thread context
 609  */
 610 void nes_queue_mgt_skbs(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp)
 611 {
 612         struct nes_rskb_cb *cb;
 613
 614         cb = (struct nes_rskb_cb *)&skb->cb[0];
 615         cb->nesqp = nesqp;
 616         skb_queue_tail(&nesvnic->mgt_skb_list, skb);
 617         wake_up_interruptible(&nesvnic->mgt_wait_queue);
 618 }
 619
 620 void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp)
 621 {
 622         struct sk_buff *skb;
 623         unsigned long flags;
 624         atomic_inc(&pau_qps_destroyed);
 625
 626         /* Free packets that have not yet been forwarded */
 627         /* Lock is acquired by skb_dequeue when removing the skb */
 628         spin_lock_irqsave(&nesqp->pau_lock, flags);
 629         while (skb_queue_len(&nesqp->pau_list)) {
 630                 skb = skb_dequeue(&nesqp->pau_list);
 631                 nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE);
 632                 nes_rem_ref_cm_node(nesqp->cm_node);
 633         }
 634         spin_unlock_irqrestore(&nesqp->pau_lock, flags);
 635 }
 636
 637 static void nes_chg_qh_handler(struct nes_device *nesdev, struct nes_cqp_request *cqp_request)
 638 {
 639         struct pau_qh_chg *qh_chg = cqp_request->cqp_callback_pointer;
 640         struct nes_cqp_request *new_request;
 641         struct nes_hw_cqp_wqe *cqp_wqe;
 642         struct nes_adapter *nesadapter;
 643         struct nes_qp *nesqp;
 644         struct nes_v4_quad nes_quad;
 645         u32 crc_value;
 646         u64 u64temp;
 647
 648         nesadapter = nesdev->nesadapter;
 649         nesqp = qh_chg->nesqp;
 650
 651         /* Should we handle the bad completion */
 652         if (cqp_request->major_code)
 653                 WARN(1, PFX "Invalid cqp_request major_code=0x%x\n",
 654                        cqp_request->major_code);
 655
 656         switch (nesqp->pau_state) {
 657         case PAU_DEL_QH:
 658                 /* Old hash code deleted, now set the new one */
 659                 nesqp->pau_state = PAU_ADD_LB_QH;
 660                 new_request = nes_get_cqp_request(nesdev);
 661                 if (new_request == NULL) {
 662                         nes_debug(NES_DBG_PAU, "Failed to get a new_request.\n");
 663                         WARN_ON(1);
 664                         return;
 665                 }
 666
 667                 memset(&nes_quad, 0, sizeof(nes_quad));
 668                 nes_quad.DstIpAdrIndex =
 669                         cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24);
 670                 nes_quad.SrcIpadr = cpu_to_be32(0x7f000001);
 671                 nes_quad.TcpPorts[0] = swab16(nesqp->nesqp_context->tcpPorts[1]);
 672                 nes_quad.TcpPorts[1] = swab16(nesqp->nesqp_context->tcpPorts[0]);
 673
 674                 /* Produce hash key */
 675                 crc_value = get_crc_value(&nes_quad);
 676                 nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff);
 677                 nes_debug(NES_DBG_PAU, "new HTE Index = 0x%08X, CRC = 0x%08X\n",
 678                           nesqp->hte_index, nesqp->hte_index & nesadapter->hte_index_mask);
 679
 680                 nesqp->hte_index &= nesadapter->hte_index_mask;
 681                 nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index);
 682                 nesqp->nesqp_context->ip0 = cpu_to_le32(0x7f000001);
 683                 nesqp->nesqp_context->rcv_nxt = cpu_to_le32(nesqp->pau_rcv_nxt);
 684
 685                 cqp_wqe = &new_request->cqp_wqe;
 686                 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
 687                 set_wqe_32bit_value(cqp_wqe->wqe_words,
 688                                     NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH |
 689                                     NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS);
 690                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
 691                 u64temp = (u64)nesqp->nesqp_context_pbase;
 692                 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
 693
 694                 nes_debug(NES_DBG_PAU, "Waiting for CQP completion for adding the quad hash.\n");
 695
 696                 new_request->cqp_callback_pointer = qh_chg;
 697                 new_request->callback = 1;
 698                 new_request->cqp_callback = nes_chg_qh_handler;
 699                 atomic_set(&new_request->refcount, 1);
 700                 nes_post_cqp_request(nesdev, new_request);
 701                 break;
 702
 703         case PAU_ADD_LB_QH:
 704                 /* Start processing the queued fpdu's */
 705                 nesqp->pau_state = PAU_READY;
 706                 process_fpdus(qh_chg->nesvnic, qh_chg->nesqp);
 707                 kfree(qh_chg);
 708                 break;
 709         }
 710 }
 711
 712 /**
 713  * nes_change_quad_hash
 714  */
 715 static int nes_change_quad_hash(struct nes_device *nesdev,
 716                                 struct nes_vnic *nesvnic, struct nes_qp *nesqp)
 717 {
 718         struct nes_cqp_request *cqp_request = NULL;
 719         struct pau_qh_chg *qh_chg = NULL;
 720         u64 u64temp;
 721         struct nes_hw_cqp_wqe *cqp_wqe;
 722         int ret = 0;
 723
 724         cqp_request = nes_get_cqp_request(nesdev);
 725         if (cqp_request == NULL) {
 726                 nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n");
 727                 ret = -ENOMEM;
 728                 goto chg_qh_err;
 729         }
 730
 731         qh_chg = kmalloc(sizeof *qh_chg, GFP_ATOMIC);
 732         if (qh_chg == NULL) {
 733                 nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n");
 734                 ret = -ENOMEM;
 735                 goto chg_qh_err;
 736         }
 737         qh_chg->nesdev = nesdev;
 738         qh_chg->nesvnic = nesvnic;
 739         qh_chg->nesqp = nesqp;
 740         nesqp->pau_state = PAU_DEL_QH;
 741
 742         cqp_wqe = &cqp_request->cqp_wqe;
 743         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
 744         set_wqe_32bit_value(cqp_wqe->wqe_words,
 745                             NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH | NES_CQP_QP_DEL_HTE |
 746                             NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS);
 747         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);
 748         u64temp = (u64)nesqp->nesqp_context_pbase;
 749         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
 750
 751         nes_debug(NES_DBG_PAU, "Waiting for CQP completion for deleting the quad hash.\n");
 752
 753         cqp_request->cqp_callback_pointer = qh_chg;
 754         cqp_request->callback = 1;
 755         cqp_request->cqp_callback = nes_chg_qh_handler;
 756         atomic_set(&cqp_request->refcount, 1);
 757         nes_post_cqp_request(nesdev, cqp_request);
 758
 759         return ret;
 760
 761 chg_qh_err:
 762         kfree(qh_chg);
 763         if (cqp_request)
 764                 nes_put_cqp_request(nesdev, cqp_request);
 765         return ret;
 766 }
 767
 768 /**
 769  * nes_mgt_ce_handler
 770  * This management code deals with any packed and unaligned (pau) fpdu's
 771  * that the hardware cannot handle.
 772  */
 773 static void nes_mgt_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
 774 {
 775         struct nes_vnic_mgt *mgtvnic = container_of(cq, struct nes_vnic_mgt, mgt_cq);
 776         struct nes_adapter *nesadapter = nesdev->nesadapter;
 777         u32 head;
 778         u32 cq_size;
 779         u32 cqe_count = 0;
 780         u32 cqe_misc;
 781         u32 qp_id = 0;
 782         u32 skbs_needed;
 783         unsigned long context;
 784         struct nes_qp *nesqp;
 785         struct sk_buff *rx_skb;
 786         struct nes_rskb_cb *cb;
 787
 788         head = cq->cq_head;
 789         cq_size = cq->cq_size;
 790
 791         while (1) {
 792                 cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
 793                 if (!(cqe_misc & NES_NIC_CQE_VALID))
 794                         break;
 795
 796                 nesqp = NULL;
 797                 if (cqe_misc & NES_NIC_CQE_ACCQP_VALID) {
 798                         qp_id = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_ACCQP_ID_IDX]);
 799                         qp_id &= 0x001fffff;
 800                         if (qp_id < nesadapter->max_qp) {
 801                                 context = (unsigned long)nesadapter->qp_table[qp_id - NES_FIRST_QPN];
 802                                 nesqp = (struct nes_qp *)context;
 803                         }
 804                 }
 805
 806                 if (nesqp) {
 807                         if (nesqp->pau_mode == false) {
 808                                 nesqp->pau_mode = true; /* First time for this qp */
 809                                 nesqp->pau_rcv_nxt = le32_to_cpu(
 810                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_HASH_RCVNXT]);
 811                                 skb_queue_head_init(&nesqp->pau_list);
 812                                 spin_lock_init(&nesqp->pau_lock);
 813                                 atomic_inc(&pau_qps_created);
 814                                 nes_change_quad_hash(nesdev, mgtvnic->nesvnic, nesqp);
 815                         }
 816
 817                         rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail];
 818                         rx_skb->len = 0;
 819                         skb_put(rx_skb, cqe_misc & 0x0000ffff);
 820                         rx_skb->protocol = eth_type_trans(rx_skb, mgtvnic->nesvnic->netdev);
 821                         cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
 822                         pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, PCI_DMA_FROMDEVICE);
 823                         cb->busaddr = 0;
 824                         mgtvnic->mgt.rq_tail++;
 825                         mgtvnic->mgt.rq_tail &= mgtvnic->mgt.rq_size - 1;
 826
 827                         nes_add_ref_cm_node(nesqp->cm_node);
 828                         nes_queue_mgt_skbs(rx_skb, mgtvnic->nesvnic, nesqp);
 829                 } else {
 830                         printk(KERN_ERR PFX "Invalid QP %d for packed/unaligned handling\n", qp_id);
 831                 }
 832
 833                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
 834                 cqe_count++;
 835                 if (++head >= cq_size)
 836                         head = 0;
 837
 838                 if (cqe_count == 255) {
 839                         /* Replenish mgt CQ */
 840                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (cqe_count << 16));
 841                         nesdev->currcq_count += cqe_count;
 842                         cqe_count = 0;
 843                 }
 844
 845                 skbs_needed = atomic_inc_return(&mgtvnic->rx_skbs_needed);
 846                 if (skbs_needed > (mgtvnic->mgt.rq_size >> 1))
 847                         nes_replenish_mgt_rq(mgtvnic);
 848         }
 849
 850         cq->cq_head = head;
 851         nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
 852                     cq->cq_number | (cqe_count << 16));
 853         nes_read32(nesdev->regs + NES_CQE_ALLOC);
 854         nesdev->currcq_count += cqe_count;
 855 }
 856
 857 /**
 858  * nes_init_mgt_qp
 859  */
 860 int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct nes_vnic *nesvnic)
 861 {
 862         struct nes_vnic_mgt *mgtvnic;
 863         u32 counter;
 864         void *vmem;
 865         dma_addr_t pmem;
 866         struct nes_hw_cqp_wqe *cqp_wqe;
 867         u32 cqp_head;
 868         unsigned long flags;
 869         struct nes_hw_nic_qp_context *mgt_context;
 870         u64 u64temp;
 871         struct nes_hw_nic_rq_wqe *mgt_rqe;
 872         struct sk_buff *skb;
 873         u32 wqe_count;
 874         struct nes_rskb_cb *cb;
 875         u32 mgt_mem_size;
 876         void *mgt_vbase;
 877         dma_addr_t mgt_pbase;
 878         int i;
 879         int ret;
 880
 881         /* Allocate space the all mgt QPs once */
 882         mgtvnic = kzalloc(NES_MGT_QP_COUNT * sizeof(struct nes_vnic_mgt), GFP_KERNEL);
 883         if (mgtvnic == NULL) {
 884                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for mgt structure\n");
 885                 return -ENOMEM;
 886         }
 887
 888         /* Allocate fragment, RQ, and CQ; Reuse CEQ based on the PCI function */
 889         /* We are not sending from this NIC so sq is not allocated */
 890         mgt_mem_size = 256 +
 891                        (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)) +
 892                        (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_cqe)) +
 893                        sizeof(struct nes_hw_nic_qp_context);
 894         mgt_mem_size = (mgt_mem_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
 895         mgt_vbase = pci_alloc_consistent(nesdev->pcidev, NES_MGT_QP_COUNT * mgt_mem_size, &mgt_pbase);
 896         if (!mgt_vbase) {
 897                 kfree(mgtvnic);
 898                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for mgt host descriptor rings\n");
 899                 return -ENOMEM;
 900         }
 901
 902         nesvnic->mgt_mem_size = NES_MGT_QP_COUNT * mgt_mem_size;
 903         nesvnic->mgt_vbase = mgt_vbase;
 904         nesvnic->mgt_pbase = mgt_pbase;
 905
 906         skb_queue_head_init(&nesvnic->mgt_skb_list);
 907         init_waitqueue_head(&nesvnic->mgt_wait_queue);
 908         nesvnic->mgt_thread = kthread_run(mgt_thread, nesvnic, "nes_mgt_thread");
 909
 910         for (i = 0; i < NES_MGT_QP_COUNT; i++) {
 911                 mgtvnic->nesvnic = nesvnic;
 912                 mgtvnic->mgt.qp_id = nesdev->mac_index + NES_MGT_QP_OFFSET + i;
 913                 memset(mgt_vbase, 0, mgt_mem_size);
 914                 nes_debug(NES_DBG_INIT, "Allocated mgt QP structures at %p (phys = %016lX), size = %u.\n",
 915                           mgt_vbase, (unsigned long)mgt_pbase, mgt_mem_size);
 916
 917                 vmem = (void *)(((unsigned long)mgt_vbase + (256 - 1)) &
 918                                 ~(unsigned long)(256 - 1));
 919                 pmem = (dma_addr_t)(((unsigned long long)mgt_pbase + (256 - 1)) &
 920                                     ~(unsigned long long)(256 - 1));
 921
 922                 spin_lock_init(&mgtvnic->mgt.rq_lock);
 923
 924                 /* setup the RQ */
 925                 mgtvnic->mgt.rq_vbase = vmem;
 926                 mgtvnic->mgt.rq_pbase = pmem;
 927                 mgtvnic->mgt.rq_head = 0;
 928                 mgtvnic->mgt.rq_tail = 0;
 929                 mgtvnic->mgt.rq_size = NES_MGT_WQ_COUNT;
 930
 931                 /* setup the CQ */
 932                 vmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe));
 933                 pmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe));
 934
 935                 mgtvnic->mgt_cq.cq_number = mgtvnic->mgt.qp_id;
 936                 mgtvnic->mgt_cq.cq_vbase = vmem;
 937                 mgtvnic->mgt_cq.cq_pbase = pmem;
 938                 mgtvnic->mgt_cq.cq_head = 0;
 939                 mgtvnic->mgt_cq.cq_size = NES_MGT_WQ_COUNT;
 940
 941                 mgtvnic->mgt_cq.ce_handler = nes_mgt_ce_handler;
 942
 943                 /* Send CreateCQ request to CQP */
 944                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
 945                 cqp_head = nesdev->cqp.sq_head;
 946
 947                 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
 948                 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
 949
 950                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
 951                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
 952                         ((u32)mgtvnic->mgt_cq.cq_size << 16));
 953                 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
 954                         mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16));
 955                 u64temp = (u64)mgtvnic->mgt_cq.cq_pbase;
 956                 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
 957                 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
 958                 u64temp = (unsigned long)&mgtvnic->mgt_cq;
 959                 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1));
 960                 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
 961                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
 962                 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
 963
 964                 if (++cqp_head >= nesdev->cqp.sq_size)
 965                         cqp_head = 0;
 966                 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
 967                 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
 968
 969                 /* Send CreateQP request to CQP */
 970                 mgt_context = (void *)(&mgtvnic->mgt_cq.cq_vbase[mgtvnic->mgt_cq.cq_size]);
 971                 mgt_context->context_words[NES_NIC_CTX_MISC_IDX] =
 972                         cpu_to_le32((u32)NES_MGT_CTX_SIZE |
 973                                     ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
 974                 nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
 975                           nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
 976                           nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
 977                 if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0)
 978                         mgt_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
 979
 980                 u64temp = (u64)mgtvnic->mgt.rq_pbase;
 981                 mgt_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
 982                 mgt_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
 983                 u64temp = (u64)mgtvnic->mgt.rq_pbase;
 984                 mgt_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
 985                 mgt_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
 986
 987                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
 988                                                                          NES_CQP_QP_TYPE_NIC);
 989                 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(mgtvnic->mgt.qp_id);
 990                 u64temp = (u64)mgtvnic->mgt_cq.cq_pbase +
 991                           (mgtvnic->mgt_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
 992                 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
 993
 994                 if (++cqp_head >= nesdev->cqp.sq_size)
 995                         cqp_head = 0;
 996                 nesdev->cqp.sq_head = cqp_head;
 997
 998                 barrier();
 999
1000                 /* Ring doorbell (2 WQEs) */
1001                 nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1002
1003                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1004                 nes_debug(NES_DBG_INIT, "Waiting for create MGT QP%u to complete.\n",
1005                           mgtvnic->mgt.qp_id);
1006
1007                 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1008                                          NES_EVENT_TIMEOUT);
1009                 nes_debug(NES_DBG_INIT, "Create MGT QP%u completed, wait_event_timeout ret = %u.\n",
1010                           mgtvnic->mgt.qp_id, ret);
1011                 if (!ret) {
1012                         nes_debug(NES_DBG_INIT, "MGT QP%u create timeout expired\n", mgtvnic->mgt.qp_id);
1013                         if (i == 0) {
1014                                 pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase,
1015                                                     nesvnic->mgt_pbase);
1016                                 kfree(mgtvnic);
1017                         } else {
1018                                 nes_destroy_mgt(nesvnic);
1019                         }
1020                         return -EIO;
1021                 }
1022
1023                 /* Populate the RQ */
1024                 for (counter = 0; counter < (NES_MGT_WQ_COUNT - 1); counter++) {
1025                         skb = dev_alloc_skb(nesvnic->max_frame_size);
1026                         if (!skb) {
1027                                 nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1028                                 return -ENOMEM;
1029                         }
1030
1031                         skb->dev = netdev;
1032
1033                         pmem = pci_map_single(nesdev->pcidev, skb->data,
1034                                               nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1035                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1036                         cb->busaddr = pmem;
1037                         cb->maplen = nesvnic->max_frame_size;
1038
1039                         mgt_rqe = &mgtvnic->mgt.rq_vbase[counter];
1040                         mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32((u32)nesvnic->max_frame_size);
1041                         mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1042                         mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1043                         mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1044                         mgtvnic->mgt.rx_skb[counter] = skb;
1045                 }
1046
1047                 init_timer(&mgtvnic->rq_wqes_timer);
1048                 mgtvnic->rq_wqes_timer.function = nes_mgt_rq_wqes_timeout;
1049                 mgtvnic->rq_wqes_timer.data = (unsigned long)mgtvnic;
1050
1051                 wqe_count = NES_MGT_WQ_COUNT - 1;
1052                 mgtvnic->mgt.rq_head = wqe_count;
1053                 barrier();
1054                 do {
1055                         counter = min(wqe_count, ((u32)255));
1056                         wqe_count -= counter;
1057                         nes_write32(nesdev->regs + NES_WQE_ALLOC, (counter << 24) | mgtvnic->mgt.qp_id);
1058                 } while (wqe_count);
1059
1060                 nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
1061                             mgtvnic->mgt_cq.cq_number);
1062                 nes_read32(nesdev->regs + NES_CQE_ALLOC);
1063
1064                 mgt_vbase += mgt_mem_size;
1065                 mgt_pbase += mgt_mem_size;
1066                 nesvnic->mgtvnic[i] = mgtvnic++;
1067         }
1068         return 0;
1069 }
1070
1071
1072 void nes_destroy_mgt(struct nes_vnic *nesvnic)
1073 {
1074         struct nes_device *nesdev = nesvnic->nesdev;
1075         struct nes_vnic_mgt *mgtvnic;
1076         struct nes_vnic_mgt *first_mgtvnic;
1077         unsigned long flags;
1078         struct nes_hw_cqp_wqe *cqp_wqe;
1079         u32 cqp_head;
1080         struct sk_buff *rx_skb;
1081         int i;
1082         int ret;
1083
1084         kthread_stop(nesvnic->mgt_thread);
1085
1086         /* Free remaining NIC receive buffers */
1087         first_mgtvnic = nesvnic->mgtvnic[0];
1088         for (i = 0; i < NES_MGT_QP_COUNT; i++) {
1089                 mgtvnic = nesvnic->mgtvnic[i];
1090                 if (mgtvnic == NULL)
1091                         continue;
1092
1093                 while (mgtvnic->mgt.rq_head != mgtvnic->mgt.rq_tail) {
1094                         rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail];
1095                         nes_mgt_free_skb(nesdev, rx_skb, PCI_DMA_FROMDEVICE);
1096                         mgtvnic->mgt.rq_tail++;
1097                         mgtvnic->mgt.rq_tail &= (mgtvnic->mgt.rq_size - 1);
1098                 }
1099
1100                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1101
1102                 /* Destroy NIC QP */
1103                 cqp_head = nesdev->cqp.sq_head;
1104                 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1105                 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1106
1107                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1108                                     (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1109                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1110                                     mgtvnic->mgt.qp_id);
1111
1112                 if (++cqp_head >= nesdev->cqp.sq_size)
1113                         cqp_head = 0;
1114
1115                 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1116
1117                 /* Destroy NIC CQ */
1118                 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1119                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1120                                     (NES_CQP_DESTROY_CQ | ((u32)mgtvnic->mgt_cq.cq_size << 16)));
1121                 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1122                                     (mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16)));
1123
1124                 if (++cqp_head >= nesdev->cqp.sq_size)
1125                         cqp_head = 0;
1126
1127                 nesdev->cqp.sq_head = cqp_head;
1128                 barrier();
1129
1130                 /* Ring doorbell (2 WQEs) */
1131                 nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1132
1133                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1134                 nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1135                           " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1136                           cqp_head, nesdev->cqp.sq_head,
1137                           nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1138
1139                 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1140                                          NES_EVENT_TIMEOUT);
1141
1142                 nes_debug(NES_DBG_SHUTDOWN, "Destroy MGT QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1143                           " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1144                           ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1145                 if (!ret)
1146                         nes_debug(NES_DBG_SHUTDOWN, "MGT QP%u destroy timeout expired\n",
1147                                   mgtvnic->mgt.qp_id);
1148
1149                 nesvnic->mgtvnic[i] = NULL;
1150         }
1151
1152         if (nesvnic->mgt_vbase) {
1153                 pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase,
1154                                     nesvnic->mgt_pbase);
1155                 nesvnic->mgt_vbase = NULL;
1156                 nesvnic->mgt_pbase = 0;
1157         }
1158
1159         kfree(first_mgtvnic);
1160 }