kernel/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/module.h>
  36 #include <linux/moduleparam.h>
  37 #include <linux/init.h>
  38 #include <linux/pci.h>
  39 #include <linux/dma-mapping.h>
  40 #include <linux/netdevice.h>
  41 #include <linux/etherdevice.h>
  42 #include <linux/if_vlan.h>
  43 #include <linux/mdio.h>
  44 #include <linux/sockios.h>
  45 #include <linux/workqueue.h>
  46 #include <linux/proc_fs.h>
  47 #include <linux/rtnetlink.h>
  48 #include <linux/firmware.h>
  49 #include <linux/log2.h>
  50 #include <linux/stringify.h>
  51 #include <linux/sched.h>
  52 #include <linux/slab.h>
  53 #include <asm/uaccess.h>
  54
  55 #include "common.h"
  56 #include "cxgb3_ioctl.h"
  57 #include "regs.h"
  58 #include "cxgb3_offload.h"
  59 #include "version.h"
  60
  61 #include "cxgb3_ctl_defs.h"
  62 #include "t3_cpl.h"
  63 #include "firmware_exports.h"
  64
  65 enum {
  66         MAX_TXQ_ENTRIES = 16384,
  67         MAX_CTRL_TXQ_ENTRIES = 1024,
  68         MAX_RSPQ_ENTRIES = 16384,
  69         MAX_RX_BUFFERS = 16384,
  70         MAX_RX_JUMBO_BUFFERS = 16384,
  71         MIN_TXQ_ENTRIES = 4,
  72         MIN_CTRL_TXQ_ENTRIES = 4,
  73         MIN_RSPQ_ENTRIES = 32,
  74         MIN_FL_ENTRIES = 32
  75 };
  76
  77 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  78
  79 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  80                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  81                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  82
  83 #define EEPROM_MAGIC 0x38E2F10C
  84
  85 #define CH_DEVICE(devid, idx) \
  86         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  87
  88 static const struct pci_device_id cxgb3_pci_tbl[] = {
  89         CH_DEVICE(0x20, 0),     /* PE9000 */
  90         CH_DEVICE(0x21, 1),     /* T302E */
  91         CH_DEVICE(0x22, 2),     /* T310E */
  92         CH_DEVICE(0x23, 3),     /* T320X */
  93         CH_DEVICE(0x24, 1),     /* T302X */
  94         CH_DEVICE(0x25, 3),     /* T320E */
  95         CH_DEVICE(0x26, 2),     /* T310X */
  96         CH_DEVICE(0x30, 2),     /* T3B10 */
  97         CH_DEVICE(0x31, 3),     /* T3B20 */
  98         CH_DEVICE(0x32, 1),     /* T3B02 */
  99         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
 100         CH_DEVICE(0x36, 3),     /* S320E-CR */
 101         CH_DEVICE(0x37, 7),     /* N320E-G2 */
 102         {0,}
 103 };
 104
 105 MODULE_DESCRIPTION(DRV_DESC);
 106 MODULE_AUTHOR("Chelsio Communications");
 107 MODULE_LICENSE("Dual BSD/GPL");
 108 MODULE_VERSION(DRV_VERSION);
 109 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 110
 111 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 112
 113 module_param(dflt_msg_enable, int, 0644);
 114 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 115
 116 /*
 117  * The driver uses the best interrupt scheme available on a platform in the
 118  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 119  * of these schemes the driver may consider as follows:
 120  *
 121  * msi = 2: choose from among all three options
 122  * msi = 1: only consider MSI and pin interrupts
 123  * msi = 0: force pin interrupts
 124  */
 125 static int msi = 2;
 126
 127 module_param(msi, int, 0644);
 128 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 129
 130 /*
 131  * The driver enables offload as a default.
 132  * To disable it, use ofld_disable = 1.
 133  */
 134
 135 static int ofld_disable = 0;
 136
 137 module_param(ofld_disable, int, 0644);
 138 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 139
 140 /*
 141  * We have work elements that we need to cancel when an interface is taken
 142  * down.  Normally the work elements would be executed by keventd but that
 143  * can deadlock because of linkwatch.  If our close method takes the rtnl
 144  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 145  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 146  * for our work to complete.  Get our own work queue to solve this.
 147  */
 148 struct workqueue_struct *cxgb3_wq;
 149
 150 /**
 151  *      link_report - show link status and link speed/duplex
 152  *      @p: the port whose settings are to be reported
 153  *
 154  *      Shows the link status, speed, and duplex of a port.
 155  */
 156 static void link_report(struct net_device *dev)
 157 {
 158         if (!netif_carrier_ok(dev))
 159                 netdev_info(dev, "link down\n");
 160         else {
 161                 const char *s = "10Mbps";
 162                 const struct port_info *p = netdev_priv(dev);
 163
 164                 switch (p->link_config.speed) {
 165                 case SPEED_10000:
 166                         s = "10Gbps";
 167                         break;
 168                 case SPEED_1000:
 169                         s = "1000Mbps";
 170                         break;
 171                 case SPEED_100:
 172                         s = "100Mbps";
 173                         break;
 174                 }
 175
 176                 netdev_info(dev, "link up, %s, %s-duplex\n",
 177                             s, p->link_config.duplex == DUPLEX_FULL
 178                             ? "full" : "half");
 179         }
 180 }
 181
 182 static void enable_tx_fifo_drain(struct adapter *adapter,
 183                                  struct port_info *pi)
 184 {
 185         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 186                          F_ENDROPPKT);
 187         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 188         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 189         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 190 }
 191
 192 static void disable_tx_fifo_drain(struct adapter *adapter,
 193                                   struct port_info *pi)
 194 {
 195         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 196                          F_ENDROPPKT, 0);
 197 }
 198
 199 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 200 {
 201         struct net_device *dev = adap->port[port_id];
 202         struct port_info *pi = netdev_priv(dev);
 203
 204         if (state == netif_carrier_ok(dev))
 205                 return;
 206
 207         if (state) {
 208                 struct cmac *mac = &pi->mac;
 209
 210                 netif_carrier_on(dev);
 211
 212                 disable_tx_fifo_drain(adap, pi);
 213
 214                 /* Clear local faults */
 215                 t3_xgm_intr_disable(adap, pi->port_id);
 216                 t3_read_reg(adap, A_XGM_INT_STATUS +
 217                                     pi->mac.offset);
 218                 t3_write_reg(adap,
 219                              A_XGM_INT_CAUSE + pi->mac.offset,
 220                              F_XGM_INT);
 221
 222                 t3_set_reg_field(adap,
 223                                  A_XGM_INT_ENABLE +
 224                                  pi->mac.offset,
 225                                  F_XGM_INT, F_XGM_INT);
 226                 t3_xgm_intr_enable(adap, pi->port_id);
 227
 228                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 229         } else {
 230                 netif_carrier_off(dev);
 231
 232                 /* Flush TX FIFO */
 233                 enable_tx_fifo_drain(adap, pi);
 234         }
 235         link_report(dev);
 236 }
 237
 238 /**
 239  *      t3_os_link_changed - handle link status changes
 240  *      @adapter: the adapter associated with the link change
 241  *      @port_id: the port index whose limk status has changed
 242  *      @link_stat: the new status of the link
 243  *      @speed: the new speed setting
 244  *      @duplex: the new duplex setting
 245  *      @pause: the new flow-control setting
 246  *
 247  *      This is the OS-dependent handler for link status changes.  The OS
 248  *      neutral handler takes care of most of the processing for these events,
 249  *      then calls this handler for any OS-specific processing.
 250  */
 251 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 252                         int speed, int duplex, int pause)
 253 {
 254         struct net_device *dev = adapter->port[port_id];
 255         struct port_info *pi = netdev_priv(dev);
 256         struct cmac *mac = &pi->mac;
 257
 258         /* Skip changes from disabled ports. */
 259         if (!netif_running(dev))
 260                 return;
 261
 262         if (link_stat != netif_carrier_ok(dev)) {
 263                 if (link_stat) {
 264                         disable_tx_fifo_drain(adapter, pi);
 265
 266                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 267
 268                         /* Clear local faults */
 269                         t3_xgm_intr_disable(adapter, pi->port_id);
 270                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 271                                     pi->mac.offset);
 272                         t3_write_reg(adapter,
 273                                      A_XGM_INT_CAUSE + pi->mac.offset,
 274                                      F_XGM_INT);
 275
 276                         t3_set_reg_field(adapter,
 277                                          A_XGM_INT_ENABLE + pi->mac.offset,
 278                                          F_XGM_INT, F_XGM_INT);
 279                         t3_xgm_intr_enable(adapter, pi->port_id);
 280
 281                         netif_carrier_on(dev);
 282                 } else {
 283                         netif_carrier_off(dev);
 284
 285                         t3_xgm_intr_disable(adapter, pi->port_id);
 286                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 287                         t3_set_reg_field(adapter,
 288                                          A_XGM_INT_ENABLE + pi->mac.offset,
 289                                          F_XGM_INT, 0);
 290
 291                         if (is_10G(adapter))
 292                                 pi->phy.ops->power_down(&pi->phy, 1);
 293
 294                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 295                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 296                         t3_link_start(&pi->phy, mac, &pi->link_config);
 297
 298                         /* Flush TX FIFO */
 299                         enable_tx_fifo_drain(adapter, pi);
 300                 }
 301
 302                 link_report(dev);
 303         }
 304 }
 305
 306 /**
 307  *      t3_os_phymod_changed - handle PHY module changes
 308  *      @phy: the PHY reporting the module change
 309  *      @mod_type: new module type
 310  *
 311  *      This is the OS-dependent handler for PHY module changes.  It is
 312  *      invoked when a PHY module is removed or inserted for any OS-specific
 313  *      processing.
 314  */
 315 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 316 {
 317         static const char *mod_str[] = {
 318                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 319         };
 320
 321         const struct net_device *dev = adap->port[port_id];
 322         const struct port_info *pi = netdev_priv(dev);
 323
 324         if (pi->phy.modtype == phy_modtype_none)
 325                 netdev_info(dev, "PHY module unplugged\n");
 326         else
 327                 netdev_info(dev, "%s PHY module inserted\n",
 328                             mod_str[pi->phy.modtype]);
 329 }
 330
 331 static void cxgb_set_rxmode(struct net_device *dev)
 332 {
 333         struct port_info *pi = netdev_priv(dev);
 334
 335         t3_mac_set_rx_mode(&pi->mac, dev);
 336 }
 337
 338 /**
 339  *      link_start - enable a port
 340  *      @dev: the device to enable
 341  *
 342  *      Performs the MAC and PHY actions needed to enable a port.
 343  */
 344 static void link_start(struct net_device *dev)
 345 {
 346         struct port_info *pi = netdev_priv(dev);
 347         struct cmac *mac = &pi->mac;
 348
 349         t3_mac_reset(mac);
 350         t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
 351         t3_mac_set_mtu(mac, dev->mtu);
 352         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
 353         t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
 354         t3_mac_set_rx_mode(mac, dev);
 355         t3_link_start(&pi->phy, mac, &pi->link_config);
 356         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 357 }
 358
 359 static inline void cxgb_disable_msi(struct adapter *adapter)
 360 {
 361         if (adapter->flags & USING_MSIX) {
 362                 pci_disable_msix(adapter->pdev);
 363                 adapter->flags &= ~USING_MSIX;
 364         } else if (adapter->flags & USING_MSI) {
 365                 pci_disable_msi(adapter->pdev);
 366                 adapter->flags &= ~USING_MSI;
 367         }
 368 }
 369
 370 /*
 371  * Interrupt handler for asynchronous events used with MSI-X.
 372  */
 373 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 374 {
 375         t3_slow_intr_handler(cookie);
 376         return IRQ_HANDLED;
 377 }
 378
 379 /*
 380  * Name the MSI-X interrupts.
 381  */
 382 static void name_msix_vecs(struct adapter *adap)
 383 {
 384         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 385
 386         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 387         adap->msix_info[0].desc[n] = 0;
 388
 389         for_each_port(adap, j) {
 390                 struct net_device *d = adap->port[j];
 391                 const struct port_info *pi = netdev_priv(d);
 392
 393                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 394                         snprintf(adap->msix_info[msi_idx].desc, n,
 395                                  "%s-%d", d->name, pi->first_qset + i);
 396                         adap->msix_info[msi_idx].desc[n] = 0;
 397                 }
 398         }
 399 }
 400
 401 static int request_msix_data_irqs(struct adapter *adap)
 402 {
 403         int i, j, err, qidx = 0;
 404
 405         for_each_port(adap, i) {
 406                 int nqsets = adap2pinfo(adap, i)->nqsets;
 407
 408                 for (j = 0; j < nqsets; ++j) {
 409                         err = request_irq(adap->msix_info[qidx + 1].vec,
 410                                           t3_intr_handler(adap,
 411                                                           adap->sge.qs[qidx].
 412                                                           rspq.polling), 0,
 413                                           adap->msix_info[qidx + 1].desc,
 414                                           &adap->sge.qs[qidx]);
 415                         if (err) {
 416                                 while (--qidx >= 0)
 417                                         free_irq(adap->msix_info[qidx + 1].vec,
 418                                                  &adap->sge.qs[qidx]);
 419                                 return err;
 420                         }
 421                         qidx++;
 422                 }
 423         }
 424         return 0;
 425 }
 426
 427 static void free_irq_resources(struct adapter *adapter)
 428 {
 429         if (adapter->flags & USING_MSIX) {
 430                 int i, n = 0;
 431
 432                 free_irq(adapter->msix_info[0].vec, adapter);
 433                 for_each_port(adapter, i)
 434                         n += adap2pinfo(adapter, i)->nqsets;
 435
 436                 for (i = 0; i < n; ++i)
 437                         free_irq(adapter->msix_info[i + 1].vec,
 438                                  &adapter->sge.qs[i]);
 439         } else
 440                 free_irq(adapter->pdev->irq, adapter);
 441 }
 442
 443 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 444                               unsigned long n)
 445 {
 446         int attempts = 10;
 447
 448         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 449                 if (!--attempts)
 450                         return -ETIMEDOUT;
 451                 msleep(10);
 452         }
 453         return 0;
 454 }
 455
 456 static int init_tp_parity(struct adapter *adap)
 457 {
 458         int i;
 459         struct sk_buff *skb;
 460         struct cpl_set_tcb_field *greq;
 461         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 462
 463         t3_tp_set_offload_mode(adap, 1);
 464
 465         for (i = 0; i < 16; i++) {
 466                 struct cpl_smt_write_req *req;
 467
 468                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 469                 if (!skb)
 470                         skb = adap->nofail_skb;
 471                 if (!skb)
 472                         goto alloc_skb_fail;
 473
 474                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 475                 memset(req, 0, sizeof(*req));
 476                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 477                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 478                 req->mtu_idx = NMTUS - 1;
 479                 req->iff = i;
 480                 t3_mgmt_tx(adap, skb);
 481                 if (skb == adap->nofail_skb) {
 482                         await_mgmt_replies(adap, cnt, i + 1);
 483                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 484                         if (!adap->nofail_skb)
 485                                 goto alloc_skb_fail;
 486                 }
 487         }
 488
 489         for (i = 0; i < 2048; i++) {
 490                 struct cpl_l2t_write_req *req;
 491
 492                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 493                 if (!skb)
 494                         skb = adap->nofail_skb;
 495                 if (!skb)
 496                         goto alloc_skb_fail;
 497
 498                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 499                 memset(req, 0, sizeof(*req));
 500                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 501                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 502                 req->params = htonl(V_L2T_W_IDX(i));
 503                 t3_mgmt_tx(adap, skb);
 504                 if (skb == adap->nofail_skb) {
 505                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 506                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 507                         if (!adap->nofail_skb)
 508                                 goto alloc_skb_fail;
 509                 }
 510         }
 511
 512         for (i = 0; i < 2048; i++) {
 513                 struct cpl_rte_write_req *req;
 514
 515                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 516                 if (!skb)
 517                         skb = adap->nofail_skb;
 518                 if (!skb)
 519                         goto alloc_skb_fail;
 520
 521                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 522                 memset(req, 0, sizeof(*req));
 523                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 524                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 525                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 526                 t3_mgmt_tx(adap, skb);
 527                 if (skb == adap->nofail_skb) {
 528                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 529                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 530                         if (!adap->nofail_skb)
 531                                 goto alloc_skb_fail;
 532                 }
 533         }
 534
 535         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 536         if (!skb)
 537                 skb = adap->nofail_skb;
 538         if (!skb)
 539                 goto alloc_skb_fail;
 540
 541         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 542         memset(greq, 0, sizeof(*greq));
 543         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 544         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 545         greq->mask = cpu_to_be64(1);
 546         t3_mgmt_tx(adap, skb);
 547
 548         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 549         if (skb == adap->nofail_skb) {
 550                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 551                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 552         }
 553
 554         t3_tp_set_offload_mode(adap, 0);
 555         return i;
 556
 557 alloc_skb_fail:
 558         t3_tp_set_offload_mode(adap, 0);
 559         return -ENOMEM;
 560 }
 561
 562 /**
 563  *      setup_rss - configure RSS
 564  *      @adap: the adapter
 565  *
 566  *      Sets up RSS to distribute packets to multiple receive queues.  We
 567  *      configure the RSS CPU lookup table to distribute to the number of HW
 568  *      receive queues, and the response queue lookup table to narrow that
 569  *      down to the response queues actually configured for each port.
 570  *      We always configure the RSS mapping for two ports since the mapping
 571  *      table has plenty of entries.
 572  */
 573 static void setup_rss(struct adapter *adap)
 574 {
 575         int i;
 576         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 577         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 578         u8 cpus[SGE_QSETS + 1];
 579         u16 rspq_map[RSS_TABLE_SIZE];
 580
 581         for (i = 0; i < SGE_QSETS; ++i)
 582                 cpus[i] = i;
 583         cpus[SGE_QSETS] = 0xff; /* terminator */
 584
 585         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 586                 rspq_map[i] = i % nq0;
 587                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 588         }
 589
 590         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 591                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 592                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 593 }
 594
 595 static void ring_dbs(struct adapter *adap)
 596 {
 597         int i, j;
 598
 599         for (i = 0; i < SGE_QSETS; i++) {
 600                 struct sge_qset *qs = &adap->sge.qs[i];
 601
 602                 if (qs->adap)
 603                         for (j = 0; j < SGE_TXQ_PER_SET; j++)
 604                                 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
 605         }
 606 }
 607
 608 static void init_napi(struct adapter *adap)
 609 {
 610         int i;
 611
 612         for (i = 0; i < SGE_QSETS; i++) {
 613                 struct sge_qset *qs = &adap->sge.qs[i];
 614
 615                 if (qs->adap)
 616                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 617                                        64);
 618         }
 619
 620         /*
 621          * netif_napi_add() can be called only once per napi_struct because it
 622          * adds each new napi_struct to a list.  Be careful not to call it a
 623          * second time, e.g., during EEH recovery, by making a note of it.
 624          */
 625         adap->flags |= NAPI_INIT;
 626 }
 627
 628 /*
 629  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 630  * both netdevices representing interfaces and the dummy ones for the extra
 631  * queues.
 632  */
 633 static void quiesce_rx(struct adapter *adap)
 634 {
 635         int i;
 636
 637         for (i = 0; i < SGE_QSETS; i++)
 638                 if (adap->sge.qs[i].adap)
 639                         napi_disable(&adap->sge.qs[i].napi);
 640 }
 641
 642 static void enable_all_napi(struct adapter *adap)
 643 {
 644         int i;
 645         for (i = 0; i < SGE_QSETS; i++)
 646                 if (adap->sge.qs[i].adap)
 647                         napi_enable(&adap->sge.qs[i].napi);
 648 }
 649
 650 /**
 651  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 652  *      @adap: the adapter
 653  *
 654  *      Determines how many sets of SGE queues to use and initializes them.
 655  *      We support multiple queue sets per port if we have MSI-X, otherwise
 656  *      just one queue set per port.
 657  */
 658 static int setup_sge_qsets(struct adapter *adap)
 659 {
 660         int i, j, err, irq_idx = 0, qset_idx = 0;
 661         unsigned int ntxq = SGE_TXQ_PER_SET;
 662
 663         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 664                 irq_idx = -1;
 665
 666         for_each_port(adap, i) {
 667                 struct net_device *dev = adap->port[i];
 668                 struct port_info *pi = netdev_priv(dev);
 669
 670                 pi->qs = &adap->sge.qs[pi->first_qset];
 671                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 672                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 673                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 674                                                              irq_idx,
 675                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 676                                 netdev_get_tx_queue(dev, j));
 677                         if (err) {
 678                                 t3_free_sge_resources(adap);
 679                                 return err;
 680                         }
 681                 }
 682         }
 683
 684         return 0;
 685 }
 686
 687 static ssize_t attr_show(struct device *d, char *buf,
 688                          ssize_t(*format) (struct net_device *, char *))
 689 {
 690         ssize_t len;
 691
 692         /* Synchronize with ioctls that may shut down the device */
 693         rtnl_lock();
 694         len = (*format) (to_net_dev(d), buf);
 695         rtnl_unlock();
 696         return len;
 697 }
 698
 699 static ssize_t attr_store(struct device *d,
 700                           const char *buf, size_t len,
 701                           ssize_t(*set) (struct net_device *, unsigned int),
 702                           unsigned int min_val, unsigned int max_val)
 703 {
 704         char *endp;
 705         ssize_t ret;
 706         unsigned int val;
 707
 708         if (!capable(CAP_NET_ADMIN))
 709                 return -EPERM;
 710
 711         val = simple_strtoul(buf, &endp, 0);
 712         if (endp == buf || val < min_val || val > max_val)
 713                 return -EINVAL;
 714
 715         rtnl_lock();
 716         ret = (*set) (to_net_dev(d), val);
 717         if (!ret)
 718                 ret = len;
 719         rtnl_unlock();
 720         return ret;
 721 }
 722
 723 #define CXGB3_SHOW(name, val_expr) \
 724 static ssize_t format_##name(struct net_device *dev, char *buf) \
 725 { \
 726         struct port_info *pi = netdev_priv(dev); \
 727         struct adapter *adap = pi->adapter; \
 728         return sprintf(buf, "%u\n", val_expr); \
 729 } \
 730 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 731                            char *buf) \
 732 { \
 733         return attr_show(d, buf, format_##name); \
 734 }
 735
 736 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 737 {
 738         struct port_info *pi = netdev_priv(dev);
 739         struct adapter *adap = pi->adapter;
 740         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 741
 742         if (adap->flags & FULL_INIT_DONE)
 743                 return -EBUSY;
 744         if (val && adap->params.rev == 0)
 745                 return -EINVAL;
 746         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 747             min_tids)
 748                 return -EINVAL;
 749         adap->params.mc5.nfilters = val;
 750         return 0;
 751 }
 752
 753 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 754                               const char *buf, size_t len)
 755 {
 756         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 757 }
 758
 759 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 760 {
 761         struct port_info *pi = netdev_priv(dev);
 762         struct adapter *adap = pi->adapter;
 763
 764         if (adap->flags & FULL_INIT_DONE)
 765                 return -EBUSY;
 766         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 767             MC5_MIN_TIDS)
 768                 return -EINVAL;
 769         adap->params.mc5.nservers = val;
 770         return 0;
 771 }
 772
 773 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 774                               const char *buf, size_t len)
 775 {
 776         return attr_store(d, buf, len, set_nservers, 0, ~0);
 777 }
 778
 779 #define CXGB3_ATTR_R(name, val_expr) \
 780 CXGB3_SHOW(name, val_expr) \
 781 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 782
 783 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 784 CXGB3_SHOW(name, val_expr) \
 785 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 786
 787 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 788 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 789 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 790
 791 static struct attribute *cxgb3_attrs[] = {
 792         &dev_attr_cam_size.attr,
 793         &dev_attr_nfilters.attr,
 794         &dev_attr_nservers.attr,
 795         NULL
 796 };
 797
 798 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 799
 800 static ssize_t tm_attr_show(struct device *d,
 801                             char *buf, int sched)
 802 {
 803         struct port_info *pi = netdev_priv(to_net_dev(d));
 804         struct adapter *adap = pi->adapter;
 805         unsigned int v, addr, bpt, cpt;
 806         ssize_t len;
 807
 808         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 809         rtnl_lock();
 810         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 811         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 812         if (sched & 1)
 813                 v >>= 16;
 814         bpt = (v >> 8) & 0xff;
 815         cpt = v & 0xff;
 816         if (!cpt)
 817                 len = sprintf(buf, "disabled\n");
 818         else {
 819                 v = (adap->params.vpd.cclk * 1000) / cpt;
 820                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 821         }
 822         rtnl_unlock();
 823         return len;
 824 }
 825
 826 static ssize_t tm_attr_store(struct device *d,
 827                              const char *buf, size_t len, int sched)
 828 {
 829         struct port_info *pi = netdev_priv(to_net_dev(d));
 830         struct adapter *adap = pi->adapter;
 831         unsigned int val;
 832         char *endp;
 833         ssize_t ret;
 834
 835         if (!capable(CAP_NET_ADMIN))
 836                 return -EPERM;
 837
 838         val = simple_strtoul(buf, &endp, 0);
 839         if (endp == buf || val > 10000000)
 840                 return -EINVAL;
 841
 842         rtnl_lock();
 843         ret = t3_config_sched(adap, val, sched);
 844         if (!ret)
 845                 ret = len;
 846         rtnl_unlock();
 847         return ret;
 848 }
 849
 850 #define TM_ATTR(name, sched) \
 851 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 852                            char *buf) \
 853 { \
 854         return tm_attr_show(d, buf, sched); \
 855 } \
 856 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 857                             const char *buf, size_t len) \
 858 { \
 859         return tm_attr_store(d, buf, len, sched); \
 860 } \
 861 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 862
 863 TM_ATTR(sched0, 0);
 864 TM_ATTR(sched1, 1);
 865 TM_ATTR(sched2, 2);
 866 TM_ATTR(sched3, 3);
 867 TM_ATTR(sched4, 4);
 868 TM_ATTR(sched5, 5);
 869 TM_ATTR(sched6, 6);
 870 TM_ATTR(sched7, 7);
 871
 872 static struct attribute *offload_attrs[] = {
 873         &dev_attr_sched0.attr,
 874         &dev_attr_sched1.attr,
 875         &dev_attr_sched2.attr,
 876         &dev_attr_sched3.attr,
 877         &dev_attr_sched4.attr,
 878         &dev_attr_sched5.attr,
 879         &dev_attr_sched6.attr,
 880         &dev_attr_sched7.attr,
 881         NULL
 882 };
 883
 884 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 885
 886 /*
 887  * Sends an sk_buff to an offload queue driver
 888  * after dealing with any active network taps.
 889  */
 890 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 891 {
 892         int ret;
 893
 894         local_bh_disable();
 895         ret = t3_offload_tx(tdev, skb);
 896         local_bh_enable();
 897         return ret;
 898 }
 899
 900 static int write_smt_entry(struct adapter *adapter, int idx)
 901 {
 902         struct cpl_smt_write_req *req;
 903         struct port_info *pi = netdev_priv(adapter->port[idx]);
 904         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 905
 906         if (!skb)
 907                 return -ENOMEM;
 908
 909         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 910         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 911         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 912         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 913         req->iff = idx;
 914         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 915         memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
 916         skb->priority = 1;
 917         offload_tx(&adapter->tdev, skb);
 918         return 0;
 919 }
 920
 921 static int init_smt(struct adapter *adapter)
 922 {
 923         int i;
 924
 925         for_each_port(adapter, i)
 926             write_smt_entry(adapter, i);
 927         return 0;
 928 }
 929
 930 static void init_port_mtus(struct adapter *adapter)
 931 {
 932         unsigned int mtus = adapter->port[0]->mtu;
 933
 934         if (adapter->port[1])
 935                 mtus |= adapter->port[1]->mtu << 16;
 936         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 937 }
 938
 939 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 940                               int hi, int port)
 941 {
 942         struct sk_buff *skb;
 943         struct mngt_pktsched_wr *req;
 944         int ret;
 945
 946         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 947         if (!skb)
 948                 skb = adap->nofail_skb;
 949         if (!skb)
 950                 return -ENOMEM;
 951
 952         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 953         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 954         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 955         req->sched = sched;
 956         req->idx = qidx;
 957         req->min = lo;
 958         req->max = hi;
 959         req->binding = port;
 960         ret = t3_mgmt_tx(adap, skb);
 961         if (skb == adap->nofail_skb) {
 962                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 963                                              GFP_KERNEL);
 964                 if (!adap->nofail_skb)
 965                         ret = -ENOMEM;
 966         }
 967
 968         return ret;
 969 }
 970
 971 static int bind_qsets(struct adapter *adap)
 972 {
 973         int i, j, err = 0;
 974
 975         for_each_port(adap, i) {
 976                 const struct port_info *pi = adap2pinfo(adap, i);
 977
 978                 for (j = 0; j < pi->nqsets; ++j) {
 979                         int ret = send_pktsched_cmd(adap, 1,
 980                                                     pi->first_qset + j, -1,
 981                                                     -1, i);
 982                         if (ret)
 983                                 err = ret;
 984                 }
 985         }
 986
 987         return err;
 988 }
 989
 990 #define FW_VERSION __stringify(FW_VERSION_MAJOR) "."                    \
 991         __stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
 992 #define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
 993 #define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "."                \
 994         __stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
 995 #define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
 996 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
 997 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
 998 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
 999 MODULE_FIRMWARE(FW_FNAME);
1000 MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
1001 MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
1002 MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
1003 MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1004 MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
1005
1006 static inline const char *get_edc_fw_name(int edc_idx)
1007 {
1008         const char *fw_name = NULL;
1009
1010         switch (edc_idx) {
1011         case EDC_OPT_AEL2005:
1012                 fw_name = AEL2005_OPT_EDC_NAME;
1013                 break;
1014         case EDC_TWX_AEL2005:
1015                 fw_name = AEL2005_TWX_EDC_NAME;
1016                 break;
1017         case EDC_TWX_AEL2020:
1018                 fw_name = AEL2020_TWX_EDC_NAME;
1019                 break;
1020         }
1021         return fw_name;
1022 }
1023
1024 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1025 {
1026         struct adapter *adapter = phy->adapter;
1027         const struct firmware *fw;
1028         char buf[64];
1029         u32 csum;
1030         const __be32 *p;
1031         u16 *cache = phy->phy_cache;
1032         int i, ret;
1033
1034         snprintf(buf, sizeof(buf), get_edc_fw_name(edc_idx));
1035
1036         ret = request_firmware(&fw, buf, &adapter->pdev->dev);
1037         if (ret < 0) {
1038                 dev_err(&adapter->pdev->dev,
1039                         "could not upgrade firmware: unable to load %s\n",
1040                         buf);
1041                 return ret;
1042         }
1043
1044         /* check size, take checksum in account */
1045         if (fw->size > size + 4) {
1046                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1047                        (unsigned int)fw->size, size + 4);
1048                 ret = -EINVAL;
1049         }
1050
1051         /* compute checksum */
1052         p = (const __be32 *)fw->data;
1053         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1054                 csum += ntohl(p[i]);
1055
1056         if (csum != 0xffffffff) {
1057                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1058                        csum);
1059                 ret = -EINVAL;
1060         }
1061
1062         for (i = 0; i < size / 4 ; i++) {
1063                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1064                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1065         }
1066
1067         release_firmware(fw);
1068
1069         return ret;
1070 }
1071
1072 static int upgrade_fw(struct adapter *adap)
1073 {
1074         int ret;
1075         const struct firmware *fw;
1076         struct device *dev = &adap->pdev->dev;
1077
1078         ret = request_firmware(&fw, FW_FNAME, dev);
1079         if (ret < 0) {
1080                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1081                         FW_FNAME);
1082                 return ret;
1083         }
1084         ret = t3_load_fw(adap, fw->data, fw->size);
1085         release_firmware(fw);
1086
1087         if (ret == 0)
1088                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1089                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1090         else
1091                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1092                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1093
1094         return ret;
1095 }
1096
1097 static inline char t3rev2char(struct adapter *adapter)
1098 {
1099         char rev = 0;
1100
1101         switch(adapter->params.rev) {
1102         case T3_REV_B:
1103         case T3_REV_B2:
1104                 rev = 'b';
1105                 break;
1106         case T3_REV_C:
1107                 rev = 'c';
1108                 break;
1109         }
1110         return rev;
1111 }
1112
1113 static int update_tpsram(struct adapter *adap)
1114 {
1115         const struct firmware *tpsram;
1116         char buf[64];
1117         struct device *dev = &adap->pdev->dev;
1118         int ret;
1119         char rev;
1120
1121         rev = t3rev2char(adap);
1122         if (!rev)
1123                 return 0;
1124
1125         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1126
1127         ret = request_firmware(&tpsram, buf, dev);
1128         if (ret < 0) {
1129                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1130                         buf);
1131                 return ret;
1132         }
1133
1134         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1135         if (ret)
1136                 goto release_tpsram;
1137
1138         ret = t3_set_proto_sram(adap, tpsram->data);
1139         if (ret == 0)
1140                 dev_info(dev,
1141                          "successful update of protocol engine "
1142                          "to %d.%d.%d\n",
1143                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1144         else
1145                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1146                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1147         if (ret)
1148                 dev_err(dev, "loading protocol SRAM failed\n");
1149
1150 release_tpsram:
1151         release_firmware(tpsram);
1152
1153         return ret;
1154 }
1155
1156 /**
1157  * t3_synchronize_rx - wait for current Rx processing on a port to complete
1158  * @adap: the adapter
1159  * @p: the port
1160  *
1161  * Ensures that current Rx processing on any of the queues associated with
1162  * the given port completes before returning.  We do this by acquiring and
1163  * releasing the locks of the response queues associated with the port.
1164  */
1165 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
1166 {
1167         int i;
1168
1169         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1170                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
1171
1172                 spin_lock_irq(&q->lock);
1173                 spin_unlock_irq(&q->lock);
1174         }
1175 }
1176
1177 static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
1178 {
1179         struct port_info *pi = netdev_priv(dev);
1180         struct adapter *adapter = pi->adapter;
1181
1182         if (adapter->params.rev > 0) {
1183                 t3_set_vlan_accel(adapter, 1 << pi->port_id,
1184                                   features & NETIF_F_HW_VLAN_CTAG_RX);
1185         } else {
1186                 /* single control for all ports */
1187                 unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_CTAG_RX;
1188
1189                 for_each_port(adapter, i)
1190                         have_vlans |=
1191                                 adapter->port[i]->features &
1192                                 NETIF_F_HW_VLAN_CTAG_RX;
1193
1194                 t3_set_vlan_accel(adapter, 1, have_vlans);
1195         }
1196         t3_synchronize_rx(adapter, pi);
1197 }
1198
1199 /**
1200  *      cxgb_up - enable the adapter
1201  *      @adapter: adapter being enabled
1202  *
1203  *      Called when the first port is enabled, this function performs the
1204  *      actions necessary to make an adapter operational, such as completing
1205  *      the initialization of HW modules, and enabling interrupts.
1206  *
1207  *      Must be called with the rtnl lock held.
1208  */
1209 static int cxgb_up(struct adapter *adap)
1210 {
1211         int i, err;
1212
1213         if (!(adap->flags & FULL_INIT_DONE)) {
1214                 err = t3_check_fw_version(adap);
1215                 if (err == -EINVAL) {
1216                         err = upgrade_fw(adap);
1217                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1218                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1219                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1220                 }
1221
1222                 err = t3_check_tpsram_version(adap);
1223                 if (err == -EINVAL) {
1224                         err = update_tpsram(adap);
1225                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1226                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1227                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1228                 }
1229
1230                 /*
1231                  * Clear interrupts now to catch errors if t3_init_hw fails.
1232                  * We clear them again later as initialization may trigger
1233                  * conditions that can interrupt.
1234                  */
1235                 t3_intr_clear(adap);
1236
1237                 err = t3_init_hw(adap, 0);
1238                 if (err)
1239                         goto out;
1240
1241                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1242                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1243
1244                 err = setup_sge_qsets(adap);
1245                 if (err)
1246                         goto out;
1247
1248                 for_each_port(adap, i)
1249                         cxgb_vlan_mode(adap->port[i], adap->port[i]->features);
1250
1251                 setup_rss(adap);
1252                 if (!(adap->flags & NAPI_INIT))
1253                         init_napi(adap);
1254
1255                 t3_start_sge_timers(adap);
1256                 adap->flags |= FULL_INIT_DONE;
1257         }
1258
1259         t3_intr_clear(adap);
1260
1261         if (adap->flags & USING_MSIX) {
1262                 name_msix_vecs(adap);
1263                 err = request_irq(adap->msix_info[0].vec,
1264                                   t3_async_intr_handler, 0,
1265                                   adap->msix_info[0].desc, adap);
1266                 if (err)
1267                         goto irq_err;
1268
1269                 err = request_msix_data_irqs(adap);
1270                 if (err) {
1271                         free_irq(adap->msix_info[0].vec, adap);
1272                         goto irq_err;
1273                 }
1274         } else if ((err = request_irq(adap->pdev->irq,
1275                                       t3_intr_handler(adap,
1276                                                       adap->sge.qs[0].rspq.
1277                                                       polling),
1278                                       (adap->flags & USING_MSI) ?
1279                                        0 : IRQF_SHARED,
1280                                       adap->name, adap)))
1281                 goto irq_err;
1282
1283         enable_all_napi(adap);
1284         t3_sge_start(adap);
1285         t3_intr_enable(adap);
1286
1287         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1288             is_offload(adap) && init_tp_parity(adap) == 0)
1289                 adap->flags |= TP_PARITY_INIT;
1290
1291         if (adap->flags & TP_PARITY_INIT) {
1292                 t3_write_reg(adap, A_TP_INT_CAUSE,
1293                              F_CMCACHEPERR | F_ARPLUTPERR);
1294                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1295         }
1296
1297         if (!(adap->flags & QUEUES_BOUND)) {
1298                 int ret = bind_qsets(adap);
1299
1300                 if (ret < 0) {
1301                         CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
1302                         t3_intr_disable(adap);
1303                         free_irq_resources(adap);
1304                         err = ret;
1305                         goto out;
1306                 }
1307                 adap->flags |= QUEUES_BOUND;
1308         }
1309
1310 out:
1311         return err;
1312 irq_err:
1313         CH_ERR(adap, "request_irq failed, err %d\n", err);
1314         goto out;
1315 }
1316
1317 /*
1318  * Release resources when all the ports and offloading have been stopped.
1319  */
1320 static void cxgb_down(struct adapter *adapter, int on_wq)
1321 {
1322         t3_sge_stop(adapter);
1323         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1324         t3_intr_disable(adapter);
1325         spin_unlock_irq(&adapter->work_lock);
1326
1327         free_irq_resources(adapter);
1328         quiesce_rx(adapter);
1329         t3_sge_stop(adapter);
1330         if (!on_wq)
1331                 flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
1332 }
1333
1334 static void schedule_chk_task(struct adapter *adap)
1335 {
1336         unsigned int timeo;
1337
1338         timeo = adap->params.linkpoll_period ?
1339             (HZ * adap->params.linkpoll_period) / 10 :
1340             adap->params.stats_update_period * HZ;
1341         if (timeo)
1342                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1343 }
1344
1345 static int offload_open(struct net_device *dev)
1346 {
1347         struct port_info *pi = netdev_priv(dev);
1348         struct adapter *adapter = pi->adapter;
1349         struct t3cdev *tdev = dev2t3cdev(dev);
1350         int adap_up = adapter->open_device_map & PORT_MASK;
1351         int err;
1352
1353         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1354                 return 0;
1355
1356         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1357                 goto out;
1358
1359         t3_tp_set_offload_mode(adapter, 1);
1360         tdev->lldev = adapter->port[0];
1361         err = cxgb3_offload_activate(adapter);
1362         if (err)
1363                 goto out;
1364
1365         init_port_mtus(adapter);
1366         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1367                      adapter->params.b_wnd,
1368                      adapter->params.rev == 0 ?
1369                      adapter->port[0]->mtu : 0xffff);
1370         init_smt(adapter);
1371
1372         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1373                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1374
1375         /* Call back all registered clients */
1376         cxgb3_add_clients(tdev);
1377
1378 out:
1379         /* restore them in case the offload module has changed them */
1380         if (err) {
1381                 t3_tp_set_offload_mode(adapter, 0);
1382                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1383                 cxgb3_set_dummy_ops(tdev);
1384         }
1385         return err;
1386 }
1387
1388 static int offload_close(struct t3cdev *tdev)
1389 {
1390         struct adapter *adapter = tdev2adap(tdev);
1391         struct t3c_data *td = T3C_DATA(tdev);
1392
1393         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1394                 return 0;
1395
1396         /* Call back all registered clients */
1397         cxgb3_remove_clients(tdev);
1398
1399         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1400
1401         /* Flush work scheduled while releasing TIDs */
1402         flush_work(&td->tid_release_task);
1403
1404         tdev->lldev = NULL;
1405         cxgb3_set_dummy_ops(tdev);
1406         t3_tp_set_offload_mode(adapter, 0);
1407         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1408
1409         if (!adapter->open_device_map)
1410                 cxgb_down(adapter, 0);
1411
1412         cxgb3_offload_deactivate(adapter);
1413         return 0;
1414 }
1415
1416 static int cxgb_open(struct net_device *dev)
1417 {
1418         struct port_info *pi = netdev_priv(dev);
1419         struct adapter *adapter = pi->adapter;
1420         int other_ports = adapter->open_device_map & PORT_MASK;
1421         int err;
1422
1423         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1424                 return err;
1425
1426         set_bit(pi->port_id, &adapter->open_device_map);
1427         if (is_offload(adapter) && !ofld_disable) {
1428                 err = offload_open(dev);
1429                 if (err)
1430                         pr_warn("Could not initialize offload capabilities\n");
1431         }
1432
1433         netif_set_real_num_tx_queues(dev, pi->nqsets);
1434         err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1435         if (err)
1436                 return err;
1437         link_start(dev);
1438         t3_port_intr_enable(adapter, pi->port_id);
1439         netif_tx_start_all_queues(dev);
1440         if (!other_ports)
1441                 schedule_chk_task(adapter);
1442
1443         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1444         return 0;
1445 }
1446
1447 static int __cxgb_close(struct net_device *dev, int on_wq)
1448 {
1449         struct port_info *pi = netdev_priv(dev);
1450         struct adapter *adapter = pi->adapter;
1451
1452
1453         if (!adapter->open_device_map)
1454                 return 0;
1455
1456         /* Stop link fault interrupts */
1457         t3_xgm_intr_disable(adapter, pi->port_id);
1458         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1459
1460         t3_port_intr_disable(adapter, pi->port_id);
1461         netif_tx_stop_all_queues(dev);
1462         pi->phy.ops->power_down(&pi->phy, 1);
1463         netif_carrier_off(dev);
1464         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1465
1466         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1467         clear_bit(pi->port_id, &adapter->open_device_map);
1468         spin_unlock_irq(&adapter->work_lock);
1469
1470         if (!(adapter->open_device_map & PORT_MASK))
1471                 cancel_delayed_work_sync(&adapter->adap_check_task);
1472
1473         if (!adapter->open_device_map)
1474                 cxgb_down(adapter, on_wq);
1475
1476         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1477         return 0;
1478 }
1479
1480 static int cxgb_close(struct net_device *dev)
1481 {
1482         return __cxgb_close(dev, 0);
1483 }
1484
1485 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1486 {
1487         struct port_info *pi = netdev_priv(dev);
1488         struct adapter *adapter = pi->adapter;
1489         struct net_device_stats *ns = &pi->netstats;
1490         const struct mac_stats *pstats;
1491
1492         spin_lock(&adapter->stats_lock);
1493         pstats = t3_mac_update_stats(&pi->mac);
1494         spin_unlock(&adapter->stats_lock);
1495
1496         ns->tx_bytes = pstats->tx_octets;
1497         ns->tx_packets = pstats->tx_frames;
1498         ns->rx_bytes = pstats->rx_octets;
1499         ns->rx_packets = pstats->rx_frames;
1500         ns->multicast = pstats->rx_mcast_frames;
1501
1502         ns->tx_errors = pstats->tx_underrun;
1503         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1504             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1505             pstats->rx_fifo_ovfl;
1506
1507         /* detailed rx_errors */
1508         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1509         ns->rx_over_errors = 0;
1510         ns->rx_crc_errors = pstats->rx_fcs_errs;
1511         ns->rx_frame_errors = pstats->rx_symbol_errs;
1512         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1513         ns->rx_missed_errors = pstats->rx_cong_drops;
1514
1515         /* detailed tx_errors */
1516         ns->tx_aborted_errors = 0;
1517         ns->tx_carrier_errors = 0;
1518         ns->tx_fifo_errors = pstats->tx_underrun;
1519         ns->tx_heartbeat_errors = 0;
1520         ns->tx_window_errors = 0;
1521         return ns;
1522 }
1523
1524 static u32 get_msglevel(struct net_device *dev)
1525 {
1526         struct port_info *pi = netdev_priv(dev);
1527         struct adapter *adapter = pi->adapter;
1528
1529         return adapter->msg_enable;
1530 }
1531
1532 static void set_msglevel(struct net_device *dev, u32 val)
1533 {
1534         struct port_info *pi = netdev_priv(dev);
1535         struct adapter *adapter = pi->adapter;
1536
1537         adapter->msg_enable = val;
1538 }
1539
1540 static const char stats_strings[][ETH_GSTRING_LEN] = {
1541         "TxOctetsOK         ",
1542         "TxFramesOK         ",
1543         "TxMulticastFramesOK",
1544         "TxBroadcastFramesOK",
1545         "TxPauseFrames      ",
1546         "TxUnderrun         ",
1547         "TxExtUnderrun      ",
1548
1549         "TxFrames64         ",
1550         "TxFrames65To127    ",
1551         "TxFrames128To255   ",
1552         "TxFrames256To511   ",
1553         "TxFrames512To1023  ",
1554         "TxFrames1024To1518 ",
1555         "TxFrames1519ToMax  ",
1556
1557         "RxOctetsOK         ",
1558         "RxFramesOK         ",
1559         "RxMulticastFramesOK",
1560         "RxBroadcastFramesOK",
1561         "RxPauseFrames      ",
1562         "RxFCSErrors        ",
1563         "RxSymbolErrors     ",
1564         "RxShortErrors      ",
1565         "RxJabberErrors     ",
1566         "RxLengthErrors     ",
1567         "RxFIFOoverflow     ",
1568
1569         "RxFrames64         ",
1570         "RxFrames65To127    ",
1571         "RxFrames128To255   ",
1572         "RxFrames256To511   ",
1573         "RxFrames512To1023  ",
1574         "RxFrames1024To1518 ",
1575         "RxFrames1519ToMax  ",
1576
1577         "PhyFIFOErrors      ",
1578         "TSO                ",
1579         "VLANextractions    ",
1580         "VLANinsertions     ",
1581         "TxCsumOffload      ",
1582         "RxCsumGood         ",
1583         "LroAggregated      ",
1584         "LroFlushed         ",
1585         "LroNoDesc          ",
1586         "RxDrops            ",
1587
1588         "CheckTXEnToggled   ",
1589         "CheckResets        ",
1590
1591         "LinkFaults         ",
1592 };
1593
1594 static int get_sset_count(struct net_device *dev, int sset)
1595 {
1596         switch (sset) {
1597         case ETH_SS_STATS:
1598                 return ARRAY_SIZE(stats_strings);
1599         default:
1600                 return -EOPNOTSUPP;
1601         }
1602 }
1603
1604 #define T3_REGMAP_SIZE (3 * 1024)
1605
1606 static int get_regs_len(struct net_device *dev)
1607 {
1608         return T3_REGMAP_SIZE;
1609 }
1610
1611 static int get_eeprom_len(struct net_device *dev)
1612 {
1613         return EEPROMSIZE;
1614 }
1615
1616 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1617 {
1618         struct port_info *pi = netdev_priv(dev);
1619         struct adapter *adapter = pi->adapter;
1620         u32 fw_vers = 0;
1621         u32 tp_vers = 0;
1622
1623         spin_lock(&adapter->stats_lock);
1624         t3_get_fw_version(adapter, &fw_vers);
1625         t3_get_tp_version(adapter, &tp_vers);
1626         spin_unlock(&adapter->stats_lock);
1627
1628         strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1629         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1630         strlcpy(info->bus_info, pci_name(adapter->pdev),
1631                 sizeof(info->bus_info));
1632         if (fw_vers)
1633                 snprintf(info->fw_version, sizeof(info->fw_version),
1634                          "%s %u.%u.%u TP %u.%u.%u",
1635                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1636                          G_FW_VERSION_MAJOR(fw_vers),
1637                          G_FW_VERSION_MINOR(fw_vers),
1638                          G_FW_VERSION_MICRO(fw_vers),
1639                          G_TP_VERSION_MAJOR(tp_vers),
1640                          G_TP_VERSION_MINOR(tp_vers),
1641                          G_TP_VERSION_MICRO(tp_vers));
1642 }
1643
1644 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1645 {
1646         if (stringset == ETH_SS_STATS)
1647                 memcpy(data, stats_strings, sizeof(stats_strings));
1648 }
1649
1650 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1651                                             struct port_info *p, int idx)
1652 {
1653         int i;
1654         unsigned long tot = 0;
1655
1656         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1657                 tot += adapter->sge.qs[i].port_stats[idx];
1658         return tot;
1659 }
1660
1661 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1662                       u64 *data)
1663 {
1664         struct port_info *pi = netdev_priv(dev);
1665         struct adapter *adapter = pi->adapter;
1666         const struct mac_stats *s;
1667
1668         spin_lock(&adapter->stats_lock);
1669         s = t3_mac_update_stats(&pi->mac);
1670         spin_unlock(&adapter->stats_lock);
1671
1672         *data++ = s->tx_octets;
1673         *data++ = s->tx_frames;
1674         *data++ = s->tx_mcast_frames;
1675         *data++ = s->tx_bcast_frames;
1676         *data++ = s->tx_pause;
1677         *data++ = s->tx_underrun;
1678         *data++ = s->tx_fifo_urun;
1679
1680         *data++ = s->tx_frames_64;
1681         *data++ = s->tx_frames_65_127;
1682         *data++ = s->tx_frames_128_255;
1683         *data++ = s->tx_frames_256_511;
1684         *data++ = s->tx_frames_512_1023;
1685         *data++ = s->tx_frames_1024_1518;
1686         *data++ = s->tx_frames_1519_max;
1687
1688         *data++ = s->rx_octets;
1689         *data++ = s->rx_frames;
1690         *data++ = s->rx_mcast_frames;
1691         *data++ = s->rx_bcast_frames;
1692         *data++ = s->rx_pause;
1693         *data++ = s->rx_fcs_errs;
1694         *data++ = s->rx_symbol_errs;
1695         *data++ = s->rx_short;
1696         *data++ = s->rx_jabber;
1697         *data++ = s->rx_too_long;
1698         *data++ = s->rx_fifo_ovfl;
1699
1700         *data++ = s->rx_frames_64;
1701         *data++ = s->rx_frames_65_127;
1702         *data++ = s->rx_frames_128_255;
1703         *data++ = s->rx_frames_256_511;
1704         *data++ = s->rx_frames_512_1023;
1705         *data++ = s->rx_frames_1024_1518;
1706         *data++ = s->rx_frames_1519_max;
1707
1708         *data++ = pi->phy.fifo_errors;
1709
1710         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1711         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1712         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1713         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1714         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1715         *data++ = 0;
1716         *data++ = 0;
1717         *data++ = 0;
1718         *data++ = s->rx_cong_drops;
1719
1720         *data++ = s->num_toggled;
1721         *data++ = s->num_resets;
1722
1723         *data++ = s->link_faults;
1724 }
1725
1726 static inline void reg_block_dump(struct adapter *ap, void *buf,
1727                                   unsigned int start, unsigned int end)
1728 {
1729         u32 *p = buf + start;
1730
1731         for (; start <= end; start += sizeof(u32))
1732                 *p++ = t3_read_reg(ap, start);
1733 }
1734
1735 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1736                      void *buf)
1737 {
1738         struct port_info *pi = netdev_priv(dev);
1739         struct adapter *ap = pi->adapter;
1740
1741         /*
1742          * Version scheme:
1743          * bits 0..9: chip version
1744          * bits 10..15: chip revision
1745          * bit 31: set for PCIe cards
1746          */
1747         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1748
1749         /*
1750          * We skip the MAC statistics registers because they are clear-on-read.
1751          * Also reading multi-register stats would need to synchronize with the
1752          * periodic mac stats accumulation.  Hard to justify the complexity.
1753          */
1754         memset(buf, 0, T3_REGMAP_SIZE);
1755         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1756         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1757         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1758         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1759         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1760         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1761                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1762         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1763                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1764 }
1765
1766 static int restart_autoneg(struct net_device *dev)
1767 {
1768         struct port_info *p = netdev_priv(dev);
1769
1770         if (!netif_running(dev))
1771                 return -EAGAIN;
1772         if (p->link_config.autoneg != AUTONEG_ENABLE)
1773                 return -EINVAL;
1774         p->phy.ops->autoneg_restart(&p->phy);
1775         return 0;
1776 }
1777
1778 static int set_phys_id(struct net_device *dev,
1779                        enum ethtool_phys_id_state state)
1780 {
1781         struct port_info *pi = netdev_priv(dev);
1782         struct adapter *adapter = pi->adapter;
1783
1784         switch (state) {
1785         case ETHTOOL_ID_ACTIVE:
1786                 return 1;       /* cycle on/off once per second */
1787
1788         case ETHTOOL_ID_OFF:
1789                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1790                 break;
1791
1792         case ETHTOOL_ID_ON:
1793         case ETHTOOL_ID_INACTIVE:
1794                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1795                          F_GPIO0_OUT_VAL);
1796         }
1797
1798         return 0;
1799 }
1800
1801 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1802 {
1803         struct port_info *p = netdev_priv(dev);
1804
1805         cmd->supported = p->link_config.supported;
1806         cmd->advertising = p->link_config.advertising;
1807
1808         if (netif_carrier_ok(dev)) {
1809                 ethtool_cmd_speed_set(cmd, p->link_config.speed);
1810                 cmd->duplex = p->link_config.duplex;
1811         } else {
1812                 ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
1813                 cmd->duplex = DUPLEX_UNKNOWN;
1814         }
1815
1816         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1817         cmd->phy_address = p->phy.mdio.prtad;
1818         cmd->transceiver = XCVR_EXTERNAL;
1819         cmd->autoneg = p->link_config.autoneg;
1820         cmd->maxtxpkt = 0;
1821         cmd->maxrxpkt = 0;
1822         return 0;
1823 }
1824
1825 static int speed_duplex_to_caps(int speed, int duplex)
1826 {
1827         int cap = 0;
1828
1829         switch (speed) {
1830         case SPEED_10:
1831                 if (duplex == DUPLEX_FULL)
1832                         cap = SUPPORTED_10baseT_Full;
1833                 else
1834                         cap = SUPPORTED_10baseT_Half;
1835                 break;
1836         case SPEED_100:
1837                 if (duplex == DUPLEX_FULL)
1838                         cap = SUPPORTED_100baseT_Full;
1839                 else
1840                         cap = SUPPORTED_100baseT_Half;
1841                 break;
1842         case SPEED_1000:
1843                 if (duplex == DUPLEX_FULL)
1844                         cap = SUPPORTED_1000baseT_Full;
1845                 else
1846                         cap = SUPPORTED_1000baseT_Half;
1847                 break;
1848         case SPEED_10000:
1849                 if (duplex == DUPLEX_FULL)
1850                         cap = SUPPORTED_10000baseT_Full;
1851         }
1852         return cap;
1853 }
1854
1855 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1856                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1857                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1858                       ADVERTISED_10000baseT_Full)
1859
1860 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1861 {
1862         struct port_info *p = netdev_priv(dev);
1863         struct link_config *lc = &p->link_config;
1864
1865         if (!(lc->supported & SUPPORTED_Autoneg)) {
1866                 /*
1867                  * PHY offers a single speed/duplex.  See if that's what's
1868                  * being requested.
1869                  */
1870                 if (cmd->autoneg == AUTONEG_DISABLE) {
1871                         u32 speed = ethtool_cmd_speed(cmd);
1872                         int cap = speed_duplex_to_caps(speed, cmd->duplex);
1873                         if (lc->supported & cap)
1874                                 return 0;
1875                 }
1876                 return -EINVAL;
1877         }
1878
1879         if (cmd->autoneg == AUTONEG_DISABLE) {
1880                 u32 speed = ethtool_cmd_speed(cmd);
1881                 int cap = speed_duplex_to_caps(speed, cmd->duplex);
1882
1883                 if (!(lc->supported & cap) || (speed == SPEED_1000))
1884                         return -EINVAL;
1885                 lc->requested_speed = speed;
1886                 lc->requested_duplex = cmd->duplex;
1887                 lc->advertising = 0;
1888         } else {
1889                 cmd->advertising &= ADVERTISED_MASK;
1890                 cmd->advertising &= lc->supported;
1891                 if (!cmd->advertising)
1892                         return -EINVAL;
1893                 lc->requested_speed = SPEED_INVALID;
1894                 lc->requested_duplex = DUPLEX_INVALID;
1895                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1896         }
1897         lc->autoneg = cmd->autoneg;
1898         if (netif_running(dev))
1899                 t3_link_start(&p->phy, &p->mac, lc);
1900         return 0;
1901 }
1902
1903 static void get_pauseparam(struct net_device *dev,
1904                            struct ethtool_pauseparam *epause)
1905 {
1906         struct port_info *p = netdev_priv(dev);
1907
1908         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1909         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1910         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1911 }
1912
1913 static int set_pauseparam(struct net_device *dev,
1914                           struct ethtool_pauseparam *epause)
1915 {
1916         struct port_info *p = netdev_priv(dev);
1917         struct link_config *lc = &p->link_config;
1918
1919         if (epause->autoneg == AUTONEG_DISABLE)
1920                 lc->requested_fc = 0;
1921         else if (lc->supported & SUPPORTED_Autoneg)
1922                 lc->requested_fc = PAUSE_AUTONEG;
1923         else
1924                 return -EINVAL;
1925
1926         if (epause->rx_pause)
1927                 lc->requested_fc |= PAUSE_RX;
1928         if (epause->tx_pause)
1929                 lc->requested_fc |= PAUSE_TX;
1930         if (lc->autoneg == AUTONEG_ENABLE) {
1931                 if (netif_running(dev))
1932                         t3_link_start(&p->phy, &p->mac, lc);
1933         } else {
1934                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1935                 if (netif_running(dev))
1936                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1937         }
1938         return 0;
1939 }
1940
1941 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1942 {
1943         struct port_info *pi = netdev_priv(dev);
1944         struct adapter *adapter = pi->adapter;
1945         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1946
1947         e->rx_max_pending = MAX_RX_BUFFERS;
1948         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1949         e->tx_max_pending = MAX_TXQ_ENTRIES;
1950
1951         e->rx_pending = q->fl_size;
1952         e->rx_mini_pending = q->rspq_size;
1953         e->rx_jumbo_pending = q->jumbo_size;
1954         e->tx_pending = q->txq_size[0];
1955 }
1956
1957 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1958 {
1959         struct port_info *pi = netdev_priv(dev);
1960         struct adapter *adapter = pi->adapter;
1961         struct qset_params *q;
1962         int i;
1963
1964         if (e->rx_pending > MAX_RX_BUFFERS ||
1965             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1966             e->tx_pending > MAX_TXQ_ENTRIES ||
1967             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1968             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1969             e->rx_pending < MIN_FL_ENTRIES ||
1970             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1971             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1972                 return -EINVAL;
1973
1974         if (adapter->flags & FULL_INIT_DONE)
1975                 return -EBUSY;
1976
1977         q = &adapter->params.sge.qset[pi->first_qset];
1978         for (i = 0; i < pi->nqsets; ++i, ++q) {
1979                 q->rspq_size = e->rx_mini_pending;
1980                 q->fl_size = e->rx_pending;
1981                 q->jumbo_size = e->rx_jumbo_pending;
1982                 q->txq_size[0] = e->tx_pending;
1983                 q->txq_size[1] = e->tx_pending;
1984                 q->txq_size[2] = e->tx_pending;
1985         }
1986         return 0;
1987 }
1988
1989 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1990 {
1991         struct port_info *pi = netdev_priv(dev);
1992         struct adapter *adapter = pi->adapter;
1993         struct qset_params *qsp;
1994         struct sge_qset *qs;
1995         int i;
1996
1997         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1998                 return -EINVAL;
1999
2000         for (i = 0; i < pi->nqsets; i++) {
2001                 qsp = &adapter->params.sge.qset[i];
2002                 qs = &adapter->sge.qs[i];
2003                 qsp->coalesce_usecs = c->rx_coalesce_usecs;
2004                 t3_update_qset_coalesce(qs, qsp);
2005         }
2006
2007         return 0;
2008 }
2009
2010 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2011 {
2012         struct port_info *pi = netdev_priv(dev);
2013         struct adapter *adapter = pi->adapter;
2014         struct qset_params *q = adapter->params.sge.qset;
2015
2016         c->rx_coalesce_usecs = q->coalesce_usecs;
2017         return 0;
2018 }
2019
2020 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2021                       u8 * data)
2022 {
2023         struct port_info *pi = netdev_priv(dev);
2024         struct adapter *adapter = pi->adapter;
2025         int i, err = 0;
2026
2027         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2028         if (!buf)
2029                 return -ENOMEM;
2030
2031         e->magic = EEPROM_MAGIC;
2032         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2033                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
2034
2035         if (!err)
2036                 memcpy(data, buf + e->offset, e->len);
2037         kfree(buf);
2038         return err;
2039 }
2040
2041 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2042                       u8 * data)
2043 {
2044         struct port_info *pi = netdev_priv(dev);
2045         struct adapter *adapter = pi->adapter;
2046         u32 aligned_offset, aligned_len;
2047         __le32 *p;
2048         u8 *buf;
2049         int err;
2050
2051         if (eeprom->magic != EEPROM_MAGIC)
2052                 return -EINVAL;
2053
2054         aligned_offset = eeprom->offset & ~3;
2055         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2056
2057         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2058                 buf = kmalloc(aligned_len, GFP_KERNEL);
2059                 if (!buf)
2060                         return -ENOMEM;
2061                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2062                 if (!err && aligned_len > 4)
2063                         err = t3_seeprom_read(adapter,
2064                                               aligned_offset + aligned_len - 4,
2065                                               (__le32 *) & buf[aligned_len - 4]);
2066                 if (err)
2067                         goto out;
2068                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2069         } else
2070                 buf = data;
2071
2072         err = t3_seeprom_wp(adapter, 0);
2073         if (err)
2074                 goto out;
2075
2076         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2077                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2078                 aligned_offset += 4;
2079         }
2080
2081         if (!err)
2082                 err = t3_seeprom_wp(adapter, 1);
2083 out:
2084         if (buf != data)
2085                 kfree(buf);
2086         return err;
2087 }
2088
2089 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2090 {
2091         wol->supported = 0;
2092         wol->wolopts = 0;
2093         memset(&wol->sopass, 0, sizeof(wol->sopass));
2094 }
2095
2096 static const struct ethtool_ops cxgb_ethtool_ops = {
2097         .get_settings = get_settings,
2098         .set_settings = set_settings,
2099         .get_drvinfo = get_drvinfo,
2100         .get_msglevel = get_msglevel,
2101         .set_msglevel = set_msglevel,
2102         .get_ringparam = get_sge_param,
2103         .set_ringparam = set_sge_param,
2104         .get_coalesce = get_coalesce,
2105         .set_coalesce = set_coalesce,
2106         .get_eeprom_len = get_eeprom_len,
2107         .get_eeprom = get_eeprom,
2108         .set_eeprom = set_eeprom,
2109         .get_pauseparam = get_pauseparam,
2110         .set_pauseparam = set_pauseparam,
2111         .get_link = ethtool_op_get_link,
2112         .get_strings = get_strings,
2113         .set_phys_id = set_phys_id,
2114         .nway_reset = restart_autoneg,
2115         .get_sset_count = get_sset_count,
2116         .get_ethtool_stats = get_stats,
2117         .get_regs_len = get_regs_len,
2118         .get_regs = get_regs,
2119         .get_wol = get_wol,
2120 };
2121
2122 static int in_range(int val, int lo, int hi)
2123 {
2124         return val < 0 || (val <= hi && val >= lo);
2125 }
2126
2127 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2128 {
2129         struct port_info *pi = netdev_priv(dev);
2130         struct adapter *adapter = pi->adapter;
2131         u32 cmd;
2132         int ret;
2133
2134         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2135                 return -EFAULT;
2136
2137         switch (cmd) {
2138         case CHELSIO_SET_QSET_PARAMS:{
2139                 int i;
2140                 struct qset_params *q;
2141                 struct ch_qset_params t;
2142                 int q1 = pi->first_qset;
2143                 int nqsets = pi->nqsets;
2144
2145                 if (!capable(CAP_NET_ADMIN))
2146                         return -EPERM;
2147                 if (copy_from_user(&t, useraddr, sizeof(t)))
2148                         return -EFAULT;
2149                 if (t.qset_idx >= SGE_QSETS)
2150                         return -EINVAL;
2151                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2152                     !in_range(t.cong_thres, 0, 255) ||
2153                     !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2154                               MAX_TXQ_ENTRIES) ||
2155                     !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2156                               MAX_TXQ_ENTRIES) ||
2157                     !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2158                               MAX_CTRL_TXQ_ENTRIES) ||
2159                     !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2160                               MAX_RX_BUFFERS) ||
2161                     !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2162                               MAX_RX_JUMBO_BUFFERS) ||
2163                     !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2164                               MAX_RSPQ_ENTRIES))
2165                         return -EINVAL;
2166
2167                 if ((adapter->flags & FULL_INIT_DONE) &&
2168                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2169                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2170                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2171                         t.polling >= 0 || t.cong_thres >= 0))
2172                         return -EBUSY;
2173
2174                 /* Allow setting of any available qset when offload enabled */
2175                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2176                         q1 = 0;
2177                         for_each_port(adapter, i) {
2178                                 pi = adap2pinfo(adapter, i);
2179                                 nqsets += pi->first_qset + pi->nqsets;
2180                         }
2181                 }
2182
2183                 if (t.qset_idx < q1)
2184                         return -EINVAL;
2185                 if (t.qset_idx > q1 + nqsets - 1)
2186                         return -EINVAL;
2187
2188                 q = &adapter->params.sge.qset[t.qset_idx];
2189
2190                 if (t.rspq_size >= 0)
2191                         q->rspq_size = t.rspq_size;
2192                 if (t.fl_size[0] >= 0)
2193                         q->fl_size = t.fl_size[0];
2194                 if (t.fl_size[1] >= 0)
2195                         q->jumbo_size = t.fl_size[1];
2196                 if (t.txq_size[0] >= 0)
2197                         q->txq_size[0] = t.txq_size[0];
2198                 if (t.txq_size[1] >= 0)
2199                         q->txq_size[1] = t.txq_size[1];
2200                 if (t.txq_size[2] >= 0)
2201                         q->txq_size[2] = t.txq_size[2];
2202                 if (t.cong_thres >= 0)
2203                         q->cong_thres = t.cong_thres;
2204                 if (t.intr_lat >= 0) {
2205                         struct sge_qset *qs =
2206                                 &adapter->sge.qs[t.qset_idx];
2207
2208                         q->coalesce_usecs = t.intr_lat;
2209                         t3_update_qset_coalesce(qs, q);
2210                 }
2211                 if (t.polling >= 0) {
2212                         if (adapter->flags & USING_MSIX)
2213                                 q->polling = t.polling;
2214                         else {
2215                                 /* No polling with INTx for T3A */
2216                                 if (adapter->params.rev == 0 &&
2217                                         !(adapter->flags & USING_MSI))
2218                                         t.polling = 0;
2219
2220                                 for (i = 0; i < SGE_QSETS; i++) {
2221                                         q = &adapter->params.sge.
2222                                                 qset[i];
2223                                         q->polling = t.polling;
2224                                 }
2225                         }
2226                 }
2227
2228                 if (t.lro >= 0) {
2229                         if (t.lro)
2230                                 dev->wanted_features |= NETIF_F_GRO;
2231                         else
2232                                 dev->wanted_features &= ~NETIF_F_GRO;
2233                         netdev_update_features(dev);
2234                 }
2235
2236                 break;
2237         }
2238         case CHELSIO_GET_QSET_PARAMS:{
2239                 struct qset_params *q;
2240                 struct ch_qset_params t;
2241                 int q1 = pi->first_qset;
2242                 int nqsets = pi->nqsets;
2243                 int i;
2244
2245                 if (copy_from_user(&t, useraddr, sizeof(t)))
2246                         return -EFAULT;
2247
2248                 /* Display qsets for all ports when offload enabled */
2249                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2250                         q1 = 0;
2251                         for_each_port(adapter, i) {
2252                                 pi = adap2pinfo(adapter, i);
2253                                 nqsets = pi->first_qset + pi->nqsets;
2254                         }
2255                 }
2256
2257                 if (t.qset_idx >= nqsets)
2258                         return -EINVAL;
2259
2260                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2261                 t.rspq_size = q->rspq_size;
2262                 t.txq_size[0] = q->txq_size[0];
2263                 t.txq_size[1] = q->txq_size[1];
2264                 t.txq_size[2] = q->txq_size[2];
2265                 t.fl_size[0] = q->fl_size;
2266                 t.fl_size[1] = q->jumbo_size;
2267                 t.polling = q->polling;
2268                 t.lro = !!(dev->features & NETIF_F_GRO);
2269                 t.intr_lat = q->coalesce_usecs;
2270                 t.cong_thres = q->cong_thres;
2271                 t.qnum = q1;
2272
2273                 if (adapter->flags & USING_MSIX)
2274                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2275                 else
2276                         t.vector = adapter->pdev->irq;
2277
2278                 if (copy_to_user(useraddr, &t, sizeof(t)))
2279                         return -EFAULT;
2280                 break;
2281         }
2282         case CHELSIO_SET_QSET_NUM:{
2283                 struct ch_reg edata;
2284                 unsigned int i, first_qset = 0, other_qsets = 0;
2285
2286                 if (!capable(CAP_NET_ADMIN))
2287                         return -EPERM;
2288                 if (adapter->flags & FULL_INIT_DONE)
2289                         return -EBUSY;
2290                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2291                         return -EFAULT;
2292                 if (edata.val < 1 ||
2293                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2294                         return -EINVAL;
2295
2296                 for_each_port(adapter, i)
2297                         if (adapter->port[i] && adapter->port[i] != dev)
2298                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2299
2300                 if (edata.val + other_qsets > SGE_QSETS)
2301                         return -EINVAL;
2302
2303                 pi->nqsets = edata.val;
2304
2305                 for_each_port(adapter, i)
2306                         if (adapter->port[i]) {
2307                                 pi = adap2pinfo(adapter, i);
2308                                 pi->first_qset = first_qset;
2309                                 first_qset += pi->nqsets;
2310                         }
2311                 break;
2312         }
2313         case CHELSIO_GET_QSET_NUM:{
2314                 struct ch_reg edata;
2315
2316                 memset(&edata, 0, sizeof(struct ch_reg));
2317
2318                 edata.cmd = CHELSIO_GET_QSET_NUM;
2319                 edata.val = pi->nqsets;
2320                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2321                         return -EFAULT;
2322                 break;
2323         }
2324         case CHELSIO_LOAD_FW:{
2325                 u8 *fw_data;
2326                 struct ch_mem_range t;
2327
2328                 if (!capable(CAP_SYS_RAWIO))
2329                         return -EPERM;
2330                 if (copy_from_user(&t, useraddr, sizeof(t)))
2331                         return -EFAULT;
2332                 /* Check t.len sanity ? */
2333                 fw_data = memdup_user(useraddr + sizeof(t), t.len);
2334                 if (IS_ERR(fw_data))
2335                         return PTR_ERR(fw_data);
2336
2337                 ret = t3_load_fw(adapter, fw_data, t.len);
2338                 kfree(fw_data);
2339                 if (ret)
2340                         return ret;
2341                 break;
2342         }
2343         case CHELSIO_SETMTUTAB:{
2344                 struct ch_mtus m;
2345                 int i;
2346
2347                 if (!is_offload(adapter))
2348                         return -EOPNOTSUPP;
2349                 if (!capable(CAP_NET_ADMIN))
2350                         return -EPERM;
2351                 if (offload_running(adapter))
2352                         return -EBUSY;
2353                 if (copy_from_user(&m, useraddr, sizeof(m)))
2354                         return -EFAULT;
2355                 if (m.nmtus != NMTUS)
2356                         return -EINVAL;
2357                 if (m.mtus[0] < 81)     /* accommodate SACK */
2358                         return -EINVAL;
2359
2360                 /* MTUs must be in ascending order */
2361                 for (i = 1; i < NMTUS; ++i)
2362                         if (m.mtus[i] < m.mtus[i - 1])
2363                                 return -EINVAL;
2364
2365                 memcpy(adapter->params.mtus, m.mtus,
2366                         sizeof(adapter->params.mtus));
2367                 break;
2368         }
2369         case CHELSIO_GET_PM:{
2370                 struct tp_params *p = &adapter->params.tp;
2371                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2372
2373                 if (!is_offload(adapter))
2374                         return -EOPNOTSUPP;
2375                 m.tx_pg_sz = p->tx_pg_size;
2376                 m.tx_num_pg = p->tx_num_pgs;
2377                 m.rx_pg_sz = p->rx_pg_size;
2378                 m.rx_num_pg = p->rx_num_pgs;
2379                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2380                 if (copy_to_user(useraddr, &m, sizeof(m)))
2381                         return -EFAULT;
2382                 break;
2383         }
2384         case CHELSIO_SET_PM:{
2385                 struct ch_pm m;
2386                 struct tp_params *p = &adapter->params.tp;
2387
2388                 if (!is_offload(adapter))
2389                         return -EOPNOTSUPP;
2390                 if (!capable(CAP_NET_ADMIN))
2391                         return -EPERM;
2392                 if (adapter->flags & FULL_INIT_DONE)
2393                         return -EBUSY;
2394                 if (copy_from_user(&m, useraddr, sizeof(m)))
2395                         return -EFAULT;
2396                 if (!is_power_of_2(m.rx_pg_sz) ||
2397                         !is_power_of_2(m.tx_pg_sz))
2398                         return -EINVAL; /* not power of 2 */
2399                 if (!(m.rx_pg_sz & 0x14000))
2400                         return -EINVAL; /* not 16KB or 64KB */
2401                 if (!(m.tx_pg_sz & 0x1554000))
2402                         return -EINVAL;
2403                 if (m.tx_num_pg == -1)
2404                         m.tx_num_pg = p->tx_num_pgs;
2405                 if (m.rx_num_pg == -1)
2406                         m.rx_num_pg = p->rx_num_pgs;
2407                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2408                         return -EINVAL;
2409                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2410                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2411                         return -EINVAL;
2412                 p->rx_pg_size = m.rx_pg_sz;
2413                 p->tx_pg_size = m.tx_pg_sz;
2414                 p->rx_num_pgs = m.rx_num_pg;
2415                 p->tx_num_pgs = m.tx_num_pg;
2416                 break;
2417         }
2418         case CHELSIO_GET_MEM:{
2419                 struct ch_mem_range t;
2420                 struct mc7 *mem;
2421                 u64 buf[32];
2422
2423                 if (!is_offload(adapter))
2424                         return -EOPNOTSUPP;
2425                 if (!(adapter->flags & FULL_INIT_DONE))
2426                         return -EIO;    /* need the memory controllers */
2427                 if (copy_from_user(&t, useraddr, sizeof(t)))
2428                         return -EFAULT;
2429                 if ((t.addr & 7) || (t.len & 7))
2430                         return -EINVAL;
2431                 if (t.mem_id == MEM_CM)
2432                         mem = &adapter->cm;
2433                 else if (t.mem_id == MEM_PMRX)
2434                         mem = &adapter->pmrx;
2435                 else if (t.mem_id == MEM_PMTX)
2436                         mem = &adapter->pmtx;
2437                 else
2438                         return -EINVAL;
2439
2440                 /*
2441                  * Version scheme:
2442                  * bits 0..9: chip version
2443                  * bits 10..15: chip revision
2444                  */
2445                 t.version = 3 | (adapter->params.rev << 10);
2446                 if (copy_to_user(useraddr, &t, sizeof(t)))
2447                         return -EFAULT;
2448
2449                 /*
2450                  * Read 256 bytes at a time as len can be large and we don't
2451                  * want to use huge intermediate buffers.
2452                  */
2453                 useraddr += sizeof(t);  /* advance to start of buffer */
2454                 while (t.len) {
2455                         unsigned int chunk =
2456                                 min_t(unsigned int, t.len, sizeof(buf));
2457
2458                         ret =
2459                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2460                                                 buf);
2461                         if (ret)
2462                                 return ret;
2463                         if (copy_to_user(useraddr, buf, chunk))
2464                                 return -EFAULT;
2465                         useraddr += chunk;
2466                         t.addr += chunk;
2467                         t.len -= chunk;
2468                 }
2469                 break;
2470         }
2471         case CHELSIO_SET_TRACE_FILTER:{
2472                 struct ch_trace t;
2473                 const struct trace_params *tp;
2474
2475                 if (!capable(CAP_NET_ADMIN))
2476                         return -EPERM;
2477                 if (!offload_running(adapter))
2478                         return -EAGAIN;
2479                 if (copy_from_user(&t, useraddr, sizeof(t)))
2480                         return -EFAULT;
2481
2482                 tp = (const struct trace_params *)&t.sip;
2483                 if (t.config_tx)
2484                         t3_config_trace_filter(adapter, tp, 0,
2485                                                 t.invert_match,
2486                                                 t.trace_tx);
2487                 if (t.config_rx)
2488                         t3_config_trace_filter(adapter, tp, 1,
2489                                                 t.invert_match,
2490                                                 t.trace_rx);
2491                 break;
2492         }
2493         default:
2494                 return -EOPNOTSUPP;
2495         }
2496         return 0;
2497 }
2498
2499 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2500 {
2501         struct mii_ioctl_data *data = if_mii(req);
2502         struct port_info *pi = netdev_priv(dev);
2503         struct adapter *adapter = pi->adapter;
2504
2505         switch (cmd) {
2506         case SIOCGMIIREG:
2507         case SIOCSMIIREG:
2508                 /* Convert phy_id from older PRTAD/DEVAD format */
2509                 if (is_10G(adapter) &&
2510                     !mdio_phy_id_is_c45(data->phy_id) &&
2511                     (data->phy_id & 0x1f00) &&
2512                     !(data->phy_id & 0xe0e0))
2513                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2514                                                        data->phy_id & 0x1f);
2515                 /* FALLTHRU */
2516         case SIOCGMIIPHY:
2517                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2518         case SIOCCHIOCTL:
2519                 return cxgb_extension_ioctl(dev, req->ifr_data);
2520         default:
2521                 return -EOPNOTSUPP;
2522         }
2523 }
2524
2525 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2526 {
2527         struct port_info *pi = netdev_priv(dev);
2528         struct adapter *adapter = pi->adapter;
2529         int ret;
2530
2531         if (new_mtu < 81)       /* accommodate SACK */
2532                 return -EINVAL;
2533         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2534                 return ret;
2535         dev->mtu = new_mtu;
2536         init_port_mtus(adapter);
2537         if (adapter->params.rev == 0 && offload_running(adapter))
2538                 t3_load_mtus(adapter, adapter->params.mtus,
2539                              adapter->params.a_wnd, adapter->params.b_wnd,
2540                              adapter->port[0]->mtu);
2541         return 0;
2542 }
2543
2544 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2545 {
2546         struct port_info *pi = netdev_priv(dev);
2547         struct adapter *adapter = pi->adapter;
2548         struct sockaddr *addr = p;
2549
2550         if (!is_valid_ether_addr(addr->sa_data))
2551                 return -EADDRNOTAVAIL;
2552
2553         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2554         t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2555         if (offload_running(adapter))
2556                 write_smt_entry(adapter, pi->port_id);
2557         return 0;
2558 }
2559
2560 static netdev_features_t cxgb_fix_features(struct net_device *dev,
2561         netdev_features_t features)
2562 {
2563         /*
2564          * Since there is no support for separate rx/tx vlan accel
2565          * enable/disable make sure tx flag is always in same state as rx.
2566          */
2567         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2568                 features |= NETIF_F_HW_VLAN_CTAG_TX;
2569         else
2570                 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2571
2572         return features;
2573 }
2574
2575 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2576 {
2577         netdev_features_t changed = dev->features ^ features;
2578
2579         if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2580                 cxgb_vlan_mode(dev, features);
2581
2582         return 0;
2583 }
2584
2585 #ifdef CONFIG_NET_POLL_CONTROLLER
2586 static void cxgb_netpoll(struct net_device *dev)
2587 {
2588         struct port_info *pi = netdev_priv(dev);
2589         struct adapter *adapter = pi->adapter;
2590         int qidx;
2591
2592         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2593                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2594                 void *source;
2595
2596                 if (adapter->flags & USING_MSIX)
2597                         source = qs;
2598                 else
2599                         source = adapter;
2600
2601                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2602         }
2603 }
2604 #endif
2605
2606 /*
2607  * Periodic accumulation of MAC statistics.
2608  */
2609 static void mac_stats_update(struct adapter *adapter)
2610 {
2611         int i;
2612
2613         for_each_port(adapter, i) {
2614                 struct net_device *dev = adapter->port[i];
2615                 struct port_info *p = netdev_priv(dev);
2616
2617                 if (netif_running(dev)) {
2618                         spin_lock(&adapter->stats_lock);
2619                         t3_mac_update_stats(&p->mac);
2620                         spin_unlock(&adapter->stats_lock);
2621                 }
2622         }
2623 }
2624
2625 static void check_link_status(struct adapter *adapter)
2626 {
2627         int i;
2628
2629         for_each_port(adapter, i) {
2630                 struct net_device *dev = adapter->port[i];
2631                 struct port_info *p = netdev_priv(dev);
2632                 int link_fault;
2633
2634                 spin_lock_irq(&adapter->work_lock);
2635                 link_fault = p->link_fault;
2636                 spin_unlock_irq(&adapter->work_lock);
2637
2638                 if (link_fault) {
2639                         t3_link_fault(adapter, i);
2640                         continue;
2641                 }
2642
2643                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2644                         t3_xgm_intr_disable(adapter, i);
2645                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2646
2647                         t3_link_changed(adapter, i);
2648                         t3_xgm_intr_enable(adapter, i);
2649                 }
2650         }
2651 }
2652
2653 static void check_t3b2_mac(struct adapter *adapter)
2654 {
2655         int i;
2656
2657         if (!rtnl_trylock())    /* synchronize with ifdown */
2658                 return;
2659
2660         for_each_port(adapter, i) {
2661                 struct net_device *dev = adapter->port[i];
2662                 struct port_info *p = netdev_priv(dev);
2663                 int status;
2664
2665                 if (!netif_running(dev))
2666                         continue;
2667
2668                 status = 0;
2669                 if (netif_running(dev) && netif_carrier_ok(dev))
2670                         status = t3b2_mac_watchdog_task(&p->mac);
2671                 if (status == 1)
2672                         p->mac.stats.num_toggled++;
2673                 else if (status == 2) {
2674                         struct cmac *mac = &p->mac;
2675
2676                         t3_mac_set_mtu(mac, dev->mtu);
2677                         t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2678                         cxgb_set_rxmode(dev);
2679                         t3_link_start(&p->phy, mac, &p->link_config);
2680                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2681                         t3_port_intr_enable(adapter, p->port_id);
2682                         p->mac.stats.num_resets++;
2683                 }
2684         }
2685         rtnl_unlock();
2686 }
2687
2688
2689 static void t3_adap_check_task(struct work_struct *work)
2690 {
2691         struct adapter *adapter = container_of(work, struct adapter,
2692                                                adap_check_task.work);
2693         const struct adapter_params *p = &adapter->params;
2694         int port;
2695         unsigned int v, status, reset;
2696
2697         adapter->check_task_cnt++;
2698
2699         check_link_status(adapter);
2700
2701         /* Accumulate MAC stats if needed */
2702         if (!p->linkpoll_period ||
2703             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2704             p->stats_update_period) {
2705                 mac_stats_update(adapter);
2706                 adapter->check_task_cnt = 0;
2707         }
2708
2709         if (p->rev == T3_REV_B2)
2710                 check_t3b2_mac(adapter);
2711
2712         /*
2713          * Scan the XGMAC's to check for various conditions which we want to
2714          * monitor in a periodic polling manner rather than via an interrupt
2715          * condition.  This is used for conditions which would otherwise flood
2716          * the system with interrupts and we only really need to know that the
2717          * conditions are "happening" ...  For each condition we count the
2718          * detection of the condition and reset it for the next polling loop.
2719          */
2720         for_each_port(adapter, port) {
2721                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2722                 u32 cause;
2723
2724                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2725                 reset = 0;
2726                 if (cause & F_RXFIFO_OVERFLOW) {
2727                         mac->stats.rx_fifo_ovfl++;
2728                         reset |= F_RXFIFO_OVERFLOW;
2729                 }
2730
2731                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2732         }
2733
2734         /*
2735          * We do the same as above for FL_EMPTY interrupts.
2736          */
2737         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2738         reset = 0;
2739
2740         if (status & F_FLEMPTY) {
2741                 struct sge_qset *qs = &adapter->sge.qs[0];
2742                 int i = 0;
2743
2744                 reset |= F_FLEMPTY;
2745
2746                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2747                     0xffff;
2748
2749                 while (v) {
2750                         qs->fl[i].empty += (v & 1);
2751                         if (i)
2752                                 qs++;
2753                         i ^= 1;
2754                         v >>= 1;
2755                 }
2756         }
2757
2758         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2759
2760         /* Schedule the next check update if any port is active. */
2761         spin_lock_irq(&adapter->work_lock);
2762         if (adapter->open_device_map & PORT_MASK)
2763                 schedule_chk_task(adapter);
2764         spin_unlock_irq(&adapter->work_lock);
2765 }
2766
2767 static void db_full_task(struct work_struct *work)
2768 {
2769         struct adapter *adapter = container_of(work, struct adapter,
2770                                                db_full_task);
2771
2772         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2773 }
2774
2775 static void db_empty_task(struct work_struct *work)
2776 {
2777         struct adapter *adapter = container_of(work, struct adapter,
2778                                                db_empty_task);
2779
2780         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2781 }
2782
2783 static void db_drop_task(struct work_struct *work)
2784 {
2785         struct adapter *adapter = container_of(work, struct adapter,
2786                                                db_drop_task);
2787         unsigned long delay = 1000;
2788         unsigned short r;
2789
2790         cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2791
2792         /*
2793          * Sleep a while before ringing the driver qset dbs.
2794          * The delay is between 1000-2023 usecs.
2795          */
2796         get_random_bytes(&r, 2);
2797         delay += r & 1023;
2798         set_current_state(TASK_UNINTERRUPTIBLE);
2799         schedule_timeout(usecs_to_jiffies(delay));
2800         ring_dbs(adapter);
2801 }
2802
2803 /*
2804  * Processes external (PHY) interrupts in process context.
2805  */
2806 static void ext_intr_task(struct work_struct *work)
2807 {
2808         struct adapter *adapter = container_of(work, struct adapter,
2809                                                ext_intr_handler_task);
2810         int i;
2811
2812         /* Disable link fault interrupts */
2813         for_each_port(adapter, i) {
2814                 struct net_device *dev = adapter->port[i];
2815                 struct port_info *p = netdev_priv(dev);
2816
2817                 t3_xgm_intr_disable(adapter, i);
2818                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2819         }
2820
2821         /* Re-enable link fault interrupts */
2822         t3_phy_intr_handler(adapter);
2823
2824         for_each_port(adapter, i)
2825                 t3_xgm_intr_enable(adapter, i);
2826
2827         /* Now reenable external interrupts */
2828         spin_lock_irq(&adapter->work_lock);
2829         if (adapter->slow_intr_mask) {
2830                 adapter->slow_intr_mask |= F_T3DBG;
2831                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2832                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2833                              adapter->slow_intr_mask);
2834         }
2835         spin_unlock_irq(&adapter->work_lock);
2836 }
2837
2838 /*
2839  * Interrupt-context handler for external (PHY) interrupts.
2840  */
2841 void t3_os_ext_intr_handler(struct adapter *adapter)
2842 {
2843         /*
2844          * Schedule a task to handle external interrupts as they may be slow
2845          * and we use a mutex to protect MDIO registers.  We disable PHY
2846          * interrupts in the meantime and let the task reenable them when
2847          * it's done.
2848          */
2849         spin_lock(&adapter->work_lock);
2850         if (adapter->slow_intr_mask) {
2851                 adapter->slow_intr_mask &= ~F_T3DBG;
2852                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2853                              adapter->slow_intr_mask);
2854                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2855         }
2856         spin_unlock(&adapter->work_lock);
2857 }
2858
2859 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2860 {
2861         struct net_device *netdev = adapter->port[port_id];
2862         struct port_info *pi = netdev_priv(netdev);
2863
2864         spin_lock(&adapter->work_lock);
2865         pi->link_fault = 1;
2866         spin_unlock(&adapter->work_lock);
2867 }
2868
2869 static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
2870 {
2871         int i, ret = 0;
2872
2873         if (is_offload(adapter) &&
2874             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2875                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2876                 offload_close(&adapter->tdev);
2877         }
2878
2879         /* Stop all ports */
2880         for_each_port(adapter, i) {
2881                 struct net_device *netdev = adapter->port[i];
2882
2883                 if (netif_running(netdev))
2884                         __cxgb_close(netdev, on_wq);
2885         }
2886
2887         /* Stop SGE timers */
2888         t3_stop_sge_timers(adapter);
2889
2890         adapter->flags &= ~FULL_INIT_DONE;
2891
2892         if (reset)
2893                 ret = t3_reset_adapter(adapter);
2894
2895         pci_disable_device(adapter->pdev);
2896
2897         return ret;
2898 }
2899
2900 static int t3_reenable_adapter(struct adapter *adapter)
2901 {
2902         if (pci_enable_device(adapter->pdev)) {
2903                 dev_err(&adapter->pdev->dev,
2904                         "Cannot re-enable PCI device after reset.\n");
2905                 goto err;
2906         }
2907         pci_set_master(adapter->pdev);
2908         pci_restore_state(adapter->pdev);
2909         pci_save_state(adapter->pdev);
2910
2911         /* Free sge resources */
2912         t3_free_sge_resources(adapter);
2913
2914         if (t3_replay_prep_adapter(adapter))
2915                 goto err;
2916
2917         return 0;
2918 err:
2919         return -1;
2920 }
2921
2922 static void t3_resume_ports(struct adapter *adapter)
2923 {
2924         int i;
2925
2926         /* Restart the ports */
2927         for_each_port(adapter, i) {
2928                 struct net_device *netdev = adapter->port[i];
2929
2930                 if (netif_running(netdev)) {
2931                         if (cxgb_open(netdev)) {
2932                                 dev_err(&adapter->pdev->dev,
2933                                         "can't bring device back up"
2934                                         " after reset\n");
2935                                 continue;
2936                         }
2937                 }
2938         }
2939
2940         if (is_offload(adapter) && !ofld_disable)
2941                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2942 }
2943
2944 /*
2945  * processes a fatal error.
2946  * Bring the ports down, reset the chip, bring the ports back up.
2947  */
2948 static void fatal_error_task(struct work_struct *work)
2949 {
2950         struct adapter *adapter = container_of(work, struct adapter,
2951                                                fatal_error_handler_task);
2952         int err = 0;
2953
2954         rtnl_lock();
2955         err = t3_adapter_error(adapter, 1, 1);
2956         if (!err)
2957                 err = t3_reenable_adapter(adapter);
2958         if (!err)
2959                 t3_resume_ports(adapter);
2960
2961         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2962         rtnl_unlock();
2963 }
2964
2965 void t3_fatal_err(struct adapter *adapter)
2966 {
2967         unsigned int fw_status[4];
2968
2969         if (adapter->flags & FULL_INIT_DONE) {
2970                 t3_sge_stop(adapter);
2971                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2972                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2973                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2974                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2975
2976                 spin_lock(&adapter->work_lock);
2977                 t3_intr_disable(adapter);
2978                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2979                 spin_unlock(&adapter->work_lock);
2980         }
2981         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2982         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2983                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2984                          fw_status[0], fw_status[1],
2985                          fw_status[2], fw_status[3]);
2986 }
2987
2988 /**
2989  * t3_io_error_detected - called when PCI error is detected
2990  * @pdev: Pointer to PCI device
2991  * @state: The current pci connection state
2992  *
2993  * This function is called after a PCI bus error affecting
2994  * this device has been detected.
2995  */
2996 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2997                                              pci_channel_state_t state)
2998 {
2999         struct adapter *adapter = pci_get_drvdata(pdev);
3000
3001         if (state == pci_channel_io_perm_failure)
3002                 return PCI_ERS_RESULT_DISCONNECT;
3003
3004         t3_adapter_error(adapter, 0, 0);
3005
3006         /* Request a slot reset. */
3007         return PCI_ERS_RESULT_NEED_RESET;
3008 }
3009
3010 /**
3011  * t3_io_slot_reset - called after the pci bus has been reset.
3012  * @pdev: Pointer to PCI device
3013  *
3014  * Restart the card from scratch, as if from a cold-boot.
3015  */
3016 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3017 {
3018         struct adapter *adapter = pci_get_drvdata(pdev);
3019
3020         if (!t3_reenable_adapter(adapter))
3021                 return PCI_ERS_RESULT_RECOVERED;
3022
3023         return PCI_ERS_RESULT_DISCONNECT;
3024 }
3025
3026 /**
3027  * t3_io_resume - called when traffic can start flowing again.
3028  * @pdev: Pointer to PCI device
3029  *
3030  * This callback is called when the error recovery driver tells us that
3031  * its OK to resume normal operation.
3032  */
3033 static void t3_io_resume(struct pci_dev *pdev)
3034 {
3035         struct adapter *adapter = pci_get_drvdata(pdev);
3036
3037         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3038                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
3039
3040         rtnl_lock();
3041         t3_resume_ports(adapter);
3042         rtnl_unlock();
3043 }
3044
3045 static const struct pci_error_handlers t3_err_handler = {
3046         .error_detected = t3_io_error_detected,
3047         .slot_reset = t3_io_slot_reset,
3048         .resume = t3_io_resume,
3049 };
3050
3051 /*
3052  * Set the number of qsets based on the number of CPUs and the number of ports,
3053  * not to exceed the number of available qsets, assuming there are enough qsets
3054  * per port in HW.
3055  */
3056 static void set_nqsets(struct adapter *adap)
3057 {
3058         int i, j = 0;
3059         int num_cpus = netif_get_num_default_rss_queues();
3060         int hwports = adap->params.nports;
3061         int nqsets = adap->msix_nvectors - 1;
3062
3063         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3064                 if (hwports == 2 &&
3065                     (hwports * nqsets > SGE_QSETS ||
3066                      num_cpus >= nqsets / hwports))
3067                         nqsets /= hwports;
3068                 if (nqsets > num_cpus)
3069                         nqsets = num_cpus;
3070                 if (nqsets < 1 || hwports == 4)
3071                         nqsets = 1;
3072         } else
3073                 nqsets = 1;
3074
3075         for_each_port(adap, i) {
3076                 struct port_info *pi = adap2pinfo(adap, i);
3077
3078                 pi->first_qset = j;
3079                 pi->nqsets = nqsets;
3080                 j = pi->first_qset + nqsets;
3081
3082                 dev_info(&adap->pdev->dev,
3083                          "Port %d using %d queue sets.\n", i, nqsets);
3084         }
3085 }
3086
3087 static int cxgb_enable_msix(struct adapter *adap)
3088 {
3089         struct msix_entry entries[SGE_QSETS + 1];
3090         int vectors;
3091         int i;
3092
3093         vectors = ARRAY_SIZE(entries);
3094         for (i = 0; i < vectors; ++i)
3095                 entries[i].entry = i;
3096
3097         vectors = pci_enable_msix_range(adap->pdev, entries,
3098                                         adap->params.nports + 1, vectors);
3099         if (vectors < 0)
3100                 return vectors;
3101
3102         for (i = 0; i < vectors; ++i)
3103                 adap->msix_info[i].vec = entries[i].vector;
3104         adap->msix_nvectors = vectors;
3105
3106         return 0;
3107 }
3108
3109 static void print_port_info(struct adapter *adap, const struct adapter_info *ai)
3110 {
3111         static const char *pci_variant[] = {
3112                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3113         };
3114
3115         int i;
3116         char buf[80];
3117
3118         if (is_pcie(adap))
3119                 snprintf(buf, sizeof(buf), "%s x%d",
3120                          pci_variant[adap->params.pci.variant],
3121                          adap->params.pci.width);
3122         else
3123                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3124                          pci_variant[adap->params.pci.variant],
3125                          adap->params.pci.speed, adap->params.pci.width);
3126
3127         for_each_port(adap, i) {
3128                 struct net_device *dev = adap->port[i];
3129                 const struct port_info *pi = netdev_priv(dev);
3130
3131                 if (!test_bit(i, &adap->registered_device_map))
3132                         continue;
3133                 netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n",
3134                             ai->desc, pi->phy.desc,
3135                             is_offload(adap) ? "R" : "", adap->params.rev, buf,
3136                             (adap->flags & USING_MSIX) ? " MSI-X" :
3137                             (adap->flags & USING_MSI) ? " MSI" : "");
3138                 if (adap->name == dev->name && adap->params.vpd.mclk)
3139                         pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3140                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3141                                t3_mc7_size(&adap->pmtx) >> 20,
3142                                t3_mc7_size(&adap->pmrx) >> 20,
3143                                adap->params.vpd.sn);
3144         }
3145 }
3146
3147 static const struct net_device_ops cxgb_netdev_ops = {
3148         .ndo_open               = cxgb_open,
3149         .ndo_stop               = cxgb_close,
3150         .ndo_start_xmit         = t3_eth_xmit,
3151         .ndo_get_stats          = cxgb_get_stats,
3152         .ndo_validate_addr      = eth_validate_addr,
3153         .ndo_set_rx_mode        = cxgb_set_rxmode,
3154         .ndo_do_ioctl           = cxgb_ioctl,
3155         .ndo_change_mtu         = cxgb_change_mtu,
3156         .ndo_set_mac_address    = cxgb_set_mac_addr,
3157         .ndo_fix_features       = cxgb_fix_features,
3158         .ndo_set_features       = cxgb_set_features,
3159 #ifdef CONFIG_NET_POLL_CONTROLLER
3160         .ndo_poll_controller    = cxgb_netpoll,
3161 #endif
3162 };
3163
3164 static void cxgb3_init_iscsi_mac(struct net_device *dev)
3165 {
3166         struct port_info *pi = netdev_priv(dev);
3167
3168         memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3169         pi->iscsic.mac_addr[3] |= 0x80;
3170 }
3171
3172 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3173 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3174                         NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3175 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
3176 {
3177         int i, err, pci_using_dac = 0;
3178         resource_size_t mmio_start, mmio_len;
3179         const struct adapter_info *ai;
3180         struct adapter *adapter = NULL;
3181         struct port_info *pi;
3182
3183         pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
3184
3185         if (!cxgb3_wq) {
3186                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3187                 if (!cxgb3_wq) {
3188                         pr_err("cannot initialize work queue\n");
3189                         return -ENOMEM;
3190                 }
3191         }
3192
3193         err = pci_enable_device(pdev);
3194         if (err) {
3195                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3196                 goto out;
3197         }
3198
3199         err = pci_request_regions(pdev, DRV_NAME);
3200         if (err) {
3201                 /* Just info, some other driver may have claimed the device. */
3202                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3203                 goto out_disable_device;
3204         }
3205
3206         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3207                 pci_using_dac = 1;
3208                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3209                 if (err) {
3210                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3211                                "coherent allocations\n");
3212                         goto out_release_regions;
3213                 }
3214         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3215                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3216                 goto out_release_regions;
3217         }
3218
3219         pci_set_master(pdev);
3220         pci_save_state(pdev);
3221
3222         mmio_start = pci_resource_start(pdev, 0);
3223         mmio_len = pci_resource_len(pdev, 0);
3224         ai = t3_get_adapter_info(ent->driver_data);
3225
3226         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3227         if (!adapter) {
3228                 err = -ENOMEM;
3229                 goto out_release_regions;
3230         }
3231
3232         adapter->nofail_skb =
3233                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3234         if (!adapter->nofail_skb) {
3235                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3236                 err = -ENOMEM;
3237                 goto out_free_adapter;
3238         }
3239
3240         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3241         if (!adapter->regs) {
3242                 dev_err(&pdev->dev, "cannot map device registers\n");
3243                 err = -ENOMEM;
3244                 goto out_free_adapter;
3245         }
3246
3247         adapter->pdev = pdev;
3248         adapter->name = pci_name(pdev);
3249         adapter->msg_enable = dflt_msg_enable;
3250         adapter->mmio_len = mmio_len;
3251
3252         mutex_init(&adapter->mdio_lock);
3253         spin_lock_init(&adapter->work_lock);
3254         spin_lock_init(&adapter->stats_lock);
3255
3256         INIT_LIST_HEAD(&adapter->adapter_list);
3257         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3258         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3259
3260         INIT_WORK(&adapter->db_full_task, db_full_task);
3261         INIT_WORK(&adapter->db_empty_task, db_empty_task);
3262         INIT_WORK(&adapter->db_drop_task, db_drop_task);
3263
3264         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3265
3266         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3267                 struct net_device *netdev;
3268
3269                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3270                 if (!netdev) {
3271                         err = -ENOMEM;
3272                         goto out_free_dev;
3273                 }
3274
3275                 SET_NETDEV_DEV(netdev, &pdev->dev);
3276
3277                 adapter->port[i] = netdev;
3278                 pi = netdev_priv(netdev);
3279                 pi->adapter = adapter;
3280                 pi->port_id = i;
3281                 netif_carrier_off(netdev);
3282                 netdev->irq = pdev->irq;
3283                 netdev->mem_start = mmio_start;
3284                 netdev->mem_end = mmio_start + mmio_len - 1;
3285                 netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
3286                         NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
3287                 netdev->features |= netdev->hw_features |
3288                                     NETIF_F_HW_VLAN_CTAG_TX;
3289                 netdev->vlan_features |= netdev->features & VLAN_FEAT;
3290                 if (pci_using_dac)
3291                         netdev->features |= NETIF_F_HIGHDMA;
3292
3293                 netdev->netdev_ops = &cxgb_netdev_ops;
3294                 netdev->ethtool_ops = &cxgb_ethtool_ops;
3295         }
3296
3297         pci_set_drvdata(pdev, adapter);
3298         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3299                 err = -ENODEV;
3300                 goto out_free_dev;
3301         }
3302
3303         /*
3304          * The card is now ready to go.  If any errors occur during device
3305          * registration we do not fail the whole card but rather proceed only
3306          * with the ports we manage to register successfully.  However we must
3307          * register at least one net device.
3308          */
3309         for_each_port(adapter, i) {
3310                 err = register_netdev(adapter->port[i]);
3311                 if (err)
3312                         dev_warn(&pdev->dev,
3313                                  "cannot register net device %s, skipping\n",
3314                                  adapter->port[i]->name);
3315                 else {
3316                         /*
3317                          * Change the name we use for messages to the name of
3318                          * the first successfully registered interface.
3319                          */
3320                         if (!adapter->registered_device_map)
3321                                 adapter->name = adapter->port[i]->name;
3322
3323                         __set_bit(i, &adapter->registered_device_map);
3324                 }
3325         }
3326         if (!adapter->registered_device_map) {
3327                 dev_err(&pdev->dev, "could not register any net devices\n");
3328                 goto out_free_dev;
3329         }
3330
3331         for_each_port(adapter, i)
3332                 cxgb3_init_iscsi_mac(adapter->port[i]);
3333
3334         /* Driver's ready. Reflect it on LEDs */
3335         t3_led_ready(adapter);
3336
3337         if (is_offload(adapter)) {
3338                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3339                 cxgb3_adapter_ofld(adapter);
3340         }
3341
3342         /* See what interrupts we'll be using */
3343         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3344                 adapter->flags |= USING_MSIX;
3345         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3346                 adapter->flags |= USING_MSI;
3347
3348         set_nqsets(adapter);
3349
3350         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3351                                  &cxgb3_attr_group);
3352
3353         print_port_info(adapter, ai);
3354         return 0;
3355
3356 out_free_dev:
3357         iounmap(adapter->regs);
3358         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3359                 if (adapter->port[i])
3360                         free_netdev(adapter->port[i]);
3361
3362 out_free_adapter:
3363         kfree(adapter);
3364
3365 out_release_regions:
3366         pci_release_regions(pdev);
3367 out_disable_device:
3368         pci_disable_device(pdev);
3369 out:
3370         return err;
3371 }
3372
3373 static void remove_one(struct pci_dev *pdev)
3374 {
3375         struct adapter *adapter = pci_get_drvdata(pdev);
3376
3377         if (adapter) {
3378                 int i;
3379
3380                 t3_sge_stop(adapter);
3381                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3382                                    &cxgb3_attr_group);
3383
3384                 if (is_offload(adapter)) {
3385                         cxgb3_adapter_unofld(adapter);
3386                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3387                                      &adapter->open_device_map))
3388                                 offload_close(&adapter->tdev);
3389                 }
3390
3391                 for_each_port(adapter, i)
3392                     if (test_bit(i, &adapter->registered_device_map))
3393                         unregister_netdev(adapter->port[i]);
3394
3395                 t3_stop_sge_timers(adapter);
3396                 t3_free_sge_resources(adapter);
3397                 cxgb_disable_msi(adapter);
3398
3399                 for_each_port(adapter, i)
3400                         if (adapter->port[i])
3401                                 free_netdev(adapter->port[i]);
3402
3403                 iounmap(adapter->regs);
3404                 if (adapter->nofail_skb)
3405                         kfree_skb(adapter->nofail_skb);
3406                 kfree(adapter);
3407                 pci_release_regions(pdev);
3408                 pci_disable_device(pdev);
3409         }
3410 }
3411
3412 static struct pci_driver driver = {
3413         .name = DRV_NAME,
3414         .id_table = cxgb3_pci_tbl,
3415         .probe = init_one,
3416         .remove = remove_one,
3417         .err_handler = &t3_err_handler,
3418 };
3419
3420 static int __init cxgb3_init_module(void)
3421 {
3422         int ret;
3423
3424         cxgb3_offload_init();
3425
3426         ret = pci_register_driver(&driver);
3427         return ret;
3428 }
3429
3430 static void __exit cxgb3_cleanup_module(void)
3431 {
3432         pci_unregister_driver(&driver);
3433         if (cxgb3_wq)
3434                 destroy_workqueue(cxgb3_wq);
3435 }
3436
3437 module_init(cxgb3_init_module);
3438 module_exit(cxgb3_cleanup_module);