kernel/drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  30 MODULE_AUTHOR("Emulex Corporation");
  31 MODULE_LICENSE("GPL");
  32
  33 /* num_vfs module param is obsolete.
  34  * Use sysfs method to enable/disable VFs.
  35  */
  36 static unsigned int num_vfs;
  37 module_param(num_vfs, uint, S_IRUGO);
  38 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  39
  40 static ushort rx_frag_size = 2048;
  41 module_param(rx_frag_size, ushort, S_IRUGO);
  42 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  43
  44 static const struct pci_device_id be_dev_ids[] = {
  45         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  47         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  48         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  51         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  52         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  53         { 0 }
  54 };
  55 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  56 /* UE Status Low CSR */
  57 static const char * const ue_status_low_desc[] = {
  58         "CEV",
  59         "CTX",
  60         "DBUF",
  61         "ERX",
  62         "Host",
  63         "MPU",
  64         "NDMA",
  65         "PTC ",
  66         "RDMA ",
  67         "RXF ",
  68         "RXIPS ",
  69         "RXULP0 ",
  70         "RXULP1 ",
  71         "RXULP2 ",
  72         "TIM ",
  73         "TPOST ",
  74         "TPRE ",
  75         "TXIPS ",
  76         "TXULP0 ",
  77         "TXULP1 ",
  78         "UC ",
  79         "WDMA ",
  80         "TXULP2 ",
  81         "HOST1 ",
  82         "P0_OB_LINK ",
  83         "P1_OB_LINK ",
  84         "HOST_GPIO ",
  85         "MBOX ",
  86         "ERX2 ",
  87         "SPARE ",
  88         "JTAG ",
  89         "MPU_INTPEND "
  90 };
  91
  92 /* UE Status High CSR */
  93 static const char * const ue_status_hi_desc[] = {
  94         "LPCMEMHOST",
  95         "MGMT_MAC",
  96         "PCS0ONLINE",
  97         "MPU_IRAM",
  98         "PCS1ONLINE",
  99         "PCTL0",
 100         "PCTL1",
 101         "PMEM",
 102         "RR",
 103         "TXPB",
 104         "RXPP",
 105         "XAUI",
 106         "TXP",
 107         "ARM",
 108         "IPC",
 109         "HOST2",
 110         "HOST3",
 111         "HOST4",
 112         "HOST5",
 113         "HOST6",
 114         "HOST7",
 115         "ECRC",
 116         "Poison TLP",
 117         "NETC",
 118         "PERIPH",
 119         "LLTXULP",
 120         "D2P",
 121         "RCON",
 122         "LDMA",
 123         "LLTXP",
 124         "LLTXPB",
 125         "Unknown"
 126 };
 127
 128 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 129 {
 130         struct be_dma_mem *mem = &q->dma_mem;
 131
 132         if (mem->va) {
 133                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 134                                   mem->dma);
 135                 mem->va = NULL;
 136         }
 137 }
 138
 139 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 140                           u16 len, u16 entry_size)
 141 {
 142         struct be_dma_mem *mem = &q->dma_mem;
 143
 144         memset(q, 0, sizeof(*q));
 145         q->len = len;
 146         q->entry_size = entry_size;
 147         mem->size = len * entry_size;
 148         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 149                                       GFP_KERNEL);
 150         if (!mem->va)
 151                 return -ENOMEM;
 152         return 0;
 153 }
 154
 155 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 156 {
 157         u32 reg, enabled;
 158
 159         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 160                               &reg);
 161         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 162
 163         if (!enabled && enable)
 164                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else if (enabled && !enable)
 166                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 167         else
 168                 return;
 169
 170         pci_write_config_dword(adapter->pdev,
 171                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 172 }
 173
 174 static void be_intr_set(struct be_adapter *adapter, bool enable)
 175 {
 176         int status = 0;
 177
 178         /* On lancer interrupts can't be controlled via this register */
 179         if (lancer_chip(adapter))
 180                 return;
 181
 182         if (adapter->eeh_error)
 183                 return;
 184
 185         status = be_cmd_intr_set(adapter, enable);
 186         if (status)
 187                 be_reg_intr_set(adapter, enable);
 188 }
 189
 190 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 191 {
 192         u32 val = 0;
 193
 194         val |= qid & DB_RQ_RING_ID_MASK;
 195         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 196
 197         wmb();
 198         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 199 }
 200
 201 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 202                           u16 posted)
 203 {
 204         u32 val = 0;
 205
 206         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 207         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 208
 209         wmb();
 210         iowrite32(val, adapter->db + txo->db_offset);
 211 }
 212
 213 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 214                          bool arm, bool clear_int, u16 num_popped)
 215 {
 216         u32 val = 0;
 217
 218         val |= qid & DB_EQ_RING_ID_MASK;
 219         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 220
 221         if (adapter->eeh_error)
 222                 return;
 223
 224         if (arm)
 225                 val |= 1 << DB_EQ_REARM_SHIFT;
 226         if (clear_int)
 227                 val |= 1 << DB_EQ_CLR_SHIFT;
 228         val |= 1 << DB_EQ_EVNT_SHIFT;
 229         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 230         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 231 }
 232
 233 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 234 {
 235         u32 val = 0;
 236
 237         val |= qid & DB_CQ_RING_ID_MASK;
 238         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 239                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 240
 241         if (adapter->eeh_error)
 242                 return;
 243
 244         if (arm)
 245                 val |= 1 << DB_CQ_REARM_SHIFT;
 246         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 247         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 248 }
 249
 250 static int be_mac_addr_set(struct net_device *netdev, void *p)
 251 {
 252         struct be_adapter *adapter = netdev_priv(netdev);
 253         struct device *dev = &adapter->pdev->dev;
 254         struct sockaddr *addr = p;
 255         int status;
 256         u8 mac[ETH_ALEN];
 257         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 258
 259         if (!is_valid_ether_addr(addr->sa_data))
 260                 return -EADDRNOTAVAIL;
 261
 262         /* Proceed further only if, User provided MAC is different
 263          * from active MAC
 264          */
 265         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 266                 return 0;
 267
 268         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 269          * privilege or if PF did not provision the new MAC address.
 270          * On BE3, this cmd will always fail if the VF doesn't have the
 271          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 272          * the MAC for the VF.
 273          */
 274         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 275                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 276         if (!status) {
 277                 curr_pmac_id = adapter->pmac_id[0];
 278
 279                 /* Delete the old programmed MAC. This call may fail if the
 280                  * old MAC was already deleted by the PF driver.
 281                  */
 282                 if (adapter->pmac_id[0] != old_pmac_id)
 283                         be_cmd_pmac_del(adapter, adapter->if_handle,
 284                                         old_pmac_id, 0);
 285         }
 286
 287         /* Decide if the new MAC is successfully activated only after
 288          * querying the FW
 289          */
 290         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 291                                        adapter->if_handle, true, 0);
 292         if (status)
 293                 goto err;
 294
 295         /* The MAC change did not happen, either due to lack of privilege
 296          * or PF didn't pre-provision.
 297          */
 298         if (!ether_addr_equal(addr->sa_data, mac)) {
 299                 status = -EPERM;
 300                 goto err;
 301         }
 302
 303         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 304         dev_info(dev, "MAC address changed to %pM\n", mac);
 305         return 0;
 306 err:
 307         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 308         return status;
 309 }
 310
 311 /* BE2 supports only v0 cmd */
 312 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 313 {
 314         if (BE2_chip(adapter)) {
 315                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 316
 317                 return &cmd->hw_stats;
 318         } else if (BE3_chip(adapter)) {
 319                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 320
 321                 return &cmd->hw_stats;
 322         } else {
 323                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 324
 325                 return &cmd->hw_stats;
 326         }
 327 }
 328
 329 /* BE2 supports only v0 cmd */
 330 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 331 {
 332         if (BE2_chip(adapter)) {
 333                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 334
 335                 return &hw_stats->erx;
 336         } else if (BE3_chip(adapter)) {
 337                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 338
 339                 return &hw_stats->erx;
 340         } else {
 341                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 342
 343                 return &hw_stats->erx;
 344         }
 345 }
 346
 347 static void populate_be_v0_stats(struct be_adapter *adapter)
 348 {
 349         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 350         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 351         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 352         struct be_port_rxf_stats_v0 *port_stats =
 353                                         &rxf_stats->port[adapter->port_num];
 354         struct be_drv_stats *drvs = &adapter->drv_stats;
 355
 356         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 357         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 358         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 359         drvs->rx_control_frames = port_stats->rx_control_frames;
 360         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 361         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 362         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 363         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 364         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 365         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 366         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 367         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 368         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 369         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 370         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 371         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 372         drvs->rx_dropped_header_too_small =
 373                 port_stats->rx_dropped_header_too_small;
 374         drvs->rx_address_filtered =
 375                                         port_stats->rx_address_filtered +
 376                                         port_stats->rx_vlan_filtered;
 377         drvs->rx_alignment_symbol_errors =
 378                 port_stats->rx_alignment_symbol_errors;
 379
 380         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 381         drvs->tx_controlframes = port_stats->tx_controlframes;
 382
 383         if (adapter->port_num)
 384                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 385         else
 386                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 387         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 388         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 389         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 390         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 391         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 392         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 393         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 394 }
 395
 396 static void populate_be_v1_stats(struct be_adapter *adapter)
 397 {
 398         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 399         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 400         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 401         struct be_port_rxf_stats_v1 *port_stats =
 402                                         &rxf_stats->port[adapter->port_num];
 403         struct be_drv_stats *drvs = &adapter->drv_stats;
 404
 405         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 406         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 407         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 408         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 409         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 410         drvs->rx_control_frames = port_stats->rx_control_frames;
 411         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 412         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 413         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 414         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 415         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 416         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 417         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 418         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 419         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 420         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 421         drvs->rx_dropped_header_too_small =
 422                 port_stats->rx_dropped_header_too_small;
 423         drvs->rx_input_fifo_overflow_drop =
 424                 port_stats->rx_input_fifo_overflow_drop;
 425         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 426         drvs->rx_alignment_symbol_errors =
 427                 port_stats->rx_alignment_symbol_errors;
 428         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 429         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 430         drvs->tx_controlframes = port_stats->tx_controlframes;
 431         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 432         drvs->jabber_events = port_stats->jabber_events;
 433         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 434         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 435         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 436         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 437         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 438         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 439         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 440 }
 441
 442 static void populate_be_v2_stats(struct be_adapter *adapter)
 443 {
 444         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 445         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 446         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 447         struct be_port_rxf_stats_v2 *port_stats =
 448                                         &rxf_stats->port[adapter->port_num];
 449         struct be_drv_stats *drvs = &adapter->drv_stats;
 450
 451         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 452         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 453         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 454         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 455         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 456         drvs->rx_control_frames = port_stats->rx_control_frames;
 457         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 458         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 459         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 460         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 461         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 462         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 463         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 464         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 465         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 466         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 467         drvs->rx_dropped_header_too_small =
 468                 port_stats->rx_dropped_header_too_small;
 469         drvs->rx_input_fifo_overflow_drop =
 470                 port_stats->rx_input_fifo_overflow_drop;
 471         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 472         drvs->rx_alignment_symbol_errors =
 473                 port_stats->rx_alignment_symbol_errors;
 474         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 475         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 476         drvs->tx_controlframes = port_stats->tx_controlframes;
 477         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 478         drvs->jabber_events = port_stats->jabber_events;
 479         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 480         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 481         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 482         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 483         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 484         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 485         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 486         if (be_roce_supported(adapter)) {
 487                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 488                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 489                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 490                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 491                 drvs->roce_drops_payload_len =
 492                         port_stats->roce_drops_payload_len;
 493         }
 494 }
 495
 496 static void populate_lancer_stats(struct be_adapter *adapter)
 497 {
 498         struct be_drv_stats *drvs = &adapter->drv_stats;
 499         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 500
 501         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 502         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 503         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 504         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 505         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 506         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 507         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 508         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 509         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 510         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 511         drvs->rx_dropped_tcp_length =
 512                                 pport_stats->rx_dropped_invalid_tcp_length;
 513         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 514         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 515         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 516         drvs->rx_dropped_header_too_small =
 517                                 pport_stats->rx_dropped_header_too_small;
 518         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 519         drvs->rx_address_filtered =
 520                                         pport_stats->rx_address_filtered +
 521                                         pport_stats->rx_vlan_filtered;
 522         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 523         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 524         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 525         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 526         drvs->jabber_events = pport_stats->rx_jabbers;
 527         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 528         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 529         drvs->rx_drops_too_many_frags =
 530                                 pport_stats->rx_drops_too_many_frags_lo;
 531 }
 532
 533 static void accumulate_16bit_val(u32 *acc, u16 val)
 534 {
 535 #define lo(x)                   (x & 0xFFFF)
 536 #define hi(x)                   (x & 0xFFFF0000)
 537         bool wrapped = val < lo(*acc);
 538         u32 newacc = hi(*acc) + val;
 539
 540         if (wrapped)
 541                 newacc += 65536;
 542         ACCESS_ONCE(*acc) = newacc;
 543 }
 544
 545 static void populate_erx_stats(struct be_adapter *adapter,
 546                                struct be_rx_obj *rxo, u32 erx_stat)
 547 {
 548         if (!BEx_chip(adapter))
 549                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 550         else
 551                 /* below erx HW counter can actually wrap around after
 552                  * 65535. Driver accumulates a 32-bit value
 553                  */
 554                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 555                                      (u16)erx_stat);
 556 }
 557
 558 void be_parse_stats(struct be_adapter *adapter)
 559 {
 560         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 561         struct be_rx_obj *rxo;
 562         int i;
 563         u32 erx_stat;
 564
 565         if (lancer_chip(adapter)) {
 566                 populate_lancer_stats(adapter);
 567         } else {
 568                 if (BE2_chip(adapter))
 569                         populate_be_v0_stats(adapter);
 570                 else if (BE3_chip(adapter))
 571                         /* for BE3 */
 572                         populate_be_v1_stats(adapter);
 573                 else
 574                         populate_be_v2_stats(adapter);
 575
 576                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 577                 for_all_rx_queues(adapter, rxo, i) {
 578                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 579                         populate_erx_stats(adapter, rxo, erx_stat);
 580                 }
 581         }
 582 }
 583
 584 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 585                                                 struct rtnl_link_stats64 *stats)
 586 {
 587         struct be_adapter *adapter = netdev_priv(netdev);
 588         struct be_drv_stats *drvs = &adapter->drv_stats;
 589         struct be_rx_obj *rxo;
 590         struct be_tx_obj *txo;
 591         u64 pkts, bytes;
 592         unsigned int start;
 593         int i;
 594
 595         for_all_rx_queues(adapter, rxo, i) {
 596                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 597
 598                 do {
 599                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 600                         pkts = rx_stats(rxo)->rx_pkts;
 601                         bytes = rx_stats(rxo)->rx_bytes;
 602                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 603                 stats->rx_packets += pkts;
 604                 stats->rx_bytes += bytes;
 605                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 606                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 607                                         rx_stats(rxo)->rx_drops_no_frags;
 608         }
 609
 610         for_all_tx_queues(adapter, txo, i) {
 611                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 612
 613                 do {
 614                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 615                         pkts = tx_stats(txo)->tx_pkts;
 616                         bytes = tx_stats(txo)->tx_bytes;
 617                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 618                 stats->tx_packets += pkts;
 619                 stats->tx_bytes += bytes;
 620         }
 621
 622         /* bad pkts received */
 623         stats->rx_errors = drvs->rx_crc_errors +
 624                 drvs->rx_alignment_symbol_errors +
 625                 drvs->rx_in_range_errors +
 626                 drvs->rx_out_range_errors +
 627                 drvs->rx_frame_too_long +
 628                 drvs->rx_dropped_too_small +
 629                 drvs->rx_dropped_too_short +
 630                 drvs->rx_dropped_header_too_small +
 631                 drvs->rx_dropped_tcp_length +
 632                 drvs->rx_dropped_runt;
 633
 634         /* detailed rx errors */
 635         stats->rx_length_errors = drvs->rx_in_range_errors +
 636                 drvs->rx_out_range_errors +
 637                 drvs->rx_frame_too_long;
 638
 639         stats->rx_crc_errors = drvs->rx_crc_errors;
 640
 641         /* frame alignment errors */
 642         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 643
 644         /* receiver fifo overrun */
 645         /* drops_no_pbuf is no per i/f, it's per BE card */
 646         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 647                                 drvs->rx_input_fifo_overflow_drop +
 648                                 drvs->rx_drops_no_pbuf;
 649         return stats;
 650 }
 651
 652 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 653 {
 654         struct net_device *netdev = adapter->netdev;
 655
 656         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 657                 netif_carrier_off(netdev);
 658                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 659         }
 660
 661         if (link_status)
 662                 netif_carrier_on(netdev);
 663         else
 664                 netif_carrier_off(netdev);
 665 }
 666
 667 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
 668 {
 669         struct be_tx_stats *stats = tx_stats(txo);
 670
 671         u64_stats_update_begin(&stats->sync);
 672         stats->tx_reqs++;
 673         stats->tx_bytes += skb->len;
 674         stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1);
 675         u64_stats_update_end(&stats->sync);
 676 }
 677
 678 /* Returns number of WRBs needed for the skb */
 679 static u32 skb_wrb_cnt(struct sk_buff *skb)
 680 {
 681         /* +1 for the header wrb */
 682         return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
 683 }
 684
 685 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 686 {
 687         wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
 688         wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
 689         wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
 690         wrb->rsvd0 = 0;
 691 }
 692
 693 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
 694  * to avoid the swap and shift/mask operations in wrb_fill().
 695  */
 696 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
 697 {
 698         wrb->frag_pa_hi = 0;
 699         wrb->frag_pa_lo = 0;
 700         wrb->frag_len = 0;
 701         wrb->rsvd0 = 0;
 702 }
 703
 704 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 705                                      struct sk_buff *skb)
 706 {
 707         u8 vlan_prio;
 708         u16 vlan_tag;
 709
 710         vlan_tag = skb_vlan_tag_get(skb);
 711         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 712         /* If vlan priority provided by OS is NOT in available bmap */
 713         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 714                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 715                                 adapter->recommended_prio;
 716
 717         return vlan_tag;
 718 }
 719
 720 /* Used only for IP tunnel packets */
 721 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 722 {
 723         return (inner_ip_hdr(skb)->version == 4) ?
 724                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 725 }
 726
 727 static u16 skb_ip_proto(struct sk_buff *skb)
 728 {
 729         return (ip_hdr(skb)->version == 4) ?
 730                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static inline bool be_is_txq_full(struct be_tx_obj *txo)
 734 {
 735         return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;
 736 }
 737
 738 static inline bool be_can_txq_wake(struct be_tx_obj *txo)
 739 {
 740         return atomic_read(&txo->q.used) < txo->q.len / 2;
 741 }
 742
 743 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)
 744 {
 745         return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;
 746 }
 747
 748 static void be_get_wrb_params_from_skb(struct be_adapter *adapter,
 749                                        struct sk_buff *skb,
 750                                        struct be_wrb_params *wrb_params)
 751 {
 752         u16 proto;
 753
 754         if (skb_is_gso(skb)) {
 755                 BE_WRB_F_SET(wrb_params->features, LSO, 1);
 756                 wrb_params->lso_mss = skb_shinfo(skb)->gso_size;
 757                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 758                         BE_WRB_F_SET(wrb_params->features, LSO6, 1);
 759         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 760                 if (skb->encapsulation) {
 761                         BE_WRB_F_SET(wrb_params->features, IPCS, 1);
 762                         proto = skb_inner_ip_proto(skb);
 763                 } else {
 764                         proto = skb_ip_proto(skb);
 765                 }
 766                 if (proto == IPPROTO_TCP)
 767                         BE_WRB_F_SET(wrb_params->features, TCPCS, 1);
 768                 else if (proto == IPPROTO_UDP)
 769                         BE_WRB_F_SET(wrb_params->features, UDPCS, 1);
 770         }
 771
 772         if (skb_vlan_tag_present(skb)) {
 773                 BE_WRB_F_SET(wrb_params->features, VLAN, 1);
 774                 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 775         }
 776
 777         BE_WRB_F_SET(wrb_params->features, CRC, 1);
 778 }
 779
 780 static void wrb_fill_hdr(struct be_adapter *adapter,
 781                          struct be_eth_hdr_wrb *hdr,
 782                          struct be_wrb_params *wrb_params,
 783                          struct sk_buff *skb)
 784 {
 785         memset(hdr, 0, sizeof(*hdr));
 786
 787         SET_TX_WRB_HDR_BITS(crc, hdr,
 788                             BE_WRB_F_GET(wrb_params->features, CRC));
 789         SET_TX_WRB_HDR_BITS(ipcs, hdr,
 790                             BE_WRB_F_GET(wrb_params->features, IPCS));
 791         SET_TX_WRB_HDR_BITS(tcpcs, hdr,
 792                             BE_WRB_F_GET(wrb_params->features, TCPCS));
 793         SET_TX_WRB_HDR_BITS(udpcs, hdr,
 794                             BE_WRB_F_GET(wrb_params->features, UDPCS));
 795
 796         SET_TX_WRB_HDR_BITS(lso, hdr,
 797                             BE_WRB_F_GET(wrb_params->features, LSO));
 798         SET_TX_WRB_HDR_BITS(lso6, hdr,
 799                             BE_WRB_F_GET(wrb_params->features, LSO6));
 800         SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss);
 801
 802         /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this
 803          * hack is not needed, the evt bit is set while ringing DB.
 804          */
 805         SET_TX_WRB_HDR_BITS(event, hdr,
 806                             BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW));
 807         SET_TX_WRB_HDR_BITS(vlan, hdr,
 808                             BE_WRB_F_GET(wrb_params->features, VLAN));
 809         SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag);
 810
 811         SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb));
 812         SET_TX_WRB_HDR_BITS(len, hdr, skb->len);
 813 }
 814
 815 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 816                           bool unmap_single)
 817 {
 818         dma_addr_t dma;
 819         u32 frag_len = le32_to_cpu(wrb->frag_len);
 820
 821
 822         dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
 823                 (u64)le32_to_cpu(wrb->frag_pa_lo);
 824         if (frag_len) {
 825                 if (unmap_single)
 826                         dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
 827                 else
 828                         dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
 829         }
 830 }
 831
 832 /* Grab a WRB header for xmit */
 833 static u16 be_tx_get_wrb_hdr(struct be_tx_obj *txo)
 834 {
 835         u16 head = txo->q.head;
 836
 837         queue_head_inc(&txo->q);
 838         return head;
 839 }
 840
 841 /* Set up the WRB header for xmit */
 842 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter,
 843                                 struct be_tx_obj *txo,
 844                                 struct be_wrb_params *wrb_params,
 845                                 struct sk_buff *skb, u16 head)
 846 {
 847         u32 num_frags = skb_wrb_cnt(skb);
 848         struct be_queue_info *txq = &txo->q;
 849         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head);
 850
 851         wrb_fill_hdr(adapter, hdr, wrb_params, skb);
 852         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 853
 854         BUG_ON(txo->sent_skb_list[head]);
 855         txo->sent_skb_list[head] = skb;
 856         txo->last_req_hdr = head;
 857         atomic_add(num_frags, &txq->used);
 858         txo->last_req_wrb_cnt = num_frags;
 859         txo->pend_wrb_cnt += num_frags;
 860 }
 861
 862 /* Setup a WRB fragment (buffer descriptor) for xmit */
 863 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,
 864                                  int len)
 865 {
 866         struct be_eth_wrb *wrb;
 867         struct be_queue_info *txq = &txo->q;
 868
 869         wrb = queue_head_node(txq);
 870         wrb_fill(wrb, busaddr, len);
 871         queue_head_inc(txq);
 872 }
 873
 874 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine
 875  * was invoked. The producer index is restored to the previous packet and the
 876  * WRBs of the current packet are unmapped. Invoked to handle tx setup errors.
 877  */
 878 static void be_xmit_restore(struct be_adapter *adapter,
 879                             struct be_tx_obj *txo, u16 head, bool map_single,
 880                             u32 copied)
 881 {
 882         struct device *dev;
 883         struct be_eth_wrb *wrb;
 884         struct be_queue_info *txq = &txo->q;
 885
 886         dev = &adapter->pdev->dev;
 887         txq->head = head;
 888
 889         /* skip the first wrb (hdr); it's not mapped */
 890         queue_head_inc(txq);
 891         while (copied) {
 892                 wrb = queue_head_node(txq);
 893                 unmap_tx_frag(dev, wrb, map_single);
 894                 map_single = false;
 895                 copied -= le32_to_cpu(wrb->frag_len);
 896                 queue_head_inc(txq);
 897         }
 898
 899         txq->head = head;
 900 }
 901
 902 /* Enqueue the given packet for transmit. This routine allocates WRBs for the
 903  * packet, dma maps the packet buffers and sets up the WRBs. Returns the number
 904  * of WRBs used up by the packet.
 905  */
 906 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
 907                            struct sk_buff *skb,
 908                            struct be_wrb_params *wrb_params)
 909 {
 910         u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
 911         struct device *dev = &adapter->pdev->dev;
 912         struct be_queue_info *txq = &txo->q;
 913         bool map_single = false;
 914         u16 head = txq->head;
 915         dma_addr_t busaddr;
 916         int len;
 917
 918         head = be_tx_get_wrb_hdr(txo);
 919
 920         if (skb->len > skb->data_len) {
 921                 len = skb_headlen(skb);
 922
 923                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 924                 if (dma_mapping_error(dev, busaddr))
 925                         goto dma_err;
 926                 map_single = true;
 927                 be_tx_setup_wrb_frag(txo, busaddr, len);
 928                 copied += len;
 929         }
 930
 931         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 932                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 933                 len = skb_frag_size(frag);
 934
 935                 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE);
 936                 if (dma_mapping_error(dev, busaddr))
 937                         goto dma_err;
 938                 be_tx_setup_wrb_frag(txo, busaddr, len);
 939                 copied += len;
 940         }
 941
 942         be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);
 943
 944         be_tx_stats_update(txo, skb);
 945         return wrb_cnt;
 946
 947 dma_err:
 948         adapter->drv_stats.dma_map_errors++;
 949         be_xmit_restore(adapter, txo, head, map_single, copied);
 950         return 0;
 951 }
 952
 953 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
 954 {
 955         return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
 956 }
 957
 958 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 959                                              struct sk_buff *skb,
 960                                              struct be_wrb_params
 961                                              *wrb_params)
 962 {
 963         u16 vlan_tag = 0;
 964
 965         skb = skb_share_check(skb, GFP_ATOMIC);
 966         if (unlikely(!skb))
 967                 return skb;
 968
 969         if (skb_vlan_tag_present(skb))
 970                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 971
 972         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 973                 if (!vlan_tag)
 974                         vlan_tag = adapter->pvid;
 975                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 976                  * skip VLAN insertion
 977                  */
 978                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
 979         }
 980
 981         if (vlan_tag) {
 982                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 983                                                 vlan_tag);
 984                 if (unlikely(!skb))
 985                         return skb;
 986                 skb->vlan_tci = 0;
 987         }
 988
 989         /* Insert the outer VLAN, if any */
 990         if (adapter->qnq_vid) {
 991                 vlan_tag = adapter->qnq_vid;
 992                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 993                                                 vlan_tag);
 994                 if (unlikely(!skb))
 995                         return skb;
 996                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
 997         }
 998
 999         return skb;
1000 }
1001
1002 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
1003 {
1004         struct ethhdr *eh = (struct ethhdr *)skb->data;
1005         u16 offset = ETH_HLEN;
1006
1007         if (eh->h_proto == htons(ETH_P_IPV6)) {
1008                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
1009
1010                 offset += sizeof(struct ipv6hdr);
1011                 if (ip6h->nexthdr != NEXTHDR_TCP &&
1012                     ip6h->nexthdr != NEXTHDR_UDP) {
1013                         struct ipv6_opt_hdr *ehdr =
1014                                 (struct ipv6_opt_hdr *)(skb->data + offset);
1015
1016                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
1017                         if (ehdr->hdrlen == 0xff)
1018                                 return true;
1019                 }
1020         }
1021         return false;
1022 }
1023
1024 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
1025 {
1026         return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
1027 }
1028
1029 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
1030 {
1031         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
1032 }
1033
1034 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
1035                                                   struct sk_buff *skb,
1036                                                   struct be_wrb_params
1037                                                   *wrb_params)
1038 {
1039         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1040         unsigned int eth_hdr_len;
1041         struct iphdr *ip;
1042
1043         /* For padded packets, BE HW modifies tot_len field in IP header
1044          * incorrecly when VLAN tag is inserted by HW.
1045          * For padded packets, Lancer computes incorrect checksum.
1046          */
1047         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
1048                                                 VLAN_ETH_HLEN : ETH_HLEN;
1049         if (skb->len <= 60 &&
1050             (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
1051             is_ipv4_pkt(skb)) {
1052                 ip = (struct iphdr *)ip_hdr(skb);
1053                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
1054         }
1055
1056         /* If vlan tag is already inlined in the packet, skip HW VLAN
1057          * tagging in pvid-tagging mode
1058          */
1059         if (be_pvid_tagging_enabled(adapter) &&
1060             veh->h_vlan_proto == htons(ETH_P_8021Q))
1061                 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1062
1063         /* HW has a bug wherein it will calculate CSUM for VLAN
1064          * pkts even though it is disabled.
1065          * Manually insert VLAN in pkt.
1066          */
1067         if (skb->ip_summed != CHECKSUM_PARTIAL &&
1068             skb_vlan_tag_present(skb)) {
1069                 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1070                 if (unlikely(!skb))
1071                         goto err;
1072         }
1073
1074         /* HW may lockup when VLAN HW tagging is requested on
1075          * certain ipv6 packets. Drop such pkts if the HW workaround to
1076          * skip HW tagging is not enabled by FW.
1077          */
1078         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
1079                      (adapter->pvid || adapter->qnq_vid) &&
1080                      !qnq_async_evt_rcvd(adapter)))
1081                 goto tx_drop;
1082
1083         /* Manual VLAN tag insertion to prevent:
1084          * ASIC lockup when the ASIC inserts VLAN tag into
1085          * certain ipv6 packets. Insert VLAN tags in driver,
1086          * and set event, completion, vlan bits accordingly
1087          * in the Tx WRB.
1088          */
1089         if (be_ipv6_tx_stall_chk(adapter, skb) &&
1090             be_vlan_tag_tx_chk(adapter, skb)) {
1091                 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1092                 if (unlikely(!skb))
1093                         goto err;
1094         }
1095
1096         return skb;
1097 tx_drop:
1098         dev_kfree_skb_any(skb);
1099 err:
1100         return NULL;
1101 }
1102
1103 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1104                                            struct sk_buff *skb,
1105                                            struct be_wrb_params *wrb_params)
1106 {
1107         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1108          * less may cause a transmit stall on that port. So the work-around is
1109          * to pad short packets (<= 32 bytes) to a 36-byte length.
1110          */
1111         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1112                 if (skb_put_padto(skb, 36))
1113                         return NULL;
1114         }
1115
1116         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1117                 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params);
1118                 if (!skb)
1119                         return NULL;
1120         }
1121
1122         return skb;
1123 }
1124
1125 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1126 {
1127         struct be_queue_info *txq = &txo->q;
1128         struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1129
1130         /* Mark the last request eventable if it hasn't been marked already */
1131         if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1132                 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1133
1134         /* compose a dummy wrb if there are odd set of wrbs to notify */
1135         if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1136                 wrb_fill_dummy(queue_head_node(txq));
1137                 queue_head_inc(txq);
1138                 atomic_inc(&txq->used);
1139                 txo->pend_wrb_cnt++;
1140                 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1141                                            TX_HDR_WRB_NUM_SHIFT);
1142                 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1143                                           TX_HDR_WRB_NUM_SHIFT);
1144         }
1145         be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1146         txo->pend_wrb_cnt = 0;
1147 }
1148
1149 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1150 {
1151         struct be_adapter *adapter = netdev_priv(netdev);
1152         u16 q_idx = skb_get_queue_mapping(skb);
1153         struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1154         struct be_wrb_params wrb_params = { 0 };
1155         bool flush = !skb->xmit_more;
1156         u16 wrb_cnt;
1157
1158         skb = be_xmit_workarounds(adapter, skb, &wrb_params);
1159         if (unlikely(!skb))
1160                 goto drop;
1161
1162         be_get_wrb_params_from_skb(adapter, skb, &wrb_params);
1163
1164         wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);
1165         if (unlikely(!wrb_cnt)) {
1166                 dev_kfree_skb_any(skb);
1167                 goto drop;
1168         }
1169
1170         if (be_is_txq_full(txo)) {
1171                 netif_stop_subqueue(netdev, q_idx);
1172                 tx_stats(txo)->tx_stops++;
1173         }
1174
1175         if (flush || __netif_subqueue_stopped(netdev, q_idx))
1176                 be_xmit_flush(adapter, txo);
1177
1178         return NETDEV_TX_OK;
1179 drop:
1180         tx_stats(txo)->tx_drv_drops++;
1181         /* Flush the already enqueued tx requests */
1182         if (flush && txo->pend_wrb_cnt)
1183                 be_xmit_flush(adapter, txo);
1184
1185         return NETDEV_TX_OK;
1186 }
1187
1188 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1189 {
1190         struct be_adapter *adapter = netdev_priv(netdev);
1191         struct device *dev = &adapter->pdev->dev;
1192
1193         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1194                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1195                          BE_MIN_MTU, BE_MAX_MTU);
1196                 return -EINVAL;
1197         }
1198
1199         dev_info(dev, "MTU changed from %d to %d bytes\n",
1200                  netdev->mtu, new_mtu);
1201         netdev->mtu = new_mtu;
1202         return 0;
1203 }
1204
1205 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1206 {
1207         return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1208                         BE_IF_FLAGS_ALL_PROMISCUOUS;
1209 }
1210
1211 static int be_set_vlan_promisc(struct be_adapter *adapter)
1212 {
1213         struct device *dev = &adapter->pdev->dev;
1214         int status;
1215
1216         if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1217                 return 0;
1218
1219         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1220         if (!status) {
1221                 dev_info(dev, "Enabled VLAN promiscuous mode\n");
1222                 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1223         } else {
1224                 dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1225         }
1226         return status;
1227 }
1228
1229 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1230 {
1231         struct device *dev = &adapter->pdev->dev;
1232         int status;
1233
1234         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1235         if (!status) {
1236                 dev_info(dev, "Disabling VLAN promiscuous mode\n");
1237                 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1238         }
1239         return status;
1240 }
1241
1242 /*
1243  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1244  * If the user configures more, place BE in vlan promiscuous mode.
1245  */
1246 static int be_vid_config(struct be_adapter *adapter)
1247 {
1248         struct device *dev = &adapter->pdev->dev;
1249         u16 vids[BE_NUM_VLANS_SUPPORTED];
1250         u16 num = 0, i = 0;
1251         int status = 0;
1252
1253         /* No need to further configure vids if in promiscuous mode */
1254         if (be_in_all_promisc(adapter))
1255                 return 0;
1256
1257         if (adapter->vlans_added > be_max_vlans(adapter))
1258                 return be_set_vlan_promisc(adapter);
1259
1260         /* Construct VLAN Table to give to HW */
1261         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1262                 vids[num++] = cpu_to_le16(i);
1263
1264         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0);
1265         if (status) {
1266                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1267                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1268                 if (addl_status(status) ==
1269                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1270                         return be_set_vlan_promisc(adapter);
1271         } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1272                 status = be_clear_vlan_promisc(adapter);
1273         }
1274         return status;
1275 }
1276
1277 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1278 {
1279         struct be_adapter *adapter = netdev_priv(netdev);
1280         int status = 0;
1281
1282         /* Packets with VID 0 are always received by Lancer by default */
1283         if (lancer_chip(adapter) && vid == 0)
1284                 return status;
1285
1286         if (test_bit(vid, adapter->vids))
1287                 return status;
1288
1289         set_bit(vid, adapter->vids);
1290         adapter->vlans_added++;
1291
1292         status = be_vid_config(adapter);
1293         if (status) {
1294                 adapter->vlans_added--;
1295                 clear_bit(vid, adapter->vids);
1296         }
1297
1298         return status;
1299 }
1300
1301 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1302 {
1303         struct be_adapter *adapter = netdev_priv(netdev);
1304
1305         /* Packets with VID 0 are always received by Lancer by default */
1306         if (lancer_chip(adapter) && vid == 0)
1307                 return 0;
1308
1309         clear_bit(vid, adapter->vids);
1310         adapter->vlans_added--;
1311
1312         return be_vid_config(adapter);
1313 }
1314
1315 static void be_clear_all_promisc(struct be_adapter *adapter)
1316 {
1317         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF);
1318         adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
1319 }
1320
1321 static void be_set_all_promisc(struct be_adapter *adapter)
1322 {
1323         be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1324         adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1325 }
1326
1327 static void be_set_mc_promisc(struct be_adapter *adapter)
1328 {
1329         int status;
1330
1331         if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1332                 return;
1333
1334         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1335         if (!status)
1336                 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1337 }
1338
1339 static void be_set_mc_list(struct be_adapter *adapter)
1340 {
1341         int status;
1342
1343         status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1344         if (!status)
1345                 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1346         else
1347                 be_set_mc_promisc(adapter);
1348 }
1349
1350 static void be_set_uc_list(struct be_adapter *adapter)
1351 {
1352         struct netdev_hw_addr *ha;
1353         int i = 1; /* First slot is claimed by the Primary MAC */
1354
1355         for (; adapter->uc_macs > 0; adapter->uc_macs--, i++)
1356                 be_cmd_pmac_del(adapter, adapter->if_handle,
1357                                 adapter->pmac_id[i], 0);
1358
1359         if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) {
1360                 be_set_all_promisc(adapter);
1361                 return;
1362         }
1363
1364         netdev_for_each_uc_addr(ha, adapter->netdev) {
1365                 adapter->uc_macs++; /* First slot is for Primary MAC */
1366                 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle,
1367                                 &adapter->pmac_id[adapter->uc_macs], 0);
1368         }
1369 }
1370
1371 static void be_clear_uc_list(struct be_adapter *adapter)
1372 {
1373         int i;
1374
1375         for (i = 1; i < (adapter->uc_macs + 1); i++)
1376                 be_cmd_pmac_del(adapter, adapter->if_handle,
1377                                 adapter->pmac_id[i], 0);
1378         adapter->uc_macs = 0;
1379 }
1380
1381 static void be_set_rx_mode(struct net_device *netdev)
1382 {
1383         struct be_adapter *adapter = netdev_priv(netdev);
1384
1385         if (netdev->flags & IFF_PROMISC) {
1386                 be_set_all_promisc(adapter);
1387                 return;
1388         }
1389
1390         /* Interface was previously in promiscuous mode; disable it */
1391         if (be_in_all_promisc(adapter)) {
1392                 be_clear_all_promisc(adapter);
1393                 if (adapter->vlans_added)
1394                         be_vid_config(adapter);
1395         }
1396
1397         /* Enable multicast promisc if num configured exceeds what we support */
1398         if (netdev->flags & IFF_ALLMULTI ||
1399             netdev_mc_count(netdev) > be_max_mc(adapter)) {
1400                 be_set_mc_promisc(adapter);
1401                 return;
1402         }
1403
1404         if (netdev_uc_count(netdev) != adapter->uc_macs)
1405                 be_set_uc_list(adapter);
1406
1407         be_set_mc_list(adapter);
1408 }
1409
1410 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1411 {
1412         struct be_adapter *adapter = netdev_priv(netdev);
1413         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1414         int status;
1415
1416         if (!sriov_enabled(adapter))
1417                 return -EPERM;
1418
1419         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1420                 return -EINVAL;
1421
1422         /* Proceed further only if user provided MAC is different
1423          * from active MAC
1424          */
1425         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1426                 return 0;
1427
1428         if (BEx_chip(adapter)) {
1429                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1430                                 vf + 1);
1431
1432                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1433                                          &vf_cfg->pmac_id, vf + 1);
1434         } else {
1435                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1436                                         vf + 1);
1437         }
1438
1439         if (status) {
1440                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1441                         mac, vf, status);
1442                 return be_cmd_status(status);
1443         }
1444
1445         ether_addr_copy(vf_cfg->mac_addr, mac);
1446
1447         return 0;
1448 }
1449
1450 static int be_get_vf_config(struct net_device *netdev, int vf,
1451                             struct ifla_vf_info *vi)
1452 {
1453         struct be_adapter *adapter = netdev_priv(netdev);
1454         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1455
1456         if (!sriov_enabled(adapter))
1457                 return -EPERM;
1458
1459         if (vf >= adapter->num_vfs)
1460                 return -EINVAL;
1461
1462         vi->vf = vf;
1463         vi->max_tx_rate = vf_cfg->tx_rate;
1464         vi->min_tx_rate = 0;
1465         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1466         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1467         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1468         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1469
1470         return 0;
1471 }
1472
1473 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan)
1474 {
1475         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1476         u16 vids[BE_NUM_VLANS_SUPPORTED];
1477         int vf_if_id = vf_cfg->if_handle;
1478         int status;
1479
1480         /* Enable Transparent VLAN Tagging */
1481         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0);
1482         if (status)
1483                 return status;
1484
1485         /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */
1486         vids[0] = 0;
1487         status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1);
1488         if (!status)
1489                 dev_info(&adapter->pdev->dev,
1490                          "Cleared guest VLANs on VF%d", vf);
1491
1492         /* After TVT is enabled, disallow VFs to program VLAN filters */
1493         if (vf_cfg->privileges & BE_PRIV_FILTMGMT) {
1494                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges &
1495                                                   ~BE_PRIV_FILTMGMT, vf + 1);
1496                 if (!status)
1497                         vf_cfg->privileges &= ~BE_PRIV_FILTMGMT;
1498         }
1499         return 0;
1500 }
1501
1502 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf)
1503 {
1504         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1505         struct device *dev = &adapter->pdev->dev;
1506         int status;
1507
1508         /* Reset Transparent VLAN Tagging. */
1509         status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1,
1510                                        vf_cfg->if_handle, 0);
1511         if (status)
1512                 return status;
1513
1514         /* Allow VFs to program VLAN filtering */
1515         if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
1516                 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges |
1517                                                   BE_PRIV_FILTMGMT, vf + 1);
1518                 if (!status) {
1519                         vf_cfg->privileges |= BE_PRIV_FILTMGMT;
1520                         dev_info(dev, "VF%d: FILTMGMT priv enabled", vf);
1521                 }
1522         }
1523
1524         dev_info(dev,
1525                  "Disable/re-enable i/f in VM to clear Transparent VLAN tag");
1526         return 0;
1527 }
1528
1529 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1530 {
1531         struct be_adapter *adapter = netdev_priv(netdev);
1532         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1533         int status;
1534
1535         if (!sriov_enabled(adapter))
1536                 return -EPERM;
1537
1538         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1539                 return -EINVAL;
1540
1541         if (vlan || qos) {
1542                 vlan |= qos << VLAN_PRIO_SHIFT;
1543                 status = be_set_vf_tvt(adapter, vf, vlan);
1544         } else {
1545                 status = be_clear_vf_tvt(adapter, vf);
1546         }
1547
1548         if (status) {
1549                 dev_err(&adapter->pdev->dev,
1550                         "VLAN %d config on VF %d failed : %#x\n", vlan, vf,
1551                         status);
1552                 return be_cmd_status(status);
1553         }
1554
1555         vf_cfg->vlan_tag = vlan;
1556         return 0;
1557 }
1558
1559 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1560                              int min_tx_rate, int max_tx_rate)
1561 {
1562         struct be_adapter *adapter = netdev_priv(netdev);
1563         struct device *dev = &adapter->pdev->dev;
1564         int percent_rate, status = 0;
1565         u16 link_speed = 0;
1566         u8 link_status;
1567
1568         if (!sriov_enabled(adapter))
1569                 return -EPERM;
1570
1571         if (vf >= adapter->num_vfs)
1572                 return -EINVAL;
1573
1574         if (min_tx_rate)
1575                 return -EINVAL;
1576
1577         if (!max_tx_rate)
1578                 goto config_qos;
1579
1580         status = be_cmd_link_status_query(adapter, &link_speed,
1581                                           &link_status, 0);
1582         if (status)
1583                 goto err;
1584
1585         if (!link_status) {
1586                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1587                 status = -ENETDOWN;
1588                 goto err;
1589         }
1590
1591         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1592                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1593                         link_speed);
1594                 status = -EINVAL;
1595                 goto err;
1596         }
1597
1598         /* On Skyhawk the QOS setting must be done only as a % value */
1599         percent_rate = link_speed / 100;
1600         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1601                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1602                         percent_rate);
1603                 status = -EINVAL;
1604                 goto err;
1605         }
1606
1607 config_qos:
1608         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1609         if (status)
1610                 goto err;
1611
1612         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1613         return 0;
1614
1615 err:
1616         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1617                 max_tx_rate, vf);
1618         return be_cmd_status(status);
1619 }
1620
1621 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1622                                 int link_state)
1623 {
1624         struct be_adapter *adapter = netdev_priv(netdev);
1625         int status;
1626
1627         if (!sriov_enabled(adapter))
1628                 return -EPERM;
1629
1630         if (vf >= adapter->num_vfs)
1631                 return -EINVAL;
1632
1633         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1634         if (status) {
1635                 dev_err(&adapter->pdev->dev,
1636                         "Link state change on VF %d failed: %#x\n", vf, status);
1637                 return be_cmd_status(status);
1638         }
1639
1640         adapter->vf_cfg[vf].plink_tracking = link_state;
1641
1642         return 0;
1643 }
1644
1645 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1646                           ulong now)
1647 {
1648         aic->rx_pkts_prev = rx_pkts;
1649         aic->tx_reqs_prev = tx_pkts;
1650         aic->jiffies = now;
1651 }
1652
1653 static void be_eqd_update(struct be_adapter *adapter)
1654 {
1655         struct be_set_eqd set_eqd[MAX_EVT_QS];
1656         int eqd, i, num = 0, start;
1657         struct be_aic_obj *aic;
1658         struct be_eq_obj *eqo;
1659         struct be_rx_obj *rxo;
1660         struct be_tx_obj *txo;
1661         u64 rx_pkts, tx_pkts;
1662         ulong now;
1663         u32 pps, delta;
1664
1665         for_all_evt_queues(adapter, eqo, i) {
1666                 aic = &adapter->aic_obj[eqo->idx];
1667                 if (!aic->enable) {
1668                         if (aic->jiffies)
1669                                 aic->jiffies = 0;
1670                         eqd = aic->et_eqd;
1671                         goto modify_eqd;
1672                 }
1673
1674                 rxo = &adapter->rx_obj[eqo->idx];
1675                 do {
1676                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1677                         rx_pkts = rxo->stats.rx_pkts;
1678                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1679
1680                 txo = &adapter->tx_obj[eqo->idx];
1681                 do {
1682                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1683                         tx_pkts = txo->stats.tx_reqs;
1684                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1685
1686                 /* Skip, if wrapped around or first calculation */
1687                 now = jiffies;
1688                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1689                     rx_pkts < aic->rx_pkts_prev ||
1690                     tx_pkts < aic->tx_reqs_prev) {
1691                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1692                         continue;
1693                 }
1694
1695                 delta = jiffies_to_msecs(now - aic->jiffies);
1696                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1697                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1698                 eqd = (pps / 15000) << 2;
1699
1700                 if (eqd < 8)
1701                         eqd = 0;
1702                 eqd = min_t(u32, eqd, aic->max_eqd);
1703                 eqd = max_t(u32, eqd, aic->min_eqd);
1704
1705                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1706 modify_eqd:
1707                 if (eqd != aic->prev_eqd) {
1708                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1709                         set_eqd[num].eq_id = eqo->q.id;
1710                         aic->prev_eqd = eqd;
1711                         num++;
1712                 }
1713         }
1714
1715         if (num)
1716                 be_cmd_modify_eqd(adapter, set_eqd, num);
1717 }
1718
1719 static void be_rx_stats_update(struct be_rx_obj *rxo,
1720                                struct be_rx_compl_info *rxcp)
1721 {
1722         struct be_rx_stats *stats = rx_stats(rxo);
1723
1724         u64_stats_update_begin(&stats->sync);
1725         stats->rx_compl++;
1726         stats->rx_bytes += rxcp->pkt_size;
1727         stats->rx_pkts++;
1728         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1729                 stats->rx_mcast_pkts++;
1730         if (rxcp->err)
1731                 stats->rx_compl_err++;
1732         u64_stats_update_end(&stats->sync);
1733 }
1734
1735 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1736 {
1737         /* L4 checksum is not reliable for non TCP/UDP packets.
1738          * Also ignore ipcksm for ipv6 pkts
1739          */
1740         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1741                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1742 }
1743
1744 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1745 {
1746         struct be_adapter *adapter = rxo->adapter;
1747         struct be_rx_page_info *rx_page_info;
1748         struct be_queue_info *rxq = &rxo->q;
1749         u16 frag_idx = rxq->tail;
1750
1751         rx_page_info = &rxo->page_info_tbl[frag_idx];
1752         BUG_ON(!rx_page_info->page);
1753
1754         if (rx_page_info->last_frag) {
1755                 dma_unmap_page(&adapter->pdev->dev,
1756                                dma_unmap_addr(rx_page_info, bus),
1757                                adapter->big_page_size, DMA_FROM_DEVICE);
1758                 rx_page_info->last_frag = false;
1759         } else {
1760                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1761                                         dma_unmap_addr(rx_page_info, bus),
1762                                         rx_frag_size, DMA_FROM_DEVICE);
1763         }
1764
1765         queue_tail_inc(rxq);
1766         atomic_dec(&rxq->used);
1767         return rx_page_info;
1768 }
1769
1770 /* Throwaway the data in the Rx completion */
1771 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1772                                 struct be_rx_compl_info *rxcp)
1773 {
1774         struct be_rx_page_info *page_info;
1775         u16 i, num_rcvd = rxcp->num_rcvd;
1776
1777         for (i = 0; i < num_rcvd; i++) {
1778                 page_info = get_rx_page_info(rxo);
1779                 put_page(page_info->page);
1780                 memset(page_info, 0, sizeof(*page_info));
1781         }
1782 }
1783
1784 /*
1785  * skb_fill_rx_data forms a complete skb for an ether frame
1786  * indicated by rxcp.
1787  */
1788 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1789                              struct be_rx_compl_info *rxcp)
1790 {
1791         struct be_rx_page_info *page_info;
1792         u16 i, j;
1793         u16 hdr_len, curr_frag_len, remaining;
1794         u8 *start;
1795
1796         page_info = get_rx_page_info(rxo);
1797         start = page_address(page_info->page) + page_info->page_offset;
1798         prefetch(start);
1799
1800         /* Copy data in the first descriptor of this completion */
1801         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1802
1803         skb->len = curr_frag_len;
1804         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1805                 memcpy(skb->data, start, curr_frag_len);
1806                 /* Complete packet has now been moved to data */
1807                 put_page(page_info->page);
1808                 skb->data_len = 0;
1809                 skb->tail += curr_frag_len;
1810         } else {
1811                 hdr_len = ETH_HLEN;
1812                 memcpy(skb->data, start, hdr_len);
1813                 skb_shinfo(skb)->nr_frags = 1;
1814                 skb_frag_set_page(skb, 0, page_info->page);
1815                 skb_shinfo(skb)->frags[0].page_offset =
1816                                         page_info->page_offset + hdr_len;
1817                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1818                                   curr_frag_len - hdr_len);
1819                 skb->data_len = curr_frag_len - hdr_len;
1820                 skb->truesize += rx_frag_size;
1821                 skb->tail += hdr_len;
1822         }
1823         page_info->page = NULL;
1824
1825         if (rxcp->pkt_size <= rx_frag_size) {
1826                 BUG_ON(rxcp->num_rcvd != 1);
1827                 return;
1828         }
1829
1830         /* More frags present for this completion */
1831         remaining = rxcp->pkt_size - curr_frag_len;
1832         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1833                 page_info = get_rx_page_info(rxo);
1834                 curr_frag_len = min(remaining, rx_frag_size);
1835
1836                 /* Coalesce all frags from the same physical page in one slot */
1837                 if (page_info->page_offset == 0) {
1838                         /* Fresh page */
1839                         j++;
1840                         skb_frag_set_page(skb, j, page_info->page);
1841                         skb_shinfo(skb)->frags[j].page_offset =
1842                                                         page_info->page_offset;
1843                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1844                         skb_shinfo(skb)->nr_frags++;
1845                 } else {
1846                         put_page(page_info->page);
1847                 }
1848
1849                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1850                 skb->len += curr_frag_len;
1851                 skb->data_len += curr_frag_len;
1852                 skb->truesize += rx_frag_size;
1853                 remaining -= curr_frag_len;
1854                 page_info->page = NULL;
1855         }
1856         BUG_ON(j > MAX_SKB_FRAGS);
1857 }
1858
1859 /* Process the RX completion indicated by rxcp when GRO is disabled */
1860 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1861                                 struct be_rx_compl_info *rxcp)
1862 {
1863         struct be_adapter *adapter = rxo->adapter;
1864         struct net_device *netdev = adapter->netdev;
1865         struct sk_buff *skb;
1866
1867         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1868         if (unlikely(!skb)) {
1869                 rx_stats(rxo)->rx_drops_no_skbs++;
1870                 be_rx_compl_discard(rxo, rxcp);
1871                 return;
1872         }
1873
1874         skb_fill_rx_data(rxo, skb, rxcp);
1875
1876         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1877                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1878         else
1879                 skb_checksum_none_assert(skb);
1880
1881         skb->protocol = eth_type_trans(skb, netdev);
1882         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1883         if (netdev->features & NETIF_F_RXHASH)
1884                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1885
1886         skb->csum_level = rxcp->tunneled;
1887         skb_mark_napi_id(skb, napi);
1888
1889         if (rxcp->vlanf)
1890                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1891
1892         netif_receive_skb(skb);
1893 }
1894
1895 /* Process the RX completion indicated by rxcp when GRO is enabled */
1896 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1897                                     struct napi_struct *napi,
1898                                     struct be_rx_compl_info *rxcp)
1899 {
1900         struct be_adapter *adapter = rxo->adapter;
1901         struct be_rx_page_info *page_info;
1902         struct sk_buff *skb = NULL;
1903         u16 remaining, curr_frag_len;
1904         u16 i, j;
1905
1906         skb = napi_get_frags(napi);
1907         if (!skb) {
1908                 be_rx_compl_discard(rxo, rxcp);
1909                 return;
1910         }
1911
1912         remaining = rxcp->pkt_size;
1913         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1914                 page_info = get_rx_page_info(rxo);
1915
1916                 curr_frag_len = min(remaining, rx_frag_size);
1917
1918                 /* Coalesce all frags from the same physical page in one slot */
1919                 if (i == 0 || page_info->page_offset == 0) {
1920                         /* First frag or Fresh page */
1921                         j++;
1922                         skb_frag_set_page(skb, j, page_info->page);
1923                         skb_shinfo(skb)->frags[j].page_offset =
1924                                                         page_info->page_offset;
1925                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1926                 } else {
1927                         put_page(page_info->page);
1928                 }
1929                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1930                 skb->truesize += rx_frag_size;
1931                 remaining -= curr_frag_len;
1932                 memset(page_info, 0, sizeof(*page_info));
1933         }
1934         BUG_ON(j > MAX_SKB_FRAGS);
1935
1936         skb_shinfo(skb)->nr_frags = j + 1;
1937         skb->len = rxcp->pkt_size;
1938         skb->data_len = rxcp->pkt_size;
1939         skb->ip_summed = CHECKSUM_UNNECESSARY;
1940         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1941         if (adapter->netdev->features & NETIF_F_RXHASH)
1942                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1943
1944         skb->csum_level = rxcp->tunneled;
1945         skb_mark_napi_id(skb, napi);
1946
1947         if (rxcp->vlanf)
1948                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1949
1950         napi_gro_frags(napi);
1951 }
1952
1953 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1954                                  struct be_rx_compl_info *rxcp)
1955 {
1956         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1957         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1958         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1959         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1960         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1961         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1962         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1963         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1964         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1965         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1966         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1967         if (rxcp->vlanf) {
1968                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1969                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1970         }
1971         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1972         rxcp->tunneled =
1973                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1974 }
1975
1976 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1977                                  struct be_rx_compl_info *rxcp)
1978 {
1979         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1980         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1981         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1982         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1983         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1984         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1985         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1986         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1987         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1988         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1989         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1990         if (rxcp->vlanf) {
1991                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1992                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1993         }
1994         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1995         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1996 }
1997
1998 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1999 {
2000         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
2001         struct be_rx_compl_info *rxcp = &rxo->rxcp;
2002         struct be_adapter *adapter = rxo->adapter;
2003
2004         /* For checking the valid bit it is Ok to use either definition as the
2005          * valid bit is at the same position in both v0 and v1 Rx compl */
2006         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
2007                 return NULL;
2008
2009         rmb();
2010         be_dws_le_to_cpu(compl, sizeof(*compl));
2011
2012         if (adapter->be3_native)
2013                 be_parse_rx_compl_v1(compl, rxcp);
2014         else
2015                 be_parse_rx_compl_v0(compl, rxcp);
2016
2017         if (rxcp->ip_frag)
2018                 rxcp->l4_csum = 0;
2019
2020         if (rxcp->vlanf) {
2021                 /* In QNQ modes, if qnq bit is not set, then the packet was
2022                  * tagged only with the transparent outer vlan-tag and must
2023                  * not be treated as a vlan packet by host
2024                  */
2025                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
2026                         rxcp->vlanf = 0;
2027
2028                 if (!lancer_chip(adapter))
2029                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
2030
2031                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
2032                     !test_bit(rxcp->vlan_tag, adapter->vids))
2033                         rxcp->vlanf = 0;
2034         }
2035
2036         /* As the compl has been parsed, reset it; we wont touch it again */
2037         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
2038
2039         queue_tail_inc(&rxo->cq);
2040         return rxcp;
2041 }
2042
2043 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
2044 {
2045         u32 order = get_order(size);
2046
2047         if (order > 0)
2048                 gfp |= __GFP_COMP;
2049         return  alloc_pages(gfp, order);
2050 }
2051
2052 /*
2053  * Allocate a page, split it to fragments of size rx_frag_size and post as
2054  * receive buffers to BE
2055  */
2056 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
2057 {
2058         struct be_adapter *adapter = rxo->adapter;
2059         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
2060         struct be_queue_info *rxq = &rxo->q;
2061         struct page *pagep = NULL;
2062         struct device *dev = &adapter->pdev->dev;
2063         struct be_eth_rx_d *rxd;
2064         u64 page_dmaaddr = 0, frag_dmaaddr;
2065         u32 posted, page_offset = 0, notify = 0;
2066
2067         page_info = &rxo->page_info_tbl[rxq->head];
2068         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
2069                 if (!pagep) {
2070                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
2071                         if (unlikely(!pagep)) {
2072                                 rx_stats(rxo)->rx_post_fail++;
2073                                 break;
2074                         }
2075                         page_dmaaddr = dma_map_page(dev, pagep, 0,
2076                                                     adapter->big_page_size,
2077                                                     DMA_FROM_DEVICE);
2078                         if (dma_mapping_error(dev, page_dmaaddr)) {
2079                                 put_page(pagep);
2080                                 pagep = NULL;
2081                                 adapter->drv_stats.dma_map_errors++;
2082                                 break;
2083                         }
2084                         page_offset = 0;
2085                 } else {
2086                         get_page(pagep);
2087                         page_offset += rx_frag_size;
2088                 }
2089                 page_info->page_offset = page_offset;
2090                 page_info->page = pagep;
2091
2092                 rxd = queue_head_node(rxq);
2093                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
2094                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
2095                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
2096
2097                 /* Any space left in the current big page for another frag? */
2098                 if ((page_offset + rx_frag_size + rx_frag_size) >
2099                                         adapter->big_page_size) {
2100                         pagep = NULL;
2101                         page_info->last_frag = true;
2102                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
2103                 } else {
2104                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
2105                 }
2106
2107                 prev_page_info = page_info;
2108                 queue_head_inc(rxq);
2109                 page_info = &rxo->page_info_tbl[rxq->head];
2110         }
2111
2112         /* Mark the last frag of a page when we break out of the above loop
2113          * with no more slots available in the RXQ
2114          */
2115         if (pagep) {
2116                 prev_page_info->last_frag = true;
2117                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
2118         }
2119
2120         if (posted) {
2121                 atomic_add(posted, &rxq->used);
2122                 if (rxo->rx_post_starved)
2123                         rxo->rx_post_starved = false;
2124                 do {
2125                         notify = min(MAX_NUM_POST_ERX_DB, posted);
2126                         be_rxq_notify(adapter, rxq->id, notify);
2127                         posted -= notify;
2128                 } while (posted);
2129         } else if (atomic_read(&rxq->used) == 0) {
2130                 /* Let be_worker replenish when memory is available */
2131                 rxo->rx_post_starved = true;
2132         }
2133 }
2134
2135 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo)
2136 {
2137         struct be_queue_info *tx_cq = &txo->cq;
2138         struct be_tx_compl_info *txcp = &txo->txcp;
2139         struct be_eth_tx_compl *compl = queue_tail_node(tx_cq);
2140
2141         if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2142                 return NULL;
2143
2144         /* Ensure load ordering of valid bit dword and other dwords below */
2145         rmb();
2146         be_dws_le_to_cpu(compl, sizeof(*compl));
2147
2148         txcp->status = GET_TX_COMPL_BITS(status, compl);
2149         txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl);
2150
2151         compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2152         queue_tail_inc(tx_cq);
2153         return txcp;
2154 }
2155
2156 static u16 be_tx_compl_process(struct be_adapter *adapter,
2157                                struct be_tx_obj *txo, u16 last_index)
2158 {
2159         struct sk_buff **sent_skbs = txo->sent_skb_list;
2160         struct be_queue_info *txq = &txo->q;
2161         u16 frag_index, num_wrbs = 0;
2162         struct sk_buff *skb = NULL;
2163         bool unmap_skb_hdr = false;
2164         struct be_eth_wrb *wrb;
2165
2166         do {
2167                 if (sent_skbs[txq->tail]) {
2168                         /* Free skb from prev req */
2169                         if (skb)
2170                                 dev_consume_skb_any(skb);
2171                         skb = sent_skbs[txq->tail];
2172                         sent_skbs[txq->tail] = NULL;
2173                         queue_tail_inc(txq);  /* skip hdr wrb */
2174                         num_wrbs++;
2175                         unmap_skb_hdr = true;
2176                 }
2177                 wrb = queue_tail_node(txq);
2178                 frag_index = txq->tail;
2179                 unmap_tx_frag(&adapter->pdev->dev, wrb,
2180                               (unmap_skb_hdr && skb_headlen(skb)));
2181                 unmap_skb_hdr = false;
2182                 queue_tail_inc(txq);
2183                 num_wrbs++;
2184         } while (frag_index != last_index);
2185         dev_consume_skb_any(skb);
2186
2187         return num_wrbs;
2188 }
2189
2190 /* Return the number of events in the event queue */
2191 static inline int events_get(struct be_eq_obj *eqo)
2192 {
2193         struct be_eq_entry *eqe;
2194         int num = 0;
2195
2196         do {
2197                 eqe = queue_tail_node(&eqo->q);
2198                 if (eqe->evt == 0)
2199                         break;
2200
2201                 rmb();
2202                 eqe->evt = 0;
2203                 num++;
2204                 queue_tail_inc(&eqo->q);
2205         } while (true);
2206
2207         return num;
2208 }
2209
2210 /* Leaves the EQ is disarmed state */
2211 static void be_eq_clean(struct be_eq_obj *eqo)
2212 {
2213         int num = events_get(eqo);
2214
2215         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2216 }
2217
2218 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2219 {
2220         struct be_rx_page_info *page_info;
2221         struct be_queue_info *rxq = &rxo->q;
2222         struct be_queue_info *rx_cq = &rxo->cq;
2223         struct be_rx_compl_info *rxcp;
2224         struct be_adapter *adapter = rxo->adapter;
2225         int flush_wait = 0;
2226
2227         /* Consume pending rx completions.
2228          * Wait for the flush completion (identified by zero num_rcvd)
2229          * to arrive. Notify CQ even when there are no more CQ entries
2230          * for HW to flush partially coalesced CQ entries.
2231          * In Lancer, there is no need to wait for flush compl.
2232          */
2233         for (;;) {
2234                 rxcp = be_rx_compl_get(rxo);
2235                 if (!rxcp) {
2236                         if (lancer_chip(adapter))
2237                                 break;
2238
2239                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2240                                 dev_warn(&adapter->pdev->dev,
2241                                          "did not receive flush compl\n");
2242                                 break;
2243                         }
2244                         be_cq_notify(adapter, rx_cq->id, true, 0);
2245                         mdelay(1);
2246                 } else {
2247                         be_rx_compl_discard(rxo, rxcp);
2248                         be_cq_notify(adapter, rx_cq->id, false, 1);
2249                         if (rxcp->num_rcvd == 0)
2250                                 break;
2251                 }
2252         }
2253
2254         /* After cleanup, leave the CQ in unarmed state */
2255         be_cq_notify(adapter, rx_cq->id, false, 0);
2256
2257         /* Then free posted rx buffers that were not used */
2258         while (atomic_read(&rxq->used) > 0) {
2259                 page_info = get_rx_page_info(rxo);
2260                 put_page(page_info->page);
2261                 memset(page_info, 0, sizeof(*page_info));
2262         }
2263         BUG_ON(atomic_read(&rxq->used));
2264         rxq->tail = 0;
2265         rxq->head = 0;
2266 }
2267
2268 static void be_tx_compl_clean(struct be_adapter *adapter)
2269 {
2270         u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2271         struct device *dev = &adapter->pdev->dev;
2272         struct be_tx_compl_info *txcp;
2273         struct be_queue_info *txq;
2274         struct be_tx_obj *txo;
2275         int i, pending_txqs;
2276
2277         /* Stop polling for compls when HW has been silent for 10ms */
2278         do {
2279                 pending_txqs = adapter->num_tx_qs;
2280
2281                 for_all_tx_queues(adapter, txo, i) {
2282                         cmpl = 0;
2283                         num_wrbs = 0;
2284                         txq = &txo->q;
2285                         while ((txcp = be_tx_compl_get(txo))) {
2286                                 num_wrbs +=
2287                                         be_tx_compl_process(adapter, txo,
2288                                                             txcp->end_index);
2289                                 cmpl++;
2290                         }
2291                         if (cmpl) {
2292                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2293                                 atomic_sub(num_wrbs, &txq->used);
2294                                 timeo = 0;
2295                         }
2296                         if (!be_is_tx_compl_pending(txo))
2297                                 pending_txqs--;
2298                 }
2299
2300                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2301                         break;
2302
2303                 mdelay(1);
2304         } while (true);
2305
2306         /* Free enqueued TX that was never notified to HW */
2307         for_all_tx_queues(adapter, txo, i) {
2308                 txq = &txo->q;
2309
2310                 if (atomic_read(&txq->used)) {
2311                         dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2312                                  i, atomic_read(&txq->used));
2313                         notified_idx = txq->tail;
2314                         end_idx = txq->tail;
2315                         index_adv(&end_idx, atomic_read(&txq->used) - 1,
2316                                   txq->len);
2317                         /* Use the tx-compl process logic to handle requests
2318                          * that were not sent to the HW.
2319                          */
2320                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2321                         atomic_sub(num_wrbs, &txq->used);
2322                         BUG_ON(atomic_read(&txq->used));
2323                         txo->pend_wrb_cnt = 0;
2324                         /* Since hw was never notified of these requests,
2325                          * reset TXQ indices
2326                          */
2327                         txq->head = notified_idx;
2328                         txq->tail = notified_idx;
2329                 }
2330         }
2331 }
2332
2333 static void be_evt_queues_destroy(struct be_adapter *adapter)
2334 {
2335         struct be_eq_obj *eqo;
2336         int i;
2337
2338         for_all_evt_queues(adapter, eqo, i) {
2339                 if (eqo->q.created) {
2340                         be_eq_clean(eqo);
2341                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2342                         napi_hash_del(&eqo->napi);
2343                         netif_napi_del(&eqo->napi);
2344                 }
2345                 free_cpumask_var(eqo->affinity_mask);
2346                 be_queue_free(adapter, &eqo->q);
2347         }
2348 }
2349
2350 static int be_evt_queues_create(struct be_adapter *adapter)
2351 {
2352         struct be_queue_info *eq;
2353         struct be_eq_obj *eqo;
2354         struct be_aic_obj *aic;
2355         int i, rc;
2356
2357         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2358                                     adapter->cfg_num_qs);
2359
2360         for_all_evt_queues(adapter, eqo, i) {
2361                 int numa_node = dev_to_node(&adapter->pdev->dev);
2362                 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL))
2363                         return -ENOMEM;
2364                 cpumask_set_cpu(cpumask_local_spread(i, numa_node),
2365                                 eqo->affinity_mask);
2366                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2367                                BE_NAPI_WEIGHT);
2368                 napi_hash_add(&eqo->napi);
2369                 aic = &adapter->aic_obj[i];
2370                 eqo->adapter = adapter;
2371                 eqo->idx = i;
2372                 aic->max_eqd = BE_MAX_EQD;
2373                 aic->enable = true;
2374
2375                 eq = &eqo->q;
2376                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2377                                     sizeof(struct be_eq_entry));
2378                 if (rc)
2379                         return rc;
2380
2381                 rc = be_cmd_eq_create(adapter, eqo);
2382                 if (rc)
2383                         return rc;
2384         }
2385         return 0;
2386 }
2387
2388 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2389 {
2390         struct be_queue_info *q;
2391
2392         q = &adapter->mcc_obj.q;
2393         if (q->created)
2394                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2395         be_queue_free(adapter, q);
2396
2397         q = &adapter->mcc_obj.cq;
2398         if (q->created)
2399                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2400         be_queue_free(adapter, q);
2401 }
2402
2403 /* Must be called only after TX qs are created as MCC shares TX EQ */
2404 static int be_mcc_queues_create(struct be_adapter *adapter)
2405 {
2406         struct be_queue_info *q, *cq;
2407
2408         cq = &adapter->mcc_obj.cq;
2409         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2410                            sizeof(struct be_mcc_compl)))
2411                 goto err;
2412
2413         /* Use the default EQ for MCC completions */
2414         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2415                 goto mcc_cq_free;
2416
2417         q = &adapter->mcc_obj.q;
2418         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2419                 goto mcc_cq_destroy;
2420
2421         if (be_cmd_mccq_create(adapter, q, cq))
2422                 goto mcc_q_free;
2423
2424         return 0;
2425
2426 mcc_q_free:
2427         be_queue_free(adapter, q);
2428 mcc_cq_destroy:
2429         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2430 mcc_cq_free:
2431         be_queue_free(adapter, cq);
2432 err:
2433         return -1;
2434 }
2435
2436 static void be_tx_queues_destroy(struct be_adapter *adapter)
2437 {
2438         struct be_queue_info *q;
2439         struct be_tx_obj *txo;
2440         u8 i;
2441
2442         for_all_tx_queues(adapter, txo, i) {
2443                 q = &txo->q;
2444                 if (q->created)
2445                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2446                 be_queue_free(adapter, q);
2447
2448                 q = &txo->cq;
2449                 if (q->created)
2450                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2451                 be_queue_free(adapter, q);
2452         }
2453 }
2454
2455 static int be_tx_qs_create(struct be_adapter *adapter)
2456 {
2457         struct be_queue_info *cq;
2458         struct be_tx_obj *txo;
2459         struct be_eq_obj *eqo;
2460         int status, i;
2461
2462         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2463
2464         for_all_tx_queues(adapter, txo, i) {
2465                 cq = &txo->cq;
2466                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2467                                         sizeof(struct be_eth_tx_compl));
2468                 if (status)
2469                         return status;
2470
2471                 u64_stats_init(&txo->stats.sync);
2472                 u64_stats_init(&txo->stats.sync_compl);
2473
2474                 /* If num_evt_qs is less than num_tx_qs, then more than
2475                  * one txq share an eq
2476                  */
2477                 eqo = &adapter->eq_obj[i % adapter->num_evt_qs];
2478                 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3);
2479                 if (status)
2480                         return status;
2481
2482                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2483                                         sizeof(struct be_eth_wrb));
2484                 if (status)
2485                         return status;
2486
2487                 status = be_cmd_txq_create(adapter, txo);
2488                 if (status)
2489                         return status;
2490
2491                 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask,
2492                                     eqo->idx);
2493         }
2494
2495         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2496                  adapter->num_tx_qs);
2497         return 0;
2498 }
2499
2500 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2501 {
2502         struct be_queue_info *q;
2503         struct be_rx_obj *rxo;
2504         int i;
2505
2506         for_all_rx_queues(adapter, rxo, i) {
2507                 q = &rxo->cq;
2508                 if (q->created)
2509                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2510                 be_queue_free(adapter, q);
2511         }
2512 }
2513
2514 static int be_rx_cqs_create(struct be_adapter *adapter)
2515 {
2516         struct be_queue_info *eq, *cq;
2517         struct be_rx_obj *rxo;
2518         int rc, i;
2519
2520         /* We can create as many RSS rings as there are EQs. */
2521         adapter->num_rss_qs = adapter->num_evt_qs;
2522
2523         /* We'll use RSS only if atleast 2 RSS rings are supported. */
2524         if (adapter->num_rss_qs <= 1)
2525                 adapter->num_rss_qs = 0;
2526
2527         adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq;
2528
2529         /* When the interface is not capable of RSS rings (and there is no
2530          * need to create a default RXQ) we'll still need one RXQ
2531          */
2532         if (adapter->num_rx_qs == 0)
2533                 adapter->num_rx_qs = 1;
2534
2535         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2536         for_all_rx_queues(adapter, rxo, i) {
2537                 rxo->adapter = adapter;
2538                 cq = &rxo->cq;
2539                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2540                                     sizeof(struct be_eth_rx_compl));
2541                 if (rc)
2542                         return rc;
2543
2544                 u64_stats_init(&rxo->stats.sync);
2545                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2546                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2547                 if (rc)
2548                         return rc;
2549         }
2550
2551         dev_info(&adapter->pdev->dev,
2552                  "created %d RX queue(s)\n", adapter->num_rx_qs);
2553         return 0;
2554 }
2555
2556 static irqreturn_t be_intx(int irq, void *dev)
2557 {
2558         struct be_eq_obj *eqo = dev;
2559         struct be_adapter *adapter = eqo->adapter;
2560         int num_evts = 0;
2561
2562         /* IRQ is not expected when NAPI is scheduled as the EQ
2563          * will not be armed.
2564          * But, this can happen on Lancer INTx where it takes
2565          * a while to de-assert INTx or in BE2 where occasionaly
2566          * an interrupt may be raised even when EQ is unarmed.
2567          * If NAPI is already scheduled, then counting & notifying
2568          * events will orphan them.
2569          */
2570         if (napi_schedule_prep(&eqo->napi)) {
2571                 num_evts = events_get(eqo);
2572                 __napi_schedule(&eqo->napi);
2573                 if (num_evts)
2574                         eqo->spurious_intr = 0;
2575         }
2576         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2577
2578         /* Return IRQ_HANDLED only for the the first spurious intr
2579          * after a valid intr to stop the kernel from branding
2580          * this irq as a bad one!
2581          */
2582         if (num_evts || eqo->spurious_intr++ == 0)
2583                 return IRQ_HANDLED;
2584         else
2585                 return IRQ_NONE;
2586 }
2587
2588 static irqreturn_t be_msix(int irq, void *dev)
2589 {
2590         struct be_eq_obj *eqo = dev;
2591
2592         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2593         napi_schedule(&eqo->napi);
2594         return IRQ_HANDLED;
2595 }
2596
2597 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2598 {
2599         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2600 }
2601
2602 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2603                          int budget, int polling)
2604 {
2605         struct be_adapter *adapter = rxo->adapter;
2606         struct be_queue_info *rx_cq = &rxo->cq;
2607         struct be_rx_compl_info *rxcp;
2608         u32 work_done;
2609         u32 frags_consumed = 0;
2610
2611         for (work_done = 0; work_done < budget; work_done++) {
2612                 rxcp = be_rx_compl_get(rxo);
2613                 if (!rxcp)
2614                         break;
2615
2616                 /* Is it a flush compl that has no data */
2617                 if (unlikely(rxcp->num_rcvd == 0))
2618                         goto loop_continue;
2619
2620                 /* Discard compl with partial DMA Lancer B0 */
2621                 if (unlikely(!rxcp->pkt_size)) {
2622                         be_rx_compl_discard(rxo, rxcp);
2623                         goto loop_continue;
2624                 }
2625
2626                 /* On BE drop pkts that arrive due to imperfect filtering in
2627                  * promiscuous mode on some skews
2628                  */
2629                 if (unlikely(rxcp->port != adapter->port_num &&
2630                              !lancer_chip(adapter))) {
2631                         be_rx_compl_discard(rxo, rxcp);
2632                         goto loop_continue;
2633                 }
2634
2635                 /* Don't do gro when we're busy_polling */
2636                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2637                         be_rx_compl_process_gro(rxo, napi, rxcp);
2638                 else
2639                         be_rx_compl_process(rxo, napi, rxcp);
2640
2641 loop_continue:
2642                 frags_consumed += rxcp->num_rcvd;
2643                 be_rx_stats_update(rxo, rxcp);
2644         }
2645
2646         if (work_done) {
2647                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2648
2649                 /* When an rx-obj gets into post_starved state, just
2650                  * let be_worker do the posting.
2651                  */
2652                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2653                     !rxo->rx_post_starved)
2654                         be_post_rx_frags(rxo, GFP_ATOMIC,
2655                                          max_t(u32, MAX_RX_POST,
2656                                                frags_consumed));
2657         }
2658
2659         return work_done;
2660 }
2661
2662 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)
2663 {
2664         switch (status) {
2665         case BE_TX_COMP_HDR_PARSE_ERR:
2666                 tx_stats(txo)->tx_hdr_parse_err++;
2667                 break;
2668         case BE_TX_COMP_NDMA_ERR:
2669                 tx_stats(txo)->tx_dma_err++;
2670                 break;
2671         case BE_TX_COMP_ACL_ERR:
2672                 tx_stats(txo)->tx_spoof_check_err++;
2673                 break;
2674         }
2675 }
2676
2677 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)
2678 {
2679         switch (status) {
2680         case LANCER_TX_COMP_LSO_ERR:
2681                 tx_stats(txo)->tx_tso_err++;
2682                 break;
2683         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2684         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2685                 tx_stats(txo)->tx_spoof_check_err++;
2686                 break;
2687         case LANCER_TX_COMP_QINQ_ERR:
2688                 tx_stats(txo)->tx_qinq_err++;
2689                 break;
2690         case LANCER_TX_COMP_PARITY_ERR:
2691                 tx_stats(txo)->tx_internal_parity_err++;
2692                 break;
2693         case LANCER_TX_COMP_DMA_ERR:
2694                 tx_stats(txo)->tx_dma_err++;
2695                 break;
2696         }
2697 }
2698
2699 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2700                           int idx)
2701 {
2702         int num_wrbs = 0, work_done = 0;
2703         struct be_tx_compl_info *txcp;
2704
2705         while ((txcp = be_tx_compl_get(txo))) {
2706                 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);
2707                 work_done++;
2708
2709                 if (txcp->status) {
2710                         if (lancer_chip(adapter))
2711                                 lancer_update_tx_err(txo, txcp->status);
2712                         else
2713                                 be_update_tx_err(txo, txcp->status);
2714                 }
2715         }
2716
2717         if (work_done) {
2718                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2719                 atomic_sub(num_wrbs, &txo->q.used);
2720
2721                 /* As Tx wrbs have been freed up, wake up netdev queue
2722                  * if it was stopped due to lack of tx wrbs.  */
2723                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2724                     be_can_txq_wake(txo)) {
2725                         netif_wake_subqueue(adapter->netdev, idx);
2726                 }
2727
2728                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2729                 tx_stats(txo)->tx_compl += work_done;
2730                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2731         }
2732 }
2733
2734 #ifdef CONFIG_NET_RX_BUSY_POLL
2735 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2736 {
2737         bool status = true;
2738
2739         spin_lock(&eqo->lock); /* BH is already disabled */
2740         if (eqo->state & BE_EQ_LOCKED) {
2741                 WARN_ON(eqo->state & BE_EQ_NAPI);
2742                 eqo->state |= BE_EQ_NAPI_YIELD;
2743                 status = false;
2744         } else {
2745                 eqo->state = BE_EQ_NAPI;
2746         }
2747         spin_unlock(&eqo->lock);
2748         return status;
2749 }
2750
2751 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2752 {
2753         spin_lock(&eqo->lock); /* BH is already disabled */
2754
2755         WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
2756         eqo->state = BE_EQ_IDLE;
2757
2758         spin_unlock(&eqo->lock);
2759 }
2760
2761 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2762 {
2763         bool status = true;
2764
2765         spin_lock_bh(&eqo->lock);
2766         if (eqo->state & BE_EQ_LOCKED) {
2767                 eqo->state |= BE_EQ_POLL_YIELD;
2768                 status = false;
2769         } else {
2770                 eqo->state |= BE_EQ_POLL;
2771         }
2772         spin_unlock_bh(&eqo->lock);
2773         return status;
2774 }
2775
2776 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2777 {
2778         spin_lock_bh(&eqo->lock);
2779
2780         WARN_ON(eqo->state & (BE_EQ_NAPI));
2781         eqo->state = BE_EQ_IDLE;
2782
2783         spin_unlock_bh(&eqo->lock);
2784 }
2785
2786 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2787 {
2788         spin_lock_init(&eqo->lock);
2789         eqo->state = BE_EQ_IDLE;
2790 }
2791
2792 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2793 {
2794         local_bh_disable();
2795
2796         /* It's enough to just acquire napi lock on the eqo to stop
2797          * be_busy_poll() from processing any queueus.
2798          */
2799         while (!be_lock_napi(eqo))
2800                 mdelay(1);
2801
2802         local_bh_enable();
2803 }
2804
2805 #else /* CONFIG_NET_RX_BUSY_POLL */
2806
2807 static inline bool be_lock_napi(struct be_eq_obj *eqo)
2808 {
2809         return true;
2810 }
2811
2812 static inline void be_unlock_napi(struct be_eq_obj *eqo)
2813 {
2814 }
2815
2816 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
2817 {
2818         return false;
2819 }
2820
2821 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
2822 {
2823 }
2824
2825 static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
2826 {
2827 }
2828
2829 static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
2830 {
2831 }
2832 #endif /* CONFIG_NET_RX_BUSY_POLL */
2833
2834 int be_poll(struct napi_struct *napi, int budget)
2835 {
2836         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2837         struct be_adapter *adapter = eqo->adapter;
2838         int max_work = 0, work, i, num_evts;
2839         struct be_rx_obj *rxo;
2840         struct be_tx_obj *txo;
2841
2842         num_evts = events_get(eqo);
2843
2844         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2845                 be_process_tx(adapter, txo, i);
2846
2847         if (be_lock_napi(eqo)) {
2848                 /* This loop will iterate twice for EQ0 in which
2849                  * completions of the last RXQ (default one) are also processed
2850                  * For other EQs the loop iterates only once
2851                  */
2852                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2853                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2854                         max_work = max(work, max_work);
2855                 }
2856                 be_unlock_napi(eqo);
2857         } else {
2858                 max_work = budget;
2859         }
2860
2861         if (is_mcc_eqo(eqo))
2862                 be_process_mcc(adapter);
2863
2864         if (max_work < budget) {
2865                 napi_complete(napi);
2866                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2867         } else {
2868                 /* As we'll continue in polling mode, count and clear events */
2869                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2870         }
2871         return max_work;
2872 }
2873
2874 #ifdef CONFIG_NET_RX_BUSY_POLL
2875 static int be_busy_poll(struct napi_struct *napi)
2876 {
2877         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2878         struct be_adapter *adapter = eqo->adapter;
2879         struct be_rx_obj *rxo;
2880         int i, work = 0;
2881
2882         if (!be_lock_busy_poll(eqo))
2883                 return LL_FLUSH_BUSY;
2884
2885         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2886                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2887                 if (work)
2888                         break;
2889         }
2890
2891         be_unlock_busy_poll(eqo);
2892         return work;
2893 }
2894 #endif
2895
2896 void be_detect_error(struct be_adapter *adapter)
2897 {
2898         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2899         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2900         u32 i;
2901         bool error_detected = false;
2902         struct device *dev = &adapter->pdev->dev;
2903         struct net_device *netdev = adapter->netdev;
2904
2905         if (be_hw_error(adapter))
2906                 return;
2907
2908         if (lancer_chip(adapter)) {
2909                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2910                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2911                         sliport_err1 = ioread32(adapter->db +
2912                                                 SLIPORT_ERROR1_OFFSET);
2913                         sliport_err2 = ioread32(adapter->db +
2914                                                 SLIPORT_ERROR2_OFFSET);
2915                         adapter->hw_error = true;
2916                         error_detected = true;
2917                         /* Do not log error messages if its a FW reset */
2918                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2919                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2920                                 dev_info(dev, "Firmware update in progress\n");
2921                         } else {
2922                                 dev_err(dev, "Error detected in the card\n");
2923                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2924                                         sliport_status);
2925                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2926                                         sliport_err1);
2927                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2928                                         sliport_err2);
2929                         }
2930                 }
2931         } else {
2932                 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW);
2933                 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH);
2934                 ue_lo_mask = ioread32(adapter->pcicfg +
2935                                       PCICFG_UE_STATUS_LOW_MASK);
2936                 ue_hi_mask = ioread32(adapter->pcicfg +
2937                                       PCICFG_UE_STATUS_HI_MASK);
2938
2939                 ue_lo = (ue_lo & ~ue_lo_mask);
2940                 ue_hi = (ue_hi & ~ue_hi_mask);
2941
2942                 /* On certain platforms BE hardware can indicate spurious UEs.
2943                  * Allow HW to stop working completely in case of a real UE.
2944                  * Hence not setting the hw_error for UE detection.
2945                  */
2946
2947                 if (ue_lo || ue_hi) {
2948                         error_detected = true;
2949                         dev_err(dev,
2950                                 "Unrecoverable Error detected in the adapter");
2951                         dev_err(dev, "Please reboot server to recover");
2952                         if (skyhawk_chip(adapter))
2953                                 adapter->hw_error = true;
2954                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2955                                 if (ue_lo & 1)
2956                                         dev_err(dev, "UE: %s bit set\n",
2957                                                 ue_status_low_desc[i]);
2958                         }
2959                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2960                                 if (ue_hi & 1)
2961                                         dev_err(dev, "UE: %s bit set\n",
2962                                                 ue_status_hi_desc[i]);
2963                         }
2964                 }
2965         }
2966         if (error_detected)
2967                 netif_carrier_off(netdev);
2968 }
2969
2970 static void be_msix_disable(struct be_adapter *adapter)
2971 {
2972         if (msix_enabled(adapter)) {
2973                 pci_disable_msix(adapter->pdev);
2974                 adapter->num_msix_vec = 0;
2975                 adapter->num_msix_roce_vec = 0;
2976         }
2977 }
2978
2979 static int be_msix_enable(struct be_adapter *adapter)
2980 {
2981         int i, num_vec;
2982         struct device *dev = &adapter->pdev->dev;
2983
2984         /* If RoCE is supported, program the max number of NIC vectors that
2985          * may be configured via set-channels, along with vectors needed for
2986          * RoCe. Else, just program the number we'll use initially.
2987          */
2988         if (be_roce_supported(adapter))
2989                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2990                                 2 * num_online_cpus());
2991         else
2992                 num_vec = adapter->cfg_num_qs;
2993
2994         for (i = 0; i < num_vec; i++)
2995                 adapter->msix_entries[i].entry = i;
2996
2997         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2998                                         MIN_MSIX_VECTORS, num_vec);
2999         if (num_vec < 0)
3000                 goto fail;
3001
3002         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
3003                 adapter->num_msix_roce_vec = num_vec / 2;
3004                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
3005                          adapter->num_msix_roce_vec);
3006         }
3007
3008         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
3009
3010         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
3011                  adapter->num_msix_vec);
3012         return 0;
3013
3014 fail:
3015         dev_warn(dev, "MSIx enable failed\n");
3016
3017         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
3018         if (!be_physfn(adapter))
3019                 return num_vec;
3020         return 0;
3021 }
3022
3023 static inline int be_msix_vec_get(struct be_adapter *adapter,
3024                                   struct be_eq_obj *eqo)
3025 {
3026         return adapter->msix_entries[eqo->msix_idx].vector;
3027 }
3028
3029 static int be_msix_register(struct be_adapter *adapter)
3030 {
3031         struct net_device *netdev = adapter->netdev;
3032         struct be_eq_obj *eqo;
3033         int status, i, vec;
3034
3035         for_all_evt_queues(adapter, eqo, i) {
3036                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
3037                 vec = be_msix_vec_get(adapter, eqo);
3038                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
3039                 if (status)
3040                         goto err_msix;
3041
3042                 irq_set_affinity_hint(vec, eqo->affinity_mask);
3043         }
3044
3045         return 0;
3046 err_msix:
3047         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
3048                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
3049         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
3050                  status);
3051         be_msix_disable(adapter);
3052         return status;
3053 }
3054
3055 static int be_irq_register(struct be_adapter *adapter)
3056 {
3057         struct net_device *netdev = adapter->netdev;
3058         int status;
3059
3060         if (msix_enabled(adapter)) {
3061                 status = be_msix_register(adapter);
3062                 if (status == 0)
3063                         goto done;
3064                 /* INTx is not supported for VF */
3065                 if (!be_physfn(adapter))
3066                         return status;
3067         }
3068
3069         /* INTx: only the first EQ is used */
3070         netdev->irq = adapter->pdev->irq;
3071         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
3072                              &adapter->eq_obj[0]);
3073         if (status) {
3074                 dev_err(&adapter->pdev->dev,
3075                         "INTx request IRQ failed - err %d\n", status);
3076                 return status;
3077         }
3078 done:
3079         adapter->isr_registered = true;
3080         return 0;
3081 }
3082
3083 static void be_irq_unregister(struct be_adapter *adapter)
3084 {
3085         struct net_device *netdev = adapter->netdev;
3086         struct be_eq_obj *eqo;
3087         int i, vec;
3088
3089         if (!adapter->isr_registered)
3090                 return;
3091
3092         /* INTx */
3093         if (!msix_enabled(adapter)) {
3094                 free_irq(netdev->irq, &adapter->eq_obj[0]);
3095                 goto done;
3096         }
3097
3098         /* MSIx */
3099         for_all_evt_queues(adapter, eqo, i) {
3100                 vec = be_msix_vec_get(adapter, eqo);
3101                 irq_set_affinity_hint(vec, NULL);
3102                 free_irq(vec, eqo);
3103         }
3104
3105 done:
3106         adapter->isr_registered = false;
3107 }
3108
3109 static void be_rx_qs_destroy(struct be_adapter *adapter)
3110 {
3111         struct be_queue_info *q;
3112         struct be_rx_obj *rxo;
3113         int i;
3114
3115         for_all_rx_queues(adapter, rxo, i) {
3116                 q = &rxo->q;
3117                 if (q->created) {
3118                         be_cmd_rxq_destroy(adapter, q);
3119                         be_rx_cq_clean(rxo);
3120                 }
3121                 be_queue_free(adapter, q);
3122         }
3123 }
3124
3125 static int be_close(struct net_device *netdev)
3126 {
3127         struct be_adapter *adapter = netdev_priv(netdev);
3128         struct be_eq_obj *eqo;
3129         int i;
3130
3131         /* This protection is needed as be_close() may be called even when the
3132          * adapter is in cleared state (after eeh perm failure)
3133          */
3134         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
3135                 return 0;
3136
3137         be_roce_dev_close(adapter);
3138
3139         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
3140                 for_all_evt_queues(adapter, eqo, i) {
3141                         napi_disable(&eqo->napi);
3142                         be_disable_busy_poll(eqo);
3143                 }
3144                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
3145         }
3146
3147         be_async_mcc_disable(adapter);
3148
3149         /* Wait for all pending tx completions to arrive so that
3150          * all tx skbs are freed.
3151          */
3152         netif_tx_disable(netdev);
3153         be_tx_compl_clean(adapter);
3154
3155         be_rx_qs_destroy(adapter);
3156         be_clear_uc_list(adapter);
3157
3158         for_all_evt_queues(adapter, eqo, i) {
3159                 if (msix_enabled(adapter))
3160                         synchronize_irq(be_msix_vec_get(adapter, eqo));
3161                 else
3162                         synchronize_irq(netdev->irq);
3163                 be_eq_clean(eqo);
3164         }
3165
3166         be_irq_unregister(adapter);
3167
3168         return 0;
3169 }
3170
3171 static int be_rx_qs_create(struct be_adapter *adapter)
3172 {
3173         struct rss_info *rss = &adapter->rss_info;
3174         u8 rss_key[RSS_HASH_KEY_LEN];
3175         struct be_rx_obj *rxo;
3176         int rc, i, j;
3177
3178         for_all_rx_queues(adapter, rxo, i) {
3179                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3180                                     sizeof(struct be_eth_rx_d));
3181                 if (rc)
3182                         return rc;
3183         }
3184
3185         if (adapter->need_def_rxq || !adapter->num_rss_qs) {
3186                 rxo = default_rxo(adapter);
3187                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3188                                        rx_frag_size, adapter->if_handle,
3189                                        false, &rxo->rss_id);
3190                 if (rc)
3191                         return rc;
3192         }
3193
3194         for_all_rss_queues(adapter, rxo, i) {
3195                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3196                                        rx_frag_size, adapter->if_handle,
3197                                        true, &rxo->rss_id);
3198                 if (rc)
3199                         return rc;
3200         }
3201
3202         if (be_multi_rxq(adapter)) {
3203                 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) {
3204                         for_all_rss_queues(adapter, rxo, i) {
3205                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
3206                                         break;
3207                                 rss->rsstable[j + i] = rxo->rss_id;
3208                                 rss->rss_queue[j + i] = i;
3209                         }
3210                 }
3211                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3212                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3213
3214                 if (!BEx_chip(adapter))
3215                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3216                                 RSS_ENABLE_UDP_IPV6;
3217         } else {
3218                 /* Disable RSS, if only default RX Q is created */
3219                 rss->rss_flags = RSS_ENABLE_NONE;
3220         }
3221
3222         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3223         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3224                                128, rss_key);
3225         if (rc) {
3226                 rss->rss_flags = RSS_ENABLE_NONE;
3227                 return rc;
3228         }
3229
3230         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3231
3232         /* First time posting */
3233         for_all_rx_queues(adapter, rxo, i)
3234                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
3235         return 0;
3236 }
3237
3238 static int be_open(struct net_device *netdev)
3239 {
3240         struct be_adapter *adapter = netdev_priv(netdev);
3241         struct be_eq_obj *eqo;
3242         struct be_rx_obj *rxo;
3243         struct be_tx_obj *txo;
3244         u8 link_status;
3245         int status, i;
3246
3247         status = be_rx_qs_create(adapter);
3248         if (status)
3249                 goto err;
3250
3251         status = be_irq_register(adapter);
3252         if (status)
3253                 goto err;
3254
3255         for_all_rx_queues(adapter, rxo, i)
3256                 be_cq_notify(adapter, rxo->cq.id, true, 0);
3257
3258         for_all_tx_queues(adapter, txo, i)
3259                 be_cq_notify(adapter, txo->cq.id, true, 0);
3260
3261         be_async_mcc_enable(adapter);
3262
3263         for_all_evt_queues(adapter, eqo, i) {
3264                 napi_enable(&eqo->napi);
3265                 be_enable_busy_poll(eqo);
3266                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
3267         }
3268         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3269
3270         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3271         if (!status)
3272                 be_link_status_update(adapter, link_status);
3273
3274         netif_tx_start_all_queues(netdev);
3275         be_roce_dev_open(adapter);
3276
3277 #ifdef CONFIG_BE2NET_VXLAN
3278         if (skyhawk_chip(adapter))
3279                 vxlan_get_rx_port(netdev);
3280 #endif
3281
3282         return 0;
3283 err:
3284         be_close(adapter->netdev);
3285         return -EIO;
3286 }
3287
3288 static int be_setup_wol(struct be_adapter *adapter, bool enable)
3289 {
3290         struct be_dma_mem cmd;
3291         int status = 0;
3292         u8 mac[ETH_ALEN];
3293
3294         eth_zero_addr(mac);
3295
3296         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
3297         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
3298                                      GFP_KERNEL);
3299         if (!cmd.va)
3300                 return -ENOMEM;
3301
3302         if (enable) {
3303                 status = pci_write_config_dword(adapter->pdev,
3304                                                 PCICFG_PM_CONTROL_OFFSET,
3305                                                 PCICFG_PM_CONTROL_MASK);
3306                 if (status) {
3307                         dev_err(&adapter->pdev->dev,
3308                                 "Could not enable Wake-on-lan\n");
3309                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
3310                                           cmd.dma);
3311                         return status;
3312                 }
3313                 status = be_cmd_enable_magic_wol(adapter,
3314                                                  adapter->netdev->dev_addr,
3315                                                  &cmd);
3316                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3317                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3318         } else {
3319                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3320                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3321                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3322         }
3323
3324         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3325         return status;
3326 }
3327
3328 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3329 {
3330         u32 addr;
3331
3332         addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3333
3334         mac[5] = (u8)(addr & 0xFF);
3335         mac[4] = (u8)((addr >> 8) & 0xFF);
3336         mac[3] = (u8)((addr >> 16) & 0xFF);
3337         /* Use the OUI from the current MAC address */
3338         memcpy(mac, adapter->netdev->dev_addr, 3);
3339 }
3340
3341 /*
3342  * Generate a seed MAC address from the PF MAC Address using jhash.
3343  * MAC Address for VFs are assigned incrementally starting from the seed.
3344  * These addresses are programmed in the ASIC by the PF and the VF driver
3345  * queries for the MAC address during its probe.
3346  */
3347 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3348 {
3349         u32 vf;
3350         int status = 0;
3351         u8 mac[ETH_ALEN];
3352         struct be_vf_cfg *vf_cfg;
3353
3354         be_vf_eth_addr_generate(adapter, mac);
3355
3356         for_all_vfs(adapter, vf_cfg, vf) {
3357                 if (BEx_chip(adapter))
3358                         status = be_cmd_pmac_add(adapter, mac,
3359                                                  vf_cfg->if_handle,
3360                                                  &vf_cfg->pmac_id, vf + 1);
3361                 else
3362                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3363                                                 vf + 1);
3364
3365                 if (status)
3366                         dev_err(&adapter->pdev->dev,
3367                                 "Mac address assignment failed for VF %d\n",
3368                                 vf);
3369                 else
3370                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3371
3372                 mac[5] += 1;
3373         }
3374         return status;
3375 }
3376
3377 static int be_vfs_mac_query(struct be_adapter *adapter)
3378 {
3379         int status, vf;
3380         u8 mac[ETH_ALEN];
3381         struct be_vf_cfg *vf_cfg;
3382
3383         for_all_vfs(adapter, vf_cfg, vf) {
3384                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3385                                                mac, vf_cfg->if_handle,
3386                                                false, vf+1);
3387                 if (status)
3388                         return status;
3389                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3390         }
3391         return 0;
3392 }
3393
3394 static void be_vf_clear(struct be_adapter *adapter)
3395 {
3396         struct be_vf_cfg *vf_cfg;
3397         u32 vf;
3398
3399         if (pci_vfs_assigned(adapter->pdev)) {
3400                 dev_warn(&adapter->pdev->dev,
3401                          "VFs are assigned to VMs: not disabling VFs\n");
3402                 goto done;
3403         }
3404
3405         pci_disable_sriov(adapter->pdev);
3406
3407         for_all_vfs(adapter, vf_cfg, vf) {
3408                 if (BEx_chip(adapter))
3409                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3410                                         vf_cfg->pmac_id, vf + 1);
3411                 else
3412                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3413                                        vf + 1);
3414
3415                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3416         }
3417 done:
3418         kfree(adapter->vf_cfg);
3419         adapter->num_vfs = 0;
3420         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3421 }
3422
3423 static void be_clear_queues(struct be_adapter *adapter)
3424 {
3425         be_mcc_queues_destroy(adapter);
3426         be_rx_cqs_destroy(adapter);
3427         be_tx_queues_destroy(adapter);
3428         be_evt_queues_destroy(adapter);
3429 }
3430
3431 static void be_cancel_worker(struct be_adapter *adapter)
3432 {
3433         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3434                 cancel_delayed_work_sync(&adapter->work);
3435                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3436         }
3437 }
3438
3439 static void be_cancel_err_detection(struct be_adapter *adapter)
3440 {
3441         if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) {
3442                 cancel_delayed_work_sync(&adapter->be_err_detection_work);
3443                 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED;
3444         }
3445 }
3446
3447 static void be_mac_clear(struct be_adapter *adapter)
3448 {
3449         if (adapter->pmac_id) {
3450                 be_cmd_pmac_del(adapter, adapter->if_handle,
3451                                 adapter->pmac_id[0], 0);
3452                 kfree(adapter->pmac_id);
3453                 adapter->pmac_id = NULL;
3454         }
3455 }
3456
3457 #ifdef CONFIG_BE2NET_VXLAN
3458 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3459 {
3460         struct net_device *netdev = adapter->netdev;
3461
3462         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3463                 be_cmd_manage_iface(adapter, adapter->if_handle,
3464                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3465
3466         if (adapter->vxlan_port)
3467                 be_cmd_set_vxlan_port(adapter, 0);
3468
3469         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3470         adapter->vxlan_port = 0;
3471
3472         netdev->hw_enc_features = 0;
3473         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3474         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3475 }
3476 #endif
3477
3478 static u16 be_calculate_vf_qs(struct be_adapter *adapter, u16 num_vfs)
3479 {
3480         struct be_resources res = adapter->pool_res;
3481         u16 num_vf_qs = 1;
3482
3483         /* Distribute the queue resources equally among the PF and it's VFs
3484          * Do not distribute queue resources in multi-channel configuration.
3485          */
3486         if (num_vfs && !be_is_mc(adapter)) {
3487                 /* If number of VFs requested is 8 less than max supported,
3488                  * assign 8 queue pairs to the PF and divide the remaining
3489                  * resources evenly among the VFs
3490                  */
3491                 if (num_vfs < (be_max_vfs(adapter) - 8))
3492                         num_vf_qs = (res.max_rss_qs - 8) / num_vfs;
3493                 else
3494                         num_vf_qs = res.max_rss_qs / num_vfs;
3495
3496                 /* Skyhawk-R chip supports only MAX_RSS_IFACES RSS capable
3497                  * interfaces per port. Provide RSS on VFs, only if number
3498                  * of VFs requested is less than MAX_RSS_IFACES limit.
3499                  */
3500                 if (num_vfs >= MAX_RSS_IFACES)
3501                         num_vf_qs = 1;
3502         }
3503         return num_vf_qs;
3504 }
3505
3506 static int be_clear(struct be_adapter *adapter)
3507 {
3508         struct pci_dev *pdev = adapter->pdev;
3509         u16 num_vf_qs;
3510
3511         be_cancel_worker(adapter);
3512
3513         if (sriov_enabled(adapter))
3514                 be_vf_clear(adapter);
3515
3516         /* Re-configure FW to distribute resources evenly across max-supported
3517          * number of VFs, only when VFs are not already enabled.
3518          */
3519         if (skyhawk_chip(adapter) && be_physfn(adapter) &&
3520             !pci_vfs_assigned(pdev)) {
3521                 num_vf_qs = be_calculate_vf_qs(adapter,
3522                                                pci_sriov_get_totalvfs(pdev));
3523                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3524                                         pci_sriov_get_totalvfs(pdev),
3525                                         num_vf_qs);
3526         }
3527
3528 #ifdef CONFIG_BE2NET_VXLAN
3529         be_disable_vxlan_offloads(adapter);
3530 #endif
3531         /* delete the primary mac along with the uc-mac list */
3532         be_mac_clear(adapter);
3533
3534         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3535
3536         be_clear_queues(adapter);
3537
3538         be_msix_disable(adapter);
3539         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3540         return 0;
3541 }
3542
3543 static int be_if_create(struct be_adapter *adapter, u32 *if_handle,
3544                         u32 cap_flags, u32 vf)
3545 {
3546         u32 en_flags;
3547
3548         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3549                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS |
3550                    BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS;
3551
3552         en_flags &= cap_flags;
3553
3554         return be_cmd_if_create(adapter, cap_flags, en_flags, if_handle, vf);
3555 }
3556
3557 static int be_vfs_if_create(struct be_adapter *adapter)
3558 {
3559         struct be_resources res = {0};
3560         struct be_vf_cfg *vf_cfg;
3561         u32 cap_flags, vf;
3562         int status;
3563
3564         /* If a FW profile exists, then cap_flags are updated */
3565         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3566                     BE_IF_FLAGS_MULTICAST;
3567
3568         for_all_vfs(adapter, vf_cfg, vf) {
3569                 if (!BE3_chip(adapter)) {
3570                         status = be_cmd_get_profile_config(adapter, &res,
3571                                                            RESOURCE_LIMITS,
3572                                                            vf + 1);
3573                         if (!status) {
3574                                 cap_flags = res.if_cap_flags;
3575                                 /* Prevent VFs from enabling VLAN promiscuous
3576                                  * mode
3577                                  */
3578                                 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
3579                         }
3580                 }
3581
3582                 status = be_if_create(adapter, &vf_cfg->if_handle,
3583                                       cap_flags, vf + 1);
3584                 if (status)
3585                         return status;
3586         }
3587
3588         return 0;
3589 }
3590
3591 static int be_vf_setup_init(struct be_adapter *adapter)
3592 {
3593         struct be_vf_cfg *vf_cfg;
3594         int vf;
3595
3596         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3597                                   GFP_KERNEL);
3598         if (!adapter->vf_cfg)
3599                 return -ENOMEM;
3600
3601         for_all_vfs(adapter, vf_cfg, vf) {
3602                 vf_cfg->if_handle = -1;
3603                 vf_cfg->pmac_id = -1;
3604         }
3605         return 0;
3606 }
3607
3608 static int be_vf_setup(struct be_adapter *adapter)
3609 {
3610         struct device *dev = &adapter->pdev->dev;
3611         struct be_vf_cfg *vf_cfg;
3612         int status, old_vfs, vf;
3613
3614         old_vfs = pci_num_vf(adapter->pdev);
3615
3616         status = be_vf_setup_init(adapter);
3617         if (status)
3618                 goto err;
3619
3620         if (old_vfs) {
3621                 for_all_vfs(adapter, vf_cfg, vf) {
3622                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3623                         if (status)
3624                                 goto err;
3625                 }
3626
3627                 status = be_vfs_mac_query(adapter);
3628                 if (status)
3629                         goto err;
3630         } else {
3631                 status = be_vfs_if_create(adapter);
3632                 if (status)
3633                         goto err;
3634
3635                 status = be_vf_eth_addr_config(adapter);
3636                 if (status)
3637                         goto err;
3638         }
3639
3640         for_all_vfs(adapter, vf_cfg, vf) {
3641                 /* Allow VFs to programs MAC/VLAN filters */
3642                 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges,
3643                                                   vf + 1);
3644                 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
3645                         status = be_cmd_set_fn_privileges(adapter,
3646                                                           vf_cfg->privileges |
3647                                                           BE_PRIV_FILTMGMT,
3648                                                           vf + 1);
3649                         if (!status) {
3650                                 vf_cfg->privileges |= BE_PRIV_FILTMGMT;
3651                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3652                                          vf);
3653                         }
3654                 }
3655
3656                 /* Allow full available bandwidth */
3657                 if (!old_vfs)
3658                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3659
3660                 if (!old_vfs) {
3661                         be_cmd_enable_vf(adapter, vf + 1);
3662                         be_cmd_set_logical_link_config(adapter,
3663                                                        IFLA_VF_LINK_STATE_AUTO,
3664                                                        vf+1);
3665                 }
3666         }
3667
3668         if (!old_vfs) {
3669                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3670                 if (status) {
3671                         dev_err(dev, "SRIOV enable failed\n");
3672                         adapter->num_vfs = 0;
3673                         goto err;
3674                 }
3675         }
3676
3677         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3678         return 0;
3679 err:
3680         dev_err(dev, "VF setup failed\n");
3681         be_vf_clear(adapter);
3682         return status;
3683 }
3684
3685 /* Converting function_mode bits on BE3 to SH mc_type enums */
3686
3687 static u8 be_convert_mc_type(u32 function_mode)
3688 {
3689         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3690                 return vNIC1;
3691         else if (function_mode & QNQ_MODE)
3692                 return FLEX10;
3693         else if (function_mode & VNIC_MODE)
3694                 return vNIC2;
3695         else if (function_mode & UMC_ENABLED)
3696                 return UMC;
3697         else
3698                 return MC_NONE;
3699 }
3700
3701 /* On BE2/BE3 FW does not suggest the supported limits */
3702 static void BEx_get_resources(struct be_adapter *adapter,
3703                               struct be_resources *res)
3704 {
3705         bool use_sriov = adapter->num_vfs ? 1 : 0;
3706
3707         if (be_physfn(adapter))
3708                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3709         else
3710                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3711
3712         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3713
3714         if (be_is_mc(adapter)) {
3715                 /* Assuming that there are 4 channels per port,
3716                  * when multi-channel is enabled
3717                  */
3718                 if (be_is_qnq_mode(adapter))
3719                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3720                 else
3721                         /* In a non-qnq multichannel mode, the pvid
3722                          * takes up one vlan entry
3723                          */
3724                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3725         } else {
3726                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3727         }
3728
3729         res->max_mcast_mac = BE_MAX_MC;
3730
3731         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3732          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3733          *    *only* if it is RSS-capable.
3734          */
3735         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3736             !be_physfn(adapter) || (be_is_mc(adapter) &&
3737             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3738                 res->max_tx_qs = 1;
3739         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3740                 struct be_resources super_nic_res = {0};
3741
3742                 /* On a SuperNIC profile, the driver needs to use the
3743                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3744                  */
3745                 be_cmd_get_profile_config(adapter, &super_nic_res,
3746                                           RESOURCE_LIMITS, 0);
3747                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3748                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3749         } else {
3750                 res->max_tx_qs = BE3_MAX_TX_QS;
3751         }
3752
3753         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3754             !use_sriov && be_physfn(adapter))
3755                 res->max_rss_qs = (adapter->be3_native) ?
3756                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3757         res->max_rx_qs = res->max_rss_qs + 1;
3758
3759         if (be_physfn(adapter))
3760                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3761                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3762         else
3763                 res->max_evt_qs = 1;
3764
3765         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3766         res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS;
3767         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3768                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3769 }
3770
3771 static void be_setup_init(struct be_adapter *adapter)
3772 {
3773         adapter->vlan_prio_bmap = 0xff;
3774         adapter->phy.link_speed = -1;
3775         adapter->if_handle = -1;
3776         adapter->be3_native = false;
3777         adapter->if_flags = 0;
3778         if (be_physfn(adapter))
3779                 adapter->cmd_privileges = MAX_PRIVILEGES;
3780         else
3781                 adapter->cmd_privileges = MIN_PRIVILEGES;
3782 }
3783
3784 static int be_get_sriov_config(struct be_adapter *adapter)
3785 {
3786         struct be_resources res = {0};
3787         int max_vfs, old_vfs;
3788
3789         be_cmd_get_profile_config(adapter, &res, RESOURCE_LIMITS, 0);
3790
3791         /* Some old versions of BE3 FW don't report max_vfs value */
3792         if (BE3_chip(adapter) && !res.max_vfs) {
3793                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3794                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3795         }
3796
3797         adapter->pool_res = res;
3798
3799         /* If during previous unload of the driver, the VFs were not disabled,
3800          * then we cannot rely on the PF POOL limits for the TotalVFs value.
3801          * Instead use the TotalVFs value stored in the pci-dev struct.
3802          */
3803         old_vfs = pci_num_vf(adapter->pdev);
3804         if (old_vfs) {
3805                 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n",
3806                          old_vfs);
3807
3808                 adapter->pool_res.max_vfs =
3809                         pci_sriov_get_totalvfs(adapter->pdev);
3810                 adapter->num_vfs = old_vfs;
3811         }
3812
3813         return 0;
3814 }
3815
3816 static void be_alloc_sriov_res(struct be_adapter *adapter)
3817 {
3818         int old_vfs = pci_num_vf(adapter->pdev);
3819         u16 num_vf_qs;
3820         int status;
3821
3822         be_get_sriov_config(adapter);
3823
3824         if (!old_vfs)
3825                 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3826
3827         /* When the HW is in SRIOV capable configuration, the PF-pool
3828          * resources are given to PF during driver load, if there are no
3829          * old VFs. This facility is not available in BE3 FW.
3830          * Also, this is done by FW in Lancer chip.
3831          */
3832         if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
3833                 num_vf_qs = be_calculate_vf_qs(adapter, 0);
3834                 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0,
3835                                                  num_vf_qs);
3836                 if (status)
3837                         dev_err(&adapter->pdev->dev,
3838                                 "Failed to optimize SRIOV resources\n");
3839         }
3840 }
3841
3842 static int be_get_resources(struct be_adapter *adapter)
3843 {
3844         struct device *dev = &adapter->pdev->dev;
3845         struct be_resources res = {0};
3846         int status;
3847
3848         if (BEx_chip(adapter)) {
3849                 BEx_get_resources(adapter, &res);
3850                 adapter->res = res;
3851         }
3852
3853         /* For Lancer, SH etc read per-function resource limits from FW.
3854          * GET_FUNC_CONFIG returns per function guaranteed limits.
3855          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3856          */
3857         if (!BEx_chip(adapter)) {
3858                 status = be_cmd_get_func_config(adapter, &res);
3859                 if (status)
3860                         return status;
3861
3862                 /* If a deafault RXQ must be created, we'll use up one RSSQ*/
3863                 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs &&
3864                     !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS))
3865                         res.max_rss_qs -= 1;
3866
3867                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3868                 if (be_roce_supported(adapter))
3869                         res.max_evt_qs /= 2;
3870                 adapter->res = res;
3871         }
3872
3873         /* If FW supports RSS default queue, then skip creating non-RSS
3874          * queue for non-IP traffic.
3875          */
3876         adapter->need_def_rxq = (be_if_cap_flags(adapter) &
3877                                  BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1;
3878
3879         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3880                  be_max_txqs(adapter), be_max_rxqs(adapter),
3881                  be_max_rss(adapter), be_max_eqs(adapter),
3882                  be_max_vfs(adapter));
3883         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3884                  be_max_uc(adapter), be_max_mc(adapter),
3885                  be_max_vlans(adapter));
3886
3887         /* Sanitize cfg_num_qs based on HW and platform limits */
3888         adapter->cfg_num_qs = min_t(u16, netif_get_num_default_rss_queues(),
3889                                     be_max_qs(adapter));
3890         return 0;
3891 }
3892
3893 static int be_get_config(struct be_adapter *adapter)
3894 {
3895         int status, level;
3896         u16 profile_id;
3897
3898         status = be_cmd_get_cntl_attributes(adapter);
3899         if (status)
3900                 return status;
3901
3902         status = be_cmd_query_fw_cfg(adapter);
3903         if (status)
3904                 return status;
3905
3906         if (BEx_chip(adapter)) {
3907                 level = be_cmd_get_fw_log_level(adapter);
3908                 adapter->msg_enable =
3909                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
3910         }
3911
3912         be_cmd_get_acpi_wol_cap(adapter);
3913
3914         be_cmd_query_port_name(adapter);
3915
3916         if (be_physfn(adapter)) {
3917                 status = be_cmd_get_active_profile(adapter, &profile_id);
3918                 if (!status)
3919                         dev_info(&adapter->pdev->dev,
3920                                  "Using profile 0x%x\n", profile_id);
3921         }
3922
3923         status = be_get_resources(adapter);
3924         if (status)
3925                 return status;
3926
3927         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3928                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3929         if (!adapter->pmac_id)
3930                 return -ENOMEM;
3931
3932         return 0;
3933 }
3934
3935 static int be_mac_setup(struct be_adapter *adapter)
3936 {
3937         u8 mac[ETH_ALEN];
3938         int status;
3939
3940         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3941                 status = be_cmd_get_perm_mac(adapter, mac);
3942                 if (status)
3943                         return status;
3944
3945                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3946                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3947         } else {
3948                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3949                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3950         }
3951
3952         /* For BE3-R VFs, the PF programs the initial MAC address */
3953         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3954                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3955                                 &adapter->pmac_id[0], 0);
3956         return 0;
3957 }
3958
3959 static void be_schedule_worker(struct be_adapter *adapter)
3960 {
3961         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3962         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3963 }
3964
3965 static void be_schedule_err_detection(struct be_adapter *adapter)
3966 {
3967         schedule_delayed_work(&adapter->be_err_detection_work,
3968                               msecs_to_jiffies(1000));
3969         adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED;
3970 }
3971
3972 static int be_setup_queues(struct be_adapter *adapter)
3973 {
3974         struct net_device *netdev = adapter->netdev;
3975         int status;
3976
3977         status = be_evt_queues_create(adapter);
3978         if (status)
3979                 goto err;
3980
3981         status = be_tx_qs_create(adapter);
3982         if (status)
3983                 goto err;
3984
3985         status = be_rx_cqs_create(adapter);
3986         if (status)
3987                 goto err;
3988
3989         status = be_mcc_queues_create(adapter);
3990         if (status)
3991                 goto err;
3992
3993         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3994         if (status)
3995                 goto err;
3996
3997         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3998         if (status)
3999                 goto err;
4000
4001         return 0;
4002 err:
4003         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
4004         return status;
4005 }
4006
4007 int be_update_queues(struct be_adapter *adapter)
4008 {
4009         struct net_device *netdev = adapter->netdev;
4010         int status;
4011
4012         if (netif_running(netdev))
4013                 be_close(netdev);
4014
4015         be_cancel_worker(adapter);
4016
4017         /* If any vectors have been shared with RoCE we cannot re-program
4018          * the MSIx table.
4019          */
4020         if (!adapter->num_msix_roce_vec)
4021                 be_msix_disable(adapter);
4022
4023         be_clear_queues(adapter);
4024
4025         if (!msix_enabled(adapter)) {
4026                 status = be_msix_enable(adapter);
4027                 if (status)
4028                         return status;
4029         }
4030
4031         status = be_setup_queues(adapter);
4032         if (status)
4033                 return status;
4034
4035         be_schedule_worker(adapter);
4036
4037         if (netif_running(netdev))
4038                 status = be_open(netdev);
4039
4040         return status;
4041 }
4042
4043 static inline int fw_major_num(const char *fw_ver)
4044 {
4045         int fw_major = 0, i;
4046
4047         i = sscanf(fw_ver, "%d.", &fw_major);
4048         if (i != 1)
4049                 return 0;
4050
4051         return fw_major;
4052 }
4053
4054 /* If any VFs are already enabled don't FLR the PF */
4055 static bool be_reset_required(struct be_adapter *adapter)
4056 {
4057         return pci_num_vf(adapter->pdev) ? false : true;
4058 }
4059
4060 /* Wait for the FW to be ready and perform the required initialization */
4061 static int be_func_init(struct be_adapter *adapter)
4062 {
4063         int status;
4064
4065         status = be_fw_wait_ready(adapter);
4066         if (status)
4067                 return status;
4068
4069         if (be_reset_required(adapter)) {
4070                 status = be_cmd_reset_function(adapter);
4071                 if (status)
4072                         return status;
4073
4074                 /* Wait for interrupts to quiesce after an FLR */
4075                 msleep(100);
4076
4077                 /* We can clear all errors when function reset succeeds */
4078                 be_clear_all_error(adapter);
4079         }
4080
4081         /* Tell FW we're ready to fire cmds */
4082         status = be_cmd_fw_init(adapter);
4083         if (status)
4084                 return status;
4085
4086         /* Allow interrupts for other ULPs running on NIC function */
4087         be_intr_set(adapter, true);
4088
4089         return 0;
4090 }
4091
4092 static int be_setup(struct be_adapter *adapter)
4093 {
4094         struct device *dev = &adapter->pdev->dev;
4095         int status;
4096
4097         status = be_func_init(adapter);
4098         if (status)
4099                 return status;
4100
4101         be_setup_init(adapter);
4102
4103         if (!lancer_chip(adapter))
4104                 be_cmd_req_native_mode(adapter);
4105
4106         if (!BE2_chip(adapter) && be_physfn(adapter))
4107                 be_alloc_sriov_res(adapter);
4108
4109         status = be_get_config(adapter);
4110         if (status)
4111                 goto err;
4112
4113         status = be_msix_enable(adapter);
4114         if (status)
4115                 goto err;
4116
4117         status = be_if_create(adapter, &adapter->if_handle,
4118                               be_if_cap_flags(adapter), 0);
4119         if (status)
4120                 goto err;
4121
4122         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
4123         rtnl_lock();
4124         status = be_setup_queues(adapter);
4125         rtnl_unlock();
4126         if (status)
4127                 goto err;
4128
4129         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
4130
4131         status = be_mac_setup(adapter);
4132         if (status)
4133                 goto err;
4134
4135         be_cmd_get_fw_ver(adapter);
4136         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
4137
4138         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
4139                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
4140                         adapter->fw_ver);
4141                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
4142         }
4143
4144         if (adapter->vlans_added)
4145                 be_vid_config(adapter);
4146
4147         be_set_rx_mode(adapter->netdev);
4148
4149         status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
4150                                          adapter->rx_fc);
4151         if (status)
4152                 be_cmd_get_flow_control(adapter, &adapter->tx_fc,
4153                                         &adapter->rx_fc);
4154
4155         dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
4156                  adapter->tx_fc, adapter->rx_fc);
4157
4158         if (be_physfn(adapter))
4159                 be_cmd_set_logical_link_config(adapter,
4160                                                IFLA_VF_LINK_STATE_AUTO, 0);
4161
4162         if (adapter->num_vfs)
4163                 be_vf_setup(adapter);
4164
4165         status = be_cmd_get_phy_info(adapter);
4166         if (!status && be_pause_supported(adapter))
4167                 adapter->phy.fc_autoneg = 1;
4168
4169         be_schedule_worker(adapter);
4170         adapter->flags |= BE_FLAGS_SETUP_DONE;
4171         return 0;
4172 err:
4173         be_clear(adapter);
4174         return status;
4175 }
4176
4177 #ifdef CONFIG_NET_POLL_CONTROLLER
4178 static void be_netpoll(struct net_device *netdev)
4179 {
4180         struct be_adapter *adapter = netdev_priv(netdev);
4181         struct be_eq_obj *eqo;
4182         int i;
4183
4184         for_all_evt_queues(adapter, eqo, i) {
4185                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
4186                 napi_schedule(&eqo->napi);
4187         }
4188 }
4189 #endif
4190
4191 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
4192
4193 static bool phy_flashing_required(struct be_adapter *adapter)
4194 {
4195         return (adapter->phy.phy_type == PHY_TYPE_TN_8022 &&
4196                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
4197 }
4198
4199 static bool is_comp_in_ufi(struct be_adapter *adapter,
4200                            struct flash_section_info *fsec, int type)
4201 {
4202         int i = 0, img_type = 0;
4203         struct flash_section_info_g2 *fsec_g2 = NULL;
4204
4205         if (BE2_chip(adapter))
4206                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
4207
4208         for (i = 0; i < MAX_FLASH_COMP; i++) {
4209                 if (fsec_g2)
4210                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
4211                 else
4212                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
4213
4214                 if (img_type == type)
4215                         return true;
4216         }
4217         return false;
4218
4219 }
4220
4221 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
4222                                                 int header_size,
4223                                                 const struct firmware *fw)
4224 {
4225         struct flash_section_info *fsec = NULL;
4226         const u8 *p = fw->data;
4227
4228         p += header_size;
4229         while (p < (fw->data + fw->size)) {
4230                 fsec = (struct flash_section_info *)p;
4231                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
4232                         return fsec;
4233                 p += 32;
4234         }
4235         return NULL;
4236 }
4237
4238 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
4239                               u32 img_offset, u32 img_size, int hdr_size,
4240                               u16 img_optype, bool *crc_match)
4241 {
4242         u32 crc_offset;
4243         int status;
4244         u8 crc[4];
4245
4246         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset,
4247                                       img_size - 4);
4248         if (status)
4249                 return status;
4250
4251         crc_offset = hdr_size + img_offset + img_size - 4;
4252
4253         /* Skip flashing, if crc of flashed region matches */
4254         if (!memcmp(crc, p + crc_offset, 4))
4255                 *crc_match = true;
4256         else
4257                 *crc_match = false;
4258
4259         return status;
4260 }
4261
4262 static int be_flash(struct be_adapter *adapter, const u8 *img,
4263                     struct be_dma_mem *flash_cmd, int optype, int img_size,
4264                     u32 img_offset)
4265 {
4266         u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0;
4267         struct be_cmd_write_flashrom *req = flash_cmd->va;
4268         int status;
4269
4270         while (total_bytes) {
4271                 num_bytes = min_t(u32, 32*1024, total_bytes);
4272
4273                 total_bytes -= num_bytes;
4274
4275                 if (!total_bytes) {
4276                         if (optype == OPTYPE_PHY_FW)
4277                                 flash_op = FLASHROM_OPER_PHY_FLASH;
4278                         else
4279                                 flash_op = FLASHROM_OPER_FLASH;
4280                 } else {
4281                         if (optype == OPTYPE_PHY_FW)
4282                                 flash_op = FLASHROM_OPER_PHY_SAVE;
4283                         else
4284                                 flash_op = FLASHROM_OPER_SAVE;
4285                 }
4286
4287                 memcpy(req->data_buf, img, num_bytes);
4288                 img += num_bytes;
4289                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
4290                                                flash_op, img_offset +
4291                                                bytes_sent, num_bytes);
4292                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
4293                     optype == OPTYPE_PHY_FW)
4294                         break;
4295                 else if (status)
4296                         return status;
4297
4298                 bytes_sent += num_bytes;
4299         }
4300         return 0;
4301 }
4302
4303 /* For BE2, BE3 and BE3-R */
4304 static int be_flash_BEx(struct be_adapter *adapter,
4305                         const struct firmware *fw,
4306                         struct be_dma_mem *flash_cmd, int num_of_images)
4307 {
4308         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
4309         struct device *dev = &adapter->pdev->dev;
4310         struct flash_section_info *fsec = NULL;
4311         int status, i, filehdr_size, num_comp;
4312         const struct flash_comp *pflashcomp;
4313         bool crc_match;
4314         const u8 *p;
4315
4316         struct flash_comp gen3_flash_types[] = {
4317                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
4318                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
4319                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
4320                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
4321                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
4322                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
4323                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
4324                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
4325                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
4326                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
4327                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
4328                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
4329                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
4330                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
4331                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
4332                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
4333                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
4334                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
4335                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
4336                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
4337         };
4338
4339         struct flash_comp gen2_flash_types[] = {
4340                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
4341                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
4342                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
4343                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
4344                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
4345                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
4346                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
4347                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
4348                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
4349                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
4350                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
4351                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
4352                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
4353                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
4354                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
4355                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
4356         };
4357
4358         if (BE3_chip(adapter)) {
4359                 pflashcomp = gen3_flash_types;
4360                 filehdr_size = sizeof(struct flash_file_hdr_g3);
4361                 num_comp = ARRAY_SIZE(gen3_flash_types);
4362         } else {
4363                 pflashcomp = gen2_flash_types;
4364                 filehdr_size = sizeof(struct flash_file_hdr_g2);
4365                 num_comp = ARRAY_SIZE(gen2_flash_types);
4366                 img_hdrs_size = 0;
4367         }
4368
4369         /* Get flash section info*/
4370         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4371         if (!fsec) {
4372                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4373                 return -1;
4374         }
4375         for (i = 0; i < num_comp; i++) {
4376                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
4377                         continue;
4378
4379                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
4380                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
4381                         continue;
4382
4383                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
4384                     !phy_flashing_required(adapter))
4385                                 continue;
4386
4387                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
4388                         status = be_check_flash_crc(adapter, fw->data,
4389                                                     pflashcomp[i].offset,
4390                                                     pflashcomp[i].size,
4391                                                     filehdr_size +
4392                                                     img_hdrs_size,
4393                                                     OPTYPE_REDBOOT, &crc_match);
4394                         if (status) {
4395                                 dev_err(dev,
4396                                         "Could not get CRC for 0x%x region\n",
4397                                         pflashcomp[i].optype);
4398                                 continue;
4399                         }
4400
4401                         if (crc_match)
4402                                 continue;
4403                 }
4404
4405                 p = fw->data + filehdr_size + pflashcomp[i].offset +
4406                         img_hdrs_size;
4407                 if (p + pflashcomp[i].size > fw->data + fw->size)
4408                         return -1;
4409
4410                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
4411                                   pflashcomp[i].size, 0);
4412                 if (status) {
4413                         dev_err(dev, "Flashing section type 0x%x failed\n",
4414                                 pflashcomp[i].img_type);
4415                         return status;
4416                 }
4417         }
4418         return 0;
4419 }
4420
4421 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
4422 {
4423         u32 img_type = le32_to_cpu(fsec_entry.type);
4424         u16 img_optype = le16_to_cpu(fsec_entry.optype);
4425
4426         if (img_optype != 0xFFFF)
4427                 return img_optype;
4428
4429         switch (img_type) {
4430         case IMAGE_FIRMWARE_iSCSI:
4431                 img_optype = OPTYPE_ISCSI_ACTIVE;
4432                 break;
4433         case IMAGE_BOOT_CODE:
4434                 img_optype = OPTYPE_REDBOOT;
4435                 break;
4436         case IMAGE_OPTION_ROM_ISCSI:
4437                 img_optype = OPTYPE_BIOS;
4438                 break;
4439         case IMAGE_OPTION_ROM_PXE:
4440                 img_optype = OPTYPE_PXE_BIOS;
4441                 break;
4442         case IMAGE_OPTION_ROM_FCoE:
4443                 img_optype = OPTYPE_FCOE_BIOS;
4444                 break;
4445         case IMAGE_FIRMWARE_BACKUP_iSCSI:
4446                 img_optype = OPTYPE_ISCSI_BACKUP;
4447                 break;
4448         case IMAGE_NCSI:
4449                 img_optype = OPTYPE_NCSI_FW;
4450                 break;
4451         case IMAGE_FLASHISM_JUMPVECTOR:
4452                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
4453                 break;
4454         case IMAGE_FIRMWARE_PHY:
4455                 img_optype = OPTYPE_SH_PHY_FW;
4456                 break;
4457         case IMAGE_REDBOOT_DIR:
4458                 img_optype = OPTYPE_REDBOOT_DIR;
4459                 break;
4460         case IMAGE_REDBOOT_CONFIG:
4461                 img_optype = OPTYPE_REDBOOT_CONFIG;
4462                 break;
4463         case IMAGE_UFI_DIR:
4464                 img_optype = OPTYPE_UFI_DIR;
4465                 break;
4466         default:
4467                 break;
4468         }
4469
4470         return img_optype;
4471 }
4472
4473 static int be_flash_skyhawk(struct be_adapter *adapter,
4474                             const struct firmware *fw,
4475                             struct be_dma_mem *flash_cmd, int num_of_images)
4476 {
4477         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4478         bool crc_match, old_fw_img, flash_offset_support = true;
4479         struct device *dev = &adapter->pdev->dev;
4480         struct flash_section_info *fsec = NULL;
4481         u32 img_offset, img_size, img_type;
4482         u16 img_optype, flash_optype;
4483         int status, i, filehdr_size;
4484         const u8 *p;
4485
4486         filehdr_size = sizeof(struct flash_file_hdr_g3);
4487         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4488         if (!fsec) {
4489                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4490                 return -EINVAL;
4491         }
4492
4493 retry_flash:
4494         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4495                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4496                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4497                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4498                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4499                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4500
4501                 if (img_optype == 0xFFFF)
4502                         continue;
4503
4504                 if (flash_offset_support)
4505                         flash_optype = OPTYPE_OFFSET_SPECIFIED;
4506                 else
4507                         flash_optype = img_optype;
4508
4509                 /* Don't bother verifying CRC if an old FW image is being
4510                  * flashed
4511                  */
4512                 if (old_fw_img)
4513                         goto flash;
4514
4515                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4516                                             img_size, filehdr_size +
4517                                             img_hdrs_size, flash_optype,
4518                                             &crc_match);
4519                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4520                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4521                         /* The current FW image on the card does not support
4522                          * OFFSET based flashing. Retry using older mechanism
4523                          * of OPTYPE based flashing
4524                          */
4525                         if (flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4526                                 flash_offset_support = false;
4527                                 goto retry_flash;
4528                         }
4529
4530                         /* The current FW image on the card does not recognize
4531                          * the new FLASH op_type. The FW download is partially
4532                          * complete. Reboot the server now to enable FW image
4533                          * to recognize the new FLASH op_type. To complete the
4534                          * remaining process, download the same FW again after
4535                          * the reboot.
4536                          */
4537                         dev_err(dev, "Flash incomplete. Reset the server\n");
4538                         dev_err(dev, "Download FW image again after reset\n");
4539                         return -EAGAIN;
4540                 } else if (status) {
4541                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4542                                 img_optype);
4543                         return -EFAULT;
4544                 }
4545
4546                 if (crc_match)
4547                         continue;
4548
4549 flash:
4550                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4551                 if (p + img_size > fw->data + fw->size)
4552                         return -1;
4553
4554                 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size,
4555                                   img_offset);
4556
4557                 /* The current FW image on the card does not support OFFSET
4558                  * based flashing. Retry using older mechanism of OPTYPE based
4559                  * flashing
4560                  */
4561                 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD &&
4562                     flash_optype == OPTYPE_OFFSET_SPECIFIED) {
4563                         flash_offset_support = false;
4564                         goto retry_flash;
4565                 }
4566
4567                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4568                  * UFI_DIR region
4569                  */
4570                 if (old_fw_img &&
4571                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4572                      (img_optype == OPTYPE_UFI_DIR &&
4573                       base_status(status) == MCC_STATUS_FAILED))) {
4574                         continue;
4575                 } else if (status) {
4576                         dev_err(dev, "Flashing section type 0x%x failed\n",
4577                                 img_type);
4578                         return -EFAULT;
4579                 }
4580         }
4581         return 0;
4582 }
4583
4584 static int lancer_fw_download(struct be_adapter *adapter,
4585                               const struct firmware *fw)
4586 {
4587 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4588 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4589         struct device *dev = &adapter->pdev->dev;
4590         struct be_dma_mem flash_cmd;
4591         const u8 *data_ptr = NULL;
4592         u8 *dest_image_ptr = NULL;
4593         size_t image_size = 0;
4594         u32 chunk_size = 0;
4595         u32 data_written = 0;
4596         u32 offset = 0;
4597         int status = 0;
4598         u8 add_status = 0;
4599         u8 change_status;
4600
4601         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4602                 dev_err(dev, "FW image size should be multiple of 4\n");
4603                 return -EINVAL;
4604         }
4605
4606         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4607                                 + LANCER_FW_DOWNLOAD_CHUNK;
4608         flash_cmd.va = dma_zalloc_coherent(dev, flash_cmd.size,
4609                                            &flash_cmd.dma, GFP_KERNEL);
4610         if (!flash_cmd.va)
4611                 return -ENOMEM;
4612
4613         dest_image_ptr = flash_cmd.va +
4614                                 sizeof(struct lancer_cmd_req_write_object);
4615         image_size = fw->size;
4616         data_ptr = fw->data;
4617
4618         while (image_size) {
4619                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4620
4621                 /* Copy the image chunk content. */
4622                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4623
4624                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4625                                                  chunk_size, offset,
4626                                                  LANCER_FW_DOWNLOAD_LOCATION,
4627                                                  &data_written, &change_status,
4628                                                  &add_status);
4629                 if (status)
4630                         break;
4631
4632                 offset += data_written;
4633                 data_ptr += data_written;
4634                 image_size -= data_written;
4635         }
4636
4637         if (!status) {
4638                 /* Commit the FW written */
4639                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4640                                                  0, offset,
4641                                                  LANCER_FW_DOWNLOAD_LOCATION,
4642                                                  &data_written, &change_status,
4643                                                  &add_status);
4644         }
4645
4646         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4647         if (status) {
4648                 dev_err(dev, "Firmware load error\n");
4649                 return be_cmd_status(status);
4650         }
4651
4652         dev_info(dev, "Firmware flashed successfully\n");
4653
4654         if (change_status == LANCER_FW_RESET_NEEDED) {
4655                 dev_info(dev, "Resetting adapter to activate new FW\n");
4656                 status = lancer_physdev_ctrl(adapter,
4657                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4658                 if (status) {
4659                         dev_err(dev, "Adapter busy, could not reset FW\n");
4660                         dev_err(dev, "Reboot server to activate new FW\n");
4661                 }
4662         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4663                 dev_info(dev, "Reboot server to activate new FW\n");
4664         }
4665
4666         return 0;
4667 }
4668
4669 #define BE2_UFI         2
4670 #define BE3_UFI         3
4671 #define BE3R_UFI        10
4672 #define SH_UFI          4
4673 #define SH_P2_UFI       11
4674
4675 static int be_get_ufi_type(struct be_adapter *adapter,
4676                            struct flash_file_hdr_g3 *fhdr)
4677 {
4678         if (!fhdr) {
4679                 dev_err(&adapter->pdev->dev, "Invalid FW UFI file");
4680                 return -1;
4681         }
4682
4683         /* First letter of the build version is used to identify
4684          * which chip this image file is meant for.
4685          */
4686         switch (fhdr->build[0]) {
4687         case BLD_STR_UFI_TYPE_SH:
4688                 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI :
4689                                                                 SH_UFI;
4690         case BLD_STR_UFI_TYPE_BE3:
4691                 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI :
4692                                                                 BE3_UFI;
4693         case BLD_STR_UFI_TYPE_BE2:
4694                 return BE2_UFI;
4695         default:
4696                 return -1;
4697         }
4698 }
4699
4700 /* Check if the flash image file is compatible with the adapter that
4701  * is being flashed.
4702  * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type.
4703  * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type.
4704  */
4705 static bool be_check_ufi_compatibility(struct be_adapter *adapter,
4706                                        struct flash_file_hdr_g3 *fhdr)
4707 {
4708         int ufi_type = be_get_ufi_type(adapter, fhdr);
4709
4710         switch (ufi_type) {
4711         case SH_P2_UFI:
4712                 return skyhawk_chip(adapter);
4713         case SH_UFI:
4714                 return (skyhawk_chip(adapter) &&
4715                         adapter->asic_rev < ASIC_REV_P2);
4716         case BE3R_UFI:
4717                 return BE3_chip(adapter);
4718         case BE3_UFI:
4719                 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0);
4720         case BE2_UFI:
4721                 return BE2_chip(adapter);
4722         default:
4723                 return false;
4724         }
4725 }
4726
4727 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4728 {
4729         struct device *dev = &adapter->pdev->dev;
4730         struct flash_file_hdr_g3 *fhdr3;
4731         struct image_hdr *img_hdr_ptr;
4732         int status = 0, i, num_imgs;
4733         struct be_dma_mem flash_cmd;
4734
4735         fhdr3 = (struct flash_file_hdr_g3 *)fw->data;
4736         if (!be_check_ufi_compatibility(adapter, fhdr3)) {
4737                 dev_err(dev, "Flash image is not compatible with adapter\n");
4738                 return -EINVAL;
4739         }
4740
4741         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4742         flash_cmd.va = dma_zalloc_coherent(dev, flash_cmd.size, &flash_cmd.dma,
4743                                            GFP_KERNEL);
4744         if (!flash_cmd.va)
4745                 return -ENOMEM;
4746
4747         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4748         for (i = 0; i < num_imgs; i++) {
4749                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4750                                 (sizeof(struct flash_file_hdr_g3) +
4751                                  i * sizeof(struct image_hdr)));
4752                 if (!BE2_chip(adapter) &&
4753                     le32_to_cpu(img_hdr_ptr->imageid) != 1)
4754                         continue;
4755
4756                 if (skyhawk_chip(adapter))
4757                         status = be_flash_skyhawk(adapter, fw, &flash_cmd,
4758                                                   num_imgs);
4759                 else
4760                         status = be_flash_BEx(adapter, fw, &flash_cmd,
4761                                               num_imgs);
4762         }
4763
4764         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4765         if (!status)
4766                 dev_info(dev, "Firmware flashed successfully\n");
4767
4768         return status;
4769 }
4770
4771 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4772 {
4773         const struct firmware *fw;
4774         int status;
4775
4776         if (!netif_running(adapter->netdev)) {
4777                 dev_err(&adapter->pdev->dev,
4778                         "Firmware load not allowed (interface is down)\n");
4779                 return -ENETDOWN;
4780         }
4781
4782         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4783         if (status)
4784                 goto fw_exit;
4785
4786         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4787
4788         if (lancer_chip(adapter))
4789                 status = lancer_fw_download(adapter, fw);
4790         else
4791                 status = be_fw_download(adapter, fw);
4792
4793         if (!status)
4794                 be_cmd_get_fw_ver(adapter);
4795
4796 fw_exit:
4797         release_firmware(fw);
4798         return status;
4799 }
4800
4801 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4802                                  u16 flags)
4803 {
4804         struct be_adapter *adapter = netdev_priv(dev);
4805         struct nlattr *attr, *br_spec;
4806         int rem;
4807         int status = 0;
4808         u16 mode = 0;
4809
4810         if (!sriov_enabled(adapter))
4811                 return -EOPNOTSUPP;
4812
4813         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4814         if (!br_spec)
4815                 return -EINVAL;
4816
4817         nla_for_each_nested(attr, br_spec, rem) {
4818                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4819                         continue;
4820
4821                 if (nla_len(attr) < sizeof(mode))
4822                         return -EINVAL;
4823
4824                 mode = nla_get_u16(attr);
4825                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4826                         return -EINVAL;
4827
4828                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4829                                                adapter->if_handle,
4830                                                mode == BRIDGE_MODE_VEPA ?
4831                                                PORT_FWD_TYPE_VEPA :
4832                                                PORT_FWD_TYPE_VEB);
4833                 if (status)
4834                         goto err;
4835
4836                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4837                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4838
4839                 return status;
4840         }
4841 err:
4842         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4843                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4844
4845         return status;
4846 }
4847
4848 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4849                                  struct net_device *dev, u32 filter_mask,
4850                                  int nlflags)
4851 {
4852         struct be_adapter *adapter = netdev_priv(dev);
4853         int status = 0;
4854         u8 hsw_mode;
4855
4856         if (!sriov_enabled(adapter))
4857                 return 0;
4858
4859         /* BE and Lancer chips support VEB mode only */
4860         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4861                 hsw_mode = PORT_FWD_TYPE_VEB;
4862         } else {
4863                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4864                                                adapter->if_handle, &hsw_mode);
4865                 if (status)
4866                         return 0;
4867         }
4868
4869         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4870                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4871                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4872                                        0, 0, nlflags);
4873 }
4874
4875 #ifdef CONFIG_BE2NET_VXLAN
4876 /* VxLAN offload Notes:
4877  *
4878  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4879  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4880  * is expected to work across all types of IP tunnels once exported. Skyhawk
4881  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4882  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4883  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4884  * those other tunnels are unexported on the fly through ndo_features_check().
4885  *
4886  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4887  * adds more than one port, disable offloads and don't re-enable them again
4888  * until after all the tunnels are removed.
4889  */
4890 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4891                               __be16 port)
4892 {
4893         struct be_adapter *adapter = netdev_priv(netdev);
4894         struct device *dev = &adapter->pdev->dev;
4895         int status;
4896
4897         if (lancer_chip(adapter) || BEx_chip(adapter))
4898                 return;
4899
4900         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4901                 dev_info(dev,
4902                          "Only one UDP port supported for VxLAN offloads\n");
4903                 dev_info(dev, "Disabling VxLAN offloads\n");
4904                 adapter->vxlan_port_count++;
4905                 goto err;
4906         }
4907
4908         if (adapter->vxlan_port_count++ >= 1)
4909                 return;
4910
4911         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4912                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4913         if (status) {
4914                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4915                 goto err;
4916         }
4917
4918         status = be_cmd_set_vxlan_port(adapter, port);
4919         if (status) {
4920                 dev_warn(dev, "Failed to add VxLAN port\n");
4921                 goto err;
4922         }
4923         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4924         adapter->vxlan_port = port;
4925
4926         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4927                                    NETIF_F_TSO | NETIF_F_TSO6 |
4928                                    NETIF_F_GSO_UDP_TUNNEL;
4929         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4930         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4931
4932         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4933                  be16_to_cpu(port));
4934         return;
4935 err:
4936         be_disable_vxlan_offloads(adapter);
4937 }
4938
4939 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4940                               __be16 port)
4941 {
4942         struct be_adapter *adapter = netdev_priv(netdev);
4943
4944         if (lancer_chip(adapter) || BEx_chip(adapter))
4945                 return;
4946
4947         if (adapter->vxlan_port != port)
4948                 goto done;
4949
4950         be_disable_vxlan_offloads(adapter);
4951
4952         dev_info(&adapter->pdev->dev,
4953                  "Disabled VxLAN offloads for UDP port %d\n",
4954                  be16_to_cpu(port));
4955 done:
4956         adapter->vxlan_port_count--;
4957 }
4958
4959 static netdev_features_t be_features_check(struct sk_buff *skb,
4960                                            struct net_device *dev,
4961                                            netdev_features_t features)
4962 {
4963         struct be_adapter *adapter = netdev_priv(dev);
4964         u8 l4_hdr = 0;
4965
4966         /* The code below restricts offload features for some tunneled packets.
4967          * Offload features for normal (non tunnel) packets are unchanged.
4968          */
4969         if (!skb->encapsulation ||
4970             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4971                 return features;
4972
4973         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4974          * should disable tunnel offload features if it's not a VxLAN packet,
4975          * as tunnel offloads have been enabled only for VxLAN. This is done to
4976          * allow other tunneled traffic like GRE work fine while VxLAN
4977          * offloads are configured in Skyhawk-R.
4978          */
4979         switch (vlan_get_protocol(skb)) {
4980         case htons(ETH_P_IP):
4981                 l4_hdr = ip_hdr(skb)->protocol;
4982                 break;
4983         case htons(ETH_P_IPV6):
4984                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4985                 break;
4986         default:
4987                 return features;
4988         }
4989
4990         if (l4_hdr != IPPROTO_UDP ||
4991             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4992             skb->inner_protocol != htons(ETH_P_TEB) ||
4993             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4994             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4995                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4996
4997         return features;
4998 }
4999 #endif
5000
5001 static const struct net_device_ops be_netdev_ops = {
5002         .ndo_open               = be_open,
5003         .ndo_stop               = be_close,
5004         .ndo_start_xmit         = be_xmit,
5005         .ndo_set_rx_mode        = be_set_rx_mode,
5006         .ndo_set_mac_address    = be_mac_addr_set,
5007         .ndo_change_mtu         = be_change_mtu,
5008         .ndo_get_stats64        = be_get_stats64,
5009         .ndo_validate_addr      = eth_validate_addr,
5010         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
5011         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
5012         .ndo_set_vf_mac         = be_set_vf_mac,
5013         .ndo_set_vf_vlan        = be_set_vf_vlan,
5014         .ndo_set_vf_rate        = be_set_vf_tx_rate,
5015         .ndo_get_vf_config      = be_get_vf_config,
5016         .ndo_set_vf_link_state  = be_set_vf_link_state,
5017 #ifdef CONFIG_NET_POLL_CONTROLLER
5018         .ndo_poll_controller    = be_netpoll,
5019 #endif
5020         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
5021         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
5022 #ifdef CONFIG_NET_RX_BUSY_POLL
5023         .ndo_busy_poll          = be_busy_poll,
5024 #endif
5025 #ifdef CONFIG_BE2NET_VXLAN
5026         .ndo_add_vxlan_port     = be_add_vxlan_port,
5027         .ndo_del_vxlan_port     = be_del_vxlan_port,
5028         .ndo_features_check     = be_features_check,
5029 #endif
5030 };
5031
5032 static void be_netdev_init(struct net_device *netdev)
5033 {
5034         struct be_adapter *adapter = netdev_priv(netdev);
5035
5036         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5037                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
5038                 NETIF_F_HW_VLAN_CTAG_TX;
5039         if (be_multi_rxq(adapter))
5040                 netdev->hw_features |= NETIF_F_RXHASH;
5041
5042         netdev->features |= netdev->hw_features |
5043                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
5044
5045         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5046                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
5047
5048         netdev->priv_flags |= IFF_UNICAST_FLT;
5049
5050         netdev->flags |= IFF_MULTICAST;
5051
5052         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
5053
5054         netdev->netdev_ops = &be_netdev_ops;
5055
5056         netdev->ethtool_ops = &be_ethtool_ops;
5057 }
5058
5059 static void be_cleanup(struct be_adapter *adapter)
5060 {
5061         struct net_device *netdev = adapter->netdev;
5062
5063         rtnl_lock();
5064         netif_device_detach(netdev);
5065         if (netif_running(netdev))
5066                 be_close(netdev);
5067         rtnl_unlock();
5068
5069         be_clear(adapter);
5070 }
5071
5072 static int be_resume(struct be_adapter *adapter)
5073 {
5074         struct net_device *netdev = adapter->netdev;
5075         int status;
5076
5077         status = be_setup(adapter);
5078         if (status)
5079                 return status;
5080
5081         if (netif_running(netdev)) {
5082                 status = be_open(netdev);
5083                 if (status)
5084                         return status;
5085         }
5086
5087         netif_device_attach(netdev);
5088
5089         return 0;
5090 }
5091
5092 static int be_err_recover(struct be_adapter *adapter)
5093 {
5094         struct device *dev = &adapter->pdev->dev;
5095         int status;
5096
5097         status = be_resume(adapter);
5098         if (status)
5099                 goto err;
5100
5101         dev_info(dev, "Adapter recovery successful\n");
5102         return 0;
5103 err:
5104         if (be_physfn(adapter))
5105                 dev_err(dev, "Adapter recovery failed\n");
5106         else
5107                 dev_err(dev, "Re-trying adapter recovery\n");
5108
5109         return status;
5110 }
5111
5112 static void be_err_detection_task(struct work_struct *work)
5113 {
5114         struct be_adapter *adapter =
5115                                 container_of(work, struct be_adapter,
5116                                              be_err_detection_work.work);
5117         int status = 0;
5118
5119         be_detect_error(adapter);
5120
5121         if (adapter->hw_error) {
5122                 be_cleanup(adapter);
5123
5124                 /* As of now error recovery support is in Lancer only */
5125                 if (lancer_chip(adapter))
5126                         status = be_err_recover(adapter);
5127         }
5128
5129         /* Always attempt recovery on VFs */
5130         if (!status || be_virtfn(adapter))
5131                 be_schedule_err_detection(adapter);
5132 }
5133
5134 static void be_log_sfp_info(struct be_adapter *adapter)
5135 {
5136         int status;
5137
5138         status = be_cmd_query_sfp_info(adapter);
5139         if (!status) {
5140                 dev_err(&adapter->pdev->dev,
5141                         "Unqualified SFP+ detected on %c from %s part no: %s",
5142                         adapter->port_name, adapter->phy.vendor_name,
5143                         adapter->phy.vendor_pn);
5144         }
5145         adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP;
5146 }
5147
5148 static void be_worker(struct work_struct *work)
5149 {
5150         struct be_adapter *adapter =
5151                 container_of(work, struct be_adapter, work.work);
5152         struct be_rx_obj *rxo;
5153         int i;
5154
5155         /* when interrupts are not yet enabled, just reap any pending
5156          * mcc completions
5157          */
5158         if (!netif_running(adapter->netdev)) {
5159                 local_bh_disable();
5160                 be_process_mcc(adapter);
5161                 local_bh_enable();
5162                 goto reschedule;
5163         }
5164
5165         if (!adapter->stats_cmd_sent) {
5166                 if (lancer_chip(adapter))
5167                         lancer_cmd_get_pport_stats(adapter,
5168                                                    &adapter->stats_cmd);
5169                 else
5170                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
5171         }
5172
5173         if (be_physfn(adapter) &&
5174             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5175                 be_cmd_get_die_temperature(adapter);
5176
5177         for_all_rx_queues(adapter, rxo, i) {
5178                 /* Replenish RX-queues starved due to memory
5179                  * allocation failures.
5180                  */
5181                 if (rxo->rx_post_starved)
5182                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5183         }
5184
5185         be_eqd_update(adapter);
5186
5187         if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP)
5188                 be_log_sfp_info(adapter);
5189
5190 reschedule:
5191         adapter->work_counter++;
5192         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
5193 }
5194
5195 static void be_unmap_pci_bars(struct be_adapter *adapter)
5196 {
5197         if (adapter->csr)
5198                 pci_iounmap(adapter->pdev, adapter->csr);
5199         if (adapter->db)
5200                 pci_iounmap(adapter->pdev, adapter->db);
5201 }
5202
5203 static int db_bar(struct be_adapter *adapter)
5204 {
5205         if (lancer_chip(adapter) || !be_physfn(adapter))
5206                 return 0;
5207         else
5208                 return 4;
5209 }
5210
5211 static int be_roce_map_pci_bars(struct be_adapter *adapter)
5212 {
5213         if (skyhawk_chip(adapter)) {
5214                 adapter->roce_db.size = 4096;
5215                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
5216                                                               db_bar(adapter));
5217                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
5218                                                                db_bar(adapter));
5219         }
5220         return 0;
5221 }
5222
5223 static int be_map_pci_bars(struct be_adapter *adapter)
5224 {
5225         struct pci_dev *pdev = adapter->pdev;
5226         u8 __iomem *addr;
5227         u32 sli_intf;
5228
5229         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
5230         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
5231                                 SLI_INTF_FAMILY_SHIFT;
5232         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
5233
5234         if (BEx_chip(adapter) && be_physfn(adapter)) {
5235                 adapter->csr = pci_iomap(pdev, 2, 0);
5236                 if (!adapter->csr)
5237                         return -ENOMEM;
5238         }
5239
5240         addr = pci_iomap(pdev, db_bar(adapter), 0);
5241         if (!addr)
5242                 goto pci_map_err;
5243         adapter->db = addr;
5244
5245         if (skyhawk_chip(adapter) || BEx_chip(adapter)) {
5246                 if (be_physfn(adapter)) {
5247                         /* PCICFG is the 2nd BAR in BE2 */
5248                         addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0);
5249                         if (!addr)
5250                                 goto pci_map_err;
5251                         adapter->pcicfg = addr;
5252                 } else {
5253                         adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET;
5254                 }
5255         }
5256
5257         be_roce_map_pci_bars(adapter);
5258         return 0;
5259
5260 pci_map_err:
5261         dev_err(&pdev->dev, "Error in mapping PCI BARs\n");
5262         be_unmap_pci_bars(adapter);
5263         return -ENOMEM;
5264 }
5265
5266 static void be_drv_cleanup(struct be_adapter *adapter)
5267 {
5268         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
5269         struct device *dev = &adapter->pdev->dev;
5270
5271         if (mem->va)
5272                 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5273
5274         mem = &adapter->rx_filter;
5275         if (mem->va)
5276                 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5277
5278         mem = &adapter->stats_cmd;
5279         if (mem->va)
5280                 dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5281 }
5282
5283 /* Allocate and initialize various fields in be_adapter struct */
5284 static int be_drv_init(struct be_adapter *adapter)
5285 {
5286         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
5287         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
5288         struct be_dma_mem *rx_filter = &adapter->rx_filter;
5289         struct be_dma_mem *stats_cmd = &adapter->stats_cmd;
5290         struct device *dev = &adapter->pdev->dev;
5291         int status = 0;
5292
5293         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
5294         mbox_mem_alloc->va = dma_zalloc_coherent(dev, mbox_mem_alloc->size,
5295                                                  &mbox_mem_alloc->dma,
5296                                                  GFP_KERNEL);
5297         if (!mbox_mem_alloc->va)
5298                 return -ENOMEM;
5299
5300         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
5301         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
5302         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
5303
5304         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
5305         rx_filter->va = dma_zalloc_coherent(dev, rx_filter->size,
5306                                             &rx_filter->dma, GFP_KERNEL);
5307         if (!rx_filter->va) {
5308                 status = -ENOMEM;
5309                 goto free_mbox;
5310         }
5311
5312         if (lancer_chip(adapter))
5313                 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
5314         else if (BE2_chip(adapter))
5315                 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
5316         else if (BE3_chip(adapter))
5317                 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
5318         else
5319                 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
5320         stats_cmd->va = dma_zalloc_coherent(dev, stats_cmd->size,
5321                                             &stats_cmd->dma, GFP_KERNEL);
5322         if (!stats_cmd->va) {
5323                 status = -ENOMEM;
5324                 goto free_rx_filter;
5325         }
5326
5327         mutex_init(&adapter->mbox_lock);
5328         spin_lock_init(&adapter->mcc_lock);
5329         spin_lock_init(&adapter->mcc_cq_lock);
5330         init_completion(&adapter->et_cmd_compl);
5331
5332         pci_save_state(adapter->pdev);
5333
5334         INIT_DELAYED_WORK(&adapter->work, be_worker);
5335         INIT_DELAYED_WORK(&adapter->be_err_detection_work,
5336                           be_err_detection_task);
5337
5338         adapter->rx_fc = true;
5339         adapter->tx_fc = true;
5340
5341         /* Must be a power of 2 or else MODULO will BUG_ON */
5342         adapter->be_get_temp_freq = 64;
5343
5344         return 0;
5345
5346 free_rx_filter:
5347         dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma);
5348 free_mbox:
5349         dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va,
5350                           mbox_mem_alloc->dma);
5351         return status;
5352 }
5353
5354 static void be_remove(struct pci_dev *pdev)
5355 {
5356         struct be_adapter *adapter = pci_get_drvdata(pdev);
5357
5358         if (!adapter)
5359                 return;
5360
5361         be_roce_dev_remove(adapter);
5362         be_intr_set(adapter, false);
5363
5364         be_cancel_err_detection(adapter);
5365
5366         unregister_netdev(adapter->netdev);
5367
5368         be_clear(adapter);
5369
5370         /* tell fw we're done with firing cmds */
5371         be_cmd_fw_clean(adapter);
5372
5373         be_unmap_pci_bars(adapter);
5374         be_drv_cleanup(adapter);
5375
5376         pci_disable_pcie_error_reporting(pdev);
5377
5378         pci_release_regions(pdev);
5379         pci_disable_device(pdev);
5380
5381         free_netdev(adapter->netdev);
5382 }
5383
5384 static char *mc_name(struct be_adapter *adapter)
5385 {
5386         char *str = ""; /* default */
5387
5388         switch (adapter->mc_type) {
5389         case UMC:
5390                 str = "UMC";
5391                 break;
5392         case FLEX10:
5393                 str = "FLEX10";
5394                 break;
5395         case vNIC1:
5396                 str = "vNIC-1";
5397                 break;
5398         case nPAR:
5399                 str = "nPAR";
5400                 break;
5401         case UFP:
5402                 str = "UFP";
5403                 break;
5404         case vNIC2:
5405                 str = "vNIC-2";
5406                 break;
5407         default:
5408                 str = "";
5409         }
5410
5411         return str;
5412 }
5413
5414 static inline char *func_name(struct be_adapter *adapter)
5415 {
5416         return be_physfn(adapter) ? "PF" : "VF";
5417 }
5418
5419 static inline char *nic_name(struct pci_dev *pdev)
5420 {
5421         switch (pdev->device) {
5422         case OC_DEVICE_ID1:
5423                 return OC_NAME;
5424         case OC_DEVICE_ID2:
5425                 return OC_NAME_BE;
5426         case OC_DEVICE_ID3:
5427         case OC_DEVICE_ID4:
5428                 return OC_NAME_LANCER;
5429         case BE_DEVICE_ID2:
5430                 return BE3_NAME;
5431         case OC_DEVICE_ID5:
5432         case OC_DEVICE_ID6:
5433                 return OC_NAME_SH;
5434         default:
5435                 return BE_NAME;
5436         }
5437 }
5438
5439 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5440 {
5441         struct be_adapter *adapter;
5442         struct net_device *netdev;
5443         int status = 0;
5444
5445         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5446
5447         status = pci_enable_device(pdev);
5448         if (status)
5449                 goto do_none;
5450
5451         status = pci_request_regions(pdev, DRV_NAME);
5452         if (status)
5453                 goto disable_dev;
5454         pci_set_master(pdev);
5455
5456         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5457         if (!netdev) {
5458                 status = -ENOMEM;
5459                 goto rel_reg;
5460         }
5461         adapter = netdev_priv(netdev);
5462         adapter->pdev = pdev;
5463         pci_set_drvdata(pdev, adapter);
5464         adapter->netdev = netdev;
5465         SET_NETDEV_DEV(netdev, &pdev->dev);
5466
5467         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5468         if (!status) {
5469                 netdev->features |= NETIF_F_HIGHDMA;
5470         } else {
5471                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5472                 if (status) {
5473                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5474                         goto free_netdev;
5475                 }
5476         }
5477
5478         status = pci_enable_pcie_error_reporting(pdev);
5479         if (!status)
5480                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5481
5482         status = be_map_pci_bars(adapter);
5483         if (status)
5484                 goto free_netdev;
5485
5486         status = be_drv_init(adapter);
5487         if (status)
5488                 goto unmap_bars;
5489
5490         status = be_setup(adapter);
5491         if (status)
5492                 goto drv_cleanup;
5493
5494         be_netdev_init(netdev);
5495         status = register_netdev(netdev);
5496         if (status != 0)
5497                 goto unsetup;
5498
5499         be_roce_dev_add(adapter);
5500
5501         be_schedule_err_detection(adapter);
5502
5503         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5504                  func_name(adapter), mc_name(adapter), adapter->port_name);
5505
5506         return 0;
5507
5508 unsetup:
5509         be_clear(adapter);
5510 drv_cleanup:
5511         be_drv_cleanup(adapter);
5512 unmap_bars:
5513         be_unmap_pci_bars(adapter);
5514 free_netdev:
5515         free_netdev(netdev);
5516 rel_reg:
5517         pci_release_regions(pdev);
5518 disable_dev:
5519         pci_disable_device(pdev);
5520 do_none:
5521         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5522         return status;
5523 }
5524
5525 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5526 {
5527         struct be_adapter *adapter = pci_get_drvdata(pdev);
5528
5529         if (adapter->wol_en)
5530                 be_setup_wol(adapter, true);
5531
5532         be_intr_set(adapter, false);
5533         be_cancel_err_detection(adapter);
5534
5535         be_cleanup(adapter);
5536
5537         pci_save_state(pdev);
5538         pci_disable_device(pdev);
5539         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5540         return 0;
5541 }
5542
5543 static int be_pci_resume(struct pci_dev *pdev)
5544 {
5545         struct be_adapter *adapter = pci_get_drvdata(pdev);
5546         int status = 0;
5547
5548         status = pci_enable_device(pdev);
5549         if (status)
5550                 return status;
5551
5552         pci_set_power_state(pdev, PCI_D0);
5553         pci_restore_state(pdev);
5554
5555         status = be_resume(adapter);
5556         if (status)
5557                 return status;
5558
5559         be_schedule_err_detection(adapter);
5560
5561         if (adapter->wol_en)
5562                 be_setup_wol(adapter, false);
5563
5564         return 0;
5565 }
5566
5567 /*
5568  * An FLR will stop BE from DMAing any data.
5569  */
5570 static void be_shutdown(struct pci_dev *pdev)
5571 {
5572         struct be_adapter *adapter = pci_get_drvdata(pdev);
5573
5574         if (!adapter)
5575                 return;
5576
5577         be_roce_dev_shutdown(adapter);
5578         cancel_delayed_work_sync(&adapter->work);
5579         be_cancel_err_detection(adapter);
5580
5581         netif_device_detach(adapter->netdev);
5582
5583         be_cmd_reset_function(adapter);
5584
5585         pci_disable_device(pdev);
5586 }
5587
5588 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5589                                             pci_channel_state_t state)
5590 {
5591         struct be_adapter *adapter = pci_get_drvdata(pdev);
5592
5593         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5594
5595         if (!adapter->eeh_error) {
5596                 adapter->eeh_error = true;
5597
5598                 be_cancel_err_detection(adapter);
5599
5600                 be_cleanup(adapter);
5601         }
5602
5603         if (state == pci_channel_io_perm_failure)
5604                 return PCI_ERS_RESULT_DISCONNECT;
5605
5606         pci_disable_device(pdev);
5607
5608         /* The error could cause the FW to trigger a flash debug dump.
5609          * Resetting the card while flash dump is in progress
5610          * can cause it not to recover; wait for it to finish.
5611          * Wait only for first function as it is needed only once per
5612          * adapter.
5613          */
5614         if (pdev->devfn == 0)
5615                 ssleep(30);
5616
5617         return PCI_ERS_RESULT_NEED_RESET;
5618 }
5619
5620 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5621 {
5622         struct be_adapter *adapter = pci_get_drvdata(pdev);
5623         int status;
5624
5625         dev_info(&adapter->pdev->dev, "EEH reset\n");
5626
5627         status = pci_enable_device(pdev);
5628         if (status)
5629                 return PCI_ERS_RESULT_DISCONNECT;
5630
5631         pci_set_master(pdev);
5632         pci_set_power_state(pdev, PCI_D0);
5633         pci_restore_state(pdev);
5634
5635         /* Check if card is ok and fw is ready */
5636         dev_info(&adapter->pdev->dev,
5637                  "Waiting for FW to be ready after EEH reset\n");
5638         status = be_fw_wait_ready(adapter);
5639         if (status)
5640                 return PCI_ERS_RESULT_DISCONNECT;
5641
5642         pci_cleanup_aer_uncorrect_error_status(pdev);
5643         be_clear_all_error(adapter);
5644         return PCI_ERS_RESULT_RECOVERED;
5645 }
5646
5647 static void be_eeh_resume(struct pci_dev *pdev)
5648 {
5649         int status = 0;
5650         struct be_adapter *adapter = pci_get_drvdata(pdev);
5651
5652         dev_info(&adapter->pdev->dev, "EEH resume\n");
5653
5654         pci_save_state(pdev);
5655
5656         status = be_resume(adapter);
5657         if (status)
5658                 goto err;
5659
5660         be_schedule_err_detection(adapter);
5661         return;
5662 err:
5663         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5664 }
5665
5666 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
5667 {
5668         struct be_adapter *adapter = pci_get_drvdata(pdev);
5669         u16 num_vf_qs;
5670         int status;
5671
5672         if (!num_vfs)
5673                 be_vf_clear(adapter);
5674
5675         adapter->num_vfs = num_vfs;
5676
5677         if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) {
5678                 dev_warn(&pdev->dev,
5679                          "Cannot disable VFs while they are assigned\n");
5680                 return -EBUSY;
5681         }
5682
5683         /* When the HW is in SRIOV capable configuration, the PF-pool resources
5684          * are equally distributed across the max-number of VFs. The user may
5685          * request only a subset of the max-vfs to be enabled.
5686          * Based on num_vfs, redistribute the resources across num_vfs so that
5687          * each VF will have access to more number of resources.
5688          * This facility is not available in BE3 FW.
5689          * Also, this is done by FW in Lancer chip.
5690          */
5691         if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) {
5692                 num_vf_qs = be_calculate_vf_qs(adapter, adapter->num_vfs);
5693                 status = be_cmd_set_sriov_config(adapter, adapter->pool_res,
5694                                                  adapter->num_vfs, num_vf_qs);
5695                 if (status)
5696                         dev_err(&pdev->dev,
5697                                 "Failed to optimize SR-IOV resources\n");
5698         }
5699
5700         status = be_get_resources(adapter);
5701         if (status)
5702                 return be_cmd_status(status);
5703
5704         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
5705         rtnl_lock();
5706         status = be_update_queues(adapter);
5707         rtnl_unlock();
5708         if (status)
5709                 return be_cmd_status(status);
5710
5711         if (adapter->num_vfs)
5712                 status = be_vf_setup(adapter);
5713
5714         if (!status)
5715                 return adapter->num_vfs;
5716
5717         return 0;
5718 }
5719
5720 static const struct pci_error_handlers be_eeh_handlers = {
5721         .error_detected = be_eeh_err_detected,
5722         .slot_reset = be_eeh_reset,
5723         .resume = be_eeh_resume,
5724 };
5725
5726 static struct pci_driver be_driver = {
5727         .name = DRV_NAME,
5728         .id_table = be_dev_ids,
5729         .probe = be_probe,
5730         .remove = be_remove,
5731         .suspend = be_suspend,
5732         .resume = be_pci_resume,
5733         .shutdown = be_shutdown,
5734         .sriov_configure = be_pci_sriov_configure,
5735         .err_handler = &be_eeh_handlers
5736 };
5737
5738 static int __init be_init_module(void)
5739 {
5740         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5741             rx_frag_size != 2048) {
5742                 printk(KERN_WARNING DRV_NAME
5743                         " : Module param rx_frag_size must be 2048/4096/8192."
5744                         " Using 2048\n");
5745                 rx_frag_size = 2048;
5746         }
5747
5748         if (num_vfs > 0) {
5749                 pr_info(DRV_NAME " : Module param num_vfs is obsolete.");
5750                 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n");
5751         }
5752
5753         return pci_register_driver(&be_driver);
5754 }
5755 module_init(be_init_module);
5756
5757 static void __exit be_exit_module(void)
5758 {
5759         pci_unregister_driver(&be_driver);
5760 }
5761 module_exit(be_exit_module);