kernel/drivers/ntb/ntb_hw.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8  *
   9  *   This program is free software; you can redistribute it and/or modify
  10  *   it under the terms of version 2 of the GNU General Public License as
  11  *   published by the Free Software Foundation.
  12  *
  13  *   BSD LICENSE
  14  *
  15  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  16  *
  17  *   Redistribution and use in source and binary forms, with or without
  18  *   modification, are permitted provided that the following conditions
  19  *   are met:
  20  *
  21  *     * Redistributions of source code must retain the above copyright
  22  *       notice, this list of conditions and the following disclaimer.
  23  *     * Redistributions in binary form must reproduce the above copy
  24  *       notice, this list of conditions and the following disclaimer in
  25  *       the documentation and/or other materials provided with the
  26  *       distribution.
  27  *     * Neither the name of Intel Corporation nor the names of its
  28  *       contributors may be used to endorse or promote products derived
  29  *       from this software without specific prior written permission.
  30  *
  31  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  32  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  33  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  34  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  35  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  36  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  37  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  38  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  39  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  41  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42  *
  43  * Intel PCIe NTB Linux driver
  44  *
  45  * Contact Information:
  46  * Jon Mason <jon.mason@intel.com>
  47  */
  48 #include <linux/debugfs.h>
  49 #include <linux/delay.h>
  50 #include <linux/init.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/module.h>
  53 #include <linux/pci.h>
  54 #include <linux/random.h>
  55 #include <linux/slab.h>
  56 #include "ntb_hw.h"
  57 #include "ntb_regs.h"
  58
  59 #define NTB_NAME        "Intel(R) PCI-E Non-Transparent Bridge Driver"
  60 #define NTB_VER         "1.0"
  61
  62 MODULE_DESCRIPTION(NTB_NAME);
  63 MODULE_VERSION(NTB_VER);
  64 MODULE_LICENSE("Dual BSD/GPL");
  65 MODULE_AUTHOR("Intel Corporation");
  66
  67 enum {
  68         NTB_CONN_TRANSPARENT = 0,
  69         NTB_CONN_B2B,
  70         NTB_CONN_RP,
  71 };
  72
  73 enum {
  74         NTB_DEV_USD = 0,
  75         NTB_DEV_DSD,
  76 };
  77
  78 enum {
  79         SNB_HW = 0,
  80         BWD_HW,
  81 };
  82
  83 static struct dentry *debugfs_dir;
  84
  85 #define BWD_LINK_RECOVERY_TIME  500
  86
  87 /* Translate memory window 0,1,2 to BAR 2,4,5 */
  88 #define MW_TO_BAR(mw)   (mw == 0 ? 2 : (mw == 1 ? 4 : 5))
  89
  90 static const struct pci_device_id ntb_pci_tbl[] = {
  91         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
  92         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
  93         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
  94         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
  95         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
  96         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
  97         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
  98         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
  99         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
 100         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
 101         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
 102         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
 103         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
 104         {0}
 105 };
 106 MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
 107
 108 static int is_ntb_xeon(struct ntb_device *ndev)
 109 {
 110         switch (ndev->pdev->device) {
 111         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
 112         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
 113         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
 114         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
 115         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
 116         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
 117         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
 118         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
 119         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
 120         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
 121         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
 122         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
 123                 return 1;
 124         default:
 125                 return 0;
 126         }
 127
 128         return 0;
 129 }
 130
 131 static int is_ntb_atom(struct ntb_device *ndev)
 132 {
 133         switch (ndev->pdev->device) {
 134         case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
 135                 return 1;
 136         default:
 137                 return 0;
 138         }
 139
 140         return 0;
 141 }
 142
 143 static void ntb_set_errata_flags(struct ntb_device *ndev)
 144 {
 145         switch (ndev->pdev->device) {
 146         /*
 147          * this workaround applies to all platform up to IvyBridge
 148          * Haswell has splitbar support and use a different workaround
 149          */
 150         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
 151         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
 152         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
 153         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
 154         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
 155         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
 156         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
 157         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
 158         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
 159         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
 160         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
 161         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
 162                 ndev->wa_flags |= WA_SNB_ERR;
 163                 break;
 164         }
 165 }
 166
 167 /**
 168  * ntb_register_event_callback() - register event callback
 169  * @ndev: pointer to ntb_device instance
 170  * @func: callback function to register
 171  *
 172  * This function registers a callback for any HW driver events such as link
 173  * up/down, power management notices and etc.
 174  *
 175  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 176  */
 177 int ntb_register_event_callback(struct ntb_device *ndev,
 178                                 void (*func)(void *handle,
 179                                              enum ntb_hw_event event))
 180 {
 181         if (ndev->event_cb)
 182                 return -EINVAL;
 183
 184         ndev->event_cb = func;
 185
 186         return 0;
 187 }
 188
 189 /**
 190  * ntb_unregister_event_callback() - unregisters the event callback
 191  * @ndev: pointer to ntb_device instance
 192  *
 193  * This function unregisters the existing callback from transport
 194  */
 195 void ntb_unregister_event_callback(struct ntb_device *ndev)
 196 {
 197         ndev->event_cb = NULL;
 198 }
 199
 200 static void ntb_irq_work(unsigned long data)
 201 {
 202         struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data;
 203         int rc;
 204
 205         rc = db_cb->callback(db_cb->data, db_cb->db_num);
 206         if (rc)
 207                 tasklet_schedule(&db_cb->irq_work);
 208         else {
 209                 struct ntb_device *ndev = db_cb->ndev;
 210                 unsigned long mask;
 211
 212                 mask = readw(ndev->reg_ofs.ldb_mask);
 213                 clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
 214                 writew(mask, ndev->reg_ofs.ldb_mask);
 215         }
 216 }
 217
 218 /**
 219  * ntb_register_db_callback() - register a callback for doorbell interrupt
 220  * @ndev: pointer to ntb_device instance
 221  * @idx: doorbell index to register callback, zero based
 222  * @data: pointer to be returned to caller with every callback
 223  * @func: callback function to register
 224  *
 225  * This function registers a callback function for the doorbell interrupt
 226  * on the primary side. The function will unmask the doorbell as well to
 227  * allow interrupt.
 228  *
 229  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 230  */
 231 int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
 232                              void *data, int (*func)(void *data, int db_num))
 233 {
 234         unsigned long mask;
 235
 236         if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
 237                 dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
 238                 return -EINVAL;
 239         }
 240
 241         ndev->db_cb[idx].callback = func;
 242         ndev->db_cb[idx].data = data;
 243         ndev->db_cb[idx].ndev = ndev;
 244
 245         tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work,
 246                      (unsigned long) &ndev->db_cb[idx]);
 247
 248         /* unmask interrupt */
 249         mask = readw(ndev->reg_ofs.ldb_mask);
 250         clear_bit(idx * ndev->bits_per_vector, &mask);
 251         writew(mask, ndev->reg_ofs.ldb_mask);
 252
 253         return 0;
 254 }
 255
 256 /**
 257  * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
 258  * @ndev: pointer to ntb_device instance
 259  * @idx: doorbell index to register callback, zero based
 260  *
 261  * This function unregisters a callback function for the doorbell interrupt
 262  * on the primary side. The function will also mask the said doorbell.
 263  */
 264 void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
 265 {
 266         unsigned long mask;
 267
 268         if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
 269                 return;
 270
 271         mask = readw(ndev->reg_ofs.ldb_mask);
 272         set_bit(idx * ndev->bits_per_vector, &mask);
 273         writew(mask, ndev->reg_ofs.ldb_mask);
 274
 275         tasklet_disable(&ndev->db_cb[idx].irq_work);
 276
 277         ndev->db_cb[idx].callback = NULL;
 278 }
 279
 280 /**
 281  * ntb_find_transport() - find the transport pointer
 282  * @transport: pointer to pci device
 283  *
 284  * Given the pci device pointer, return the transport pointer passed in when
 285  * the transport attached when it was inited.
 286  *
 287  * RETURNS: pointer to transport.
 288  */
 289 void *ntb_find_transport(struct pci_dev *pdev)
 290 {
 291         struct ntb_device *ndev = pci_get_drvdata(pdev);
 292         return ndev->ntb_transport;
 293 }
 294
 295 /**
 296  * ntb_register_transport() - Register NTB transport with NTB HW driver
 297  * @transport: transport identifier
 298  *
 299  * This function allows a transport to reserve the hardware driver for
 300  * NTB usage.
 301  *
 302  * RETURNS: pointer to ntb_device, NULL on error.
 303  */
 304 struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
 305 {
 306         struct ntb_device *ndev = pci_get_drvdata(pdev);
 307
 308         if (ndev->ntb_transport)
 309                 return NULL;
 310
 311         ndev->ntb_transport = transport;
 312         return ndev;
 313 }
 314
 315 /**
 316  * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
 317  * @ndev - ntb_device of the transport to be freed
 318  *
 319  * This function unregisters the transport from the HW driver and performs any
 320  * necessary cleanups.
 321  */
 322 void ntb_unregister_transport(struct ntb_device *ndev)
 323 {
 324         int i;
 325
 326         if (!ndev->ntb_transport)
 327                 return;
 328
 329         for (i = 0; i < ndev->max_cbs; i++)
 330                 ntb_unregister_db_callback(ndev, i);
 331
 332         ntb_unregister_event_callback(ndev);
 333         ndev->ntb_transport = NULL;
 334 }
 335
 336 /**
 337  * ntb_write_local_spad() - write to the secondary scratchpad register
 338  * @ndev: pointer to ntb_device instance
 339  * @idx: index to the scratchpad register, 0 based
 340  * @val: the data value to put into the register
 341  *
 342  * This function allows writing of a 32bit value to the indexed scratchpad
 343  * register. This writes over the data mirrored to the local scratchpad register
 344  * by the remote system.
 345  *
 346  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 347  */
 348 int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
 349 {
 350         if (idx >= ndev->limits.max_spads)
 351                 return -EINVAL;
 352
 353         dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
 354                 val, idx);
 355         writel(val, ndev->reg_ofs.spad_read + idx * 4);
 356
 357         return 0;
 358 }
 359
 360 /**
 361  * ntb_read_local_spad() - read from the primary scratchpad register
 362  * @ndev: pointer to ntb_device instance
 363  * @idx: index to scratchpad register, 0 based
 364  * @val: pointer to 32bit integer for storing the register value
 365  *
 366  * This function allows reading of the 32bit scratchpad register on
 367  * the primary (internal) side.  This allows the local system to read data
 368  * written and mirrored to the scratchpad register by the remote system.
 369  *
 370  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 371  */
 372 int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
 373 {
 374         if (idx >= ndev->limits.max_spads)
 375                 return -EINVAL;
 376
 377         *val = readl(ndev->reg_ofs.spad_write + idx * 4);
 378         dev_dbg(&ndev->pdev->dev,
 379                 "Reading %x from local scratch pad index %d\n", *val, idx);
 380
 381         return 0;
 382 }
 383
 384 /**
 385  * ntb_write_remote_spad() - write to the secondary scratchpad register
 386  * @ndev: pointer to ntb_device instance
 387  * @idx: index to the scratchpad register, 0 based
 388  * @val: the data value to put into the register
 389  *
 390  * This function allows writing of a 32bit value to the indexed scratchpad
 391  * register. The register resides on the secondary (external) side.  This allows
 392  * the local system to write data to be mirrored to the remote systems
 393  * scratchpad register.
 394  *
 395  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 396  */
 397 int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
 398 {
 399         if (idx >= ndev->limits.max_spads)
 400                 return -EINVAL;
 401
 402         dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
 403                 val, idx);
 404         writel(val, ndev->reg_ofs.spad_write + idx * 4);
 405
 406         return 0;
 407 }
 408
 409 /**
 410  * ntb_read_remote_spad() - read from the primary scratchpad register
 411  * @ndev: pointer to ntb_device instance
 412  * @idx: index to scratchpad register, 0 based
 413  * @val: pointer to 32bit integer for storing the register value
 414  *
 415  * This function allows reading of the 32bit scratchpad register on
 416  * the primary (internal) side.  This alloows the local system to read the data
 417  * it wrote to be mirrored on the remote system.
 418  *
 419  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 420  */
 421 int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
 422 {
 423         if (idx >= ndev->limits.max_spads)
 424                 return -EINVAL;
 425
 426         *val = readl(ndev->reg_ofs.spad_read + idx * 4);
 427         dev_dbg(&ndev->pdev->dev,
 428                 "Reading %x from remote scratch pad index %d\n", *val, idx);
 429
 430         return 0;
 431 }
 432
 433 /**
 434  * ntb_get_mw_base() - get addr for the NTB memory window
 435  * @ndev: pointer to ntb_device instance
 436  * @mw: memory window number
 437  *
 438  * This function provides the base address of the memory window specified.
 439  *
 440  * RETURNS: address, or NULL on error.
 441  */
 442 resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
 443 {
 444         if (mw >= ntb_max_mw(ndev))
 445                 return 0;
 446
 447         return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
 448 }
 449
 450 /**
 451  * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
 452  * @ndev: pointer to ntb_device instance
 453  * @mw: memory window number
 454  *
 455  * This function provides the base virtual address of the memory window
 456  * specified.
 457  *
 458  * RETURNS: pointer to virtual address, or NULL on error.
 459  */
 460 void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
 461 {
 462         if (mw >= ntb_max_mw(ndev))
 463                 return NULL;
 464
 465         return ndev->mw[mw].vbase;
 466 }
 467
 468 /**
 469  * ntb_get_mw_size() - return size of NTB memory window
 470  * @ndev: pointer to ntb_device instance
 471  * @mw: memory window number
 472  *
 473  * This function provides the physical size of the memory window specified
 474  *
 475  * RETURNS: the size of the memory window or zero on error
 476  */
 477 u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
 478 {
 479         if (mw >= ntb_max_mw(ndev))
 480                 return 0;
 481
 482         return ndev->mw[mw].bar_sz;
 483 }
 484
 485 /**
 486  * ntb_set_mw_addr - set the memory window address
 487  * @ndev: pointer to ntb_device instance
 488  * @mw: memory window number
 489  * @addr: base address for data
 490  *
 491  * This function sets the base physical address of the memory window.  This
 492  * memory address is where data from the remote system will be transfered into
 493  * or out of depending on how the transport is configured.
 494  */
 495 void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
 496 {
 497         if (mw >= ntb_max_mw(ndev))
 498                 return;
 499
 500         dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
 501                 MW_TO_BAR(mw));
 502
 503         ndev->mw[mw].phys_addr = addr;
 504
 505         switch (MW_TO_BAR(mw)) {
 506         case NTB_BAR_23:
 507                 writeq(addr, ndev->reg_ofs.bar2_xlat);
 508                 break;
 509         case NTB_BAR_4:
 510                 if (ndev->split_bar)
 511                         writel(addr, ndev->reg_ofs.bar4_xlat);
 512                 else
 513                         writeq(addr, ndev->reg_ofs.bar4_xlat);
 514                 break;
 515         case NTB_BAR_5:
 516                 writel(addr, ndev->reg_ofs.bar5_xlat);
 517                 break;
 518         }
 519 }
 520
 521 /**
 522  * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
 523  * @ndev: pointer to ntb_device instance
 524  * @db: doorbell to ring
 525  *
 526  * This function allows triggering of a doorbell on the secondary/external
 527  * side that will initiate an interrupt on the remote host
 528  *
 529  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 530  */
 531 void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
 532 {
 533         dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
 534
 535         if (ndev->hw_type == BWD_HW)
 536                 writeq((u64) 1 << db, ndev->reg_ofs.rdb);
 537         else
 538                 writew(((1 << ndev->bits_per_vector) - 1) <<
 539                        (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
 540 }
 541
 542 static void bwd_recover_link(struct ntb_device *ndev)
 543 {
 544         u32 status;
 545
 546         /* Driver resets the NTB ModPhy lanes - magic! */
 547         writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
 548         writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
 549         writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
 550         writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
 551
 552         /* Driver waits 100ms to allow the NTB ModPhy to settle */
 553         msleep(100);
 554
 555         /* Clear AER Errors, write to clear */
 556         status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
 557         dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
 558         status &= PCI_ERR_COR_REP_ROLL;
 559         writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
 560
 561         /* Clear unexpected electrical idle event in LTSSM, write to clear */
 562         status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
 563         dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
 564         status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
 565         writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
 566
 567         /* Clear DeSkew Buffer error, write to clear */
 568         status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
 569         dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
 570         status |= BWD_DESKEWSTS_DBERR;
 571         writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
 572
 573         status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 574         dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
 575         status &= BWD_IBIST_ERR_OFLOW;
 576         writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 577
 578         /* Releases the NTB state machine to allow the link to retrain */
 579         status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 580         dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
 581         status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
 582         writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 583 }
 584
 585 static void ntb_link_event(struct ntb_device *ndev, int link_state)
 586 {
 587         unsigned int event;
 588
 589         if (ndev->link_status == link_state)
 590                 return;
 591
 592         if (link_state == NTB_LINK_UP) {
 593                 u16 status;
 594
 595                 dev_info(&ndev->pdev->dev, "Link Up\n");
 596                 ndev->link_status = NTB_LINK_UP;
 597                 event = NTB_EVENT_HW_LINK_UP;
 598
 599                 if (is_ntb_atom(ndev) ||
 600                     ndev->conn_type == NTB_CONN_TRANSPARENT)
 601                         status = readw(ndev->reg_ofs.lnk_stat);
 602                 else {
 603                         int rc = pci_read_config_word(ndev->pdev,
 604                                                       SNB_LINK_STATUS_OFFSET,
 605                                                       &status);
 606                         if (rc)
 607                                 return;
 608                 }
 609
 610                 ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
 611                 ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
 612                 dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
 613                          ndev->link_width, ndev->link_speed);
 614         } else {
 615                 dev_info(&ndev->pdev->dev, "Link Down\n");
 616                 ndev->link_status = NTB_LINK_DOWN;
 617                 event = NTB_EVENT_HW_LINK_DOWN;
 618                 /* Don't modify link width/speed, we need it in link recovery */
 619         }
 620
 621         /* notify the upper layer if we have an event change */
 622         if (ndev->event_cb)
 623                 ndev->event_cb(ndev->ntb_transport, event);
 624 }
 625
 626 static int ntb_link_status(struct ntb_device *ndev)
 627 {
 628         int link_state;
 629
 630         if (is_ntb_atom(ndev)) {
 631                 u32 ntb_cntl;
 632
 633                 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
 634                 if (ntb_cntl & BWD_CNTL_LINK_DOWN)
 635                         link_state = NTB_LINK_DOWN;
 636                 else
 637                         link_state = NTB_LINK_UP;
 638         } else {
 639                 u16 status;
 640                 int rc;
 641
 642                 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
 643                                           &status);
 644                 if (rc)
 645                         return rc;
 646
 647                 if (status & NTB_LINK_STATUS_ACTIVE)
 648                         link_state = NTB_LINK_UP;
 649                 else
 650                         link_state = NTB_LINK_DOWN;
 651         }
 652
 653         ntb_link_event(ndev, link_state);
 654
 655         return 0;
 656 }
 657
 658 static void bwd_link_recovery(struct work_struct *work)
 659 {
 660         struct ntb_device *ndev = container_of(work, struct ntb_device,
 661                                                lr_timer.work);
 662         u32 status32;
 663
 664         bwd_recover_link(ndev);
 665         /* There is a potential race between the 2 NTB devices recovering at the
 666          * same time.  If the times are the same, the link will not recover and
 667          * the driver will be stuck in this loop forever.  Add a random interval
 668          * to the recovery time to prevent this race.
 669          */
 670         msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
 671
 672         status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
 673         if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
 674                 goto retry;
 675
 676         status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
 677         if (status32 & BWD_IBIST_ERR_OFLOW)
 678                 goto retry;
 679
 680         status32 = readl(ndev->reg_ofs.lnk_cntl);
 681         if (!(status32 & BWD_CNTL_LINK_DOWN)) {
 682                 unsigned char speed, width;
 683                 u16 status16;
 684
 685                 status16 = readw(ndev->reg_ofs.lnk_stat);
 686                 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
 687                 speed = (status16 & NTB_LINK_SPEED_MASK);
 688                 if (ndev->link_width != width || ndev->link_speed != speed)
 689                         goto retry;
 690         }
 691
 692         schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
 693         return;
 694
 695 retry:
 696         schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
 697 }
 698
 699 /* BWD doesn't have link status interrupt, poll on that platform */
 700 static void bwd_link_poll(struct work_struct *work)
 701 {
 702         struct ntb_device *ndev = container_of(work, struct ntb_device,
 703                                                hb_timer.work);
 704         unsigned long ts = jiffies;
 705
 706         /* If we haven't gotten an interrupt in a while, check the BWD link
 707          * status bit
 708          */
 709         if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
 710                 int rc = ntb_link_status(ndev);
 711                 if (rc)
 712                         dev_err(&ndev->pdev->dev,
 713                                 "Error determining link status\n");
 714
 715                 /* Check to see if a link error is the cause of the link down */
 716                 if (ndev->link_status == NTB_LINK_DOWN) {
 717                         u32 status32 = readl(ndev->reg_base +
 718                                              BWD_LTSSMSTATEJMP_OFFSET);
 719                         if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
 720                                 schedule_delayed_work(&ndev->lr_timer, 0);
 721                                 return;
 722                         }
 723                 }
 724         }
 725
 726         schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
 727 }
 728
 729 static int ntb_xeon_setup(struct ntb_device *ndev)
 730 {
 731         switch (ndev->conn_type) {
 732         case NTB_CONN_B2B:
 733                 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 734                 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
 735                 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
 736                 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
 737                 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
 738                 if (ndev->split_bar)
 739                         ndev->reg_ofs.bar5_xlat =
 740                                 ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
 741                 ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
 742
 743                 /* There is a Xeon hardware errata related to writes to
 744                  * SDOORBELL or B2BDOORBELL in conjunction with inbound access
 745                  * to NTB MMIO Space, which may hang the system.  To workaround
 746                  * this use the second memory window to access the interrupt and
 747                  * scratch pad registers on the remote system.
 748                  */
 749                 if (ndev->wa_flags & WA_SNB_ERR) {
 750                         if (!ndev->mw[ndev->limits.max_mw - 1].bar_sz)
 751                                 return -EINVAL;
 752
 753                         ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
 754                         ndev->reg_ofs.spad_write =
 755                                 ndev->mw[ndev->limits.max_mw - 1].vbase +
 756                                 SNB_SPAD_OFFSET;
 757                         ndev->reg_ofs.rdb =
 758                                 ndev->mw[ndev->limits.max_mw - 1].vbase +
 759                                 SNB_PDOORBELL_OFFSET;
 760
 761                         /* Set the Limit register to 4k, the minimum size, to
 762                          * prevent an illegal access
 763                          */
 764                         writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
 765                                SNB_PBAR4LMT_OFFSET);
 766                         /* HW errata on the Limit registers.  They can only be
 767                          * written when the base register is 4GB aligned and
 768                          * < 32bit.  This should already be the case based on
 769                          * the driver defaults, but write the Limit registers
 770                          * first just in case.
 771                          */
 772
 773                         ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
 774                 } else {
 775                         /* HW Errata on bit 14 of b2bdoorbell register.  Writes
 776                          * will not be mirrored to the remote system.  Shrink
 777                          * the number of bits by one, since bit 14 is the last
 778                          * bit.
 779                          */
 780                         ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1;
 781                         ndev->reg_ofs.spad_write = ndev->reg_base +
 782                                                    SNB_B2B_SPAD_OFFSET;
 783                         ndev->reg_ofs.rdb = ndev->reg_base +
 784                                             SNB_B2B_DOORBELL_OFFSET;
 785
 786                         /* Disable the Limit register, just incase it is set to
 787                          * something silly. A 64bit write should handle it
 788                          * regardless of whether it has a split BAR or not.
 789                          */
 790                         writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
 791                         /* HW errata on the Limit registers.  They can only be
 792                          * written when the base register is 4GB aligned and
 793                          * < 32bit.  This should already be the case based on
 794                          * the driver defaults, but write the Limit registers
 795                          * first just in case.
 796                          */
 797                         if (ndev->split_bar)
 798                                 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
 799                         else
 800                                 ndev->limits.max_mw = SNB_MAX_MW;
 801                 }
 802
 803                 /* The Xeon errata workaround requires setting SBAR Base
 804                  * addresses to known values, so that the PBAR XLAT can be
 805                  * pointed at SBAR0 of the remote system.
 806                  */
 807                 if (ndev->dev_type == NTB_DEV_USD) {
 808                         writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
 809                                SNB_PBAR2XLAT_OFFSET);
 810                         if (ndev->wa_flags & WA_SNB_ERR)
 811                                 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
 812                                        SNB_PBAR4XLAT_OFFSET);
 813                         else {
 814                                 if (ndev->split_bar) {
 815                                         writel(SNB_MBAR4_DSD_ADDR,
 816                                                ndev->reg_base +
 817                                                SNB_PBAR4XLAT_OFFSET);
 818                                         writel(SNB_MBAR5_DSD_ADDR,
 819                                                ndev->reg_base +
 820                                                SNB_PBAR5XLAT_OFFSET);
 821                                 } else
 822                                         writeq(SNB_MBAR4_DSD_ADDR,
 823                                                ndev->reg_base +
 824                                                SNB_PBAR4XLAT_OFFSET);
 825
 826                                 /* B2B_XLAT_OFFSET is a 64bit register, but can
 827                                  * only take 32bit writes
 828                                  */
 829                                 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
 830                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
 831                                 writel(SNB_MBAR01_DSD_ADDR >> 32,
 832                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
 833                         }
 834
 835                         writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
 836                                SNB_SBAR0BASE_OFFSET);
 837                         writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
 838                                SNB_SBAR2BASE_OFFSET);
 839                         if (ndev->split_bar) {
 840                                 writel(SNB_MBAR4_USD_ADDR, ndev->reg_base +
 841                                        SNB_SBAR4BASE_OFFSET);
 842                                 writel(SNB_MBAR5_USD_ADDR, ndev->reg_base +
 843                                        SNB_SBAR5BASE_OFFSET);
 844                         } else
 845                                 writeq(SNB_MBAR4_USD_ADDR, ndev->reg_base +
 846                                        SNB_SBAR4BASE_OFFSET);
 847                 } else {
 848                         writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
 849                                SNB_PBAR2XLAT_OFFSET);
 850                         if (ndev->wa_flags & WA_SNB_ERR)
 851                                 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
 852                                        SNB_PBAR4XLAT_OFFSET);
 853                         else {
 854                                 if (ndev->split_bar) {
 855                                         writel(SNB_MBAR4_USD_ADDR,
 856                                                ndev->reg_base +
 857                                                SNB_PBAR4XLAT_OFFSET);
 858                                         writel(SNB_MBAR5_USD_ADDR,
 859                                                ndev->reg_base +
 860                                                SNB_PBAR5XLAT_OFFSET);
 861                                 } else
 862                                         writeq(SNB_MBAR4_USD_ADDR,
 863                                                ndev->reg_base +
 864                                                SNB_PBAR4XLAT_OFFSET);
 865
 866                                 /*
 867                                  * B2B_XLAT_OFFSET is a 64bit register, but can
 868                                  * only take 32bit writes
 869                                  */
 870                                 writel(SNB_MBAR01_USD_ADDR & 0xffffffff,
 871                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
 872                                 writel(SNB_MBAR01_USD_ADDR >> 32,
 873                                        ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
 874                         }
 875                         writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
 876                                SNB_SBAR0BASE_OFFSET);
 877                         writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
 878                                SNB_SBAR2BASE_OFFSET);
 879                         if (ndev->split_bar) {
 880                                 writel(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
 881                                        SNB_SBAR4BASE_OFFSET);
 882                                 writel(SNB_MBAR5_DSD_ADDR, ndev->reg_base +
 883                                        SNB_SBAR5BASE_OFFSET);
 884                         } else
 885                                 writeq(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
 886                                        SNB_SBAR4BASE_OFFSET);
 887
 888                 }
 889                 break;
 890         case NTB_CONN_RP:
 891                 if (ndev->wa_flags & WA_SNB_ERR) {
 892                         dev_err(&ndev->pdev->dev,
 893                                 "NTB-RP disabled due to hardware errata.\n");
 894                         return -EINVAL;
 895                 }
 896
 897                 /* Scratch pads need to have exclusive access from the primary
 898                  * or secondary side.  Halve the num spads so that each side can
 899                  * have an equal amount.
 900                  */
 901                 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
 902                 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
 903                 /* Note: The SDOORBELL is the cause of the errata.  You REALLY
 904                  * don't want to touch it.
 905                  */
 906                 ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
 907                 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 908                 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
 909                 /* Offset the start of the spads to correspond to whether it is
 910                  * primary or secondary
 911                  */
 912                 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
 913                                            ndev->limits.max_spads * 4;
 914                 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
 915                 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
 916                 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
 917                 if (ndev->split_bar) {
 918                         ndev->reg_ofs.bar5_xlat =
 919                                 ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
 920                         ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
 921                 } else
 922                         ndev->limits.max_mw = SNB_MAX_MW;
 923                 break;
 924         case NTB_CONN_TRANSPARENT:
 925                 if (ndev->wa_flags & WA_SNB_ERR) {
 926                         dev_err(&ndev->pdev->dev,
 927                                 "NTB-TRANSPARENT disabled due to hardware errata.\n");
 928                         return -EINVAL;
 929                 }
 930
 931                 /* Scratch pads need to have exclusive access from the primary
 932                  * or secondary side.  Halve the num spads so that each side can
 933                  * have an equal amount.
 934                  */
 935                 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
 936                 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
 937                 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
 938                 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
 939                 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
 940                 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
 941                 /* Offset the start of the spads to correspond to whether it is
 942                  * primary or secondary
 943                  */
 944                 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
 945                                           ndev->limits.max_spads * 4;
 946                 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
 947                 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
 948
 949                 if (ndev->split_bar) {
 950                         ndev->reg_ofs.bar5_xlat =
 951                                 ndev->reg_base + SNB_PBAR5XLAT_OFFSET;
 952                         ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
 953                 } else
 954                         ndev->limits.max_mw = SNB_MAX_MW;
 955                 break;
 956         default:
 957                 /*
 958                  * we should never hit this. the detect function should've
 959                  * take cared of everything.
 960                  */
 961                 return -EINVAL;
 962         }
 963
 964         ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
 965         ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
 966         ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
 967
 968         ndev->limits.msix_cnt = SNB_MSIX_CNT;
 969         ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
 970
 971         return 0;
 972 }
 973
 974 static int ntb_bwd_setup(struct ntb_device *ndev)
 975 {
 976         int rc;
 977         u32 val;
 978
 979         ndev->hw_type = BWD_HW;
 980
 981         rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
 982         if (rc)
 983                 return rc;
 984
 985         switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
 986         case NTB_CONN_B2B:
 987                 ndev->conn_type = NTB_CONN_B2B;
 988                 break;
 989         case NTB_CONN_RP:
 990         default:
 991                 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
 992                 return -EINVAL;
 993         }
 994
 995         if (val & BWD_PPD_DEV_TYPE)
 996                 ndev->dev_type = NTB_DEV_DSD;
 997         else
 998                 ndev->dev_type = NTB_DEV_USD;
 999
1000         /* Initiate PCI-E link training */
1001         rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
1002                                     val | BWD_PPD_INIT_LINK);
1003         if (rc)
1004                 return rc;
1005
1006         ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
1007         ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
1008         ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
1009         ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
1010         ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
1011         ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
1012         ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
1013         ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
1014         ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
1015         ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
1016         ndev->limits.max_mw = BWD_MAX_MW;
1017         ndev->limits.max_spads = BWD_MAX_SPADS;
1018         ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
1019         ndev->limits.msix_cnt = BWD_MSIX_CNT;
1020         ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
1021
1022         /* Since bwd doesn't have a link interrupt, setup a poll timer */
1023         INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
1024         INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
1025         schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
1026
1027         return 0;
1028 }
1029
1030 static int ntb_device_setup(struct ntb_device *ndev)
1031 {
1032         int rc;
1033
1034         if (is_ntb_xeon(ndev))
1035                 rc = ntb_xeon_setup(ndev);
1036         else if (is_ntb_atom(ndev))
1037                 rc = ntb_bwd_setup(ndev);
1038         else
1039                 rc = -ENODEV;
1040
1041         if (rc)
1042                 return rc;
1043
1044         if (ndev->conn_type == NTB_CONN_B2B)
1045                 /* Enable Bus Master and Memory Space on the secondary side */
1046                 writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1047                        ndev->reg_ofs.spci_cmd);
1048
1049         return 0;
1050 }
1051
1052 static void ntb_device_free(struct ntb_device *ndev)
1053 {
1054         if (is_ntb_atom(ndev)) {
1055                 cancel_delayed_work_sync(&ndev->hb_timer);
1056                 cancel_delayed_work_sync(&ndev->lr_timer);
1057         }
1058 }
1059
1060 static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
1061 {
1062         struct ntb_db_cb *db_cb = data;
1063         struct ntb_device *ndev = db_cb->ndev;
1064         unsigned long mask;
1065
1066         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
1067                 db_cb->db_num);
1068
1069         mask = readw(ndev->reg_ofs.ldb_mask);
1070         set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
1071         writew(mask, ndev->reg_ofs.ldb_mask);
1072
1073         tasklet_schedule(&db_cb->irq_work);
1074
1075         /* No need to check for the specific HB irq, any interrupt means
1076          * we're connected.
1077          */
1078         ndev->last_ts = jiffies;
1079
1080         writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
1081
1082         return IRQ_HANDLED;
1083 }
1084
1085 static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
1086 {
1087         struct ntb_db_cb *db_cb = data;
1088         struct ntb_device *ndev = db_cb->ndev;
1089         unsigned long mask;
1090
1091         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
1092                 db_cb->db_num);
1093
1094         mask = readw(ndev->reg_ofs.ldb_mask);
1095         set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
1096         writew(mask, ndev->reg_ofs.ldb_mask);
1097
1098         tasklet_schedule(&db_cb->irq_work);
1099
1100         /* On Sandybridge, there are 16 bits in the interrupt register
1101          * but only 4 vectors.  So, 5 bits are assigned to the first 3
1102          * vectors, with the 4th having a single bit for link
1103          * interrupts.
1104          */
1105         writew(((1 << ndev->bits_per_vector) - 1) <<
1106                (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
1107
1108         return IRQ_HANDLED;
1109 }
1110
1111 /* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
1112 static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
1113 {
1114         struct ntb_device *ndev = dev;
1115         int rc;
1116
1117         dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
1118
1119         rc = ntb_link_status(ndev);
1120         if (rc)
1121                 dev_err(&ndev->pdev->dev, "Error determining link status\n");
1122
1123         /* bit 15 is always the link bit */
1124         writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb);
1125
1126         return IRQ_HANDLED;
1127 }
1128
1129 static irqreturn_t ntb_interrupt(int irq, void *dev)
1130 {
1131         struct ntb_device *ndev = dev;
1132         unsigned int i = 0;
1133
1134         if (is_ntb_atom(ndev)) {
1135                 u64 ldb = readq(ndev->reg_ofs.ldb);
1136
1137                 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
1138
1139                 while (ldb) {
1140                         i = __ffs(ldb);
1141                         ldb &= ldb - 1;
1142                         bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
1143                 }
1144         } else {
1145                 u16 ldb = readw(ndev->reg_ofs.ldb);
1146
1147                 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
1148
1149                 if (ldb & SNB_DB_HW_LINK) {
1150                         xeon_event_msix_irq(irq, dev);
1151                         ldb &= ~SNB_DB_HW_LINK;
1152                 }
1153
1154                 while (ldb) {
1155                         i = __ffs(ldb);
1156                         ldb &= ldb - 1;
1157                         xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
1158                 }
1159         }
1160
1161         return IRQ_HANDLED;
1162 }
1163
1164 static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries)
1165 {
1166         struct pci_dev *pdev = ndev->pdev;
1167         struct msix_entry *msix;
1168         int rc, i;
1169
1170         if (msix_entries < ndev->limits.msix_cnt)
1171                 return -ENOSPC;
1172
1173         rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries);
1174         if (rc < 0)
1175                 return rc;
1176
1177         for (i = 0; i < msix_entries; i++) {
1178                 msix = &ndev->msix_entries[i];
1179                 WARN_ON(!msix->vector);
1180
1181                 if (i == msix_entries - 1) {
1182                         rc = request_irq(msix->vector,
1183                                          xeon_event_msix_irq, 0,
1184                                          "ntb-event-msix", ndev);
1185                         if (rc)
1186                                 goto err;
1187                 } else {
1188                         rc = request_irq(msix->vector,
1189                                          xeon_callback_msix_irq, 0,
1190                                          "ntb-callback-msix",
1191                                          &ndev->db_cb[i]);
1192                         if (rc)
1193                                 goto err;
1194                 }
1195         }
1196
1197         ndev->num_msix = msix_entries;
1198         ndev->max_cbs = msix_entries - 1;
1199
1200         return 0;
1201
1202 err:
1203         while (--i >= 0) {
1204                 /* Code never reaches here for entry nr 'ndev->num_msix - 1' */
1205                 msix = &ndev->msix_entries[i];
1206                 free_irq(msix->vector, &ndev->db_cb[i]);
1207         }
1208
1209         pci_disable_msix(pdev);
1210         ndev->num_msix = 0;
1211
1212         return rc;
1213 }
1214
1215 static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries)
1216 {
1217         struct pci_dev *pdev = ndev->pdev;
1218         struct msix_entry *msix;
1219         int rc, i;
1220
1221         msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries,
1222                                              1, msix_entries);
1223         if (msix_entries < 0)
1224                 return msix_entries;
1225
1226         for (i = 0; i < msix_entries; i++) {
1227                 msix = &ndev->msix_entries[i];
1228                 WARN_ON(!msix->vector);
1229
1230                 rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
1231                                  "ntb-callback-msix", &ndev->db_cb[i]);
1232                 if (rc)
1233                         goto err;
1234         }
1235
1236         ndev->num_msix = msix_entries;
1237         ndev->max_cbs = msix_entries;
1238
1239         return 0;
1240
1241 err:
1242         while (--i >= 0)
1243                 free_irq(msix->vector, &ndev->db_cb[i]);
1244
1245         pci_disable_msix(pdev);
1246         ndev->num_msix = 0;
1247
1248         return rc;
1249 }
1250
1251 static int ntb_setup_msix(struct ntb_device *ndev)
1252 {
1253         struct pci_dev *pdev = ndev->pdev;
1254         int msix_entries;
1255         int rc, i;
1256
1257         msix_entries = pci_msix_vec_count(pdev);
1258         if (msix_entries < 0) {
1259                 rc = msix_entries;
1260                 goto err;
1261         } else if (msix_entries > ndev->limits.msix_cnt) {
1262                 rc = -EINVAL;
1263                 goto err;
1264         }
1265
1266         ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
1267                                      GFP_KERNEL);
1268         if (!ndev->msix_entries) {
1269                 rc = -ENOMEM;
1270                 goto err;
1271         }
1272
1273         for (i = 0; i < msix_entries; i++)
1274                 ndev->msix_entries[i].entry = i;
1275
1276         if (is_ntb_atom(ndev))
1277                 rc = ntb_setup_bwd_msix(ndev, msix_entries);
1278         else
1279                 rc = ntb_setup_snb_msix(ndev, msix_entries);
1280         if (rc)
1281                 goto err1;
1282
1283         return 0;
1284
1285 err1:
1286         kfree(ndev->msix_entries);
1287 err:
1288         dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
1289         return rc;
1290 }
1291
1292 static int ntb_setup_msi(struct ntb_device *ndev)
1293 {
1294         struct pci_dev *pdev = ndev->pdev;
1295         int rc;
1296
1297         rc = pci_enable_msi(pdev);
1298         if (rc)
1299                 return rc;
1300
1301         rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
1302         if (rc) {
1303                 pci_disable_msi(pdev);
1304                 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
1305                 return rc;
1306         }
1307
1308         return 0;
1309 }
1310
1311 static int ntb_setup_intx(struct ntb_device *ndev)
1312 {
1313         struct pci_dev *pdev = ndev->pdev;
1314         int rc;
1315
1316         pci_msi_off(pdev);
1317
1318         /* Verify intx is enabled */
1319         pci_intx(pdev, 1);
1320
1321         rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
1322                          ndev);
1323         if (rc)
1324                 return rc;
1325
1326         return 0;
1327 }
1328
1329 static int ntb_setup_interrupts(struct ntb_device *ndev)
1330 {
1331         int rc;
1332
1333         /* On BWD, disable all interrupts.  On SNB, disable all but Link
1334          * Interrupt.  The rest will be unmasked as callbacks are registered.
1335          */
1336         if (is_ntb_atom(ndev))
1337                 writeq(~0, ndev->reg_ofs.ldb_mask);
1338         else {
1339                 u16 var = 1 << SNB_LINK_DB;
1340                 writew(~var, ndev->reg_ofs.ldb_mask);
1341         }
1342
1343         rc = ntb_setup_msix(ndev);
1344         if (!rc)
1345                 goto done;
1346
1347         ndev->bits_per_vector = 1;
1348         ndev->max_cbs = ndev->limits.max_db_bits;
1349
1350         rc = ntb_setup_msi(ndev);
1351         if (!rc)
1352                 goto done;
1353
1354         rc = ntb_setup_intx(ndev);
1355         if (rc) {
1356                 dev_err(&ndev->pdev->dev, "no usable interrupts\n");
1357                 return rc;
1358         }
1359
1360 done:
1361         return 0;
1362 }
1363
1364 static void ntb_free_interrupts(struct ntb_device *ndev)
1365 {
1366         struct pci_dev *pdev = ndev->pdev;
1367
1368         /* mask interrupts */
1369         if (is_ntb_atom(ndev))
1370                 writeq(~0, ndev->reg_ofs.ldb_mask);
1371         else
1372                 writew(~0, ndev->reg_ofs.ldb_mask);
1373
1374         if (ndev->num_msix) {
1375                 struct msix_entry *msix;
1376                 u32 i;
1377
1378                 for (i = 0; i < ndev->num_msix; i++) {
1379                         msix = &ndev->msix_entries[i];
1380                         if (is_ntb_xeon(ndev) && i == ndev->num_msix - 1)
1381                                 free_irq(msix->vector, ndev);
1382                         else
1383                                 free_irq(msix->vector, &ndev->db_cb[i]);
1384                 }
1385                 pci_disable_msix(pdev);
1386                 kfree(ndev->msix_entries);
1387         } else {
1388                 free_irq(pdev->irq, ndev);
1389
1390                 if (pci_dev_msi_enabled(pdev))
1391                         pci_disable_msi(pdev);
1392         }
1393 }
1394
1395 static int ntb_create_callbacks(struct ntb_device *ndev)
1396 {
1397         int i;
1398
1399         /* Chicken-egg issue.  We won't know how many callbacks are necessary
1400          * until we see how many MSI-X vectors we get, but these pointers need
1401          * to be passed into the MSI-X register function.  So, we allocate the
1402          * max, knowing that they might not all be used, to work around this.
1403          */
1404         ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
1405                               sizeof(struct ntb_db_cb),
1406                               GFP_KERNEL);
1407         if (!ndev->db_cb)
1408                 return -ENOMEM;
1409
1410         for (i = 0; i < ndev->limits.max_db_bits; i++) {
1411                 ndev->db_cb[i].db_num = i;
1412                 ndev->db_cb[i].ndev = ndev;
1413         }
1414
1415         return 0;
1416 }
1417
1418 static void ntb_free_callbacks(struct ntb_device *ndev)
1419 {
1420         int i;
1421
1422         for (i = 0; i < ndev->limits.max_db_bits; i++)
1423                 ntb_unregister_db_callback(ndev, i);
1424
1425         kfree(ndev->db_cb);
1426 }
1427
1428 static ssize_t ntb_debugfs_read(struct file *filp, char __user *ubuf,
1429                                 size_t count, loff_t *offp)
1430 {
1431         struct ntb_device *ndev;
1432         char *buf;
1433         ssize_t ret, offset, out_count;
1434
1435         out_count = 500;
1436
1437         buf = kmalloc(out_count, GFP_KERNEL);
1438         if (!buf)
1439                 return -ENOMEM;
1440
1441         ndev = filp->private_data;
1442         offset = 0;
1443         offset += snprintf(buf + offset, out_count - offset,
1444                            "NTB Device Information:\n");
1445         offset += snprintf(buf + offset, out_count - offset,
1446                            "Connection Type - \t\t%s\n",
1447                            ndev->conn_type == NTB_CONN_TRANSPARENT ?
1448                            "Transparent" : (ndev->conn_type == NTB_CONN_B2B) ?
1449                            "Back to back" : "Root Port");
1450         offset += snprintf(buf + offset, out_count - offset,
1451                            "Device Type - \t\t\t%s\n",
1452                            ndev->dev_type == NTB_DEV_USD ?
1453                            "DSD/USP" : "USD/DSP");
1454         offset += snprintf(buf + offset, out_count - offset,
1455                            "Max Number of Callbacks - \t%u\n",
1456                            ntb_max_cbs(ndev));
1457         offset += snprintf(buf + offset, out_count - offset,
1458                            "Link Status - \t\t\t%s\n",
1459                            ntb_hw_link_status(ndev) ? "Up" : "Down");
1460         if (ntb_hw_link_status(ndev)) {
1461                 offset += snprintf(buf + offset, out_count - offset,
1462                                    "Link Speed - \t\t\tPCI-E Gen %u\n",
1463                                    ndev->link_speed);
1464                 offset += snprintf(buf + offset, out_count - offset,
1465                                    "Link Width - \t\t\tx%u\n",
1466                                    ndev->link_width);
1467         }
1468
1469         if (is_ntb_xeon(ndev)) {
1470                 u32 status32;
1471                 u16 status16;
1472                 int rc;
1473
1474                 offset += snprintf(buf + offset, out_count - offset,
1475                                    "\nNTB Device Statistics:\n");
1476                 offset += snprintf(buf + offset, out_count - offset,
1477                                    "Upstream Memory Miss - \t%u\n",
1478                                    readw(ndev->reg_base +
1479                                          SNB_USMEMMISS_OFFSET));
1480
1481                 offset += snprintf(buf + offset, out_count - offset,
1482                                    "\nNTB Hardware Errors:\n");
1483
1484                 rc = pci_read_config_word(ndev->pdev, SNB_DEVSTS_OFFSET,
1485                                           &status16);
1486                 if (!rc)
1487                         offset += snprintf(buf + offset, out_count - offset,
1488                                            "DEVSTS - \t%#06x\n", status16);
1489
1490                 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
1491                                           &status16);
1492                 if (!rc)
1493                         offset += snprintf(buf + offset, out_count - offset,
1494                                            "LNKSTS - \t%#06x\n", status16);
1495
1496                 rc = pci_read_config_dword(ndev->pdev, SNB_UNCERRSTS_OFFSET,
1497                                            &status32);
1498                 if (!rc)
1499                         offset += snprintf(buf + offset, out_count - offset,
1500                                            "UNCERRSTS - \t%#010x\n", status32);
1501
1502                 rc = pci_read_config_dword(ndev->pdev, SNB_CORERRSTS_OFFSET,
1503                                            &status32);
1504                 if (!rc)
1505                         offset += snprintf(buf + offset, out_count - offset,
1506                                            "CORERRSTS - \t%#010x\n", status32);
1507         }
1508
1509         if (offset > out_count)
1510                 offset = out_count;
1511
1512         ret = simple_read_from_buffer(ubuf, count, offp, buf, offset);
1513         kfree(buf);
1514         return ret;
1515 }
1516
1517 static const struct file_operations ntb_debugfs_info = {
1518         .owner = THIS_MODULE,
1519         .open = simple_open,
1520         .read = ntb_debugfs_read,
1521 };
1522
1523 static void ntb_setup_debugfs(struct ntb_device *ndev)
1524 {
1525         if (!debugfs_initialized())
1526                 return;
1527
1528         if (!debugfs_dir)
1529                 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1530
1531         ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
1532                                                debugfs_dir);
1533         if (ndev->debugfs_dir)
1534                 ndev->debugfs_info = debugfs_create_file("info", S_IRUSR,
1535                                                          ndev->debugfs_dir,
1536                                                          ndev,
1537                                                          &ntb_debugfs_info);
1538 }
1539
1540 static void ntb_free_debugfs(struct ntb_device *ndev)
1541 {
1542         debugfs_remove_recursive(ndev->debugfs_dir);
1543
1544         if (debugfs_dir && simple_empty(debugfs_dir)) {
1545                 debugfs_remove_recursive(debugfs_dir);
1546                 debugfs_dir = NULL;
1547         }
1548 }
1549
1550 static void ntb_hw_link_up(struct ntb_device *ndev)
1551 {
1552         if (ndev->conn_type == NTB_CONN_TRANSPARENT)
1553                 ntb_link_event(ndev, NTB_LINK_UP);
1554         else {
1555                 u32 ntb_cntl;
1556
1557                 /* Let's bring the NTB link up */
1558                 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1559                 ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
1560                 ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
1561                 ntb_cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
1562                 if (ndev->split_bar)
1563                         ntb_cntl |= NTB_CNTL_P2S_BAR5_SNOOP |
1564                                     NTB_CNTL_S2P_BAR5_SNOOP;
1565
1566                 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1567         }
1568 }
1569
1570 static void ntb_hw_link_down(struct ntb_device *ndev)
1571 {
1572         u32 ntb_cntl;
1573
1574         if (ndev->conn_type == NTB_CONN_TRANSPARENT) {
1575                 ntb_link_event(ndev, NTB_LINK_DOWN);
1576                 return;
1577         }
1578
1579         /* Bring NTB link down */
1580         ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1581         ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
1582         ntb_cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
1583         if (ndev->split_bar)
1584                 ntb_cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP |
1585                               NTB_CNTL_S2P_BAR5_SNOOP);
1586         ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
1587         writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1588 }
1589
1590 static void ntb_max_mw_detect(struct ntb_device *ndev)
1591 {
1592         if (ndev->split_bar)
1593                 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
1594         else
1595                 ndev->limits.max_mw = SNB_MAX_MW;
1596 }
1597
1598 static int ntb_xeon_detect(struct ntb_device *ndev)
1599 {
1600         int rc, bars_mask;
1601         u32 bars;
1602         u8 ppd;
1603
1604         ndev->hw_type = SNB_HW;
1605
1606         rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &ppd);
1607         if (rc)
1608                 return -EIO;
1609
1610         if (ppd & SNB_PPD_DEV_TYPE)
1611                 ndev->dev_type = NTB_DEV_USD;
1612         else
1613                 ndev->dev_type = NTB_DEV_DSD;
1614
1615         ndev->split_bar = (ppd & SNB_PPD_SPLIT_BAR) ? 1 : 0;
1616
1617         switch (ppd & SNB_PPD_CONN_TYPE) {
1618         case NTB_CONN_B2B:
1619                 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
1620                 ndev->conn_type = NTB_CONN_B2B;
1621                 break;
1622         case NTB_CONN_RP:
1623                 dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
1624                 ndev->conn_type = NTB_CONN_RP;
1625                 break;
1626         case NTB_CONN_TRANSPARENT:
1627                 dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
1628                 ndev->conn_type = NTB_CONN_TRANSPARENT;
1629                 /*
1630                  * This mode is default to USD/DSP. HW does not report
1631                  * properly in transparent mode as it has no knowledge of
1632                  * NTB. We will just force correct here.
1633                  */
1634                 ndev->dev_type = NTB_DEV_USD;
1635
1636                 /*
1637                  * This is a way for transparent BAR to figure out if we
1638                  * are doing split BAR or not. There is no way for the hw
1639                  * on the transparent side to know and set the PPD.
1640                  */
1641                 bars_mask = pci_select_bars(ndev->pdev, IORESOURCE_MEM);
1642                 bars = hweight32(bars_mask);
1643                 if (bars == (HSX_SPLITBAR_MAX_MW + 1))
1644                         ndev->split_bar = 1;
1645
1646                 break;
1647         default:
1648                 dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", ppd);
1649                 return -ENODEV;
1650         }
1651
1652         ntb_max_mw_detect(ndev);
1653
1654         return 0;
1655 }
1656
1657 static int ntb_atom_detect(struct ntb_device *ndev)
1658 {
1659         int rc;
1660         u32 ppd;
1661
1662         ndev->hw_type = BWD_HW;
1663         ndev->limits.max_mw = BWD_MAX_MW;
1664
1665         rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &ppd);
1666         if (rc)
1667                 return rc;
1668
1669         switch ((ppd & BWD_PPD_CONN_TYPE) >> 8) {
1670         case NTB_CONN_B2B:
1671                 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
1672                 ndev->conn_type = NTB_CONN_B2B;
1673                 break;
1674         case NTB_CONN_RP:
1675         default:
1676                 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
1677                 return -EINVAL;
1678         }
1679
1680         if (ppd & BWD_PPD_DEV_TYPE)
1681                 ndev->dev_type = NTB_DEV_DSD;
1682         else
1683                 ndev->dev_type = NTB_DEV_USD;
1684
1685         return 0;
1686 }
1687
1688 static int ntb_device_detect(struct ntb_device *ndev)
1689 {
1690         int rc;
1691
1692         if (is_ntb_xeon(ndev))
1693                 rc = ntb_xeon_detect(ndev);
1694         else if (is_ntb_atom(ndev))
1695                 rc = ntb_atom_detect(ndev);
1696         else
1697                 rc = -ENODEV;
1698
1699         dev_info(&ndev->pdev->dev, "Device Type = %s\n",
1700                  ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
1701
1702         return 0;
1703 }
1704
1705 static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1706 {
1707         struct ntb_device *ndev;
1708         int rc, i;
1709
1710         ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
1711         if (!ndev)
1712                 return -ENOMEM;
1713
1714         ndev->pdev = pdev;
1715
1716         ntb_set_errata_flags(ndev);
1717
1718         ndev->link_status = NTB_LINK_DOWN;
1719         pci_set_drvdata(pdev, ndev);
1720         ntb_setup_debugfs(ndev);
1721
1722         rc = pci_enable_device(pdev);
1723         if (rc)
1724                 goto err;
1725
1726         pci_set_master(ndev->pdev);
1727
1728         rc = ntb_device_detect(ndev);
1729         if (rc)
1730                 goto err;
1731
1732         ndev->mw = kcalloc(ndev->limits.max_mw, sizeof(struct ntb_mw),
1733                            GFP_KERNEL);
1734         if (!ndev->mw) {
1735                 rc = -ENOMEM;
1736                 goto err1;
1737         }
1738
1739         if (ndev->split_bar)
1740                 rc = pci_request_selected_regions(pdev, NTB_SPLITBAR_MASK,
1741                                                   KBUILD_MODNAME);
1742         else
1743                 rc = pci_request_selected_regions(pdev, NTB_BAR_MASK,
1744                                                   KBUILD_MODNAME);
1745
1746         if (rc)
1747                 goto err2;
1748
1749         ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
1750         if (!ndev->reg_base) {
1751                 dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
1752                 rc = -EIO;
1753                 goto err3;
1754         }
1755
1756         for (i = 0; i < ndev->limits.max_mw; i++) {
1757                 ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
1758
1759                 /*
1760                  * with the errata we need to steal last of the memory
1761                  * windows for workarounds and they point to MMIO registers.
1762                  */
1763                 if ((ndev->wa_flags & WA_SNB_ERR) &&
1764                     (i == (ndev->limits.max_mw - 1))) {
1765                         ndev->mw[i].vbase =
1766                                 ioremap_nocache(pci_resource_start(pdev,
1767                                                         MW_TO_BAR(i)),
1768                                                 ndev->mw[i].bar_sz);
1769                 } else {
1770                         ndev->mw[i].vbase =
1771                                 ioremap_wc(pci_resource_start(pdev,
1772                                                         MW_TO_BAR(i)),
1773                                            ndev->mw[i].bar_sz);
1774                 }
1775
1776                 dev_info(&pdev->dev, "MW %d size %llu\n", i,
1777                          (unsigned long long) ndev->mw[i].bar_sz);
1778                 if (!ndev->mw[i].vbase) {
1779                         dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
1780                                  MW_TO_BAR(i));
1781                         rc = -EIO;
1782                         goto err4;
1783                 }
1784         }
1785
1786         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1787         if (rc) {
1788                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1789                 if (rc)
1790                         goto err4;
1791
1792                 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1793         }
1794
1795         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1796         if (rc) {
1797                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1798                 if (rc)
1799                         goto err4;
1800
1801                 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1802         }
1803
1804         rc = ntb_device_setup(ndev);
1805         if (rc)
1806                 goto err4;
1807
1808         rc = ntb_create_callbacks(ndev);
1809         if (rc)
1810                 goto err5;
1811
1812         rc = ntb_setup_interrupts(ndev);
1813         if (rc)
1814                 goto err6;
1815
1816         /* The scratchpad registers keep the values between rmmod/insmod,
1817          * blast them now
1818          */
1819         for (i = 0; i < ndev->limits.max_spads; i++) {
1820                 ntb_write_local_spad(ndev, i, 0);
1821                 ntb_write_remote_spad(ndev, i, 0);
1822         }
1823
1824         rc = ntb_transport_init(pdev);
1825         if (rc)
1826                 goto err7;
1827
1828         ntb_hw_link_up(ndev);
1829
1830         return 0;
1831
1832 err7:
1833         ntb_free_interrupts(ndev);
1834 err6:
1835         ntb_free_callbacks(ndev);
1836 err5:
1837         ntb_device_free(ndev);
1838 err4:
1839         for (i--; i >= 0; i--)
1840                 iounmap(ndev->mw[i].vbase);
1841         iounmap(ndev->reg_base);
1842 err3:
1843         if (ndev->split_bar)
1844                 pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK);
1845         else
1846                 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1847 err2:
1848         kfree(ndev->mw);
1849 err1:
1850         pci_disable_device(pdev);
1851 err:
1852         ntb_free_debugfs(ndev);
1853         kfree(ndev);
1854
1855         dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
1856         return rc;
1857 }
1858
1859 static void ntb_pci_remove(struct pci_dev *pdev)
1860 {
1861         struct ntb_device *ndev = pci_get_drvdata(pdev);
1862         int i;
1863
1864         ntb_hw_link_down(ndev);
1865
1866         ntb_transport_free(ndev->ntb_transport);
1867
1868         ntb_free_interrupts(ndev);
1869         ntb_free_callbacks(ndev);
1870         ntb_device_free(ndev);
1871
1872         /* need to reset max_mw limits so we can unmap properly */
1873         if (ndev->hw_type == SNB_HW)
1874                 ntb_max_mw_detect(ndev);
1875
1876         for (i = 0; i < ndev->limits.max_mw; i++)
1877                 iounmap(ndev->mw[i].vbase);
1878
1879         kfree(ndev->mw);
1880         iounmap(ndev->reg_base);
1881         if (ndev->split_bar)
1882                 pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK);
1883         else
1884                 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1885         pci_disable_device(pdev);
1886         ntb_free_debugfs(ndev);
1887         kfree(ndev);
1888 }
1889
1890 static struct pci_driver ntb_pci_driver = {
1891         .name = KBUILD_MODNAME,
1892         .id_table = ntb_pci_tbl,
1893         .probe = ntb_pci_probe,
1894         .remove = ntb_pci_remove,
1895 };
1896
1897 module_pci_driver(ntb_pci_driver);