kernel/arch/powerpc/platforms/powernv/pci.c

   1 /*
   2  * Support PCI/PCIe on PowerNV platforms
   3  *
   4  * Currently supports only P5IOC2
   5  *
   6  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License
  10  * as published by the Free Software Foundation; either version
  11  * 2 of the License, or (at your option) any later version.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/delay.h>
  17 #include <linux/string.h>
  18 #include <linux/init.h>
  19 #include <linux/irq.h>
  20 #include <linux/io.h>
  21 #include <linux/msi.h>
  22 #include <linux/iommu.h>
  23
  24 #include <asm/sections.h>
  25 #include <asm/io.h>
  26 #include <asm/prom.h>
  27 #include <asm/pci-bridge.h>
  28 #include <asm/machdep.h>
  29 #include <asm/msi_bitmap.h>
  30 #include <asm/ppc-pci.h>
  31 #include <asm/opal.h>
  32 #include <asm/iommu.h>
  33 #include <asm/tce.h>
  34 #include <asm/firmware.h>
  35 #include <asm/eeh_event.h>
  36 #include <asm/eeh.h>
  37
  38 #include "powernv.h"
  39 #include "pci.h"
  40
  41 /* Delay in usec */
  42 #define PCI_RESET_DELAY_US      3000000
  43
  44 #define cfg_dbg(fmt...) do { } while(0)
  45 //#define cfg_dbg(fmt...)       printk(fmt)
  46
  47 #ifdef CONFIG_PCI_MSI
  48 static int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
  49 {
  50         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
  51         struct pnv_phb *phb = hose->private_data;
  52         struct msi_desc *entry;
  53         struct msi_msg msg;
  54         int hwirq;
  55         unsigned int virq;
  56         int rc;
  57
  58         if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
  59                 return -ENODEV;
  60
  61         if (pdev->no_64bit_msi && !phb->msi32_support)
  62                 return -ENODEV;
  63
  64         list_for_each_entry(entry, &pdev->msi_list, list) {
  65                 if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
  66                         pr_warn("%s: Supports only 64-bit MSIs\n",
  67                                 pci_name(pdev));
  68                         return -ENXIO;
  69                 }
  70                 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
  71                 if (hwirq < 0) {
  72                         pr_warn("%s: Failed to find a free MSI\n",
  73                                 pci_name(pdev));
  74                         return -ENOSPC;
  75                 }
  76                 virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
  77                 if (virq == NO_IRQ) {
  78                         pr_warn("%s: Failed to map MSI to linux irq\n",
  79                                 pci_name(pdev));
  80                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  81                         return -ENOMEM;
  82                 }
  83                 rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
  84                                     virq, entry->msi_attrib.is_64, &msg);
  85                 if (rc) {
  86                         pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
  87                         irq_dispose_mapping(virq);
  88                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
  89                         return rc;
  90                 }
  91                 irq_set_msi_desc(virq, entry);
  92                 pci_write_msi_msg(virq, &msg);
  93         }
  94         return 0;
  95 }
  96
  97 static void pnv_teardown_msi_irqs(struct pci_dev *pdev)
  98 {
  99         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 100         struct pnv_phb *phb = hose->private_data;
 101         struct msi_desc *entry;
 102
 103         if (WARN_ON(!phb))
 104                 return;
 105
 106         list_for_each_entry(entry, &pdev->msi_list, list) {
 107                 if (entry->irq == NO_IRQ)
 108                         continue;
 109                 irq_set_msi_desc(entry->irq, NULL);
 110                 msi_bitmap_free_hwirqs(&phb->msi_bmp,
 111                         virq_to_hw(entry->irq) - phb->msi_base, 1);
 112                 irq_dispose_mapping(entry->irq);
 113         }
 114 }
 115 #endif /* CONFIG_PCI_MSI */
 116
 117 static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
 118                                          struct OpalIoPhbErrorCommon *common)
 119 {
 120         struct OpalIoP7IOCPhbErrorData *data;
 121         int i;
 122
 123         data = (struct OpalIoP7IOCPhbErrorData *)common;
 124         pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
 125                 hose->global_number, be32_to_cpu(common->version));
 126
 127         if (data->brdgCtl)
 128                 pr_info("brdgCtl:     %08x\n",
 129                         be32_to_cpu(data->brdgCtl));
 130         if (data->portStatusReg || data->rootCmplxStatus ||
 131             data->busAgentStatus)
 132                 pr_info("UtlSts:      %08x %08x %08x\n",
 133                         be32_to_cpu(data->portStatusReg),
 134                         be32_to_cpu(data->rootCmplxStatus),
 135                         be32_to_cpu(data->busAgentStatus));
 136         if (data->deviceStatus || data->slotStatus   ||
 137             data->linkStatus   || data->devCmdStatus ||
 138             data->devSecStatus)
 139                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 140                         be32_to_cpu(data->deviceStatus),
 141                         be32_to_cpu(data->slotStatus),
 142                         be32_to_cpu(data->linkStatus),
 143                         be32_to_cpu(data->devCmdStatus),
 144                         be32_to_cpu(data->devSecStatus));
 145         if (data->rootErrorStatus   || data->uncorrErrorStatus ||
 146             data->corrErrorStatus)
 147                 pr_info("RootErrSts:  %08x %08x %08x\n",
 148                         be32_to_cpu(data->rootErrorStatus),
 149                         be32_to_cpu(data->uncorrErrorStatus),
 150                         be32_to_cpu(data->corrErrorStatus));
 151         if (data->tlpHdr1 || data->tlpHdr2 ||
 152             data->tlpHdr3 || data->tlpHdr4)
 153                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 154                         be32_to_cpu(data->tlpHdr1),
 155                         be32_to_cpu(data->tlpHdr2),
 156                         be32_to_cpu(data->tlpHdr3),
 157                         be32_to_cpu(data->tlpHdr4));
 158         if (data->sourceId || data->errorClass ||
 159             data->correlator)
 160                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 161                         be32_to_cpu(data->sourceId),
 162                         be64_to_cpu(data->errorClass),
 163                         be64_to_cpu(data->correlator));
 164         if (data->p7iocPlssr || data->p7iocCsr)
 165                 pr_info("PhbSts:      %016llx %016llx\n",
 166                         be64_to_cpu(data->p7iocPlssr),
 167                         be64_to_cpu(data->p7iocCsr));
 168         if (data->lemFir)
 169                 pr_info("Lem:         %016llx %016llx %016llx\n",
 170                         be64_to_cpu(data->lemFir),
 171                         be64_to_cpu(data->lemErrorMask),
 172                         be64_to_cpu(data->lemWOF));
 173         if (data->phbErrorStatus)
 174                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 175                         be64_to_cpu(data->phbErrorStatus),
 176                         be64_to_cpu(data->phbFirstErrorStatus),
 177                         be64_to_cpu(data->phbErrorLog0),
 178                         be64_to_cpu(data->phbErrorLog1));
 179         if (data->mmioErrorStatus)
 180                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 181                         be64_to_cpu(data->mmioErrorStatus),
 182                         be64_to_cpu(data->mmioFirstErrorStatus),
 183                         be64_to_cpu(data->mmioErrorLog0),
 184                         be64_to_cpu(data->mmioErrorLog1));
 185         if (data->dma0ErrorStatus)
 186                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 187                         be64_to_cpu(data->dma0ErrorStatus),
 188                         be64_to_cpu(data->dma0FirstErrorStatus),
 189                         be64_to_cpu(data->dma0ErrorLog0),
 190                         be64_to_cpu(data->dma0ErrorLog1));
 191         if (data->dma1ErrorStatus)
 192                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 193                         be64_to_cpu(data->dma1ErrorStatus),
 194                         be64_to_cpu(data->dma1FirstErrorStatus),
 195                         be64_to_cpu(data->dma1ErrorLog0),
 196                         be64_to_cpu(data->dma1ErrorLog1));
 197
 198         for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
 199                 if ((data->pestA[i] >> 63) == 0 &&
 200                     (data->pestB[i] >> 63) == 0)
 201                         continue;
 202
 203                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 204                         i, be64_to_cpu(data->pestA[i]),
 205                         be64_to_cpu(data->pestB[i]));
 206         }
 207 }
 208
 209 static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
 210                                         struct OpalIoPhbErrorCommon *common)
 211 {
 212         struct OpalIoPhb3ErrorData *data;
 213         int i;
 214
 215         data = (struct OpalIoPhb3ErrorData*)common;
 216         pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
 217                 hose->global_number, be32_to_cpu(common->version));
 218         if (data->brdgCtl)
 219                 pr_info("brdgCtl:     %08x\n",
 220                         be32_to_cpu(data->brdgCtl));
 221         if (data->portStatusReg || data->rootCmplxStatus ||
 222             data->busAgentStatus)
 223                 pr_info("UtlSts:      %08x %08x %08x\n",
 224                         be32_to_cpu(data->portStatusReg),
 225                         be32_to_cpu(data->rootCmplxStatus),
 226                         be32_to_cpu(data->busAgentStatus));
 227         if (data->deviceStatus || data->slotStatus   ||
 228             data->linkStatus   || data->devCmdStatus ||
 229             data->devSecStatus)
 230                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
 231                         be32_to_cpu(data->deviceStatus),
 232                         be32_to_cpu(data->slotStatus),
 233                         be32_to_cpu(data->linkStatus),
 234                         be32_to_cpu(data->devCmdStatus),
 235                         be32_to_cpu(data->devSecStatus));
 236         if (data->rootErrorStatus || data->uncorrErrorStatus ||
 237             data->corrErrorStatus)
 238                 pr_info("RootErrSts:  %08x %08x %08x\n",
 239                         be32_to_cpu(data->rootErrorStatus),
 240                         be32_to_cpu(data->uncorrErrorStatus),
 241                         be32_to_cpu(data->corrErrorStatus));
 242         if (data->tlpHdr1 || data->tlpHdr2 ||
 243             data->tlpHdr3 || data->tlpHdr4)
 244                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
 245                         be32_to_cpu(data->tlpHdr1),
 246                         be32_to_cpu(data->tlpHdr2),
 247                         be32_to_cpu(data->tlpHdr3),
 248                         be32_to_cpu(data->tlpHdr4));
 249         if (data->sourceId || data->errorClass ||
 250             data->correlator)
 251                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
 252                         be32_to_cpu(data->sourceId),
 253                         be64_to_cpu(data->errorClass),
 254                         be64_to_cpu(data->correlator));
 255         if (data->nFir)
 256                 pr_info("nFir:        %016llx %016llx %016llx\n",
 257                         be64_to_cpu(data->nFir),
 258                         be64_to_cpu(data->nFirMask),
 259                         be64_to_cpu(data->nFirWOF));
 260         if (data->phbPlssr || data->phbCsr)
 261                 pr_info("PhbSts:      %016llx %016llx\n",
 262                         be64_to_cpu(data->phbPlssr),
 263                         be64_to_cpu(data->phbCsr));
 264         if (data->lemFir)
 265                 pr_info("Lem:         %016llx %016llx %016llx\n",
 266                         be64_to_cpu(data->lemFir),
 267                         be64_to_cpu(data->lemErrorMask),
 268                         be64_to_cpu(data->lemWOF));
 269         if (data->phbErrorStatus)
 270                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
 271                         be64_to_cpu(data->phbErrorStatus),
 272                         be64_to_cpu(data->phbFirstErrorStatus),
 273                         be64_to_cpu(data->phbErrorLog0),
 274                         be64_to_cpu(data->phbErrorLog1));
 275         if (data->mmioErrorStatus)
 276                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
 277                         be64_to_cpu(data->mmioErrorStatus),
 278                         be64_to_cpu(data->mmioFirstErrorStatus),
 279                         be64_to_cpu(data->mmioErrorLog0),
 280                         be64_to_cpu(data->mmioErrorLog1));
 281         if (data->dma0ErrorStatus)
 282                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
 283                         be64_to_cpu(data->dma0ErrorStatus),
 284                         be64_to_cpu(data->dma0FirstErrorStatus),
 285                         be64_to_cpu(data->dma0ErrorLog0),
 286                         be64_to_cpu(data->dma0ErrorLog1));
 287         if (data->dma1ErrorStatus)
 288                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
 289                         be64_to_cpu(data->dma1ErrorStatus),
 290                         be64_to_cpu(data->dma1FirstErrorStatus),
 291                         be64_to_cpu(data->dma1ErrorLog0),
 292                         be64_to_cpu(data->dma1ErrorLog1));
 293
 294         for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
 295                 if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
 296                     (be64_to_cpu(data->pestB[i]) >> 63) == 0)
 297                         continue;
 298
 299                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
 300                                 i, be64_to_cpu(data->pestA[i]),
 301                                 be64_to_cpu(data->pestB[i]));
 302         }
 303 }
 304
 305 void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
 306                                 unsigned char *log_buff)
 307 {
 308         struct OpalIoPhbErrorCommon *common;
 309
 310         if (!hose || !log_buff)
 311                 return;
 312
 313         common = (struct OpalIoPhbErrorCommon *)log_buff;
 314         switch (be32_to_cpu(common->ioType)) {
 315         case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
 316                 pnv_pci_dump_p7ioc_diag_data(hose, common);
 317                 break;
 318         case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
 319                 pnv_pci_dump_phb3_diag_data(hose, common);
 320                 break;
 321         default:
 322                 pr_warn("%s: Unrecognized ioType %d\n",
 323                         __func__, be32_to_cpu(common->ioType));
 324         }
 325 }
 326
 327 static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
 328 {
 329         unsigned long flags, rc;
 330         int has_diag, ret = 0;
 331
 332         spin_lock_irqsave(&phb->lock, flags);
 333
 334         /* Fetch PHB diag-data */
 335         rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
 336                                          PNV_PCI_DIAG_BUF_SIZE);
 337         has_diag = (rc == OPAL_SUCCESS);
 338
 339         /* If PHB supports compound PE, to handle it */
 340         if (phb->unfreeze_pe) {
 341                 ret = phb->unfreeze_pe(phb,
 342                                        pe_no,
 343                                        OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 344         } else {
 345                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
 346                                              pe_no,
 347                                              OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 348                 if (rc) {
 349                         pr_warn("%s: Failure %ld clearing frozen "
 350                                 "PHB#%x-PE#%x\n",
 351                                 __func__, rc, phb->hose->global_number,
 352                                 pe_no);
 353                         ret = -EIO;
 354                 }
 355         }
 356
 357         /*
 358          * For now, let's only display the diag buffer when we fail to clear
 359          * the EEH status. We'll do more sensible things later when we have
 360          * proper EEH support. We need to make sure we don't pollute ourselves
 361          * with the normal errors generated when probing empty slots
 362          */
 363         if (has_diag && ret)
 364                 pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);
 365
 366         spin_unlock_irqrestore(&phb->lock, flags);
 367 }
 368
 369 static void pnv_pci_config_check_eeh(struct pci_dn *pdn)
 370 {
 371         struct pnv_phb *phb = pdn->phb->private_data;
 372         u8      fstate;
 373         __be16  pcierr;
 374         int     pe_no;
 375         s64     rc;
 376
 377         /*
 378          * Get the PE#. During the PCI probe stage, we might not
 379          * setup that yet. So all ER errors should be mapped to
 380          * reserved PE.
 381          */
 382         pe_no = pdn->pe_number;
 383         if (pe_no == IODA_INVALID_PE) {
 384                 if (phb->type == PNV_PHB_P5IOC2)
 385                         pe_no = 0;
 386                 else
 387                         pe_no = phb->ioda.reserved_pe;
 388         }
 389
 390         /*
 391          * Fetch frozen state. If the PHB support compound PE,
 392          * we need handle that case.
 393          */
 394         if (phb->get_pe_state) {
 395                 fstate = phb->get_pe_state(phb, pe_no);
 396         } else {
 397                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
 398                                                 pe_no,
 399                                                 &fstate,
 400                                                 &pcierr,
 401                                                 NULL);
 402                 if (rc) {
 403                         pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
 404                                 __func__, rc, phb->hose->global_number, pe_no);
 405                         return;
 406                 }
 407         }
 408
 409         cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
 410                 (pdn->busno << 8) | (pdn->devfn), pe_no, fstate);
 411
 412         /* Clear the frozen state if applicable */
 413         if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
 414             fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
 415             fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
 416                 /*
 417                  * If PHB supports compound PE, freeze it for
 418                  * consistency.
 419                  */
 420                 if (phb->freeze_pe)
 421                         phb->freeze_pe(phb, pe_no);
 422
 423                 pnv_pci_handle_eeh_config(phb, pe_no);
 424         }
 425 }
 426
 427 int pnv_pci_cfg_read(struct pci_dn *pdn,
 428                      int where, int size, u32 *val)
 429 {
 430         struct pnv_phb *phb = pdn->phb->private_data;
 431         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 432         s64 rc;
 433
 434         switch (size) {
 435         case 1: {
 436                 u8 v8;
 437                 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
 438                 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
 439                 break;
 440         }
 441         case 2: {
 442                 __be16 v16;
 443                 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
 444                                                    &v16);
 445                 *val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
 446                 break;
 447         }
 448         case 4: {
 449                 __be32 v32;
 450                 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
 451                 *val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
 452                 break;
 453         }
 454         default:
 455                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 456         }
 457
 458         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 459                 __func__, pdn->busno, pdn->devfn, where, size, *val);
 460         return PCIBIOS_SUCCESSFUL;
 461 }
 462
 463 int pnv_pci_cfg_write(struct pci_dn *pdn,
 464                       int where, int size, u32 val)
 465 {
 466         struct pnv_phb *phb = pdn->phb->private_data;
 467         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
 468
 469         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
 470                 pdn->busno, pdn->devfn, where, size, val);
 471         switch (size) {
 472         case 1:
 473                 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
 474                 break;
 475         case 2:
 476                 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
 477                 break;
 478         case 4:
 479                 opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
 480                 break;
 481         default:
 482                 return PCIBIOS_FUNC_NOT_SUPPORTED;
 483         }
 484
 485         return PCIBIOS_SUCCESSFUL;
 486 }
 487
 488 #if CONFIG_EEH
 489 static bool pnv_pci_cfg_check(struct pci_dn *pdn)
 490 {
 491         struct eeh_dev *edev = NULL;
 492         struct pnv_phb *phb = pdn->phb->private_data;
 493
 494         /* EEH not enabled ? */
 495         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 496                 return true;
 497
 498         /* PE reset or device removed ? */
 499         edev = pdn->edev;
 500         if (edev) {
 501                 if (edev->pe &&
 502                     (edev->pe->state & EEH_PE_CFG_BLOCKED))
 503                         return false;
 504
 505                 if (edev->mode & EEH_DEV_REMOVED)
 506                         return false;
 507         }
 508
 509         return true;
 510 }
 511 #else
 512 static inline pnv_pci_cfg_check(struct pci_dn *pdn)
 513 {
 514         return true;
 515 }
 516 #endif /* CONFIG_EEH */
 517
 518 static int pnv_pci_read_config(struct pci_bus *bus,
 519                                unsigned int devfn,
 520                                int where, int size, u32 *val)
 521 {
 522         struct pci_dn *pdn;
 523         struct pnv_phb *phb;
 524         int ret;
 525
 526         *val = 0xFFFFFFFF;
 527         pdn = pci_get_pdn_by_devfn(bus, devfn);
 528         if (!pdn)
 529                 return PCIBIOS_DEVICE_NOT_FOUND;
 530
 531         if (!pnv_pci_cfg_check(pdn))
 532                 return PCIBIOS_DEVICE_NOT_FOUND;
 533
 534         ret = pnv_pci_cfg_read(pdn, where, size, val);
 535         phb = pdn->phb->private_data;
 536         if (phb->flags & PNV_PHB_FLAG_EEH && pdn->edev) {
 537                 if (*val == EEH_IO_ERROR_VALUE(size) &&
 538                     eeh_dev_check_failure(pdn->edev))
 539                         return PCIBIOS_DEVICE_NOT_FOUND;
 540         } else {
 541                 pnv_pci_config_check_eeh(pdn);
 542         }
 543
 544         return ret;
 545 }
 546
 547 static int pnv_pci_write_config(struct pci_bus *bus,
 548                                 unsigned int devfn,
 549                                 int where, int size, u32 val)
 550 {
 551         struct pci_dn *pdn;
 552         struct pnv_phb *phb;
 553         int ret;
 554
 555         pdn = pci_get_pdn_by_devfn(bus, devfn);
 556         if (!pdn)
 557                 return PCIBIOS_DEVICE_NOT_FOUND;
 558
 559         if (!pnv_pci_cfg_check(pdn))
 560                 return PCIBIOS_DEVICE_NOT_FOUND;
 561
 562         ret = pnv_pci_cfg_write(pdn, where, size, val);
 563         phb = pdn->phb->private_data;
 564         if (!(phb->flags & PNV_PHB_FLAG_EEH))
 565                 pnv_pci_config_check_eeh(pdn);
 566
 567         return ret;
 568 }
 569
 570 struct pci_ops pnv_pci_ops = {
 571         .read  = pnv_pci_read_config,
 572         .write = pnv_pci_write_config,
 573 };
 574
 575 static int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
 576                          unsigned long uaddr, enum dma_data_direction direction,
 577                          struct dma_attrs *attrs, bool rm)
 578 {
 579         u64 proto_tce;
 580         __be64 *tcep, *tces;
 581         u64 rpn;
 582
 583         proto_tce = TCE_PCI_READ; // Read allowed
 584
 585         if (direction != DMA_TO_DEVICE)
 586                 proto_tce |= TCE_PCI_WRITE;
 587
 588         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 589         rpn = __pa(uaddr) >> tbl->it_page_shift;
 590
 591         while (npages--)
 592                 *(tcep++) = cpu_to_be64(proto_tce |
 593                                 (rpn++ << tbl->it_page_shift));
 594
 595         /* Some implementations won't cache invalid TCEs and thus may not
 596          * need that flush. We'll probably turn it_type into a bit mask
 597          * of flags if that becomes the case
 598          */
 599         if (tbl->it_type & TCE_PCI_SWINV_CREATE)
 600                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 601
 602         return 0;
 603 }
 604
 605 static int pnv_tce_build_vm(struct iommu_table *tbl, long index, long npages,
 606                             unsigned long uaddr,
 607                             enum dma_data_direction direction,
 608                             struct dma_attrs *attrs)
 609 {
 610         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs,
 611                         false);
 612 }
 613
 614 static void pnv_tce_free(struct iommu_table *tbl, long index, long npages,
 615                 bool rm)
 616 {
 617         __be64 *tcep, *tces;
 618
 619         tces = tcep = ((__be64 *)tbl->it_base) + index - tbl->it_offset;
 620
 621         while (npages--)
 622                 *(tcep++) = cpu_to_be64(0);
 623
 624         if (tbl->it_type & TCE_PCI_SWINV_FREE)
 625                 pnv_pci_ioda_tce_invalidate(tbl, tces, tcep - 1, rm);
 626 }
 627
 628 static void pnv_tce_free_vm(struct iommu_table *tbl, long index, long npages)
 629 {
 630         pnv_tce_free(tbl, index, npages, false);
 631 }
 632
 633 static unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
 634 {
 635         return ((u64 *)tbl->it_base)[index - tbl->it_offset];
 636 }
 637
 638 static int pnv_tce_build_rm(struct iommu_table *tbl, long index, long npages,
 639                             unsigned long uaddr,
 640                             enum dma_data_direction direction,
 641                             struct dma_attrs *attrs)
 642 {
 643         return pnv_tce_build(tbl, index, npages, uaddr, direction, attrs, true);
 644 }
 645
 646 static void pnv_tce_free_rm(struct iommu_table *tbl, long index, long npages)
 647 {
 648         pnv_tce_free(tbl, index, npages, true);
 649 }
 650
 651 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
 652                                void *tce_mem, u64 tce_size,
 653                                u64 dma_offset, unsigned page_shift)
 654 {
 655         tbl->it_blocksize = 16;
 656         tbl->it_base = (unsigned long)tce_mem;
 657         tbl->it_page_shift = page_shift;
 658         tbl->it_offset = dma_offset >> tbl->it_page_shift;
 659         tbl->it_index = 0;
 660         tbl->it_size = tce_size >> 3;
 661         tbl->it_busno = 0;
 662         tbl->it_type = TCE_PCI;
 663 }
 664
 665 static void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
 666 {
 667         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 668         struct pnv_phb *phb = hose->private_data;
 669 #ifdef CONFIG_PCI_IOV
 670         struct pnv_ioda_pe *pe;
 671         struct pci_dn *pdn;
 672
 673         /* Fix the VF pdn PE number */
 674         if (pdev->is_virtfn) {
 675                 pdn = pci_get_pdn(pdev);
 676                 WARN_ON(pdn->pe_number != IODA_INVALID_PE);
 677                 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
 678                         if (pe->rid == ((pdev->bus->number << 8) |
 679                             (pdev->devfn & 0xff))) {
 680                                 pdn->pe_number = pe->pe_number;
 681                                 pe->pdev = pdev;
 682                                 break;
 683                         }
 684                 }
 685         }
 686 #endif /* CONFIG_PCI_IOV */
 687
 688         if (phb && phb->dma_dev_setup)
 689                 phb->dma_dev_setup(phb, pdev);
 690 }
 691
 692 int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask)
 693 {
 694         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 695         struct pnv_phb *phb = hose->private_data;
 696
 697         if (phb && phb->dma_set_mask)
 698                 return phb->dma_set_mask(phb, pdev, dma_mask);
 699         return __dma_set_mask(&pdev->dev, dma_mask);
 700 }
 701
 702 u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev)
 703 {
 704         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 705         struct pnv_phb *phb = hose->private_data;
 706
 707         if (phb && phb->dma_get_required_mask)
 708                 return phb->dma_get_required_mask(phb, pdev);
 709
 710         return __dma_get_required_mask(&pdev->dev);
 711 }
 712
 713 void pnv_pci_shutdown(void)
 714 {
 715         struct pci_controller *hose;
 716
 717         list_for_each_entry(hose, &hose_list, list_node) {
 718                 struct pnv_phb *phb = hose->private_data;
 719
 720                 if (phb && phb->shutdown)
 721                         phb->shutdown(phb);
 722         }
 723 }
 724
 725 /* Fixup wrong class code in p7ioc and p8 root complex */
 726 static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
 727 {
 728         dev->class = PCI_CLASS_BRIDGE_PCI << 8;
 729 }
 730 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
 731
 732 void __init pnv_pci_init(void)
 733 {
 734         struct device_node *np;
 735         bool found_ioda = false;
 736
 737         pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
 738
 739         /* If we don't have OPAL, eg. in sim, just skip PCI probe */
 740         if (!firmware_has_feature(FW_FEATURE_OPAL))
 741                 return;
 742
 743         /* Look for IODA IO-Hubs. We don't support mixing IODA
 744          * and p5ioc2 due to the need to change some global
 745          * probing flags
 746          */
 747         for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
 748                 pnv_pci_init_ioda_hub(np);
 749                 found_ioda = true;
 750         }
 751
 752         /* Look for p5ioc2 IO-Hubs */
 753         if (!found_ioda)
 754                 for_each_compatible_node(np, NULL, "ibm,p5ioc2")
 755                         pnv_pci_init_p5ioc2_hub(np);
 756
 757         /* Look for ioda2 built-in PHB3's */
 758         for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
 759                 pnv_pci_init_ioda2_phb(np);
 760
 761         /* Setup the linkage between OF nodes and PHBs */
 762         pci_devs_phb_init();
 763
 764         /* Configure IOMMU DMA hooks */
 765         ppc_md.tce_build = pnv_tce_build_vm;
 766         ppc_md.tce_free = pnv_tce_free_vm;
 767         ppc_md.tce_build_rm = pnv_tce_build_rm;
 768         ppc_md.tce_free_rm = pnv_tce_free_rm;
 769         ppc_md.tce_get = pnv_tce_get;
 770         set_pci_dma_ops(&dma_iommu_ops);
 771
 772         /* Configure MSIs */
 773 #ifdef CONFIG_PCI_MSI
 774         ppc_md.setup_msi_irqs = pnv_setup_msi_irqs;
 775         ppc_md.teardown_msi_irqs = pnv_teardown_msi_irqs;
 776 #endif
 777 }
 778
 779 machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);
 780
 781 struct pci_controller_ops pnv_pci_controller_ops = {
 782         .dma_dev_setup = pnv_pci_dma_dev_setup,
 783 };