~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/platforms/powernv/pci.c

Version: ~ [ linux-5.6-rc7 ] ~ [ linux-5.5.11 ] ~ [ linux-5.4.27 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.112 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.174 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.217 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.217 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.82 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Support PCI/PCIe on PowerNV platforms
  3  *
  4  * Copyright 2011 Benjamin Herrenschmidt, IBM Corp.
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License
  8  * as published by the Free Software Foundation; either version
  9  * 2 of the License, or (at your option) any later version.
 10  */
 11 
 12 #include <linux/kernel.h>
 13 #include <linux/pci.h>
 14 #include <linux/delay.h>
 15 #include <linux/string.h>
 16 #include <linux/init.h>
 17 #include <linux/irq.h>
 18 #include <linux/io.h>
 19 #include <linux/msi.h>
 20 #include <linux/iommu.h>
 21 
 22 #include <asm/sections.h>
 23 #include <asm/io.h>
 24 #include <asm/prom.h>
 25 #include <asm/pci-bridge.h>
 26 #include <asm/machdep.h>
 27 #include <asm/msi_bitmap.h>
 28 #include <asm/ppc-pci.h>
 29 #include <asm/opal.h>
 30 #include <asm/iommu.h>
 31 #include <asm/tce.h>
 32 #include <asm/firmware.h>
 33 #include <asm/eeh_event.h>
 34 #include <asm/eeh.h>
 35 
 36 #include "powernv.h"
 37 #include "pci.h"
 38 
 39 /* Delay in usec */
 40 #define PCI_RESET_DELAY_US      3000000
 41 
 42 #define cfg_dbg(fmt...) do { } while(0)
 43 //#define cfg_dbg(fmt...)       printk(fmt)
 44 
 45 #ifdef CONFIG_PCI_MSI
 46 int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type)
 47 {
 48         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 49         struct pnv_phb *phb = hose->private_data;
 50         struct msi_desc *entry;
 51         struct msi_msg msg;
 52         int hwirq;
 53         unsigned int virq;
 54         int rc;
 55 
 56         if (WARN_ON(!phb) || !phb->msi_bmp.bitmap)
 57                 return -ENODEV;
 58 
 59         if (pdev->no_64bit_msi && !phb->msi32_support)
 60                 return -ENODEV;
 61 
 62         for_each_pci_msi_entry(entry, pdev) {
 63                 if (!entry->msi_attrib.is_64 && !phb->msi32_support) {
 64                         pr_warn("%s: Supports only 64-bit MSIs\n",
 65                                 pci_name(pdev));
 66                         return -ENXIO;
 67                 }
 68                 hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, 1);
 69                 if (hwirq < 0) {
 70                         pr_warn("%s: Failed to find a free MSI\n",
 71                                 pci_name(pdev));
 72                         return -ENOSPC;
 73                 }
 74                 virq = irq_create_mapping(NULL, phb->msi_base + hwirq);
 75                 if (virq == NO_IRQ) {
 76                         pr_warn("%s: Failed to map MSI to linux irq\n",
 77                                 pci_name(pdev));
 78                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
 79                         return -ENOMEM;
 80                 }
 81                 rc = phb->msi_setup(phb, pdev, phb->msi_base + hwirq,
 82                                     virq, entry->msi_attrib.is_64, &msg);
 83                 if (rc) {
 84                         pr_warn("%s: Failed to setup MSI\n", pci_name(pdev));
 85                         irq_dispose_mapping(virq);
 86                         msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, 1);
 87                         return rc;
 88                 }
 89                 irq_set_msi_desc(virq, entry);
 90                 pci_write_msi_msg(virq, &msg);
 91         }
 92         return 0;
 93 }
 94 
 95 void pnv_teardown_msi_irqs(struct pci_dev *pdev)
 96 {
 97         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
 98         struct pnv_phb *phb = hose->private_data;
 99         struct msi_desc *entry;
100         irq_hw_number_t hwirq;
101 
102         if (WARN_ON(!phb))
103                 return;
104 
105         for_each_pci_msi_entry(entry, pdev) {
106                 if (entry->irq == NO_IRQ)
107                         continue;
108                 hwirq = virq_to_hw(entry->irq);
109                 irq_set_msi_desc(entry->irq, NULL);
110                 irq_dispose_mapping(entry->irq);
111                 msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, 1);
112         }
113 }
114 #endif /* CONFIG_PCI_MSI */
115 
116 static void pnv_pci_dump_p7ioc_diag_data(struct pci_controller *hose,
117                                          struct OpalIoPhbErrorCommon *common)
118 {
119         struct OpalIoP7IOCPhbErrorData *data;
120         int i;
121 
122         data = (struct OpalIoP7IOCPhbErrorData *)common;
123         pr_info("P7IOC PHB#%d Diag-data (Version: %d)\n",
124                 hose->global_number, be32_to_cpu(common->version));
125 
126         if (data->brdgCtl)
127                 pr_info("brdgCtl:     %08x\n",
128                         be32_to_cpu(data->brdgCtl));
129         if (data->portStatusReg || data->rootCmplxStatus ||
130             data->busAgentStatus)
131                 pr_info("UtlSts:      %08x %08x %08x\n",
132                         be32_to_cpu(data->portStatusReg),
133                         be32_to_cpu(data->rootCmplxStatus),
134                         be32_to_cpu(data->busAgentStatus));
135         if (data->deviceStatus || data->slotStatus   ||
136             data->linkStatus   || data->devCmdStatus ||
137             data->devSecStatus)
138                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
139                         be32_to_cpu(data->deviceStatus),
140                         be32_to_cpu(data->slotStatus),
141                         be32_to_cpu(data->linkStatus),
142                         be32_to_cpu(data->devCmdStatus),
143                         be32_to_cpu(data->devSecStatus));
144         if (data->rootErrorStatus   || data->uncorrErrorStatus ||
145             data->corrErrorStatus)
146                 pr_info("RootErrSts:  %08x %08x %08x\n",
147                         be32_to_cpu(data->rootErrorStatus),
148                         be32_to_cpu(data->uncorrErrorStatus),
149                         be32_to_cpu(data->corrErrorStatus));
150         if (data->tlpHdr1 || data->tlpHdr2 ||
151             data->tlpHdr3 || data->tlpHdr4)
152                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
153                         be32_to_cpu(data->tlpHdr1),
154                         be32_to_cpu(data->tlpHdr2),
155                         be32_to_cpu(data->tlpHdr3),
156                         be32_to_cpu(data->tlpHdr4));
157         if (data->sourceId || data->errorClass ||
158             data->correlator)
159                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
160                         be32_to_cpu(data->sourceId),
161                         be64_to_cpu(data->errorClass),
162                         be64_to_cpu(data->correlator));
163         if (data->p7iocPlssr || data->p7iocCsr)
164                 pr_info("PhbSts:      %016llx %016llx\n",
165                         be64_to_cpu(data->p7iocPlssr),
166                         be64_to_cpu(data->p7iocCsr));
167         if (data->lemFir)
168                 pr_info("Lem:         %016llx %016llx %016llx\n",
169                         be64_to_cpu(data->lemFir),
170                         be64_to_cpu(data->lemErrorMask),
171                         be64_to_cpu(data->lemWOF));
172         if (data->phbErrorStatus)
173                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
174                         be64_to_cpu(data->phbErrorStatus),
175                         be64_to_cpu(data->phbFirstErrorStatus),
176                         be64_to_cpu(data->phbErrorLog0),
177                         be64_to_cpu(data->phbErrorLog1));
178         if (data->mmioErrorStatus)
179                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
180                         be64_to_cpu(data->mmioErrorStatus),
181                         be64_to_cpu(data->mmioFirstErrorStatus),
182                         be64_to_cpu(data->mmioErrorLog0),
183                         be64_to_cpu(data->mmioErrorLog1));
184         if (data->dma0ErrorStatus)
185                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
186                         be64_to_cpu(data->dma0ErrorStatus),
187                         be64_to_cpu(data->dma0FirstErrorStatus),
188                         be64_to_cpu(data->dma0ErrorLog0),
189                         be64_to_cpu(data->dma0ErrorLog1));
190         if (data->dma1ErrorStatus)
191                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
192                         be64_to_cpu(data->dma1ErrorStatus),
193                         be64_to_cpu(data->dma1FirstErrorStatus),
194                         be64_to_cpu(data->dma1ErrorLog0),
195                         be64_to_cpu(data->dma1ErrorLog1));
196 
197         for (i = 0; i < OPAL_P7IOC_NUM_PEST_REGS; i++) {
198                 if ((data->pestA[i] >> 63) == 0 &&
199                     (data->pestB[i] >> 63) == 0)
200                         continue;
201 
202                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
203                         i, be64_to_cpu(data->pestA[i]),
204                         be64_to_cpu(data->pestB[i]));
205         }
206 }
207 
208 static void pnv_pci_dump_phb3_diag_data(struct pci_controller *hose,
209                                         struct OpalIoPhbErrorCommon *common)
210 {
211         struct OpalIoPhb3ErrorData *data;
212         int i;
213 
214         data = (struct OpalIoPhb3ErrorData*)common;
215         pr_info("PHB3 PHB#%d Diag-data (Version: %d)\n",
216                 hose->global_number, be32_to_cpu(common->version));
217         if (data->brdgCtl)
218                 pr_info("brdgCtl:     %08x\n",
219                         be32_to_cpu(data->brdgCtl));
220         if (data->portStatusReg || data->rootCmplxStatus ||
221             data->busAgentStatus)
222                 pr_info("UtlSts:      %08x %08x %08x\n",
223                         be32_to_cpu(data->portStatusReg),
224                         be32_to_cpu(data->rootCmplxStatus),
225                         be32_to_cpu(data->busAgentStatus));
226         if (data->deviceStatus || data->slotStatus   ||
227             data->linkStatus   || data->devCmdStatus ||
228             data->devSecStatus)
229                 pr_info("RootSts:     %08x %08x %08x %08x %08x\n",
230                         be32_to_cpu(data->deviceStatus),
231                         be32_to_cpu(data->slotStatus),
232                         be32_to_cpu(data->linkStatus),
233                         be32_to_cpu(data->devCmdStatus),
234                         be32_to_cpu(data->devSecStatus));
235         if (data->rootErrorStatus || data->uncorrErrorStatus ||
236             data->corrErrorStatus)
237                 pr_info("RootErrSts:  %08x %08x %08x\n",
238                         be32_to_cpu(data->rootErrorStatus),
239                         be32_to_cpu(data->uncorrErrorStatus),
240                         be32_to_cpu(data->corrErrorStatus));
241         if (data->tlpHdr1 || data->tlpHdr2 ||
242             data->tlpHdr3 || data->tlpHdr4)
243                 pr_info("RootErrLog:  %08x %08x %08x %08x\n",
244                         be32_to_cpu(data->tlpHdr1),
245                         be32_to_cpu(data->tlpHdr2),
246                         be32_to_cpu(data->tlpHdr3),
247                         be32_to_cpu(data->tlpHdr4));
248         if (data->sourceId || data->errorClass ||
249             data->correlator)
250                 pr_info("RootErrLog1: %08x %016llx %016llx\n",
251                         be32_to_cpu(data->sourceId),
252                         be64_to_cpu(data->errorClass),
253                         be64_to_cpu(data->correlator));
254         if (data->nFir)
255                 pr_info("nFir:        %016llx %016llx %016llx\n",
256                         be64_to_cpu(data->nFir),
257                         be64_to_cpu(data->nFirMask),
258                         be64_to_cpu(data->nFirWOF));
259         if (data->phbPlssr || data->phbCsr)
260                 pr_info("PhbSts:      %016llx %016llx\n",
261                         be64_to_cpu(data->phbPlssr),
262                         be64_to_cpu(data->phbCsr));
263         if (data->lemFir)
264                 pr_info("Lem:         %016llx %016llx %016llx\n",
265                         be64_to_cpu(data->lemFir),
266                         be64_to_cpu(data->lemErrorMask),
267                         be64_to_cpu(data->lemWOF));
268         if (data->phbErrorStatus)
269                 pr_info("PhbErr:      %016llx %016llx %016llx %016llx\n",
270                         be64_to_cpu(data->phbErrorStatus),
271                         be64_to_cpu(data->phbFirstErrorStatus),
272                         be64_to_cpu(data->phbErrorLog0),
273                         be64_to_cpu(data->phbErrorLog1));
274         if (data->mmioErrorStatus)
275                 pr_info("OutErr:      %016llx %016llx %016llx %016llx\n",
276                         be64_to_cpu(data->mmioErrorStatus),
277                         be64_to_cpu(data->mmioFirstErrorStatus),
278                         be64_to_cpu(data->mmioErrorLog0),
279                         be64_to_cpu(data->mmioErrorLog1));
280         if (data->dma0ErrorStatus)
281                 pr_info("InAErr:      %016llx %016llx %016llx %016llx\n",
282                         be64_to_cpu(data->dma0ErrorStatus),
283                         be64_to_cpu(data->dma0FirstErrorStatus),
284                         be64_to_cpu(data->dma0ErrorLog0),
285                         be64_to_cpu(data->dma0ErrorLog1));
286         if (data->dma1ErrorStatus)
287                 pr_info("InBErr:      %016llx %016llx %016llx %016llx\n",
288                         be64_to_cpu(data->dma1ErrorStatus),
289                         be64_to_cpu(data->dma1FirstErrorStatus),
290                         be64_to_cpu(data->dma1ErrorLog0),
291                         be64_to_cpu(data->dma1ErrorLog1));
292 
293         for (i = 0; i < OPAL_PHB3_NUM_PEST_REGS; i++) {
294                 if ((be64_to_cpu(data->pestA[i]) >> 63) == 0 &&
295                     (be64_to_cpu(data->pestB[i]) >> 63) == 0)
296                         continue;
297 
298                 pr_info("PE[%3d] A/B: %016llx %016llx\n",
299                                 i, be64_to_cpu(data->pestA[i]),
300                                 be64_to_cpu(data->pestB[i]));
301         }
302 }
303 
304 void pnv_pci_dump_phb_diag_data(struct pci_controller *hose,
305                                 unsigned char *log_buff)
306 {
307         struct OpalIoPhbErrorCommon *common;
308 
309         if (!hose || !log_buff)
310                 return;
311 
312         common = (struct OpalIoPhbErrorCommon *)log_buff;
313         switch (be32_to_cpu(common->ioType)) {
314         case OPAL_PHB_ERROR_DATA_TYPE_P7IOC:
315                 pnv_pci_dump_p7ioc_diag_data(hose, common);
316                 break;
317         case OPAL_PHB_ERROR_DATA_TYPE_PHB3:
318                 pnv_pci_dump_phb3_diag_data(hose, common);
319                 break;
320         default:
321                 pr_warn("%s: Unrecognized ioType %d\n",
322                         __func__, be32_to_cpu(common->ioType));
323         }
324 }
325 
326 static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no)
327 {
328         unsigned long flags, rc;
329         int has_diag, ret = 0;
330 
331         spin_lock_irqsave(&phb->lock, flags);
332 
333         /* Fetch PHB diag-data */
334         rc = opal_pci_get_phb_diag_data2(phb->opal_id, phb->diag.blob,
335                                          PNV_PCI_DIAG_BUF_SIZE);
336         has_diag = (rc == OPAL_SUCCESS);
337 
338         /* If PHB supports compound PE, to handle it */
339         if (phb->unfreeze_pe) {
340                 ret = phb->unfreeze_pe(phb,
341                                        pe_no,
342                                        OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
343         } else {
344                 rc = opal_pci_eeh_freeze_clear(phb->opal_id,
345                                              pe_no,
346                                              OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
347                 if (rc) {
348                         pr_warn("%s: Failure %ld clearing frozen "
349                                 "PHB#%x-PE#%x\n",
350                                 __func__, rc, phb->hose->global_number,
351                                 pe_no);
352                         ret = -EIO;
353                 }
354         }
355 
356         /*
357          * For now, let's only display the diag buffer when we fail to clear
358          * the EEH status. We'll do more sensible things later when we have
359          * proper EEH support. We need to make sure we don't pollute ourselves
360          * with the normal errors generated when probing empty slots
361          */
362         if (has_diag && ret)
363                 pnv_pci_dump_phb_diag_data(phb->hose, phb->diag.blob);
364 
365         spin_unlock_irqrestore(&phb->lock, flags);
366 }
367 
368 static void pnv_pci_config_check_eeh(struct pci_dn *pdn)
369 {
370         struct pnv_phb *phb = pdn->phb->private_data;
371         u8      fstate;
372         __be16  pcierr;
373         int     pe_no;
374         s64     rc;
375 
376         /*
377          * Get the PE#. During the PCI probe stage, we might not
378          * setup that yet. So all ER errors should be mapped to
379          * reserved PE.
380          */
381         pe_no = pdn->pe_number;
382         if (pe_no == IODA_INVALID_PE) {
383                 if (phb->type == PNV_PHB_P5IOC2)
384                         pe_no = 0;
385                 else
386                         pe_no = phb->ioda.reserved_pe;
387         }
388 
389         /*
390          * Fetch frozen state. If the PHB support compound PE,
391          * we need handle that case.
392          */
393         if (phb->get_pe_state) {
394                 fstate = phb->get_pe_state(phb, pe_no);
395         } else {
396                 rc = opal_pci_eeh_freeze_status(phb->opal_id,
397                                                 pe_no,
398                                                 &fstate,
399                                                 &pcierr,
400                                                 NULL);
401                 if (rc) {
402                         pr_warn("%s: Failure %lld getting PHB#%x-PE#%x state\n",
403                                 __func__, rc, phb->hose->global_number, pe_no);
404                         return;
405                 }
406         }
407 
408         cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n",
409                 (pdn->busno << 8) | (pdn->devfn), pe_no, fstate);
410 
411         /* Clear the frozen state if applicable */
412         if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE ||
413             fstate == OPAL_EEH_STOPPED_DMA_FREEZE  ||
414             fstate == OPAL_EEH_STOPPED_MMIO_DMA_FREEZE) {
415                 /*
416                  * If PHB supports compound PE, freeze it for
417                  * consistency.
418                  */
419                 if (phb->freeze_pe)
420                         phb->freeze_pe(phb, pe_no);
421 
422                 pnv_pci_handle_eeh_config(phb, pe_no);
423         }
424 }
425 
426 int pnv_pci_cfg_read(struct pci_dn *pdn,
427                      int where, int size, u32 *val)
428 {
429         struct pnv_phb *phb = pdn->phb->private_data;
430         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
431         s64 rc;
432 
433         switch (size) {
434         case 1: {
435                 u8 v8;
436                 rc = opal_pci_config_read_byte(phb->opal_id, bdfn, where, &v8);
437                 *val = (rc == OPAL_SUCCESS) ? v8 : 0xff;
438                 break;
439         }
440         case 2: {
441                 __be16 v16;
442                 rc = opal_pci_config_read_half_word(phb->opal_id, bdfn, where,
443                                                    &v16);
444                 *val = (rc == OPAL_SUCCESS) ? be16_to_cpu(v16) : 0xffff;
445                 break;
446         }
447         case 4: {
448                 __be32 v32;
449                 rc = opal_pci_config_read_word(phb->opal_id, bdfn, where, &v32);
450                 *val = (rc == OPAL_SUCCESS) ? be32_to_cpu(v32) : 0xffffffff;
451                 break;
452         }
453         default:
454                 return PCIBIOS_FUNC_NOT_SUPPORTED;
455         }
456 
457         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
458                 __func__, pdn->busno, pdn->devfn, where, size, *val);
459         return PCIBIOS_SUCCESSFUL;
460 }
461 
462 int pnv_pci_cfg_write(struct pci_dn *pdn,
463                       int where, int size, u32 val)
464 {
465         struct pnv_phb *phb = pdn->phb->private_data;
466         u32 bdfn = (pdn->busno << 8) | pdn->devfn;
467 
468         cfg_dbg("%s: bus: %x devfn: %x +%x/%x -> %08x\n",
469                 pdn->busno, pdn->devfn, where, size, val);
470         switch (size) {
471         case 1:
472                 opal_pci_config_write_byte(phb->opal_id, bdfn, where, val);
473                 break;
474         case 2:
475                 opal_pci_config_write_half_word(phb->opal_id, bdfn, where, val);
476                 break;
477         case 4:
478                 opal_pci_config_write_word(phb->opal_id, bdfn, where, val);
479                 break;
480         default:
481                 return PCIBIOS_FUNC_NOT_SUPPORTED;
482         }
483 
484         return PCIBIOS_SUCCESSFUL;
485 }
486 
487 #if CONFIG_EEH
488 static bool pnv_pci_cfg_check(struct pci_dn *pdn)
489 {
490         struct eeh_dev *edev = NULL;
491         struct pnv_phb *phb = pdn->phb->private_data;
492 
493         /* EEH not enabled ? */
494         if (!(phb->flags & PNV_PHB_FLAG_EEH))
495                 return true;
496 
497         /* PE reset or device removed ? */
498         edev = pdn->edev;
499         if (edev) {
500                 if (edev->pe &&
501                     (edev->pe->state & EEH_PE_CFG_BLOCKED))
502                         return false;
503 
504                 if (edev->mode & EEH_DEV_REMOVED)
505                         return false;
506         }
507 
508         return true;
509 }
510 #else
511 static inline pnv_pci_cfg_check(struct pci_dn *pdn)
512 {
513         return true;
514 }
515 #endif /* CONFIG_EEH */
516 
517 static int pnv_pci_read_config(struct pci_bus *bus,
518                                unsigned int devfn,
519                                int where, int size, u32 *val)
520 {
521         struct pci_dn *pdn;
522         struct pnv_phb *phb;
523         int ret;
524 
525         *val = 0xFFFFFFFF;
526         pdn = pci_get_pdn_by_devfn(bus, devfn);
527         if (!pdn)
528                 return PCIBIOS_DEVICE_NOT_FOUND;
529 
530         if (!pnv_pci_cfg_check(pdn))
531                 return PCIBIOS_DEVICE_NOT_FOUND;
532 
533         ret = pnv_pci_cfg_read(pdn, where, size, val);
534         phb = pdn->phb->private_data;
535         if (phb->flags & PNV_PHB_FLAG_EEH && pdn->edev) {
536                 if (*val == EEH_IO_ERROR_VALUE(size) &&
537                     eeh_dev_check_failure(pdn->edev))
538                         return PCIBIOS_DEVICE_NOT_FOUND;
539         } else {
540                 pnv_pci_config_check_eeh(pdn);
541         }
542 
543         return ret;
544 }
545 
546 static int pnv_pci_write_config(struct pci_bus *bus,
547                                 unsigned int devfn,
548                                 int where, int size, u32 val)
549 {
550         struct pci_dn *pdn;
551         struct pnv_phb *phb;
552         int ret;
553 
554         pdn = pci_get_pdn_by_devfn(bus, devfn);
555         if (!pdn)
556                 return PCIBIOS_DEVICE_NOT_FOUND;
557 
558         if (!pnv_pci_cfg_check(pdn))
559                 return PCIBIOS_DEVICE_NOT_FOUND;
560 
561         ret = pnv_pci_cfg_write(pdn, where, size, val);
562         phb = pdn->phb->private_data;
563         if (!(phb->flags & PNV_PHB_FLAG_EEH))
564                 pnv_pci_config_check_eeh(pdn);
565 
566         return ret;
567 }
568 
569 struct pci_ops pnv_pci_ops = {
570         .read  = pnv_pci_read_config,
571         .write = pnv_pci_write_config,
572 };
573 
574 static __be64 *pnv_tce(struct iommu_table *tbl, long idx)
575 {
576         __be64 *tmp = ((__be64 *)tbl->it_base);
577         int  level = tbl->it_indirect_levels;
578         const long shift = ilog2(tbl->it_level_size);
579         unsigned long mask = (tbl->it_level_size - 1) << (level * shift);
580 
581         while (level) {
582                 int n = (idx & mask) >> (level * shift);
583                 unsigned long tce = be64_to_cpu(tmp[n]);
584 
585                 tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE));
586                 idx &= ~mask;
587                 mask >>= shift;
588                 --level;
589         }
590 
591         return tmp + idx;
592 }
593 
594 int pnv_tce_build(struct iommu_table *tbl, long index, long npages,
595                 unsigned long uaddr, enum dma_data_direction direction,
596                 struct dma_attrs *attrs)
597 {
598         u64 proto_tce = iommu_direction_to_tce_perm(direction);
599         u64 rpn = __pa(uaddr) >> tbl->it_page_shift;
600         long i;
601 
602         if (proto_tce & TCE_PCI_WRITE)
603                 proto_tce |= TCE_PCI_READ;
604 
605         for (i = 0; i < npages; i++) {
606                 unsigned long newtce = proto_tce |
607                         ((rpn + i) << tbl->it_page_shift);
608                 unsigned long idx = index - tbl->it_offset + i;
609 
610                 *(pnv_tce(tbl, idx)) = cpu_to_be64(newtce);
611         }
612 
613         return 0;
614 }
615 
616 #ifdef CONFIG_IOMMU_API
617 int pnv_tce_xchg(struct iommu_table *tbl, long index,
618                 unsigned long *hpa, enum dma_data_direction *direction)
619 {
620         u64 proto_tce = iommu_direction_to_tce_perm(*direction);
621         unsigned long newtce = *hpa | proto_tce, oldtce;
622         unsigned long idx = index - tbl->it_offset;
623 
624         BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));
625 
626         if (newtce & TCE_PCI_WRITE)
627                 newtce |= TCE_PCI_READ;
628 
629         oldtce = xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce));
630         *hpa = be64_to_cpu(oldtce) & ~(TCE_PCI_READ | TCE_PCI_WRITE);
631         *direction = iommu_tce_direction(oldtce);
632 
633         return 0;
634 }
635 #endif
636 
637 void pnv_tce_free(struct iommu_table *tbl, long index, long npages)
638 {
639         long i;
640 
641         for (i = 0; i < npages; i++) {
642                 unsigned long idx = index - tbl->it_offset + i;
643 
644                 *(pnv_tce(tbl, idx)) = cpu_to_be64(0);
645         }
646 }
647 
648 unsigned long pnv_tce_get(struct iommu_table *tbl, long index)
649 {
650         return *(pnv_tce(tbl, index - tbl->it_offset));
651 }
652 
653 struct iommu_table *pnv_pci_table_alloc(int nid)
654 {
655         struct iommu_table *tbl;
656 
657         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, nid);
658         INIT_LIST_HEAD_RCU(&tbl->it_group_list);
659 
660         return tbl;
661 }
662 
663 long pnv_pci_link_table_and_group(int node, int num,
664                 struct iommu_table *tbl,
665                 struct iommu_table_group *table_group)
666 {
667         struct iommu_table_group_link *tgl = NULL;
668 
669         if (WARN_ON(!tbl || !table_group))
670                 return -EINVAL;
671 
672         tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL,
673                         node);
674         if (!tgl)
675                 return -ENOMEM;
676 
677         tgl->table_group = table_group;
678         list_add_rcu(&tgl->next, &tbl->it_group_list);
679 
680         table_group->tables[num] = tbl;
681 
682         return 0;
683 }
684 
685 static void pnv_iommu_table_group_link_free(struct rcu_head *head)
686 {
687         struct iommu_table_group_link *tgl = container_of(head,
688                         struct iommu_table_group_link, rcu);
689 
690         kfree(tgl);
691 }
692 
693 void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,
694                 struct iommu_table_group *table_group)
695 {
696         long i;
697         bool found;
698         struct iommu_table_group_link *tgl;
699 
700         if (!tbl || !table_group)
701                 return;
702 
703         /* Remove link to a group from table's list of attached groups */
704         found = false;
705         list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {
706                 if (tgl->table_group == table_group) {
707                         list_del_rcu(&tgl->next);
708                         call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free);
709                         found = true;
710                         break;
711                 }
712         }
713         if (WARN_ON(!found))
714                 return;
715 
716         /* Clean a pointer to iommu_table in iommu_table_group::tables[] */
717         found = false;
718         for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
719                 if (table_group->tables[i] == tbl) {
720                         table_group->tables[i] = NULL;
721                         found = true;
722                         break;
723                 }
724         }
725         WARN_ON(!found);
726 }
727 
728 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,
729                                void *tce_mem, u64 tce_size,
730                                u64 dma_offset, unsigned page_shift)
731 {
732         tbl->it_blocksize = 16;
733         tbl->it_base = (unsigned long)tce_mem;
734         tbl->it_page_shift = page_shift;
735         tbl->it_offset = dma_offset >> tbl->it_page_shift;
736         tbl->it_index = 0;
737         tbl->it_size = tce_size >> 3;
738         tbl->it_busno = 0;
739         tbl->it_type = TCE_PCI;
740 }
741 
742 void pnv_pci_dma_dev_setup(struct pci_dev *pdev)
743 {
744         struct pci_controller *hose = pci_bus_to_host(pdev->bus);
745         struct pnv_phb *phb = hose->private_data;
746 #ifdef CONFIG_PCI_IOV
747         struct pnv_ioda_pe *pe;
748         struct pci_dn *pdn;
749 
750         /* Fix the VF pdn PE number */
751         if (pdev->is_virtfn) {
752                 pdn = pci_get_pdn(pdev);
753                 WARN_ON(pdn->pe_number != IODA_INVALID_PE);
754                 list_for_each_entry(pe, &phb->ioda.pe_list, list) {
755                         if (pe->rid == ((pdev->bus->number << 8) |
756                             (pdev->devfn & 0xff))) {
757                                 pdn->pe_number = pe->pe_number;
758                                 pe->pdev = pdev;
759                                 break;
760                         }
761                 }
762         }
763 #endif /* CONFIG_PCI_IOV */
764 
765         if (phb && phb->dma_dev_setup)
766                 phb->dma_dev_setup(phb, pdev);
767 }
768 
769 void pnv_pci_dma_bus_setup(struct pci_bus *bus)
770 {
771         struct pci_controller *hose = bus->sysdata;
772         struct pnv_phb *phb = hose->private_data;
773         struct pnv_ioda_pe *pe;
774 
775         list_for_each_entry(pe, &phb->ioda.pe_list, list) {
776                 if (!(pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)))
777                         continue;
778 
779                 if (!pe->pbus)
780                         continue;
781 
782                 if (bus->number == ((pe->rid >> 8) & 0xFF)) {
783                         pe->pbus = bus;
784                         break;
785                 }
786         }
787 }
788 
789 void pnv_pci_shutdown(void)
790 {
791         struct pci_controller *hose;
792 
793         list_for_each_entry(hose, &hose_list, list_node)
794                 if (hose->controller_ops.shutdown)
795                         hose->controller_ops.shutdown(hose);
796 }
797 
798 /* Fixup wrong class code in p7ioc and p8 root complex */
799 static void pnv_p7ioc_rc_quirk(struct pci_dev *dev)
800 {
801         dev->class = PCI_CLASS_BRIDGE_PCI << 8;
802 }
803 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk);
804 
805 void __init pnv_pci_init(void)
806 {
807         struct device_node *np;
808         bool found_ioda = false;
809 
810         pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN);
811 
812         /* If we don't have OPAL, eg. in sim, just skip PCI probe */
813         if (!firmware_has_feature(FW_FEATURE_OPAL))
814                 return;
815 
816         /* Look for IODA IO-Hubs. We don't support mixing IODA
817          * and p5ioc2 due to the need to change some global
818          * probing flags
819          */
820         for_each_compatible_node(np, NULL, "ibm,ioda-hub") {
821                 pnv_pci_init_ioda_hub(np);
822                 found_ioda = true;
823         }
824 
825         /* Look for p5ioc2 IO-Hubs */
826         if (!found_ioda)
827                 for_each_compatible_node(np, NULL, "ibm,p5ioc2")
828                         pnv_pci_init_p5ioc2_hub(np);
829 
830         /* Look for ioda2 built-in PHB3's */
831         for_each_compatible_node(np, NULL, "ibm,ioda2-phb")
832                 pnv_pci_init_ioda2_phb(np);
833 
834         /* Look for NPU PHBs */
835         for_each_compatible_node(np, NULL, "ibm,ioda2-npu-phb")
836                 pnv_pci_init_npu_phb(np);
837 
838         /* Setup the linkage between OF nodes and PHBs */
839         pci_devs_phb_init();
840 
841         /* Configure IOMMU DMA hooks */
842         set_pci_dma_ops(&dma_iommu_ops);
843 }
844 
845 machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init);
846 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp