~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/platforms/powernv/npu-dma.c

Version: ~ [ linux-5.5-rc6 ] ~ [ linux-5.4.11 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.95 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.164 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.209 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.209 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * This file implements the DMA operations for NVLink devices. The NPU
  3  * devices all point to the same iommu table as the parent PCI device.
  4  *
  5  * Copyright Alistair Popple, IBM Corporation 2015.
  6  *
  7  * This program is free software; you can redistribute it and/or
  8  * modify it under the terms of version 2 of the GNU General Public
  9  * License as published by the Free Software Foundation.
 10  */
 11 
 12 #include <linux/slab.h>
 13 #include <linux/mmu_notifier.h>
 14 #include <linux/mmu_context.h>
 15 #include <linux/of.h>
 16 #include <linux/export.h>
 17 #include <linux/pci.h>
 18 #include <linux/memblock.h>
 19 #include <linux/iommu.h>
 20 
 21 #include <asm/tlb.h>
 22 #include <asm/powernv.h>
 23 #include <asm/reg.h>
 24 #include <asm/opal.h>
 25 #include <asm/io.h>
 26 #include <asm/iommu.h>
 27 #include <asm/pnv-pci.h>
 28 #include <asm/msi_bitmap.h>
 29 #include <asm/opal.h>
 30 
 31 #include "powernv.h"
 32 #include "pci.h"
 33 
 34 #define npu_to_phb(x) container_of(x, struct pnv_phb, npu)
 35 
 36 /*
 37  * spinlock to protect initialisation of an npu_context for a particular
 38  * mm_struct.
 39  */
 40 static DEFINE_SPINLOCK(npu_context_lock);
 41 
 42 /*
 43  * When an address shootdown range exceeds this threshold we invalidate the
 44  * entire TLB on the GPU for the given PID rather than each specific address in
 45  * the range.
 46  */
 47 #define ATSD_THRESHOLD (2*1024*1024)
 48 
 49 /*
 50  * Other types of TCE cache invalidation are not functional in the
 51  * hardware.
 52  */
 53 static struct pci_dev *get_pci_dev(struct device_node *dn)
 54 {
 55         struct pci_dn *pdn = PCI_DN(dn);
 56 
 57         return pci_get_domain_bus_and_slot(pci_domain_nr(pdn->phb->bus),
 58                                            pdn->busno, pdn->devfn);
 59 }
 60 
 61 /* Given a NPU device get the associated PCI device. */
 62 struct pci_dev *pnv_pci_get_gpu_dev(struct pci_dev *npdev)
 63 {
 64         struct device_node *dn;
 65         struct pci_dev *gpdev;
 66 
 67         if (WARN_ON(!npdev))
 68                 return NULL;
 69 
 70         if (WARN_ON(!npdev->dev.of_node))
 71                 return NULL;
 72 
 73         /* Get assoicated PCI device */
 74         dn = of_parse_phandle(npdev->dev.of_node, "ibm,gpu", 0);
 75         if (!dn)
 76                 return NULL;
 77 
 78         gpdev = get_pci_dev(dn);
 79         of_node_put(dn);
 80 
 81         return gpdev;
 82 }
 83 EXPORT_SYMBOL(pnv_pci_get_gpu_dev);
 84 
 85 /* Given the real PCI device get a linked NPU device. */
 86 struct pci_dev *pnv_pci_get_npu_dev(struct pci_dev *gpdev, int index)
 87 {
 88         struct device_node *dn;
 89         struct pci_dev *npdev;
 90 
 91         if (WARN_ON(!gpdev))
 92                 return NULL;
 93 
 94         /* Not all PCI devices have device-tree nodes */
 95         if (!gpdev->dev.of_node)
 96                 return NULL;
 97 
 98         /* Get assoicated PCI device */
 99         dn = of_parse_phandle(gpdev->dev.of_node, "ibm,npu", index);
100         if (!dn)
101                 return NULL;
102 
103         npdev = get_pci_dev(dn);
104         of_node_put(dn);
105 
106         return npdev;
107 }
108 EXPORT_SYMBOL(pnv_pci_get_npu_dev);
109 
110 #define NPU_DMA_OP_UNSUPPORTED()                                        \
111         dev_err_once(dev, "%s operation unsupported for NVLink devices\n", \
112                 __func__)
113 
114 static void *dma_npu_alloc(struct device *dev, size_t size,
115                            dma_addr_t *dma_handle, gfp_t flag,
116                            unsigned long attrs)
117 {
118         NPU_DMA_OP_UNSUPPORTED();
119         return NULL;
120 }
121 
122 static void dma_npu_free(struct device *dev, size_t size,
123                          void *vaddr, dma_addr_t dma_handle,
124                          unsigned long attrs)
125 {
126         NPU_DMA_OP_UNSUPPORTED();
127 }
128 
129 static dma_addr_t dma_npu_map_page(struct device *dev, struct page *page,
130                                    unsigned long offset, size_t size,
131                                    enum dma_data_direction direction,
132                                    unsigned long attrs)
133 {
134         NPU_DMA_OP_UNSUPPORTED();
135         return 0;
136 }
137 
138 static int dma_npu_map_sg(struct device *dev, struct scatterlist *sglist,
139                           int nelems, enum dma_data_direction direction,
140                           unsigned long attrs)
141 {
142         NPU_DMA_OP_UNSUPPORTED();
143         return 0;
144 }
145 
146 static int dma_npu_dma_supported(struct device *dev, u64 mask)
147 {
148         NPU_DMA_OP_UNSUPPORTED();
149         return 0;
150 }
151 
152 static u64 dma_npu_get_required_mask(struct device *dev)
153 {
154         NPU_DMA_OP_UNSUPPORTED();
155         return 0;
156 }
157 
158 static const struct dma_map_ops dma_npu_ops = {
159         .map_page               = dma_npu_map_page,
160         .map_sg                 = dma_npu_map_sg,
161         .alloc                  = dma_npu_alloc,
162         .free                   = dma_npu_free,
163         .dma_supported          = dma_npu_dma_supported,
164         .get_required_mask      = dma_npu_get_required_mask,
165 };
166 
167 /*
168  * Returns the PE assoicated with the PCI device of the given
169  * NPU. Returns the linked pci device if pci_dev != NULL.
170  */
171 static struct pnv_ioda_pe *get_gpu_pci_dev_and_pe(struct pnv_ioda_pe *npe,
172                                                   struct pci_dev **gpdev)
173 {
174         struct pnv_phb *phb;
175         struct pci_controller *hose;
176         struct pci_dev *pdev;
177         struct pnv_ioda_pe *pe;
178         struct pci_dn *pdn;
179 
180         pdev = pnv_pci_get_gpu_dev(npe->pdev);
181         if (!pdev)
182                 return NULL;
183 
184         pdn = pci_get_pdn(pdev);
185         if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
186                 return NULL;
187 
188         hose = pci_bus_to_host(pdev->bus);
189         phb = hose->private_data;
190         pe = &phb->ioda.pe_array[pdn->pe_number];
191 
192         if (gpdev)
193                 *gpdev = pdev;
194 
195         return pe;
196 }
197 
198 long pnv_npu_set_window(struct pnv_ioda_pe *npe, int num,
199                 struct iommu_table *tbl)
200 {
201         struct pnv_phb *phb = npe->phb;
202         int64_t rc;
203         const unsigned long size = tbl->it_indirect_levels ?
204                 tbl->it_level_size : tbl->it_size;
205         const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;
206         const __u64 win_size = tbl->it_size << tbl->it_page_shift;
207 
208         pe_info(npe, "Setting up window %llx..%llx pg=%lx\n",
209                         start_addr, start_addr + win_size - 1,
210                         IOMMU_PAGE_SIZE(tbl));
211 
212         rc = opal_pci_map_pe_dma_window(phb->opal_id,
213                         npe->pe_number,
214                         npe->pe_number,
215                         tbl->it_indirect_levels + 1,
216                         __pa(tbl->it_base),
217                         size << 3,
218                         IOMMU_PAGE_SIZE(tbl));
219         if (rc) {
220                 pe_err(npe, "Failed to configure TCE table, err %lld\n", rc);
221                 return rc;
222         }
223         pnv_pci_ioda2_tce_invalidate_entire(phb, false);
224 
225         /* Add the table to the list so its TCE cache will get invalidated */
226         pnv_pci_link_table_and_group(phb->hose->node, num,
227                         tbl, &npe->table_group);
228 
229         return 0;
230 }
231 
232 long pnv_npu_unset_window(struct pnv_ioda_pe *npe, int num)
233 {
234         struct pnv_phb *phb = npe->phb;
235         int64_t rc;
236 
237         pe_info(npe, "Removing DMA window\n");
238 
239         rc = opal_pci_map_pe_dma_window(phb->opal_id, npe->pe_number,
240                         npe->pe_number,
241                         0/* levels */, 0/* table address */,
242                         0/* table size */, 0/* page size */);
243         if (rc) {
244                 pe_err(npe, "Unmapping failed, ret = %lld\n", rc);
245                 return rc;
246         }
247         pnv_pci_ioda2_tce_invalidate_entire(phb, false);
248 
249         pnv_pci_unlink_table_and_group(npe->table_group.tables[num],
250                         &npe->table_group);
251 
252         return 0;
253 }
254 
255 /*
256  * Enables 32 bit DMA on NPU.
257  */
258 static void pnv_npu_dma_set_32(struct pnv_ioda_pe *npe)
259 {
260         struct pci_dev *gpdev;
261         struct pnv_ioda_pe *gpe;
262         int64_t rc;
263 
264         /*
265          * Find the assoicated PCI devices and get the dma window
266          * information from there.
267          */
268         if (!npe->pdev || !(npe->flags & PNV_IODA_PE_DEV))
269                 return;
270 
271         gpe = get_gpu_pci_dev_and_pe(npe, &gpdev);
272         if (!gpe)
273                 return;
274 
275         rc = pnv_npu_set_window(npe, 0, gpe->table_group.tables[0]);
276 
277         /*
278          * We don't initialise npu_pe->tce32_table as we always use
279          * dma_npu_ops which are nops.
280          */
281         set_dma_ops(&npe->pdev->dev, &dma_npu_ops);
282 }
283 
284 /*
285  * Enables bypass mode on the NPU. The NPU only supports one
286  * window per link, so bypass needs to be explicitly enabled or
287  * disabled. Unlike for a PHB3 bypass and non-bypass modes can't be
288  * active at the same time.
289  */
290 static int pnv_npu_dma_set_bypass(struct pnv_ioda_pe *npe)
291 {
292         struct pnv_phb *phb = npe->phb;
293         int64_t rc = 0;
294         phys_addr_t top = memblock_end_of_DRAM();
295 
296         if (phb->type != PNV_PHB_NPU_NVLINK || !npe->pdev)
297                 return -EINVAL;
298 
299         rc = pnv_npu_unset_window(npe, 0);
300         if (rc != OPAL_SUCCESS)
301                 return rc;
302 
303         /* Enable the bypass window */
304 
305         top = roundup_pow_of_two(top);
306         dev_info(&npe->pdev->dev, "Enabling bypass for PE %x\n",
307                         npe->pe_number);
308         rc = opal_pci_map_pe_dma_window_real(phb->opal_id,
309                         npe->pe_number, npe->pe_number,
310                         0 /* bypass base */, top);
311 
312         if (rc == OPAL_SUCCESS)
313                 pnv_pci_ioda2_tce_invalidate_entire(phb, false);
314 
315         return rc;
316 }
317 
318 void pnv_npu_try_dma_set_bypass(struct pci_dev *gpdev, bool bypass)
319 {
320         int i;
321         struct pnv_phb *phb;
322         struct pci_dn *pdn;
323         struct pnv_ioda_pe *npe;
324         struct pci_dev *npdev;
325 
326         for (i = 0; ; ++i) {
327                 npdev = pnv_pci_get_npu_dev(gpdev, i);
328 
329                 if (!npdev)
330                         break;
331 
332                 pdn = pci_get_pdn(npdev);
333                 if (WARN_ON(!pdn || pdn->pe_number == IODA_INVALID_PE))
334                         return;
335 
336                 phb = pci_bus_to_host(npdev->bus)->private_data;
337 
338                 /* We only do bypass if it's enabled on the linked device */
339                 npe = &phb->ioda.pe_array[pdn->pe_number];
340 
341                 if (bypass) {
342                         dev_info(&npdev->dev,
343                                         "Using 64-bit DMA iommu bypass\n");
344                         pnv_npu_dma_set_bypass(npe);
345                 } else {
346                         dev_info(&npdev->dev, "Using 32-bit DMA via iommu\n");
347                         pnv_npu_dma_set_32(npe);
348                 }
349         }
350 }
351 
352 /* Switch ownership from platform code to external user (e.g. VFIO) */
353 void pnv_npu_take_ownership(struct pnv_ioda_pe *npe)
354 {
355         struct pnv_phb *phb = npe->phb;
356         int64_t rc;
357 
358         /*
359          * Note: NPU has just a single TVE in the hardware which means that
360          * while used by the kernel, it can have either 32bit window or
361          * DMA bypass but never both. So we deconfigure 32bit window only
362          * if it was enabled at the moment of ownership change.
363          */
364         if (npe->table_group.tables[0]) {
365                 pnv_npu_unset_window(npe, 0);
366                 return;
367         }
368 
369         /* Disable bypass */
370         rc = opal_pci_map_pe_dma_window_real(phb->opal_id,
371                         npe->pe_number, npe->pe_number,
372                         0 /* bypass base */, 0);
373         if (rc) {
374                 pe_err(npe, "Failed to disable bypass, err %lld\n", rc);
375                 return;
376         }
377         pnv_pci_ioda2_tce_invalidate_entire(npe->phb, false);
378 }
379 
380 struct pnv_ioda_pe *pnv_pci_npu_setup_iommu(struct pnv_ioda_pe *npe)
381 {
382         struct pnv_phb *phb = npe->phb;
383         struct pci_bus *pbus = phb->hose->bus;
384         struct pci_dev *npdev, *gpdev = NULL, *gptmp;
385         struct pnv_ioda_pe *gpe = get_gpu_pci_dev_and_pe(npe, &gpdev);
386 
387         if (!gpe || !gpdev)
388                 return NULL;
389 
390         list_for_each_entry(npdev, &pbus->devices, bus_list) {
391                 gptmp = pnv_pci_get_gpu_dev(npdev);
392 
393                 if (gptmp != gpdev)
394                         continue;
395 
396                 pe_info(gpe, "Attached NPU %s\n", dev_name(&npdev->dev));
397                 iommu_group_add_device(gpe->table_group.group, &npdev->dev);
398         }
399 
400         return gpe;
401 }
402 
403 /* Maximum number of nvlinks per npu */
404 #define NV_MAX_LINKS 6
405 
406 /* Maximum index of npu2 hosts in the system. Always < NV_MAX_NPUS */
407 static int max_npu2_index;
408 
409 struct npu_context {
410         struct mm_struct *mm;
411         struct pci_dev *npdev[NV_MAX_NPUS][NV_MAX_LINKS];
412         struct mmu_notifier mn;
413         struct kref kref;
414         bool nmmu_flush;
415 
416         /* Callback to stop translation requests on a given GPU */
417         void (*release_cb)(struct npu_context *context, void *priv);
418 
419         /*
420          * Private pointer passed to the above callback for usage by
421          * device drivers.
422          */
423         void *priv;
424 };
425 
426 struct mmio_atsd_reg {
427         struct npu *npu;
428         int reg;
429 };
430 
431 /*
432  * Find a free MMIO ATSD register and mark it in use. Return -ENOSPC
433  * if none are available.
434  */
435 static int get_mmio_atsd_reg(struct npu *npu)
436 {
437         int i;
438 
439         for (i = 0; i < npu->mmio_atsd_count; i++) {
440                 if (!test_bit(i, &npu->mmio_atsd_usage))
441                         if (!test_and_set_bit_lock(i, &npu->mmio_atsd_usage))
442                                 return i;
443         }
444 
445         return -ENOSPC;
446 }
447 
448 static void put_mmio_atsd_reg(struct npu *npu, int reg)
449 {
450         clear_bit_unlock(reg, &npu->mmio_atsd_usage);
451 }
452 
453 /* MMIO ATSD register offsets */
454 #define XTS_ATSD_AVA  1
455 #define XTS_ATSD_STAT 2
456 
457 static void mmio_launch_invalidate(struct mmio_atsd_reg *mmio_atsd_reg,
458                                 unsigned long launch, unsigned long va)
459 {
460         struct npu *npu = mmio_atsd_reg->npu;
461         int reg = mmio_atsd_reg->reg;
462 
463         __raw_writeq_be(va, npu->mmio_atsd_regs[reg] + XTS_ATSD_AVA);
464         eieio();
465         __raw_writeq_be(launch, npu->mmio_atsd_regs[reg]);
466 }
467 
468 static void mmio_invalidate_pid(struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS],
469                                 unsigned long pid, bool flush)
470 {
471         int i;
472         unsigned long launch;
473 
474         for (i = 0; i <= max_npu2_index; i++) {
475                 if (mmio_atsd_reg[i].reg < 0)
476                         continue;
477 
478                 /* IS set to invalidate matching PID */
479                 launch = PPC_BIT(12);
480 
481                 /* PRS set to process-scoped */
482                 launch |= PPC_BIT(13);
483 
484                 /* AP */
485                 launch |= (u64)
486                         mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17);
487 
488                 /* PID */
489                 launch |= pid << PPC_BITLSHIFT(38);
490 
491                 /* No flush */
492                 launch |= !flush << PPC_BITLSHIFT(39);
493 
494                 /* Invalidating the entire process doesn't use a va */
495                 mmio_launch_invalidate(&mmio_atsd_reg[i], launch, 0);
496         }
497 }
498 
499 static void mmio_invalidate_va(struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS],
500                         unsigned long va, unsigned long pid, bool flush)
501 {
502         int i;
503         unsigned long launch;
504 
505         for (i = 0; i <= max_npu2_index; i++) {
506                 if (mmio_atsd_reg[i].reg < 0)
507                         continue;
508 
509                 /* IS set to invalidate target VA */
510                 launch = 0;
511 
512                 /* PRS set to process scoped */
513                 launch |= PPC_BIT(13);
514 
515                 /* AP */
516                 launch |= (u64)
517                         mmu_get_ap(mmu_virtual_psize) << PPC_BITLSHIFT(17);
518 
519                 /* PID */
520                 launch |= pid << PPC_BITLSHIFT(38);
521 
522                 /* No flush */
523                 launch |= !flush << PPC_BITLSHIFT(39);
524 
525                 mmio_launch_invalidate(&mmio_atsd_reg[i], launch, va);
526         }
527 }
528 
529 #define mn_to_npu_context(x) container_of(x, struct npu_context, mn)
530 
531 static void mmio_invalidate_wait(
532         struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS])
533 {
534         struct npu *npu;
535         int i, reg;
536 
537         /* Wait for all invalidations to complete */
538         for (i = 0; i <= max_npu2_index; i++) {
539                 if (mmio_atsd_reg[i].reg < 0)
540                         continue;
541 
542                 /* Wait for completion */
543                 npu = mmio_atsd_reg[i].npu;
544                 reg = mmio_atsd_reg[i].reg;
545                 while (__raw_readq(npu->mmio_atsd_regs[reg] + XTS_ATSD_STAT))
546                         cpu_relax();
547         }
548 }
549 
550 /*
551  * Acquires all the address translation shootdown (ATSD) registers required to
552  * launch an ATSD on all links this npu_context is active on.
553  */
554 static void acquire_atsd_reg(struct npu_context *npu_context,
555                         struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS])
556 {
557         int i, j;
558         struct npu *npu;
559         struct pci_dev *npdev;
560         struct pnv_phb *nphb;
561 
562         for (i = 0; i <= max_npu2_index; i++) {
563                 mmio_atsd_reg[i].reg = -1;
564                 for (j = 0; j < NV_MAX_LINKS; j++) {
565                         /*
566                          * There are no ordering requirements with respect to
567                          * the setup of struct npu_context, but to ensure
568                          * consistent behaviour we need to ensure npdev[][] is
569                          * only read once.
570                          */
571                         npdev = READ_ONCE(npu_context->npdev[i][j]);
572                         if (!npdev)
573                                 continue;
574 
575                         nphb = pci_bus_to_host(npdev->bus)->private_data;
576                         npu = &nphb->npu;
577                         mmio_atsd_reg[i].npu = npu;
578                         mmio_atsd_reg[i].reg = get_mmio_atsd_reg(npu);
579                         while (mmio_atsd_reg[i].reg < 0) {
580                                 mmio_atsd_reg[i].reg = get_mmio_atsd_reg(npu);
581                                 cpu_relax();
582                         }
583                         break;
584                 }
585         }
586 }
587 
588 /*
589  * Release previously acquired ATSD registers. To avoid deadlocks the registers
590  * must be released in the same order they were acquired above in
591  * acquire_atsd_reg.
592  */
593 static void release_atsd_reg(struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS])
594 {
595         int i;
596 
597         for (i = 0; i <= max_npu2_index; i++) {
598                 /*
599                  * We can't rely on npu_context->npdev[][] being the same here
600                  * as when acquire_atsd_reg() was called, hence we use the
601                  * values stored in mmio_atsd_reg during the acquire phase
602                  * rather than re-reading npdev[][].
603                  */
604                 if (mmio_atsd_reg[i].reg < 0)
605                         continue;
606 
607                 put_mmio_atsd_reg(mmio_atsd_reg[i].npu, mmio_atsd_reg[i].reg);
608         }
609 }
610 
611 /*
612  * Invalidate either a single address or an entire PID depending on
613  * the value of va.
614  */
615 static void mmio_invalidate(struct npu_context *npu_context, int va,
616                         unsigned long address, bool flush)
617 {
618         struct mmio_atsd_reg mmio_atsd_reg[NV_MAX_NPUS];
619         unsigned long pid = npu_context->mm->context.id;
620 
621         if (npu_context->nmmu_flush)
622                 /*
623                  * Unfortunately the nest mmu does not support flushing specific
624                  * addresses so we have to flush the whole mm once before
625                  * shooting down the GPU translation.
626                  */
627                 flush_all_mm(npu_context->mm);
628 
629         /*
630          * Loop over all the NPUs this process is active on and launch
631          * an invalidate.
632          */
633         acquire_atsd_reg(npu_context, mmio_atsd_reg);
634         if (va)
635                 mmio_invalidate_va(mmio_atsd_reg, address, pid, flush);
636         else
637                 mmio_invalidate_pid(mmio_atsd_reg, pid, flush);
638 
639         mmio_invalidate_wait(mmio_atsd_reg);
640         if (flush) {
641                 /*
642                  * The GPU requires two flush ATSDs to ensure all entries have
643                  * been flushed. We use PID 0 as it will never be used for a
644                  * process on the GPU.
645                  */
646                 mmio_invalidate_pid(mmio_atsd_reg, 0, true);
647                 mmio_invalidate_wait(mmio_atsd_reg);
648                 mmio_invalidate_pid(mmio_atsd_reg, 0, true);
649                 mmio_invalidate_wait(mmio_atsd_reg);
650         }
651         release_atsd_reg(mmio_atsd_reg);
652 }
653 
654 static void pnv_npu2_mn_release(struct mmu_notifier *mn,
655                                 struct mm_struct *mm)
656 {
657         struct npu_context *npu_context = mn_to_npu_context(mn);
658 
659         /* Call into device driver to stop requests to the NMMU */
660         if (npu_context->release_cb)
661                 npu_context->release_cb(npu_context, npu_context->priv);
662 
663         /*
664          * There should be no more translation requests for this PID, but we
665          * need to ensure any entries for it are removed from the TLB.
666          */
667         mmio_invalidate(npu_context, 0, 0, true);
668 }
669 
670 static void pnv_npu2_mn_change_pte(struct mmu_notifier *mn,
671                                 struct mm_struct *mm,
672                                 unsigned long address,
673                                 pte_t pte)
674 {
675         struct npu_context *npu_context = mn_to_npu_context(mn);
676 
677         mmio_invalidate(npu_context, 1, address, true);
678 }
679 
680 static void pnv_npu2_mn_invalidate_range(struct mmu_notifier *mn,
681                                         struct mm_struct *mm,
682                                         unsigned long start, unsigned long end)
683 {
684         struct npu_context *npu_context = mn_to_npu_context(mn);
685         unsigned long address;
686 
687         if (end - start > ATSD_THRESHOLD) {
688                 /*
689                  * Just invalidate the entire PID if the address range is too
690                  * large.
691                  */
692                 mmio_invalidate(npu_context, 0, 0, true);
693         } else {
694                 for (address = start; address < end; address += PAGE_SIZE)
695                         mmio_invalidate(npu_context, 1, address, false);
696 
697                 /* Do the flush only on the final addess == end */
698                 mmio_invalidate(npu_context, 1, address, true);
699         }
700 }
701 
702 static const struct mmu_notifier_ops nv_nmmu_notifier_ops = {
703         .release = pnv_npu2_mn_release,
704         .change_pte = pnv_npu2_mn_change_pte,
705         .invalidate_range = pnv_npu2_mn_invalidate_range,
706 };
707 
708 /*
709  * Call into OPAL to setup the nmmu context for the current task in
710  * the NPU. This must be called to setup the context tables before the
711  * GPU issues ATRs. pdev should be a pointed to PCIe GPU device.
712  *
713  * A release callback should be registered to allow a device driver to
714  * be notified that it should not launch any new translation requests
715  * as the final TLB invalidate is about to occur.
716  *
717  * Returns an error if there no contexts are currently available or a
718  * npu_context which should be passed to pnv_npu2_handle_fault().
719  *
720  * mmap_sem must be held in write mode and must not be called from interrupt
721  * context.
722  */
723 struct npu_context *pnv_npu2_init_context(struct pci_dev *gpdev,
724                         unsigned long flags,
725                         void (*cb)(struct npu_context *, void *),
726                         void *priv)
727 {
728         int rc;
729         u32 nvlink_index;
730         struct device_node *nvlink_dn;
731         struct mm_struct *mm = current->mm;
732         struct pnv_phb *nphb;
733         struct npu *npu;
734         struct npu_context *npu_context;
735 
736         /*
737          * At present we don't support GPUs connected to multiple NPUs and I'm
738          * not sure the hardware does either.
739          */
740         struct pci_dev *npdev = pnv_pci_get_npu_dev(gpdev, 0);
741 
742         if (!firmware_has_feature(FW_FEATURE_OPAL))
743                 return ERR_PTR(-ENODEV);
744 
745         if (!npdev)
746                 /* No nvlink associated with this GPU device */
747                 return ERR_PTR(-ENODEV);
748 
749         nvlink_dn = of_parse_phandle(npdev->dev.of_node, "ibm,nvlink", 0);
750         if (WARN_ON(of_property_read_u32(nvlink_dn, "ibm,npu-link-index",
751                                                         &nvlink_index)))
752                 return ERR_PTR(-ENODEV);
753 
754         if (!mm || mm->context.id == 0) {
755                 /*
756                  * Kernel thread contexts are not supported and context id 0 is
757                  * reserved on the GPU.
758                  */
759                 return ERR_PTR(-EINVAL);
760         }
761 
762         nphb = pci_bus_to_host(npdev->bus)->private_data;
763         npu = &nphb->npu;
764 
765         /*
766          * Setup the NPU context table for a particular GPU. These need to be
767          * per-GPU as we need the tables to filter ATSDs when there are no
768          * active contexts on a particular GPU. It is safe for these to be
769          * called concurrently with destroy as the OPAL call takes appropriate
770          * locks and refcounts on init/destroy.
771          */
772         rc = opal_npu_init_context(nphb->opal_id, mm->context.id, flags,
773                                 PCI_DEVID(gpdev->bus->number, gpdev->devfn));
774         if (rc < 0)
775                 return ERR_PTR(-ENOSPC);
776 
777         /*
778          * We store the npu pci device so we can more easily get at the
779          * associated npus.
780          */
781         spin_lock(&npu_context_lock);
782         npu_context = mm->context.npu_context;
783         if (npu_context) {
784                 if (npu_context->release_cb != cb ||
785                         npu_context->priv != priv) {
786                         spin_unlock(&npu_context_lock);
787                         opal_npu_destroy_context(nphb->opal_id, mm->context.id,
788                                                 PCI_DEVID(gpdev->bus->number,
789                                                         gpdev->devfn));
790                         return ERR_PTR(-EINVAL);
791                 }
792 
793                 WARN_ON(!kref_get_unless_zero(&npu_context->kref));
794         }
795         spin_unlock(&npu_context_lock);
796 
797         if (!npu_context) {
798                 /*
799                  * We can set up these fields without holding the
800                  * npu_context_lock as the npu_context hasn't been returned to
801                  * the caller meaning it can't be destroyed. Parallel allocation
802                  * is protected against by mmap_sem.
803                  */
804                 rc = -ENOMEM;
805                 npu_context = kzalloc(sizeof(struct npu_context), GFP_KERNEL);
806                 if (npu_context) {
807                         kref_init(&npu_context->kref);
808                         npu_context->mm = mm;
809                         npu_context->mn.ops = &nv_nmmu_notifier_ops;
810                         rc = __mmu_notifier_register(&npu_context->mn, mm);
811                 }
812 
813                 if (rc) {
814                         kfree(npu_context);
815                         opal_npu_destroy_context(nphb->opal_id, mm->context.id,
816                                         PCI_DEVID(gpdev->bus->number,
817                                                 gpdev->devfn));
818                         return ERR_PTR(rc);
819                 }
820 
821                 mm->context.npu_context = npu_context;
822         }
823 
824         npu_context->release_cb = cb;
825         npu_context->priv = priv;
826 
827         /*
828          * npdev is a pci_dev pointer setup by the PCI code. We assign it to
829          * npdev[][] to indicate to the mmu notifiers that an invalidation
830          * should also be sent over this nvlink. The notifiers don't use any
831          * other fields in npu_context, so we just need to ensure that when they
832          * deference npu_context->npdev[][] it is either a valid pointer or
833          * NULL.
834          */
835         WRITE_ONCE(npu_context->npdev[npu->index][nvlink_index], npdev);
836 
837         if (!nphb->npu.nmmu_flush) {
838                 /*
839                  * If we're not explicitly flushing ourselves we need to mark
840                  * the thread for global flushes
841                  */
842                 npu_context->nmmu_flush = false;
843                 mm_context_add_copro(mm);
844         } else
845                 npu_context->nmmu_flush = true;
846 
847         return npu_context;
848 }
849 EXPORT_SYMBOL(pnv_npu2_init_context);
850 
851 static void pnv_npu2_release_context(struct kref *kref)
852 {
853         struct npu_context *npu_context =
854                 container_of(kref, struct npu_context, kref);
855 
856         if (!npu_context->nmmu_flush)
857                 mm_context_remove_copro(npu_context->mm);
858 
859         npu_context->mm->context.npu_context = NULL;
860 }
861 
862 /*
863  * Destroy a context on the given GPU. May free the npu_context if it is no
864  * longer active on any GPUs. Must not be called from interrupt context.
865  */
866 void pnv_npu2_destroy_context(struct npu_context *npu_context,
867                         struct pci_dev *gpdev)
868 {
869         int removed;
870         struct pnv_phb *nphb;
871         struct npu *npu;
872         struct pci_dev *npdev = pnv_pci_get_npu_dev(gpdev, 0);
873         struct device_node *nvlink_dn;
874         u32 nvlink_index;
875 
876         if (WARN_ON(!npdev))
877                 return;
878 
879         if (!firmware_has_feature(FW_FEATURE_OPAL))
880                 return;
881 
882         nphb = pci_bus_to_host(npdev->bus)->private_data;
883         npu = &nphb->npu;
884         nvlink_dn = of_parse_phandle(npdev->dev.of_node, "ibm,nvlink", 0);
885         if (WARN_ON(of_property_read_u32(nvlink_dn, "ibm,npu-link-index",
886                                                         &nvlink_index)))
887                 return;
888         WRITE_ONCE(npu_context->npdev[npu->index][nvlink_index], NULL);
889         opal_npu_destroy_context(nphb->opal_id, npu_context->mm->context.id,
890                                 PCI_DEVID(gpdev->bus->number, gpdev->devfn));
891         spin_lock(&npu_context_lock);
892         removed = kref_put(&npu_context->kref, pnv_npu2_release_context);
893         spin_unlock(&npu_context_lock);
894 
895         /*
896          * We need to do this outside of pnv_npu2_release_context so that it is
897          * outside the spinlock as mmu_notifier_destroy uses SRCU.
898          */
899         if (removed) {
900                 mmu_notifier_unregister(&npu_context->mn,
901                                         npu_context->mm);
902 
903                 kfree(npu_context);
904         }
905 
906 }
907 EXPORT_SYMBOL(pnv_npu2_destroy_context);
908 
909 /*
910  * Assumes mmap_sem is held for the contexts associated mm.
911  */
912 int pnv_npu2_handle_fault(struct npu_context *context, uintptr_t *ea,
913                         unsigned long *flags, unsigned long *status, int count)
914 {
915         u64 rc = 0, result = 0;
916         int i, is_write;
917         struct page *page[1];
918 
919         /* mmap_sem should be held so the struct_mm must be present */
920         struct mm_struct *mm = context->mm;
921 
922         if (!firmware_has_feature(FW_FEATURE_OPAL))
923                 return -ENODEV;
924 
925         WARN_ON(!rwsem_is_locked(&mm->mmap_sem));
926 
927         for (i = 0; i < count; i++) {
928                 is_write = flags[i] & NPU2_WRITE;
929                 rc = get_user_pages_remote(NULL, mm, ea[i], 1,
930                                         is_write ? FOLL_WRITE : 0,
931                                         page, NULL, NULL);
932 
933                 /*
934                  * To support virtualised environments we will have to do an
935                  * access to the page to ensure it gets faulted into the
936                  * hypervisor. For the moment virtualisation is not supported in
937                  * other areas so leave the access out.
938                  */
939                 if (rc != 1) {
940                         status[i] = rc;
941                         result = -EFAULT;
942                         continue;
943                 }
944 
945                 status[i] = 0;
946                 put_page(page[0]);
947         }
948 
949         return result;
950 }
951 EXPORT_SYMBOL(pnv_npu2_handle_fault);
952 
953 int pnv_npu2_init(struct pnv_phb *phb)
954 {
955         unsigned int i;
956         u64 mmio_atsd;
957         struct device_node *dn;
958         struct pci_dev *gpdev;
959         static int npu_index;
960         uint64_t rc = 0;
961 
962         phb->npu.nmmu_flush =
963                 of_property_read_bool(phb->hose->dn, "ibm,nmmu-flush");
964         for_each_child_of_node(phb->hose->dn, dn) {
965                 gpdev = pnv_pci_get_gpu_dev(get_pci_dev(dn));
966                 if (gpdev) {
967                         rc = opal_npu_map_lpar(phb->opal_id,
968                                 PCI_DEVID(gpdev->bus->number, gpdev->devfn),
969                                 0, 0);
970                         if (rc)
971                                 dev_err(&gpdev->dev,
972                                         "Error %lld mapping device to LPAR\n",
973                                         rc);
974                 }
975         }
976 
977         for (i = 0; !of_property_read_u64_index(phb->hose->dn, "ibm,mmio-atsd",
978                                                         i, &mmio_atsd); i++)
979                 phb->npu.mmio_atsd_regs[i] = ioremap(mmio_atsd, 32);
980 
981         pr_info("NPU%lld: Found %d MMIO ATSD registers", phb->opal_id, i);
982         phb->npu.mmio_atsd_count = i;
983         phb->npu.mmio_atsd_usage = 0;
984         npu_index++;
985         if (WARN_ON(npu_index >= NV_MAX_NPUS))
986                 return -ENOSPC;
987         max_npu2_index = npu_index;
988         phb->npu.index = npu_index;
989 
990         return 0;
991 }
992 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp