~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/sysdev/dart_iommu.c

Version: ~ [ linux-5.3-rc5 ] ~ [ linux-5.2.9 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.67 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.139 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.189 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.189 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.72 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * arch/powerpc/sysdev/dart_iommu.c
  3  *
  4  * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
  5  * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
  6  *                    IBM Corporation
  7  *
  8  * Based on pSeries_iommu.c:
  9  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
 10  * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
 11  *
 12  * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
 13  *
 14  *
 15  * This program is free software; you can redistribute it and/or modify
 16  * it under the terms of the GNU General Public License as published by
 17  * the Free Software Foundation; either version 2 of the License, or
 18  * (at your option) any later version.
 19  *
 20  * This program is distributed in the hope that it will be useful,
 21  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 22  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 23  * GNU General Public License for more details.
 24  *
 25  * You should have received a copy of the GNU General Public License
 26  * along with this program; if not, write to the Free Software
 27  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 28  */
 29 
 30 #include <linux/init.h>
 31 #include <linux/types.h>
 32 #include <linux/mm.h>
 33 #include <linux/spinlock.h>
 34 #include <linux/string.h>
 35 #include <linux/pci.h>
 36 #include <linux/dma-mapping.h>
 37 #include <linux/vmalloc.h>
 38 #include <linux/suspend.h>
 39 #include <linux/memblock.h>
 40 #include <linux/gfp.h>
 41 #include <asm/io.h>
 42 #include <asm/prom.h>
 43 #include <asm/iommu.h>
 44 #include <asm/pci-bridge.h>
 45 #include <asm/machdep.h>
 46 #include <asm/cacheflush.h>
 47 #include <asm/ppc-pci.h>
 48 
 49 #include "dart.h"
 50 
 51 /* Physical base address and size of the DART table */
 52 unsigned long dart_tablebase; /* exported to htab_initialize */
 53 static unsigned long dart_tablesize;
 54 
 55 /* Virtual base address of the DART table */
 56 static u32 *dart_vbase;
 57 #ifdef CONFIG_PM
 58 static u32 *dart_copy;
 59 #endif
 60 
 61 /* Mapped base address for the dart */
 62 static unsigned int __iomem *dart;
 63 
 64 /* Dummy val that entries are set to when unused */
 65 static unsigned int dart_emptyval;
 66 
 67 static struct iommu_table iommu_table_dart;
 68 static int iommu_table_dart_inited;
 69 static int dart_dirty;
 70 static int dart_is_u4;
 71 
 72 #define DART_U4_BYPASS_BASE     0x8000000000ull
 73 
 74 #define DBG(...)
 75 
 76 static DEFINE_SPINLOCK(invalidate_lock);
 77 
 78 static inline void dart_tlb_invalidate_all(void)
 79 {
 80         unsigned long l = 0;
 81         unsigned int reg, inv_bit;
 82         unsigned long limit;
 83         unsigned long flags;
 84 
 85         spin_lock_irqsave(&invalidate_lock, flags);
 86 
 87         DBG("dart: flush\n");
 88 
 89         /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
 90          * control register and wait for it to clear.
 91          *
 92          * Gotcha: Sometimes, the DART won't detect that the bit gets
 93          * set. If so, clear it and set it again.
 94          */
 95 
 96         limit = 0;
 97 
 98         inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB;
 99 retry:
100         l = 0;
101         reg = DART_IN(DART_CNTL);
102         reg |= inv_bit;
103         DART_OUT(DART_CNTL, reg);
104 
105         while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit))
106                 l++;
107         if (l == (1L << limit)) {
108                 if (limit < 4) {
109                         limit++;
110                         reg = DART_IN(DART_CNTL);
111                         reg &= ~inv_bit;
112                         DART_OUT(DART_CNTL, reg);
113                         goto retry;
114                 } else
115                         panic("DART: TLB did not flush after waiting a long "
116                               "time. Buggy U3 ?");
117         }
118 
119         spin_unlock_irqrestore(&invalidate_lock, flags);
120 }
121 
122 static inline void dart_tlb_invalidate_one(unsigned long bus_rpn)
123 {
124         unsigned int reg;
125         unsigned int l, limit;
126         unsigned long flags;
127 
128         spin_lock_irqsave(&invalidate_lock, flags);
129 
130         reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE |
131                 (bus_rpn & DART_CNTL_U4_IONE_MASK);
132         DART_OUT(DART_CNTL, reg);
133 
134         limit = 0;
135 wait_more:
136         l = 0;
137         while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) {
138                 rmb();
139                 l++;
140         }
141 
142         if (l == (1L << limit)) {
143                 if (limit < 4) {
144                         limit++;
145                         goto wait_more;
146                 } else
147                         panic("DART: TLB did not flush after waiting a long "
148                               "time. Buggy U4 ?");
149         }
150 
151         spin_unlock_irqrestore(&invalidate_lock, flags);
152 }
153 
154 static void dart_flush(struct iommu_table *tbl)
155 {
156         mb();
157         if (dart_dirty) {
158                 dart_tlb_invalidate_all();
159                 dart_dirty = 0;
160         }
161 }
162 
163 static int dart_build(struct iommu_table *tbl, long index,
164                        long npages, unsigned long uaddr,
165                        enum dma_data_direction direction,
166                        struct dma_attrs *attrs)
167 {
168         unsigned int *dp;
169         unsigned int rpn;
170         long l;
171 
172         DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr);
173 
174         dp = ((unsigned int*)tbl->it_base) + index;
175 
176         /* On U3, all memory is contiguous, so we can move this
177          * out of the loop.
178          */
179         l = npages;
180         while (l--) {
181                 rpn = __pa(uaddr) >> DART_PAGE_SHIFT;
182 
183                 *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK);
184 
185                 uaddr += DART_PAGE_SIZE;
186         }
187 
188         /* make sure all updates have reached memory */
189         mb();
190         in_be32((unsigned __iomem *)dp);
191         mb();
192 
193         if (dart_is_u4) {
194                 rpn = index;
195                 while (npages--)
196                         dart_tlb_invalidate_one(rpn++);
197         } else {
198                 dart_dirty = 1;
199         }
200         return 0;
201 }
202 
203 
204 static void dart_free(struct iommu_table *tbl, long index, long npages)
205 {
206         unsigned int *dp;
207 
208         /* We don't worry about flushing the TLB cache. The only drawback of
209          * not doing it is that we won't catch buggy device drivers doing
210          * bad DMAs, but then no 32-bit architecture ever does either.
211          */
212 
213         DBG("dart: free at: %lx, %lx\n", index, npages);
214 
215         dp  = ((unsigned int *)tbl->it_base) + index;
216 
217         while (npages--)
218                 *(dp++) = dart_emptyval;
219 }
220 
221 
222 static int __init dart_init(struct device_node *dart_node)
223 {
224         unsigned int i;
225         unsigned long tmp, base, size;
226         struct resource r;
227 
228         if (dart_tablebase == 0 || dart_tablesize == 0) {
229                 printk(KERN_INFO "DART: table not allocated, using "
230                        "direct DMA\n");
231                 return -ENODEV;
232         }
233 
234         if (of_address_to_resource(dart_node, 0, &r))
235                 panic("DART: can't get register base ! ");
236 
237         /* Make sure nothing from the DART range remains in the CPU cache
238          * from a previous mapping that existed before the kernel took
239          * over
240          */
241         flush_dcache_phys_range(dart_tablebase,
242                                 dart_tablebase + dart_tablesize);
243 
244         /* Allocate a spare page to map all invalid DART pages. We need to do
245          * that to work around what looks like a problem with the HT bridge
246          * prefetching into invalid pages and corrupting data
247          */
248         tmp = memblock_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE);
249         dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) &
250                                          DARTMAP_RPNMASK);
251 
252         /* Map in DART registers */
253         dart = ioremap(r.start, resource_size(&r));
254         if (dart == NULL)
255                 panic("DART: Cannot map registers!");
256 
257         /* Map in DART table */
258         dart_vbase = ioremap(__pa(dart_tablebase), dart_tablesize);
259 
260         /* Fill initial table */
261         for (i = 0; i < dart_tablesize/4; i++)
262                 dart_vbase[i] = dart_emptyval;
263 
264         /* Initialize DART with table base and enable it. */
265         base = dart_tablebase >> DART_PAGE_SHIFT;
266         size = dart_tablesize >> DART_PAGE_SHIFT;
267         if (dart_is_u4) {
268                 size &= DART_SIZE_U4_SIZE_MASK;
269                 DART_OUT(DART_BASE_U4, base);
270                 DART_OUT(DART_SIZE_U4, size);
271                 DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE);
272         } else {
273                 size &= DART_CNTL_U3_SIZE_MASK;
274                 DART_OUT(DART_CNTL,
275                          DART_CNTL_U3_ENABLE |
276                          (base << DART_CNTL_U3_BASE_SHIFT) |
277                          (size << DART_CNTL_U3_SIZE_SHIFT));
278         }
279 
280         /* Invalidate DART to get rid of possible stale TLBs */
281         dart_tlb_invalidate_all();
282 
283         printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n",
284                dart_is_u4 ? "U4" : "U3");
285 
286         return 0;
287 }
288 
289 static struct iommu_table_ops iommu_dart_ops = {
290         .set = dart_build,
291         .clear = dart_free,
292         .flush = dart_flush,
293 };
294 
295 static void iommu_table_dart_setup(void)
296 {
297         iommu_table_dart.it_busno = 0;
298         iommu_table_dart.it_offset = 0;
299         /* it_size is in number of entries */
300         iommu_table_dart.it_size = dart_tablesize / sizeof(u32);
301         iommu_table_dart.it_page_shift = IOMMU_PAGE_SHIFT_4K;
302 
303         /* Initialize the common IOMMU code */
304         iommu_table_dart.it_base = (unsigned long)dart_vbase;
305         iommu_table_dart.it_index = 0;
306         iommu_table_dart.it_blocksize = 1;
307         iommu_table_dart.it_ops = &iommu_dart_ops;
308         iommu_init_table(&iommu_table_dart, -1);
309 
310         /* Reserve the last page of the DART to avoid possible prefetch
311          * past the DART mapped area
312          */
313         set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map);
314 }
315 
316 static void dma_dev_setup_dart(struct device *dev)
317 {
318         /* We only have one iommu table on the mac for now, which makes
319          * things simple. Setup all PCI devices to point to this table
320          */
321         if (get_dma_ops(dev) == &dma_direct_ops)
322                 set_dma_offset(dev, DART_U4_BYPASS_BASE);
323         else
324                 set_iommu_table_base(dev, &iommu_table_dart);
325 }
326 
327 static void pci_dma_dev_setup_dart(struct pci_dev *dev)
328 {
329         dma_dev_setup_dart(&dev->dev);
330 }
331 
332 static void pci_dma_bus_setup_dart(struct pci_bus *bus)
333 {
334         if (!iommu_table_dart_inited) {
335                 iommu_table_dart_inited = 1;
336                 iommu_table_dart_setup();
337         }
338 }
339 
340 static bool dart_device_on_pcie(struct device *dev)
341 {
342         struct device_node *np = of_node_get(dev->of_node);
343 
344         while(np) {
345                 if (of_device_is_compatible(np, "U4-pcie") ||
346                     of_device_is_compatible(np, "u4-pcie")) {
347                         of_node_put(np);
348                         return true;
349                 }
350                 np = of_get_next_parent(np);
351         }
352         return false;
353 }
354 
355 static int dart_dma_set_mask(struct device *dev, u64 dma_mask)
356 {
357         if (!dev->dma_mask || !dma_supported(dev, dma_mask))
358                 return -EIO;
359 
360         /* U4 supports a DART bypass, we use it for 64-bit capable
361          * devices to improve performances. However, that only works
362          * for devices connected to U4 own PCIe interface, not bridged
363          * through hypertransport. We need the device to support at
364          * least 40 bits of addresses.
365          */
366         if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) {
367                 dev_info(dev, "Using 64-bit DMA iommu bypass\n");
368                 set_dma_ops(dev, &dma_direct_ops);
369         } else {
370                 dev_info(dev, "Using 32-bit DMA via iommu\n");
371                 set_dma_ops(dev, &dma_iommu_ops);
372         }
373         dma_dev_setup_dart(dev);
374 
375         *dev->dma_mask = dma_mask;
376         return 0;
377 }
378 
379 void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops)
380 {
381         struct device_node *dn;
382 
383         /* Find the DART in the device-tree */
384         dn = of_find_compatible_node(NULL, "dart", "u3-dart");
385         if (dn == NULL) {
386                 dn = of_find_compatible_node(NULL, "dart", "u4-dart");
387                 if (dn == NULL)
388                         return; /* use default direct_dma_ops */
389                 dart_is_u4 = 1;
390         }
391 
392         /* Initialize the DART HW */
393         if (dart_init(dn) != 0)
394                 goto bail;
395 
396         /* Setup bypass if supported */
397         if (dart_is_u4)
398                 ppc_md.dma_set_mask = dart_dma_set_mask;
399 
400         controller_ops->dma_dev_setup = pci_dma_dev_setup_dart;
401         controller_ops->dma_bus_setup = pci_dma_bus_setup_dart;
402 
403         /* Setup pci_dma ops */
404         set_pci_dma_ops(&dma_iommu_ops);
405         return;
406 
407  bail:
408         /* If init failed, use direct iommu and null setup functions */
409         controller_ops->dma_dev_setup = NULL;
410         controller_ops->dma_bus_setup = NULL;
411 
412         /* Setup pci_dma ops */
413         set_pci_dma_ops(&dma_direct_ops);
414 }
415 
416 #ifdef CONFIG_PM
417 static void iommu_dart_save(void)
418 {
419         memcpy(dart_copy, dart_vbase, 2*1024*1024);
420 }
421 
422 static void iommu_dart_restore(void)
423 {
424         memcpy(dart_vbase, dart_copy, 2*1024*1024);
425         dart_tlb_invalidate_all();
426 }
427 
428 static int __init iommu_init_late_dart(void)
429 {
430         unsigned long tbasepfn;
431         struct page *p;
432 
433         /* if no dart table exists then we won't need to save it
434          * and the area has also not been reserved */
435         if (!dart_tablebase)
436                 return 0;
437 
438         tbasepfn = __pa(dart_tablebase) >> PAGE_SHIFT;
439         register_nosave_region_late(tbasepfn,
440                                     tbasepfn + ((1<<24) >> PAGE_SHIFT));
441 
442         /* For suspend we need to copy the dart contents because
443          * it is not part of the regular mapping (see above) and
444          * thus not saved automatically. The memory for this copy
445          * must be allocated early because we need 2 MB. */
446         p = alloc_pages(GFP_KERNEL, 21 - PAGE_SHIFT);
447         BUG_ON(!p);
448         dart_copy = page_address(p);
449 
450         ppc_md.iommu_save = iommu_dart_save;
451         ppc_md.iommu_restore = iommu_dart_restore;
452 
453         return 0;
454 }
455 
456 late_initcall(iommu_init_late_dart);
457 #endif
458 
459 void __init alloc_dart_table(void)
460 {
461         /* Only reserve DART space if machine has more than 1GB of RAM
462          * or if requested with iommu=on on cmdline.
463          *
464          * 1GB of RAM is picked as limit because some default devices
465          * (i.e. Airport Extreme) have 30 bit address range limits.
466          */
467 
468         if (iommu_is_off)
469                 return;
470 
471         if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull)
472                 return;
473 
474         /* 512 pages (2MB) is max DART tablesize. */
475         dart_tablesize = 1UL << 21;
476         /* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
477          * will blow up an entire large page anyway in the kernel mapping
478          */
479         dart_tablebase = (unsigned long)
480                 __va(memblock_alloc_base(1UL<<24, 1UL<<24, 0x80000000L));
481         /*
482          * The DART space is later unmapped from the kernel linear mapping and
483          * accessing dart_tablebase during kmemleak scanning will fault.
484          */
485         kmemleak_no_scan((void *)dart_tablebase);
486 
487         printk(KERN_INFO "DART table allocated at: %lx\n", dart_tablebase);
488 }
489 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp