~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/alpha/kernel/pci_iommu.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.11 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.84 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.154 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.201 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.201 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.77 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *      linux/arch/alpha/kernel/pci_iommu.c
  3  */
  4 
  5 #include <linux/kernel.h>
  6 #include <linux/mm.h>
  7 #include <linux/pci.h>
  8 #include <linux/gfp.h>
  9 #include <linux/bootmem.h>
 10 #include <linux/export.h>
 11 #include <linux/scatterlist.h>
 12 #include <linux/log2.h>
 13 #include <linux/dma-mapping.h>
 14 #include <linux/iommu-helper.h>
 15 
 16 #include <asm/io.h>
 17 #include <asm/hwrpb.h>
 18 
 19 #include "proto.h"
 20 #include "pci_impl.h"
 21 
 22 
 23 #define DEBUG_ALLOC 0
 24 #if DEBUG_ALLOC > 0
 25 # define DBGA(args...)          printk(KERN_DEBUG args)
 26 #else
 27 # define DBGA(args...)
 28 #endif
 29 #if DEBUG_ALLOC > 1
 30 # define DBGA2(args...)         printk(KERN_DEBUG args)
 31 #else
 32 # define DBGA2(args...)
 33 #endif
 34 
 35 #define DEBUG_NODIRECT 0
 36 
 37 #define ISA_DMA_MASK            0x00ffffff
 38 
 39 static inline unsigned long
 40 mk_iommu_pte(unsigned long paddr)
 41 {
 42         return (paddr >> (PAGE_SHIFT-1)) | 1;
 43 }
 44 
 45 /* Return the minimum of MAX or the first power of two larger
 46    than main memory.  */
 47 
 48 unsigned long
 49 size_for_memory(unsigned long max)
 50 {
 51         unsigned long mem = max_low_pfn << PAGE_SHIFT;
 52         if (mem < max)
 53                 max = roundup_pow_of_two(mem);
 54         return max;
 55 }
 56 
 57 struct pci_iommu_arena * __init
 58 iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base,
 59                      unsigned long window_size, unsigned long align)
 60 {
 61         unsigned long mem_size;
 62         struct pci_iommu_arena *arena;
 63 
 64         mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long));
 65 
 66         /* Note that the TLB lookup logic uses bitwise concatenation,
 67            not addition, so the required arena alignment is based on
 68            the size of the window.  Retain the align parameter so that
 69            particular systems can over-align the arena.  */
 70         if (align < mem_size)
 71                 align = mem_size;
 72 
 73 
 74 #ifdef CONFIG_DISCONTIGMEM
 75 
 76         arena = alloc_bootmem_node(NODE_DATA(nid), sizeof(*arena));
 77         if (!NODE_DATA(nid) || !arena) {
 78                 printk("%s: couldn't allocate arena from node %d\n"
 79                        "    falling back to system-wide allocation\n",
 80                        __func__, nid);
 81                 arena = alloc_bootmem(sizeof(*arena));
 82         }
 83 
 84         arena->ptes = __alloc_bootmem_node(NODE_DATA(nid), mem_size, align, 0);
 85         if (!NODE_DATA(nid) || !arena->ptes) {
 86                 printk("%s: couldn't allocate arena ptes from node %d\n"
 87                        "    falling back to system-wide allocation\n",
 88                        __func__, nid);
 89                 arena->ptes = __alloc_bootmem(mem_size, align, 0);
 90         }
 91 
 92 #else /* CONFIG_DISCONTIGMEM */
 93 
 94         arena = alloc_bootmem(sizeof(*arena));
 95         arena->ptes = __alloc_bootmem(mem_size, align, 0);
 96 
 97 #endif /* CONFIG_DISCONTIGMEM */
 98 
 99         spin_lock_init(&arena->lock);
100         arena->hose = hose;
101         arena->dma_base = base;
102         arena->size = window_size;
103         arena->next_entry = 0;
104 
105         /* Align allocations to a multiple of a page size.  Not needed
106            unless there are chip bugs.  */
107         arena->align_entry = 1;
108 
109         return arena;
110 }
111 
112 struct pci_iommu_arena * __init
113 iommu_arena_new(struct pci_controller *hose, dma_addr_t base,
114                 unsigned long window_size, unsigned long align)
115 {
116         return iommu_arena_new_node(0, hose, base, window_size, align);
117 }
118 
119 /* Must be called with the arena lock held */
120 static long
121 iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena,
122                        long n, long mask)
123 {
124         unsigned long *ptes;
125         long i, p, nent;
126         int pass = 0;
127         unsigned long base;
128         unsigned long boundary_size;
129 
130         base = arena->dma_base >> PAGE_SHIFT;
131         if (dev) {
132                 boundary_size = dma_get_seg_boundary(dev) + 1;
133                 boundary_size >>= PAGE_SHIFT;
134         } else {
135                 boundary_size = 1UL << (32 - PAGE_SHIFT);
136         }
137 
138         /* Search forward for the first mask-aligned sequence of N free ptes */
139         ptes = arena->ptes;
140         nent = arena->size >> PAGE_SHIFT;
141         p = ALIGN(arena->next_entry, mask + 1);
142         i = 0;
143 
144 again:
145         while (i < n && p+i < nent) {
146                 if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) {
147                         p = ALIGN(p + 1, mask + 1);
148                         goto again;
149                 }
150 
151                 if (ptes[p+i])
152                         p = ALIGN(p + i + 1, mask + 1), i = 0;
153                 else
154                         i = i + 1;
155         }
156 
157         if (i < n) {
158                 if (pass < 1) {
159                         /*
160                          * Reached the end.  Flush the TLB and restart
161                          * the search from the beginning.
162                         */
163                         alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
164 
165                         pass++;
166                         p = 0;
167                         i = 0;
168                         goto again;
169                 } else
170                         return -1;
171         }
172 
173         /* Success. It's the responsibility of the caller to mark them
174            in use before releasing the lock */
175         return p;
176 }
177 
178 static long
179 iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n,
180                   unsigned int align)
181 {
182         unsigned long flags;
183         unsigned long *ptes;
184         long i, p, mask;
185 
186         spin_lock_irqsave(&arena->lock, flags);
187 
188         /* Search for N empty ptes */
189         ptes = arena->ptes;
190         mask = max(align, arena->align_entry) - 1;
191         p = iommu_arena_find_pages(dev, arena, n, mask);
192         if (p < 0) {
193                 spin_unlock_irqrestore(&arena->lock, flags);
194                 return -1;
195         }
196 
197         /* Success.  Mark them all in use, ie not zero and invalid
198            for the iommu tlb that could load them from under us.
199            The chip specific bits will fill this in with something
200            kosher when we return.  */
201         for (i = 0; i < n; ++i)
202                 ptes[p+i] = IOMMU_INVALID_PTE;
203 
204         arena->next_entry = p + n;
205         spin_unlock_irqrestore(&arena->lock, flags);
206 
207         return p;
208 }
209 
210 static void
211 iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n)
212 {
213         unsigned long *p;
214         long i;
215 
216         p = arena->ptes + ofs;
217         for (i = 0; i < n; ++i)
218                 p[i] = 0;
219 }
220 
221 /*
222  * True if the machine supports DAC addressing, and DEV can
223  * make use of it given MASK.
224  */
225 static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask)
226 {
227         dma_addr_t dac_offset = alpha_mv.pci_dac_offset;
228         int ok = 1;
229 
230         /* If this is not set, the machine doesn't support DAC at all.  */
231         if (dac_offset == 0)
232                 ok = 0;
233 
234         /* The device has to be able to address our DAC bit.  */
235         if ((dac_offset & dev->dma_mask) != dac_offset)
236                 ok = 0;
237 
238         /* If both conditions above are met, we are fine. */
239         DBGA("pci_dac_dma_supported %s from %pf\n",
240              ok ? "yes" : "no", __builtin_return_address(0));
241 
242         return ok;
243 }
244 
245 /* Map a single buffer of the indicated size for PCI DMA in streaming
246    mode.  The 32-bit PCI bus mastering address to use is returned.
247    Once the device is given the dma address, the device owns this memory
248    until either pci_unmap_single or pci_dma_sync_single is performed.  */
249 
250 static dma_addr_t
251 pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size,
252                  int dac_allowed)
253 {
254         struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
255         dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
256         struct pci_iommu_arena *arena;
257         long npages, dma_ofs, i;
258         unsigned long paddr;
259         dma_addr_t ret;
260         unsigned int align = 0;
261         struct device *dev = pdev ? &pdev->dev : NULL;
262 
263         paddr = __pa(cpu_addr);
264 
265 #if !DEBUG_NODIRECT
266         /* First check to see if we can use the direct map window.  */
267         if (paddr + size + __direct_map_base - 1 <= max_dma
268             && paddr + size <= __direct_map_size) {
269                 ret = paddr + __direct_map_base;
270 
271                 DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %pf\n",
272                       cpu_addr, size, ret, __builtin_return_address(0));
273 
274                 return ret;
275         }
276 #endif
277 
278         /* Next, use DAC if selected earlier.  */
279         if (dac_allowed) {
280                 ret = paddr + alpha_mv.pci_dac_offset;
281 
282                 DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %pf\n",
283                       cpu_addr, size, ret, __builtin_return_address(0));
284 
285                 return ret;
286         }
287 
288         /* If the machine doesn't define a pci_tbi routine, we have to
289            assume it doesn't support sg mapping, and, since we tried to
290            use direct_map above, it now must be considered an error. */
291         if (! alpha_mv.mv_pci_tbi) {
292                 printk_once(KERN_WARNING "pci_map_single: no HW sg\n");
293                 return 0;
294         }
295 
296         arena = hose->sg_pci;
297         if (!arena || arena->dma_base + arena->size - 1 > max_dma)
298                 arena = hose->sg_isa;
299 
300         npages = iommu_num_pages(paddr, size, PAGE_SIZE);
301 
302         /* Force allocation to 64KB boundary for ISA bridges. */
303         if (pdev && pdev == isa_bridge)
304                 align = 8;
305         dma_ofs = iommu_arena_alloc(dev, arena, npages, align);
306         if (dma_ofs < 0) {
307                 printk(KERN_WARNING "pci_map_single failed: "
308                        "could not allocate dma page tables\n");
309                 return 0;
310         }
311 
312         paddr &= PAGE_MASK;
313         for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
314                 arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr);
315 
316         ret = arena->dma_base + dma_ofs * PAGE_SIZE;
317         ret += (unsigned long)cpu_addr & ~PAGE_MASK;
318 
319         DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %pf\n",
320               cpu_addr, size, npages, ret, __builtin_return_address(0));
321 
322         return ret;
323 }
324 
325 /* Helper for generic DMA-mapping functions. */
326 static struct pci_dev *alpha_gendev_to_pci(struct device *dev)
327 {
328         if (dev && dev_is_pci(dev))
329                 return to_pci_dev(dev);
330 
331         /* Assume that non-PCI devices asking for DMA are either ISA or EISA,
332            BUG() otherwise. */
333         BUG_ON(!isa_bridge);
334 
335         /* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA
336            bridge is bus master then). */
337         if (!dev || !dev->dma_mask || !*dev->dma_mask)
338                 return isa_bridge;
339 
340         /* For EISA bus masters, return isa_bridge (it might have smaller
341            dma_mask due to wiring limitations). */
342         if (*dev->dma_mask >= isa_bridge->dma_mask)
343                 return isa_bridge;
344 
345         /* This assumes ISA bus master with dma_mask 0xffffff. */
346         return NULL;
347 }
348 
349 static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page,
350                                      unsigned long offset, size_t size,
351                                      enum dma_data_direction dir,
352                                      unsigned long attrs)
353 {
354         struct pci_dev *pdev = alpha_gendev_to_pci(dev);
355         int dac_allowed;
356 
357         BUG_ON(dir == PCI_DMA_NONE);
358 
359         dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; 
360         return pci_map_single_1(pdev, (char *)page_address(page) + offset, 
361                                 size, dac_allowed);
362 }
363 
364 /* Unmap a single streaming mode DMA translation.  The DMA_ADDR and
365    SIZE must match what was provided for in a previous pci_map_single
366    call.  All other usages are undefined.  After this call, reads by
367    the cpu to the buffer are guaranteed to see whatever the device
368    wrote there.  */
369 
370 static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr,
371                                  size_t size, enum dma_data_direction dir,
372                                  unsigned long attrs)
373 {
374         unsigned long flags;
375         struct pci_dev *pdev = alpha_gendev_to_pci(dev);
376         struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
377         struct pci_iommu_arena *arena;
378         long dma_ofs, npages;
379 
380         BUG_ON(dir == PCI_DMA_NONE);
381 
382         if (dma_addr >= __direct_map_base
383             && dma_addr < __direct_map_base + __direct_map_size) {
384                 /* Nothing to do.  */
385 
386                 DBGA2("pci_unmap_single: direct [%llx,%zx] from %pf\n",
387                       dma_addr, size, __builtin_return_address(0));
388 
389                 return;
390         }
391 
392         if (dma_addr > 0xffffffff) {
393                 DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %pf\n",
394                       dma_addr, size, __builtin_return_address(0));
395                 return;
396         }
397 
398         arena = hose->sg_pci;
399         if (!arena || dma_addr < arena->dma_base)
400                 arena = hose->sg_isa;
401 
402         dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT;
403         if (dma_ofs * PAGE_SIZE >= arena->size) {
404                 printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx "
405                        " base %llx size %x\n",
406                        dma_addr, arena->dma_base, arena->size);
407                 return;
408                 BUG();
409         }
410 
411         npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
412 
413         spin_lock_irqsave(&arena->lock, flags);
414 
415         iommu_arena_free(arena, dma_ofs, npages);
416 
417         /* If we're freeing ptes above the `next_entry' pointer (they
418            may have snuck back into the TLB since the last wrap flush),
419            we need to flush the TLB before reallocating the latter.  */
420         if (dma_ofs >= arena->next_entry)
421                 alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1);
422 
423         spin_unlock_irqrestore(&arena->lock, flags);
424 
425         DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %pf\n",
426               dma_addr, size, npages, __builtin_return_address(0));
427 }
428 
429 /* Allocate and map kernel buffer using consistent mode DMA for PCI
430    device.  Returns non-NULL cpu-view pointer to the buffer if
431    successful and sets *DMA_ADDRP to the pci side dma address as well,
432    else DMA_ADDRP is undefined.  */
433 
434 static void *alpha_pci_alloc_coherent(struct device *dev, size_t size,
435                                       dma_addr_t *dma_addrp, gfp_t gfp,
436                                       unsigned long attrs)
437 {
438         struct pci_dev *pdev = alpha_gendev_to_pci(dev);
439         void *cpu_addr;
440         long order = get_order(size);
441 
442         gfp &= ~GFP_DMA;
443 
444 try_again:
445         cpu_addr = (void *)__get_free_pages(gfp, order);
446         if (! cpu_addr) {
447                 printk(KERN_INFO "pci_alloc_consistent: "
448                        "get_free_pages failed from %pf\n",
449                         __builtin_return_address(0));
450                 /* ??? Really atomic allocation?  Otherwise we could play
451                    with vmalloc and sg if we can't find contiguous memory.  */
452                 return NULL;
453         }
454         memset(cpu_addr, 0, size);
455 
456         *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0);
457         if (*dma_addrp == 0) {
458                 free_pages((unsigned long)cpu_addr, order);
459                 if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA))
460                         return NULL;
461                 /* The address doesn't fit required mask and we
462                    do not have iommu. Try again with GFP_DMA. */
463                 gfp |= GFP_DMA;
464                 goto try_again;
465         }
466 
467         DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %pf\n",
468               size, cpu_addr, *dma_addrp, __builtin_return_address(0));
469 
470         return cpu_addr;
471 }
472 
473 /* Free and unmap a consistent DMA buffer.  CPU_ADDR and DMA_ADDR must
474    be values that were returned from pci_alloc_consistent.  SIZE must
475    be the same as what as passed into pci_alloc_consistent.
476    References to the memory and mappings associated with CPU_ADDR or
477    DMA_ADDR past this call are illegal.  */
478 
479 static void alpha_pci_free_coherent(struct device *dev, size_t size,
480                                     void *cpu_addr, dma_addr_t dma_addr,
481                                     unsigned long attrs)
482 {
483         struct pci_dev *pdev = alpha_gendev_to_pci(dev);
484         pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
485         free_pages((unsigned long)cpu_addr, get_order(size));
486 
487         DBGA2("pci_free_consistent: [%llx,%zx] from %pf\n",
488               dma_addr, size, __builtin_return_address(0));
489 }
490 
491 /* Classify the elements of the scatterlist.  Write dma_address
492    of each element with:
493         0   : Followers all physically adjacent.
494         1   : Followers all virtually adjacent.
495         -1  : Not leader, physically adjacent to previous.
496         -2  : Not leader, virtually adjacent to previous.
497    Write dma_length of each leader with the combined lengths of
498    the mergable followers.  */
499 
500 #define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG)))
501 #define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG))
502 
503 static void
504 sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end,
505             int virt_ok)
506 {
507         unsigned long next_paddr;
508         struct scatterlist *leader;
509         long leader_flag, leader_length;
510         unsigned int max_seg_size;
511 
512         leader = sg;
513         leader_flag = 0;
514         leader_length = leader->length;
515         next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length;
516 
517         /* we will not marge sg without device. */
518         max_seg_size = dev ? dma_get_max_seg_size(dev) : 0;
519         for (++sg; sg < end; ++sg) {
520                 unsigned long addr, len;
521                 addr = SG_ENT_PHYS_ADDRESS(sg);
522                 len = sg->length;
523 
524                 if (leader_length + len > max_seg_size)
525                         goto new_segment;
526 
527                 if (next_paddr == addr) {
528                         sg->dma_address = -1;
529                         leader_length += len;
530                 } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
531                         sg->dma_address = -2;
532                         leader_flag = 1;
533                         leader_length += len;
534                 } else {
535 new_segment:
536                         leader->dma_address = leader_flag;
537                         leader->dma_length = leader_length;
538                         leader = sg;
539                         leader_flag = 0;
540                         leader_length = len;
541                 }
542 
543                 next_paddr = addr + len;
544         }
545 
546         leader->dma_address = leader_flag;
547         leader->dma_length = leader_length;
548 }
549 
550 /* Given a scatterlist leader, choose an allocation method and fill
551    in the blanks.  */
552 
553 static int
554 sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end,
555         struct scatterlist *out, struct pci_iommu_arena *arena,
556         dma_addr_t max_dma, int dac_allowed)
557 {
558         unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader);
559         long size = leader->dma_length;
560         struct scatterlist *sg;
561         unsigned long *ptes;
562         long npages, dma_ofs, i;
563 
564 #if !DEBUG_NODIRECT
565         /* If everything is physically contiguous, and the addresses
566            fall into the direct-map window, use it.  */
567         if (leader->dma_address == 0
568             && paddr + size + __direct_map_base - 1 <= max_dma
569             && paddr + size <= __direct_map_size) {
570                 out->dma_address = paddr + __direct_map_base;
571                 out->dma_length = size;
572 
573                 DBGA("    sg_fill: [%p,%lx] -> direct %llx\n",
574                      __va(paddr), size, out->dma_address);
575 
576                 return 0;
577         }
578 #endif
579 
580         /* If physically contiguous and DAC is available, use it.  */
581         if (leader->dma_address == 0 && dac_allowed) {
582                 out->dma_address = paddr + alpha_mv.pci_dac_offset;
583                 out->dma_length = size;
584 
585                 DBGA("    sg_fill: [%p,%lx] -> DAC %llx\n",
586                      __va(paddr), size, out->dma_address);
587 
588                 return 0;
589         }
590 
591         /* Otherwise, we'll use the iommu to make the pages virtually
592            contiguous.  */
593 
594         paddr &= ~PAGE_MASK;
595         npages = iommu_num_pages(paddr, size, PAGE_SIZE);
596         dma_ofs = iommu_arena_alloc(dev, arena, npages, 0);
597         if (dma_ofs < 0) {
598                 /* If we attempted a direct map above but failed, die.  */
599                 if (leader->dma_address == 0)
600                         return -1;
601 
602                 /* Otherwise, break up the remaining virtually contiguous
603                    hunks into individual direct maps and retry.  */
604                 sg_classify(dev, leader, end, 0);
605                 return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed);
606         }
607 
608         out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr;
609         out->dma_length = size;
610 
611         DBGA("    sg_fill: [%p,%lx] -> sg %llx np %ld\n",
612              __va(paddr), size, out->dma_address, npages);
613 
614         /* All virtually contiguous.  We need to find the length of each
615            physically contiguous subsegment to fill in the ptes.  */
616         ptes = &arena->ptes[dma_ofs];
617         sg = leader;
618         do {
619 #if DEBUG_ALLOC > 0
620                 struct scatterlist *last_sg = sg;
621 #endif
622 
623                 size = sg->length;
624                 paddr = SG_ENT_PHYS_ADDRESS(sg);
625 
626                 while (sg+1 < end && (int) sg[1].dma_address == -1) {
627                         size += sg[1].length;
628                         sg++;
629                 }
630 
631                 npages = iommu_num_pages(paddr, size, PAGE_SIZE);
632 
633                 paddr &= PAGE_MASK;
634                 for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
635                         *ptes++ = mk_iommu_pte(paddr);
636 
637 #if DEBUG_ALLOC > 0
638                 DBGA("    (%ld) [%p,%x] np %ld\n",
639                      last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
640                      last_sg->length, npages);
641                 while (++last_sg <= sg) {
642                         DBGA("        (%ld) [%p,%x] cont\n",
643                              last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
644                              last_sg->length);
645                 }
646 #endif
647         } while (++sg < end && (int) sg->dma_address < 0);
648 
649         return 1;
650 }
651 
652 static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg,
653                             int nents, enum dma_data_direction dir,
654                             unsigned long attrs)
655 {
656         struct pci_dev *pdev = alpha_gendev_to_pci(dev);
657         struct scatterlist *start, *end, *out;
658         struct pci_controller *hose;
659         struct pci_iommu_arena *arena;
660         dma_addr_t max_dma;
661         int dac_allowed;
662 
663         BUG_ON(dir == PCI_DMA_NONE);
664 
665         dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
666 
667         /* Fast path single entry scatterlists.  */
668         if (nents == 1) {
669                 sg->dma_length = sg->length;
670                 sg->dma_address
671                   = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg),
672                                      sg->length, dac_allowed);
673                 return sg->dma_address != 0;
674         }
675 
676         start = sg;
677         end = sg + nents;
678 
679         /* First, prepare information about the entries.  */
680         sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0);
681 
682         /* Second, figure out where we're going to map things.  */
683         if (alpha_mv.mv_pci_tbi) {
684                 hose = pdev ? pdev->sysdata : pci_isa_hose;
685                 max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
686                 arena = hose->sg_pci;
687                 if (!arena || arena->dma_base + arena->size - 1 > max_dma)
688                         arena = hose->sg_isa;
689         } else {
690                 max_dma = -1;
691                 arena = NULL;
692                 hose = NULL;
693         }
694 
695         /* Third, iterate over the scatterlist leaders and allocate
696            dma space as needed.  */
697         for (out = sg; sg < end; ++sg) {
698                 if ((int) sg->dma_address < 0)
699                         continue;
700                 if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0)
701                         goto error;
702                 out++;
703         }
704 
705         /* Mark the end of the list for pci_unmap_sg.  */
706         if (out < end)
707                 out->dma_length = 0;
708 
709         if (out - start == 0)
710                 printk(KERN_WARNING "pci_map_sg failed: no entries?\n");
711         DBGA("pci_map_sg: %ld entries\n", out - start);
712 
713         return out - start;
714 
715  error:
716         printk(KERN_WARNING "pci_map_sg failed: "
717                "could not allocate dma page tables\n");
718 
719         /* Some allocation failed while mapping the scatterlist
720            entries.  Unmap them now.  */
721         if (out > start)
722                 pci_unmap_sg(pdev, start, out - start, dir);
723         return 0;
724 }
725 
726 /* Unmap a set of streaming mode DMA translations.  Again, cpu read
727    rules concerning calls here are the same as for pci_unmap_single()
728    above.  */
729 
730 static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg,
731                                int nents, enum dma_data_direction dir,
732                                unsigned long attrs)
733 {
734         struct pci_dev *pdev = alpha_gendev_to_pci(dev);
735         unsigned long flags;
736         struct pci_controller *hose;
737         struct pci_iommu_arena *arena;
738         struct scatterlist *end;
739         dma_addr_t max_dma;
740         dma_addr_t fbeg, fend;
741 
742         BUG_ON(dir == PCI_DMA_NONE);
743 
744         if (! alpha_mv.mv_pci_tbi)
745                 return;
746 
747         hose = pdev ? pdev->sysdata : pci_isa_hose;
748         max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
749         arena = hose->sg_pci;
750         if (!arena || arena->dma_base + arena->size - 1 > max_dma)
751                 arena = hose->sg_isa;
752 
753         fbeg = -1, fend = 0;
754 
755         spin_lock_irqsave(&arena->lock, flags);
756 
757         for (end = sg + nents; sg < end; ++sg) {
758                 dma_addr_t addr;
759                 size_t size;
760                 long npages, ofs;
761                 dma_addr_t tend;
762 
763                 addr = sg->dma_address;
764                 size = sg->dma_length;
765                 if (!size)
766                         break;
767 
768                 if (addr > 0xffffffff) {
769                         /* It's a DAC address -- nothing to do.  */
770                         DBGA("    (%ld) DAC [%llx,%zx]\n",
771                               sg - end + nents, addr, size);
772                         continue;
773                 }
774 
775                 if (addr >= __direct_map_base
776                     && addr < __direct_map_base + __direct_map_size) {
777                         /* Nothing to do.  */
778                         DBGA("    (%ld) direct [%llx,%zx]\n",
779                               sg - end + nents, addr, size);
780                         continue;
781                 }
782 
783                 DBGA("    (%ld) sg [%llx,%zx]\n",
784                      sg - end + nents, addr, size);
785 
786                 npages = iommu_num_pages(addr, size, PAGE_SIZE);
787                 ofs = (addr - arena->dma_base) >> PAGE_SHIFT;
788                 iommu_arena_free(arena, ofs, npages);
789 
790                 tend = addr + size - 1;
791                 if (fbeg > addr) fbeg = addr;
792                 if (fend < tend) fend = tend;
793         }
794 
795         /* If we're freeing ptes above the `next_entry' pointer (they
796            may have snuck back into the TLB since the last wrap flush),
797            we need to flush the TLB before reallocating the latter.  */
798         if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry)
799                 alpha_mv.mv_pci_tbi(hose, fbeg, fend);
800 
801         spin_unlock_irqrestore(&arena->lock, flags);
802 
803         DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg));
804 }
805 
806 /* Return whether the given PCI device DMA address mask can be
807    supported properly.  */
808 
809 static int alpha_pci_supported(struct device *dev, u64 mask)
810 {
811         struct pci_dev *pdev = alpha_gendev_to_pci(dev);
812         struct pci_controller *hose;
813         struct pci_iommu_arena *arena;
814 
815         /* If there exists a direct map, and the mask fits either
816            the entire direct mapped space or the total system memory as
817            shifted by the map base */
818         if (__direct_map_size != 0
819             && (__direct_map_base + __direct_map_size - 1 <= mask ||
820                 __direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask))
821                 return 1;
822 
823         /* Check that we have a scatter-gather arena that fits.  */
824         hose = pdev ? pdev->sysdata : pci_isa_hose;
825         arena = hose->sg_isa;
826         if (arena && arena->dma_base + arena->size - 1 <= mask)
827                 return 1;
828         arena = hose->sg_pci;
829         if (arena && arena->dma_base + arena->size - 1 <= mask)
830                 return 1;
831 
832         /* As last resort try ZONE_DMA.  */
833         if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask)
834                 return 1;
835 
836         return 0;
837 }
838 
839 
840 /*
841  * AGP GART extensions to the IOMMU
842  */
843 int
844 iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask) 
845 {
846         unsigned long flags;
847         unsigned long *ptes;
848         long i, p;
849 
850         if (!arena) return -EINVAL;
851 
852         spin_lock_irqsave(&arena->lock, flags);
853 
854         /* Search for N empty ptes.  */
855         ptes = arena->ptes;
856         p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask);
857         if (p < 0) {
858                 spin_unlock_irqrestore(&arena->lock, flags);
859                 return -1;
860         }
861 
862         /* Success.  Mark them all reserved (ie not zero and invalid)
863            for the iommu tlb that could load them from under us.
864            They will be filled in with valid bits by _bind() */
865         for (i = 0; i < pg_count; ++i)
866                 ptes[p+i] = IOMMU_RESERVED_PTE;
867 
868         arena->next_entry = p + pg_count;
869         spin_unlock_irqrestore(&arena->lock, flags);
870 
871         return p;
872 }
873 
874 int 
875 iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count)
876 {
877         unsigned long *ptes;
878         long i;
879 
880         if (!arena) return -EINVAL;
881 
882         ptes = arena->ptes;
883 
884         /* Make sure they're all reserved first... */
885         for(i = pg_start; i < pg_start + pg_count; i++)
886                 if (ptes[i] != IOMMU_RESERVED_PTE)
887                         return -EBUSY;
888 
889         iommu_arena_free(arena, pg_start, pg_count);
890         return 0;
891 }
892 
893 int
894 iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count, 
895            struct page **pages)
896 {
897         unsigned long flags;
898         unsigned long *ptes;
899         long i, j;
900 
901         if (!arena) return -EINVAL;
902         
903         spin_lock_irqsave(&arena->lock, flags);
904 
905         ptes = arena->ptes;
906 
907         for(j = pg_start; j < pg_start + pg_count; j++) {
908                 if (ptes[j] != IOMMU_RESERVED_PTE) {
909                         spin_unlock_irqrestore(&arena->lock, flags);
910                         return -EBUSY;
911                 }
912         }
913                 
914         for(i = 0, j = pg_start; i < pg_count; i++, j++)
915                 ptes[j] = mk_iommu_pte(page_to_phys(pages[i]));
916 
917         spin_unlock_irqrestore(&arena->lock, flags);
918 
919         return 0;
920 }
921 
922 int
923 iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count)
924 {
925         unsigned long *p;
926         long i;
927 
928         if (!arena) return -EINVAL;
929 
930         p = arena->ptes + pg_start;
931         for(i = 0; i < pg_count; i++)
932                 p[i] = IOMMU_RESERVED_PTE;
933 
934         return 0;
935 }
936 
937 static int alpha_pci_mapping_error(struct device *dev, dma_addr_t dma_addr)
938 {
939         return dma_addr == 0;
940 }
941 
942 struct dma_map_ops alpha_pci_ops = {
943         .alloc                  = alpha_pci_alloc_coherent,
944         .free                   = alpha_pci_free_coherent,
945         .map_page               = alpha_pci_map_page,
946         .unmap_page             = alpha_pci_unmap_page,
947         .map_sg                 = alpha_pci_map_sg,
948         .unmap_sg               = alpha_pci_unmap_sg,
949         .mapping_error          = alpha_pci_mapping_error,
950         .dma_supported          = alpha_pci_supported,
951 };
952 
953 struct dma_map_ops *dma_ops = &alpha_pci_ops;
954 EXPORT_SYMBOL(dma_ops);
955 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp