~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/mm/dma-mapping.c

Version: ~ [ linux-5.13-rc5 ] ~ [ linux-5.12.9 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.42 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.124 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.193 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.235 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.271 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.271 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/arch/arm/mm/dma-mapping.c
  3  *
  4  *  Copyright (C) 2000-2004 Russell King
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  *
 10  *  DMA uncached mapping support.
 11  */
 12 #include <linux/bootmem.h>
 13 #include <linux/module.h>
 14 #include <linux/mm.h>
 15 #include <linux/gfp.h>
 16 #include <linux/errno.h>
 17 #include <linux/list.h>
 18 #include <linux/init.h>
 19 #include <linux/device.h>
 20 #include <linux/dma-mapping.h>
 21 #include <linux/dma-contiguous.h>
 22 #include <linux/highmem.h>
 23 #include <linux/memblock.h>
 24 #include <linux/slab.h>
 25 #include <linux/iommu.h>
 26 #include <linux/io.h>
 27 #include <linux/vmalloc.h>
 28 #include <linux/sizes.h>
 29 
 30 #include <asm/memory.h>
 31 #include <asm/highmem.h>
 32 #include <asm/cacheflush.h>
 33 #include <asm/tlbflush.h>
 34 #include <asm/mach/arch.h>
 35 #include <asm/dma-iommu.h>
 36 #include <asm/mach/map.h>
 37 #include <asm/system_info.h>
 38 #include <asm/dma-contiguous.h>
 39 
 40 #include "mm.h"
 41 
 42 /*
 43  * The DMA API is built upon the notion of "buffer ownership".  A buffer
 44  * is either exclusively owned by the CPU (and therefore may be accessed
 45  * by it) or exclusively owned by the DMA device.  These helper functions
 46  * represent the transitions between these two ownership states.
 47  *
 48  * Note, however, that on later ARMs, this notion does not work due to
 49  * speculative prefetches.  We model our approach on the assumption that
 50  * the CPU does do speculative prefetches, which means we clean caches
 51  * before transfers and delay cache invalidation until transfer completion.
 52  *
 53  */
 54 static void __dma_page_cpu_to_dev(struct page *, unsigned long,
 55                 size_t, enum dma_data_direction);
 56 static void __dma_page_dev_to_cpu(struct page *, unsigned long,
 57                 size_t, enum dma_data_direction);
 58 
 59 /**
 60  * arm_dma_map_page - map a portion of a page for streaming DMA
 61  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 62  * @page: page that buffer resides in
 63  * @offset: offset into page for start of buffer
 64  * @size: size of buffer to map
 65  * @dir: DMA transfer direction
 66  *
 67  * Ensure that any data held in the cache is appropriately discarded
 68  * or written back.
 69  *
 70  * The device owns this memory once this call has completed.  The CPU
 71  * can regain ownership by calling dma_unmap_page().
 72  */
 73 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page,
 74              unsigned long offset, size_t size, enum dma_data_direction dir,
 75              struct dma_attrs *attrs)
 76 {
 77         if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
 78                 __dma_page_cpu_to_dev(page, offset, size, dir);
 79         return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 80 }
 81 
 82 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page,
 83              unsigned long offset, size_t size, enum dma_data_direction dir,
 84              struct dma_attrs *attrs)
 85 {
 86         return pfn_to_dma(dev, page_to_pfn(page)) + offset;
 87 }
 88 
 89 /**
 90  * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page()
 91  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 92  * @handle: DMA address of buffer
 93  * @size: size of buffer (same as passed to dma_map_page)
 94  * @dir: DMA transfer direction (same as passed to dma_map_page)
 95  *
 96  * Unmap a page streaming mode DMA translation.  The handle and size
 97  * must match what was provided in the previous dma_map_page() call.
 98  * All other usages are undefined.
 99  *
100  * After this call, reads by the CPU to the buffer are guaranteed to see
101  * whatever the device wrote there.
102  */
103 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle,
104                 size_t size, enum dma_data_direction dir,
105                 struct dma_attrs *attrs)
106 {
107         if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
108                 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)),
109                                       handle & ~PAGE_MASK, size, dir);
110 }
111 
112 static void arm_dma_sync_single_for_cpu(struct device *dev,
113                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
114 {
115         unsigned int offset = handle & (PAGE_SIZE - 1);
116         struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
117         __dma_page_dev_to_cpu(page, offset, size, dir);
118 }
119 
120 static void arm_dma_sync_single_for_device(struct device *dev,
121                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
122 {
123         unsigned int offset = handle & (PAGE_SIZE - 1);
124         struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset));
125         __dma_page_cpu_to_dev(page, offset, size, dir);
126 }
127 
128 struct dma_map_ops arm_dma_ops = {
129         .alloc                  = arm_dma_alloc,
130         .free                   = arm_dma_free,
131         .mmap                   = arm_dma_mmap,
132         .get_sgtable            = arm_dma_get_sgtable,
133         .map_page               = arm_dma_map_page,
134         .unmap_page             = arm_dma_unmap_page,
135         .map_sg                 = arm_dma_map_sg,
136         .unmap_sg               = arm_dma_unmap_sg,
137         .sync_single_for_cpu    = arm_dma_sync_single_for_cpu,
138         .sync_single_for_device = arm_dma_sync_single_for_device,
139         .sync_sg_for_cpu        = arm_dma_sync_sg_for_cpu,
140         .sync_sg_for_device     = arm_dma_sync_sg_for_device,
141         .set_dma_mask           = arm_dma_set_mask,
142 };
143 EXPORT_SYMBOL(arm_dma_ops);
144 
145 static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
146         dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs);
147 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
148                                   dma_addr_t handle, struct dma_attrs *attrs);
149 
150 struct dma_map_ops arm_coherent_dma_ops = {
151         .alloc                  = arm_coherent_dma_alloc,
152         .free                   = arm_coherent_dma_free,
153         .mmap                   = arm_dma_mmap,
154         .get_sgtable            = arm_dma_get_sgtable,
155         .map_page               = arm_coherent_dma_map_page,
156         .map_sg                 = arm_dma_map_sg,
157         .set_dma_mask           = arm_dma_set_mask,
158 };
159 EXPORT_SYMBOL(arm_coherent_dma_ops);
160 
161 static int __dma_supported(struct device *dev, u64 mask, bool warn)
162 {
163         unsigned long max_dma_pfn;
164 
165         /*
166          * If the mask allows for more memory than we can address,
167          * and we actually have that much memory, then we must
168          * indicate that DMA to this device is not supported.
169          */
170         if (sizeof(mask) != sizeof(dma_addr_t) &&
171             mask > (dma_addr_t)~0 &&
172             dma_to_pfn(dev, ~0) < max_pfn) {
173                 if (warn) {
174                         dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n",
175                                  mask);
176                         dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n");
177                 }
178                 return 0;
179         }
180 
181         max_dma_pfn = min(max_pfn, arm_dma_pfn_limit);
182 
183         /*
184          * Translate the device's DMA mask to a PFN limit.  This
185          * PFN number includes the page which we can DMA to.
186          */
187         if (dma_to_pfn(dev, mask) < max_dma_pfn) {
188                 if (warn)
189                         dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n",
190                                  mask,
191                                  dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1,
192                                  max_dma_pfn + 1);
193                 return 0;
194         }
195 
196         return 1;
197 }
198 
199 static u64 get_coherent_dma_mask(struct device *dev)
200 {
201         u64 mask = (u64)DMA_BIT_MASK(32);
202 
203         if (dev) {
204                 mask = dev->coherent_dma_mask;
205 
206                 /*
207                  * Sanity check the DMA mask - it must be non-zero, and
208                  * must be able to be satisfied by a DMA allocation.
209                  */
210                 if (mask == 0) {
211                         dev_warn(dev, "coherent DMA mask is unset\n");
212                         return 0;
213                 }
214 
215                 if (!__dma_supported(dev, mask, true))
216                         return 0;
217         }
218 
219         return mask;
220 }
221 
222 static void __dma_clear_buffer(struct page *page, size_t size)
223 {
224         /*
225          * Ensure that the allocated pages are zeroed, and that any data
226          * lurking in the kernel direct-mapped region is invalidated.
227          */
228         if (PageHighMem(page)) {
229                 phys_addr_t base = __pfn_to_phys(page_to_pfn(page));
230                 phys_addr_t end = base + size;
231                 while (size > 0) {
232                         void *ptr = kmap_atomic(page);
233                         memset(ptr, 0, PAGE_SIZE);
234                         dmac_flush_range(ptr, ptr + PAGE_SIZE);
235                         kunmap_atomic(ptr);
236                         page++;
237                         size -= PAGE_SIZE;
238                 }
239                 outer_flush_range(base, end);
240         } else {
241                 void *ptr = page_address(page);
242                 memset(ptr, 0, size);
243                 dmac_flush_range(ptr, ptr + size);
244                 outer_flush_range(__pa(ptr), __pa(ptr) + size);
245         }
246 }
247 
248 /*
249  * Allocate a DMA buffer for 'dev' of size 'size' using the
250  * specified gfp mask.  Note that 'size' must be page aligned.
251  */
252 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, gfp_t gfp)
253 {
254         unsigned long order = get_order(size);
255         struct page *page, *p, *e;
256 
257         page = alloc_pages(gfp, order);
258         if (!page)
259                 return NULL;
260 
261         /*
262          * Now split the huge page and free the excess pages
263          */
264         split_page(page, order);
265         for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++)
266                 __free_page(p);
267 
268         __dma_clear_buffer(page, size);
269 
270         return page;
271 }
272 
273 /*
274  * Free a DMA buffer.  'size' must be page aligned.
275  */
276 static void __dma_free_buffer(struct page *page, size_t size)
277 {
278         struct page *e = page + (size >> PAGE_SHIFT);
279 
280         while (page < e) {
281                 __free_page(page);
282                 page++;
283         }
284 }
285 
286 #ifdef CONFIG_MMU
287 #ifdef CONFIG_HUGETLB_PAGE
288 #warning ARM Coherent DMA allocator does not (yet) support huge TLB
289 #endif
290 
291 static void *__alloc_from_contiguous(struct device *dev, size_t size,
292                                      pgprot_t prot, struct page **ret_page,
293                                      const void *caller);
294 
295 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
296                                  pgprot_t prot, struct page **ret_page,
297                                  const void *caller);
298 
299 static void *
300 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
301         const void *caller)
302 {
303         struct vm_struct *area;
304         unsigned long addr;
305 
306         /*
307          * DMA allocation can be mapped to user space, so lets
308          * set VM_USERMAP flags too.
309          */
310         area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
311                                   caller);
312         if (!area)
313                 return NULL;
314         addr = (unsigned long)area->addr;
315         area->phys_addr = __pfn_to_phys(page_to_pfn(page));
316 
317         if (ioremap_page_range(addr, addr + size, area->phys_addr, prot)) {
318                 vunmap((void *)addr);
319                 return NULL;
320         }
321         return (void *)addr;
322 }
323 
324 static void __dma_free_remap(void *cpu_addr, size_t size)
325 {
326         unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP;
327         struct vm_struct *area = find_vm_area(cpu_addr);
328         if (!area || (area->flags & flags) != flags) {
329                 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
330                 return;
331         }
332         unmap_kernel_range((unsigned long)cpu_addr, size);
333         vunmap(cpu_addr);
334 }
335 
336 #define DEFAULT_DMA_COHERENT_POOL_SIZE  SZ_256K
337 
338 struct dma_pool {
339         size_t size;
340         spinlock_t lock;
341         unsigned long *bitmap;
342         unsigned long nr_pages;
343         void *vaddr;
344         struct page **pages;
345 };
346 
347 static struct dma_pool atomic_pool = {
348         .size = DEFAULT_DMA_COHERENT_POOL_SIZE,
349 };
350 
351 static int __init early_coherent_pool(char *p)
352 {
353         atomic_pool.size = memparse(p, &p);
354         return 0;
355 }
356 early_param("coherent_pool", early_coherent_pool);
357 
358 void __init init_dma_coherent_pool_size(unsigned long size)
359 {
360         /*
361          * Catch any attempt to set the pool size too late.
362          */
363         BUG_ON(atomic_pool.vaddr);
364 
365         /*
366          * Set architecture specific coherent pool size only if
367          * it has not been changed by kernel command line parameter.
368          */
369         if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
370                 atomic_pool.size = size;
371 }
372 
373 /*
374  * Initialise the coherent pool for atomic allocations.
375  */
376 static int __init atomic_pool_init(void)
377 {
378         struct dma_pool *pool = &atomic_pool;
379         pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
380         gfp_t gfp = GFP_KERNEL | GFP_DMA;
381         unsigned long nr_pages = pool->size >> PAGE_SHIFT;
382         unsigned long *bitmap;
383         struct page *page;
384         struct page **pages;
385         void *ptr;
386         int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
387 
388         bitmap = kzalloc(bitmap_size, GFP_KERNEL);
389         if (!bitmap)
390                 goto no_bitmap;
391 
392         pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
393         if (!pages)
394                 goto no_pages;
395 
396         if (IS_ENABLED(CONFIG_DMA_CMA))
397                 ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page,
398                                               atomic_pool_init);
399         else
400                 ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page,
401                                            atomic_pool_init);
402         if (ptr) {
403                 int i;
404 
405                 for (i = 0; i < nr_pages; i++)
406                         pages[i] = page + i;
407 
408                 spin_lock_init(&pool->lock);
409                 pool->vaddr = ptr;
410                 pool->pages = pages;
411                 pool->bitmap = bitmap;
412                 pool->nr_pages = nr_pages;
413                 pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
414                        (unsigned)pool->size / 1024);
415                 return 0;
416         }
417 
418         kfree(pages);
419 no_pages:
420         kfree(bitmap);
421 no_bitmap:
422         pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
423                (unsigned)pool->size / 1024);
424         return -ENOMEM;
425 }
426 /*
427  * CMA is activated by core_initcall, so we must be called after it.
428  */
429 postcore_initcall(atomic_pool_init);
430 
431 struct dma_contig_early_reserve {
432         phys_addr_t base;
433         unsigned long size;
434 };
435 
436 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
437 
438 static int dma_mmu_remap_num __initdata;
439 
440 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
441 {
442         dma_mmu_remap[dma_mmu_remap_num].base = base;
443         dma_mmu_remap[dma_mmu_remap_num].size = size;
444         dma_mmu_remap_num++;
445 }
446 
447 void __init dma_contiguous_remap(void)
448 {
449         int i;
450         for (i = 0; i < dma_mmu_remap_num; i++) {
451                 phys_addr_t start = dma_mmu_remap[i].base;
452                 phys_addr_t end = start + dma_mmu_remap[i].size;
453                 struct map_desc map;
454                 unsigned long addr;
455 
456                 if (end > arm_lowmem_limit)
457                         end = arm_lowmem_limit;
458                 if (start >= end)
459                         continue;
460 
461                 map.pfn = __phys_to_pfn(start);
462                 map.virtual = __phys_to_virt(start);
463                 map.length = end - start;
464                 map.type = MT_MEMORY_DMA_READY;
465 
466                 /*
467                  * Clear previous low-memory mapping to ensure that the
468                  * TLB does not see any conflicting entries, then flush
469                  * the TLB of the old entries before creating new mappings.
470                  *
471                  * This ensures that any speculatively loaded TLB entries
472                  * (even though they may be rare) can not cause any problems,
473                  * and ensures that this code is architecturally compliant.
474                  */
475                 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end);
476                      addr += PMD_SIZE)
477                         pmd_clear(pmd_off_k(addr));
478 
479                 flush_tlb_kernel_range(__phys_to_virt(start),
480                                        __phys_to_virt(end));
481 
482                 iotable_init(&map, 1);
483         }
484 }
485 
486 static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr,
487                             void *data)
488 {
489         struct page *page = virt_to_page(addr);
490         pgprot_t prot = *(pgprot_t *)data;
491 
492         set_pte_ext(pte, mk_pte(page, prot), 0);
493         return 0;
494 }
495 
496 static void __dma_remap(struct page *page, size_t size, pgprot_t prot)
497 {
498         unsigned long start = (unsigned long) page_address(page);
499         unsigned end = start + size;
500 
501         apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot);
502         flush_tlb_kernel_range(start, end);
503 }
504 
505 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
506                                  pgprot_t prot, struct page **ret_page,
507                                  const void *caller)
508 {
509         struct page *page;
510         void *ptr;
511         page = __dma_alloc_buffer(dev, size, gfp);
512         if (!page)
513                 return NULL;
514 
515         ptr = __dma_alloc_remap(page, size, gfp, prot, caller);
516         if (!ptr) {
517                 __dma_free_buffer(page, size);
518                 return NULL;
519         }
520 
521         *ret_page = page;
522         return ptr;
523 }
524 
525 static void *__alloc_from_pool(size_t size, struct page **ret_page)
526 {
527         struct dma_pool *pool = &atomic_pool;
528         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
529         unsigned int pageno;
530         unsigned long flags;
531         void *ptr = NULL;
532         unsigned long align_mask;
533 
534         if (!pool->vaddr) {
535                 WARN(1, "coherent pool not initialised!\n");
536                 return NULL;
537         }
538 
539         /*
540          * Align the region allocation - allocations from pool are rather
541          * small, so align them to their order in pages, minimum is a page
542          * size. This helps reduce fragmentation of the DMA space.
543          */
544         align_mask = (1 << get_order(size)) - 1;
545 
546         spin_lock_irqsave(&pool->lock, flags);
547         pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
548                                             0, count, align_mask);
549         if (pageno < pool->nr_pages) {
550                 bitmap_set(pool->bitmap, pageno, count);
551                 ptr = pool->vaddr + PAGE_SIZE * pageno;
552                 *ret_page = pool->pages[pageno];
553         } else {
554                 pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
555                             "Please increase it with coherent_pool= kernel parameter!\n",
556                             (unsigned)pool->size / 1024);
557         }
558         spin_unlock_irqrestore(&pool->lock, flags);
559 
560         return ptr;
561 }
562 
563 static bool __in_atomic_pool(void *start, size_t size)
564 {
565         struct dma_pool *pool = &atomic_pool;
566         void *end = start + size;
567         void *pool_start = pool->vaddr;
568         void *pool_end = pool->vaddr + pool->size;
569 
570         if (start < pool_start || start >= pool_end)
571                 return false;
572 
573         if (end <= pool_end)
574                 return true;
575 
576         WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
577              start, end - 1, pool_start, pool_end - 1);
578 
579         return false;
580 }
581 
582 static int __free_from_pool(void *start, size_t size)
583 {
584         struct dma_pool *pool = &atomic_pool;
585         unsigned long pageno, count;
586         unsigned long flags;
587 
588         if (!__in_atomic_pool(start, size))
589                 return 0;
590 
591         pageno = (start - pool->vaddr) >> PAGE_SHIFT;
592         count = size >> PAGE_SHIFT;
593 
594         spin_lock_irqsave(&pool->lock, flags);
595         bitmap_clear(pool->bitmap, pageno, count);
596         spin_unlock_irqrestore(&pool->lock, flags);
597 
598         return 1;
599 }
600 
601 static void *__alloc_from_contiguous(struct device *dev, size_t size,
602                                      pgprot_t prot, struct page **ret_page,
603                                      const void *caller)
604 {
605         unsigned long order = get_order(size);
606         size_t count = size >> PAGE_SHIFT;
607         struct page *page;
608         void *ptr;
609 
610         page = dma_alloc_from_contiguous(dev, count, order);
611         if (!page)
612                 return NULL;
613 
614         __dma_clear_buffer(page, size);
615 
616         if (PageHighMem(page)) {
617                 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
618                 if (!ptr) {
619                         dma_release_from_contiguous(dev, page, count);
620                         return NULL;
621                 }
622         } else {
623                 __dma_remap(page, size, prot);
624                 ptr = page_address(page);
625         }
626         *ret_page = page;
627         return ptr;
628 }
629 
630 static void __free_from_contiguous(struct device *dev, struct page *page,
631                                    void *cpu_addr, size_t size)
632 {
633         if (PageHighMem(page))
634                 __dma_free_remap(cpu_addr, size);
635         else
636                 __dma_remap(page, size, PAGE_KERNEL);
637         dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT);
638 }
639 
640 static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot)
641 {
642         prot = dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs) ?
643                             pgprot_writecombine(prot) :
644                             pgprot_dmacoherent(prot);
645         return prot;
646 }
647 
648 #define nommu() 0
649 
650 #else   /* !CONFIG_MMU */
651 
652 #define nommu() 1
653 
654 #define __get_dma_pgprot(attrs, prot)   __pgprot(0)
655 #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c)      NULL
656 #define __alloc_from_pool(size, ret_page)                       NULL
657 #define __alloc_from_contiguous(dev, size, prot, ret, c)        NULL
658 #define __free_from_pool(cpu_addr, size)                        0
659 #define __free_from_contiguous(dev, page, cpu_addr, size)       do { } while (0)
660 #define __dma_free_remap(cpu_addr, size)                        do { } while (0)
661 
662 #endif  /* CONFIG_MMU */
663 
664 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp,
665                                    struct page **ret_page)
666 {
667         struct page *page;
668         page = __dma_alloc_buffer(dev, size, gfp);
669         if (!page)
670                 return NULL;
671 
672         *ret_page = page;
673         return page_address(page);
674 }
675 
676 
677 
678 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
679                          gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller)
680 {
681         u64 mask = get_coherent_dma_mask(dev);
682         struct page *page = NULL;
683         void *addr;
684 
685 #ifdef CONFIG_DMA_API_DEBUG
686         u64 limit = (mask + 1) & ~mask;
687         if (limit && size >= limit) {
688                 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
689                         size, mask);
690                 return NULL;
691         }
692 #endif
693 
694         if (!mask)
695                 return NULL;
696 
697         if (mask < 0xffffffffULL)
698                 gfp |= GFP_DMA;
699 
700         /*
701          * Following is a work-around (a.k.a. hack) to prevent pages
702          * with __GFP_COMP being passed to split_page() which cannot
703          * handle them.  The real problem is that this flag probably
704          * should be 0 on ARM as it is not supported on this
705          * platform; see CONFIG_HUGETLBFS.
706          */
707         gfp &= ~(__GFP_COMP);
708 
709         *handle = DMA_ERROR_CODE;
710         size = PAGE_ALIGN(size);
711 
712         if (is_coherent || nommu())
713                 addr = __alloc_simple_buffer(dev, size, gfp, &page);
714         else if (!(gfp & __GFP_WAIT))
715                 addr = __alloc_from_pool(size, &page);
716         else if (!IS_ENABLED(CONFIG_DMA_CMA))
717                 addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller);
718         else
719                 addr = __alloc_from_contiguous(dev, size, prot, &page, caller);
720 
721         if (addr)
722                 *handle = pfn_to_dma(dev, page_to_pfn(page));
723 
724         return addr;
725 }
726 
727 /*
728  * Allocate DMA-coherent memory space and return both the kernel remapped
729  * virtual and bus address for that space.
730  */
731 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
732                     gfp_t gfp, struct dma_attrs *attrs)
733 {
734         pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
735         void *memory;
736 
737         if (dma_alloc_from_coherent(dev, size, handle, &memory))
738                 return memory;
739 
740         return __dma_alloc(dev, size, handle, gfp, prot, false,
741                            __builtin_return_address(0));
742 }
743 
744 static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
745         dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
746 {
747         pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
748         void *memory;
749 
750         if (dma_alloc_from_coherent(dev, size, handle, &memory))
751                 return memory;
752 
753         return __dma_alloc(dev, size, handle, gfp, prot, true,
754                            __builtin_return_address(0));
755 }
756 
757 /*
758  * Create userspace mapping for the DMA-coherent memory.
759  */
760 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
761                  void *cpu_addr, dma_addr_t dma_addr, size_t size,
762                  struct dma_attrs *attrs)
763 {
764         int ret = -ENXIO;
765 #ifdef CONFIG_MMU
766         unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
767         unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
768         unsigned long pfn = dma_to_pfn(dev, dma_addr);
769         unsigned long off = vma->vm_pgoff;
770 
771         vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
772 
773         if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
774                 return ret;
775 
776         if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
777                 ret = remap_pfn_range(vma, vma->vm_start,
778                                       pfn + off,
779                                       vma->vm_end - vma->vm_start,
780                                       vma->vm_page_prot);
781         }
782 #endif  /* CONFIG_MMU */
783 
784         return ret;
785 }
786 
787 /*
788  * Free a buffer as defined by the above mapping.
789  */
790 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
791                            dma_addr_t handle, struct dma_attrs *attrs,
792                            bool is_coherent)
793 {
794         struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
795 
796         if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
797                 return;
798 
799         size = PAGE_ALIGN(size);
800 
801         if (is_coherent || nommu()) {
802                 __dma_free_buffer(page, size);
803         } else if (__free_from_pool(cpu_addr, size)) {
804                 return;
805         } else if (!IS_ENABLED(CONFIG_DMA_CMA)) {
806                 __dma_free_remap(cpu_addr, size);
807                 __dma_free_buffer(page, size);
808         } else {
809                 /*
810                  * Non-atomic allocations cannot be freed with IRQs disabled
811                  */
812                 WARN_ON(irqs_disabled());
813                 __free_from_contiguous(dev, page, cpu_addr, size);
814         }
815 }
816 
817 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
818                   dma_addr_t handle, struct dma_attrs *attrs)
819 {
820         __arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
821 }
822 
823 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
824                                   dma_addr_t handle, struct dma_attrs *attrs)
825 {
826         __arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
827 }
828 
829 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
830                  void *cpu_addr, dma_addr_t handle, size_t size,
831                  struct dma_attrs *attrs)
832 {
833         struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
834         int ret;
835 
836         ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
837         if (unlikely(ret))
838                 return ret;
839 
840         sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
841         return 0;
842 }
843 
844 static void dma_cache_maint_page(struct page *page, unsigned long offset,
845         size_t size, enum dma_data_direction dir,
846         void (*op)(const void *, size_t, int))
847 {
848         unsigned long pfn;
849         size_t left = size;
850 
851         pfn = page_to_pfn(page) + offset / PAGE_SIZE;
852         offset %= PAGE_SIZE;
853 
854         /*
855          * A single sg entry may refer to multiple physically contiguous
856          * pages.  But we still need to process highmem pages individually.
857          * If highmem is not configured then the bulk of this loop gets
858          * optimized out.
859          */
860         do {
861                 size_t len = left;
862                 void *vaddr;
863 
864                 page = pfn_to_page(pfn);
865 
866                 if (PageHighMem(page)) {
867                         if (len + offset > PAGE_SIZE)
868                                 len = PAGE_SIZE - offset;
869 
870                         if (cache_is_vipt_nonaliasing()) {
871                                 vaddr = kmap_atomic(page);
872                                 op(vaddr + offset, len, dir);
873                                 kunmap_atomic(vaddr);
874                         } else {
875                                 vaddr = kmap_high_get(page);
876                                 if (vaddr) {
877                                         op(vaddr + offset, len, dir);
878                                         kunmap_high(page);
879                                 }
880                         }
881                 } else {
882                         vaddr = page_address(page) + offset;
883                         op(vaddr, len, dir);
884                 }
885                 offset = 0;
886                 pfn++;
887                 left -= len;
888         } while (left);
889 }
890 
891 /*
892  * Make an area consistent for devices.
893  * Note: Drivers should NOT use this function directly, as it will break
894  * platforms with CONFIG_DMABOUNCE.
895  * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
896  */
897 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off,
898         size_t size, enum dma_data_direction dir)
899 {
900         unsigned long paddr;
901 
902         dma_cache_maint_page(page, off, size, dir, dmac_map_area);
903 
904         paddr = page_to_phys(page) + off;
905         if (dir == DMA_FROM_DEVICE) {
906                 outer_inv_range(paddr, paddr + size);
907         } else {
908                 outer_clean_range(paddr, paddr + size);
909         }
910         /* FIXME: non-speculating: flush on bidirectional mappings? */
911 }
912 
913 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
914         size_t size, enum dma_data_direction dir)
915 {
916         unsigned long paddr = page_to_phys(page) + off;
917 
918         /* FIXME: non-speculating: not required */
919         /* don't bother invalidating if DMA to device */
920         if (dir != DMA_TO_DEVICE)
921                 outer_inv_range(paddr, paddr + size);
922 
923         dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
924 
925         /*
926          * Mark the D-cache clean for these pages to avoid extra flushing.
927          */
928         if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) {
929                 unsigned long pfn;
930                 size_t left = size;
931 
932                 pfn = page_to_pfn(page) + off / PAGE_SIZE;
933                 off %= PAGE_SIZE;
934                 if (off) {
935                         pfn++;
936                         left -= PAGE_SIZE - off;
937                 }
938                 while (left >= PAGE_SIZE) {
939                         page = pfn_to_page(pfn++);
940                         set_bit(PG_dcache_clean, &page->flags);
941                         left -= PAGE_SIZE;
942                 }
943         }
944 }
945 
946 /**
947  * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA
948  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
949  * @sg: list of buffers
950  * @nents: number of buffers to map
951  * @dir: DMA transfer direction
952  *
953  * Map a set of buffers described by scatterlist in streaming mode for DMA.
954  * This is the scatter-gather version of the dma_map_single interface.
955  * Here the scatter gather list elements are each tagged with the
956  * appropriate dma address and length.  They are obtained via
957  * sg_dma_{address,length}.
958  *
959  * Device ownership issues as mentioned for dma_map_single are the same
960  * here.
961  */
962 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
963                 enum dma_data_direction dir, struct dma_attrs *attrs)
964 {
965         struct dma_map_ops *ops = get_dma_ops(dev);
966         struct scatterlist *s;
967         int i, j;
968 
969         for_each_sg(sg, s, nents, i) {
970 #ifdef CONFIG_NEED_SG_DMA_LENGTH
971                 s->dma_length = s->length;
972 #endif
973                 s->dma_address = ops->map_page(dev, sg_page(s), s->offset,
974                                                 s->length, dir, attrs);
975                 if (dma_mapping_error(dev, s->dma_address))
976                         goto bad_mapping;
977         }
978         return nents;
979 
980  bad_mapping:
981         for_each_sg(sg, s, i, j)
982                 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
983         return 0;
984 }
985 
986 /**
987  * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
988  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
989  * @sg: list of buffers
990  * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
991  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
992  *
993  * Unmap a set of streaming mode DMA translations.  Again, CPU access
994  * rules concerning calls here are the same as for dma_unmap_single().
995  */
996 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
997                 enum dma_data_direction dir, struct dma_attrs *attrs)
998 {
999         struct dma_map_ops *ops = get_dma_ops(dev);
1000         struct scatterlist *s;
1001 
1002         int i;
1003 
1004         for_each_sg(sg, s, nents, i)
1005                 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs);
1006 }
1007 
1008 /**
1009  * arm_dma_sync_sg_for_cpu
1010  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1011  * @sg: list of buffers
1012  * @nents: number of buffers to map (returned from dma_map_sg)
1013  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1014  */
1015 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1016                         int nents, enum dma_data_direction dir)
1017 {
1018         struct dma_map_ops *ops = get_dma_ops(dev);
1019         struct scatterlist *s;
1020         int i;
1021 
1022         for_each_sg(sg, s, nents, i)
1023                 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length,
1024                                          dir);
1025 }
1026 
1027 /**
1028  * arm_dma_sync_sg_for_device
1029  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
1030  * @sg: list of buffers
1031  * @nents: number of buffers to map (returned from dma_map_sg)
1032  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1033  */
1034 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1035                         int nents, enum dma_data_direction dir)
1036 {
1037         struct dma_map_ops *ops = get_dma_ops(dev);
1038         struct scatterlist *s;
1039         int i;
1040 
1041         for_each_sg(sg, s, nents, i)
1042                 ops->sync_single_for_device(dev, sg_dma_address(s), s->length,
1043                                             dir);
1044 }
1045 
1046 /*
1047  * Return whether the given device DMA address mask can be supported
1048  * properly.  For example, if your device can only drive the low 24-bits
1049  * during bus mastering, then you would pass 0x00ffffff as the mask
1050  * to this function.
1051  */
1052 int dma_supported(struct device *dev, u64 mask)
1053 {
1054         return __dma_supported(dev, mask, false);
1055 }
1056 EXPORT_SYMBOL(dma_supported);
1057 
1058 int arm_dma_set_mask(struct device *dev, u64 dma_mask)
1059 {
1060         if (!dev->dma_mask || !dma_supported(dev, dma_mask))
1061                 return -EIO;
1062 
1063         *dev->dma_mask = dma_mask;
1064 
1065         return 0;
1066 }
1067 
1068 #define PREALLOC_DMA_DEBUG_ENTRIES      4096
1069 
1070 static int __init dma_debug_do_init(void)
1071 {
1072         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
1073         return 0;
1074 }
1075 fs_initcall(dma_debug_do_init);
1076 
1077 #ifdef CONFIG_ARM_DMA_USE_IOMMU
1078 
1079 /* IOMMU */
1080 
1081 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,
1082                                       size_t size)
1083 {
1084         unsigned int order = get_order(size);
1085         unsigned int align = 0;
1086         unsigned int count, start;
1087         unsigned long flags;
1088 
1089         if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT)
1090                 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT;
1091 
1092         count = ((PAGE_ALIGN(size) >> PAGE_SHIFT) +
1093                  (1 << mapping->order) - 1) >> mapping->order;
1094 
1095         if (order > mapping->order)
1096                 align = (1 << (order - mapping->order)) - 1;
1097 
1098         spin_lock_irqsave(&mapping->lock, flags);
1099         start = bitmap_find_next_zero_area(mapping->bitmap, mapping->bits, 0,
1100                                            count, align);
1101         if (start > mapping->bits) {
1102                 spin_unlock_irqrestore(&mapping->lock, flags);
1103                 return DMA_ERROR_CODE;
1104         }
1105 
1106         bitmap_set(mapping->bitmap, start, count);
1107         spin_unlock_irqrestore(&mapping->lock, flags);
1108 
1109         return mapping->base + (start << (mapping->order + PAGE_SHIFT));
1110 }
1111 
1112 static inline void __free_iova(struct dma_iommu_mapping *mapping,
1113                                dma_addr_t addr, size_t size)
1114 {
1115         unsigned int start = (addr - mapping->base) >>
1116                              (mapping->order + PAGE_SHIFT);
1117         unsigned int count = ((size >> PAGE_SHIFT) +
1118                               (1 << mapping->order) - 1) >> mapping->order;
1119         unsigned long flags;
1120 
1121         spin_lock_irqsave(&mapping->lock, flags);
1122         bitmap_clear(mapping->bitmap, start, count);
1123         spin_unlock_irqrestore(&mapping->lock, flags);
1124 }
1125 
1126 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size,
1127                                           gfp_t gfp, struct dma_attrs *attrs)
1128 {
1129         struct page **pages;
1130         int count = size >> PAGE_SHIFT;
1131         int array_size = count * sizeof(struct page *);
1132         int i = 0;
1133 
1134         if (array_size <= PAGE_SIZE)
1135                 pages = kzalloc(array_size, gfp);
1136         else
1137                 pages = vzalloc(array_size);
1138         if (!pages)
1139                 return NULL;
1140 
1141         if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs))
1142         {
1143                 unsigned long order = get_order(size);
1144                 struct page *page;
1145 
1146                 page = dma_alloc_from_contiguous(dev, count, order);
1147                 if (!page)
1148                         goto error;
1149 
1150                 __dma_clear_buffer(page, size);
1151 
1152                 for (i = 0; i < count; i++)
1153                         pages[i] = page + i;
1154 
1155                 return pages;
1156         }
1157 
1158         /*
1159          * IOMMU can map any pages, so himem can also be used here
1160          */
1161         gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
1162 
1163         while (count) {
1164                 int j, order = __fls(count);
1165 
1166                 pages[i] = alloc_pages(gfp, order);
1167                 while (!pages[i] && order)
1168                         pages[i] = alloc_pages(gfp, --order);
1169                 if (!pages[i])
1170                         goto error;
1171 
1172                 if (order) {
1173                         split_page(pages[i], order);
1174                         j = 1 << order;
1175                         while (--j)
1176                                 pages[i + j] = pages[i] + j;
1177                 }
1178 
1179                 __dma_clear_buffer(pages[i], PAGE_SIZE << order);
1180                 i += 1 << order;
1181                 count -= 1 << order;
1182         }
1183 
1184         return pages;
1185 error:
1186         while (i--)
1187                 if (pages[i])
1188                         __free_pages(pages[i], 0);
1189         if (array_size <= PAGE_SIZE)
1190                 kfree(pages);
1191         else
1192                 vfree(pages);
1193         return NULL;
1194 }
1195 
1196 static int __iommu_free_buffer(struct device *dev, struct page **pages,
1197                                size_t size, struct dma_attrs *attrs)
1198 {
1199         int count = size >> PAGE_SHIFT;
1200         int array_size = count * sizeof(struct page *);
1201         int i;
1202 
1203         if (dma_get_attr(DMA_ATTR_FORCE_CONTIGUOUS, attrs)) {
1204                 dma_release_from_contiguous(dev, pages[0], count);
1205         } else {
1206                 for (i = 0; i < count; i++)
1207                         if (pages[i])
1208                                 __free_pages(pages[i], 0);
1209         }
1210 
1211         if (array_size <= PAGE_SIZE)
1212                 kfree(pages);
1213         else
1214                 vfree(pages);
1215         return 0;
1216 }
1217 
1218 /*
1219  * Create a CPU mapping for a specified pages
1220  */
1221 static void *
1222 __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot,
1223                     const void *caller)
1224 {
1225         unsigned int i, nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1226         struct vm_struct *area;
1227         unsigned long p;
1228 
1229         area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP,
1230                                   caller);
1231         if (!area)
1232                 return NULL;
1233 
1234         area->pages = pages;
1235         area->nr_pages = nr_pages;
1236         p = (unsigned long)area->addr;
1237 
1238         for (i = 0; i < nr_pages; i++) {
1239                 phys_addr_t phys = __pfn_to_phys(page_to_pfn(pages[i]));
1240                 if (ioremap_page_range(p, p + PAGE_SIZE, phys, prot))
1241                         goto err;
1242                 p += PAGE_SIZE;
1243         }
1244         return area->addr;
1245 err:
1246         unmap_kernel_range((unsigned long)area->addr, size);
1247         vunmap(area->addr);
1248         return NULL;
1249 }
1250 
1251 /*
1252  * Create a mapping in device IO address space for specified pages
1253  */
1254 static dma_addr_t
1255 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size)
1256 {
1257         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1258         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1259         dma_addr_t dma_addr, iova;
1260         int i, ret = DMA_ERROR_CODE;
1261 
1262         dma_addr = __alloc_iova(mapping, size);
1263         if (dma_addr == DMA_ERROR_CODE)
1264                 return dma_addr;
1265 
1266         iova = dma_addr;
1267         for (i = 0; i < count; ) {
1268                 unsigned int next_pfn = page_to_pfn(pages[i]) + 1;
1269                 phys_addr_t phys = page_to_phys(pages[i]);
1270                 unsigned int len, j;
1271 
1272                 for (j = i + 1; j < count; j++, next_pfn++)
1273                         if (page_to_pfn(pages[j]) != next_pfn)
1274                                 break;
1275 
1276                 len = (j - i) << PAGE_SHIFT;
1277                 ret = iommu_map(mapping->domain, iova, phys, len,
1278                                 IOMMU_READ|IOMMU_WRITE);
1279                 if (ret < 0)
1280                         goto fail;
1281                 iova += len;
1282                 i = j;
1283         }
1284         return dma_addr;
1285 fail:
1286         iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);
1287         __free_iova(mapping, dma_addr, size);
1288         return DMA_ERROR_CODE;
1289 }
1290 
1291 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size)
1292 {
1293         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1294 
1295         /*
1296          * add optional in-page offset from iova to size and align
1297          * result to page size
1298          */
1299         size = PAGE_ALIGN((iova & ~PAGE_MASK) + size);
1300         iova &= PAGE_MASK;
1301 
1302         iommu_unmap(mapping->domain, iova, size);
1303         __free_iova(mapping, iova, size);
1304         return 0;
1305 }
1306 
1307 static struct page **__atomic_get_pages(void *addr)
1308 {
1309         struct dma_pool *pool = &atomic_pool;
1310         struct page **pages = pool->pages;
1311         int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
1312 
1313         return pages + offs;
1314 }
1315 
1316 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)
1317 {
1318         struct vm_struct *area;
1319 
1320         if (__in_atomic_pool(cpu_addr, PAGE_SIZE))
1321                 return __atomic_get_pages(cpu_addr);
1322 
1323         if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1324                 return cpu_addr;
1325 
1326         area = find_vm_area(cpu_addr);
1327         if (area && (area->flags & VM_ARM_DMA_CONSISTENT))
1328                 return area->pages;
1329         return NULL;
1330 }
1331 
1332 static void *__iommu_alloc_atomic(struct device *dev, size_t size,
1333                                   dma_addr_t *handle)
1334 {
1335         struct page *page;
1336         void *addr;
1337 
1338         addr = __alloc_from_pool(size, &page);
1339         if (!addr)
1340                 return NULL;
1341 
1342         *handle = __iommu_create_mapping(dev, &page, size);
1343         if (*handle == DMA_ERROR_CODE)
1344                 goto err_mapping;
1345 
1346         return addr;
1347 
1348 err_mapping:
1349         __free_from_pool(addr, size);
1350         return NULL;
1351 }
1352 
1353 static void __iommu_free_atomic(struct device *dev, void *cpu_addr,
1354                                 dma_addr_t handle, size_t size)
1355 {
1356         __iommu_remove_mapping(dev, handle, size);
1357         __free_from_pool(cpu_addr, size);
1358 }
1359 
1360 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
1361             dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
1362 {
1363         pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
1364         struct page **pages;
1365         void *addr = NULL;
1366 
1367         *handle = DMA_ERROR_CODE;
1368         size = PAGE_ALIGN(size);
1369 
1370         if (!(gfp & __GFP_WAIT))
1371                 return __iommu_alloc_atomic(dev, size, handle);
1372 
1373         /*
1374          * Following is a work-around (a.k.a. hack) to prevent pages
1375          * with __GFP_COMP being passed to split_page() which cannot
1376          * handle them.  The real problem is that this flag probably
1377          * should be 0 on ARM as it is not supported on this
1378          * platform; see CONFIG_HUGETLBFS.
1379          */
1380         gfp &= ~(__GFP_COMP);
1381 
1382         pages = __iommu_alloc_buffer(dev, size, gfp, attrs);
1383         if (!pages)
1384                 return NULL;
1385 
1386         *handle = __iommu_create_mapping(dev, pages, size);
1387         if (*handle == DMA_ERROR_CODE)
1388                 goto err_buffer;
1389 
1390         if (dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs))
1391                 return pages;
1392 
1393         addr = __iommu_alloc_remap(pages, size, gfp, prot,
1394                                    __builtin_return_address(0));
1395         if (!addr)
1396                 goto err_mapping;
1397 
1398         return addr;
1399 
1400 err_mapping:
1401         __iommu_remove_mapping(dev, *handle, size);
1402 err_buffer:
1403         __iommu_free_buffer(dev, pages, size, attrs);
1404         return NULL;
1405 }
1406 
1407 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
1408                     void *cpu_addr, dma_addr_t dma_addr, size_t size,
1409                     struct dma_attrs *attrs)
1410 {
1411         unsigned long uaddr = vma->vm_start;
1412         unsigned long usize = vma->vm_end - vma->vm_start;
1413         struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1414         unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1415         unsigned long off = vma->vm_pgoff;
1416 
1417         vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
1418 
1419         if (!pages)
1420                 return -ENXIO;
1421 
1422         if (off >= nr_pages || (usize >> PAGE_SHIFT) > nr_pages - off)
1423                 return -ENXIO;
1424 
1425         pages += off;
1426 
1427         do {
1428                 int ret = vm_insert_page(vma, uaddr, *pages++);
1429                 if (ret) {
1430                         pr_err("Remapping memory failed: %d\n", ret);
1431                         return ret;
1432                 }
1433                 uaddr += PAGE_SIZE;
1434                 usize -= PAGE_SIZE;
1435         } while (usize > 0);
1436 
1437         return 0;
1438 }
1439 
1440 /*
1441  * free a page as defined by the above mapping.
1442  * Must not be called with IRQs disabled.
1443  */
1444 void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
1445                           dma_addr_t handle, struct dma_attrs *attrs)
1446 {
1447         struct page **pages;
1448         size = PAGE_ALIGN(size);
1449 
1450         if (__in_atomic_pool(cpu_addr, size)) {
1451                 __iommu_free_atomic(dev, cpu_addr, handle, size);
1452                 return;
1453         }
1454 
1455         pages = __iommu_get_pages(cpu_addr, attrs);
1456         if (!pages) {
1457                 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
1458                 return;
1459         }
1460 
1461         if (!dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs)) {
1462                 unmap_kernel_range((unsigned long)cpu_addr, size);
1463                 vunmap(cpu_addr);
1464         }
1465 
1466         __iommu_remove_mapping(dev, handle, size);
1467         __iommu_free_buffer(dev, pages, size, attrs);
1468 }
1469 
1470 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt,
1471                                  void *cpu_addr, dma_addr_t dma_addr,
1472                                  size_t size, struct dma_attrs *attrs)
1473 {
1474         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
1475         struct page **pages = __iommu_get_pages(cpu_addr, attrs);
1476 
1477         if (!pages)
1478                 return -ENXIO;
1479 
1480         return sg_alloc_table_from_pages(sgt, pages, count, 0, size,
1481                                          GFP_KERNEL);
1482 }
1483 
1484 static int __dma_direction_to_prot(enum dma_data_direction dir)
1485 {
1486         int prot;
1487 
1488         switch (dir) {
1489         case DMA_BIDIRECTIONAL:
1490                 prot = IOMMU_READ | IOMMU_WRITE;
1491                 break;
1492         case DMA_TO_DEVICE:
1493                 prot = IOMMU_READ;
1494                 break;
1495         case DMA_FROM_DEVICE:
1496                 prot = IOMMU_WRITE;
1497                 break;
1498         default:
1499                 prot = 0;
1500         }
1501 
1502         return prot;
1503 }
1504 
1505 /*
1506  * Map a part of the scatter-gather list into contiguous io address space
1507  */
1508 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
1509                           size_t size, dma_addr_t *handle,
1510                           enum dma_data_direction dir, struct dma_attrs *attrs,
1511                           bool is_coherent)
1512 {
1513         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1514         dma_addr_t iova, iova_base;
1515         int ret = 0;
1516         unsigned int count;
1517         struct scatterlist *s;
1518         int prot;
1519 
1520         size = PAGE_ALIGN(size);
1521         *handle = DMA_ERROR_CODE;
1522 
1523         iova_base = iova = __alloc_iova(mapping, size);
1524         if (iova == DMA_ERROR_CODE)
1525                 return -ENOMEM;
1526 
1527         for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) {
1528                 phys_addr_t phys = page_to_phys(sg_page(s));
1529                 unsigned int len = PAGE_ALIGN(s->offset + s->length);
1530 
1531                 if (!is_coherent &&
1532                         !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1533                         __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1534 
1535                 prot = __dma_direction_to_prot(dir);
1536 
1537                 ret = iommu_map(mapping->domain, iova, phys, len, prot);
1538                 if (ret < 0)
1539                         goto fail;
1540                 count += len >> PAGE_SHIFT;
1541                 iova += len;
1542         }
1543         *handle = iova_base;
1544 
1545         return 0;
1546 fail:
1547         iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);
1548         __free_iova(mapping, iova_base, size);
1549         return ret;
1550 }
1551 
1552 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents,
1553                      enum dma_data_direction dir, struct dma_attrs *attrs,
1554                      bool is_coherent)
1555 {
1556         struct scatterlist *s = sg, *dma = sg, *start = sg;
1557         int i, count = 0;
1558         unsigned int offset = s->offset;
1559         unsigned int size = s->offset + s->length;
1560         unsigned int max = dma_get_max_seg_size(dev);
1561 
1562         for (i = 1; i < nents; i++) {
1563                 s = sg_next(s);
1564 
1565                 s->dma_address = DMA_ERROR_CODE;
1566                 s->dma_length = 0;
1567 
1568                 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) {
1569                         if (__map_sg_chunk(dev, start, size, &dma->dma_address,
1570                             dir, attrs, is_coherent) < 0)
1571                                 goto bad_mapping;
1572 
1573                         dma->dma_address += offset;
1574                         dma->dma_length = size - offset;
1575 
1576                         size = offset = s->offset;
1577                         start = s;
1578                         dma = sg_next(dma);
1579                         count += 1;
1580                 }
1581                 size += s->length;
1582         }
1583         if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs,
1584                 is_coherent) < 0)
1585                 goto bad_mapping;
1586 
1587         dma->dma_address += offset;
1588         dma->dma_length = size - offset;
1589 
1590         return count+1;
1591 
1592 bad_mapping:
1593         for_each_sg(sg, s, count, i)
1594                 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s));
1595         return 0;
1596 }
1597 
1598 /**
1599  * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1600  * @dev: valid struct device pointer
1601  * @sg: list of buffers
1602  * @nents: number of buffers to map
1603  * @dir: DMA transfer direction
1604  *
1605  * Map a set of i/o coherent buffers described by scatterlist in streaming
1606  * mode for DMA. The scatter gather list elements are merged together (if
1607  * possible) and tagged with the appropriate dma address and length. They are
1608  * obtained via sg_dma_{address,length}.
1609  */
1610 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1611                 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1612 {
1613         return __iommu_map_sg(dev, sg, nents, dir, attrs, true);
1614 }
1615 
1616 /**
1617  * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA
1618  * @dev: valid struct device pointer
1619  * @sg: list of buffers
1620  * @nents: number of buffers to map
1621  * @dir: DMA transfer direction
1622  *
1623  * Map a set of buffers described by scatterlist in streaming mode for DMA.
1624  * The scatter gather list elements are merged together (if possible) and
1625  * tagged with the appropriate dma address and length. They are obtained via
1626  * sg_dma_{address,length}.
1627  */
1628 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg,
1629                 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1630 {
1631         return __iommu_map_sg(dev, sg, nents, dir, attrs, false);
1632 }
1633 
1634 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1635                 int nents, enum dma_data_direction dir, struct dma_attrs *attrs,
1636                 bool is_coherent)
1637 {
1638         struct scatterlist *s;
1639         int i;
1640 
1641         for_each_sg(sg, s, nents, i) {
1642                 if (sg_dma_len(s))
1643                         __iommu_remove_mapping(dev, sg_dma_address(s),
1644                                                sg_dma_len(s));
1645                 if (!is_coherent &&
1646                     !dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1647                         __dma_page_dev_to_cpu(sg_page(s), s->offset,
1648                                               s->length, dir);
1649         }
1650 }
1651 
1652 /**
1653  * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1654  * @dev: valid struct device pointer
1655  * @sg: list of buffers
1656  * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1657  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1658  *
1659  * Unmap a set of streaming mode DMA translations.  Again, CPU access
1660  * rules concerning calls here are the same as for dma_unmap_single().
1661  */
1662 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg,
1663                 int nents, enum dma_data_direction dir, struct dma_attrs *attrs)
1664 {
1665         __iommu_unmap_sg(dev, sg, nents, dir, attrs, true);
1666 }
1667 
1668 /**
1669  * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
1670  * @dev: valid struct device pointer
1671  * @sg: list of buffers
1672  * @nents: number of buffers to unmap (same as was passed to dma_map_sg)
1673  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1674  *
1675  * Unmap a set of streaming mode DMA translations.  Again, CPU access
1676  * rules concerning calls here are the same as for dma_unmap_single().
1677  */
1678 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
1679                         enum dma_data_direction dir, struct dma_attrs *attrs)
1680 {
1681         __iommu_unmap_sg(dev, sg, nents, dir, attrs, false);
1682 }
1683 
1684 /**
1685  * arm_iommu_sync_sg_for_cpu
1686  * @dev: valid struct device pointer
1687  * @sg: list of buffers
1688  * @nents: number of buffers to map (returned from dma_map_sg)
1689  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1690  */
1691 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1692                         int nents, enum dma_data_direction dir)
1693 {
1694         struct scatterlist *s;
1695         int i;
1696 
1697         for_each_sg(sg, s, nents, i)
1698                 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir);
1699 
1700 }
1701 
1702 /**
1703  * arm_iommu_sync_sg_for_device
1704  * @dev: valid struct device pointer
1705  * @sg: list of buffers
1706  * @nents: number of buffers to map (returned from dma_map_sg)
1707  * @dir: DMA transfer direction (same as was passed to dma_map_sg)
1708  */
1709 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1710                         int nents, enum dma_data_direction dir)
1711 {
1712         struct scatterlist *s;
1713         int i;
1714 
1715         for_each_sg(sg, s, nents, i)
1716                 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir);
1717 }
1718 
1719 
1720 /**
1721  * arm_coherent_iommu_map_page
1722  * @dev: valid struct device pointer
1723  * @page: page that buffer resides in
1724  * @offset: offset into page for start of buffer
1725  * @size: size of buffer to map
1726  * @dir: DMA transfer direction
1727  *
1728  * Coherent IOMMU aware version of arm_dma_map_page()
1729  */
1730 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page,
1731              unsigned long offset, size_t size, enum dma_data_direction dir,
1732              struct dma_attrs *attrs)
1733 {
1734         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1735         dma_addr_t dma_addr;
1736         int ret, prot, len = PAGE_ALIGN(size + offset);
1737 
1738         dma_addr = __alloc_iova(mapping, len);
1739         if (dma_addr == DMA_ERROR_CODE)
1740                 return dma_addr;
1741 
1742         prot = __dma_direction_to_prot(dir);
1743 
1744         ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);
1745         if (ret < 0)
1746                 goto fail;
1747 
1748         return dma_addr + offset;
1749 fail:
1750         __free_iova(mapping, dma_addr, len);
1751         return DMA_ERROR_CODE;
1752 }
1753 
1754 /**
1755  * arm_iommu_map_page
1756  * @dev: valid struct device pointer
1757  * @page: page that buffer resides in
1758  * @offset: offset into page for start of buffer
1759  * @size: size of buffer to map
1760  * @dir: DMA transfer direction
1761  *
1762  * IOMMU aware version of arm_dma_map_page()
1763  */
1764 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page,
1765              unsigned long offset, size_t size, enum dma_data_direction dir,
1766              struct dma_attrs *attrs)
1767 {
1768         if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1769                 __dma_page_cpu_to_dev(page, offset, size, dir);
1770 
1771         return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs);
1772 }
1773 
1774 /**
1775  * arm_coherent_iommu_unmap_page
1776  * @dev: valid struct device pointer
1777  * @handle: DMA address of buffer
1778  * @size: size of buffer (same as passed to dma_map_page)
1779  * @dir: DMA transfer direction (same as passed to dma_map_page)
1780  *
1781  * Coherent IOMMU aware version of arm_dma_unmap_page()
1782  */
1783 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1784                 size_t size, enum dma_data_direction dir,
1785                 struct dma_attrs *attrs)
1786 {
1787         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1788         dma_addr_t iova = handle & PAGE_MASK;
1789         int offset = handle & ~PAGE_MASK;
1790         int len = PAGE_ALIGN(size + offset);
1791 
1792         if (!iova)
1793                 return;
1794 
1795         iommu_unmap(mapping->domain, iova, len);
1796         __free_iova(mapping, iova, len);
1797 }
1798 
1799 /**
1800  * arm_iommu_unmap_page
1801  * @dev: valid struct device pointer
1802  * @handle: DMA address of buffer
1803  * @size: size of buffer (same as passed to dma_map_page)
1804  * @dir: DMA transfer direction (same as passed to dma_map_page)
1805  *
1806  * IOMMU aware version of arm_dma_unmap_page()
1807  */
1808 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle,
1809                 size_t size, enum dma_data_direction dir,
1810                 struct dma_attrs *attrs)
1811 {
1812         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1813         dma_addr_t iova = handle & PAGE_MASK;
1814         struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1815         int offset = handle & ~PAGE_MASK;
1816         int len = PAGE_ALIGN(size + offset);
1817 
1818         if (!iova)
1819                 return;
1820 
1821         if (!dma_get_attr(DMA_ATTR_SKIP_CPU_SYNC, attrs))
1822                 __dma_page_dev_to_cpu(page, offset, size, dir);
1823 
1824         iommu_unmap(mapping->domain, iova, len);
1825         __free_iova(mapping, iova, len);
1826 }
1827 
1828 static void arm_iommu_sync_single_for_cpu(struct device *dev,
1829                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1830 {
1831         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1832         dma_addr_t iova = handle & PAGE_MASK;
1833         struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1834         unsigned int offset = handle & ~PAGE_MASK;
1835 
1836         if (!iova)
1837                 return;
1838 
1839         __dma_page_dev_to_cpu(page, offset, size, dir);
1840 }
1841 
1842 static void arm_iommu_sync_single_for_device(struct device *dev,
1843                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
1844 {
1845         struct dma_iommu_mapping *mapping = dev->archdata.mapping;
1846         dma_addr_t iova = handle & PAGE_MASK;
1847         struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));
1848         unsigned int offset = handle & ~PAGE_MASK;
1849 
1850         if (!iova)
1851                 return;
1852 
1853         __dma_page_cpu_to_dev(page, offset, size, dir);
1854 }
1855 
1856 struct dma_map_ops iommu_ops = {
1857         .alloc          = arm_iommu_alloc_attrs,
1858         .free           = arm_iommu_free_attrs,
1859         .mmap           = arm_iommu_mmap_attrs,
1860         .get_sgtable    = arm_iommu_get_sgtable,
1861 
1862         .map_page               = arm_iommu_map_page,
1863         .unmap_page             = arm_iommu_unmap_page,
1864         .sync_single_for_cpu    = arm_iommu_sync_single_for_cpu,
1865         .sync_single_for_device = arm_iommu_sync_single_for_device,
1866 
1867         .map_sg                 = arm_iommu_map_sg,
1868         .unmap_sg               = arm_iommu_unmap_sg,
1869         .sync_sg_for_cpu        = arm_iommu_sync_sg_for_cpu,
1870         .sync_sg_for_device     = arm_iommu_sync_sg_for_device,
1871 
1872         .set_dma_mask           = arm_dma_set_mask,
1873 };
1874 
1875 struct dma_map_ops iommu_coherent_ops = {
1876         .alloc          = arm_iommu_alloc_attrs,
1877         .free           = arm_iommu_free_attrs,
1878         .mmap           = arm_iommu_mmap_attrs,
1879         .get_sgtable    = arm_iommu_get_sgtable,
1880 
1881         .map_page       = arm_coherent_iommu_map_page,
1882         .unmap_page     = arm_coherent_iommu_unmap_page,
1883 
1884         .map_sg         = arm_coherent_iommu_map_sg,
1885         .unmap_sg       = arm_coherent_iommu_unmap_sg,
1886 
1887         .set_dma_mask   = arm_dma_set_mask,
1888 };
1889 
1890 /**
1891  * arm_iommu_create_mapping
1892  * @bus: pointer to the bus holding the client device (for IOMMU calls)
1893  * @base: start address of the valid IO address space
1894  * @size: size of the valid IO address space
1895  * @order: accuracy of the IO addresses allocations
1896  *
1897  * Creates a mapping structure which holds information about used/unused
1898  * IO address ranges, which is required to perform memory allocation and
1899  * mapping with IOMMU aware functions.
1900  *
1901  * The client device need to be attached to the mapping with
1902  * arm_iommu_attach_device function.
1903  */
1904 struct dma_iommu_mapping *
1905 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, size_t size,
1906                          int order)
1907 {
1908         unsigned int count = size >> (PAGE_SHIFT + order);
1909         unsigned int bitmap_size = BITS_TO_LONGS(count) * sizeof(long);
1910         struct dma_iommu_mapping *mapping;
1911         int err = -ENOMEM;
1912 
1913         if (!count)
1914                 return ERR_PTR(-EINVAL);
1915 
1916         mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);
1917         if (!mapping)
1918                 goto err;
1919 
1920         mapping->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
1921         if (!mapping->bitmap)
1922                 goto err2;
1923 
1924         mapping->base = base;
1925         mapping->bits = BITS_PER_BYTE * bitmap_size;
1926         mapping->order = order;
1927         spin_lock_init(&mapping->lock);
1928 
1929         mapping->domain = iommu_domain_alloc(bus);
1930         if (!mapping->domain)
1931                 goto err3;
1932 
1933         kref_init(&mapping->kref);
1934         return mapping;
1935 err3:
1936         kfree(mapping->bitmap);
1937 err2:
1938         kfree(mapping);
1939 err:
1940         return ERR_PTR(err);
1941 }
1942 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping);
1943 
1944 static void release_iommu_mapping(struct kref *kref)
1945 {
1946         struct dma_iommu_mapping *mapping =
1947                 container_of(kref, struct dma_iommu_mapping, kref);
1948 
1949         iommu_domain_free(mapping->domain);
1950         kfree(mapping->bitmap);
1951         kfree(mapping);
1952 }
1953 
1954 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)
1955 {
1956         if (mapping)
1957                 kref_put(&mapping->kref, release_iommu_mapping);
1958 }
1959 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
1960 
1961 /**
1962  * arm_iommu_attach_device
1963  * @dev: valid struct device pointer
1964  * @mapping: io address space mapping structure (returned from
1965  *      arm_iommu_create_mapping)
1966  *
1967  * Attaches specified io address space mapping to the provided device,
1968  * this replaces the dma operations (dma_map_ops pointer) with the
1969  * IOMMU aware version. More than one client might be attached to
1970  * the same io address space mapping.
1971  */
1972 int arm_iommu_attach_device(struct device *dev,
1973                             struct dma_iommu_mapping *mapping)
1974 {
1975         int err;
1976 
1977         err = iommu_attach_device(mapping->domain, dev);
1978         if (err)
1979                 return err;
1980 
1981         kref_get(&mapping->kref);
1982         dev->archdata.mapping = mapping;
1983         set_dma_ops(dev, &iommu_ops);
1984 
1985         pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
1986         return 0;
1987 }
1988 EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
1989 
1990 /**
1991  * arm_iommu_detach_device
1992  * @dev: valid struct device pointer
1993  *
1994  * Detaches the provided device from a previously attached map.
1995  * This voids the dma operations (dma_map_ops pointer)
1996  */
1997 void arm_iommu_detach_device(struct device *dev)
1998 {
1999         struct dma_iommu_mapping *mapping;
2000 
2001         mapping = to_dma_iommu_mapping(dev);
2002         if (!mapping) {
2003                 dev_warn(dev, "Not attached\n");
2004                 return;
2005         }
2006 
2007         iommu_detach_device(mapping->domain, dev);
2008         kref_put(&mapping->kref, release_iommu_mapping);
2009         dev->archdata.mapping = NULL;
2010         set_dma_ops(dev, NULL);
2011 
2012         pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
2013 }
2014 EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
2015 
2016 #endif
2017 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp