~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/metag/kernel/dma.c

Version: ~ [ linux-5.4-rc7 ] ~ [ linux-5.3.10 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.83 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.153 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.200 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.200 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.76 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  Meta version derived from arch/powerpc/lib/dma-noncoherent.c
  3  *    Copyright (C) 2008 Imagination Technologies Ltd.
  4  *
  5  *  PowerPC version derived from arch/arm/mm/consistent.c
  6  *    Copyright (C) 2001 Dan Malek (dmalek@jlc.net)
  7  *
  8  *  Copyright (C) 2000 Russell King
  9  *
 10  * Consistent memory allocators.  Used for DMA devices that want to
 11  * share uncached memory with the processor core.  The function return
 12  * is the virtual address and 'dma_handle' is the physical address.
 13  * Mostly stolen from the ARM port, with some changes for PowerPC.
 14  *                                              -- Dan
 15  *
 16  * Reorganized to get rid of the arch-specific consistent_* functions
 17  * and provide non-coherent implementations for the DMA API. -Matt
 18  *
 19  * Added in_interrupt() safe dma_alloc_coherent()/dma_free_coherent()
 20  * implementation. This is pulled straight from ARM and barely
 21  * modified. -Matt
 22  *
 23  * This program is free software; you can redistribute it and/or modify
 24  * it under the terms of the GNU General Public License version 2 as
 25  * published by the Free Software Foundation.
 26  */
 27 
 28 #include <linux/sched.h>
 29 #include <linux/kernel.h>
 30 #include <linux/errno.h>
 31 #include <linux/export.h>
 32 #include <linux/string.h>
 33 #include <linux/types.h>
 34 #include <linux/highmem.h>
 35 #include <linux/dma-mapping.h>
 36 #include <linux/slab.h>
 37 
 38 #include <asm/tlbflush.h>
 39 #include <asm/mmu.h>
 40 
 41 #define CONSISTENT_OFFSET(x)    (((unsigned long)(x) - CONSISTENT_START) \
 42                                         >> PAGE_SHIFT)
 43 
 44 static u64 get_coherent_dma_mask(struct device *dev)
 45 {
 46         u64 mask = ~0ULL;
 47 
 48         if (dev) {
 49                 mask = dev->coherent_dma_mask;
 50 
 51                 /*
 52                  * Sanity check the DMA mask - it must be non-zero, and
 53                  * must be able to be satisfied by a DMA allocation.
 54                  */
 55                 if (mask == 0) {
 56                         dev_warn(dev, "coherent DMA mask is unset\n");
 57                         return 0;
 58                 }
 59         }
 60 
 61         return mask;
 62 }
 63 /*
 64  * This is the page table (2MB) covering uncached, DMA consistent allocations
 65  */
 66 static pte_t *consistent_pte;
 67 static DEFINE_SPINLOCK(consistent_lock);
 68 
 69 /*
 70  * VM region handling support.
 71  *
 72  * This should become something generic, handling VM region allocations for
 73  * vmalloc and similar (ioremap, module space, etc).
 74  *
 75  * I envisage vmalloc()'s supporting vm_struct becoming:
 76  *
 77  *  struct vm_struct {
 78  *    struct metag_vm_region    region;
 79  *    unsigned long     flags;
 80  *    struct page       **pages;
 81  *    unsigned int      nr_pages;
 82  *    unsigned long     phys_addr;
 83  *  };
 84  *
 85  * get_vm_area() would then call metag_vm_region_alloc with an appropriate
 86  * struct metag_vm_region head (eg):
 87  *
 88  *  struct metag_vm_region vmalloc_head = {
 89  *      .vm_list        = LIST_HEAD_INIT(vmalloc_head.vm_list),
 90  *      .vm_start       = VMALLOC_START,
 91  *      .vm_end         = VMALLOC_END,
 92  *  };
 93  *
 94  * However, vmalloc_head.vm_start is variable (typically, it is dependent on
 95  * the amount of RAM found at boot time.)  I would imagine that get_vm_area()
 96  * would have to initialise this each time prior to calling
 97  * metag_vm_region_alloc().
 98  */
 99 struct metag_vm_region {
100         struct list_head vm_list;
101         unsigned long vm_start;
102         unsigned long vm_end;
103         struct page             *vm_pages;
104         int                     vm_active;
105 };
106 
107 static struct metag_vm_region consistent_head = {
108         .vm_list = LIST_HEAD_INIT(consistent_head.vm_list),
109         .vm_start = CONSISTENT_START,
110         .vm_end = CONSISTENT_END,
111 };
112 
113 static struct metag_vm_region *metag_vm_region_alloc(struct metag_vm_region
114                                                      *head, size_t size,
115                                                      gfp_t gfp)
116 {
117         unsigned long addr = head->vm_start, end = head->vm_end - size;
118         unsigned long flags;
119         struct metag_vm_region *c, *new;
120 
121         new = kmalloc(sizeof(struct metag_vm_region), gfp);
122         if (!new)
123                 goto out;
124 
125         spin_lock_irqsave(&consistent_lock, flags);
126 
127         list_for_each_entry(c, &head->vm_list, vm_list) {
128                 if ((addr + size) < addr)
129                         goto nospc;
130                 if ((addr + size) <= c->vm_start)
131                         goto found;
132                 addr = c->vm_end;
133                 if (addr > end)
134                         goto nospc;
135         }
136 
137 found:
138         /*
139          * Insert this entry _before_ the one we found.
140          */
141         list_add_tail(&new->vm_list, &c->vm_list);
142         new->vm_start = addr;
143         new->vm_end = addr + size;
144         new->vm_active = 1;
145 
146         spin_unlock_irqrestore(&consistent_lock, flags);
147         return new;
148 
149 nospc:
150         spin_unlock_irqrestore(&consistent_lock, flags);
151         kfree(new);
152 out:
153         return NULL;
154 }
155 
156 static struct metag_vm_region *metag_vm_region_find(struct metag_vm_region
157                                                     *head, unsigned long addr)
158 {
159         struct metag_vm_region *c;
160 
161         list_for_each_entry(c, &head->vm_list, vm_list) {
162                 if (c->vm_active && c->vm_start == addr)
163                         goto out;
164         }
165         c = NULL;
166 out:
167         return c;
168 }
169 
170 /*
171  * Allocate DMA-coherent memory space and return both the kernel remapped
172  * virtual and bus address for that space.
173  */
174 void *dma_alloc_coherent(struct device *dev, size_t size,
175                          dma_addr_t *handle, gfp_t gfp)
176 {
177         struct page *page;
178         struct metag_vm_region *c;
179         unsigned long order;
180         u64 mask = get_coherent_dma_mask(dev);
181         u64 limit;
182 
183         if (!consistent_pte) {
184                 pr_err("%s: not initialised\n", __func__);
185                 dump_stack();
186                 return NULL;
187         }
188 
189         if (!mask)
190                 goto no_page;
191         size = PAGE_ALIGN(size);
192         limit = (mask + 1) & ~mask;
193         if ((limit && size >= limit)
194             || size >= (CONSISTENT_END - CONSISTENT_START)) {
195                 pr_warn("coherent allocation too big (requested %#x mask %#Lx)\n",
196                         size, mask);
197                 return NULL;
198         }
199 
200         order = get_order(size);
201 
202         if (mask != 0xffffffff)
203                 gfp |= GFP_DMA;
204 
205         page = alloc_pages(gfp, order);
206         if (!page)
207                 goto no_page;
208 
209         /*
210          * Invalidate any data that might be lurking in the
211          * kernel direct-mapped region for device DMA.
212          */
213         {
214                 void *kaddr = page_address(page);
215                 memset(kaddr, 0, size);
216                 flush_dcache_region(kaddr, size);
217         }
218 
219         /*
220          * Allocate a virtual address in the consistent mapping region.
221          */
222         c = metag_vm_region_alloc(&consistent_head, size,
223                                   gfp & ~(__GFP_DMA | __GFP_HIGHMEM));
224         if (c) {
225                 unsigned long vaddr = c->vm_start;
226                 pte_t *pte = consistent_pte + CONSISTENT_OFFSET(vaddr);
227                 struct page *end = page + (1 << order);
228 
229                 c->vm_pages = page;
230                 split_page(page, order);
231 
232                 /*
233                  * Set the "dma handle"
234                  */
235                 *handle = page_to_bus(page);
236 
237                 do {
238                         BUG_ON(!pte_none(*pte));
239 
240                         SetPageReserved(page);
241                         set_pte_at(&init_mm, vaddr,
242                                    pte, mk_pte(page,
243                                                pgprot_writecombine
244                                                (PAGE_KERNEL)));
245                         page++;
246                         pte++;
247                         vaddr += PAGE_SIZE;
248                 } while (size -= PAGE_SIZE);
249 
250                 /*
251                  * Free the otherwise unused pages.
252                  */
253                 while (page < end) {
254                         __free_page(page);
255                         page++;
256                 }
257 
258                 return (void *)c->vm_start;
259         }
260 
261         if (page)
262                 __free_pages(page, order);
263 no_page:
264         return NULL;
265 }
266 EXPORT_SYMBOL(dma_alloc_coherent);
267 
268 /*
269  * free a page as defined by the above mapping.
270  */
271 void dma_free_coherent(struct device *dev, size_t size,
272                        void *vaddr, dma_addr_t dma_handle)
273 {
274         struct metag_vm_region *c;
275         unsigned long flags, addr;
276         pte_t *ptep;
277 
278         size = PAGE_ALIGN(size);
279 
280         spin_lock_irqsave(&consistent_lock, flags);
281 
282         c = metag_vm_region_find(&consistent_head, (unsigned long)vaddr);
283         if (!c)
284                 goto no_area;
285 
286         c->vm_active = 0;
287         if ((c->vm_end - c->vm_start) != size) {
288                 pr_err("%s: freeing wrong coherent size (%ld != %d)\n",
289                        __func__, c->vm_end - c->vm_start, size);
290                 dump_stack();
291                 size = c->vm_end - c->vm_start;
292         }
293 
294         ptep = consistent_pte + CONSISTENT_OFFSET(c->vm_start);
295         addr = c->vm_start;
296         do {
297                 pte_t pte = ptep_get_and_clear(&init_mm, addr, ptep);
298                 unsigned long pfn;
299 
300                 ptep++;
301                 addr += PAGE_SIZE;
302 
303                 if (!pte_none(pte) && pte_present(pte)) {
304                         pfn = pte_pfn(pte);
305 
306                         if (pfn_valid(pfn)) {
307                                 struct page *page = pfn_to_page(pfn);
308                                 __free_reserved_page(page);
309                                 continue;
310                         }
311                 }
312 
313                 pr_crit("%s: bad page in kernel page table\n",
314                         __func__);
315         } while (size -= PAGE_SIZE);
316 
317         flush_tlb_kernel_range(c->vm_start, c->vm_end);
318 
319         list_del(&c->vm_list);
320 
321         spin_unlock_irqrestore(&consistent_lock, flags);
322 
323         kfree(c);
324         return;
325 
326 no_area:
327         spin_unlock_irqrestore(&consistent_lock, flags);
328         pr_err("%s: trying to free invalid coherent area: %p\n",
329                __func__, vaddr);
330         dump_stack();
331 }
332 EXPORT_SYMBOL(dma_free_coherent);
333 
334 
335 static int dma_mmap(struct device *dev, struct vm_area_struct *vma,
336                     void *cpu_addr, dma_addr_t dma_addr, size_t size)
337 {
338         int ret = -ENXIO;
339 
340         unsigned long flags, user_size, kern_size;
341         struct metag_vm_region *c;
342 
343         user_size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
344 
345         spin_lock_irqsave(&consistent_lock, flags);
346         c = metag_vm_region_find(&consistent_head, (unsigned long)cpu_addr);
347         spin_unlock_irqrestore(&consistent_lock, flags);
348 
349         if (c) {
350                 unsigned long off = vma->vm_pgoff;
351 
352                 kern_size = (c->vm_end - c->vm_start) >> PAGE_SHIFT;
353 
354                 if (off < kern_size &&
355                     user_size <= (kern_size - off)) {
356                         ret = remap_pfn_range(vma, vma->vm_start,
357                                               page_to_pfn(c->vm_pages) + off,
358                                               user_size << PAGE_SHIFT,
359                                               vma->vm_page_prot);
360                 }
361         }
362 
363 
364         return ret;
365 }
366 
367 int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma,
368                       void *cpu_addr, dma_addr_t dma_addr, size_t size)
369 {
370         vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
371         return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
372 }
373 EXPORT_SYMBOL(dma_mmap_coherent);
374 
375 int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
376                           void *cpu_addr, dma_addr_t dma_addr, size_t size)
377 {
378         vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
379         return dma_mmap(dev, vma, cpu_addr, dma_addr, size);
380 }
381 EXPORT_SYMBOL(dma_mmap_writecombine);
382 
383 
384 
385 
386 /*
387  * Initialise the consistent memory allocation.
388  */
389 static int __init dma_alloc_init(void)
390 {
391         pgd_t *pgd, *pgd_k;
392         pud_t *pud, *pud_k;
393         pmd_t *pmd, *pmd_k;
394         pte_t *pte;
395         int ret = 0;
396 
397         do {
398                 int offset = pgd_index(CONSISTENT_START);
399                 pgd = pgd_offset(&init_mm, CONSISTENT_START);
400                 pud = pud_alloc(&init_mm, pgd, CONSISTENT_START);
401                 pmd = pmd_alloc(&init_mm, pud, CONSISTENT_START);
402                 WARN_ON(!pmd_none(*pmd));
403 
404                 pte = pte_alloc_kernel(pmd, CONSISTENT_START);
405                 if (!pte) {
406                         pr_err("%s: no pte tables\n", __func__);
407                         ret = -ENOMEM;
408                         break;
409                 }
410 
411                 pgd_k = ((pgd_t *) mmu_get_base()) + offset;
412                 pud_k = pud_offset(pgd_k, CONSISTENT_START);
413                 pmd_k = pmd_offset(pud_k, CONSISTENT_START);
414                 set_pmd(pmd_k, *pmd);
415 
416                 consistent_pte = pte;
417         } while (0);
418 
419         return ret;
420 }
421 early_initcall(dma_alloc_init);
422 
423 /*
424  * make an area consistent to devices.
425  */
426 void dma_sync_for_device(void *vaddr, size_t size, int dma_direction)
427 {
428         /*
429          * Ensure any writes get through the write combiner. This is necessary
430          * even with DMA_FROM_DEVICE, or the write may dirty the cache after
431          * we've invalidated it and get written back during the DMA.
432          */
433 
434         barrier();
435 
436         switch (dma_direction) {
437         case DMA_BIDIRECTIONAL:
438                 /*
439                  * Writeback to ensure the device can see our latest changes and
440                  * so that we have no dirty lines, and invalidate the cache
441                  * lines too in preparation for receiving the buffer back
442                  * (dma_sync_for_cpu) later.
443                  */
444                 flush_dcache_region(vaddr, size);
445                 break;
446         case DMA_TO_DEVICE:
447                 /*
448                  * Writeback to ensure the device can see our latest changes.
449                  * There's no need to invalidate as the device shouldn't write
450                  * to the buffer.
451                  */
452                 writeback_dcache_region(vaddr, size);
453                 break;
454         case DMA_FROM_DEVICE:
455                 /*
456                  * Invalidate to ensure we have no dirty lines that could get
457                  * written back during the DMA. It's also safe to flush
458                  * (writeback) here if necessary.
459                  */
460                 invalidate_dcache_region(vaddr, size);
461                 break;
462         case DMA_NONE:
463                 BUG();
464         }
465 
466         wmb();
467 }
468 EXPORT_SYMBOL(dma_sync_for_device);
469 
470 /*
471  * make an area consistent to the core.
472  */
473 void dma_sync_for_cpu(void *vaddr, size_t size, int dma_direction)
474 {
475         /*
476          * Hardware L2 cache prefetch doesn't occur across 4K physical
477          * boundaries, however according to Documentation/DMA-API-HOWTO.txt
478          * kmalloc'd memory is DMA'able, so accesses in nearby memory could
479          * trigger a cache fill in the DMA buffer.
480          *
481          * This should never cause dirty lines, so a flush or invalidate should
482          * be safe to allow us to see data from the device.
483          */
484         if (_meta_l2c_pf_is_enabled()) {
485                 switch (dma_direction) {
486                 case DMA_BIDIRECTIONAL:
487                 case DMA_FROM_DEVICE:
488                         invalidate_dcache_region(vaddr, size);
489                         break;
490                 case DMA_TO_DEVICE:
491                         /* The device shouldn't have written to the buffer */
492                         break;
493                 case DMA_NONE:
494                         BUG();
495                 }
496         }
497 
498         rmb();
499 }
500 EXPORT_SYMBOL(dma_sync_for_cpu);
501 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp