~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/common/dmabounce.c

Version: ~ [ linux-5.11-rc3 ] ~ [ linux-5.10.7 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.89 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.167 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.215 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.251 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.251 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  *  arch/arm/common/dmabounce.c
  4  *
  5  *  Special dma_{map/unmap/dma_sync}_* routines for systems that have
  6  *  limited DMA windows. These functions utilize bounce buffers to
  7  *  copy data to/from buffers located outside the DMA region. This
  8  *  only works for systems in which DMA memory is at the bottom of
  9  *  RAM, the remainder of memory is at the top and the DMA memory
 10  *  can be marked as ZONE_DMA. Anything beyond that such as discontiguous
 11  *  DMA windows will require custom implementations that reserve memory
 12  *  areas at early bootup.
 13  *
 14  *  Original version by Brad Parker (brad@heeltoe.com)
 15  *  Re-written by Christopher Hoover <ch@murgatroid.com>
 16  *  Made generic by Deepak Saxena <dsaxena@plexity.net>
 17  *
 18  *  Copyright (C) 2002 Hewlett Packard Company.
 19  *  Copyright (C) 2004 MontaVista Software, Inc.
 20  */
 21 
 22 #include <linux/module.h>
 23 #include <linux/init.h>
 24 #include <linux/slab.h>
 25 #include <linux/page-flags.h>
 26 #include <linux/device.h>
 27 #include <linux/dma-mapping.h>
 28 #include <linux/dmapool.h>
 29 #include <linux/list.h>
 30 #include <linux/scatterlist.h>
 31 
 32 #include <asm/cacheflush.h>
 33 #include <asm/dma-iommu.h>
 34 
 35 #undef STATS
 36 
 37 #ifdef STATS
 38 #define DO_STATS(X) do { X ; } while (0)
 39 #else
 40 #define DO_STATS(X) do { } while (0)
 41 #endif
 42 
 43 /* ************************************************** */
 44 
 45 struct safe_buffer {
 46         struct list_head node;
 47 
 48         /* original request */
 49         void            *ptr;
 50         size_t          size;
 51         int             direction;
 52 
 53         /* safe buffer info */
 54         struct dmabounce_pool *pool;
 55         void            *safe;
 56         dma_addr_t      safe_dma_addr;
 57 };
 58 
 59 struct dmabounce_pool {
 60         unsigned long   size;
 61         struct dma_pool *pool;
 62 #ifdef STATS
 63         unsigned long   allocs;
 64 #endif
 65 };
 66 
 67 struct dmabounce_device_info {
 68         struct device *dev;
 69         struct list_head safe_buffers;
 70 #ifdef STATS
 71         unsigned long total_allocs;
 72         unsigned long map_op_count;
 73         unsigned long bounce_count;
 74         int attr_res;
 75 #endif
 76         struct dmabounce_pool   small;
 77         struct dmabounce_pool   large;
 78 
 79         rwlock_t lock;
 80 
 81         int (*needs_bounce)(struct device *, dma_addr_t, size_t);
 82 };
 83 
 84 #ifdef STATS
 85 static ssize_t dmabounce_show(struct device *dev, struct device_attribute *attr,
 86                               char *buf)
 87 {
 88         struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
 89         return sprintf(buf, "%lu %lu %lu %lu %lu %lu\n",
 90                 device_info->small.allocs,
 91                 device_info->large.allocs,
 92                 device_info->total_allocs - device_info->small.allocs -
 93                         device_info->large.allocs,
 94                 device_info->total_allocs,
 95                 device_info->map_op_count,
 96                 device_info->bounce_count);
 97 }
 98 
 99 static DEVICE_ATTR(dmabounce_stats, 0400, dmabounce_show, NULL);
100 #endif
101 
102 
103 /* allocate a 'safe' buffer and keep track of it */
104 static inline struct safe_buffer *
105 alloc_safe_buffer(struct dmabounce_device_info *device_info, void *ptr,
106                   size_t size, enum dma_data_direction dir)
107 {
108         struct safe_buffer *buf;
109         struct dmabounce_pool *pool;
110         struct device *dev = device_info->dev;
111         unsigned long flags;
112 
113         dev_dbg(dev, "%s(ptr=%p, size=%d, dir=%d)\n",
114                 __func__, ptr, size, dir);
115 
116         if (size <= device_info->small.size) {
117                 pool = &device_info->small;
118         } else if (size <= device_info->large.size) {
119                 pool = &device_info->large;
120         } else {
121                 pool = NULL;
122         }
123 
124         buf = kmalloc(sizeof(struct safe_buffer), GFP_ATOMIC);
125         if (buf == NULL) {
126                 dev_warn(dev, "%s: kmalloc failed\n", __func__);
127                 return NULL;
128         }
129 
130         buf->ptr = ptr;
131         buf->size = size;
132         buf->direction = dir;
133         buf->pool = pool;
134 
135         if (pool) {
136                 buf->safe = dma_pool_alloc(pool->pool, GFP_ATOMIC,
137                                            &buf->safe_dma_addr);
138         } else {
139                 buf->safe = dma_alloc_coherent(dev, size, &buf->safe_dma_addr,
140                                                GFP_ATOMIC);
141         }
142 
143         if (buf->safe == NULL) {
144                 dev_warn(dev,
145                          "%s: could not alloc dma memory (size=%d)\n",
146                          __func__, size);
147                 kfree(buf);
148                 return NULL;
149         }
150 
151 #ifdef STATS
152         if (pool)
153                 pool->allocs++;
154         device_info->total_allocs++;
155 #endif
156 
157         write_lock_irqsave(&device_info->lock, flags);
158         list_add(&buf->node, &device_info->safe_buffers);
159         write_unlock_irqrestore(&device_info->lock, flags);
160 
161         return buf;
162 }
163 
164 /* determine if a buffer is from our "safe" pool */
165 static inline struct safe_buffer *
166 find_safe_buffer(struct dmabounce_device_info *device_info, dma_addr_t safe_dma_addr)
167 {
168         struct safe_buffer *b, *rb = NULL;
169         unsigned long flags;
170 
171         read_lock_irqsave(&device_info->lock, flags);
172 
173         list_for_each_entry(b, &device_info->safe_buffers, node)
174                 if (b->safe_dma_addr <= safe_dma_addr &&
175                     b->safe_dma_addr + b->size > safe_dma_addr) {
176                         rb = b;
177                         break;
178                 }
179 
180         read_unlock_irqrestore(&device_info->lock, flags);
181         return rb;
182 }
183 
184 static inline void
185 free_safe_buffer(struct dmabounce_device_info *device_info, struct safe_buffer *buf)
186 {
187         unsigned long flags;
188 
189         dev_dbg(device_info->dev, "%s(buf=%p)\n", __func__, buf);
190 
191         write_lock_irqsave(&device_info->lock, flags);
192 
193         list_del(&buf->node);
194 
195         write_unlock_irqrestore(&device_info->lock, flags);
196 
197         if (buf->pool)
198                 dma_pool_free(buf->pool->pool, buf->safe, buf->safe_dma_addr);
199         else
200                 dma_free_coherent(device_info->dev, buf->size, buf->safe,
201                                     buf->safe_dma_addr);
202 
203         kfree(buf);
204 }
205 
206 /* ************************************************** */
207 
208 static struct safe_buffer *find_safe_buffer_dev(struct device *dev,
209                 dma_addr_t dma_addr, const char *where)
210 {
211         if (!dev || !dev->archdata.dmabounce)
212                 return NULL;
213         if (dma_mapping_error(dev, dma_addr)) {
214                 dev_err(dev, "Trying to %s invalid mapping\n", where);
215                 return NULL;
216         }
217         return find_safe_buffer(dev->archdata.dmabounce, dma_addr);
218 }
219 
220 static int needs_bounce(struct device *dev, dma_addr_t dma_addr, size_t size)
221 {
222         if (!dev || !dev->archdata.dmabounce)
223                 return 0;
224 
225         if (dev->dma_mask) {
226                 unsigned long limit, mask = *dev->dma_mask;
227 
228                 limit = (mask + 1) & ~mask;
229                 if (limit && size > limit) {
230                         dev_err(dev, "DMA mapping too big (requested %#x "
231                                 "mask %#Lx)\n", size, *dev->dma_mask);
232                         return -E2BIG;
233                 }
234 
235                 /* Figure out if we need to bounce from the DMA mask. */
236                 if ((dma_addr | (dma_addr + size - 1)) & ~mask)
237                         return 1;
238         }
239 
240         return !!dev->archdata.dmabounce->needs_bounce(dev, dma_addr, size);
241 }
242 
243 static inline dma_addr_t map_single(struct device *dev, void *ptr, size_t size,
244                                     enum dma_data_direction dir,
245                                     unsigned long attrs)
246 {
247         struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
248         struct safe_buffer *buf;
249 
250         if (device_info)
251                 DO_STATS ( device_info->map_op_count++ );
252 
253         buf = alloc_safe_buffer(device_info, ptr, size, dir);
254         if (buf == NULL) {
255                 dev_err(dev, "%s: unable to map unsafe buffer %p!\n",
256                        __func__, ptr);
257                 return DMA_MAPPING_ERROR;
258         }
259 
260         dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
261                 __func__, buf->ptr, virt_to_dma(dev, buf->ptr),
262                 buf->safe, buf->safe_dma_addr);
263 
264         if ((dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) &&
265             !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
266                 dev_dbg(dev, "%s: copy unsafe %p to safe %p, size %d\n",
267                         __func__, ptr, buf->safe, size);
268                 memcpy(buf->safe, ptr, size);
269         }
270 
271         return buf->safe_dma_addr;
272 }
273 
274 static inline void unmap_single(struct device *dev, struct safe_buffer *buf,
275                                 size_t size, enum dma_data_direction dir,
276                                 unsigned long attrs)
277 {
278         BUG_ON(buf->size != size);
279         BUG_ON(buf->direction != dir);
280 
281         dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped to %p (dma=%#x)\n",
282                 __func__, buf->ptr, virt_to_dma(dev, buf->ptr),
283                 buf->safe, buf->safe_dma_addr);
284 
285         DO_STATS(dev->archdata.dmabounce->bounce_count++);
286 
287         if ((dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) &&
288             !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) {
289                 void *ptr = buf->ptr;
290 
291                 dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n",
292                         __func__, buf->safe, ptr, size);
293                 memcpy(ptr, buf->safe, size);
294 
295                 /*
296                  * Since we may have written to a page cache page,
297                  * we need to ensure that the data will be coherent
298                  * with user mappings.
299                  */
300                 __cpuc_flush_dcache_area(ptr, size);
301         }
302         free_safe_buffer(dev->archdata.dmabounce, buf);
303 }
304 
305 /* ************************************************** */
306 
307 /*
308  * see if a buffer address is in an 'unsafe' range.  if it is
309  * allocate a 'safe' buffer and copy the unsafe buffer into it.
310  * substitute the safe buffer for the unsafe one.
311  * (basically move the buffer from an unsafe area to a safe one)
312  */
313 static dma_addr_t dmabounce_map_page(struct device *dev, struct page *page,
314                 unsigned long offset, size_t size, enum dma_data_direction dir,
315                 unsigned long attrs)
316 {
317         dma_addr_t dma_addr;
318         int ret;
319 
320         dev_dbg(dev, "%s(page=%p,off=%#lx,size=%zx,dir=%x)\n",
321                 __func__, page, offset, size, dir);
322 
323         dma_addr = pfn_to_dma(dev, page_to_pfn(page)) + offset;
324 
325         ret = needs_bounce(dev, dma_addr, size);
326         if (ret < 0)
327                 return DMA_MAPPING_ERROR;
328 
329         if (ret == 0) {
330                 arm_dma_ops.sync_single_for_device(dev, dma_addr, size, dir);
331                 return dma_addr;
332         }
333 
334         if (PageHighMem(page)) {
335                 dev_err(dev, "DMA buffer bouncing of HIGHMEM pages is not supported\n");
336                 return DMA_MAPPING_ERROR;
337         }
338 
339         return map_single(dev, page_address(page) + offset, size, dir, attrs);
340 }
341 
342 /*
343  * see if a mapped address was really a "safe" buffer and if so, copy
344  * the data from the safe buffer back to the unsafe buffer and free up
345  * the safe buffer.  (basically return things back to the way they
346  * should be)
347  */
348 static void dmabounce_unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
349                 enum dma_data_direction dir, unsigned long attrs)
350 {
351         struct safe_buffer *buf;
352 
353         dev_dbg(dev, "%s(dma=%#x,size=%d,dir=%x)\n",
354                 __func__, dma_addr, size, dir);
355 
356         buf = find_safe_buffer_dev(dev, dma_addr, __func__);
357         if (!buf) {
358                 arm_dma_ops.sync_single_for_cpu(dev, dma_addr, size, dir);
359                 return;
360         }
361 
362         unmap_single(dev, buf, size, dir, attrs);
363 }
364 
365 static int __dmabounce_sync_for_cpu(struct device *dev, dma_addr_t addr,
366                 size_t sz, enum dma_data_direction dir)
367 {
368         struct safe_buffer *buf;
369         unsigned long off;
370 
371         dev_dbg(dev, "%s(dma=%#x,sz=%zx,dir=%x)\n",
372                 __func__, addr, sz, dir);
373 
374         buf = find_safe_buffer_dev(dev, addr, __func__);
375         if (!buf)
376                 return 1;
377 
378         off = addr - buf->safe_dma_addr;
379 
380         BUG_ON(buf->direction != dir);
381 
382         dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x off=%#lx) mapped to %p (dma=%#x)\n",
383                 __func__, buf->ptr, virt_to_dma(dev, buf->ptr), off,
384                 buf->safe, buf->safe_dma_addr);
385 
386         DO_STATS(dev->archdata.dmabounce->bounce_count++);
387 
388         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
389                 dev_dbg(dev, "%s: copy back safe %p to unsafe %p size %d\n",
390                         __func__, buf->safe + off, buf->ptr + off, sz);
391                 memcpy(buf->ptr + off, buf->safe + off, sz);
392         }
393         return 0;
394 }
395 
396 static void dmabounce_sync_for_cpu(struct device *dev,
397                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
398 {
399         if (!__dmabounce_sync_for_cpu(dev, handle, size, dir))
400                 return;
401 
402         arm_dma_ops.sync_single_for_cpu(dev, handle, size, dir);
403 }
404 
405 static int __dmabounce_sync_for_device(struct device *dev, dma_addr_t addr,
406                 size_t sz, enum dma_data_direction dir)
407 {
408         struct safe_buffer *buf;
409         unsigned long off;
410 
411         dev_dbg(dev, "%s(dma=%#x,sz=%zx,dir=%x)\n",
412                 __func__, addr, sz, dir);
413 
414         buf = find_safe_buffer_dev(dev, addr, __func__);
415         if (!buf)
416                 return 1;
417 
418         off = addr - buf->safe_dma_addr;
419 
420         BUG_ON(buf->direction != dir);
421 
422         dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x off=%#lx) mapped to %p (dma=%#x)\n",
423                 __func__, buf->ptr, virt_to_dma(dev, buf->ptr), off,
424                 buf->safe, buf->safe_dma_addr);
425 
426         DO_STATS(dev->archdata.dmabounce->bounce_count++);
427 
428         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) {
429                 dev_dbg(dev, "%s: copy out unsafe %p to safe %p, size %d\n",
430                         __func__,buf->ptr + off, buf->safe + off, sz);
431                 memcpy(buf->safe + off, buf->ptr + off, sz);
432         }
433         return 0;
434 }
435 
436 static void dmabounce_sync_for_device(struct device *dev,
437                 dma_addr_t handle, size_t size, enum dma_data_direction dir)
438 {
439         if (!__dmabounce_sync_for_device(dev, handle, size, dir))
440                 return;
441 
442         arm_dma_ops.sync_single_for_device(dev, handle, size, dir);
443 }
444 
445 static int dmabounce_dma_supported(struct device *dev, u64 dma_mask)
446 {
447         if (dev->archdata.dmabounce)
448                 return 0;
449 
450         return arm_dma_ops.dma_supported(dev, dma_mask);
451 }
452 
453 static const struct dma_map_ops dmabounce_ops = {
454         .alloc                  = arm_dma_alloc,
455         .free                   = arm_dma_free,
456         .mmap                   = arm_dma_mmap,
457         .get_sgtable            = arm_dma_get_sgtable,
458         .map_page               = dmabounce_map_page,
459         .unmap_page             = dmabounce_unmap_page,
460         .sync_single_for_cpu    = dmabounce_sync_for_cpu,
461         .sync_single_for_device = dmabounce_sync_for_device,
462         .map_sg                 = arm_dma_map_sg,
463         .unmap_sg               = arm_dma_unmap_sg,
464         .sync_sg_for_cpu        = arm_dma_sync_sg_for_cpu,
465         .sync_sg_for_device     = arm_dma_sync_sg_for_device,
466         .dma_supported          = dmabounce_dma_supported,
467 };
468 
469 static int dmabounce_init_pool(struct dmabounce_pool *pool, struct device *dev,
470                 const char *name, unsigned long size)
471 {
472         pool->size = size;
473         DO_STATS(pool->allocs = 0);
474         pool->pool = dma_pool_create(name, dev, size,
475                                      0 /* byte alignment */,
476                                      0 /* no page-crossing issues */);
477 
478         return pool->pool ? 0 : -ENOMEM;
479 }
480 
481 int dmabounce_register_dev(struct device *dev, unsigned long small_buffer_size,
482                 unsigned long large_buffer_size,
483                 int (*needs_bounce_fn)(struct device *, dma_addr_t, size_t))
484 {
485         struct dmabounce_device_info *device_info;
486         int ret;
487 
488         device_info = kmalloc(sizeof(struct dmabounce_device_info), GFP_ATOMIC);
489         if (!device_info) {
490                 dev_err(dev,
491                         "Could not allocated dmabounce_device_info\n");
492                 return -ENOMEM;
493         }
494 
495         ret = dmabounce_init_pool(&device_info->small, dev,
496                                   "small_dmabounce_pool", small_buffer_size);
497         if (ret) {
498                 dev_err(dev,
499                         "dmabounce: could not allocate DMA pool for %ld byte objects\n",
500                         small_buffer_size);
501                 goto err_free;
502         }
503 
504         if (large_buffer_size) {
505                 ret = dmabounce_init_pool(&device_info->large, dev,
506                                           "large_dmabounce_pool",
507                                           large_buffer_size);
508                 if (ret) {
509                         dev_err(dev,
510                                 "dmabounce: could not allocate DMA pool for %ld byte objects\n",
511                                 large_buffer_size);
512                         goto err_destroy;
513                 }
514         }
515 
516         device_info->dev = dev;
517         INIT_LIST_HEAD(&device_info->safe_buffers);
518         rwlock_init(&device_info->lock);
519         device_info->needs_bounce = needs_bounce_fn;
520 
521 #ifdef STATS
522         device_info->total_allocs = 0;
523         device_info->map_op_count = 0;
524         device_info->bounce_count = 0;
525         device_info->attr_res = device_create_file(dev, &dev_attr_dmabounce_stats);
526 #endif
527 
528         dev->archdata.dmabounce = device_info;
529         set_dma_ops(dev, &dmabounce_ops);
530 
531         dev_info(dev, "dmabounce: registered device\n");
532 
533         return 0;
534 
535  err_destroy:
536         dma_pool_destroy(device_info->small.pool);
537  err_free:
538         kfree(device_info);
539         return ret;
540 }
541 EXPORT_SYMBOL(dmabounce_register_dev);
542 
543 void dmabounce_unregister_dev(struct device *dev)
544 {
545         struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
546 
547         dev->archdata.dmabounce = NULL;
548         set_dma_ops(dev, NULL);
549 
550         if (!device_info) {
551                 dev_warn(dev,
552                          "Never registered with dmabounce but attempting"
553                          "to unregister!\n");
554                 return;
555         }
556 
557         if (!list_empty(&device_info->safe_buffers)) {
558                 dev_err(dev,
559                         "Removing from dmabounce with pending buffers!\n");
560                 BUG();
561         }
562 
563         if (device_info->small.pool)
564                 dma_pool_destroy(device_info->small.pool);
565         if (device_info->large.pool)
566                 dma_pool_destroy(device_info->large.pool);
567 
568 #ifdef STATS
569         if (device_info->attr_res == 0)
570                 device_remove_file(dev, &dev_attr_dmabounce_stats);
571 #endif
572 
573         kfree(device_info);
574 
575         dev_info(dev, "dmabounce: device unregistered\n");
576 }
577 EXPORT_SYMBOL(dmabounce_unregister_dev);
578 
579 MODULE_AUTHOR("Christopher Hoover <ch@hpl.hp.com>, Deepak Saxena <dsaxena@plexity.net>");
580 MODULE_DESCRIPTION("Special dma_{map/unmap/dma_sync}_* routines for systems with limited DMA windows");
581 MODULE_LICENSE("GPL");
582 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp