~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/pci/pci_dma.c

Version: ~ [ linux-5.7-rc7 ] ~ [ linux-5.6.14 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.42 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.124 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.181 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.224 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.224 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.84 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright IBM Corp. 2012
  3  *
  4  * Author(s):
  5  *   Jan Glauber <jang@linux.vnet.ibm.com>
  6  */
  7 
  8 #include <linux/kernel.h>
  9 #include <linux/slab.h>
 10 #include <linux/export.h>
 11 #include <linux/iommu-helper.h>
 12 #include <linux/dma-mapping.h>
 13 #include <linux/vmalloc.h>
 14 #include <linux/pci.h>
 15 #include <asm/pci_dma.h>
 16 
 17 static struct kmem_cache *dma_region_table_cache;
 18 static struct kmem_cache *dma_page_table_cache;
 19 
 20 static unsigned long *dma_alloc_cpu_table(void)
 21 {
 22         unsigned long *table, *entry;
 23 
 24         table = kmem_cache_alloc(dma_region_table_cache, GFP_ATOMIC);
 25         if (!table)
 26                 return NULL;
 27 
 28         for (entry = table; entry < table + ZPCI_TABLE_ENTRIES; entry++)
 29                 *entry = ZPCI_TABLE_INVALID | ZPCI_TABLE_PROTECTED;
 30         return table;
 31 }
 32 
 33 static void dma_free_cpu_table(void *table)
 34 {
 35         kmem_cache_free(dma_region_table_cache, table);
 36 }
 37 
 38 static unsigned long *dma_alloc_page_table(void)
 39 {
 40         unsigned long *table, *entry;
 41 
 42         table = kmem_cache_alloc(dma_page_table_cache, GFP_ATOMIC);
 43         if (!table)
 44                 return NULL;
 45 
 46         for (entry = table; entry < table + ZPCI_PT_ENTRIES; entry++)
 47                 *entry = ZPCI_PTE_INVALID | ZPCI_TABLE_PROTECTED;
 48         return table;
 49 }
 50 
 51 static void dma_free_page_table(void *table)
 52 {
 53         kmem_cache_free(dma_page_table_cache, table);
 54 }
 55 
 56 static unsigned long *dma_get_seg_table_origin(unsigned long *entry)
 57 {
 58         unsigned long *sto;
 59 
 60         if (reg_entry_isvalid(*entry))
 61                 sto = get_rt_sto(*entry);
 62         else {
 63                 sto = dma_alloc_cpu_table();
 64                 if (!sto)
 65                         return NULL;
 66 
 67                 set_rt_sto(entry, sto);
 68                 validate_rt_entry(entry);
 69                 entry_clr_protected(entry);
 70         }
 71         return sto;
 72 }
 73 
 74 static unsigned long *dma_get_page_table_origin(unsigned long *entry)
 75 {
 76         unsigned long *pto;
 77 
 78         if (reg_entry_isvalid(*entry))
 79                 pto = get_st_pto(*entry);
 80         else {
 81                 pto = dma_alloc_page_table();
 82                 if (!pto)
 83                         return NULL;
 84                 set_st_pto(entry, pto);
 85                 validate_st_entry(entry);
 86                 entry_clr_protected(entry);
 87         }
 88         return pto;
 89 }
 90 
 91 static unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr)
 92 {
 93         unsigned long *sto, *pto;
 94         unsigned int rtx, sx, px;
 95 
 96         rtx = calc_rtx(dma_addr);
 97         sto = dma_get_seg_table_origin(&rto[rtx]);
 98         if (!sto)
 99                 return NULL;
100 
101         sx = calc_sx(dma_addr);
102         pto = dma_get_page_table_origin(&sto[sx]);
103         if (!pto)
104                 return NULL;
105 
106         px = calc_px(dma_addr);
107         return &pto[px];
108 }
109 
110 static void dma_update_cpu_trans(struct zpci_dev *zdev, void *page_addr,
111                                  dma_addr_t dma_addr, int flags)
112 {
113         unsigned long *entry;
114 
115         entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);
116         if (!entry) {
117                 WARN_ON_ONCE(1);
118                 return;
119         }
120 
121         if (flags & ZPCI_PTE_INVALID) {
122                 invalidate_pt_entry(entry);
123                 return;
124         } else {
125                 set_pt_pfaa(entry, page_addr);
126                 validate_pt_entry(entry);
127         }
128 
129         if (flags & ZPCI_TABLE_PROTECTED)
130                 entry_set_protected(entry);
131         else
132                 entry_clr_protected(entry);
133 }
134 
135 static int dma_update_trans(struct zpci_dev *zdev, unsigned long pa,
136                             dma_addr_t dma_addr, size_t size, int flags)
137 {
138         unsigned int nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
139         u8 *page_addr = (u8 *) (pa & PAGE_MASK);
140         dma_addr_t start_dma_addr = dma_addr;
141         unsigned long irq_flags;
142         int i, rc = 0;
143 
144         if (!nr_pages)
145                 return -EINVAL;
146 
147         spin_lock_irqsave(&zdev->dma_table_lock, irq_flags);
148         if (!zdev->dma_table)
149                 goto no_refresh;
150 
151         for (i = 0; i < nr_pages; i++) {
152                 dma_update_cpu_trans(zdev, page_addr, dma_addr, flags);
153                 page_addr += PAGE_SIZE;
154                 dma_addr += PAGE_SIZE;
155         }
156 
157         /*
158          * rpcit is not required to establish new translations when previously
159          * invalid translation-table entries are validated, however it is
160          * required when altering previously valid entries.
161          */
162         if (!zdev->tlb_refresh &&
163             ((flags & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID))
164                 /*
165                  * TODO: also need to check that the old entry is indeed INVALID
166                  * and not only for one page but for the whole range...
167                  * -> now we WARN_ON in that case but with lazy unmap that
168                  * needs to be redone!
169                  */
170                 goto no_refresh;
171 
172         rc = zpci_refresh_trans((u64) zdev->fh << 32, start_dma_addr,
173                                 nr_pages * PAGE_SIZE);
174 
175 no_refresh:
176         spin_unlock_irqrestore(&zdev->dma_table_lock, irq_flags);
177         return rc;
178 }
179 
180 static void dma_free_seg_table(unsigned long entry)
181 {
182         unsigned long *sto = get_rt_sto(entry);
183         int sx;
184 
185         for (sx = 0; sx < ZPCI_TABLE_ENTRIES; sx++)
186                 if (reg_entry_isvalid(sto[sx]))
187                         dma_free_page_table(get_st_pto(sto[sx]));
188 
189         dma_free_cpu_table(sto);
190 }
191 
192 static void dma_cleanup_tables(struct zpci_dev *zdev)
193 {
194         unsigned long *table;
195         int rtx;
196 
197         if (!zdev || !zdev->dma_table)
198                 return;
199 
200         table = zdev->dma_table;
201         for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++)
202                 if (reg_entry_isvalid(table[rtx]))
203                         dma_free_seg_table(table[rtx]);
204 
205         dma_free_cpu_table(table);
206         zdev->dma_table = NULL;
207 }
208 
209 static unsigned long __dma_alloc_iommu(struct zpci_dev *zdev, unsigned long start,
210                                    int size)
211 {
212         unsigned long boundary_size = 0x1000000;
213 
214         return iommu_area_alloc(zdev->iommu_bitmap, zdev->iommu_pages,
215                                 start, size, 0, boundary_size, 0);
216 }
217 
218 static unsigned long dma_alloc_iommu(struct zpci_dev *zdev, int size)
219 {
220         unsigned long offset, flags;
221 
222         spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);
223         offset = __dma_alloc_iommu(zdev, zdev->next_bit, size);
224         if (offset == -1)
225                 offset = __dma_alloc_iommu(zdev, 0, size);
226 
227         if (offset != -1) {
228                 zdev->next_bit = offset + size;
229                 if (zdev->next_bit >= zdev->iommu_pages)
230                         zdev->next_bit = 0;
231         }
232         spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
233         return offset;
234 }
235 
236 static void dma_free_iommu(struct zpci_dev *zdev, unsigned long offset, int size)
237 {
238         unsigned long flags;
239 
240         spin_lock_irqsave(&zdev->iommu_bitmap_lock, flags);
241         if (!zdev->iommu_bitmap)
242                 goto out;
243         bitmap_clear(zdev->iommu_bitmap, offset, size);
244         if (offset >= zdev->next_bit)
245                 zdev->next_bit = offset + size;
246 out:
247         spin_unlock_irqrestore(&zdev->iommu_bitmap_lock, flags);
248 }
249 
250 int dma_set_mask(struct device *dev, u64 mask)
251 {
252         if (!dev->dma_mask || !dma_supported(dev, mask))
253                 return -EIO;
254 
255         *dev->dma_mask = mask;
256         return 0;
257 }
258 EXPORT_SYMBOL_GPL(dma_set_mask);
259 
260 static dma_addr_t s390_dma_map_pages(struct device *dev, struct page *page,
261                                      unsigned long offset, size_t size,
262                                      enum dma_data_direction direction,
263                                      struct dma_attrs *attrs)
264 {
265         struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));
266         unsigned long nr_pages, iommu_page_index;
267         unsigned long pa = page_to_phys(page) + offset;
268         int flags = ZPCI_PTE_VALID;
269         dma_addr_t dma_addr;
270 
271         /* This rounds up number of pages based on size and offset */
272         nr_pages = iommu_num_pages(pa, size, PAGE_SIZE);
273         iommu_page_index = dma_alloc_iommu(zdev, nr_pages);
274         if (iommu_page_index == -1)
275                 goto out_err;
276 
277         /* Use rounded up size */
278         size = nr_pages * PAGE_SIZE;
279 
280         dma_addr = zdev->start_dma + iommu_page_index * PAGE_SIZE;
281         if (dma_addr + size > zdev->end_dma)
282                 goto out_free;
283 
284         if (direction == DMA_NONE || direction == DMA_TO_DEVICE)
285                 flags |= ZPCI_TABLE_PROTECTED;
286 
287         if (!dma_update_trans(zdev, pa, dma_addr, size, flags)) {
288                 atomic64_add(nr_pages, (atomic64_t *) &zdev->fmb->mapped_pages);
289                 return dma_addr + (offset & ~PAGE_MASK);
290         }
291 
292 out_free:
293         dma_free_iommu(zdev, iommu_page_index, nr_pages);
294 out_err:
295         zpci_err("map error:\n");
296         zpci_err_hex(&pa, sizeof(pa));
297         return DMA_ERROR_CODE;
298 }
299 
300 static void s390_dma_unmap_pages(struct device *dev, dma_addr_t dma_addr,
301                                  size_t size, enum dma_data_direction direction,
302                                  struct dma_attrs *attrs)
303 {
304         struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));
305         unsigned long iommu_page_index;
306         int npages;
307 
308         npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
309         dma_addr = dma_addr & PAGE_MASK;
310         if (dma_update_trans(zdev, 0, dma_addr, npages * PAGE_SIZE,
311                              ZPCI_TABLE_PROTECTED | ZPCI_PTE_INVALID)) {
312                 zpci_err("unmap error:\n");
313                 zpci_err_hex(&dma_addr, sizeof(dma_addr));
314         }
315 
316         atomic64_add(npages, (atomic64_t *) &zdev->fmb->unmapped_pages);
317         iommu_page_index = (dma_addr - zdev->start_dma) >> PAGE_SHIFT;
318         dma_free_iommu(zdev, iommu_page_index, npages);
319 }
320 
321 static void *s390_dma_alloc(struct device *dev, size_t size,
322                             dma_addr_t *dma_handle, gfp_t flag,
323                             struct dma_attrs *attrs)
324 {
325         struct zpci_dev *zdev = get_zdev(to_pci_dev(dev));
326         struct page *page;
327         unsigned long pa;
328         dma_addr_t map;
329 
330         size = PAGE_ALIGN(size);
331         page = alloc_pages(flag, get_order(size));
332         if (!page)
333                 return NULL;
334 
335         atomic64_add(size / PAGE_SIZE, (atomic64_t *) &zdev->fmb->allocated_pages);
336         pa = page_to_phys(page);
337         memset((void *) pa, 0, size);
338 
339         map = s390_dma_map_pages(dev, page, pa % PAGE_SIZE,
340                                  size, DMA_BIDIRECTIONAL, NULL);
341         if (dma_mapping_error(dev, map)) {
342                 free_pages(pa, get_order(size));
343                 return NULL;
344         }
345 
346         if (dma_handle)
347                 *dma_handle = map;
348         return (void *) pa;
349 }
350 
351 static void s390_dma_free(struct device *dev, size_t size,
352                           void *pa, dma_addr_t dma_handle,
353                           struct dma_attrs *attrs)
354 {
355         s390_dma_unmap_pages(dev, dma_handle, PAGE_ALIGN(size),
356                              DMA_BIDIRECTIONAL, NULL);
357         free_pages((unsigned long) pa, get_order(size));
358 }
359 
360 static int s390_dma_map_sg(struct device *dev, struct scatterlist *sg,
361                            int nr_elements, enum dma_data_direction dir,
362                            struct dma_attrs *attrs)
363 {
364         int mapped_elements = 0;
365         struct scatterlist *s;
366         int i;
367 
368         for_each_sg(sg, s, nr_elements, i) {
369                 struct page *page = sg_page(s);
370                 s->dma_address = s390_dma_map_pages(dev, page, s->offset,
371                                                     s->length, dir, NULL);
372                 if (!dma_mapping_error(dev, s->dma_address)) {
373                         s->dma_length = s->length;
374                         mapped_elements++;
375                 } else
376                         goto unmap;
377         }
378 out:
379         return mapped_elements;
380 
381 unmap:
382         for_each_sg(sg, s, mapped_elements, i) {
383                 if (s->dma_address)
384                         s390_dma_unmap_pages(dev, s->dma_address, s->dma_length,
385                                              dir, NULL);
386                 s->dma_address = 0;
387                 s->dma_length = 0;
388         }
389         mapped_elements = 0;
390         goto out;
391 }
392 
393 static void s390_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
394                               int nr_elements, enum dma_data_direction dir,
395                               struct dma_attrs *attrs)
396 {
397         struct scatterlist *s;
398         int i;
399 
400         for_each_sg(sg, s, nr_elements, i) {
401                 s390_dma_unmap_pages(dev, s->dma_address, s->dma_length, dir, NULL);
402                 s->dma_address = 0;
403                 s->dma_length = 0;
404         }
405 }
406 
407 int zpci_dma_init_device(struct zpci_dev *zdev)
408 {
409         int rc;
410 
411         spin_lock_init(&zdev->iommu_bitmap_lock);
412         spin_lock_init(&zdev->dma_table_lock);
413 
414         zdev->dma_table = dma_alloc_cpu_table();
415         if (!zdev->dma_table) {
416                 rc = -ENOMEM;
417                 goto out_clean;
418         }
419 
420         zdev->iommu_size = (unsigned long) high_memory - PAGE_OFFSET;
421         zdev->iommu_pages = zdev->iommu_size >> PAGE_SHIFT;
422         zdev->iommu_bitmap = vzalloc(zdev->iommu_pages / 8);
423         if (!zdev->iommu_bitmap) {
424                 rc = -ENOMEM;
425                 goto out_reg;
426         }
427 
428         rc = zpci_register_ioat(zdev,
429                                 0,
430                                 zdev->start_dma + PAGE_OFFSET,
431                                 zdev->start_dma + zdev->iommu_size - 1,
432                                 (u64) zdev->dma_table);
433         if (rc)
434                 goto out_reg;
435         return 0;
436 
437 out_reg:
438         dma_free_cpu_table(zdev->dma_table);
439 out_clean:
440         return rc;
441 }
442 
443 void zpci_dma_exit_device(struct zpci_dev *zdev)
444 {
445         zpci_unregister_ioat(zdev, 0);
446         dma_cleanup_tables(zdev);
447         vfree(zdev->iommu_bitmap);
448         zdev->iommu_bitmap = NULL;
449         zdev->next_bit = 0;
450 }
451 
452 static int __init dma_alloc_cpu_table_caches(void)
453 {
454         dma_region_table_cache = kmem_cache_create("PCI_DMA_region_tables",
455                                         ZPCI_TABLE_SIZE, ZPCI_TABLE_ALIGN,
456                                         0, NULL);
457         if (!dma_region_table_cache)
458                 return -ENOMEM;
459 
460         dma_page_table_cache = kmem_cache_create("PCI_DMA_page_tables",
461                                         ZPCI_PT_SIZE, ZPCI_PT_ALIGN,
462                                         0, NULL);
463         if (!dma_page_table_cache) {
464                 kmem_cache_destroy(dma_region_table_cache);
465                 return -ENOMEM;
466         }
467         return 0;
468 }
469 
470 int __init zpci_dma_init(void)
471 {
472         return dma_alloc_cpu_table_caches();
473 }
474 
475 void zpci_dma_exit(void)
476 {
477         kmem_cache_destroy(dma_page_table_cache);
478         kmem_cache_destroy(dma_region_table_cache);
479 }
480 
481 #define PREALLOC_DMA_DEBUG_ENTRIES      (1 << 16)
482 
483 static int __init dma_debug_do_init(void)
484 {
485         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
486         return 0;
487 }
488 fs_initcall(dma_debug_do_init);
489 
490 struct dma_map_ops s390_dma_ops = {
491         .alloc          = s390_dma_alloc,
492         .free           = s390_dma_free,
493         .map_sg         = s390_dma_map_sg,
494         .unmap_sg       = s390_dma_unmap_sg,
495         .map_page       = s390_dma_map_pages,
496         .unmap_page     = s390_dma_unmap_pages,
497         /* if we support direct DMA this must be conditional */
498         .is_phys        = 0,
499         /* dma_supported is unconditionally true without a callback */
500 };
501 EXPORT_SYMBOL_GPL(s390_dma_ops);
502 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp