~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/sparc/mm/iommu.c

Version: ~ [ linux-5.8-rc4 ] ~ [ linux-5.7.7 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.50 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.131 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.187 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.229 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.229 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * iommu.c:  IOMMU specific routines for memory management.
  3  *
  4  * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
  5  * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
  6  * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
  7  * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
  8  */
  9  
 10 #include <linux/kernel.h>
 11 #include <linux/init.h>
 12 #include <linux/mm.h>
 13 #include <linux/slab.h>
 14 #include <linux/highmem.h>      /* pte_offset_map => kmap_atomic */
 15 #include <linux/scatterlist.h>
 16 #include <linux/of.h>
 17 #include <linux/of_device.h>
 18 
 19 #include <asm/pgalloc.h>
 20 #include <asm/pgtable.h>
 21 #include <asm/io.h>
 22 #include <asm/mxcc.h>
 23 #include <asm/mbus.h>
 24 #include <asm/cacheflush.h>
 25 #include <asm/tlbflush.h>
 26 #include <asm/bitext.h>
 27 #include <asm/iommu.h>
 28 #include <asm/dma.h>
 29 
 30 /*
 31  * This can be sized dynamically, but we will do this
 32  * only when we have a guidance about actual I/O pressures.
 33  */
 34 #define IOMMU_RNGE      IOMMU_RNGE_256MB
 35 #define IOMMU_START     0xF0000000
 36 #define IOMMU_WINSIZE   (256*1024*1024U)
 37 #define IOMMU_NPTES     (IOMMU_WINSIZE/PAGE_SIZE)       /* 64K PTEs, 256KB */
 38 #define IOMMU_ORDER     6                               /* 4096 * (1<<6) */
 39 
 40 /* srmmu.c */
 41 extern int viking_mxcc_present;
 42 extern int flush_page_for_dma_global;
 43 static int viking_flush;
 44 /* viking.S */
 45 extern void viking_flush_page(unsigned long page);
 46 extern void viking_mxcc_flush_page(unsigned long page);
 47 
 48 /*
 49  * Values precomputed according to CPU type.
 50  */
 51 static unsigned int ioperm_noc;         /* Consistent mapping iopte flags */
 52 static pgprot_t dvma_prot;              /* Consistent mapping pte flags */
 53 
 54 #define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
 55 #define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)
 56 
 57 static void __init sbus_iommu_init(struct platform_device *op)
 58 {
 59         struct iommu_struct *iommu;
 60         unsigned int impl, vers;
 61         unsigned long *bitmap;
 62         unsigned long tmp;
 63 
 64         iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
 65         if (!iommu) {
 66                 prom_printf("Unable to allocate iommu structure\n");
 67                 prom_halt();
 68         }
 69 
 70         iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
 71                                  "iommu_regs");
 72         if (!iommu->regs) {
 73                 prom_printf("Cannot map IOMMU registers\n");
 74                 prom_halt();
 75         }
 76         impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28;
 77         vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24;
 78         tmp = iommu->regs->control;
 79         tmp &= ~(IOMMU_CTRL_RNGE);
 80         tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
 81         iommu->regs->control = tmp;
 82         iommu_invalidate(iommu->regs);
 83         iommu->start = IOMMU_START;
 84         iommu->end = 0xffffffff;
 85 
 86         /* Allocate IOMMU page table */
 87         /* Stupid alignment constraints give me a headache. 
 88            We need 256K or 512K or 1M or 2M area aligned to
 89            its size and current gfp will fortunately give
 90            it to us. */
 91         tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
 92         if (!tmp) {
 93                 prom_printf("Unable to allocate iommu table [0x%lx]\n",
 94                             IOMMU_NPTES * sizeof(iopte_t));
 95                 prom_halt();
 96         }
 97         iommu->page_table = (iopte_t *)tmp;
 98 
 99         /* Initialize new table. */
100         memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
101         flush_cache_all();
102         flush_tlb_all();
103         iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4;
104         iommu_invalidate(iommu->regs);
105 
106         bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
107         if (!bitmap) {
108                 prom_printf("Unable to allocate iommu bitmap [%d]\n",
109                             (int)(IOMMU_NPTES>>3));
110                 prom_halt();
111         }
112         bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
113         /* To be coherent on HyperSparc, the page color of DVMA
114          * and physical addresses must match.
115          */
116         if (srmmu_modtype == HyperSparc)
117                 iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
118         else
119                 iommu->usemap.num_colors = 1;
120 
121         printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
122                impl, vers, iommu->page_table,
123                (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
124 
125         op->dev.archdata.iommu = iommu;
126 }
127 
128 static int __init iommu_init(void)
129 {
130         struct device_node *dp;
131 
132         for_each_node_by_name(dp, "iommu") {
133                 struct platform_device *op = of_find_device_by_node(dp);
134 
135                 sbus_iommu_init(op);
136                 of_propagate_archdata(op);
137         }
138 
139         return 0;
140 }
141 
142 subsys_initcall(iommu_init);
143 
144 /* Flush the iotlb entries to ram. */
145 /* This could be better if we didn't have to flush whole pages. */
146 static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
147 {
148         unsigned long start;
149         unsigned long end;
150 
151         start = (unsigned long)iopte;
152         end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
153         start &= PAGE_MASK;
154         if (viking_mxcc_present) {
155                 while(start < end) {
156                         viking_mxcc_flush_page(start);
157                         start += PAGE_SIZE;
158                 }
159         } else if (viking_flush) {
160                 while(start < end) {
161                         viking_flush_page(start);
162                         start += PAGE_SIZE;
163                 }
164         } else {
165                 while(start < end) {
166                         __flush_page_to_ram(start);
167                         start += PAGE_SIZE;
168                 }
169         }
170 }
171 
172 static u32 iommu_get_one(struct device *dev, struct page *page, int npages)
173 {
174         struct iommu_struct *iommu = dev->archdata.iommu;
175         int ioptex;
176         iopte_t *iopte, *iopte0;
177         unsigned int busa, busa0;
178         int i;
179 
180         /* page color = pfn of page */
181         ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
182         if (ioptex < 0)
183                 panic("iommu out");
184         busa0 = iommu->start + (ioptex << PAGE_SHIFT);
185         iopte0 = &iommu->page_table[ioptex];
186 
187         busa = busa0;
188         iopte = iopte0;
189         for (i = 0; i < npages; i++) {
190                 iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
191                 iommu_invalidate_page(iommu->regs, busa);
192                 busa += PAGE_SIZE;
193                 iopte++;
194                 page++;
195         }
196 
197         iommu_flush_iotlb(iopte0, npages);
198 
199         return busa0;
200 }
201 
202 static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len)
203 {
204         unsigned long off;
205         int npages;
206         struct page *page;
207         u32 busa;
208 
209         off = (unsigned long)vaddr & ~PAGE_MASK;
210         npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
211         page = virt_to_page((unsigned long)vaddr & PAGE_MASK);
212         busa = iommu_get_one(dev, page, npages);
213         return busa + off;
214 }
215 
216 static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len)
217 {
218         flush_page_for_dma(0);
219         return iommu_get_scsi_one(dev, vaddr, len);
220 }
221 
222 static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned long len)
223 {
224         unsigned long page = ((unsigned long) vaddr) & PAGE_MASK;
225 
226         while(page < ((unsigned long)(vaddr + len))) {
227                 flush_page_for_dma(page);
228                 page += PAGE_SIZE;
229         }
230         return iommu_get_scsi_one(dev, vaddr, len);
231 }
232 
233 static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz)
234 {
235         int n;
236 
237         flush_page_for_dma(0);
238         while (sz != 0) {
239                 --sz;
240                 n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
241                 sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
242                 sg->dma_length = sg->length;
243                 sg = sg_next(sg);
244         }
245 }
246 
247 static void iommu_get_scsi_sgl_pflush(struct device *dev, struct scatterlist *sg, int sz)
248 {
249         unsigned long page, oldpage = 0;
250         int n, i;
251 
252         while(sz != 0) {
253                 --sz;
254 
255                 n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
256 
257                 /*
258                  * We expect unmapped highmem pages to be not in the cache.
259                  * XXX Is this a good assumption?
260                  * XXX What if someone else unmaps it here and races us?
261                  */
262                 if ((page = (unsigned long) page_address(sg_page(sg))) != 0) {
263                         for (i = 0; i < n; i++) {
264                                 if (page != oldpage) {  /* Already flushed? */
265                                         flush_page_for_dma(page);
266                                         oldpage = page;
267                                 }
268                                 page += PAGE_SIZE;
269                         }
270                 }
271 
272                 sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
273                 sg->dma_length = sg->length;
274                 sg = sg_next(sg);
275         }
276 }
277 
278 static void iommu_release_one(struct device *dev, u32 busa, int npages)
279 {
280         struct iommu_struct *iommu = dev->archdata.iommu;
281         int ioptex;
282         int i;
283 
284         BUG_ON(busa < iommu->start);
285         ioptex = (busa - iommu->start) >> PAGE_SHIFT;
286         for (i = 0; i < npages; i++) {
287                 iopte_val(iommu->page_table[ioptex + i]) = 0;
288                 iommu_invalidate_page(iommu->regs, busa);
289                 busa += PAGE_SIZE;
290         }
291         bit_map_clear(&iommu->usemap, ioptex, npages);
292 }
293 
294 static void iommu_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len)
295 {
296         unsigned long off;
297         int npages;
298 
299         off = vaddr & ~PAGE_MASK;
300         npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
301         iommu_release_one(dev, vaddr & PAGE_MASK, npages);
302 }
303 
304 static void iommu_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
305 {
306         int n;
307 
308         while(sz != 0) {
309                 --sz;
310 
311                 n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
312                 iommu_release_one(dev, sg->dma_address & PAGE_MASK, n);
313                 sg->dma_address = 0x21212121;
314                 sg = sg_next(sg);
315         }
316 }
317 
318 #ifdef CONFIG_SBUS
319 static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va,
320                               unsigned long addr, int len)
321 {
322         struct iommu_struct *iommu = dev->archdata.iommu;
323         unsigned long page, end;
324         iopte_t *iopte = iommu->page_table;
325         iopte_t *first;
326         int ioptex;
327 
328         BUG_ON((va & ~PAGE_MASK) != 0);
329         BUG_ON((addr & ~PAGE_MASK) != 0);
330         BUG_ON((len & ~PAGE_MASK) != 0);
331 
332         /* page color = physical address */
333         ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
334                 addr >> PAGE_SHIFT);
335         if (ioptex < 0)
336                 panic("iommu out");
337 
338         iopte += ioptex;
339         first = iopte;
340         end = addr + len;
341         while(addr < end) {
342                 page = va;
343                 {
344                         pgd_t *pgdp;
345                         pmd_t *pmdp;
346                         pte_t *ptep;
347 
348                         if (viking_mxcc_present)
349                                 viking_mxcc_flush_page(page);
350                         else if (viking_flush)
351                                 viking_flush_page(page);
352                         else
353                                 __flush_page_to_ram(page);
354 
355                         pgdp = pgd_offset(&init_mm, addr);
356                         pmdp = pmd_offset(pgdp, addr);
357                         ptep = pte_offset_map(pmdp, addr);
358 
359                         set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
360                 }
361                 iopte_val(*iopte++) =
362                     MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
363                 addr += PAGE_SIZE;
364                 va += PAGE_SIZE;
365         }
366         /* P3: why do we need this?
367          *
368          * DAVEM: Because there are several aspects, none of which
369          *        are handled by a single interface.  Some cpus are
370          *        completely not I/O DMA coherent, and some have
371          *        virtually indexed caches.  The driver DMA flushing
372          *        methods handle the former case, but here during
373          *        IOMMU page table modifications, and usage of non-cacheable
374          *        cpu mappings of pages potentially in the cpu caches, we have
375          *        to handle the latter case as well.
376          */
377         flush_cache_all();
378         iommu_flush_iotlb(first, len >> PAGE_SHIFT);
379         flush_tlb_all();
380         iommu_invalidate(iommu->regs);
381 
382         *pba = iommu->start + (ioptex << PAGE_SHIFT);
383         return 0;
384 }
385 
386 static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len)
387 {
388         struct iommu_struct *iommu = dev->archdata.iommu;
389         iopte_t *iopte = iommu->page_table;
390         unsigned long end;
391         int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
392 
393         BUG_ON((busa & ~PAGE_MASK) != 0);
394         BUG_ON((len & ~PAGE_MASK) != 0);
395 
396         iopte += ioptex;
397         end = busa + len;
398         while (busa < end) {
399                 iopte_val(*iopte++) = 0;
400                 busa += PAGE_SIZE;
401         }
402         flush_tlb_all();
403         iommu_invalidate(iommu->regs);
404         bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
405 }
406 #endif
407 
408 static const struct sparc32_dma_ops iommu_dma_gflush_ops = {
409         .get_scsi_one           = iommu_get_scsi_one_gflush,
410         .get_scsi_sgl           = iommu_get_scsi_sgl_gflush,
411         .release_scsi_one       = iommu_release_scsi_one,
412         .release_scsi_sgl       = iommu_release_scsi_sgl,
413 #ifdef CONFIG_SBUS
414         .map_dma_area           = iommu_map_dma_area,
415         .unmap_dma_area         = iommu_unmap_dma_area,
416 #endif
417 };
418 
419 static const struct sparc32_dma_ops iommu_dma_pflush_ops = {
420         .get_scsi_one           = iommu_get_scsi_one_pflush,
421         .get_scsi_sgl           = iommu_get_scsi_sgl_pflush,
422         .release_scsi_one       = iommu_release_scsi_one,
423         .release_scsi_sgl       = iommu_release_scsi_sgl,
424 #ifdef CONFIG_SBUS
425         .map_dma_area           = iommu_map_dma_area,
426         .unmap_dma_area         = iommu_unmap_dma_area,
427 #endif
428 };
429 
430 void __init ld_mmu_iommu(void)
431 {
432         if (flush_page_for_dma_global) {
433                 /* flush_page_for_dma flushes everything, no matter of what page is it */
434                 sparc32_dma_ops = &iommu_dma_gflush_ops;
435         } else {
436                 sparc32_dma_ops = &iommu_dma_pflush_ops;
437         }
438 
439         if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
440                 dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
441                 ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
442         } else {
443                 dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
444                 ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
445         }
446 }
447 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp