~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/pci-dma.c

Version: ~ [ linux-5.10-rc5 ] ~ [ linux-5.9.10 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.79 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.159 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.208 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.245 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.245 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include <linux/dma-mapping.h>
  2 #include <linux/dma-debug.h>
  3 #include <linux/dmar.h>
  4 #include <linux/export.h>
  5 #include <linux/bootmem.h>
  6 #include <linux/gfp.h>
  7 #include <linux/pci.h>
  8 #include <linux/kmemleak.h>
  9 
 10 #include <asm/proto.h>
 11 #include <asm/dma.h>
 12 #include <asm/iommu.h>
 13 #include <asm/gart.h>
 14 #include <asm/calgary.h>
 15 #include <asm/x86_init.h>
 16 #include <asm/iommu_table.h>
 17 
 18 static int forbid_dac __read_mostly;
 19 
 20 const struct dma_map_ops *dma_ops = &nommu_dma_ops;
 21 EXPORT_SYMBOL(dma_ops);
 22 
 23 static int iommu_sac_force __read_mostly;
 24 
 25 #ifdef CONFIG_IOMMU_DEBUG
 26 int panic_on_overflow __read_mostly = 1;
 27 int force_iommu __read_mostly = 1;
 28 #else
 29 int panic_on_overflow __read_mostly = 0;
 30 int force_iommu __read_mostly = 0;
 31 #endif
 32 
 33 int iommu_merge __read_mostly = 0;
 34 
 35 int no_iommu __read_mostly;
 36 /* Set this to 1 if there is a HW IOMMU in the system */
 37 int iommu_detected __read_mostly = 0;
 38 
 39 /*
 40  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
 41  * If this variable is 1, IOMMU implementations do no DMA translation for
 42  * devices and allow every device to access to whole physical memory. This is
 43  * useful if a user wants to use an IOMMU only for KVM device assignment to
 44  * guests and not for driver dma translation.
 45  */
 46 int iommu_pass_through __read_mostly;
 47 
 48 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
 49 
 50 /* Dummy device used for NULL arguments (normally ISA). */
 51 struct device x86_dma_fallback_dev = {
 52         .init_name = "fallback device",
 53         .coherent_dma_mask = ISA_DMA_BIT_MASK,
 54         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
 55 };
 56 EXPORT_SYMBOL(x86_dma_fallback_dev);
 57 
 58 /* Number of entries preallocated for DMA-API debugging */
 59 #define PREALLOC_DMA_DEBUG_ENTRIES       65536
 60 
 61 void __init pci_iommu_alloc(void)
 62 {
 63         struct iommu_table_entry *p;
 64 
 65         sort_iommu_table(__iommu_table, __iommu_table_end);
 66         check_iommu_entries(__iommu_table, __iommu_table_end);
 67 
 68         for (p = __iommu_table; p < __iommu_table_end; p++) {
 69                 if (p && p->detect && p->detect() > 0) {
 70                         p->flags |= IOMMU_DETECTED;
 71                         if (p->early_init)
 72                                 p->early_init();
 73                         if (p->flags & IOMMU_FINISH_IF_DETECTED)
 74                                 break;
 75                 }
 76         }
 77 }
 78 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 79                                  dma_addr_t *dma_addr, gfp_t flag,
 80                                  unsigned long attrs)
 81 {
 82         unsigned long dma_mask;
 83         struct page *page;
 84         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 85         dma_addr_t addr;
 86 
 87         dma_mask = dma_alloc_coherent_mask(dev, flag);
 88 
 89         flag &= ~__GFP_ZERO;
 90 again:
 91         page = NULL;
 92         /* CMA can be used only in the context which permits sleeping */
 93         if (gfpflags_allow_blocking(flag)) {
 94                 page = dma_alloc_from_contiguous(dev, count, get_order(size),
 95                                                  flag);
 96                 if (page && page_to_phys(page) + size > dma_mask) {
 97                         dma_release_from_contiguous(dev, page, count);
 98                         page = NULL;
 99                 }
100         }
101         /* fallback */
102         if (!page)
103                 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
104         if (!page)
105                 return NULL;
106 
107         addr = page_to_phys(page);
108         if (addr + size > dma_mask) {
109                 __free_pages(page, get_order(size));
110 
111                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
112                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
113                         goto again;
114                 }
115 
116                 return NULL;
117         }
118         memset(page_address(page), 0, size);
119         *dma_addr = addr;
120         return page_address(page);
121 }
122 
123 void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr,
124                                dma_addr_t dma_addr, unsigned long attrs)
125 {
126         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
127         struct page *page = virt_to_page(vaddr);
128 
129         if (!dma_release_from_contiguous(dev, page, count))
130                 free_pages((unsigned long)vaddr, get_order(size));
131 }
132 
133 bool arch_dma_alloc_attrs(struct device **dev, gfp_t *gfp)
134 {
135         if (!*dev)
136                 *dev = &x86_dma_fallback_dev;
137 
138         *gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
139         *gfp = dma_alloc_coherent_gfp_flags(*dev, *gfp);
140 
141         if (!is_device_dma_capable(*dev))
142                 return false;
143         return true;
144 
145 }
146 EXPORT_SYMBOL(arch_dma_alloc_attrs);
147 
148 /*
149  * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
150  * parameter documentation.
151  */
152 static __init int iommu_setup(char *p)
153 {
154         iommu_merge = 1;
155 
156         if (!p)
157                 return -EINVAL;
158 
159         while (*p) {
160                 if (!strncmp(p, "off", 3))
161                         no_iommu = 1;
162                 /* gart_parse_options has more force support */
163                 if (!strncmp(p, "force", 5))
164                         force_iommu = 1;
165                 if (!strncmp(p, "noforce", 7)) {
166                         iommu_merge = 0;
167                         force_iommu = 0;
168                 }
169 
170                 if (!strncmp(p, "biomerge", 8)) {
171                         iommu_merge = 1;
172                         force_iommu = 1;
173                 }
174                 if (!strncmp(p, "panic", 5))
175                         panic_on_overflow = 1;
176                 if (!strncmp(p, "nopanic", 7))
177                         panic_on_overflow = 0;
178                 if (!strncmp(p, "merge", 5)) {
179                         iommu_merge = 1;
180                         force_iommu = 1;
181                 }
182                 if (!strncmp(p, "nomerge", 7))
183                         iommu_merge = 0;
184                 if (!strncmp(p, "forcesac", 8))
185                         iommu_sac_force = 1;
186                 if (!strncmp(p, "allowdac", 8))
187                         forbid_dac = 0;
188                 if (!strncmp(p, "nodac", 5))
189                         forbid_dac = 1;
190                 if (!strncmp(p, "usedac", 6)) {
191                         forbid_dac = -1;
192                         return 1;
193                 }
194 #ifdef CONFIG_SWIOTLB
195                 if (!strncmp(p, "soft", 4))
196                         swiotlb = 1;
197 #endif
198                 if (!strncmp(p, "pt", 2))
199                         iommu_pass_through = 1;
200 
201                 gart_parse_options(p);
202 
203 #ifdef CONFIG_CALGARY_IOMMU
204                 if (!strncmp(p, "calgary", 7))
205                         use_calgary = 1;
206 #endif /* CONFIG_CALGARY_IOMMU */
207 
208                 p += strcspn(p, ",");
209                 if (*p == ',')
210                         ++p;
211         }
212         return 0;
213 }
214 early_param("iommu", iommu_setup);
215 
216 int x86_dma_supported(struct device *dev, u64 mask)
217 {
218 #ifdef CONFIG_PCI
219         if (mask > 0xffffffff && forbid_dac > 0) {
220                 dev_info(dev, "PCI: Disallowing DAC for device\n");
221                 return 0;
222         }
223 #endif
224 
225         /* Copied from i386. Doesn't make much sense, because it will
226            only work for pci_alloc_coherent.
227            The caller just has to use GFP_DMA in this case. */
228         if (mask < DMA_BIT_MASK(24))
229                 return 0;
230 
231         /* Tell the device to use SAC when IOMMU force is on.  This
232            allows the driver to use cheaper accesses in some cases.
233 
234            Problem with this is that if we overflow the IOMMU area and
235            return DAC as fallback address the device may not handle it
236            correctly.
237 
238            As a special case some controllers have a 39bit address
239            mode that is as efficient as 32bit (aic79xx). Don't force
240            SAC for these.  Assume all masks <= 40 bits are of this
241            type. Normally this doesn't make any difference, but gives
242            more gentle handling of IOMMU overflow. */
243         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
244                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
245                 return 0;
246         }
247 
248         return 1;
249 }
250 
251 static int __init pci_iommu_init(void)
252 {
253         struct iommu_table_entry *p;
254         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
255 
256 #ifdef CONFIG_PCI
257         dma_debug_add_bus(&pci_bus_type);
258 #endif
259         x86_init.iommu.iommu_init();
260 
261         for (p = __iommu_table; p < __iommu_table_end; p++) {
262                 if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
263                         p->late_init();
264         }
265 
266         return 0;
267 }
268 /* Must execute after PCI subsystem */
269 rootfs_initcall(pci_iommu_init);
270 
271 #ifdef CONFIG_PCI
272 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
273 
274 static void via_no_dac(struct pci_dev *dev)
275 {
276         if (forbid_dac == 0) {
277                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
278                 forbid_dac = 1;
279         }
280 }
281 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID,
282                                 PCI_CLASS_BRIDGE_PCI, 8, via_no_dac);
283 #endif
284 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp