~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/pci-dma.c

Version: ~ [ linux-5.13-rc7 ] ~ [ linux-5.12.12 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.45 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.127 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.195 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.237 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.273 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.273 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include <linux/dma-mapping.h>
  2 #include <linux/dma-debug.h>
  3 #include <linux/dmar.h>
  4 #include <linux/export.h>
  5 #include <linux/bootmem.h>
  6 #include <linux/gfp.h>
  7 #include <linux/pci.h>
  8 #include <linux/kmemleak.h>
  9 
 10 #include <asm/proto.h>
 11 #include <asm/dma.h>
 12 #include <asm/iommu.h>
 13 #include <asm/gart.h>
 14 #include <asm/calgary.h>
 15 #include <asm/x86_init.h>
 16 #include <asm/iommu_table.h>
 17 
 18 static int forbid_dac __read_mostly;
 19 
 20 struct dma_map_ops *dma_ops = &nommu_dma_ops;
 21 EXPORT_SYMBOL(dma_ops);
 22 
 23 static int iommu_sac_force __read_mostly;
 24 
 25 #ifdef CONFIG_IOMMU_DEBUG
 26 int panic_on_overflow __read_mostly = 1;
 27 int force_iommu __read_mostly = 1;
 28 #else
 29 int panic_on_overflow __read_mostly = 0;
 30 int force_iommu __read_mostly = 0;
 31 #endif
 32 
 33 int iommu_merge __read_mostly = 0;
 34 
 35 int no_iommu __read_mostly;
 36 /* Set this to 1 if there is a HW IOMMU in the system */
 37 int iommu_detected __read_mostly = 0;
 38 
 39 /*
 40  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
 41  * If this variable is 1, IOMMU implementations do no DMA translation for
 42  * devices and allow every device to access to whole physical memory. This is
 43  * useful if a user wants to use an IOMMU only for KVM device assignment to
 44  * guests and not for driver dma translation.
 45  */
 46 int iommu_pass_through __read_mostly;
 47 
 48 extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
 49 
 50 /* Dummy device used for NULL arguments (normally ISA). */
 51 struct device x86_dma_fallback_dev = {
 52         .init_name = "fallback device",
 53         .coherent_dma_mask = ISA_DMA_BIT_MASK,
 54         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
 55 };
 56 EXPORT_SYMBOL(x86_dma_fallback_dev);
 57 
 58 /* Number of entries preallocated for DMA-API debugging */
 59 #define PREALLOC_DMA_DEBUG_ENTRIES       65536
 60 
 61 int dma_set_mask(struct device *dev, u64 mask)
 62 {
 63         if (!dev->dma_mask || !dma_supported(dev, mask))
 64                 return -EIO;
 65 
 66         *dev->dma_mask = mask;
 67 
 68         return 0;
 69 }
 70 EXPORT_SYMBOL(dma_set_mask);
 71 
 72 void __init pci_iommu_alloc(void)
 73 {
 74         struct iommu_table_entry *p;
 75 
 76         sort_iommu_table(__iommu_table, __iommu_table_end);
 77         check_iommu_entries(__iommu_table, __iommu_table_end);
 78 
 79         for (p = __iommu_table; p < __iommu_table_end; p++) {
 80                 if (p && p->detect && p->detect() > 0) {
 81                         p->flags |= IOMMU_DETECTED;
 82                         if (p->early_init)
 83                                 p->early_init();
 84                         if (p->flags & IOMMU_FINISH_IF_DETECTED)
 85                                 break;
 86                 }
 87         }
 88 }
 89 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 90                                  dma_addr_t *dma_addr, gfp_t flag,
 91                                  struct dma_attrs *attrs)
 92 {
 93         unsigned long dma_mask;
 94         struct page *page;
 95         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 96         dma_addr_t addr;
 97 
 98         dma_mask = dma_alloc_coherent_mask(dev, flag);
 99 
100         flag |= __GFP_ZERO;
101 again:
102         page = NULL;
103         /* CMA can be used only in the context which permits sleeping */
104         if (flag & __GFP_WAIT)
105                 page = dma_alloc_from_contiguous(dev, count, get_order(size));
106         /* fallback */
107         if (!page)
108                 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
109         if (!page)
110                 return NULL;
111 
112         addr = page_to_phys(page);
113         if (addr + size > dma_mask) {
114                 __free_pages(page, get_order(size));
115 
116                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
117                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
118                         goto again;
119                 }
120 
121                 return NULL;
122         }
123 
124         *dma_addr = addr;
125         return page_address(page);
126 }
127 
128 void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr,
129                                dma_addr_t dma_addr, struct dma_attrs *attrs)
130 {
131         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
132         struct page *page = virt_to_page(vaddr);
133 
134         if (!dma_release_from_contiguous(dev, page, count))
135                 free_pages((unsigned long)vaddr, get_order(size));
136 }
137 
138 /*
139  * See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
140  * parameter documentation.
141  */
142 static __init int iommu_setup(char *p)
143 {
144         iommu_merge = 1;
145 
146         if (!p)
147                 return -EINVAL;
148 
149         while (*p) {
150                 if (!strncmp(p, "off", 3))
151                         no_iommu = 1;
152                 /* gart_parse_options has more force support */
153                 if (!strncmp(p, "force", 5))
154                         force_iommu = 1;
155                 if (!strncmp(p, "noforce", 7)) {
156                         iommu_merge = 0;
157                         force_iommu = 0;
158                 }
159 
160                 if (!strncmp(p, "biomerge", 8)) {
161                         iommu_merge = 1;
162                         force_iommu = 1;
163                 }
164                 if (!strncmp(p, "panic", 5))
165                         panic_on_overflow = 1;
166                 if (!strncmp(p, "nopanic", 7))
167                         panic_on_overflow = 0;
168                 if (!strncmp(p, "merge", 5)) {
169                         iommu_merge = 1;
170                         force_iommu = 1;
171                 }
172                 if (!strncmp(p, "nomerge", 7))
173                         iommu_merge = 0;
174                 if (!strncmp(p, "forcesac", 8))
175                         iommu_sac_force = 1;
176                 if (!strncmp(p, "allowdac", 8))
177                         forbid_dac = 0;
178                 if (!strncmp(p, "nodac", 5))
179                         forbid_dac = 1;
180                 if (!strncmp(p, "usedac", 6)) {
181                         forbid_dac = -1;
182                         return 1;
183                 }
184 #ifdef CONFIG_SWIOTLB
185                 if (!strncmp(p, "soft", 4))
186                         swiotlb = 1;
187 #endif
188                 if (!strncmp(p, "pt", 2))
189                         iommu_pass_through = 1;
190 
191                 gart_parse_options(p);
192 
193 #ifdef CONFIG_CALGARY_IOMMU
194                 if (!strncmp(p, "calgary", 7))
195                         use_calgary = 1;
196 #endif /* CONFIG_CALGARY_IOMMU */
197 
198                 p += strcspn(p, ",");
199                 if (*p == ',')
200                         ++p;
201         }
202         return 0;
203 }
204 early_param("iommu", iommu_setup);
205 
206 int dma_supported(struct device *dev, u64 mask)
207 {
208         struct dma_map_ops *ops = get_dma_ops(dev);
209 
210 #ifdef CONFIG_PCI
211         if (mask > 0xffffffff && forbid_dac > 0) {
212                 dev_info(dev, "PCI: Disallowing DAC for device\n");
213                 return 0;
214         }
215 #endif
216 
217         if (ops->dma_supported)
218                 return ops->dma_supported(dev, mask);
219 
220         /* Copied from i386. Doesn't make much sense, because it will
221            only work for pci_alloc_coherent.
222            The caller just has to use GFP_DMA in this case. */
223         if (mask < DMA_BIT_MASK(24))
224                 return 0;
225 
226         /* Tell the device to use SAC when IOMMU force is on.  This
227            allows the driver to use cheaper accesses in some cases.
228 
229            Problem with this is that if we overflow the IOMMU area and
230            return DAC as fallback address the device may not handle it
231            correctly.
232 
233            As a special case some controllers have a 39bit address
234            mode that is as efficient as 32bit (aic79xx). Don't force
235            SAC for these.  Assume all masks <= 40 bits are of this
236            type. Normally this doesn't make any difference, but gives
237            more gentle handling of IOMMU overflow. */
238         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
239                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
240                 return 0;
241         }
242 
243         return 1;
244 }
245 EXPORT_SYMBOL(dma_supported);
246 
247 static int __init pci_iommu_init(void)
248 {
249         struct iommu_table_entry *p;
250         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
251 
252 #ifdef CONFIG_PCI
253         dma_debug_add_bus(&pci_bus_type);
254 #endif
255         x86_init.iommu.iommu_init();
256 
257         for (p = __iommu_table; p < __iommu_table_end; p++) {
258                 if (p && (p->flags & IOMMU_DETECTED) && p->late_init)
259                         p->late_init();
260         }
261 
262         return 0;
263 }
264 /* Must execute after PCI subsystem */
265 rootfs_initcall(pci_iommu_init);
266 
267 #ifdef CONFIG_PCI
268 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
269 
270 static void via_no_dac(struct pci_dev *dev)
271 {
272         if (forbid_dac == 0) {
273                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
274                 forbid_dac = 1;
275         }
276 }
277 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID,
278                                 PCI_CLASS_BRIDGE_PCI, 8, via_no_dac);
279 #endif
280 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp