~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/mm/ioremap.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Re-map IO memory to kernel address space so that we can access it.
  3  * This is needed for high PCI addresses that aren't mapped in the
  4  * 640k-1MB IO memory area on PC's
  5  *
  6  * (C) Copyright 1995 1996 Linus Torvalds
  7  */
  8 
  9 #include <linux/bootmem.h>
 10 #include <linux/init.h>
 11 #include <linux/io.h>
 12 #include <linux/ioport.h>
 13 #include <linux/slab.h>
 14 #include <linux/vmalloc.h>
 15 #include <linux/mmiotrace.h>
 16 #include <linux/mem_encrypt.h>
 17 #include <linux/efi.h>
 18 
 19 #include <asm/set_memory.h>
 20 #include <asm/e820/api.h>
 21 #include <asm/fixmap.h>
 22 #include <asm/pgtable.h>
 23 #include <asm/tlbflush.h>
 24 #include <asm/pgalloc.h>
 25 #include <asm/pat.h>
 26 #include <asm/setup.h>
 27 
 28 #include "physaddr.h"
 29 
 30 struct ioremap_mem_flags {
 31         bool system_ram;
 32         bool desc_other;
 33 };
 34 
 35 /*
 36  * Fix up the linear direct mapping of the kernel to avoid cache attribute
 37  * conflicts.
 38  */
 39 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
 40                         enum page_cache_mode pcm)
 41 {
 42         unsigned long nrpages = size >> PAGE_SHIFT;
 43         int err;
 44 
 45         switch (pcm) {
 46         case _PAGE_CACHE_MODE_UC:
 47         default:
 48                 err = _set_memory_uc(vaddr, nrpages);
 49                 break;
 50         case _PAGE_CACHE_MODE_WC:
 51                 err = _set_memory_wc(vaddr, nrpages);
 52                 break;
 53         case _PAGE_CACHE_MODE_WT:
 54                 err = _set_memory_wt(vaddr, nrpages);
 55                 break;
 56         case _PAGE_CACHE_MODE_WB:
 57                 err = _set_memory_wb(vaddr, nrpages);
 58                 break;
 59         }
 60 
 61         return err;
 62 }
 63 
 64 static bool __ioremap_check_ram(struct resource *res)
 65 {
 66         unsigned long start_pfn, stop_pfn;
 67         unsigned long i;
 68 
 69         if ((res->flags & IORESOURCE_SYSTEM_RAM) != IORESOURCE_SYSTEM_RAM)
 70                 return false;
 71 
 72         start_pfn = (res->start + PAGE_SIZE - 1) >> PAGE_SHIFT;
 73         stop_pfn = (res->end + 1) >> PAGE_SHIFT;
 74         if (stop_pfn > start_pfn) {
 75                 for (i = 0; i < (stop_pfn - start_pfn); ++i)
 76                         if (pfn_valid(start_pfn + i) &&
 77                             !PageReserved(pfn_to_page(start_pfn + i)))
 78                                 return true;
 79         }
 80 
 81         return false;
 82 }
 83 
 84 static int __ioremap_check_desc_other(struct resource *res)
 85 {
 86         return (res->desc != IORES_DESC_NONE);
 87 }
 88 
 89 static int __ioremap_res_check(struct resource *res, void *arg)
 90 {
 91         struct ioremap_mem_flags *flags = arg;
 92 
 93         if (!flags->system_ram)
 94                 flags->system_ram = __ioremap_check_ram(res);
 95 
 96         if (!flags->desc_other)
 97                 flags->desc_other = __ioremap_check_desc_other(res);
 98 
 99         return flags->system_ram && flags->desc_other;
100 }
101 
102 /*
103  * To avoid multiple resource walks, this function walks resources marked as
104  * IORESOURCE_MEM and IORESOURCE_BUSY and looking for system RAM and/or a
105  * resource described not as IORES_DESC_NONE (e.g. IORES_DESC_ACPI_TABLES).
106  */
107 static void __ioremap_check_mem(resource_size_t addr, unsigned long size,
108                                 struct ioremap_mem_flags *flags)
109 {
110         u64 start, end;
111 
112         start = (u64)addr;
113         end = start + size - 1;
114         memset(flags, 0, sizeof(*flags));
115 
116         walk_mem_res(start, end, flags, __ioremap_res_check);
117 }
118 
119 /*
120  * Remap an arbitrary physical address space into the kernel virtual
121  * address space. It transparently creates kernel huge I/O mapping when
122  * the physical address is aligned by a huge page size (1GB or 2MB) and
123  * the requested size is at least the huge page size.
124  *
125  * NOTE: MTRRs can override PAT memory types with a 4KB granularity.
126  * Therefore, the mapping code falls back to use a smaller page toward 4KB
127  * when a mapping range is covered by non-WB type of MTRRs.
128  *
129  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
130  * have to convert them into an offset in a page-aligned mapping, but the
131  * caller shouldn't need to know that small detail.
132  */
133 static void __iomem *__ioremap_caller(resource_size_t phys_addr,
134                 unsigned long size, enum page_cache_mode pcm, void *caller)
135 {
136         unsigned long offset, vaddr;
137         resource_size_t last_addr;
138         const resource_size_t unaligned_phys_addr = phys_addr;
139         const unsigned long unaligned_size = size;
140         struct ioremap_mem_flags mem_flags;
141         struct vm_struct *area;
142         enum page_cache_mode new_pcm;
143         pgprot_t prot;
144         int retval;
145         void __iomem *ret_addr;
146 
147         /* Don't allow wraparound or zero size */
148         last_addr = phys_addr + size - 1;
149         if (!size || last_addr < phys_addr)
150                 return NULL;
151 
152         if (!phys_addr_valid(phys_addr)) {
153                 printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
154                        (unsigned long long)phys_addr);
155                 WARN_ON_ONCE(1);
156                 return NULL;
157         }
158 
159         __ioremap_check_mem(phys_addr, size, &mem_flags);
160 
161         /*
162          * Don't allow anybody to remap normal RAM that we're using..
163          */
164         if (mem_flags.system_ram) {
165                 WARN_ONCE(1, "ioremap on RAM at %pa - %pa\n",
166                           &phys_addr, &last_addr);
167                 return NULL;
168         }
169 
170         /*
171          * Mappings have to be page-aligned
172          */
173         offset = phys_addr & ~PAGE_MASK;
174         phys_addr &= PHYSICAL_PAGE_MASK;
175         size = PAGE_ALIGN(last_addr+1) - phys_addr;
176 
177         retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
178                                                 pcm, &new_pcm);
179         if (retval) {
180                 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
181                 return NULL;
182         }
183 
184         if (pcm != new_pcm) {
185                 if (!is_new_memtype_allowed(phys_addr, size, pcm, new_pcm)) {
186                         printk(KERN_ERR
187                 "ioremap error for 0x%llx-0x%llx, requested 0x%x, got 0x%x\n",
188                                 (unsigned long long)phys_addr,
189                                 (unsigned long long)(phys_addr + size),
190                                 pcm, new_pcm);
191                         goto err_free_memtype;
192                 }
193                 pcm = new_pcm;
194         }
195 
196         /*
197          * If the page being mapped is in memory and SEV is active then
198          * make sure the memory encryption attribute is enabled in the
199          * resulting mapping.
200          */
201         prot = PAGE_KERNEL_IO;
202         if (sev_active() && mem_flags.desc_other)
203                 prot = pgprot_encrypted(prot);
204 
205         switch (pcm) {
206         case _PAGE_CACHE_MODE_UC:
207         default:
208                 prot = __pgprot(pgprot_val(prot) |
209                                 cachemode2protval(_PAGE_CACHE_MODE_UC));
210                 break;
211         case _PAGE_CACHE_MODE_UC_MINUS:
212                 prot = __pgprot(pgprot_val(prot) |
213                                 cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS));
214                 break;
215         case _PAGE_CACHE_MODE_WC:
216                 prot = __pgprot(pgprot_val(prot) |
217                                 cachemode2protval(_PAGE_CACHE_MODE_WC));
218                 break;
219         case _PAGE_CACHE_MODE_WT:
220                 prot = __pgprot(pgprot_val(prot) |
221                                 cachemode2protval(_PAGE_CACHE_MODE_WT));
222                 break;
223         case _PAGE_CACHE_MODE_WB:
224                 break;
225         }
226 
227         /*
228          * Ok, go for it..
229          */
230         area = get_vm_area_caller(size, VM_IOREMAP, caller);
231         if (!area)
232                 goto err_free_memtype;
233         area->phys_addr = phys_addr;
234         vaddr = (unsigned long) area->addr;
235 
236         if (kernel_map_sync_memtype(phys_addr, size, pcm))
237                 goto err_free_area;
238 
239         if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
240                 goto err_free_area;
241 
242         ret_addr = (void __iomem *) (vaddr + offset);
243         mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
244 
245         /*
246          * Check if the request spans more than any BAR in the iomem resource
247          * tree.
248          */
249         if (iomem_map_sanity_check(unaligned_phys_addr, unaligned_size))
250                 pr_warn("caller %pS mapping multiple BARs\n", caller);
251 
252         return ret_addr;
253 err_free_area:
254         free_vm_area(area);
255 err_free_memtype:
256         free_memtype(phys_addr, phys_addr + size);
257         return NULL;
258 }
259 
260 /**
261  * ioremap_nocache     -   map bus memory into CPU space
262  * @phys_addr:    bus address of the memory
263  * @size:      size of the resource to map
264  *
265  * ioremap_nocache performs a platform specific sequence of operations to
266  * make bus memory CPU accessible via the readb/readw/readl/writeb/
267  * writew/writel functions and the other mmio helpers. The returned
268  * address is not guaranteed to be usable directly as a virtual
269  * address.
270  *
271  * This version of ioremap ensures that the memory is marked uncachable
272  * on the CPU as well as honouring existing caching rules from things like
273  * the PCI bus. Note that there are other caches and buffers on many
274  * busses. In particular driver authors should read up on PCI writes
275  *
276  * It's useful if some control registers are in such an area and
277  * write combining or read caching is not desirable:
278  *
279  * Must be freed with iounmap.
280  */
281 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
282 {
283         /*
284          * Ideally, this should be:
285          *      pat_enabled() ? _PAGE_CACHE_MODE_UC : _PAGE_CACHE_MODE_UC_MINUS;
286          *
287          * Till we fix all X drivers to use ioremap_wc(), we will use
288          * UC MINUS. Drivers that are certain they need or can already
289          * be converted over to strong UC can use ioremap_uc().
290          */
291         enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC_MINUS;
292 
293         return __ioremap_caller(phys_addr, size, pcm,
294                                 __builtin_return_address(0));
295 }
296 EXPORT_SYMBOL(ioremap_nocache);
297 
298 /**
299  * ioremap_uc     -   map bus memory into CPU space as strongly uncachable
300  * @phys_addr:    bus address of the memory
301  * @size:      size of the resource to map
302  *
303  * ioremap_uc performs a platform specific sequence of operations to
304  * make bus memory CPU accessible via the readb/readw/readl/writeb/
305  * writew/writel functions and the other mmio helpers. The returned
306  * address is not guaranteed to be usable directly as a virtual
307  * address.
308  *
309  * This version of ioremap ensures that the memory is marked with a strong
310  * preference as completely uncachable on the CPU when possible. For non-PAT
311  * systems this ends up setting page-attribute flags PCD=1, PWT=1. For PAT
312  * systems this will set the PAT entry for the pages as strong UC.  This call
313  * will honor existing caching rules from things like the PCI bus. Note that
314  * there are other caches and buffers on many busses. In particular driver
315  * authors should read up on PCI writes.
316  *
317  * It's useful if some control registers are in such an area and
318  * write combining or read caching is not desirable:
319  *
320  * Must be freed with iounmap.
321  */
322 void __iomem *ioremap_uc(resource_size_t phys_addr, unsigned long size)
323 {
324         enum page_cache_mode pcm = _PAGE_CACHE_MODE_UC;
325 
326         return __ioremap_caller(phys_addr, size, pcm,
327                                 __builtin_return_address(0));
328 }
329 EXPORT_SYMBOL_GPL(ioremap_uc);
330 
331 /**
332  * ioremap_wc   -       map memory into CPU space write combined
333  * @phys_addr:  bus address of the memory
334  * @size:       size of the resource to map
335  *
336  * This version of ioremap ensures that the memory is marked write combining.
337  * Write combining allows faster writes to some hardware devices.
338  *
339  * Must be freed with iounmap.
340  */
341 void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
342 {
343         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WC,
344                                         __builtin_return_address(0));
345 }
346 EXPORT_SYMBOL(ioremap_wc);
347 
348 /**
349  * ioremap_wt   -       map memory into CPU space write through
350  * @phys_addr:  bus address of the memory
351  * @size:       size of the resource to map
352  *
353  * This version of ioremap ensures that the memory is marked write through.
354  * Write through stores data into memory while keeping the cache up-to-date.
355  *
356  * Must be freed with iounmap.
357  */
358 void __iomem *ioremap_wt(resource_size_t phys_addr, unsigned long size)
359 {
360         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WT,
361                                         __builtin_return_address(0));
362 }
363 EXPORT_SYMBOL(ioremap_wt);
364 
365 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
366 {
367         return __ioremap_caller(phys_addr, size, _PAGE_CACHE_MODE_WB,
368                                 __builtin_return_address(0));
369 }
370 EXPORT_SYMBOL(ioremap_cache);
371 
372 void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
373                                 unsigned long prot_val)
374 {
375         return __ioremap_caller(phys_addr, size,
376                                 pgprot2cachemode(__pgprot(prot_val)),
377                                 __builtin_return_address(0));
378 }
379 EXPORT_SYMBOL(ioremap_prot);
380 
381 /**
382  * iounmap - Free a IO remapping
383  * @addr: virtual address from ioremap_*
384  *
385  * Caller must ensure there is only one unmapping for the same pointer.
386  */
387 void iounmap(volatile void __iomem *addr)
388 {
389         struct vm_struct *p, *o;
390 
391         if ((void __force *)addr <= high_memory)
392                 return;
393 
394         /*
395          * The PCI/ISA range special-casing was removed from __ioremap()
396          * so this check, in theory, can be removed. However, there are
397          * cases where iounmap() is called for addresses not obtained via
398          * ioremap() (vga16fb for example). Add a warning so that these
399          * cases can be caught and fixed.
400          */
401         if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
402             (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) {
403                 WARN(1, "iounmap() called for ISA range not obtained using ioremap()\n");
404                 return;
405         }
406 
407         mmiotrace_iounmap(addr);
408 
409         addr = (volatile void __iomem *)
410                 (PAGE_MASK & (unsigned long __force)addr);
411 
412         /* Use the vm area unlocked, assuming the caller
413            ensures there isn't another iounmap for the same address
414            in parallel. Reuse of the virtual address is prevented by
415            leaving it in the global lists until we're done with it.
416            cpa takes care of the direct mappings. */
417         p = find_vm_area((void __force *)addr);
418 
419         if (!p) {
420                 printk(KERN_ERR "iounmap: bad address %p\n", addr);
421                 dump_stack();
422                 return;
423         }
424 
425         free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
426 
427         /* Finally remove it */
428         o = remove_vm_area((void __force *)addr);
429         BUG_ON(p != o || o == NULL);
430         kfree(p);
431 }
432 EXPORT_SYMBOL(iounmap);
433 
434 int __init arch_ioremap_pud_supported(void)
435 {
436 #ifdef CONFIG_X86_64
437         return boot_cpu_has(X86_FEATURE_GBPAGES);
438 #else
439         return 0;
440 #endif
441 }
442 
443 int __init arch_ioremap_pmd_supported(void)
444 {
445         return boot_cpu_has(X86_FEATURE_PSE);
446 }
447 
448 /*
449  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
450  * access
451  */
452 void *xlate_dev_mem_ptr(phys_addr_t phys)
453 {
454         unsigned long start  = phys &  PAGE_MASK;
455         unsigned long offset = phys & ~PAGE_MASK;
456         void *vaddr;
457 
458         /* memremap() maps if RAM, otherwise falls back to ioremap() */
459         vaddr = memremap(start, PAGE_SIZE, MEMREMAP_WB);
460 
461         /* Only add the offset on success and return NULL if memremap() failed */
462         if (vaddr)
463                 vaddr += offset;
464 
465         return vaddr;
466 }
467 
468 void unxlate_dev_mem_ptr(phys_addr_t phys, void *addr)
469 {
470         memunmap((void *)((unsigned long)addr & PAGE_MASK));
471 }
472 
473 /*
474  * Examine the physical address to determine if it is an area of memory
475  * that should be mapped decrypted.  If the memory is not part of the
476  * kernel usable area it was accessed and created decrypted, so these
477  * areas should be mapped decrypted. And since the encryption key can
478  * change across reboots, persistent memory should also be mapped
479  * decrypted.
480  *
481  * If SEV is active, that implies that BIOS/UEFI also ran encrypted so
482  * only persistent memory should be mapped decrypted.
483  */
484 static bool memremap_should_map_decrypted(resource_size_t phys_addr,
485                                           unsigned long size)
486 {
487         int is_pmem;
488 
489         /*
490          * Check if the address is part of a persistent memory region.
491          * This check covers areas added by E820, EFI and ACPI.
492          */
493         is_pmem = region_intersects(phys_addr, size, IORESOURCE_MEM,
494                                     IORES_DESC_PERSISTENT_MEMORY);
495         if (is_pmem != REGION_DISJOINT)
496                 return true;
497 
498         /*
499          * Check if the non-volatile attribute is set for an EFI
500          * reserved area.
501          */
502         if (efi_enabled(EFI_BOOT)) {
503                 switch (efi_mem_type(phys_addr)) {
504                 case EFI_RESERVED_TYPE:
505                         if (efi_mem_attributes(phys_addr) & EFI_MEMORY_NV)
506                                 return true;
507                         break;
508                 default:
509                         break;
510                 }
511         }
512 
513         /* Check if the address is outside kernel usable area */
514         switch (e820__get_entry_type(phys_addr, phys_addr + size - 1)) {
515         case E820_TYPE_RESERVED:
516         case E820_TYPE_ACPI:
517         case E820_TYPE_NVS:
518         case E820_TYPE_UNUSABLE:
519                 /* For SEV, these areas are encrypted */
520                 if (sev_active())
521                         break;
522                 /* Fallthrough */
523 
524         case E820_TYPE_PRAM:
525                 return true;
526         default:
527                 break;
528         }
529 
530         return false;
531 }
532 
533 /*
534  * Examine the physical address to determine if it is EFI data. Check
535  * it against the boot params structure and EFI tables and memory types.
536  */
537 static bool memremap_is_efi_data(resource_size_t phys_addr,
538                                  unsigned long size)
539 {
540         u64 paddr;
541 
542         /* Check if the address is part of EFI boot/runtime data */
543         if (!efi_enabled(EFI_BOOT))
544                 return false;
545 
546         paddr = boot_params.efi_info.efi_memmap_hi;
547         paddr <<= 32;
548         paddr |= boot_params.efi_info.efi_memmap;
549         if (phys_addr == paddr)
550                 return true;
551 
552         paddr = boot_params.efi_info.efi_systab_hi;
553         paddr <<= 32;
554         paddr |= boot_params.efi_info.efi_systab;
555         if (phys_addr == paddr)
556                 return true;
557 
558         if (efi_is_table_address(phys_addr))
559                 return true;
560 
561         switch (efi_mem_type(phys_addr)) {
562         case EFI_BOOT_SERVICES_DATA:
563         case EFI_RUNTIME_SERVICES_DATA:
564                 return true;
565         default:
566                 break;
567         }
568 
569         return false;
570 }
571 
572 /*
573  * Examine the physical address to determine if it is boot data by checking
574  * it against the boot params setup_data chain.
575  */
576 static bool memremap_is_setup_data(resource_size_t phys_addr,
577                                    unsigned long size)
578 {
579         struct setup_data *data;
580         u64 paddr, paddr_next;
581 
582         paddr = boot_params.hdr.setup_data;
583         while (paddr) {
584                 unsigned int len;
585 
586                 if (phys_addr == paddr)
587                         return true;
588 
589                 data = memremap(paddr, sizeof(*data),
590                                 MEMREMAP_WB | MEMREMAP_DEC);
591 
592                 paddr_next = data->next;
593                 len = data->len;
594 
595                 memunmap(data);
596 
597                 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
598                         return true;
599 
600                 paddr = paddr_next;
601         }
602 
603         return false;
604 }
605 
606 /*
607  * Examine the physical address to determine if it is boot data by checking
608  * it against the boot params setup_data chain (early boot version).
609  */
610 static bool __init early_memremap_is_setup_data(resource_size_t phys_addr,
611                                                 unsigned long size)
612 {
613         struct setup_data *data;
614         u64 paddr, paddr_next;
615 
616         paddr = boot_params.hdr.setup_data;
617         while (paddr) {
618                 unsigned int len;
619 
620                 if (phys_addr == paddr)
621                         return true;
622 
623                 data = early_memremap_decrypted(paddr, sizeof(*data));
624 
625                 paddr_next = data->next;
626                 len = data->len;
627 
628                 early_memunmap(data, sizeof(*data));
629 
630                 if ((phys_addr > paddr) && (phys_addr < (paddr + len)))
631                         return true;
632 
633                 paddr = paddr_next;
634         }
635 
636         return false;
637 }
638 
639 /*
640  * Architecture function to determine if RAM remap is allowed. By default, a
641  * RAM remap will map the data as encrypted. Determine if a RAM remap should
642  * not be done so that the data will be mapped decrypted.
643  */
644 bool arch_memremap_can_ram_remap(resource_size_t phys_addr, unsigned long size,
645                                  unsigned long flags)
646 {
647         if (!mem_encrypt_active())
648                 return true;
649 
650         if (flags & MEMREMAP_ENC)
651                 return true;
652 
653         if (flags & MEMREMAP_DEC)
654                 return false;
655 
656         if (sme_active()) {
657                 if (memremap_is_setup_data(phys_addr, size) ||
658                     memremap_is_efi_data(phys_addr, size))
659                         return false;
660         }
661 
662         return !memremap_should_map_decrypted(phys_addr, size);
663 }
664 
665 /*
666  * Architecture override of __weak function to adjust the protection attributes
667  * used when remapping memory. By default, early_memremap() will map the data
668  * as encrypted. Determine if an encrypted mapping should not be done and set
669  * the appropriate protection attributes.
670  */
671 pgprot_t __init early_memremap_pgprot_adjust(resource_size_t phys_addr,
672                                              unsigned long size,
673                                              pgprot_t prot)
674 {
675         bool encrypted_prot;
676 
677         if (!mem_encrypt_active())
678                 return prot;
679 
680         encrypted_prot = true;
681 
682         if (sme_active()) {
683                 if (early_memremap_is_setup_data(phys_addr, size) ||
684                     memremap_is_efi_data(phys_addr, size))
685                         encrypted_prot = false;
686         }
687 
688         if (encrypted_prot && memremap_should_map_decrypted(phys_addr, size))
689                 encrypted_prot = false;
690 
691         return encrypted_prot ? pgprot_encrypted(prot)
692                               : pgprot_decrypted(prot);
693 }
694 
695 bool phys_mem_access_encrypted(unsigned long phys_addr, unsigned long size)
696 {
697         return arch_memremap_can_ram_remap(phys_addr, size, 0);
698 }
699 
700 #ifdef CONFIG_ARCH_USE_MEMREMAP_PROT
701 /* Remap memory with encryption */
702 void __init *early_memremap_encrypted(resource_size_t phys_addr,
703                                       unsigned long size)
704 {
705         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC);
706 }
707 
708 /*
709  * Remap memory with encryption and write-protected - cannot be called
710  * before pat_init() is called
711  */
712 void __init *early_memremap_encrypted_wp(resource_size_t phys_addr,
713                                          unsigned long size)
714 {
715         /* Be sure the write-protect PAT entry is set for write-protect */
716         if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
717                 return NULL;
718 
719         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_ENC_WP);
720 }
721 
722 /* Remap memory without encryption */
723 void __init *early_memremap_decrypted(resource_size_t phys_addr,
724                                       unsigned long size)
725 {
726         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC);
727 }
728 
729 /*
730  * Remap memory without encryption and write-protected - cannot be called
731  * before pat_init() is called
732  */
733 void __init *early_memremap_decrypted_wp(resource_size_t phys_addr,
734                                          unsigned long size)
735 {
736         /* Be sure the write-protect PAT entry is set for write-protect */
737         if (__pte2cachemode_tbl[_PAGE_CACHE_MODE_WP] != _PAGE_CACHE_MODE_WP)
738                 return NULL;
739 
740         return early_memremap_prot(phys_addr, size, __PAGE_KERNEL_NOENC_WP);
741 }
742 #endif  /* CONFIG_ARCH_USE_MEMREMAP_PROT */
743 
744 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
745 
746 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
747 {
748         /* Don't assume we're using swapper_pg_dir at this point */
749         pgd_t *base = __va(read_cr3_pa());
750         pgd_t *pgd = &base[pgd_index(addr)];
751         p4d_t *p4d = p4d_offset(pgd, addr);
752         pud_t *pud = pud_offset(p4d, addr);
753         pmd_t *pmd = pmd_offset(pud, addr);
754 
755         return pmd;
756 }
757 
758 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
759 {
760         return &bm_pte[pte_index(addr)];
761 }
762 
763 bool __init is_early_ioremap_ptep(pte_t *ptep)
764 {
765         return ptep >= &bm_pte[0] && ptep < &bm_pte[PAGE_SIZE/sizeof(pte_t)];
766 }
767 
768 void __init early_ioremap_init(void)
769 {
770         pmd_t *pmd;
771 
772 #ifdef CONFIG_X86_64
773         BUILD_BUG_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
774 #else
775         WARN_ON((fix_to_virt(0) + PAGE_SIZE) & ((1 << PMD_SHIFT) - 1));
776 #endif
777 
778         early_ioremap_setup();
779 
780         pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
781         memset(bm_pte, 0, sizeof(bm_pte));
782         pmd_populate_kernel(&init_mm, pmd, bm_pte);
783 
784         /*
785          * The boot-ioremap range spans multiple pmds, for which
786          * we are not prepared:
787          */
788 #define __FIXADDR_TOP (-PAGE_SIZE)
789         BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT)
790                      != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT));
791 #undef __FIXADDR_TOP
792         if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
793                 WARN_ON(1);
794                 printk(KERN_WARNING "pmd %p != %p\n",
795                        pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
796                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
797                         fix_to_virt(FIX_BTMAP_BEGIN));
798                 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
799                         fix_to_virt(FIX_BTMAP_END));
800 
801                 printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
802                 printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
803                        FIX_BTMAP_BEGIN);
804         }
805 }
806 
807 void __init __early_set_fixmap(enum fixed_addresses idx,
808                                phys_addr_t phys, pgprot_t flags)
809 {
810         unsigned long addr = __fix_to_virt(idx);
811         pte_t *pte;
812 
813         if (idx >= __end_of_fixed_addresses) {
814                 BUG();
815                 return;
816         }
817         pte = early_ioremap_pte(addr);
818 
819         if (pgprot_val(flags))
820                 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
821         else
822                 pte_clear(&init_mm, addr, pte);
823         __flush_tlb_one_kernel(addr);
824 }
825 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp