~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/platform/efi/efi.c

Version: ~ [ linux-5.12 ] ~ [ linux-5.11.16 ] ~ [ linux-5.10.32 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.114 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.188 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.231 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.267 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.267 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Common EFI (Extensible Firmware Interface) support functions
  4  * Based on Extensible Firmware Interface Specification version 1.0
  5  *
  6  * Copyright (C) 1999 VA Linux Systems
  7  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
  8  * Copyright (C) 1999-2002 Hewlett-Packard Co.
  9  *      David Mosberger-Tang <davidm@hpl.hp.com>
 10  *      Stephane Eranian <eranian@hpl.hp.com>
 11  * Copyright (C) 2005-2008 Intel Co.
 12  *      Fenghua Yu <fenghua.yu@intel.com>
 13  *      Bibo Mao <bibo.mao@intel.com>
 14  *      Chandramouli Narayanan <mouli@linux.intel.com>
 15  *      Huang Ying <ying.huang@intel.com>
 16  * Copyright (C) 2013 SuSE Labs
 17  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
 18  *
 19  * Copied from efi_32.c to eliminate the duplicated code between EFI
 20  * 32/64 support code. --ying 2007-10-26
 21  *
 22  * All EFI Runtime Services are not implemented yet as EFI only
 23  * supports physical mode addressing on SoftSDV. This is to be fixed
 24  * in a future version.  --drummond 1999-07-20
 25  *
 26  * Implemented EFI runtime services and virtual mode calls.  --davidm
 27  *
 28  * Goutham Rao: <goutham.rao@intel.com>
 29  *      Skip non-WB memory and ignore empty memory ranges.
 30  */
 31 
 32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 33 
 34 #include <linux/kernel.h>
 35 #include <linux/init.h>
 36 #include <linux/efi.h>
 37 #include <linux/efi-bgrt.h>
 38 #include <linux/export.h>
 39 #include <linux/bootmem.h>
 40 #include <linux/slab.h>
 41 #include <linux/memblock.h>
 42 #include <linux/spinlock.h>
 43 #include <linux/uaccess.h>
 44 #include <linux/time.h>
 45 #include <linux/io.h>
 46 #include <linux/reboot.h>
 47 #include <linux/bcd.h>
 48 
 49 #include <asm/setup.h>
 50 #include <asm/efi.h>
 51 #include <asm/e820/api.h>
 52 #include <asm/time.h>
 53 #include <asm/set_memory.h>
 54 #include <asm/tlbflush.h>
 55 #include <asm/x86_init.h>
 56 #include <asm/uv/uv.h>
 57 
 58 static struct efi efi_phys __initdata;
 59 static efi_system_table_t efi_systab __initdata;
 60 
 61 static efi_config_table_type_t arch_tables[] __initdata = {
 62 #ifdef CONFIG_X86_UV
 63         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
 64 #endif
 65         {NULL_GUID, NULL, NULL},
 66 };
 67 
 68 u64 efi_setup;          /* efi setup_data physical address */
 69 
 70 static int add_efi_memmap __initdata;
 71 static int __init setup_add_efi_memmap(char *arg)
 72 {
 73         add_efi_memmap = 1;
 74         return 0;
 75 }
 76 early_param("add_efi_memmap", setup_add_efi_memmap);
 77 
 78 static efi_status_t __init phys_efi_set_virtual_address_map(
 79         unsigned long memory_map_size,
 80         unsigned long descriptor_size,
 81         u32 descriptor_version,
 82         efi_memory_desc_t *virtual_map)
 83 {
 84         efi_status_t status;
 85         unsigned long flags;
 86         pgd_t *save_pgd;
 87 
 88         save_pgd = efi_call_phys_prolog();
 89 
 90         /* Disable interrupts around EFI calls: */
 91         local_irq_save(flags);
 92         status = efi_call_phys(efi_phys.set_virtual_address_map,
 93                                memory_map_size, descriptor_size,
 94                                descriptor_version, virtual_map);
 95         local_irq_restore(flags);
 96 
 97         efi_call_phys_epilog(save_pgd);
 98 
 99         return status;
100 }
101 
102 void __init efi_find_mirror(void)
103 {
104         efi_memory_desc_t *md;
105         u64 mirror_size = 0, total_size = 0;
106 
107         for_each_efi_memory_desc(md) {
108                 unsigned long long start = md->phys_addr;
109                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
110 
111                 total_size += size;
112                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
113                         memblock_mark_mirror(start, size);
114                         mirror_size += size;
115                 }
116         }
117         if (mirror_size)
118                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
119                         mirror_size>>20, total_size>>20);
120 }
121 
122 /*
123  * Tell the kernel about the EFI memory map.  This might include
124  * more than the max 128 entries that can fit in the e820 legacy
125  * (zeropage) memory map.
126  */
127 
128 static void __init do_add_efi_memmap(void)
129 {
130         efi_memory_desc_t *md;
131 
132         for_each_efi_memory_desc(md) {
133                 unsigned long long start = md->phys_addr;
134                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
135                 int e820_type;
136 
137                 switch (md->type) {
138                 case EFI_LOADER_CODE:
139                 case EFI_LOADER_DATA:
140                 case EFI_BOOT_SERVICES_CODE:
141                 case EFI_BOOT_SERVICES_DATA:
142                 case EFI_CONVENTIONAL_MEMORY:
143                         if (md->attribute & EFI_MEMORY_WB)
144                                 e820_type = E820_TYPE_RAM;
145                         else
146                                 e820_type = E820_TYPE_RESERVED;
147                         break;
148                 case EFI_ACPI_RECLAIM_MEMORY:
149                         e820_type = E820_TYPE_ACPI;
150                         break;
151                 case EFI_ACPI_MEMORY_NVS:
152                         e820_type = E820_TYPE_NVS;
153                         break;
154                 case EFI_UNUSABLE_MEMORY:
155                         e820_type = E820_TYPE_UNUSABLE;
156                         break;
157                 case EFI_PERSISTENT_MEMORY:
158                         e820_type = E820_TYPE_PMEM;
159                         break;
160                 default:
161                         /*
162                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
163                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
164                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
165                          */
166                         e820_type = E820_TYPE_RESERVED;
167                         break;
168                 }
169                 e820__range_add(start, size, e820_type);
170         }
171         e820__update_table(e820_table);
172 }
173 
174 int __init efi_memblock_x86_reserve_range(void)
175 {
176         struct efi_info *e = &boot_params.efi_info;
177         struct efi_memory_map_data data;
178         phys_addr_t pmap;
179         int rv;
180 
181         if (efi_enabled(EFI_PARAVIRT))
182                 return 0;
183 
184 #ifdef CONFIG_X86_32
185         /* Can't handle data above 4GB at this time */
186         if (e->efi_memmap_hi) {
187                 pr_err("Memory map is above 4GB, disabling EFI.\n");
188                 return -EINVAL;
189         }
190         pmap =  e->efi_memmap;
191 #else
192         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
193 #endif
194         data.phys_map           = pmap;
195         data.size               = e->efi_memmap_size;
196         data.desc_size          = e->efi_memdesc_size;
197         data.desc_version       = e->efi_memdesc_version;
198 
199         rv = efi_memmap_init_early(&data);
200         if (rv)
201                 return rv;
202 
203         if (add_efi_memmap)
204                 do_add_efi_memmap();
205 
206         WARN(efi.memmap.desc_version != 1,
207              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
208              efi.memmap.desc_version);
209 
210         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
211 
212         return 0;
213 }
214 
215 #define OVERFLOW_ADDR_SHIFT     (64 - EFI_PAGE_SHIFT)
216 #define OVERFLOW_ADDR_MASK      (U64_MAX << OVERFLOW_ADDR_SHIFT)
217 #define U64_HIGH_BIT            (~(U64_MAX >> 1))
218 
219 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
220 {
221         u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
222         u64 end_hi = 0;
223         char buf[64];
224 
225         if (md->num_pages == 0) {
226                 end = 0;
227         } else if (md->num_pages > EFI_PAGES_MAX ||
228                    EFI_PAGES_MAX - md->num_pages <
229                    (md->phys_addr >> EFI_PAGE_SHIFT)) {
230                 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
231                         >> OVERFLOW_ADDR_SHIFT;
232 
233                 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
234                         end_hi += 1;
235         } else {
236                 return true;
237         }
238 
239         pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
240 
241         if (end_hi) {
242                 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
243                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
244                         md->phys_addr, end_hi, end);
245         } else {
246                 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
247                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
248                         md->phys_addr, end);
249         }
250         return false;
251 }
252 
253 static void __init efi_clean_memmap(void)
254 {
255         efi_memory_desc_t *out = efi.memmap.map;
256         const efi_memory_desc_t *in = out;
257         const efi_memory_desc_t *end = efi.memmap.map_end;
258         int i, n_removal;
259 
260         for (i = n_removal = 0; in < end; i++) {
261                 if (efi_memmap_entry_valid(in, i)) {
262                         if (out != in)
263                                 memcpy(out, in, efi.memmap.desc_size);
264                         out = (void *)out + efi.memmap.desc_size;
265                 } else {
266                         n_removal++;
267                 }
268                 in = (void *)in + efi.memmap.desc_size;
269         }
270 
271         if (n_removal > 0) {
272                 u64 size = efi.memmap.nr_map - n_removal;
273 
274                 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
275                 efi_memmap_install(efi.memmap.phys_map, size);
276         }
277 }
278 
279 void __init efi_print_memmap(void)
280 {
281         efi_memory_desc_t *md;
282         int i = 0;
283 
284         for_each_efi_memory_desc(md) {
285                 char buf[64];
286 
287                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
288                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
289                         md->phys_addr,
290                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
291                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
292         }
293 }
294 
295 static int __init efi_systab_init(void *phys)
296 {
297         if (efi_enabled(EFI_64BIT)) {
298                 efi_system_table_64_t *systab64;
299                 struct efi_setup_data *data = NULL;
300                 u64 tmp = 0;
301 
302                 if (efi_setup) {
303                         data = early_memremap(efi_setup, sizeof(*data));
304                         if (!data)
305                                 return -ENOMEM;
306                 }
307                 systab64 = early_memremap((unsigned long)phys,
308                                          sizeof(*systab64));
309                 if (systab64 == NULL) {
310                         pr_err("Couldn't map the system table!\n");
311                         if (data)
312                                 early_memunmap(data, sizeof(*data));
313                         return -ENOMEM;
314                 }
315 
316                 efi_systab.hdr = systab64->hdr;
317                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
318                                               systab64->fw_vendor;
319                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
320                 efi_systab.fw_revision = systab64->fw_revision;
321                 efi_systab.con_in_handle = systab64->con_in_handle;
322                 tmp |= systab64->con_in_handle;
323                 efi_systab.con_in = systab64->con_in;
324                 tmp |= systab64->con_in;
325                 efi_systab.con_out_handle = systab64->con_out_handle;
326                 tmp |= systab64->con_out_handle;
327                 efi_systab.con_out = systab64->con_out;
328                 tmp |= systab64->con_out;
329                 efi_systab.stderr_handle = systab64->stderr_handle;
330                 tmp |= systab64->stderr_handle;
331                 efi_systab.stderr = systab64->stderr;
332                 tmp |= systab64->stderr;
333                 efi_systab.runtime = data ?
334                                      (void *)(unsigned long)data->runtime :
335                                      (void *)(unsigned long)systab64->runtime;
336                 tmp |= data ? data->runtime : systab64->runtime;
337                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
338                 tmp |= systab64->boottime;
339                 efi_systab.nr_tables = systab64->nr_tables;
340                 efi_systab.tables = data ? (unsigned long)data->tables :
341                                            systab64->tables;
342                 tmp |= data ? data->tables : systab64->tables;
343 
344                 early_memunmap(systab64, sizeof(*systab64));
345                 if (data)
346                         early_memunmap(data, sizeof(*data));
347 #ifdef CONFIG_X86_32
348                 if (tmp >> 32) {
349                         pr_err("EFI data located above 4GB, disabling EFI.\n");
350                         return -EINVAL;
351                 }
352 #endif
353         } else {
354                 efi_system_table_32_t *systab32;
355 
356                 systab32 = early_memremap((unsigned long)phys,
357                                          sizeof(*systab32));
358                 if (systab32 == NULL) {
359                         pr_err("Couldn't map the system table!\n");
360                         return -ENOMEM;
361                 }
362 
363                 efi_systab.hdr = systab32->hdr;
364                 efi_systab.fw_vendor = systab32->fw_vendor;
365                 efi_systab.fw_revision = systab32->fw_revision;
366                 efi_systab.con_in_handle = systab32->con_in_handle;
367                 efi_systab.con_in = systab32->con_in;
368                 efi_systab.con_out_handle = systab32->con_out_handle;
369                 efi_systab.con_out = systab32->con_out;
370                 efi_systab.stderr_handle = systab32->stderr_handle;
371                 efi_systab.stderr = systab32->stderr;
372                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
373                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
374                 efi_systab.nr_tables = systab32->nr_tables;
375                 efi_systab.tables = systab32->tables;
376 
377                 early_memunmap(systab32, sizeof(*systab32));
378         }
379 
380         efi.systab = &efi_systab;
381 
382         /*
383          * Verify the EFI Table
384          */
385         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
386                 pr_err("System table signature incorrect!\n");
387                 return -EINVAL;
388         }
389         if ((efi.systab->hdr.revision >> 16) == 0)
390                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
391                        efi.systab->hdr.revision >> 16,
392                        efi.systab->hdr.revision & 0xffff);
393 
394         return 0;
395 }
396 
397 static int __init efi_runtime_init32(void)
398 {
399         efi_runtime_services_32_t *runtime;
400 
401         runtime = early_memremap((unsigned long)efi.systab->runtime,
402                         sizeof(efi_runtime_services_32_t));
403         if (!runtime) {
404                 pr_err("Could not map the runtime service table!\n");
405                 return -ENOMEM;
406         }
407 
408         /*
409          * We will only need *early* access to the SetVirtualAddressMap
410          * EFI runtime service. All other runtime services will be called
411          * via the virtual mapping.
412          */
413         efi_phys.set_virtual_address_map =
414                         (efi_set_virtual_address_map_t *)
415                         (unsigned long)runtime->set_virtual_address_map;
416         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
417 
418         return 0;
419 }
420 
421 static int __init efi_runtime_init64(void)
422 {
423         efi_runtime_services_64_t *runtime;
424 
425         runtime = early_memremap((unsigned long)efi.systab->runtime,
426                         sizeof(efi_runtime_services_64_t));
427         if (!runtime) {
428                 pr_err("Could not map the runtime service table!\n");
429                 return -ENOMEM;
430         }
431 
432         /*
433          * We will only need *early* access to the SetVirtualAddressMap
434          * EFI runtime service. All other runtime services will be called
435          * via the virtual mapping.
436          */
437         efi_phys.set_virtual_address_map =
438                         (efi_set_virtual_address_map_t *)
439                         (unsigned long)runtime->set_virtual_address_map;
440         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
441 
442         return 0;
443 }
444 
445 static int __init efi_runtime_init(void)
446 {
447         int rv;
448 
449         /*
450          * Check out the runtime services table. We need to map
451          * the runtime services table so that we can grab the physical
452          * address of several of the EFI runtime functions, needed to
453          * set the firmware into virtual mode.
454          *
455          * When EFI_PARAVIRT is in force then we could not map runtime
456          * service memory region because we do not have direct access to it.
457          * However, runtime services are available through proxy functions
458          * (e.g. in case of Xen dom0 EFI implementation they call special
459          * hypercall which executes relevant EFI functions) and that is why
460          * they are always enabled.
461          */
462 
463         if (!efi_enabled(EFI_PARAVIRT)) {
464                 if (efi_enabled(EFI_64BIT))
465                         rv = efi_runtime_init64();
466                 else
467                         rv = efi_runtime_init32();
468 
469                 if (rv)
470                         return rv;
471         }
472 
473         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
474 
475         return 0;
476 }
477 
478 void __init efi_init(void)
479 {
480         efi_char16_t *c16;
481         char vendor[100] = "unknown";
482         int i = 0;
483         void *tmp;
484 
485 #ifdef CONFIG_X86_32
486         if (boot_params.efi_info.efi_systab_hi ||
487             boot_params.efi_info.efi_memmap_hi) {
488                 pr_info("Table located above 4GB, disabling EFI.\n");
489                 return;
490         }
491         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
492 #else
493         efi_phys.systab = (efi_system_table_t *)
494                           (boot_params.efi_info.efi_systab |
495                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
496 #endif
497 
498         if (efi_systab_init(efi_phys.systab))
499                 return;
500 
501         efi.config_table = (unsigned long)efi.systab->tables;
502         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
503         efi.runtime      = (unsigned long)efi.systab->runtime;
504 
505         /*
506          * Show what we know for posterity
507          */
508         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
509         if (c16) {
510                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
511                         vendor[i] = *c16++;
512                 vendor[i] = '\0';
513         } else
514                 pr_err("Could not map the firmware vendor!\n");
515         early_memunmap(tmp, 2);
516 
517         pr_info("EFI v%u.%.02u by %s\n",
518                 efi.systab->hdr.revision >> 16,
519                 efi.systab->hdr.revision & 0xffff, vendor);
520 
521         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
522                 return;
523 
524         if (efi_config_init(arch_tables))
525                 return;
526 
527         /*
528          * Note: We currently don't support runtime services on an EFI
529          * that doesn't match the kernel 32/64-bit mode.
530          */
531 
532         if (!efi_runtime_supported())
533                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
534         else {
535                 if (efi_runtime_disabled() || efi_runtime_init()) {
536                         efi_memmap_unmap();
537                         return;
538                 }
539         }
540 
541         efi_clean_memmap();
542 
543         if (efi_enabled(EFI_DBG))
544                 efi_print_memmap();
545 }
546 
547 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
548 {
549         u64 addr, npages;
550 
551         addr = md->virt_addr;
552         npages = md->num_pages;
553 
554         memrange_efi_to_native(&addr, &npages);
555 
556         if (executable)
557                 set_memory_x(addr, npages);
558         else
559                 set_memory_nx(addr, npages);
560 }
561 
562 void __init runtime_code_page_mkexec(void)
563 {
564         efi_memory_desc_t *md;
565 
566         /* Make EFI runtime service code area executable */
567         for_each_efi_memory_desc(md) {
568                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
569                         continue;
570 
571                 efi_set_executable(md, true);
572         }
573 }
574 
575 void __init efi_memory_uc(u64 addr, unsigned long size)
576 {
577         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
578         u64 npages;
579 
580         npages = round_up(size, page_shift) / page_shift;
581         memrange_efi_to_native(&addr, &npages);
582         set_memory_uc(addr, npages);
583 }
584 
585 void __init old_map_region(efi_memory_desc_t *md)
586 {
587         u64 start_pfn, end_pfn, end;
588         unsigned long size;
589         void *va;
590 
591         start_pfn = PFN_DOWN(md->phys_addr);
592         size      = md->num_pages << PAGE_SHIFT;
593         end       = md->phys_addr + size;
594         end_pfn   = PFN_UP(end);
595 
596         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
597                 va = __va(md->phys_addr);
598 
599                 if (!(md->attribute & EFI_MEMORY_WB))
600                         efi_memory_uc((u64)(unsigned long)va, size);
601         } else
602                 va = efi_ioremap(md->phys_addr, size,
603                                  md->type, md->attribute);
604 
605         md->virt_addr = (u64) (unsigned long) va;
606         if (!va)
607                 pr_err("ioremap of 0x%llX failed!\n",
608                        (unsigned long long)md->phys_addr);
609 }
610 
611 /* Merge contiguous regions of the same type and attribute */
612 static void __init efi_merge_regions(void)
613 {
614         efi_memory_desc_t *md, *prev_md = NULL;
615 
616         for_each_efi_memory_desc(md) {
617                 u64 prev_size;
618 
619                 if (!prev_md) {
620                         prev_md = md;
621                         continue;
622                 }
623 
624                 if (prev_md->type != md->type ||
625                     prev_md->attribute != md->attribute) {
626                         prev_md = md;
627                         continue;
628                 }
629 
630                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
631 
632                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
633                         prev_md->num_pages += md->num_pages;
634                         md->type = EFI_RESERVED_TYPE;
635                         md->attribute = 0;
636                         continue;
637                 }
638                 prev_md = md;
639         }
640 }
641 
642 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
643 {
644         unsigned long size;
645         u64 end, systab;
646 
647         size = md->num_pages << EFI_PAGE_SHIFT;
648         end = md->phys_addr + size;
649         systab = (u64)(unsigned long)efi_phys.systab;
650         if (md->phys_addr <= systab && systab < end) {
651                 systab += md->virt_addr - md->phys_addr;
652                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
653         }
654 }
655 
656 static void *realloc_pages(void *old_memmap, int old_shift)
657 {
658         void *ret;
659 
660         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
661         if (!ret)
662                 goto out;
663 
664         /*
665          * A first-time allocation doesn't have anything to copy.
666          */
667         if (!old_memmap)
668                 return ret;
669 
670         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
671 
672 out:
673         free_pages((unsigned long)old_memmap, old_shift);
674         return ret;
675 }
676 
677 /*
678  * Iterate the EFI memory map in reverse order because the regions
679  * will be mapped top-down. The end result is the same as if we had
680  * mapped things forward, but doesn't require us to change the
681  * existing implementation of efi_map_region().
682  */
683 static inline void *efi_map_next_entry_reverse(void *entry)
684 {
685         /* Initial call */
686         if (!entry)
687                 return efi.memmap.map_end - efi.memmap.desc_size;
688 
689         entry -= efi.memmap.desc_size;
690         if (entry < efi.memmap.map)
691                 return NULL;
692 
693         return entry;
694 }
695 
696 /*
697  * efi_map_next_entry - Return the next EFI memory map descriptor
698  * @entry: Previous EFI memory map descriptor
699  *
700  * This is a helper function to iterate over the EFI memory map, which
701  * we do in different orders depending on the current configuration.
702  *
703  * To begin traversing the memory map @entry must be %NULL.
704  *
705  * Returns %NULL when we reach the end of the memory map.
706  */
707 static void *efi_map_next_entry(void *entry)
708 {
709         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
710                 /*
711                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
712                  * config table feature requires us to map all entries
713                  * in the same order as they appear in the EFI memory
714                  * map. That is to say, entry N must have a lower
715                  * virtual address than entry N+1. This is because the
716                  * firmware toolchain leaves relative references in
717                  * the code/data sections, which are split and become
718                  * separate EFI memory regions. Mapping things
719                  * out-of-order leads to the firmware accessing
720                  * unmapped addresses.
721                  *
722                  * Since we need to map things this way whether or not
723                  * the kernel actually makes use of
724                  * EFI_PROPERTIES_TABLE, let's just switch to this
725                  * scheme by default for 64-bit.
726                  */
727                 return efi_map_next_entry_reverse(entry);
728         }
729 
730         /* Initial call */
731         if (!entry)
732                 return efi.memmap.map;
733 
734         entry += efi.memmap.desc_size;
735         if (entry >= efi.memmap.map_end)
736                 return NULL;
737 
738         return entry;
739 }
740 
741 static bool should_map_region(efi_memory_desc_t *md)
742 {
743         /*
744          * Runtime regions always require runtime mappings (obviously).
745          */
746         if (md->attribute & EFI_MEMORY_RUNTIME)
747                 return true;
748 
749         /*
750          * 32-bit EFI doesn't suffer from the bug that requires us to
751          * reserve boot services regions, and mixed mode support
752          * doesn't exist for 32-bit kernels.
753          */
754         if (IS_ENABLED(CONFIG_X86_32))
755                 return false;
756 
757         /*
758          * Map all of RAM so that we can access arguments in the 1:1
759          * mapping when making EFI runtime calls.
760          */
761         if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
762                 if (md->type == EFI_CONVENTIONAL_MEMORY ||
763                     md->type == EFI_LOADER_DATA ||
764                     md->type == EFI_LOADER_CODE)
765                         return true;
766         }
767 
768         /*
769          * Map boot services regions as a workaround for buggy
770          * firmware that accesses them even when they shouldn't.
771          *
772          * See efi_{reserve,free}_boot_services().
773          */
774         if (md->type == EFI_BOOT_SERVICES_CODE ||
775             md->type == EFI_BOOT_SERVICES_DATA)
776                 return true;
777 
778         return false;
779 }
780 
781 /*
782  * Map the efi memory ranges of the runtime services and update new_mmap with
783  * virtual addresses.
784  */
785 static void * __init efi_map_regions(int *count, int *pg_shift)
786 {
787         void *p, *new_memmap = NULL;
788         unsigned long left = 0;
789         unsigned long desc_size;
790         efi_memory_desc_t *md;
791 
792         desc_size = efi.memmap.desc_size;
793 
794         p = NULL;
795         while ((p = efi_map_next_entry(p))) {
796                 md = p;
797 
798                 if (!should_map_region(md))
799                         continue;
800 
801                 efi_map_region(md);
802                 get_systab_virt_addr(md);
803 
804                 if (left < desc_size) {
805                         new_memmap = realloc_pages(new_memmap, *pg_shift);
806                         if (!new_memmap)
807                                 return NULL;
808 
809                         left += PAGE_SIZE << *pg_shift;
810                         (*pg_shift)++;
811                 }
812 
813                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
814 
815                 left -= desc_size;
816                 (*count)++;
817         }
818 
819         return new_memmap;
820 }
821 
822 static void __init kexec_enter_virtual_mode(void)
823 {
824 #ifdef CONFIG_KEXEC_CORE
825         efi_memory_desc_t *md;
826         unsigned int num_pages;
827 
828         efi.systab = NULL;
829 
830         /*
831          * We don't do virtual mode, since we don't do runtime services, on
832          * non-native EFI. With efi=old_map, we don't do runtime services in
833          * kexec kernel because in the initial boot something else might
834          * have been mapped at these virtual addresses.
835          */
836         if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
837                 efi_memmap_unmap();
838                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
839                 return;
840         }
841 
842         if (efi_alloc_page_tables()) {
843                 pr_err("Failed to allocate EFI page tables\n");
844                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
845                 return;
846         }
847 
848         /*
849         * Map efi regions which were passed via setup_data. The virt_addr is a
850         * fixed addr which was used in first kernel of a kexec boot.
851         */
852         for_each_efi_memory_desc(md) {
853                 efi_map_region_fixed(md); /* FIXME: add error handling */
854                 get_systab_virt_addr(md);
855         }
856 
857         /*
858          * Unregister the early EFI memmap from efi_init() and install
859          * the new EFI memory map.
860          */
861         efi_memmap_unmap();
862 
863         if (efi_memmap_init_late(efi.memmap.phys_map,
864                                  efi.memmap.desc_size * efi.memmap.nr_map)) {
865                 pr_err("Failed to remap late EFI memory map\n");
866                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
867                 return;
868         }
869 
870         BUG_ON(!efi.systab);
871 
872         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
873         num_pages >>= PAGE_SHIFT;
874 
875         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
876                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
877                 return;
878         }
879 
880         efi_sync_low_kernel_mappings();
881 
882         /*
883          * Now that EFI is in virtual mode, update the function
884          * pointers in the runtime service table to the new virtual addresses.
885          *
886          * Call EFI services through wrapper functions.
887          */
888         efi.runtime_version = efi_systab.hdr.revision;
889 
890         efi_native_runtime_setup();
891 
892         efi.set_virtual_address_map = NULL;
893 
894         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
895                 runtime_code_page_mkexec();
896 
897         /* clean DUMMY object */
898         efi_delete_dummy_variable();
899 #endif
900 }
901 
902 /*
903  * This function will switch the EFI runtime services to virtual mode.
904  * Essentially, we look through the EFI memmap and map every region that
905  * has the runtime attribute bit set in its memory descriptor into the
906  * efi_pgd page table.
907  *
908  * The old method which used to update that memory descriptor with the
909  * virtual address obtained from ioremap() is still supported when the
910  * kernel is booted with efi=old_map on its command line. Same old
911  * method enabled the runtime services to be called without having to
912  * thunk back into physical mode for every invocation.
913  *
914  * The new method does a pagetable switch in a preemption-safe manner
915  * so that we're in a different address space when calling a runtime
916  * function. For function arguments passing we do copy the PUDs of the
917  * kernel page table into efi_pgd prior to each call.
918  *
919  * Specially for kexec boot, efi runtime maps in previous kernel should
920  * be passed in via setup_data. In that case runtime ranges will be mapped
921  * to the same virtual addresses as the first kernel, see
922  * kexec_enter_virtual_mode().
923  */
924 static void __init __efi_enter_virtual_mode(void)
925 {
926         int count = 0, pg_shift = 0;
927         void *new_memmap = NULL;
928         efi_status_t status;
929         unsigned long pa;
930 
931         efi.systab = NULL;
932 
933         if (efi_alloc_page_tables()) {
934                 pr_err("Failed to allocate EFI page tables\n");
935                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
936                 return;
937         }
938 
939         efi_merge_regions();
940         new_memmap = efi_map_regions(&count, &pg_shift);
941         if (!new_memmap) {
942                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
943                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
944                 return;
945         }
946 
947         pa = __pa(new_memmap);
948 
949         /*
950          * Unregister the early EFI memmap from efi_init() and install
951          * the new EFI memory map that we are about to pass to the
952          * firmware via SetVirtualAddressMap().
953          */
954         efi_memmap_unmap();
955 
956         if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
957                 pr_err("Failed to remap late EFI memory map\n");
958                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
959                 return;
960         }
961 
962         if (efi_enabled(EFI_DBG)) {
963                 pr_info("EFI runtime memory map:\n");
964                 efi_print_memmap();
965         }
966 
967         BUG_ON(!efi.systab);
968 
969         if (efi_setup_page_tables(pa, 1 << pg_shift)) {
970                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
971                 return;
972         }
973 
974         efi_sync_low_kernel_mappings();
975 
976         if (efi_is_native()) {
977                 status = phys_efi_set_virtual_address_map(
978                                 efi.memmap.desc_size * count,
979                                 efi.memmap.desc_size,
980                                 efi.memmap.desc_version,
981                                 (efi_memory_desc_t *)pa);
982         } else {
983                 status = efi_thunk_set_virtual_address_map(
984                                 efi_phys.set_virtual_address_map,
985                                 efi.memmap.desc_size * count,
986                                 efi.memmap.desc_size,
987                                 efi.memmap.desc_version,
988                                 (efi_memory_desc_t *)pa);
989         }
990 
991         if (status != EFI_SUCCESS) {
992                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
993                          status);
994                 panic("EFI call to SetVirtualAddressMap() failed!");
995         }
996 
997         /*
998          * Now that EFI is in virtual mode, update the function
999          * pointers in the runtime service table to the new virtual addresses.
1000          *
1001          * Call EFI services through wrapper functions.
1002          */
1003         efi.runtime_version = efi_systab.hdr.revision;
1004 
1005         if (efi_is_native())
1006                 efi_native_runtime_setup();
1007         else
1008                 efi_thunk_runtime_setup();
1009 
1010         efi.set_virtual_address_map = NULL;
1011 
1012         /*
1013          * Apply more restrictive page table mapping attributes now that
1014          * SVAM() has been called and the firmware has performed all
1015          * necessary relocation fixups for the new virtual addresses.
1016          */
1017         efi_runtime_update_mappings();
1018 
1019         /* clean DUMMY object */
1020         efi_delete_dummy_variable();
1021 }
1022 
1023 void __init efi_enter_virtual_mode(void)
1024 {
1025         if (efi_enabled(EFI_PARAVIRT))
1026                 return;
1027 
1028         if (efi_setup)
1029                 kexec_enter_virtual_mode();
1030         else
1031                 __efi_enter_virtual_mode();
1032 
1033         efi_dump_pagetable();
1034 }
1035 
1036 static int __init arch_parse_efi_cmdline(char *str)
1037 {
1038         if (!str) {
1039                 pr_warn("need at least one option\n");
1040                 return -EINVAL;
1041         }
1042 
1043         if (parse_option_str(str, "old_map"))
1044                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1045 
1046         return 0;
1047 }
1048 early_param("efi", arch_parse_efi_cmdline);
1049 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp