kernel/arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/rtc.h>
  55 #include <asm/uv/uv.h>
  56
  57 #define EFI_DEBUG
  58
  59 struct efi_memory_map memmap;
  60
  61 static struct efi efi_phys __initdata;
  62 static efi_system_table_t efi_systab __initdata;
  63
  64 static efi_config_table_type_t arch_tables[] __initdata = {
  65 #ifdef CONFIG_X86_UV
  66         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  67 #endif
  68         {NULL_GUID, NULL, NULL},
  69 };
  70
  71 u64 efi_setup;          /* efi setup_data physical address */
  72
  73 static int add_efi_memmap __initdata;
  74 static int __init setup_add_efi_memmap(char *arg)
  75 {
  76         add_efi_memmap = 1;
  77         return 0;
  78 }
  79 early_param("add_efi_memmap", setup_add_efi_memmap);
  80
  81 static efi_status_t __init phys_efi_set_virtual_address_map(
  82         unsigned long memory_map_size,
  83         unsigned long descriptor_size,
  84         u32 descriptor_version,
  85         efi_memory_desc_t *virtual_map)
  86 {
  87         efi_status_t status;
  88         unsigned long flags;
  89         pgd_t *save_pgd;
  90
  91         save_pgd = efi_call_phys_prolog();
  92
  93         /* Disable interrupts around EFI calls: */
  94         local_irq_save(flags);
  95         status = efi_call_phys(efi_phys.set_virtual_address_map,
  96                                memory_map_size, descriptor_size,
  97                                descriptor_version, virtual_map);
  98         local_irq_restore(flags);
  99
 100         efi_call_phys_epilog(save_pgd);
 101
 102         return status;
 103 }
 104
 105 void efi_get_time(struct timespec *now)
 106 {
 107         efi_status_t status;
 108         efi_time_t eft;
 109         efi_time_cap_t cap;
 110
 111         status = efi.get_time(&eft, &cap);
 112         if (status != EFI_SUCCESS)
 113                 pr_err("Oops: efitime: can't read time!\n");
 114
 115         now->tv_sec = mktime(eft.year, eft.month, eft.day, eft.hour,
 116                              eft.minute, eft.second);
 117         now->tv_nsec = 0;
 118 }
 119
 120 /*
 121  * Tell the kernel about the EFI memory map.  This might include
 122  * more than the max 128 entries that can fit in the e820 legacy
 123  * (zeropage) memory map.
 124  */
 125
 126 static void __init do_add_efi_memmap(void)
 127 {
 128         void *p;
 129
 130         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 131                 efi_memory_desc_t *md = p;
 132                 unsigned long long start = md->phys_addr;
 133                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 134                 int e820_type;
 135
 136                 switch (md->type) {
 137                 case EFI_LOADER_CODE:
 138                 case EFI_LOADER_DATA:
 139                 case EFI_BOOT_SERVICES_CODE:
 140                 case EFI_BOOT_SERVICES_DATA:
 141                 case EFI_CONVENTIONAL_MEMORY:
 142                         if (md->attribute & EFI_MEMORY_WB)
 143                                 e820_type = E820_RAM;
 144                         else
 145                                 e820_type = E820_RESERVED;
 146                         break;
 147                 case EFI_ACPI_RECLAIM_MEMORY:
 148                         e820_type = E820_ACPI;
 149                         break;
 150                 case EFI_ACPI_MEMORY_NVS:
 151                         e820_type = E820_NVS;
 152                         break;
 153                 case EFI_UNUSABLE_MEMORY:
 154                         e820_type = E820_UNUSABLE;
 155                         break;
 156                 default:
 157                         /*
 158                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 159                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 160                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 161                          */
 162                         e820_type = E820_RESERVED;
 163                         break;
 164                 }
 165                 e820_add_region(start, size, e820_type);
 166         }
 167         sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &e820.nr_map);
 168 }
 169
 170 int __init efi_memblock_x86_reserve_range(void)
 171 {
 172         struct efi_info *e = &boot_params.efi_info;
 173         unsigned long pmap;
 174
 175         if (efi_enabled(EFI_PARAVIRT))
 176                 return 0;
 177
 178 #ifdef CONFIG_X86_32
 179         /* Can't handle data above 4GB at this time */
 180         if (e->efi_memmap_hi) {
 181                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 182                 return -EINVAL;
 183         }
 184         pmap =  e->efi_memmap;
 185 #else
 186         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 187 #endif
 188         memmap.phys_map         = (void *)pmap;
 189         memmap.nr_map           = e->efi_memmap_size /
 190                                   e->efi_memdesc_size;
 191         memmap.desc_size        = e->efi_memdesc_size;
 192         memmap.desc_version     = e->efi_memdesc_version;
 193
 194         memblock_reserve(pmap, memmap.nr_map * memmap.desc_size);
 195
 196         efi.memmap = &memmap;
 197
 198         return 0;
 199 }
 200
 201 static void __init print_efi_memmap(void)
 202 {
 203 #ifdef EFI_DEBUG
 204         efi_memory_desc_t *md;
 205         void *p;
 206         int i;
 207
 208         for (p = memmap.map, i = 0;
 209              p < memmap.map_end;
 210              p += memmap.desc_size, i++) {
 211                 char buf[64];
 212
 213                 md = p;
 214                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx) (%lluMB)\n",
 215                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 216                         md->phys_addr,
 217                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT),
 218                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 219         }
 220 #endif  /*  EFI_DEBUG  */
 221 }
 222
 223 void __init efi_unmap_memmap(void)
 224 {
 225         clear_bit(EFI_MEMMAP, &efi.flags);
 226         if (memmap.map) {
 227                 early_memunmap(memmap.map, memmap.nr_map * memmap.desc_size);
 228                 memmap.map = NULL;
 229         }
 230 }
 231
 232 static int __init efi_systab_init(void *phys)
 233 {
 234         if (efi_enabled(EFI_64BIT)) {
 235                 efi_system_table_64_t *systab64;
 236                 struct efi_setup_data *data = NULL;
 237                 u64 tmp = 0;
 238
 239                 if (efi_setup) {
 240                         data = early_memremap(efi_setup, sizeof(*data));
 241                         if (!data)
 242                                 return -ENOMEM;
 243                 }
 244                 systab64 = early_memremap((unsigned long)phys,
 245                                          sizeof(*systab64));
 246                 if (systab64 == NULL) {
 247                         pr_err("Couldn't map the system table!\n");
 248                         if (data)
 249                                 early_memunmap(data, sizeof(*data));
 250                         return -ENOMEM;
 251                 }
 252
 253                 efi_systab.hdr = systab64->hdr;
 254                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 255                                               systab64->fw_vendor;
 256                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 257                 efi_systab.fw_revision = systab64->fw_revision;
 258                 efi_systab.con_in_handle = systab64->con_in_handle;
 259                 tmp |= systab64->con_in_handle;
 260                 efi_systab.con_in = systab64->con_in;
 261                 tmp |= systab64->con_in;
 262                 efi_systab.con_out_handle = systab64->con_out_handle;
 263                 tmp |= systab64->con_out_handle;
 264                 efi_systab.con_out = systab64->con_out;
 265                 tmp |= systab64->con_out;
 266                 efi_systab.stderr_handle = systab64->stderr_handle;
 267                 tmp |= systab64->stderr_handle;
 268                 efi_systab.stderr = systab64->stderr;
 269                 tmp |= systab64->stderr;
 270                 efi_systab.runtime = data ?
 271                                      (void *)(unsigned long)data->runtime :
 272                                      (void *)(unsigned long)systab64->runtime;
 273                 tmp |= data ? data->runtime : systab64->runtime;
 274                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 275                 tmp |= systab64->boottime;
 276                 efi_systab.nr_tables = systab64->nr_tables;
 277                 efi_systab.tables = data ? (unsigned long)data->tables :
 278                                            systab64->tables;
 279                 tmp |= data ? data->tables : systab64->tables;
 280
 281                 early_memunmap(systab64, sizeof(*systab64));
 282                 if (data)
 283                         early_memunmap(data, sizeof(*data));
 284 #ifdef CONFIG_X86_32
 285                 if (tmp >> 32) {
 286                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 287                         return -EINVAL;
 288                 }
 289 #endif
 290         } else {
 291                 efi_system_table_32_t *systab32;
 292
 293                 systab32 = early_memremap((unsigned long)phys,
 294                                          sizeof(*systab32));
 295                 if (systab32 == NULL) {
 296                         pr_err("Couldn't map the system table!\n");
 297                         return -ENOMEM;
 298                 }
 299
 300                 efi_systab.hdr = systab32->hdr;
 301                 efi_systab.fw_vendor = systab32->fw_vendor;
 302                 efi_systab.fw_revision = systab32->fw_revision;
 303                 efi_systab.con_in_handle = systab32->con_in_handle;
 304                 efi_systab.con_in = systab32->con_in;
 305                 efi_systab.con_out_handle = systab32->con_out_handle;
 306                 efi_systab.con_out = systab32->con_out;
 307                 efi_systab.stderr_handle = systab32->stderr_handle;
 308                 efi_systab.stderr = systab32->stderr;
 309                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 310                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 311                 efi_systab.nr_tables = systab32->nr_tables;
 312                 efi_systab.tables = systab32->tables;
 313
 314                 early_memunmap(systab32, sizeof(*systab32));
 315         }
 316
 317         efi.systab = &efi_systab;
 318
 319         /*
 320          * Verify the EFI Table
 321          */
 322         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 323                 pr_err("System table signature incorrect!\n");
 324                 return -EINVAL;
 325         }
 326         if ((efi.systab->hdr.revision >> 16) == 0)
 327                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 328                        efi.systab->hdr.revision >> 16,
 329                        efi.systab->hdr.revision & 0xffff);
 330
 331         set_bit(EFI_SYSTEM_TABLES, &efi.flags);
 332
 333         return 0;
 334 }
 335
 336 static int __init efi_runtime_init32(void)
 337 {
 338         efi_runtime_services_32_t *runtime;
 339
 340         runtime = early_memremap((unsigned long)efi.systab->runtime,
 341                         sizeof(efi_runtime_services_32_t));
 342         if (!runtime) {
 343                 pr_err("Could not map the runtime service table!\n");
 344                 return -ENOMEM;
 345         }
 346
 347         /*
 348          * We will only need *early* access to the SetVirtualAddressMap
 349          * EFI runtime service. All other runtime services will be called
 350          * via the virtual mapping.
 351          */
 352         efi_phys.set_virtual_address_map =
 353                         (efi_set_virtual_address_map_t *)
 354                         (unsigned long)runtime->set_virtual_address_map;
 355         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 356
 357         return 0;
 358 }
 359
 360 static int __init efi_runtime_init64(void)
 361 {
 362         efi_runtime_services_64_t *runtime;
 363
 364         runtime = early_memremap((unsigned long)efi.systab->runtime,
 365                         sizeof(efi_runtime_services_64_t));
 366         if (!runtime) {
 367                 pr_err("Could not map the runtime service table!\n");
 368                 return -ENOMEM;
 369         }
 370
 371         /*
 372          * We will only need *early* access to the SetVirtualAddressMap
 373          * EFI runtime service. All other runtime services will be called
 374          * via the virtual mapping.
 375          */
 376         efi_phys.set_virtual_address_map =
 377                         (efi_set_virtual_address_map_t *)
 378                         (unsigned long)runtime->set_virtual_address_map;
 379         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 380
 381         return 0;
 382 }
 383
 384 static int __init efi_runtime_init(void)
 385 {
 386         int rv;
 387
 388         /*
 389          * Check out the runtime services table. We need to map
 390          * the runtime services table so that we can grab the physical
 391          * address of several of the EFI runtime functions, needed to
 392          * set the firmware into virtual mode.
 393          *
 394          * When EFI_PARAVIRT is in force then we could not map runtime
 395          * service memory region because we do not have direct access to it.
 396          * However, runtime services are available through proxy functions
 397          * (e.g. in case of Xen dom0 EFI implementation they call special
 398          * hypercall which executes relevant EFI functions) and that is why
 399          * they are always enabled.
 400          */
 401
 402         if (!efi_enabled(EFI_PARAVIRT)) {
 403                 if (efi_enabled(EFI_64BIT))
 404                         rv = efi_runtime_init64();
 405                 else
 406                         rv = efi_runtime_init32();
 407
 408                 if (rv)
 409                         return rv;
 410         }
 411
 412         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 413
 414         return 0;
 415 }
 416
 417 static int __init efi_memmap_init(void)
 418 {
 419         if (efi_enabled(EFI_PARAVIRT))
 420                 return 0;
 421
 422         /* Map the EFI memory map */
 423         memmap.map = early_memremap((unsigned long)memmap.phys_map,
 424                                    memmap.nr_map * memmap.desc_size);
 425         if (memmap.map == NULL) {
 426                 pr_err("Could not map the memory map!\n");
 427                 return -ENOMEM;
 428         }
 429         memmap.map_end = memmap.map + (memmap.nr_map * memmap.desc_size);
 430
 431         if (add_efi_memmap)
 432                 do_add_efi_memmap();
 433
 434         set_bit(EFI_MEMMAP, &efi.flags);
 435
 436         return 0;
 437 }
 438
 439 void __init efi_init(void)
 440 {
 441         efi_char16_t *c16;
 442         char vendor[100] = "unknown";
 443         int i = 0;
 444         void *tmp;
 445
 446 #ifdef CONFIG_X86_32
 447         if (boot_params.efi_info.efi_systab_hi ||
 448             boot_params.efi_info.efi_memmap_hi) {
 449                 pr_info("Table located above 4GB, disabling EFI.\n");
 450                 return;
 451         }
 452         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 453 #else
 454         efi_phys.systab = (efi_system_table_t *)
 455                           (boot_params.efi_info.efi_systab |
 456                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 457 #endif
 458
 459         if (efi_systab_init(efi_phys.systab))
 460                 return;
 461
 462         efi.config_table = (unsigned long)efi.systab->tables;
 463         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 464         efi.runtime      = (unsigned long)efi.systab->runtime;
 465
 466         /*
 467          * Show what we know for posterity
 468          */
 469         c16 = tmp = early_memremap(efi.systab->fw_vendor, 2);
 470         if (c16) {
 471                 for (i = 0; i < sizeof(vendor) - 1 && *c16; ++i)
 472                         vendor[i] = *c16++;
 473                 vendor[i] = '\0';
 474         } else
 475                 pr_err("Could not map the firmware vendor!\n");
 476         early_memunmap(tmp, 2);
 477
 478         pr_info("EFI v%u.%.02u by %s\n",
 479                 efi.systab->hdr.revision >> 16,
 480                 efi.systab->hdr.revision & 0xffff, vendor);
 481
 482         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 483                 return;
 484
 485         if (efi_config_init(arch_tables))
 486                 return;
 487
 488         /*
 489          * Note: We currently don't support runtime services on an EFI
 490          * that doesn't match the kernel 32/64-bit mode.
 491          */
 492
 493         if (!efi_runtime_supported())
 494                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 495         else {
 496                 if (efi_runtime_disabled() || efi_runtime_init())
 497                         return;
 498         }
 499         if (efi_memmap_init())
 500                 return;
 501
 502         if (efi_enabled(EFI_DBG))
 503                 print_efi_memmap();
 504 }
 505
 506 void __init efi_late_init(void)
 507 {
 508         efi_bgrt_init();
 509 }
 510
 511 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 512 {
 513         u64 addr, npages;
 514
 515         addr = md->virt_addr;
 516         npages = md->num_pages;
 517
 518         memrange_efi_to_native(&addr, &npages);
 519
 520         if (executable)
 521                 set_memory_x(addr, npages);
 522         else
 523                 set_memory_nx(addr, npages);
 524 }
 525
 526 void __init runtime_code_page_mkexec(void)
 527 {
 528         efi_memory_desc_t *md;
 529         void *p;
 530
 531         /* Make EFI runtime service code area executable */
 532         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 533                 md = p;
 534
 535                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 536                         continue;
 537
 538                 efi_set_executable(md, true);
 539         }
 540 }
 541
 542 void __init efi_memory_uc(u64 addr, unsigned long size)
 543 {
 544         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 545         u64 npages;
 546
 547         npages = round_up(size, page_shift) / page_shift;
 548         memrange_efi_to_native(&addr, &npages);
 549         set_memory_uc(addr, npages);
 550 }
 551
 552 void __init old_map_region(efi_memory_desc_t *md)
 553 {
 554         u64 start_pfn, end_pfn, end;
 555         unsigned long size;
 556         void *va;
 557
 558         start_pfn = PFN_DOWN(md->phys_addr);
 559         size      = md->num_pages << PAGE_SHIFT;
 560         end       = md->phys_addr + size;
 561         end_pfn   = PFN_UP(end);
 562
 563         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 564                 va = __va(md->phys_addr);
 565
 566                 if (!(md->attribute & EFI_MEMORY_WB))
 567                         efi_memory_uc((u64)(unsigned long)va, size);
 568         } else
 569                 va = efi_ioremap(md->phys_addr, size,
 570                                  md->type, md->attribute);
 571
 572         md->virt_addr = (u64) (unsigned long) va;
 573         if (!va)
 574                 pr_err("ioremap of 0x%llX failed!\n",
 575                        (unsigned long long)md->phys_addr);
 576 }
 577
 578 /* Merge contiguous regions of the same type and attribute */
 579 static void __init efi_merge_regions(void)
 580 {
 581         void *p;
 582         efi_memory_desc_t *md, *prev_md = NULL;
 583
 584         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 585                 u64 prev_size;
 586                 md = p;
 587
 588                 if (!prev_md) {
 589                         prev_md = md;
 590                         continue;
 591                 }
 592
 593                 if (prev_md->type != md->type ||
 594                     prev_md->attribute != md->attribute) {
 595                         prev_md = md;
 596                         continue;
 597                 }
 598
 599                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 600
 601                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 602                         prev_md->num_pages += md->num_pages;
 603                         md->type = EFI_RESERVED_TYPE;
 604                         md->attribute = 0;
 605                         continue;
 606                 }
 607                 prev_md = md;
 608         }
 609 }
 610
 611 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 612 {
 613         unsigned long size;
 614         u64 end, systab;
 615
 616         size = md->num_pages << EFI_PAGE_SHIFT;
 617         end = md->phys_addr + size;
 618         systab = (u64)(unsigned long)efi_phys.systab;
 619         if (md->phys_addr <= systab && systab < end) {
 620                 systab += md->virt_addr - md->phys_addr;
 621                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 622         }
 623 }
 624
 625 static void __init save_runtime_map(void)
 626 {
 627 #ifdef CONFIG_KEXEC
 628         efi_memory_desc_t *md;
 629         void *tmp, *p, *q = NULL;
 630         int count = 0;
 631
 632         if (efi_enabled(EFI_OLD_MEMMAP))
 633                 return;
 634
 635         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 636                 md = p;
 637
 638                 if (!(md->attribute & EFI_MEMORY_RUNTIME) ||
 639                     (md->type == EFI_BOOT_SERVICES_CODE) ||
 640                     (md->type == EFI_BOOT_SERVICES_DATA))
 641                         continue;
 642                 tmp = krealloc(q, (count + 1) * memmap.desc_size, GFP_KERNEL);
 643                 if (!tmp)
 644                         goto out;
 645                 q = tmp;
 646
 647                 memcpy(q + count * memmap.desc_size, md, memmap.desc_size);
 648                 count++;
 649         }
 650
 651         efi_runtime_map_setup(q, count, memmap.desc_size);
 652         return;
 653
 654 out:
 655         kfree(q);
 656         pr_err("Error saving runtime map, efi runtime on kexec non-functional!!\n");
 657 #endif
 658 }
 659
 660 static void *realloc_pages(void *old_memmap, int old_shift)
 661 {
 662         void *ret;
 663
 664         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 665         if (!ret)
 666                 goto out;
 667
 668         /*
 669          * A first-time allocation doesn't have anything to copy.
 670          */
 671         if (!old_memmap)
 672                 return ret;
 673
 674         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 675
 676 out:
 677         free_pages((unsigned long)old_memmap, old_shift);
 678         return ret;
 679 }
 680
 681 /*
 682  * Map the efi memory ranges of the runtime services and update new_mmap with
 683  * virtual addresses.
 684  */
 685 static void * __init efi_map_regions(int *count, int *pg_shift)
 686 {
 687         void *p, *new_memmap = NULL;
 688         unsigned long left = 0;
 689         efi_memory_desc_t *md;
 690
 691         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 692                 md = p;
 693                 if (!(md->attribute & EFI_MEMORY_RUNTIME)) {
 694 #ifdef CONFIG_X86_64
 695                         if (md->type != EFI_BOOT_SERVICES_CODE &&
 696                             md->type != EFI_BOOT_SERVICES_DATA)
 697 #endif
 698                                 continue;
 699                 }
 700
 701                 efi_map_region(md);
 702                 get_systab_virt_addr(md);
 703
 704                 if (left < memmap.desc_size) {
 705                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 706                         if (!new_memmap)
 707                                 return NULL;
 708
 709                         left += PAGE_SIZE << *pg_shift;
 710                         (*pg_shift)++;
 711                 }
 712
 713                 memcpy(new_memmap + (*count * memmap.desc_size), md,
 714                        memmap.desc_size);
 715
 716                 left -= memmap.desc_size;
 717                 (*count)++;
 718         }
 719
 720         return new_memmap;
 721 }
 722
 723 static void __init kexec_enter_virtual_mode(void)
 724 {
 725 #ifdef CONFIG_KEXEC
 726         efi_memory_desc_t *md;
 727         void *p;
 728
 729         efi.systab = NULL;
 730
 731         /*
 732          * We don't do virtual mode, since we don't do runtime services, on
 733          * non-native EFI
 734          */
 735         if (!efi_is_native()) {
 736                 efi_unmap_memmap();
 737                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 738                 return;
 739         }
 740
 741         /*
 742         * Map efi regions which were passed via setup_data. The virt_addr is a
 743         * fixed addr which was used in first kernel of a kexec boot.
 744         */
 745         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 746                 md = p;
 747                 efi_map_region_fixed(md); /* FIXME: add error handling */
 748                 get_systab_virt_addr(md);
 749         }
 750
 751         save_runtime_map();
 752
 753         BUG_ON(!efi.systab);
 754
 755         efi_sync_low_kernel_mappings();
 756
 757         /*
 758          * Now that EFI is in virtual mode, update the function
 759          * pointers in the runtime service table to the new virtual addresses.
 760          *
 761          * Call EFI services through wrapper functions.
 762          */
 763         efi.runtime_version = efi_systab.hdr.revision;
 764
 765         efi_native_runtime_setup();
 766
 767         efi.set_virtual_address_map = NULL;
 768
 769         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 770                 runtime_code_page_mkexec();
 771
 772         /* clean DUMMY object */
 773         efi_delete_dummy_variable();
 774 #endif
 775 }
 776
 777 /*
 778  * This function will switch the EFI runtime services to virtual mode.
 779  * Essentially, we look through the EFI memmap and map every region that
 780  * has the runtime attribute bit set in its memory descriptor into the
 781  * ->trampoline_pgd page table using a top-down VA allocation scheme.
 782  *
 783  * The old method which used to update that memory descriptor with the
 784  * virtual address obtained from ioremap() is still supported when the
 785  * kernel is booted with efi=old_map on its command line. Same old
 786  * method enabled the runtime services to be called without having to
 787  * thunk back into physical mode for every invocation.
 788  *
 789  * The new method does a pagetable switch in a preemption-safe manner
 790  * so that we're in a different address space when calling a runtime
 791  * function. For function arguments passing we do copy the PGDs of the
 792  * kernel page table into ->trampoline_pgd prior to each call.
 793  *
 794  * Specially for kexec boot, efi runtime maps in previous kernel should
 795  * be passed in via setup_data. In that case runtime ranges will be mapped
 796  * to the same virtual addresses as the first kernel, see
 797  * kexec_enter_virtual_mode().
 798  */
 799 static void __init __efi_enter_virtual_mode(void)
 800 {
 801         int count = 0, pg_shift = 0;
 802         void *new_memmap = NULL;
 803         efi_status_t status;
 804
 805         efi.systab = NULL;
 806
 807         efi_merge_regions();
 808         new_memmap = efi_map_regions(&count, &pg_shift);
 809         if (!new_memmap) {
 810                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 811                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 812                 return;
 813         }
 814
 815         save_runtime_map();
 816
 817         BUG_ON(!efi.systab);
 818
 819         if (efi_setup_page_tables(__pa(new_memmap), 1 << pg_shift)) {
 820                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 821                 return;
 822         }
 823
 824         efi_sync_low_kernel_mappings();
 825         efi_dump_pagetable();
 826
 827         if (efi_is_native()) {
 828                 status = phys_efi_set_virtual_address_map(
 829                                 memmap.desc_size * count,
 830                                 memmap.desc_size,
 831                                 memmap.desc_version,
 832                                 (efi_memory_desc_t *)__pa(new_memmap));
 833         } else {
 834                 status = efi_thunk_set_virtual_address_map(
 835                                 efi_phys.set_virtual_address_map,
 836                                 memmap.desc_size * count,
 837                                 memmap.desc_size,
 838                                 memmap.desc_version,
 839                                 (efi_memory_desc_t *)__pa(new_memmap));
 840         }
 841
 842         if (status != EFI_SUCCESS) {
 843                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 844                          status);
 845                 panic("EFI call to SetVirtualAddressMap() failed!");
 846         }
 847
 848         /*
 849          * Now that EFI is in virtual mode, update the function
 850          * pointers in the runtime service table to the new virtual addresses.
 851          *
 852          * Call EFI services through wrapper functions.
 853          */
 854         efi.runtime_version = efi_systab.hdr.revision;
 855
 856         if (efi_is_native())
 857                 efi_native_runtime_setup();
 858         else
 859                 efi_thunk_runtime_setup();
 860
 861         efi.set_virtual_address_map = NULL;
 862
 863         efi_runtime_mkexec();
 864
 865         /*
 866          * We mapped the descriptor array into the EFI pagetable above but we're
 867          * not unmapping it here. Here's why:
 868          *
 869          * We're copying select PGDs from the kernel page table to the EFI page
 870          * table and when we do so and make changes to those PGDs like unmapping
 871          * stuff from them, those changes appear in the kernel page table and we
 872          * go boom.
 873          *
 874          * From setup_real_mode():
 875          *
 876          * ...
 877          * trampoline_pgd[0] = init_level4_pgt[pgd_index(__PAGE_OFFSET)].pgd;
 878          *
 879          * In this particular case, our allocation is in PGD 0 of the EFI page
 880          * table but we've copied that PGD from PGD[272] of the EFI page table:
 881          *
 882          *      pgd_index(__PAGE_OFFSET = 0xffff880000000000) = 272
 883          *
 884          * where the direct memory mapping in kernel space is.
 885          *
 886          * new_memmap's VA comes from that direct mapping and thus clearing it,
 887          * it would get cleared in the kernel page table too.
 888          *
 889          * efi_cleanup_page_tables(__pa(new_memmap), 1 << pg_shift);
 890          */
 891         free_pages((unsigned long)new_memmap, pg_shift);
 892
 893         /* clean DUMMY object */
 894         efi_delete_dummy_variable();
 895 }
 896
 897 void __init efi_enter_virtual_mode(void)
 898 {
 899         if (efi_enabled(EFI_PARAVIRT))
 900                 return;
 901
 902         if (efi_setup)
 903                 kexec_enter_virtual_mode();
 904         else
 905                 __efi_enter_virtual_mode();
 906 }
 907
 908 /*
 909  * Convenience functions to obtain memory types and attributes
 910  */
 911 u32 efi_mem_type(unsigned long phys_addr)
 912 {
 913         efi_memory_desc_t *md;
 914         void *p;
 915
 916         if (!efi_enabled(EFI_MEMMAP))
 917                 return 0;
 918
 919         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 920                 md = p;
 921                 if ((md->phys_addr <= phys_addr) &&
 922                     (phys_addr < (md->phys_addr +
 923                                   (md->num_pages << EFI_PAGE_SHIFT))))
 924                         return md->type;
 925         }
 926         return 0;
 927 }
 928
 929 u64 efi_mem_attributes(unsigned long phys_addr)
 930 {
 931         efi_memory_desc_t *md;
 932         void *p;
 933
 934         if (!efi_enabled(EFI_MEMMAP))
 935                 return 0;
 936
 937         for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
 938                 md = p;
 939                 if ((md->phys_addr <= phys_addr) &&
 940                     (phys_addr < (md->phys_addr +
 941                                   (md->num_pages << EFI_PAGE_SHIFT))))
 942                         return md->attribute;
 943         }
 944         return 0;
 945 }
 946
 947 static int __init arch_parse_efi_cmdline(char *str)
 948 {
 949         if (parse_option_str(str, "old_map"))
 950                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
 951         if (parse_option_str(str, "debug"))
 952                 set_bit(EFI_DBG, &efi.flags);
 953
 954         return 0;
 955 }
 956 early_param("efi", arch_parse_efi_cmdline);