diff options
Diffstat (limited to 'drivers/firmware/efi/libstub')
41 files changed, 3328 insertions, 1804 deletions
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index b1601aad7e1a..7d15a85d579f 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -5,33 +5,38 @@ # things like ftrace and stack-protector are likely to cause trouble if left # enabled, even if doing so doesn't break the build. # + +# non-x86 reuses KBUILD_CFLAGS, x86 does not +cflags-y := $(KBUILD_CFLAGS) + cflags-$(CONFIG_X86_32) := -march=i386 cflags-$(CONFIG_X86_64) := -mcmodel=small -cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ \ +cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -std=gnu11 -fms-extensions \ -fPIC -fno-strict-aliasing -mno-red-zone \ -mno-mmx -mno-sse -fshort-wchar \ -Wno-pointer-sign \ $(call cc-disable-warning, address-of-packed-member) \ - $(call cc-disable-warning, gnu) \ + $(if $(CONFIG_CC_IS_CLANG),-Wno-gnu -Wno-microsoft-anon-tag) \ -fno-asynchronous-unwind-tables \ $(CLANG_FLAGS) # arm64 uses the full KBUILD_CFLAGS so it's necessary to explicitly # disable the stackleak plugin -cflags-$(CONFIG_ARM64) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ - -fpie $(DISABLE_STACKLEAK_PLUGIN) \ - $(call cc-option,-mbranch-protection=none) -cflags-$(CONFIG_ARM) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ - -fno-builtin -fpic \ - $(call cc-option,-mno-single-pic-base) -cflags-$(CONFIG_RISCV) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ - -fpic -cflags-$(CONFIG_LOONGARCH) := $(subst $(CC_FLAGS_FTRACE),,$(KBUILD_CFLAGS)) \ - -fpie +cflags-$(CONFIG_ARM64) += -fpie $(DISABLE_KSTACK_ERASE) \ + -fno-unwind-tables -fno-asynchronous-unwind-tables +cflags-$(CONFIG_ARM) += -DEFI_HAVE_STRLEN -DEFI_HAVE_STRNLEN \ + -DEFI_HAVE_MEMCHR -DEFI_HAVE_STRRCHR \ + -DEFI_HAVE_STRCMP -fno-builtin -fpic \ + $(call cc-option,-mno-single-pic-base) \ + $(DISABLE_KSTACK_ERASE) +cflags-$(CONFIG_RISCV) += -fpic -DNO_ALTERNATIVE -mno-relax \ + $(DISABLE_KSTACK_ERASE) +cflags-$(CONFIG_LOONGARCH) += -fpie $(DISABLE_KSTACK_ERASE) cflags-$(CONFIG_EFI_PARAMS_FROM_FDT) += -I$(srctree)/scripts/dtc/libfdt -KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \ +KBUILD_CFLAGS := $(subst $(CC_FLAGS_FTRACE),,$(cflags-y)) \ + -Os -DDISABLE_BRANCH_PROFILING \ -include $(srctree)/include/linux/hidden.h \ -D__NO_FORTIFY \ -ffreestanding \ @@ -53,21 +58,16 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_CFI), $(KBUILD_CFLAGS)) # disable LTO KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS)) -GCOV_PROFILE := n -# Sanitizer runtimes are unavailable and cannot be linked here. -KASAN_SANITIZE := n -KCSAN_SANITIZE := n -KMSAN_SANITIZE := n -UBSAN_SANITIZE := n -OBJECT_FILES_NON_STANDARD := y +# The .data section would be renamed to .data.efistub, therefore, remove +# `-fdata-sections` flag from KBUILD_CFLAGS_KERNEL +KBUILD_CFLAGS_KERNEL := $(filter-out -fdata-sections, $(KBUILD_CFLAGS_KERNEL)) -# Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. -KCOV_INSTRUMENT := n +KBUILD_AFLAGS := $(KBUILD_CFLAGS) -D__ASSEMBLY__ lib-y := efi-stub-helper.o gop.o secureboot.o tpm.o \ file.o mem.o random.o randomalloc.o pci.o \ skip_spaces.o lib-cmdline.o lib-ctype.o \ - alignedmem.o relocate.o vsprintf.o + alignedmem.o relocate.o printk.o vsprintf.o # include the stub's libfdt dependencies from lib/ when needed libfdt-deps := fdt_rw.c fdt_ro.c fdt_wip.c fdt.c \ @@ -79,35 +79,37 @@ lib-$(CONFIG_EFI_PARAMS_FROM_FDT) += fdt.o \ $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE $(call if_changed_rule,cc_o_c) -lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o +lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o \ + screen_info.o efi-stub-entry.o lib-$(CONFIG_ARM) += arm32-stub.o -lib-$(CONFIG_ARM64) += arm64-stub.o -lib-$(CONFIG_X86) += x86-stub.o -lib-$(CONFIG_RISCV) += riscv-stub.o -lib-$(CONFIG_LOONGARCH) += loongarch-stub.o +lib-$(CONFIG_ARM64) += kaslr.o arm64.o arm64-stub.o smbios.o +lib-$(CONFIG_X86) += x86-stub.o smbios.o +lib-$(CONFIG_X86_64) += x86-5lvl.o +lib-$(CONFIG_RISCV) += kaslr.o riscv.o riscv-stub.o +lib-$(CONFIG_LOONGARCH) += loongarch.o loongarch-stub.o CFLAGS_arm32-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) -zboot-obj-$(CONFIG_RISCV) := lib-clz_ctz.o lib-ashldi3.o +zboot-obj-y := zboot-decompress-gzip.o +CFLAGS_zboot-decompress-gzip.o += -I$(srctree)/lib/zlib_inflate +zboot-obj-$(CONFIG_KERNEL_ZSTD) := zboot-decompress-zstd.o lib-xxhash.o +CFLAGS_zboot-decompress-zstd.o += -I$(srctree)/lib/zstd + +zboot-obj-$(CONFIG_RISCV) += lib-clz_ctz.o lib-ashldi3.o lib-$(CONFIG_EFI_ZBOOT) += zboot.o $(zboot-obj-y) -extra-y := $(lib-y) +lib-$(CONFIG_UNACCEPTED_MEMORY) += unaccepted_memory.o bitmap.o find.o + +targets := $(lib-y) lib-y := $(patsubst %.o,%.stub.o,$(lib-y)) # Even when -mbranch-protection=none is set, Clang will generate a # .note.gnu.property for code-less object files (like lib/ctype.c), # so work around this by explicitly removing the unwanted section. -# https://bugs.llvm.org/show_bug.cgi?id=46480 +# https://llvm.org/pr46480 STUBCOPY_FLAGS-y += --remove-section=.note.gnu.property -# -# For x86, bootloaders like systemd-boot or grub-efi do not zero-initialize the -# .bss section, so the .bss section of the EFI stub needs to be included in the -# .data section of the compressed kernel to ensure initialization. Rename the -# .bss section here so it's easy to pick out in the linker script. -# -STUBCOPY_FLAGS-$(CONFIG_X86) += --rename-section .bss=.bss.efistub,load,alloc STUBCOPY_RELOC-$(CONFIG_X86_32) := R_386_32 STUBCOPY_RELOC-$(CONFIG_X86_64) := R_X86_64_64 @@ -140,10 +142,10 @@ STUBCOPY_RELOC-$(CONFIG_ARM64) := R_AARCH64_ABS # For RISC-V, we don't need anything special other than arm64. Keep all the # symbols in .init section and make sure that no absolute symbols references -# doesn't exist. +# exist. STUBCOPY_FLAGS-$(CONFIG_RISCV) += --prefix-alloc-sections=.init \ --prefix-symbols=__efistub_ -STUBCOPY_RELOC-$(CONFIG_RISCV) := R_RISCV_HI20 +STUBCOPY_RELOC-$(CONFIG_RISCV) := -E R_RISCV_HI20\|R_RISCV_$(BITS)\|R_RISCV_RELAX # For LoongArch, keep all the symbols in .init section and make sure that no # absolute symbols references exist. diff --git a/drivers/firmware/efi/libstub/Makefile.zboot b/drivers/firmware/efi/libstub/Makefile.zboot index 3340b385a05b..832deee36e48 100644 --- a/drivers/firmware/efi/libstub/Makefile.zboot +++ b/drivers/firmware/efi/libstub/Makefile.zboot @@ -1,41 +1,53 @@ # SPDX-License-Identifier: GPL-2.0 # to be include'd by arch/$(ARCH)/boot/Makefile after setting -# EFI_ZBOOT_PAYLOAD, EFI_ZBOOT_BFD_TARGET and EFI_ZBOOT_MACH_TYPE +# EFI_ZBOOT_PAYLOAD, EFI_ZBOOT_BFD_TARGET, EFI_ZBOOT_MACH_TYPE and +# EFI_ZBOOT_FORWARD_CFI -comp-type-$(CONFIG_KERNEL_GZIP) := gzip -comp-type-$(CONFIG_KERNEL_LZ4) := lz4 -comp-type-$(CONFIG_KERNEL_LZMA) := lzma -comp-type-$(CONFIG_KERNEL_LZO) := lzo -comp-type-$(CONFIG_KERNEL_XZ) := xzkern -comp-type-$(CONFIG_KERNEL_ZSTD) := zstd22 +quiet_cmd_copy_and_pad = PAD $@ + cmd_copy_and_pad = cp $< $@; \ + truncate -s $$(hexdump -s16 -n4 -e '"%u"' $<) $@ + +# Pad the file to the size of the uncompressed image in memory, including BSS +$(obj)/vmlinux.bin: $(obj)/$(EFI_ZBOOT_PAYLOAD) FORCE + $(call if_changed,copy_and_pad) # in GZIP, the appended le32 carrying the uncompressed size is part of the # format, but in other cases, we just append it at the end for convenience, # causing the original tools to complain when checking image integrity. -# So disregard it when calculating the payload size in the zimage header. -zboot-method-y := $(comp-type-y)_with_size -zboot-size-len-y := 4 +comp-type-y := gzip +zboot-method-y := gzip +zboot-size-len-y := 0 -zboot-method-$(CONFIG_KERNEL_GZIP) := gzip -zboot-size-len-$(CONFIG_KERNEL_GZIP) := 0 +comp-type-$(CONFIG_KERNEL_ZSTD) := zstd +zboot-method-$(CONFIG_KERNEL_ZSTD) := zstd22_with_size +zboot-size-len-$(CONFIG_KERNEL_ZSTD) := 4 -$(obj)/vmlinuz: $(obj)/$(EFI_ZBOOT_PAYLOAD) FORCE +$(obj)/vmlinuz: $(obj)/vmlinux.bin FORCE $(call if_changed,$(zboot-method-y)) -OBJCOPYFLAGS_vmlinuz.o := -I binary -O $(EFI_ZBOOT_BFD_TARGET) \ +# avoid eager evaluation to prevent references to non-existent build artifacts +OBJCOPYFLAGS_vmlinuz.o = -I binary -O $(EFI_ZBOOT_BFD_TARGET) $(EFI_ZBOOT_OBJCOPY_FLAGS) \ --rename-section .data=.gzdata,load,alloc,readonly,contents $(obj)/vmlinuz.o: $(obj)/vmlinuz FORCE $(call if_changed,objcopy) +aflags-zboot-header-$(EFI_ZBOOT_FORWARD_CFI) := \ + -DPE_DLL_CHAR_EX=IMAGE_DLLCHARACTERISTICS_EX_FORWARD_CFI_COMPAT + AFLAGS_zboot-header.o += -DMACHINE_TYPE=IMAGE_FILE_MACHINE_$(EFI_ZBOOT_MACH_TYPE) \ - -DZBOOT_EFI_PATH="\"$(realpath $(obj)/vmlinuz.efi.elf)\"" \ + -DZBOOT_EFI_PATH="\"$(abspath $(obj)/vmlinuz.efi.elf)\"" \ -DZBOOT_SIZE_LEN=$(zboot-size-len-y) \ - -DCOMP_TYPE="\"$(comp-type-y)\"" + -DCOMP_TYPE="\"$(comp-type-y)\"" \ + $(aflags-zboot-header-y) $(obj)/zboot-header.o: $(srctree)/drivers/firmware/efi/libstub/zboot-header.S FORCE $(call if_changed_rule,as_o_S) +ifneq ($(CONFIG_EFI_SBAT_FILE),) +$(obj)/zboot-header.o: $(CONFIG_EFI_SBAT_FILE) +endif + ZBOOT_DEPS := $(obj)/zboot-header.o $(objtree)/drivers/firmware/efi/libstub/lib.a LDFLAGS_vmlinuz.efi.elf := -T $(srctree)/drivers/firmware/efi/libstub/zboot.lds @@ -46,4 +58,4 @@ OBJCOPYFLAGS_vmlinuz.efi := -O binary $(obj)/vmlinuz.efi: $(obj)/vmlinuz.efi.elf FORCE $(call if_changed,objcopy) -targets += zboot-header.o vmlinuz vmlinuz.o vmlinuz.efi.elf vmlinuz.efi +targets += zboot-header.o vmlinux.bin vmlinuz vmlinuz.o vmlinuz.efi.elf vmlinuz.efi diff --git a/drivers/firmware/efi/libstub/alignedmem.c b/drivers/firmware/efi/libstub/alignedmem.c index 1de9878ddd3a..31928bd87e0f 100644 --- a/drivers/firmware/efi/libstub/alignedmem.c +++ b/drivers/firmware/efi/libstub/alignedmem.c @@ -14,6 +14,7 @@ * @max: the address that the last allocated memory page shall not * exceed * @align: minimum alignment of the base of the allocation + * @memory_type: the type of memory to allocate * * Allocate pages as EFI_LOADER_DATA. The allocated pages are aligned according * to @align, which should be >= EFI_ALLOC_ALIGN. The last allocated page will @@ -22,12 +23,15 @@ * Return: status code */ efi_status_t efi_allocate_pages_aligned(unsigned long size, unsigned long *addr, - unsigned long max, unsigned long align) + unsigned long max, unsigned long align, + int memory_type) { efi_physical_addr_t alloc_addr; efi_status_t status; int slack; + max = min(max, EFI_ALLOC_LIMIT); + if (align < EFI_ALLOC_ALIGN) align = EFI_ALLOC_ALIGN; @@ -36,7 +40,7 @@ efi_status_t efi_allocate_pages_aligned(unsigned long size, unsigned long *addr, slack = align / EFI_PAGE_SIZE - 1; status = efi_bs_call(allocate_pages, EFI_ALLOCATE_MAX_ADDRESS, - EFI_LOADER_DATA, size / EFI_PAGE_SIZE + slack, + memory_type, size / EFI_PAGE_SIZE + slack, &alloc_addr); if (status != EFI_SUCCESS) return status; diff --git a/drivers/firmware/efi/libstub/arm32-stub.c b/drivers/firmware/efi/libstub/arm32-stub.c index 0131e3aaa605..1073dd947516 100644 --- a/drivers/firmware/efi/libstub/arm32-stub.c +++ b/drivers/firmware/efi/libstub/arm32-stub.c @@ -76,43 +76,6 @@ void efi_handle_post_ebs_state(void) &efi_entry_state->sctlr_after_ebs); } -static efi_guid_t screen_info_guid = LINUX_EFI_ARM_SCREEN_INFO_TABLE_GUID; - -struct screen_info *alloc_screen_info(void) -{ - struct screen_info *si; - efi_status_t status; - - /* - * Unlike on arm64, where we can directly fill out the screen_info - * structure from the stub, we need to allocate a buffer to hold - * its contents while we hand over to the kernel proper from the - * decompressor. - */ - status = efi_bs_call(allocate_pool, EFI_RUNTIME_SERVICES_DATA, - sizeof(*si), (void **)&si); - - if (status != EFI_SUCCESS) - return NULL; - - status = efi_bs_call(install_configuration_table, - &screen_info_guid, si); - if (status == EFI_SUCCESS) - return si; - - efi_bs_call(free_pool, si); - return NULL; -} - -void free_screen_info(struct screen_info *si) -{ - if (!si) - return; - - efi_bs_call(install_configuration_table, &screen_info_guid, NULL); - efi_bs_call(free_pool, si); -} - efi_status_t handle_kernel_image(unsigned long *image_addr, unsigned long *image_size, unsigned long *reserve_addr, diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 259e4b852d63..2c3869356147 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -11,73 +11,9 @@ #include <asm/efi.h> #include <asm/memory.h> #include <asm/sections.h> -#include <asm/sysreg.h> #include "efistub.h" -efi_status_t check_platform_features(void) -{ - u64 tg; - - /* - * If we have 48 bits of VA space for TTBR0 mappings, we can map the - * UEFI runtime regions 1:1 and so calling SetVirtualAddressMap() is - * unnecessary. - */ - if (VA_BITS_MIN >= 48) - efi_novamap = true; - - /* UEFI mandates support for 4 KB granularity, no need to check */ - if (IS_ENABLED(CONFIG_ARM64_4K_PAGES)) - return EFI_SUCCESS; - - tg = (read_cpuid(ID_AA64MMFR0_EL1) >> ID_AA64MMFR0_EL1_TGRAN_SHIFT) & 0xf; - if (tg < ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN || tg > ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX) { - if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) - efi_err("This 64 KB granular kernel is not supported by your CPU\n"); - else - efi_err("This 16 KB granular kernel is not supported by your CPU\n"); - return EFI_UNSUPPORTED; - } - return EFI_SUCCESS; -} - -/* - * Distro versions of GRUB may ignore the BSS allocation entirely (i.e., fail - * to provide space, and fail to zero it). Check for this condition by double - * checking that the first and the last byte of the image are covered by the - * same EFI memory map entry. - */ -static bool check_image_region(u64 base, u64 size) -{ - struct efi_boot_memmap *map; - efi_status_t status; - bool ret = false; - int map_offset; - - status = efi_get_memory_map(&map, false); - if (status != EFI_SUCCESS) - return false; - - for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { - efi_memory_desc_t *md = (void *)map->map + map_offset; - u64 end = md->phys_addr + md->num_pages * EFI_PAGE_SIZE; - - /* - * Find the region that covers base, and return whether - * it covers base+size bytes. - */ - if (base >= md->phys_addr && base < end) { - ret = (base + size) <= end; - break; - } - } - - efi_bs_call(free_pool, map); - - return ret; -} - efi_status_t handle_kernel_image(unsigned long *image_addr, unsigned long *image_size, unsigned long *reserve_addr, @@ -85,92 +21,43 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, efi_loaded_image_t *image, efi_handle_t image_handle) { - efi_status_t status; - unsigned long kernel_size, kernel_memsize = 0; - u32 phys_seed = 0; - - /* - * Although relocatable kernels can fix up the misalignment with - * respect to MIN_KIMG_ALIGN, the resulting virtual text addresses are - * subtly out of sync with those recorded in the vmlinux when kaslr is - * disabled but the image required relocation anyway. Therefore retain - * 2M alignment if KASLR was explicitly disabled, even if it was not - * going to be activated to begin with. - */ - u64 min_kimg_align = efi_nokaslr ? MIN_KIMG_ALIGN : EFI_KIMG_ALIGN; - - if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { - efi_guid_t li_fixed_proto = LINUX_EFI_LOADED_IMAGE_FIXED_GUID; - void *p; + unsigned long kernel_size, kernel_codesize, kernel_memsize; - if (efi_nokaslr) { - efi_info("KASLR disabled on kernel command line\n"); - } else if (efi_bs_call(handle_protocol, image_handle, - &li_fixed_proto, &p) == EFI_SUCCESS) { - efi_info("Image placement fixed by loader\n"); - } else { - status = efi_get_random_bytes(sizeof(phys_seed), - (u8 *)&phys_seed); - if (status == EFI_NOT_FOUND) { - efi_info("EFI_RNG_PROTOCOL unavailable\n"); - efi_nokaslr = true; - } else if (status != EFI_SUCCESS) { - efi_err("efi_get_random_bytes() failed (0x%lx)\n", - status); - efi_nokaslr = true; - } - } - } - - if (image->image_base != _text) + if (image->image_base != _text) { efi_err("FIRMWARE BUG: efi_loaded_image_t::image_base has bogus value\n"); + image->image_base = _text; + } if (!IS_ALIGNED((u64)_text, SEGMENT_ALIGN)) efi_err("FIRMWARE BUG: kernel image not aligned on %dk boundary\n", SEGMENT_ALIGN >> 10); kernel_size = _edata - _text; + kernel_codesize = __inittext_end - _text; kernel_memsize = kernel_size + (_end - _edata); *reserve_size = kernel_memsize; + *image_addr = (unsigned long)_text; - if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && phys_seed != 0) { - /* - * If KASLR is enabled, and we have some randomness available, - * locate the kernel at a randomized offset in physical memory. - */ - status = efi_random_alloc(*reserve_size, min_kimg_align, - reserve_addr, phys_seed); - if (status != EFI_SUCCESS) - efi_warn("efi_random_alloc() failed: 0x%lx\n", status); - } else { - status = EFI_OUT_OF_RESOURCES; - } - - if (status != EFI_SUCCESS) { - if (!check_image_region((u64)_text, kernel_memsize)) { - efi_err("FIRMWARE BUG: Image BSS overlaps adjacent EFI memory region\n"); - } else if (IS_ALIGNED((u64)_text, min_kimg_align)) { - /* - * Just execute from wherever we were loaded by the - * UEFI PE/COFF loader if the alignment is suitable. - */ - *image_addr = (u64)_text; - *reserve_size = 0; - return EFI_SUCCESS; - } + return efi_kaslr_relocate_kernel(image_addr, reserve_addr, reserve_size, + kernel_size, kernel_codesize, kernel_memsize, + efi_kaslr_get_phys_seed(image_handle)); +} - status = efi_allocate_pages_aligned(*reserve_size, reserve_addr, - ULONG_MAX, min_kimg_align); +asmlinkage void primary_entry(void); - if (status != EFI_SUCCESS) { - efi_err("Failed to relocate kernel\n"); - *reserve_size = 0; - return status; - } - } - - *image_addr = *reserve_addr; - memcpy((void *)*image_addr, _text, kernel_size); +unsigned long primary_entry_offset(void) +{ + /* + * When built as part of the kernel, the EFI stub cannot branch to the + * kernel proper via the image header, as the PE/COFF header is + * strictly not part of the in-memory presentation of the image, only + * of the file representation. So instead, we need to jump to the + * actual entrypoint in the .text region of the image. + */ + return (char *)primary_entry - _text; +} - return EFI_SUCCESS; +void efi_icache_sync(unsigned long start, unsigned long end) +{ + caches_clean_inval_pou(start, end); } diff --git a/drivers/firmware/efi/libstub/arm64.c b/drivers/firmware/efi/libstub/arm64.c new file mode 100644 index 000000000000..e57cd3de0a00 --- /dev/null +++ b/drivers/firmware/efi/libstub/arm64.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013, 2014 Linaro Ltd; <roy.franz@linaro.org> + * + * This file implements the EFI boot stub for the arm64 kernel. + * Adapted from ARM version by Mark Salter <msalter@redhat.com> + */ + + +#include <linux/efi.h> +#include <asm/efi.h> +#include <asm/image.h> +#include <asm/memory.h> +#include <asm/sysreg.h> + +#include "efistub.h" + +static bool system_needs_vamap(void) +{ + const struct efi_smbios_type4_record *record; + const u32 __aligned(1) *socid; + const u8 *version; + + /* + * Ampere eMAG, Altra, and Altra Max machines crash in SetTime() if + * SetVirtualAddressMap() has not been called prior. Most Altra systems + * can be identified by the SMCCC soc ID, which is conveniently exposed + * via the type 4 SMBIOS records. Otherwise, test the processor version + * field. eMAG systems all appear to have the processor version field + * set to "eMAG". + */ + record = (struct efi_smbios_type4_record *)efi_get_smbios_record(4); + if (!record) + return false; + + socid = (u32 *)record->processor_id; + switch (*socid & 0xffff000f) { + static char const altra[] = "Ampere(TM) Altra(TM) Processor"; + static char const emag[] = "eMAG"; + + default: + version = efi_get_smbios_string(record, processor_version); + if (!version || (strncmp(version, altra, sizeof(altra) - 1) && + strncmp(version, emag, sizeof(emag) - 1))) + break; + + fallthrough; + + case 0x0a160001: // Altra + case 0x0a160002: // Altra Max + efi_warn("Working around broken SetVirtualAddressMap()\n"); + return true; + } + + return false; +} + +efi_status_t check_platform_features(void) +{ + u64 tg; + + /* + * If we have 48 bits of VA space for TTBR0 mappings, we can map the + * UEFI runtime regions 1:1 and so calling SetVirtualAddressMap() is + * unnecessary. + */ + if (VA_BITS_MIN >= 48 && !system_needs_vamap()) + efi_novamap = true; + + /* UEFI mandates support for 4 KB granularity, no need to check */ + if (IS_ENABLED(CONFIG_ARM64_4K_PAGES)) + return EFI_SUCCESS; + + tg = (read_cpuid(ID_AA64MMFR0_EL1) >> ID_AA64MMFR0_EL1_TGRAN_SHIFT) & 0xf; + if (tg < ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN || tg > ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX) { + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) + efi_err("This 64 KB granular kernel is not supported by your CPU\n"); + else + efi_err("This 16 KB granular kernel is not supported by your CPU\n"); + return EFI_UNSUPPORTED; + } + return EFI_SUCCESS; +} + +#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE +#define DCTYPE "civac" +#else +#define DCTYPE "cvau" +#endif + +u32 __weak code_size; + +void efi_cache_sync_image(unsigned long image_base, + unsigned long alloc_size) +{ + u32 ctr = read_cpuid_effective_cachetype(); + u64 lsize = 4 << cpuid_feature_extract_unsigned_field(ctr, + CTR_EL0_DminLine_SHIFT); + + /* only perform the cache maintenance if needed for I/D coherency */ + if (!(ctr & BIT(CTR_EL0_IDC_SHIFT))) { + unsigned long base = image_base; + unsigned long size = code_size; + + do { + asm("dc " DCTYPE ", %0" :: "r"(base)); + base += lsize; + size -= lsize; + } while (size >= lsize); + } + + asm("ic ialluis"); + dsb(ish); + isb(); + + efi_remap_image(image_base, alloc_size, code_size); +} + +unsigned long __weak primary_entry_offset(void) +{ + /* + * By default, we can invoke the kernel via the branch instruction in + * the image header, so offset #0. This will be overridden by the EFI + * stub build that is linked into the core kernel, as in that case, the + * image header may not have been loaded into memory, or may be mapped + * with non-executable permissions. + */ + return 0; +} + +void __noreturn efi_enter_kernel(unsigned long entrypoint, + unsigned long fdt_addr, + unsigned long fdt_size) +{ + void (* __noreturn enter_kernel)(u64, u64, u64, u64); + + enter_kernel = (void *)entrypoint + primary_entry_offset(); + enter_kernel(fdt_addr, 0, 0, 0); +} diff --git a/drivers/firmware/efi/libstub/bitmap.c b/drivers/firmware/efi/libstub/bitmap.c new file mode 100644 index 000000000000..5c9bba0d549b --- /dev/null +++ b/drivers/firmware/efi/libstub/bitmap.c @@ -0,0 +1,41 @@ +#include <linux/bitmap.h> + +void __bitmap_set(unsigned long *map, unsigned int start, int len) +{ + unsigned long *p = map + BIT_WORD(start); + const unsigned int size = start + len; + int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); + + while (len - bits_to_set >= 0) { + *p |= mask_to_set; + len -= bits_to_set; + bits_to_set = BITS_PER_LONG; + mask_to_set = ~0UL; + p++; + } + if (len) { + mask_to_set &= BITMAP_LAST_WORD_MASK(size); + *p |= mask_to_set; + } +} + +void __bitmap_clear(unsigned long *map, unsigned int start, int len) +{ + unsigned long *p = map + BIT_WORD(start); + const unsigned int size = start + len; + int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); + unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); + + while (len - bits_to_clear >= 0) { + *p &= ~mask_to_clear; + len -= bits_to_clear; + bits_to_clear = BITS_PER_LONG; + mask_to_clear = ~0UL; + p++; + } + if (len) { + mask_to_clear &= BITMAP_LAST_WORD_MASK(size); + *p &= ~mask_to_clear; + } +} diff --git a/drivers/firmware/efi/libstub/efi-stub-entry.c b/drivers/firmware/efi/libstub/efi-stub-entry.c new file mode 100644 index 000000000000..a6c049835190 --- /dev/null +++ b/drivers/firmware/efi/libstub/efi-stub-entry.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/efi.h> +#include <linux/screen_info.h> + +#include <asm/efi.h> + +#include "efistub.h" + +static unsigned long screen_info_offset; + +struct screen_info *alloc_screen_info(void) +{ + if (IS_ENABLED(CONFIG_ARM)) + return __alloc_screen_info(); + + if (IS_ENABLED(CONFIG_X86) || + IS_ENABLED(CONFIG_EFI_EARLYCON) || + IS_ENABLED(CONFIG_SYSFB)) + return (void *)&screen_info + screen_info_offset; + + return NULL; +} + +/* + * EFI entry point for the generic EFI stub used by ARM, arm64, RISC-V and + * LoongArch. This is the entrypoint that is described in the PE/COFF header + * of the core kernel. + */ +efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, + efi_system_table_t *systab) +{ + efi_loaded_image_t *image; + efi_status_t status; + unsigned long image_addr; + unsigned long image_size = 0; + /* addr/point and size pairs for memory management*/ + char *cmdline_ptr = NULL; + efi_guid_t loaded_image_proto = LOADED_IMAGE_PROTOCOL_GUID; + unsigned long reserve_addr = 0; + unsigned long reserve_size = 0; + + WRITE_ONCE(efi_system_table, systab); + + /* Check if we were booted by the EFI firmware */ + if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) + return EFI_INVALID_PARAMETER; + + /* + * Get a handle to the loaded image protocol. This is used to get + * information about the running image, such as size and the command + * line. + */ + status = efi_bs_call(handle_protocol, handle, &loaded_image_proto, + (void *)&image); + if (status != EFI_SUCCESS) { + efi_err("Failed to get loaded image protocol\n"); + return status; + } + + status = efi_handle_cmdline(image, &cmdline_ptr); + if (status != EFI_SUCCESS) + return status; + + efi_info("Booting Linux Kernel...\n"); + + status = handle_kernel_image(&image_addr, &image_size, + &reserve_addr, + &reserve_size, + image, handle); + if (status != EFI_SUCCESS) { + efi_err("Failed to relocate kernel\n"); + return status; + } + + screen_info_offset = image_addr - (unsigned long)image->image_base; + + status = efi_stub_common(handle, image, image_addr, cmdline_ptr); + + efi_free(image_size, image_addr); + efi_free(reserve_size, reserve_addr); + + return status; +} diff --git a/drivers/firmware/efi/libstub/efi-stub-helper.c b/drivers/firmware/efi/libstub/efi-stub-helper.c index 0c493521b25b..7aa2f9ad2935 100644 --- a/drivers/firmware/efi/libstub/efi-stub-helper.c +++ b/drivers/firmware/efi/libstub/efi-stub-helper.c @@ -9,10 +9,9 @@ #include <linux/stdarg.h> -#include <linux/ctype.h> #include <linux/efi.h> #include <linux/kernel.h> -#include <linux/printk.h> /* For CONSOLE_LOGLEVEL_* */ +#include <linux/overflow.h> #include <asm/efi.h> #include <asm/setup.h> @@ -20,159 +19,20 @@ bool efi_nochunk; bool efi_nokaslr = !IS_ENABLED(CONFIG_RANDOMIZE_BASE); -int efi_loglevel = CONSOLE_LOGLEVEL_DEFAULT; bool efi_novamap; static bool efi_noinitrd; static bool efi_nosoftreserve; static bool efi_disable_pci_dma = IS_ENABLED(CONFIG_EFI_DISABLE_PCI_DMA); +int efi_mem_encrypt; + bool __pure __efi_soft_reserve_enabled(void) { return !efi_nosoftreserve; } /** - * efi_char16_puts() - Write a UCS-2 encoded string to the console - * @str: UCS-2 encoded string - */ -void efi_char16_puts(efi_char16_t *str) -{ - efi_call_proto(efi_table_attr(efi_system_table, con_out), - output_string, str); -} - -static -u32 utf8_to_utf32(const u8 **s8) -{ - u32 c32; - u8 c0, cx; - size_t clen, i; - - c0 = cx = *(*s8)++; - /* - * The position of the most-significant 0 bit gives us the length of - * a multi-octet encoding. - */ - for (clen = 0; cx & 0x80; ++clen) - cx <<= 1; - /* - * If the 0 bit is in position 8, this is a valid single-octet - * encoding. If the 0 bit is in position 7 or positions 1-3, the - * encoding is invalid. - * In either case, we just return the first octet. - */ - if (clen < 2 || clen > 4) - return c0; - /* Get the bits from the first octet. */ - c32 = cx >> clen--; - for (i = 0; i < clen; ++i) { - /* Trailing octets must have 10 in most significant bits. */ - cx = (*s8)[i] ^ 0x80; - if (cx & 0xc0) - return c0; - c32 = (c32 << 6) | cx; - } - /* - * Check for validity: - * - The character must be in the Unicode range. - * - It must not be a surrogate. - * - It must be encoded using the correct number of octets. - */ - if (c32 > 0x10ffff || - (c32 & 0xf800) == 0xd800 || - clen != (c32 >= 0x80) + (c32 >= 0x800) + (c32 >= 0x10000)) - return c0; - *s8 += clen; - return c32; -} - -/** - * efi_puts() - Write a UTF-8 encoded string to the console - * @str: UTF-8 encoded string - */ -void efi_puts(const char *str) -{ - efi_char16_t buf[128]; - size_t pos = 0, lim = ARRAY_SIZE(buf); - const u8 *s8 = (const u8 *)str; - u32 c32; - - while (*s8) { - if (*s8 == '\n') - buf[pos++] = L'\r'; - c32 = utf8_to_utf32(&s8); - if (c32 < 0x10000) { - /* Characters in plane 0 use a single word. */ - buf[pos++] = c32; - } else { - /* - * Characters in other planes encode into a surrogate - * pair. - */ - buf[pos++] = (0xd800 - (0x10000 >> 10)) + (c32 >> 10); - buf[pos++] = 0xdc00 + (c32 & 0x3ff); - } - if (*s8 == '\0' || pos >= lim - 2) { - buf[pos] = L'\0'; - efi_char16_puts(buf); - pos = 0; - } - } -} - -/** - * efi_printk() - Print a kernel message - * @fmt: format string - * - * The first letter of the format string is used to determine the logging level - * of the message. If the level is less then the current EFI logging level, the - * message is suppressed. The message will be truncated to 255 bytes. - * - * Return: number of printed characters - */ -int efi_printk(const char *fmt, ...) -{ - char printf_buf[256]; - va_list args; - int printed; - int loglevel = printk_get_level(fmt); - - switch (loglevel) { - case '0' ... '9': - loglevel -= '0'; - break; - default: - /* - * Use loglevel -1 for cases where we just want to print to - * the screen. - */ - loglevel = -1; - break; - } - - if (loglevel >= efi_loglevel) - return 0; - - if (loglevel >= 0) - efi_puts("EFI stub: "); - - fmt = printk_skip_level(fmt); - - va_start(args, fmt); - printed = vsnprintf(printf_buf, sizeof(printf_buf), fmt, args); - va_end(args); - - efi_puts(printf_buf); - if (printed >= sizeof(printf_buf)) { - efi_puts("[Message truncated]\n"); - return -1; - } - - return printed; -} - -/** * efi_parse_options() - Parse EFI command line options * @cmdline: kernel command line * @@ -187,9 +47,10 @@ int efi_printk(const char *fmt, ...) */ efi_status_t efi_parse_options(char const *cmdline) { - size_t len; + char *buf __free(efi_pool) = NULL; efi_status_t status; - char *str, *buf; + size_t len; + char *str; if (!cmdline) return EFI_SUCCESS; @@ -216,6 +77,14 @@ efi_status_t efi_parse_options(char const *cmdline) efi_loglevel = CONSOLE_LOGLEVEL_QUIET; } else if (!strcmp(param, "noinitrd")) { efi_noinitrd = true; + } else if (IS_ENABLED(CONFIG_X86_64) && !strcmp(param, "no5lvl")) { + efi_no5lvl = true; + } else if (IS_ENABLED(CONFIG_ARCH_HAS_MEM_ENCRYPT) && + !strcmp(param, "mem_encrypt") && val) { + if (parse_option_str(val, "on")) + efi_mem_encrypt = 1; + else if (parse_option_str(val, "off")) + efi_mem_encrypt = -1; } else if (!strcmp(param, "efi") && val) { efi_nochunk = parse_option_str(val, "nochunk"); efi_novamap |= parse_option_str(val, "novamap"); @@ -234,7 +103,6 @@ efi_status_t efi_parse_options(char const *cmdline) efi_parse_option_graphics(val + strlen("efifb:")); } } - efi_bs_call(free_pool, buf); return EFI_SUCCESS; } @@ -334,7 +202,7 @@ void efi_apply_loadoptions_quirk(const void **load_options, u32 *load_options_si *load_options_size = load_option_unpacked.optional_data_size; } -enum efistub_event { +enum efistub_event_type { EFISTUB_EVT_INITRD, EFISTUB_EVT_LOAD_OPTIONS, EFISTUB_EVT_COUNT, @@ -360,54 +228,94 @@ static const struct { }, }; +static_assert(sizeof(efi_tcg2_event_t) == sizeof(efi_cc_event_t)); + +union efistub_event { + efi_tcg2_event_t tcg2_data; + efi_cc_event_t cc_data; +}; + +struct efistub_measured_event { + union efistub_event event_data; + TCG_PCClientTaggedEvent tagged_event __packed; +}; + static efi_status_t efi_measure_tagged_event(unsigned long load_addr, unsigned long load_size, - enum efistub_event event) + enum efistub_event_type event) { + union { + efi_status_t + (__efiapi *hash_log_extend_event)(void *, u64, efi_physical_addr_t, + u64, const union efistub_event *); + struct { u32 hash_log_extend_event; } mixed_mode; + } method; + struct efistub_measured_event *evt __free(efi_pool) = NULL; + int size = struct_size(evt, tagged_event.tagged_event_data, + events[event].event_data_len); efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID; efi_tcg2_protocol_t *tcg2 = NULL; + union efistub_event ev; efi_status_t status; + void *protocol; efi_bs_call(locate_protocol, &tcg2_guid, NULL, (void **)&tcg2); if (tcg2) { - struct efi_measured_event { - efi_tcg2_event_t event_data; - efi_tcg2_tagged_event_t tagged_event; - u8 tagged_event_data[]; - } *evt; - int size = sizeof(*evt) + events[event].event_data_len; - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)&evt); - if (status != EFI_SUCCESS) - goto fail; - - evt->event_data = (struct efi_tcg2_event){ + ev.tcg2_data = (struct efi_tcg2_event){ .event_size = size, - .event_header.header_size = sizeof(evt->event_data.event_header), + .event_header.header_size = sizeof(ev.tcg2_data.event_header), .event_header.header_version = EFI_TCG2_EVENT_HEADER_VERSION, .event_header.pcr_index = events[event].pcr_index, .event_header.event_type = EV_EVENT_TAG, }; - - evt->tagged_event = (struct efi_tcg2_tagged_event){ - .tagged_event_id = events[event].event_id, - .tagged_event_data_size = events[event].event_data_len, + protocol = tcg2; + method.hash_log_extend_event = + (void *)efi_table_attr(tcg2, hash_log_extend_event); + } else { + efi_guid_t cc_guid = EFI_CC_MEASUREMENT_PROTOCOL_GUID; + efi_cc_protocol_t *cc = NULL; + + efi_bs_call(locate_protocol, &cc_guid, NULL, (void **)&cc); + if (!cc) + return EFI_UNSUPPORTED; + + ev.cc_data = (struct efi_cc_event){ + .event_size = size, + .event_header.header_size = sizeof(ev.cc_data.event_header), + .event_header.header_version = EFI_CC_EVENT_HEADER_VERSION, + .event_header.event_type = EV_EVENT_TAG, }; - memcpy(evt->tagged_event_data, events[event].event_data, - events[event].event_data_len); - - status = efi_call_proto(tcg2, hash_log_extend_event, 0, - load_addr, load_size, &evt->event_data); - efi_bs_call(free_pool, evt); - + status = efi_call_proto(cc, map_pcr_to_mr_index, + events[event].pcr_index, + &ev.cc_data.event_header.mr_index); if (status != EFI_SUCCESS) goto fail; - return EFI_SUCCESS; + + protocol = cc; + method.hash_log_extend_event = + (void *)efi_table_attr(cc, hash_log_extend_event); } - return EFI_UNSUPPORTED; + status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, (void **)&evt); + if (status != EFI_SUCCESS) + goto fail; + + *evt = (struct efistub_measured_event) { + .event_data = ev, + .tagged_event.tagged_event_id = events[event].event_id, + .tagged_event.tagged_event_data_size = events[event].event_data_len, + }; + + memcpy(evt->tagged_event.tagged_event_data, events[event].event_data, + events[event].event_data_len); + + status = efi_fn_call(&method, hash_log_extend_event, protocol, 0, + load_addr, load_size, &evt->event_data); + + if (status == EFI_SUCCESS) + return EFI_SUCCESS; + fail: efi_warn("Failed to measure data for event %d: 0x%lx\n", event, status); return status; @@ -418,7 +326,7 @@ fail: * Size of memory allocated return in *cmd_line_len. * Returns NULL on error. */ -char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len) +char *efi_convert_cmdline(efi_loaded_image_t *image) { const efi_char16_t *options = efi_table_attr(image, load_options); u32 options_size = efi_table_attr(image, load_options_size); @@ -496,7 +404,6 @@ char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len) snprintf((char *)cmdline_addr, options_bytes, "%.*ls", options_bytes - 1, options); - *cmd_line_len = options_bytes; return (char *)cmdline_addr; } @@ -521,6 +428,9 @@ efi_status_t efi_exit_boot_services(void *handle, void *priv, struct efi_boot_memmap *map; efi_status_t status; + if (efi_disable_pci_dma) + efi_pci_disable_bridge_busmaster(); + status = efi_get_memory_map(&map, true); if (status != EFI_SUCCESS) return status; @@ -531,9 +441,6 @@ efi_status_t efi_exit_boot_services(void *handle, void *priv, return status; } - if (efi_disable_pci_dma) - efi_pci_disable_bridge_busmaster(); - status = efi_bs_call(exit_boot_services, handle, map->map_key); if (status == EFI_INVALID_PARAMETER) { @@ -626,8 +533,8 @@ static const struct { /** * efi_load_initrd_dev_path() - load the initrd from the Linux initrd device path - * @load_addr: pointer to store the address where the initrd was loaded - * @load_size: pointer to store the size of the loaded initrd + * @initrd: pointer of struct to store the address where the initrd was loaded + * and the size of the loaded initrd * @max: upper limit for the initrd memory allocation * * Return: @@ -681,8 +588,7 @@ efi_status_t efi_load_initrd_cmdline(efi_loaded_image_t *image, unsigned long soft_limit, unsigned long hard_limit) { - if (!IS_ENABLED(CONFIG_EFI_GENERIC_STUB_INITRD_CMDLINE_LOADER) || - (IS_ENABLED(CONFIG_X86) && (!efi_is_native() || image == NULL))) + if (image == NULL) return EFI_UNSUPPORTED; return handle_cmdline_files(image, L"initrd=", sizeof(L"initrd=") - 2, @@ -695,6 +601,7 @@ efi_status_t efi_load_initrd_cmdline(efi_loaded_image_t *image, * @image: EFI loaded image protocol * @soft_limit: preferred address for loading the initrd * @hard_limit: upper limit address for loading the initrd + * @out: pointer to store the address of the initrd table * * Return: status code */ @@ -713,10 +620,6 @@ efi_status_t efi_load_initrd(efi_loaded_image_t *image, status = efi_load_initrd_dev_path(&initrd, hard_limit); if (status == EFI_SUCCESS) { efi_info("Loaded initrd from LINUX_EFI_INITRD_MEDIA_GUID device path\n"); - if (initrd.size > 0 && - efi_measure_tagged_event(initrd.base, initrd.size, - EFISTUB_EVT_INITRD) == EFI_SUCCESS) - efi_info("Measured initrd data into PCR 9\n"); } else if (status == EFI_NOT_FOUND) { status = efi_load_initrd_cmdline(image, &initrd, soft_limit, hard_limit); @@ -729,6 +632,11 @@ efi_status_t efi_load_initrd(efi_loaded_image_t *image, if (status != EFI_SUCCESS) goto failed; + if (initrd.size > 0 && + efi_measure_tagged_event(initrd.base, initrd.size, + EFISTUB_EVT_INITRD) == EFI_SUCCESS) + efi_info("Measured initrd data into PCR 9\n"); + status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, sizeof(initrd), (void **)&tbl); if (status != EFI_SUCCESS) @@ -795,3 +703,70 @@ efi_status_t efi_wait_for_key(unsigned long usec, efi_input_key_t *key) return status; } + +/** + * efi_remap_image - Remap a loaded image with the appropriate permissions + * for code and data + * + * @image_base: the base of the image in memory + * @alloc_size: the size of the area in memory occupied by the image + * @code_size: the size of the leading part of the image containing code + * and read-only data + * + * efi_remap_image() uses the EFI memory attribute protocol to remap the code + * region of the loaded image read-only/executable, and the remainder + * read-write/non-executable. The code region is assumed to start at the base + * of the image, and will therefore cover the PE/COFF header as well. + */ +void efi_remap_image(unsigned long image_base, unsigned alloc_size, + unsigned long code_size) +{ + efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID; + efi_memory_attribute_protocol_t *memattr; + efi_status_t status; + u64 attr; + + /* + * If the firmware implements the EFI_MEMORY_ATTRIBUTE_PROTOCOL, let's + * invoke it to remap the text/rodata region of the decompressed image + * as read-only and the data/bss region as non-executable. + */ + status = efi_bs_call(locate_protocol, &guid, NULL, (void **)&memattr); + if (status != EFI_SUCCESS) + return; + + // Get the current attributes for the entire region + status = memattr->get_memory_attributes(memattr, image_base, + alloc_size, &attr); + if (status != EFI_SUCCESS) { + efi_warn("Failed to retrieve memory attributes for image region: 0x%lx\n", + status); + return; + } + + // Mark the code region as read-only + status = memattr->set_memory_attributes(memattr, image_base, code_size, + EFI_MEMORY_RO); + if (status != EFI_SUCCESS) { + efi_warn("Failed to remap code region read-only\n"); + return; + } + + // If the entire region was already mapped as non-exec, clear the + // attribute from the code region. Otherwise, set it on the data + // region. + if (attr & EFI_MEMORY_XP) { + status = memattr->clear_memory_attributes(memattr, image_base, + code_size, + EFI_MEMORY_XP); + if (status != EFI_SUCCESS) + efi_warn("Failed to remap code region executable\n"); + } else { + status = memattr->set_memory_attributes(memattr, + image_base + code_size, + alloc_size - code_size, + EFI_MEMORY_XP); + if (status != EFI_SUCCESS) + efi_warn("Failed to remap data region non-executable\n"); + } +} diff --git a/drivers/firmware/efi/libstub/efi-stub.c b/drivers/firmware/efi/libstub/efi-stub.c index cf474f0dd261..9cb814c5ba1b 100644 --- a/drivers/firmware/efi/libstub/efi-stub.c +++ b/drivers/firmware/efi/libstub/efi-stub.c @@ -10,6 +10,7 @@ */ #include <linux/efi.h> +#include <linux/screen_info.h> #include <asm/efi.h> #include "efistub.h" @@ -35,15 +36,6 @@ * as well to minimize the code churn. */ #define EFI_RT_VIRTUAL_BASE SZ_512M -#define EFI_RT_VIRTUAL_SIZE SZ_512M - -#ifdef CONFIG_ARM64 -# define EFI_RT_VIRTUAL_LIMIT DEFAULT_MAP_WINDOW_64 -#elif defined(CONFIG_RISCV) || defined(CONFIG_LOONGARCH) -# define EFI_RT_VIRTUAL_LIMIT TASK_SIZE_MIN -#else /* Only if TASK_SIZE is a constant */ -# define EFI_RT_VIRTUAL_LIMIT TASK_SIZE -#endif /* * Some architectures map the EFI regions into the kernel's linear map using a @@ -56,27 +48,22 @@ static u64 virtmap_base = EFI_RT_VIRTUAL_BASE; static bool flat_va_mapping = (EFI_RT_VIRTUAL_OFFSET != 0); +void __weak free_screen_info(struct screen_info *si) +{ +} + static struct screen_info *setup_graphics(void) { - efi_guid_t gop_proto = EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID; - efi_status_t status; - unsigned long size; - void **gop_handle = NULL; - struct screen_info *si = NULL; + struct screen_info *si, tmp = {}; - size = 0; - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &gop_proto, NULL, &size, gop_handle); - if (status == EFI_BUFFER_TOO_SMALL) { - si = alloc_screen_info(); - if (!si) - return NULL; - status = efi_setup_gop(si, &gop_proto, size); - if (status != EFI_SUCCESS) { - free_screen_info(si); - return NULL; - } - } + if (efi_setup_graphics(&tmp, NULL) != EFI_SUCCESS) + return NULL; + + si = alloc_screen_info(); + if (!si) + return NULL; + + *si = tmp; return si; } @@ -115,96 +102,59 @@ static u32 get_supported_rt_services(void) return supported; } -/* - * EFI entry point for the arm/arm64 EFI stubs. This is the entrypoint - * that is described in the PE/COFF header. Most of the code is the same - * for both archictectures, with the arch-specific code provided in the - * handle_kernel_image() function. - */ -efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg) +efi_status_t efi_handle_cmdline(efi_loaded_image_t *image, char **cmdline_ptr) { - efi_loaded_image_t *image; + char *cmdline __free(efi_pool) = NULL; efi_status_t status; - unsigned long image_addr; - unsigned long image_size = 0; - /* addr/point and size pairs for memory management*/ - char *cmdline_ptr = NULL; - int cmdline_size = 0; - efi_guid_t loaded_image_proto = LOADED_IMAGE_PROTOCOL_GUID; - unsigned long reserve_addr = 0; - unsigned long reserve_size = 0; - struct screen_info *si; - efi_properties_table_t *prop_tbl; - - efi_system_table = sys_table_arg; - - /* Check if we were booted by the EFI firmware */ - if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) { - status = EFI_INVALID_PARAMETER; - goto fail; - } - - status = check_platform_features(); - if (status != EFI_SUCCESS) - goto fail; - - /* - * Get a handle to the loaded image protocol. This is used to get - * information about the running image, such as size and the command - * line. - */ - status = efi_bs_call(handle_protocol, handle, &loaded_image_proto, - (void *)&image); - if (status != EFI_SUCCESS) { - efi_err("Failed to get loaded image protocol\n"); - goto fail; - } /* * Get the command line from EFI, using the LOADED_IMAGE * protocol. We are going to copy the command line into the * device tree, so this can be allocated anywhere. */ - cmdline_ptr = efi_convert_cmdline(image, &cmdline_size); - if (!cmdline_ptr) { + cmdline = efi_convert_cmdline(image); + if (!cmdline) { efi_err("getting command line via LOADED_IMAGE_PROTOCOL\n"); - status = EFI_OUT_OF_RESOURCES; - goto fail; + return EFI_OUT_OF_RESOURCES; } - if (IS_ENABLED(CONFIG_CMDLINE_EXTEND) || - IS_ENABLED(CONFIG_CMDLINE_FORCE) || - cmdline_size == 0) { - status = efi_parse_options(CONFIG_CMDLINE); + if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) { + status = efi_parse_options(cmdline); if (status != EFI_SUCCESS) { - efi_err("Failed to parse options\n"); - goto fail_free_cmdline; + efi_err("Failed to parse EFI load options\n"); + return status; } } - if (!IS_ENABLED(CONFIG_CMDLINE_FORCE) && cmdline_size > 0) { - status = efi_parse_options(cmdline_ptr); + if (IS_ENABLED(CONFIG_CMDLINE_EXTEND) || + IS_ENABLED(CONFIG_CMDLINE_FORCE) || + cmdline[0] == 0) { + status = efi_parse_options(CONFIG_CMDLINE); if (status != EFI_SUCCESS) { - efi_err("Failed to parse options\n"); - goto fail_free_cmdline; + efi_err("Failed to parse built-in command line\n"); + return status; } } - efi_info("Booting Linux Kernel...\n"); + *cmdline_ptr = no_free_ptr(cmdline); + return EFI_SUCCESS; +} - si = setup_graphics(); +efi_status_t efi_stub_common(efi_handle_t handle, + efi_loaded_image_t *image, + unsigned long image_addr, + char *cmdline_ptr) +{ + struct screen_info *si; + efi_status_t status; - status = handle_kernel_image(&image_addr, &image_size, - &reserve_addr, - &reserve_size, - image, handle); - if (status != EFI_SUCCESS) { - efi_err("Failed to relocate kernel\n"); - goto fail_free_screeninfo; - } + status = check_platform_features(); + if (status != EFI_SUCCESS) + return status; + + si = setup_graphics(); - efi_retrieve_tpm2_eventlog(); + efi_retrieve_eventlog(); /* Ask the firmware to clear memory on unclean shutdown */ efi_enable_reset_attack_mitigation(); @@ -214,53 +164,15 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, efi_random_get_seed(); - /* - * If the NX PE data feature is enabled in the properties table, we - * should take care not to create a virtual mapping that changes the - * relative placement of runtime services code and data regions, as - * they may belong to the same PE/COFF executable image in memory. - * The easiest way to achieve that is to simply use a 1:1 mapping. - */ - prop_tbl = get_efi_config_table(EFI_PROPERTIES_TABLE_GUID); - flat_va_mapping |= prop_tbl && - (prop_tbl->memory_protection_attribute & - EFI_PROPERTIES_RUNTIME_MEMORY_PROTECTION_NON_EXECUTABLE_PE_DATA); - /* force efi_novamap if SetVirtualAddressMap() is unsupported */ efi_novamap |= !(get_supported_rt_services() & EFI_RT_SUPPORTED_SET_VIRTUAL_ADDRESS_MAP); - /* hibernation expects the runtime regions to stay in the same place */ - if (!IS_ENABLED(CONFIG_HIBERNATION) && !efi_nokaslr && !flat_va_mapping) { - /* - * Randomize the base of the UEFI runtime services region. - * Preserve the 2 MB alignment of the region by taking a - * shift of 21 bit positions into account when scaling - * the headroom value using a 32-bit random value. - */ - static const u64 headroom = EFI_RT_VIRTUAL_LIMIT - - EFI_RT_VIRTUAL_BASE - - EFI_RT_VIRTUAL_SIZE; - u32 rnd; - - status = efi_get_random_bytes(sizeof(rnd), (u8 *)&rnd); - if (status == EFI_SUCCESS) { - virtmap_base = EFI_RT_VIRTUAL_BASE + - (((headroom >> 21) * rnd) >> (32 - 21)); - } - } - install_memreserve_table(); status = efi_boot_kernel(handle, image, image_addr, cmdline_ptr); - efi_free(image_size, image_addr); - efi_free(reserve_size, reserve_addr); -fail_free_screeninfo: free_screen_info(si); -fail_free_cmdline: - efi_bs_call(free_pool, cmdline_ptr); -fail: return status; } diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index a30fb5d8ef05..b2fb0c3fa721 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -4,6 +4,7 @@ #define _DRIVERS_FIRMWARE_EFI_EFISTUB_H #include <linux/compiler.h> +#include <linux/cleanup.h> #include <linux/efi.h> #include <linux/kernel.h> #include <linux/kern_levels.h> @@ -29,11 +30,19 @@ #define EFI_ALLOC_ALIGN EFI_PAGE_SIZE #endif +#ifndef EFI_ALLOC_LIMIT +#define EFI_ALLOC_LIMIT ULONG_MAX +#endif + +struct edid_info; +struct screen_info; + +extern bool efi_no5lvl; extern bool efi_nochunk; extern bool efi_nokaslr; extern int efi_loglevel; +extern int efi_mem_encrypt; extern bool efi_novamap; - extern const efi_system_table_t *efi_system_table; typedef union efi_dxe_services_table efi_dxe_services_table_t; @@ -44,15 +53,23 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, #ifndef ARCH_HAS_EFISTUB_WRAPPERS -#define efi_is_native() (true) -#define efi_bs_call(func, ...) efi_system_table->boottime->func(__VA_ARGS__) -#define efi_rt_call(func, ...) efi_system_table->runtime->func(__VA_ARGS__) -#define efi_dxe_call(func, ...) efi_dxe_table->func(__VA_ARGS__) -#define efi_table_attr(inst, attr) (inst->attr) -#define efi_call_proto(inst, func, ...) inst->func(inst, ##__VA_ARGS__) +#define efi_is_native() (true) +#define efi_table_attr(inst, attr) (inst)->attr +#define efi_fn_call(inst, func, ...) (inst)->func(__VA_ARGS__) #endif +#define efi_call_proto(inst, func, ...) ({ \ + __typeof__(inst) __inst = (inst); \ + efi_fn_call(__inst, func, __inst, ##__VA_ARGS__); \ +}) +#define efi_bs_call(func, ...) \ + efi_fn_call(efi_table_attr(efi_system_table, boottime), func, ##__VA_ARGS__) +#define efi_rt_call(func, ...) \ + efi_fn_call(efi_table_attr(efi_system_table, runtime), func, ##__VA_ARGS__) +#define efi_dxe_call(func, ...) \ + efi_fn_call(efi_dxe_table, func, ##__VA_ARGS__) + #define efi_info(fmt, ...) \ efi_printk(KERN_INFO fmt, ##__VA_ARGS__) #define efi_warn(fmt, ...) \ @@ -109,11 +126,10 @@ efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, #define efi_get_handle_num(size) \ ((size) / (efi_is_native() ? sizeof(efi_handle_t) : sizeof(u32))) -#define for_each_efi_handle(handle, array, size, i) \ - for (i = 0; \ - i < efi_get_handle_num(size) && \ - ((handle = efi_get_handle_at((array), i)) || true); \ - i++) +#define for_each_efi_handle(handle, array, num) \ + for (int __i = 0; __i < (num) && \ + ((handle = efi_get_handle_at((array), __i)) || true); \ + __i++) static inline void efi_set_u64_split(u64 data, u32 *lo, u32 *hi) @@ -158,7 +174,7 @@ void efi_set_u64_split(u64 data, u32 *lo, u32 *hi) * the EFI memory map. Other related structures, e.g. x86 e820ext, need * to factor in this headroom requirement as well. */ -#define EFI_MMAP_NR_SLACK_SLOTS 8 +#define EFI_MMAP_NR_SLACK_SLOTS 32 typedef struct efi_generic_dev_path efi_device_path_protocol_t; @@ -179,6 +195,21 @@ union efi_device_path_to_text_protocol { typedef union efi_device_path_to_text_protocol efi_device_path_to_text_protocol_t; +union efi_device_path_from_text_protocol { + struct { + efi_device_path_protocol_t * + (__efiapi *convert_text_to_device_node)(const efi_char16_t *); + efi_device_path_protocol_t * + (__efiapi *convert_text_to_device_path)(const efi_char16_t *); + }; + struct { + u32 convert_text_to_device_node; + u32 convert_text_to_device_path; + } mixed_mode; +}; + +typedef union efi_device_path_from_text_protocol efi_device_path_from_text_protocol_t; + typedef void *efi_event_t; /* Note that notifications won't work in mixed mode */ typedef void (__efiapi *efi_event_notify_t)(efi_event_t, void *); @@ -286,7 +317,9 @@ union efi_boot_services { void *close_protocol; void *open_protocol_information; void *protocols_per_handle; - void *locate_handle_buffer; + efi_status_t (__efiapi *locate_handle_buffer)(int, efi_guid_t *, + void *, unsigned long *, + efi_handle_t **); efi_status_t (__efiapi *locate_protocol)(efi_guid_t *, void *, void **); efi_status_t (__efiapi *install_multiple_protocol_interfaces)(efi_handle_t *, ...); @@ -415,6 +448,26 @@ union efi_dxe_services_table { } mixed_mode; }; +typedef union efi_memory_attribute_protocol efi_memory_attribute_protocol_t; + +union efi_memory_attribute_protocol { + struct { + efi_status_t (__efiapi *get_memory_attributes)( + efi_memory_attribute_protocol_t *, efi_physical_addr_t, u64, u64 *); + + efi_status_t (__efiapi *set_memory_attributes)( + efi_memory_attribute_protocol_t *, efi_physical_addr_t, u64, u64); + + efi_status_t (__efiapi *clear_memory_attributes)( + efi_memory_attribute_protocol_t *, efi_physical_addr_t, u64, u64); + }; + struct { + u32 get_memory_attributes; + u32 set_memory_attributes; + u32 clear_memory_attributes; + } mixed_mode; +}; + typedef union efi_uga_draw_protocol efi_uga_draw_protocol_t; union efi_uga_draw_protocol { @@ -528,6 +581,32 @@ union efi_graphics_output_protocol { } mixed_mode; }; +typedef union efi_edid_discovered_protocol efi_edid_discovered_protocol_t; + +union efi_edid_discovered_protocol { + struct { + u32 size_of_edid; + u8 *edid; + }; + struct { + u32 size_of_edid; + u32 edid; + } mixed_mode; +}; + +typedef union efi_edid_active_protocol efi_edid_active_protocol_t; + +union efi_edid_active_protocol { + struct { + u32 size_of_edid; + u8 *edid; + }; + struct { + u32 size_of_edid; + u32 edid; + } mixed_mode; +}; + typedef union { struct { u32 revision; @@ -572,36 +651,63 @@ typedef struct { efi_char16_t filename[]; } efi_file_info_t; -typedef struct efi_file_protocol efi_file_protocol_t; - -struct efi_file_protocol { - u64 revision; - efi_status_t (__efiapi *open) (efi_file_protocol_t *, - efi_file_protocol_t **, - efi_char16_t *, u64, u64); - efi_status_t (__efiapi *close) (efi_file_protocol_t *); - efi_status_t (__efiapi *delete) (efi_file_protocol_t *); - efi_status_t (__efiapi *read) (efi_file_protocol_t *, - unsigned long *, void *); - efi_status_t (__efiapi *write) (efi_file_protocol_t *, - unsigned long, void *); - efi_status_t (__efiapi *get_position)(efi_file_protocol_t *, u64 *); - efi_status_t (__efiapi *set_position)(efi_file_protocol_t *, u64); - efi_status_t (__efiapi *get_info) (efi_file_protocol_t *, - efi_guid_t *, unsigned long *, - void *); - efi_status_t (__efiapi *set_info) (efi_file_protocol_t *, - efi_guid_t *, unsigned long, - void *); - efi_status_t (__efiapi *flush) (efi_file_protocol_t *); +typedef union efi_file_protocol efi_file_protocol_t; + +union efi_file_protocol { + struct { + u64 revision; + efi_status_t (__efiapi *open) (efi_file_protocol_t *, + efi_file_protocol_t **, + efi_char16_t *, u64, + u64); + efi_status_t (__efiapi *close) (efi_file_protocol_t *); + efi_status_t (__efiapi *delete) (efi_file_protocol_t *); + efi_status_t (__efiapi *read) (efi_file_protocol_t *, + unsigned long *, + void *); + efi_status_t (__efiapi *write) (efi_file_protocol_t *, + unsigned long, void *); + efi_status_t (__efiapi *get_position)(efi_file_protocol_t *, + u64 *); + efi_status_t (__efiapi *set_position)(efi_file_protocol_t *, + u64); + efi_status_t (__efiapi *get_info) (efi_file_protocol_t *, + efi_guid_t *, + unsigned long *, + void *); + efi_status_t (__efiapi *set_info) (efi_file_protocol_t *, + efi_guid_t *, + unsigned long, + void *); + efi_status_t (__efiapi *flush) (efi_file_protocol_t *); + }; + struct { + u64 revision; + u32 open; + u32 close; + u32 delete; + u32 read; + u32 write; + u32 get_position; + u32 set_position; + u32 get_info; + u32 set_info; + u32 flush; + } mixed_mode; }; -typedef struct efi_simple_file_system_protocol efi_simple_file_system_protocol_t; +typedef union efi_simple_file_system_protocol efi_simple_file_system_protocol_t; -struct efi_simple_file_system_protocol { - u64 revision; - int (__efiapi *open_volume)(efi_simple_file_system_protocol_t *, - efi_file_protocol_t **); +union efi_simple_file_system_protocol { + struct { + u64 revision; + efi_status_t (__efiapi *open_volume)(efi_simple_file_system_protocol_t *, + efi_file_protocol_t **); + }; + struct { + u64 revision; + u32 open_volume; + } mixed_mode; }; #define EFI_FILE_MODE_READ 0x0000000000000001 @@ -768,14 +874,14 @@ struct efi_tcg2_event { /* u8[] event follows here */ } __packed; -struct efi_tcg2_tagged_event { - u32 tagged_event_id; - u32 tagged_event_data_size; - /* u8 tagged event data follows here */ -} __packed; +/* from TCG PC Client Platform Firmware Profile Specification */ +typedef struct tdTCG_PCClientTaggedEvent { + u32 tagged_event_id; + u32 tagged_event_data_size; + u8 tagged_event_data[]; +} TCG_PCClientTaggedEvent; typedef struct efi_tcg2_event efi_tcg2_event_t; -typedef struct efi_tcg2_tagged_event efi_tcg2_tagged_event_t; typedef union efi_tcg2_protocol efi_tcg2_protocol_t; union efi_tcg2_protocol { @@ -807,6 +913,87 @@ union efi_tcg2_protocol { } mixed_mode; }; +typedef struct { + u8 major; + u8 minor; +} efi_cc_version_t; + +typedef struct { + u8 type; + u8 sub_type; +} efi_cc_type_t; + +/* EFI CC type/subtype defines */ +#define EFI_CC_TYPE_NONE 0 +#define EFI_CC_TYPE_AMD_SEV 1 +#define EFI_CC_TYPE_INTEL_TDX 2 + +typedef u32 efi_cc_mr_index_t; + +struct efi_cc_event { + u32 event_size; + struct { + u32 header_size; + u16 header_version; + u32 mr_index; + u32 event_type; + } __packed event_header; + /* u8[] event follows here */ +} __packed; + +typedef struct efi_cc_event efi_cc_event_t; + +typedef u32 efi_cc_event_log_bitmap_t; +typedef u32 efi_cc_event_log_format_t; +typedef u32 efi_cc_event_algorithm_bitmap_t; + +typedef struct { + u8 size; + efi_cc_version_t structure_version; + efi_cc_version_t protocol_version; + efi_cc_event_algorithm_bitmap_t hash_algorithm_bitmap; + efi_cc_event_log_bitmap_t supported_event_logs; + efi_cc_type_t cc_type; +} efi_cc_boot_service_cap_t; + +#define EFI_CC_EVENT_HEADER_VERSION 1 + +#define EFI_CC_BOOT_HASH_ALG_SHA384 0x00000004 + +#define EFI_CC_EVENT_LOG_FORMAT_TCG_2 0x00000002 + +typedef union efi_cc_protocol efi_cc_protocol_t; + +union efi_cc_protocol { + struct { + efi_status_t + (__efiapi *get_capability)(efi_cc_protocol_t *, + efi_cc_boot_service_cap_t *); + + efi_status_t + (__efiapi *get_event_log)(efi_cc_protocol_t *, + efi_cc_event_log_format_t, + efi_physical_addr_t *, + efi_physical_addr_t *, + efi_bool_t *); + + efi_status_t + (__efiapi *hash_log_extend_event)(efi_cc_protocol_t *, u64, + efi_physical_addr_t, u64, + const efi_cc_event_t *); + + efi_status_t + (__efiapi *map_pcr_to_mr_index)(efi_cc_protocol_t *, u32, + efi_cc_mr_index_t *); + }; + struct { + u32 get_capability; + u32 get_event_log; + u32 hash_log_extend_event; + u32 map_pcr_to_mr_index; + } mixed_mode; +}; + struct riscv_efi_boot_protocol { u64 revision; @@ -880,7 +1067,11 @@ void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size, efi_status_t efi_get_random_bytes(unsigned long size, u8 *out); efi_status_t efi_random_alloc(unsigned long size, unsigned long align, - unsigned long *addr, unsigned long random_seed); + unsigned long *addr, unsigned long random_seed, + int memory_type, unsigned long alloc_min, + unsigned long alloc_max); + +efi_status_t efi_random_get_seed(void); efi_status_t check_platform_features(void); @@ -893,10 +1084,11 @@ void efi_puts(const char *str); __printf(1, 2) int efi_printk(char const *fmt, ...); void efi_free(unsigned long size, unsigned long addr); +DEFINE_FREE(efi_pool, void *, if (_T) efi_bs_call(free_pool, _T)); void efi_apply_loadoptions_quirk(const void **load_options, u32 *load_options_size); -char *efi_convert_cmdline(efi_loaded_image_t *image, int *cmd_line_len); +char *efi_convert_cmdline(efi_loaded_image_t *image); efi_status_t efi_get_memory_map(struct efi_boot_memmap **map, bool install_cfg_tbl); @@ -905,7 +1097,8 @@ efi_status_t efi_allocate_pages(unsigned long size, unsigned long *addr, unsigned long max); efi_status_t efi_allocate_pages_aligned(unsigned long size, unsigned long *addr, - unsigned long max, unsigned long align); + unsigned long max, unsigned long align, + int memory_type); efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, unsigned long *addr, unsigned long min); @@ -921,8 +1114,7 @@ efi_status_t efi_parse_options(char const *cmdline); void efi_parse_option_graphics(char *option); -efi_status_t efi_setup_gop(struct screen_info *si, efi_guid_t *proto, - unsigned long size); +efi_status_t efi_setup_graphics(struct screen_info *si, struct edid_info *edid); efi_status_t handle_cmdline_files(efi_loaded_image_t *image, const efi_char16_t *optstr, @@ -958,6 +1150,14 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, efi_loaded_image_t *image, efi_handle_t image_handle); +/* shared entrypoint between the normal stub and the zboot stub */ +efi_status_t efi_stub_common(efi_handle_t handle, + efi_loaded_image_t *image, + unsigned long image_addr, + char *cmdline_ptr); + +efi_status_t efi_handle_cmdline(efi_loaded_image_t *image, char **cmdline_ptr); + asmlinkage void __noreturn efi_enter_kernel(unsigned long entrypoint, unsigned long fdt_addr, unsigned long fdt_size); @@ -973,6 +1173,97 @@ static inline void efi_enable_reset_attack_mitigation(void) { } #endif -void efi_retrieve_tpm2_eventlog(void); +void efi_retrieve_eventlog(void); + +struct screen_info *alloc_screen_info(void); +struct screen_info *__alloc_screen_info(void); +void free_screen_info(struct screen_info *si); + +void efi_cache_sync_image(unsigned long image_base, + unsigned long alloc_size); + +struct efi_smbios_record { + u8 type; + u8 length; + u16 handle; +}; + +const struct efi_smbios_record *efi_get_smbios_record(u8 type); + +struct efi_smbios_type1_record { + struct efi_smbios_record header; + + u8 manufacturer; + u8 product_name; + u8 version; + u8 serial_number; + efi_guid_t uuid; + u8 wakeup_type; + u8 sku_number; + u8 family; +}; + +struct efi_smbios_type4_record { + struct efi_smbios_record header; + + u8 socket; + u8 processor_type; + u8 processor_family; + u8 processor_manufacturer; + u8 processor_id[8]; + u8 processor_version; + u8 voltage; + u16 external_clock; + u16 max_speed; + u16 current_speed; + u8 status; + u8 processor_upgrade; + u16 l1_cache_handle; + u16 l2_cache_handle; + u16 l3_cache_handle; + u8 serial_number; + u8 asset_tag; + u8 part_number; + u8 core_count; + u8 enabled_core_count; + u8 thread_count; + u16 processor_characteristics; + u16 processor_family2; + u16 core_count2; + u16 enabled_core_count2; + u16 thread_count2; + u16 thread_enabled; +}; + +#define efi_get_smbios_string(__record, __field) ({ \ + __typeof__(__record) __rec = __record; \ + __efi_get_smbios_string(&__rec->header, &__rec->__field); \ +}) + +const u8 *__efi_get_smbios_string(const struct efi_smbios_record *record, + const u8 *offset); + +void efi_remap_image(unsigned long image_base, unsigned alloc_size, + unsigned long code_size); +efi_status_t efi_kaslr_relocate_kernel(unsigned long *image_addr, + unsigned long *reserve_addr, + unsigned long *reserve_size, + unsigned long kernel_size, + unsigned long kernel_codesize, + unsigned long kernel_memsize, + u32 phys_seed); +u32 efi_kaslr_get_phys_seed(efi_handle_t image_handle); + +asmlinkage efi_status_t __efiapi +efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab); + +efi_status_t allocate_unaccepted_bitmap(__u32 nr_desc, + struct efi_boot_memmap *map); +void process_unaccepted_memory(u64 start, u64 end); +void accept_memory(phys_addr_t start, unsigned long size); +void arch_accept_memory(phys_addr_t start, phys_addr_t end); + +efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size); +efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen); #endif diff --git a/drivers/firmware/efi/libstub/fdt.c b/drivers/firmware/efi/libstub/fdt.c index 70e9789ff9de..6a337f1f8787 100644 --- a/drivers/firmware/efi/libstub/fdt.c +++ b/drivers/firmware/efi/libstub/fdt.c @@ -335,8 +335,8 @@ fail_free_new_fdt: fail: efi_free(fdt_size, fdt_addr); - - efi_bs_call(free_pool, priv.runtime_map); + if (!efi_novamap) + efi_bs_call(free_pool, priv.runtime_map); return EFI_LOAD_ERROR; } diff --git a/drivers/firmware/efi/libstub/file.c b/drivers/firmware/efi/libstub/file.c index f756c61396e9..bd626d55dcbc 100644 --- a/drivers/firmware/efi/libstub/file.c +++ b/drivers/firmware/efi/libstub/file.c @@ -43,18 +43,26 @@ static efi_status_t efi_open_file(efi_file_protocol_t *volume, efi_file_protocol_t *fh; unsigned long info_sz; efi_status_t status; + efi_char16_t *c; - status = volume->open(volume, &fh, fi->filename, EFI_FILE_MODE_READ, 0); + /* Replace UNIX dir separators with EFI standard ones */ + for (c = fi->filename; *c != L'\0'; c++) { + if (*c == L'/') + *c = L'\\'; + } + + status = efi_call_proto(volume, open, &fh, fi->filename, + EFI_FILE_MODE_READ, 0); if (status != EFI_SUCCESS) { efi_err("Failed to open file: %ls\n", fi->filename); return status; } info_sz = sizeof(struct finfo); - status = fh->get_info(fh, &info_guid, &info_sz, fi); + status = efi_call_proto(fh, get_info, &info_guid, &info_sz, fi); if (status != EFI_SUCCESS) { efi_err("Failed to get file info\n"); - fh->close(fh); + efi_call_proto(fh, close); return status; } @@ -66,36 +74,18 @@ static efi_status_t efi_open_file(efi_file_protocol_t *volume, static efi_status_t efi_open_volume(efi_loaded_image_t *image, efi_file_protocol_t **fh) { - struct efi_vendor_dev_path *dp = image->file_path; - efi_guid_t li_proto = LOADED_IMAGE_PROTOCOL_GUID; efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID; efi_simple_file_system_protocol_t *io; efi_status_t status; - // If we are using EFI zboot, we should look for the file system - // protocol on the parent image's handle instead - if (IS_ENABLED(CONFIG_EFI_ZBOOT) && - image->parent_handle != NULL && - dp != NULL && - dp->header.type == EFI_DEV_MEDIA && - dp->header.sub_type == EFI_DEV_MEDIA_VENDOR && - !efi_guidcmp(dp->vendorguid, LINUX_EFI_ZBOOT_MEDIA_GUID)) { - status = efi_bs_call(handle_protocol, image->parent_handle, - &li_proto, (void *)&image); - if (status != EFI_SUCCESS) { - efi_err("Failed to locate parent image handle\n"); - return status; - } - } - - status = efi_bs_call(handle_protocol, image->device_handle, &fs_proto, - (void **)&io); + status = efi_bs_call(handle_protocol, efi_table_attr(image, device_handle), + &fs_proto, (void **)&io); if (status != EFI_SUCCESS) { efi_err("Failed to handle fs_proto\n"); return status; } - status = io->open_volume(io, fh); + status = efi_call_proto(io, open_volume, fh); if (status != EFI_SUCCESS) efi_err("Failed to open volume\n"); @@ -129,16 +119,68 @@ static int find_file_option(const efi_char16_t *cmdline, int cmdline_len, if (c == L'\0' || c == L'\n' || c == L' ') break; - else if (c == L'/') - /* Replace UNIX dir separators with EFI standard ones */ - *result++ = L'\\'; - else - *result++ = c; + *result++ = c; } *result = L'\0'; return i; } +static efi_status_t efi_open_device_path(efi_file_protocol_t **volume, + struct finfo *fi) +{ + efi_guid_t text_to_dp_guid = EFI_DEVICE_PATH_FROM_TEXT_PROTOCOL_GUID; + static efi_device_path_from_text_protocol_t *text_to_dp = NULL; + efi_guid_t fs_proto = EFI_FILE_SYSTEM_GUID; + efi_device_path_protocol_t *initrd_dp; + efi_simple_file_system_protocol_t *io; + struct efi_file_path_dev_path *fpath; + efi_handle_t handle; + efi_status_t status; + + /* See if the text to device path protocol exists */ + if (!text_to_dp && + efi_bs_call(locate_protocol, &text_to_dp_guid, NULL, + (void **)&text_to_dp) != EFI_SUCCESS) + return EFI_UNSUPPORTED; + + + /* Convert the filename wide string into a device path */ + initrd_dp = efi_fn_call(text_to_dp, convert_text_to_device_path, + fi->filename); + + /* Check whether the device path in question implements simple FS */ + if ((efi_bs_call(locate_device_path, &fs_proto, &initrd_dp, &handle) ?: + efi_bs_call(handle_protocol, handle, &fs_proto, (void **)&io)) + != EFI_SUCCESS) + return EFI_NOT_FOUND; + + /* Check whether the remaining device path is a file device path */ + if (initrd_dp->type != EFI_DEV_MEDIA || + initrd_dp->sub_type != EFI_DEV_MEDIA_FILE) { + efi_warn("Unexpected device path node type: (%x, %x)\n", + initrd_dp->type, initrd_dp->sub_type); + return EFI_LOAD_ERROR; + } + + /* Copy the remaining file path into the fi structure */ + fpath = (struct efi_file_path_dev_path *)initrd_dp; + memcpy(fi->filename, fpath->filename, + min(sizeof(fi->filename), + fpath->header.length - sizeof(fpath->header))); + + status = efi_call_proto(io, open_volume, volume); + if (status != EFI_SUCCESS) + efi_err("Failed to open volume\n"); + + return status; +} + +#ifndef CONFIG_CMDLINE +#define CONFIG_CMDLINE +#endif + +static const efi_char16_t builtin_cmdline[] = L"" CONFIG_CMDLINE; + /* * Check the cmdline for a LILO-style file= arguments. * @@ -153,14 +195,17 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, unsigned long *load_addr, unsigned long *load_size) { - const efi_char16_t *cmdline = image->load_options; - u32 cmdline_len = image->load_options_size; + const bool ignore_load_options = IS_ENABLED(CONFIG_CMDLINE_OVERRIDE) || + IS_ENABLED(CONFIG_CMDLINE_FORCE); + const efi_char16_t *cmdline = efi_table_attr(image, load_options); + u32 cmdline_len = efi_table_attr(image, load_options_size); unsigned long efi_chunk_size = ULONG_MAX; efi_file_protocol_t *volume = NULL; efi_file_protocol_t *file; unsigned long alloc_addr; unsigned long alloc_size; efi_status_t status; + bool twopass; int offset; if (!load_addr || !load_size) @@ -173,6 +218,16 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, efi_chunk_size = EFI_READ_CHUNK_SIZE; alloc_addr = alloc_size = 0; + + if (!ignore_load_options && cmdline_len > 0) { + twopass = IS_ENABLED(CONFIG_CMDLINE_BOOL) || + IS_ENABLED(CONFIG_CMDLINE_EXTEND); + } else { +do_builtin: cmdline = builtin_cmdline; + cmdline_len = ARRAY_SIZE(builtin_cmdline) - 1; + twopass = false; + } + do { struct finfo fi; unsigned long size; @@ -188,11 +243,13 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, cmdline += offset; cmdline_len -= offset; - if (!volume) { + status = efi_open_device_path(&volume, &fi); + if (status == EFI_UNSUPPORTED || status == EFI_NOT_FOUND) + /* try the volume that holds the kernel itself */ status = efi_open_volume(image, &volume); - if (status != EFI_SUCCESS) - return status; - } + + if (status != EFI_SUCCESS) + goto err_free_alloc; status = efi_open_file(volume, &fi, &file, &size); if (status != EFI_SUCCESS) @@ -240,7 +297,7 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, while (size) { unsigned long chunksize = min(size, efi_chunk_size); - status = file->read(file, &chunksize, addr); + status = efi_call_proto(file, read, &chunksize, addr); if (status != EFI_SUCCESS) { efi_err("Failed to read file\n"); goto err_close_file; @@ -248,24 +305,27 @@ efi_status_t handle_cmdline_files(efi_loaded_image_t *image, addr += chunksize; size -= chunksize; } - file->close(file); + efi_call_proto(file, close); + efi_call_proto(volume, close); } while (offset > 0); + if (twopass) + goto do_builtin; + *load_addr = alloc_addr; *load_size = alloc_size; - if (volume) - volume->close(volume); - if (*load_size == 0) return EFI_NOT_READY; return EFI_SUCCESS; err_close_file: - file->close(file); + efi_call_proto(file, close); err_close_volume: - volume->close(volume); + efi_call_proto(volume, close); + +err_free_alloc: efi_free(alloc_size, alloc_addr); return status; } diff --git a/drivers/firmware/efi/libstub/find.c b/drivers/firmware/efi/libstub/find.c new file mode 100644 index 000000000000..4e7740d28987 --- /dev/null +++ b/drivers/firmware/efi/libstub/find.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/bitmap.h> +#include <linux/math.h> +#include <linux/minmax.h> + +/* + * Common helper for find_next_bit() function family + * @FETCH: The expression that fetches and pre-processes each word of bitmap(s) + * @MUNGE: The expression that post-processes a word containing found bit (may be empty) + * @size: The bitmap size in bits + * @start: The bitnumber to start searching at + */ +#define FIND_NEXT_BIT(FETCH, MUNGE, size, start) \ +({ \ + unsigned long mask, idx, tmp, sz = (size), __start = (start); \ + \ + if (unlikely(__start >= sz)) \ + goto out; \ + \ + mask = MUNGE(BITMAP_FIRST_WORD_MASK(__start)); \ + idx = __start / BITS_PER_LONG; \ + \ + for (tmp = (FETCH) & mask; !tmp; tmp = (FETCH)) { \ + if ((idx + 1) * BITS_PER_LONG >= sz) \ + goto out; \ + idx++; \ + } \ + \ + sz = min(idx * BITS_PER_LONG + __ffs(MUNGE(tmp)), sz); \ +out: \ + sz; \ +}) + +unsigned long _find_next_bit(const unsigned long *addr, unsigned long nbits, unsigned long start) +{ + return FIND_NEXT_BIT(addr[idx], /* nop */, nbits, start); +} + +unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits, + unsigned long start) +{ + return FIND_NEXT_BIT(~addr[idx], /* nop */, nbits, start); +} diff --git a/drivers/firmware/efi/libstub/gop.c b/drivers/firmware/efi/libstub/gop.c index ea5da307d542..72d74436a7a4 100644 --- a/drivers/firmware/efi/libstub/gop.c +++ b/drivers/firmware/efi/libstub/gop.c @@ -12,6 +12,7 @@ #include <linux/string.h> #include <asm/efi.h> #include <asm/setup.h> +#include <video/edid.h> #include "efistub.h" @@ -133,13 +134,11 @@ void efi_parse_option_graphics(char *option) static u32 choose_mode_modenum(efi_graphics_output_protocol_t *gop) { - efi_status_t status; - + efi_graphics_output_mode_info_t *info __free(efi_pool) = NULL; efi_graphics_output_protocol_mode_t *mode; - efi_graphics_output_mode_info_t *info; unsigned long info_size; - u32 max_mode, cur_mode; + efi_status_t status; int pf; mode = efi_table_attr(gop, mode); @@ -154,17 +153,13 @@ static u32 choose_mode_modenum(efi_graphics_output_protocol_t *gop) return cur_mode; } - status = efi_call_proto(gop, query_mode, cmdline.mode, - &info_size, &info); + status = efi_call_proto(gop, query_mode, cmdline.mode, &info_size, &info); if (status != EFI_SUCCESS) { efi_err("Couldn't get mode information\n"); return cur_mode; } pf = info->pixel_format; - - efi_bs_call(free_pool, info); - if (pf == PIXEL_BLT_ONLY || pf >= PIXEL_FORMAT_MAX) { efi_err("Invalid PixelFormat\n"); return cur_mode; @@ -173,6 +168,28 @@ static u32 choose_mode_modenum(efi_graphics_output_protocol_t *gop) return cmdline.mode; } +static u32 choose_mode(efi_graphics_output_protocol_t *gop, + bool (*match)(const efi_graphics_output_mode_info_t *, u32, void *), + void *ctx) +{ + efi_graphics_output_protocol_mode_t *mode = efi_table_attr(gop, mode); + u32 max_mode = efi_table_attr(mode, max_mode); + + for (u32 m = 0; m < max_mode; m++) { + efi_graphics_output_mode_info_t *info __free(efi_pool) = NULL; + unsigned long info_size; + efi_status_t status; + + status = efi_call_proto(gop, query_mode, m, &info_size, &info); + if (status != EFI_SUCCESS) + continue; + + if (match(info, m, ctx)) + return m; + } + return (unsigned long)ctx; +} + static u8 pixel_bpp(int pixel_format, efi_pixel_bitmask_t pixel_info) { if (pixel_format == PIXEL_BIT_MASK) { @@ -185,192 +202,117 @@ static u8 pixel_bpp(int pixel_format, efi_pixel_bitmask_t pixel_info) return 32; } -static u32 choose_mode_res(efi_graphics_output_protocol_t *gop) +static bool match_res(const efi_graphics_output_mode_info_t *info, u32 mode, void *ctx) { - efi_status_t status; + efi_pixel_bitmask_t pi = info->pixel_information; + int pf = info->pixel_format; - efi_graphics_output_protocol_mode_t *mode; - efi_graphics_output_mode_info_t *info; - unsigned long info_size; - - u32 max_mode, cur_mode; - int pf; - efi_pixel_bitmask_t pi; - u32 m, w, h; + if (pf == PIXEL_BLT_ONLY || pf >= PIXEL_FORMAT_MAX) + return false; - mode = efi_table_attr(gop, mode); + return cmdline.res.width == info->horizontal_resolution && + cmdline.res.height == info->vertical_resolution && + (cmdline.res.format < 0 || cmdline.res.format == pf) && + (!cmdline.res.depth || cmdline.res.depth == pixel_bpp(pf, pi)); +} - cur_mode = efi_table_attr(mode, mode); - info = efi_table_attr(mode, info); - pf = info->pixel_format; - pi = info->pixel_information; - w = info->horizontal_resolution; - h = info->vertical_resolution; +static u32 choose_mode_res(efi_graphics_output_protocol_t *gop) +{ + efi_graphics_output_protocol_mode_t *mode = efi_table_attr(gop, mode); + unsigned long cur_mode = efi_table_attr(mode, mode); - if (w == cmdline.res.width && h == cmdline.res.height && - (cmdline.res.format < 0 || cmdline.res.format == pf) && - (!cmdline.res.depth || cmdline.res.depth == pixel_bpp(pf, pi))) + if (match_res(efi_table_attr(mode, info), cur_mode, NULL)) return cur_mode; - max_mode = efi_table_attr(mode, max_mode); - - for (m = 0; m < max_mode; m++) { - if (m == cur_mode) - continue; - - status = efi_call_proto(gop, query_mode, m, - &info_size, &info); - if (status != EFI_SUCCESS) - continue; + return choose_mode(gop, match_res, (void *)cur_mode); +} - pf = info->pixel_format; - pi = info->pixel_information; - w = info->horizontal_resolution; - h = info->vertical_resolution; +struct match { + u32 mode; + u32 area; + u8 depth; +}; - efi_bs_call(free_pool, info); +static bool match_auto(const efi_graphics_output_mode_info_t *info, u32 mode, void *ctx) +{ + u32 area = info->horizontal_resolution * info->vertical_resolution; + efi_pixel_bitmask_t pi = info->pixel_information; + int pf = info->pixel_format; + u8 depth = pixel_bpp(pf, pi); + struct match *m = ctx; - if (pf == PIXEL_BLT_ONLY || pf >= PIXEL_FORMAT_MAX) - continue; - if (w == cmdline.res.width && h == cmdline.res.height && - (cmdline.res.format < 0 || cmdline.res.format == pf) && - (!cmdline.res.depth || cmdline.res.depth == pixel_bpp(pf, pi))) - return m; - } + if (pf == PIXEL_BLT_ONLY || pf >= PIXEL_FORMAT_MAX) + return false; - efi_err("Couldn't find requested mode\n"); + if (area > m->area || (area == m->area && depth > m->depth)) + *m = (struct match){ mode, area, depth }; - return cur_mode; + return false; } static u32 choose_mode_auto(efi_graphics_output_protocol_t *gop) { - efi_status_t status; - - efi_graphics_output_protocol_mode_t *mode; - efi_graphics_output_mode_info_t *info; - unsigned long info_size; - - u32 max_mode, cur_mode, best_mode, area; - u8 depth; - int pf; - efi_pixel_bitmask_t pi; - u32 m, w, h, a; - u8 d; - - mode = efi_table_attr(gop, mode); - - cur_mode = efi_table_attr(mode, mode); - max_mode = efi_table_attr(mode, max_mode); - - info = efi_table_attr(mode, info); - - pf = info->pixel_format; - pi = info->pixel_information; - w = info->horizontal_resolution; - h = info->vertical_resolution; - - best_mode = cur_mode; - area = w * h; - depth = pixel_bpp(pf, pi); + struct match match = {}; - for (m = 0; m < max_mode; m++) { - if (m == cur_mode) - continue; + choose_mode(gop, match_auto, &match); - status = efi_call_proto(gop, query_mode, m, - &info_size, &info); - if (status != EFI_SUCCESS) - continue; + return match.mode; +} - pf = info->pixel_format; - pi = info->pixel_information; - w = info->horizontal_resolution; - h = info->vertical_resolution; +static bool match_list(const efi_graphics_output_mode_info_t *info, u32 mode, void *ctx) +{ + efi_pixel_bitmask_t pi = info->pixel_information; + u32 cur_mode = (unsigned long)ctx; + int pf = info->pixel_format; + const char *dstr; + u8 depth = 0; + bool valid; - efi_bs_call(free_pool, info); + valid = !(pf == PIXEL_BLT_ONLY || pf >= PIXEL_FORMAT_MAX); - if (pf == PIXEL_BLT_ONLY || pf >= PIXEL_FORMAT_MAX) - continue; - a = w * h; - if (a < area) - continue; - d = pixel_bpp(pf, pi); - if (a > area || d > depth) { - best_mode = m; - area = a; - depth = d; - } + switch (pf) { + case PIXEL_RGB_RESERVED_8BIT_PER_COLOR: + dstr = "rgb"; + break; + case PIXEL_BGR_RESERVED_8BIT_PER_COLOR: + dstr = "bgr"; + break; + case PIXEL_BIT_MASK: + dstr = ""; + depth = pixel_bpp(pf, pi); + break; + case PIXEL_BLT_ONLY: + dstr = "blt"; + break; + default: + dstr = "xxx"; + break; } - return best_mode; + efi_printk("Mode %3u %c%c: Resolution %ux%u-%s%.0hhu\n", + mode, + (mode == cur_mode) ? '*' : ' ', + !valid ? '-' : ' ', + info->horizontal_resolution, + info->vertical_resolution, + dstr, depth); + + return false; } static u32 choose_mode_list(efi_graphics_output_protocol_t *gop) { - efi_status_t status; - - efi_graphics_output_protocol_mode_t *mode; - efi_graphics_output_mode_info_t *info; - unsigned long info_size; - - u32 max_mode, cur_mode; - int pf; - efi_pixel_bitmask_t pi; - u32 m, w, h; - u8 d; - const char *dstr; - bool valid; + efi_graphics_output_protocol_mode_t *mode = efi_table_attr(gop, mode); + unsigned long cur_mode = efi_table_attr(mode, mode); + u32 max_mode = efi_table_attr(mode, max_mode); efi_input_key_t key; - - mode = efi_table_attr(gop, mode); - - cur_mode = efi_table_attr(mode, mode); - max_mode = efi_table_attr(mode, max_mode); + efi_status_t status; efi_printk("Available graphics modes are 0-%u\n", max_mode-1); efi_puts(" * = current mode\n" " - = unusable mode\n"); - for (m = 0; m < max_mode; m++) { - status = efi_call_proto(gop, query_mode, m, - &info_size, &info); - if (status != EFI_SUCCESS) - continue; - pf = info->pixel_format; - pi = info->pixel_information; - w = info->horizontal_resolution; - h = info->vertical_resolution; - - efi_bs_call(free_pool, info); - - valid = !(pf == PIXEL_BLT_ONLY || pf >= PIXEL_FORMAT_MAX); - d = 0; - switch (pf) { - case PIXEL_RGB_RESERVED_8BIT_PER_COLOR: - dstr = "rgb"; - break; - case PIXEL_BGR_RESERVED_8BIT_PER_COLOR: - dstr = "bgr"; - break; - case PIXEL_BIT_MASK: - dstr = ""; - d = pixel_bpp(pf, pi); - break; - case PIXEL_BLT_ONLY: - dstr = "blt"; - break; - default: - dstr = "xxx"; - break; - } - - efi_printk("Mode %3u %c%c: Resolution %ux%u-%s%.0hhu\n", - m, - m == cur_mode ? '*' : ' ', - !valid ? '-' : ' ', - w, h, dstr, d); - } + choose_mode(gop, match_list, (void *)cur_mode); efi_puts("\nPress any key to continue (or wait 10 seconds)\n"); status = efi_wait_for_key(10 * EFI_USEC_PER_SEC, &key); @@ -426,24 +368,31 @@ static void find_bits(u32 mask, u8 *pos, u8 *size) *size = __fls(mask) - *pos + 1; } -static void -setup_pixel_info(struct screen_info *si, u32 pixels_per_scan_line, - efi_pixel_bitmask_t pixel_info, int pixel_format) +static void setup_screen_info(struct screen_info *si, const efi_graphics_output_protocol_t *gop) { - if (pixel_format == PIXEL_BIT_MASK) { - find_bits(pixel_info.red_mask, - &si->red_pos, &si->red_size); - find_bits(pixel_info.green_mask, - &si->green_pos, &si->green_size); - find_bits(pixel_info.blue_mask, - &si->blue_pos, &si->blue_size); - find_bits(pixel_info.reserved_mask, - &si->rsvd_pos, &si->rsvd_size); - si->lfb_depth = si->red_size + si->green_size + - si->blue_size + si->rsvd_size; - si->lfb_linelength = (pixels_per_scan_line * si->lfb_depth) / 8; + const efi_graphics_output_protocol_mode_t *mode = efi_table_attr(gop, mode); + const efi_graphics_output_mode_info_t *info = efi_table_attr(mode, info); + + si->orig_video_isVGA = VIDEO_TYPE_EFI; + + si->lfb_width = info->horizontal_resolution; + si->lfb_height = info->vertical_resolution; + + efi_set_u64_split(efi_table_attr(mode, frame_buffer_base), + &si->lfb_base, &si->ext_lfb_base); + if (si->ext_lfb_base) + si->capabilities |= VIDEO_CAPABILITY_64BIT_BASE; + si->pages = 1; + + if (info->pixel_format == PIXEL_BIT_MASK) { + find_bits(info->pixel_information.red_mask, &si->red_pos, &si->red_size); + find_bits(info->pixel_information.green_mask, &si->green_pos, &si->green_size); + find_bits(info->pixel_information.blue_mask, &si->blue_pos, &si->blue_size); + find_bits(info->pixel_information.reserved_mask, &si->rsvd_pos, &si->rsvd_size); + si->lfb_depth = si->red_size + si->green_size + si->blue_size + si->rsvd_size; + si->lfb_linelength = (info->pixels_per_scan_line * si->lfb_depth) / 8; } else { - if (pixel_format == PIXEL_RGB_RESERVED_8BIT_PER_COLOR) { + if (info->pixel_format == PIXEL_RGB_RESERVED_8BIT_PER_COLOR) { si->red_pos = 0; si->blue_pos = 16; } else /* PIXEL_BGR_RESERVED_8BIT_PER_COLOR */ { @@ -453,34 +402,46 @@ setup_pixel_info(struct screen_info *si, u32 pixels_per_scan_line, si->green_pos = 8; si->rsvd_pos = 24; - si->red_size = si->green_size = - si->blue_size = si->rsvd_size = 8; - + si->red_size = 8; + si->green_size = 8; + si->blue_size = 8; + si->rsvd_size = 8; si->lfb_depth = 32; - si->lfb_linelength = pixels_per_scan_line * 4; + si->lfb_linelength = info->pixels_per_scan_line * 4; } + + si->lfb_size = si->lfb_linelength * si->lfb_height; + si->capabilities |= VIDEO_CAPABILITY_SKIP_QUIRKS; } -static efi_graphics_output_protocol_t * -find_gop(efi_guid_t *proto, unsigned long size, void **handles) +static void setup_edid_info(struct edid_info *edid, u32 gop_size_of_edid, u8 *gop_edid) +{ + if (!gop_edid || gop_size_of_edid < 128) + memset(edid->dummy, 0, sizeof(edid->dummy)); + else + memcpy(edid->dummy, gop_edid, min(gop_size_of_edid, sizeof(edid->dummy))); +} + +static efi_handle_t find_handle_with_primary_gop(unsigned long num, const efi_handle_t handles[], + efi_graphics_output_protocol_t **found_gop) { efi_graphics_output_protocol_t *first_gop; - efi_handle_t h; - int i; + efi_handle_t h, first_gop_handle; + first_gop_handle = NULL; first_gop = NULL; - for_each_efi_handle(h, handles, size, i) { + for_each_efi_handle(h, handles, num) { efi_status_t status; efi_graphics_output_protocol_t *gop; efi_graphics_output_protocol_mode_t *mode; efi_graphics_output_mode_info_t *info; - - efi_guid_t conout_proto = EFI_CONSOLE_OUT_DEVICE_GUID; void *dummy = NULL; - status = efi_bs_call(handle_protocol, h, proto, (void **)&gop); + status = efi_bs_call(handle_protocol, h, + &EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID, + (void **)&gop); if (status != EFI_SUCCESS) continue; @@ -500,81 +461,72 @@ find_gop(efi_guid_t *proto, unsigned long size, void **handles) * Once we've found a GOP supporting ConOut, * don't bother looking any further. */ - status = efi_bs_call(handle_protocol, h, &conout_proto, &dummy); - if (status == EFI_SUCCESS) - return gop; - - if (!first_gop) + status = efi_bs_call(handle_protocol, h, + &EFI_CONSOLE_OUT_DEVICE_GUID, &dummy); + if (status == EFI_SUCCESS) { + if (found_gop) + *found_gop = gop; + return h; + } else if (!first_gop_handle) { + first_gop_handle = h; first_gop = gop; + } } - return first_gop; + if (found_gop) + *found_gop = first_gop; + return first_gop_handle; } -static efi_status_t setup_gop(struct screen_info *si, efi_guid_t *proto, - unsigned long size, void **handles) +efi_status_t efi_setup_graphics(struct screen_info *si, struct edid_info *edid) { + efi_handle_t *handles __free(efi_pool) = NULL; + efi_handle_t handle; efi_graphics_output_protocol_t *gop; - efi_graphics_output_protocol_mode_t *mode; - efi_graphics_output_mode_info_t *info; + efi_status_t status; + unsigned long num; - gop = find_gop(proto, size, handles); + status = efi_bs_call(locate_handle_buffer, EFI_LOCATE_BY_PROTOCOL, + &EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID, NULL, &num, + &handles); + if (status != EFI_SUCCESS) + return status; - /* Did we find any GOPs? */ - if (!gop) + handle = find_handle_with_primary_gop(num, handles, &gop); + if (!handle) return EFI_NOT_FOUND; /* Change mode if requested */ set_mode(gop); /* EFI framebuffer */ - mode = efi_table_attr(gop, mode); - info = efi_table_attr(mode, info); - - si->orig_video_isVGA = VIDEO_TYPE_EFI; - - si->lfb_width = info->horizontal_resolution; - si->lfb_height = info->vertical_resolution; - - efi_set_u64_split(efi_table_attr(mode, frame_buffer_base), - &si->lfb_base, &si->ext_lfb_base); - if (si->ext_lfb_base) - si->capabilities |= VIDEO_CAPABILITY_64BIT_BASE; - - si->pages = 1; - - setup_pixel_info(si, info->pixels_per_scan_line, - info->pixel_information, info->pixel_format); - - si->lfb_size = si->lfb_linelength * si->lfb_height; + if (si) + setup_screen_info(si, gop); + + /* Display EDID for primary GOP */ + if (edid) { + efi_edid_discovered_protocol_t *discovered_edid; + efi_edid_active_protocol_t *active_edid; + u32 gop_size_of_edid = 0; + u8 *gop_edid = NULL; + + status = efi_bs_call(handle_protocol, handle, &EFI_EDID_ACTIVE_PROTOCOL_GUID, + (void **)&active_edid); + if (status == EFI_SUCCESS) { + gop_size_of_edid = active_edid->size_of_edid; + gop_edid = active_edid->edid; + } else { + status = efi_bs_call(handle_protocol, handle, + &EFI_EDID_DISCOVERED_PROTOCOL_GUID, + (void **)&discovered_edid); + if (status == EFI_SUCCESS) { + gop_size_of_edid = discovered_edid->size_of_edid; + gop_edid = discovered_edid->edid; + } + } - si->capabilities |= VIDEO_CAPABILITY_SKIP_QUIRKS; + setup_edid_info(edid, gop_size_of_edid, gop_edid); + } return EFI_SUCCESS; } - -/* - * See if we have Graphics Output Protocol - */ -efi_status_t efi_setup_gop(struct screen_info *si, efi_guid_t *proto, - unsigned long size) -{ - efi_status_t status; - void **gop_handle = NULL; - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)&gop_handle); - if (status != EFI_SUCCESS) - return status; - - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, proto, NULL, - &size, gop_handle); - if (status != EFI_SUCCESS) - goto free_handle; - - status = setup_gop(si, proto, size, gop_handle); - -free_handle: - efi_bs_call(free_pool, gop_handle); - return status; -} diff --git a/drivers/firmware/efi/libstub/intrinsics.c b/drivers/firmware/efi/libstub/intrinsics.c index a04ab39292b6..418cd2e6dccc 100644 --- a/drivers/firmware/efi/libstub/intrinsics.c +++ b/drivers/firmware/efi/libstub/intrinsics.c @@ -15,8 +15,31 @@ void *__memmove(void *__dest, const void *__src, size_t count) __alias(memmove); void *__memset(void *s, int c, size_t count) __alias(memset); #endif +static void *efistub_memmove(u8 *dst, const u8 *src, size_t len) +{ + if (src > dst || dst >= (src + len)) + for (size_t i = 0; i < len; i++) + dst[i] = src[i]; + else + for (ssize_t i = len - 1; i >= 0; i--) + dst[i] = src[i]; + + return dst; +} + +static void *efistub_memset(void *dst, int c, size_t len) +{ + for (u8 *d = dst; len--; d++) + *d = c; + + return dst; +} + void *memcpy(void *dst, const void *src, size_t len) { + if (efi_table_attr(efi_system_table, boottime) == NULL) + return efistub_memmove(dst, src, len); + efi_bs_call(copy_mem, dst, src, len); return dst; } @@ -25,6 +48,27 @@ extern void *memmove(void *dst, const void *src, size_t len) __alias(memcpy); void *memset(void *dst, int c, size_t len) { + if (efi_table_attr(efi_system_table, boottime) == NULL) + return efistub_memset(dst, c, len); + efi_bs_call(set_mem, dst, len, c & U8_MAX); return dst; } + +/** + * memcmp - Compare two areas of memory + * @cs: One area of memory + * @ct: Another area of memory + * @count: The size of the area. + */ +#undef memcmp +int memcmp(const void *cs, const void *ct, size_t count) +{ + const unsigned char *su1, *su2; + int res = 0; + + for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--) + if ((res = *su1 - *su2) != 0) + break; + return res; +} diff --git a/drivers/firmware/efi/libstub/kaslr.c b/drivers/firmware/efi/libstub/kaslr.c new file mode 100644 index 000000000000..4bc963e999eb --- /dev/null +++ b/drivers/firmware/efi/libstub/kaslr.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Helper functions used by the EFI stub on multiple + * architectures to deal with physical address space randomization. + */ +#include <linux/efi.h> + +#include "efistub.h" + +/** + * efi_kaslr_get_phys_seed() - Get random seed for physical kernel KASLR + * @image_handle: Handle to the image + * + * If KASLR is not disabled, obtain a random seed using EFI_RNG_PROTOCOL + * that will be used to move the kernel physical mapping. + * + * Return: the random seed + */ +u32 efi_kaslr_get_phys_seed(efi_handle_t image_handle) +{ + efi_guid_t li_fixed_proto = LINUX_EFI_LOADED_IMAGE_FIXED_GUID; + void *p; + + if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE)) + return 0; + + if (efi_nokaslr) { + efi_info("KASLR disabled on kernel command line\n"); + } else if (efi_bs_call(handle_protocol, image_handle, + &li_fixed_proto, &p) == EFI_SUCCESS) { + efi_info("Image placement fixed by loader\n"); + } else { + efi_status_t status; + u32 phys_seed; + + status = efi_get_random_bytes(sizeof(phys_seed), + (u8 *)&phys_seed); + if (status == EFI_SUCCESS) + return phys_seed; + + if (status == EFI_NOT_FOUND) + efi_info("EFI_RNG_PROTOCOL unavailable\n"); + else + efi_err("efi_get_random_bytes() failed (0x%lx)\n", status); + + efi_nokaslr = true; + } + + return 0; +} + +/* + * Distro versions of GRUB may ignore the BSS allocation entirely (i.e., fail + * to provide space, and fail to zero it). Check for this condition by double + * checking that the first and the last byte of the image are covered by the + * same EFI memory map entry. + */ +static bool check_image_region(u64 base, u64 size) +{ + struct efi_boot_memmap *map __free(efi_pool) = NULL; + efi_status_t status; + bool ret = false; + int map_offset; + + status = efi_get_memory_map(&map, false); + if (status != EFI_SUCCESS) + return false; + + for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { + efi_memory_desc_t *md = (void *)map->map + map_offset; + u64 end = md->phys_addr + md->num_pages * EFI_PAGE_SIZE; + + /* + * Find the region that covers base, and return whether + * it covers base+size bytes. + */ + if (base >= md->phys_addr && base < end) { + ret = (base + size) <= end; + break; + } + } + + return ret; +} + +/** + * efi_kaslr_relocate_kernel() - Relocate the kernel (random if KASLR enabled) + * @image_addr: Pointer to the current kernel location + * @reserve_addr: Pointer to the relocated kernel location + * @reserve_size: Size of the relocated kernel + * @kernel_size: Size of the text + data + * @kernel_codesize: Size of the text + * @kernel_memsize: Size of the text + data + bss + * @phys_seed: Random seed used for the relocation + * + * If KASLR is not enabled, this function relocates the kernel to a fixed + * address (or leave it as its current location). If KASLR is enabled, the + * kernel physical location is randomized using the seed in parameter. + * + * Return: status code, EFI_SUCCESS if relocation is successful + */ +efi_status_t efi_kaslr_relocate_kernel(unsigned long *image_addr, + unsigned long *reserve_addr, + unsigned long *reserve_size, + unsigned long kernel_size, + unsigned long kernel_codesize, + unsigned long kernel_memsize, + u32 phys_seed) +{ + efi_status_t status; + u64 min_kimg_align = efi_get_kimg_min_align(); + + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && phys_seed != 0) { + /* + * If KASLR is enabled, and we have some randomness available, + * locate the kernel at a randomized offset in physical memory. + */ + status = efi_random_alloc(*reserve_size, min_kimg_align, + reserve_addr, phys_seed, + EFI_LOADER_CODE, 0, EFI_ALLOC_LIMIT); + if (status != EFI_SUCCESS) + efi_warn("efi_random_alloc() failed: 0x%lx\n", status); + } else { + status = EFI_OUT_OF_RESOURCES; + } + + if (status != EFI_SUCCESS) { + if (!check_image_region(*image_addr, kernel_memsize)) { + efi_err("FIRMWARE BUG: Image BSS overlaps adjacent EFI memory region\n"); + } else if (IS_ALIGNED(*image_addr, min_kimg_align) && + (unsigned long)_end < EFI_ALLOC_LIMIT) { + /* + * Just execute from wherever we were loaded by the + * UEFI PE/COFF loader if the placement is suitable. + */ + *reserve_size = 0; + return EFI_SUCCESS; + } + + status = efi_allocate_pages_aligned(*reserve_size, reserve_addr, + ULONG_MAX, min_kimg_align, + EFI_LOADER_CODE); + + if (status != EFI_SUCCESS) { + efi_err("Failed to relocate kernel\n"); + *reserve_size = 0; + return status; + } + } + + memcpy((void *)*reserve_addr, (void *)*image_addr, kernel_size); + *image_addr = *reserve_addr; + efi_icache_sync(*image_addr, *image_addr + kernel_codesize); + efi_remap_image(*image_addr, *reserve_size, kernel_codesize); + + return status; +} diff --git a/drivers/firmware/efi/libstub/loongarch-stub.c b/drivers/firmware/efi/libstub/loongarch-stub.c index 32329f2a92f9..736b6aae323d 100644 --- a/drivers/firmware/efi/libstub/loongarch-stub.c +++ b/drivers/firmware/efi/libstub/loongarch-stub.c @@ -8,19 +8,11 @@ #include <asm/efi.h> #include <asm/addrspace.h> #include "efistub.h" - -typedef void __noreturn (*kernel_entry_t)(bool efi, unsigned long cmdline, - unsigned long systab); +#include "loongarch-stub.h" extern int kernel_asize; extern int kernel_fsize; -extern int kernel_offset; -extern kernel_entry_t kernel_entry; - -efi_status_t check_platform_features(void) -{ - return EFI_SUCCESS; -} +extern int kernel_entry; efi_status_t handle_kernel_image(unsigned long *image_addr, unsigned long *image_size, @@ -32,10 +24,10 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, efi_status_t status; unsigned long kernel_addr = 0; - kernel_addr = (unsigned long)&kernel_offset - kernel_offset; + kernel_addr = (unsigned long)image->image_base; status = efi_relocate_kernel(&kernel_addr, kernel_fsize, kernel_asize, - PHYSADDR(VMLINUX_LOAD_ADDRESS), SZ_2M, 0x0); + EFI_KIMG_PREFERRED_ADDRESS, efi_get_kimg_min_align(), 0x0); *image_addr = kernel_addr; *image_size = kernel_asize; @@ -43,60 +35,10 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, return status; } -struct exit_boot_struct { - efi_memory_desc_t *runtime_map; - int runtime_entry_count; -}; - -static efi_status_t exit_boot_func(struct efi_boot_memmap *map, void *priv) +unsigned long kernel_entry_address(unsigned long kernel_addr, + efi_loaded_image_t *image) { - struct exit_boot_struct *p = priv; - - /* - * Update the memory map with virtual addresses. The function will also - * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME - * entries so that we can pass it straight to SetVirtualAddressMap() - */ - efi_get_virtmap(map->map, map->map_size, map->desc_size, - p->runtime_map, &p->runtime_entry_count); - - return EFI_SUCCESS; -} - -efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, - unsigned long kernel_addr, char *cmdline_ptr) -{ - kernel_entry_t real_kernel_entry; - struct exit_boot_struct priv; - unsigned long desc_size; - efi_status_t status; - u32 desc_ver; - - status = efi_alloc_virtmap(&priv.runtime_map, &desc_size, &desc_ver); - if (status != EFI_SUCCESS) { - efi_err("Unable to retrieve UEFI memory map.\n"); - return status; - } - - efi_info("Exiting boot services\n"); - - efi_novamap = false; - status = efi_exit_boot_services(handle, &priv, exit_boot_func); - if (status != EFI_SUCCESS) - return status; - - /* Install the new virtual address map */ - efi_rt_call(set_virtual_address_map, - priv.runtime_entry_count * desc_size, desc_size, - desc_ver, priv.runtime_map); - - /* Config Direct Mapping */ - csr_write64(CSR_DMW0_INIT, LOONGARCH_CSR_DMWIN0); - csr_write64(CSR_DMW1_INIT, LOONGARCH_CSR_DMWIN1); - - real_kernel_entry = (kernel_entry_t) - ((unsigned long)&kernel_entry - kernel_addr + VMLINUX_LOAD_ADDRESS); + unsigned long base = (unsigned long)image->image_base; - real_kernel_entry(true, (unsigned long)cmdline_ptr, - (unsigned long)efi_system_table); + return (unsigned long)&kernel_entry - base + kernel_addr; } diff --git a/drivers/firmware/efi/libstub/loongarch-stub.h b/drivers/firmware/efi/libstub/loongarch-stub.h new file mode 100644 index 000000000000..cd015955a015 --- /dev/null +++ b/drivers/firmware/efi/libstub/loongarch-stub.h @@ -0,0 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +unsigned long kernel_entry_address(unsigned long kernel_addr, + efi_loaded_image_t *image); diff --git a/drivers/firmware/efi/libstub/loongarch.c b/drivers/firmware/efi/libstub/loongarch.c new file mode 100644 index 000000000000..3782d0a187d1 --- /dev/null +++ b/drivers/firmware/efi/libstub/loongarch.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Yun Liu <liuyun@loongson.cn> + * Huacai Chen <chenhuacai@loongson.cn> + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited + */ + +#include <asm/efi.h> +#include <asm/addrspace.h> +#include "efistub.h" +#include "loongarch-stub.h" + +typedef void __noreturn (*kernel_entry_t)(bool efi, unsigned long cmdline, + unsigned long systab); + +efi_status_t check_platform_features(void) +{ + return EFI_SUCCESS; +} + +struct exit_boot_struct { + efi_memory_desc_t *runtime_map; + int runtime_entry_count; +}; + +static efi_status_t exit_boot_func(struct efi_boot_memmap *map, void *priv) +{ + struct exit_boot_struct *p = priv; + + /* + * Update the memory map with virtual addresses. The function will also + * populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME + * entries so that we can pass it straight to SetVirtualAddressMap() + */ + efi_get_virtmap(map->map, map->map_size, map->desc_size, + p->runtime_map, &p->runtime_entry_count); + + return EFI_SUCCESS; +} + +unsigned long __weak kernel_entry_address(unsigned long kernel_addr, + efi_loaded_image_t *image) +{ + return *(unsigned long *)(kernel_addr + 8) - PHYSADDR(VMLINUX_LOAD_ADDRESS) + kernel_addr; +} + +efi_status_t efi_boot_kernel(void *handle, efi_loaded_image_t *image, + unsigned long kernel_addr, char *cmdline_ptr) +{ + kernel_entry_t real_kernel_entry; + struct exit_boot_struct priv; + unsigned long desc_size; + efi_status_t status; + u32 desc_ver; + + status = efi_alloc_virtmap(&priv.runtime_map, &desc_size, &desc_ver); + if (status != EFI_SUCCESS) { + efi_err("Unable to retrieve UEFI memory map.\n"); + return status; + } + + efi_info("Exiting boot services\n"); + + efi_novamap = false; + status = efi_exit_boot_services(handle, &priv, exit_boot_func); + if (status != EFI_SUCCESS) + return status; + + /* Install the new virtual address map */ + efi_rt_call(set_virtual_address_map, + priv.runtime_entry_count * desc_size, desc_size, + desc_ver, priv.runtime_map); + + /* Config Direct Mapping */ + csr_write64(CSR_DMW0_INIT, LOONGARCH_CSR_DMWIN0); + csr_write64(CSR_DMW1_INIT, LOONGARCH_CSR_DMWIN1); + csr_write64(CSR_DMW2_INIT, LOONGARCH_CSR_DMWIN2); + csr_write64(CSR_DMW3_INIT, LOONGARCH_CSR_DMWIN3); + + real_kernel_entry = (void *)kernel_entry_address(kernel_addr, image); + + real_kernel_entry(true, (unsigned long)cmdline_ptr, + (unsigned long)efi_system_table); +} diff --git a/drivers/firmware/efi/libstub/mem.c b/drivers/firmware/efi/libstub/mem.c index 45841ef55a9f..9c82259eea81 100644 --- a/drivers/firmware/efi/libstub/mem.c +++ b/drivers/firmware/efi/libstub/mem.c @@ -20,10 +20,10 @@ efi_status_t efi_get_memory_map(struct efi_boot_memmap **map, bool install_cfg_tbl) { + struct efi_boot_memmap tmp, *m __free(efi_pool) = NULL; int memtype = install_cfg_tbl ? EFI_ACPI_RECLAIM_MEMORY : EFI_LOADER_DATA; efi_guid_t tbl_guid = LINUX_EFI_BOOT_MEMMAP_GUID; - struct efi_boot_memmap *m, tmp; efi_status_t status; unsigned long size; @@ -48,24 +48,20 @@ efi_status_t efi_get_memory_map(struct efi_boot_memmap **map, */ status = efi_bs_call(install_configuration_table, &tbl_guid, m); if (status != EFI_SUCCESS) - goto free_map; + return status; } m->buff_size = m->map_size = size; status = efi_bs_call(get_memory_map, &m->map_size, m->map, &m->map_key, &m->desc_size, &m->desc_ver); - if (status != EFI_SUCCESS) - goto uninstall_table; + if (status != EFI_SUCCESS) { + if (install_cfg_tbl) + efi_bs_call(install_configuration_table, &tbl_guid, NULL); + return status; + } - *map = m; + *map = no_free_ptr(m); return EFI_SUCCESS; - -uninstall_table: - if (install_cfg_tbl) - efi_bs_call(install_configuration_table, &tbl_guid, NULL); -free_map: - efi_bs_call(free_pool, m); - return status; } /** @@ -89,9 +85,12 @@ efi_status_t efi_allocate_pages(unsigned long size, unsigned long *addr, efi_physical_addr_t alloc_addr; efi_status_t status; + max = min(max, EFI_ALLOC_LIMIT); + if (EFI_ALLOC_ALIGN > EFI_PAGE_SIZE) return efi_allocate_pages_aligned(size, addr, max, - EFI_ALLOC_ALIGN); + EFI_ALLOC_ALIGN, + EFI_LOADER_DATA); alloc_addr = ALIGN_DOWN(max + 1, EFI_ALLOC_ALIGN) - 1; status = efi_bs_call(allocate_pages, EFI_ALLOCATE_MAX_ADDRESS, diff --git a/drivers/firmware/efi/libstub/pci.c b/drivers/firmware/efi/libstub/pci.c index 99fb25d2bcf5..1dccf77958d3 100644 --- a/drivers/firmware/efi/libstub/pci.c +++ b/drivers/firmware/efi/libstub/pci.c @@ -16,37 +16,20 @@ void efi_pci_disable_bridge_busmaster(void) { efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID; - unsigned long pci_handle_size = 0; - efi_handle_t *pci_handle = NULL; + efi_handle_t *pci_handle __free(efi_pool) = NULL; + unsigned long pci_handle_num; efi_handle_t handle; efi_status_t status; u16 class, command; - int i; - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, &pci_proto, - NULL, &pci_handle_size, NULL); - - if (status != EFI_BUFFER_TOO_SMALL) { - if (status != EFI_SUCCESS && status != EFI_NOT_FOUND) - efi_err("Failed to locate PCI I/O handles'\n"); - return; - } - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, pci_handle_size, - (void **)&pci_handle); + status = efi_bs_call(locate_handle_buffer, EFI_LOCATE_BY_PROTOCOL, + &pci_proto, NULL, &pci_handle_num, &pci_handle); if (status != EFI_SUCCESS) { - efi_err("Failed to allocate memory for 'pci_handle'\n"); + efi_err("Failed to locate PCI I/O handles\n"); return; } - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, &pci_proto, - NULL, &pci_handle_size, pci_handle); - if (status != EFI_SUCCESS) { - efi_err("Failed to locate PCI I/O handles'\n"); - goto free_handle; - } - - for_each_efi_handle(handle, pci_handle, pci_handle_size, i) { + for_each_efi_handle(handle, pci_handle, pci_handle_num) { efi_pci_io_protocol_t *pci; unsigned long segment_nr, bus_nr, device_nr, func_nr; @@ -82,7 +65,7 @@ void efi_pci_disable_bridge_busmaster(void) efi_bs_call(disconnect_controller, handle, NULL, NULL); } - for_each_efi_handle(handle, pci_handle, pci_handle_size, i) { + for_each_efi_handle(handle, pci_handle, pci_handle_num) { efi_pci_io_protocol_t *pci; status = efi_bs_call(handle_protocol, handle, &pci_proto, @@ -108,7 +91,4 @@ void efi_pci_disable_bridge_busmaster(void) if (status != EFI_SUCCESS) efi_err("Failed to disable PCI busmastering\n"); } - -free_handle: - efi_bs_call(free_pool, pci_handle); } diff --git a/drivers/firmware/efi/libstub/printk.c b/drivers/firmware/efi/libstub/printk.c new file mode 100644 index 000000000000..bc599212c05d --- /dev/null +++ b/drivers/firmware/efi/libstub/printk.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/stdarg.h> + +#include <linux/ctype.h> +#include <linux/efi.h> +#include <linux/kernel.h> +#include <linux/kern_levels.h> +#include <asm/efi.h> +#include <asm/setup.h> + +#include "efistub.h" + +int efi_loglevel = LOGLEVEL_NOTICE; + +/** + * efi_char16_puts() - Write a UCS-2 encoded string to the console + * @str: UCS-2 encoded string + */ +void efi_char16_puts(efi_char16_t *str) +{ + efi_call_proto(efi_table_attr(efi_system_table, con_out), + output_string, str); +} + +static +u32 utf8_to_utf32(const u8 **s8) +{ + u32 c32; + u8 c0, cx; + size_t clen, i; + + c0 = cx = *(*s8)++; + /* + * The position of the most-significant 0 bit gives us the length of + * a multi-octet encoding. + */ + for (clen = 0; cx & 0x80; ++clen) + cx <<= 1; + /* + * If the 0 bit is in position 8, this is a valid single-octet + * encoding. If the 0 bit is in position 7 or positions 1-3, the + * encoding is invalid. + * In either case, we just return the first octet. + */ + if (clen < 2 || clen > 4) + return c0; + /* Get the bits from the first octet. */ + c32 = cx >> clen--; + for (i = 0; i < clen; ++i) { + /* Trailing octets must have 10 in most significant bits. */ + cx = (*s8)[i] ^ 0x80; + if (cx & 0xc0) + return c0; + c32 = (c32 << 6) | cx; + } + /* + * Check for validity: + * - The character must be in the Unicode range. + * - It must not be a surrogate. + * - It must be encoded using the correct number of octets. + */ + if (c32 > 0x10ffff || + (c32 & 0xf800) == 0xd800 || + clen != (c32 >= 0x80) + (c32 >= 0x800) + (c32 >= 0x10000)) + return c0; + *s8 += clen; + return c32; +} + +/** + * efi_puts() - Write a UTF-8 encoded string to the console + * @str: UTF-8 encoded string + */ +void efi_puts(const char *str) +{ + efi_char16_t buf[128]; + size_t pos = 0, lim = ARRAY_SIZE(buf); + const u8 *s8 = (const u8 *)str; + u32 c32; + + while (*s8) { + if (*s8 == '\n') + buf[pos++] = L'\r'; + c32 = utf8_to_utf32(&s8); + if (c32 < 0x10000) { + /* Characters in plane 0 use a single word. */ + buf[pos++] = c32; + } else { + /* + * Characters in other planes encode into a surrogate + * pair. + */ + buf[pos++] = (0xd800 - (0x10000 >> 10)) + (c32 >> 10); + buf[pos++] = 0xdc00 + (c32 & 0x3ff); + } + if (*s8 == '\0' || pos >= lim - 2) { + buf[pos] = L'\0'; + efi_char16_puts(buf); + pos = 0; + } + } +} + +/** + * efi_printk() - Print a kernel message + * @fmt: format string + * + * The first letter of the format string is used to determine the logging level + * of the message. If the level is less then the current EFI logging level, the + * message is suppressed. The message will be truncated to 255 bytes. + * + * Return: number of printed characters + */ +int efi_printk(const char *fmt, ...) +{ + char printf_buf[256]; + va_list args; + int printed; + int loglevel = printk_get_level(fmt); + + switch (loglevel) { + case '0' ... '9': + loglevel -= '0'; + break; + default: + /* + * Use loglevel -1 for cases where we just want to print to + * the screen. + */ + loglevel = -1; + break; + } + + if (loglevel >= efi_loglevel) + return 0; + + if (loglevel >= 0) + efi_puts("EFI stub: "); + + fmt = printk_skip_level(fmt); + + va_start(args, fmt); + printed = vsnprintf(printf_buf, sizeof(printf_buf), fmt, args); + va_end(args); + + efi_puts(printf_buf); + if (printed >= sizeof(printf_buf)) { + efi_puts("[Message truncated]\n"); + return -1; + } + + return printed; +} diff --git a/drivers/firmware/efi/libstub/random.c b/drivers/firmware/efi/libstub/random.c index 33ab56769595..7109b8a2dcba 100644 --- a/drivers/firmware/efi/libstub/random.c +++ b/drivers/firmware/efi/libstub/random.c @@ -67,47 +67,113 @@ efi_status_t efi_random_get_seed(void) efi_guid_t rng_proto = EFI_RNG_PROTOCOL_GUID; efi_guid_t rng_algo_raw = EFI_RNG_ALGORITHM_RAW; efi_guid_t rng_table_guid = LINUX_EFI_RANDOM_SEED_TABLE_GUID; + struct linux_efi_random_seed *prev_seed, *seed = NULL; + int prev_seed_size = 0, seed_size = EFI_RANDOM_SEED_SIZE; + unsigned long nv_seed_size = 0, offset = 0; efi_rng_protocol_t *rng = NULL; - struct linux_efi_random_seed *seed = NULL; efi_status_t status; status = efi_bs_call(locate_protocol, &rng_proto, NULL, (void **)&rng); if (status != EFI_SUCCESS) + seed_size = 0; + + // Call GetVariable() with a zero length buffer to obtain the size + get_efi_var(L"RandomSeed", &rng_table_guid, NULL, &nv_seed_size, NULL); + if (!seed_size && !nv_seed_size) return status; + seed_size += nv_seed_size; + + /* + * Check whether a seed was provided by a prior boot stage. In that + * case, instead of overwriting it, let's create a new buffer that can + * hold both, and concatenate the existing and the new seeds. + * Note that we should read the seed size with caution, in case the + * table got corrupted in memory somehow. + */ + prev_seed = get_efi_config_table(rng_table_guid); + if (prev_seed && prev_seed->size <= 512U) { + prev_seed_size = prev_seed->size; + seed_size += prev_seed_size; + } + /* * Use EFI_ACPI_RECLAIM_MEMORY here so that it is guaranteed that the * allocation will survive a kexec reboot (although we refresh the seed * beforehand) */ status = efi_bs_call(allocate_pool, EFI_ACPI_RECLAIM_MEMORY, - sizeof(*seed) + EFI_RANDOM_SEED_SIZE, + struct_size(seed, bits, seed_size), (void **)&seed); - if (status != EFI_SUCCESS) - return status; - - status = efi_call_proto(rng, get_rng, &rng_algo_raw, - EFI_RANDOM_SEED_SIZE, seed->bits); + if (status != EFI_SUCCESS) { + efi_warn("Failed to allocate memory for RNG seed.\n"); + goto err_warn; + } - if (status == EFI_UNSUPPORTED) - /* - * Use whatever algorithm we have available if the raw algorithm - * is not implemented. - */ - status = efi_call_proto(rng, get_rng, NULL, + if (rng) { + status = efi_call_proto(rng, get_rng, &rng_algo_raw, EFI_RANDOM_SEED_SIZE, seed->bits); - if (status != EFI_SUCCESS) + if (status == EFI_UNSUPPORTED) + /* + * Use whatever algorithm we have available if the raw algorithm + * is not implemented. + */ + status = efi_call_proto(rng, get_rng, NULL, + EFI_RANDOM_SEED_SIZE, seed->bits); + + if (status == EFI_SUCCESS) + offset = EFI_RANDOM_SEED_SIZE; + } + + if (nv_seed_size) { + status = get_efi_var(L"RandomSeed", &rng_table_guid, NULL, + &nv_seed_size, seed->bits + offset); + + if (status == EFI_SUCCESS) + /* + * We delete the seed here, and /hope/ that this causes + * EFI to also zero out its representation on disk. + * This is somewhat idealistic, but overwriting the + * variable with zeros is likely just as fraught too. + * TODO: in the future, maybe we can hash it forward + * instead, and write a new seed. + */ + status = set_efi_var(L"RandomSeed", &rng_table_guid, 0, + 0, NULL); + + if (status == EFI_SUCCESS) + offset += nv_seed_size; + else + memzero_explicit(seed->bits + offset, nv_seed_size); + } + + if (!offset) goto err_freepool; - seed->size = EFI_RANDOM_SEED_SIZE; + if (prev_seed_size) { + memcpy(seed->bits + offset, prev_seed->bits, prev_seed_size); + offset += prev_seed_size; + } + + seed->size = offset; status = efi_bs_call(install_configuration_table, &rng_table_guid, seed); if (status != EFI_SUCCESS) goto err_freepool; + if (prev_seed_size) { + /* wipe and free the old seed if we managed to install the new one */ + memzero_explicit(prev_seed->bits, prev_seed_size); + efi_bs_call(free_pool, prev_seed); + } return EFI_SUCCESS; err_freepool: + memzero_explicit(seed, struct_size(seed, bits, seed_size)); efi_bs_call(free_pool, seed); + efi_warn("Failed to obtain seed from EFI_RNG_PROTOCOL or EFI variable\n"); +err_warn: + if (prev_seed) + efi_warn("Retaining bootloader-supplied seed only"); return status; } diff --git a/drivers/firmware/efi/libstub/randomalloc.c b/drivers/firmware/efi/libstub/randomalloc.c index 9fb5869896be..fd80b2f3233a 100644 --- a/drivers/firmware/efi/libstub/randomalloc.c +++ b/drivers/firmware/efi/libstub/randomalloc.c @@ -16,7 +16,8 @@ */ static unsigned long get_entry_num_slots(efi_memory_desc_t *md, unsigned long size, - unsigned long align_shift) + unsigned long align_shift, + u64 alloc_min, u64 alloc_max) { unsigned long align = 1UL << align_shift; u64 first_slot, last_slot, region_end; @@ -24,16 +25,19 @@ static unsigned long get_entry_num_slots(efi_memory_desc_t *md, if (md->type != EFI_CONVENTIONAL_MEMORY) return 0; + if (md->attribute & EFI_MEMORY_HOT_PLUGGABLE) + return 0; + if (efi_soft_reserve_enabled() && (md->attribute & EFI_MEMORY_SP)) return 0; region_end = min(md->phys_addr + md->num_pages * EFI_PAGE_SIZE - 1, - (u64)ULONG_MAX); + alloc_max); if (region_end < size) return 0; - first_slot = round_up(md->phys_addr, align); + first_slot = round_up(max(md->phys_addr, alloc_min), align); last_slot = round_down(region_end - size + 1, align); if (first_slot > last_slot) @@ -53,11 +57,14 @@ static unsigned long get_entry_num_slots(efi_memory_desc_t *md, efi_status_t efi_random_alloc(unsigned long size, unsigned long align, unsigned long *addr, - unsigned long random_seed) + unsigned long random_seed, + int memory_type, + unsigned long alloc_min, + unsigned long alloc_max) { + struct efi_boot_memmap *map __free(efi_pool) = NULL; unsigned long total_slots = 0, target_slot; unsigned long total_mirrored_slots = 0; - struct efi_boot_memmap *map; efi_status_t status; int map_offset; @@ -68,6 +75,10 @@ efi_status_t efi_random_alloc(unsigned long size, if (align < EFI_ALLOC_ALIGN) align = EFI_ALLOC_ALIGN; + /* Avoid address 0x0, as it can be mistaken for NULL */ + if (alloc_min == 0) + alloc_min = align; + size = round_up(size, EFI_ALLOC_ALIGN); /* count the suitable slots in each memory map entry */ @@ -75,7 +86,8 @@ efi_status_t efi_random_alloc(unsigned long size, efi_memory_desc_t *md = (void *)map->map + map_offset; unsigned long slots; - slots = get_entry_num_slots(md, size, ilog2(align)); + slots = get_entry_num_slots(md, size, ilog2(align), alloc_min, + alloc_max); MD_NUM_SLOTS(md) = slots; total_slots += slots; if (md->attribute & EFI_MEMORY_MORE_RELIABLE) @@ -100,6 +112,7 @@ efi_status_t efi_random_alloc(unsigned long size, * to calculate the randomly chosen address, and allocate it directly * using EFI_ALLOCATE_ADDRESS. */ + status = EFI_OUT_OF_RESOURCES; for (map_offset = 0; map_offset < map->map_size; map_offset += map->desc_size) { efi_memory_desc_t *md = (void *)map->map + map_offset; efi_physical_addr_t target; @@ -114,17 +127,15 @@ efi_status_t efi_random_alloc(unsigned long size, continue; } - target = round_up(md->phys_addr, align) + target_slot * align; + target = round_up(max_t(u64, md->phys_addr, alloc_min), align) + target_slot * align; pages = size / EFI_PAGE_SIZE; status = efi_bs_call(allocate_pages, EFI_ALLOCATE_ADDRESS, - EFI_LOADER_DATA, pages, &target); + memory_type, pages, &target); if (status == EFI_SUCCESS) *addr = target; break; } - efi_bs_call(free_pool, map); - return status; } diff --git a/drivers/firmware/efi/libstub/relocate.c b/drivers/firmware/efi/libstub/relocate.c index bf6fbd5d22a1..d4264bfb6dc1 100644 --- a/drivers/firmware/efi/libstub/relocate.c +++ b/drivers/firmware/efi/libstub/relocate.c @@ -23,14 +23,14 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, unsigned long *addr, unsigned long min) { - struct efi_boot_memmap *map; + struct efi_boot_memmap *map __free(efi_pool) = NULL; efi_status_t status; unsigned long nr_pages; int i; status = efi_get_memory_map(&map, false); if (status != EFI_SUCCESS) - goto fail; + return status; /* * Enforce minimum alignment that EFI or Linux requires when @@ -48,11 +48,14 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, unsigned long m = (unsigned long)map->map; u64 start, end; - desc = efi_early_memdesc_ptr(m, map->desc_size, i); + desc = efi_memdesc_ptr(m, map->desc_size, i); if (desc->type != EFI_CONVENTIONAL_MEMORY) continue; + if (desc->attribute & EFI_MEMORY_HOT_PLUGGABLE) + continue; + if (efi_soft_reserve_enabled() && (desc->attribute & EFI_MEMORY_SP)) continue; @@ -79,11 +82,9 @@ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align, } if (i == map->map_size / map->desc_size) - status = EFI_NOT_FOUND; + return EFI_NOT_FOUND; - efi_bs_call(free_pool, map); -fail: - return status; + return EFI_SUCCESS; } /** diff --git a/drivers/firmware/efi/libstub/riscv-stub.c b/drivers/firmware/efi/libstub/riscv-stub.c index b450ebf95977..e7d9204baee3 100644 --- a/drivers/firmware/efi/libstub/riscv-stub.c +++ b/drivers/firmware/efi/libstub/riscv-stub.c @@ -4,100 +4,23 @@ */ #include <linux/efi.h> -#include <linux/libfdt.h> #include <asm/efi.h> #include <asm/sections.h> -#include <asm/unaligned.h> +#include <linux/unaligned.h> #include "efistub.h" -/* - * RISC-V requires the kernel image to placed 2 MB aligned base for 64 bit and - * 4MB for 32 bit. - */ -#ifdef CONFIG_64BIT -#define MIN_KIMG_ALIGN SZ_2M -#else -#define MIN_KIMG_ALIGN SZ_4M -#endif - -typedef void __noreturn (*jump_kernel_func)(unsigned long, unsigned long); - -static unsigned long hartid; - -static int get_boot_hartid_from_fdt(void) +unsigned long stext_offset(void) { - const void *fdt; - int chosen_node, len; - const void *prop; - - fdt = get_efi_config_table(DEVICE_TREE_GUID); - if (!fdt) - return -EINVAL; - - chosen_node = fdt_path_offset(fdt, "/chosen"); - if (chosen_node < 0) - return -EINVAL; - - prop = fdt_getprop((void *)fdt, chosen_node, "boot-hartid", &len); - if (!prop) - return -EINVAL; - - if (len == sizeof(u32)) - hartid = (unsigned long) fdt32_to_cpu(*(fdt32_t *)prop); - else if (len == sizeof(u64)) - hartid = (unsigned long) fdt64_to_cpu(__get_unaligned_t(fdt64_t, prop)); - else - return -EINVAL; - - return 0; -} - -static efi_status_t get_boot_hartid_from_efi(void) -{ - efi_guid_t boot_protocol_guid = RISCV_EFI_BOOT_PROTOCOL_GUID; - struct riscv_efi_boot_protocol *boot_protocol; - efi_status_t status; - - status = efi_bs_call(locate_protocol, &boot_protocol_guid, NULL, - (void **)&boot_protocol); - if (status != EFI_SUCCESS) - return status; - return efi_call_proto(boot_protocol, get_boot_hartid, &hartid); -} - -efi_status_t check_platform_features(void) -{ - efi_status_t status; - int ret; - - status = get_boot_hartid_from_efi(); - if (status != EFI_SUCCESS) { - ret = get_boot_hartid_from_fdt(); - if (ret) { - efi_err("Failed to get boot hartid!\n"); - return EFI_UNSUPPORTED; - } - } - return EFI_SUCCESS; -} - -void __noreturn efi_enter_kernel(unsigned long entrypoint, unsigned long fdt, - unsigned long fdt_size) -{ - unsigned long stext_offset = _start_kernel - _start; - unsigned long kernel_entry = entrypoint + stext_offset; - jump_kernel_func jump_kernel = (jump_kernel_func)kernel_entry; - /* - * Jump to real kernel here with following constraints. - * 1. MMU should be disabled. - * 2. a0 should contain hartid - * 3. a1 should DT address + * When built as part of the kernel, the EFI stub cannot branch to the + * kernel proper via the image header, as the PE/COFF header is + * strictly not part of the in-memory presentation of the image, only + * of the file representation. So instead, we need to jump to the + * actual entrypoint in the .text region of the image. */ - csr_write(CSR_SATP, 0); - jump_kernel(hartid, fdt); + return _start_kernel - _start; } efi_status_t handle_kernel_image(unsigned long *image_addr, @@ -107,31 +30,29 @@ efi_status_t handle_kernel_image(unsigned long *image_addr, efi_loaded_image_t *image, efi_handle_t image_handle) { - unsigned long kernel_size = 0; - unsigned long preferred_addr; + unsigned long kernel_size, kernel_codesize, kernel_memsize; efi_status_t status; kernel_size = _edata - _start; + kernel_codesize = __init_text_end - _start; + kernel_memsize = kernel_size + (_end - _edata); *image_addr = (unsigned long)_start; - *image_size = kernel_size + (_end - _edata); - - /* - * RISC-V kernel maps PAGE_OFFSET virtual address to the same physical - * address where kernel is booted. That's why kernel should boot from - * as low as possible to avoid wastage of memory. Currently, dram_base - * is occupied by the firmware. So the preferred address for kernel to - * boot is next aligned address. If preferred address is not available, - * relocate_kernel will fall back to efi_low_alloc_above to allocate - * lowest possible memory region as long as the address and size meets - * the alignment constraints. - */ - preferred_addr = MIN_KIMG_ALIGN; - status = efi_relocate_kernel(image_addr, kernel_size, *image_size, - preferred_addr, MIN_KIMG_ALIGN, 0x0); + *image_size = kernel_memsize; + *reserve_size = *image_size; + status = efi_kaslr_relocate_kernel(image_addr, + reserve_addr, reserve_size, + kernel_size, kernel_codesize, kernel_memsize, + efi_kaslr_get_phys_seed(image_handle)); if (status != EFI_SUCCESS) { efi_err("Failed to relocate kernel\n"); *image_size = 0; } + return status; } + +void efi_icache_sync(unsigned long start, unsigned long end) +{ + asm volatile ("fence.i" ::: "memory"); +} diff --git a/drivers/firmware/efi/libstub/riscv.c b/drivers/firmware/efi/libstub/riscv.c new file mode 100644 index 000000000000..f66f33ceb99e --- /dev/null +++ b/drivers/firmware/efi/libstub/riscv.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +#include <linux/efi.h> +#include <linux/libfdt.h> + +#include <asm/efi.h> +#include <linux/unaligned.h> + +#include "efistub.h" + +typedef void __noreturn (*jump_kernel_func)(unsigned long, unsigned long); + +static unsigned long hartid; + +static int get_boot_hartid_from_fdt(void) +{ + const void *fdt; + int chosen_node, len; + const void *prop; + + fdt = get_efi_config_table(DEVICE_TREE_GUID); + if (!fdt) + return -EINVAL; + + chosen_node = fdt_path_offset(fdt, "/chosen"); + if (chosen_node < 0) + return -EINVAL; + + prop = fdt_getprop((void *)fdt, chosen_node, "boot-hartid", &len); + if (!prop) + return -EINVAL; + + if (len == sizeof(u32)) + hartid = (unsigned long) fdt32_to_cpu(*(fdt32_t *)prop); + else if (len == sizeof(u64)) + hartid = (unsigned long) fdt64_to_cpu(__get_unaligned_t(fdt64_t, prop)); + else + return -EINVAL; + + return 0; +} + +static efi_status_t get_boot_hartid_from_efi(void) +{ + efi_guid_t boot_protocol_guid = RISCV_EFI_BOOT_PROTOCOL_GUID; + struct riscv_efi_boot_protocol *boot_protocol; + efi_status_t status; + + status = efi_bs_call(locate_protocol, &boot_protocol_guid, NULL, + (void **)&boot_protocol); + if (status != EFI_SUCCESS) + return status; + return efi_call_proto(boot_protocol, get_boot_hartid, &hartid); +} + +efi_status_t check_platform_features(void) +{ + efi_status_t status; + int ret; + + status = get_boot_hartid_from_efi(); + if (status != EFI_SUCCESS) { + ret = get_boot_hartid_from_fdt(); + if (ret) { + efi_err("Failed to get boot hartid!\n"); + return EFI_UNSUPPORTED; + } + } + return EFI_SUCCESS; +} + +unsigned long __weak stext_offset(void) +{ + /* + * This fallback definition is used by the EFI zboot stub, which loads + * the entire image so it can branch via the image header at offset #0. + */ + return 0; +} + +void __noreturn efi_enter_kernel(unsigned long entrypoint, unsigned long fdt, + unsigned long fdt_size) +{ + unsigned long kernel_entry = entrypoint + stext_offset(); + jump_kernel_func jump_kernel = (jump_kernel_func)kernel_entry; + + /* + * Jump to real kernel here with following constraints. + * 1. MMU should be disabled. + * 2. a0 should contain hartid + * 3. a1 should DT address + */ + csr_write(CSR_SATP, 0); + jump_kernel(hartid, fdt); +} diff --git a/drivers/firmware/efi/libstub/screen_info.c b/drivers/firmware/efi/libstub/screen_info.c new file mode 100644 index 000000000000..5d3a1e32d177 --- /dev/null +++ b/drivers/firmware/efi/libstub/screen_info.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/efi.h> +#include <linux/screen_info.h> + +#include <asm/efi.h> + +#include "efistub.h" + +/* + * There are two ways of populating the core kernel's struct screen_info via the stub: + * - using a configuration table, like below, which relies on the EFI init code + * to locate the table and copy the contents; + * - by linking directly to the core kernel's copy of the global symbol. + * + * The latter is preferred because it makes the EFIFB earlycon available very + * early, but it only works if the EFI stub is part of the core kernel image + * itself. The zboot decompressor can only use the configuration table + * approach. + */ + +static efi_guid_t screen_info_guid = LINUX_EFI_SCREEN_INFO_TABLE_GUID; + +struct screen_info *__alloc_screen_info(void) +{ + struct screen_info *si; + efi_status_t status; + + status = efi_bs_call(allocate_pool, EFI_ACPI_RECLAIM_MEMORY, + sizeof(*si), (void **)&si); + + if (status != EFI_SUCCESS) + return NULL; + + memset(si, 0, sizeof(*si)); + + status = efi_bs_call(install_configuration_table, + &screen_info_guid, si); + if (status == EFI_SUCCESS) + return si; + + efi_bs_call(free_pool, si); + return NULL; +} + +void free_screen_info(struct screen_info *si) +{ + if (!si) + return; + + efi_bs_call(install_configuration_table, &screen_info_guid, NULL); + efi_bs_call(free_pool, si); +} diff --git a/drivers/firmware/efi/libstub/smbios.c b/drivers/firmware/efi/libstub/smbios.c new file mode 100644 index 000000000000..f31410d7e7e1 --- /dev/null +++ b/drivers/firmware/efi/libstub/smbios.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2022 Google LLC +// Author: Ard Biesheuvel <ardb@google.com> + +#include <linux/efi.h> + +#include "efistub.h" + +typedef union efi_smbios_protocol efi_smbios_protocol_t; + +union efi_smbios_protocol { + struct { + efi_status_t (__efiapi *add)(efi_smbios_protocol_t *, efi_handle_t, + u16 *, struct efi_smbios_record *); + efi_status_t (__efiapi *update_string)(efi_smbios_protocol_t *, u16 *, + unsigned long *, u8 *); + efi_status_t (__efiapi *remove)(efi_smbios_protocol_t *, u16); + efi_status_t (__efiapi *get_next)(efi_smbios_protocol_t *, u16 *, u8 *, + struct efi_smbios_record **, + efi_handle_t *); + + u8 major_version; + u8 minor_version; + }; + struct { + u32 add; + u32 update_string; + u32 remove; + u32 get_next; + + u8 major_version; + u8 minor_version; + } mixed_mode; +}; + +const struct efi_smbios_record *efi_get_smbios_record(u8 type) +{ + struct efi_smbios_record *record; + efi_smbios_protocol_t *smbios; + efi_status_t status; + u16 handle = 0xfffe; + + status = efi_bs_call(locate_protocol, &EFI_SMBIOS_PROTOCOL_GUID, NULL, + (void **)&smbios) ?: + efi_call_proto(smbios, get_next, &handle, &type, &record, NULL); + if (status != EFI_SUCCESS) + return NULL; + return record; +} + +const u8 *__efi_get_smbios_string(const struct efi_smbios_record *record, + const u8 *offset) +{ + const u8 *strtable; + + if (!record) + return NULL; + + strtable = (u8 *)record + record->length; + for (int i = 1; i < *offset; i++) { + int len = strlen(strtable); + + if (!len) + return NULL; + strtable += len + 1; + } + return strtable; +} diff --git a/drivers/firmware/efi/libstub/string.c b/drivers/firmware/efi/libstub/string.c index 5d13e43869ee..168fe8e79abc 100644 --- a/drivers/firmware/efi/libstub/string.c +++ b/drivers/firmware/efi/libstub/string.c @@ -11,7 +11,37 @@ #include <linux/types.h> #include <linux/string.h> -#ifndef __HAVE_ARCH_STRSTR +#ifndef EFI_HAVE_STRLEN +/** + * strlen - Find the length of a string + * @s: The string to be sized + */ +size_t strlen(const char *s) +{ + const char *sc; + + for (sc = s; *sc != '\0'; ++sc) + /* nothing */; + return sc - s; +} +#endif + +#ifndef EFI_HAVE_STRNLEN +/** + * strnlen - Find the length of a length-limited string + * @s: The string to be sized + * @count: The maximum number of bytes to search + */ +size_t strnlen(const char *s, size_t count) +{ + const char *sc; + + for (sc = s; count-- && *sc != '\0'; ++sc) + /* nothing */; + return sc - s; +} +#endif + /** * strstr - Find the first substring in a %NUL terminated string * @s1: The string to be searched @@ -33,9 +63,29 @@ char *strstr(const char *s1, const char *s2) } return NULL; } + +#ifndef EFI_HAVE_STRCMP +/** + * strcmp - Compare two strings + * @cs: One string + * @ct: Another string + */ +int strcmp(const char *cs, const char *ct) +{ + unsigned char c1, c2; + + while (1) { + c1 = *cs++; + c2 = *ct++; + if (c1 != c2) + return c1 < c2 ? -1 : 1; + if (!c1) + break; + } + return 0; +} #endif -#ifndef __HAVE_ARCH_STRNCMP /** * strncmp - Compare two length-limited strings * @cs: One string @@ -57,7 +107,6 @@ int strncmp(const char *cs, const char *ct, size_t count) } return 0; } -#endif /* Works only for digits and letters, but small and fast */ #define TOLOWER(x) ((x) | 0x20) @@ -113,3 +162,43 @@ long simple_strtol(const char *cp, char **endp, unsigned int base) return simple_strtoull(cp, endp, base); } + +#ifdef CONFIG_EFI_PARAMS_FROM_FDT +#ifndef EFI_HAVE_STRRCHR +/** + * strrchr - Find the last occurrence of a character in a string + * @s: The string to be searched + * @c: The character to search for + */ +char *strrchr(const char *s, int c) +{ + const char *last = NULL; + do { + if (*s == (char)c) + last = s; + } while (*s++); + return (char *)last; +} +#endif +#ifndef EFI_HAVE_MEMCHR +/** + * memchr - Find a character in an area of memory. + * @s: The memory area + * @c: The byte to search for + * @n: The size of the area. + * + * returns the address of the first occurrence of @c, or %NULL + * if @c is not found + */ +void *memchr(const void *s, int c, size_t n) +{ + const unsigned char *p = s; + while (n-- != 0) { + if ((unsigned char)c == *p++) { + return (void *)(p - 1); + } + } + return NULL; +} +#endif +#endif diff --git a/drivers/firmware/efi/libstub/tpm.c b/drivers/firmware/efi/libstub/tpm.c index 7acbac16eae0..a5c6c4f163fc 100644 --- a/drivers/firmware/efi/libstub/tpm.c +++ b/drivers/firmware/efi/libstub/tpm.c @@ -47,38 +47,17 @@ void efi_enable_reset_attack_mitigation(void) #endif -void efi_retrieve_tpm2_eventlog(void) +static void efi_retrieve_tcg2_eventlog(int version, efi_physical_addr_t log_location, + efi_physical_addr_t log_last_entry, + efi_bool_t truncated, + struct efi_tcg2_final_events_table *final_events_table) { - efi_guid_t tcg2_guid = EFI_TCG2_PROTOCOL_GUID; efi_guid_t linux_eventlog_guid = LINUX_EFI_TPM_EVENT_LOG_GUID; efi_status_t status; - efi_physical_addr_t log_location = 0, log_last_entry = 0; struct linux_efi_tpm_eventlog *log_tbl = NULL; - struct efi_tcg2_final_events_table *final_events_table = NULL; unsigned long first_entry_addr, last_entry_addr; size_t log_size, last_entry_size; - efi_bool_t truncated; - int version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_2; - efi_tcg2_protocol_t *tcg2_protocol = NULL; - int final_events_size = 0; - - status = efi_bs_call(locate_protocol, &tcg2_guid, NULL, - (void **)&tcg2_protocol); - if (status != EFI_SUCCESS) - return; - - status = efi_call_proto(tcg2_protocol, get_event_log, version, - &log_location, &log_last_entry, &truncated); - - if (status != EFI_SUCCESS || !log_location) { - version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2; - status = efi_call_proto(tcg2_protocol, get_event_log, version, - &log_location, &log_last_entry, - &truncated); - if (status != EFI_SUCCESS || !log_location) - return; - - } + u32 final_events_size = 0; first_entry_addr = (unsigned long) log_location; @@ -93,8 +72,10 @@ void efi_retrieve_tpm2_eventlog(void) * get_event_log only returns the address of the last entry. * We need to calculate its size to deduce the full size of * the logs. + * + * CC Event log also uses TCG2 format, handle it same as TPM2. */ - if (version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2) { + if (version > EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2) { /* * The TCG2 log format has variable length entries, * and the information to decode the hash algorithms @@ -115,7 +96,7 @@ void efi_retrieve_tpm2_eventlog(void) } /* Allocate space for the logs and copy them. */ - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, + status = efi_bs_call(allocate_pool, EFI_ACPI_RECLAIM_MEMORY, sizeof(*log_tbl) + log_size, (void **)&log_tbl); if (status != EFI_SUCCESS) { @@ -127,13 +108,11 @@ void efi_retrieve_tpm2_eventlog(void) * Figure out whether any events have already been logged to the * final events structure, and if so how much space they take up */ - if (version == EFI_TCG2_EVENT_LOG_FORMAT_TCG_2) - final_events_table = get_efi_config_table(LINUX_EFI_TPM_FINAL_LOG_GUID); if (final_events_table && final_events_table->nr_events) { struct tcg_pcr_event2_head *header; - int offset; + u32 offset; void *data; - int event_size; + u32 event_size; int i = final_events_table->nr_events; data = (void *)final_events_table; @@ -145,6 +124,9 @@ void efi_retrieve_tpm2_eventlog(void) event_size = __calc_tpm2_event_size(header, (void *)(long)log_location, false); + /* If calc fails this is a malformed log */ + if (!event_size) + break; final_events_size += event_size; i--; } @@ -165,3 +147,50 @@ void efi_retrieve_tpm2_eventlog(void) err_free: efi_bs_call(free_pool, log_tbl); } + +void efi_retrieve_eventlog(void) +{ + struct efi_tcg2_final_events_table *final_events_table = NULL; + efi_physical_addr_t log_location = 0, log_last_entry = 0; + efi_guid_t tpm2_guid = EFI_TCG2_PROTOCOL_GUID; + int version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_2; + efi_tcg2_protocol_t *tpm2 = NULL; + efi_bool_t truncated; + efi_status_t status; + + status = efi_bs_call(locate_protocol, &tpm2_guid, NULL, (void **)&tpm2); + if (status == EFI_SUCCESS) { + status = efi_call_proto(tpm2, get_event_log, version, &log_location, + &log_last_entry, &truncated); + + if (status != EFI_SUCCESS || !log_location) { + version = EFI_TCG2_EVENT_LOG_FORMAT_TCG_1_2; + status = efi_call_proto(tpm2, get_event_log, version, + &log_location, &log_last_entry, + &truncated); + } else { + final_events_table = + get_efi_config_table(EFI_TCG2_FINAL_EVENTS_TABLE_GUID); + } + } else { + efi_guid_t cc_guid = EFI_CC_MEASUREMENT_PROTOCOL_GUID; + efi_cc_protocol_t *cc = NULL; + + status = efi_bs_call(locate_protocol, &cc_guid, NULL, (void **)&cc); + if (status != EFI_SUCCESS) + return; + + version = EFI_CC_EVENT_LOG_FORMAT_TCG_2; + status = efi_call_proto(cc, get_event_log, version, &log_location, + &log_last_entry, &truncated); + + final_events_table = + get_efi_config_table(EFI_CC_FINAL_EVENTS_TABLE_GUID); + } + + if (status != EFI_SUCCESS || !log_location) + return; + + efi_retrieve_tcg2_eventlog(version, log_location, log_last_entry, + truncated, final_events_table); +} diff --git a/drivers/firmware/efi/libstub/unaccepted_memory.c b/drivers/firmware/efi/libstub/unaccepted_memory.c new file mode 100644 index 000000000000..757dbe734a47 --- /dev/null +++ b/drivers/firmware/efi/libstub/unaccepted_memory.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/efi.h> +#include <asm/efi.h> +#include "efistub.h" + +struct efi_unaccepted_memory *unaccepted_table; + +efi_status_t allocate_unaccepted_bitmap(__u32 nr_desc, + struct efi_boot_memmap *map) +{ + efi_guid_t unaccepted_table_guid = LINUX_EFI_UNACCEPTED_MEM_TABLE_GUID; + u64 unaccepted_start = ULLONG_MAX, unaccepted_end = 0, bitmap_size; + efi_status_t status; + int i; + + /* Check if the table is already installed */ + unaccepted_table = get_efi_config_table(unaccepted_table_guid); + if (unaccepted_table) { + if (unaccepted_table->version != 1) { + efi_err("Unknown version of unaccepted memory table\n"); + return EFI_UNSUPPORTED; + } + return EFI_SUCCESS; + } + + /* Check if there's any unaccepted memory and find the max address */ + for (i = 0; i < nr_desc; i++) { + efi_memory_desc_t *d; + unsigned long m = (unsigned long)map->map; + + d = efi_memdesc_ptr(m, map->desc_size, i); + if (d->type != EFI_UNACCEPTED_MEMORY) + continue; + + unaccepted_start = min(unaccepted_start, d->phys_addr); + unaccepted_end = max(unaccepted_end, + d->phys_addr + d->num_pages * PAGE_SIZE); + } + + if (unaccepted_start == ULLONG_MAX) + return EFI_SUCCESS; + + unaccepted_start = round_down(unaccepted_start, + EFI_UNACCEPTED_UNIT_SIZE); + unaccepted_end = round_up(unaccepted_end, EFI_UNACCEPTED_UNIT_SIZE); + + /* + * If unaccepted memory is present, allocate a bitmap to track what + * memory has to be accepted before access. + * + * One bit in the bitmap represents 2MiB in the address space: + * A 4k bitmap can track 64GiB of physical address space. + * + * In the worst case scenario -- a huge hole in the middle of the + * address space -- It needs 256MiB to handle 4PiB of the address + * space. + * + * The bitmap will be populated in setup_e820() according to the memory + * map after efi_exit_boot_services(). + */ + bitmap_size = DIV_ROUND_UP(unaccepted_end - unaccepted_start, + EFI_UNACCEPTED_UNIT_SIZE * BITS_PER_BYTE); + + status = efi_bs_call(allocate_pool, EFI_ACPI_RECLAIM_MEMORY, + sizeof(*unaccepted_table) + bitmap_size, + (void **)&unaccepted_table); + if (status != EFI_SUCCESS) { + efi_err("Failed to allocate unaccepted memory config table\n"); + return status; + } + + unaccepted_table->version = 1; + unaccepted_table->unit_size = EFI_UNACCEPTED_UNIT_SIZE; + unaccepted_table->phys_base = unaccepted_start; + unaccepted_table->size = bitmap_size; + memset(unaccepted_table->bitmap, 0, bitmap_size); + + status = efi_bs_call(install_configuration_table, + &unaccepted_table_guid, unaccepted_table); + if (status != EFI_SUCCESS) { + efi_bs_call(free_pool, unaccepted_table); + efi_err("Failed to install unaccepted memory config table!\n"); + } + + return status; +} + +/* + * The accepted memory bitmap only works at unit_size granularity. Take + * unaligned start/end addresses and either: + * 1. Accepts the memory immediately and in its entirety + * 2. Accepts unaligned parts, and marks *some* aligned part unaccepted + * + * The function will never reach the bitmap_set() with zero bits to set. + */ +void process_unaccepted_memory(u64 start, u64 end) +{ + u64 unit_size = unaccepted_table->unit_size; + u64 unit_mask = unaccepted_table->unit_size - 1; + u64 bitmap_size = unaccepted_table->size; + + /* + * Ensure that at least one bit will be set in the bitmap by + * immediately accepting all regions under 2*unit_size. This is + * imprecise and may immediately accept some areas that could + * have been represented in the bitmap. But, results in simpler + * code below + * + * Consider case like this (assuming unit_size == 2MB): + * + * | 4k | 2044k | 2048k | + * ^ 0x0 ^ 2MB ^ 4MB + * + * Only the first 4k has been accepted. The 0MB->2MB region can not be + * represented in the bitmap. The 2MB->4MB region can be represented in + * the bitmap. But, the 0MB->4MB region is <2*unit_size and will be + * immediately accepted in its entirety. + */ + if (end - start < 2 * unit_size) { + arch_accept_memory(start, end); + return; + } + + /* + * No matter how the start and end are aligned, at least one unaccepted + * unit_size area will remain to be marked in the bitmap. + */ + + /* Immediately accept a <unit_size piece at the start: */ + if (start & unit_mask) { + arch_accept_memory(start, round_up(start, unit_size)); + start = round_up(start, unit_size); + } + + /* Immediately accept a <unit_size piece at the end: */ + if (end & unit_mask) { + arch_accept_memory(round_down(end, unit_size), end); + end = round_down(end, unit_size); + } + + /* + * Accept part of the range that before phys_base and cannot be recorded + * into the bitmap. + */ + if (start < unaccepted_table->phys_base) { + arch_accept_memory(start, + min(unaccepted_table->phys_base, end)); + start = unaccepted_table->phys_base; + } + + /* Nothing to record */ + if (end < unaccepted_table->phys_base) + return; + + /* Translate to offsets from the beginning of the bitmap */ + start -= unaccepted_table->phys_base; + end -= unaccepted_table->phys_base; + + /* Accept memory that doesn't fit into bitmap */ + if (end > bitmap_size * unit_size * BITS_PER_BYTE) { + unsigned long phys_start, phys_end; + + phys_start = bitmap_size * unit_size * BITS_PER_BYTE + + unaccepted_table->phys_base; + phys_end = end + unaccepted_table->phys_base; + + arch_accept_memory(phys_start, phys_end); + end = bitmap_size * unit_size * BITS_PER_BYTE; + } + + /* + * 'start' and 'end' are now both unit_size-aligned. + * Record the range as being unaccepted: + */ + bitmap_set(unaccepted_table->bitmap, + start / unit_size, (end - start) / unit_size); +} + +void accept_memory(phys_addr_t start, unsigned long size) +{ + unsigned long range_start, range_end; + phys_addr_t end = start + size; + unsigned long bitmap_size; + u64 unit_size; + + if (!unaccepted_table) + return; + + unit_size = unaccepted_table->unit_size; + + /* + * Only care for the part of the range that is represented + * in the bitmap. + */ + if (start < unaccepted_table->phys_base) + start = unaccepted_table->phys_base; + if (end < unaccepted_table->phys_base) + return; + + /* Translate to offsets from the beginning of the bitmap */ + start -= unaccepted_table->phys_base; + end -= unaccepted_table->phys_base; + + /* Make sure not to overrun the bitmap */ + if (end > unaccepted_table->size * unit_size * BITS_PER_BYTE) + end = unaccepted_table->size * unit_size * BITS_PER_BYTE; + + range_start = start / unit_size; + bitmap_size = DIV_ROUND_UP(end, unit_size); + + for_each_set_bitrange_from(range_start, range_end, + unaccepted_table->bitmap, bitmap_size) { + unsigned long phys_start, phys_end; + + phys_start = range_start * unit_size + unaccepted_table->phys_base; + phys_end = range_end * unit_size + unaccepted_table->phys_base; + + arch_accept_memory(phys_start, phys_end); + bitmap_clear(unaccepted_table->bitmap, + range_start, range_end - range_start); + } +} diff --git a/drivers/firmware/efi/libstub/x86-5lvl.c b/drivers/firmware/efi/libstub/x86-5lvl.c new file mode 100644 index 000000000000..c00d0ae7ed5d --- /dev/null +++ b/drivers/firmware/efi/libstub/x86-5lvl.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/efi.h> + +#include <asm/boot.h> +#include <asm/desc.h> +#include <asm/efi.h> + +#include "efistub.h" +#include "x86-stub.h" + +bool efi_no5lvl; + +static void (*la57_toggle)(void *cr3); + +static const struct desc_struct gdt[] = { + [GDT_ENTRY_KERNEL32_CS] = GDT_ENTRY_INIT(DESC_CODE32, 0, 0xfffff), + [GDT_ENTRY_KERNEL_CS] = GDT_ENTRY_INIT(DESC_CODE64, 0, 0xfffff), +}; + +/* + * Enabling (or disabling) 5 level paging is tricky, because it can only be + * done from 32-bit mode with paging disabled. This means not only that the + * code itself must be running from 32-bit addressable physical memory, but + * also that the root page table must be 32-bit addressable, as programming + * a 64-bit value into CR3 when running in 32-bit mode is not supported. + */ +efi_status_t efi_setup_5level_paging(void) +{ + u8 tmpl_size = (u8 *)&trampoline_ljmp_imm_offset - (u8 *)&trampoline_32bit_src; + efi_status_t status; + u8 *la57_code; + + if (!efi_is_64bit()) + return EFI_SUCCESS; + + /* check for 5 level paging support */ + if (native_cpuid_eax(0) < 7 || + !(native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) + return EFI_SUCCESS; + + /* allocate some 32-bit addressable memory for code and a page table */ + status = efi_allocate_pages(2 * PAGE_SIZE, (unsigned long *)&la57_code, + U32_MAX); + if (status != EFI_SUCCESS) + return status; + + la57_toggle = memcpy(la57_code, trampoline_32bit_src, tmpl_size); + memset(la57_code + tmpl_size, 0x90, PAGE_SIZE - tmpl_size); + + /* + * To avoid the need to allocate a 32-bit addressable stack, the + * trampoline uses a LJMP instruction to switch back to long mode. + * LJMP takes an absolute destination address, which needs to be + * fixed up at runtime. + */ + *(u32 *)&la57_code[trampoline_ljmp_imm_offset] += (unsigned long)la57_code; + + efi_adjust_memory_range_protection((unsigned long)la57_toggle, PAGE_SIZE); + + return EFI_SUCCESS; +} + +void efi_5level_switch(void) +{ + bool want_la57 = !efi_no5lvl; + bool have_la57 = native_read_cr4() & X86_CR4_LA57; + bool need_toggle = want_la57 ^ have_la57; + u64 *pgt = (void *)la57_toggle + PAGE_SIZE; + pgd_t *cr3 = (pgd_t *)native_read_cr3_pa(); + u64 *new_cr3; + + if (!la57_toggle || !need_toggle) + return; + + if (!have_la57) { + /* + * 5 level paging will be enabled, so a root level page needs + * to be allocated from the 32-bit addressable physical region, + * with its first entry referring to the existing hierarchy. + */ + new_cr3 = memset(pgt, 0, PAGE_SIZE); + new_cr3[0] = (u64)cr3 | _PAGE_TABLE_NOENC; + } else { + /* take the new root table pointer from the current entry #0 */ + new_cr3 = (u64 *)(native_pgd_val(cr3[0]) & PTE_PFN_MASK); + + /* copy the new root table if it is not 32-bit addressable */ + if ((u64)new_cr3 > U32_MAX) + new_cr3 = memcpy(pgt, new_cr3, PAGE_SIZE); + } + + native_load_gdt(&(struct desc_ptr){ sizeof(gdt) - 1, (u64)gdt }); + + la57_toggle(new_cr3); +} diff --git a/drivers/firmware/efi/libstub/x86-stub.c b/drivers/firmware/efi/libstub/x86-stub.c index 33a7811e12c6..cef32e2c82d8 100644 --- a/drivers/firmware/efi/libstub/x86-stub.c +++ b/drivers/firmware/efi/libstub/x86-stub.c @@ -15,21 +15,34 @@ #include <asm/setup.h> #include <asm/desc.h> #include <asm/boot.h> +#include <asm/kaslr.h> +#include <asm/sev.h> #include "efistub.h" +#include "x86-stub.h" -/* Maximum physical address for 64-bit kernel with 4-level paging */ -#define MAXMEM_X86_64_4LEVEL (1ull << 46) +extern char _bss[], _ebss[]; const efi_system_table_t *efi_system_table; const efi_dxe_services_table_t *efi_dxe_table; -extern u32 image_offset; static efi_loaded_image_t *image = NULL; +static efi_memory_attribute_protocol_t *memattr; + +typedef union sev_memory_acceptance_protocol sev_memory_acceptance_protocol_t; +union sev_memory_acceptance_protocol { + struct { + efi_status_t (__efiapi * allow_unaccepted_memory)( + sev_memory_acceptance_protocol_t *); + }; + struct { + u32 allow_unaccepted_memory; + } mixed_mode; +}; static efi_status_t preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom) { - struct pci_setup_rom *rom = NULL; + struct pci_setup_rom *rom __free(efi_pool) = NULL; efi_status_t status; unsigned long size; uint64_t romsize; @@ -61,15 +74,14 @@ preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom) rom->data.type = SETUP_PCI; rom->data.len = size - sizeof(struct setup_data); rom->data.next = 0; - rom->pcilen = pci->romsize; - *__rom = rom; + rom->pcilen = romsize; status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16, PCI_VENDOR_ID, 1, &rom->vendor); if (status != EFI_SUCCESS) { efi_err("Failed to read rom->vendor\n"); - goto free_struct; + return status; } status = efi_call_proto(pci, pci.read, EfiPciIoWidthUint16, @@ -77,21 +89,18 @@ preserve_pci_rom_image(efi_pci_io_protocol_t *pci, struct pci_setup_rom **__rom) if (status != EFI_SUCCESS) { efi_err("Failed to read rom->devid\n"); - goto free_struct; + return status; } status = efi_call_proto(pci, get_location, &rom->segment, &rom->bus, &rom->device, &rom->function); if (status != EFI_SUCCESS) - goto free_struct; + return status; memcpy(rom->romdata, romimage, romsize); - return status; - -free_struct: - efi_bs_call(free_pool, rom); - return status; + *__rom = no_free_ptr(rom); + return EFI_SUCCESS; } /* @@ -106,38 +115,23 @@ free_struct: static void setup_efi_pci(struct boot_params *params) { efi_status_t status; - void **pci_handle = NULL; + efi_handle_t *pci_handle __free(efi_pool) = NULL; efi_guid_t pci_proto = EFI_PCI_IO_PROTOCOL_GUID; - unsigned long size = 0; struct setup_data *data; + unsigned long num; efi_handle_t h; - int i; - - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &pci_proto, NULL, &size, pci_handle); - - if (status == EFI_BUFFER_TOO_SMALL) { - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)&pci_handle); - - if (status != EFI_SUCCESS) { - efi_err("Failed to allocate memory for 'pci_handle'\n"); - return; - } - - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &pci_proto, NULL, &size, pci_handle); - } + status = efi_bs_call(locate_handle_buffer, EFI_LOCATE_BY_PROTOCOL, + &pci_proto, NULL, &num, &pci_handle); if (status != EFI_SUCCESS) - goto free_handle; + return; data = (struct setup_data *)(unsigned long)params->hdr.setup_data; while (data && data->next) data = (struct setup_data *)(unsigned long)data->next; - for_each_efi_handle(h, pci_handle, size, i) { + for_each_efi_handle(h, pci_handle, num) { efi_pci_io_protocol_t *pci = NULL; struct pci_setup_rom *rom; @@ -157,9 +151,6 @@ static void setup_efi_pci(struct boot_params *params) data = (struct setup_data *)rom; } - -free_handle: - efi_bs_call(free_pool, pci_handle); } static void retrieve_apple_device_properties(struct boot_params *boot_params) @@ -212,8 +203,169 @@ static void retrieve_apple_device_properties(struct boot_params *boot_params) } } -static void -adjust_memory_range_protection(unsigned long start, unsigned long size) +struct smbios_entry_point { + u8 anchor[4]; + u8 ep_checksum; + u8 ep_length; + u8 major_version; + u8 minor_version; + u16 max_size_entry; + u8 ep_rev; + u8 reserved[5]; + + struct __packed { + u8 anchor[5]; + u8 checksum; + u16 st_length; + u32 st_address; + u16 number_of_entries; + u8 bcd_rev; + } intm; +}; + +static bool verify_ep_checksum(const void *ptr, int length) +{ + u8 sum = 0; + + for (int i = 0; i < length; i++) + sum += ((u8 *)ptr)[i]; + + return sum == 0; +} + +static bool verify_ep_integrity(const struct smbios_entry_point *ep) +{ + if (memcmp(ep->anchor, "_SM_", sizeof(ep->anchor)) != 0) + return false; + + if (memcmp(ep->intm.anchor, "_DMI_", sizeof(ep->intm.anchor)) != 0) + return false; + + if (!verify_ep_checksum(ep, ep->ep_length) || + !verify_ep_checksum(&ep->intm, sizeof(ep->intm))) + return false; + + return true; +} + +static const struct efi_smbios_record *search_record(void *table, u32 length, + u8 type) +{ + const u8 *p, *end; + + p = (u8 *)table; + end = p + length; + + while (p + sizeof(struct efi_smbios_record) < end) { + const struct efi_smbios_record *hdr = + (struct efi_smbios_record *)p; + const u8 *next; + + if (hdr->type == type) + return hdr; + + /* Type 127 = End-of-Table */ + if (hdr->type == 0x7F) + return NULL; + + /* Jumping to the unformed section */ + next = p + hdr->length; + + /* Unformed section ends with 0000h */ + while ((next[0] != 0 || next[1] != 0) && next + 1 < end) + next++; + + next += 2; + p = next; + } + + return NULL; +} + +static const struct efi_smbios_record *get_table_record(u8 type) +{ + const struct smbios_entry_point *ep; + + /* + * Locate the legacy 32-bit SMBIOS entrypoint in memory, and parse it + * directly. Needed by some Macs that do not implement the EFI protocol. + */ + ep = get_efi_config_table(SMBIOS_TABLE_GUID); + if (!ep) + return NULL; + + if (!verify_ep_integrity(ep)) + return NULL; + + return search_record((void *)(unsigned long)ep->intm.st_address, + ep->intm.st_length, type); +} + +static bool apple_match_product_name(void) +{ + static const char type1_product_matches[][15] = { + "MacBookPro11,3", + "MacBookPro11,5", + "MacBookPro13,3", + "MacBookPro14,3", + "MacBookPro15,1", + "MacBookPro15,3", + "MacBookPro16,1", + "MacBookPro16,4", + }; + const struct efi_smbios_type1_record *record; + const u8 *product; + + record = (struct efi_smbios_type1_record *) + (efi_get_smbios_record(1) ?: get_table_record(1)); + if (!record) + return false; + + product = efi_get_smbios_string(record, product_name); + if (!product) + return false; + + for (int i = 0; i < ARRAY_SIZE(type1_product_matches); i++) { + if (!strcmp(product, type1_product_matches[i])) + return true; + } + + return false; +} + +static void apple_set_os(void) +{ + struct { + unsigned long version; + efi_status_t (__efiapi *set_os_version)(const char *); + efi_status_t (__efiapi *set_os_vendor)(const char *); + } *set_os; + efi_status_t status; + + if (!efi_is_64bit() || !apple_match_product_name()) + return; + + status = efi_bs_call(locate_protocol, &APPLE_SET_OS_PROTOCOL_GUID, NULL, + (void **)&set_os); + if (status != EFI_SUCCESS) + return; + + if (set_os->version >= 2) { + status = set_os->set_os_vendor("Apple Inc."); + if (status != EFI_SUCCESS) + efi_err("Failed to set OS vendor via apple_set_os\n"); + } + + if (set_os->version > 0) { + /* The version being set doesn't seem to matter */ + status = set_os->set_os_version("Mac OS X 10.9"); + if (status != EFI_SUCCESS) + efi_err("Failed to set OS version via apple_set_os\n"); + } +} + +efi_status_t efi_adjust_memory_range_protection(unsigned long start, + unsigned long size) { efi_status_t status; efi_gcd_memory_space_desc_t desc; @@ -221,14 +373,33 @@ adjust_memory_range_protection(unsigned long start, unsigned long size) unsigned long rounded_start, rounded_end; unsigned long unprotect_start, unprotect_size; - if (efi_dxe_table == NULL) - return; - rounded_start = rounddown(start, EFI_PAGE_SIZE); rounded_end = roundup(start + size, EFI_PAGE_SIZE); + if (memattr != NULL) { + status = efi_call_proto(memattr, set_memory_attributes, + rounded_start, + rounded_end - rounded_start, + EFI_MEMORY_RO); + if (status != EFI_SUCCESS) { + efi_warn("Failed to set EFI_MEMORY_RO attribute\n"); + return status; + } + + status = efi_call_proto(memattr, clear_memory_attributes, + rounded_start, + rounded_end - rounded_start, + EFI_MEMORY_XP); + if (status != EFI_SUCCESS) + efi_warn("Failed to clear EFI_MEMORY_XP attribute\n"); + return status; + } + + if (efi_dxe_table == NULL) + return EFI_SUCCESS; + /* - * Don't modify memory region attributes, they are + * Don't modify memory region attributes, if they are * already suitable, to lower the possibility to * encounter firmware bugs. */ @@ -238,16 +409,18 @@ adjust_memory_range_protection(unsigned long start, unsigned long size) status = efi_dxe_call(get_memory_space_descriptor, start, &desc); if (status != EFI_SUCCESS) - return; + break; next = desc.base_address + desc.length; /* - * Only system memory is suitable for trampoline/kernel image placement, - * so only this type of memory needs its attributes to be modified. + * Only system memory and more reliable memory are suitable for + * trampoline/kernel image placement. So only those memory types + * may need to have attributes modified. */ - if (desc.gcd_memory_type != EfiGcdMemoryTypeSystemMemory || + if ((desc.gcd_memory_type != EfiGcdMemoryTypeSystemMemory && + desc.gcd_memory_type != EfiGcdMemoryTypeMoreReliable) || (desc.attributes & (EFI_MEMORY_RO | EFI_MEMORY_XP)) == 0) continue; @@ -263,181 +436,62 @@ adjust_memory_range_protection(unsigned long start, unsigned long size) unprotect_start, unprotect_start + unprotect_size, status); + break; } } + return EFI_SUCCESS; } -/* - * Trampoline takes 2 pages and can be loaded in first megabyte of memory - * with its end placed between 128k and 640k where BIOS might start. - * (see arch/x86/boot/compressed/pgtable_64.c) - * - * We cannot find exact trampoline placement since memory map - * can be modified by UEFI, and it can alter the computed address. - */ - -#define TRAMPOLINE_PLACEMENT_BASE ((128 - 8)*1024) -#define TRAMPOLINE_PLACEMENT_SIZE (640*1024 - (128 - 8)*1024) +static void setup_unaccepted_memory(void) +{ + efi_guid_t mem_acceptance_proto = OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID; + sev_memory_acceptance_protocol_t *proto; + efi_status_t status; -void startup_32(struct boot_params *boot_params); + if (!IS_ENABLED(CONFIG_UNACCEPTED_MEMORY)) + return; -static void -setup_memory_protection(unsigned long image_base, unsigned long image_size) -{ /* - * Allow execution of possible trampoline used - * for switching between 4- and 5-level page tables - * and relocated kernel image. + * Enable unaccepted memory before calling exit boot services in order + * for the UEFI to not accept all memory on EBS. */ + status = efi_bs_call(locate_protocol, &mem_acceptance_proto, NULL, + (void **)&proto); + if (status != EFI_SUCCESS) + return; - adjust_memory_range_protection(TRAMPOLINE_PLACEMENT_BASE, - TRAMPOLINE_PLACEMENT_SIZE); + status = efi_call_proto(proto, allow_unaccepted_memory); + if (status != EFI_SUCCESS) + efi_err("Memory acceptance protocol failed\n"); +} -#ifdef CONFIG_64BIT - if (image_base != (unsigned long)startup_32) - adjust_memory_range_protection(image_base, image_size); -#else - /* - * Clear protection flags on a whole range of possible - * addresses used for KASLR. We don't need to do that - * on x86_64, since KASLR/extraction is performed after - * dedicated identity page tables are built and we only - * need to remove possible protection on relocated image - * itself disregarding further relocations. - */ - adjust_memory_range_protection(LOAD_PHYSICAL_ADDR, - KERNEL_IMAGE_SIZE - LOAD_PHYSICAL_ADDR); -#endif +static efi_char16_t *efistub_fw_vendor(void) +{ + unsigned long vendor = efi_table_attr(efi_system_table, fw_vendor); + + return (efi_char16_t *)vendor; } static const efi_char16_t apple[] = L"Apple"; -static void setup_quirks(struct boot_params *boot_params, - unsigned long image_base, - unsigned long image_size) +static void setup_quirks(struct boot_params *boot_params) { - efi_char16_t *fw_vendor = (efi_char16_t *)(unsigned long) - efi_table_attr(efi_system_table, fw_vendor); - - if (!memcmp(fw_vendor, apple, sizeof(apple))) { + if (!memcmp(efistub_fw_vendor(), apple, sizeof(apple))) { if (IS_ENABLED(CONFIG_APPLE_PROPERTIES)) retrieve_apple_device_properties(boot_params); - } - - if (IS_ENABLED(CONFIG_EFI_DXE_MEM_ATTRIBUTES)) - setup_memory_protection(image_base, image_size); -} - -/* - * See if we have Universal Graphics Adapter (UGA) protocol - */ -static efi_status_t -setup_uga(struct screen_info *si, efi_guid_t *uga_proto, unsigned long size) -{ - efi_status_t status; - u32 width, height; - void **uga_handle = NULL; - efi_uga_draw_protocol_t *uga = NULL, *first_uga; - efi_handle_t handle; - int i; - - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, size, - (void **)&uga_handle); - if (status != EFI_SUCCESS) - return status; - - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - uga_proto, NULL, &size, uga_handle); - if (status != EFI_SUCCESS) - goto free_handle; - - height = 0; - width = 0; - - first_uga = NULL; - for_each_efi_handle(handle, uga_handle, size, i) { - efi_guid_t pciio_proto = EFI_PCI_IO_PROTOCOL_GUID; - u32 w, h, depth, refresh; - void *pciio; - status = efi_bs_call(handle_protocol, handle, uga_proto, - (void **)&uga); - if (status != EFI_SUCCESS) - continue; - - pciio = NULL; - efi_bs_call(handle_protocol, handle, &pciio_proto, &pciio); - - status = efi_call_proto(uga, get_mode, &w, &h, &depth, &refresh); - if (status == EFI_SUCCESS && (!first_uga || pciio)) { - width = w; - height = h; - - /* - * Once we've found a UGA supporting PCIIO, - * don't bother looking any further. - */ - if (pciio) - break; - - first_uga = uga; - } + apple_set_os(); } - - if (!width && !height) - goto free_handle; - - /* EFI framebuffer */ - si->orig_video_isVGA = VIDEO_TYPE_EFI; - - si->lfb_depth = 32; - si->lfb_width = width; - si->lfb_height = height; - - si->red_size = 8; - si->red_pos = 16; - si->green_size = 8; - si->green_pos = 8; - si->blue_size = 8; - si->blue_pos = 0; - si->rsvd_size = 8; - si->rsvd_pos = 24; - -free_handle: - efi_bs_call(free_pool, uga_handle); - - return status; } static void setup_graphics(struct boot_params *boot_params) { - efi_guid_t graphics_proto = EFI_GRAPHICS_OUTPUT_PROTOCOL_GUID; - struct screen_info *si; - efi_guid_t uga_proto = EFI_UGA_PROTOCOL_GUID; - efi_status_t status; - unsigned long size; - void **gop_handle = NULL; - void **uga_handle = NULL; - - si = &boot_params->screen_info; - memset(si, 0, sizeof(*si)); + struct screen_info *si = memset(&boot_params->screen_info, 0, sizeof(*si)); + struct edid_info *edid = memset(&boot_params->edid_info, 0, sizeof(*edid)); - size = 0; - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &graphics_proto, NULL, &size, gop_handle); - if (status == EFI_BUFFER_TOO_SMALL) - status = efi_setup_gop(si, &graphics_proto, size); - - if (status != EFI_SUCCESS) { - size = 0; - status = efi_bs_call(locate_handle, EFI_LOCATE_BY_PROTOCOL, - &uga_proto, NULL, &size, uga_handle); - if (status == EFI_BUFFER_TOO_SMALL) - setup_uga(si, &uga_proto, size); - } + efi_setup_graphics(si, edid); } - static void __noreturn efi_exit(efi_handle_t handle, efi_status_t status) { efi_bs_call(exit, handle, status, 0, NULL); @@ -445,91 +499,53 @@ static void __noreturn efi_exit(efi_handle_t handle, efi_status_t status) asm("hlt"); } -void __noreturn efi_stub_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg, - struct boot_params *boot_params); - /* * Because the x86 boot code expects to be passed a boot_params we * need to create one ourselves (usually the bootloader would create * one for us). */ -efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, - efi_system_table_t *sys_table_arg) +static efi_status_t efi_allocate_bootparams(efi_handle_t handle, + struct boot_params **bp) { + efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; struct boot_params *boot_params; struct setup_header *hdr; - void *image_base; - efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; - int options_size = 0; efi_status_t status; + unsigned long alloc; char *cmdline_ptr; - efi_system_table = sys_table_arg; - - /* Check if we were booted by the EFI firmware */ - if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) - efi_exit(handle, EFI_INVALID_PARAMETER); - status = efi_bs_call(handle_protocol, handle, &proto, (void **)&image); if (status != EFI_SUCCESS) { efi_err("Failed to get handle for LOADED_IMAGE_PROTOCOL\n"); - efi_exit(handle, status); - } - - image_base = efi_table_attr(image, image_base); - image_offset = (void *)startup_32 - image_base; - - status = efi_allocate_pages(sizeof(struct boot_params), - (unsigned long *)&boot_params, ULONG_MAX); - if (status != EFI_SUCCESS) { - efi_err("Failed to allocate lowmem for boot params\n"); - efi_exit(handle, status); + return status; } - memset(boot_params, 0x0, sizeof(struct boot_params)); - - hdr = &boot_params->hdr; + status = efi_allocate_pages(PARAM_SIZE, &alloc, ULONG_MAX); + if (status != EFI_SUCCESS) + return status; - /* Copy the setup header from the second sector to boot_params */ - memcpy(&hdr->jump, image_base + 512, - sizeof(struct setup_header) - offsetof(struct setup_header, jump)); + boot_params = memset((void *)alloc, 0x0, PARAM_SIZE); + hdr = &boot_params->hdr; - /* - * Fill out some of the header fields ourselves because the - * EFI firmware loader doesn't load the first sector. - */ + /* Assign the setup_header fields that the kernel actually cares about */ hdr->root_flags = 1; hdr->vid_mode = 0xffff; - hdr->boot_flag = 0xAA55; hdr->type_of_loader = 0x21; + hdr->initrd_addr_max = INT_MAX; /* Convert unicode cmdline to ascii */ - cmdline_ptr = efi_convert_cmdline(image, &options_size); - if (!cmdline_ptr) - goto fail; - - efi_set_u64_split((unsigned long)cmdline_ptr, - &hdr->cmd_line_ptr, &boot_params->ext_cmd_line_ptr); - - hdr->ramdisk_image = 0; - hdr->ramdisk_size = 0; - - /* - * Disregard any setup data that was provided by the bootloader: - * setup_data could be pointing anywhere, and we have no way of - * authenticating or validating the payload. - */ - hdr->setup_data = 0; - - efi_stub_entry(handle, sys_table_arg, boot_params); - /* not reached */ + cmdline_ptr = efi_convert_cmdline(image); + if (!cmdline_ptr) { + efi_free(PARAM_SIZE, alloc); + return EFI_OUT_OF_RESOURCES; + } -fail: - efi_free(sizeof(struct boot_params), (unsigned long)boot_params); + efi_set_u64_split((unsigned long)cmdline_ptr, &hdr->cmd_line_ptr, + &boot_params->ext_cmd_line_ptr); - efi_exit(handle, status); + *bp = boot_params; + return EFI_SUCCESS; } static void add_e820ext(struct boot_params *params, @@ -574,7 +590,7 @@ setup_e820(struct boot_params *params, struct setup_data *e820ext, u32 e820ext_s m |= (u64)efi->efi_memmap_hi << 32; #endif - d = efi_early_memdesc_ptr(m, efi->efi_memdesc_size, i); + d = efi_memdesc_ptr(m, efi->efi_memdesc_size, i); switch (d->type) { case EFI_RESERVED_TYPE: case EFI_RUNTIME_SERVICES_CODE: @@ -613,6 +629,13 @@ setup_e820(struct boot_params *params, struct setup_data *e820ext, u32 e820ext_s e820_type = E820_TYPE_PMEM; break; + case EFI_UNACCEPTED_MEMORY: + if (!IS_ENABLED(CONFIG_UNACCEPTED_MEMORY)) + continue; + e820_type = E820_TYPE_RAM; + process_unaccepted_memory(d->phys_addr, + d->phys_addr + PAGE_SIZE * d->num_pages); + break; default: continue; } @@ -681,27 +704,27 @@ static efi_status_t allocate_e820(struct boot_params *params, struct setup_data **e820ext, u32 *e820ext_size) { - unsigned long map_size, desc_size, map_key; + struct efi_boot_memmap *map __free(efi_pool) = NULL; efi_status_t status; - __u32 nr_desc, desc_version; - - /* Only need the size of the mem map and size of each mem descriptor */ - map_size = 0; - status = efi_bs_call(get_memory_map, &map_size, NULL, &map_key, - &desc_size, &desc_version); - if (status != EFI_BUFFER_TOO_SMALL) - return (status != EFI_SUCCESS) ? status : EFI_UNSUPPORTED; + __u32 nr_desc; - nr_desc = map_size / desc_size + EFI_MMAP_NR_SLACK_SLOTS; + status = efi_get_memory_map(&map, false); + if (status != EFI_SUCCESS) + return status; - if (nr_desc > ARRAY_SIZE(params->e820_table)) { - u32 nr_e820ext = nr_desc - ARRAY_SIZE(params->e820_table); + nr_desc = map->map_size / map->desc_size; + if (nr_desc > ARRAY_SIZE(params->e820_table) - EFI_MMAP_NR_SLACK_SLOTS) { + u32 nr_e820ext = nr_desc - ARRAY_SIZE(params->e820_table) + + EFI_MMAP_NR_SLACK_SLOTS; status = alloc_e820ext(nr_e820ext, e820ext, e820ext_size); if (status != EFI_SUCCESS) return status; } + if (IS_ENABLED(CONFIG_UNACCEPTED_MEMORY)) + return allocate_unaccepted_bitmap(nr_desc, map); + return EFI_SUCCESS; } @@ -760,19 +783,141 @@ static efi_status_t exit_boot(struct boot_params *boot_params, void *handle) return EFI_SUCCESS; } +static bool have_unsupported_snp_features(void) +{ + u64 unsupported; + + unsupported = snp_get_unsupported_features(sev_get_status()); + if (unsupported) { + efi_err("Unsupported SEV-SNP features detected: 0x%llx\n", + unsupported); + return true; + } + return false; +} + +static void efi_get_seed(void *seed, int size) +{ + efi_get_random_bytes(size, seed); + + /* + * This only updates seed[0] when running on 32-bit, but in that case, + * seed[1] is not used anyway, as there is no virtual KASLR on 32-bit. + */ + *(unsigned long *)seed ^= kaslr_get_random_long("EFI"); +} + +static void error(char *str) +{ + efi_warn("Decompression failed: %s\n", str); +} + +static const char *cmdline_memmap_override; + +static efi_status_t parse_options(const char *cmdline) +{ + static const char opts[][14] = { + "mem=", "memmap=", "hugepages=" + }; + + for (int i = 0; i < ARRAY_SIZE(opts); i++) { + const char *p = strstr(cmdline, opts[i]); + + if (p == cmdline || (p > cmdline && isspace(p[-1]))) { + cmdline_memmap_override = opts[i]; + break; + } + } + + return efi_parse_options(cmdline); +} + +static efi_status_t efi_decompress_kernel(unsigned long *kernel_entry, + struct boot_params *boot_params) +{ + unsigned long virt_addr = LOAD_PHYSICAL_ADDR; + unsigned long addr, alloc_size, entry; + efi_status_t status; + u32 seed[2] = {}; + + boot_params_ptr = boot_params; + + /* determine the required size of the allocation */ + alloc_size = ALIGN(max_t(unsigned long, output_len, kernel_total_size), + MIN_KERNEL_ALIGN); + + if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && !efi_nokaslr) { + u64 range = KERNEL_IMAGE_SIZE - LOAD_PHYSICAL_ADDR - kernel_total_size; + static const efi_char16_t ami[] = L"American Megatrends"; + + efi_get_seed(seed, sizeof(seed)); + + virt_addr += (range * seed[1]) >> 32; + virt_addr &= ~(CONFIG_PHYSICAL_ALIGN - 1); + + /* + * Older Dell systems with AMI UEFI firmware v2.0 may hang + * while decompressing the kernel if physical address + * randomization is enabled. + * + * https://bugzilla.kernel.org/show_bug.cgi?id=218173 + */ + if (efi_system_table->hdr.revision <= EFI_2_00_SYSTEM_TABLE_REVISION && + !memcmp(efistub_fw_vendor(), ami, sizeof(ami))) { + efi_debug("AMI firmware v2.0 or older detected - disabling physical KASLR\n"); + seed[0] = 0; + } else if (cmdline_memmap_override) { + efi_info("%s detected on the kernel command line - disabling physical KASLR\n", + cmdline_memmap_override); + seed[0] = 0; + } + + boot_params->hdr.loadflags |= KASLR_FLAG; + } + + status = efi_random_alloc(alloc_size, CONFIG_PHYSICAL_ALIGN, &addr, + seed[0], EFI_LOADER_CODE, + LOAD_PHYSICAL_ADDR, + EFI_X86_KERNEL_ALLOC_LIMIT); + if (status != EFI_SUCCESS) + return status; + + entry = decompress_kernel((void *)addr, virt_addr, error); + if (entry == ULONG_MAX) { + efi_free(alloc_size, addr); + return EFI_LOAD_ERROR; + } + + *kernel_entry = addr + entry; + + return efi_adjust_memory_range_protection(addr, kernel_text_size) ?: + efi_adjust_memory_range_protection(addr + kernel_inittext_offset, + kernel_inittext_size); +} + +static void __noreturn enter_kernel(unsigned long kernel_addr, + struct boot_params *boot_params) +{ + /* enter decompressed kernel with boot_params pointer in RSI/ESI */ + asm("jmp *%0"::"r"(kernel_addr), "S"(boot_params)); + + unreachable(); +} + /* - * On success, we return the address of startup_32, which has potentially been - * relocated by efi_relocate_kernel. - * On failure, we exit to the firmware via efi_exit instead of returning. + * On success, this routine will jump to the relocated image directly and never + * return. On failure, it will exit to the firmware via efi_exit() instead of + * returning. */ -asmlinkage unsigned long efi_main(efi_handle_t handle, - efi_system_table_t *sys_table_arg, - struct boot_params *boot_params) +void __noreturn efi_stub_entry(efi_handle_t handle, + efi_system_table_t *sys_table_arg, + struct boot_params *boot_params) + { - unsigned long bzimage_addr = (unsigned long)startup_32; - unsigned long buffer_start, buffer_end; - struct setup_header *hdr = &boot_params->hdr; + efi_guid_t guid = EFI_MEMORY_ATTRIBUTE_PROTOCOL_GUID; const struct linux_efi_initrd *initrd = NULL; + unsigned long kernel_entry; + struct setup_header *hdr; efi_status_t status; efi_system_table = sys_table_arg; @@ -780,69 +925,37 @@ asmlinkage unsigned long efi_main(efi_handle_t handle, if (efi_system_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) efi_exit(handle, EFI_INVALID_PARAMETER); - efi_dxe_table = get_efi_config_table(EFI_DXE_SERVICES_TABLE_GUID); - if (efi_dxe_table && - efi_dxe_table->hdr.signature != EFI_DXE_SERVICES_TABLE_SIGNATURE) { - efi_warn("Ignoring DXE services table: invalid signature\n"); - efi_dxe_table = NULL; + if (!IS_ENABLED(CONFIG_EFI_HANDOVER_PROTOCOL) || !boot_params) { + status = efi_allocate_bootparams(handle, &boot_params); + if (status != EFI_SUCCESS) + efi_exit(handle, status); } - /* - * If the kernel isn't already loaded at a suitable address, - * relocate it. - * - * It must be loaded above LOAD_PHYSICAL_ADDR. - * - * The maximum address for 64-bit is 1 << 46 for 4-level paging. This - * is defined as the macro MAXMEM, but unfortunately that is not a - * compile-time constant if 5-level paging is configured, so we instead - * define our own macro for use here. - * - * For 32-bit, the maximum address is complicated to figure out, for - * now use KERNEL_IMAGE_SIZE, which will be 512MiB, the same as what - * KASLR uses. - * - * Also relocate it if image_offset is zero, i.e. the kernel wasn't - * loaded by LoadImage, but rather by a bootloader that called the - * handover entry. The reason we must always relocate in this case is - * to handle the case of systemd-boot booting a unified kernel image, - * which is a PE executable that contains the bzImage and an initrd as - * COFF sections. The initrd section is placed after the bzImage - * without ensuring that there are at least init_size bytes available - * for the bzImage, and thus the compressed kernel's startup code may - * overwrite the initrd unless it is moved out of the way. - */ + hdr = &boot_params->hdr; - buffer_start = ALIGN(bzimage_addr - image_offset, - hdr->kernel_alignment); - buffer_end = buffer_start + hdr->init_size; - - if ((buffer_start < LOAD_PHYSICAL_ADDR) || - (IS_ENABLED(CONFIG_X86_32) && buffer_end > KERNEL_IMAGE_SIZE) || - (IS_ENABLED(CONFIG_X86_64) && buffer_end > MAXMEM_X86_64_4LEVEL) || - (image_offset == 0)) { - extern char _bss[]; - - status = efi_relocate_kernel(&bzimage_addr, - (unsigned long)_bss - bzimage_addr, - hdr->init_size, - hdr->pref_address, - hdr->kernel_alignment, - LOAD_PHYSICAL_ADDR); - if (status != EFI_SUCCESS) { - efi_err("efi_relocate_kernel() failed!\n"); - goto fail; + if (have_unsupported_snp_features()) + efi_exit(handle, EFI_UNSUPPORTED); + + if (IS_ENABLED(CONFIG_EFI_DXE_MEM_ATTRIBUTES)) { + efi_dxe_table = get_efi_config_table(EFI_DXE_SERVICES_TABLE_GUID); + if (efi_dxe_table && + efi_dxe_table->hdr.signature != EFI_DXE_SERVICES_TABLE_SIGNATURE) { + efi_warn("Ignoring DXE services table: invalid signature\n"); + efi_dxe_table = NULL; } - /* - * Now that we've copied the kernel elsewhere, we no longer - * have a set up block before startup_32(), so reset image_offset - * to zero in case it was set earlier. - */ - image_offset = 0; + } + + /* grab the memory attributes protocol if it exists */ + efi_bs_call(locate_protocol, &guid, NULL, (void **)&memattr); + + status = efi_setup_5level_paging(); + if (status != EFI_SUCCESS) { + efi_err("efi_setup_5level_paging() failed!\n"); + goto fail; } #ifdef CONFIG_CMDLINE_BOOL - status = efi_parse_options(CONFIG_CMDLINE); + status = parse_options(CONFIG_CMDLINE); if (status != EFI_SUCCESS) { efi_err("Failed to parse options\n"); goto fail; @@ -851,13 +964,22 @@ asmlinkage unsigned long efi_main(efi_handle_t handle, if (!IS_ENABLED(CONFIG_CMDLINE_OVERRIDE)) { unsigned long cmdline_paddr = ((u64)hdr->cmd_line_ptr | ((u64)boot_params->ext_cmd_line_ptr << 32)); - status = efi_parse_options((char *)cmdline_paddr); + status = parse_options((char *)cmdline_paddr); if (status != EFI_SUCCESS) { efi_err("Failed to parse options\n"); goto fail; } } + if (efi_mem_encrypt > 0) + hdr->xloadflags |= XLF_MEM_ENCRYPTION; + + status = efi_decompress_kernel(&kernel_entry, boot_params); + if (status != EFI_SUCCESS) { + efi_err("Failed to decompress kernel\n"); + goto fail; + } + /* * At this point, an initrd may already have been loaded by the * bootloader and passed via bootparams. We permit an initrd loaded @@ -891,13 +1013,15 @@ asmlinkage unsigned long efi_main(efi_handle_t handle, efi_random_get_seed(); - efi_retrieve_tpm2_eventlog(); + efi_retrieve_eventlog(); setup_graphics(boot_params); setup_efi_pci(boot_params); - setup_quirks(boot_params, bzimage_addr, buffer_end - buffer_start); + setup_quirks(boot_params); + + setup_unaccepted_memory(); status = exit_boot(boot_params, handle); if (status != EFI_SUCCESS) { @@ -905,9 +1029,42 @@ asmlinkage unsigned long efi_main(efi_handle_t handle, goto fail; } - return bzimage_addr; + /* + * Call the SEV init code while still running with the firmware's + * GDT/IDT, so #VC exceptions will be handled by EFI. + */ + sev_enable(boot_params); + + efi_5level_switch(); + + enter_kernel(kernel_entry, boot_params); fail: - efi_err("efi_main() failed!\n"); + efi_err("efi_stub_entry() failed!\n"); efi_exit(handle, status); } + +efi_status_t __efiapi efi_pe_entry(efi_handle_t handle, + efi_system_table_t *sys_table_arg) +{ + efi_stub_entry(handle, sys_table_arg, NULL); +} + +#ifdef CONFIG_EFI_HANDOVER_PROTOCOL +void efi_handover_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg, + struct boot_params *boot_params) +{ + memset(_bss, 0, _ebss - _bss); + efi_stub_entry(handle, sys_table_arg, boot_params); +} + +#ifndef CONFIG_EFI_MIXED +extern __alias(efi_handover_entry) +void efi32_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg, + struct boot_params *boot_params); + +extern __alias(efi_handover_entry) +void efi64_stub_entry(efi_handle_t handle, efi_system_table_t *sys_table_arg, + struct boot_params *boot_params); +#endif +#endif diff --git a/drivers/firmware/efi/libstub/x86-stub.h b/drivers/firmware/efi/libstub/x86-stub.h new file mode 100644 index 000000000000..1c20e99a6494 --- /dev/null +++ b/drivers/firmware/efi/libstub/x86-stub.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include <linux/efi.h> + +extern void trampoline_32bit_src(void *, bool); +extern const u16 trampoline_ljmp_imm_offset; + +efi_status_t efi_adjust_memory_range_protection(unsigned long start, + unsigned long size); + +#ifdef CONFIG_X86_64 +efi_status_t efi_setup_5level_paging(void); +void efi_5level_switch(void); +#else +static inline efi_status_t efi_setup_5level_paging(void) { return EFI_SUCCESS; } +static inline void efi_5level_switch(void) {} +#endif diff --git a/drivers/firmware/efi/libstub/zboot-decompress-gzip.c b/drivers/firmware/efi/libstub/zboot-decompress-gzip.c new file mode 100644 index 000000000000..e97a7e9d3c98 --- /dev/null +++ b/drivers/firmware/efi/libstub/zboot-decompress-gzip.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/efi.h> +#include <linux/zlib.h> + +#include <asm/efi.h> + +#include "efistub.h" + +#include "inftrees.c" +#include "inffast.c" +#include "inflate.c" + +extern unsigned char _gzdata_start[], _gzdata_end[]; +extern u32 __aligned(1) payload_size; + +static struct z_stream_s stream; + +efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size) +{ + efi_status_t status; + int rc; + + /* skip the 10 byte header, assume no recorded filename */ + stream.next_in = _gzdata_start + 10; + stream.avail_in = _gzdata_end - stream.next_in; + + status = efi_allocate_pages(zlib_inflate_workspacesize(), + (unsigned long *)&stream.workspace, + ULONG_MAX); + if (status != EFI_SUCCESS) + return status; + + rc = zlib_inflateInit2(&stream, -MAX_WBITS); + if (rc != Z_OK) { + efi_err("failed to initialize GZIP decompressor: %d\n", rc); + status = EFI_LOAD_ERROR; + goto out; + } + + *alloc_size = payload_size; + return EFI_SUCCESS; +out: + efi_free(zlib_inflate_workspacesize(), (unsigned long)stream.workspace); + return status; +} + +efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen) +{ + int rc; + + stream.next_out = out; + stream.avail_out = outlen; + + rc = zlib_inflate(&stream, 0); + zlib_inflateEnd(&stream); + + efi_free(zlib_inflate_workspacesize(), (unsigned long)stream.workspace); + + if (rc != Z_STREAM_END) { + efi_err("GZIP decompression failed with status %d\n", rc); + return EFI_LOAD_ERROR; + } + + efi_cache_sync_image((unsigned long)out, outlen); + + return EFI_SUCCESS; +} diff --git a/drivers/firmware/efi/libstub/zboot-decompress-zstd.c b/drivers/firmware/efi/libstub/zboot-decompress-zstd.c new file mode 100644 index 000000000000..bde9d94dd2e3 --- /dev/null +++ b/drivers/firmware/efi/libstub/zboot-decompress-zstd.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/efi.h> +#include <linux/zstd.h> + +#include <asm/efi.h> + +#include "decompress_sources.h" +#include "efistub.h" + +extern unsigned char _gzdata_start[], _gzdata_end[]; +extern u32 __aligned(1) payload_size; + +static size_t wksp_size; +static void *wksp; + +efi_status_t efi_zboot_decompress_init(unsigned long *alloc_size) +{ + efi_status_t status; + + wksp_size = zstd_dctx_workspace_bound(); + status = efi_allocate_pages(wksp_size, (unsigned long *)&wksp, ULONG_MAX); + if (status != EFI_SUCCESS) + return status; + + *alloc_size = payload_size; + return EFI_SUCCESS; +} + +efi_status_t efi_zboot_decompress(u8 *out, unsigned long outlen) +{ + zstd_dctx *dctx = zstd_init_dctx(wksp, wksp_size); + size_t ret; + int retval; + + ret = zstd_decompress_dctx(dctx, out, outlen, _gzdata_start, + _gzdata_end - _gzdata_start - 4); + efi_free(wksp_size, (unsigned long)wksp); + + retval = zstd_get_error_code(ret); + if (retval) { + efi_err("ZSTD-decompression failed with status %d\n", retval); + return EFI_LOAD_ERROR; + } + + efi_cache_sync_image((unsigned long)out, outlen); + + return EFI_SUCCESS; +} diff --git a/drivers/firmware/efi/libstub/zboot-header.S b/drivers/firmware/efi/libstub/zboot-header.S index 9e6fe061ab07..b6431edd0fc9 100644 --- a/drivers/firmware/efi/libstub/zboot-header.S +++ b/drivers/firmware/efi/libstub/zboot-header.S @@ -4,27 +4,28 @@ #ifdef CONFIG_64BIT .set .Lextra_characteristics, 0x0 - .set .Lpe_opt_magic, PE_OPT_MAGIC_PE32PLUS + .set .Lpe_opt_magic, IMAGE_NT_OPTIONAL_HDR64_MAGIC #else .set .Lextra_characteristics, IMAGE_FILE_32BIT_MACHINE - .set .Lpe_opt_magic, PE_OPT_MAGIC_PE32 + .set .Lpe_opt_magic, IMAGE_NT_OPTIONAL_HDR32_MAGIC #endif .section ".head", "a" .globl __efistub_efi_zboot_header __efistub_efi_zboot_header: .Ldoshdr: - .long MZ_MAGIC + .long IMAGE_DOS_SIGNATURE .ascii "zimg" // image type .long __efistub__gzdata_start - .Ldoshdr // payload offset .long __efistub__gzdata_size - ZBOOT_SIZE_LEN // payload size .long 0, 0 // reserved .asciz COMP_TYPE // compression type - .org .Ldoshdr + 0x3c + .org .Ldoshdr + 0x38 + .long LINUX_PE_MAGIC .long .Lpehdr - .Ldoshdr // PE header offset .Lpehdr: - .long PE_MAGIC + .long IMAGE_NT_SIGNATURE .short MACHINE_TYPE .short .Lsection_count .long 0 @@ -62,7 +63,7 @@ __efistub_efi_zboot_header: .long .Lefi_header_end - .Ldoshdr .long 0 .short IMAGE_SUBSYSTEM_EFI_APPLICATION - .short 0 + .short IMAGE_DLLCHARACTERISTICS_NX_COMPAT #ifdef CONFIG_64BIT .quad 0, 0, 0, 0 #else @@ -77,9 +78,36 @@ __efistub_efi_zboot_header: .quad 0 // ExceptionTable .quad 0 // CertificationTable .quad 0 // BaseRelocationTable -#ifdef CONFIG_DEBUG_EFI +#if defined(PE_DLL_CHAR_EX) || defined(CONFIG_DEBUG_EFI) .long .Lefi_debug_table - .Ldoshdr // DebugTable .long .Lefi_debug_table_size + + .section ".rodata", "a" + .p2align 2 +.Lefi_debug_table: + // EFI_IMAGE_DEBUG_DIRECTORY_ENTRY[] +#ifdef PE_DLL_CHAR_EX + .long 0 // Characteristics + .long 0 // TimeDateStamp + .short 0 // MajorVersion + .short 0 // MinorVersion + .long IMAGE_DEBUG_TYPE_EX_DLLCHARACTERISTICS // Type + .long 4 // SizeOfData + .long 0 // RVA + .long .Lefi_dll_characteristics_ex - .Ldoshdr // FileOffset +#endif +#ifdef CONFIG_DEBUG_EFI + .long 0 // Characteristics + .long 0 // TimeDateStamp + .short 0 // MajorVersion + .short 0 // MinorVersion + .long IMAGE_DEBUG_TYPE_CODEVIEW // Type + .long .Lefi_debug_entry_size // SizeOfData + .long 0 // RVA + .long .Lefi_debug_entry - .Ldoshdr // FileOffset +#endif + .set .Lefi_debug_table_size, . - .Lefi_debug_table + .previous #endif .Lsection_table: @@ -95,11 +123,29 @@ __efistub_efi_zboot_header: IMAGE_SCN_MEM_READ | \ IMAGE_SCN_MEM_EXECUTE +#ifdef CONFIG_EFI_SBAT + .ascii ".sbat\0\0\0" + .long __sbat_size + .long _sbat - .Ldoshdr + .long __sbat_size + .long _sbat - .Ldoshdr + + .long 0, 0 + .short 0, 0 + .long IMAGE_SCN_CNT_INITIALIZED_DATA | \ + IMAGE_SCN_MEM_READ | \ + IMAGE_SCN_MEM_DISCARDABLE + + .pushsection ".sbat", "a", @progbits + .incbin CONFIG_EFI_SBAT_FILE + .popsection +#endif + .ascii ".data\0\0\0" .long __data_size - .long _etext - .Ldoshdr + .long _data - .Ldoshdr .long __data_rawsize - .long _etext - .Ldoshdr + .long _data - .Ldoshdr .long 0, 0 .short 0, 0 @@ -109,23 +155,11 @@ __efistub_efi_zboot_header: .set .Lsection_count, (. - .Lsection_table) / 40 +#ifdef PE_DLL_CHAR_EX +.Lefi_dll_characteristics_ex: + .long PE_DLL_CHAR_EX +#endif #ifdef CONFIG_DEBUG_EFI - .section ".rodata", "a" - .align 2 -.Lefi_debug_table: - // EFI_IMAGE_DEBUG_DIRECTORY_ENTRY - .long 0 // Characteristics - .long 0 // TimeDateStamp - .short 0 // MajorVersion - .short 0 // MinorVersion - .long IMAGE_DEBUG_TYPE_CODEVIEW // Type - .long .Lefi_debug_entry_size // SizeOfData - .long 0 // RVA - .long .Lefi_debug_entry - .Ldoshdr // FileOffset - - .set .Lefi_debug_table_size, . - .Lefi_debug_table - .previous - .Lefi_debug_entry: // EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY .ascii "NB10" // Signature diff --git a/drivers/firmware/efi/libstub/zboot.c b/drivers/firmware/efi/libstub/zboot.c index ea72c8f27da6..c47ace06f010 100644 --- a/drivers/firmware/efi/libstub/zboot.c +++ b/drivers/firmware/efi/libstub/zboot.c @@ -3,300 +3,98 @@ #include <linux/efi.h> #include <linux/pe.h> #include <asm/efi.h> -#include <asm/unaligned.h> +#include <linux/unaligned.h> #include "efistub.h" -static unsigned char zboot_heap[SZ_256K] __aligned(64); -static unsigned long free_mem_ptr, free_mem_end_ptr; - -#define STATIC static -#if defined(CONFIG_KERNEL_GZIP) -#include "../../../../lib/decompress_inflate.c" -#elif defined(CONFIG_KERNEL_LZ4) -#include "../../../../lib/decompress_unlz4.c" -#elif defined(CONFIG_KERNEL_LZMA) -#include "../../../../lib/decompress_unlzma.c" -#elif defined(CONFIG_KERNEL_LZO) -#include "../../../../lib/decompress_unlzo.c" -#elif defined(CONFIG_KERNEL_XZ) -#undef memcpy -#define memcpy memcpy -#undef memmove -#define memmove memmove -#include "../../../../lib/decompress_unxz.c" -#elif defined(CONFIG_KERNEL_ZSTD) -#include "../../../../lib/decompress_unzstd.c" -#endif - -extern char efi_zboot_header[]; -extern char _gzdata_start[], _gzdata_end[]; - -static void log(efi_char16_t str[]) -{ - efi_call_proto(efi_table_attr(efi_system_table, con_out), - output_string, L"EFI decompressor: "); - efi_call_proto(efi_table_attr(efi_system_table, con_out), - output_string, str); - efi_call_proto(efi_table_attr(efi_system_table, con_out), - output_string, L"\n"); -} - -static void error(char *x) -{ - log(L"error() called from decompressor library\n"); -} - -// Local version to avoid pulling in memcmp() -static bool guids_eq(const efi_guid_t *a, const efi_guid_t *b) -{ - const u32 *l = (u32 *)a; - const u32 *r = (u32 *)b; - - return l[0] == r[0] && l[1] == r[1] && l[2] == r[2] && l[3] == r[3]; -} - -static efi_status_t __efiapi -load_file(efi_load_file_protocol_t *this, efi_device_path_protocol_t *rem, - bool boot_policy, unsigned long *bufsize, void *buffer) -{ - unsigned long compressed_size = _gzdata_end - _gzdata_start; - struct efi_vendor_dev_path *vendor_dp; - bool decompress = false; - unsigned long size; - int ret; - - if (rem == NULL || bufsize == NULL) - return EFI_INVALID_PARAMETER; - - if (boot_policy) - return EFI_UNSUPPORTED; - - // Look for our vendor media device node in the remaining file path - if (rem->type == EFI_DEV_MEDIA && - rem->sub_type == EFI_DEV_MEDIA_VENDOR) { - vendor_dp = container_of(rem, struct efi_vendor_dev_path, header); - if (!guids_eq(&vendor_dp->vendorguid, &LINUX_EFI_ZBOOT_MEDIA_GUID)) - return EFI_NOT_FOUND; - - decompress = true; - rem = (void *)(vendor_dp + 1); - } - - if (rem->type != EFI_DEV_END_PATH || - rem->sub_type != EFI_DEV_END_ENTIRE) - return EFI_NOT_FOUND; - - // The uncompressed size of the payload is appended to the raw bit - // stream, and may therefore appear misaligned in memory - size = decompress ? get_unaligned_le32(_gzdata_end - 4) - : compressed_size; - if (buffer == NULL || *bufsize < size) { - *bufsize = size; - return EFI_BUFFER_TOO_SMALL; - } - - if (decompress) { - ret = __decompress(_gzdata_start, compressed_size, NULL, NULL, - buffer, size, NULL, error); - if (ret < 0) { - log(L"Decompression failed"); - return EFI_DEVICE_ERROR; - } - } else { - memcpy(buffer, _gzdata_start, compressed_size); - } - - return EFI_SUCCESS; -} - -// Return the length in bytes of the device path up to the first end node. -static int device_path_length(const efi_device_path_protocol_t *dp) -{ - int len = 0; - - while (dp->type != EFI_DEV_END_PATH) { - len += dp->length; - dp = (void *)((u8 *)dp + dp->length); - } - return len; -} - -static void append_rel_offset_node(efi_device_path_protocol_t **dp, - unsigned long start, unsigned long end) +static unsigned long alloc_preferred_address(unsigned long alloc_size) { - struct efi_rel_offset_dev_path *rodp = (void *)*dp; - - rodp->header.type = EFI_DEV_MEDIA; - rodp->header.sub_type = EFI_DEV_MEDIA_REL_OFFSET; - rodp->header.length = sizeof(struct efi_rel_offset_dev_path); - rodp->reserved = 0; - rodp->starting_offset = start; - rodp->ending_offset = end; +#ifdef EFI_KIMG_PREFERRED_ADDRESS + efi_physical_addr_t efi_addr = EFI_KIMG_PREFERRED_ADDRESS; - *dp = (void *)(rodp + 1); + if (efi_bs_call(allocate_pages, EFI_ALLOCATE_ADDRESS, EFI_LOADER_DATA, + alloc_size / EFI_PAGE_SIZE, &efi_addr) == EFI_SUCCESS) + return efi_addr; +#endif + return ULONG_MAX; } -static void append_ven_media_node(efi_device_path_protocol_t **dp, - efi_guid_t *guid) +void __weak efi_cache_sync_image(unsigned long image_base, + unsigned long alloc_size) { - struct efi_vendor_dev_path *vmdp = (void *)*dp; - - vmdp->header.type = EFI_DEV_MEDIA; - vmdp->header.sub_type = EFI_DEV_MEDIA_VENDOR; - vmdp->header.length = sizeof(struct efi_vendor_dev_path); - vmdp->vendorguid = *guid; - - *dp = (void *)(vmdp + 1); + // Provided by the arch to perform the cache maintenance necessary for + // executable code loaded into memory to be safe for execution. } -static void append_end_node(efi_device_path_protocol_t **dp) +struct screen_info *alloc_screen_info(void) { - (*dp)->type = EFI_DEV_END_PATH; - (*dp)->sub_type = EFI_DEV_END_ENTIRE; - (*dp)->length = sizeof(struct efi_generic_dev_path); - - ++*dp; + return __alloc_screen_info(); } asmlinkage efi_status_t __efiapi efi_zboot_entry(efi_handle_t handle, efi_system_table_t *systab) { - struct efi_mem_mapped_dev_path mmdp = { - .header.type = EFI_DEV_HW, - .header.sub_type = EFI_DEV_MEM_MAPPED, - .header.length = sizeof(struct efi_mem_mapped_dev_path) - }; - efi_device_path_protocol_t *parent_dp, *dpp, *lf2_dp, *li_dp; - efi_load_file2_protocol_t zboot_load_file2; - efi_loaded_image_t *parent, *child; - unsigned long exit_data_size; - efi_handle_t child_handle; - efi_handle_t zboot_handle; - efi_char16_t *exit_data; + char *cmdline_ptr __free(efi_pool) = NULL; + unsigned long image_base, alloc_size; + efi_loaded_image_t *image; efi_status_t status; - void *dp_alloc; - int dp_len; WRITE_ONCE(efi_system_table, systab); - free_mem_ptr = (unsigned long)&zboot_heap; - free_mem_end_ptr = free_mem_ptr + sizeof(zboot_heap); - - exit_data = NULL; - exit_data_size = 0; - status = efi_bs_call(handle_protocol, handle, - &LOADED_IMAGE_PROTOCOL_GUID, (void **)&parent); + &LOADED_IMAGE_PROTOCOL_GUID, (void **)&image); if (status != EFI_SUCCESS) { - log(L"Failed to locate parent's loaded image protocol"); + efi_err("Failed to locate parent's loaded image protocol\n"); return status; } - status = efi_bs_call(handle_protocol, handle, - &LOADED_IMAGE_DEVICE_PATH_PROTOCOL_GUID, - (void **)&parent_dp); - if (status != EFI_SUCCESS || parent_dp == NULL) { - // Create a MemoryMapped() device path node to describe - // the parent image if no device path was provided. - mmdp.memory_type = parent->image_code_type; - mmdp.starting_addr = (unsigned long)parent->image_base; - mmdp.ending_addr = (unsigned long)parent->image_base + - parent->image_size - 1; - parent_dp = &mmdp.header; - dp_len = sizeof(mmdp); - } else { - dp_len = device_path_length(parent_dp); - } - - // Allocate some pool memory for device path protocol data - status = efi_bs_call(allocate_pool, EFI_LOADER_DATA, - 2 * (dp_len + sizeof(struct efi_rel_offset_dev_path) + - sizeof(struct efi_generic_dev_path)) + - sizeof(struct efi_vendor_dev_path), - (void **)&dp_alloc); - if (status != EFI_SUCCESS) { - log(L"Failed to allocate device path pool memory"); + status = efi_handle_cmdline(image, &cmdline_ptr); + if (status != EFI_SUCCESS) return status; - } - - // Create a device path describing the compressed payload in this image - // <...parent_dp...>/Offset(<start>, <end>) - lf2_dp = memcpy(dp_alloc, parent_dp, dp_len); - dpp = (void *)((u8 *)lf2_dp + dp_len); - append_rel_offset_node(&dpp, - (unsigned long)(_gzdata_start - efi_zboot_header), - (unsigned long)(_gzdata_end - efi_zboot_header - 1)); - append_end_node(&dpp); - // Create a device path describing the decompressed payload in this image - // <...parent_dp...>/Offset(<start>, <end>)/VenMedia(ZBOOT_MEDIA_GUID) - dp_len += sizeof(struct efi_rel_offset_dev_path); - li_dp = memcpy(dpp, lf2_dp, dp_len); - dpp = (void *)((u8 *)li_dp + dp_len); - append_ven_media_node(&dpp, &LINUX_EFI_ZBOOT_MEDIA_GUID); - append_end_node(&dpp); + efi_info("Decompressing Linux Kernel...\n"); - zboot_handle = NULL; - zboot_load_file2.load_file = load_file; - status = efi_bs_call(install_multiple_protocol_interfaces, - &zboot_handle, - &EFI_DEVICE_PATH_PROTOCOL_GUID, lf2_dp, - &EFI_LOAD_FILE2_PROTOCOL_GUID, &zboot_load_file2, - NULL); - if (status != EFI_SUCCESS) { - log(L"Failed to install LoadFile2 protocol and device path"); - goto free_dpalloc; - } - - status = efi_bs_call(load_image, false, handle, li_dp, NULL, 0, - &child_handle); - if (status != EFI_SUCCESS) { - log(L"Failed to load image"); - goto uninstall_lf2; - } - - status = efi_bs_call(handle_protocol, child_handle, - &LOADED_IMAGE_PROTOCOL_GUID, (void **)&child); - if (status != EFI_SUCCESS) { - log(L"Failed to locate child's loaded image protocol"); - goto unload_image; - } - - // Copy the kernel command line - child->load_options = parent->load_options; - child->load_options_size = parent->load_options_size; + status = efi_zboot_decompress_init(&alloc_size); + if (status != EFI_SUCCESS) + return status; - status = efi_bs_call(start_image, child_handle, &exit_data_size, - &exit_data); - if (status != EFI_SUCCESS) { - log(L"StartImage() returned with error"); - if (exit_data_size > 0) - log(exit_data); + // If the architecture has a preferred address for the image, + // try that first. + image_base = alloc_preferred_address(alloc_size); + if (image_base == ULONG_MAX) { + unsigned long min_kimg_align = efi_get_kimg_min_align(); + u32 seed = U32_MAX; + + if (!IS_ENABLED(CONFIG_RANDOMIZE_BASE)) { + // Setting the random seed to 0x0 is the same as + // allocating as low as possible + seed = 0; + } else if (efi_nokaslr) { + efi_info("KASLR disabled on kernel command line\n"); + } else { + status = efi_get_random_bytes(sizeof(seed), (u8 *)&seed); + if (status == EFI_NOT_FOUND) { + efi_info("EFI_RNG_PROTOCOL unavailable\n"); + efi_nokaslr = true; + } else if (status != EFI_SUCCESS) { + efi_err("efi_get_random_bytes() failed (0x%lx)\n", + status); + efi_nokaslr = true; + } + } - // If StartImage() returns EFI_SECURITY_VIOLATION, the image is - // not unloaded so we need to do it by hand. - if (status == EFI_SECURITY_VIOLATION) -unload_image: - efi_bs_call(unload_image, child_handle); + status = efi_random_alloc(alloc_size, min_kimg_align, &image_base, + seed, EFI_LOADER_CODE, 0, EFI_ALLOC_LIMIT); + if (status != EFI_SUCCESS) { + efi_err("Failed to allocate memory\n"); + return status; + } } -uninstall_lf2: - efi_bs_call(uninstall_multiple_protocol_interfaces, - zboot_handle, - &EFI_DEVICE_PATH_PROTOCOL_GUID, lf2_dp, - &EFI_LOAD_FILE2_PROTOCOL_GUID, &zboot_load_file2, - NULL); - -free_dpalloc: - efi_bs_call(free_pool, dp_alloc); - - efi_bs_call(exit, handle, status, exit_data_size, exit_data); + // Decompress the payload into the newly allocated buffer + status = efi_zboot_decompress((void *)image_base, alloc_size) ?: + efi_stub_common(handle, image, image_base, cmdline_ptr); - // Free ExitData in case Exit() returned with a failure code, - // but return the original status code. - log(L"Exit() returned with failure code"); - if (exit_data != NULL) - efi_bs_call(free_pool, exit_data); + efi_free(alloc_size, image_base); return status; } diff --git a/drivers/firmware/efi/libstub/zboot.lds b/drivers/firmware/efi/libstub/zboot.lds index 93d33f68333b..367907eb7d86 100644 --- a/drivers/firmware/efi/libstub/zboot.lds +++ b/drivers/firmware/efi/libstub/zboot.lds @@ -2,6 +2,8 @@ ENTRY(__efistub_efi_zboot_header); +PROVIDE(zboot_code_size = ABSOLUTE(0)); + SECTIONS { .head : ALIGN(4096) { @@ -15,13 +17,27 @@ SECTIONS .rodata : ALIGN(8) { __efistub__gzdata_start = .; *(.gzdata) + __efistub_payload_size = . - 4; __efistub__gzdata_end = .; *(.rodata* .init.rodata* .srodata*) + + . = ALIGN(4); + __efistub_code_size = .; + LONG(zboot_code_size); + _etext = ALIGN(4096); . = _etext; } + .sbat : ALIGN(4096) { + _sbat = .; + *(.sbat) + _esbat = ALIGN(4096); + . = _esbat; + } + .data : ALIGN(4096) { + _data = .; *(.data* .init.data*) _edata = ALIGN(512); . = _edata; @@ -34,6 +50,7 @@ SECTIONS } /DISCARD/ : { + *(.discard .discard.*) *(.modinfo .init.modinfo) } } @@ -41,5 +58,6 @@ SECTIONS PROVIDE(__efistub__gzdata_size = ABSOLUTE(__efistub__gzdata_end - __efistub__gzdata_start)); -PROVIDE(__data_rawsize = ABSOLUTE(_edata - _etext)); -PROVIDE(__data_size = ABSOLUTE(_end - _etext)); +PROVIDE(__data_rawsize = ABSOLUTE(_edata - _data)); +PROVIDE(__data_size = ABSOLUTE(_end - _data)); +PROVIDE(__sbat_size = ABSOLUTE(_esbat - _sbat)); |