diff options
Diffstat (limited to 'scripts')
32 files changed, 2335 insertions, 401 deletions
diff --git a/scripts/.gitignore b/scripts/.gitignore index 6e9ce6720a05..3dbb8bb2457b 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -5,6 +5,8 @@ /kallsyms /module.lds /recordmcount +/rustdoc_test_builder +/rustdoc_test_gen /sign-file /sorttable /target.json diff --git a/scripts/Makefile b/scripts/Makefile index 32b6ba722728..576cf64be667 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -9,6 +9,8 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable hostprogs-always-$(CONFIG_ASN1) += asn1_compiler hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert +hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += rustdoc_test_builder +hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += rustdoc_test_gen always-$(CONFIG_RUST) += target.json filechk_rust_target = $< < include/config/auto.conf @@ -18,6 +20,8 @@ $(obj)/target.json: scripts/generate_rust_target include/config/auto.conf FORCE hostprogs += generate_rust_target generate_rust_target-rust := y +rustdoc_test_builder-rust := y +rustdoc_test_gen-rust := y HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include HOSTLDLIBS_sorttable = -lpthread diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 6413342a03f4..82e3fb19fdaf 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -264,6 +264,9 @@ $(obj)/%.lst: $(src)/%.c FORCE rust_allowed_features := new_uninit +# `--out-dir` is required to avoid temporaries being created by `rustc` in the +# current working directory, which may be not accessible in the out-of-tree +# modules case. rust_common_cmd = \ RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \ -Zallow-features=$(rust_allowed_features) \ @@ -272,7 +275,7 @@ rust_common_cmd = \ --extern alloc --extern kernel \ --crate-type rlib -L $(objtree)/rust/ \ --crate-name $(basename $(notdir $@)) \ - --emit=dep-info=$(depfile) + --out-dir $(dir $@) --emit=dep-info=$(depfile) # `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit # will be used. We explicitly request `-Ccodegen-units=1` in any case, and diff --git a/scripts/Makefile.host b/scripts/Makefile.host index 7aea9005e497..8f7f842b54f9 100644 --- a/scripts/Makefile.host +++ b/scripts/Makefile.host @@ -86,7 +86,11 @@ hostc_flags = -Wp,-MMD,$(depfile) \ hostcxx_flags = -Wp,-MMD,$(depfile) \ $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \ $(HOSTCXXFLAGS_$(target-stem).o) -hostrust_flags = --emit=dep-info=$(depfile) \ + +# `--out-dir` is required to avoid temporaries being created by `rustc` in the +# current working directory, which may be not accessible in the out-of-tree +# modules case. +hostrust_flags = --out-dir $(dir $@) --emit=dep-info=$(depfile) \ $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \ $(HOSTRUSTFLAGS_$(target-stem)) diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index fc19f67039bd..b3a6aa8fbe8c 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -41,8 +41,6 @@ quiet_cmd_btf_ko = BTF [M] $@ cmd_btf_ko = \ if [ ! -f vmlinux ]; then \ printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \ - elif [ -n "$(CONFIG_RUST)" ] && $(srctree)/scripts/is_rust_module.sh $@; then \ - printf "Skipping BTF generation for %s because it's a Rust module\n" $@ 1>&2; \ else \ LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J $(PAHOLE_FLAGS) --btf_base vmlinux $@; \ $(RESOLVE_BTFIDS) -b vmlinux $@; \ diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter index 36303afa9dfc..888ce286a351 100755 --- a/scripts/bloat-o-meter +++ b/scripts/bloat-o-meter @@ -100,12 +100,12 @@ def print_result(symboltype, symbolformat): print("Total: Before=%d, After=%d, chg %+.2f%%" % (otot, ntot, percent)) if args.c: - print_result("Function", "tT") - print_result("Data", "dDbB") + print_result("Function", "tTwW") + print_result("Data", "dDbBvV") print_result("RO Data", "rR") elif args.d: - print_result("Data", "dDbBrR") + print_result("Data", "dDbBrRvV") elif args.t: - print_result("Function", "tT") + print_result("Function", "tTwW") else: - print_result("Function", "tTdDbBrR") + print_result("Function", "tTdDbBrRvVwW") diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 880fde13d9b8..7d16f863edf1 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -74,6 +74,8 @@ my $git_command ='export LANGUAGE=en_US.UTF-8; git'; my $tabsize = 8; my ${CONFIG_} = "CONFIG_"; +my %maybe_linker_symbol; # for externs in c exceptions, when seen in *vmlinux.lds.h + sub help { my ($exitcode) = @_; @@ -3270,7 +3272,7 @@ sub process { # A Fixes:, link or signature tag line $commit_log_possible_stack_dump)) { WARN("COMMIT_LOG_LONG_LINE", - "Possible unwrapped commit description (prefer a maximum 75 chars per line)\n" . $herecurr); + "Prefer a maximum 75 chars per line (possible unwrapped commit description?)\n" . $herecurr); $commit_log_long_line = 1; } @@ -6051,6 +6053,9 @@ sub process { # check for line continuations outside of #defines, preprocessor #, and asm + } elsif ($realfile =~ m@/vmlinux.lds.h$@) { + $line =~ s/(\w+)/$maybe_linker_symbol{$1}++/ge; + #print "REAL: $realfile\nln: $line\nkeys:", sort keys %maybe_linker_symbol; } else { if ($prevline !~ /^..*\\$/ && $line !~ /^\+\s*\#.*\\$/ && # preprocessor @@ -7120,6 +7125,21 @@ sub process { } } elsif ($realfile =~ /\.c$/ && defined $stat && + $stat =~ /^\+extern struct\s+(\w+)\s+(\w+)\[\];/) + { + my ($st_type, $st_name) = ($1, $2); + + for my $s (keys %maybe_linker_symbol) { + #print "Linker symbol? $st_name : $s\n"; + goto LIKELY_LINKER_SYMBOL + if $st_name =~ /$s/; + } + WARN("AVOID_EXTERNS", + "found a file-scoped extern type:$st_type name:$st_name in .c file\n" + . "is this a linker symbol ?\n" . $herecurr); + LIKELY_LINKER_SYMBOL: + + } elsif ($realfile =~ /\.c$/ && defined $stat && $stat =~ /^.\s*extern\s+/) { WARN("AVOID_EXTERNS", @@ -7457,6 +7477,30 @@ sub process { } } +# Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). + our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_trace | + rcu_read_lock_trace_held | + rcu_read_unlock_trace | + call_rcu_tasks_trace | + synchronize_rcu_tasks_trace | + rcu_barrier_tasks_trace | + rcu_request_urgent_qs_task + )}; + our $rcu_trace_paths = qr{(?x: + kernel/bpf/ | + include/linux/bpf | + net/bpf/ | + kernel/rcu/ | + include/linux/rcu + )}; + if ($line =~ /\b($rcu_trace_funcs)\s*\(/) { + if ($realfile !~ m{^$rcu_trace_paths}) { + WARN("RCU_TASKS_TRACE", + "use of RCU tasks trace is incorrect outside BPF or core RCU code\n" . $herecurr); + } + } + # check for lockdep_set_novalidate_class if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || $line =~ /__lockdep_no_validate__\s*\)/ ) { diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py index 15ba56527acd..a84cc5737c2c 100755 --- a/scripts/clang-tools/gen_compile_commands.py +++ b/scripts/clang-tools/gen_compile_commands.py @@ -19,7 +19,7 @@ _DEFAULT_OUTPUT = 'compile_commands.json' _DEFAULT_LOG_LEVEL = 'WARNING' _FILENAME_PATTERN = r'^\..*\.cmd$' -_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.c) *(;|$)' +_LINE_PATTERN = r'^savedcmd_[^ ]*\.o := (.* )([^ ]*\.[cS]) *(;|$)' _VALID_LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] # The tools/ directory adopts a different build system, and produces .cmd # files in a different format. Do not support it. diff --git a/scripts/coccinelle/api/debugfs/debugfs_simple_attr.cocci b/scripts/coccinelle/api/debugfs/debugfs_simple_attr.cocci deleted file mode 100644 index 7c312310547c..000000000000 --- a/scripts/coccinelle/api/debugfs/debugfs_simple_attr.cocci +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/// Use DEFINE_DEBUGFS_ATTRIBUTE rather than DEFINE_SIMPLE_ATTRIBUTE -/// for debugfs files. -/// -//# Rationale: DEFINE_SIMPLE_ATTRIBUTE + debugfs_create_file() -//# imposes some significant overhead as compared to -//# DEFINE_DEBUGFS_ATTRIBUTE + debugfs_create_file_unsafe(). -// -// Copyright (C): 2016 Nicolai Stange -// Options: --no-includes -// - -virtual context -virtual patch -virtual org -virtual report - -@dsa@ -declarer name DEFINE_SIMPLE_ATTRIBUTE; -identifier dsa_fops; -expression dsa_get, dsa_set, dsa_fmt; -position p; -@@ -DEFINE_SIMPLE_ATTRIBUTE@p(dsa_fops, dsa_get, dsa_set, dsa_fmt); - -@dcf@ -expression name, mode, parent, data; -identifier dsa.dsa_fops; -@@ -debugfs_create_file(name, mode, parent, data, &dsa_fops) - - -@context_dsa depends on context && dcf@ -declarer name DEFINE_DEBUGFS_ATTRIBUTE; -identifier dsa.dsa_fops; -expression dsa.dsa_get, dsa.dsa_set, dsa.dsa_fmt; -@@ -* DEFINE_SIMPLE_ATTRIBUTE(dsa_fops, dsa_get, dsa_set, dsa_fmt); - - -@patch_dcf depends on patch expression@ -expression name, mode, parent, data; -identifier dsa.dsa_fops; -@@ -- debugfs_create_file(name, mode, parent, data, &dsa_fops) -+ debugfs_create_file_unsafe(name, mode, parent, data, &dsa_fops) - -@patch_dsa depends on patch_dcf && patch@ -identifier dsa.dsa_fops; -expression dsa.dsa_get, dsa.dsa_set, dsa.dsa_fmt; -@@ -- DEFINE_SIMPLE_ATTRIBUTE(dsa_fops, dsa_get, dsa_set, dsa_fmt); -+ DEFINE_DEBUGFS_ATTRIBUTE(dsa_fops, dsa_get, dsa_set, dsa_fmt); - - -@script:python depends on org && dcf@ -fops << dsa.dsa_fops; -p << dsa.p; -@@ -msg="%s should be defined with DEFINE_DEBUGFS_ATTRIBUTE" % (fops) -coccilib.org.print_todo(p[0], msg) - -@script:python depends on report && dcf@ -fops << dsa.dsa_fops; -p << dsa.p; -@@ -msg="WARNING: %s should be defined with DEFINE_DEBUGFS_ATTRIBUTE" % (fops) -coccilib.report.print_report(p[0], msg) diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 84c730da36dd..1ae39b9f4a95 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -440,4 +440,8 @@ static inline void debug_gimple_stmt(const_gimple s) #define SET_DECL_MODE(decl, mode) DECL_MODE(decl) = (mode) #endif +#if BUILDING_GCC_VERSION >= 14000 +#define last_stmt(x) last_nondebug_stmt(x) +#endif + #endif diff --git a/scripts/gdb/linux/constants.py.in b/scripts/gdb/linux/constants.py.in index 50a92c4e9984..e3517d4ab8ec 100644 --- a/scripts/gdb/linux/constants.py.in +++ b/scripts/gdb/linux/constants.py.in @@ -18,8 +18,11 @@ #include <linux/irq.h> #include <linux/mount.h> #include <linux/of_fdt.h> +#include <linux/page_ext.h> #include <linux/radix-tree.h> +#include <linux/slab.h> #include <linux/threads.h> +#include <linux/vmalloc.h> /* We need to stringify expanded macros so that they can be parsed */ @@ -64,6 +67,9 @@ LX_GDBPARSED(IRQ_HIDDEN) /* linux/module.h */ LX_GDBPARSED(MOD_TEXT) +LX_GDBPARSED(MOD_DATA) +LX_GDBPARSED(MOD_RODATA) +LX_GDBPARSED(MOD_RO_AFTER_INIT) /* linux/mount.h */ LX_VALUE(MNT_NOSUID) @@ -86,6 +92,28 @@ LX_GDBPARSED(RADIX_TREE_MAP_SIZE) LX_GDBPARSED(RADIX_TREE_MAP_SHIFT) LX_GDBPARSED(RADIX_TREE_MAP_MASK) +/* linux/vmalloc.h */ +LX_VALUE(VM_IOREMAP) +LX_VALUE(VM_ALLOC) +LX_VALUE(VM_MAP) +LX_VALUE(VM_USERMAP) +LX_VALUE(VM_DMA_COHERENT) + +/* linux/page_ext.h */ +if IS_BUILTIN(CONFIG_PAGE_OWNER): + LX_GDBPARSED(PAGE_EXT_OWNER) + LX_GDBPARSED(PAGE_EXT_OWNER_ALLOCATED) + +/* linux/slab.h */ +LX_GDBPARSED(SLAB_RED_ZONE) +LX_GDBPARSED(SLAB_POISON) +LX_GDBPARSED(SLAB_KMALLOC) +LX_GDBPARSED(SLAB_HWCACHE_ALIGN) +LX_GDBPARSED(SLAB_CACHE_DMA) +LX_GDBPARSED(SLAB_CACHE_DMA32) +LX_GDBPARSED(SLAB_STORE_USER) +LX_GDBPARSED(SLAB_PANIC) + /* Kernel Configs */ LX_CONFIG(CONFIG_GENERIC_CLOCKEVENTS) LX_CONFIG(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) @@ -102,3 +130,30 @@ LX_CONFIG(CONFIG_X86_MCE_AMD) LX_CONFIG(CONFIG_X86_MCE) LX_CONFIG(CONFIG_X86_IO_APIC) LX_CONFIG(CONFIG_HAVE_KVM) +LX_CONFIG(CONFIG_NUMA) +LX_CONFIG(CONFIG_ARM64) +LX_CONFIG(CONFIG_ARM64_4K_PAGES) +LX_CONFIG(CONFIG_ARM64_16K_PAGES) +LX_CONFIG(CONFIG_ARM64_64K_PAGES) +if IS_BUILTIN(CONFIG_ARM64): + LX_VALUE(CONFIG_ARM64_PA_BITS) + LX_VALUE(CONFIG_ARM64_VA_BITS) + LX_VALUE(CONFIG_ARM64_PAGE_SHIFT) + LX_VALUE(CONFIG_ARCH_FORCE_MAX_ORDER) +LX_CONFIG(CONFIG_SPARSEMEM) +LX_CONFIG(CONFIG_SPARSEMEM_EXTREME) +LX_CONFIG(CONFIG_SPARSEMEM_VMEMMAP) +LX_CONFIG(CONFIG_KASAN) +LX_CONFIG(CONFIG_KASAN_GENERIC) +LX_CONFIG(CONFIG_KASAN_SW_TAGS) +LX_CONFIG(CONFIG_KASAN_HW_TAGS) +if IS_BUILTIN(CONFIG_KASAN_GENERIC) or IS_BUILTIN(CONFIG_KASAN_SW_TAGS): + LX_VALUE(CONFIG_KASAN_SHADOW_OFFSET) +LX_CONFIG(CONFIG_VMAP_STACK) +if IS_BUILTIN(CONFIG_NUMA): + LX_VALUE(CONFIG_NODES_SHIFT) +LX_CONFIG(CONFIG_DEBUG_VIRTUAL) +LX_CONFIG(CONFIG_STACKDEPOT) +LX_CONFIG(CONFIG_PAGE_OWNER) +LX_CONFIG(CONFIG_SLUB_DEBUG) +LX_CONFIG(CONFIG_SLAB_FREELIST_HARDENED) diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py index 30d837f3dfae..ad5641dcb068 100644 --- a/scripts/gdb/linux/mm.py +++ b/scripts/gdb/linux/mm.py @@ -1,222 +1,398 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: GPL-2.0 # -# gdb helper commands and functions for Linux kernel debugging -# -# routines to introspect page table +# Copyright (c) 2023 MediaTek Inc. # # Authors: -# Dmitrii Bundin <dmitrii.bundin.a@gmail.com> +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> # import gdb +import math +from linux import utils, constants + +def DIV_ROUND_UP(n,d): + return ((n) + (d) - 1) // (d) -from linux import utils +def test_bit(nr, addr): + if addr.dereference() & (0x1 << nr): + return True + else: + return False -PHYSICAL_ADDRESS_MASK = gdb.parse_and_eval('0xfffffffffffff') +class page_ops(): + ops = None + def __init__(self): + if not constants.LX_CONFIG_SPARSEMEM_VMEMMAP: + raise gdb.GdbError('Only support CONFIG_SPARSEMEM_VMEMMAP now') + if constants.LX_CONFIG_ARM64 and utils.is_target_arch('aarch64'): + self.ops = aarch64_page_ops() + else: + raise gdb.GdbError('Only support aarch64 now') +class aarch64_page_ops(): + def __init__(self): + self.SUBSECTION_SHIFT = 21 + self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT + self.MODULES_VSIZE = 128 * 1024 * 1024 -def page_mask(level=1): - # 4KB - if level == 1: - return gdb.parse_and_eval('(u64) ~0xfff') - # 2MB - elif level == 2: - return gdb.parse_and_eval('(u64) ~0x1fffff') - # 1GB - elif level == 3: - return gdb.parse_and_eval('(u64) ~0x3fffffff') - else: - raise Exception(f'Unknown page level: {level}') - - -#page_offset_base in case CONFIG_DYNAMIC_MEMORY_LAYOUT is disabled -POB_NO_DYNAMIC_MEM_LAYOUT = '0xffff888000000000' -def _page_offset_base(): - pob_symbol = gdb.lookup_global_symbol('page_offset_base') - pob = pob_symbol.name if pob_symbol else POB_NO_DYNAMIC_MEM_LAYOUT - return gdb.parse_and_eval(pob) - - -def is_bit_defined_tupled(data, offset): - return offset, bool(data >> offset & 1) - -def content_tupled(data, bit_start, bit_end): - return (bit_start, bit_end), data >> bit_start & ((1 << (1 + bit_end - bit_start)) - 1) - -def entry_va(level, phys_addr, translating_va): - def start_bit(level): - if level == 5: - return 48 - elif level == 4: - return 39 - elif level == 3: - return 30 - elif level == 2: - return 21 - elif level == 1: - return 12 + if constants.LX_CONFIG_ARM64_64K_PAGES: + self.SECTION_SIZE_BITS = 29 + else: + self.SECTION_SIZE_BITS = 27 + self.MAX_PHYSMEM_BITS = constants.LX_CONFIG_ARM64_VA_BITS + + self.PAGE_SHIFT = constants.LX_CONFIG_ARM64_PAGE_SHIFT + self.PAGE_SIZE = 1 << self.PAGE_SHIFT + self.PAGE_MASK = (~(self.PAGE_SIZE - 1)) & ((1 << 64) - 1) + + self.VA_BITS = constants.LX_CONFIG_ARM64_VA_BITS + if self.VA_BITS > 48: + self.VA_BITS_MIN = 48 + self.vabits_actual = gdb.parse_and_eval('vabits_actual') + else: + self.VA_BITS_MIN = self.VA_BITS + self.vabits_actual = self.VA_BITS + self.kimage_voffset = gdb.parse_and_eval('kimage_voffset') & ((1 << 64) - 1) + + self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS + + if str(constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER).isdigit(): + self.MAX_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER + else: + self.MAX_ORDER = 11 + + self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_ORDER - 1) + self.PFN_SECTION_SHIFT = self.SECTION_SIZE_BITS - self.PAGE_SHIFT + self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT + self.PAGES_PER_SECTION = 1 << self.PFN_SECTION_SHIFT + self.PAGE_SECTION_MASK = (~(self.PAGES_PER_SECTION - 1)) & ((1 << 64) - 1) + + if constants.LX_CONFIG_SPARSEMEM_EXTREME: + self.SECTIONS_PER_ROOT = self.PAGE_SIZE // gdb.lookup_type("struct mem_section").sizeof + else: + self.SECTIONS_PER_ROOT = 1 + + self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT) + self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1 + self.SUBSECTION_SHIFT = 21 + self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT + self.PFN_SUBSECTION_SHIFT = self.SUBSECTION_SHIFT - self.PAGE_SHIFT + self.PAGES_PER_SUBSECTION = 1 << self.PFN_SUBSECTION_SHIFT + + self.SECTION_HAS_MEM_MAP = 1 << int(gdb.parse_and_eval('SECTION_HAS_MEM_MAP_BIT')) + self.SECTION_IS_EARLY = 1 << int(gdb.parse_and_eval('SECTION_IS_EARLY_BIT')) + + self.struct_page_size = utils.get_page_type().sizeof + self.STRUCT_PAGE_MAX_SHIFT = (int)(math.log(self.struct_page_size, 2)) + + self.PAGE_OFFSET = self._PAGE_OFFSET(self.VA_BITS) + self.MODULES_VADDR = self._PAGE_END(self.VA_BITS_MIN) + self.MODULES_END = self.MODULES_VADDR + self.MODULES_VSIZE + + self.VMEMMAP_SHIFT = (self.PAGE_SHIFT - self.STRUCT_PAGE_MAX_SHIFT) + self.VMEMMAP_SIZE = ((self._PAGE_END(self.VA_BITS_MIN) - self.PAGE_OFFSET) >> self.VMEMMAP_SHIFT) + self.VMEMMAP_START = (-(1 << (self.VA_BITS - self.VMEMMAP_SHIFT))) & 0xffffffffffffffff + self.VMEMMAP_END = self.VMEMMAP_START + self.VMEMMAP_SIZE + + self.VMALLOC_START = self.MODULES_END + self.VMALLOC_END = self.VMEMMAP_START - 256 * 1024 * 1024 + + self.memstart_addr = gdb.parse_and_eval("memstart_addr") + self.PHYS_OFFSET = self.memstart_addr + self.vmemmap = gdb.Value(self.VMEMMAP_START).cast(utils.get_page_type().pointer()) - (self.memstart_addr >> self.PAGE_SHIFT) + + self.KERNEL_START = gdb.parse_and_eval("_text") + self.KERNEL_END = gdb.parse_and_eval("_end") + + if constants.LX_CONFIG_KASAN_GENERIC or constants.LX_CONFIG_KASAN_SW_TAGS: + if constants.LX_CONFIG_KASAN_GENERIC: + self.KASAN_SHADOW_SCALE_SHIFT = 3 else: - raise Exception(f'Unknown level {level}') - - entry_offset = ((translating_va >> start_bit(level)) & 511) * 8 - entry_va = _page_offset_base() + phys_addr + entry_offset - return entry_va - -class Cr3(): - def __init__(self, cr3, page_levels): - self.cr3 = cr3 - self.page_levels = page_levels - self.page_level_write_through = is_bit_defined_tupled(cr3, 3) - self.page_level_cache_disabled = is_bit_defined_tupled(cr3, 4) - self.next_entry_physical_address = cr3 & PHYSICAL_ADDRESS_MASK & page_mask() - - def next_entry(self, va): - next_level = self.page_levels - return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) - - def mk_string(self): - return f"""\ -cr3: - {'cr3 binary data': <30} {hex(self.cr3)} - {'next entry physical address': <30} {hex(self.next_entry_physical_address)} - --- - {'bit' : <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} - {'bit' : <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} -""" - - -class PageHierarchyEntry(): - def __init__(self, address, level): - data = int.from_bytes( - memoryview(gdb.selected_inferior().read_memory(address, 8)), - "little" - ) - if level == 1: - self.is_page = True - self.entry_present = is_bit_defined_tupled(data, 0) - self.read_write = is_bit_defined_tupled(data, 1) - self.user_access_allowed = is_bit_defined_tupled(data, 2) - self.page_level_write_through = is_bit_defined_tupled(data, 3) - self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) - self.entry_was_accessed = is_bit_defined_tupled(data, 5) - self.dirty = is_bit_defined_tupled(data, 6) - self.pat = is_bit_defined_tupled(data, 7) - self.global_translation = is_bit_defined_tupled(data, 8) - self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) - self.next_entry_physical_address = None - self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) - self.protection_key = content_tupled(data, 59, 62) - self.executed_disable = is_bit_defined_tupled(data, 63) + self.KASAN_SHADOW_SCALE_SHIFT = 4 + self.KASAN_SHADOW_OFFSET = constants.LX_CONFIG_KASAN_SHADOW_OFFSET + self.KASAN_SHADOW_END = (1 << (64 - self.KASAN_SHADOW_SCALE_SHIFT)) + self.KASAN_SHADOW_OFFSET + self.PAGE_END = self.KASAN_SHADOW_END - (1 << (self.vabits_actual - self.KASAN_SHADOW_SCALE_SHIFT)) + else: + self.PAGE_END = self._PAGE_END(self.VA_BITS_MIN) + + if constants.LX_CONFIG_NUMA and constants.LX_CONFIG_NODES_SHIFT: + self.NODE_SHIFT = constants.LX_CONFIG_NODES_SHIFT + else: + self.NODE_SHIFT = 0 + + self.MAX_NUMNODES = 1 << self.NODE_SHIFT + + def SECTION_NR_TO_ROOT(self, sec): + return sec // self.SECTIONS_PER_ROOT + + def __nr_to_section(self, nr): + root = self.SECTION_NR_TO_ROOT(nr) + mem_section = gdb.parse_and_eval("mem_section") + return mem_section[root][nr & self.SECTION_ROOT_MASK] + + def pfn_to_section_nr(self, pfn): + return pfn >> self.PFN_SECTION_SHIFT + + def section_nr_to_pfn(self, sec): + return sec << self.PFN_SECTION_SHIFT + + def __pfn_to_section(self, pfn): + return self.__nr_to_section(self.pfn_to_section_nr(pfn)) + + def pfn_to_section(self, pfn): + return self.__pfn_to_section(pfn) + + def subsection_map_index(self, pfn): + return (pfn & ~(self.PAGE_SECTION_MASK)) // self.PAGES_PER_SUBSECTION + + def pfn_section_valid(self, ms, pfn): + if constants.LX_CONFIG_SPARSEMEM_VMEMMAP: + idx = self.subsection_map_index(pfn) + return test_bit(idx, ms['usage']['subsection_map']) + else: + return True + + def valid_section(self, mem_section): + if mem_section != None and (mem_section['section_mem_map'] & self.SECTION_HAS_MEM_MAP): + return True + return False + + def early_section(self, mem_section): + if mem_section != None and (mem_section['section_mem_map'] & self.SECTION_IS_EARLY): + return True + return False + + def pfn_valid(self, pfn): + ms = None + if self.PHYS_PFN(self.PFN_PHYS(pfn)) != pfn: + return False + if self.pfn_to_section_nr(pfn) >= self.NR_MEM_SECTIONS: + return False + ms = self.__pfn_to_section(pfn) + + if not self.valid_section(ms): + return False + return self.early_section(ms) or self.pfn_section_valid(ms, pfn) + + def _PAGE_OFFSET(self, va): + return (-(1 << (va))) & 0xffffffffffffffff + + def _PAGE_END(self, va): + return (-(1 << (va - 1))) & 0xffffffffffffffff + + def kasan_reset_tag(self, addr): + if constants.LX_CONFIG_KASAN_SW_TAGS or constants.LX_CONFIG_KASAN_HW_TAGS: + return int(addr) | (0xff << 56) + else: + return addr + + def __is_lm_address(self, addr): + if (addr - self.PAGE_OFFSET) < (self.PAGE_END - self.PAGE_OFFSET): + return True + else: + return False + def __lm_to_phys(self, addr): + return addr - self.PAGE_OFFSET + self.PHYS_OFFSET + + def __kimg_to_phys(self, addr): + return addr - self.kimage_voffset + + def __virt_to_phys_nodebug(self, va): + untagged_va = self.kasan_reset_tag(va) + if self.__is_lm_address(untagged_va): + return self.__lm_to_phys(untagged_va) + else: + return self.__kimg_to_phys(untagged_va) + + def __virt_to_phys(self, va): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + if not self.__is_lm_address(self.kasan_reset_tag(va)): + raise gdb.GdbError("Warning: virt_to_phys used for non-linear address: 0x%lx\n" % va) + return self.__virt_to_phys_nodebug(va) + + def virt_to_phys(self, va): + return self.__virt_to_phys(va) + + def PFN_PHYS(self, pfn): + return pfn << self.PAGE_SHIFT + + def PHYS_PFN(self, phys): + return phys >> self.PAGE_SHIFT + + def __phys_to_virt(self, pa): + return (pa - self.PHYS_OFFSET) | self.PAGE_OFFSET + + def __phys_to_pfn(self, pa): + return self.PHYS_PFN(pa) + + def __pfn_to_phys(self, pfn): + return self.PFN_PHYS(pfn) + + def __pa_symbol_nodebug(self, x): + return self.__kimg_to_phys(x) + + def __phys_addr_symbol(self, x): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + if x < self.KERNEL_START or x > self.KERNEL_END: + raise gdb.GdbError("0x%x exceed kernel range" % x) + return self.__pa_symbol_nodebug(x) + + def __pa_symbol(self, x): + return self.__phys_addr_symbol(x) + + def __va(self, pa): + return self.__phys_to_virt(pa) + + def pfn_to_kaddr(self, pfn): + return self.__va(pfn << self.PAGE_SHIFT) + + def virt_to_pfn(self, va): + return self.__phys_to_pfn(self.__virt_to_phys(va)) + + def sym_to_pfn(self, x): + return self.__phys_to_pfn(self.__pa_symbol(x)) + + def page_to_pfn(self, page): + return int(page.cast(utils.get_page_type().pointer()) - self.vmemmap.cast(utils.get_page_type().pointer())) + + def page_to_phys(self, page): + return self.__pfn_to_phys(self.page_to_pfn(page)) + + def pfn_to_page(self, pfn): + return (self.vmemmap + pfn).cast(utils.get_page_type().pointer()) + + def page_to_virt(self, page): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + return self.__va(self.page_to_phys(page)) else: - page_size = is_bit_defined_tupled(data, 7) - page_size_bit = page_size[1] - self.is_page = page_size_bit - self.entry_present = is_bit_defined_tupled(data, 0) - self.read_write = is_bit_defined_tupled(data, 1) - self.user_access_allowed = is_bit_defined_tupled(data, 2) - self.page_level_write_through = is_bit_defined_tupled(data, 3) - self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) - self.entry_was_accessed = is_bit_defined_tupled(data, 5) - self.page_size = page_size - self.dirty = is_bit_defined_tupled( - data, 6) if page_size_bit else None - self.global_translation = is_bit_defined_tupled( - data, 8) if page_size_bit else None - self.pat = is_bit_defined_tupled( - data, 12) if page_size_bit else None - self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) if page_size_bit else None - self.next_entry_physical_address = None if page_size_bit else data & PHYSICAL_ADDRESS_MASK & page_mask() - self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) - self.protection_key = content_tupled(data, 59, 62) if page_size_bit else None - self.executed_disable = is_bit_defined_tupled(data, 63) - self.address = address - self.page_entry_binary_data = data - self.page_hierarchy_level = level - - def next_entry(self, va): - if self.is_page or not self.entry_present[1]: - return None - - next_level = self.page_hierarchy_level - 1 - return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) - - - def mk_string(self): - if not self.entry_present[1]: - return f"""\ -level {self.page_hierarchy_level}: - {'entry address': <30} {hex(self.address)} - {'page entry binary data': <30} {hex(self.page_entry_binary_data)} - --- - PAGE ENTRY IS NOT PRESENT! -""" - elif self.is_page: - def page_size_line(ps_bit, ps, level): - return "" if level == 1 else f"{'bit': <3} {ps_bit: <5} {'page size': <30} {ps}" - - return f"""\ -level {self.page_hierarchy_level}: - {'entry address': <30} {hex(self.address)} - {'page entry binary data': <30} {hex(self.page_entry_binary_data)} - {'page size': <30} {'1GB' if self.page_hierarchy_level == 3 else '2MB' if self.page_hierarchy_level == 2 else '4KB' if self.page_hierarchy_level == 1 else 'Unknown page size for level:' + self.page_hierarchy_level} - {'page physical address': <30} {hex(self.page_physical_address)} - --- - {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} - {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} - {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} - {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} - {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} - {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} - {"" if self.page_hierarchy_level == 1 else f"{'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]}"} - {'bit': <4} {self.dirty[0]: <10} {'page dirty': <30} {self.dirty[1]} - {'bit': <4} {self.global_translation[0]: <10} {'global translation': <30} {self.global_translation[1]} - {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} - {'bit': <4} {self.pat[0]: <10} {'pat': <30} {self.pat[1]} - {'bits': <4} {str(self.protection_key[0]): <10} {'protection key': <30} {self.protection_key[1]} - {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} -""" + __idx = int((page.cast(gdb.lookup_type("unsigned long")) - self.VMEMMAP_START).cast(utils.get_ulong_type())) // self.struct_page_size + return self.PAGE_OFFSET + (__idx * self.PAGE_SIZE) + + def virt_to_page(self, va): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + return self.pfn_to_page(self.virt_to_pfn(va)) else: - return f"""\ -level {self.page_hierarchy_level}: - {'entry address': <30} {hex(self.address)} - {'page entry binary data': <30} {hex(self.page_entry_binary_data)} - {'next entry physical address': <30} {hex(self.next_entry_physical_address)} - --- - {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} - {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} - {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} - {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} - {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} - {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} - {'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]} - {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} - {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} -""" - - -class TranslateVM(gdb.Command): - """Prints the entire paging structure used to translate a given virtual address. - -Having an address space of the currently executed process translates the virtual address -and prints detailed information of all paging structure levels used for the transaltion. -Currently supported arch: x86""" + __idx = int(self.kasan_reset_tag(va) - self.PAGE_OFFSET) // self.PAGE_SIZE + addr = self.VMEMMAP_START + (__idx * self.struct_page_size) + return gdb.Value(addr).cast(utils.get_page_type().pointer()) + + def page_address(self, page): + return self.page_to_virt(page) + + def folio_address(self, folio): + return self.page_address(folio['page'].address) + +class LxPFN2Page(gdb.Command): + """PFN to struct page""" def __init__(self): - super(TranslateVM, self).__init__('translate-vm', gdb.COMMAND_USER) + super(LxPFN2Page, self).__init__("lx-pfn_to_page", gdb.COMMAND_USER) def invoke(self, arg, from_tty): - if utils.is_target_arch("x86"): - vm_address = gdb.parse_and_eval(f'{arg}') - cr3_data = gdb.parse_and_eval('$cr3') - cr4 = gdb.parse_and_eval('$cr4') - page_levels = 5 if cr4 & (1 << 12) else 4 - page_entry = Cr3(cr3_data, page_levels) - while page_entry: - gdb.write(page_entry.mk_string()) - page_entry = page_entry.next_entry(vm_address) - else: - gdb.GdbError("Virtual address translation is not" - "supported for this arch") + argv = gdb.string_to_argv(arg) + pfn = int(argv[0]) + page = page_ops().ops.pfn_to_page(pfn) + gdb.write("pfn_to_page(0x%x) = 0x%x\n" % (pfn, page)) + +LxPFN2Page() + +class LxPage2PFN(gdb.Command): + """struct page to PFN""" + + def __init__(self): + super(LxPage2PFN, self).__init__("lx-page_to_pfn", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + struct_page_addr = int(argv[0], 16) + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + pfn = page_ops().ops.page_to_pfn(page) + gdb.write("page_to_pfn(0x%x) = 0x%x\n" % (page, pfn)) + +LxPage2PFN() + +class LxPageAddress(gdb.Command): + """struct page to linear mapping address""" + + def __init__(self): + super(LxPageAddress, self).__init__("lx-page_address", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + struct_page_addr = int(argv[0], 16) + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + addr = page_ops().ops.page_address(page) + gdb.write("page_address(0x%x) = 0x%x\n" % (page, addr)) +LxPageAddress() + +class LxPage2Phys(gdb.Command): + """struct page to physical address""" + + def __init__(self): + super(LxPage2Phys, self).__init__("lx-page_to_phys", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + struct_page_addr = int(argv[0], 16) + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + phys_addr = page_ops().ops.page_to_phys(page) + gdb.write("page_to_phys(0x%x) = 0x%x\n" % (page, phys_addr)) + +LxPage2Phys() + +class LxVirt2Phys(gdb.Command): + """virtual address to physical address""" + + def __init__(self): + super(LxVirt2Phys, self).__init__("lx-virt_to_phys", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + linear_addr = int(argv[0], 16) + phys_addr = page_ops().ops.virt_to_phys(linear_addr) + gdb.write("virt_to_phys(0x%x) = 0x%x\n" % (linear_addr, phys_addr)) + +LxVirt2Phys() + +class LxVirt2Page(gdb.Command): + """virtual address to struct page""" + + def __init__(self): + super(LxVirt2Page, self).__init__("lx-virt_to_page", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + linear_addr = int(argv[0], 16) + page = page_ops().ops.virt_to_page(linear_addr) + gdb.write("virt_to_page(0x%x) = 0x%x\n" % (linear_addr, page)) + +LxVirt2Page() + +class LxSym2PFN(gdb.Command): + """symbol address to PFN""" + + def __init__(self): + super(LxSym2PFN, self).__init__("lx-sym_to_pfn", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + sym_addr = int(argv[0], 16) + pfn = page_ops().ops.sym_to_pfn(sym_addr) + gdb.write("sym_to_pfn(0x%x) = %d\n" % (sym_addr, pfn)) + +LxSym2PFN() + +class LxPFN2Kaddr(gdb.Command): + """PFN to kernel address""" + + def __init__(self): + super(LxPFN2Kaddr, self).__init__("lx-pfn_to_kaddr", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + pfn = int(argv[0]) + kaddr = page_ops().ops.pfn_to_kaddr(pfn) + gdb.write("pfn_to_kaddr(%d) = 0x%x\n" % (pfn, kaddr)) -TranslateVM() +LxPFN2Kaddr() diff --git a/scripts/gdb/linux/modules.py b/scripts/gdb/linux/modules.py index 261f28640f4c..298dfcc25eae 100644 --- a/scripts/gdb/linux/modules.py +++ b/scripts/gdb/linux/modules.py @@ -73,11 +73,17 @@ class LxLsmod(gdb.Command): " " if utils.get_long_type().sizeof == 8 else "")) for module in module_list(): - layout = module['mem'][constants.LX_MOD_TEXT] + text = module['mem'][constants.LX_MOD_TEXT] + text_addr = str(text['base']).split()[0] + total_size = 0 + + for i in range(constants.LX_MOD_TEXT, constants.LX_MOD_RO_AFTER_INIT + 1): + total_size += module['mem'][i]['size'] + gdb.write("{address} {name:<19} {size:>8} {ref}".format( - address=str(layout['base']).split()[0], + address=text_addr, name=module['name'].string(), - size=str(layout['size']), + size=str(total_size), ref=str(module['refcnt']['counter'] - 1))) t = self._module_use_type.get_type().pointer() @@ -91,5 +97,35 @@ class LxLsmod(gdb.Command): gdb.write("\n") - LxLsmod() + +def help(): + t = """Usage: lx-getmod-by-textaddr [Heximal Address] + Example: lx-getmod-by-textaddr 0xffff800002d305ac\n""" + gdb.write("Unrecognized command\n") + raise gdb.GdbError(t) + +class LxFindTextAddrinMod(gdb.Command): + '''Look up loaded kernel module by text address.''' + + def __init__(self): + super(LxFindTextAddrinMod, self).__init__('lx-getmod-by-textaddr', gdb.COMMAND_SUPPORT) + + def invoke(self, arg, from_tty): + args = gdb.string_to_argv(arg) + + if len(args) != 1: + help() + + addr = gdb.Value(int(args[0], 16)).cast(utils.get_ulong_type()) + for mod in module_list(): + mod_text_start = mod['mem'][constants.LX_MOD_TEXT]['base'] + mod_text_end = mod_text_start + mod['mem'][constants.LX_MOD_TEXT]['size'].cast(utils.get_ulong_type()) + + if addr >= mod_text_start and addr < mod_text_end: + s = "0x%x" % addr + " is in " + mod['name'].string() + ".ko\n" + gdb.write(s) + return + gdb.write("0x%x is not in any module text section\n" % addr) + +LxFindTextAddrinMod() diff --git a/scripts/gdb/linux/page_owner.py b/scripts/gdb/linux/page_owner.py new file mode 100644 index 000000000000..844fd5d0c912 --- /dev/null +++ b/scripts/gdb/linux/page_owner.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +from linux import utils, stackdepot, constants, mm + +if constants.LX_CONFIG_PAGE_OWNER: + page_ext_t = utils.CachedType('struct page_ext') + page_owner_t = utils.CachedType('struct page_owner') + + PAGE_OWNER_STACK_DEPTH = 16 + PAGE_EXT_OWNER = constants.LX_PAGE_EXT_OWNER + PAGE_EXT_INVALID = 0x1 + PAGE_EXT_OWNER_ALLOCATED = constants.LX_PAGE_EXT_OWNER_ALLOCATED + +def help(): + t = """Usage: lx-dump-page-owner [Option] + Option: + --pfn [Decimal pfn] + Example: + lx-dump-page-owner --pfn 655360\n""" + gdb.write("Unrecognized command\n") + raise gdb.GdbError(t) + +class DumpPageOwner(gdb.Command): + """Dump page owner""" + + min_pfn = None + max_pfn = None + p_ops = None + migrate_reason_names = None + + def __init__(self): + super(DumpPageOwner, self).__init__("lx-dump-page-owner", gdb.COMMAND_SUPPORT) + + def invoke(self, args, from_tty): + if not constants.LX_CONFIG_PAGE_OWNER: + raise gdb.GdbError('CONFIG_PAGE_OWNER does not enable') + + page_owner_inited = gdb.parse_and_eval('page_owner_inited') + if page_owner_inited['key']['enabled']['counter'] != 0x1: + raise gdb.GdbError('page_owner_inited is not enabled') + + self.p_ops = mm.page_ops().ops + self.get_page_owner_info() + argv = gdb.string_to_argv(args) + if len(argv) == 0: + self.read_page_owner() + elif len(argv) == 2: + if argv[0] == "--pfn": + pfn = int(argv[1]) + self.read_page_owner_by_addr(self.p_ops.pfn_to_page(pfn)) + else: + help() + else: + help() + + def get_page_owner_info(self): + self.min_pfn = int(gdb.parse_and_eval("min_low_pfn")) + self.max_pfn = int(gdb.parse_and_eval("max_pfn")) + self.page_ext_size = int(gdb.parse_and_eval("page_ext_size")) + self.migrate_reason_names = gdb.parse_and_eval('migrate_reason_names') + + def page_ext_invalid(self, page_ext): + if page_ext == gdb.Value(0): + return True + if page_ext.cast(utils.get_ulong_type()) & PAGE_EXT_INVALID == PAGE_EXT_INVALID: + return True + return False + + def get_entry(self, base, index): + return (base.cast(utils.get_ulong_type()) + self.page_ext_size * index).cast(page_ext_t.get_type().pointer()) + + def lookup_page_ext(self, page): + pfn = self.p_ops.page_to_pfn(page) + section = self.p_ops.pfn_to_section(pfn) + page_ext = section["page_ext"] + if self.page_ext_invalid(page_ext): + return gdb.Value(0) + return self.get_entry(page_ext, pfn) + + def page_ext_get(self, page): + page_ext = self.lookup_page_ext(page) + if page_ext != gdb.Value(0): + return page_ext + else: + return gdb.Value(0) + + def get_page_owner(self, page_ext): + addr = page_ext.cast(utils.get_ulong_type()) + gdb.parse_and_eval("page_owner_ops")["offset"].cast(utils.get_ulong_type()) + return addr.cast(page_owner_t.get_type().pointer()) + + def read_page_owner_by_addr(self, struct_page_addr): + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + pfn = self.p_ops.page_to_pfn(page) + + if pfn < self.min_pfn or pfn > self.max_pfn or (not self.p_ops.pfn_valid(pfn)): + gdb.write("pfn is invalid\n") + return + + page = self.p_ops.pfn_to_page(pfn) + page_ext = self.page_ext_get(page) + + if page_ext == gdb.Value(0): + gdb.write("page_ext is null\n") + return + + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER)): + gdb.write("page_owner flag is invalid\n") + raise gdb.GdbError('page_owner info is not present (never set?)\n') + + if mm.test_bit(PAGE_EXT_OWNER_ALLOCATED, page_ext['flags'].address): + gdb.write('page_owner tracks the page as allocated\n') + else: + gdb.write('page_owner tracks the page as freed\n') + + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER_ALLOCATED)): + gdb.write("page_owner is not allocated\n") + + try: + page_owner = self.get_page_owner(page_ext) + gdb.write("Page last allocated via order %d, gfp_mask: 0x%x, pid: %d, tgid: %d (%s), ts %u ns, free_ts %u ns\n" %\ + (page_owner["order"], page_owner["gfp_mask"],\ + page_owner["pid"], page_owner["tgid"], page_owner["comm"],\ + page_owner["ts_nsec"], page_owner["free_ts_nsec"])) + gdb.write("PFN: %d, Flags: 0x%x\n" % (pfn, page['flags'])) + if page_owner["handle"] == 0: + gdb.write('page_owner allocation stack trace missing\n') + else: + stackdepot.stack_depot_print(page_owner["handle"]) + + if page_owner["free_handle"] == 0: + gdb.write('page_owner free stack trace missing\n') + else: + gdb.write('page last free stack trace:\n') + stackdepot.stack_depot_print(page_owner["free_handle"]) + if page_owner['last_migrate_reason'] != -1: + gdb.write('page has been migrated, last migrate reason: %s\n' % self.migrate_reason_names[page_owner['last_migrate_reason']]) + except: + gdb.write("\n") + + def read_page_owner(self): + pfn = self.min_pfn + + # Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area + while ((not self.p_ops.pfn_valid(pfn)) and (pfn & (self.p_ops.MAX_ORDER_NR_PAGES - 1))) != 0: + pfn += 1 + + while pfn < self.max_pfn: + # + # If the new page is in a new MAX_ORDER_NR_PAGES area, + # validate the area as existing, skip it if not + # + if ((pfn & (self.p_ops.MAX_ORDER_NR_PAGES - 1)) == 0) and (not self.p_ops.pfn_valid(pfn)): + pfn += (self.p_ops.MAX_ORDER_NR_PAGES - 1) + continue; + + page = self.p_ops.pfn_to_page(pfn) + page_ext = self.page_ext_get(page) + if page_ext == gdb.Value(0): + pfn += 1 + continue + + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER)): + pfn += 1 + continue + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER_ALLOCATED)): + pfn += 1 + continue + + try: + page_owner = self.get_page_owner(page_ext) + gdb.write("Page allocated via order %d, gfp_mask: 0x%x, pid: %d, tgid: %d (%s), ts %u ns, free_ts %u ns\n" %\ + (page_owner["order"], page_owner["gfp_mask"],\ + page_owner["pid"], page_owner["tgid"], page_owner["comm"],\ + page_owner["ts_nsec"], page_owner["free_ts_nsec"])) + gdb.write("PFN: %d, Flags: 0x%x\n" % (pfn, page['flags'])) + stackdepot.stack_depot_print(page_owner["handle"]) + pfn += (1 << page_owner["order"]) + continue + except: + gdb.write("\n") + pfn += 1 + +DumpPageOwner() diff --git a/scripts/gdb/linux/pgtable.py b/scripts/gdb/linux/pgtable.py new file mode 100644 index 000000000000..30d837f3dfae --- /dev/null +++ b/scripts/gdb/linux/pgtable.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# gdb helper commands and functions for Linux kernel debugging +# +# routines to introspect page table +# +# Authors: +# Dmitrii Bundin <dmitrii.bundin.a@gmail.com> +# + +import gdb + +from linux import utils + +PHYSICAL_ADDRESS_MASK = gdb.parse_and_eval('0xfffffffffffff') + + +def page_mask(level=1): + # 4KB + if level == 1: + return gdb.parse_and_eval('(u64) ~0xfff') + # 2MB + elif level == 2: + return gdb.parse_and_eval('(u64) ~0x1fffff') + # 1GB + elif level == 3: + return gdb.parse_and_eval('(u64) ~0x3fffffff') + else: + raise Exception(f'Unknown page level: {level}') + + +#page_offset_base in case CONFIG_DYNAMIC_MEMORY_LAYOUT is disabled +POB_NO_DYNAMIC_MEM_LAYOUT = '0xffff888000000000' +def _page_offset_base(): + pob_symbol = gdb.lookup_global_symbol('page_offset_base') + pob = pob_symbol.name if pob_symbol else POB_NO_DYNAMIC_MEM_LAYOUT + return gdb.parse_and_eval(pob) + + +def is_bit_defined_tupled(data, offset): + return offset, bool(data >> offset & 1) + +def content_tupled(data, bit_start, bit_end): + return (bit_start, bit_end), data >> bit_start & ((1 << (1 + bit_end - bit_start)) - 1) + +def entry_va(level, phys_addr, translating_va): + def start_bit(level): + if level == 5: + return 48 + elif level == 4: + return 39 + elif level == 3: + return 30 + elif level == 2: + return 21 + elif level == 1: + return 12 + else: + raise Exception(f'Unknown level {level}') + + entry_offset = ((translating_va >> start_bit(level)) & 511) * 8 + entry_va = _page_offset_base() + phys_addr + entry_offset + return entry_va + +class Cr3(): + def __init__(self, cr3, page_levels): + self.cr3 = cr3 + self.page_levels = page_levels + self.page_level_write_through = is_bit_defined_tupled(cr3, 3) + self.page_level_cache_disabled = is_bit_defined_tupled(cr3, 4) + self.next_entry_physical_address = cr3 & PHYSICAL_ADDRESS_MASK & page_mask() + + def next_entry(self, va): + next_level = self.page_levels + return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) + + def mk_string(self): + return f"""\ +cr3: + {'cr3 binary data': <30} {hex(self.cr3)} + {'next entry physical address': <30} {hex(self.next_entry_physical_address)} + --- + {'bit' : <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} + {'bit' : <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} +""" + + +class PageHierarchyEntry(): + def __init__(self, address, level): + data = int.from_bytes( + memoryview(gdb.selected_inferior().read_memory(address, 8)), + "little" + ) + if level == 1: + self.is_page = True + self.entry_present = is_bit_defined_tupled(data, 0) + self.read_write = is_bit_defined_tupled(data, 1) + self.user_access_allowed = is_bit_defined_tupled(data, 2) + self.page_level_write_through = is_bit_defined_tupled(data, 3) + self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) + self.entry_was_accessed = is_bit_defined_tupled(data, 5) + self.dirty = is_bit_defined_tupled(data, 6) + self.pat = is_bit_defined_tupled(data, 7) + self.global_translation = is_bit_defined_tupled(data, 8) + self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) + self.next_entry_physical_address = None + self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) + self.protection_key = content_tupled(data, 59, 62) + self.executed_disable = is_bit_defined_tupled(data, 63) + else: + page_size = is_bit_defined_tupled(data, 7) + page_size_bit = page_size[1] + self.is_page = page_size_bit + self.entry_present = is_bit_defined_tupled(data, 0) + self.read_write = is_bit_defined_tupled(data, 1) + self.user_access_allowed = is_bit_defined_tupled(data, 2) + self.page_level_write_through = is_bit_defined_tupled(data, 3) + self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) + self.entry_was_accessed = is_bit_defined_tupled(data, 5) + self.page_size = page_size + self.dirty = is_bit_defined_tupled( + data, 6) if page_size_bit else None + self.global_translation = is_bit_defined_tupled( + data, 8) if page_size_bit else None + self.pat = is_bit_defined_tupled( + data, 12) if page_size_bit else None + self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) if page_size_bit else None + self.next_entry_physical_address = None if page_size_bit else data & PHYSICAL_ADDRESS_MASK & page_mask() + self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) + self.protection_key = content_tupled(data, 59, 62) if page_size_bit else None + self.executed_disable = is_bit_defined_tupled(data, 63) + self.address = address + self.page_entry_binary_data = data + self.page_hierarchy_level = level + + def next_entry(self, va): + if self.is_page or not self.entry_present[1]: + return None + + next_level = self.page_hierarchy_level - 1 + return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) + + + def mk_string(self): + if not self.entry_present[1]: + return f"""\ +level {self.page_hierarchy_level}: + {'entry address': <30} {hex(self.address)} + {'page entry binary data': <30} {hex(self.page_entry_binary_data)} + --- + PAGE ENTRY IS NOT PRESENT! +""" + elif self.is_page: + def page_size_line(ps_bit, ps, level): + return "" if level == 1 else f"{'bit': <3} {ps_bit: <5} {'page size': <30} {ps}" + + return f"""\ +level {self.page_hierarchy_level}: + {'entry address': <30} {hex(self.address)} + {'page entry binary data': <30} {hex(self.page_entry_binary_data)} + {'page size': <30} {'1GB' if self.page_hierarchy_level == 3 else '2MB' if self.page_hierarchy_level == 2 else '4KB' if self.page_hierarchy_level == 1 else 'Unknown page size for level:' + self.page_hierarchy_level} + {'page physical address': <30} {hex(self.page_physical_address)} + --- + {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} + {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} + {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} + {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} + {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} + {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} + {"" if self.page_hierarchy_level == 1 else f"{'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]}"} + {'bit': <4} {self.dirty[0]: <10} {'page dirty': <30} {self.dirty[1]} + {'bit': <4} {self.global_translation[0]: <10} {'global translation': <30} {self.global_translation[1]} + {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} + {'bit': <4} {self.pat[0]: <10} {'pat': <30} {self.pat[1]} + {'bits': <4} {str(self.protection_key[0]): <10} {'protection key': <30} {self.protection_key[1]} + {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} +""" + else: + return f"""\ +level {self.page_hierarchy_level}: + {'entry address': <30} {hex(self.address)} + {'page entry binary data': <30} {hex(self.page_entry_binary_data)} + {'next entry physical address': <30} {hex(self.next_entry_physical_address)} + --- + {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} + {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} + {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} + {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} + {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} + {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} + {'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]} + {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} + {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} +""" + + +class TranslateVM(gdb.Command): + """Prints the entire paging structure used to translate a given virtual address. + +Having an address space of the currently executed process translates the virtual address +and prints detailed information of all paging structure levels used for the transaltion. +Currently supported arch: x86""" + + def __init__(self): + super(TranslateVM, self).__init__('translate-vm', gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + if utils.is_target_arch("x86"): + vm_address = gdb.parse_and_eval(f'{arg}') + cr3_data = gdb.parse_and_eval('$cr3') + cr4 = gdb.parse_and_eval('$cr4') + page_levels = 5 if cr4 & (1 << 12) else 4 + page_entry = Cr3(cr3_data, page_levels) + while page_entry: + gdb.write(page_entry.mk_string()) + page_entry = page_entry.next_entry(vm_address) + else: + gdb.GdbError("Virtual address translation is not" + "supported for this arch") + + +TranslateVM() diff --git a/scripts/gdb/linux/slab.py b/scripts/gdb/linux/slab.py new file mode 100644 index 000000000000..f012ba38c7d9 --- /dev/null +++ b/scripts/gdb/linux/slab.py @@ -0,0 +1,326 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +import re +import traceback +from linux import lists, utils, stackdepot, constants, mm + +SLAB_RED_ZONE = constants.LX_SLAB_RED_ZONE +SLAB_POISON = constants.LX_SLAB_POISON +SLAB_KMALLOC = constants.LX_SLAB_KMALLOC +SLAB_HWCACHE_ALIGN = constants.LX_SLAB_HWCACHE_ALIGN +SLAB_CACHE_DMA = constants.LX_SLAB_CACHE_DMA +SLAB_CACHE_DMA32 = constants.LX_SLAB_CACHE_DMA32 +SLAB_STORE_USER = constants.LX_SLAB_STORE_USER +SLAB_PANIC = constants.LX_SLAB_PANIC + +OO_SHIFT = 16 +OO_MASK = (1 << OO_SHIFT) - 1 + +if constants.LX_CONFIG_SLUB_DEBUG: + slab_type = utils.CachedType("struct slab") + slab_ptr_type = slab_type.get_type().pointer() + kmem_cache_type = utils.CachedType("struct kmem_cache") + kmem_cache_ptr_type = kmem_cache_type.get_type().pointer() + freeptr_t = utils.CachedType("freeptr_t") + freeptr_t_ptr = freeptr_t.get_type().pointer() + + track_type = gdb.lookup_type('struct track') + track_alloc = int(gdb.parse_and_eval('TRACK_ALLOC')) + track_free = int(gdb.parse_and_eval('TRACK_FREE')) + +def slab_folio(slab): + return slab.cast(gdb.lookup_type("struct folio").pointer()) + +def slab_address(slab): + p_ops = mm.page_ops().ops + folio = slab_folio(slab) + return p_ops.folio_address(folio) + +def for_each_object(cache, addr, slab_objects): + p = addr + if cache['flags'] & SLAB_RED_ZONE: + p += int(cache['red_left_pad']) + while p < addr + (slab_objects * cache['size']): + yield p + p = p + int(cache['size']) + +def get_info_end(cache): + if (cache['offset'] >= cache['inuse']): + return cache['inuse'] + gdb.lookup_type("void").pointer().sizeof + else: + return cache['inuse'] + +def get_orig_size(cache, obj): + if cache['flags'] & SLAB_STORE_USER and cache['flags'] & SLAB_KMALLOC: + p = mm.page_ops().ops.kasan_reset_tag(obj) + p += get_info_end(cache) + p += gdb.lookup_type('struct track').sizeof * 2 + p = p.cast(utils.get_uint_type().pointer()) + return p.dereference() + else: + return cache['object_size'] + +def get_track(cache, object_pointer, alloc): + p = object_pointer + get_info_end(cache) + p += (alloc * track_type.sizeof) + return p + +def oo_objects(x): + return int(x['x']) & OO_MASK + +def oo_order(x): + return int(x['x']) >> OO_SHIFT + +def reciprocal_divide(a, R): + t = (a * int(R['m'])) >> 32 + return (t + ((a - t) >> int(R['sh1']))) >> int(R['sh2']) + +def __obj_to_index(cache, addr, obj): + return reciprocal_divide(int(mm.page_ops().ops.kasan_reset_tag(obj)) - addr, cache['reciprocal_size']) + +def swab64(x): + result = (((x & 0x00000000000000ff) << 56) | \ + ((x & 0x000000000000ff00) << 40) | \ + ((x & 0x0000000000ff0000) << 24) | \ + ((x & 0x00000000ff000000) << 8) | \ + ((x & 0x000000ff00000000) >> 8) | \ + ((x & 0x0000ff0000000000) >> 24) | \ + ((x & 0x00ff000000000000) >> 40) | \ + ((x & 0xff00000000000000) >> 56)) + return result + +def freelist_ptr_decode(cache, ptr, ptr_addr): + if constants.LX_CONFIG_SLAB_FREELIST_HARDENED: + return ptr['v'] ^ cache['random'] ^ swab64(int(ptr_addr)) + else: + return ptr['v'] + +def get_freepointer(cache, obj): + obj = mm.page_ops().ops.kasan_reset_tag(obj) + ptr_addr = obj + cache['offset'] + p = ptr_addr.cast(freeptr_t_ptr).dereference() + return freelist_ptr_decode(cache, p, ptr_addr) + +def loc_exist(loc_track, addr, handle, waste): + for loc in loc_track: + if loc['addr'] == addr and loc['handle'] == handle and loc['waste'] == waste: + return loc + return None + +def add_location(loc_track, cache, track, orig_size): + jiffies = gdb.parse_and_eval("jiffies_64") + age = jiffies - track['when'] + handle = 0 + waste = cache['object_size'] - int(orig_size) + pid = int(track['pid']) + cpuid = int(track['cpu']) + addr = track['addr'] + if constants.LX_CONFIG_STACKDEPOT: + handle = track['handle'] + + loc = loc_exist(loc_track, addr, handle, waste) + if loc: + loc['count'] += 1 + if track['when']: + loc['sum_time'] += age + loc['min_time'] = min(loc['min_time'], age) + loc['max_time'] = max(loc['max_time'], age) + loc['min_pid'] = min(loc['min_pid'], pid) + loc['max_pid'] = max(loc['max_pid'], pid) + loc['cpus'].add(cpuid) + else: + loc_track.append({ + 'count' : 1, + 'addr' : addr, + 'sum_time' : age, + 'min_time' : age, + 'max_time' : age, + 'min_pid' : pid, + 'max_pid' : pid, + 'handle' : handle, + 'waste' : waste, + 'cpus' : {cpuid} + } + ) + +def slabtrace(alloc, cache_name): + + def __fill_map(obj_map, cache, slab): + p = slab['freelist'] + addr = slab_address(slab) + while p != gdb.Value(0): + index = __obj_to_index(cache, addr, p) + obj_map[index] = True # free objects + p = get_freepointer(cache, p) + + # process every slab page on the slab_list (partial and full list) + def process_slab(loc_track, slab_list, alloc, cache): + for slab in lists.list_for_each_entry(slab_list, slab_ptr_type, "slab_list"): + obj_map[:] = [False] * oo_objects(cache['oo']) + __fill_map(obj_map, cache, slab) + addr = slab_address(slab) + for object_pointer in for_each_object(cache, addr, slab['objects']): + if obj_map[__obj_to_index(cache, addr, object_pointer)] == True: + continue + p = get_track(cache, object_pointer, alloc) + track = gdb.Value(p).cast(track_type.pointer()) + if alloc == track_alloc: + size = get_orig_size(cache, object_pointer) + else: + size = cache['object_size'] + add_location(loc_track, cache, track, size) + continue + + slab_caches = gdb.parse_and_eval("slab_caches") + if mm.page_ops().ops.MAX_NUMNODES > 1: + nr_node_ids = int(gdb.parse_and_eval("nr_node_ids")) + else: + nr_node_ids = 1 + + target_cache = None + loc_track = [] + + for cache in lists.list_for_each_entry(slab_caches, kmem_cache_ptr_type, 'list'): + if cache['name'].string() == cache_name: + target_cache = cache + break + + obj_map = [False] * oo_objects(target_cache['oo']) + + if target_cache['flags'] & SLAB_STORE_USER: + for i in range(0, nr_node_ids): + cache_node = target_cache['node'][i] + if cache_node['nr_slabs']['counter'] == 0: + continue + process_slab(loc_track, cache_node['partial'], alloc, target_cache) + process_slab(loc_track, cache_node['full'], alloc, target_cache) + else: + raise gdb.GdbError("SLAB_STORE_USER is not set in %s" % target_cache['name'].string()) + + for loc in sorted(loc_track, key=lambda x:x['count'], reverse=True): + if loc['addr']: + addr = loc['addr'].cast(utils.get_ulong_type().pointer()) + gdb.write("%d %s" % (loc['count'], str(addr).split(' ')[-1])) + else: + gdb.write("%d <not-available>" % loc['count']) + + if loc['waste']: + gdb.write(" waste=%d/%d" % (loc['count'] * loc['waste'], loc['waste'])) + + if loc['sum_time'] != loc['min_time']: + gdb.write(" age=%d/%d/%d" % (loc['min_time'], loc['sum_time']/loc['count'], loc['max_time'])) + else: + gdb.write(" age=%d" % loc['min_time']) + + if loc['min_pid'] != loc['max_pid']: + gdb.write(" pid=%d-%d" % (loc['min_pid'], loc['max_pid'])) + else: + gdb.write(" pid=%d" % loc['min_pid']) + + if constants.LX_NR_CPUS > 1: + nr_cpu = gdb.parse_and_eval('__num_online_cpus')['counter'] + if nr_cpu > 1: + gdb.write(" cpus=") + for i in loc['cpus']: + gdb.write("%d," % i) + gdb.write("\n") + if constants.LX_CONFIG_STACKDEPOT: + if loc['handle']: + stackdepot.stack_depot_print(loc['handle']) + gdb.write("\n") + +def help(): + t = """Usage: lx-slabtrace --cache_name [cache_name] [Options] + Options: + --alloc + print information of allocation trace of the allocated objects + --free + print information of freeing trace of the allocated objects + Example: + lx-slabtrace --cache_name kmalloc-1k --alloc + lx-slabtrace --cache_name kmalloc-1k --free\n""" + gdb.write("Unrecognized command\n") + raise gdb.GdbError(t) + +class LxSlabTrace(gdb.Command): + """Show specific cache slabtrace""" + + def __init__(self): + super(LxSlabTrace, self).__init__("lx-slabtrace", gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + if not constants.LX_CONFIG_SLUB_DEBUG: + raise gdb.GdbError("CONFIG_SLUB_DEBUG is not enabled") + + argv = gdb.string_to_argv(arg) + alloc = track_alloc # default show alloc_traces + + if len(argv) == 3: + if argv[2] == '--alloc': + alloc = track_alloc + elif argv[2] == '--free': + alloc = track_free + else: + help() + if len(argv) >= 2 and argv[0] == '--cache_name': + slabtrace(alloc, argv[1]) + else: + help() +LxSlabTrace() + +def slabinfo(): + nr_node_ids = None + + if not constants.LX_CONFIG_SLUB_DEBUG: + raise gdb.GdbError("CONFIG_SLUB_DEBUG is not enabled") + + def count_free(slab): + total_free = 0 + for slab in lists.list_for_each_entry(slab, slab_ptr_type, 'slab_list'): + total_free += int(slab['objects'] - slab['inuse']) + return total_free + + gdb.write("{:^18} | {:^20} | {:^12} | {:^12} | {:^8} | {:^11} | {:^13}\n".format('Pointer', 'name', 'active_objs', 'num_objs', 'objsize', 'objperslab', 'pagesperslab')) + gdb.write("{:-^18} | {:-^20} | {:-^12} | {:-^12} | {:-^8} | {:-^11} | {:-^13}\n".format('', '', '', '', '', '', '')) + + slab_caches = gdb.parse_and_eval("slab_caches") + if mm.page_ops().ops.MAX_NUMNODES > 1: + nr_node_ids = int(gdb.parse_and_eval("nr_node_ids")) + else: + nr_node_ids = 1 + + for cache in lists.list_for_each_entry(slab_caches, kmem_cache_ptr_type, 'list'): + nr_objs = 0 + nr_free = 0 + nr_slabs = 0 + for i in range(0, nr_node_ids): + cache_node = cache['node'][i] + try: + nr_slabs += cache_node['nr_slabs']['counter'] + nr_objs = int(cache_node['total_objects']['counter']) + nr_free = count_free(cache_node['partial']) + except: + raise gdb.GdbError(traceback.format_exc()) + active_objs = nr_objs - nr_free + num_objs = nr_objs + active_slabs = nr_slabs + objects_per_slab = oo_objects(cache['oo']) + cache_order = oo_order(cache['oo']) + gdb.write("{:18s} | {:20.19s} | {:12} | {:12} | {:8} | {:11} | {:13}\n".format(hex(cache), cache['name'].string(), str(active_objs), str(num_objs), str(cache['size']), str(objects_per_slab), str(1 << cache_order))) + +class LxSlabInfo(gdb.Command): + """Show slabinfo""" + + def __init__(self): + super(LxSlabInfo, self).__init__("lx-slabinfo", gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + slabinfo() +LxSlabInfo() diff --git a/scripts/gdb/linux/stackdepot.py b/scripts/gdb/linux/stackdepot.py new file mode 100644 index 000000000000..047d329a6a12 --- /dev/null +++ b/scripts/gdb/linux/stackdepot.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +from linux import utils, constants + +if constants.LX_CONFIG_STACKDEPOT: + stack_record_type = utils.CachedType('struct stack_record') + DEPOT_STACK_ALIGN = 4 + +def stack_depot_fetch(handle): + global DEPOT_STACK_ALIGN + global stack_record_type + + stack_depot_disabled = gdb.parse_and_eval('stack_depot_disabled') + + if stack_depot_disabled: + raise gdb.GdbError("stack_depot_disabled\n") + + handle_parts_t = gdb.lookup_type("union handle_parts") + parts = handle.cast(handle_parts_t) + offset = parts['offset'] << DEPOT_STACK_ALIGN + pool_index_cached = gdb.parse_and_eval('pool_index') + + if parts['pool_index'] > pool_index_cached: + gdb.write("pool index %d out of bounds (%d) for stack id 0x%08x\n" % (parts['pool_index'], pool_index_cached, handle)) + return gdb.Value(0), 0 + + stack_pools = gdb.parse_and_eval('stack_pools') + + try: + pool = stack_pools[parts['pool_index']] + stack = (pool + gdb.Value(offset).cast(utils.get_size_t_type())).cast(stack_record_type.get_type().pointer()) + size = int(stack['size'].cast(utils.get_ulong_type())) + return stack['entries'], size + except Exception as e: + gdb.write("%s\n" % e) + return gdb.Value(0), 0 + +def stack_depot_print(handle): + if not constants.LX_CONFIG_STACKDEPOT: + raise gdb.GdbError("CONFIG_STACKDEPOT is not enabled") + + entries, nr_entries = stack_depot_fetch(handle) + if nr_entries > 0: + for i in range(0, nr_entries): + try: + gdb.execute("x /i 0x%x" % (int(entries[i]))) + except Exception as e: + gdb.write("%s\n" % e) diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index fdad3f32c747..5179edd1b627 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -89,29 +89,34 @@ lx-symbols command.""" return name return None - def _section_arguments(self, module): + def _section_arguments(self, module, module_addr): try: sect_attrs = module['sect_attrs'].dereference() except gdb.error: - return "" + return str(module_addr) + attrs = sect_attrs['attrs'] section_name_to_address = { attrs[n]['battr']['attr']['name'].string(): attrs[n]['address'] for n in range(int(sect_attrs['nsections']))} + + textaddr = section_name_to_address.get(".text", module_addr) args = [] for section_name in [".data", ".data..read_mostly", ".rodata", ".bss", - ".text", ".text.hot", ".text.unlikely"]: + ".text.hot", ".text.unlikely"]: address = section_name_to_address.get(section_name) if address: args.append(" -s {name} {addr}".format( name=section_name, addr=str(address))) - return "".join(args) + return "{textaddr} {sections}".format( + textaddr=textaddr, sections="".join(args)) - def load_module_symbols(self, module): + def load_module_symbols(self, module, module_file=None): module_name = module['name'].string() module_addr = str(module['mem'][constants.LX_MOD_TEXT]['base']).split()[0] - module_file = self._get_module_file(module_name) + if not module_file: + module_file = self._get_module_file(module_name) if not module_file and not self.module_files_updated: self._update_module_files() module_file = self._get_module_file(module_name) @@ -125,16 +130,28 @@ lx-symbols command.""" module_addr = hex(int(module_addr, 0) + plt_offset + plt_size) gdb.write("loading @{addr}: {filename}\n".format( addr=module_addr, filename=module_file)) - cmdline = "add-symbol-file {filename} {addr}{sections}".format( + cmdline = "add-symbol-file {filename} {sections}".format( filename=module_file, - addr=module_addr, - sections=self._section_arguments(module)) + sections=self._section_arguments(module, module_addr)) gdb.execute(cmdline, to_string=True) if module_name not in self.loaded_modules: self.loaded_modules.append(module_name) else: gdb.write("no module object found for '{0}'\n".format(module_name)) + def load_ko_symbols(self, mod_path): + self.loaded_modules = [] + module_list = modules.module_list() + + for module in module_list: + module_name = module['name'].string() + module_pattern = ".*/{0}\.ko(?:.debug)?$".format( + module_name.replace("_", r"[_\-]")) + if re.match(module_pattern, mod_path) and os.path.exists(mod_path): + self.load_module_symbols(module, mod_path) + return + raise gdb.GdbError("%s is not a valid .ko\n" % mod_path) + def load_all_symbols(self): gdb.write("loading vmlinux\n") @@ -173,6 +190,11 @@ lx-symbols command.""" self.module_files = [] self.module_files_updated = False + argv = gdb.string_to_argv(arg) + if len(argv) == 1: + self.load_ko_symbols(argv[0]) + return + self.load_all_symbols() if hasattr(gdb, 'Breakpoint'): diff --git a/scripts/gdb/linux/utils.py b/scripts/gdb/linux/utils.py index 9f44df13761e..7d5278d815fa 100644 --- a/scripts/gdb/linux/utils.py +++ b/scripts/gdb/linux/utils.py @@ -35,12 +35,32 @@ class CachedType: long_type = CachedType("long") +ulong_type = CachedType("unsigned long") +uint_type = CachedType("unsigned int") atomic_long_type = CachedType("atomic_long_t") +size_t_type = CachedType("size_t") +struct_page_type = CachedType("struct page") + +def get_uint_type(): + global uint_type + return uint_type.get_type() + +def get_page_type(): + global struct_page_type + return struct_page_type.get_type() def get_long_type(): global long_type return long_type.get_type() +def get_ulong_type(): + global ulong_type + return ulong_type.get_type() + +def get_size_t_type(): + global size_t_type + return size_t_type.get_type() + def offset_of(typeobj, field): element = gdb.Value(0).cast(typeobj) return int(str(element[field].address).split()[0], 16) diff --git a/scripts/gdb/linux/vmalloc.py b/scripts/gdb/linux/vmalloc.py new file mode 100644 index 000000000000..48e4a4fae7bb --- /dev/null +++ b/scripts/gdb/linux/vmalloc.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +import re +from linux import lists, utils, stackdepot, constants, mm + +vmap_area_type = utils.CachedType('struct vmap_area') +vmap_area_ptr_type = vmap_area_type.get_type().pointer() + +def is_vmalloc_addr(x): + pg_ops = mm.page_ops().ops + addr = pg_ops.kasan_reset_tag(x) + return addr >= pg_ops.VMALLOC_START and addr < pg_ops.VMALLOC_END + +class LxVmallocInfo(gdb.Command): + """Show vmallocinfo""" + + def __init__(self): + super(LxVmallocInfo, self).__init__("lx-vmallocinfo", gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + vmap_area_list = gdb.parse_and_eval('vmap_area_list') + for vmap_area in lists.list_for_each_entry(vmap_area_list, vmap_area_ptr_type, "list"): + if not vmap_area['vm']: + gdb.write("0x%x-0x%x %10d vm_map_ram\n" % (vmap_area['va_start'], vmap_area['va_end'], + vmap_area['va_end'] - vmap_area['va_start'])) + continue + v = vmap_area['vm'] + gdb.write("0x%x-0x%x %10d" % (v['addr'], v['addr'] + v['size'], v['size'])) + if v['caller']: + gdb.write(" %s" % str(v['caller']).split(' ')[-1]) + if v['nr_pages']: + gdb.write(" pages=%d" % v['nr_pages']) + if v['phys_addr']: + gdb.write(" phys=0x%x" % v['phys_addr']) + if v['flags'] & constants.LX_VM_IOREMAP: + gdb.write(" ioremap") + if v['flags'] & constants.LX_VM_ALLOC: + gdb.write(" vmalloc") + if v['flags'] & constants.LX_VM_MAP: + gdb.write(" vmap") + if v['flags'] & constants.LX_VM_USERMAP: + gdb.write(" user") + if v['flags'] & constants.LX_VM_DMA_COHERENT: + gdb.write(" dma-coherent") + if is_vmalloc_addr(v['pages']): + gdb.write(" vpages") + gdb.write("\n") + +LxVmallocInfo() diff --git a/scripts/gdb/vmlinux-gdb.py b/scripts/gdb/vmlinux-gdb.py index 2d32308c3f7a..fc53cdf286f1 100644 --- a/scripts/gdb/vmlinux-gdb.py +++ b/scripts/gdb/vmlinux-gdb.py @@ -41,6 +41,11 @@ else: import linux.genpd import linux.device import linux.vfs - import linux.mm + import linux.pgtable import linux.radixtree import linux.interrupts + import linux.mm + import linux.stackdepot + import linux.page_owner + import linux.slab + import linux.vmalloc diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index 946e250c1b2a..fc52bc41d3e7 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -6,10 +6,19 @@ import argparse import json import logging +import os import pathlib import sys -def generate_crates(srctree, objtree, sysroot_src): +def args_crates_cfgs(cfgs): + crates_cfgs = {} + for cfg in cfgs: + crate, vals = cfg.split("=", 1) + crates_cfgs[crate] = vals.replace("--cfg", "").split() + + return crates_cfgs + +def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs): # Generate the configuration list. cfg = [] with open(objtree / "include" / "generated" / "rustc_cfg") as fd: @@ -23,6 +32,7 @@ def generate_crates(srctree, objtree, sysroot_src): # Avoid O(n^2) iterations by keeping a map of indexes. crates = [] crates_indexes = {} + crates_cfgs = args_crates_cfgs(cfgs) def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False): crates_indexes[display_name] = len(crates) @@ -44,6 +54,7 @@ def generate_crates(srctree, objtree, sysroot_src): "core", sysroot_src / "core" / "src" / "lib.rs", [], + cfg=crates_cfgs.get("core", []), is_workspace_member=False, ) @@ -57,6 +68,7 @@ def generate_crates(srctree, objtree, sysroot_src): "alloc", srctree / "rust" / "alloc" / "lib.rs", ["core", "compiler_builtins"], + cfg=crates_cfgs.get("alloc", []), ) append_crate( @@ -65,7 +77,7 @@ def generate_crates(srctree, objtree, sysroot_src): [], is_proc_macro=True, ) - crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so" + crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so" append_crate( "build_error", @@ -95,19 +107,26 @@ def generate_crates(srctree, objtree, sysroot_src): "exclude_dirs": [], } + def is_root_crate(build_file, target): + try: + return f"{target}.o" in open(build_file).read() + except FileNotFoundError: + return False + # Then, the rest outside of `rust/`. # # We explicitly mention the top-level folders we want to cover. - for folder in ("samples", "drivers"): - for path in (srctree / folder).rglob("*.rs"): + extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers")) + if external_src is not None: + extra_dirs = [external_src] + for folder in extra_dirs: + for path in folder.rglob("*.rs"): logging.info("Checking %s", path) name = path.name.replace(".rs", "") # Skip those that are not crate roots. - try: - if f"{name}.o" not in open(path.parent / "Makefile").read(): - continue - except FileNotFoundError: + if not is_root_crate(path.parent / "Makefile", name) and \ + not is_root_crate(path.parent / "Kbuild", name): continue logging.info("Adding %s", name) @@ -123,9 +142,11 @@ def generate_crates(srctree, objtree, sysroot_src): def main(): parser = argparse.ArgumentParser() parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument('--cfgs', action='append', default=[]) parser.add_argument("srctree", type=pathlib.Path) parser.add_argument("objtree", type=pathlib.Path) parser.add_argument("sysroot_src", type=pathlib.Path) + parser.add_argument("exttree", type=pathlib.Path, nargs="?") args = parser.parse_args() logging.basicConfig( @@ -134,7 +155,7 @@ def main(): ) rust_project = { - "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src), + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs), "sysroot_src": str(args.sysroot_src), } diff --git a/scripts/headers_install.sh b/scripts/headers_install.sh index 36b56b746fce..afdddc82f02b 100755 --- a/scripts/headers_install.sh +++ b/scripts/headers_install.sh @@ -76,7 +76,6 @@ arch/arc/include/uapi/asm/swab.h:CONFIG_ARC_HAS_SWAPE arch/arm/include/uapi/asm/ptrace.h:CONFIG_CPU_ENDIAN_BE8 arch/hexagon/include/uapi/asm/ptrace.h:CONFIG_HEXAGON_ARCH_VERSION arch/hexagon/include/uapi/asm/user.h:CONFIG_HEXAGON_ARCH_VERSION -arch/ia64/include/uapi/asm/cmpxchg.h:CONFIG_IA64_DEBUG_CMPXCHG arch/m68k/include/uapi/asm/ptrace.h:CONFIG_COLDFIRE arch/nios2/include/uapi/asm/swab.h:CONFIG_NIOS2_CI_SWAB_NO arch/nios2/include/uapi/asm/swab.h:CONFIG_NIOS2_CI_SWAB_SUPPORT diff --git a/scripts/is_rust_module.sh b/scripts/is_rust_module.sh deleted file mode 100755 index 464761a7cf7f..000000000000 --- a/scripts/is_rust_module.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# -# is_rust_module.sh module.ko -# -# Returns `0` if `module.ko` is a Rust module, `1` otherwise. - -set -e - -# Using the `16_` prefix ensures other symbols with the same substring -# are not picked up (even if it would be unlikely). The last part is -# used just in case LLVM decides to use the `.` suffix. -# -# In the future, checking for the `.comment` section may be another -# option, see https://github.com/rust-lang/rust/pull/97550. -${NM} "$*" | grep -qE '^[0-9a-fA-F]+ [Rr] _R[^[:space:]]+16___IS_RUST_MODULE[^[:space:]]*$' diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index d387c9381650..653b92f6d4c8 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -129,6 +129,7 @@ static struct sym_entry *read_symbol(FILE *in, char **buf, size_t *buf_len) ssize_t readlen; struct sym_entry *sym; + errno = 0; readlen = getline(buf, buf_len, in); if (readlen < 0) { if (errno) { @@ -349,10 +350,10 @@ static void cleanup_symbol_name(char *s) * ASCII[_] = 5f * ASCII[a-z] = 61,7a * - * As above, replacing '.' with '\0' does not affect the main sorting, - * but it helps us with subsorting. + * As above, replacing the first '.' in ".llvm." with '\0' does not + * affect the main sorting, but it helps us with subsorting. */ - p = strchr(s, '.'); + p = strstr(s, ".llvm."); if (p) *p = '\0'; } diff --git a/scripts/kconfig/gconf.c b/scripts/kconfig/gconf.c index 17adabfd6e6b..9709aca3a30f 100644 --- a/scripts/kconfig/gconf.c +++ b/scripts/kconfig/gconf.c @@ -636,7 +636,7 @@ void on_introduction1_activate(GtkMenuItem * menuitem, gpointer user_data) { GtkWidget *dialog; const gchar *intro_text = - "Welcome to gkc, the GTK+ graphical configuration tool\n" + "Welcome to gconfig, the GTK+ graphical configuration tool.\n" "For each option, a blank box indicates the feature is disabled, a\n" "check indicates it is enabled, and a dot indicates that it is to\n" "be compiled as a module. Clicking on the box will cycle through the three states.\n" @@ -647,10 +647,7 @@ void on_introduction1_activate(GtkMenuItem * menuitem, gpointer user_data) "Although there is no cross reference yet to help you figure out\n" "what other options must be enabled to support the option you\n" "are interested in, you can still view the help of a grayed-out\n" - "option.\n" - "\n" - "Toggling Show Debug Info under the Options menu will show \n" - "the dependencies, which you can then match by examining other options."; + "option."; dialog = gtk_message_dialog_new(GTK_WINDOW(main_wnd), GTK_DIALOG_DESTROY_WITH_PARENT, @@ -667,7 +664,7 @@ void on_about1_activate(GtkMenuItem * menuitem, gpointer user_data) { GtkWidget *dialog; const gchar *about_text = - "gkc is copyright (c) 2002 Romain Lievin <roms@lpg.ticalc.org>.\n" + "gconfig is copyright (c) 2002 Romain Lievin <roms@lpg.ticalc.org>.\n" "Based on the source code from Roman Zippel.\n"; dialog = gtk_message_dialog_new(GTK_WINDOW(main_wnd), @@ -685,7 +682,7 @@ void on_license1_activate(GtkMenuItem * menuitem, gpointer user_data) { GtkWidget *dialog; const gchar *license_text = - "gkc is released under the terms of the GNU GPL v2.\n" + "gconfig is released under the terms of the GNU GPL v2.\n" "For more information, please see the source code or\n" "visit http://www.fsf.org/licenses/licenses.html\n"; diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 2ade63149466..d65ab8bfeaf4 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -31,10 +31,10 @@ llvm) fi ;; rustc) - echo 1.68.2 + echo 1.71.1 ;; bindgen) - echo 0.56.0 + echo 0.65.1 ;; *) echo "$1: unknown tool" >&2 diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index aebbf1913970..117018946b57 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -2,8 +2,6 @@ # SPDX-License-Identifier: GPL-2.0 # # Tests whether a suitable Rust toolchain is available. -# -# Pass `-v` for human output and more checks (as warnings). set -e @@ -21,102 +19,208 @@ get_canonical_version() echo $((100000 * $1 + 100 * $2 + $3)) } +# Print a reference to the Quick Start guide in the documentation. +print_docs_reference() +{ + echo >&2 "***" + echo >&2 "*** Please see Documentation/rust/quick-start.rst for details" + echo >&2 "*** on how to set up the Rust support." + echo >&2 "***" +} + +# Print an explanation about the fact that the script is meant to be called from Kbuild. +print_kbuild_explanation() +{ + echo >&2 "***" + echo >&2 "*** This script is intended to be called from Kbuild." + echo >&2 "*** Please use the 'rustavailable' target to call it instead." + echo >&2 "*** Otherwise, the results may not be meaningful." + exit 1 +} + +# If the script fails for any reason, or if there was any warning, then +# print a reference to the documentation on exit. +warning=0 +trap 'if [ $? -ne 0 ] || [ $warning -ne 0 ]; then print_docs_reference; fi' EXIT + +# Check that the expected environment variables are set. +if [ -z "${RUSTC+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'RUSTC' is not set." + print_kbuild_explanation +fi + +if [ -z "${BINDGEN+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'BINDGEN' is not set." + print_kbuild_explanation +fi + +if [ -z "${CC+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'CC' is not set." + print_kbuild_explanation +fi + # Check that the Rust compiler exists. if ! command -v "$RUSTC" >/dev/null; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust compiler '$RUSTC' could not be found." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' could not be found." + echo >&2 "***" exit 1 fi # Check that the Rust bindings generator exists. if ! command -v "$BINDGEN" >/dev/null; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found." + echo >&2 "***" exit 1 fi # Check that the Rust compiler version is suitable. # # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_compiler_output=$( \ + LC_ALL=C "$RUSTC" --version 2>/dev/null +) || rust_compiler_code=$? +if [ -n "$rust_compiler_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$RUSTC' to check the Rust compiler version failed with" + echo >&2 "*** code $rust_compiler_code. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_compiler_output" + echo >&2 "***" + exit 1 +fi rust_compiler_version=$( \ - LC_ALL=C "$RUSTC" --version 2>/dev/null \ - | head -n 1 \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + echo "$rust_compiler_output" \ + | sed -nE '1s:.*rustc ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$rust_compiler_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$RUSTC' to check the Rust compiler version did not return" + echo >&2 "*** an expected output. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_compiler_output" + echo >&2 "***" + exit 1 +fi rust_compiler_min_version=$($min_tool_version rustc) rust_compiler_cversion=$(get_canonical_version $rust_compiler_version) rust_compiler_min_cversion=$(get_canonical_version $rust_compiler_min_version) if [ "$rust_compiler_cversion" -lt "$rust_compiler_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust compiler '$RUSTC' is too old." - echo >&2 "*** Your version: $rust_compiler_version" - echo >&2 "*** Minimum version: $rust_compiler_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' is too old." + echo >&2 "*** Your version: $rust_compiler_version" + echo >&2 "*** Minimum version: $rust_compiler_min_version" + echo >&2 "***" exit 1 fi -if [ "$1" = -v ] && [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then +if [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then echo >&2 "***" echo >&2 "*** Rust compiler '$RUSTC' is too new. This may or may not work." echo >&2 "*** Your version: $rust_compiler_version" echo >&2 "*** Expected version: $rust_compiler_min_version" echo >&2 "***" + warning=1 fi # Check that the Rust bindings generator is suitable. # # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_bindings_generator_output=$( \ + LC_ALL=C "$BINDGEN" --version 2>/dev/null +) || rust_bindings_generator_code=$? +if [ -n "$rust_bindings_generator_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the Rust bindings generator version failed with" + echo >&2 "*** code $rust_bindings_generator_code. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_bindings_generator_output" + echo >&2 "***" + exit 1 +fi rust_bindings_generator_version=$( \ - LC_ALL=C "$BINDGEN" --version 2>/dev/null \ - | head -n 1 \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + echo "$rust_bindings_generator_output" \ + | sed -nE '1s:.*bindgen ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$rust_bindings_generator_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the bindings generator version did not return" + echo >&2 "*** an expected output. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_bindings_generator_output" + echo >&2 "***" + exit 1 +fi rust_bindings_generator_min_version=$($min_tool_version bindgen) rust_bindings_generator_cversion=$(get_canonical_version $rust_bindings_generator_version) rust_bindings_generator_min_cversion=$(get_canonical_version $rust_bindings_generator_min_version) if [ "$rust_bindings_generator_cversion" -lt "$rust_bindings_generator_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' is too old." - echo >&2 "*** Your version: $rust_bindings_generator_version" - echo >&2 "*** Minimum version: $rust_bindings_generator_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' is too old." + echo >&2 "*** Your version: $rust_bindings_generator_version" + echo >&2 "*** Minimum version: $rust_bindings_generator_min_version" + echo >&2 "***" exit 1 fi -if [ "$1" = -v ] && [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then +if [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then echo >&2 "***" echo >&2 "*** Rust bindings generator '$BINDGEN' is too new. This may or may not work." echo >&2 "*** Your version: $rust_bindings_generator_version" echo >&2 "*** Expected version: $rust_bindings_generator_min_version" echo >&2 "***" + warning=1 fi # Check that the `libclang` used by the Rust bindings generator is suitable. +# +# In order to do that, first invoke `bindgen` to get the `libclang` version +# found by `bindgen`. This step may already fail if, for instance, `libclang` +# is not found, thus inform the user in such a case. +bindgen_libclang_output=$( \ + LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null +) || bindgen_libclang_code=$? +if [ -n "$bindgen_libclang_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the libclang version (used by the Rust" + echo >&2 "*** bindings generator) failed with code $bindgen_libclang_code. This may be caused by" + echo >&2 "*** a failure to locate libclang. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$bindgen_libclang_output" + echo >&2 "***" + exit 1 +fi + +# `bindgen` returned successfully, thus use the output to check that the version +# of the `libclang` found by the Rust bindings generator is suitable. +# +# Unlike other version checks, note that this one does not necessarily appear +# in the first line of the output, thus no `sed` address is provided. bindgen_libclang_version=$( \ - LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null \ - | grep -F 'clang version ' \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ - | head -n 1 \ + echo "$bindgen_libclang_output" \ + | sed -nE 's:.*clang version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$bindgen_libclang_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the libclang version (used by the Rust" + echo >&2 "*** bindings generator) did not return an expected output. See output" + echo >&2 "*** and docs below for details:" + echo >&2 "***" + echo >&2 "$bindgen_libclang_output" + echo >&2 "***" + exit 1 +fi bindgen_libclang_min_version=$($min_tool_version llvm) bindgen_libclang_cversion=$(get_canonical_version $bindgen_libclang_version) bindgen_libclang_min_cversion=$(get_canonical_version $bindgen_libclang_min_version) if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old." - echo >&2 "*** Your version: $bindgen_libclang_version" - echo >&2 "*** Minimum version: $bindgen_libclang_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old." + echo >&2 "*** Your version: $bindgen_libclang_version" + echo >&2 "*** Minimum version: $bindgen_libclang_min_version" + echo >&2 "***" exit 1 fi @@ -125,21 +229,20 @@ fi # # In the future, we might be able to perform a full version check, see # https://github.com/rust-lang/rust-bindgen/issues/2138. -if [ "$1" = -v ]; then - cc_name=$($(dirname $0)/cc-version.sh "$CC" | cut -f1 -d' ') - if [ "$cc_name" = Clang ]; then - clang_version=$( \ - LC_ALL=C "$CC" --version 2>/dev/null \ - | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' - ) - if [ "$clang_version" != "$bindgen_libclang_version" ]; then - echo >&2 "***" - echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')" - echo >&2 "*** version does not match Clang's. This may be a problem." - echo >&2 "*** libclang version: $bindgen_libclang_version" - echo >&2 "*** Clang version: $clang_version" - echo >&2 "***" - fi +cc_name=$($(dirname $0)/cc-version.sh $CC | cut -f1 -d' ') +if [ "$cc_name" = Clang ]; then + clang_version=$( \ + LC_ALL=C $CC --version 2>/dev/null \ + | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' + ) + if [ "$clang_version" != "$bindgen_libclang_version" ]; then + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')" + echo >&2 "*** version does not match Clang's. This may be a problem." + echo >&2 "*** libclang version: $bindgen_libclang_version" + echo >&2 "*** Clang version: $clang_version" + echo >&2 "***" + warning=1 fi fi @@ -150,11 +253,9 @@ rustc_sysroot=$("$RUSTC" $KRUSTFLAGS --print sysroot) rustc_src=${RUST_LIB_SRC:-"$rustc_sysroot/lib/rustlib/src/rust/library"} rustc_src_core="$rustc_src/core/src/lib.rs" if [ ! -e "$rustc_src_core" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Source code for the 'core' standard library could not be found" - echo >&2 "*** at '$rustc_src_core'." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Source code for the 'core' standard library could not be found" + echo >&2 "*** at '$rustc_src_core'." + echo >&2 "***" exit 1 fi diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py new file mode 100755 index 000000000000..57613fe5ed75 --- /dev/null +++ b/scripts/rust_is_available_test.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +"""Tests the `rust_is_available.sh` script. + +Some of the tests require the real programs to be available in `$PATH` +under their canonical name (and with the expected versions). +""" + +import enum +import os +import pathlib +import stat +import subprocess +import tempfile +import unittest + +class TestRustIsAvailable(unittest.TestCase): + @enum.unique + class Expected(enum.Enum): + SUCCESS = enum.auto() + SUCCESS_WITH_WARNINGS = enum.auto() + SUCCESS_WITH_EXTRA_OUTPUT = enum.auto() + FAILURE = enum.auto() + + @classmethod + def generate_executable(cls, content): + path = pathlib.Path(cls.tempdir.name) + name = str(len(tuple(path.iterdir()))) + path = path / name + with open(path, "w") as file_: + file_.write(content) + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + return path + + @classmethod + def generate_clang(cls, stdout): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "-E" in " ".join(sys.argv): + print({repr("Clang " + " ".join(cls.llvm_default_version.split(" ")))}) +else: + print({repr(stdout)}) +""") + + @classmethod + def generate_rustc(cls, stdout): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "--print sysroot" in " ".join(sys.argv): + print({repr(cls.rust_default_sysroot)}) +else: + print({repr(stdout)}) +""") + + @classmethod + def generate_bindgen(cls, version_stdout, libclang_stderr): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): + print({repr(libclang_stderr)}, file=sys.stderr) +else: + print({repr(version_stdout)}) +""") + + @classmethod + def generate_bindgen_version(cls, stdout): + return cls.generate_bindgen(stdout, cls.bindgen_default_bindgen_libclang_stderr) + + @classmethod + def generate_bindgen_libclang(cls, stderr): + return cls.generate_bindgen(cls.bindgen_default_bindgen_version_stdout, stderr) + + @classmethod + def setUpClass(cls): + cls.tempdir = tempfile.TemporaryDirectory() + + cls.missing = pathlib.Path(cls.tempdir.name) / "missing" + + cls.nonexecutable = pathlib.Path(cls.tempdir.name) / "nonexecutable" + with open(cls.nonexecutable, "w") as file_: + file_.write("nonexecutable") + + cls.unexpected_binary = "true" + + cls.rustc_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "rustc")).decode().strip() + cls.bindgen_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "bindgen")).decode().strip() + cls.llvm_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "llvm")).decode().strip() + cls.rust_default_sysroot = subprocess.check_output(("rustc", "--print", "sysroot")).decode().strip() + + cls.bindgen_default_bindgen_version_stdout = f"bindgen {cls.bindgen_default_version}" + cls.bindgen_default_bindgen_libclang_stderr = f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {cls.llvm_default_version} [-W#pragma-messages], err: false" + + cls.default_rustc = cls.generate_rustc(f"rustc {cls.rustc_default_version}") + cls.default_bindgen = cls.generate_bindgen(cls.bindgen_default_bindgen_version_stdout, cls.bindgen_default_bindgen_libclang_stderr) + cls.default_cc = cls.generate_clang(f"clang version {cls.llvm_default_version}") + + def run_script(self, expected, override_env): + env = { + "RUSTC": self.default_rustc, + "BINDGEN": self.default_bindgen, + "CC": self.default_cc, + } + + for key, value in override_env.items(): + if value is None: + del env[key] + continue + env[key] = value + + result = subprocess.run("scripts/rust_is_available.sh", env=env, capture_output=True) + + # The script should never output anything to `stdout`. + self.assertEqual(result.stdout, b"") + + if expected == self.Expected.SUCCESS: + # When expecting a success, the script should return 0 + # and it should not output anything to `stderr`. + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_EXTRA_OUTPUT: + # When expecting a success with extra output (that is not warnings, + # which is the common case), the script should return 0 and it + # should output at least something to `stderr` (the output should + # be checked further by the test). + self.assertEqual(result.returncode, 0) + self.assertNotEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_WARNINGS: + # When expecting a success with warnings, the script should return 0 + # and it should output at least the instructions to `stderr`. + self.assertEqual(result.returncode, 0) + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) + else: + # When expecting a failure, the script should return non-0 + # and it should output at least the instructions to `stderr`. + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) + + # The output will generally be UTF-8 (i.e. unless the user has + # put strange values in the environment). + result.stderr = result.stderr.decode() + + return result + + def test_rustc_unset(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": None }) + self.assertIn("Environment variable 'RUSTC' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_bindgen_unset(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": None }) + self.assertIn("Environment variable 'BINDGEN' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_cc_unset(self): + result = self.run_script(self.Expected.FAILURE, { "CC": None }) + self.assertIn("Environment variable 'CC' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_rustc_missing(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.missing }) + self.assertIn(f"Rust compiler '{self.missing}' could not be found.", result.stderr) + + def test_bindgen_missing(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.missing }) + self.assertIn(f"Rust bindings generator '{self.missing}' could not be found.", result.stderr) + + def test_rustc_nonexecutable(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.nonexecutable }) + self.assertIn(f"Running '{self.nonexecutable}' to check the Rust compiler version failed with", result.stderr) + + def test_rustc_unexpected_binary(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.unexpected_binary }) + self.assertIn(f"Running '{self.unexpected_binary}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_unexpected_name(self): + rustc = self.generate_rustc(f"unexpected {self.rustc_default_version} (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_unexpected_version(self): + rustc = self.generate_rustc("rustc unexpected (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_no_minor(self): + rustc = self.generate_rustc(f"rustc {'.'.join(self.rustc_default_version.split('.')[:2])} (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_old_version(self): + rustc = self.generate_rustc("rustc 1.60.0 (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Rust compiler '{rustc}' is too old.", result.stderr) + + def test_rustc_new_version(self): + rustc = self.generate_rustc("rustc 1.999.0 (a8314ef7d 2099-06-27)") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "RUSTC": rustc }) + self.assertIn(f"Rust compiler '{rustc}' is too new. This may or may not work.", result.stderr) + + def test_bindgen_nonexecutable(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.nonexecutable }) + self.assertIn(f"Running '{self.nonexecutable}' to check the Rust bindings generator version failed with", result.stderr) + + def test_bindgen_unexpected_binary(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.unexpected_binary }) + self.assertIn(f"Running '{self.unexpected_binary}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_unexpected_name(self): + bindgen = self.generate_bindgen_version(f"unexpected {self.bindgen_default_version}") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_unexpected_version(self): + bindgen = self.generate_bindgen_version("bindgen unexpected") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_no_minor(self): + bindgen = self.generate_bindgen_version(f"bindgen {'.'.join(self.bindgen_default_version.split('.')[:2])}") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_old_version(self): + bindgen = self.generate_bindgen_version("bindgen 0.50.0") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Rust bindings generator '{bindgen}' is too old.", result.stderr) + + def test_bindgen_new_version(self): + bindgen = self.generate_bindgen_version("bindgen 0.999.0") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn(f"Rust bindings generator '{bindgen}' is too new. This may or may not work.", result.stderr) + + def test_bindgen_libclang_failure(self): + for env in ( + { "LLVM_CONFIG_PATH": self.missing }, + { "LIBCLANG_PATH": self.missing }, + { "CLANG_PATH": self.missing }, + ): + with self.subTest(env=env): + result = self.run_script(self.Expected.FAILURE, env | { "PATH": os.environ["PATH"], "BINDGEN": "bindgen" }) + self.assertIn("Running 'bindgen' to check the libclang version (used by the Rust", result.stderr) + self.assertIn("bindings generator) failed with code ", result.stderr) + + def test_bindgen_libclang_unexpected_version(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version unexpected [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the libclang version (used by the Rust", result.stderr) + self.assertIn("bindings generator) did not return an expected output. See output", result.stderr) + + def test_bindgen_libclang_old_version(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 10.0.0 [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"libclang (used by the Rust bindings generator '{bindgen}') is too old.", result.stderr) + + def test_clang_matches_bindgen_libclang_different_bindgen(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 999.0.0 [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn("version does not match Clang's. This may be a problem.", result.stderr) + + def test_clang_matches_bindgen_libclang_different_clang(self): + cc = self.generate_clang("clang version 999.0.0") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "CC": cc }) + self.assertIn("version does not match Clang's. This may be a problem.", result.stderr) + + def test_rustc_src_core_krustflags(self): + result = self.run_script(self.Expected.FAILURE, { "PATH": os.environ["PATH"], "RUSTC": "rustc", "KRUSTFLAGS": f"--sysroot={self.missing}" }) + self.assertIn("Source code for the 'core' standard library could not be found", result.stderr) + + def test_rustc_src_core_rustlibsrc(self): + result = self.run_script(self.Expected.FAILURE, { "RUST_LIB_SRC": self.missing }) + self.assertIn("Source code for the 'core' standard library could not be found", result.stderr) + + def test_success_cc_unknown(self): + result = self.run_script(self.Expected.SUCCESS_WITH_EXTRA_OUTPUT, { "CC": self.missing }) + self.assertIn("unknown C compiler", result.stderr) + + def test_success_cc_multiple_arguments_ccache(self): + clang = self.generate_clang(f"""Ubuntu clang version {self.llvm_default_version}-1ubuntu1 +Target: x86_64-pc-linux-gnu +Thread model: posix +InstalledDir: /usr/bin +""") + result = self.run_script(self.Expected.SUCCESS, { "CC": f"{clang} clang" }) + + def test_success_rustc_version(self): + for rustc_stdout in ( + f"rustc {self.rustc_default_version} (a8314ef7d 2022-06-27)", + f"rustc {self.rustc_default_version}-dev (a8314ef7d 2022-06-27)", + f"rustc {self.rustc_default_version}-1.60.0 (a8314ef7d 2022-06-27)", + ): + with self.subTest(rustc_stdout=rustc_stdout): + rustc = self.generate_rustc(rustc_stdout) + result = self.run_script(self.Expected.SUCCESS, { "RUSTC": rustc }) + + def test_success_bindgen_version(self): + for bindgen_stdout in ( + f"bindgen {self.bindgen_default_version}", + f"bindgen {self.bindgen_default_version}-dev", + f"bindgen {self.bindgen_default_version}-0.999.0", + ): + with self.subTest(bindgen_stdout=bindgen_stdout): + bindgen = self.generate_bindgen_version(bindgen_stdout) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_success_bindgen_libclang(self): + for stderr in ( + f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1) [-W#pragma-messages], err: false", + f"/home/jd/Documents/dev/kernel-module-flake/linux-6.1/outputs/dev/lib/modules/6.1.0-development/source/scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} [-W#pragma-messages], err: false", + f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (Fedora 13.0.0-3.fc35) [-W#pragma-messages], err: false", + f""" +/nix/store/dsd5gz46hdbdk2rfdimqddhq6m8m8fqs-bash-5.1-p16/bin/bash: warning: setlocale: LC_ALL: cannot change locale (c) +scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} [-W#pragma-messages], err: false +""", + f""" +/nix/store/dsd5gz46hdbdk2rfdimqddhq6m8m8fqs-bash-5.1.0-p16/bin/bash: warning: setlocale: LC_ALL: cannot change locale (c) +/home/jd/Documents/dev/kernel-module-flake/linux-6.1/outputs/dev/lib/modules/6.1.0-development/source/scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (Fedora 13.0.0-3.fc35) [-W#pragma-messages], err: false +""" + ): + with self.subTest(stderr=stderr): + bindgen = self.generate_bindgen_libclang(stderr) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_success_clang_version(self): + for clang_stdout in ( + f"clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1)", + f"clang version {self.llvm_default_version}-dev", + f"clang version {self.llvm_default_version}-2~ubuntu20.04.1", + f"Ubuntu clang version {self.llvm_default_version}-2~ubuntu20.04.1", + ): + with self.subTest(clang_stdout=clang_stdout): + clang = self.generate_clang(clang_stdout) + result = self.run_script(self.Expected.SUCCESS, { "CC": clang }) + + def test_success_real_programs(self): + for cc in ["gcc", "clang"]: + with self.subTest(cc=cc): + result = self.run_script(self.Expected.SUCCESS, { + "PATH": os.environ["PATH"], + "RUSTC": "rustc", + "BINDGEN": "bindgen", + "CC": cc, + }) + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/rustdoc_test_builder.rs b/scripts/rustdoc_test_builder.rs new file mode 100644 index 000000000000..e5894652f12c --- /dev/null +++ b/scripts/rustdoc_test_builder.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Test builder for `rustdoc`-generated tests. +//! +//! This script is a hack to extract the test from `rustdoc`'s output. Ideally, `rustdoc` would +//! have an option to generate this information instead, e.g. as JSON output. +//! +//! The `rustdoc`-generated test names look like `{file}_{line}_{number}`, e.g. +//! `...path_rust_kernel_sync_arc_rs_42_0`. `number` is the "test number", needed in cases like +//! a macro that expands into items with doctests is invoked several times within the same line. +//! +//! However, since these names are used for bisection in CI, the line number makes it not stable +//! at all. In the future, we would like `rustdoc` to give us the Rust item path associated with +//! the test, plus a "test number" (for cases with several examples per item) and generate a name +//! from that. For the moment, we generate ourselves a new name, `{file}_{number}` instead, in +//! the `gen` script (done there since we need to be aware of all the tests in a given file). + +use std::io::Read; + +fn main() { + let mut stdin = std::io::stdin().lock(); + let mut body = String::new(); + stdin.read_to_string(&mut body).unwrap(); + + // Find the generated function name looking for the inner function inside `main()`. + // + // The line we are looking for looks like one of the following: + // + // ``` + // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_28_0() { + // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl core::fmt::Debug> { + // ``` + // + // It should be unlikely that doctest code matches such lines (when code is formatted properly). + let rustdoc_function_name = body + .lines() + .find_map(|line| { + Some( + line.split_once("fn main() {")? + .1 + .split_once("fn ")? + .1 + .split_once("()")? + .0, + ) + .filter(|x| x.chars().all(|c| c.is_alphanumeric() || c == '_')) + }) + .expect("No test function found in `rustdoc`'s output."); + + // Qualify `Result` to avoid the collision with our own `Result` coming from the prelude. + let body = body.replace( + &format!("{rustdoc_function_name}() -> Result<(), impl core::fmt::Debug> {{"), + &format!("{rustdoc_function_name}() -> core::result::Result<(), impl core::fmt::Debug> {{"), + ); + + // For tests that get generated with `Result`, like above, `rustdoc` generates an `unwrap()` on + // the return value to check there were no returned errors. Instead, we use our assert macro + // since we want to just fail the test, not panic the kernel. + // + // We save the result in a variable so that the failed assertion message looks nicer. + let body = body.replace( + &format!("}} {rustdoc_function_name}().unwrap() }}"), + &format!("}} let test_return_value = {rustdoc_function_name}(); assert!(test_return_value.is_ok()); }}"), + ); + + // Figure out a smaller test name based on the generated function name. + let name = rustdoc_function_name.split_once("_rust_kernel_").unwrap().1; + + let path = format!("rust/test/doctests/kernel/{name}"); + + std::fs::write(path, body.as_bytes()).unwrap(); +} diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs new file mode 100644 index 000000000000..5ebd42ae4a3f --- /dev/null +++ b/scripts/rustdoc_test_gen.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generates KUnit tests from saved `rustdoc`-generated tests. +//! +//! KUnit passes a context (`struct kunit *`) to each test, which should be forwarded to the other +//! KUnit functions and macros. +//! +//! However, we want to keep this as an implementation detail because: +//! +//! - Test code should not care about the implementation. +//! +//! - Documentation looks worse if it needs to carry extra details unrelated to the piece +//! being described. +//! +//! - Test code should be able to define functions and call them, without having to carry +//! the context. +//! +//! - Later on, we may want to be able to test non-kernel code (e.g. `core`, `alloc` or +//! third-party crates) which likely use the standard library `assert*!` macros. +//! +//! For this reason, instead of the passed context, `kunit_get_current_test()` is used instead +//! (i.e. `current->kunit_test`). +//! +//! Note that this means other threads/tasks potentially spawned by a given test, if failing, will +//! report the failure in the kernel log but will not fail the actual test. Saving the pointer in +//! e.g. a `static` per test does not fully solve the issue either, because currently KUnit does +//! not support assertions (only expectations) from other tasks. Thus leave that feature for +//! the future, which simplifies the code here too. We could also simply not allow `assert`s in +//! other tasks, but that seems overly constraining, and we do want to support them, eventually. + +use std::{ + fs, + fs::File, + io::{BufWriter, Read, Write}, + path::{Path, PathBuf}, +}; + +/// Find the real path to the original file based on the `file` portion of the test name. +/// +/// `rustdoc` generated `file`s look like `sync_locked_by_rs`. Underscores (except the last one) +/// may represent an actual underscore in a directory/file, or a path separator. Thus the actual +/// file might be `sync_locked_by.rs`, `sync/locked_by.rs`, `sync_locked/by.rs` or +/// `sync/locked/by.rs`. This function walks the file system to determine which is the real one. +/// +/// This does require that ambiguities do not exist, but that seems fair, especially since this is +/// all supposed to be temporary until `rustdoc` gives us proper metadata to build this. If such +/// ambiguities are detected, they are diagnosed and the script panics. +fn find_real_path<'a>(srctree: &Path, valid_paths: &'a mut Vec<PathBuf>, file: &str) -> &'a str { + valid_paths.clear(); + + let potential_components: Vec<&str> = file.strip_suffix("_rs").unwrap().split('_').collect(); + + find_candidates(srctree, valid_paths, Path::new(""), &potential_components); + fn find_candidates( + srctree: &Path, + valid_paths: &mut Vec<PathBuf>, + prefix: &Path, + potential_components: &[&str], + ) { + // The base case: check whether all the potential components left, joined by underscores, + // is a file. + let joined_potential_components = potential_components.join("_") + ".rs"; + if srctree + .join("rust/kernel") + .join(prefix) + .join(&joined_potential_components) + .is_file() + { + // Avoid `srctree` here in order to keep paths relative to it in the KTAP output. + valid_paths.push( + Path::new("rust/kernel") + .join(prefix) + .join(joined_potential_components), + ); + } + + // In addition, check whether each component prefix, joined by underscores, is a directory. + // If not, there is no need to check for combinations with that prefix. + for i in 1..potential_components.len() { + let (components_prefix, components_rest) = potential_components.split_at(i); + let prefix = prefix.join(components_prefix.join("_")); + if srctree.join("rust/kernel").join(&prefix).is_dir() { + find_candidates(srctree, valid_paths, &prefix, components_rest); + } + } + } + + assert!( + valid_paths.len() > 0, + "No path candidates found. This is likely a bug in the build system, or some files went \ + away while compiling." + ); + + if valid_paths.len() > 1 { + eprintln!("Several path candidates found:"); + for path in valid_paths { + eprintln!(" {path:?}"); + } + panic!( + "Several path candidates found, please resolve the ambiguity by renaming a file or \ + folder." + ); + } + + valid_paths[0].to_str().unwrap() +} + +fn main() { + let srctree = std::env::var("srctree").unwrap(); + let srctree = Path::new(&srctree); + + let mut paths = fs::read_dir("rust/test/doctests/kernel") + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::<Vec<_>>(); + + // Sort paths. + paths.sort(); + + let mut rust_tests = String::new(); + let mut c_test_declarations = String::new(); + let mut c_test_cases = String::new(); + let mut body = String::new(); + let mut last_file = String::new(); + let mut number = 0; + let mut valid_paths: Vec<PathBuf> = Vec::new(); + let mut real_path: &str = ""; + for path in paths { + // The `name` follows the `{file}_{line}_{number}` pattern (see description in + // `scripts/rustdoc_test_builder.rs`). Discard the `number`. + let name = path.file_name().unwrap().to_str().unwrap().to_string(); + + // Extract the `file` and the `line`, discarding the `number`. + let (file, line) = name.rsplit_once('_').unwrap().0.rsplit_once('_').unwrap(); + + // Generate an ID sequence ("test number") for each one in the file. + if file == last_file { + number += 1; + } else { + number = 0; + last_file = file.to_string(); + + // Figure out the real path, only once per file. + real_path = find_real_path(srctree, &mut valid_paths, file); + } + + // Generate a KUnit name (i.e. test name and C symbol) for this test. + // + // We avoid the line number, like `rustdoc` does, to make things slightly more stable for + // bisection purposes. However, to aid developers in mapping back what test failed, we will + // print a diagnostics line in the KTAP report. + let kunit_name = format!("rust_doctest_kernel_{file}_{number}"); + + // Read the test's text contents to dump it below. + body.clear(); + File::open(path).unwrap().read_to_string(&mut body).unwrap(); + + // Calculate how many lines before `main` function (including the `main` function line). + let body_offset = body + .lines() + .take_while(|line| !line.contains("fn main() {")) + .count() + + 1; + + use std::fmt::Write; + write!( + rust_tests, + r#"/// Generated `{name}` KUnit test case from a Rust documentation test. +#[no_mangle] +pub extern "C" fn {kunit_name}(__kunit_test: *mut kernel::bindings::kunit) {{ + /// Overrides the usual [`assert!`] macro with one that calls KUnit instead. + #[allow(unused)] + macro_rules! assert {{ + ($cond:expr $(,)?) => {{{{ + kernel::kunit_assert!("{kunit_name}", "{real_path}", __DOCTEST_ANCHOR - {line}, $cond); + }}}} + }} + + /// Overrides the usual [`assert_eq!`] macro with one that calls KUnit instead. + #[allow(unused)] + macro_rules! assert_eq {{ + ($left:expr, $right:expr $(,)?) => {{{{ + kernel::kunit_assert_eq!("{kunit_name}", "{real_path}", __DOCTEST_ANCHOR - {line}, $left, $right); + }}}} + }} + + // Many tests need the prelude, so provide it by default. + #[allow(unused)] + use kernel::prelude::*; + + // Unconditionally print the location of the original doctest (i.e. rather than the location in + // the generated file) so that developers can easily map the test back to the source code. + // + // This information is also printed when assertions fail, but this helps in the successful cases + // when the user is running KUnit manually, or when passing `--raw_output` to `kunit.py`. + // + // This follows the syntax for declaring test metadata in the proposed KTAP v2 spec, which may + // be used for the proposed KUnit test attributes API. Thus hopefully this will make migration + // easier later on. + kernel::kunit::info(format_args!(" # {kunit_name}.location: {real_path}:{line}\n")); + + /// The anchor where the test code body starts. + #[allow(unused)] + static __DOCTEST_ANCHOR: i32 = core::line!() as i32 + {body_offset} + 1; + {{ + {body} + main(); + }} +}} + +"# + ) + .unwrap(); + + write!(c_test_declarations, "void {kunit_name}(struct kunit *);\n").unwrap(); + write!(c_test_cases, " KUNIT_CASE({kunit_name}),\n").unwrap(); + } + + let rust_tests = rust_tests.trim(); + let c_test_declarations = c_test_declarations.trim(); + let c_test_cases = c_test_cases.trim(); + + write!( + BufWriter::new(File::create("rust/doctests_kernel_generated.rs").unwrap()), + r#"//! `kernel` crate documentation tests. + +const __LOG_PREFIX: &[u8] = b"rust_doctests_kernel\0"; + +{rust_tests} +"# + ) + .unwrap(); + + write!( + BufWriter::new(File::create("rust/doctests_kernel_generated_kunit.c").unwrap()), + r#"/* + * `kernel` crate documentation tests. + */ + +#include <kunit/test.h> + +{c_test_declarations} + +static struct kunit_case test_cases[] = {{ + {c_test_cases} + {{ }} +}}; + +static struct kunit_suite test_suite = {{ + .name = "rust_doctests_kernel", + .test_cases = test_cases, +}}; + +kunit_test_suite(test_suite); + +MODULE_LICENSE("GPL"); +"# + ) + .unwrap(); +} diff --git a/scripts/spelling.txt b/scripts/spelling.txt index fc7ba95e86a0..855c4863124b 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -1541,7 +1541,6 @@ temeprature||temperature temorary||temporary temproarily||temporarily temperture||temperature -thead||thread theads||threads therfore||therefore thier||their |