diff options
Diffstat (limited to 'scripts')
30 files changed, 2325 insertions, 390 deletions
diff --git a/scripts/.gitignore b/scripts/.gitignore index 6e9ce6720a05..3dbb8bb2457b 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -5,6 +5,8 @@ /kallsyms /module.lds /recordmcount +/rustdoc_test_builder +/rustdoc_test_gen /sign-file /sorttable /target.json diff --git a/scripts/Makefile b/scripts/Makefile index 32b6ba722728..576cf64be667 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -9,6 +9,8 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable hostprogs-always-$(CONFIG_ASN1) += asn1_compiler hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert +hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += rustdoc_test_builder +hostprogs-always-$(CONFIG_RUST_KERNEL_DOCTESTS) += rustdoc_test_gen always-$(CONFIG_RUST) += target.json filechk_rust_target = $< < include/config/auto.conf @@ -18,6 +20,8 @@ $(obj)/target.json: scripts/generate_rust_target include/config/auto.conf FORCE hostprogs += generate_rust_target generate_rust_target-rust := y +rustdoc_test_builder-rust := y +rustdoc_test_gen-rust := y HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include HOSTLDLIBS_sorttable = -lpthread diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal index fc19f67039bd..b3a6aa8fbe8c 100644 --- a/scripts/Makefile.modfinal +++ b/scripts/Makefile.modfinal @@ -41,8 +41,6 @@ quiet_cmd_btf_ko = BTF [M] $@ cmd_btf_ko = \ if [ ! -f vmlinux ]; then \ printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \ - elif [ -n "$(CONFIG_RUST)" ] && $(srctree)/scripts/is_rust_module.sh $@; then \ - printf "Skipping BTF generation for %s because it's a Rust module\n" $@ 1>&2; \ else \ LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J $(PAHOLE_FLAGS) --btf_base vmlinux $@; \ $(RESOLVE_BTFIDS) -b vmlinux $@; \ diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter index 36303afa9dfc..888ce286a351 100755 --- a/scripts/bloat-o-meter +++ b/scripts/bloat-o-meter @@ -100,12 +100,12 @@ def print_result(symboltype, symbolformat): print("Total: Before=%d, After=%d, chg %+.2f%%" % (otot, ntot, percent)) if args.c: - print_result("Function", "tT") - print_result("Data", "dDbB") + print_result("Function", "tTwW") + print_result("Data", "dDbBvV") print_result("RO Data", "rR") elif args.d: - print_result("Data", "dDbBrR") + print_result("Data", "dDbBrRvV") elif args.t: - print_result("Function", "tT") + print_result("Function", "tTwW") else: - print_result("Function", "tTdDbBrR") + print_result("Function", "tTdDbBrRvVwW") diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 880fde13d9b8..7d16f863edf1 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -74,6 +74,8 @@ my $git_command ='export LANGUAGE=en_US.UTF-8; git'; my $tabsize = 8; my ${CONFIG_} = "CONFIG_"; +my %maybe_linker_symbol; # for externs in c exceptions, when seen in *vmlinux.lds.h + sub help { my ($exitcode) = @_; @@ -3270,7 +3272,7 @@ sub process { # A Fixes:, link or signature tag line $commit_log_possible_stack_dump)) { WARN("COMMIT_LOG_LONG_LINE", - "Possible unwrapped commit description (prefer a maximum 75 chars per line)\n" . $herecurr); + "Prefer a maximum 75 chars per line (possible unwrapped commit description?)\n" . $herecurr); $commit_log_long_line = 1; } @@ -6051,6 +6053,9 @@ sub process { # check for line continuations outside of #defines, preprocessor #, and asm + } elsif ($realfile =~ m@/vmlinux.lds.h$@) { + $line =~ s/(\w+)/$maybe_linker_symbol{$1}++/ge; + #print "REAL: $realfile\nln: $line\nkeys:", sort keys %maybe_linker_symbol; } else { if ($prevline !~ /^..*\\$/ && $line !~ /^\+\s*\#.*\\$/ && # preprocessor @@ -7120,6 +7125,21 @@ sub process { } } elsif ($realfile =~ /\.c$/ && defined $stat && + $stat =~ /^\+extern struct\s+(\w+)\s+(\w+)\[\];/) + { + my ($st_type, $st_name) = ($1, $2); + + for my $s (keys %maybe_linker_symbol) { + #print "Linker symbol? $st_name : $s\n"; + goto LIKELY_LINKER_SYMBOL + if $st_name =~ /$s/; + } + WARN("AVOID_EXTERNS", + "found a file-scoped extern type:$st_type name:$st_name in .c file\n" + . "is this a linker symbol ?\n" . $herecurr); + LIKELY_LINKER_SYMBOL: + + } elsif ($realfile =~ /\.c$/ && defined $stat && $stat =~ /^.\s*extern\s+/) { WARN("AVOID_EXTERNS", @@ -7457,6 +7477,30 @@ sub process { } } +# Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). + our $rcu_trace_funcs = qr{(?x: + rcu_read_lock_trace | + rcu_read_lock_trace_held | + rcu_read_unlock_trace | + call_rcu_tasks_trace | + synchronize_rcu_tasks_trace | + rcu_barrier_tasks_trace | + rcu_request_urgent_qs_task + )}; + our $rcu_trace_paths = qr{(?x: + kernel/bpf/ | + include/linux/bpf | + net/bpf/ | + kernel/rcu/ | + include/linux/rcu + )}; + if ($line =~ /\b($rcu_trace_funcs)\s*\(/) { + if ($realfile !~ m{^$rcu_trace_paths}) { + WARN("RCU_TASKS_TRACE", + "use of RCU tasks trace is incorrect outside BPF or core RCU code\n" . $herecurr); + } + } + # check for lockdep_set_novalidate_class if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || $line =~ /__lockdep_no_validate__\s*\)/ ) { diff --git a/scripts/coccinelle/api/debugfs/debugfs_simple_attr.cocci b/scripts/coccinelle/api/debugfs/debugfs_simple_attr.cocci deleted file mode 100644 index 7c312310547c..000000000000 --- a/scripts/coccinelle/api/debugfs/debugfs_simple_attr.cocci +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/// Use DEFINE_DEBUGFS_ATTRIBUTE rather than DEFINE_SIMPLE_ATTRIBUTE -/// for debugfs files. -/// -//# Rationale: DEFINE_SIMPLE_ATTRIBUTE + debugfs_create_file() -//# imposes some significant overhead as compared to -//# DEFINE_DEBUGFS_ATTRIBUTE + debugfs_create_file_unsafe(). -// -// Copyright (C): 2016 Nicolai Stange -// Options: --no-includes -// - -virtual context -virtual patch -virtual org -virtual report - -@dsa@ -declarer name DEFINE_SIMPLE_ATTRIBUTE; -identifier dsa_fops; -expression dsa_get, dsa_set, dsa_fmt; -position p; -@@ -DEFINE_SIMPLE_ATTRIBUTE@p(dsa_fops, dsa_get, dsa_set, dsa_fmt); - -@dcf@ -expression name, mode, parent, data; -identifier dsa.dsa_fops; -@@ -debugfs_create_file(name, mode, parent, data, &dsa_fops) - - -@context_dsa depends on context && dcf@ -declarer name DEFINE_DEBUGFS_ATTRIBUTE; -identifier dsa.dsa_fops; -expression dsa.dsa_get, dsa.dsa_set, dsa.dsa_fmt; -@@ -* DEFINE_SIMPLE_ATTRIBUTE(dsa_fops, dsa_get, dsa_set, dsa_fmt); - - -@patch_dcf depends on patch expression@ -expression name, mode, parent, data; -identifier dsa.dsa_fops; -@@ -- debugfs_create_file(name, mode, parent, data, &dsa_fops) -+ debugfs_create_file_unsafe(name, mode, parent, data, &dsa_fops) - -@patch_dsa depends on patch_dcf && patch@ -identifier dsa.dsa_fops; -expression dsa.dsa_get, dsa.dsa_set, dsa.dsa_fmt; -@@ -- DEFINE_SIMPLE_ATTRIBUTE(dsa_fops, dsa_get, dsa_set, dsa_fmt); -+ DEFINE_DEBUGFS_ATTRIBUTE(dsa_fops, dsa_get, dsa_set, dsa_fmt); - - -@script:python depends on org && dcf@ -fops << dsa.dsa_fops; -p << dsa.p; -@@ -msg="%s should be defined with DEFINE_DEBUGFS_ATTRIBUTE" % (fops) -coccilib.org.print_todo(p[0], msg) - -@script:python depends on report && dcf@ -fops << dsa.dsa_fops; -p << dsa.p; -@@ -msg="WARNING: %s should be defined with DEFINE_DEBUGFS_ATTRIBUTE" % (fops) -coccilib.report.print_report(p[0], msg) diff --git a/scripts/dtc/dt-extract-compatibles b/scripts/dtc/dt-extract-compatibles index a1119762ed08..9df9f1face83 100755 --- a/scripts/dtc/dt-extract-compatibles +++ b/scripts/dtc/dt-extract-compatibles @@ -25,8 +25,8 @@ def parse_of_declare_macros(data): def parse_of_device_id(data): """ Find all compatible strings in of_device_id structs """ compat_list = [] - for m in re.finditer(r'of_device_id\s+[a-zA-Z0-9_]+\[\]\s*=\s*({.*?);', data): - compat_list += re.findall(r'\.compatible\s+=\s+"([a-zA-Z0-9_\-,]+)"', m[1]) + for m in re.finditer(r'of_device_id(\s+\S+)?\s+\S+\[\](\s+\S+)?\s*=\s*({.*?);', data): + compat_list += re.findall(r'\.compatible\s+=\s+"(\S+)"', m[3]) return compat_list diff --git a/scripts/gcc-plugins/gcc-common.h b/scripts/gcc-plugins/gcc-common.h index 84c730da36dd..1ae39b9f4a95 100644 --- a/scripts/gcc-plugins/gcc-common.h +++ b/scripts/gcc-plugins/gcc-common.h @@ -440,4 +440,8 @@ static inline void debug_gimple_stmt(const_gimple s) #define SET_DECL_MODE(decl, mode) DECL_MODE(decl) = (mode) #endif +#if BUILDING_GCC_VERSION >= 14000 +#define last_stmt(x) last_nondebug_stmt(x) +#endif + #endif diff --git a/scripts/gdb/linux/constants.py.in b/scripts/gdb/linux/constants.py.in index 50a92c4e9984..e3517d4ab8ec 100644 --- a/scripts/gdb/linux/constants.py.in +++ b/scripts/gdb/linux/constants.py.in @@ -18,8 +18,11 @@ #include <linux/irq.h> #include <linux/mount.h> #include <linux/of_fdt.h> +#include <linux/page_ext.h> #include <linux/radix-tree.h> +#include <linux/slab.h> #include <linux/threads.h> +#include <linux/vmalloc.h> /* We need to stringify expanded macros so that they can be parsed */ @@ -64,6 +67,9 @@ LX_GDBPARSED(IRQ_HIDDEN) /* linux/module.h */ LX_GDBPARSED(MOD_TEXT) +LX_GDBPARSED(MOD_DATA) +LX_GDBPARSED(MOD_RODATA) +LX_GDBPARSED(MOD_RO_AFTER_INIT) /* linux/mount.h */ LX_VALUE(MNT_NOSUID) @@ -86,6 +92,28 @@ LX_GDBPARSED(RADIX_TREE_MAP_SIZE) LX_GDBPARSED(RADIX_TREE_MAP_SHIFT) LX_GDBPARSED(RADIX_TREE_MAP_MASK) +/* linux/vmalloc.h */ +LX_VALUE(VM_IOREMAP) +LX_VALUE(VM_ALLOC) +LX_VALUE(VM_MAP) +LX_VALUE(VM_USERMAP) +LX_VALUE(VM_DMA_COHERENT) + +/* linux/page_ext.h */ +if IS_BUILTIN(CONFIG_PAGE_OWNER): + LX_GDBPARSED(PAGE_EXT_OWNER) + LX_GDBPARSED(PAGE_EXT_OWNER_ALLOCATED) + +/* linux/slab.h */ +LX_GDBPARSED(SLAB_RED_ZONE) +LX_GDBPARSED(SLAB_POISON) +LX_GDBPARSED(SLAB_KMALLOC) +LX_GDBPARSED(SLAB_HWCACHE_ALIGN) +LX_GDBPARSED(SLAB_CACHE_DMA) +LX_GDBPARSED(SLAB_CACHE_DMA32) +LX_GDBPARSED(SLAB_STORE_USER) +LX_GDBPARSED(SLAB_PANIC) + /* Kernel Configs */ LX_CONFIG(CONFIG_GENERIC_CLOCKEVENTS) LX_CONFIG(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) @@ -102,3 +130,30 @@ LX_CONFIG(CONFIG_X86_MCE_AMD) LX_CONFIG(CONFIG_X86_MCE) LX_CONFIG(CONFIG_X86_IO_APIC) LX_CONFIG(CONFIG_HAVE_KVM) +LX_CONFIG(CONFIG_NUMA) +LX_CONFIG(CONFIG_ARM64) +LX_CONFIG(CONFIG_ARM64_4K_PAGES) +LX_CONFIG(CONFIG_ARM64_16K_PAGES) +LX_CONFIG(CONFIG_ARM64_64K_PAGES) +if IS_BUILTIN(CONFIG_ARM64): + LX_VALUE(CONFIG_ARM64_PA_BITS) + LX_VALUE(CONFIG_ARM64_VA_BITS) + LX_VALUE(CONFIG_ARM64_PAGE_SHIFT) + LX_VALUE(CONFIG_ARCH_FORCE_MAX_ORDER) +LX_CONFIG(CONFIG_SPARSEMEM) +LX_CONFIG(CONFIG_SPARSEMEM_EXTREME) +LX_CONFIG(CONFIG_SPARSEMEM_VMEMMAP) +LX_CONFIG(CONFIG_KASAN) +LX_CONFIG(CONFIG_KASAN_GENERIC) +LX_CONFIG(CONFIG_KASAN_SW_TAGS) +LX_CONFIG(CONFIG_KASAN_HW_TAGS) +if IS_BUILTIN(CONFIG_KASAN_GENERIC) or IS_BUILTIN(CONFIG_KASAN_SW_TAGS): + LX_VALUE(CONFIG_KASAN_SHADOW_OFFSET) +LX_CONFIG(CONFIG_VMAP_STACK) +if IS_BUILTIN(CONFIG_NUMA): + LX_VALUE(CONFIG_NODES_SHIFT) +LX_CONFIG(CONFIG_DEBUG_VIRTUAL) +LX_CONFIG(CONFIG_STACKDEPOT) +LX_CONFIG(CONFIG_PAGE_OWNER) +LX_CONFIG(CONFIG_SLUB_DEBUG) +LX_CONFIG(CONFIG_SLAB_FREELIST_HARDENED) diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py index 30d837f3dfae..ad5641dcb068 100644 --- a/scripts/gdb/linux/mm.py +++ b/scripts/gdb/linux/mm.py @@ -1,222 +1,398 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: GPL-2.0 # -# gdb helper commands and functions for Linux kernel debugging -# -# routines to introspect page table +# Copyright (c) 2023 MediaTek Inc. # # Authors: -# Dmitrii Bundin <dmitrii.bundin.a@gmail.com> +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> # import gdb +import math +from linux import utils, constants + +def DIV_ROUND_UP(n,d): + return ((n) + (d) - 1) // (d) -from linux import utils +def test_bit(nr, addr): + if addr.dereference() & (0x1 << nr): + return True + else: + return False -PHYSICAL_ADDRESS_MASK = gdb.parse_and_eval('0xfffffffffffff') +class page_ops(): + ops = None + def __init__(self): + if not constants.LX_CONFIG_SPARSEMEM_VMEMMAP: + raise gdb.GdbError('Only support CONFIG_SPARSEMEM_VMEMMAP now') + if constants.LX_CONFIG_ARM64 and utils.is_target_arch('aarch64'): + self.ops = aarch64_page_ops() + else: + raise gdb.GdbError('Only support aarch64 now') +class aarch64_page_ops(): + def __init__(self): + self.SUBSECTION_SHIFT = 21 + self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT + self.MODULES_VSIZE = 128 * 1024 * 1024 -def page_mask(level=1): - # 4KB - if level == 1: - return gdb.parse_and_eval('(u64) ~0xfff') - # 2MB - elif level == 2: - return gdb.parse_and_eval('(u64) ~0x1fffff') - # 1GB - elif level == 3: - return gdb.parse_and_eval('(u64) ~0x3fffffff') - else: - raise Exception(f'Unknown page level: {level}') - - -#page_offset_base in case CONFIG_DYNAMIC_MEMORY_LAYOUT is disabled -POB_NO_DYNAMIC_MEM_LAYOUT = '0xffff888000000000' -def _page_offset_base(): - pob_symbol = gdb.lookup_global_symbol('page_offset_base') - pob = pob_symbol.name if pob_symbol else POB_NO_DYNAMIC_MEM_LAYOUT - return gdb.parse_and_eval(pob) - - -def is_bit_defined_tupled(data, offset): - return offset, bool(data >> offset & 1) - -def content_tupled(data, bit_start, bit_end): - return (bit_start, bit_end), data >> bit_start & ((1 << (1 + bit_end - bit_start)) - 1) - -def entry_va(level, phys_addr, translating_va): - def start_bit(level): - if level == 5: - return 48 - elif level == 4: - return 39 - elif level == 3: - return 30 - elif level == 2: - return 21 - elif level == 1: - return 12 + if constants.LX_CONFIG_ARM64_64K_PAGES: + self.SECTION_SIZE_BITS = 29 + else: + self.SECTION_SIZE_BITS = 27 + self.MAX_PHYSMEM_BITS = constants.LX_CONFIG_ARM64_VA_BITS + + self.PAGE_SHIFT = constants.LX_CONFIG_ARM64_PAGE_SHIFT + self.PAGE_SIZE = 1 << self.PAGE_SHIFT + self.PAGE_MASK = (~(self.PAGE_SIZE - 1)) & ((1 << 64) - 1) + + self.VA_BITS = constants.LX_CONFIG_ARM64_VA_BITS + if self.VA_BITS > 48: + self.VA_BITS_MIN = 48 + self.vabits_actual = gdb.parse_and_eval('vabits_actual') + else: + self.VA_BITS_MIN = self.VA_BITS + self.vabits_actual = self.VA_BITS + self.kimage_voffset = gdb.parse_and_eval('kimage_voffset') & ((1 << 64) - 1) + + self.SECTIONS_SHIFT = self.MAX_PHYSMEM_BITS - self.SECTION_SIZE_BITS + + if str(constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER).isdigit(): + self.MAX_ORDER = constants.LX_CONFIG_ARCH_FORCE_MAX_ORDER + else: + self.MAX_ORDER = 11 + + self.MAX_ORDER_NR_PAGES = 1 << (self.MAX_ORDER - 1) + self.PFN_SECTION_SHIFT = self.SECTION_SIZE_BITS - self.PAGE_SHIFT + self.NR_MEM_SECTIONS = 1 << self.SECTIONS_SHIFT + self.PAGES_PER_SECTION = 1 << self.PFN_SECTION_SHIFT + self.PAGE_SECTION_MASK = (~(self.PAGES_PER_SECTION - 1)) & ((1 << 64) - 1) + + if constants.LX_CONFIG_SPARSEMEM_EXTREME: + self.SECTIONS_PER_ROOT = self.PAGE_SIZE // gdb.lookup_type("struct mem_section").sizeof + else: + self.SECTIONS_PER_ROOT = 1 + + self.NR_SECTION_ROOTS = DIV_ROUND_UP(self.NR_MEM_SECTIONS, self.SECTIONS_PER_ROOT) + self.SECTION_ROOT_MASK = self.SECTIONS_PER_ROOT - 1 + self.SUBSECTION_SHIFT = 21 + self.SEBSECTION_SIZE = 1 << self.SUBSECTION_SHIFT + self.PFN_SUBSECTION_SHIFT = self.SUBSECTION_SHIFT - self.PAGE_SHIFT + self.PAGES_PER_SUBSECTION = 1 << self.PFN_SUBSECTION_SHIFT + + self.SECTION_HAS_MEM_MAP = 1 << int(gdb.parse_and_eval('SECTION_HAS_MEM_MAP_BIT')) + self.SECTION_IS_EARLY = 1 << int(gdb.parse_and_eval('SECTION_IS_EARLY_BIT')) + + self.struct_page_size = utils.get_page_type().sizeof + self.STRUCT_PAGE_MAX_SHIFT = (int)(math.log(self.struct_page_size, 2)) + + self.PAGE_OFFSET = self._PAGE_OFFSET(self.VA_BITS) + self.MODULES_VADDR = self._PAGE_END(self.VA_BITS_MIN) + self.MODULES_END = self.MODULES_VADDR + self.MODULES_VSIZE + + self.VMEMMAP_SHIFT = (self.PAGE_SHIFT - self.STRUCT_PAGE_MAX_SHIFT) + self.VMEMMAP_SIZE = ((self._PAGE_END(self.VA_BITS_MIN) - self.PAGE_OFFSET) >> self.VMEMMAP_SHIFT) + self.VMEMMAP_START = (-(1 << (self.VA_BITS - self.VMEMMAP_SHIFT))) & 0xffffffffffffffff + self.VMEMMAP_END = self.VMEMMAP_START + self.VMEMMAP_SIZE + + self.VMALLOC_START = self.MODULES_END + self.VMALLOC_END = self.VMEMMAP_START - 256 * 1024 * 1024 + + self.memstart_addr = gdb.parse_and_eval("memstart_addr") + self.PHYS_OFFSET = self.memstart_addr + self.vmemmap = gdb.Value(self.VMEMMAP_START).cast(utils.get_page_type().pointer()) - (self.memstart_addr >> self.PAGE_SHIFT) + + self.KERNEL_START = gdb.parse_and_eval("_text") + self.KERNEL_END = gdb.parse_and_eval("_end") + + if constants.LX_CONFIG_KASAN_GENERIC or constants.LX_CONFIG_KASAN_SW_TAGS: + if constants.LX_CONFIG_KASAN_GENERIC: + self.KASAN_SHADOW_SCALE_SHIFT = 3 else: - raise Exception(f'Unknown level {level}') - - entry_offset = ((translating_va >> start_bit(level)) & 511) * 8 - entry_va = _page_offset_base() + phys_addr + entry_offset - return entry_va - -class Cr3(): - def __init__(self, cr3, page_levels): - self.cr3 = cr3 - self.page_levels = page_levels - self.page_level_write_through = is_bit_defined_tupled(cr3, 3) - self.page_level_cache_disabled = is_bit_defined_tupled(cr3, 4) - self.next_entry_physical_address = cr3 & PHYSICAL_ADDRESS_MASK & page_mask() - - def next_entry(self, va): - next_level = self.page_levels - return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) - - def mk_string(self): - return f"""\ -cr3: - {'cr3 binary data': <30} {hex(self.cr3)} - {'next entry physical address': <30} {hex(self.next_entry_physical_address)} - --- - {'bit' : <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} - {'bit' : <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} -""" - - -class PageHierarchyEntry(): - def __init__(self, address, level): - data = int.from_bytes( - memoryview(gdb.selected_inferior().read_memory(address, 8)), - "little" - ) - if level == 1: - self.is_page = True - self.entry_present = is_bit_defined_tupled(data, 0) - self.read_write = is_bit_defined_tupled(data, 1) - self.user_access_allowed = is_bit_defined_tupled(data, 2) - self.page_level_write_through = is_bit_defined_tupled(data, 3) - self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) - self.entry_was_accessed = is_bit_defined_tupled(data, 5) - self.dirty = is_bit_defined_tupled(data, 6) - self.pat = is_bit_defined_tupled(data, 7) - self.global_translation = is_bit_defined_tupled(data, 8) - self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) - self.next_entry_physical_address = None - self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) - self.protection_key = content_tupled(data, 59, 62) - self.executed_disable = is_bit_defined_tupled(data, 63) + self.KASAN_SHADOW_SCALE_SHIFT = 4 + self.KASAN_SHADOW_OFFSET = constants.LX_CONFIG_KASAN_SHADOW_OFFSET + self.KASAN_SHADOW_END = (1 << (64 - self.KASAN_SHADOW_SCALE_SHIFT)) + self.KASAN_SHADOW_OFFSET + self.PAGE_END = self.KASAN_SHADOW_END - (1 << (self.vabits_actual - self.KASAN_SHADOW_SCALE_SHIFT)) + else: + self.PAGE_END = self._PAGE_END(self.VA_BITS_MIN) + + if constants.LX_CONFIG_NUMA and constants.LX_CONFIG_NODES_SHIFT: + self.NODE_SHIFT = constants.LX_CONFIG_NODES_SHIFT + else: + self.NODE_SHIFT = 0 + + self.MAX_NUMNODES = 1 << self.NODE_SHIFT + + def SECTION_NR_TO_ROOT(self, sec): + return sec // self.SECTIONS_PER_ROOT + + def __nr_to_section(self, nr): + root = self.SECTION_NR_TO_ROOT(nr) + mem_section = gdb.parse_and_eval("mem_section") + return mem_section[root][nr & self.SECTION_ROOT_MASK] + + def pfn_to_section_nr(self, pfn): + return pfn >> self.PFN_SECTION_SHIFT + + def section_nr_to_pfn(self, sec): + return sec << self.PFN_SECTION_SHIFT + + def __pfn_to_section(self, pfn): + return self.__nr_to_section(self.pfn_to_section_nr(pfn)) + + def pfn_to_section(self, pfn): + return self.__pfn_to_section(pfn) + + def subsection_map_index(self, pfn): + return (pfn & ~(self.PAGE_SECTION_MASK)) // self.PAGES_PER_SUBSECTION + + def pfn_section_valid(self, ms, pfn): + if constants.LX_CONFIG_SPARSEMEM_VMEMMAP: + idx = self.subsection_map_index(pfn) + return test_bit(idx, ms['usage']['subsection_map']) + else: + return True + + def valid_section(self, mem_section): + if mem_section != None and (mem_section['section_mem_map'] & self.SECTION_HAS_MEM_MAP): + return True + return False + + def early_section(self, mem_section): + if mem_section != None and (mem_section['section_mem_map'] & self.SECTION_IS_EARLY): + return True + return False + + def pfn_valid(self, pfn): + ms = None + if self.PHYS_PFN(self.PFN_PHYS(pfn)) != pfn: + return False + if self.pfn_to_section_nr(pfn) >= self.NR_MEM_SECTIONS: + return False + ms = self.__pfn_to_section(pfn) + + if not self.valid_section(ms): + return False + return self.early_section(ms) or self.pfn_section_valid(ms, pfn) + + def _PAGE_OFFSET(self, va): + return (-(1 << (va))) & 0xffffffffffffffff + + def _PAGE_END(self, va): + return (-(1 << (va - 1))) & 0xffffffffffffffff + + def kasan_reset_tag(self, addr): + if constants.LX_CONFIG_KASAN_SW_TAGS or constants.LX_CONFIG_KASAN_HW_TAGS: + return int(addr) | (0xff << 56) + else: + return addr + + def __is_lm_address(self, addr): + if (addr - self.PAGE_OFFSET) < (self.PAGE_END - self.PAGE_OFFSET): + return True + else: + return False + def __lm_to_phys(self, addr): + return addr - self.PAGE_OFFSET + self.PHYS_OFFSET + + def __kimg_to_phys(self, addr): + return addr - self.kimage_voffset + + def __virt_to_phys_nodebug(self, va): + untagged_va = self.kasan_reset_tag(va) + if self.__is_lm_address(untagged_va): + return self.__lm_to_phys(untagged_va) + else: + return self.__kimg_to_phys(untagged_va) + + def __virt_to_phys(self, va): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + if not self.__is_lm_address(self.kasan_reset_tag(va)): + raise gdb.GdbError("Warning: virt_to_phys used for non-linear address: 0x%lx\n" % va) + return self.__virt_to_phys_nodebug(va) + + def virt_to_phys(self, va): + return self.__virt_to_phys(va) + + def PFN_PHYS(self, pfn): + return pfn << self.PAGE_SHIFT + + def PHYS_PFN(self, phys): + return phys >> self.PAGE_SHIFT + + def __phys_to_virt(self, pa): + return (pa - self.PHYS_OFFSET) | self.PAGE_OFFSET + + def __phys_to_pfn(self, pa): + return self.PHYS_PFN(pa) + + def __pfn_to_phys(self, pfn): + return self.PFN_PHYS(pfn) + + def __pa_symbol_nodebug(self, x): + return self.__kimg_to_phys(x) + + def __phys_addr_symbol(self, x): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + if x < self.KERNEL_START or x > self.KERNEL_END: + raise gdb.GdbError("0x%x exceed kernel range" % x) + return self.__pa_symbol_nodebug(x) + + def __pa_symbol(self, x): + return self.__phys_addr_symbol(x) + + def __va(self, pa): + return self.__phys_to_virt(pa) + + def pfn_to_kaddr(self, pfn): + return self.__va(pfn << self.PAGE_SHIFT) + + def virt_to_pfn(self, va): + return self.__phys_to_pfn(self.__virt_to_phys(va)) + + def sym_to_pfn(self, x): + return self.__phys_to_pfn(self.__pa_symbol(x)) + + def page_to_pfn(self, page): + return int(page.cast(utils.get_page_type().pointer()) - self.vmemmap.cast(utils.get_page_type().pointer())) + + def page_to_phys(self, page): + return self.__pfn_to_phys(self.page_to_pfn(page)) + + def pfn_to_page(self, pfn): + return (self.vmemmap + pfn).cast(utils.get_page_type().pointer()) + + def page_to_virt(self, page): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + return self.__va(self.page_to_phys(page)) else: - page_size = is_bit_defined_tupled(data, 7) - page_size_bit = page_size[1] - self.is_page = page_size_bit - self.entry_present = is_bit_defined_tupled(data, 0) - self.read_write = is_bit_defined_tupled(data, 1) - self.user_access_allowed = is_bit_defined_tupled(data, 2) - self.page_level_write_through = is_bit_defined_tupled(data, 3) - self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) - self.entry_was_accessed = is_bit_defined_tupled(data, 5) - self.page_size = page_size - self.dirty = is_bit_defined_tupled( - data, 6) if page_size_bit else None - self.global_translation = is_bit_defined_tupled( - data, 8) if page_size_bit else None - self.pat = is_bit_defined_tupled( - data, 12) if page_size_bit else None - self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) if page_size_bit else None - self.next_entry_physical_address = None if page_size_bit else data & PHYSICAL_ADDRESS_MASK & page_mask() - self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) - self.protection_key = content_tupled(data, 59, 62) if page_size_bit else None - self.executed_disable = is_bit_defined_tupled(data, 63) - self.address = address - self.page_entry_binary_data = data - self.page_hierarchy_level = level - - def next_entry(self, va): - if self.is_page or not self.entry_present[1]: - return None - - next_level = self.page_hierarchy_level - 1 - return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) - - - def mk_string(self): - if not self.entry_present[1]: - return f"""\ -level {self.page_hierarchy_level}: - {'entry address': <30} {hex(self.address)} - {'page entry binary data': <30} {hex(self.page_entry_binary_data)} - --- - PAGE ENTRY IS NOT PRESENT! -""" - elif self.is_page: - def page_size_line(ps_bit, ps, level): - return "" if level == 1 else f"{'bit': <3} {ps_bit: <5} {'page size': <30} {ps}" - - return f"""\ -level {self.page_hierarchy_level}: - {'entry address': <30} {hex(self.address)} - {'page entry binary data': <30} {hex(self.page_entry_binary_data)} - {'page size': <30} {'1GB' if self.page_hierarchy_level == 3 else '2MB' if self.page_hierarchy_level == 2 else '4KB' if self.page_hierarchy_level == 1 else 'Unknown page size for level:' + self.page_hierarchy_level} - {'page physical address': <30} {hex(self.page_physical_address)} - --- - {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} - {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} - {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} - {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} - {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} - {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} - {"" if self.page_hierarchy_level == 1 else f"{'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]}"} - {'bit': <4} {self.dirty[0]: <10} {'page dirty': <30} {self.dirty[1]} - {'bit': <4} {self.global_translation[0]: <10} {'global translation': <30} {self.global_translation[1]} - {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} - {'bit': <4} {self.pat[0]: <10} {'pat': <30} {self.pat[1]} - {'bits': <4} {str(self.protection_key[0]): <10} {'protection key': <30} {self.protection_key[1]} - {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} -""" + __idx = int((page.cast(gdb.lookup_type("unsigned long")) - self.VMEMMAP_START).cast(utils.get_ulong_type())) // self.struct_page_size + return self.PAGE_OFFSET + (__idx * self.PAGE_SIZE) + + def virt_to_page(self, va): + if constants.LX_CONFIG_DEBUG_VIRTUAL: + return self.pfn_to_page(self.virt_to_pfn(va)) else: - return f"""\ -level {self.page_hierarchy_level}: - {'entry address': <30} {hex(self.address)} - {'page entry binary data': <30} {hex(self.page_entry_binary_data)} - {'next entry physical address': <30} {hex(self.next_entry_physical_address)} - --- - {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} - {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} - {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} - {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} - {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} - {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} - {'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]} - {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} - {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} -""" - - -class TranslateVM(gdb.Command): - """Prints the entire paging structure used to translate a given virtual address. - -Having an address space of the currently executed process translates the virtual address -and prints detailed information of all paging structure levels used for the transaltion. -Currently supported arch: x86""" + __idx = int(self.kasan_reset_tag(va) - self.PAGE_OFFSET) // self.PAGE_SIZE + addr = self.VMEMMAP_START + (__idx * self.struct_page_size) + return gdb.Value(addr).cast(utils.get_page_type().pointer()) + + def page_address(self, page): + return self.page_to_virt(page) + + def folio_address(self, folio): + return self.page_address(folio['page'].address) + +class LxPFN2Page(gdb.Command): + """PFN to struct page""" def __init__(self): - super(TranslateVM, self).__init__('translate-vm', gdb.COMMAND_USER) + super(LxPFN2Page, self).__init__("lx-pfn_to_page", gdb.COMMAND_USER) def invoke(self, arg, from_tty): - if utils.is_target_arch("x86"): - vm_address = gdb.parse_and_eval(f'{arg}') - cr3_data = gdb.parse_and_eval('$cr3') - cr4 = gdb.parse_and_eval('$cr4') - page_levels = 5 if cr4 & (1 << 12) else 4 - page_entry = Cr3(cr3_data, page_levels) - while page_entry: - gdb.write(page_entry.mk_string()) - page_entry = page_entry.next_entry(vm_address) - else: - gdb.GdbError("Virtual address translation is not" - "supported for this arch") + argv = gdb.string_to_argv(arg) + pfn = int(argv[0]) + page = page_ops().ops.pfn_to_page(pfn) + gdb.write("pfn_to_page(0x%x) = 0x%x\n" % (pfn, page)) + +LxPFN2Page() + +class LxPage2PFN(gdb.Command): + """struct page to PFN""" + + def __init__(self): + super(LxPage2PFN, self).__init__("lx-page_to_pfn", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + struct_page_addr = int(argv[0], 16) + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + pfn = page_ops().ops.page_to_pfn(page) + gdb.write("page_to_pfn(0x%x) = 0x%x\n" % (page, pfn)) + +LxPage2PFN() + +class LxPageAddress(gdb.Command): + """struct page to linear mapping address""" + + def __init__(self): + super(LxPageAddress, self).__init__("lx-page_address", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + struct_page_addr = int(argv[0], 16) + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + addr = page_ops().ops.page_address(page) + gdb.write("page_address(0x%x) = 0x%x\n" % (page, addr)) +LxPageAddress() + +class LxPage2Phys(gdb.Command): + """struct page to physical address""" + + def __init__(self): + super(LxPage2Phys, self).__init__("lx-page_to_phys", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + struct_page_addr = int(argv[0], 16) + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + phys_addr = page_ops().ops.page_to_phys(page) + gdb.write("page_to_phys(0x%x) = 0x%x\n" % (page, phys_addr)) + +LxPage2Phys() + +class LxVirt2Phys(gdb.Command): + """virtual address to physical address""" + + def __init__(self): + super(LxVirt2Phys, self).__init__("lx-virt_to_phys", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + linear_addr = int(argv[0], 16) + phys_addr = page_ops().ops.virt_to_phys(linear_addr) + gdb.write("virt_to_phys(0x%x) = 0x%x\n" % (linear_addr, phys_addr)) + +LxVirt2Phys() + +class LxVirt2Page(gdb.Command): + """virtual address to struct page""" + + def __init__(self): + super(LxVirt2Page, self).__init__("lx-virt_to_page", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + linear_addr = int(argv[0], 16) + page = page_ops().ops.virt_to_page(linear_addr) + gdb.write("virt_to_page(0x%x) = 0x%x\n" % (linear_addr, page)) + +LxVirt2Page() + +class LxSym2PFN(gdb.Command): + """symbol address to PFN""" + + def __init__(self): + super(LxSym2PFN, self).__init__("lx-sym_to_pfn", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + sym_addr = int(argv[0], 16) + pfn = page_ops().ops.sym_to_pfn(sym_addr) + gdb.write("sym_to_pfn(0x%x) = %d\n" % (sym_addr, pfn)) + +LxSym2PFN() + +class LxPFN2Kaddr(gdb.Command): + """PFN to kernel address""" + + def __init__(self): + super(LxPFN2Kaddr, self).__init__("lx-pfn_to_kaddr", gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + pfn = int(argv[0]) + kaddr = page_ops().ops.pfn_to_kaddr(pfn) + gdb.write("pfn_to_kaddr(%d) = 0x%x\n" % (pfn, kaddr)) -TranslateVM() +LxPFN2Kaddr() diff --git a/scripts/gdb/linux/modules.py b/scripts/gdb/linux/modules.py index 261f28640f4c..298dfcc25eae 100644 --- a/scripts/gdb/linux/modules.py +++ b/scripts/gdb/linux/modules.py @@ -73,11 +73,17 @@ class LxLsmod(gdb.Command): " " if utils.get_long_type().sizeof == 8 else "")) for module in module_list(): - layout = module['mem'][constants.LX_MOD_TEXT] + text = module['mem'][constants.LX_MOD_TEXT] + text_addr = str(text['base']).split()[0] + total_size = 0 + + for i in range(constants.LX_MOD_TEXT, constants.LX_MOD_RO_AFTER_INIT + 1): + total_size += module['mem'][i]['size'] + gdb.write("{address} {name:<19} {size:>8} {ref}".format( - address=str(layout['base']).split()[0], + address=text_addr, name=module['name'].string(), - size=str(layout['size']), + size=str(total_size), ref=str(module['refcnt']['counter'] - 1))) t = self._module_use_type.get_type().pointer() @@ -91,5 +97,35 @@ class LxLsmod(gdb.Command): gdb.write("\n") - LxLsmod() + +def help(): + t = """Usage: lx-getmod-by-textaddr [Heximal Address] + Example: lx-getmod-by-textaddr 0xffff800002d305ac\n""" + gdb.write("Unrecognized command\n") + raise gdb.GdbError(t) + +class LxFindTextAddrinMod(gdb.Command): + '''Look up loaded kernel module by text address.''' + + def __init__(self): + super(LxFindTextAddrinMod, self).__init__('lx-getmod-by-textaddr', gdb.COMMAND_SUPPORT) + + def invoke(self, arg, from_tty): + args = gdb.string_to_argv(arg) + + if len(args) != 1: + help() + + addr = gdb.Value(int(args[0], 16)).cast(utils.get_ulong_type()) + for mod in module_list(): + mod_text_start = mod['mem'][constants.LX_MOD_TEXT]['base'] + mod_text_end = mod_text_start + mod['mem'][constants.LX_MOD_TEXT]['size'].cast(utils.get_ulong_type()) + + if addr >= mod_text_start and addr < mod_text_end: + s = "0x%x" % addr + " is in " + mod['name'].string() + ".ko\n" + gdb.write(s) + return + gdb.write("0x%x is not in any module text section\n" % addr) + +LxFindTextAddrinMod() diff --git a/scripts/gdb/linux/page_owner.py b/scripts/gdb/linux/page_owner.py new file mode 100644 index 000000000000..844fd5d0c912 --- /dev/null +++ b/scripts/gdb/linux/page_owner.py @@ -0,0 +1,190 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +from linux import utils, stackdepot, constants, mm + +if constants.LX_CONFIG_PAGE_OWNER: + page_ext_t = utils.CachedType('struct page_ext') + page_owner_t = utils.CachedType('struct page_owner') + + PAGE_OWNER_STACK_DEPTH = 16 + PAGE_EXT_OWNER = constants.LX_PAGE_EXT_OWNER + PAGE_EXT_INVALID = 0x1 + PAGE_EXT_OWNER_ALLOCATED = constants.LX_PAGE_EXT_OWNER_ALLOCATED + +def help(): + t = """Usage: lx-dump-page-owner [Option] + Option: + --pfn [Decimal pfn] + Example: + lx-dump-page-owner --pfn 655360\n""" + gdb.write("Unrecognized command\n") + raise gdb.GdbError(t) + +class DumpPageOwner(gdb.Command): + """Dump page owner""" + + min_pfn = None + max_pfn = None + p_ops = None + migrate_reason_names = None + + def __init__(self): + super(DumpPageOwner, self).__init__("lx-dump-page-owner", gdb.COMMAND_SUPPORT) + + def invoke(self, args, from_tty): + if not constants.LX_CONFIG_PAGE_OWNER: + raise gdb.GdbError('CONFIG_PAGE_OWNER does not enable') + + page_owner_inited = gdb.parse_and_eval('page_owner_inited') + if page_owner_inited['key']['enabled']['counter'] != 0x1: + raise gdb.GdbError('page_owner_inited is not enabled') + + self.p_ops = mm.page_ops().ops + self.get_page_owner_info() + argv = gdb.string_to_argv(args) + if len(argv) == 0: + self.read_page_owner() + elif len(argv) == 2: + if argv[0] == "--pfn": + pfn = int(argv[1]) + self.read_page_owner_by_addr(self.p_ops.pfn_to_page(pfn)) + else: + help() + else: + help() + + def get_page_owner_info(self): + self.min_pfn = int(gdb.parse_and_eval("min_low_pfn")) + self.max_pfn = int(gdb.parse_and_eval("max_pfn")) + self.page_ext_size = int(gdb.parse_and_eval("page_ext_size")) + self.migrate_reason_names = gdb.parse_and_eval('migrate_reason_names') + + def page_ext_invalid(self, page_ext): + if page_ext == gdb.Value(0): + return True + if page_ext.cast(utils.get_ulong_type()) & PAGE_EXT_INVALID == PAGE_EXT_INVALID: + return True + return False + + def get_entry(self, base, index): + return (base.cast(utils.get_ulong_type()) + self.page_ext_size * index).cast(page_ext_t.get_type().pointer()) + + def lookup_page_ext(self, page): + pfn = self.p_ops.page_to_pfn(page) + section = self.p_ops.pfn_to_section(pfn) + page_ext = section["page_ext"] + if self.page_ext_invalid(page_ext): + return gdb.Value(0) + return self.get_entry(page_ext, pfn) + + def page_ext_get(self, page): + page_ext = self.lookup_page_ext(page) + if page_ext != gdb.Value(0): + return page_ext + else: + return gdb.Value(0) + + def get_page_owner(self, page_ext): + addr = page_ext.cast(utils.get_ulong_type()) + gdb.parse_and_eval("page_owner_ops")["offset"].cast(utils.get_ulong_type()) + return addr.cast(page_owner_t.get_type().pointer()) + + def read_page_owner_by_addr(self, struct_page_addr): + page = gdb.Value(struct_page_addr).cast(utils.get_page_type().pointer()) + pfn = self.p_ops.page_to_pfn(page) + + if pfn < self.min_pfn or pfn > self.max_pfn or (not self.p_ops.pfn_valid(pfn)): + gdb.write("pfn is invalid\n") + return + + page = self.p_ops.pfn_to_page(pfn) + page_ext = self.page_ext_get(page) + + if page_ext == gdb.Value(0): + gdb.write("page_ext is null\n") + return + + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER)): + gdb.write("page_owner flag is invalid\n") + raise gdb.GdbError('page_owner info is not present (never set?)\n') + + if mm.test_bit(PAGE_EXT_OWNER_ALLOCATED, page_ext['flags'].address): + gdb.write('page_owner tracks the page as allocated\n') + else: + gdb.write('page_owner tracks the page as freed\n') + + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER_ALLOCATED)): + gdb.write("page_owner is not allocated\n") + + try: + page_owner = self.get_page_owner(page_ext) + gdb.write("Page last allocated via order %d, gfp_mask: 0x%x, pid: %d, tgid: %d (%s), ts %u ns, free_ts %u ns\n" %\ + (page_owner["order"], page_owner["gfp_mask"],\ + page_owner["pid"], page_owner["tgid"], page_owner["comm"],\ + page_owner["ts_nsec"], page_owner["free_ts_nsec"])) + gdb.write("PFN: %d, Flags: 0x%x\n" % (pfn, page['flags'])) + if page_owner["handle"] == 0: + gdb.write('page_owner allocation stack trace missing\n') + else: + stackdepot.stack_depot_print(page_owner["handle"]) + + if page_owner["free_handle"] == 0: + gdb.write('page_owner free stack trace missing\n') + else: + gdb.write('page last free stack trace:\n') + stackdepot.stack_depot_print(page_owner["free_handle"]) + if page_owner['last_migrate_reason'] != -1: + gdb.write('page has been migrated, last migrate reason: %s\n' % self.migrate_reason_names[page_owner['last_migrate_reason']]) + except: + gdb.write("\n") + + def read_page_owner(self): + pfn = self.min_pfn + + # Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area + while ((not self.p_ops.pfn_valid(pfn)) and (pfn & (self.p_ops.MAX_ORDER_NR_PAGES - 1))) != 0: + pfn += 1 + + while pfn < self.max_pfn: + # + # If the new page is in a new MAX_ORDER_NR_PAGES area, + # validate the area as existing, skip it if not + # + if ((pfn & (self.p_ops.MAX_ORDER_NR_PAGES - 1)) == 0) and (not self.p_ops.pfn_valid(pfn)): + pfn += (self.p_ops.MAX_ORDER_NR_PAGES - 1) + continue; + + page = self.p_ops.pfn_to_page(pfn) + page_ext = self.page_ext_get(page) + if page_ext == gdb.Value(0): + pfn += 1 + continue + + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER)): + pfn += 1 + continue + if not (page_ext['flags'] & (1 << PAGE_EXT_OWNER_ALLOCATED)): + pfn += 1 + continue + + try: + page_owner = self.get_page_owner(page_ext) + gdb.write("Page allocated via order %d, gfp_mask: 0x%x, pid: %d, tgid: %d (%s), ts %u ns, free_ts %u ns\n" %\ + (page_owner["order"], page_owner["gfp_mask"],\ + page_owner["pid"], page_owner["tgid"], page_owner["comm"],\ + page_owner["ts_nsec"], page_owner["free_ts_nsec"])) + gdb.write("PFN: %d, Flags: 0x%x\n" % (pfn, page['flags'])) + stackdepot.stack_depot_print(page_owner["handle"]) + pfn += (1 << page_owner["order"]) + continue + except: + gdb.write("\n") + pfn += 1 + +DumpPageOwner() diff --git a/scripts/gdb/linux/pgtable.py b/scripts/gdb/linux/pgtable.py new file mode 100644 index 000000000000..30d837f3dfae --- /dev/null +++ b/scripts/gdb/linux/pgtable.py @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# gdb helper commands and functions for Linux kernel debugging +# +# routines to introspect page table +# +# Authors: +# Dmitrii Bundin <dmitrii.bundin.a@gmail.com> +# + +import gdb + +from linux import utils + +PHYSICAL_ADDRESS_MASK = gdb.parse_and_eval('0xfffffffffffff') + + +def page_mask(level=1): + # 4KB + if level == 1: + return gdb.parse_and_eval('(u64) ~0xfff') + # 2MB + elif level == 2: + return gdb.parse_and_eval('(u64) ~0x1fffff') + # 1GB + elif level == 3: + return gdb.parse_and_eval('(u64) ~0x3fffffff') + else: + raise Exception(f'Unknown page level: {level}') + + +#page_offset_base in case CONFIG_DYNAMIC_MEMORY_LAYOUT is disabled +POB_NO_DYNAMIC_MEM_LAYOUT = '0xffff888000000000' +def _page_offset_base(): + pob_symbol = gdb.lookup_global_symbol('page_offset_base') + pob = pob_symbol.name if pob_symbol else POB_NO_DYNAMIC_MEM_LAYOUT + return gdb.parse_and_eval(pob) + + +def is_bit_defined_tupled(data, offset): + return offset, bool(data >> offset & 1) + +def content_tupled(data, bit_start, bit_end): + return (bit_start, bit_end), data >> bit_start & ((1 << (1 + bit_end - bit_start)) - 1) + +def entry_va(level, phys_addr, translating_va): + def start_bit(level): + if level == 5: + return 48 + elif level == 4: + return 39 + elif level == 3: + return 30 + elif level == 2: + return 21 + elif level == 1: + return 12 + else: + raise Exception(f'Unknown level {level}') + + entry_offset = ((translating_va >> start_bit(level)) & 511) * 8 + entry_va = _page_offset_base() + phys_addr + entry_offset + return entry_va + +class Cr3(): + def __init__(self, cr3, page_levels): + self.cr3 = cr3 + self.page_levels = page_levels + self.page_level_write_through = is_bit_defined_tupled(cr3, 3) + self.page_level_cache_disabled = is_bit_defined_tupled(cr3, 4) + self.next_entry_physical_address = cr3 & PHYSICAL_ADDRESS_MASK & page_mask() + + def next_entry(self, va): + next_level = self.page_levels + return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) + + def mk_string(self): + return f"""\ +cr3: + {'cr3 binary data': <30} {hex(self.cr3)} + {'next entry physical address': <30} {hex(self.next_entry_physical_address)} + --- + {'bit' : <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} + {'bit' : <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} +""" + + +class PageHierarchyEntry(): + def __init__(self, address, level): + data = int.from_bytes( + memoryview(gdb.selected_inferior().read_memory(address, 8)), + "little" + ) + if level == 1: + self.is_page = True + self.entry_present = is_bit_defined_tupled(data, 0) + self.read_write = is_bit_defined_tupled(data, 1) + self.user_access_allowed = is_bit_defined_tupled(data, 2) + self.page_level_write_through = is_bit_defined_tupled(data, 3) + self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) + self.entry_was_accessed = is_bit_defined_tupled(data, 5) + self.dirty = is_bit_defined_tupled(data, 6) + self.pat = is_bit_defined_tupled(data, 7) + self.global_translation = is_bit_defined_tupled(data, 8) + self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) + self.next_entry_physical_address = None + self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) + self.protection_key = content_tupled(data, 59, 62) + self.executed_disable = is_bit_defined_tupled(data, 63) + else: + page_size = is_bit_defined_tupled(data, 7) + page_size_bit = page_size[1] + self.is_page = page_size_bit + self.entry_present = is_bit_defined_tupled(data, 0) + self.read_write = is_bit_defined_tupled(data, 1) + self.user_access_allowed = is_bit_defined_tupled(data, 2) + self.page_level_write_through = is_bit_defined_tupled(data, 3) + self.page_level_cache_disabled = is_bit_defined_tupled(data, 4) + self.entry_was_accessed = is_bit_defined_tupled(data, 5) + self.page_size = page_size + self.dirty = is_bit_defined_tupled( + data, 6) if page_size_bit else None + self.global_translation = is_bit_defined_tupled( + data, 8) if page_size_bit else None + self.pat = is_bit_defined_tupled( + data, 12) if page_size_bit else None + self.page_physical_address = data & PHYSICAL_ADDRESS_MASK & page_mask(level) if page_size_bit else None + self.next_entry_physical_address = None if page_size_bit else data & PHYSICAL_ADDRESS_MASK & page_mask() + self.hlat_restart_with_ordinary = is_bit_defined_tupled(data, 11) + self.protection_key = content_tupled(data, 59, 62) if page_size_bit else None + self.executed_disable = is_bit_defined_tupled(data, 63) + self.address = address + self.page_entry_binary_data = data + self.page_hierarchy_level = level + + def next_entry(self, va): + if self.is_page or not self.entry_present[1]: + return None + + next_level = self.page_hierarchy_level - 1 + return PageHierarchyEntry(entry_va(next_level, self.next_entry_physical_address, va), next_level) + + + def mk_string(self): + if not self.entry_present[1]: + return f"""\ +level {self.page_hierarchy_level}: + {'entry address': <30} {hex(self.address)} + {'page entry binary data': <30} {hex(self.page_entry_binary_data)} + --- + PAGE ENTRY IS NOT PRESENT! +""" + elif self.is_page: + def page_size_line(ps_bit, ps, level): + return "" if level == 1 else f"{'bit': <3} {ps_bit: <5} {'page size': <30} {ps}" + + return f"""\ +level {self.page_hierarchy_level}: + {'entry address': <30} {hex(self.address)} + {'page entry binary data': <30} {hex(self.page_entry_binary_data)} + {'page size': <30} {'1GB' if self.page_hierarchy_level == 3 else '2MB' if self.page_hierarchy_level == 2 else '4KB' if self.page_hierarchy_level == 1 else 'Unknown page size for level:' + self.page_hierarchy_level} + {'page physical address': <30} {hex(self.page_physical_address)} + --- + {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} + {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} + {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} + {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} + {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} + {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} + {"" if self.page_hierarchy_level == 1 else f"{'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]}"} + {'bit': <4} {self.dirty[0]: <10} {'page dirty': <30} {self.dirty[1]} + {'bit': <4} {self.global_translation[0]: <10} {'global translation': <30} {self.global_translation[1]} + {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} + {'bit': <4} {self.pat[0]: <10} {'pat': <30} {self.pat[1]} + {'bits': <4} {str(self.protection_key[0]): <10} {'protection key': <30} {self.protection_key[1]} + {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} +""" + else: + return f"""\ +level {self.page_hierarchy_level}: + {'entry address': <30} {hex(self.address)} + {'page entry binary data': <30} {hex(self.page_entry_binary_data)} + {'next entry physical address': <30} {hex(self.next_entry_physical_address)} + --- + {'bit': <4} {self.entry_present[0]: <10} {'entry present': <30} {self.entry_present[1]} + {'bit': <4} {self.read_write[0]: <10} {'read/write access allowed': <30} {self.read_write[1]} + {'bit': <4} {self.user_access_allowed[0]: <10} {'user access allowed': <30} {self.user_access_allowed[1]} + {'bit': <4} {self.page_level_write_through[0]: <10} {'page level write through': <30} {self.page_level_write_through[1]} + {'bit': <4} {self.page_level_cache_disabled[0]: <10} {'page level cache disabled': <30} {self.page_level_cache_disabled[1]} + {'bit': <4} {self.entry_was_accessed[0]: <10} {'entry has been accessed': <30} {self.entry_was_accessed[1]} + {'bit': <4} {self.page_size[0]: <10} {'page size': <30} {self.page_size[1]} + {'bit': <4} {self.hlat_restart_with_ordinary[0]: <10} {'restart to ordinary': <30} {self.hlat_restart_with_ordinary[1]} + {'bit': <4} {self.executed_disable[0]: <10} {'execute disable': <30} {self.executed_disable[1]} +""" + + +class TranslateVM(gdb.Command): + """Prints the entire paging structure used to translate a given virtual address. + +Having an address space of the currently executed process translates the virtual address +and prints detailed information of all paging structure levels used for the transaltion. +Currently supported arch: x86""" + + def __init__(self): + super(TranslateVM, self).__init__('translate-vm', gdb.COMMAND_USER) + + def invoke(self, arg, from_tty): + if utils.is_target_arch("x86"): + vm_address = gdb.parse_and_eval(f'{arg}') + cr3_data = gdb.parse_and_eval('$cr3') + cr4 = gdb.parse_and_eval('$cr4') + page_levels = 5 if cr4 & (1 << 12) else 4 + page_entry = Cr3(cr3_data, page_levels) + while page_entry: + gdb.write(page_entry.mk_string()) + page_entry = page_entry.next_entry(vm_address) + else: + gdb.GdbError("Virtual address translation is not" + "supported for this arch") + + +TranslateVM() diff --git a/scripts/gdb/linux/slab.py b/scripts/gdb/linux/slab.py new file mode 100644 index 000000000000..f012ba38c7d9 --- /dev/null +++ b/scripts/gdb/linux/slab.py @@ -0,0 +1,326 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +import re +import traceback +from linux import lists, utils, stackdepot, constants, mm + +SLAB_RED_ZONE = constants.LX_SLAB_RED_ZONE +SLAB_POISON = constants.LX_SLAB_POISON +SLAB_KMALLOC = constants.LX_SLAB_KMALLOC +SLAB_HWCACHE_ALIGN = constants.LX_SLAB_HWCACHE_ALIGN +SLAB_CACHE_DMA = constants.LX_SLAB_CACHE_DMA +SLAB_CACHE_DMA32 = constants.LX_SLAB_CACHE_DMA32 +SLAB_STORE_USER = constants.LX_SLAB_STORE_USER +SLAB_PANIC = constants.LX_SLAB_PANIC + +OO_SHIFT = 16 +OO_MASK = (1 << OO_SHIFT) - 1 + +if constants.LX_CONFIG_SLUB_DEBUG: + slab_type = utils.CachedType("struct slab") + slab_ptr_type = slab_type.get_type().pointer() + kmem_cache_type = utils.CachedType("struct kmem_cache") + kmem_cache_ptr_type = kmem_cache_type.get_type().pointer() + freeptr_t = utils.CachedType("freeptr_t") + freeptr_t_ptr = freeptr_t.get_type().pointer() + + track_type = gdb.lookup_type('struct track') + track_alloc = int(gdb.parse_and_eval('TRACK_ALLOC')) + track_free = int(gdb.parse_and_eval('TRACK_FREE')) + +def slab_folio(slab): + return slab.cast(gdb.lookup_type("struct folio").pointer()) + +def slab_address(slab): + p_ops = mm.page_ops().ops + folio = slab_folio(slab) + return p_ops.folio_address(folio) + +def for_each_object(cache, addr, slab_objects): + p = addr + if cache['flags'] & SLAB_RED_ZONE: + p += int(cache['red_left_pad']) + while p < addr + (slab_objects * cache['size']): + yield p + p = p + int(cache['size']) + +def get_info_end(cache): + if (cache['offset'] >= cache['inuse']): + return cache['inuse'] + gdb.lookup_type("void").pointer().sizeof + else: + return cache['inuse'] + +def get_orig_size(cache, obj): + if cache['flags'] & SLAB_STORE_USER and cache['flags'] & SLAB_KMALLOC: + p = mm.page_ops().ops.kasan_reset_tag(obj) + p += get_info_end(cache) + p += gdb.lookup_type('struct track').sizeof * 2 + p = p.cast(utils.get_uint_type().pointer()) + return p.dereference() + else: + return cache['object_size'] + +def get_track(cache, object_pointer, alloc): + p = object_pointer + get_info_end(cache) + p += (alloc * track_type.sizeof) + return p + +def oo_objects(x): + return int(x['x']) & OO_MASK + +def oo_order(x): + return int(x['x']) >> OO_SHIFT + +def reciprocal_divide(a, R): + t = (a * int(R['m'])) >> 32 + return (t + ((a - t) >> int(R['sh1']))) >> int(R['sh2']) + +def __obj_to_index(cache, addr, obj): + return reciprocal_divide(int(mm.page_ops().ops.kasan_reset_tag(obj)) - addr, cache['reciprocal_size']) + +def swab64(x): + result = (((x & 0x00000000000000ff) << 56) | \ + ((x & 0x000000000000ff00) << 40) | \ + ((x & 0x0000000000ff0000) << 24) | \ + ((x & 0x00000000ff000000) << 8) | \ + ((x & 0x000000ff00000000) >> 8) | \ + ((x & 0x0000ff0000000000) >> 24) | \ + ((x & 0x00ff000000000000) >> 40) | \ + ((x & 0xff00000000000000) >> 56)) + return result + +def freelist_ptr_decode(cache, ptr, ptr_addr): + if constants.LX_CONFIG_SLAB_FREELIST_HARDENED: + return ptr['v'] ^ cache['random'] ^ swab64(int(ptr_addr)) + else: + return ptr['v'] + +def get_freepointer(cache, obj): + obj = mm.page_ops().ops.kasan_reset_tag(obj) + ptr_addr = obj + cache['offset'] + p = ptr_addr.cast(freeptr_t_ptr).dereference() + return freelist_ptr_decode(cache, p, ptr_addr) + +def loc_exist(loc_track, addr, handle, waste): + for loc in loc_track: + if loc['addr'] == addr and loc['handle'] == handle and loc['waste'] == waste: + return loc + return None + +def add_location(loc_track, cache, track, orig_size): + jiffies = gdb.parse_and_eval("jiffies_64") + age = jiffies - track['when'] + handle = 0 + waste = cache['object_size'] - int(orig_size) + pid = int(track['pid']) + cpuid = int(track['cpu']) + addr = track['addr'] + if constants.LX_CONFIG_STACKDEPOT: + handle = track['handle'] + + loc = loc_exist(loc_track, addr, handle, waste) + if loc: + loc['count'] += 1 + if track['when']: + loc['sum_time'] += age + loc['min_time'] = min(loc['min_time'], age) + loc['max_time'] = max(loc['max_time'], age) + loc['min_pid'] = min(loc['min_pid'], pid) + loc['max_pid'] = max(loc['max_pid'], pid) + loc['cpus'].add(cpuid) + else: + loc_track.append({ + 'count' : 1, + 'addr' : addr, + 'sum_time' : age, + 'min_time' : age, + 'max_time' : age, + 'min_pid' : pid, + 'max_pid' : pid, + 'handle' : handle, + 'waste' : waste, + 'cpus' : {cpuid} + } + ) + +def slabtrace(alloc, cache_name): + + def __fill_map(obj_map, cache, slab): + p = slab['freelist'] + addr = slab_address(slab) + while p != gdb.Value(0): + index = __obj_to_index(cache, addr, p) + obj_map[index] = True # free objects + p = get_freepointer(cache, p) + + # process every slab page on the slab_list (partial and full list) + def process_slab(loc_track, slab_list, alloc, cache): + for slab in lists.list_for_each_entry(slab_list, slab_ptr_type, "slab_list"): + obj_map[:] = [False] * oo_objects(cache['oo']) + __fill_map(obj_map, cache, slab) + addr = slab_address(slab) + for object_pointer in for_each_object(cache, addr, slab['objects']): + if obj_map[__obj_to_index(cache, addr, object_pointer)] == True: + continue + p = get_track(cache, object_pointer, alloc) + track = gdb.Value(p).cast(track_type.pointer()) + if alloc == track_alloc: + size = get_orig_size(cache, object_pointer) + else: + size = cache['object_size'] + add_location(loc_track, cache, track, size) + continue + + slab_caches = gdb.parse_and_eval("slab_caches") + if mm.page_ops().ops.MAX_NUMNODES > 1: + nr_node_ids = int(gdb.parse_and_eval("nr_node_ids")) + else: + nr_node_ids = 1 + + target_cache = None + loc_track = [] + + for cache in lists.list_for_each_entry(slab_caches, kmem_cache_ptr_type, 'list'): + if cache['name'].string() == cache_name: + target_cache = cache + break + + obj_map = [False] * oo_objects(target_cache['oo']) + + if target_cache['flags'] & SLAB_STORE_USER: + for i in range(0, nr_node_ids): + cache_node = target_cache['node'][i] + if cache_node['nr_slabs']['counter'] == 0: + continue + process_slab(loc_track, cache_node['partial'], alloc, target_cache) + process_slab(loc_track, cache_node['full'], alloc, target_cache) + else: + raise gdb.GdbError("SLAB_STORE_USER is not set in %s" % target_cache['name'].string()) + + for loc in sorted(loc_track, key=lambda x:x['count'], reverse=True): + if loc['addr']: + addr = loc['addr'].cast(utils.get_ulong_type().pointer()) + gdb.write("%d %s" % (loc['count'], str(addr).split(' ')[-1])) + else: + gdb.write("%d <not-available>" % loc['count']) + + if loc['waste']: + gdb.write(" waste=%d/%d" % (loc['count'] * loc['waste'], loc['waste'])) + + if loc['sum_time'] != loc['min_time']: + gdb.write(" age=%d/%d/%d" % (loc['min_time'], loc['sum_time']/loc['count'], loc['max_time'])) + else: + gdb.write(" age=%d" % loc['min_time']) + + if loc['min_pid'] != loc['max_pid']: + gdb.write(" pid=%d-%d" % (loc['min_pid'], loc['max_pid'])) + else: + gdb.write(" pid=%d" % loc['min_pid']) + + if constants.LX_NR_CPUS > 1: + nr_cpu = gdb.parse_and_eval('__num_online_cpus')['counter'] + if nr_cpu > 1: + gdb.write(" cpus=") + for i in loc['cpus']: + gdb.write("%d," % i) + gdb.write("\n") + if constants.LX_CONFIG_STACKDEPOT: + if loc['handle']: + stackdepot.stack_depot_print(loc['handle']) + gdb.write("\n") + +def help(): + t = """Usage: lx-slabtrace --cache_name [cache_name] [Options] + Options: + --alloc + print information of allocation trace of the allocated objects + --free + print information of freeing trace of the allocated objects + Example: + lx-slabtrace --cache_name kmalloc-1k --alloc + lx-slabtrace --cache_name kmalloc-1k --free\n""" + gdb.write("Unrecognized command\n") + raise gdb.GdbError(t) + +class LxSlabTrace(gdb.Command): + """Show specific cache slabtrace""" + + def __init__(self): + super(LxSlabTrace, self).__init__("lx-slabtrace", gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + if not constants.LX_CONFIG_SLUB_DEBUG: + raise gdb.GdbError("CONFIG_SLUB_DEBUG is not enabled") + + argv = gdb.string_to_argv(arg) + alloc = track_alloc # default show alloc_traces + + if len(argv) == 3: + if argv[2] == '--alloc': + alloc = track_alloc + elif argv[2] == '--free': + alloc = track_free + else: + help() + if len(argv) >= 2 and argv[0] == '--cache_name': + slabtrace(alloc, argv[1]) + else: + help() +LxSlabTrace() + +def slabinfo(): + nr_node_ids = None + + if not constants.LX_CONFIG_SLUB_DEBUG: + raise gdb.GdbError("CONFIG_SLUB_DEBUG is not enabled") + + def count_free(slab): + total_free = 0 + for slab in lists.list_for_each_entry(slab, slab_ptr_type, 'slab_list'): + total_free += int(slab['objects'] - slab['inuse']) + return total_free + + gdb.write("{:^18} | {:^20} | {:^12} | {:^12} | {:^8} | {:^11} | {:^13}\n".format('Pointer', 'name', 'active_objs', 'num_objs', 'objsize', 'objperslab', 'pagesperslab')) + gdb.write("{:-^18} | {:-^20} | {:-^12} | {:-^12} | {:-^8} | {:-^11} | {:-^13}\n".format('', '', '', '', '', '', '')) + + slab_caches = gdb.parse_and_eval("slab_caches") + if mm.page_ops().ops.MAX_NUMNODES > 1: + nr_node_ids = int(gdb.parse_and_eval("nr_node_ids")) + else: + nr_node_ids = 1 + + for cache in lists.list_for_each_entry(slab_caches, kmem_cache_ptr_type, 'list'): + nr_objs = 0 + nr_free = 0 + nr_slabs = 0 + for i in range(0, nr_node_ids): + cache_node = cache['node'][i] + try: + nr_slabs += cache_node['nr_slabs']['counter'] + nr_objs = int(cache_node['total_objects']['counter']) + nr_free = count_free(cache_node['partial']) + except: + raise gdb.GdbError(traceback.format_exc()) + active_objs = nr_objs - nr_free + num_objs = nr_objs + active_slabs = nr_slabs + objects_per_slab = oo_objects(cache['oo']) + cache_order = oo_order(cache['oo']) + gdb.write("{:18s} | {:20.19s} | {:12} | {:12} | {:8} | {:11} | {:13}\n".format(hex(cache), cache['name'].string(), str(active_objs), str(num_objs), str(cache['size']), str(objects_per_slab), str(1 << cache_order))) + +class LxSlabInfo(gdb.Command): + """Show slabinfo""" + + def __init__(self): + super(LxSlabInfo, self).__init__("lx-slabinfo", gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + slabinfo() +LxSlabInfo() diff --git a/scripts/gdb/linux/stackdepot.py b/scripts/gdb/linux/stackdepot.py new file mode 100644 index 000000000000..047d329a6a12 --- /dev/null +++ b/scripts/gdb/linux/stackdepot.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +from linux import utils, constants + +if constants.LX_CONFIG_STACKDEPOT: + stack_record_type = utils.CachedType('struct stack_record') + DEPOT_STACK_ALIGN = 4 + +def stack_depot_fetch(handle): + global DEPOT_STACK_ALIGN + global stack_record_type + + stack_depot_disabled = gdb.parse_and_eval('stack_depot_disabled') + + if stack_depot_disabled: + raise gdb.GdbError("stack_depot_disabled\n") + + handle_parts_t = gdb.lookup_type("union handle_parts") + parts = handle.cast(handle_parts_t) + offset = parts['offset'] << DEPOT_STACK_ALIGN + pool_index_cached = gdb.parse_and_eval('pool_index') + + if parts['pool_index'] > pool_index_cached: + gdb.write("pool index %d out of bounds (%d) for stack id 0x%08x\n" % (parts['pool_index'], pool_index_cached, handle)) + return gdb.Value(0), 0 + + stack_pools = gdb.parse_and_eval('stack_pools') + + try: + pool = stack_pools[parts['pool_index']] + stack = (pool + gdb.Value(offset).cast(utils.get_size_t_type())).cast(stack_record_type.get_type().pointer()) + size = int(stack['size'].cast(utils.get_ulong_type())) + return stack['entries'], size + except Exception as e: + gdb.write("%s\n" % e) + return gdb.Value(0), 0 + +def stack_depot_print(handle): + if not constants.LX_CONFIG_STACKDEPOT: + raise gdb.GdbError("CONFIG_STACKDEPOT is not enabled") + + entries, nr_entries = stack_depot_fetch(handle) + if nr_entries > 0: + for i in range(0, nr_entries): + try: + gdb.execute("x /i 0x%x" % (int(entries[i]))) + except Exception as e: + gdb.write("%s\n" % e) diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index fdad3f32c747..5179edd1b627 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -89,29 +89,34 @@ lx-symbols command.""" return name return None - def _section_arguments(self, module): + def _section_arguments(self, module, module_addr): try: sect_attrs = module['sect_attrs'].dereference() except gdb.error: - return "" + return str(module_addr) + attrs = sect_attrs['attrs'] section_name_to_address = { attrs[n]['battr']['attr']['name'].string(): attrs[n]['address'] for n in range(int(sect_attrs['nsections']))} + + textaddr = section_name_to_address.get(".text", module_addr) args = [] for section_name in [".data", ".data..read_mostly", ".rodata", ".bss", - ".text", ".text.hot", ".text.unlikely"]: + ".text.hot", ".text.unlikely"]: address = section_name_to_address.get(section_name) if address: args.append(" -s {name} {addr}".format( name=section_name, addr=str(address))) - return "".join(args) + return "{textaddr} {sections}".format( + textaddr=textaddr, sections="".join(args)) - def load_module_symbols(self, module): + def load_module_symbols(self, module, module_file=None): module_name = module['name'].string() module_addr = str(module['mem'][constants.LX_MOD_TEXT]['base']).split()[0] - module_file = self._get_module_file(module_name) + if not module_file: + module_file = self._get_module_file(module_name) if not module_file and not self.module_files_updated: self._update_module_files() module_file = self._get_module_file(module_name) @@ -125,16 +130,28 @@ lx-symbols command.""" module_addr = hex(int(module_addr, 0) + plt_offset + plt_size) gdb.write("loading @{addr}: {filename}\n".format( addr=module_addr, filename=module_file)) - cmdline = "add-symbol-file {filename} {addr}{sections}".format( + cmdline = "add-symbol-file {filename} {sections}".format( filename=module_file, - addr=module_addr, - sections=self._section_arguments(module)) + sections=self._section_arguments(module, module_addr)) gdb.execute(cmdline, to_string=True) if module_name not in self.loaded_modules: self.loaded_modules.append(module_name) else: gdb.write("no module object found for '{0}'\n".format(module_name)) + def load_ko_symbols(self, mod_path): + self.loaded_modules = [] + module_list = modules.module_list() + + for module in module_list: + module_name = module['name'].string() + module_pattern = ".*/{0}\.ko(?:.debug)?$".format( + module_name.replace("_", r"[_\-]")) + if re.match(module_pattern, mod_path) and os.path.exists(mod_path): + self.load_module_symbols(module, mod_path) + return + raise gdb.GdbError("%s is not a valid .ko\n" % mod_path) + def load_all_symbols(self): gdb.write("loading vmlinux\n") @@ -173,6 +190,11 @@ lx-symbols command.""" self.module_files = [] self.module_files_updated = False + argv = gdb.string_to_argv(arg) + if len(argv) == 1: + self.load_ko_symbols(argv[0]) + return + self.load_all_symbols() if hasattr(gdb, 'Breakpoint'): diff --git a/scripts/gdb/linux/utils.py b/scripts/gdb/linux/utils.py index 9f44df13761e..7d5278d815fa 100644 --- a/scripts/gdb/linux/utils.py +++ b/scripts/gdb/linux/utils.py @@ -35,12 +35,32 @@ class CachedType: long_type = CachedType("long") +ulong_type = CachedType("unsigned long") +uint_type = CachedType("unsigned int") atomic_long_type = CachedType("atomic_long_t") +size_t_type = CachedType("size_t") +struct_page_type = CachedType("struct page") + +def get_uint_type(): + global uint_type + return uint_type.get_type() + +def get_page_type(): + global struct_page_type + return struct_page_type.get_type() def get_long_type(): global long_type return long_type.get_type() +def get_ulong_type(): + global ulong_type + return ulong_type.get_type() + +def get_size_t_type(): + global size_t_type + return size_t_type.get_type() + def offset_of(typeobj, field): element = gdb.Value(0).cast(typeobj) return int(str(element[field].address).split()[0], 16) diff --git a/scripts/gdb/linux/vmalloc.py b/scripts/gdb/linux/vmalloc.py new file mode 100644 index 000000000000..48e4a4fae7bb --- /dev/null +++ b/scripts/gdb/linux/vmalloc.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2023 MediaTek Inc. +# +# Authors: +# Kuan-Ying Lee <Kuan-Ying.Lee@mediatek.com> +# + +import gdb +import re +from linux import lists, utils, stackdepot, constants, mm + +vmap_area_type = utils.CachedType('struct vmap_area') +vmap_area_ptr_type = vmap_area_type.get_type().pointer() + +def is_vmalloc_addr(x): + pg_ops = mm.page_ops().ops + addr = pg_ops.kasan_reset_tag(x) + return addr >= pg_ops.VMALLOC_START and addr < pg_ops.VMALLOC_END + +class LxVmallocInfo(gdb.Command): + """Show vmallocinfo""" + + def __init__(self): + super(LxVmallocInfo, self).__init__("lx-vmallocinfo", gdb.COMMAND_DATA) + + def invoke(self, arg, from_tty): + vmap_area_list = gdb.parse_and_eval('vmap_area_list') + for vmap_area in lists.list_for_each_entry(vmap_area_list, vmap_area_ptr_type, "list"): + if not vmap_area['vm']: + gdb.write("0x%x-0x%x %10d vm_map_ram\n" % (vmap_area['va_start'], vmap_area['va_end'], + vmap_area['va_end'] - vmap_area['va_start'])) + continue + v = vmap_area['vm'] + gdb.write("0x%x-0x%x %10d" % (v['addr'], v['addr'] + v['size'], v['size'])) + if v['caller']: + gdb.write(" %s" % str(v['caller']).split(' ')[-1]) + if v['nr_pages']: + gdb.write(" pages=%d" % v['nr_pages']) + if v['phys_addr']: + gdb.write(" phys=0x%x" % v['phys_addr']) + if v['flags'] & constants.LX_VM_IOREMAP: + gdb.write(" ioremap") + if v['flags'] & constants.LX_VM_ALLOC: + gdb.write(" vmalloc") + if v['flags'] & constants.LX_VM_MAP: + gdb.write(" vmap") + if v['flags'] & constants.LX_VM_USERMAP: + gdb.write(" user") + if v['flags'] & constants.LX_VM_DMA_COHERENT: + gdb.write(" dma-coherent") + if is_vmalloc_addr(v['pages']): + gdb.write(" vpages") + gdb.write("\n") + +LxVmallocInfo() diff --git a/scripts/gdb/vmlinux-gdb.py b/scripts/gdb/vmlinux-gdb.py index 2d32308c3f7a..fc53cdf286f1 100644 --- a/scripts/gdb/vmlinux-gdb.py +++ b/scripts/gdb/vmlinux-gdb.py @@ -41,6 +41,11 @@ else: import linux.genpd import linux.device import linux.vfs - import linux.mm + import linux.pgtable import linux.radixtree import linux.interrupts + import linux.mm + import linux.stackdepot + import linux.page_owner + import linux.slab + import linux.vmalloc diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py index 946e250c1b2a..fc52bc41d3e7 100755 --- a/scripts/generate_rust_analyzer.py +++ b/scripts/generate_rust_analyzer.py @@ -6,10 +6,19 @@ import argparse import json import logging +import os import pathlib import sys -def generate_crates(srctree, objtree, sysroot_src): +def args_crates_cfgs(cfgs): + crates_cfgs = {} + for cfg in cfgs: + crate, vals = cfg.split("=", 1) + crates_cfgs[crate] = vals.replace("--cfg", "").split() + + return crates_cfgs + +def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs): # Generate the configuration list. cfg = [] with open(objtree / "include" / "generated" / "rustc_cfg") as fd: @@ -23,6 +32,7 @@ def generate_crates(srctree, objtree, sysroot_src): # Avoid O(n^2) iterations by keeping a map of indexes. crates = [] crates_indexes = {} + crates_cfgs = args_crates_cfgs(cfgs) def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False): crates_indexes[display_name] = len(crates) @@ -44,6 +54,7 @@ def generate_crates(srctree, objtree, sysroot_src): "core", sysroot_src / "core" / "src" / "lib.rs", [], + cfg=crates_cfgs.get("core", []), is_workspace_member=False, ) @@ -57,6 +68,7 @@ def generate_crates(srctree, objtree, sysroot_src): "alloc", srctree / "rust" / "alloc" / "lib.rs", ["core", "compiler_builtins"], + cfg=crates_cfgs.get("alloc", []), ) append_crate( @@ -65,7 +77,7 @@ def generate_crates(srctree, objtree, sysroot_src): [], is_proc_macro=True, ) - crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so" + crates[-1]["proc_macro_dylib_path"] = f"{objtree}/rust/libmacros.so" append_crate( "build_error", @@ -95,19 +107,26 @@ def generate_crates(srctree, objtree, sysroot_src): "exclude_dirs": [], } + def is_root_crate(build_file, target): + try: + return f"{target}.o" in open(build_file).read() + except FileNotFoundError: + return False + # Then, the rest outside of `rust/`. # # We explicitly mention the top-level folders we want to cover. - for folder in ("samples", "drivers"): - for path in (srctree / folder).rglob("*.rs"): + extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers")) + if external_src is not None: + extra_dirs = [external_src] + for folder in extra_dirs: + for path in folder.rglob("*.rs"): logging.info("Checking %s", path) name = path.name.replace(".rs", "") # Skip those that are not crate roots. - try: - if f"{name}.o" not in open(path.parent / "Makefile").read(): - continue - except FileNotFoundError: + if not is_root_crate(path.parent / "Makefile", name) and \ + not is_root_crate(path.parent / "Kbuild", name): continue logging.info("Adding %s", name) @@ -123,9 +142,11 @@ def generate_crates(srctree, objtree, sysroot_src): def main(): parser = argparse.ArgumentParser() parser.add_argument('--verbose', '-v', action='store_true') + parser.add_argument('--cfgs', action='append', default=[]) parser.add_argument("srctree", type=pathlib.Path) parser.add_argument("objtree", type=pathlib.Path) parser.add_argument("sysroot_src", type=pathlib.Path) + parser.add_argument("exttree", type=pathlib.Path, nargs="?") args = parser.parse_args() logging.basicConfig( @@ -134,7 +155,7 @@ def main(): ) rust_project = { - "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src), + "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src, args.exttree, args.cfgs), "sysroot_src": str(args.sysroot_src), } diff --git a/scripts/headers_install.sh b/scripts/headers_install.sh index 36b56b746fce..afdddc82f02b 100755 --- a/scripts/headers_install.sh +++ b/scripts/headers_install.sh @@ -76,7 +76,6 @@ arch/arc/include/uapi/asm/swab.h:CONFIG_ARC_HAS_SWAPE arch/arm/include/uapi/asm/ptrace.h:CONFIG_CPU_ENDIAN_BE8 arch/hexagon/include/uapi/asm/ptrace.h:CONFIG_HEXAGON_ARCH_VERSION arch/hexagon/include/uapi/asm/user.h:CONFIG_HEXAGON_ARCH_VERSION -arch/ia64/include/uapi/asm/cmpxchg.h:CONFIG_IA64_DEBUG_CMPXCHG arch/m68k/include/uapi/asm/ptrace.h:CONFIG_COLDFIRE arch/nios2/include/uapi/asm/swab.h:CONFIG_NIOS2_CI_SWAB_NO arch/nios2/include/uapi/asm/swab.h:CONFIG_NIOS2_CI_SWAB_SUPPORT diff --git a/scripts/is_rust_module.sh b/scripts/is_rust_module.sh deleted file mode 100755 index 464761a7cf7f..000000000000 --- a/scripts/is_rust_module.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: GPL-2.0 -# -# is_rust_module.sh module.ko -# -# Returns `0` if `module.ko` is a Rust module, `1` otherwise. - -set -e - -# Using the `16_` prefix ensures other symbols with the same substring -# are not picked up (even if it would be unlikely). The last part is -# used just in case LLVM decides to use the `.` suffix. -# -# In the future, checking for the `.comment` section may be another -# option, see https://github.com/rust-lang/rust/pull/97550. -${NM} "$*" | grep -qE '^[0-9a-fA-F]+ [Rr] _R[^[:space:]]+16___IS_RUST_MODULE[^[:space:]]*$' diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 16c87938b316..653b92f6d4c8 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -129,6 +129,7 @@ static struct sym_entry *read_symbol(FILE *in, char **buf, size_t *buf_len) ssize_t readlen; struct sym_entry *sym; + errno = 0; readlen = getline(buf, buf_len, in); if (readlen < 0) { if (errno) { diff --git a/scripts/kernel-doc b/scripts/kernel-doc index d0116c6939dc..6e199a745ccb 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1168,6 +1168,10 @@ sub dump_struct($$) { $members =~ s/DECLARE_KFIFO_PTR\s*\($args,\s*$args\)/$2 \*$1/gos; # replace DECLARE_FLEX_ARRAY $members =~ s/(?:__)?DECLARE_FLEX_ARRAY\s*\($args,\s*$args\)/$1 $2\[\]/gos; + #replace DEFINE_DMA_UNMAP_ADDR + $members =~ s/DEFINE_DMA_UNMAP_ADDR\s*\($args\)/dma_addr_t $1/gos; + #replace DEFINE_DMA_UNMAP_LEN + $members =~ s/DEFINE_DMA_UNMAP_LEN\s*\($args\)/__u32 $1/gos; my $declaration = $members; # Split nested struct/union elements as newer ones @@ -1349,6 +1353,7 @@ sub dump_enum($$) { my %_members; $members =~ s/\s+$//; + $members =~ s/\([^;]*?[\)]//g; foreach my $arg (split ',', $members) { $arg =~ s/^\s*(\w+).*/$1/; diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index 2ade63149466..d65ab8bfeaf4 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -31,10 +31,10 @@ llvm) fi ;; rustc) - echo 1.68.2 + echo 1.71.1 ;; bindgen) - echo 0.56.0 + echo 0.65.1 ;; *) echo "$1: unknown tool" >&2 diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index aebbf1913970..117018946b57 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -2,8 +2,6 @@ # SPDX-License-Identifier: GPL-2.0 # # Tests whether a suitable Rust toolchain is available. -# -# Pass `-v` for human output and more checks (as warnings). set -e @@ -21,102 +19,208 @@ get_canonical_version() echo $((100000 * $1 + 100 * $2 + $3)) } +# Print a reference to the Quick Start guide in the documentation. +print_docs_reference() +{ + echo >&2 "***" + echo >&2 "*** Please see Documentation/rust/quick-start.rst for details" + echo >&2 "*** on how to set up the Rust support." + echo >&2 "***" +} + +# Print an explanation about the fact that the script is meant to be called from Kbuild. +print_kbuild_explanation() +{ + echo >&2 "***" + echo >&2 "*** This script is intended to be called from Kbuild." + echo >&2 "*** Please use the 'rustavailable' target to call it instead." + echo >&2 "*** Otherwise, the results may not be meaningful." + exit 1 +} + +# If the script fails for any reason, or if there was any warning, then +# print a reference to the documentation on exit. +warning=0 +trap 'if [ $? -ne 0 ] || [ $warning -ne 0 ]; then print_docs_reference; fi' EXIT + +# Check that the expected environment variables are set. +if [ -z "${RUSTC+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'RUSTC' is not set." + print_kbuild_explanation +fi + +if [ -z "${BINDGEN+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'BINDGEN' is not set." + print_kbuild_explanation +fi + +if [ -z "${CC+x}" ]; then + echo >&2 "***" + echo >&2 "*** Environment variable 'CC' is not set." + print_kbuild_explanation +fi + # Check that the Rust compiler exists. if ! command -v "$RUSTC" >/dev/null; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust compiler '$RUSTC' could not be found." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' could not be found." + echo >&2 "***" exit 1 fi # Check that the Rust bindings generator exists. if ! command -v "$BINDGEN" >/dev/null; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found." + echo >&2 "***" exit 1 fi # Check that the Rust compiler version is suitable. # # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_compiler_output=$( \ + LC_ALL=C "$RUSTC" --version 2>/dev/null +) || rust_compiler_code=$? +if [ -n "$rust_compiler_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$RUSTC' to check the Rust compiler version failed with" + echo >&2 "*** code $rust_compiler_code. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_compiler_output" + echo >&2 "***" + exit 1 +fi rust_compiler_version=$( \ - LC_ALL=C "$RUSTC" --version 2>/dev/null \ - | head -n 1 \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + echo "$rust_compiler_output" \ + | sed -nE '1s:.*rustc ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$rust_compiler_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$RUSTC' to check the Rust compiler version did not return" + echo >&2 "*** an expected output. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_compiler_output" + echo >&2 "***" + exit 1 +fi rust_compiler_min_version=$($min_tool_version rustc) rust_compiler_cversion=$(get_canonical_version $rust_compiler_version) rust_compiler_min_cversion=$(get_canonical_version $rust_compiler_min_version) if [ "$rust_compiler_cversion" -lt "$rust_compiler_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust compiler '$RUSTC' is too old." - echo >&2 "*** Your version: $rust_compiler_version" - echo >&2 "*** Minimum version: $rust_compiler_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust compiler '$RUSTC' is too old." + echo >&2 "*** Your version: $rust_compiler_version" + echo >&2 "*** Minimum version: $rust_compiler_min_version" + echo >&2 "***" exit 1 fi -if [ "$1" = -v ] && [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then +if [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then echo >&2 "***" echo >&2 "*** Rust compiler '$RUSTC' is too new. This may or may not work." echo >&2 "*** Your version: $rust_compiler_version" echo >&2 "*** Expected version: $rust_compiler_min_version" echo >&2 "***" + warning=1 fi # Check that the Rust bindings generator is suitable. # # Non-stable and distributions' versions may have a version suffix, e.g. `-dev`. +rust_bindings_generator_output=$( \ + LC_ALL=C "$BINDGEN" --version 2>/dev/null +) || rust_bindings_generator_code=$? +if [ -n "$rust_bindings_generator_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the Rust bindings generator version failed with" + echo >&2 "*** code $rust_bindings_generator_code. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_bindings_generator_output" + echo >&2 "***" + exit 1 +fi rust_bindings_generator_version=$( \ - LC_ALL=C "$BINDGEN" --version 2>/dev/null \ - | head -n 1 \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ + echo "$rust_bindings_generator_output" \ + | sed -nE '1s:.*bindgen ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$rust_bindings_generator_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the bindings generator version did not return" + echo >&2 "*** an expected output. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$rust_bindings_generator_output" + echo >&2 "***" + exit 1 +fi rust_bindings_generator_min_version=$($min_tool_version bindgen) rust_bindings_generator_cversion=$(get_canonical_version $rust_bindings_generator_version) rust_bindings_generator_min_cversion=$(get_canonical_version $rust_bindings_generator_min_version) if [ "$rust_bindings_generator_cversion" -lt "$rust_bindings_generator_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Rust bindings generator '$BINDGEN' is too old." - echo >&2 "*** Your version: $rust_bindings_generator_version" - echo >&2 "*** Minimum version: $rust_bindings_generator_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' is too old." + echo >&2 "*** Your version: $rust_bindings_generator_version" + echo >&2 "*** Minimum version: $rust_bindings_generator_min_version" + echo >&2 "***" exit 1 fi -if [ "$1" = -v ] && [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then +if [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then echo >&2 "***" echo >&2 "*** Rust bindings generator '$BINDGEN' is too new. This may or may not work." echo >&2 "*** Your version: $rust_bindings_generator_version" echo >&2 "*** Expected version: $rust_bindings_generator_min_version" echo >&2 "***" + warning=1 fi # Check that the `libclang` used by the Rust bindings generator is suitable. +# +# In order to do that, first invoke `bindgen` to get the `libclang` version +# found by `bindgen`. This step may already fail if, for instance, `libclang` +# is not found, thus inform the user in such a case. +bindgen_libclang_output=$( \ + LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null +) || bindgen_libclang_code=$? +if [ -n "$bindgen_libclang_code" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the libclang version (used by the Rust" + echo >&2 "*** bindings generator) failed with code $bindgen_libclang_code. This may be caused by" + echo >&2 "*** a failure to locate libclang. See output and docs below for details:" + echo >&2 "***" + echo >&2 "$bindgen_libclang_output" + echo >&2 "***" + exit 1 +fi + +# `bindgen` returned successfully, thus use the output to check that the version +# of the `libclang` found by the Rust bindings generator is suitable. +# +# Unlike other version checks, note that this one does not necessarily appear +# in the first line of the output, thus no `sed` address is provided. bindgen_libclang_version=$( \ - LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null \ - | grep -F 'clang version ' \ - | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \ - | head -n 1 \ + echo "$bindgen_libclang_output" \ + | sed -nE 's:.*clang version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' ) +if [ -z "$bindgen_libclang_version" ]; then + echo >&2 "***" + echo >&2 "*** Running '$BINDGEN' to check the libclang version (used by the Rust" + echo >&2 "*** bindings generator) did not return an expected output. See output" + echo >&2 "*** and docs below for details:" + echo >&2 "***" + echo >&2 "$bindgen_libclang_output" + echo >&2 "***" + exit 1 +fi bindgen_libclang_min_version=$($min_tool_version llvm) bindgen_libclang_cversion=$(get_canonical_version $bindgen_libclang_version) bindgen_libclang_min_cversion=$(get_canonical_version $bindgen_libclang_min_version) if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old." - echo >&2 "*** Your version: $bindgen_libclang_version" - echo >&2 "*** Minimum version: $bindgen_libclang_min_version" - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old." + echo >&2 "*** Your version: $bindgen_libclang_version" + echo >&2 "*** Minimum version: $bindgen_libclang_min_version" + echo >&2 "***" exit 1 fi @@ -125,21 +229,20 @@ fi # # In the future, we might be able to perform a full version check, see # https://github.com/rust-lang/rust-bindgen/issues/2138. -if [ "$1" = -v ]; then - cc_name=$($(dirname $0)/cc-version.sh "$CC" | cut -f1 -d' ') - if [ "$cc_name" = Clang ]; then - clang_version=$( \ - LC_ALL=C "$CC" --version 2>/dev/null \ - | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' - ) - if [ "$clang_version" != "$bindgen_libclang_version" ]; then - echo >&2 "***" - echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')" - echo >&2 "*** version does not match Clang's. This may be a problem." - echo >&2 "*** libclang version: $bindgen_libclang_version" - echo >&2 "*** Clang version: $clang_version" - echo >&2 "***" - fi +cc_name=$($(dirname $0)/cc-version.sh $CC | cut -f1 -d' ') +if [ "$cc_name" = Clang ]; then + clang_version=$( \ + LC_ALL=C $CC --version 2>/dev/null \ + | sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p' + ) + if [ "$clang_version" != "$bindgen_libclang_version" ]; then + echo >&2 "***" + echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')" + echo >&2 "*** version does not match Clang's. This may be a problem." + echo >&2 "*** libclang version: $bindgen_libclang_version" + echo >&2 "*** Clang version: $clang_version" + echo >&2 "***" + warning=1 fi fi @@ -150,11 +253,9 @@ rustc_sysroot=$("$RUSTC" $KRUSTFLAGS --print sysroot) rustc_src=${RUST_LIB_SRC:-"$rustc_sysroot/lib/rustlib/src/rust/library"} rustc_src_core="$rustc_src/core/src/lib.rs" if [ ! -e "$rustc_src_core" ]; then - if [ "$1" = -v ]; then - echo >&2 "***" - echo >&2 "*** Source code for the 'core' standard library could not be found" - echo >&2 "*** at '$rustc_src_core'." - echo >&2 "***" - fi + echo >&2 "***" + echo >&2 "*** Source code for the 'core' standard library could not be found" + echo >&2 "*** at '$rustc_src_core'." + echo >&2 "***" exit 1 fi diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py new file mode 100755 index 000000000000..57613fe5ed75 --- /dev/null +++ b/scripts/rust_is_available_test.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +"""Tests the `rust_is_available.sh` script. + +Some of the tests require the real programs to be available in `$PATH` +under their canonical name (and with the expected versions). +""" + +import enum +import os +import pathlib +import stat +import subprocess +import tempfile +import unittest + +class TestRustIsAvailable(unittest.TestCase): + @enum.unique + class Expected(enum.Enum): + SUCCESS = enum.auto() + SUCCESS_WITH_WARNINGS = enum.auto() + SUCCESS_WITH_EXTRA_OUTPUT = enum.auto() + FAILURE = enum.auto() + + @classmethod + def generate_executable(cls, content): + path = pathlib.Path(cls.tempdir.name) + name = str(len(tuple(path.iterdir()))) + path = path / name + with open(path, "w") as file_: + file_.write(content) + os.chmod(path, os.stat(path).st_mode | stat.S_IXUSR) + return path + + @classmethod + def generate_clang(cls, stdout): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "-E" in " ".join(sys.argv): + print({repr("Clang " + " ".join(cls.llvm_default_version.split(" ")))}) +else: + print({repr(stdout)}) +""") + + @classmethod + def generate_rustc(cls, stdout): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "--print sysroot" in " ".join(sys.argv): + print({repr(cls.rust_default_sysroot)}) +else: + print({repr(stdout)}) +""") + + @classmethod + def generate_bindgen(cls, version_stdout, libclang_stderr): + return cls.generate_executable(f"""#!/usr/bin/env python3 +import sys +if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): + print({repr(libclang_stderr)}, file=sys.stderr) +else: + print({repr(version_stdout)}) +""") + + @classmethod + def generate_bindgen_version(cls, stdout): + return cls.generate_bindgen(stdout, cls.bindgen_default_bindgen_libclang_stderr) + + @classmethod + def generate_bindgen_libclang(cls, stderr): + return cls.generate_bindgen(cls.bindgen_default_bindgen_version_stdout, stderr) + + @classmethod + def setUpClass(cls): + cls.tempdir = tempfile.TemporaryDirectory() + + cls.missing = pathlib.Path(cls.tempdir.name) / "missing" + + cls.nonexecutable = pathlib.Path(cls.tempdir.name) / "nonexecutable" + with open(cls.nonexecutable, "w") as file_: + file_.write("nonexecutable") + + cls.unexpected_binary = "true" + + cls.rustc_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "rustc")).decode().strip() + cls.bindgen_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "bindgen")).decode().strip() + cls.llvm_default_version = subprocess.check_output(("scripts/min-tool-version.sh", "llvm")).decode().strip() + cls.rust_default_sysroot = subprocess.check_output(("rustc", "--print", "sysroot")).decode().strip() + + cls.bindgen_default_bindgen_version_stdout = f"bindgen {cls.bindgen_default_version}" + cls.bindgen_default_bindgen_libclang_stderr = f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {cls.llvm_default_version} [-W#pragma-messages], err: false" + + cls.default_rustc = cls.generate_rustc(f"rustc {cls.rustc_default_version}") + cls.default_bindgen = cls.generate_bindgen(cls.bindgen_default_bindgen_version_stdout, cls.bindgen_default_bindgen_libclang_stderr) + cls.default_cc = cls.generate_clang(f"clang version {cls.llvm_default_version}") + + def run_script(self, expected, override_env): + env = { + "RUSTC": self.default_rustc, + "BINDGEN": self.default_bindgen, + "CC": self.default_cc, + } + + for key, value in override_env.items(): + if value is None: + del env[key] + continue + env[key] = value + + result = subprocess.run("scripts/rust_is_available.sh", env=env, capture_output=True) + + # The script should never output anything to `stdout`. + self.assertEqual(result.stdout, b"") + + if expected == self.Expected.SUCCESS: + # When expecting a success, the script should return 0 + # and it should not output anything to `stderr`. + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_EXTRA_OUTPUT: + # When expecting a success with extra output (that is not warnings, + # which is the common case), the script should return 0 and it + # should output at least something to `stderr` (the output should + # be checked further by the test). + self.assertEqual(result.returncode, 0) + self.assertNotEqual(result.stderr, b"") + elif expected == self.Expected.SUCCESS_WITH_WARNINGS: + # When expecting a success with warnings, the script should return 0 + # and it should output at least the instructions to `stderr`. + self.assertEqual(result.returncode, 0) + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) + else: + # When expecting a failure, the script should return non-0 + # and it should output at least the instructions to `stderr`. + self.assertNotEqual(result.returncode, 0) + self.assertIn(b"Please see Documentation/rust/quick-start.rst for details", result.stderr) + + # The output will generally be UTF-8 (i.e. unless the user has + # put strange values in the environment). + result.stderr = result.stderr.decode() + + return result + + def test_rustc_unset(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": None }) + self.assertIn("Environment variable 'RUSTC' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_bindgen_unset(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": None }) + self.assertIn("Environment variable 'BINDGEN' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_cc_unset(self): + result = self.run_script(self.Expected.FAILURE, { "CC": None }) + self.assertIn("Environment variable 'CC' is not set.", result.stderr) + self.assertIn("This script is intended to be called from Kbuild.", result.stderr) + + def test_rustc_missing(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.missing }) + self.assertIn(f"Rust compiler '{self.missing}' could not be found.", result.stderr) + + def test_bindgen_missing(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.missing }) + self.assertIn(f"Rust bindings generator '{self.missing}' could not be found.", result.stderr) + + def test_rustc_nonexecutable(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.nonexecutable }) + self.assertIn(f"Running '{self.nonexecutable}' to check the Rust compiler version failed with", result.stderr) + + def test_rustc_unexpected_binary(self): + result = self.run_script(self.Expected.FAILURE, { "RUSTC": self.unexpected_binary }) + self.assertIn(f"Running '{self.unexpected_binary}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_unexpected_name(self): + rustc = self.generate_rustc(f"unexpected {self.rustc_default_version} (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_unexpected_version(self): + rustc = self.generate_rustc("rustc unexpected (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_no_minor(self): + rustc = self.generate_rustc(f"rustc {'.'.join(self.rustc_default_version.split('.')[:2])} (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Running '{rustc}' to check the Rust compiler version did not return", result.stderr) + + def test_rustc_old_version(self): + rustc = self.generate_rustc("rustc 1.60.0 (a8314ef7d 2022-06-27)") + result = self.run_script(self.Expected.FAILURE, { "RUSTC": rustc }) + self.assertIn(f"Rust compiler '{rustc}' is too old.", result.stderr) + + def test_rustc_new_version(self): + rustc = self.generate_rustc("rustc 1.999.0 (a8314ef7d 2099-06-27)") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "RUSTC": rustc }) + self.assertIn(f"Rust compiler '{rustc}' is too new. This may or may not work.", result.stderr) + + def test_bindgen_nonexecutable(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.nonexecutable }) + self.assertIn(f"Running '{self.nonexecutable}' to check the Rust bindings generator version failed with", result.stderr) + + def test_bindgen_unexpected_binary(self): + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": self.unexpected_binary }) + self.assertIn(f"Running '{self.unexpected_binary}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_unexpected_name(self): + bindgen = self.generate_bindgen_version(f"unexpected {self.bindgen_default_version}") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_unexpected_version(self): + bindgen = self.generate_bindgen_version("bindgen unexpected") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_no_minor(self): + bindgen = self.generate_bindgen_version(f"bindgen {'.'.join(self.bindgen_default_version.split('.')[:2])}") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the bindings generator version did not return", result.stderr) + + def test_bindgen_old_version(self): + bindgen = self.generate_bindgen_version("bindgen 0.50.0") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Rust bindings generator '{bindgen}' is too old.", result.stderr) + + def test_bindgen_new_version(self): + bindgen = self.generate_bindgen_version("bindgen 0.999.0") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn(f"Rust bindings generator '{bindgen}' is too new. This may or may not work.", result.stderr) + + def test_bindgen_libclang_failure(self): + for env in ( + { "LLVM_CONFIG_PATH": self.missing }, + { "LIBCLANG_PATH": self.missing }, + { "CLANG_PATH": self.missing }, + ): + with self.subTest(env=env): + result = self.run_script(self.Expected.FAILURE, env | { "PATH": os.environ["PATH"], "BINDGEN": "bindgen" }) + self.assertIn("Running 'bindgen' to check the libclang version (used by the Rust", result.stderr) + self.assertIn("bindings generator) failed with code ", result.stderr) + + def test_bindgen_libclang_unexpected_version(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version unexpected [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"Running '{bindgen}' to check the libclang version (used by the Rust", result.stderr) + self.assertIn("bindings generator) did not return an expected output. See output", result.stderr) + + def test_bindgen_libclang_old_version(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 10.0.0 [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.FAILURE, { "BINDGEN": bindgen }) + self.assertIn(f"libclang (used by the Rust bindings generator '{bindgen}') is too old.", result.stderr) + + def test_clang_matches_bindgen_libclang_different_bindgen(self): + bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 999.0.0 [-W#pragma-messages], err: false") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) + self.assertIn("version does not match Clang's. This may be a problem.", result.stderr) + + def test_clang_matches_bindgen_libclang_different_clang(self): + cc = self.generate_clang("clang version 999.0.0") + result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "CC": cc }) + self.assertIn("version does not match Clang's. This may be a problem.", result.stderr) + + def test_rustc_src_core_krustflags(self): + result = self.run_script(self.Expected.FAILURE, { "PATH": os.environ["PATH"], "RUSTC": "rustc", "KRUSTFLAGS": f"--sysroot={self.missing}" }) + self.assertIn("Source code for the 'core' standard library could not be found", result.stderr) + + def test_rustc_src_core_rustlibsrc(self): + result = self.run_script(self.Expected.FAILURE, { "RUST_LIB_SRC": self.missing }) + self.assertIn("Source code for the 'core' standard library could not be found", result.stderr) + + def test_success_cc_unknown(self): + result = self.run_script(self.Expected.SUCCESS_WITH_EXTRA_OUTPUT, { "CC": self.missing }) + self.assertIn("unknown C compiler", result.stderr) + + def test_success_cc_multiple_arguments_ccache(self): + clang = self.generate_clang(f"""Ubuntu clang version {self.llvm_default_version}-1ubuntu1 +Target: x86_64-pc-linux-gnu +Thread model: posix +InstalledDir: /usr/bin +""") + result = self.run_script(self.Expected.SUCCESS, { "CC": f"{clang} clang" }) + + def test_success_rustc_version(self): + for rustc_stdout in ( + f"rustc {self.rustc_default_version} (a8314ef7d 2022-06-27)", + f"rustc {self.rustc_default_version}-dev (a8314ef7d 2022-06-27)", + f"rustc {self.rustc_default_version}-1.60.0 (a8314ef7d 2022-06-27)", + ): + with self.subTest(rustc_stdout=rustc_stdout): + rustc = self.generate_rustc(rustc_stdout) + result = self.run_script(self.Expected.SUCCESS, { "RUSTC": rustc }) + + def test_success_bindgen_version(self): + for bindgen_stdout in ( + f"bindgen {self.bindgen_default_version}", + f"bindgen {self.bindgen_default_version}-dev", + f"bindgen {self.bindgen_default_version}-0.999.0", + ): + with self.subTest(bindgen_stdout=bindgen_stdout): + bindgen = self.generate_bindgen_version(bindgen_stdout) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_success_bindgen_libclang(self): + for stderr in ( + f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1) [-W#pragma-messages], err: false", + f"/home/jd/Documents/dev/kernel-module-flake/linux-6.1/outputs/dev/lib/modules/6.1.0-development/source/scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} [-W#pragma-messages], err: false", + f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (Fedora 13.0.0-3.fc35) [-W#pragma-messages], err: false", + f""" +/nix/store/dsd5gz46hdbdk2rfdimqddhq6m8m8fqs-bash-5.1-p16/bin/bash: warning: setlocale: LC_ALL: cannot change locale (c) +scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} [-W#pragma-messages], err: false +""", + f""" +/nix/store/dsd5gz46hdbdk2rfdimqddhq6m8m8fqs-bash-5.1.0-p16/bin/bash: warning: setlocale: LC_ALL: cannot change locale (c) +/home/jd/Documents/dev/kernel-module-flake/linux-6.1/outputs/dev/lib/modules/6.1.0-development/source/scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {self.llvm_default_version} (Fedora 13.0.0-3.fc35) [-W#pragma-messages], err: false +""" + ): + with self.subTest(stderr=stderr): + bindgen = self.generate_bindgen_libclang(stderr) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen }) + + def test_success_clang_version(self): + for clang_stdout in ( + f"clang version {self.llvm_default_version} (https://github.com/llvm/llvm-project.git 4a2c05b05ed07f1f620e94f6524a8b4b2760a0b1)", + f"clang version {self.llvm_default_version}-dev", + f"clang version {self.llvm_default_version}-2~ubuntu20.04.1", + f"Ubuntu clang version {self.llvm_default_version}-2~ubuntu20.04.1", + ): + with self.subTest(clang_stdout=clang_stdout): + clang = self.generate_clang(clang_stdout) + result = self.run_script(self.Expected.SUCCESS, { "CC": clang }) + + def test_success_real_programs(self): + for cc in ["gcc", "clang"]: + with self.subTest(cc=cc): + result = self.run_script(self.Expected.SUCCESS, { + "PATH": os.environ["PATH"], + "RUSTC": "rustc", + "BINDGEN": "bindgen", + "CC": cc, + }) + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/rustdoc_test_builder.rs b/scripts/rustdoc_test_builder.rs new file mode 100644 index 000000000000..e5894652f12c --- /dev/null +++ b/scripts/rustdoc_test_builder.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Test builder for `rustdoc`-generated tests. +//! +//! This script is a hack to extract the test from `rustdoc`'s output. Ideally, `rustdoc` would +//! have an option to generate this information instead, e.g. as JSON output. +//! +//! The `rustdoc`-generated test names look like `{file}_{line}_{number}`, e.g. +//! `...path_rust_kernel_sync_arc_rs_42_0`. `number` is the "test number", needed in cases like +//! a macro that expands into items with doctests is invoked several times within the same line. +//! +//! However, since these names are used for bisection in CI, the line number makes it not stable +//! at all. In the future, we would like `rustdoc` to give us the Rust item path associated with +//! the test, plus a "test number" (for cases with several examples per item) and generate a name +//! from that. For the moment, we generate ourselves a new name, `{file}_{number}` instead, in +//! the `gen` script (done there since we need to be aware of all the tests in a given file). + +use std::io::Read; + +fn main() { + let mut stdin = std::io::stdin().lock(); + let mut body = String::new(); + stdin.read_to_string(&mut body).unwrap(); + + // Find the generated function name looking for the inner function inside `main()`. + // + // The line we are looking for looks like one of the following: + // + // ``` + // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_28_0() { + // fn main() { #[allow(non_snake_case)] fn _doctest_main_rust_kernel_file_rs_37_0() -> Result<(), impl core::fmt::Debug> { + // ``` + // + // It should be unlikely that doctest code matches such lines (when code is formatted properly). + let rustdoc_function_name = body + .lines() + .find_map(|line| { + Some( + line.split_once("fn main() {")? + .1 + .split_once("fn ")? + .1 + .split_once("()")? + .0, + ) + .filter(|x| x.chars().all(|c| c.is_alphanumeric() || c == '_')) + }) + .expect("No test function found in `rustdoc`'s output."); + + // Qualify `Result` to avoid the collision with our own `Result` coming from the prelude. + let body = body.replace( + &format!("{rustdoc_function_name}() -> Result<(), impl core::fmt::Debug> {{"), + &format!("{rustdoc_function_name}() -> core::result::Result<(), impl core::fmt::Debug> {{"), + ); + + // For tests that get generated with `Result`, like above, `rustdoc` generates an `unwrap()` on + // the return value to check there were no returned errors. Instead, we use our assert macro + // since we want to just fail the test, not panic the kernel. + // + // We save the result in a variable so that the failed assertion message looks nicer. + let body = body.replace( + &format!("}} {rustdoc_function_name}().unwrap() }}"), + &format!("}} let test_return_value = {rustdoc_function_name}(); assert!(test_return_value.is_ok()); }}"), + ); + + // Figure out a smaller test name based on the generated function name. + let name = rustdoc_function_name.split_once("_rust_kernel_").unwrap().1; + + let path = format!("rust/test/doctests/kernel/{name}"); + + std::fs::write(path, body.as_bytes()).unwrap(); +} diff --git a/scripts/rustdoc_test_gen.rs b/scripts/rustdoc_test_gen.rs new file mode 100644 index 000000000000..5ebd42ae4a3f --- /dev/null +++ b/scripts/rustdoc_test_gen.rs @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Generates KUnit tests from saved `rustdoc`-generated tests. +//! +//! KUnit passes a context (`struct kunit *`) to each test, which should be forwarded to the other +//! KUnit functions and macros. +//! +//! However, we want to keep this as an implementation detail because: +//! +//! - Test code should not care about the implementation. +//! +//! - Documentation looks worse if it needs to carry extra details unrelated to the piece +//! being described. +//! +//! - Test code should be able to define functions and call them, without having to carry +//! the context. +//! +//! - Later on, we may want to be able to test non-kernel code (e.g. `core`, `alloc` or +//! third-party crates) which likely use the standard library `assert*!` macros. +//! +//! For this reason, instead of the passed context, `kunit_get_current_test()` is used instead +//! (i.e. `current->kunit_test`). +//! +//! Note that this means other threads/tasks potentially spawned by a given test, if failing, will +//! report the failure in the kernel log but will not fail the actual test. Saving the pointer in +//! e.g. a `static` per test does not fully solve the issue either, because currently KUnit does +//! not support assertions (only expectations) from other tasks. Thus leave that feature for +//! the future, which simplifies the code here too. We could also simply not allow `assert`s in +//! other tasks, but that seems overly constraining, and we do want to support them, eventually. + +use std::{ + fs, + fs::File, + io::{BufWriter, Read, Write}, + path::{Path, PathBuf}, +}; + +/// Find the real path to the original file based on the `file` portion of the test name. +/// +/// `rustdoc` generated `file`s look like `sync_locked_by_rs`. Underscores (except the last one) +/// may represent an actual underscore in a directory/file, or a path separator. Thus the actual +/// file might be `sync_locked_by.rs`, `sync/locked_by.rs`, `sync_locked/by.rs` or +/// `sync/locked/by.rs`. This function walks the file system to determine which is the real one. +/// +/// This does require that ambiguities do not exist, but that seems fair, especially since this is +/// all supposed to be temporary until `rustdoc` gives us proper metadata to build this. If such +/// ambiguities are detected, they are diagnosed and the script panics. +fn find_real_path<'a>(srctree: &Path, valid_paths: &'a mut Vec<PathBuf>, file: &str) -> &'a str { + valid_paths.clear(); + + let potential_components: Vec<&str> = file.strip_suffix("_rs").unwrap().split('_').collect(); + + find_candidates(srctree, valid_paths, Path::new(""), &potential_components); + fn find_candidates( + srctree: &Path, + valid_paths: &mut Vec<PathBuf>, + prefix: &Path, + potential_components: &[&str], + ) { + // The base case: check whether all the potential components left, joined by underscores, + // is a file. + let joined_potential_components = potential_components.join("_") + ".rs"; + if srctree + .join("rust/kernel") + .join(prefix) + .join(&joined_potential_components) + .is_file() + { + // Avoid `srctree` here in order to keep paths relative to it in the KTAP output. + valid_paths.push( + Path::new("rust/kernel") + .join(prefix) + .join(joined_potential_components), + ); + } + + // In addition, check whether each component prefix, joined by underscores, is a directory. + // If not, there is no need to check for combinations with that prefix. + for i in 1..potential_components.len() { + let (components_prefix, components_rest) = potential_components.split_at(i); + let prefix = prefix.join(components_prefix.join("_")); + if srctree.join("rust/kernel").join(&prefix).is_dir() { + find_candidates(srctree, valid_paths, &prefix, components_rest); + } + } + } + + assert!( + valid_paths.len() > 0, + "No path candidates found. This is likely a bug in the build system, or some files went \ + away while compiling." + ); + + if valid_paths.len() > 1 { + eprintln!("Several path candidates found:"); + for path in valid_paths { + eprintln!(" {path:?}"); + } + panic!( + "Several path candidates found, please resolve the ambiguity by renaming a file or \ + folder." + ); + } + + valid_paths[0].to_str().unwrap() +} + +fn main() { + let srctree = std::env::var("srctree").unwrap(); + let srctree = Path::new(&srctree); + + let mut paths = fs::read_dir("rust/test/doctests/kernel") + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::<Vec<_>>(); + + // Sort paths. + paths.sort(); + + let mut rust_tests = String::new(); + let mut c_test_declarations = String::new(); + let mut c_test_cases = String::new(); + let mut body = String::new(); + let mut last_file = String::new(); + let mut number = 0; + let mut valid_paths: Vec<PathBuf> = Vec::new(); + let mut real_path: &str = ""; + for path in paths { + // The `name` follows the `{file}_{line}_{number}` pattern (see description in + // `scripts/rustdoc_test_builder.rs`). Discard the `number`. + let name = path.file_name().unwrap().to_str().unwrap().to_string(); + + // Extract the `file` and the `line`, discarding the `number`. + let (file, line) = name.rsplit_once('_').unwrap().0.rsplit_once('_').unwrap(); + + // Generate an ID sequence ("test number") for each one in the file. + if file == last_file { + number += 1; + } else { + number = 0; + last_file = file.to_string(); + + // Figure out the real path, only once per file. + real_path = find_real_path(srctree, &mut valid_paths, file); + } + + // Generate a KUnit name (i.e. test name and C symbol) for this test. + // + // We avoid the line number, like `rustdoc` does, to make things slightly more stable for + // bisection purposes. However, to aid developers in mapping back what test failed, we will + // print a diagnostics line in the KTAP report. + let kunit_name = format!("rust_doctest_kernel_{file}_{number}"); + + // Read the test's text contents to dump it below. + body.clear(); + File::open(path).unwrap().read_to_string(&mut body).unwrap(); + + // Calculate how many lines before `main` function (including the `main` function line). + let body_offset = body + .lines() + .take_while(|line| !line.contains("fn main() {")) + .count() + + 1; + + use std::fmt::Write; + write!( + rust_tests, + r#"/// Generated `{name}` KUnit test case from a Rust documentation test. +#[no_mangle] +pub extern "C" fn {kunit_name}(__kunit_test: *mut kernel::bindings::kunit) {{ + /// Overrides the usual [`assert!`] macro with one that calls KUnit instead. + #[allow(unused)] + macro_rules! assert {{ + ($cond:expr $(,)?) => {{{{ + kernel::kunit_assert!("{kunit_name}", "{real_path}", __DOCTEST_ANCHOR - {line}, $cond); + }}}} + }} + + /// Overrides the usual [`assert_eq!`] macro with one that calls KUnit instead. + #[allow(unused)] + macro_rules! assert_eq {{ + ($left:expr, $right:expr $(,)?) => {{{{ + kernel::kunit_assert_eq!("{kunit_name}", "{real_path}", __DOCTEST_ANCHOR - {line}, $left, $right); + }}}} + }} + + // Many tests need the prelude, so provide it by default. + #[allow(unused)] + use kernel::prelude::*; + + // Unconditionally print the location of the original doctest (i.e. rather than the location in + // the generated file) so that developers can easily map the test back to the source code. + // + // This information is also printed when assertions fail, but this helps in the successful cases + // when the user is running KUnit manually, or when passing `--raw_output` to `kunit.py`. + // + // This follows the syntax for declaring test metadata in the proposed KTAP v2 spec, which may + // be used for the proposed KUnit test attributes API. Thus hopefully this will make migration + // easier later on. + kernel::kunit::info(format_args!(" # {kunit_name}.location: {real_path}:{line}\n")); + + /// The anchor where the test code body starts. + #[allow(unused)] + static __DOCTEST_ANCHOR: i32 = core::line!() as i32 + {body_offset} + 1; + {{ + {body} + main(); + }} +}} + +"# + ) + .unwrap(); + + write!(c_test_declarations, "void {kunit_name}(struct kunit *);\n").unwrap(); + write!(c_test_cases, " KUNIT_CASE({kunit_name}),\n").unwrap(); + } + + let rust_tests = rust_tests.trim(); + let c_test_declarations = c_test_declarations.trim(); + let c_test_cases = c_test_cases.trim(); + + write!( + BufWriter::new(File::create("rust/doctests_kernel_generated.rs").unwrap()), + r#"//! `kernel` crate documentation tests. + +const __LOG_PREFIX: &[u8] = b"rust_doctests_kernel\0"; + +{rust_tests} +"# + ) + .unwrap(); + + write!( + BufWriter::new(File::create("rust/doctests_kernel_generated_kunit.c").unwrap()), + r#"/* + * `kernel` crate documentation tests. + */ + +#include <kunit/test.h> + +{c_test_declarations} + +static struct kunit_case test_cases[] = {{ + {c_test_cases} + {{ }} +}}; + +static struct kunit_suite test_suite = {{ + .name = "rust_doctests_kernel", + .test_cases = test_cases, +}}; + +kunit_test_suite(test_suite); + +MODULE_LICENSE("GPL"); +"# + ) + .unwrap(); +} diff --git a/scripts/spelling.txt b/scripts/spelling.txt index fc7ba95e86a0..855c4863124b 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -1541,7 +1541,6 @@ temeprature||temperature temorary||temporary temproarily||temporarily temperture||temperature -thead||thread theads||threads therfore||therefore thier||their |