diff options
Diffstat (limited to 'lib')
124 files changed, 12891 insertions, 1826 deletions
diff --git a/lib/.gitignore b/lib/.gitignore index 5e7fa54c4536..e5e217b8307b 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -4,3 +4,5 @@ /gen_crc32table /gen_crc64table /oid_registry_data.c +/test_fortify.log +/test_fortify/*.log diff --git a/lib/Kconfig b/lib/Kconfig index ac3b30697b2b..5e7165e6a346 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -50,12 +50,18 @@ config HAVE_ARCH_BITREVERSE This option enables the use of hardware bit-reversal instructions on architectures which support such operations. -config GENERIC_STRNCPY_FROM_USER +config ARCH_HAS_STRNCPY_FROM_USER bool -config GENERIC_STRNLEN_USER +config ARCH_HAS_STRNLEN_USER bool +config GENERIC_STRNCPY_FROM_USER + def_bool !ARCH_HAS_STRNCPY_FROM_USER + +config GENERIC_STRNLEN_USER + def_bool !ARCH_HAS_STRNLEN_USER + config GENERIC_NET_UTILS bool @@ -102,6 +108,20 @@ config INDIRECT_PIO When in doubt, say N. +config INDIRECT_IOMEM + bool + help + This is selected by other options/architectures to provide the + emulated iomem accessors. + +config INDIRECT_IOMEM_FALLBACK + bool + depends on INDIRECT_IOMEM + help + If INDIRECT_IOMEM is selected, this enables falling back to plain + mmio accesses when the IO memory address is not a registered + emulated region. + config CRC_CCITT tristate "CRC-CCITT functions" help @@ -669,9 +689,6 @@ config PARMAN config OBJAGG tristate "objagg" if COMPILE_TEST -config STRING_SELFTEST - tristate "Test string functions" - endmenu config GENERIC_IOREMAP diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 1c9857fdb1a0..6fdbf9613aec 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -35,6 +35,17 @@ config PRINTK_CALLER no option to enable/disable at the kernel command line parameter or sysfs interface. +config STACKTRACE_BUILD_ID + bool "Show build ID information in stacktraces" + depends on PRINTK + help + Selecting this option adds build ID information for symbols in + stacktraces printed with the printk format '%p[SR]b'. + + This option is intended for distros where debuginfo is not easily + accessible but can be downloaded given the build ID of the vmlinux or + kernel module where the function is located. + config CONSOLE_LOGLEVEL_DEFAULT int "Default console loglevel (1-15)" range 1 15 @@ -284,7 +295,7 @@ config DEBUG_INFO_DWARF4 config DEBUG_INFO_DWARF5 bool "Generate DWARF Version 5 debuginfo" - depends on GCC_VERSION >= 50000 || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502))) + depends on !CC_IS_CLANG || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502))) depends on !DEBUG_INFO_BTF help Generate DWARF v5 debug info. Requires binutils 2.35.2, gcc 5.0+ (gcc @@ -313,9 +324,6 @@ config DEBUG_INFO_BTF config PAHOLE_HAS_SPLIT_BTF def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "119") -config PAHOLE_HAS_ZEROSIZE_PERCPU_SUPPORT - def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "122") - config DEBUG_INFO_BTF_MODULES def_bool y depends on DEBUG_INFO_BTF && MODULES && PAHOLE_HAS_SPLIT_BTF @@ -338,7 +346,7 @@ config FRAME_WARN int "Warn for stack frames larger than" range 0 8192 default 2048 if GCC_PLUGIN_LATENT_ENTROPY - default 1280 if (!64BIT && PARISC) + default 1536 if (!64BIT && (PARISC || XTENSA)) default 1024 if (!64BIT && !PARISC) default 2048 if 64BIT help @@ -357,6 +365,7 @@ config STRIP_ASM_SYMS config READABLE_ASM bool "Generate readable assembler code" depends on DEBUG_KERNEL + depends on CC_IS_GCC help Disable some compiler optimizations that tend to generate human unreadable assembler output. This may make the kernel slightly slower, but it helps @@ -375,6 +384,7 @@ config HEADERS_INSTALL config DEBUG_SECTION_MISMATCH bool "Enable full Section mismatch analysis" + depends on CC_IS_GCC help The section mismatch analysis checks if there are illegal references from one section to another section. @@ -403,8 +413,8 @@ config SECTION_MISMATCH_WARN_ONLY If unsure, say Y. -config DEBUG_FORCE_FUNCTION_ALIGN_32B - bool "Force all function address 32B aligned" if EXPERT +config DEBUG_FORCE_FUNCTION_ALIGN_64B + bool "Force all function address 64B aligned" if EXPERT help There are cases that a commit from one domain changes the function address alignment of other domains, and cause magic performance @@ -448,7 +458,7 @@ config STACK_VALIDATION config VMLINUX_VALIDATION bool - depends on STACK_VALIDATION && DEBUG_ENTRY && !PARAVIRT + depends on STACK_VALIDATION && DEBUG_ENTRY default y config VMLINUX_MAP @@ -1054,7 +1064,6 @@ config HARDLOCKUP_DETECTOR depends on HAVE_HARDLOCKUP_DETECTOR_PERF || HAVE_HARDLOCKUP_DETECTOR_ARCH select LOCKUP_DETECTOR select HARDLOCKUP_DETECTOR_PERF if HAVE_HARDLOCKUP_DETECTOR_PERF - select HARDLOCKUP_DETECTOR_ARCH if HAVE_HARDLOCKUP_DETECTOR_ARCH help Say Y here to enable the kernel to act as a watchdog to detect hard lockups. @@ -1227,7 +1236,7 @@ config PROVE_LOCKING depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select LOCKDEP select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT select DEBUG_RT_MUTEXES if RT_MUTEXES select DEBUG_RWSEMS select DEBUG_WW_MUTEX_SLOWPATH @@ -1282,7 +1291,7 @@ config PROVE_RAW_LOCK_NESTING option expect lockdep splats until these problems have been fully addressed which is work in progress. This config switch allows to identify and analyze these problems. It will be removed and the - check permanentely enabled once the main issues have been fixed. + check permanently enabled once the main issues have been fixed. If unsure, select N. @@ -1291,7 +1300,7 @@ config LOCK_STAT depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select LOCKDEP select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT select DEBUG_RT_MUTEXES if RT_MUTEXES select DEBUG_LOCK_ALLOC default n @@ -1327,7 +1336,7 @@ config DEBUG_SPINLOCK config DEBUG_MUTEXES bool "Mutex debugging: basic checks" - depends on DEBUG_KERNEL + depends on DEBUG_KERNEL && !PREEMPT_RT help This feature allows mutex semantics violations to be detected and reported. @@ -1337,7 +1346,8 @@ config DEBUG_WW_MUTEX_SLOWPATH depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select DEBUG_LOCK_ALLOC select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT + select DEBUG_RT_MUTEXES if PREEMPT_RT help This feature enables slowpath testing for w/w mutex users by injecting additional -EDEADLK wound/backoff cases. Together with @@ -1360,7 +1370,7 @@ config DEBUG_LOCK_ALLOC bool "Lock debugging: detect incorrect freeing of live locks" depends on DEBUG_KERNEL && LOCK_DEBUGGING_SUPPORT select DEBUG_SPINLOCK - select DEBUG_MUTEXES + select DEBUG_MUTEXES if !PREEMPT_RT select DEBUG_RT_MUTEXES if RT_MUTEXES select LOCKDEP help @@ -1448,7 +1458,7 @@ config DEBUG_LOCKING_API_SELFTESTS Say Y here if you want the kernel to run a short self-test during bootup. The self-test checks whether common types of locking bugs are detected by debugging mechanisms or not. (if you disable - lock debugging then those bugs wont be detected of course.) + lock debugging then those bugs won't be detected of course.) The following locking APIs are covered: spinlocks, rwlocks, mutexes and rwsems. @@ -1671,33 +1681,6 @@ config DEBUG_WQ_FORCE_RR_CPU feature by default. When enabled, memory and cache locality will be impacted. -config DEBUG_BLOCK_EXT_DEVT - bool "Force extended block device numbers and spread them" - depends on DEBUG_KERNEL - depends on BLOCK - default n - help - BIG FAT WARNING: ENABLING THIS OPTION MIGHT BREAK BOOTING ON - SOME DISTRIBUTIONS. DO NOT ENABLE THIS UNLESS YOU KNOW WHAT - YOU ARE DOING. Distros, please enable this and fix whatever - is broken. - - Conventionally, block device numbers are allocated from - predetermined contiguous area. However, extended block area - may introduce non-contiguous block device numbers. This - option forces most block device numbers to be allocated from - the extended space and spreads them to discover kernel or - userland code paths which assume predetermined contiguous - device number allocation. - - Note that turning on this debug option shuffles all the - device numbers for all IDE and SCSI devices including libata - ones, so root partition specified using device number - directly (via rdev or root=MAJ:MIN) won't work anymore. - Textual device names (root=/dev/sdXn) will continue to work. - - Say N if you are unsure. - config CPU_HOTPLUG_STATE_CONTROL bool "Enable CPU hotplug state control" depends on DEBUG_KERNEL @@ -1928,7 +1911,7 @@ config FAIL_IO_TIMEOUT thus exercising the error handling. Only works with drivers that use the generic timeout handling, - for others it wont do anything. + for others it won't do anything. config FAIL_FUTEX bool "Fault-injection capability for futexes" @@ -1963,6 +1946,13 @@ config FAIL_MMC_REQUEST and to test how the mmc host driver handles retries from the block device. +config FAIL_SUNRPC + bool "Fault-injection capability for SunRPC" + depends on FAULT_INJECTION_DEBUG_FS && SUNRPC_DEBUG + help + Provide fault-injection capability for SunRPC and + its consumers. + config FAULT_INJECTION_STACKTRACE_FILTER bool "stacktrace filter for fault-injection capabilities" depends on FAULT_INJECTION_DEBUG_FS && STACKTRACE_SUPPORT @@ -2049,8 +2039,9 @@ config LKDTM Documentation/fault-injection/provoke-crashes.rst config TEST_LIST_SORT - tristate "Linked list sorting test" - depends on DEBUG_KERNEL || m + tristate "Linked list sorting test" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS help Enable this to turn on 'list_sort()' function test. This test is executed only once during system boot (so affects only boot time), @@ -2069,8 +2060,9 @@ config TEST_MIN_HEAP If unsure, say N. config TEST_SORT - tristate "Array-based sort test" - depends on DEBUG_KERNEL || m + tristate "Array-based sort test" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS help This option enables the self-test function of 'sort()' at boot, or at module load time. @@ -2088,9 +2080,10 @@ config TEST_DIV64 If unsure, say N. config KPROBES_SANITY_TEST - bool "Kprobes sanity tests" + tristate "Kprobes sanity tests" depends on DEBUG_KERNEL depends on KPROBES + depends on KUNIT help This option provides for testing basic kprobes functionality on boot. Samples of kprobe and kretprobe are inserted and @@ -2171,6 +2164,9 @@ config ASYNC_RAID6_TEST config TEST_HEXDUMP tristate "Test functions located in the hexdump module at runtime" +config STRING_SELFTEST + tristate "Test string functions at runtime" + config TEST_STRING_HELPERS tristate "Test functions located in the string_helpers module at runtime" @@ -2446,6 +2442,28 @@ config SLUB_KUNIT_TEST If unsure, say N. +config RATIONAL_KUNIT_TEST + tristate "KUnit test for rational.c" if !KUNIT_ALL_TESTS + depends on KUNIT && RATIONAL + default KUNIT_ALL_TESTS + help + This builds the rational math unit test. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config MEMCPY_KUNIT_TEST + tristate "Test memcpy(), memmove(), and memset() functions at runtime" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Builds unit tests for memcpy(), memmove(), and memset() functions. + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config TEST_UDELAY tristate "udelay test driver" help diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 1e2d10f86011..cdc842d090db 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -66,6 +66,7 @@ choice config KASAN_GENERIC bool "Generic mode" depends on HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC + depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS help @@ -86,6 +87,7 @@ config KASAN_GENERIC config KASAN_SW_TAGS bool "Software tag-based mode" depends on HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS + depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS help diff --git a/lib/Kconfig.kcsan b/lib/Kconfig.kcsan index 0440f373248e..e0a93ffdef30 100644 --- a/lib/Kconfig.kcsan +++ b/lib/Kconfig.kcsan @@ -40,10 +40,14 @@ menuconfig KCSAN if KCSAN -# Compiler capabilities that should not fail the test if they are unavailable. config CC_HAS_TSAN_COMPOUND_READ_BEFORE_WRITE def_bool (CC_IS_CLANG && $(cc-option,-fsanitize=thread -mllvm -tsan-compound-read-before-write=1)) || \ (CC_IS_GCC && $(cc-option,-fsanitize=thread --param tsan-compound-read-before-write=1)) + help + The compiler instruments plain compound read-write operations + differently (++, --, +=, -=, |=, &=, etc.), which allows KCSAN to + distinguish them from other plain accesses. This is currently + supported by Clang 12 or later. config KCSAN_VERBOSE bool "Show verbose reports with more information about system state" @@ -58,9 +62,6 @@ config KCSAN_VERBOSE generated from any one of them, system stability may suffer due to deadlocks or recursion. If in doubt, say N. -config KCSAN_DEBUG - bool "Debugging of KCSAN internals" - config KCSAN_SELFTEST bool "Perform short selftests on boot" default y @@ -149,7 +150,8 @@ config KCSAN_SKIP_WATCH_RANDOMIZE KCSAN_WATCH_SKIP. config KCSAN_INTERRUPT_WATCHER - bool "Interruptible watchers" + bool "Interruptible watchers" if !KCSAN_STRICT + default KCSAN_STRICT help If enabled, a task that set up a watchpoint may be interrupted while delayed. This option will allow KCSAN to detect races between @@ -169,13 +171,9 @@ config KCSAN_REPORT_ONCE_IN_MS reporting to avoid flooding the console with reports. Setting this to 0 disables rate limiting. -# The main purpose of the below options is to control reported data races (e.g. -# in fuzzer configs), and are not expected to be switched frequently by other -# users. We could turn some of them into boot parameters, but given they should -# not be switched normally, let's keep them here to simplify configuration. -# -# The defaults below are chosen to be very conservative, and may miss certain -# bugs. +# The main purpose of the below options is to control reported data races, and +# are not expected to be switched frequently by non-testers or at runtime. +# The defaults are chosen to be conservative, and can miss certain bugs. config KCSAN_REPORT_RACE_UNKNOWN_ORIGIN bool "Report races of unknown origin" @@ -186,9 +184,17 @@ config KCSAN_REPORT_RACE_UNKNOWN_ORIGIN reported if it was only possible to infer a race due to a data value change while an access is being delayed on a watchpoint. +config KCSAN_STRICT + bool "Strict data-race checking" + help + KCSAN will report data races with the strictest possible rules, which + closely aligns with the rules defined by the Linux-kernel memory + consistency model (LKMM). + config KCSAN_REPORT_VALUE_CHANGE_ONLY bool "Only report races where watcher observed a data value change" default y + depends on !KCSAN_STRICT help If enabled and a conflicting write is observed via a watchpoint, but the data value of the memory location was observed to remain @@ -197,6 +203,7 @@ config KCSAN_REPORT_VALUE_CHANGE_ONLY config KCSAN_ASSUME_PLAIN_WRITES_ATOMIC bool "Assume that plain aligned writes up to word size are atomic" default y + depends on !KCSAN_STRICT help Assume that plain aligned writes up to word size are atomic by default, and also not subject to other unsafe compiler optimizations @@ -209,6 +216,7 @@ config KCSAN_ASSUME_PLAIN_WRITES_ATOMIC config KCSAN_IGNORE_ATOMICS bool "Do not instrument marked atomic accesses" + depends on !KCSAN_STRICT help Never instrument marked atomic accesses. This option can be used for additional filtering. Conflicting marked atomic reads and plain @@ -224,4 +232,14 @@ config KCSAN_IGNORE_ATOMICS due to two conflicting plain writes will be reported (aligned and unaligned, if CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n). +config KCSAN_PERMISSIVE + bool "Enable all additional permissive rules" + depends on KCSAN_REPORT_VALUE_CHANGE_ONLY + help + Enable additional permissive rules to ignore certain classes of data + races (also see kernel/kcsan/permissive.h). None of the permissive + rules imply that such data races are generally safe, but can be used + to further reduce reported data races due to data-racy patterns + common across the kernel. + endif # KCSAN diff --git a/lib/Makefile b/lib/Makefile index 6d765d5fb8ac..364c23f15578 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -100,6 +100,7 @@ obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o obj-$(CONFIG_TEST_LOCKUP) += test_lockup.o obj-$(CONFIG_TEST_HMM) += test_hmm.o obj-$(CONFIG_TEST_FREE_PAGES) += test_free_pages.o +obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o # # CFLAGS for compiling floating point code inside the kernel. x86/Makefile turns @@ -148,6 +149,8 @@ obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o lib-y += logic_pio.o +lib-$(CONFIG_INDIRECT_IOMEM) += logic_iomem.o + obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o obj-$(CONFIG_BTREE) += btree.o @@ -349,12 +352,46 @@ obj-$(CONFIG_OBJAGG) += objagg.o obj-$(CONFIG_PLDMFW) += pldmfw/ # KUnit tests -CFLAGS_bitfield_kunit.o := $(call cc-option,-Wframe-larger-than=10240) +CFLAGS_bitfield_kunit.o := $(DISABLE_STRUCTLEAK_PLUGIN) obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o obj-$(CONFIG_LIST_KUNIT_TEST) += list-test.o obj-$(CONFIG_LINEAR_RANGES_TEST) += test_linear_ranges.o obj-$(CONFIG_BITS_TEST) += test_bits.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o +obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o + +# FORTIFY_SOURCE compile-time behavior tests +TEST_FORTIFY_SRCS = $(wildcard $(srctree)/$(src)/test_fortify/*-*.c) +TEST_FORTIFY_LOGS = $(patsubst $(srctree)/$(src)/%.c, %.log, $(TEST_FORTIFY_SRCS)) +TEST_FORTIFY_LOG = test_fortify.log + +quiet_cmd_test_fortify = TEST $@ + cmd_test_fortify = $(CONFIG_SHELL) $(srctree)/scripts/test_fortify.sh \ + $< $@ "$(NM)" $(CC) $(c_flags) \ + $(call cc-disable-warning,fortify-source) + +targets += $(TEST_FORTIFY_LOGS) +clean-files += $(TEST_FORTIFY_LOGS) +clean-files += $(addsuffix .o, $(TEST_FORTIFY_LOGS)) +$(obj)/test_fortify/%.log: $(src)/test_fortify/%.c \ + $(src)/test_fortify/test_fortify.h \ + $(srctree)/include/linux/fortify-string.h \ + $(srctree)/scripts/test_fortify.sh \ + FORCE + $(call if_changed,test_fortify) + +quiet_cmd_gen_fortify_log = GEN $@ + cmd_gen_fortify_log = cat </dev/null $(filter-out FORCE,$^) 2>/dev/null > $@ || true + +targets += $(TEST_FORTIFY_LOG) +clean-files += $(TEST_FORTIFY_LOG) +$(obj)/$(TEST_FORTIFY_LOG): $(addprefix $(obj)/, $(TEST_FORTIFY_LOGS)) FORCE + $(call if_changed,gen_fortify_log) + +# Fake dependency to trigger the fortify tests. +ifeq ($(CONFIG_FORTIFY_SOURCE),y) +$(obj)/string.o: $(obj)/$(TEST_FORTIFY_LOG) +endif diff --git a/lib/asn1_encoder.c b/lib/asn1_encoder.c index 41e71aae3ef6..27bbe891714f 100644 --- a/lib/asn1_encoder.c +++ b/lib/asn1_encoder.c @@ -181,7 +181,7 @@ EXPORT_SYMBOL_GPL(asn1_encode_oid); /** * asn1_encode_length() - encode a length to follow an ASN.1 tag * @data: pointer to encode at - * @data_len: pointer to remaning length (adjusted by routine) + * @data_len: pointer to remaining length (adjusted by routine) * @len: length to encode * * This routine can encode lengths up to 65535 using the ASN.1 rules. diff --git a/lib/assoc_array.c b/lib/assoc_array.c index 04c98799c3ba..079c72e26493 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -741,8 +741,7 @@ all_leaves_cluster_together: keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s0 = kzalloc(struct_size(new_s0, index_key, keylen), GFP_KERNEL); if (!new_s0) return false; edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s0); @@ -849,8 +848,8 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, keylen = round_up(diff, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s0 = kzalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s0 = kzalloc(struct_size(new_s0, index_key, keylen), + GFP_KERNEL); if (!new_s0) return false; edit->new_meta[1] = assoc_array_shortcut_to_ptr(new_s0); @@ -864,7 +863,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, new_n0->parent_slot = 0; memcpy(new_s0->index_key, shortcut->index_key, - keylen * sizeof(unsigned long)); + flex_array_size(new_s0, index_key, keylen)); blank = ULONG_MAX << (diff & ASSOC_ARRAY_KEY_CHUNK_MASK); pr_devel("blank off [%zu] %d: %lx\n", keylen - 1, diff, blank); @@ -899,8 +898,8 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s1 = kzalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s1 = kzalloc(struct_size(new_s1, index_key, keylen), + GFP_KERNEL); if (!new_s1) return false; edit->new_meta[2] = assoc_array_shortcut_to_ptr(new_s1); @@ -913,7 +912,7 @@ static bool assoc_array_insert_mid_shortcut(struct assoc_array_edit *edit, new_n0->slots[sc_slot] = assoc_array_shortcut_to_ptr(new_s1); memcpy(new_s1->index_key, shortcut->index_key, - keylen * sizeof(unsigned long)); + flex_array_size(new_s1, index_key, keylen)); edit->set[1].ptr = &side->back_pointer; edit->set[1].to = assoc_array_shortcut_to_ptr(new_s1); @@ -1490,13 +1489,12 @@ descend: shortcut = assoc_array_ptr_to_shortcut(cursor); keylen = round_up(shortcut->skip_to_level, ASSOC_ARRAY_KEY_CHUNK_SIZE); keylen >>= ASSOC_ARRAY_KEY_CHUNK_SHIFT; - new_s = kmalloc(sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long), GFP_KERNEL); + new_s = kmalloc(struct_size(new_s, index_key, keylen), + GFP_KERNEL); if (!new_s) goto enomem; pr_devel("dup shortcut %p -> %p\n", shortcut, new_s); - memcpy(new_s, shortcut, (sizeof(struct assoc_array_shortcut) + - keylen * sizeof(unsigned long))); + memcpy(new_s, shortcut, struct_size(new_s, index_key, keylen)); new_s->back_pointer = new_parent; new_s->parent_slot = shortcut->parent_slot; *new_ptr_pp = new_parent = assoc_array_shortcut_to_ptr(new_s); diff --git a/lib/audit.c b/lib/audit.c index 5004bff928a7..738bda22dd39 100644 --- a/lib/audit.c +++ b/lib/audit.c @@ -45,23 +45,27 @@ int audit_classify_syscall(int abi, unsigned syscall) switch(syscall) { #ifdef __NR_open case __NR_open: - return 2; + return AUDITSC_OPEN; #endif #ifdef __NR_openat case __NR_openat: - return 3; + return AUDITSC_OPENAT; #endif #ifdef __NR_socketcall case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; #endif #ifdef __NR_execveat case __NR_execveat: #endif case __NR_execve: - return 5; + return AUDITSC_EXECVE; +#ifdef __NR_openat2 + case __NR_openat2: + return AUDITSC_OPENAT2; +#endif default: - return 0; + return AUDITSC_NATIVE; } } diff --git a/lib/bitmap.c b/lib/bitmap.c index 74ceb02f45e3..926408883456 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -487,6 +487,127 @@ int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, } EXPORT_SYMBOL(bitmap_print_to_pagebuf); +/** + * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string + * @list: indicates whether the bitmap must be list + * true: print in decimal list format + * false: print in hexadecimal bitmask format + */ +static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + const char *fmt = list ? "%*pbl\n" : "%*pb\n"; + ssize_t size; + void *data; + + data = kasprintf(GFP_KERNEL, fmt, nmaskbits, maskp); + if (!data) + return -ENOMEM; + + size = memory_read_from_buffer(buf, count, &off, data, strlen(data) + 1); + kfree(data); + + return size; +} + +/** + * bitmap_print_bitmask_to_buf - convert bitmap to hex bitmask format ASCII string + * + * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper + * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal + * bitmask and decimal list to userspace by sysfs ABI. + * Drivers might be using a normal attribute for this kind of ABIs. A + * normal attribute typically has show entry as below: + * static ssize_t example_attribute_show(struct device *dev, + * struct device_attribute *attr, char *buf) + * { + * ... + * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); + * } + * show entry of attribute has no offset and count parameters and this + * means the file is limited to one page only. + * bitmap_print_to_pagebuf() API works terribly well for this kind of + * normal attribute with buf parameter and without offset, count: + * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, + * int nmaskbits) + * { + * } + * The problem is once we have a large bitmap, we have a chance to get a + * bitmask or list more than one page. Especially for list, it could be + * as complex as 0,3,5,7,9,... We have no simple way to know it exact size. + * It turns out bin_attribute is a way to break this limit. bin_attribute + * has show entry as below: + * static ssize_t + * example_bin_attribute_show(struct file *filp, struct kobject *kobj, + * struct bin_attribute *attr, char *buf, + * loff_t offset, size_t count) + * { + * ... + * } + * With the new offset and count parameters, this makes sysfs ABI be able + * to support file size more than one page. For example, offset could be + * >= 4096. + * bitmap_print_bitmask_to_buf(), bitmap_print_list_to_buf() wit their + * cpumap wrapper cpumap_print_bitmask_to_buf(), cpumap_print_list_to_buf() + * make those drivers be able to support large bitmask and list after they + * move to use bin_attribute. In result, we have to pass the corresponding + * parameters such as off, count from bin_attribute show entry to this API. + * + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print + * + * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() + * is similar with cpumap_print_to_pagebuf(), the difference is that + * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption + * the destination buffer is exactly one page and won't be more than one page. + * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other + * hand, mainly serves bin_attribute which doesn't work with exact one page, + * and it can break the size limit of converted decimal list and hexadecimal + * bitmask. + * + * WARNING! + * + * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). + * It is intended to workaround sysfs limitations discussed above and should be + * used carefully in general case for the following reasons: + * - Time complexity is O(nbits^2/count), comparing to O(nbits) for snprintf(). + * - Memory complexity is O(nbits), comparing to O(1) for snprintf(). + * - @off and @count are NOT offset and number of bits to print. + * - If printing part of bitmap as list, the resulting string is not a correct + * list representation of bitmap. Particularly, some bits within or out of + * related interval may be erroneously set or unset. The format of the string + * may be broken, so bitmap_parselist-like parser may fail parsing it. + * - If printing the whole bitmap as list by parts, user must ensure the order + * of calls of the function such that the offset is incremented linearly. + * - If printing the whole bitmap as list by parts, user must keep bitmap + * unchanged between the very first and very last call. Otherwise concatenated + * result may be incorrect, and format may be broken. + * + * Returns the number of characters actually printed to @buf + */ +int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(false, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_bitmask_to_buf); + +/** + * bitmap_print_list_to_buf - convert bitmap to decimal list format ASCII string + * + * Everything is same with the above bitmap_print_bitmask_to_buf() except + * the print format. + */ +int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, + int nmaskbits, loff_t off, size_t count) +{ + return bitmap_print_to_buf(true, buf, maskp, nmaskbits, off, count); +} +EXPORT_SYMBOL(bitmap_print_list_to_buf); + /* * Region 9-38:4/10 describes the following bitmap structure: * 0 9 12 18 38 N @@ -581,6 +702,14 @@ static const char *bitmap_parse_region(const char *str, struct region *r) { unsigned int lastbit = r->nbits - 1; + if (!strncasecmp(str, "all", 3)) { + r->start = 0; + r->end = lastbit; + str += 3; + + goto check_pattern; + } + str = bitmap_getnum(str, &r->start, lastbit); if (IS_ERR(str)) return str; @@ -595,6 +724,7 @@ static const char *bitmap_parse_region(const char *str, struct region *r) if (IS_ERR(str)) return str; +check_pattern: if (end_of_region(*str)) goto no_pattern; @@ -784,8 +914,6 @@ int bitmap_parse(const char *start, unsigned int buflen, } EXPORT_SYMBOL(bitmap_parse); - -#ifdef CONFIG_NUMA /** * bitmap_pos_to_ord - find ordinal of set bit at given position in bitmap * @buf: pointer to a bitmap @@ -894,6 +1022,7 @@ void bitmap_remap(unsigned long *dst, const unsigned long *src, set_bit(bitmap_ord_to_pos(new, n % w, nbits), dst); } } +EXPORT_SYMBOL(bitmap_remap); /** * bitmap_bitremap - Apply map defined by a pair of bitmaps to a single bit @@ -931,7 +1060,9 @@ int bitmap_bitremap(int oldbit, const unsigned long *old, else return bitmap_ord_to_pos(new, n % w, bits); } +EXPORT_SYMBOL(bitmap_bitremap); +#ifdef CONFIG_NUMA /** * bitmap_onto - translate one bitmap relative to another * @dst: resulting translated bitmap @@ -1267,6 +1398,19 @@ unsigned long *bitmap_zalloc(unsigned int nbits, gfp_t flags) } EXPORT_SYMBOL(bitmap_zalloc); +unsigned long *bitmap_alloc_node(unsigned int nbits, gfp_t flags, int node) +{ + return kmalloc_array_node(BITS_TO_LONGS(nbits), sizeof(unsigned long), + flags, node); +} +EXPORT_SYMBOL(bitmap_alloc_node); + +unsigned long *bitmap_zalloc_node(unsigned int nbits, gfp_t flags, int node) +{ + return bitmap_alloc_node(nbits, flags | __GFP_ZERO, node); +} +EXPORT_SYMBOL(bitmap_zalloc_node); + void bitmap_free(const unsigned long *bitmap) { kfree(bitmap); diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 9f8c70a98fcf..70e0d52ffd24 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -4,16 +4,24 @@ * Masami Hiramatsu <mhiramat@kernel.org> */ -#define pr_fmt(fmt) "bootconfig: " fmt - +#ifdef __KERNEL__ #include <linux/bootconfig.h> #include <linux/bug.h> #include <linux/ctype.h> #include <linux/errno.h> #include <linux/kernel.h> #include <linux/memblock.h> -#include <linux/printk.h> #include <linux/string.h> +#else /* !__KERNEL__ */ +/* + * NOTE: This is only for tools/bootconfig, because tools/bootconfig will + * run the parser sanity test. + * This does NOT mean lib/bootconfig.c is available in the user space. + * However, if you change this file, please make sure the tools/bootconfig + * has no issue on building and running. + */ +#include <linux/bootconfig.h> +#endif /* * Extra Boot Config (XBC) is given as tree-structured ascii text of @@ -34,6 +42,50 @@ static int xbc_err_pos __initdata; static int open_brace[XBC_DEPTH_MAX] __initdata; static int brace_index __initdata; +#ifdef __KERNEL__ +static inline void * __init xbc_alloc_mem(size_t size) +{ + return memblock_alloc(size, SMP_CACHE_BYTES); +} + +static inline void __init xbc_free_mem(void *addr, size_t size) +{ + memblock_free_ptr(addr, size); +} + +#else /* !__KERNEL__ */ + +static inline void *xbc_alloc_mem(size_t size) +{ + return malloc(size); +} + +static inline void xbc_free_mem(void *addr, size_t size) +{ + free(addr); +} +#endif +/** + * xbc_get_info() - Get the information of loaded boot config + * @node_size: A pointer to store the number of nodes. + * @data_size: A pointer to store the size of bootconfig data. + * + * Get the number of used nodes in @node_size if it is not NULL, + * and the size of bootconfig data in @data_size if it is not NULL. + * Return 0 if the boot config is initialized, or return -ENODEV. + */ +int __init xbc_get_info(int *node_size, size_t *data_size) +{ + if (!xbc_data) + return -ENODEV; + + if (node_size) + *node_size = xbc_node_num; + if (data_size) + *data_size = xbc_data_size; + return 0; +} + static int __init xbc_parse_error(const char *msg, const char *p) { xbc_err_msg = msg; @@ -142,21 +194,21 @@ xbc_node_match_prefix(struct xbc_node *node, const char **prefix) } /** - * xbc_node_find_child() - Find a child node which matches given key + * xbc_node_find_subkey() - Find a subkey node which matches given key * @parent: An XBC node. * @key: A key string. * - * Search a node under @parent which matches @key. The @key can contain + * Search a key node under @parent which matches @key. The @key can contain * several words jointed with '.'. If @parent is NULL, this searches the * node from whole tree. Return NULL if no node is matched. */ struct xbc_node * __init -xbc_node_find_child(struct xbc_node *parent, const char *key) +xbc_node_find_subkey(struct xbc_node *parent, const char *key) { struct xbc_node *node; if (parent) - node = xbc_node_get_child(parent); + node = xbc_node_get_subkey(parent); else node = xbc_root_node(); @@ -164,7 +216,7 @@ xbc_node_find_child(struct xbc_node *parent, const char *key) if (!xbc_node_match_prefix(node, &key)) node = xbc_node_get_next(node); else if (*key != '\0') - node = xbc_node_get_child(node); + node = xbc_node_get_subkey(node); else break; } @@ -191,7 +243,7 @@ const char * __init xbc_node_find_value(struct xbc_node *parent, const char *key, struct xbc_node **vnode) { - struct xbc_node *node = xbc_node_find_child(parent, key); + struct xbc_node *node = xbc_node_find_subkey(parent, key); if (!node || !xbc_node_is_key(node)) return NULL; @@ -226,7 +278,7 @@ int __init xbc_node_compose_key_after(struct xbc_node *root, struct xbc_node *node, char *buf, size_t size) { - u16 keys[XBC_DEPTH_MAX]; + uint16_t keys[XBC_DEPTH_MAX]; int depth = 0, ret = 0, total = 0; if (!node || node == root) @@ -274,6 +326,8 @@ int __init xbc_node_compose_key_after(struct xbc_node *root, struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, struct xbc_node *node) { + struct xbc_node *next; + if (unlikely(!xbc_data)) return NULL; @@ -282,6 +336,13 @@ struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, if (!node) node = xbc_nodes; } else { + /* Leaf node may have a subkey */ + next = xbc_node_get_subkey(node); + if (next) { + node = next; + goto found; + } + if (node == root) /* @root was a leaf, no child node. */ return NULL; @@ -296,6 +357,7 @@ struct xbc_node * __init xbc_node_find_next_leaf(struct xbc_node *root, node = xbc_node_get_next(node); } +found: while (node && !xbc_node_is_leaf(node)) node = xbc_node_get_child(node); @@ -331,21 +393,21 @@ const char * __init xbc_node_find_next_key_value(struct xbc_node *root, /* XBC parse and tree build */ -static int __init xbc_init_node(struct xbc_node *node, char *data, u32 flag) +static int __init xbc_init_node(struct xbc_node *node, char *data, uint32_t flag) { unsigned long offset = data - xbc_data; if (WARN_ON(offset >= XBC_DATA_MAX)) return -EINVAL; - node->data = (u16)offset | flag; + node->data = (uint16_t)offset | flag; node->child = 0; node->next = 0; return 0; } -static struct xbc_node * __init xbc_add_node(char *data, u32 flag) +static struct xbc_node * __init xbc_add_node(char *data, uint32_t flag) { struct xbc_node *node; @@ -367,18 +429,28 @@ static inline __init struct xbc_node *xbc_last_sibling(struct xbc_node *node) return node; } -static struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) +static inline __init struct xbc_node *xbc_last_child(struct xbc_node *node) +{ + while (node->child) + node = xbc_node_get_child(node); + + return node; +} + +static struct xbc_node * __init __xbc_add_sibling(char *data, uint32_t flag, bool head) { struct xbc_node *sib, *node = xbc_add_node(data, flag); if (node) { if (!last_parent) { + /* Ignore @head in this case */ node->parent = XBC_NODE_MAX; sib = xbc_last_sibling(xbc_nodes); sib->next = xbc_node_index(node); } else { node->parent = xbc_node_index(last_parent); - if (!last_parent->child) { + if (!last_parent->child || head) { + node->next = last_parent->child; last_parent->child = xbc_node_index(node); } else { sib = xbc_node_get_child(last_parent); @@ -392,7 +464,17 @@ static struct xbc_node * __init xbc_add_sibling(char *data, u32 flag) return node; } -static inline __init struct xbc_node *xbc_add_child(char *data, u32 flag) +static inline struct xbc_node * __init xbc_add_sibling(char *data, uint32_t flag) +{ + return __xbc_add_sibling(data, flag, false); +} + +static inline struct xbc_node * __init xbc_add_head_sibling(char *data, uint32_t flag) +{ + return __xbc_add_sibling(data, flag, true); +} + +static inline __init struct xbc_node *xbc_add_child(char *data, uint32_t flag) { struct xbc_node *node = xbc_add_sibling(data, flag); @@ -517,17 +599,20 @@ static int __init xbc_parse_array(char **__v) char *next; int c = 0; + if (last_parent->child) + last_parent = xbc_node_get_child(last_parent); + do { c = __xbc_parse_value(__v, &next); if (c < 0) return c; - node = xbc_add_sibling(*__v, XBC_VALUE); + node = xbc_add_child(*__v, XBC_VALUE); if (!node) return -ENOMEM; *__v = next; } while (c == ','); - node->next = 0; + node->child = 0; return c; } @@ -557,8 +642,9 @@ static int __init __xbc_add_key(char *k) node = find_match_node(xbc_nodes, k); else { child = xbc_node_get_child(last_parent); + /* Since the value node is the first child, skip it. */ if (child && xbc_node_is_value(child)) - return xbc_parse_error("Subkey is mixed with value", k); + child = xbc_node_get_next(child); node = find_match_node(child, k); } @@ -601,23 +687,29 @@ static int __init xbc_parse_kv(char **k, char *v, int op) if (ret) return ret; - child = xbc_node_get_child(last_parent); - if (child) { - if (xbc_node_is_key(child)) - return xbc_parse_error("Value is mixed with subkey", v); - else if (op == '=') - return xbc_parse_error("Value is redefined", v); - } - c = __xbc_parse_value(&v, &next); if (c < 0) return c; - if (op == ':' && child) { - xbc_init_node(child, v, XBC_VALUE); - } else if (!xbc_add_sibling(v, XBC_VALUE)) + child = xbc_node_get_child(last_parent); + if (child && xbc_node_is_value(child)) { + if (op == '=') + return xbc_parse_error("Value is redefined", v); + if (op == ':') { + unsigned short nidx = child->next; + + xbc_init_node(child, v, XBC_VALUE); + child->next = nidx; /* keep subkeys */ + goto array; + } + /* op must be '+' */ + last_parent = xbc_last_child(child); + } + /* The value node should always be the first child */ + if (!xbc_add_head_sibling(v, XBC_VALUE)) return -ENOMEM; +array: if (c == ',') { /* Array */ c = xbc_parse_array(&next); if (c < 0) @@ -740,41 +832,94 @@ static int __init xbc_verify_tree(void) return 0; } +/* Need to setup xbc_data and xbc_nodes before call this. */ +static int __init xbc_parse_tree(void) +{ + char *p, *q; + int ret = 0, c; + + last_parent = NULL; + p = xbc_data; + do { + q = strpbrk(p, "{}=+;:\n#"); + if (!q) { + p = skip_spaces(p); + if (*p != '\0') + ret = xbc_parse_error("No delimiter", p); + break; + } + + c = *q; + *q++ = '\0'; + switch (c) { + case ':': + case '+': + if (*q++ != '=') { + ret = xbc_parse_error(c == '+' ? + "Wrong '+' operator" : + "Wrong ':' operator", + q - 2); + break; + } + fallthrough; + case '=': + ret = xbc_parse_kv(&p, q, c); + break; + case '{': + ret = xbc_open_brace(&p, q); + break; + case '#': + q = skip_comment(q); + fallthrough; + case ';': + case '\n': + ret = xbc_parse_key(&p, q); + break; + case '}': + ret = xbc_close_brace(&p, q); + break; + } + } while (!ret); + + return ret; +} + /** - * xbc_destroy_all() - Clean up all parsed bootconfig + * xbc_exit() - Clean up all parsed bootconfig * * This clears all data structures of parsed bootconfig on memory. * If you need to reuse xbc_init() with new boot config, you can * use this. */ -void __init xbc_destroy_all(void) +void __init xbc_exit(void) { + xbc_free_mem(xbc_data, xbc_data_size); xbc_data = NULL; xbc_data_size = 0; xbc_node_num = 0; - memblock_free(__pa(xbc_nodes), sizeof(struct xbc_node) * XBC_NODE_MAX); + xbc_free_mem(xbc_nodes, sizeof(struct xbc_node) * XBC_NODE_MAX); xbc_nodes = NULL; brace_index = 0; } /** * xbc_init() - Parse given XBC file and build XBC internal tree - * @buf: boot config text + * @data: The boot config text original data + * @size: The size of @data * @emsg: A pointer of const char * to store the error message * @epos: A pointer of int to store the error position * - * This parses the boot config text in @buf. @buf must be a - * null terminated string and smaller than XBC_DATA_MAX. + * This parses the boot config text in @data. @size must be smaller + * than XBC_DATA_MAX. * Return the number of stored nodes (>0) if succeeded, or -errno * if there is any error. * In error cases, @emsg will be updated with an error message and * @epos will be updated with the error position which is the byte offset * of @buf. If the error is not a parser error, @epos will be -1. */ -int __init xbc_init(char *buf, const char **emsg, int *epos) +int __init xbc_init(const char *data, size_t size, const char **emsg, int *epos) { - char *p, *q; - int ret, c; + int ret; if (epos) *epos = -1; @@ -784,69 +929,33 @@ int __init xbc_init(char *buf, const char **emsg, int *epos) *emsg = "Bootconfig is already initialized"; return -EBUSY; } - - ret = strlen(buf); - if (ret > XBC_DATA_MAX - 1 || ret == 0) { + if (size > XBC_DATA_MAX || size == 0) { if (emsg) - *emsg = ret ? "Config data is too big" : + *emsg = size ? "Config data is too big" : "Config data is empty"; return -ERANGE; } - xbc_nodes = memblock_alloc(sizeof(struct xbc_node) * XBC_NODE_MAX, - SMP_CACHE_BYTES); + xbc_data = xbc_alloc_mem(size + 1); + if (!xbc_data) { + if (emsg) + *emsg = "Failed to allocate bootconfig data"; + return -ENOMEM; + } + memcpy(xbc_data, data, size); + xbc_data[size] = '\0'; + xbc_data_size = size + 1; + + xbc_nodes = xbc_alloc_mem(sizeof(struct xbc_node) * XBC_NODE_MAX); if (!xbc_nodes) { if (emsg) *emsg = "Failed to allocate bootconfig nodes"; + xbc_exit(); return -ENOMEM; } memset(xbc_nodes, 0, sizeof(struct xbc_node) * XBC_NODE_MAX); - xbc_data = buf; - xbc_data_size = ret + 1; - last_parent = NULL; - - p = buf; - do { - q = strpbrk(p, "{}=+;:\n#"); - if (!q) { - p = skip_spaces(p); - if (*p != '\0') - ret = xbc_parse_error("No delimiter", p); - break; - } - - c = *q; - *q++ = '\0'; - switch (c) { - case ':': - case '+': - if (*q++ != '=') { - ret = xbc_parse_error(c == '+' ? - "Wrong '+' operator" : - "Wrong ':' operator", - q - 2); - break; - } - fallthrough; - case '=': - ret = xbc_parse_kv(&p, q, c); - break; - case '{': - ret = xbc_open_brace(&p, q); - break; - case '#': - q = skip_comment(q); - fallthrough; - case ';': - case '\n': - ret = xbc_parse_key(&p, q); - break; - case '}': - ret = xbc_close_brace(&p, q); - break; - } - } while (!ret); + ret = xbc_parse_tree(); if (!ret) ret = xbc_verify_tree(); @@ -855,27 +964,9 @@ int __init xbc_init(char *buf, const char **emsg, int *epos) *epos = xbc_err_pos; if (emsg) *emsg = xbc_err_msg; - xbc_destroy_all(); + xbc_exit(); } else ret = xbc_node_num; return ret; } - -/** - * xbc_debug_dump() - Dump current XBC node list - * - * Dump the current XBC node list on printk buffer for debug. - */ -void __init xbc_debug_dump(void) -{ - int i; - - for (i = 0; i < xbc_node_num; i++) { - pr_debug("[%d] %s (%s) .next=%d, .child=%d .parent=%d\n", i, - xbc_node_get_data(xbc_nodes + i), - xbc_node_is_value(xbc_nodes + i) ? "value" : "key", - xbc_nodes[i].next, xbc_nodes[i].child, - xbc_nodes[i].parent); - } -} diff --git a/lib/buildid.c b/lib/buildid.c index 6156997c3895..dfc62625cae4 100644 --- a/lib/buildid.c +++ b/lib/buildid.c @@ -1,36 +1,31 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/buildid.h> +#include <linux/cache.h> #include <linux/elf.h> +#include <linux/kernel.h> #include <linux/pagemap.h> #define BUILD_ID 3 + /* * Parse build id from the note segment. This logic can be shared between * 32-bit and 64-bit system, because Elf32_Nhdr and Elf64_Nhdr are * identical. */ -static inline int parse_build_id(void *page_addr, - unsigned char *build_id, - __u32 *size, - void *note_start, - Elf32_Word note_size) +static int parse_build_id_buf(unsigned char *build_id, + __u32 *size, + const void *note_start, + Elf32_Word note_size) { Elf32_Word note_offs = 0, new_offs; - /* check for overflow */ - if (note_start < page_addr || note_start + note_size < note_start) - return -EINVAL; - - /* only supports note that fits in the first page */ - if (note_start + note_size > page_addr + PAGE_SIZE) - return -EINVAL; - while (note_offs + sizeof(Elf32_Nhdr) < note_size) { Elf32_Nhdr *nhdr = (Elf32_Nhdr *)(note_start + note_offs); if (nhdr->n_type == BUILD_ID && nhdr->n_namesz == sizeof("GNU") && + !strcmp((char *)(nhdr + 1), "GNU") && nhdr->n_descsz > 0 && nhdr->n_descsz <= BUILD_ID_SIZE_MAX) { memcpy(build_id, @@ -49,11 +44,29 @@ static inline int parse_build_id(void *page_addr, break; note_offs = new_offs; } + return -EINVAL; } +static inline int parse_build_id(const void *page_addr, + unsigned char *build_id, + __u32 *size, + const void *note_start, + Elf32_Word note_size) +{ + /* check for overflow */ + if (note_start < page_addr || note_start + note_size < note_start) + return -EINVAL; + + /* only supports note that fits in the first page */ + if (note_start + note_size > page_addr + PAGE_SIZE) + return -EINVAL; + + return parse_build_id_buf(build_id, size, note_start, note_size); +} + /* Parse build ID from 32-bit ELF */ -static int get_build_id_32(void *page_addr, unsigned char *build_id, +static int get_build_id_32(const void *page_addr, unsigned char *build_id, __u32 *size) { Elf32_Ehdr *ehdr = (Elf32_Ehdr *)page_addr; @@ -78,7 +91,7 @@ static int get_build_id_32(void *page_addr, unsigned char *build_id, } /* Parse build ID from 64-bit ELF */ -static int get_build_id_64(void *page_addr, unsigned char *build_id, +static int get_build_id_64(const void *page_addr, unsigned char *build_id, __u32 *size) { Elf64_Ehdr *ehdr = (Elf64_Ehdr *)page_addr; @@ -108,7 +121,7 @@ static int get_build_id_64(void *page_addr, unsigned char *build_id, * @build_id: buffer to store build id, at least BUILD_ID_SIZE long * @size: returns actual build id size in case of success * - * Returns 0 on success, otherwise error (< 0). + * Return: 0 on success, -EINVAL otherwise */ int build_id_parse(struct vm_area_struct *vma, unsigned char *build_id, __u32 *size) @@ -147,3 +160,32 @@ out: put_page(page); return ret; } + +/** + * build_id_parse_buf - Get build ID from a buffer + * @buf: Elf note section(s) to parse + * @buf_size: Size of @buf in bytes + * @build_id: Build ID parsed from @buf, at least BUILD_ID_SIZE_MAX long + * + * Return: 0 on success, -EINVAL otherwise + */ +int build_id_parse_buf(const void *buf, unsigned char *build_id, u32 buf_size) +{ + return parse_build_id_buf(build_id, NULL, buf, buf_size); +} + +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) || IS_ENABLED(CONFIG_CRASH_CORE) +unsigned char vmlinux_build_id[BUILD_ID_SIZE_MAX] __ro_after_init; + +/** + * init_vmlinux_build_id - Compute and stash the running kernel's build ID + */ +void __init init_vmlinux_build_id(void) +{ + extern const void __start_notes __weak; + extern const void __stop_notes __weak; + unsigned int size = &__stop_notes - &__start_notes; + + build_id_parse_buf(&__start_notes, vmlinux_build_id, size); +} +#endif diff --git a/lib/cmdline_kunit.c b/lib/cmdline_kunit.c index 018bfc8113c4..a72a2c16066e 100644 --- a/lib/cmdline_kunit.c +++ b/lib/cmdline_kunit.c @@ -124,7 +124,7 @@ static void cmdline_do_one_range_test(struct kunit *test, const char *in, n, e[0], r[0]); p = memchr_inv(&r[1], 0, sizeof(r) - sizeof(r[0])); - KUNIT_EXPECT_PTR_EQ_MSG(test, p, (int *)0, "in test %u at %u out of bound", n, p - r); + KUNIT_EXPECT_PTR_EQ_MSG(test, p, NULL, "in test %u at %u out of bound", n, p - r); } static void cmdline_test_range(struct kunit *test) diff --git a/lib/compat_audit.c b/lib/compat_audit.c index 77eabad69b4a..3d6b8996f027 100644 --- a/lib/compat_audit.c +++ b/lib/compat_audit.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include <linux/init.h> #include <linux/types.h> +#include <linux/audit_arch.h> #include <asm/unistd32.h> unsigned compat_dir_class[] = { @@ -33,19 +34,23 @@ int audit_classify_compat_syscall(int abi, unsigned syscall) switch (syscall) { #ifdef __NR_open case __NR_open: - return 2; + return AUDITSC_OPEN; #endif #ifdef __NR_openat case __NR_openat: - return 3; + return AUDITSC_OPENAT; #endif #ifdef __NR_socketcall case __NR_socketcall: - return 4; + return AUDITSC_SOCKETCALL; #endif case __NR_execve: - return 5; + return AUDITSC_EXECVE; +#ifdef __NR_openat2 + case __NR_openat2: + return AUDITSC_OPENAT2; +#endif default: - return 1; + return AUDITSC_COMPAT; } } diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 14c032de276e..545ccbddf6a1 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -128,3 +128,6 @@ config CRYPTO_LIB_CHACHA20POLY1305 config CRYPTO_LIB_SHA256 tristate + +config CRYPTO_LIB_SM4 + tristate diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 3a435629d9ce..73205ed269ba 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -38,6 +38,9 @@ libpoly1305-y += poly1305.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o +obj-$(CONFIG_CRYPTO_LIB_SM4) += libsm4.o +libsm4-y := sm4.o + ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) libblake2s-y += blake2s-selftest.o libchacha20poly1305-y += chacha20poly1305-selftest.o diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index c64ac8bfb6a9..4055aa593ec4 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -73,7 +73,7 @@ void blake2s256_hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen, } EXPORT_SYMBOL(blake2s256_hmac); -static int __init mod_init(void) +static int __init blake2s_mod_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && WARN_ON(!blake2s_selftest())) @@ -81,12 +81,12 @@ static int __init mod_init(void) return 0; } -static void __exit mod_exit(void) +static void __exit blake2s_mod_exit(void) { } -module_init(mod_init); -module_exit(mod_exit); +module_init(blake2s_mod_init); +module_exit(blake2s_mod_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("BLAKE2s hash function"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index c2fcdb98cc02..fa6a9440fc95 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -354,7 +354,7 @@ bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len } EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); -static int __init mod_init(void) +static int __init chacha20poly1305_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && WARN_ON(!chacha20poly1305_selftest())) @@ -362,12 +362,12 @@ static int __init mod_init(void) return 0; } -static void __exit mod_exit(void) +static void __exit chacha20poly1305_exit(void) { } -module_init(mod_init); -module_exit(mod_exit); +module_init(chacha20poly1305_init); +module_exit(chacha20poly1305_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction"); MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>"); diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index fb29739e8c29..064b352c6907 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -13,7 +13,7 @@ #include <linux/module.h> #include <linux/init.h> -static int __init mod_init(void) +static int __init curve25519_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && WARN_ON(!curve25519_selftest())) @@ -21,12 +21,12 @@ static int __init mod_init(void) return 0; } -static void __exit mod_exit(void) +static void __exit curve25519_exit(void) { } -module_init(mod_init); -module_exit(mod_exit); +module_init(curve25519_init); +module_exit(curve25519_exit); MODULE_LICENSE("GPL v2"); MODULE_DESCRIPTION("Curve25519 scalar multiplication"); diff --git a/lib/crypto/sm4.c b/lib/crypto/sm4.c new file mode 100644 index 000000000000..284e62576d0c --- /dev/null +++ b/lib/crypto/sm4.c @@ -0,0 +1,176 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SM4, as specified in + * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html + * + * Copyright (C) 2018 ARM Limited or its affiliates. + * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> + */ + +#include <linux/module.h> +#include <asm/unaligned.h> +#include <crypto/sm4.h> + +static const u32 fk[4] = { + 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc +}; + +static const u32 ____cacheline_aligned ck[32] = { + 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269, + 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9, + 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249, + 0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9, + 0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229, + 0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299, + 0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209, + 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279 +}; + +static const u8 ____cacheline_aligned sbox[256] = { + 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, + 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05, + 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, + 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99, + 0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, + 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62, + 0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, + 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6, + 0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, + 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8, + 0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, + 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35, + 0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, + 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87, + 0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, + 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e, + 0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, + 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1, + 0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, + 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3, + 0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, + 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f, + 0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, + 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51, + 0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, + 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8, + 0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, + 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0, + 0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, + 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84, + 0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, + 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48 +}; + +static inline u32 sm4_t_non_lin_sub(u32 x) +{ + u32 out; + + out = (u32)sbox[x & 0xff]; + out |= (u32)sbox[(x >> 8) & 0xff] << 8; + out |= (u32)sbox[(x >> 16) & 0xff] << 16; + out |= (u32)sbox[(x >> 24) & 0xff] << 24; + + return out; +} + +static inline u32 sm4_key_lin_sub(u32 x) +{ + return x ^ rol32(x, 13) ^ rol32(x, 23); +} + +static inline u32 sm4_enc_lin_sub(u32 x) +{ + return x ^ rol32(x, 2) ^ rol32(x, 10) ^ rol32(x, 18) ^ rol32(x, 24); +} + +static inline u32 sm4_key_sub(u32 x) +{ + return sm4_key_lin_sub(sm4_t_non_lin_sub(x)); +} + +static inline u32 sm4_enc_sub(u32 x) +{ + return sm4_enc_lin_sub(sm4_t_non_lin_sub(x)); +} + +static inline u32 sm4_round(u32 x0, u32 x1, u32 x2, u32 x3, u32 rk) +{ + return x0 ^ sm4_enc_sub(x1 ^ x2 ^ x3 ^ rk); +} + + +/** + * sm4_expandkey - Expands the SM4 key as described in GB/T 32907-2016 + * @ctx: The location where the computed key will be stored. + * @in_key: The supplied key. + * @key_len: The length of the supplied key. + * + * Returns 0 on success. The function fails only if an invalid key size (or + * pointer) is supplied. + */ +int sm4_expandkey(struct sm4_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + u32 rk[4]; + const u32 *key = (u32 *)in_key; + int i; + + if (key_len != SM4_KEY_SIZE) + return -EINVAL; + + rk[0] = get_unaligned_be32(&key[0]) ^ fk[0]; + rk[1] = get_unaligned_be32(&key[1]) ^ fk[1]; + rk[2] = get_unaligned_be32(&key[2]) ^ fk[2]; + rk[3] = get_unaligned_be32(&key[3]) ^ fk[3]; + + for (i = 0; i < 32; i += 4) { + rk[0] ^= sm4_key_sub(rk[1] ^ rk[2] ^ rk[3] ^ ck[i + 0]); + rk[1] ^= sm4_key_sub(rk[2] ^ rk[3] ^ rk[0] ^ ck[i + 1]); + rk[2] ^= sm4_key_sub(rk[3] ^ rk[0] ^ rk[1] ^ ck[i + 2]); + rk[3] ^= sm4_key_sub(rk[0] ^ rk[1] ^ rk[2] ^ ck[i + 3]); + + ctx->rkey_enc[i + 0] = rk[0]; + ctx->rkey_enc[i + 1] = rk[1]; + ctx->rkey_enc[i + 2] = rk[2]; + ctx->rkey_enc[i + 3] = rk[3]; + ctx->rkey_dec[31 - 0 - i] = rk[0]; + ctx->rkey_dec[31 - 1 - i] = rk[1]; + ctx->rkey_dec[31 - 2 - i] = rk[2]; + ctx->rkey_dec[31 - 3 - i] = rk[3]; + } + + return 0; +} +EXPORT_SYMBOL_GPL(sm4_expandkey); + +/** + * sm4_crypt_block - Encrypt or decrypt a single SM4 block + * @rk: The rkey_enc for encrypt or rkey_dec for decrypt + * @out: Buffer to store output data + * @in: Buffer containing the input data + */ +void sm4_crypt_block(const u32 *rk, u8 *out, const u8 *in) +{ + u32 x[4], i; + + x[0] = get_unaligned_be32(in + 0 * 4); + x[1] = get_unaligned_be32(in + 1 * 4); + x[2] = get_unaligned_be32(in + 2 * 4); + x[3] = get_unaligned_be32(in + 3 * 4); + + for (i = 0; i < 32; i += 4) { + x[0] = sm4_round(x[0], x[1], x[2], x[3], rk[i + 0]); + x[1] = sm4_round(x[1], x[2], x[3], x[0], rk[i + 1]); + x[2] = sm4_round(x[2], x[3], x[0], x[1], rk[i + 2]); + x[3] = sm4_round(x[3], x[0], x[1], x[2], rk[i + 3]); + } + + put_unaligned_be32(x[3 - 0], out + 0 * 4); + put_unaligned_be32(x[3 - 1], out + 1 * 4); + put_unaligned_be32(x[3 - 2], out + 2 * 4); + put_unaligned_be32(x[3 - 3], out + 3 * 4); +} +EXPORT_SYMBOL_GPL(sm4_crypt_block); + +MODULE_DESCRIPTION("Generic SM4 library"); +MODULE_LICENSE("GPL v2"); diff --git a/lib/debug_info.c b/lib/debug_info.c index 36daf753293c..cc4723c74af5 100644 --- a/lib/debug_info.c +++ b/lib/debug_info.c @@ -5,8 +5,6 @@ * CONFIG_DEBUG_INFO_REDUCED. Please do not add actual code. However, * adding appropriate #includes is fine. */ -#include <stdarg.h> - #include <linux/cred.h> #include <linux/crypto.h> #include <linux/dcache.h> @@ -22,6 +20,7 @@ #include <linux/net.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/stdarg.h> #include <linux/types.h> #include <net/addrconf.h> #include <net/sock.h> diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 9e14ae02306b..6946f8e204e3 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -557,7 +557,12 @@ __debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack struct debug_obj *obj; unsigned long flags; - fill_pool(); + /* + * On RT enabled kernels the pool refill must happen in preemptible + * context: + */ + if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible()) + fill_pool(); db = get_bucket((unsigned long) addr); diff --git a/lib/decompress_bunzip2.c b/lib/decompress_bunzip2.c index c72c865032fa..3518e7394eca 100644 --- a/lib/decompress_bunzip2.c +++ b/lib/decompress_bunzip2.c @@ -80,7 +80,7 @@ /* This is what we know about each Huffman coding group */ struct group_data { - /* We have an extra slot at the end of limit[] for a sentinal value. */ + /* We have an extra slot at the end of limit[] for a sentinel value. */ int limit[MAX_HUFCODE_BITS+1]; int base[MAX_HUFCODE_BITS]; int permute[MAX_SYMBOLS]; @@ -337,7 +337,7 @@ static int INIT get_next_block(struct bunzip_data *bd) pp <<= 1; base[i+1] = pp-(t += temp[i]); } - limit[maxLen+1] = INT_MAX; /* Sentinal value for + limit[maxLen+1] = INT_MAX; /* Sentinel value for * reading next sym. */ limit[maxLen] = pp+temp[maxLen]-1; base[minLen] = 0; @@ -385,7 +385,7 @@ static int INIT get_next_block(struct bunzip_data *bd) bd->inbufBits = (bd->inbufBits << 8)|bd->inbuf[bd->inbufPos++]; bd->inbufBitCount += 8; - }; + } bd->inbufBitCount -= hufGroup->maxLen; j = (bd->inbufBits >> bd->inbufBitCount)& ((1 << hufGroup->maxLen)-1); diff --git a/lib/decompress_unlz4.c b/lib/decompress_unlz4.c index c0cfcfd486be..e6327391b6b6 100644 --- a/lib/decompress_unlz4.c +++ b/lib/decompress_unlz4.c @@ -112,6 +112,9 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, error("data corrupted"); goto exit_2; } + } else if (size < 4) { + /* empty or end-of-file */ + goto exit_3; } chunksize = get_unaligned_le32(inp); @@ -125,6 +128,10 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, continue; } + if (!fill && chunksize == 0) { + /* empty or end-of-file */ + goto exit_3; + } if (posp) *posp += 4; @@ -184,6 +191,7 @@ STATIC inline int INIT unlz4(u8 *input, long in_len, } } +exit_3: ret = 0; exit_2: if (!input) diff --git a/lib/decompress_unlzo.c b/lib/decompress_unlzo.c index 1f439a622076..64c1358500ce 100644 --- a/lib/decompress_unlzo.c +++ b/lib/decompress_unlzo.c @@ -43,7 +43,6 @@ STATIC inline long INIT parse_header(u8 *input, long *skip, long in_len) int l; u8 *parse = input; u8 *end = input + in_len; - u8 level = 0; u16 version; /* @@ -65,7 +64,7 @@ STATIC inline long INIT parse_header(u8 *input, long *skip, long in_len) version = get_unaligned_be16(parse); parse += 7; if (version >= 0x0940) - level = *parse++; + parse++; if (get_unaligned_be32(parse) & HEADER_HAS_FILTER) parse += 8; /* flags + filter info */ else diff --git a/lib/decompress_unxz.c b/lib/decompress_unxz.c index 25d59a95bd66..9f4262ee33a5 100644 --- a/lib/decompress_unxz.c +++ b/lib/decompress_unxz.c @@ -20,10 +20,10 @@ * * The worst case for in-place decompression is that the beginning of * the file is compressed extremely well, and the rest of the file is - * uncompressible. Thus, we must look for worst-case expansion when the - * compressor is encoding uncompressible data. + * incompressible. Thus, we must look for worst-case expansion when the + * compressor is encoding incompressible data. * - * The structure of the .xz file in case of a compresed kernel is as follows. + * The structure of the .xz file in case of a compressed kernel is as follows. * Sizes (as bytes) of the fields are in parenthesis. * * Stream Header (12) @@ -58,7 +58,7 @@ * uncompressed size of the payload is in practice never less than the * payload size itself. The LZMA2 format would allow uncompressed size * to be less than the payload size, but no sane compressor creates such - * files. LZMA2 supports storing uncompressible data in uncompressed form, + * files. LZMA2 supports storing incompressible data in uncompressed form, * so there's never a need to create payloads whose uncompressed size is * smaller than the compressed size. * @@ -167,8 +167,8 @@ * memeq and memzero are not used much and any remotely sane implementation * is fast enough. memcpy/memmove speed matters in multi-call mode, but * the kernel image is decompressed in single-call mode, in which only - * memcpy speed can matter and only if there is a lot of uncompressible data - * (LZMA2 stores uncompressible chunks in uncompressed form). Thus, the + * memmove speed can matter and only if there is a lot of incompressible data + * (LZMA2 stores incompressible chunks in uncompressed form). Thus, the * functions below should just be kept small; it's probably not worth * optimizing for speed. */ diff --git a/lib/decompress_unzstd.c b/lib/decompress_unzstd.c index 790abc472f5b..6b629ab31c1e 100644 --- a/lib/decompress_unzstd.c +++ b/lib/decompress_unzstd.c @@ -16,7 +16,7 @@ * uncompressible. Thus, we must look for worst-case expansion when the * compressor is encoding uncompressible data. * - * The structure of the .zst file in case of a compresed kernel is as follows. + * The structure of the .zst file in case of a compressed kernel is as follows. * Maximum sizes (as bytes) of the fields are in parenthesis. * * Frame Header: (18) @@ -56,7 +56,7 @@ /* * Preboot environments #include "path/to/decompress_unzstd.c". * All of the source files we depend on must be #included. - * zstd's only source dependeny is xxhash, which has no source + * zstd's only source dependency is xxhash, which has no source * dependencies. * * When UNZSTD_PREBOOT is defined we declare __decompress(), which is diff --git a/lib/devmem_is_allowed.c b/lib/devmem_is_allowed.c index c0d67c541849..60be9e24bd57 100644 --- a/lib/devmem_is_allowed.c +++ b/lib/devmem_is_allowed.c @@ -19,7 +19,7 @@ */ int devmem_is_allowed(unsigned long pfn) { - if (iomem_is_exclusive(pfn << PAGE_SHIFT)) + if (iomem_is_exclusive(PFN_PHYS(pfn))) return 0; if (!page_is_ram(pfn)) return 1; diff --git a/lib/devres.c b/lib/devres.c index 4679dbb1bf5f..b0e1c6702c71 100644 --- a/lib/devres.c +++ b/lib/devres.c @@ -157,8 +157,10 @@ __devm_ioremap_resource(struct device *dev, const struct resource *res, dev_name(dev), res->name); else pretty_name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL); - if (!pretty_name) + if (!pretty_name) { + dev_err(dev, "can't generate pretty name for resource %pR\n", res); return IOMEM_ERR_PTR(-ENOMEM); + } if (!devm_request_mem_region(dev, res->start, size, pretty_name)) { dev_err(dev, "can't request region for resource %pR\n", res); @@ -353,7 +355,7 @@ static void pcim_iomap_release(struct device *gendev, void *res) * detach. * * This function might sleep when the table is first allocated but can - * be safely called without context and guaranteed to succed once + * be safely called without context and guaranteed to succeed once * allocated. */ void __iomem * const *pcim_iomap_table(struct pci_dev *pdev) diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 27f16872320d..6b7f1bf6715d 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -5,6 +5,7 @@ */ #include <linux/kernel.h> +#include <linux/buildid.h> #include <linux/export.h> #include <linux/sched.h> #include <linux/sched/debug.h> @@ -36,6 +37,14 @@ void __init dump_stack_set_arch_desc(const char *fmt, ...) va_end(args); } +#if IS_ENABLED(CONFIG_STACKTRACE_BUILD_ID) +#define BUILD_ID_FMT " %20phN" +#define BUILD_ID_VAL vmlinux_build_id +#else +#define BUILD_ID_FMT "%s" +#define BUILD_ID_VAL "" +#endif + /** * dump_stack_print_info - print generic debug info for dump_stack() * @log_lvl: log level @@ -45,13 +54,13 @@ void __init dump_stack_set_arch_desc(const char *fmt, ...) */ void dump_stack_print_info(const char *log_lvl) { - printk("%sCPU: %d PID: %d Comm: %.20s %s%s %s %.*s\n", + printk("%sCPU: %d PID: %d Comm: %.20s %s%s %s %.*s" BUILD_ID_FMT "\n", log_lvl, raw_smp_processor_id(), current->pid, current->comm, kexec_crash_loaded() ? "Kdump: loaded " : "", print_tainted(), init_utsname()->release, (int)strcspn(init_utsname()->version, " "), - init_utsname()->version); + init_utsname()->version, BUILD_ID_VAL); if (dump_stack_arch_desc_str[0] != '\0') printk("%sHardware name: %s\n", @@ -80,7 +89,8 @@ static void __dump_stack(const char *log_lvl) } /** - * dump_stack - dump the current task information and its stack trace + * dump_stack_lvl - dump the current task information and its stack trace + * @log_lvl: log level * * Architectures can override this implementation by implementing its own. */ diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index 641767b0dce2..cb5abb42c16a 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -991,7 +991,7 @@ static int ddebug_dyndbg_param_cb(char *param, char *val, ddebug_exec_queries((val ? val : "+p"), modname); - return 0; /* query failure shouldnt stop module load */ + return 0; /* query failure shouldn't stop module load */ } /* handle both dyndbg and $module.dyndbg params at boot */ @@ -1117,9 +1117,9 @@ static int __init dynamic_debug_init(void) goto out_err; ddebug_init_success = 1; - vpr_info("%d modules, %d entries and %d bytes in ddebug tables, %d bytes in __dyndbg section\n", - modct, entries, (int)(modct * sizeof(struct ddebug_table)), - (int)(entries * sizeof(struct _ddebug))); + vpr_info("%d prdebugs in %d modules, %d KiB in ddebug tables, %d kiB in __dyndbg section\n", + entries, modct, (int)((modct * sizeof(struct ddebug_table)) >> 10), + (int)((entries * sizeof(struct _ddebug)) >> 10)); /* apply ddebug_query boot param, dont unload tables on err */ if (ddebug_setup_string[0] != '\0') { diff --git a/lib/error-inject.c b/lib/error-inject.c index c73651b15b76..2ff5ef689d72 100644 --- a/lib/error-inject.c +++ b/lib/error-inject.c @@ -8,6 +8,7 @@ #include <linux/mutex.h> #include <linux/list.h> #include <linux/slab.h> +#include <asm/sections.h> /* Whitelist of symbols that can be overridden for error injection. */ static LIST_HEAD(error_injection_list); @@ -64,7 +65,7 @@ static void populate_error_injection_list(struct error_injection_entry *start, mutex_lock(&ei_mutex); for (iter = start; iter < end; iter++) { - entry = arch_deref_entry_point((void *)iter->addr); + entry = (unsigned long)dereference_symbol_descriptor((void *)iter->addr); if (!kernel_text_address(entry) || !kallsyms_lookup_size_offset(entry, &size, &offset)) { diff --git a/lib/flex_proportions.c b/lib/flex_proportions.c index 451543937524..53e7eb1dd76c 100644 --- a/lib/flex_proportions.c +++ b/lib/flex_proportions.c @@ -217,11 +217,12 @@ static void fprop_reflect_period_percpu(struct fprop_global *p, } /* Event of type pl happened */ -void __fprop_inc_percpu(struct fprop_global *p, struct fprop_local_percpu *pl) +void __fprop_add_percpu(struct fprop_global *p, struct fprop_local_percpu *pl, + long nr) { fprop_reflect_period_percpu(p, pl); - percpu_counter_add_batch(&pl->events, 1, PROP_BATCH); - percpu_counter_add(&p->events, 1); + percpu_counter_add_batch(&pl->events, nr, PROP_BATCH); + percpu_counter_add(&p->events, nr); } void fprop_fraction_percpu(struct fprop_global *p, @@ -253,20 +254,29 @@ void fprop_fraction_percpu(struct fprop_global *p, } /* - * Like __fprop_inc_percpu() except that event is counted only if the given + * Like __fprop_add_percpu() except that event is counted only if the given * type has fraction smaller than @max_frac/FPROP_FRAC_BASE */ -void __fprop_inc_percpu_max(struct fprop_global *p, - struct fprop_local_percpu *pl, int max_frac) +void __fprop_add_percpu_max(struct fprop_global *p, + struct fprop_local_percpu *pl, int max_frac, long nr) { if (unlikely(max_frac < FPROP_FRAC_BASE)) { unsigned long numerator, denominator; + s64 tmp; fprop_fraction_percpu(p, pl, &numerator, &denominator); - if (numerator > - (((u64)denominator) * max_frac) >> FPROP_FRAC_SHIFT) + /* Adding 'nr' to fraction exceeds max_frac/FPROP_FRAC_BASE? */ + tmp = (u64)denominator * max_frac - + ((u64)numerator << FPROP_FRAC_SHIFT); + if (tmp < 0) { + /* Maximum fraction already exceeded? */ return; + } else if (tmp < nr * (FPROP_FRAC_BASE - max_frac)) { + /* Add just enough for the fraction to saturate */ + nr = div_u64(tmp + FPROP_FRAC_BASE - max_frac - 1, + FPROP_FRAC_BASE - max_frac); + } } - __fprop_inc_percpu(p, pl); + __fprop_add_percpu(p, pl, nr); } diff --git a/lib/fonts/font_pearl_8x8.c b/lib/fonts/font_pearl_8x8.c index b1678ed0017c..ae98ca17982e 100644 --- a/lib/fonts/font_pearl_8x8.c +++ b/lib/fonts/font_pearl_8x8.c @@ -3,7 +3,7 @@ /* */ /* Font file generated by cpi2fnt */ /* ------------------------------ */ -/* Combined with the alpha-numeric */ +/* Combined with the alphanumeric */ /* portion of Greg Harp's old PEARL */ /* font (from earlier versions of */ /* linux-m86k) by John Shifflett */ diff --git a/lib/iov_iter.c b/lib/iov_iter.c index c701b7a187f2..755c10c5138c 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -16,170 +16,137 @@ #define PIPE_PARANOIA /* for now */ -#define iterate_iovec(i, n, __v, __p, skip, STEP) { \ - size_t left; \ - size_t wanted = n; \ - __p = i->iov; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } else { \ - left = 0; \ - } \ - while (unlikely(!left && n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - left = (STEP); \ - __v.iov_len -= left; \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted - n; \ -} - -#define iterate_kvec(i, n, __v, __p, skip, STEP) { \ - size_t wanted = n; \ - __p = i->kvec; \ - __v.iov_len = min(n, __p->iov_len - skip); \ - if (likely(__v.iov_len)) { \ - __v.iov_base = __p->iov_base + skip; \ - (void)(STEP); \ - skip += __v.iov_len; \ - n -= __v.iov_len; \ - } \ - while (unlikely(n)) { \ - __p++; \ - __v.iov_len = min(n, __p->iov_len); \ - if (unlikely(!__v.iov_len)) \ - continue; \ - __v.iov_base = __p->iov_base; \ - (void)(STEP); \ - skip = __v.iov_len; \ - n -= __v.iov_len; \ - } \ - n = wanted; \ -} - -#define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ - struct bvec_iter __start; \ - __start.bi_size = n; \ - __start.bi_bvec_done = skip; \ - __start.bi_idx = 0; \ - for_each_bvec(__v, i->bvec, __bi, __start) { \ - (void)(STEP); \ - } \ -} - -#define iterate_xarray(i, n, __v, skip, STEP) { \ +/* covers iovec and kvec alike */ +#define iterate_iovec(i, n, base, len, off, __p, STEP) { \ + size_t off = 0; \ + size_t skip = i->iov_offset; \ + do { \ + len = min(n, __p->iov_len - skip); \ + if (likely(len)) { \ + base = __p->iov_base + skip; \ + len -= (STEP); \ + off += len; \ + skip += len; \ + n -= len; \ + if (skip < __p->iov_len) \ + break; \ + } \ + __p++; \ + skip = 0; \ + } while (n); \ + i->iov_offset = skip; \ + n = off; \ +} + +#define iterate_bvec(i, n, base, len, off, p, STEP) { \ + size_t off = 0; \ + unsigned skip = i->iov_offset; \ + while (n) { \ + unsigned offset = p->bv_offset + skip; \ + unsigned left; \ + void *kaddr = kmap_local_page(p->bv_page + \ + offset / PAGE_SIZE); \ + base = kaddr + offset % PAGE_SIZE; \ + len = min(min(n, (size_t)(p->bv_len - skip)), \ + (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ + left = (STEP); \ + kunmap_local(kaddr); \ + len -= left; \ + off += len; \ + skip += len; \ + if (skip == p->bv_len) { \ + skip = 0; \ + p++; \ + } \ + n -= len; \ + if (left) \ + break; \ + } \ + i->iov_offset = skip; \ + n = off; \ +} + +#define iterate_xarray(i, n, base, len, __off, STEP) { \ + __label__ __out; \ + size_t __off = 0; \ struct page *head = NULL; \ - size_t wanted = n, seg, offset; \ - loff_t start = i->xarray_start + skip; \ - pgoff_t index = start >> PAGE_SHIFT; \ + loff_t start = i->xarray_start + i->iov_offset; \ + unsigned offset = start % PAGE_SIZE; \ + pgoff_t index = start / PAGE_SIZE; \ int j; \ \ XA_STATE(xas, i->xarray, index); \ \ - rcu_read_lock(); \ - xas_for_each(&xas, head, ULONG_MAX) { \ - if (xas_retry(&xas, head)) \ - continue; \ - if (WARN_ON(xa_is_value(head))) \ - break; \ - if (WARN_ON(PageHuge(head))) \ - break; \ + rcu_read_lock(); \ + xas_for_each(&xas, head, ULONG_MAX) { \ + unsigned left; \ + if (xas_retry(&xas, head)) \ + continue; \ + if (WARN_ON(xa_is_value(head))) \ + break; \ + if (WARN_ON(PageHuge(head))) \ + break; \ for (j = (head->index < index) ? index - head->index : 0; \ - j < thp_nr_pages(head); j++) { \ - __v.bv_page = head + j; \ - offset = (i->xarray_start + skip) & ~PAGE_MASK; \ - seg = PAGE_SIZE - offset; \ - __v.bv_offset = offset; \ - __v.bv_len = min(n, seg); \ - (void)(STEP); \ - n -= __v.bv_len; \ - skip += __v.bv_len; \ - if (n == 0) \ - break; \ - } \ - if (n == 0) \ - break; \ - } \ - rcu_read_unlock(); \ - n = wanted - n; \ -} - -#define iterate_all_kinds(i, n, v, I, B, K, X) { \ - if (likely(n)) { \ - size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ - struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - } else if (unlikely(i->type & ITER_KVEC)) { \ - const struct kvec *kvec; \ - struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - } else if (unlikely(i->type & ITER_XARRAY)) { \ - struct bio_vec v; \ - iterate_xarray(i, n, v, skip, (X)); \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ + j < thp_nr_pages(head); j++) { \ + void *kaddr = kmap_local_page(head + j); \ + base = kaddr + offset; \ + len = PAGE_SIZE - offset; \ + len = min(n, len); \ + left = (STEP); \ + kunmap_local(kaddr); \ + len -= left; \ + __off += len; \ + n -= len; \ + if (left || n == 0) \ + goto __out; \ + offset = 0; \ } \ } \ +__out: \ + rcu_read_unlock(); \ + i->iov_offset += __off; \ + n = __off; \ } -#define iterate_and_advance(i, n, v, I, B, K, X) { \ +#define __iterate_and_advance(i, n, base, len, off, I, K) { \ if (unlikely(i->count < n)) \ n = i->count; \ - if (i->count) { \ - size_t skip = i->iov_offset; \ - if (unlikely(i->type & ITER_BVEC)) { \ + if (likely(n)) { \ + if (likely(iter_is_iovec(i))) { \ + const struct iovec *iov = i->iov; \ + void __user *base; \ + size_t len; \ + iterate_iovec(i, n, base, len, off, \ + iov, (I)) \ + i->nr_segs -= iov - i->iov; \ + i->iov = iov; \ + } else if (iov_iter_is_bvec(i)) { \ const struct bio_vec *bvec = i->bvec; \ - struct bio_vec v; \ - struct bvec_iter __bi; \ - iterate_bvec(i, n, v, __bi, skip, (B)) \ - i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ - i->nr_segs -= i->bvec - bvec; \ - skip = __bi.bi_bvec_done; \ - } else if (unlikely(i->type & ITER_KVEC)) { \ - const struct kvec *kvec; \ - struct kvec v; \ - iterate_kvec(i, n, v, kvec, skip, (K)) \ - if (skip == kvec->iov_len) { \ - kvec++; \ - skip = 0; \ - } \ + void *base; \ + size_t len; \ + iterate_bvec(i, n, base, len, off, \ + bvec, (K)) \ + i->nr_segs -= bvec - i->bvec; \ + i->bvec = bvec; \ + } else if (iov_iter_is_kvec(i)) { \ + const struct kvec *kvec = i->kvec; \ + void *base; \ + size_t len; \ + iterate_iovec(i, n, base, len, off, \ + kvec, (K)) \ i->nr_segs -= kvec - i->kvec; \ i->kvec = kvec; \ - } else if (unlikely(i->type & ITER_DISCARD)) { \ - skip += n; \ - } else if (unlikely(i->type & ITER_XARRAY)) { \ - struct bio_vec v; \ - iterate_xarray(i, n, v, skip, (X)) \ - } else { \ - const struct iovec *iov; \ - struct iovec v; \ - iterate_iovec(i, n, v, iov, skip, (I)) \ - if (skip == iov->iov_len) { \ - iov++; \ - skip = 0; \ - } \ - i->nr_segs -= iov - i->iov; \ - i->iov = iov; \ + } else if (iov_iter_is_xarray(i)) { \ + void *base; \ + size_t len; \ + iterate_xarray(i, n, base, len, off, \ + (K)) \ } \ i->count -= n; \ - i->iov_offset = skip; \ } \ } +#define iterate_and_advance(i, n, base, len, off, I, K) \ + __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0)) static int copyout(void __user *to, const void *from, size_t n) { @@ -469,19 +436,25 @@ out: * Return 0 on success, or non-zero if the memory could not be accessed (i.e. * because it is an invalid address). */ -int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) +int iov_iter_fault_in_readable(const struct iov_iter *i, size_t bytes) { - size_t skip = i->iov_offset; - const struct iovec *iov; - int err; - struct iovec v; + if (iter_is_iovec(i)) { + const struct iovec *p; + size_t skip; - if (!(i->type & (ITER_BVEC|ITER_KVEC))) { - iterate_iovec(i, bytes, v, iov, skip, ({ - err = fault_in_pages_readable(v.iov_base, v.iov_len); + if (bytes > i->count) + bytes = i->count; + for (p = i->iov, skip = i->iov_offset; bytes; p++, skip = 0) { + size_t len = min(bytes, p->iov_len - skip); + int err; + + if (unlikely(!len)) + continue; + err = fault_in_pages_readable(p->iov_base + skip, len); if (unlikely(err)) - return err; - 0;})) + return err; + bytes -= len; + } } return 0; } @@ -492,19 +465,14 @@ void iov_iter_init(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - direction &= READ | WRITE; - - /* It will get better. Eventually... */ - if (uaccess_kernel()) { - i->type = ITER_KVEC | direction; - i->kvec = (struct kvec *)iov; - } else { - i->type = ITER_IOVEC | direction; - i->iov = iov; - } - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter) { + .iter_type = ITER_IOVEC, + .data_source = direction, + .iov = iov, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_init); @@ -613,55 +581,45 @@ static __wsum csum_and_memcpy(void *to, const void *from, size_t len, } static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, - struct csum_state *csstate, - struct iov_iter *i) + struct iov_iter *i, __wsum *sump) { struct pipe_inode_info *pipe = i->pipe; unsigned int p_mask = pipe->ring_size - 1; - __wsum sum = csstate->csum; - size_t off = csstate->off; + __wsum sum = *sump; + size_t off = 0; unsigned int i_head; - size_t n, r; + size_t r; if (!sanity(i)) return 0; - bytes = n = push_pipe(i, bytes, &i_head, &r); - if (unlikely(!n)) - return 0; - do { - size_t chunk = min_t(size_t, n, PAGE_SIZE - r); - char *p = kmap_atomic(pipe->bufs[i_head & p_mask].page); - sum = csum_and_memcpy(p + r, addr, chunk, sum, off); - kunmap_atomic(p); + bytes = push_pipe(i, bytes, &i_head, &r); + while (bytes) { + size_t chunk = min_t(size_t, bytes, PAGE_SIZE - r); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); + sum = csum_and_memcpy(p + r, addr + off, chunk, sum, off); + kunmap_local(p); i->head = i_head; i->iov_offset = r + chunk; - n -= chunk; + bytes -= chunk; off += chunk; - addr += chunk; r = 0; i_head++; - } while (n); - i->count -= bytes; - csstate->csum = sum; - csstate->off = off; - return bytes; + } + *sump = sum; + i->count -= off; + return off; } size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; if (unlikely(iov_iter_is_pipe(i))) return copy_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len), - memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), - memcpy_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len) + iterate_and_advance(i, bytes, base, len, off, + copyout(base, addr + off, len), + memcpy(base, addr + off, len) ) return bytes; @@ -678,19 +636,6 @@ static int copyout_mc(void __user *to, const void *from, size_t n) return n; } -static unsigned long copy_mc_to_page(struct page *page, size_t offset, - const char *from, size_t len) -{ - unsigned long ret; - char *to; - - to = kmap_atomic(page); - ret = copy_mc_to_kernel(to + offset, from, len); - kunmap_atomic(to); - - return ret; -} - static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { @@ -702,25 +647,23 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, if (!sanity(i)) return 0; - bytes = n = push_pipe(i, bytes, &i_head, &off); - if (unlikely(!n)) - return 0; - do { + n = push_pipe(i, bytes, &i_head, &off); + while (n) { size_t chunk = min_t(size_t, n, PAGE_SIZE - off); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); unsigned long rem; - - rem = copy_mc_to_page(pipe->bufs[i_head & p_mask].page, - off, addr, chunk); + rem = copy_mc_to_kernel(p + off, addr + xfer, chunk); + chunk -= rem; + kunmap_local(p); i->head = i_head; - i->iov_offset = off + chunk - rem; - xfer += chunk - rem; + i->iov_offset = off + chunk; + xfer += chunk; if (rem) break; n -= chunk; - addr += chunk; off = 0; i_head++; - } while (n); + } i->count -= xfer; return xfer; } @@ -729,7 +672,7 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, * _copy_mc_to_iter - copy to iter with source memory error exception handling * @addr: source kernel address * @bytes: total transfer length - * @iter: destination iterator + * @i: destination iterator * * The pmem driver deploys this for the dax operation * (dax_copy_to_iter()) for dax reads (bypass page-cache and the @@ -747,49 +690,18 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies. * Compare to copy_to_iter() where only ITER_IOVEC attempts might return * a short copy. + * + * Return: number of bytes copied (may be %0) */ size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) { - const char *from = addr; - unsigned long rem, curr_addr, s_addr = (unsigned long) addr; - if (unlikely(iov_iter_is_pipe(i))) return copy_mc_pipe_to_iter(addr, bytes, i); if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len, - v.iov_len), - ({ - rem = copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }), - ({ - rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len) - - v.iov_len, v.iov_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - return bytes; - } - }), - ({ - rem = copy_mc_to_page(v.bv_page, v.bv_offset, - (from += v.bv_len) - v.bv_len, v.bv_len); - if (rem) { - curr_addr = (unsigned long) from; - bytes = curr_addr - s_addr - rem; - rcu_read_unlock(); - i->iov_offset += bytes; - i->count -= bytes; - return bytes; - } - }) + __iterate_and_advance(i, bytes, base, len, off, + copyout_mc(base, addr + off, len), + copy_mc_to_kernel(base, addr + off, len) ) return bytes; @@ -799,70 +711,30 @@ EXPORT_SYMBOL_GPL(_copy_mc_to_iter); size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } if (iter_is_iovec(i)) might_fault(); - iterate_and_advance(i, bytes, v, - copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + iterate_and_advance(i, bytes, base, len, off, + copyin(addr + off, base, len), + memcpy(addr + off, base, len) ) return bytes; } EXPORT_SYMBOL(_copy_from_iter); -bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) -{ - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - - if (iter_is_iovec(i)) - might_fault(); - iterate_all_kinds(i, bytes, v, ({ - if (copyin((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) - ) - - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(_copy_from_iter_full); - size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, - __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + iterate_and_advance(i, bytes, base, len, off, + __copy_from_user_inatomic_nocache(addr + off, base, len), + memcpy(addr + off, base, len) ) return bytes; @@ -874,7 +746,7 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); * _copy_from_iter_flushcache - write destination through cpu cache * @addr: destination kernel address * @bytes: total transfer length - * @iter: source iterator + * @i: source iterator * * The pmem driver arranges for filesystem-dax to use this facility via * dax_copy_from_iter() for ensuring that writes to persistent memory @@ -883,23 +755,18 @@ EXPORT_SYMBOL(_copy_from_iter_nocache); * all iterator types. The _copy_from_iter_nocache() only attempts to * bypass the cache for the ITER_IOVEC case, and on some archs may use * instructions that strand dirty-data in the cache. + * + * Return: number of bytes copied (may be %0) */ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) { - char *to = addr; if (unlikely(iov_iter_is_pipe(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, - __copy_from_user_flushcache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, - v.iov_len), - memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + iterate_and_advance(i, bytes, base, len, off, + __copy_from_user_flushcache(addr + off, base, len), + memcpy_flushcache(addr + off, base, len) ) return bytes; @@ -907,32 +774,6 @@ size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache); #endif -bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) -{ - char *to = addr; - if (unlikely(iov_iter_is_pipe(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len)) - return false; - 0;}), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) - ) - - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(_copy_from_iter_full_nocache); - static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) { struct page *head; @@ -957,22 +798,51 @@ static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) return false; } +static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes, + struct iov_iter *i) +{ + if (likely(iter_is_iovec(i))) + return copy_page_to_iter_iovec(page, offset, bytes, i); + if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { + void *kaddr = kmap_local_page(page); + size_t wanted = _copy_to_iter(kaddr + offset, bytes, i); + kunmap_local(kaddr); + return wanted; + } + if (iov_iter_is_pipe(i)) + return copy_page_to_iter_pipe(page, offset, bytes, i); + if (unlikely(iov_iter_is_discard(i))) { + if (unlikely(i->count < bytes)) + bytes = i->count; + i->count -= bytes; + return bytes; + } + WARN_ON(1); + return 0; +} + size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { + size_t res = 0; if (unlikely(!page_copy_sane(page, offset, bytes))) return 0; - if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) { - void *kaddr = kmap_atomic(page); - size_t wanted = copy_to_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); - return wanted; - } else if (unlikely(iov_iter_is_discard(i))) - return bytes; - else if (likely(!iov_iter_is_pipe(i))) - return copy_page_to_iter_iovec(page, offset, bytes, i); - else - return copy_page_to_iter_pipe(page, offset, bytes, i); + page += offset / PAGE_SIZE; // first subpage + offset %= PAGE_SIZE; + while (1) { + size_t n = __copy_page_to_iter(page, offset, + min(bytes, (size_t)PAGE_SIZE - offset), i); + res += n; + bytes -= n; + if (!bytes || !n) + break; + offset += n; + if (offset == PAGE_SIZE) { + page++; + offset = 0; + } + } + return res; } EXPORT_SYMBOL(copy_page_to_iter); @@ -981,17 +851,16 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, { if (unlikely(!page_copy_sane(page, offset, bytes))) return 0; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return 0; - } - if (i->type & (ITER_BVEC | ITER_KVEC | ITER_XARRAY)) { - void *kaddr = kmap_atomic(page); + if (likely(iter_is_iovec(i))) + return copy_page_from_iter_iovec(page, offset, bytes, i); + if (iov_iter_is_bvec(i) || iov_iter_is_kvec(i) || iov_iter_is_xarray(i)) { + void *kaddr = kmap_local_page(page); size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); - kunmap_atomic(kaddr); + kunmap_local(kaddr); return wanted; - } else - return copy_page_from_iter_iovec(page, offset, bytes, i); + } + WARN_ON(1); + return 0; } EXPORT_SYMBOL(copy_page_from_iter); @@ -1011,7 +880,9 @@ static size_t pipe_zero(size_t bytes, struct iov_iter *i) do { size_t chunk = min_t(size_t, n, PAGE_SIZE - off); - memzero_page(pipe->bufs[i_head & p_mask].page, off, chunk); + char *p = kmap_local_page(pipe->bufs[i_head & p_mask].page); + memset(p + off, 0, chunk); + kunmap_local(p); i->head = i_head; i->iov_offset = off + chunk; n -= chunk; @@ -1026,19 +897,17 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i) { if (unlikely(iov_iter_is_pipe(i))) return pipe_zero(bytes, i); - iterate_and_advance(i, bytes, v, - clear_user(v.iov_base, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len), - memset(v.iov_base, 0, v.iov_len), - memzero_page(v.bv_page, v.bv_offset, v.bv_len) + iterate_and_advance(i, bytes, base, len, count, + clear_user(base, len), + memset(base, 0, len) ) return bytes; } EXPORT_SYMBOL(iov_iter_zero); -size_t iov_iter_copy_from_user_atomic(struct page *page, - struct iov_iter *i, unsigned long offset, size_t bytes) +size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes, + struct iov_iter *i) { char *kaddr = kmap_atomic(page), *p = kaddr + offset; if (unlikely(!page_copy_sane(page, offset, bytes))) { @@ -1050,18 +919,14 @@ size_t iov_iter_copy_from_user_atomic(struct page *page, WARN_ON(1); return 0; } - iterate_all_kinds(i, bytes, v, - copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len), - memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), - memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, - v.bv_offset, v.bv_len) + iterate_and_advance(i, bytes, base, len, off, + copyin(p + off, base, len), + memcpy(p + off, base, len) ) kunmap_atomic(kaddr); return bytes; } -EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); +EXPORT_SYMBOL(copy_page_from_iter_atomic); static inline void pipe_truncate(struct iov_iter *i) { @@ -1092,8 +957,6 @@ static inline void pipe_truncate(struct iov_iter *i) static void pipe_advance(struct iov_iter *i, size_t size) { struct pipe_inode_info *pipe = i->pipe; - if (unlikely(i->count < size)) - size = i->count; if (size) { struct pipe_buffer *buf; unsigned int p_mask = pipe->ring_size - 1; @@ -1132,27 +995,42 @@ static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) i->iov_offset = bi.bi_bvec_done; } -void iov_iter_advance(struct iov_iter *i, size_t size) +static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) { - if (unlikely(iov_iter_is_pipe(i))) { - pipe_advance(i, size); - return; - } - if (unlikely(iov_iter_is_discard(i))) { - i->count -= size; + const struct iovec *iov, *end; + + if (!i->count) return; + i->count -= size; + + size += i->iov_offset; // from beginning of current segment + for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) { + if (likely(size < iov->iov_len)) + break; + size -= iov->iov_len; } - if (unlikely(iov_iter_is_xarray(i))) { - size = min(size, i->count); + i->iov_offset = size; + i->nr_segs -= iov - i->iov; + i->iov = iov; +} + +void iov_iter_advance(struct iov_iter *i, size_t size) +{ + if (unlikely(i->count < size)) + size = i->count; + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { + /* iovec and kvec have identical layouts */ + iov_iter_iovec_advance(i, size); + } else if (iov_iter_is_bvec(i)) { + iov_iter_bvec_advance(i, size); + } else if (iov_iter_is_pipe(i)) { + pipe_advance(i, size); + } else if (unlikely(iov_iter_is_xarray(i))) { i->iov_offset += size; i->count -= size; - return; - } - if (iov_iter_is_bvec(i)) { - iov_iter_bvec_advance(i, size); - return; + } else if (iov_iter_is_discard(i)) { + i->count -= size; } - iterate_and_advance(i, size, v, 0, 0, 0, 0) } EXPORT_SYMBOL(iov_iter_advance); @@ -1234,16 +1112,13 @@ EXPORT_SYMBOL(iov_iter_revert); */ size_t iov_iter_single_seg_count(const struct iov_iter *i) { - if (unlikely(iov_iter_is_pipe(i))) - return i->count; // it is a silly place, anyway - if (i->nr_segs == 1) - return i->count; - if (unlikely(iov_iter_is_discard(i) || iov_iter_is_xarray(i))) - return i->count; - if (iov_iter_is_bvec(i)) - return min(i->count, i->bvec->bv_len - i->iov_offset); - else - return min(i->count, i->iov->iov_len - i->iov_offset); + if (i->nr_segs > 1) { + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return min(i->count, i->iov->iov_len - i->iov_offset); + if (iov_iter_is_bvec(i)) + return min(i->count, i->bvec->bv_len - i->iov_offset); + } + return i->count; } EXPORT_SYMBOL(iov_iter_single_seg_count); @@ -1252,11 +1127,14 @@ void iov_iter_kvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_KVEC | (direction & (READ | WRITE)); - i->kvec = kvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_KVEC, + .data_source = direction, + .kvec = kvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_kvec); @@ -1265,11 +1143,14 @@ void iov_iter_bvec(struct iov_iter *i, unsigned int direction, size_t count) { WARN_ON(direction & ~(READ | WRITE)); - i->type = ITER_BVEC | (direction & (READ | WRITE)); - i->bvec = bvec; - i->nr_segs = nr_segs; - i->iov_offset = 0; - i->count = count; + *i = (struct iov_iter){ + .iter_type = ITER_BVEC, + .data_source = direction, + .bvec = bvec, + .nr_segs = nr_segs, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_bvec); @@ -1279,12 +1160,15 @@ void iov_iter_pipe(struct iov_iter *i, unsigned int direction, { BUG_ON(direction != READ); WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size)); - i->type = ITER_PIPE | READ; - i->pipe = pipe; - i->head = pipe->head; - i->iov_offset = 0; - i->count = count; - i->start_head = i->head; + *i = (struct iov_iter){ + .iter_type = ITER_PIPE, + .data_source = false, + .pipe = pipe, + .head = pipe->head, + .start_head = pipe->head, + .iov_offset = 0, + .count = count + }; } EXPORT_SYMBOL(iov_iter_pipe); @@ -1305,11 +1189,14 @@ void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray, loff_t start, size_t count) { BUG_ON(direction & ~1); - i->type = ITER_XARRAY | (direction & (READ | WRITE)); - i->xarray = xarray; - i->xarray_start = start; - i->count = count; - i->iov_offset = 0; + *i = (struct iov_iter) { + .iter_type = ITER_XARRAY, + .data_source = direction, + .xarray = xarray, + .xarray_start = start, + .count = count, + .iov_offset = 0 + }; } EXPORT_SYMBOL(iov_iter_xarray); @@ -1325,56 +1212,103 @@ EXPORT_SYMBOL(iov_iter_xarray); void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) { BUG_ON(direction != READ); - i->type = ITER_DISCARD | READ; - i->count = count; - i->iov_offset = 0; + *i = (struct iov_iter){ + .iter_type = ITER_DISCARD, + .data_source = false, + .count = count, + .iov_offset = 0 + }; } EXPORT_SYMBOL(iov_iter_discard); -unsigned long iov_iter_alignment(const struct iov_iter *i) +static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) { unsigned long res = 0; size_t size = i->count; + size_t skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->iov[k].iov_len - skip; + if (len) { + res |= (unsigned long)i->iov[k].iov_base + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + } + return res; +} - if (unlikely(iov_iter_is_pipe(i))) { +static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) +{ + unsigned res = 0; + size_t size = i->count; + unsigned skip = i->iov_offset; + unsigned k; + + for (k = 0; k < i->nr_segs; k++, skip = 0) { + size_t len = i->bvec[k].bv_len - skip; + res |= (unsigned long)i->bvec[k].bv_offset + skip; + if (len > size) + len = size; + res |= len; + size -= len; + if (!size) + break; + } + return res; +} + +unsigned long iov_iter_alignment(const struct iov_iter *i) +{ + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_iter_alignment_iovec(i); + + if (iov_iter_is_bvec(i)) + return iov_iter_alignment_bvec(i); + + if (iov_iter_is_pipe(i)) { unsigned int p_mask = i->pipe->ring_size - 1; + size_t size = i->count; if (size && i->iov_offset && allocated(&i->pipe->bufs[i->head & p_mask])) return size | i->iov_offset; return size; } - if (unlikely(iov_iter_is_xarray(i))) + + if (iov_iter_is_xarray(i)) return (i->xarray_start + i->iov_offset) | i->count; - iterate_all_kinds(i, size, v, - (res |= (unsigned long)v.iov_base | v.iov_len, 0), - res |= v.bv_offset | v.bv_len, - res |= (unsigned long)v.iov_base | v.iov_len, - res |= v.bv_offset | v.bv_len - ) - return res; + + return 0; } EXPORT_SYMBOL(iov_iter_alignment); unsigned long iov_iter_gap_alignment(const struct iov_iter *i) { unsigned long res = 0; + unsigned long v = 0; size_t size = i->count; + unsigned k; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); + if (WARN_ON(!iter_is_iovec(i))) return ~0U; - } - iterate_all_kinds(i, size, v, - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0), 0), - (res |= (!res ? 0 : (unsigned long)v.bv_offset) | - (size != v.bv_len ? size : 0)), - (res |= (!res ? 0 : (unsigned long)v.iov_base) | - (size != v.iov_len ? size : 0)), - (res |= (!res ? 0 : (unsigned long)v.bv_offset) | - (size != v.bv_len ? size : 0)) - ); + for (k = 0; k < i->nr_segs; k++) { + if (i->iov[k].iov_len) { + unsigned long base = (unsigned long)i->iov[k].iov_base; + if (v) // if not the first one + res |= base | v; // this start | previous end + v = base + i->iov[k].iov_len; + if (size <= i->iov[k].iov_len) + break; + size -= i->iov[k].iov_len; + } + } return res; } EXPORT_SYMBOL(iov_iter_gap_alignment); @@ -1409,9 +1343,6 @@ static ssize_t pipe_get_pages(struct iov_iter *i, unsigned int iter_head, npages; size_t capacity; - if (!maxsize) - return 0; - if (!sanity(i)) return -EFAULT; @@ -1492,29 +1423,67 @@ static ssize_t iter_xarray_get_pages(struct iov_iter *i, return actual; } +/* must be done on non-empty ITER_IOVEC one */ +static unsigned long first_iovec_segment(const struct iov_iter *i, + size_t *size, size_t *start, + size_t maxsize, unsigned maxpages) +{ + size_t skip; + long k; + + for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { + unsigned long addr = (unsigned long)i->iov[k].iov_base + skip; + size_t len = i->iov[k].iov_len - skip; + + if (unlikely(!len)) + continue; + if (len > maxsize) + len = maxsize; + len += (*start = addr % PAGE_SIZE); + if (len > maxpages * PAGE_SIZE) + len = maxpages * PAGE_SIZE; + *size = len; + return addr & PAGE_MASK; + } + BUG(); // if it had been empty, we wouldn't get called +} + +/* must be done on non-empty ITER_BVEC one */ +static struct page *first_bvec_segment(const struct iov_iter *i, + size_t *size, size_t *start, + size_t maxsize, unsigned maxpages) +{ + struct page *page; + size_t skip = i->iov_offset, len; + + len = i->bvec->bv_len - skip; + if (len > maxsize) + len = maxsize; + skip += i->bvec->bv_offset; + page = i->bvec->bv_page + skip / PAGE_SIZE; + len += (*start = skip % PAGE_SIZE); + if (len > maxpages * PAGE_SIZE) + len = maxpages * PAGE_SIZE; + *size = len; + return page; +} + ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages, size_t maxsize, unsigned maxpages, size_t *start) { + size_t len; + int n, res; + if (maxsize > i->count) maxsize = i->count; + if (!maxsize) + return 0; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages(i, pages, maxsize, maxpages, start); - if (unlikely(iov_iter_is_xarray(i))) - return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; - - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; - int res; + if (likely(iter_is_iovec(i))) { + unsigned long addr; - if (len > maxpages * PAGE_SIZE) - len = maxpages * PAGE_SIZE; - addr &= ~(PAGE_SIZE - 1); + addr = first_iovec_segment(i, &len, start, maxsize, maxpages); n = DIV_ROUND_UP(len, PAGE_SIZE); res = get_user_pages_fast(addr, n, iov_iter_rw(i) != WRITE ? FOLL_WRITE : 0, @@ -1522,17 +1491,21 @@ ssize_t iov_iter_get_pages(struct iov_iter *i, if (unlikely(res < 0)) return res; return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - get_page(*pages = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }), - 0 - ) - return 0; + } + if (iov_iter_is_bvec(i)) { + struct page *page; + + page = first_bvec_segment(i, &len, start, maxsize, maxpages); + n = DIV_ROUND_UP(len, PAGE_SIZE); + while (n--) + get_page(*pages++ = page++); + return len - *start; + } + if (iov_iter_is_pipe(i)) + return pipe_get_pages(i, pages, maxsize, maxpages, start); + if (iov_iter_is_xarray(i)) + return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); + return -EFAULT; } EXPORT_SYMBOL(iov_iter_get_pages); @@ -1549,9 +1522,6 @@ static ssize_t pipe_get_pages_alloc(struct iov_iter *i, unsigned int iter_head, npages; ssize_t n; - if (!maxsize) - return 0; - if (!sanity(i)) return -EFAULT; @@ -1624,24 +1594,18 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, size_t *start) { struct page **p; + size_t len; + int n, res; if (maxsize > i->count) maxsize = i->count; + if (!maxsize) + return 0; - if (unlikely(iov_iter_is_pipe(i))) - return pipe_get_pages_alloc(i, pages, maxsize, start); - if (unlikely(iov_iter_is_xarray(i))) - return iter_xarray_get_pages_alloc(i, pages, maxsize, start); - if (unlikely(iov_iter_is_discard(i))) - return -EFAULT; - - iterate_all_kinds(i, maxsize, v, ({ - unsigned long addr = (unsigned long)v.iov_base; - size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); - int n; - int res; + if (likely(iter_is_iovec(i))) { + unsigned long addr; - addr &= ~(PAGE_SIZE - 1); + addr = first_iovec_segment(i, &len, start, maxsize, ~0U); n = DIV_ROUND_UP(len, PAGE_SIZE); p = get_pages_array(n); if (!p) @@ -1654,61 +1618,42 @@ ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, } *pages = p; return (res == n ? len : res * PAGE_SIZE) - *start; - 0;}),({ - /* can't be more than PAGE_SIZE */ - *start = v.bv_offset; - *pages = p = get_pages_array(1); + } + if (iov_iter_is_bvec(i)) { + struct page *page; + + page = first_bvec_segment(i, &len, start, maxsize, ~0U); + n = DIV_ROUND_UP(len, PAGE_SIZE); + *pages = p = get_pages_array(n); if (!p) return -ENOMEM; - get_page(*p = v.bv_page); - return v.bv_len; - }),({ - return -EFAULT; - }), 0 - ) - return 0; + while (n--) + get_page(*p++ = page++); + return len - *start; + } + if (iov_iter_is_pipe(i)) + return pipe_get_pages_alloc(i, pages, maxsize, start); + if (iov_iter_is_xarray(i)) + return iter_xarray_get_pages_alloc(i, pages, maxsize, start); + return -EFAULT; } EXPORT_SYMBOL(iov_iter_get_pages_alloc); size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i) { - char *to = addr; __wsum sum, next; - size_t off = 0; sum = *csum; if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { WARN_ON(1); return 0; } - iterate_and_advance(i, bytes, v, ({ - next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len); - if (next) { - sum = csum_block_add(sum, next, off); - off += v.iov_len; - } - next ? 0 : v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, - sum, off); - off += v.iov_len; + iterate_and_advance(i, bytes, base, len, off, ({ + next = csum_and_copy_from_user(base, addr + off, len); + sum = csum_block_add(sum, next, off); + next ? 0 : len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; + sum = csum_and_memcpy(addr + off, base, len, sum, off); }) ) *csum = sum; @@ -1716,104 +1661,30 @@ size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, } EXPORT_SYMBOL(csum_and_copy_from_iter); -bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, - struct iov_iter *i) -{ - char *to = addr; - __wsum sum, next; - size_t off = 0; - sum = *csum; - if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { - WARN_ON(1); - return false; - } - if (unlikely(i->count < bytes)) - return false; - iterate_all_kinds(i, bytes, v, ({ - next = csum_and_copy_from_user(v.iov_base, - (to += v.iov_len) - v.iov_len, - v.iov_len); - if (!next) - return false; - sum = csum_block_add(sum, next, off); - off += v.iov_len; - 0; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy((to += v.iov_len) - v.iov_len, - v.iov_base, v.iov_len, - sum, off); - off += v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy((to += v.bv_len) - v.bv_len, - p + v.bv_offset, v.bv_len, - sum, off); - kunmap_atomic(p); - off += v.bv_len; - }) - ) - *csum = sum; - iov_iter_advance(i, bytes); - return true; -} -EXPORT_SYMBOL(csum_and_copy_from_iter_full); - size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, struct iov_iter *i) { struct csum_state *csstate = _csstate; - const char *from = addr; __wsum sum, next; - size_t off; - - if (unlikely(iov_iter_is_pipe(i))) - return csum_and_copy_to_pipe_iter(addr, bytes, _csstate, i); - sum = csstate->csum; - off = csstate->off; if (unlikely(iov_iter_is_discard(i))) { WARN_ON(1); /* for now */ return 0; } - iterate_and_advance(i, bytes, v, ({ - next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, - v.iov_base, - v.iov_len); - if (next) { - sum = csum_block_add(sum, next, off); - off += v.iov_len; - } - next ? 0 : v.iov_len; - }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; - }),({ - sum = csum_and_memcpy(v.iov_base, - (from += v.iov_len) - v.iov_len, - v.iov_len, sum, off); - off += v.iov_len; + + sum = csum_shift(csstate->csum, csstate->off); + if (unlikely(iov_iter_is_pipe(i))) + bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum); + else iterate_and_advance(i, bytes, base, len, off, ({ + next = csum_and_copy_to_user(addr + off, base, len); + sum = csum_block_add(sum, next, off); + next ? 0 : len; }), ({ - char *p = kmap_atomic(v.bv_page); - sum = csum_and_memcpy(p + v.bv_offset, - (from += v.bv_len) - v.bv_len, - v.bv_len, sum, off); - kunmap_atomic(p); - off += v.bv_len; + sum = csum_and_memcpy(base, addr + off, len, sum, off); }) ) - csstate->csum = sum; - csstate->off = off; + csstate->csum = csum_shift(sum, csstate->off); + csstate->off += bytes; return bytes; } EXPORT_SYMBOL(csum_and_copy_to_iter); @@ -1837,19 +1708,56 @@ size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, } EXPORT_SYMBOL(hash_and_copy_to_iter); -int iov_iter_npages(const struct iov_iter *i, int maxpages) +static int iov_npages(const struct iov_iter *i, int maxpages) { - size_t size = i->count; + size_t skip = i->iov_offset, size = i->count; + const struct iovec *p; int npages = 0; - if (!size) - return 0; - if (unlikely(iov_iter_is_discard(i))) - return 0; + for (p = i->iov; size; skip = 0, p++) { + unsigned offs = offset_in_page(p->iov_base + skip); + size_t len = min(p->iov_len - skip, size); - if (unlikely(iov_iter_is_pipe(i))) { - struct pipe_inode_info *pipe = i->pipe; + if (len) { + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) + return maxpages; + } + } + return npages; +} + +static int bvec_npages(const struct iov_iter *i, int maxpages) +{ + size_t skip = i->iov_offset, size = i->count; + const struct bio_vec *p; + int npages = 0; + + for (p = i->bvec; size; skip = 0, p++) { + unsigned offs = (p->bv_offset + skip) % PAGE_SIZE; + size_t len = min(p->bv_len - skip, size); + + size -= len; + npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); + if (unlikely(npages > maxpages)) + return maxpages; + } + return npages; +} + +int iov_iter_npages(const struct iov_iter *i, int maxpages) +{ + if (unlikely(!i->count)) + return 0; + /* iovec and kvec have identical layouts */ + if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) + return iov_npages(i, maxpages); + if (iov_iter_is_bvec(i)) + return bvec_npages(i, maxpages); + if (iov_iter_is_pipe(i)) { unsigned int iter_head; + int npages; size_t off; if (!sanity(i)) @@ -1857,44 +1765,15 @@ int iov_iter_npages(const struct iov_iter *i, int maxpages) data_start(i, &iter_head, &off); /* some of this one + all after this one */ - npages = pipe_space_for_user(iter_head, pipe->tail, pipe); - if (npages >= maxpages) - return maxpages; - } else if (unlikely(iov_iter_is_xarray(i))) { - unsigned offset; - - offset = (i->xarray_start + i->iov_offset) & ~PAGE_MASK; - - npages = 1; - if (size > PAGE_SIZE - offset) { - size -= PAGE_SIZE - offset; - npages += size >> PAGE_SHIFT; - size &= ~PAGE_MASK; - if (size) - npages++; - } - if (npages >= maxpages) - return maxpages; - } else iterate_all_kinds(i, size, v, ({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) - return maxpages; - 0;}),({ - npages++; - if (npages >= maxpages) - return maxpages; - }),({ - unsigned long p = (unsigned long)v.iov_base; - npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) - - p / PAGE_SIZE; - if (npages >= maxpages) - return maxpages; - }), - 0 - ) - return npages; + npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); + return min(npages, maxpages); + } + if (iov_iter_is_xarray(i)) { + unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; + int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); + return min(npages, maxpages); + } + return 0; } EXPORT_SYMBOL(iov_iter_npages); @@ -2094,29 +1973,38 @@ int import_single_range(int rw, void __user *buf, size_t len, } EXPORT_SYMBOL(import_single_range); -int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, - int (*f)(struct kvec *vec, void *context), - void *context) +/** + * iov_iter_restore() - Restore a &struct iov_iter to the same state as when + * iov_iter_save_state() was called. + * + * @i: &struct iov_iter to restore + * @state: state to restore from + * + * Used after iov_iter_save_state() to bring restore @i, if operations may + * have advanced it. + * + * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC + */ +void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) { - struct kvec w; - int err = -EINVAL; - if (!bytes) - return 0; - - iterate_all_kinds(i, bytes, v, -EINVAL, ({ - w.iov_base = kmap(v.bv_page) + v.bv_offset; - w.iov_len = v.bv_len; - err = f(&w, context); - kunmap(v.bv_page); - err;}), ({ - w = v; - err = f(&w, context);}), ({ - w.iov_base = kmap(v.bv_page) + v.bv_offset; - w.iov_len = v.bv_len; - err = f(&w, context); - kunmap(v.bv_page); - err;}) - ) - return err; + if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && + !iov_iter_is_kvec(i)) + return; + i->iov_offset = state->iov_offset; + i->count = state->count; + /* + * For the *vec iters, nr_segs + iov is constant - if we increment + * the vec, then we also decrement the nr_segs count. Hence we don't + * need to track both of these, just one is enough and we can deduct + * the other from that. ITER_KVEC and ITER_IOVEC are the same struct + * size, so we can just increment the iov pointer as they are unionzed. + * ITER_BVEC _may_ be the same size on some archs, but on others it is + * not. Be safe and handle it separately. + */ + BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); + if (iov_iter_is_bvec(i)) + i->bvec -= state->nr_segs - i->nr_segs; + else + i->iov -= state->nr_segs - i->nr_segs; + i->nr_segs = state->nr_segs; } -EXPORT_SYMBOL(iov_iter_for_each_range); diff --git a/lib/kasprintf.c b/lib/kasprintf.c index bacf7b83ccf0..cd2f5974ed98 100644 --- a/lib/kasprintf.c +++ b/lib/kasprintf.c @@ -5,7 +5,7 @@ * Copyright (C) 1991, 1992 Linus Torvalds */ -#include <stdarg.h> +#include <linux/stdarg.h> #include <linux/export.h> #include <linux/slab.h> #include <linux/types.h> diff --git a/lib/kfifo.c b/lib/kfifo.c index 70dab9ac7827..12f5a347aa13 100644 --- a/lib/kfifo.c +++ b/lib/kfifo.c @@ -415,7 +415,7 @@ static unsigned int __kfifo_peek_n(struct __kfifo *fifo, size_t recsize) ) /* - * __kfifo_poke_n internal helper function for storeing the length of + * __kfifo_poke_n internal helper function for storing the length of * the record into the fifo */ static void __kfifo_poke_n(struct __kfifo *fifo, unsigned int n, size_t recsize) diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 0b5fe8b41173..059b8b00dc53 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -14,11 +14,12 @@ */ #include <linux/ctype.h> #include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/math64.h> #include <linux/export.h> +#include <linux/kstrtox.h> +#include <linux/math64.h> #include <linux/types.h> #include <linux/uaccess.h> + #include "kstrtox.h" const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index 9214c493d8b7..b71db0abc12b 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -64,7 +64,7 @@ static int debugfs_print_results(struct seq_file *seq, void *v) debugfs_print_result(seq, suite, test_case); seq_printf(seq, "%s %d - %s\n", - kunit_status_to_string(success), 1, suite->name); + kunit_status_to_ok_not_ok(success), 1, suite->name); return 0; } diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 15832ed44668..acd1de436f59 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 +#include <linux/reboot.h> #include <kunit/test.h> #include <linux/glob.h> #include <linux/moduleparam.h> @@ -13,13 +14,17 @@ extern struct kunit_suite * const * const __kunit_suites_end[]; #if IS_BUILTIN(CONFIG_KUNIT) -static char *filter_glob; -module_param(filter_glob, charp, 0); +static char *filter_glob_param; +module_param_named(filter_glob, filter_glob_param, charp, 0); MODULE_PARM_DESC(filter_glob, "Filter which KUnit test suites run at boot-time, e.g. list*"); +static char *kunit_shutdown; +core_param(kunit_shutdown, kunit_shutdown, charp, 0644); + static struct kunit_suite * const * -kunit_filter_subsuite(struct kunit_suite * const * const subsuite) +kunit_filter_subsuite(struct kunit_suite * const * const subsuite, + const char *filter_glob) { int i, n = 0; struct kunit_suite **filtered; @@ -52,19 +57,14 @@ struct suite_set { struct kunit_suite * const * const *end; }; -static struct suite_set kunit_filter_suites(void) +static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, + const char *filter_glob) { int i; struct kunit_suite * const **copy, * const *filtered_subsuite; struct suite_set filtered; - const size_t max = __kunit_suites_end - __kunit_suites_start; - - if (!filter_glob) { - filtered.start = __kunit_suites_start; - filtered.end = __kunit_suites_end; - return filtered; - } + const size_t max = suite_set->end - suite_set->start; copy = kmalloc_array(max, sizeof(*filtered.start), GFP_KERNEL); filtered.start = copy; @@ -74,7 +74,7 @@ static struct suite_set kunit_filter_suites(void) } for (i = 0; i < max; ++i) { - filtered_subsuite = kunit_filter_subsuite(__kunit_suites_start[i]); + filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], filter_glob); if (filtered_subsuite) *copy++ = filtered_subsuite; } @@ -82,6 +82,20 @@ static struct suite_set kunit_filter_suites(void) return filtered; } +static void kunit_handle_shutdown(void) +{ + if (!kunit_shutdown) + return; + + if (!strcmp(kunit_shutdown, "poweroff")) + kernel_power_off(); + else if (!strcmp(kunit_shutdown, "halt")) + kernel_halt(); + else if (!strcmp(kunit_shutdown, "reboot")) + kernel_restart(NULL); + +} + static void kunit_print_tap_header(struct suite_set *suite_set) { struct kunit_suite * const * const *suites, * const *subsuite; @@ -98,21 +112,32 @@ static void kunit_print_tap_header(struct suite_set *suite_set) int kunit_run_all_tests(void) { struct kunit_suite * const * const *suites; + struct suite_set suite_set = { + .start = __kunit_suites_start, + .end = __kunit_suites_end, + }; - struct suite_set suite_set = kunit_filter_suites(); + if (filter_glob_param) + suite_set = kunit_filter_suites(&suite_set, filter_glob_param); kunit_print_tap_header(&suite_set); for (suites = suite_set.start; suites < suite_set.end; suites++) __kunit_test_suites_init(*suites); - if (filter_glob) { /* a copy was made of each array */ + if (filter_glob_param) { /* a copy was made of each array */ for (suites = suite_set.start; suites < suite_set.end; suites++) kfree(*suites); kfree(suite_set.start); } + kunit_handle_shutdown(); + return 0; } +#if IS_BUILTIN(CONFIG_KUNIT_TEST) +#include "executor_test.c" +#endif + #endif /* IS_BUILTIN(CONFIG_KUNIT) */ diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c new file mode 100644 index 000000000000..e14a18af573d --- /dev/null +++ b/lib/kunit/executor_test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for the KUnit executor. + * + * Copyright (C) 2021, Google LLC. + * Author: Daniel Latypov <dlatypov@google.com> + */ + +#include <kunit/test.h> + +static void kfree_at_end(struct kunit *test, const void *to_free); +static struct kunit_suite *alloc_fake_suite(struct kunit *test, + const char *suite_name); + +static void filter_subsuite_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; + struct kunit_suite * const *filtered; + + subsuite[0] = alloc_fake_suite(test, "suite1"); + subsuite[1] = alloc_fake_suite(test, "suite2"); + + /* Want: suite1, suite2, NULL -> suite2, NULL */ + filtered = kunit_filter_subsuite(subsuite, "suite2*"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered); + kfree_at_end(test, filtered); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered[0]); + KUNIT_EXPECT_STREQ(test, (const char *)filtered[0]->name, "suite2"); + + KUNIT_EXPECT_FALSE(test, filtered[1]); +} + +static void filter_subsuite_to_empty_test(struct kunit *test) +{ + struct kunit_suite *subsuite[3] = {NULL, NULL, NULL}; + struct kunit_suite * const *filtered; + + subsuite[0] = alloc_fake_suite(test, "suite1"); + subsuite[1] = alloc_fake_suite(test, "suite2"); + + filtered = kunit_filter_subsuite(subsuite, "not_found"); + kfree_at_end(test, filtered); /* just in case */ + + KUNIT_EXPECT_FALSE_MSG(test, filtered, + "should be NULL to indicate no match"); +} + +static void kfree_subsuites_at_end(struct kunit *test, struct suite_set *suite_set) +{ + struct kunit_suite * const * const *suites; + + kfree_at_end(test, suite_set->start); + for (suites = suite_set->start; suites < suite_set->end; suites++) + kfree_at_end(test, *suites); +} + +static void filter_suites_test(struct kunit *test) +{ + /* Suites per-file are stored as a NULL terminated array */ + struct kunit_suite *subsuites[2][2] = { + {NULL, NULL}, + {NULL, NULL}, + }; + /* Match the memory layout of suite_set */ + struct kunit_suite * const * const suites[2] = { + subsuites[0], subsuites[1], + }; + + const struct suite_set suite_set = { + .start = suites, + .end = suites + 2, + }; + struct suite_set filtered = {.start = NULL, .end = NULL}; + + /* Emulate two files, each having one suite */ + subsuites[0][0] = alloc_fake_suite(test, "suite0"); + subsuites[1][0] = alloc_fake_suite(test, "suite1"); + + /* Filter out suite1 */ + filtered = kunit_filter_suites(&suite_set, "suite0"); + kfree_subsuites_at_end(test, &filtered); /* let us use ASSERTs without leaking */ + KUNIT_ASSERT_EQ(test, filtered.end - filtered.start, (ptrdiff_t)1); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start[0]); + KUNIT_EXPECT_STREQ(test, (const char *)filtered.start[0][0]->name, "suite0"); +} + +static struct kunit_case executor_test_cases[] = { + KUNIT_CASE(filter_subsuite_test), + KUNIT_CASE(filter_subsuite_to_empty_test), + KUNIT_CASE(filter_suites_test), + {} +}; + +static struct kunit_suite executor_test_suite = { + .name = "kunit_executor_test", + .test_cases = executor_test_cases, +}; + +kunit_test_suites(&executor_test_suite); + +/* Test helpers */ + +static void kfree_res_free(struct kunit_resource *res) +{ + kfree(res->data); +} + +/* Use the resource API to register a call to kfree(to_free). + * Since we never actually use the resource, it's safe to use on const data. + */ +static void kfree_at_end(struct kunit *test, const void *to_free) +{ + /* kfree() handles NULL already, but avoid allocating a no-op cleanup. */ + if (IS_ERR_OR_NULL(to_free)) + return; + kunit_alloc_resource(test, NULL, kfree_res_free, GFP_KERNEL, + (void *)to_free); +} + +static struct kunit_suite *alloc_fake_suite(struct kunit *test, + const char *suite_name) +{ + struct kunit_suite *suite; + + /* We normally never expect to allocate suites, hence the non-const cast. */ + suite = kunit_kzalloc(test, sizeof(*suite), GFP_KERNEL); + strncpy((char *)suite->name, suite_name, sizeof(suite->name) - 1); + + return suite; +} diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c index be1164ecc476..51099b0ca29c 100644 --- a/lib/kunit/kunit-example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -41,6 +41,35 @@ static int example_test_init(struct kunit *test) } /* + * This test should always be skipped. + */ +static void example_skip_test(struct kunit *test) +{ + /* This line should run */ + kunit_info(test, "You should not see a line below."); + + /* Skip (and abort) the test */ + kunit_skip(test, "this test should be skipped"); + + /* This line should not execute */ + KUNIT_FAIL(test, "You should not see this line."); +} + +/* + * This test should always be marked skipped. + */ +static void example_mark_skipped_test(struct kunit *test) +{ + /* This line should run */ + kunit_info(test, "You should see a line below."); + + /* Skip (but do not abort) the test */ + kunit_mark_skipped(test, "this test should be skipped"); + + /* This line should run */ + kunit_info(test, "You should see this line."); +} +/* * Here we make a list of all the test cases we want to add to the test suite * below. */ @@ -52,6 +81,8 @@ static struct kunit_case example_test_cases[] = { * test suite. */ KUNIT_CASE(example_simple_test), + KUNIT_CASE(example_skip_test), + KUNIT_CASE(example_mark_skipped_test), {} }; diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c index 69f902440a0e..d69efcbed624 100644 --- a/lib/kunit/kunit-test.c +++ b/lib/kunit/kunit-test.c @@ -437,7 +437,47 @@ static void kunit_log_test(struct kunit *test) #endif } +static void kunit_status_set_failure_test(struct kunit *test) +{ + struct kunit fake; + + kunit_init_test(&fake, "fake test", NULL); + + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SUCCESS); + kunit_set_failure(&fake); + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_FAILURE); +} + +static void kunit_status_mark_skipped_test(struct kunit *test) +{ + struct kunit fake; + + kunit_init_test(&fake, "fake test", NULL); + + /* Before: Should be SUCCESS with no comment. */ + KUNIT_EXPECT_EQ(test, fake.status, KUNIT_SUCCESS); + KUNIT_EXPECT_STREQ(test, fake.status_comment, ""); + + /* Mark the test as skipped. */ + kunit_mark_skipped(&fake, "Accepts format string: %s", "YES"); + + /* After: Should be SKIPPED with our comment. */ + KUNIT_EXPECT_EQ(test, fake.status, (enum kunit_status)KUNIT_SKIPPED); + KUNIT_EXPECT_STREQ(test, fake.status_comment, "Accepts format string: YES"); +} + +static struct kunit_case kunit_status_test_cases[] = { + KUNIT_CASE(kunit_status_set_failure_test), + KUNIT_CASE(kunit_status_mark_skipped_test), + {} +}; + +static struct kunit_suite kunit_status_test_suite = { + .name = "kunit_status", + .test_cases = kunit_status_test_cases, +}; + kunit_test_suites(&kunit_try_catch_test_suite, &kunit_resource_test_suite, - &kunit_log_test_suite); + &kunit_log_test_suite, &kunit_status_test_suite); MODULE_LICENSE("GPL v2"); diff --git a/lib/kunit/string-stream.h b/lib/kunit/string-stream.h index fe98a00b75a9..43f9508a55b4 100644 --- a/lib/kunit/string-stream.h +++ b/lib/kunit/string-stream.h @@ -11,7 +11,7 @@ #include <linux/spinlock.h> #include <linux/types.h> -#include <stdarg.h> +#include <linux/stdarg.h> struct string_stream_fragment { struct kunit *test; @@ -35,9 +35,9 @@ struct string_stream *alloc_string_stream(struct kunit *test, gfp_t gfp); int __printf(2, 3) string_stream_add(struct string_stream *stream, const char *fmt, ...); -int string_stream_vadd(struct string_stream *stream, - const char *fmt, - va_list args); +int __printf(2, 0) string_stream_vadd(struct string_stream *stream, + const char *fmt, + va_list args); char *string_stream_get_string(struct string_stream *stream); diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 45f068864d76..f246b847024e 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -10,6 +10,7 @@ #include <kunit/test-bug.h> #include <linux/kernel.h> #include <linux/kref.h> +#include <linux/moduleparam.h> #include <linux/sched/debug.h> #include <linux/sched.h> @@ -52,6 +53,51 @@ EXPORT_SYMBOL_GPL(__kunit_fail_current_test); #endif /* + * KUnit statistic mode: + * 0 - disabled + * 1 - only when there is more than one subtest + * 2 - enabled + */ +static int kunit_stats_enabled = 1; +module_param_named(stats_enabled, kunit_stats_enabled, int, 0644); +MODULE_PARM_DESC(stats_enabled, + "Print test stats: never (0), only for multiple subtests (1), or always (2)"); + +struct kunit_result_stats { + unsigned long passed; + unsigned long skipped; + unsigned long failed; + unsigned long total; +}; + +static bool kunit_should_print_stats(struct kunit_result_stats stats) +{ + if (kunit_stats_enabled == 0) + return false; + + if (kunit_stats_enabled == 2) + return true; + + return (stats.total > 1); +} + +static void kunit_print_test_stats(struct kunit *test, + struct kunit_result_stats stats) +{ + if (!kunit_should_print_stats(stats)) + return; + + kunit_log(KERN_INFO, test, + KUNIT_SUBTEST_INDENT + "# %s: pass:%lu fail:%lu skip:%lu total:%lu", + test->name, + stats.passed, + stats.failed, + stats.skipped, + stats.total); +} + +/* * Append formatted message to log, size of which is limited to * KUNIT_LOG_SIZE bytes (including null terminating byte). */ @@ -98,12 +144,14 @@ static void kunit_print_subtest_start(struct kunit_suite *suite) static void kunit_print_ok_not_ok(void *test_or_suite, bool is_test, - bool is_ok, + enum kunit_status status, size_t test_number, - const char *description) + const char *description, + const char *directive) { struct kunit_suite *suite = is_test ? NULL : test_or_suite; struct kunit *test = is_test ? test_or_suite : NULL; + const char *directive_header = (status == KUNIT_SKIPPED) ? " # SKIP " : ""; /* * We do not log the test suite results as doing so would @@ -114,25 +162,31 @@ static void kunit_print_ok_not_ok(void *test_or_suite, * representation. */ if (suite) - pr_info("%s %zd - %s\n", - kunit_status_to_string(is_ok), - test_number, description); + pr_info("%s %zd - %s%s%s\n", + kunit_status_to_ok_not_ok(status), + test_number, description, directive_header, + (status == KUNIT_SKIPPED) ? directive : ""); else - kunit_log(KERN_INFO, test, KUNIT_SUBTEST_INDENT "%s %zd - %s", - kunit_status_to_string(is_ok), - test_number, description); + kunit_log(KERN_INFO, test, + KUNIT_SUBTEST_INDENT "%s %zd - %s%s%s", + kunit_status_to_ok_not_ok(status), + test_number, description, directive_header, + (status == KUNIT_SKIPPED) ? directive : ""); } -bool kunit_suite_has_succeeded(struct kunit_suite *suite) +enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) { const struct kunit_case *test_case; + enum kunit_status status = KUNIT_SKIPPED; kunit_suite_for_each_test_case(suite, test_case) { - if (!test_case->success) - return false; + if (test_case->status == KUNIT_FAILURE) + return KUNIT_FAILURE; + else if (test_case->status == KUNIT_SUCCESS) + status = KUNIT_SUCCESS; } - return true; + return status; } EXPORT_SYMBOL_GPL(kunit_suite_has_succeeded); @@ -143,7 +197,8 @@ static void kunit_print_subtest_end(struct kunit_suite *suite) kunit_print_ok_not_ok((void *)suite, false, kunit_suite_has_succeeded(suite), kunit_suite_counter++, - suite->name); + suite->name, + suite->status_comment); } unsigned int kunit_test_case_num(struct kunit_suite *suite, @@ -252,7 +307,8 @@ void kunit_init_test(struct kunit *test, const char *name, char *log) test->log = log; if (test->log) test->log[0] = '\0'; - test->success = true; + test->status = KUNIT_SUCCESS; + test->status_comment[0] = '\0'; } EXPORT_SYMBOL_GPL(kunit_init_test); @@ -376,19 +432,77 @@ static void kunit_run_case_catch_errors(struct kunit_suite *suite, context.test_case = test_case; kunit_try_catch_run(try_catch, &context); - test_case->success = test->success; + /* Propagate the parameter result to the test case. */ + if (test->status == KUNIT_FAILURE) + test_case->status = KUNIT_FAILURE; + else if (test_case->status != KUNIT_FAILURE && test->status == KUNIT_SUCCESS) + test_case->status = KUNIT_SUCCESS; +} + +static void kunit_print_suite_stats(struct kunit_suite *suite, + struct kunit_result_stats suite_stats, + struct kunit_result_stats param_stats) +{ + if (kunit_should_print_stats(suite_stats)) { + kunit_log(KERN_INFO, suite, + "# %s: pass:%lu fail:%lu skip:%lu total:%lu", + suite->name, + suite_stats.passed, + suite_stats.failed, + suite_stats.skipped, + suite_stats.total); + } + + if (kunit_should_print_stats(param_stats)) { + kunit_log(KERN_INFO, suite, + "# Totals: pass:%lu fail:%lu skip:%lu total:%lu", + param_stats.passed, + param_stats.failed, + param_stats.skipped, + param_stats.total); + } +} + +static void kunit_update_stats(struct kunit_result_stats *stats, + enum kunit_status status) +{ + switch (status) { + case KUNIT_SUCCESS: + stats->passed++; + break; + case KUNIT_SKIPPED: + stats->skipped++; + break; + case KUNIT_FAILURE: + stats->failed++; + break; + } + + stats->total++; +} + +static void kunit_accumulate_stats(struct kunit_result_stats *total, + struct kunit_result_stats add) +{ + total->passed += add.passed; + total->skipped += add.skipped; + total->failed += add.failed; + total->total += add.total; } int kunit_run_tests(struct kunit_suite *suite) { char param_desc[KUNIT_PARAM_DESC_SIZE]; struct kunit_case *test_case; + struct kunit_result_stats suite_stats = { 0 }; + struct kunit_result_stats total_stats = { 0 }; kunit_print_subtest_start(suite); kunit_suite_for_each_test_case(suite, test_case) { struct kunit test = { .param_value = NULL, .param_index = 0 }; - bool test_success = true; + struct kunit_result_stats param_stats = { 0 }; + test_case->status = KUNIT_SKIPPED; if (test_case->generate_params) { /* Get initial param. */ @@ -398,7 +512,6 @@ int kunit_run_tests(struct kunit_suite *suite) do { kunit_run_case_catch_errors(suite, test_case, &test); - test_success &= test_case->success; if (test_case->generate_params) { if (param_desc[0] == '\0') { @@ -410,7 +523,7 @@ int kunit_run_tests(struct kunit_suite *suite) KUNIT_SUBTEST_INDENT "# %s: %s %d - %s", test_case->name, - kunit_status_to_string(test.success), + kunit_status_to_ok_not_ok(test.status), test.param_index + 1, param_desc); /* Get next param. */ @@ -418,13 +531,23 @@ int kunit_run_tests(struct kunit_suite *suite) test.param_value = test_case->generate_params(test.param_value, param_desc); test.param_index++; } + + kunit_update_stats(¶m_stats, test.status); + } while (test.param_value); - kunit_print_ok_not_ok(&test, true, test_success, + kunit_print_test_stats(&test, param_stats); + + kunit_print_ok_not_ok(&test, true, test_case->status, kunit_test_case_num(suite, test_case), - test_case->name); + test_case->name, + test.status_comment); + + kunit_update_stats(&suite_stats, test_case->status); + kunit_accumulate_stats(&total_stats, param_stats); } + kunit_print_suite_stats(suite, suite_stats, total_stats); kunit_print_subtest_end(suite); return 0; @@ -434,6 +557,7 @@ EXPORT_SYMBOL_GPL(kunit_run_tests); static void kunit_init_suite(struct kunit_suite *suite) { kunit_debugfs_create_suite(suite); + suite->status_comment[0] = '\0'; } int __kunit_test_suites_init(struct kunit_suite * const * const suites) @@ -576,41 +700,43 @@ int kunit_destroy_resource(struct kunit *test, kunit_resource_match_t match, } EXPORT_SYMBOL_GPL(kunit_destroy_resource); -struct kunit_kmalloc_params { +struct kunit_kmalloc_array_params { + size_t n; size_t size; gfp_t gfp; }; -static int kunit_kmalloc_init(struct kunit_resource *res, void *context) +static int kunit_kmalloc_array_init(struct kunit_resource *res, void *context) { - struct kunit_kmalloc_params *params = context; + struct kunit_kmalloc_array_params *params = context; - res->data = kmalloc(params->size, params->gfp); + res->data = kmalloc_array(params->n, params->size, params->gfp); if (!res->data) return -ENOMEM; return 0; } -static void kunit_kmalloc_free(struct kunit_resource *res) +static void kunit_kmalloc_array_free(struct kunit_resource *res) { kfree(res->data); } -void *kunit_kmalloc(struct kunit *test, size_t size, gfp_t gfp) +void *kunit_kmalloc_array(struct kunit *test, size_t n, size_t size, gfp_t gfp) { - struct kunit_kmalloc_params params = { + struct kunit_kmalloc_array_params params = { .size = size, + .n = n, .gfp = gfp }; return kunit_alloc_resource(test, - kunit_kmalloc_init, - kunit_kmalloc_free, + kunit_kmalloc_array_init, + kunit_kmalloc_array_free, gfp, ¶ms); } -EXPORT_SYMBOL_GPL(kunit_kmalloc); +EXPORT_SYMBOL_GPL(kunit_kmalloc_array); void kunit_kfree(struct kunit *test, const void *ptr) { diff --git a/lib/linear_ranges.c b/lib/linear_ranges.c index ced5c15d3f04..a1a7dfa881de 100644 --- a/lib/linear_ranges.c +++ b/lib/linear_ranges.c @@ -241,5 +241,36 @@ int linear_range_get_selector_high(const struct linear_range *r, } EXPORT_SYMBOL_GPL(linear_range_get_selector_high); +/** + * linear_range_get_selector_within - return linear range selector for value + * @r: pointer to linear range where selector is looked from + * @val: value for which the selector is searched + * @selector: address where found selector value is updated + * + * Return selector for which range value is closest match for given + * input value. Value is matching if it is equal or lower than given + * value. But return maximum selector if given value is higher than + * maximum value. + */ +void linear_range_get_selector_within(const struct linear_range *r, + unsigned int val, unsigned int *selector) +{ + if (r->min > val) { + *selector = r->min_sel; + return; + } + + if (linear_range_get_max_value(r) < val) { + *selector = r->max_sel; + return; + } + + if (r->step == 0) + *selector = r->min_sel; + else + *selector = (val - r->min) / r->step + r->min_sel; +} +EXPORT_SYMBOL_GPL(linear_range_get_selector_within); + MODULE_DESCRIPTION("linear-ranges helper"); MODULE_LICENSE("GPL"); diff --git a/lib/list_sort.c b/lib/list_sort.c index 1e1e37762799..0fb59e92ca2d 100644 --- a/lib/list_sort.c +++ b/lib/list_sort.c @@ -104,7 +104,7 @@ static void merge_final(void *priv, list_cmp_func_t cmp, struct list_head *head, * @head: the list to sort * @cmp: the elements comparison function * - * The comparison funtion @cmp must return > 0 if @a should sort after + * The comparison function @cmp must return > 0 if @a should sort after * @b ("@a > @b" if you want an ascending sort), and <= 0 if @a should * sort before @b *or* their original order should be preserved. It is * always called with the element that came first in the input in @a, diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 161108e5d2fe..71652e1c397c 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -258,7 +258,7 @@ static void init_shared_classes(void) #define WWAF(x) ww_acquire_fini(x) #define WWL(x, c) ww_mutex_lock(x, c) -#define WWT(x) ww_mutex_trylock(x) +#define WWT(x) ww_mutex_trylock(x, NULL) #define WWL1(x) ww_mutex_lock(x, NULL) #define WWU(x) ww_mutex_unlock(x) diff --git a/lib/logic_iomem.c b/lib/logic_iomem.c new file mode 100644 index 000000000000..9bdfde0c0f86 --- /dev/null +++ b/lib/logic_iomem.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Intel Corporation + * Author: Johannes Berg <johannes@sipsolutions.net> + */ +#include <linux/types.h> +#include <linux/slab.h> +#include <linux/logic_iomem.h> +#include <asm/io.h> + +struct logic_iomem_region { + const struct resource *res; + const struct logic_iomem_region_ops *ops; + struct list_head list; +}; + +struct logic_iomem_area { + const struct logic_iomem_ops *ops; + void *priv; +}; + +#define AREA_SHIFT 24 +#define MAX_AREA_SIZE (1 << AREA_SHIFT) +#define MAX_AREAS ((1ULL<<32) / MAX_AREA_SIZE) +#define AREA_BITS ((MAX_AREAS - 1) << AREA_SHIFT) +#define AREA_MASK (MAX_AREA_SIZE - 1) +#ifdef CONFIG_64BIT +#define IOREMAP_BIAS 0xDEAD000000000000UL +#define IOREMAP_MASK 0xFFFFFFFF00000000UL +#else +#define IOREMAP_BIAS 0 +#define IOREMAP_MASK 0 +#endif + +static DEFINE_MUTEX(regions_mtx); +static LIST_HEAD(regions_list); +static struct logic_iomem_area mapped_areas[MAX_AREAS]; + +int logic_iomem_add_region(struct resource *resource, + const struct logic_iomem_region_ops *ops) +{ + struct logic_iomem_region *rreg; + int err; + + if (WARN_ON(!resource || !ops)) + return -EINVAL; + + if (WARN_ON((resource->flags & IORESOURCE_TYPE_BITS) != IORESOURCE_MEM)) + return -EINVAL; + + rreg = kzalloc(sizeof(*rreg), GFP_KERNEL); + if (!rreg) + return -ENOMEM; + + err = request_resource(&iomem_resource, resource); + if (err) { + kfree(rreg); + return -ENOMEM; + } + + mutex_lock(®ions_mtx); + rreg->res = resource; + rreg->ops = ops; + list_add_tail(&rreg->list, ®ions_list); + mutex_unlock(®ions_mtx); + + return 0; +} +EXPORT_SYMBOL(logic_iomem_add_region); + +#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +static void __iomem *real_ioremap(phys_addr_t offset, size_t size) +{ + WARN(1, "invalid ioremap(0x%llx, 0x%zx)\n", + (unsigned long long)offset, size); + return NULL; +} + +static void real_iounmap(void __iomem *addr) +{ + WARN(1, "invalid iounmap for addr 0x%llx\n", + (unsigned long long __force)addr); +} +#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ + +void __iomem *ioremap(phys_addr_t offset, size_t size) +{ + void __iomem *ret = NULL; + struct logic_iomem_region *rreg, *found = NULL; + int i; + + mutex_lock(®ions_mtx); + list_for_each_entry(rreg, ®ions_list, list) { + if (rreg->res->start > offset) + continue; + if (rreg->res->end < offset + size - 1) + continue; + found = rreg; + break; + } + + if (!found) + goto out; + + for (i = 0; i < MAX_AREAS; i++) { + long offs; + + if (mapped_areas[i].ops) + continue; + + offs = rreg->ops->map(offset - found->res->start, + size, &mapped_areas[i].ops, + &mapped_areas[i].priv); + if (offs < 0) { + mapped_areas[i].ops = NULL; + break; + } + + if (WARN_ON(!mapped_areas[i].ops)) { + mapped_areas[i].ops = NULL; + break; + } + + ret = (void __iomem *)(IOREMAP_BIAS + (i << AREA_SHIFT) + offs); + break; + } +out: + mutex_unlock(®ions_mtx); + if (ret) + return ret; + return real_ioremap(offset, size); +} +EXPORT_SYMBOL(ioremap); + +static inline struct logic_iomem_area * +get_area(const volatile void __iomem *addr) +{ + unsigned long a = (unsigned long)addr; + unsigned int idx; + + if (WARN_ON((a & IOREMAP_MASK) != IOREMAP_BIAS)) + return NULL; + + idx = (a & AREA_BITS) >> AREA_SHIFT; + + if (mapped_areas[idx].ops) + return &mapped_areas[idx]; + + return NULL; +} + +void iounmap(void __iomem *addr) +{ + struct logic_iomem_area *area = get_area(addr); + + if (!area) { + real_iounmap(addr); + return; + } + + if (area->ops->unmap) + area->ops->unmap(area->priv); + + mutex_lock(®ions_mtx); + area->ops = NULL; + area->priv = NULL; + mutex_unlock(®ions_mtx); +} +EXPORT_SYMBOL(iounmap); + +#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +#define MAKE_FALLBACK(op, sz) \ +static u##sz real_raw_read ## op(const volatile void __iomem *addr) \ +{ \ + WARN(1, "Invalid read" #op " at address %llx\n", \ + (unsigned long long __force)addr); \ + return (u ## sz)~0ULL; \ +} \ + \ +static void real_raw_write ## op(u ## sz val, \ + volatile void __iomem *addr) \ +{ \ + WARN(1, "Invalid writeq" #op " of 0x%llx at address %llx\n", \ + (unsigned long long)val, (unsigned long long __force)addr);\ +} \ + +MAKE_FALLBACK(b, 8); +MAKE_FALLBACK(w, 16); +MAKE_FALLBACK(l, 32); +#ifdef CONFIG_64BIT +MAKE_FALLBACK(q, 64); +#endif + +static void real_memset_io(volatile void __iomem *addr, int value, size_t size) +{ + WARN(1, "Invalid memset_io at address 0x%llx\n", + (unsigned long long __force)addr); +} + +static void real_memcpy_fromio(void *buffer, const volatile void __iomem *addr, + size_t size) +{ + WARN(1, "Invalid memcpy_fromio at address 0x%llx\n", + (unsigned long long __force)addr); + + memset(buffer, 0xff, size); +} + +static void real_memcpy_toio(volatile void __iomem *addr, const void *buffer, + size_t size) +{ + WARN(1, "Invalid memcpy_toio at address 0x%llx\n", + (unsigned long long __force)addr); +} +#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ + +#define MAKE_OP(op, sz) \ +u##sz __raw_read ## op(const volatile void __iomem *addr) \ +{ \ + struct logic_iomem_area *area = get_area(addr); \ + \ + if (!area) \ + return real_raw_read ## op(addr); \ + \ + return (u ## sz) area->ops->read(area->priv, \ + (unsigned long)addr & AREA_MASK,\ + sz / 8); \ +} \ +EXPORT_SYMBOL(__raw_read ## op); \ + \ +void __raw_write ## op(u ## sz val, volatile void __iomem *addr) \ +{ \ + struct logic_iomem_area *area = get_area(addr); \ + \ + if (!area) { \ + real_raw_write ## op(val, addr); \ + return; \ + } \ + \ + area->ops->write(area->priv, \ + (unsigned long)addr & AREA_MASK, \ + sz / 8, val); \ +} \ +EXPORT_SYMBOL(__raw_write ## op) + +MAKE_OP(b, 8); +MAKE_OP(w, 16); +MAKE_OP(l, 32); +#ifdef CONFIG_64BIT +MAKE_OP(q, 64); +#endif + +void memset_io(volatile void __iomem *addr, int value, size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + unsigned long offs, start; + + if (!area) { + real_memset_io(addr, value, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->set) { + area->ops->set(area->priv, start, value, size); + return; + } + + for (offs = 0; offs < size; offs++) + area->ops->write(area->priv, start + offs, 1, value); +} +EXPORT_SYMBOL(memset_io); + +void memcpy_fromio(void *buffer, const volatile void __iomem *addr, + size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + u8 *buf = buffer; + unsigned long offs, start; + + if (!area) { + real_memcpy_fromio(buffer, addr, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->copy_from) { + area->ops->copy_from(area->priv, buffer, start, size); + return; + } + + for (offs = 0; offs < size; offs++) + buf[offs] = area->ops->read(area->priv, start + offs, 1); +} +EXPORT_SYMBOL(memcpy_fromio); + +void memcpy_toio(volatile void __iomem *addr, const void *buffer, size_t size) +{ + struct logic_iomem_area *area = get_area(addr); + const u8 *buf = buffer; + unsigned long offs, start; + + if (!area) { + real_memcpy_toio(addr, buffer, size); + return; + } + + start = (unsigned long)addr & AREA_MASK; + + if (area->ops->copy_to) { + area->ops->copy_to(area->priv, start, buffer, size); + return; + } + + for (offs = 0; offs < size; offs++) + area->ops->write(area->priv, start + offs, 1, buf[offs]); +} +EXPORT_SYMBOL(memcpy_toio); diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c index 8a7724a6ce2f..926f4823d5ea 100644 --- a/lib/lz4/lz4_decompress.c +++ b/lib/lz4/lz4_decompress.c @@ -481,7 +481,7 @@ int LZ4_decompress_fast(const char *source, char *dest, int originalSize) /* ===== Instantiate a few more decoding cases, used more than once. ===== */ -int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, +static int LZ4_decompress_safe_withPrefix64k(const char *source, char *dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, diff --git a/lib/math/Kconfig b/lib/math/Kconfig index f19bc9734fa7..0634b428d0cb 100644 --- a/lib/math/Kconfig +++ b/lib/math/Kconfig @@ -14,4 +14,4 @@ config PRIME_NUMBERS If unsure, say N. config RATIONAL - bool + tristate diff --git a/lib/math/Makefile b/lib/math/Makefile index 7456edb864fc..bfac26ddfc22 100644 --- a/lib/math/Makefile +++ b/lib/math/Makefile @@ -6,3 +6,4 @@ obj-$(CONFIG_PRIME_NUMBERS) += prime_numbers.o obj-$(CONFIG_RATIONAL) += rational.o obj-$(CONFIG_TEST_DIV64) += test_div64.o +obj-$(CONFIG_RATIONAL_KUNIT_TEST) += rational-test.o diff --git a/lib/math/rational-test.c b/lib/math/rational-test.c new file mode 100644 index 000000000000..01611ddff420 --- /dev/null +++ b/lib/math/rational-test.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <kunit/test.h> + +#include <linux/rational.h> + +struct rational_test_param { + unsigned long num, den; + unsigned long max_num, max_den; + unsigned long exp_num, exp_den; + + const char *name; +}; + +static const struct rational_test_param test_parameters[] = { + { 1230, 10, 100, 20, 100, 1, "Exceeds bounds, semi-convergent term > 1/2 last term" }, + { 34567,100, 120, 20, 120, 1, "Exceeds bounds, semi-convergent term < 1/2 last term" }, + { 1, 30, 100, 10, 0, 1, "Closest to zero" }, + { 1, 19, 100, 10, 1, 10, "Closest to smallest non-zero" }, + { 27,32, 16, 16, 11, 13, "Use convergent" }, + { 1155, 7735, 255, 255, 33, 221, "Exact answer" }, + { 87, 32, 70, 32, 68, 25, "Semiconvergent, numerator limit" }, + { 14533, 4626, 15000, 2400, 7433, 2366, "Semiconvergent, denominator limit" }, +}; + +static void get_desc(const struct rational_test_param *param, char *desc) +{ + strscpy(desc, param->name, KUNIT_PARAM_DESC_SIZE); +} + +/* Creates function rational_gen_params */ +KUNIT_ARRAY_PARAM(rational, test_parameters, get_desc); + +static void rational_test(struct kunit *test) +{ + const struct rational_test_param *param = (const struct rational_test_param *)test->param_value; + unsigned long n = 0, d = 0; + + rational_best_approximation(param->num, param->den, param->max_num, param->max_den, &n, &d); + KUNIT_EXPECT_EQ(test, n, param->exp_num); + KUNIT_EXPECT_EQ(test, d, param->exp_den); +} + +static struct kunit_case rational_test_cases[] = { + KUNIT_CASE_PARAM(rational_test, rational_gen_params), + {} +}; + +static struct kunit_suite rational_test_suite = { + .name = "rational", + .test_cases = rational_test_cases, +}; + +kunit_test_suites(&rational_test_suite); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/math/rational.c b/lib/math/rational.c index 9781d521963d..ec59d426ea63 100644 --- a/lib/math/rational.c +++ b/lib/math/rational.c @@ -12,6 +12,8 @@ #include <linux/compiler.h> #include <linux/export.h> #include <linux/minmax.h> +#include <linux/limits.h> +#include <linux/module.h> /* * calculate best rational approximation for a given fraction @@ -78,13 +80,18 @@ void rational_best_approximation( * found below as 't'. */ if ((n2 > max_numerator) || (d2 > max_denominator)) { - unsigned long t = min((max_numerator - n0) / n1, - (max_denominator - d0) / d1); + unsigned long t = ULONG_MAX; - /* This tests if the semi-convergent is closer - * than the previous convergent. + if (d1) + t = (max_denominator - d0) / d1; + if (n1) + t = min(t, (max_numerator - n0) / n1); + + /* This tests if the semi-convergent is closer than the previous + * convergent. If d1 is zero there is no previous convergent as this + * is the 1st iteration, so always choose the semi-convergent. */ - if (2u * t > a || (2u * t == a && d0 * dp > d1 * d)) { + if (!d1 || 2u * t > a || (2u * t == a && d0 * dp > d1 * d)) { n1 = n0 + t * n1; d1 = d0 + t * d1; } @@ -100,3 +107,5 @@ void rational_best_approximation( } EXPORT_SYMBOL(rational_best_approximation); + +MODULE_LICENSE("GPL v2"); diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c new file mode 100644 index 000000000000..62f8ffcbbaa3 --- /dev/null +++ b/lib/memcpy_kunit.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test cases for memcpy(), memmove(), and memset(). + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <kunit/test.h> +#include <linux/device.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/overflow.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/vmalloc.h> + +struct some_bytes { + union { + u8 data[32]; + struct { + u32 one; + u16 two; + u8 three; + /* 1 byte hole */ + u32 four[4]; + }; + }; +}; + +#define check(instance, v) do { \ + int i; \ + BUILD_BUG_ON(sizeof(instance.data) != 32); \ + for (i = 0; i < sizeof(instance.data); i++) { \ + KUNIT_ASSERT_EQ_MSG(test, instance.data[i], v, \ + "line %d: '%s' not initialized to 0x%02x @ %d (saw 0x%02x)\n", \ + __LINE__, #instance, v, i, instance.data[i]); \ + } \ +} while (0) + +#define compare(name, one, two) do { \ + int i; \ + BUILD_BUG_ON(sizeof(one) != sizeof(two)); \ + for (i = 0; i < sizeof(one); i++) { \ + KUNIT_EXPECT_EQ_MSG(test, one.data[i], two.data[i], \ + "line %d: %s.data[%d] (0x%02x) != %s.data[%d] (0x%02x)\n", \ + __LINE__, #one, i, one.data[i], #two, i, two.data[i]); \ + } \ + kunit_info(test, "ok: " TEST_OP "() " name "\n"); \ +} while (0) + +static void memcpy_test(struct kunit *test) +{ +#define TEST_OP "memcpy" + struct some_bytes control = { + .data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + }, + }; + struct some_bytes zero = { }; + struct some_bytes middle = { + .data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + }, + }; + struct some_bytes three = { + .data = { 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, + }, + }; + struct some_bytes dest = { }; + int count; + u8 *ptr; + + /* Verify static initializers. */ + check(control, 0x20); + check(zero, 0); + compare("static initializers", dest, zero); + + /* Verify assignment. */ + dest = control; + compare("direct assignment", dest, control); + + /* Verify complete overwrite. */ + memcpy(dest.data, zero.data, sizeof(dest.data)); + compare("complete overwrite", dest, zero); + + /* Verify middle overwrite. */ + dest = control; + memcpy(dest.data + 12, zero.data, 7); + compare("middle overwrite", dest, middle); + + /* Verify argument side-effects aren't repeated. */ + dest = control; + ptr = dest.data; + count = 1; + memcpy(ptr++, zero.data, count++); + ptr += 8; + memcpy(ptr++, zero.data, count++); + compare("argument side-effects", dest, three); +#undef TEST_OP +} + +static void memmove_test(struct kunit *test) +{ +#define TEST_OP "memmove" + struct some_bytes control = { + .data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes zero = { }; + struct some_bytes middle = { + .data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes five = { + .data = { 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x00, 0x00, 0x00, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes overlap = { + .data = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes overlap_expected = { + .data = { 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x07, + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, + }, + }; + struct some_bytes dest = { }; + int count; + u8 *ptr; + + /* Verify static initializers. */ + check(control, 0x99); + check(zero, 0); + compare("static initializers", zero, dest); + + /* Verify assignment. */ + dest = control; + compare("direct assignment", dest, control); + + /* Verify complete overwrite. */ + memmove(dest.data, zero.data, sizeof(dest.data)); + compare("complete overwrite", dest, zero); + + /* Verify middle overwrite. */ + dest = control; + memmove(dest.data + 12, zero.data, 7); + compare("middle overwrite", dest, middle); + + /* Verify argument side-effects aren't repeated. */ + dest = control; + ptr = dest.data; + count = 2; + memmove(ptr++, zero.data, count++); + ptr += 9; + memmove(ptr++, zero.data, count++); + compare("argument side-effects", dest, five); + + /* Verify overlapping overwrite is correct. */ + ptr = &overlap.data[2]; + memmove(ptr, overlap.data, 5); + compare("overlapping write", overlap, overlap_expected); +#undef TEST_OP +} + +static void memset_test(struct kunit *test) +{ +#define TEST_OP "memset" + struct some_bytes control = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + }, + }; + struct some_bytes complete = { + .data = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }, + }; + struct some_bytes middle = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x31, 0x31, 0x31, 0x31, + 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, + 0x31, 0x31, 0x31, 0x31, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + }, + }; + struct some_bytes three = { + .data = { 0x60, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x61, 0x61, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + }, + }; + struct some_bytes after = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x72, + 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, + 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, + 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, + }, + }; + struct some_bytes startat = { + .data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, + 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, + 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, + }, + }; + struct some_bytes dest = { }; + int count, value; + u8 *ptr; + + /* Verify static initializers. */ + check(control, 0x30); + check(dest, 0); + + /* Verify assignment. */ + dest = control; + compare("direct assignment", dest, control); + + /* Verify complete overwrite. */ + memset(dest.data, 0xff, sizeof(dest.data)); + compare("complete overwrite", dest, complete); + + /* Verify middle overwrite. */ + dest = control; + memset(dest.data + 4, 0x31, 16); + compare("middle overwrite", dest, middle); + + /* Verify argument side-effects aren't repeated. */ + dest = control; + ptr = dest.data; + value = 0x60; + count = 1; + memset(ptr++, value++, count++); + ptr += 8; + memset(ptr++, value++, count++); + compare("argument side-effects", dest, three); + + /* Verify memset_after() */ + dest = control; + memset_after(&dest, 0x72, three); + compare("memset_after()", dest, after); + + /* Verify memset_startat() */ + dest = control; + memset_startat(&dest, 0x79, four); + compare("memset_startat()", dest, startat); +#undef TEST_OP +} + +static struct kunit_case memcpy_test_cases[] = { + KUNIT_CASE(memset_test), + KUNIT_CASE(memcpy_test), + KUNIT_CASE(memmove_test), + {} +}; + +static struct kunit_suite memcpy_test_suite = { + .name = "memcpy", + .test_cases = memcpy_test_cases, +}; + +kunit_test_suite(memcpy_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/mpi/longlong.h b/lib/mpi/longlong.h index afbd99987cf8..b6fa1d08fb55 100644 --- a/lib/mpi/longlong.h +++ b/lib/mpi/longlong.h @@ -48,8 +48,8 @@ /* Define auxiliary asm macros. * - * 1) umul_ppmm(high_prod, low_prod, multipler, multiplicand) multiplies two - * UWtype integers MULTIPLER and MULTIPLICAND, and generates a two UWtype + * 1) umul_ppmm(high_prod, low_prod, multiplier, multiplicand) multiplies two + * UWtype integers MULTIPLIER and MULTIPLICAND, and generates a two UWtype * word product in HIGH_PROD and LOW_PROD. * * 2) __umulsidi3(a,b) multiplies two UWtype integers A and B, and returns a diff --git a/lib/mpi/mpicoder.c b/lib/mpi/mpicoder.c index 7ea225b2204f..39c4c6731094 100644 --- a/lib/mpi/mpicoder.c +++ b/lib/mpi/mpicoder.c @@ -234,11 +234,11 @@ static int count_lzeros(MPI a) } /** - * mpi_read_buffer() - read MPI to a bufer provided by user (msb first) + * mpi_read_buffer() - read MPI to a buffer provided by user (msb first) * * @a: a multi precision integer - * @buf: bufer to which the output will be written to. Needs to be at - * leaset mpi_get_size(a) long. + * @buf: buffer to which the output will be written to. Needs to be at + * least mpi_get_size(a) long. * @buf_len: size of the buf. * @nbytes: receives the actual length of the data written on success and * the data to-be-written on -EOVERFLOW in case buf_len was too diff --git a/lib/mpi/mpiutil.c b/lib/mpi/mpiutil.c index 3c63710c20c6..bc81419f400c 100644 --- a/lib/mpi/mpiutil.c +++ b/lib/mpi/mpiutil.c @@ -80,7 +80,7 @@ EXPORT_SYMBOL_GPL(mpi_const); /**************** * Note: It was a bad idea to use the number of limbs to allocate * because on a alpha the limbs are large but we normally need - * integers of n bits - So we should chnage this to bits (or bytes). + * integers of n bits - So we should change this to bits (or bytes). * * But mpi_alloc is used in a lot of places :-) */ @@ -148,7 +148,7 @@ int mpi_resize(MPI a, unsigned nlimbs) return 0; /* no need to do it */ if (a->d) { - p = kmalloc_array(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL); + p = kcalloc(nlimbs, sizeof(mpi_limb_t), GFP_KERNEL); if (!p) return -ENOMEM; memcpy(p, a->d, a->alloced * sizeof(mpi_limb_t)); diff --git a/lib/nlattr.c b/lib/nlattr.c index 1d051ef66afe..86029ad5ead4 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -619,7 +619,7 @@ static int __nla_validate_parse(const struct nlattr *head, int len, int maxtype, * Validates all attributes in the specified attribute stream against the * specified policy. Validation depends on the validate flags passed, see * &enum netlink_validation for more details on that. - * See documenation of struct nla_policy for more details. + * See documentation of struct nla_policy for more details. * * Returns 0 on success or a negative error code. */ @@ -633,7 +633,7 @@ int __nla_validate(const struct nlattr *head, int len, int maxtype, EXPORT_SYMBOL(__nla_validate); /** - * nla_policy_len - Determin the max. length of a policy + * nla_policy_len - Determine the max. length of a policy * @policy: policy to use * @n: number of policies * diff --git a/lib/oid_registry.c b/lib/oid_registry.c index 3dfaa836e7c5..e592d48b1974 100644 --- a/lib/oid_registry.c +++ b/lib/oid_registry.c @@ -124,7 +124,7 @@ EXPORT_SYMBOL_GPL(parse_OID); * @bufsize: The size of the buffer * * The OID is rendered into the buffer in "a.b.c.d" format and the number of - * bytes is returned. -EBADMSG is returned if the data could not be intepreted + * bytes is returned. -EBADMSG is returned if the data could not be interpreted * and -ENOBUFS if the buffer was too small. */ int sprint_oid(const void *data, size_t datasize, char *buffer, size_t bufsize) diff --git a/lib/once.c b/lib/once.c index 8b7d6235217e..59149bf3bfb4 100644 --- a/lib/once.c +++ b/lib/once.c @@ -3,10 +3,12 @@ #include <linux/spinlock.h> #include <linux/once.h> #include <linux/random.h> +#include <linux/module.h> struct once_work { struct work_struct work; struct static_key_true *key; + struct module *module; }; static void once_deferred(struct work_struct *w) @@ -16,10 +18,11 @@ static void once_deferred(struct work_struct *w) work = container_of(w, struct once_work, work); BUG_ON(!static_key_enabled(work->key)); static_branch_disable(work->key); + module_put(work->module); kfree(work); } -static void once_disable_jump(struct static_key_true *key) +static void once_disable_jump(struct static_key_true *key, struct module *mod) { struct once_work *w; @@ -29,6 +32,8 @@ static void once_disable_jump(struct static_key_true *key) INIT_WORK(&w->work, once_deferred); w->key = key; + w->module = mod; + __module_get(mod); schedule_work(&w->work); } @@ -53,11 +58,11 @@ bool __do_once_start(bool *done, unsigned long *flags) EXPORT_SYMBOL(__do_once_start); void __do_once_done(bool *done, struct static_key_true *once_key, - unsigned long *flags) + unsigned long *flags, struct module *mod) __releases(once_lock) { *done = true; spin_unlock_irqrestore(&once_lock, *flags); - once_disable_jump(once_key); + once_disable_jump(once_key, mod); } EXPORT_SYMBOL(__do_once_done); diff --git a/lib/packing.c b/lib/packing.c index 6ed72dccfdb5..9a72f4bbf0e2 100644 --- a/lib/packing.c +++ b/lib/packing.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 -/* Copyright (c) 2016-2018, NXP Semiconductors +/* Copyright 2016-2018 NXP * Copyright (c) 2018-2019, Vladimir Oltean <olteanv@gmail.com> */ #include <linux/packing.h> diff --git a/lib/parser.c b/lib/parser.c index f1a6d90b8c34..bcb23484100e 100644 --- a/lib/parser.c +++ b/lib/parser.c @@ -6,6 +6,7 @@ #include <linux/ctype.h> #include <linux/types.h> #include <linux/export.h> +#include <linux/kstrtox.h> #include <linux/parser.h> #include <linux/slab.h> #include <linux/string.h> diff --git a/lib/pci_iomap.c b/lib/pci_iomap.c index 2d3eb1cb73b8..ce39ce9f3526 100644 --- a/lib/pci_iomap.c +++ b/lib/pci_iomap.c @@ -134,4 +134,47 @@ void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) return pci_iomap_wc_range(dev, bar, 0, maxlen); } EXPORT_SYMBOL_GPL(pci_iomap_wc); + +/* + * pci_iounmap() somewhat illogically comes from lib/iomap.c for the + * CONFIG_GENERIC_IOMAP case, because that's the code that knows about + * the different IOMAP ranges. + * + * But if the architecture does not use the generic iomap code, and if + * it has _not_ defined it's own private pci_iounmap function, we define + * it here. + * + * NOTE! This default implementation assumes that if the architecture + * support ioport mapping (HAS_IOPORT_MAP), the ioport mapping will + * be fixed to the range [ PCI_IOBASE, PCI_IOBASE+IO_SPACE_LIMIT [, + * and does not need unmapping with 'ioport_unmap()'. + * + * If you have different rules for your architecture, you need to + * implement your own pci_iounmap() that knows the rules for where + * and how IO vs MEM get mapped. + * + * This code is odd, and the ARCH_HAS/ARCH_WANTS #define logic comes + * from legacy <asm-generic/io.h> header file behavior. In particular, + * it would seem to make sense to do the iounmap(p) for the non-IO-space + * case here regardless, but that's not what the old header file code + * did. Probably incorrectly, but this is meant to be bug-for-bug + * compatible. + */ +#if defined(ARCH_WANTS_GENERIC_PCI_IOUNMAP) + +void pci_iounmap(struct pci_dev *dev, void __iomem *p) +{ +#ifdef ARCH_HAS_GENERIC_IOPORT_MAP + uintptr_t start = (uintptr_t) PCI_IOBASE; + uintptr_t addr = (uintptr_t) p; + + if (addr >= start && addr < start + IO_SPACE_LIMIT) + return; + iounmap(p); +#endif +} +EXPORT_SYMBOL(pci_iounmap); + +#endif /* ARCH_WANTS_GENERIC_PCI_IOUNMAP */ + #endif /* CONFIG_PCI */ diff --git a/lib/pldmfw/pldmfw.c b/lib/pldmfw/pldmfw.c index e5d4b3b2af81..6e77eb6d8e72 100644 --- a/lib/pldmfw/pldmfw.c +++ b/lib/pldmfw/pldmfw.c @@ -82,7 +82,7 @@ pldm_check_fw_space(struct pldmfw_priv *data, size_t offset, size_t length) * @bytes_to_move: number of bytes to move the offset forward by * * Check that there is enough space past the current offset, and then move the - * offset forward by this ammount. + * offset forward by this amount. * * Returns: zero on success, or -EFAULT if the image is too small to fit the * expected length. diff --git a/lib/random32.c b/lib/random32.c index 4d0e05e471d7..a57a0e18819d 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -39,6 +39,7 @@ #include <linux/random.h> #include <linux/sched.h> #include <linux/bitops.h> +#include <linux/slab.h> #include <asm/unaligned.h> #include <trace/events/random.h> diff --git a/lib/reed_solomon/test_rslib.c b/lib/reed_solomon/test_rslib.c index 4eb29f365ece..d9d1c33aebda 100644 --- a/lib/reed_solomon/test_rslib.c +++ b/lib/reed_solomon/test_rslib.c @@ -385,7 +385,7 @@ static void test_bc(struct rs_control *rs, int len, int errs, /* * We check that the returned word is actually a - * codeword. The obious way to do this would be to + * codeword. The obvious way to do this would be to * compute the syndrome, but we don't want to replicate * that code here. However, all the codes are in * systematic form, and therefore we can encode the diff --git a/lib/refcount.c b/lib/refcount.c index ebac8b7d15a7..a207a8f22b3c 100644 --- a/lib/refcount.c +++ b/lib/refcount.c @@ -164,7 +164,7 @@ EXPORT_SYMBOL(refcount_dec_and_lock); * @flags: saved IRQ-flags if the is acquired * * Same as refcount_dec_and_lock() above except that the spinlock is acquired - * with disabled interupts. + * with disabled interrupts. * * Return: true and hold spinlock if able to decrement refcount to 0, false * otherwise diff --git a/lib/rhashtable.c b/lib/rhashtable.c index c949c1e3b87c..e12bbfb240b8 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -703,7 +703,7 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit); * * Returns zero if successful. * - * Returns -EAGAIN if resize event occured. Note that the iterator + * Returns -EAGAIN if resize event occurred. Note that the iterator * will rewind back to the beginning and you may use it immediately * by calling rhashtable_walk_next. * diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 47b3691058eb..2709ab825499 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -489,6 +489,57 @@ int __sbitmap_queue_get(struct sbitmap_queue *sbq) } EXPORT_SYMBOL_GPL(__sbitmap_queue_get); +unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, + unsigned int *offset) +{ + struct sbitmap *sb = &sbq->sb; + unsigned int hint, depth; + unsigned long index, nr; + int i; + + if (unlikely(sb->round_robin)) + return 0; + + depth = READ_ONCE(sb->depth); + hint = update_alloc_hint_before_get(sb, depth); + + index = SB_NR_TO_INDEX(sb, hint); + + for (i = 0; i < sb->map_nr; i++) { + struct sbitmap_word *map = &sb->map[index]; + unsigned long get_mask; + + sbitmap_deferred_clear(map); + if (map->word == (1UL << (map->depth - 1)) - 1) + continue; + + nr = find_first_zero_bit(&map->word, map->depth); + if (nr + nr_tags <= map->depth) { + atomic_long_t *ptr = (atomic_long_t *) &map->word; + int map_tags = min_t(int, nr_tags, map->depth); + unsigned long val, ret; + + get_mask = ((1UL << map_tags) - 1) << nr; + do { + val = READ_ONCE(map->word); + ret = atomic_long_cmpxchg(ptr, val, get_mask | val); + } while (ret != val); + get_mask = (get_mask & ~ret) >> nr; + if (get_mask) { + *offset = nr + (index << sb->shift); + update_alloc_hint_after_get(sb, depth, hint, + *offset + map_tags - 1); + return get_mask; + } + } + /* Jump to next index. */ + if (++index >= sb->map_nr) + index = 0; + } + + return 0; +} + int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, unsigned int shallow_depth) { @@ -577,13 +628,53 @@ void sbitmap_queue_wake_up(struct sbitmap_queue *sbq) } EXPORT_SYMBOL_GPL(sbitmap_queue_wake_up); +static inline void sbitmap_update_cpu_hint(struct sbitmap *sb, int cpu, int tag) +{ + if (likely(!sb->round_robin && tag < sb->depth)) + data_race(*per_cpu_ptr(sb->alloc_hint, cpu) = tag); +} + +void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset, + int *tags, int nr_tags) +{ + struct sbitmap *sb = &sbq->sb; + unsigned long *addr = NULL; + unsigned long mask = 0; + int i; + + smp_mb__before_atomic(); + for (i = 0; i < nr_tags; i++) { + const int tag = tags[i] - offset; + unsigned long *this_addr; + + /* since we're clearing a batch, skip the deferred map */ + this_addr = &sb->map[SB_NR_TO_INDEX(sb, tag)].word; + if (!addr) { + addr = this_addr; + } else if (addr != this_addr) { + atomic_long_andnot(mask, (atomic_long_t *) addr); + mask = 0; + addr = this_addr; + } + mask |= (1UL << SB_NR_TO_BIT(sb, tag)); + } + + if (mask) + atomic_long_andnot(mask, (atomic_long_t *) addr); + + smp_mb__after_atomic(); + sbitmap_queue_wake_up(sbq); + sbitmap_update_cpu_hint(&sbq->sb, raw_smp_processor_id(), + tags[nr_tags - 1] - offset); +} + void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, unsigned int cpu) { /* * Once the clear bit is set, the bit may be allocated out. * - * Orders READ/WRITE on the asssociated instance(such as request + * Orders READ/WRITE on the associated instance(such as request * of blk_mq) by this bit for avoiding race with re-allocation, * and its pair is the memory barrier implied in __sbitmap_get_word. * @@ -601,9 +692,7 @@ void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, */ smp_mb__after_atomic(); sbitmap_queue_wake_up(sbq); - - if (likely(!sbq->sb.round_robin && nr < sbq->sb.depth)) - *per_cpu_ptr(sbq->sb.alloc_hint, cpu) = nr; + sbitmap_update_cpu_hint(&sbq->sb, cpu, nr); } EXPORT_SYMBOL_GPL(sbitmap_queue_clear); diff --git a/lib/scatterlist.c b/lib/scatterlist.c index a59778946404..abb3432ed744 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -38,7 +38,7 @@ EXPORT_SYMBOL(sg_next); * @sg: The scatterlist * * Description: - * Allows to know how many entries are in sg, taking into acount + * Allows to know how many entries are in sg, taking into account * chaining as well * **/ @@ -59,7 +59,7 @@ EXPORT_SYMBOL(sg_nents); * * Description: * Determines the number of entries in sg that are required to meet - * the supplied length, taking into acount chaining as well + * the supplied length, taking into account chaining as well * * Returns: * the number of sg entries needed, negative error on failure @@ -182,6 +182,7 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * @nents_first_chunk: Number of entries int the (preallocated) first * scatterlist chunk, 0 means no such preallocated first chunk * @free_fn: Free function + * @num_ents: Number of entries in the table * * Description: * Free an sg table previously allocated and setup with @@ -190,7 +191,8 @@ static void sg_kfree(struct scatterlist *sg, unsigned int nents) * **/ void __sg_free_table(struct sg_table *table, unsigned int max_ents, - unsigned int nents_first_chunk, sg_free_fn *free_fn) + unsigned int nents_first_chunk, sg_free_fn *free_fn, + unsigned int num_ents) { struct scatterlist *sgl, *next; unsigned curr_max_ents = nents_first_chunk ?: max_ents; @@ -199,8 +201,8 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, return; sgl = table->sgl; - while (table->orig_nents) { - unsigned int alloc_size = table->orig_nents; + while (num_ents) { + unsigned int alloc_size = num_ents; unsigned int sg_size; /* @@ -218,7 +220,7 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, next = NULL; } - table->orig_nents -= sg_size; + num_ents -= sg_size; if (nents_first_chunk) nents_first_chunk = 0; else @@ -232,13 +234,27 @@ void __sg_free_table(struct sg_table *table, unsigned int max_ents, EXPORT_SYMBOL(__sg_free_table); /** + * sg_free_append_table - Free a previously allocated append sg table. + * @table: The mapped sg append table header + * + **/ +void sg_free_append_table(struct sg_append_table *table) +{ + __sg_free_table(&table->sgt, SG_MAX_SINGLE_ALLOC, false, sg_kfree, + table->total_nents); +} +EXPORT_SYMBOL(sg_free_append_table); + + +/** * sg_free_table - Free a previously allocated sg table * @table: The mapped sg table header * **/ void sg_free_table(struct sg_table *table) { - __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree); + __sg_free_table(table, SG_MAX_SINGLE_ALLOC, false, sg_kfree, + table->orig_nents); } EXPORT_SYMBOL(sg_free_table); @@ -359,13 +375,12 @@ int sg_alloc_table(struct sg_table *table, unsigned int nents, gfp_t gfp_mask) ret = __sg_alloc_table(table, nents, SG_MAX_SINGLE_ALLOC, NULL, 0, gfp_mask, sg_kmalloc); if (unlikely(ret)) - __sg_free_table(table, SG_MAX_SINGLE_ALLOC, 0, sg_kfree); - + sg_free_table(table); return ret; } EXPORT_SYMBOL(sg_alloc_table); -static struct scatterlist *get_next_sg(struct sg_table *table, +static struct scatterlist *get_next_sg(struct sg_append_table *table, struct scatterlist *cur, unsigned long needed_sges, gfp_t gfp_mask) @@ -386,54 +401,52 @@ static struct scatterlist *get_next_sg(struct sg_table *table, return ERR_PTR(-ENOMEM); sg_init_table(new_sg, alloc_size); if (cur) { + table->total_nents += alloc_size - 1; __sg_chain(next_sg, new_sg); - table->orig_nents += alloc_size - 1; } else { - table->sgl = new_sg; - table->orig_nents = alloc_size; - table->nents = 0; + table->sgt.sgl = new_sg; + table->total_nents = alloc_size; } return new_sg; } /** - * __sg_alloc_table_from_pages - Allocate and initialize an sg table from - * an array of pages - * @sgt: The sg table header to use - * @pages: Pointer to an array of page pointers - * @n_pages: Number of pages in the pages array + * sg_alloc_append_table_from_pages - Allocate and initialize an append sg + * table from an array of pages + * @sgt_append: The sg append table to use + * @pages: Pointer to an array of page pointers + * @n_pages: Number of pages in the pages array * @offset: Offset from start of the first page to the start of a buffer * @size: Number of valid bytes in the buffer (after offset) * @max_segment: Maximum size of a scatterlist element in bytes - * @prv: Last populated sge in sgt * @left_pages: Left pages caller have to set after this call * @gfp_mask: GFP allocation mask * * Description: - * If @prv is NULL, allocate and initialize an sg table from a list of pages, - * else reuse the scatterlist passed in at @prv. - * Contiguous ranges of the pages are squashed into a single scatterlist - * entry up to the maximum size specified in @max_segment. A user may - * provide an offset at a start and a size of valid data in a buffer - * specified by the page array. + * In the first call it allocate and initialize an sg table from a list of + * pages, else reuse the scatterlist from sgt_append. Contiguous ranges of + * the pages are squashed into a single scatterlist entry up to the maximum + * size specified in @max_segment. A user may provide an offset at a start + * and a size of valid data in a buffer specified by the page array. The + * returned sg table is released by sg_free_append_table * * Returns: - * Last SGE in sgt on success, PTR_ERR on otherwise. - * The allocation in @sgt must be released by sg_free_table. + * 0 on success, negative error on failure * * Notes: * If this function returns non-0 (eg failure), the caller must call - * sg_free_table() to cleanup any leftover allocations. + * sg_free_append_table() to cleanup any leftover allocations. + * + * In the fist call, sgt_append must by initialized. */ -struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, +int sg_alloc_append_table_from_pages(struct sg_append_table *sgt_append, struct page **pages, unsigned int n_pages, unsigned int offset, unsigned long size, unsigned int max_segment, - struct scatterlist *prv, unsigned int left_pages, - gfp_t gfp_mask) + unsigned int left_pages, gfp_t gfp_mask) { unsigned int chunks, cur_page, seg_len, i, prv_len = 0; unsigned int added_nents = 0; - struct scatterlist *s = prv; + struct scatterlist *s = sgt_append->prv; /* * The algorithm below requires max_segment to be aligned to PAGE_SIZE @@ -441,25 +454,26 @@ struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, */ max_segment = ALIGN_DOWN(max_segment, PAGE_SIZE); if (WARN_ON(max_segment < PAGE_SIZE)) - return ERR_PTR(-EINVAL); + return -EINVAL; - if (IS_ENABLED(CONFIG_ARCH_NO_SG_CHAIN) && prv) - return ERR_PTR(-EOPNOTSUPP); + if (IS_ENABLED(CONFIG_ARCH_NO_SG_CHAIN) && sgt_append->prv) + return -EOPNOTSUPP; - if (prv) { - unsigned long paddr = (page_to_pfn(sg_page(prv)) * PAGE_SIZE + - prv->offset + prv->length) / - PAGE_SIZE; + if (sgt_append->prv) { + unsigned long paddr = + (page_to_pfn(sg_page(sgt_append->prv)) * PAGE_SIZE + + sgt_append->prv->offset + sgt_append->prv->length) / + PAGE_SIZE; if (WARN_ON(offset)) - return ERR_PTR(-EINVAL); + return -EINVAL; /* Merge contiguous pages into the last SG */ - prv_len = prv->length; + prv_len = sgt_append->prv->length; while (n_pages && page_to_pfn(pages[0]) == paddr) { - if (prv->length + PAGE_SIZE > max_segment) + if (sgt_append->prv->length + PAGE_SIZE > max_segment) break; - prv->length += PAGE_SIZE; + sgt_append->prv->length += PAGE_SIZE; paddr++; pages++; n_pages--; @@ -496,15 +510,16 @@ struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, } /* Pass how many chunks might be left */ - s = get_next_sg(sgt, s, chunks - i + left_pages, gfp_mask); + s = get_next_sg(sgt_append, s, chunks - i + left_pages, + gfp_mask); if (IS_ERR(s)) { /* * Adjust entry length to be as before function was * called. */ - if (prv) - prv->length = prv_len; - return s; + if (sgt_append->prv) + sgt_append->prv->length = prv_len; + return PTR_ERR(s); } chunk_size = ((j - cur_page) << PAGE_SHIFT) - offset; sg_set_page(s, pages[cur_page], @@ -514,42 +529,58 @@ struct scatterlist *__sg_alloc_table_from_pages(struct sg_table *sgt, offset = 0; cur_page = j; } - sgt->nents += added_nents; + sgt_append->sgt.nents += added_nents; + sgt_append->sgt.orig_nents = sgt_append->sgt.nents; + sgt_append->prv = s; out: if (!left_pages) sg_mark_end(s); - return s; + return 0; } -EXPORT_SYMBOL(__sg_alloc_table_from_pages); +EXPORT_SYMBOL(sg_alloc_append_table_from_pages); /** - * sg_alloc_table_from_pages - Allocate and initialize an sg table from - * an array of pages + * sg_alloc_table_from_pages_segment - Allocate and initialize an sg table from + * an array of pages and given maximum + * segment. * @sgt: The sg table header to use * @pages: Pointer to an array of page pointers * @n_pages: Number of pages in the pages array * @offset: Offset from start of the first page to the start of a buffer * @size: Number of valid bytes in the buffer (after offset) + * @max_segment: Maximum size of a scatterlist element in bytes * @gfp_mask: GFP allocation mask * * Description: * Allocate and initialize an sg table from a list of pages. Contiguous - * ranges of the pages are squashed into a single scatterlist node. A user - * may provide an offset at a start and a size of valid data in a buffer - * specified by the page array. The returned sg table is released by - * sg_free_table. + * ranges of the pages are squashed into a single scatterlist node up to the + * maximum size specified in @max_segment. A user may provide an offset at a + * start and a size of valid data in a buffer specified by the page array. * - * Returns: + * The returned sg table is released by sg_free_table. + * + * Returns: * 0 on success, negative error on failure */ -int sg_alloc_table_from_pages(struct sg_table *sgt, struct page **pages, - unsigned int n_pages, unsigned int offset, - unsigned long size, gfp_t gfp_mask) +int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages, + unsigned int n_pages, unsigned int offset, + unsigned long size, unsigned int max_segment, + gfp_t gfp_mask) { - return PTR_ERR_OR_ZERO(__sg_alloc_table_from_pages(sgt, pages, n_pages, - offset, size, UINT_MAX, NULL, 0, gfp_mask)); + struct sg_append_table append = {}; + int err; + + err = sg_alloc_append_table_from_pages(&append, pages, n_pages, offset, + size, max_segment, 0, gfp_mask); + if (err) { + sg_free_append_table(&append); + return err; + } + memcpy(sgt, &append.sgt, sizeof(*sgt)); + WARN_ON(append.total_nents != sgt->orig_nents); + return 0; } -EXPORT_SYMBOL(sg_alloc_table_from_pages); +EXPORT_SYMBOL(sg_alloc_table_from_pages_segment); #ifdef CONFIG_SGL_ALLOC @@ -887,9 +918,8 @@ void sg_miter_stop(struct sg_mapping_iter *miter) miter->__offset += miter->consumed; miter->__remaining -= miter->consumed; - if ((miter->__flags & SG_MITER_TO_SG) && - !PageSlab(miter->page)) - flush_kernel_dcache_page(miter->page); + if (miter->__flags & SG_MITER_TO_SG) + flush_dcache_page(miter->page); if (miter->__flags & SG_MITER_ATOMIC) { WARN_ON_ONCE(preemptible()); diff --git a/lib/seq_buf.c b/lib/seq_buf.c index 707453f5d58e..0a68f7aa85d6 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -229,8 +229,10 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, WARN_ON(s->size == 0); + BUILD_BUG_ON(MAX_MEMHEX_BYTES * 2 >= HEX_CHARS); + while (len) { - start_len = min(len, HEX_CHARS - 1); + start_len = min(len, MAX_MEMHEX_BYTES); #ifdef __BIG_ENDIAN for (i = 0, j = 0; i < start_len; i++) { #else @@ -243,12 +245,14 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, break; /* j increments twice per loop */ - len -= j / 2; hex[j++] = ' '; seq_buf_putmem(s, hex, j); if (seq_buf_has_overflowed(s)) return -1; + + len -= start_len; + data += start_len; } return 0; } @@ -285,7 +289,7 @@ int seq_buf_path(struct seq_buf *s, const struct path *path, const char *esc) } /** - * seq_buf_to_user - copy the squence buffer to user space + * seq_buf_to_user - copy the sequence buffer to user space * @s: seq_buf descriptor * @ubuf: The userspace memory location to copy to * @cnt: The amount to copy diff --git a/lib/sg_pool.c b/lib/sg_pool.c index db29e5c1f790..a0b1a52cd6f7 100644 --- a/lib/sg_pool.c +++ b/lib/sg_pool.c @@ -90,7 +90,8 @@ void sg_free_table_chained(struct sg_table *table, if (nents_first_chunk == 1) nents_first_chunk = 0; - __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free); + __sg_free_table(table, SG_CHUNK_SIZE, nents_first_chunk, sg_pool_free, + table->orig_nents); } EXPORT_SYMBOL_GPL(sg_free_table_chained); diff --git a/lib/sort.c b/lib/sort.c index 3ad454411997..aa18153864d2 100644 --- a/lib/sort.c +++ b/lib/sort.c @@ -51,7 +51,7 @@ static bool is_aligned(const void *base, size_t size, unsigned char align) * which basically all CPUs have, to minimize loop overhead computations. * * For some reason, on x86 gcc 7.3.0 adds a redundant test of n at the - * bottom of the loop, even though the zero flag is stil valid from the + * bottom of the loop, even though the zero flag is still valid from the * subtract (since the intervening mov instructions don't alter the flags). * Gcc 8.1.0 doesn't have that problem. */ diff --git a/lib/stackdepot.c b/lib/stackdepot.c index df9179f4f441..0a2e417f83cb 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -11,7 +11,7 @@ * Instead, stack depot maintains a hashtable of unique stacktraces. Since alloc * and free stacks repeat a lot, we save about 100x space. * Stacks are never removed from depot, so we store them contiguously one after - * another in a contiguos memory allocation. + * another in a contiguous memory allocation. * * Author: Alexander Potapenko <glider@google.com> * Copyright (C) 2016 Google, Inc. diff --git a/lib/string.c b/lib/string.c index 7548eb715ddb..485777c9da83 100644 --- a/lib/string.c +++ b/lib/string.c @@ -6,20 +6,15 @@ */ /* - * stupid library routines.. The optimized versions should generally be found - * as inline code in <asm-xx/string.h> + * This file should be used only for "library" routines that may have + * alternative implementations on specific architectures (generally + * found in <asm-xx/string.h>), or get overloaded by FORTIFY_SOURCE. + * (Specifically, this file is built with __NO_FORTIFY.) * - * These are buggy as well.. - * - * * Fri Jun 25 1999, Ingo Oeser <ioe@informatik.tu-chemnitz.de> - * - Added strsep() which will replace strtok() soon (because strsep() is - * reentrant and should be faster). Use only strsep() in new code, please. - * - * * Sat Feb 09 2002, Jason Thomas <jason@topic.com.au>, - * Matthew Hawkins <matt@mh.dropbear.id.au> - * - Kissed strtok() goodbye + * Other helper functions should live in string_helpers.c. */ +#define __NO_FORTIFY #include <linux/types.h> #include <linux/string.h> #include <linux/ctype.h> @@ -29,6 +24,7 @@ #include <linux/errno.h> #include <linux/slab.h> +#include <asm/unaligned.h> #include <asm/byteorder.h> #include <asm/word-at-a-time.h> #include <asm/page.h> @@ -238,40 +234,6 @@ EXPORT_SYMBOL(strscpy); #endif /** - * strscpy_pad() - Copy a C-string into a sized buffer - * @dest: Where to copy the string to - * @src: Where to copy the string from - * @count: Size of destination buffer - * - * Copy the string, or as much of it as fits, into the dest buffer. The - * behavior is undefined if the string buffers overlap. The destination - * buffer is always %NUL terminated, unless it's zero-sized. - * - * If the source string is shorter than the destination buffer, zeros - * the tail of the destination buffer. - * - * For full explanation of why you may want to consider using the - * 'strscpy' functions please see the function docstring for strscpy(). - * - * Returns: - * * The number of characters copied (not including the trailing %NUL) - * * -E2BIG if count is 0 or @src was truncated. - */ -ssize_t strscpy_pad(char *dest, const char *src, size_t count) -{ - ssize_t written; - - written = strscpy(dest, src, count); - if (written < 0 || written == count - 1) - return written; - - memset(dest + written + 1, 0, count - written - 1); - - return written; -} -EXPORT_SYMBOL(strscpy_pad); - -/** * stpcpy - copy a string from src to dest returning a pointer to the new end * of dest, including src's %NUL-terminator. May overrun dest. * @dest: pointer to end of string being copied into. Must be large enough @@ -513,46 +475,6 @@ char *strnchr(const char *s, size_t count, int c) EXPORT_SYMBOL(strnchr); #endif -/** - * skip_spaces - Removes leading whitespace from @str. - * @str: The string to be stripped. - * - * Returns a pointer to the first non-whitespace character in @str. - */ -char *skip_spaces(const char *str) -{ - while (isspace(*str)) - ++str; - return (char *)str; -} -EXPORT_SYMBOL(skip_spaces); - -/** - * strim - Removes leading and trailing whitespace from @s. - * @s: The string to be stripped. - * - * Note that the first trailing whitespace is replaced with a %NUL-terminator - * in the given string @s. Returns a pointer to the first non-whitespace - * character in @s. - */ -char *strim(char *s) -{ - size_t size; - char *end; - - size = strlen(s); - if (!size) - return s; - - end = s + size - 1; - while (end >= s && isspace(*end)) - end--; - *(end + 1) = '\0'; - - return skip_spaces(s); -} -EXPORT_SYMBOL(strim); - #ifndef __HAVE_ARCH_STRLEN /** * strlen - Find the length of a string @@ -687,101 +609,6 @@ char *strsep(char **s, const char *ct) EXPORT_SYMBOL(strsep); #endif -/** - * sysfs_streq - return true if strings are equal, modulo trailing newline - * @s1: one string - * @s2: another string - * - * This routine returns true iff two strings are equal, treating both - * NUL and newline-then-NUL as equivalent string terminations. It's - * geared for use with sysfs input strings, which generally terminate - * with newlines but are compared against values without newlines. - */ -bool sysfs_streq(const char *s1, const char *s2) -{ - while (*s1 && *s1 == *s2) { - s1++; - s2++; - } - - if (*s1 == *s2) - return true; - if (!*s1 && *s2 == '\n' && !s2[1]) - return true; - if (*s1 == '\n' && !s1[1] && !*s2) - return true; - return false; -} -EXPORT_SYMBOL(sysfs_streq); - -/** - * match_string - matches given string in an array - * @array: array of strings - * @n: number of strings in the array or -1 for NULL terminated arrays - * @string: string to match with - * - * This routine will look for a string in an array of strings up to the - * n-th element in the array or until the first NULL element. - * - * Historically the value of -1 for @n, was used to search in arrays that - * are NULL terminated. However, the function does not make a distinction - * when finishing the search: either @n elements have been compared OR - * the first NULL element was found. - * - * Return: - * index of a @string in the @array if matches, or %-EINVAL otherwise. - */ -int match_string(const char * const *array, size_t n, const char *string) -{ - int index; - const char *item; - - for (index = 0; index < n; index++) { - item = array[index]; - if (!item) - break; - if (!strcmp(item, string)) - return index; - } - - return -EINVAL; -} -EXPORT_SYMBOL(match_string); - -/** - * __sysfs_match_string - matches given string in an array - * @array: array of strings - * @n: number of strings in the array or -1 for NULL terminated arrays - * @str: string to match with - * - * Returns index of @str in the @array or -EINVAL, just like match_string(). - * Uses sysfs_streq instead of strcmp for matching. - * - * This routine will look for a string in an array of strings up to the - * n-th element in the array or until the first NULL element. - * - * Historically the value of -1 for @n, was used to search in arrays that - * are NULL terminated. However, the function does not make a distinction - * when finishing the search: either @n elements have been compared OR - * the first NULL element was found. - */ -int __sysfs_match_string(const char * const *array, size_t n, const char *str) -{ - const char *item; - int index; - - for (index = 0; index < n; index++) { - item = array[index]; - if (!item) - break; - if (sysfs_streq(item, str)) - return index; - } - - return -EINVAL; -} -EXPORT_SYMBOL(__sysfs_match_string); - #ifndef __HAVE_ARCH_MEMSET /** * memset - Fill a region of memory with the given value @@ -935,6 +762,21 @@ __visible int memcmp(const void *cs, const void *ct, size_t count) const unsigned char *su1, *su2; int res = 0; +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (count >= sizeof(unsigned long)) { + const unsigned long *u1 = cs; + const unsigned long *u2 = ct; + do { + if (get_unaligned(u1) != get_unaligned(u2)) + break; + u1++; + u2++; + count -= sizeof(unsigned long); + } while (count >= sizeof(unsigned long)); + cs = u1; + ct = u2; + } +#endif for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--) if ((res = *su1 - *su2) != 0) break; @@ -977,7 +819,7 @@ void *memscan(void *addr, int c, size_t size) unsigned char *p = addr; while (size) { - if (*p == c) + if (*p == (unsigned char)c) return (void *)p; p++; size--; @@ -1125,27 +967,3 @@ void *memchr_inv(const void *start, int c, size_t bytes) return check_bytes8(start, value, bytes % 8); } EXPORT_SYMBOL(memchr_inv); - -/** - * strreplace - Replace all occurrences of character in string. - * @s: The string to operate on. - * @old: The character being replaced. - * @new: The character @old is replaced with. - * - * Returns pointer to the nul byte at the end of @s. - */ -char *strreplace(char *s, char old, char new) -{ - for (; *s; ++s) - if (*s == old) - *s = new; - return s; -} -EXPORT_SYMBOL(strreplace); - -void fortify_panic(const char *name) -{ - pr_emerg("detected buffer overflow in %s\n", name); - BUG(); -} -EXPORT_SYMBOL(fortify_panic); diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 7f2d5fbaf243..faa9d8e4e2c5 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -361,6 +361,9 @@ static bool escape_special(unsigned char c, char **dst, char *end) case '\e': to = 'e'; break; + case '"': + to = '"'; + break; default: return false; } @@ -452,18 +455,20 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * The process of escaping byte buffer includes several parts. They are applied * in the following sequence. * - * 1. The character is matched to the printable class, if asked, and in - * case of match it passes through to the output. - * 2. The character is not matched to the one from @only string and thus + * 1. The character is not matched to the one from @only string and thus * must go as-is to the output. - * 3. The character is checked if it falls into the class given by @flags. + * 2. The character is matched to the printable and ASCII classes, if asked, + * and in case of match it passes through to the output. + * 3. The character is matched to the printable or ASCII class, if asked, + * and in case of match it passes through to the output. + * 4. The character is checked if it falls into the class given by @flags. * %ESCAPE_OCTAL and %ESCAPE_HEX are going last since they cover any * character. Note that they actually can't go together, otherwise * %ESCAPE_HEX will be ignored. * * Caller must provide valid source and destination pointers. Be aware that * destination buffer will not be NULL-terminated, thus caller have to append - * it if needs. The supported flags are:: + * it if needs. The supported flags are:: * * %ESCAPE_SPACE: (special white space, not space itself) * '\f' - form feed @@ -472,6 +477,7 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * '\t' - horizontal tab * '\v' - vertical tab * %ESCAPE_SPECIAL: + * '\"' - double quote * '\\' - backslash * '\a' - alert (BEL) * '\e' - escape @@ -482,11 +488,27 @@ static bool escape_hex(unsigned char c, char **dst, char *end) * %ESCAPE_ANY: * all previous together * %ESCAPE_NP: - * escape only non-printable characters (checked by isprint) + * escape only non-printable characters, checked by isprint() * %ESCAPE_ANY_NP: * all previous together * %ESCAPE_HEX: * '\xHH' - byte with hexadecimal value HH (2 digits) + * %ESCAPE_NA: + * escape only non-ascii characters, checked by isascii() + * %ESCAPE_NAP: + * escape only non-printable or non-ascii characters + * %ESCAPE_APPEND: + * append characters from @only to be escaped by the given classes + * + * %ESCAPE_APPEND would help to pass additional characters to the escaped, when + * one of %ESCAPE_NP, %ESCAPE_NA, or %ESCAPE_NAP is provided. + * + * One notable caveat, the %ESCAPE_NAP, %ESCAPE_NP and %ESCAPE_NA have the + * higher priority than the rest of the flags (%ESCAPE_NAP is the highest). + * It doesn't make much sense to use either of them without %ESCAPE_OCTAL + * or %ESCAPE_HEX, because they cover most of the other character classes. + * %ESCAPE_NAP can utilize %ESCAPE_SPACE or %ESCAPE_SPECIAL in addition to + * the above. * * Return: * The total size of the escaped output that would be generated for @@ -500,67 +522,69 @@ int string_escape_mem(const char *src, size_t isz, char *dst, size_t osz, char *p = dst; char *end = p + osz; bool is_dict = only && *only; + bool is_append = flags & ESCAPE_APPEND; while (isz--) { unsigned char c = *src++; + bool in_dict = is_dict && strchr(only, c); /* * Apply rules in the following sequence: - * - the character is printable, when @flags has - * %ESCAPE_NP bit set * - the @only string is supplied and does not contain a * character under question + * - the character is printable and ASCII, when @flags has + * %ESCAPE_NAP bit set + * - the character is printable, when @flags has + * %ESCAPE_NP bit set + * - the character is ASCII, when @flags has + * %ESCAPE_NA bit set * - the character doesn't fall into a class of symbols * defined by given @flags * In these cases we just pass through a character to the * output buffer. + * + * When %ESCAPE_APPEND is passed, the characters from @only + * have been excluded from the %ESCAPE_NAP, %ESCAPE_NP, and + * %ESCAPE_NA cases. */ - if ((flags & ESCAPE_NP && isprint(c)) || - (is_dict && !strchr(only, c))) { - /* do nothing */ - } else { - if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) - continue; + if (!(is_append || in_dict) && is_dict && + escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end)) - continue; + if (!(is_append && in_dict) && isascii(c) && isprint(c) && + flags & ESCAPE_NAP && escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_NULL && escape_null(c, &p, end)) - continue; + if (!(is_append && in_dict) && isprint(c) && + flags & ESCAPE_NP && escape_passthrough(c, &p, end)) + continue; - /* ESCAPE_OCTAL and ESCAPE_HEX always go last */ - if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end)) - continue; + if (!(is_append && in_dict) && isascii(c) && + flags & ESCAPE_NA && escape_passthrough(c, &p, end)) + continue; - if (flags & ESCAPE_HEX && escape_hex(c, &p, end)) - continue; - } + if (flags & ESCAPE_SPACE && escape_space(c, &p, end)) + continue; - escape_passthrough(c, &p, end); - } + if (flags & ESCAPE_SPECIAL && escape_special(c, &p, end)) + continue; - return p - dst; -} -EXPORT_SYMBOL(string_escape_mem); + if (flags & ESCAPE_NULL && escape_null(c, &p, end)) + continue; -int string_escape_mem_ascii(const char *src, size_t isz, char *dst, - size_t osz) -{ - char *p = dst; - char *end = p + osz; + /* ESCAPE_OCTAL and ESCAPE_HEX always go last */ + if (flags & ESCAPE_OCTAL && escape_octal(c, &p, end)) + continue; - while (isz--) { - unsigned char c = *src++; + if (flags & ESCAPE_HEX && escape_hex(c, &p, end)) + continue; - if (!isprint(c) || !isascii(c) || c == '"' || c == '\\') - escape_hex(c, &p, end); - else - escape_passthrough(c, &p, end); + escape_passthrough(c, &p, end); } return p - dst; } -EXPORT_SYMBOL(string_escape_mem_ascii); +EXPORT_SYMBOL(string_escape_mem); /* * Return an allocated string that has been escaped of special characters @@ -672,3 +696,198 @@ void kfree_strarray(char **array, size_t n) kfree(array); } EXPORT_SYMBOL_GPL(kfree_strarray); + +/** + * strscpy_pad() - Copy a C-string into a sized buffer + * @dest: Where to copy the string to + * @src: Where to copy the string from + * @count: Size of destination buffer + * + * Copy the string, or as much of it as fits, into the dest buffer. The + * behavior is undefined if the string buffers overlap. The destination + * buffer is always %NUL terminated, unless it's zero-sized. + * + * If the source string is shorter than the destination buffer, zeros + * the tail of the destination buffer. + * + * For full explanation of why you may want to consider using the + * 'strscpy' functions please see the function docstring for strscpy(). + * + * Returns: + * * The number of characters copied (not including the trailing %NUL) + * * -E2BIG if count is 0 or @src was truncated. + */ +ssize_t strscpy_pad(char *dest, const char *src, size_t count) +{ + ssize_t written; + + written = strscpy(dest, src, count); + if (written < 0 || written == count - 1) + return written; + + memset(dest + written + 1, 0, count - written - 1); + + return written; +} +EXPORT_SYMBOL(strscpy_pad); + +/** + * skip_spaces - Removes leading whitespace from @str. + * @str: The string to be stripped. + * + * Returns a pointer to the first non-whitespace character in @str. + */ +char *skip_spaces(const char *str) +{ + while (isspace(*str)) + ++str; + return (char *)str; +} +EXPORT_SYMBOL(skip_spaces); + +/** + * strim - Removes leading and trailing whitespace from @s. + * @s: The string to be stripped. + * + * Note that the first trailing whitespace is replaced with a %NUL-terminator + * in the given string @s. Returns a pointer to the first non-whitespace + * character in @s. + */ +char *strim(char *s) +{ + size_t size; + char *end; + + size = strlen(s); + if (!size) + return s; + + end = s + size - 1; + while (end >= s && isspace(*end)) + end--; + *(end + 1) = '\0'; + + return skip_spaces(s); +} +EXPORT_SYMBOL(strim); + +/** + * sysfs_streq - return true if strings are equal, modulo trailing newline + * @s1: one string + * @s2: another string + * + * This routine returns true iff two strings are equal, treating both + * NUL and newline-then-NUL as equivalent string terminations. It's + * geared for use with sysfs input strings, which generally terminate + * with newlines but are compared against values without newlines. + */ +bool sysfs_streq(const char *s1, const char *s2) +{ + while (*s1 && *s1 == *s2) { + s1++; + s2++; + } + + if (*s1 == *s2) + return true; + if (!*s1 && *s2 == '\n' && !s2[1]) + return true; + if (*s1 == '\n' && !s1[1] && !*s2) + return true; + return false; +} +EXPORT_SYMBOL(sysfs_streq); + +/** + * match_string - matches given string in an array + * @array: array of strings + * @n: number of strings in the array or -1 for NULL terminated arrays + * @string: string to match with + * + * This routine will look for a string in an array of strings up to the + * n-th element in the array or until the first NULL element. + * + * Historically the value of -1 for @n, was used to search in arrays that + * are NULL terminated. However, the function does not make a distinction + * when finishing the search: either @n elements have been compared OR + * the first NULL element was found. + * + * Return: + * index of a @string in the @array if matches, or %-EINVAL otherwise. + */ +int match_string(const char * const *array, size_t n, const char *string) +{ + int index; + const char *item; + + for (index = 0; index < n; index++) { + item = array[index]; + if (!item) + break; + if (!strcmp(item, string)) + return index; + } + + return -EINVAL; +} +EXPORT_SYMBOL(match_string); + +/** + * __sysfs_match_string - matches given string in an array + * @array: array of strings + * @n: number of strings in the array or -1 for NULL terminated arrays + * @str: string to match with + * + * Returns index of @str in the @array or -EINVAL, just like match_string(). + * Uses sysfs_streq instead of strcmp for matching. + * + * This routine will look for a string in an array of strings up to the + * n-th element in the array or until the first NULL element. + * + * Historically the value of -1 for @n, was used to search in arrays that + * are NULL terminated. However, the function does not make a distinction + * when finishing the search: either @n elements have been compared OR + * the first NULL element was found. + */ +int __sysfs_match_string(const char * const *array, size_t n, const char *str) +{ + const char *item; + int index; + + for (index = 0; index < n; index++) { + item = array[index]; + if (!item) + break; + if (sysfs_streq(item, str)) + return index; + } + + return -EINVAL; +} +EXPORT_SYMBOL(__sysfs_match_string); + +/** + * strreplace - Replace all occurrences of character in string. + * @s: The string to operate on. + * @old: The character being replaced. + * @new: The character @old is replaced with. + * + * Returns pointer to the nul byte at the end of @s. + */ +char *strreplace(char *s, char old, char new) +{ + for (; *s; ++s) + if (*s == old) + *s = new; + return s; +} +EXPORT_SYMBOL(strreplace); + +#ifdef CONFIG_FORTIFY_SOURCE +void fortify_panic(const char *name) +{ + pr_emerg("detected buffer overflow in %s\n", name); + BUG(); +} +EXPORT_SYMBOL(fortify_panic); +#endif /* CONFIG_FORTIFY_SOURCE */ diff --git a/lib/test-string_helpers.c b/lib/test-string_helpers.c index 10360d4ea273..437d8e6b7cb1 100644 --- a/lib/test-string_helpers.c +++ b/lib/test-string_helpers.c @@ -19,7 +19,7 @@ static __init bool test_string_check_buf(const char *name, unsigned int flags, if (q_real == q_test && !memcmp(out_test, out_real, q_test)) return true; - pr_warn("Test '%s' failed: flags = %u\n", name, flags); + pr_warn("Test '%s' failed: flags = %#x\n", name, flags); print_hex_dump(KERN_WARNING, "Input: ", DUMP_PREFIX_NONE, 16, 1, in, p, true); @@ -136,31 +136,31 @@ static const struct test_string_2 escape0[] __initconst = {{ .flags = ESCAPE_SPACE | ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ .in = "\\h\\\"\a\e\\", .s1 = {{ - .out = "\\\\h\\\\\"\\a\\e\\\\", + .out = "\\\\h\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL, },{ - .out = "\\\\\\150\\\\\\042\\a\\e\\\\", + .out = "\\\\\\150\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ - .out = "\\\\\\x68\\\\\\x22\\a\\e\\\\", + .out = "\\\\\\x68\\\\\\\"\\a\\e\\\\", .flags = ESCAPE_SPECIAL | ESCAPE_HEX, },{ /* terminator */ - }}, + }} },{ .in = "\eb \\C\007\"\x90\r]", .s1 = {{ .out = "\eb \\C\007\"\x90\\r]", .flags = ESCAPE_SPACE, },{ - .out = "\\eb \\\\C\\a\"\x90\r]", + .out = "\\eb \\\\C\\a\\\"\x90\r]", .flags = ESCAPE_SPECIAL, },{ - .out = "\\eb \\\\C\\a\"\x90\\r]", + .out = "\\eb \\\\C\\a\\\"\x90\\r]", .flags = ESCAPE_SPACE | ESCAPE_SPECIAL, },{ .out = "\\033\\142\\040\\134\\103\\007\\042\\220\\015\\135", @@ -169,10 +169,10 @@ static const struct test_string_2 escape0[] __initconst = {{ .out = "\\033\\142\\040\\134\\103\\007\\042\\220\\r\\135", .flags = ESCAPE_SPACE | ESCAPE_OCTAL, },{ - .out = "\\e\\142\\040\\\\\\103\\a\\042\\220\\015\\135", + .out = "\\e\\142\\040\\\\\\103\\a\\\"\\220\\015\\135", .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ - .out = "\\e\\142\\040\\\\\\103\\a\\042\\220\\r\\135", + .out = "\\e\\142\\040\\\\\\103\\a\\\"\\220\\r\\135", .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_OCTAL, },{ .out = "\eb \\C\007\"\x90\r]", @@ -201,12 +201,26 @@ static const struct test_string_2 escape0[] __initconst = {{ .flags = ESCAPE_NP | ESCAPE_HEX, },{ /* terminator */ - }}, + }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\\220\\317\r", + .flags = ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\\x90\\xcf\r", + .flags = ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NA, + },{ + /* terminator */ + }} },{ /* terminator */ }}; -#define TEST_STRING_2_DICT_1 "b\\ \t\r" +#define TEST_STRING_2_DICT_1 "b\\ \t\r\xCF" static const struct test_string_2 escape1[] __initconst = {{ .in = "\f\\ \n\r\t\v", .s1 = {{ @@ -216,16 +230,40 @@ static const struct test_string_2 escape1[] __initconst = {{ .out = "\f\\x5c\\x20\n\\x0d\\x09\v", .flags = ESCAPE_HEX, },{ + .out = "\f\\134\\040\n\\015\\011\v", + .flags = ESCAPE_ANY | ESCAPE_APPEND, + },{ + .out = "\\014\\134\\040\\012\\015\\011\\013", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\\x0c\\x5c\\x20\\x0a\\x0d\\x09\\x0b", + .flags = ESCAPE_HEX | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\f\\134\\040\n\\015\\011\v", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NA, + },{ + .out = "\f\\x5c\\x20\n\\x0d\\x09\v", + .flags = ESCAPE_HEX | ESCAPE_APPEND | ESCAPE_NA, + },{ /* terminator */ - }}, + }} },{ - .in = "\\h\\\"\a\e\\", + .in = "\\h\\\"\a\xCF\e\\", .s1 = {{ - .out = "\\134h\\134\"\a\e\\134", + .out = "\\134h\\134\"\a\\317\e\\134", .flags = ESCAPE_OCTAL, },{ + .out = "\\134h\\134\"\a\\317\e\\134", + .flags = ESCAPE_ANY | ESCAPE_APPEND, + },{ + .out = "\\134h\\134\"\\007\\317\\033\\134", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NAP, + },{ + .out = "\\134h\\134\"\a\\317\e\\134", + .flags = ESCAPE_OCTAL | ESCAPE_APPEND | ESCAPE_NA, + },{ /* terminator */ - }}, + }} },{ .in = "\eb \\C\007\"\x90\r]", .s1 = {{ @@ -233,7 +271,89 @@ static const struct test_string_2 escape1[] __initconst = {{ .flags = ESCAPE_OCTAL, },{ /* terminator */ - }}, + }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPACE | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPECIAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_SPACE | ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_ANY | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPACE | ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NA, + },{ + .out = "\007 \eb\"\x90\\xcf\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NA, + },{ + /* terminator */ + }} +},{ + .in = "\007 \eb\"\x90\xCF\r", + .s1 = {{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\\r", + .flags = ESCAPE_SPACE | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\r", + .flags = ESCAPE_SPECIAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\xCF\\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\015", + .flags = ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\r", + .flags = ESCAPE_SPACE | ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\\015", + .flags = ESCAPE_SPECIAL | ESCAPE_OCTAL | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\317\r", + .flags = ESCAPE_ANY | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\x0d", + .flags = ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\r", + .flags = ESCAPE_SPACE | ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\x0d", + .flags = ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NAP, + },{ + .out = "\007 \eb\"\x90\\xcf\\r", + .flags = ESCAPE_SPACE | ESCAPE_SPECIAL | ESCAPE_HEX | ESCAPE_NAP, + },{ + /* terminator */ + }} },{ /* terminator */ }}; @@ -290,7 +410,7 @@ test_string_escape_overflow(const char *in, int p, unsigned int flags, const cha q_real = string_escape_mem(in, p, NULL, 0, flags, esc); if (q_real != q_test) - pr_warn("Test '%s' failed: flags = %u, osz = 0, expected %d, got %d\n", + pr_warn("Test '%s' failed: flags = %#x, osz = 0, expected %d, got %d\n", name, flags, q_test, q_real); } @@ -315,8 +435,13 @@ static __init void test_string_escape(const char *name, /* NULL injection */ if (flags & ESCAPE_NULL) { in[p++] = '\0'; - out_test[q_test++] = '\\'; - out_test[q_test++] = '0'; + /* '\0' passes isascii() test */ + if (flags & ESCAPE_NA && !(flags & ESCAPE_APPEND && esc)) { + out_test[q_test++] = '\0'; + } else { + out_test[q_test++] = '\\'; + out_test[q_test++] = '0'; + } } /* Don't try strings that have no output */ @@ -459,17 +584,17 @@ static int __init test_string_helpers_init(void) unsigned int i; pr_info("Running tests...\n"); - for (i = 0; i < UNESCAPE_ANY + 1; i++) + for (i = 0; i < UNESCAPE_ALL_MASK + 1; i++) test_string_unescape("unescape", i, false); test_string_unescape("unescape inplace", get_random_int() % (UNESCAPE_ANY + 1), true); /* Without dictionary */ - for (i = 0; i < (ESCAPE_ANY_NP | ESCAPE_HEX) + 1; i++) + for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) test_string_escape("escape 0", escape0, i, TEST_STRING_2_DICT_0); /* With dictionary */ - for (i = 0; i < (ESCAPE_ANY_NP | ESCAPE_HEX) + 1; i++) + for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) test_string_escape("escape 1", escape1, i, TEST_STRING_2_DICT_1); /* Test string_get_size() */ diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 9cd575583180..d33fa5a61b95 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -19,6 +19,7 @@ KSTM_MODULE_GLOBALS(); static char pbl_buffer[PAGE_SIZE] __initdata; +static char print_buf[PAGE_SIZE * 2] __initdata; static const unsigned long exp1[] __initconst = { BITMAP_FROM_U64(1), @@ -156,6 +157,20 @@ static bool __init __check_eq_clump8(const char *srcfile, unsigned int line, return true; } +static bool __init +__check_eq_str(const char *srcfile, unsigned int line, + const char *exp_str, const char *str, + unsigned int len) +{ + bool eq; + + eq = strncmp(exp_str, str, len) == 0; + if (!eq) + pr_err("[%s:%u] expected %s, got %s\n", srcfile, line, exp_str, str); + + return eq; +} + #define __expect_eq(suffix, ...) \ ({ \ int result = 0; \ @@ -173,6 +188,7 @@ static bool __init __check_eq_clump8(const char *srcfile, unsigned int line, #define expect_eq_pbl(...) __expect_eq(pbl, ##__VA_ARGS__) #define expect_eq_u32_array(...) __expect_eq(u32_array, ##__VA_ARGS__) #define expect_eq_clump8(...) __expect_eq(clump8, ##__VA_ARGS__) +#define expect_eq_str(...) __expect_eq(str, ##__VA_ARGS__) static void __init test_zero_clear(void) { @@ -366,6 +382,13 @@ static const struct test_bitmap_parselist parselist_tests[] __initconst = { {0, "0-31:1/3,1-31:1/3,2-31:1/3", &exp1[8 * step], 32, 0}, {0, "1-10:8/12,8-31:24/29,0-31:0/3", &exp1[9 * step], 32, 0}, + {0, "all", &exp1[8 * step], 32, 0}, + {0, "0, 1, all, ", &exp1[8 * step], 32, 0}, + {0, "all:1/2", &exp1[4 * step], 32, 0}, + {0, "ALL:1/2", &exp1[4 * step], 32, 0}, + {-EINVAL, "al", NULL, 8, 0}, + {-EINVAL, "alll", NULL, 8, 0}, + {-EINVAL, "-1", NULL, 8, 0}, {-EINVAL, "-0", NULL, 8, 0}, {-EINVAL, "10-1", NULL, 8, 0}, @@ -653,6 +676,139 @@ static void __init test_bitmap_cut(void) } } +struct test_bitmap_print { + const unsigned long *bitmap; + unsigned long nbits; + const char *mask; + const char *list; +}; + +static const unsigned long small_bitmap[] __initconst = { + BITMAP_FROM_U64(0x3333333311111111ULL), +}; + +static const char small_mask[] __initconst = "33333333,11111111\n"; +static const char small_list[] __initconst = "0,4,8,12,16,20,24,28,32-33,36-37,40-41,44-45,48-49,52-53,56-57,60-61\n"; + +static const unsigned long large_bitmap[] __initconst = { + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), + BITMAP_FROM_U64(0x3333333311111111ULL), BITMAP_FROM_U64(0x3333333311111111ULL), +}; + +static const char large_mask[] __initconst = "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111," + "33333333,11111111,33333333,11111111\n"; + +static const char large_list[] __initconst = /* more than 4KB */ + "0,4,8,12,16,20,24,28,32-33,36-37,40-41,44-45,48-49,52-53,56-57,60-61,64,68,72,76,80,84,88,92,96-97,100-101,104-1" + "05,108-109,112-113,116-117,120-121,124-125,128,132,136,140,144,148,152,156,160-161,164-165,168-169,172-173,176-1" + "77,180-181,184-185,188-189,192,196,200,204,208,212,216,220,224-225,228-229,232-233,236-237,240-241,244-245,248-2" + "49,252-253,256,260,264,268,272,276,280,284,288-289,292-293,296-297,300-301,304-305,308-309,312-313,316-317,320,3" + "24,328,332,336,340,344,348,352-353,356-357,360-361,364-365,368-369,372-373,376-377,380-381,384,388,392,396,400,4" + "04,408,412,416-417,420-421,424-425,428-429,432-433,436-437,440-441,444-445,448,452,456,460,464,468,472,476,480-4" + "81,484-485,488-489,492-493,496-497,500-501,504-505,508-509,512,516,520,524,528,532,536,540,544-545,548-549,552-5" + "53,556-557,560-561,564-565,568-569,572-573,576,580,584,588,592,596,600,604,608-609,612-613,616-617,620-621,624-6" + "25,628-629,632-633,636-637,640,644,648,652,656,660,664,668,672-673,676-677,680-681,684-685,688-689,692-693,696-6" + "97,700-701,704,708,712,716,720,724,728,732,736-737,740-741,744-745,748-749,752-753,756-757,760-761,764-765,768,7" + "72,776,780,784,788,792,796,800-801,804-805,808-809,812-813,816-817,820-821,824-825,828-829,832,836,840,844,848,8" + "52,856,860,864-865,868-869,872-873,876-877,880-881,884-885,888-889,892-893,896,900,904,908,912,916,920,924,928-9" + "29,932-933,936-937,940-941,944-945,948-949,952-953,956-957,960,964,968,972,976,980,984,988,992-993,996-997,1000-" + "1001,1004-1005,1008-1009,1012-1013,1016-1017,1020-1021,1024,1028,1032,1036,1040,1044,1048,1052,1056-1057,1060-10" + "61,1064-1065,1068-1069,1072-1073,1076-1077,1080-1081,1084-1085,1088,1092,1096,1100,1104,1108,1112,1116,1120-1121" + ",1124-1125,1128-1129,1132-1133,1136-1137,1140-1141,1144-1145,1148-1149,1152,1156,1160,1164,1168,1172,1176,1180,1" + "184-1185,1188-1189,1192-1193,1196-1197,1200-1201,1204-1205,1208-1209,1212-1213,1216,1220,1224,1228,1232,1236,124" + "0,1244,1248-1249,1252-1253,1256-1257,1260-1261,1264-1265,1268-1269,1272-1273,1276-1277,1280,1284,1288,1292,1296," + "1300,1304,1308,1312-1313,1316-1317,1320-1321,1324-1325,1328-1329,1332-1333,1336-1337,1340-1341,1344,1348,1352,13" + "56,1360,1364,1368,1372,1376-1377,1380-1381,1384-1385,1388-1389,1392-1393,1396-1397,1400-1401,1404-1405,1408,1412" + ",1416,1420,1424,1428,1432,1436,1440-1441,1444-1445,1448-1449,1452-1453,1456-1457,1460-1461,1464-1465,1468-1469,1" + "472,1476,1480,1484,1488,1492,1496,1500,1504-1505,1508-1509,1512-1513,1516-1517,1520-1521,1524-1525,1528-1529,153" + "2-1533,1536,1540,1544,1548,1552,1556,1560,1564,1568-1569,1572-1573,1576-1577,1580-1581,1584-1585,1588-1589,1592-" + "1593,1596-1597,1600,1604,1608,1612,1616,1620,1624,1628,1632-1633,1636-1637,1640-1641,1644-1645,1648-1649,1652-16" + "53,1656-1657,1660-1661,1664,1668,1672,1676,1680,1684,1688,1692,1696-1697,1700-1701,1704-1705,1708-1709,1712-1713" + ",1716-1717,1720-1721,1724-1725,1728,1732,1736,1740,1744,1748,1752,1756,1760-1761,1764-1765,1768-1769,1772-1773,1" + "776-1777,1780-1781,1784-1785,1788-1789,1792,1796,1800,1804,1808,1812,1816,1820,1824-1825,1828-1829,1832-1833,183" + "6-1837,1840-1841,1844-1845,1848-1849,1852-1853,1856,1860,1864,1868,1872,1876,1880,1884,1888-1889,1892-1893,1896-" + "1897,1900-1901,1904-1905,1908-1909,1912-1913,1916-1917,1920,1924,1928,1932,1936,1940,1944,1948,1952-1953,1956-19" + "57,1960-1961,1964-1965,1968-1969,1972-1973,1976-1977,1980-1981,1984,1988,1992,1996,2000,2004,2008,2012,2016-2017" + ",2020-2021,2024-2025,2028-2029,2032-2033,2036-2037,2040-2041,2044-2045,2048,2052,2056,2060,2064,2068,2072,2076,2" + "080-2081,2084-2085,2088-2089,2092-2093,2096-2097,2100-2101,2104-2105,2108-2109,2112,2116,2120,2124,2128,2132,213" + "6,2140,2144-2145,2148-2149,2152-2153,2156-2157,2160-2161,2164-2165,2168-2169,2172-2173,2176,2180,2184,2188,2192," + "2196,2200,2204,2208-2209,2212-2213,2216-2217,2220-2221,2224-2225,2228-2229,2232-2233,2236-2237,2240,2244,2248,22" + "52,2256,2260,2264,2268,2272-2273,2276-2277,2280-2281,2284-2285,2288-2289,2292-2293,2296-2297,2300-2301,2304,2308" + ",2312,2316,2320,2324,2328,2332,2336-2337,2340-2341,2344-2345,2348-2349,2352-2353,2356-2357,2360-2361,2364-2365,2" + "368,2372,2376,2380,2384,2388,2392,2396,2400-2401,2404-2405,2408-2409,2412-2413,2416-2417,2420-2421,2424-2425,242" + "8-2429,2432,2436,2440,2444,2448,2452,2456,2460,2464-2465,2468-2469,2472-2473,2476-2477,2480-2481,2484-2485,2488-" + "2489,2492-2493,2496,2500,2504,2508,2512,2516,2520,2524,2528-2529,2532-2533,2536-2537,2540-2541,2544-2545,2548-25" + "49,2552-2553,2556-2557\n"; + +static const struct test_bitmap_print test_print[] __initconst = { + { small_bitmap, sizeof(small_bitmap) * BITS_PER_BYTE, small_mask, small_list }, + { large_bitmap, sizeof(large_bitmap) * BITS_PER_BYTE, large_mask, large_list }, +}; + +static void __init test_bitmap_print_buf(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(test_print); i++) { + const struct test_bitmap_print *t = &test_print[i]; + int n; + + n = bitmap_print_bitmask_to_buf(print_buf, t->bitmap, t->nbits, + 0, 2 * PAGE_SIZE); + expect_eq_uint(strlen(t->mask) + 1, n); + expect_eq_str(t->mask, print_buf, n); + + n = bitmap_print_list_to_buf(print_buf, t->bitmap, t->nbits, + 0, 2 * PAGE_SIZE); + expect_eq_uint(strlen(t->list) + 1, n); + expect_eq_str(t->list, print_buf, n); + + /* test by non-zero offset */ + if (strlen(t->list) > PAGE_SIZE) { + n = bitmap_print_list_to_buf(print_buf, t->bitmap, t->nbits, + PAGE_SIZE, PAGE_SIZE); + expect_eq_uint(strlen(t->list) + 1 - PAGE_SIZE, n); + expect_eq_str(t->list + PAGE_SIZE, print_buf, n); + } + } +} + static void __init selftest(void) { test_zero_clear(); @@ -665,6 +821,7 @@ static void __init selftest(void) test_mem_optimisations(); test_for_each_set_clump8(); test_bitmap_cut(); + test_bitmap_print_buf(); } KSTM_MODULE_LOADERS(test_bitmap); diff --git a/lib/test_bitops.c b/lib/test_bitops.c index 471141ddd691..3b7bcbee84db 100644 --- a/lib/test_bitops.c +++ b/lib/test_bitops.c @@ -15,7 +15,7 @@ * get_count_order/long */ -/* use an enum because thats the most common BITMAP usage */ +/* use an enum because that's the most common BITMAP usage */ enum bitops_fun { BITOPS_4 = 4, BITOPS_7 = 7, diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 4dc4dcbecd12..adae39567264 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -52,6 +52,7 @@ #define FLAG_NO_DATA BIT(0) #define FLAG_EXPECTED_FAIL BIT(1) #define FLAG_SKB_FRAG BIT(2) +#define FLAG_VERIFIER_ZEXT BIT(3) enum { CLASSIC = BIT(6), /* Old BPF instructions only. */ @@ -80,6 +81,7 @@ struct bpf_test { int expected_errcode; /* used when FLAG_EXPECTED_FAIL is set in the aux */ __u8 frag_data[MAX_DATA]; int stack_depth; /* for eBPF only, since tests don't call verifier */ + int nr_testruns; /* Custom run count, defaults to MAX_TESTRUNS if 0 */ }; /* Large test cases need separate allocation and fill handler. */ @@ -461,6 +463,2602 @@ static int bpf_fill_stxdw(struct bpf_test *self) return __bpf_fill_stxdw(self, BPF_DW); } +static int __bpf_ld_imm64(struct bpf_insn insns[2], u8 reg, s64 imm64) +{ + struct bpf_insn tmp[] = {BPF_LD_IMM64(reg, imm64)}; + + memcpy(insns, tmp, sizeof(tmp)); + return 2; +} + +/* + * Branch conversion tests. Complex operations can expand to a lot + * of instructions when JITed. This in turn may cause jump offsets + * to overflow the field size of the native instruction, triggering + * a branch conversion mechanism in some JITs. + */ +static int __bpf_fill_max_jmp(struct bpf_test *self, int jmp, int imm) +{ + struct bpf_insn *insns; + int len = S16_MAX + 5; + int i; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + i = __bpf_ld_imm64(insns, R1, 0x0123456789abcdefULL); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insns[i++] = BPF_JMP_IMM(jmp, R0, imm, S16_MAX); + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 2); + insns[i++] = BPF_EXIT_INSN(); + + while (i < len - 1) { + static const int ops[] = { + BPF_LSH, BPF_RSH, BPF_ARSH, BPF_ADD, + BPF_SUB, BPF_MUL, BPF_DIV, BPF_MOD, + }; + int op = ops[(i >> 1) % ARRAY_SIZE(ops)]; + + if (i & 1) + insns[i++] = BPF_ALU32_REG(op, R0, R1); + else + insns[i++] = BPF_ALU64_REG(op, R0, R1); + } + + insns[i++] = BPF_EXIT_INSN(); + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* Branch taken by runtime decision */ +static int bpf_fill_max_jmp_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JEQ, 1); +} + +/* Branch not taken by runtime decision */ +static int bpf_fill_max_jmp_not_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JEQ, 0); +} + +/* Branch always taken, known at JIT time */ +static int bpf_fill_max_jmp_always_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JGE, 0); +} + +/* Branch never taken, known at JIT time */ +static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) +{ + return __bpf_fill_max_jmp(self, BPF_JLT, 0); +} + +/* ALU result computation used in tests */ +static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) +{ + *res = 0; + switch (op) { + case BPF_MOV: + *res = v2; + break; + case BPF_AND: + *res = v1 & v2; + break; + case BPF_OR: + *res = v1 | v2; + break; + case BPF_XOR: + *res = v1 ^ v2; + break; + case BPF_LSH: + *res = v1 << v2; + break; + case BPF_RSH: + *res = v1 >> v2; + break; + case BPF_ARSH: + *res = v1 >> v2; + if (v2 > 0 && v1 > S64_MAX) + *res |= ~0ULL << (64 - v2); + break; + case BPF_ADD: + *res = v1 + v2; + break; + case BPF_SUB: + *res = v1 - v2; + break; + case BPF_MUL: + *res = v1 * v2; + break; + case BPF_DIV: + if (v2 == 0) + return false; + *res = div64_u64(v1, v2); + break; + case BPF_MOD: + if (v2 == 0) + return false; + div64_u64_rem(v1, v2, res); + break; + } + return true; +} + +/* Test an ALU shift operation for all valid shift values */ +static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, + u8 mode, bool alu32) +{ + static const s64 regs[] = { + 0x0123456789abcdefLL, /* dword > 0, word < 0 */ + 0xfedcba9876543210LL, /* dowrd < 0, word > 0 */ + 0xfedcba0198765432LL, /* dowrd < 0, word < 0 */ + 0x0123458967abcdefLL, /* dword > 0, word > 0 */ + }; + int bits = alu32 ? 32 : 64; + int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; + struct bpf_insn *insn; + int imm, k; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (k = 0; k < ARRAY_SIZE(regs); k++) { + s64 reg = regs[k]; + + i += __bpf_ld_imm64(&insn[i], R3, reg); + + for (imm = 0; imm < bits; imm++) { + u64 val; + + /* Perform operation */ + insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); + if (alu32) { + if (mode == BPF_K) + insn[i++] = BPF_ALU32_IMM(op, R1, imm); + else + insn[i++] = BPF_ALU32_REG(op, R1, R2); + + if (op == BPF_ARSH) + reg = (s32)reg; + else + reg = (u32)reg; + __bpf_alu_result(&val, reg, imm, op); + val = (u32)val; + } else { + if (mode == BPF_K) + insn[i++] = BPF_ALU64_IMM(op, R1, imm); + else + insn[i++] = BPF_ALU64_REG(op, R1, R2); + __bpf_alu_result(&val, reg, imm, op); + } + + /* + * When debugging a JIT that fails this test, one + * can write the immediate value to R0 here to find + * out which operand values that fail. + */ + + /* Load reference and check the result */ + i += __bpf_ld_imm64(&insn[i], R4, val); + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R4, 1); + insn[i++] = BPF_EXIT_INSN(); + } + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, false); +} + +static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, false); +} + +static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, false); +} + +static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, false); +} + +static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, false); +} + +static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, false); +} + +static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, true); +} + +static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, true); +} + +static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, true); +} + +static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, true); +} + +static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, true); +} + +static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true); +} + +/* + * Test an ALU register shift operation for all valid shift values + * for the case when the source and destination are the same. + */ +static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, + bool alu32) +{ + int bits = alu32 ? 32 : 64; + int len = 3 + 6 * bits; + struct bpf_insn *insn; + int i = 0; + u64 val; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (val = 0; val < bits; val++) { + u64 res; + + /* Perform operation */ + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); + if (alu32) + insn[i++] = BPF_ALU32_REG(op, R1, R1); + else + insn[i++] = BPF_ALU64_REG(op, R1, R1); + + /* Compute the reference result */ + __bpf_alu_result(&res, val, val, op); + if (alu32) + res = (u32)res; + i += __bpf_ld_imm64(&insn[i], R2, res); + + /* Check the actual result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, false); +} + +static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) +{ + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, true); +} + +/* + * Common operand pattern generator for exhaustive power-of-two magnitudes + * tests. The block size parameters can be adjusted to increase/reduce the + * number of combinatons tested and thereby execution speed and memory + * footprint. + */ + +static inline s64 value(int msb, int delta, int sign) +{ + return sign * (1LL << msb) + delta; +} + +static int __bpf_fill_pattern(struct bpf_test *self, void *arg, + int dbits, int sbits, int block1, int block2, + int (*emit)(struct bpf_test*, void*, + struct bpf_insn*, s64, s64)) +{ + static const int sgn[][2] = {{1, 1}, {1, -1}, {-1, 1}, {-1, -1}}; + struct bpf_insn *insns; + int di, si, bt, db, sb; + int count, len, k; + int extra = 1 + 2; + int i = 0; + + /* Total number of iterations for the two pattern */ + count = (dbits - 1) * (sbits - 1) * block1 * block1 * ARRAY_SIZE(sgn); + count += (max(dbits, sbits) - 1) * block2 * block2 * ARRAY_SIZE(sgn); + + /* Compute the maximum number of insns and allocate the buffer */ + len = extra + count * (*emit)(self, arg, NULL, 0, 0); + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Add head instruction(s) */ + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + /* + * Pattern 1: all combinations of power-of-two magnitudes and sign, + * and with a block of contiguous values around each magnitude. + */ + for (di = 0; di < dbits - 1; di++) /* Dst magnitudes */ + for (si = 0; si < sbits - 1; si++) /* Src magnitudes */ + for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ + for (db = -(block1 / 2); + db < (block1 + 1) / 2; db++) + for (sb = -(block1 / 2); + sb < (block1 + 1) / 2; sb++) { + s64 dst, src; + + dst = value(di, db, sgn[k][0]); + src = value(si, sb, sgn[k][1]); + i += (*emit)(self, arg, + &insns[i], + dst, src); + } + /* + * Pattern 2: all combinations for a larger block of values + * for each power-of-two magnitude and sign, where the magnitude is + * the same for both operands. + */ + for (bt = 0; bt < max(dbits, sbits) - 1; bt++) /* Magnitude */ + for (k = 0; k < ARRAY_SIZE(sgn); k++) /* Sign combos */ + for (db = -(block2 / 2); db < (block2 + 1) / 2; db++) + for (sb = -(block2 / 2); + sb < (block2 + 1) / 2; sb++) { + s64 dst, src; + + dst = value(bt % dbits, db, sgn[k][0]); + src = value(bt % sbits, sb, sgn[k][1]); + i += (*emit)(self, arg, &insns[i], + dst, src); + } + + /* Append tail instructions */ + insns[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insns[i++] = BPF_EXIT_INSN(); + BUG_ON(i > len); + + self->u.ptr.insns = insns; + self->u.ptr.len = i; + + return 0; +} + +/* + * Block size parameters used in pattern tests below. une as needed to + * increase/reduce the number combinations tested, see following examples. + * block values per operand MSB + * ---------------------------------------- + * 0 none + * 1 (1 << MSB) + * 2 (1 << MSB) + [-1, 0] + * 3 (1 << MSB) + [-1, 0, 1] + */ +#define PATTERN_BLOCK1 1 +#define PATTERN_BLOCK2 5 + +/* Number of test runs for a pattern test */ +#define NR_PATTERN_RUNS 1 + +/* + * Exhaustive tests of ALU operations for all combinations of power-of-two + * magnitudes of the operands, both for positive and negative values. The + * test is designed to verify e.g. the ALU and ALU64 operations for JITs that + * emit different code depending on the magnitude of the immediate value. + */ +static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 7; + + if (__bpf_alu_result(&res, dst, (s32)imm, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R3, res); + insns[i++] = BPF_ALU64_IMM(op, R1, imm); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 7; + + if (__bpf_alu_result(&res, (u32)dst, (u32)imm, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + insns[i++] = BPF_ALU32_IMM(op, R1, imm); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 9; + + if (__bpf_alu_result(&res, dst, src, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, res); + insns[i++] = BPF_ALU64_REG(op, R1, R2); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + int i = 0; + u64 res; + + if (!insns) + return 9; + + if (__bpf_alu_result(&res, (u32)dst, (u32)src, op)) { + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + insns[i++] = BPF_ALU32_REG(op, R1, R2); + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + } + + return i; +} + +static int __bpf_fill_alu64_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu64_imm); +} + +static int __bpf_fill_alu32_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu32_imm); +} + +static int __bpf_fill_alu64_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu64_reg); +} + +static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_alu32_reg); +} + +/* ALU64 immediate operations */ +static int bpf_fill_alu64_mov_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOV); +} + +static int bpf_fill_alu64_and_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_AND); +} + +static int bpf_fill_alu64_or_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_OR); +} + +static int bpf_fill_alu64_xor_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_XOR); +} + +static int bpf_fill_alu64_add_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_ADD); +} + +static int bpf_fill_alu64_sub_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_SUB); +} + +static int bpf_fill_alu64_mul_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MUL); +} + +static int bpf_fill_alu64_div_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_DIV); +} + +static int bpf_fill_alu64_mod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOD); +} + +/* ALU32 immediate operations */ +static int bpf_fill_alu32_mov_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOV); +} + +static int bpf_fill_alu32_and_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_AND); +} + +static int bpf_fill_alu32_or_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_OR); +} + +static int bpf_fill_alu32_xor_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_XOR); +} + +static int bpf_fill_alu32_add_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_ADD); +} + +static int bpf_fill_alu32_sub_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_SUB); +} + +static int bpf_fill_alu32_mul_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MUL); +} + +static int bpf_fill_alu32_div_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_DIV); +} + +static int bpf_fill_alu32_mod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOD); +} + +/* ALU64 register operations */ +static int bpf_fill_alu64_mov_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MOV); +} + +static int bpf_fill_alu64_and_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_AND); +} + +static int bpf_fill_alu64_or_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_OR); +} + +static int bpf_fill_alu64_xor_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_XOR); +} + +static int bpf_fill_alu64_add_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_ADD); +} + +static int bpf_fill_alu64_sub_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_SUB); +} + +static int bpf_fill_alu64_mul_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MUL); +} + +static int bpf_fill_alu64_div_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_DIV); +} + +static int bpf_fill_alu64_mod_reg(struct bpf_test *self) +{ + return __bpf_fill_alu64_reg(self, BPF_MOD); +} + +/* ALU32 register operations */ +static int bpf_fill_alu32_mov_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MOV); +} + +static int bpf_fill_alu32_and_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_AND); +} + +static int bpf_fill_alu32_or_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_OR); +} + +static int bpf_fill_alu32_xor_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_XOR); +} + +static int bpf_fill_alu32_add_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_ADD); +} + +static int bpf_fill_alu32_sub_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_SUB); +} + +static int bpf_fill_alu32_mul_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MUL); +} + +static int bpf_fill_alu32_div_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_DIV); +} + +static int bpf_fill_alu32_mod_reg(struct bpf_test *self) +{ + return __bpf_fill_alu32_reg(self, BPF_MOD); +} + +/* + * Test JITs that implement complex ALU operations as function + * calls, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) +{ + int len = 2 + 10 * 10; + struct bpf_insn *insns; + u64 dst, res; + int i = 0; + u32 imm; + int rd; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Operand and result values according to operation */ + if (alu32) + dst = 0x76543210U; + else + dst = 0x7edcba9876543210ULL; + imm = 0x01234567U; + + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + imm &= 31; + + __bpf_alu_result(&res, dst, imm, op); + + if (alu32) + res = (u32)res; + + /* Check all operand registers */ + for (rd = R0; rd <= R9; rd++) { + i += __bpf_ld_imm64(&insns[i], rd, dst); + + if (alu32) + insns[i++] = BPF_ALU32_IMM(op, rd, imm); + else + insns[i++] = BPF_ALU64_IMM(op, rd, imm); + + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res >> 32, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + } + + insns[i++] = BPF_MOV64_IMM(R0, 1); + insns[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* ALU64 K registers */ +static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOV, false); +} + +static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_AND, false); +} + +static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_OR, false); +} + +static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_XOR, false); +} + +static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, false); +} + +static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ADD, false); +} + +static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_SUB, false); +} + +static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MUL, false); +} + +static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, false); +} + +static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, false); +} + +/* ALU32 K registers */ +static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOV, true); +} + +static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_AND, true); +} + +static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_OR, true); +} + +static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_XOR, true); +} + +static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, true); +} + +static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_ADD, true); +} + +static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_SUB, true); +} + +static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MUL, true); +} + +static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, true); +} + +static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, true); +} + +/* + * Test JITs that implement complex ALU operations as function + * calls, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) +{ + int len = 2 + 10 * 10 * 12; + u64 dst, src, res, same; + struct bpf_insn *insns; + int rd, rs; + int i = 0; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Operand and result values according to operation */ + if (alu32) { + dst = 0x76543210U; + src = 0x01234567U; + } else { + dst = 0x7edcba9876543210ULL; + src = 0x0123456789abcdefULL; + } + + if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) + src &= 31; + + __bpf_alu_result(&res, dst, src, op); + __bpf_alu_result(&same, src, src, op); + + if (alu32) { + res = (u32)res; + same = (u32)same; + } + + /* Check all combinations of operand registers */ + for (rd = R0; rd <= R9; rd++) { + for (rs = R0; rs <= R9; rs++) { + u64 val = rd == rs ? same : res; + + i += __bpf_ld_imm64(&insns[i], rd, dst); + i += __bpf_ld_imm64(&insns[i], rs, src); + + if (alu32) + insns[i++] = BPF_ALU32_REG(op, rd, rs); + else + insns[i++] = BPF_ALU64_REG(op, rd, rs); + + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_ALU64_IMM(BPF_RSH, rd, 32); + insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, val >> 32, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + } + } + + insns[i++] = BPF_MOV64_IMM(R0, 1); + insns[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* ALU64 X register combinations */ +static int bpf_fill_alu64_mov_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOV, false); +} + +static int bpf_fill_alu64_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_AND, false); +} + +static int bpf_fill_alu64_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_OR, false); +} + +static int bpf_fill_alu64_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_XOR, false); +} + +static int bpf_fill_alu64_lsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_LSH, false); +} + +static int bpf_fill_alu64_rsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_RSH, false); +} + +static int bpf_fill_alu64_arsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, false); +} + +static int bpf_fill_alu64_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ADD, false); +} + +static int bpf_fill_alu64_sub_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_SUB, false); +} + +static int bpf_fill_alu64_mul_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MUL, false); +} + +static int bpf_fill_alu64_div_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_DIV, false); +} + +static int bpf_fill_alu64_mod_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOD, false); +} + +/* ALU32 X register combinations */ +static int bpf_fill_alu32_mov_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOV, true); +} + +static int bpf_fill_alu32_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_AND, true); +} + +static int bpf_fill_alu32_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_OR, true); +} + +static int bpf_fill_alu32_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_XOR, true); +} + +static int bpf_fill_alu32_lsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_LSH, true); +} + +static int bpf_fill_alu32_rsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_RSH, true); +} + +static int bpf_fill_alu32_arsh_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ARSH, true); +} + +static int bpf_fill_alu32_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_ADD, true); +} + +static int bpf_fill_alu32_sub_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_SUB, true); +} + +static int bpf_fill_alu32_mul_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MUL, true); +} + +static int bpf_fill_alu32_div_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_DIV, true); +} + +static int bpf_fill_alu32_mod_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_alu_reg_pairs(self, BPF_MOD, true); +} + +/* + * Exhaustive tests of atomic operations for all power-of-two operand + * magnitudes, both for positive and negative values. + */ + +static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + u64 keep, fetch, res; + int i = 0; + + if (!insns) + return 21; + + switch (op) { + case BPF_XCHG: + res = src; + break; + default: + __bpf_alu_result(&res, dst, src, BPF_OP(op)); + } + + keep = 0x0123456789abcdefULL; + if (op & BPF_FETCH) + fetch = dst; + else + fetch = src; + + i += __bpf_ld_imm64(&insns[i], R0, keep); + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, res); + i += __bpf_ld_imm64(&insns[i], R4, fetch); + i += __bpf_ld_imm64(&insns[i], R5, keep); + + insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); + insns[i++] = BPF_ATOMIC_OP(BPF_DW, op, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R1, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + u64 keep, fetch, res; + int i = 0; + + if (!insns) + return 21; + + switch (op) { + case BPF_XCHG: + res = src; + break; + default: + __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op)); + } + + keep = 0x0123456789abcdefULL; + if (op & BPF_FETCH) + fetch = (u32)dst; + else + fetch = src; + + i += __bpf_ld_imm64(&insns[i], R0, keep); + i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + i += __bpf_ld_imm64(&insns[i], R3, (u32)res); + i += __bpf_ld_imm64(&insns[i], R4, fetch); + i += __bpf_ld_imm64(&insns[i], R5, keep); + + insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); + insns[i++] = BPF_ATOMIC_OP(BPF_W, op, R10, R2, -4); + insns[i++] = BPF_LDX_MEM(BPF_W, R1, R10, -4); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R4, 1); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R5, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_cmpxchg64(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int i = 0; + + if (!insns) + return 23; + + i += __bpf_ld_imm64(&insns[i], R0, ~dst); + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + + /* Result unsuccessful */ + insns[i++] = BPF_STX_MEM(BPF_DW, R10, R1, -8); + insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + /* Result successful */ + insns[i++] = BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -8); + insns[i++] = BPF_LDX_MEM(BPF_DW, R3, R10, -8); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R2, R3, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + insns[i++] = BPF_MOV64_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_emit_cmpxchg32(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int i = 0; + + if (!insns) + return 27; + + i += __bpf_ld_imm64(&insns[i], R0, ~dst); + i += __bpf_ld_imm64(&insns[i], R1, (u32)dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + + /* Result unsuccessful */ + insns[i++] = BPF_STX_MEM(BPF_W, R10, R1, -4); + insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); + insns[i++] = BPF_ZEXT_REG(R0), /* Zext always inserted by verifier */ + insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); + + insns[i++] = BPF_JMP32_REG(BPF_JEQ, R1, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + /* Result successful */ + i += __bpf_ld_imm64(&insns[i], R0, dst); + insns[i++] = BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R2, -4); + insns[i++] = BPF_ZEXT_REG(R0), /* Zext always inserted by verifier */ + insns[i++] = BPF_LDX_MEM(BPF_W, R3, R10, -4); + + insns[i++] = BPF_JMP32_REG(BPF_JEQ, R2, R3, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + insns[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + insns[i++] = BPF_MOV32_IMM(R0, __LINE__); + insns[i++] = BPF_EXIT_INSN(); + + return i; +} + +static int __bpf_fill_atomic64(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + 0, PATTERN_BLOCK2, + &__bpf_emit_atomic64); +} + +static int __bpf_fill_atomic32(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + 0, PATTERN_BLOCK2, + &__bpf_emit_atomic32); +} + +/* 64-bit atomic operations */ +static int bpf_fill_atomic64_add(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_ADD); +} + +static int bpf_fill_atomic64_and(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_AND); +} + +static int bpf_fill_atomic64_or(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_OR); +} + +static int bpf_fill_atomic64_xor(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XOR); +} + +static int bpf_fill_atomic64_add_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic64_and_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic64_or_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xor_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xchg(struct bpf_test *self) +{ + return __bpf_fill_atomic64(self, BPF_XCHG); +} + +static int bpf_fill_cmpxchg64(struct bpf_test *self) +{ + return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, + &__bpf_emit_cmpxchg64); +} + +/* 32-bit atomic operations */ +static int bpf_fill_atomic32_add(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_ADD); +} + +static int bpf_fill_atomic32_and(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_AND); +} + +static int bpf_fill_atomic32_or(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_OR); +} + +static int bpf_fill_atomic32_xor(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XOR); +} + +static int bpf_fill_atomic32_add_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic32_and_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic32_or_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xor_fetch(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xchg(struct bpf_test *self) +{ + return __bpf_fill_atomic32(self, BPF_XCHG); +} + +static int bpf_fill_cmpxchg32(struct bpf_test *self) +{ + return __bpf_fill_pattern(self, NULL, 64, 64, 0, PATTERN_BLOCK2, + &__bpf_emit_cmpxchg32); +} + +/* + * Test JITs that implement ATOMIC operations as function calls or + * other primitives, and must re-arrange operands for argument passing. + */ +static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) +{ + struct bpf_insn *insn; + int len = 2 + 34 * 10 * 10; + u64 mem, upd, res; + int rd, rs, i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + /* Operand and memory values */ + if (width == BPF_DW) { + mem = 0x0123456789abcdefULL; + upd = 0xfedcba9876543210ULL; + } else { /* BPF_W */ + mem = 0x01234567U; + upd = 0x76543210U; + } + + /* Memory updated according to operation */ + switch (op) { + case BPF_XCHG: + res = upd; + break; + case BPF_CMPXCHG: + res = mem; + break; + default: + __bpf_alu_result(&res, mem, upd, BPF_OP(op)); + } + + /* Test all operand registers */ + for (rd = R0; rd <= R9; rd++) { + for (rs = R0; rs <= R9; rs++) { + u64 cmp, src; + + /* Initialize value in memory */ + i += __bpf_ld_imm64(&insn[i], R0, mem); + insn[i++] = BPF_STX_MEM(width, R10, R0, -8); + + /* Initialize registers in order */ + i += __bpf_ld_imm64(&insn[i], R0, ~mem); + i += __bpf_ld_imm64(&insn[i], rs, upd); + insn[i++] = BPF_MOV64_REG(rd, R10); + + /* Perform atomic operation */ + insn[i++] = BPF_ATOMIC_OP(width, op, rd, rs, -8); + if (op == BPF_CMPXCHG && width == BPF_W) + insn[i++] = BPF_ZEXT_REG(R0); + + /* Check R0 register value */ + if (op == BPF_CMPXCHG) + cmp = mem; /* Expect value from memory */ + else if (R0 == rd || R0 == rs) + cmp = 0; /* Aliased, checked below */ + else + cmp = ~mem; /* Expect value to be preserved */ + if (cmp) { + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, + (u32)cmp, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + insn[i++] = BPF_ALU64_IMM(BPF_RSH, R0, 32); + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, R0, + cmp >> 32, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check source register value */ + if (rs == R0 && op == BPF_CMPXCHG) + src = 0; /* Aliased with R0, checked above */ + else if (rs == rd && (op == BPF_CMPXCHG || + !(op & BPF_FETCH))) + src = 0; /* Aliased with rd, checked below */ + else if (op == BPF_CMPXCHG) + src = upd; /* Expect value to be preserved */ + else if (op & BPF_FETCH) + src = mem; /* Expect fetched value from mem */ + else /* no fetch */ + src = upd; /* Expect value to be preserved */ + if (src) { + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, + (u32)src, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + insn[i++] = BPF_ALU64_IMM(BPF_RSH, rs, 32); + insn[i++] = BPF_JMP32_IMM(BPF_JEQ, rs, + src >> 32, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check destination register value */ + if (!(rd == R0 && op == BPF_CMPXCHG) && + !(rd == rs && (op & BPF_FETCH))) { + insn[i++] = BPF_JMP_REG(BPF_JEQ, rd, R10, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + + /* Check value in memory */ + if (rs != rd) { /* No aliasing */ + i += __bpf_ld_imm64(&insn[i], R1, res); + } else if (op == BPF_XCHG) { /* Aliased, XCHG */ + insn[i++] = BPF_MOV64_REG(R1, R10); + } else if (op == BPF_CMPXCHG) { /* Aliased, CMPXCHG */ + i += __bpf_ld_imm64(&insn[i], R1, mem); + } else { /* Aliased, ALU oper */ + i += __bpf_ld_imm64(&insn[i], R1, mem); + insn[i++] = BPF_ALU64_REG(BPF_OP(op), R1, R10); + } + + insn[i++] = BPF_LDX_MEM(width, R0, R10, -8); + if (width == BPF_DW) + insn[i++] = BPF_JMP_REG(BPF_JEQ, R0, R1, 2); + else /* width == BPF_W */ + insn[i++] = BPF_JMP32_REG(BPF_JEQ, R0, R1, 2); + insn[i++] = BPF_MOV32_IMM(R0, __LINE__); + insn[i++] = BPF_EXIT_INSN(); + } + } + + insn[i++] = BPF_MOV64_IMM(R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = i; + BUG_ON(i > len); + + return 0; +} + +/* 64-bit atomic register tests */ +static int bpf_fill_atomic64_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD); +} + +static int bpf_fill_atomic64_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND); +} + +static int bpf_fill_atomic64_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR); +} + +static int bpf_fill_atomic64_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR); +} + +static int bpf_fill_atomic64_add_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic64_and_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic64_or_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xor_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic64_xchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_XCHG); +} + +static int bpf_fill_atomic64_cmpxchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_DW, BPF_CMPXCHG); +} + +/* 32-bit atomic register tests */ +static int bpf_fill_atomic32_add_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD); +} + +static int bpf_fill_atomic32_and_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND); +} + +static int bpf_fill_atomic32_or_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR); +} + +static int bpf_fill_atomic32_xor_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR); +} + +static int bpf_fill_atomic32_add_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_ADD | BPF_FETCH); +} + +static int bpf_fill_atomic32_and_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_AND | BPF_FETCH); +} + +static int bpf_fill_atomic32_or_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_OR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xor_fetch_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XOR | BPF_FETCH); +} + +static int bpf_fill_atomic32_xchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_XCHG); +} + +static int bpf_fill_atomic32_cmpxchg_reg_pairs(struct bpf_test *self) +{ + return __bpf_fill_atomic_reg_pairs(self, BPF_W, BPF_CMPXCHG); +} + +/* + * Test the two-instruction 64-bit immediate load operation for all + * power-of-two magnitudes of the immediate operand. For each MSB, a block + * of immediate values centered around the power-of-two MSB are tested, + * both for positive and negative values. The test is designed to verify + * the operation for JITs that emit different code depending on the magnitude + * of the immediate value. This is often the case if the native instruction + * immediate field width is narrower than 32 bits. + */ +static int bpf_fill_ld_imm64_magn(struct bpf_test *self) +{ + int block = 64; /* Increase for more tests per MSB position */ + int len = 3 + 8 * 63 * block * 2; + struct bpf_insn *insn; + int bit, adj, sign; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (bit = 0; bit <= 62; bit++) { + for (adj = -block / 2; adj < block / 2; adj++) { + for (sign = -1; sign <= 1; sign += 2) { + s64 imm = sign * ((1LL << bit) + adj); + + /* Perform operation */ + i += __bpf_ld_imm64(&insn[i], R1, imm); + + /* Load reference */ + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, + (u32)(imm >> 32)); + insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); + insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); + + /* Check result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + } + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +/* + * Test the two-instruction 64-bit immediate load operation for different + * combinations of bytes. Each byte in the 64-bit word is constructed as + * (base & mask) | (rand() & ~mask), where rand() is a deterministic LCG. + * All patterns (base1, mask1) and (base2, mask2) bytes are tested. + */ +static int __bpf_fill_ld_imm64_bytes(struct bpf_test *self, + u8 base1, u8 mask1, + u8 base2, u8 mask2) +{ + struct bpf_insn *insn; + int len = 3 + 8 * BIT(8); + int pattern, index; + u32 rand = 1; + int i = 0; + + insn = kmalloc_array(len, sizeof(*insn), GFP_KERNEL); + if (!insn) + return -ENOMEM; + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + + for (pattern = 0; pattern < BIT(8); pattern++) { + u64 imm = 0; + + for (index = 0; index < 8; index++) { + int byte; + + if (pattern & BIT(index)) + byte = (base1 & mask1) | (rand & ~mask1); + else + byte = (base2 & mask2) | (rand & ~mask2); + imm = (imm << 8) | byte; + } + + /* Update our LCG */ + rand = rand * 1664525 + 1013904223; + + /* Perform operation */ + i += __bpf_ld_imm64(&insn[i], R1, imm); + + /* Load reference */ + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R2, imm); + insn[i++] = BPF_ALU32_IMM(BPF_MOV, R3, (u32)(imm >> 32)); + insn[i++] = BPF_ALU64_IMM(BPF_LSH, R3, 32); + insn[i++] = BPF_ALU64_REG(BPF_OR, R2, R3); + + /* Check result */ + insn[i++] = BPF_JMP_REG(BPF_JEQ, R1, R2, 1); + insn[i++] = BPF_EXIT_INSN(); + } + + insn[i++] = BPF_ALU64_IMM(BPF_MOV, R0, 1); + insn[i++] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insn; + self->u.ptr.len = len; + BUG_ON(i != len); + + return 0; +} + +static int bpf_fill_ld_imm64_checker(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 0, 0xff, 0xff, 0xff); +} + +static int bpf_fill_ld_imm64_pos_neg(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0x80, 0x80); +} + +static int bpf_fill_ld_imm64_pos_zero(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 1, 0x81, 0, 0xff); +} + +static int bpf_fill_ld_imm64_neg_zero(struct bpf_test *self) +{ + return __bpf_fill_ld_imm64_bytes(self, 0x80, 0x80, 0, 0xff); +} + +/* + * Exhaustive tests of JMP operations for all combinations of power-of-two + * magnitudes of the operands, both for positive and negative values. The + * test is designed to verify e.g. the JMP and JMP32 operations for JITs that + * emit different code depending on the magnitude of the immediate value. + */ + +static bool __bpf_match_jmp_cond(s64 v1, s64 v2, u8 op) +{ + switch (op) { + case BPF_JSET: + return !!(v1 & v2); + case BPF_JEQ: + return v1 == v2; + case BPF_JNE: + return v1 != v2; + case BPF_JGT: + return (u64)v1 > (u64)v2; + case BPF_JGE: + return (u64)v1 >= (u64)v2; + case BPF_JLT: + return (u64)v1 < (u64)v2; + case BPF_JLE: + return (u64)v1 <= (u64)v2; + case BPF_JSGT: + return v1 > v2; + case BPF_JSGE: + return v1 >= v2; + case BPF_JSLT: + return v1 < v2; + case BPF_JSLE: + return v1 <= v2; + } + return false; +} + +static int __bpf_emit_jmp_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond(dst, (s32)imm, op); + int i = 0; + + insns[i++] = BPF_ALU32_IMM(BPF_MOV, R0, match); + + i += __bpf_ld_imm64(&insns[i], R1, dst); + insns[i++] = BPF_JMP_IMM(op, R1, imm, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 5 + 1; +} + +static int __bpf_emit_jmp32_imm(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 imm) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond((s32)dst, (s32)imm, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + insns[i++] = BPF_JMP32_IMM(op, R1, imm, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 5; +} + +static int __bpf_emit_jmp_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond(dst, src, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + insns[i++] = BPF_JMP_REG(op, R1, R2, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 7; +} + +static int __bpf_emit_jmp32_reg(struct bpf_test *self, void *arg, + struct bpf_insn *insns, s64 dst, s64 src) +{ + int op = *(int *)arg; + + if (insns) { + bool match = __bpf_match_jmp_cond((s32)dst, (s32)src, op); + int i = 0; + + i += __bpf_ld_imm64(&insns[i], R1, dst); + i += __bpf_ld_imm64(&insns[i], R2, src); + insns[i++] = BPF_JMP32_REG(op, R1, R2, 1); + if (!match) + insns[i++] = BPF_JMP_IMM(BPF_JA, 0, 0, 1); + insns[i++] = BPF_EXIT_INSN(); + + return i; + } + + return 7; +} + +static int __bpf_fill_jmp_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp_imm); +} + +static int __bpf_fill_jmp32_imm(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 32, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp32_imm); +} + +static int __bpf_fill_jmp_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp_reg); +} + +static int __bpf_fill_jmp32_reg(struct bpf_test *self, int op) +{ + return __bpf_fill_pattern(self, &op, 64, 64, + PATTERN_BLOCK1, PATTERN_BLOCK2, + &__bpf_emit_jmp32_reg); +} + +/* JMP immediate tests */ +static int bpf_fill_jmp_jset_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSET); +} + +static int bpf_fill_jmp_jeq_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JEQ); +} + +static int bpf_fill_jmp_jne_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JNE); +} + +static int bpf_fill_jmp_jgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JGT); +} + +static int bpf_fill_jmp_jge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JGE); +} + +static int bpf_fill_jmp_jlt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JLT); +} + +static int bpf_fill_jmp_jle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JLE); +} + +static int bpf_fill_jmp_jsgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSGT); +} + +static int bpf_fill_jmp_jsge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSGE); +} + +static int bpf_fill_jmp_jslt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSLT); +} + +static int bpf_fill_jmp_jsle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp_imm(self, BPF_JSLE); +} + +/* JMP32 immediate tests */ +static int bpf_fill_jmp32_jset_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSET); +} + +static int bpf_fill_jmp32_jeq_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JEQ); +} + +static int bpf_fill_jmp32_jne_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JNE); +} + +static int bpf_fill_jmp32_jgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JGT); +} + +static int bpf_fill_jmp32_jge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JGE); +} + +static int bpf_fill_jmp32_jlt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JLT); +} + +static int bpf_fill_jmp32_jle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JLE); +} + +static int bpf_fill_jmp32_jsgt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSGT); +} + +static int bpf_fill_jmp32_jsge_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSGE); +} + +static int bpf_fill_jmp32_jslt_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSLT); +} + +static int bpf_fill_jmp32_jsle_imm(struct bpf_test *self) +{ + return __bpf_fill_jmp32_imm(self, BPF_JSLE); +} + +/* JMP register tests */ +static int bpf_fill_jmp_jset_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSET); +} + +static int bpf_fill_jmp_jeq_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JEQ); +} + +static int bpf_fill_jmp_jne_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JNE); +} + +static int bpf_fill_jmp_jgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JGT); +} + +static int bpf_fill_jmp_jge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JGE); +} + +static int bpf_fill_jmp_jlt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JLT); +} + +static int bpf_fill_jmp_jle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JLE); +} + +static int bpf_fill_jmp_jsgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSGT); +} + +static int bpf_fill_jmp_jsge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSGE); +} + +static int bpf_fill_jmp_jslt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSLT); +} + +static int bpf_fill_jmp_jsle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp_reg(self, BPF_JSLE); +} + +/* JMP32 register tests */ +static int bpf_fill_jmp32_jset_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSET); +} + +static int bpf_fill_jmp32_jeq_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JEQ); +} + +static int bpf_fill_jmp32_jne_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JNE); +} + +static int bpf_fill_jmp32_jgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JGT); +} + +static int bpf_fill_jmp32_jge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JGE); +} + +static int bpf_fill_jmp32_jlt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JLT); +} + +static int bpf_fill_jmp32_jle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JLE); +} + +static int bpf_fill_jmp32_jsgt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSGT); +} + +static int bpf_fill_jmp32_jsge_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSGE); +} + +static int bpf_fill_jmp32_jslt_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSLT); +} + +static int bpf_fill_jmp32_jsle_reg(struct bpf_test *self) +{ + return __bpf_fill_jmp32_reg(self, BPF_JSLE); +} + +/* + * Set up a sequence of staggered jumps, forwards and backwards with + * increasing offset. This tests the conversion of relative jumps to + * JITed native jumps. On some architectures, for example MIPS, a large + * PC-relative jump offset may overflow the immediate field of the native + * conditional branch instruction, triggering a conversion to use an + * absolute jump instead. Since this changes the jump offsets, another + * offset computation pass is necessary, and that may in turn trigger + * another branch conversion. This jump sequence is particularly nasty + * in that regard. + * + * The sequence generation is parameterized by size and jump type. + * The size must be even, and the expected result is always size + 1. + * Below is an example with size=8 and result=9. + * + * ________________________Start + * R0 = 0 + * R1 = r1 + * R2 = r2 + * ,------- JMP +4 * 3______________Preamble: 4 insns + * ,----------|-ind 0- if R0 != 7 JMP 8 * 3 + 1 <--------------------. + * | | R0 = 8 | + * | | JMP +7 * 3 ------------------------. + * | ,--------|-----1- if R0 != 5 JMP 7 * 3 + 1 <--------------. | | + * | | | R0 = 6 | | | + * | | | JMP +5 * 3 ------------------. | | + * | | ,------|-----2- if R0 != 3 JMP 6 * 3 + 1 <--------. | | | | + * | | | | R0 = 4 | | | | | + * | | | | JMP +3 * 3 ------------. | | | | + * | | | ,----|-----3- if R0 != 1 JMP 5 * 3 + 1 <--. | | | | | | + * | | | | | R0 = 2 | | | | | | | + * | | | | | JMP +1 * 3 ------. | | | | | | + * | | | | ,--t=====4> if R0 != 0 JMP 4 * 3 + 1 1 2 3 4 5 6 7 8 loc + * | | | | | R0 = 1 -1 +2 -3 +4 -5 +6 -7 +8 off + * | | | | | JMP -2 * 3 ---' | | | | | | | + * | | | | | ,------5- if R0 != 2 JMP 3 * 3 + 1 <-----' | | | | | | + * | | | | | | R0 = 3 | | | | | | + * | | | | | | JMP -4 * 3 ---------' | | | | | + * | | | | | | ,----6- if R0 != 4 JMP 2 * 3 + 1 <-----------' | | | | + * | | | | | | | R0 = 5 | | | | + * | | | | | | | JMP -6 * 3 ---------------' | | | + * | | | | | | | ,--7- if R0 != 6 JMP 1 * 3 + 1 <-----------------' | | + * | | | | | | | | R0 = 7 | | + * | | Error | | | JMP -8 * 3 ---------------------' | + * | | paths | | | ,8- if R0 != 8 JMP 0 * 3 + 1 <-----------------------' + * | | | | | | | | | R0 = 9__________________Sequence: 3 * size - 1 insns + * `-+-+-+-+-+-+-+-+-> EXIT____________________Return: 1 insn + * + */ + +/* The maximum size parameter */ +#define MAX_STAGGERED_JMP_SIZE ((0x7fff / 3) & ~1) + +/* We use a reduced number of iterations to get a reasonable execution time */ +#define NR_STAGGERED_JMP_RUNS 10 + +static int __bpf_fill_staggered_jumps(struct bpf_test *self, + const struct bpf_insn *jmp, + u64 r1, u64 r2) +{ + int size = self->test[0].result - 1; + int len = 4 + 3 * (size + 1); + struct bpf_insn *insns; + int off, ind; + + insns = kmalloc_array(len, sizeof(*insns), GFP_KERNEL); + if (!insns) + return -ENOMEM; + + /* Preamble */ + insns[0] = BPF_ALU64_IMM(BPF_MOV, R0, 0); + insns[1] = BPF_ALU64_IMM(BPF_MOV, R1, r1); + insns[2] = BPF_ALU64_IMM(BPF_MOV, R2, r2); + insns[3] = BPF_JMP_IMM(BPF_JA, 0, 0, 3 * size / 2); + + /* Sequence */ + for (ind = 0, off = size; ind <= size; ind++, off -= 2) { + struct bpf_insn *ins = &insns[4 + 3 * ind]; + int loc; + + if (off == 0) + off--; + + loc = abs(off); + ins[0] = BPF_JMP_IMM(BPF_JNE, R0, loc - 1, + 3 * (size - ind) + 1); + ins[1] = BPF_ALU64_IMM(BPF_MOV, R0, loc); + ins[2] = *jmp; + ins[2].off = 3 * (off - 1); + } + + /* Return */ + insns[len - 1] = BPF_EXIT_INSN(); + + self->u.ptr.insns = insns; + self->u.ptr.len = len; + + return 0; +} + +/* 64-bit unconditional jump */ +static int bpf_fill_staggered_ja(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JA, 0, 0, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0, 0); +} + +/* 64-bit immediate jumps */ +static int bpf_fill_staggered_jeq_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JEQ, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jne_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JNE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); +} + +static int bpf_fill_staggered_jset_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSET, R1, 0x82, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); +} + +static int bpf_fill_staggered_jgt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGT, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); +} + +static int bpf_fill_staggered_jge_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JGE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jlt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLT, R1, 0x80000000, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jle_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JLE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jsgt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGT, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +static int bpf_fill_staggered_jsge_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSGE, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jslt_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLT, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jsle_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_IMM(BPF_JSLE, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +/* 64-bit register jumps */ +static int bpf_fill_staggered_jeq_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JEQ, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jne_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JNE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); +} + +static int bpf_fill_staggered_jset_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSET, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); +} + +static int bpf_fill_staggered_jgt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); +} + +static int bpf_fill_staggered_jge_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jlt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); +} + +static int bpf_fill_staggered_jle_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jsgt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); +} + +static int bpf_fill_staggered_jsge_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); +} + +static int bpf_fill_staggered_jslt_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); +} + +static int bpf_fill_staggered_jsle_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP_REG(BPF_JSLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); +} + +/* 32-bit immediate jumps */ +static int bpf_fill_staggered_jeq32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JEQ, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jne32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JNE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 0); +} + +static int bpf_fill_staggered_jset32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSET, R1, 0x82, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0); +} + +static int bpf_fill_staggered_jgt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGT, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 0); +} + +static int bpf_fill_staggered_jge32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JGE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jlt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLT, R1, 0x80000000, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jle32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JLE, R1, 1234, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0); +} + +static int bpf_fill_staggered_jsgt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGT, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +static int bpf_fill_staggered_jsge32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSGE, R1, -2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jslt32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLT, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, 0); +} + +static int bpf_fill_staggered_jsle32_imm(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_IMM(BPF_JSLE, R1, -1, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, 0); +} + +/* 32-bit register jumps */ +static int bpf_fill_staggered_jeq32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JEQ, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jne32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JNE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 4321, 1234); +} + +static int bpf_fill_staggered_jset32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSET, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x86, 0x82); +} + +static int bpf_fill_staggered_jgt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 0x80000000, 1234); +} + +static int bpf_fill_staggered_jge32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jlt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 0x80000000); +} + +static int bpf_fill_staggered_jle32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, 1234, 1234); +} + +static int bpf_fill_staggered_jsgt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -2); +} + +static int bpf_fill_staggered_jsge32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSGE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -2); +} + +static int bpf_fill_staggered_jslt32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLT, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -2, -1); +} + +static int bpf_fill_staggered_jsle32_reg(struct bpf_test *self) +{ + struct bpf_insn jmp = BPF_JMP32_REG(BPF_JSLE, R1, R2, 0); + + return __bpf_fill_staggered_jumps(self, &jmp, -1, -1); +} + + static struct bpf_test tests[] = { { "TAX", @@ -1095,7 +3693,7 @@ static struct bpf_test tests[] = { { "RET_A", .u.insns = { - /* check that unitialized X and A contain zeros */ + /* check that uninitialized X and A contain zeros */ BPF_STMT(BPF_MISC | BPF_TXA, 0), BPF_STMT(BPF_RET | BPF_A, 0) }, @@ -1916,6 +4514,22 @@ static struct bpf_test tests[] = { { }, { { 0, -1 } } }, +#ifdef CONFIG_32BIT + { + "INT: 32-bit context pointer word order and zero-extension", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_JMP32_IMM(BPF_JEQ, R1, 0, 3), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_JMP32_IMM(BPF_JNE, R1, 0, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, +#endif { "check: missing ret", .u.insns = { @@ -2361,6 +4975,48 @@ static struct bpf_test tests[] = { { { 0, 0x1 } }, }, { + "ALU_MOV_K: small negative", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "ALU_MOV_K: small negative zero extension", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU_MOV_K: large negative", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123456789), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123456789 } } + }, + { + "ALU_MOV_K: large negative zero extension", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123456789), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { "ALU64_MOV_K: dst = 2", .u.insns_int = { BPF_ALU64_IMM(BPF_MOV, R0, 2), @@ -2412,6 +5068,48 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_MOV_K: small negative", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "ALU64_MOV_K: small negative sign extension", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xffffffff } } + }, + { + "ALU64_MOV_K: large negative", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123456789), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123456789 } } + }, + { + "ALU64_MOV_K: large negative sign extension", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, -123456789), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xffffffff } } + }, /* BPF_ALU | BPF_ADD | BPF_X */ { "ALU_ADD_X: 1 + 2 = 3", @@ -2967,6 +5665,31 @@ static struct bpf_test tests[] = { { }, { { 0, 2147483647 } }, }, + { + "ALU64_MUL_X: 64x64 multiply, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0fedcba987654321LL), + BPF_LD_IMM64(R1, 0x123456789abcdef0LL), + BPF_ALU64_REG(BPF_MUL, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xe5618cf0 } } + }, + { + "ALU64_MUL_X: 64x64 multiply, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0fedcba987654321LL), + BPF_LD_IMM64(R1, 0x123456789abcdef0LL), + BPF_ALU64_REG(BPF_MUL, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x2236d88f } } + }, /* BPF_ALU | BPF_MUL | BPF_K */ { "ALU_MUL_K: 2 * 3 = 6", @@ -3077,6 +5800,29 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_MUL_K: 64x32 multiply, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xe242d208 } } + }, + { + "ALU64_MUL_K: 64x32 multiply, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_MUL, R0, 0x12345678), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xc28f5c28 } } + }, /* BPF_ALU | BPF_DIV | BPF_X */ { "ALU_DIV_X: 6 / 2 = 3", @@ -3431,6 +6177,44 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { + "ALU_AND_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_AND, R0, 15), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 4 } } + }, + { + "ALU_AND_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), + BPF_ALU32_IMM(BPF_AND, R0, 0xafbfcfdf), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xa1b2c3d4 } } + }, + { + "ALU_AND_K: Zero extension", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x0000000080a0c0e0LL), + BPF_ALU32_IMM(BPF_AND, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { "ALU64_AND_K: 3 & 2 = 2", .u.insns_int = { BPF_LD_IMM64(R0, 3), @@ -3453,7 +6237,7 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { - "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000ffff00000000", + "ALU64_AND_K: 0x0000ffffffff0000 & 0x0 = 0x0000000000000000", .u.insns_int = { BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), BPF_LD_IMM64(R3, 0x0000000000000000LL), @@ -3469,7 +6253,7 @@ static struct bpf_test tests[] = { { { 0, 0x1 } }, }, { - "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffffffff", + "ALU64_AND_K: 0x0000ffffffff0000 & -1 = 0x0000ffffffff0000", .u.insns_int = { BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), @@ -3500,6 +6284,38 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_AND_K: Sign extension 1", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x00000000090b0d0fLL), + BPF_ALU64_IMM(BPF_AND, R0, 0x0f0f0f0f), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "ALU64_AND_K: Sign extension 2", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x0123456780a0c0e0LL), + BPF_ALU64_IMM(BPF_AND, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, /* BPF_ALU | BPF_OR | BPF_X */ { "ALU_OR_X: 1 | 2 = 3", @@ -3573,6 +6389,44 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { + "ALU_OR_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_OR, R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x01020305 } } + }, + { + "ALU_OR_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_OR, R0, 0xa0b0c0d0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xa1b2c3d4 } } + }, + { + "ALU_OR_K: Zero extension", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x00000000f9fbfdffLL), + BPF_ALU32_IMM(BPF_OR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { "ALU64_OR_K: 1 | 2 = 3", .u.insns_int = { BPF_LD_IMM64(R0, 1), @@ -3595,7 +6449,7 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, }, { - "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffff00000000", + "ALU64_OR_K: 0x0000ffffffff0000 | 0x0 = 0x0000ffffffff0000", .u.insns_int = { BPF_LD_IMM64(R2, 0x0000ffffffff0000LL), BPF_LD_IMM64(R3, 0x0000ffffffff0000LL), @@ -3642,6 +6496,38 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_OR_K: Sign extension 1", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x012345678fafcfefLL), + BPF_ALU64_IMM(BPF_OR, R0, 0x0f0f0f0f), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "ALU64_OR_K: Sign extension 2", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0xfffffffff9fbfdffLL), + BPF_ALU64_IMM(BPF_OR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, /* BPF_ALU | BPF_XOR | BPF_X */ { "ALU_XOR_X: 5 ^ 6 = 3", @@ -3715,6 +6601,44 @@ static struct bpf_test tests[] = { { { 0, 0xfffffffe } }, }, { + "ALU_XOR_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x01020304), + BPF_ALU32_IMM(BPF_XOR, R0, 15), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x0102030b } } + }, + { + "ALU_XOR_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xf1f2f3f4), + BPF_ALU32_IMM(BPF_XOR, R0, 0xafbfcfdf), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x5e4d3c2b } } + }, + { + "ALU_XOR_K: Zero extension", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x00000000795b3d1fLL), + BPF_ALU32_IMM(BPF_XOR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { "ALU64_XOR_K: 5 ^ 6 = 3", .u.insns_int = { BPF_LD_IMM64(R0, 5), @@ -3726,7 +6650,7 @@ static struct bpf_test tests[] = { { { 0, 3 } }, }, { - "ALU64_XOR_K: 1 & 0xffffffff = 0xfffffffe", + "ALU64_XOR_K: 1 ^ 0xffffffff = 0xfffffffe", .u.insns_int = { BPF_LD_IMM64(R0, 1), BPF_ALU64_IMM(BPF_XOR, R0, 0xffffffff), @@ -3784,6 +6708,38 @@ static struct bpf_test tests[] = { { }, { { 0, 0x1 } }, }, + { + "ALU64_XOR_K: Sign extension 1", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0x0123456786a4c2e0LL), + BPF_ALU64_IMM(BPF_XOR, R0, 0x0f0f0f0f), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "ALU64_XOR_K: Sign extension 2", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_LD_IMM64(R1, 0xfedcba98795b3d1fLL), + BPF_ALU64_IMM(BPF_XOR, R0, 0xf0f0f0f0), + BPF_JMP_REG(BPF_JEQ, R0, R1, 2), + BPF_MOV32_IMM(R0, 2), + BPF_EXIT_INSN(), + BPF_MOV32_IMM(R0, 1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, /* BPF_ALU | BPF_LSH | BPF_X */ { "ALU_LSH_X: 1 << 1 = 2", @@ -3810,6 +6766,18 @@ static struct bpf_test tests[] = { { { 0, 0x80000000 } }, }, { + "ALU_LSH_X: 0x12345678 << 12 = 0x45678000", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU32_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x45678000 } } + }, + { "ALU64_LSH_X: 1 << 1 = 2", .u.insns_int = { BPF_LD_IMM64(R0, 1), @@ -3833,6 +6801,106 @@ static struct bpf_test tests[] = { { }, { { 0, 0x80000000 } }, }, + { + "ALU64_LSH_X: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xbcdef000 } } + }, + { + "ALU64_LSH_X: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x3456789a } } + }, + { + "ALU64_LSH_X: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_X: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x9abcdef0 } } + }, + { + "ALU64_LSH_X: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_X: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_LSH_X: Zero shift, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_LSH_X: Zero shift, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_LSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x01234567 } } + }, /* BPF_ALU | BPF_LSH | BPF_K */ { "ALU_LSH_K: 1 << 1 = 2", @@ -3857,6 +6925,28 @@ static struct bpf_test tests[] = { { { 0, 0x80000000 } }, }, { + "ALU_LSH_K: 0x12345678 << 12 = 0x45678000", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_LSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x45678000 } } + }, + { + "ALU_LSH_K: 0x12345678 << 0 = 0x12345678", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_LSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x12345678 } } + }, + { "ALU64_LSH_K: 1 << 1 = 2", .u.insns_int = { BPF_LD_IMM64(R0, 1), @@ -3878,6 +6968,86 @@ static struct bpf_test tests[] = { { }, { { 0, 0x80000000 } }, }, + { + "ALU64_LSH_K: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xbcdef000 } } + }, + { + "ALU64_LSH_K: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 12), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x3456789a } } + }, + { + "ALU64_LSH_K: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 36), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_K: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 36), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x9abcdef0 } } + }, + { + "ALU64_LSH_K: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_LSH_K: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 32), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_LSH_K: Zero shift", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_LSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, /* BPF_ALU | BPF_RSH | BPF_X */ { "ALU_RSH_X: 2 >> 1 = 1", @@ -3904,6 +7074,18 @@ static struct bpf_test tests[] = { { { 0, 1 } }, }, { + "ALU_RSH_X: 0x12345678 >> 20 = 0x123", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_MOV, R1, 20), + BPF_ALU32_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x123 } } + }, + { "ALU64_RSH_X: 2 >> 1 = 1", .u.insns_int = { BPF_LD_IMM64(R0, 2), @@ -3927,6 +7109,106 @@ static struct bpf_test tests[] = { { }, { { 0, 1 } }, }, + { + "ALU64_RSH_X: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_RSH_X: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x00081234 } } + }, + { + "ALU64_RSH_X: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x08123456 } } + }, + { + "ALU64_RSH_X: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_X: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_RSH_X: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_X: Zero shift, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_RSH_X: Zero shift, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_RSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, /* BPF_ALU | BPF_RSH | BPF_K */ { "ALU_RSH_K: 2 >> 1 = 1", @@ -3951,6 +7233,28 @@ static struct bpf_test tests[] = { { { 0, 1 } }, }, { + "ALU_RSH_K: 0x12345678 >> 20 = 0x123", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_RSH, R0, 20), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x123 } } + }, + { + "ALU_RSH_K: 0x12345678 >> 0 = 0x12345678", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x12345678), + BPF_ALU32_IMM(BPF_RSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x12345678 } } + }, + { "ALU64_RSH_K: 2 >> 1 = 1", .u.insns_int = { BPF_LD_IMM64(R0, 2), @@ -3972,9 +7276,101 @@ static struct bpf_test tests[] = { { }, { { 0, 1 } }, }, + { + "ALU64_RSH_K: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_RSH_K: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 12), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x00081234 } } + }, + { + "ALU64_RSH_K: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 36), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x08123456 } } + }, + { + "ALU64_RSH_K: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 36), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_K: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_RSH_K: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } } + }, + { + "ALU64_RSH_K: Zero shift", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, /* BPF_ALU | BPF_ARSH | BPF_X */ { - "ALU_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", + "ALU32_ARSH_X: -1234 >> 7 = -10", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -1234), + BPF_ALU32_IMM(BPF_MOV, R1, 7), + BPF_ALU32_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -10 } } + }, + { + "ALU64_ARSH_X: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", .u.insns_int = { BPF_LD_IMM64(R0, 0xff00ff0000000000LL), BPF_ALU32_IMM(BPF_MOV, R1, 40), @@ -3985,9 +7381,131 @@ static struct bpf_test tests[] = { { }, { { 0, 0xffff00ff } }, }, + { + "ALU64_ARSH_X: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_ARSH_X: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 12), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfff81234 } } + }, + { + "ALU64_ARSH_X: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xf8123456 } } + }, + { + "ALU64_ARSH_X: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 36), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_X: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_ARSH_X: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 32), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_X: Zero shift, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, + { + "ALU64_ARSH_X: Zero shift, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU32_IMM(BPF_MOV, R1, 0), + BPF_ALU64_REG(BPF_ARSH, R0, R1), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, /* BPF_ALU | BPF_ARSH | BPF_K */ { - "ALU_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", + "ALU32_ARSH_K: -1234 >> 7 = -10", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -1234), + BPF_ALU32_IMM(BPF_ARSH, R0, 7), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -10 } } + }, + { + "ALU32_ARSH_K: -1234 >> 0 = -1234", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -1234), + BPF_ALU32_IMM(BPF_ARSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1234 } } + }, + { + "ALU64_ARSH_K: 0xff00ff0000000000 >> 40 = 0xffffffffffff00ff", .u.insns_int = { BPF_LD_IMM64(R0, 0xff00ff0000000000LL), BPF_ALU64_IMM(BPF_ARSH, R0, 40), @@ -3997,6 +7515,86 @@ static struct bpf_test tests[] = { { }, { { 0, 0xffff00ff } }, }, + { + "ALU64_ARSH_K: Shift < 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_RSH, R0, 12), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x56789abc } } + }, + { + "ALU64_ARSH_K: Shift < 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 12), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfff81234 } } + }, + { + "ALU64_ARSH_K: Shift > 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 36), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xf8123456 } } + }, + { + "ALU64_ARSH_K: Shift > 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xf123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 36), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_K: Shift == 32, low word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x81234567 } } + }, + { + "ALU64_ARSH_K: Shift == 32, high word", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 32), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -1 } } + }, + { + "ALU64_ARSH_K: Zero shift", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x8123456789abcdefLL), + BPF_ALU64_IMM(BPF_ARSH, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x89abcdef } } + }, /* BPF_ALU | BPF_NEG */ { "ALU_NEG: -(3) = -3", @@ -4079,6 +7677,67 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) cpu_to_be64(0x0123456789abcdefLL) } }, }, + { + "ALU_END_FROM_BE 64: 0x0123456789abcdef >> 32 -> 0x01234567", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_be64(0x0123456789abcdefLL) >> 32) } }, + }, + /* BPF_ALU | BPF_END | BPF_FROM_BE, reversed */ + { + "ALU_END_FROM_BE 16: 0xfedcba9876543210 -> 0x3210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_be16(0x3210) } }, + }, + { + "ALU_END_FROM_BE 32: 0xfedcba9876543210 -> 0x76543210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_be32(0x76543210) } }, + }, + { + "ALU_END_FROM_BE 64: 0xfedcba9876543210 -> 0x76543210", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) cpu_to_be64(0xfedcba9876543210ULL) } }, + }, + { + "ALU_END_FROM_BE 64: 0xfedcba9876543210 >> 32 -> 0xfedcba98", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_BE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_be64(0xfedcba9876543210ULL) >> 32) } }, + }, /* BPF_ALU | BPF_END | BPF_FROM_LE */ { "ALU_END_FROM_LE 16: 0x0123456789abcdef -> 0xefcd", @@ -4116,6 +7775,321 @@ static struct bpf_test tests[] = { { }, { { 0, (u32) cpu_to_le64(0x0123456789abcdefLL) } }, }, + { + "ALU_END_FROM_LE 64: 0x0123456789abcdef >> 32 -> 0xefcdab89", + .u.insns_int = { + BPF_LD_IMM64(R0, 0x0123456789abcdefLL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_le64(0x0123456789abcdefLL) >> 32) } }, + }, + /* BPF_ALU | BPF_END | BPF_FROM_LE, reversed */ + { + "ALU_END_FROM_LE 16: 0xfedcba9876543210 -> 0x1032", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 16), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_le16(0x3210) } }, + }, + { + "ALU_END_FROM_LE 32: 0xfedcba9876543210 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 32), + BPF_ALU64_REG(BPF_MOV, R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU32_REG(BPF_ADD, R0, R1), /* R1 = 0 */ + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, cpu_to_le32(0x76543210) } }, + }, + { + "ALU_END_FROM_LE 64: 0xfedcba9876543210 -> 0x10325476", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) cpu_to_le64(0xfedcba9876543210ULL) } }, + }, + { + "ALU_END_FROM_LE 64: 0xfedcba9876543210 >> 32 -> 0x98badcfe", + .u.insns_int = { + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_ENDIAN(BPF_FROM_LE, R0, 64), + BPF_ALU64_IMM(BPF_RSH, R0, 32), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, (u32) (cpu_to_le64(0xfedcba9876543210ULL) >> 32) } }, + }, + /* BPF_LDX_MEM B/H/W/DW */ + { + "BPF_LDX_MEM | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000000000008ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_B, R0, R10, -1), +#else + BPF_LDX_MEM(BPF_B, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_B, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000000000088ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_B, R0, R10, -1), +#else + BPF_LDX_MEM(BPF_B, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000000000708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_H, R0, R10, -2), +#else + BPF_LDX_MEM(BPF_H, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_H, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000000008788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_H, R0, R10, -2), +#else + BPF_LDX_MEM(BPF_H, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_LD_IMM64(R2, 0x0000000005060708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_W, R0, R10, -4), +#else + BPF_LDX_MEM(BPF_W, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_W, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_LD_IMM64(R2, 0x0000000085868788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_LDX_MEM(BPF_W, R0, R10, -4), +#else + BPF_LDX_MEM(BPF_W, R0, R10, -8), +#endif + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* BPF_STX_MEM B/H/W/DW */ + { + "BPF_STX_MEM | BPF_B", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d0e008ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_B, R10, R2, -1), +#else + BPF_STX_MEM(BPF_B, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_B, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d0e088ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_B, R10, R2, -1), +#else + BPF_STX_MEM(BPF_B, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_H", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d00708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_H, R10, R2, -2), +#else + BPF_STX_MEM(BPF_H, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_H, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b0c0d08788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_H, R10, R2, -2), +#else + BPF_STX_MEM(BPF_H, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_W", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x0102030405060708ULL), + BPF_LD_IMM64(R3, 0x8090a0b005060708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_W, R10, R2, -4), +#else + BPF_STX_MEM(BPF_W, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_STX_MEM | BPF_W, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8090a0b0c0d0e0f0ULL), + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x8090a0b085868788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), +#ifdef __BIG_ENDIAN + BPF_STX_MEM(BPF_W, R10, R2, -4), +#else + BPF_STX_MEM(BPF_W, R10, R2, -8), +#endif + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, /* BPF_ST(X) | BPF_MEM | BPF_B/H/W/DW */ { "ST_MEM_B: Store/Load byte: max negative", @@ -4286,8 +8260,8 @@ static struct bpf_test tests[] = { .u.insns_int = { BPF_LD_IMM64(R0, 0), BPF_LD_IMM64(R1, 0xffffffffffffffffLL), - BPF_STX_MEM(BPF_W, R10, R1, -40), - BPF_LDX_MEM(BPF_W, R0, R10, -40), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_LDX_MEM(BPF_DW, R0, R10, -40), BPF_EXIT_INSN(), }, INTERNAL, @@ -4295,80 +8269,358 @@ static struct bpf_test tests[] = { { { 0, 0xffffffff } }, .stack_depth = 40, }, + { + "STX_MEM_DW: Store double word: first word in memory", + .u.insns_int = { + BPF_LD_IMM64(R0, 0), + BPF_LD_IMM64(R1, 0x0123456789abcdefLL), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_LDX_MEM(BPF_W, R0, R10, -40), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, +#ifdef __BIG_ENDIAN + { { 0, 0x01234567 } }, +#else + { { 0, 0x89abcdef } }, +#endif + .stack_depth = 40, + }, + { + "STX_MEM_DW: Store double word: second word in memory", + .u.insns_int = { + BPF_LD_IMM64(R0, 0), + BPF_LD_IMM64(R1, 0x0123456789abcdefLL), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_LDX_MEM(BPF_W, R0, R10, -36), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, +#ifdef __BIG_ENDIAN + { { 0, 0x89abcdef } }, +#else + { { 0, 0x01234567 } }, +#endif + .stack_depth = 40, + }, /* BPF_STX | BPF_ATOMIC | BPF_W/DW */ { - "STX_XADD_W: Test: 0x12 + 0x10 = 0x22", + "STX_XADD_W: X + 1 + 1 + 1 + ...", + { }, + INTERNAL, + { }, + { { 0, 4134 } }, + .fill_helper = bpf_fill_stxw, + }, + { + "STX_XADD_DW: X + 1 + 1 + 1 + ...", + { }, + INTERNAL, + { }, + { { 0, 4134 } }, + .fill_helper = bpf_fill_stxdw, + }, + /* + * Exhaustive tests of atomic operation variants. + * Individual tests are expanded from template macros for all + * combinations of ALU operation, word size and fetching. + */ +#define BPF_ATOMIC_POISON(width) ((width) == BPF_W ? (0xbaadf00dULL << 32) : 0) + +#define BPF_ATOMIC_OP_TEST1(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_LD_IMM64(R5, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(width, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R5, -40), \ + BPF_LDX_MEM(width, R0, R10, -40), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, result } }, \ + .stack_depth = 40, \ +} +#define BPF_ATOMIC_OP_TEST2(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test side effects, r10: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_ALU64_REG(BPF_MOV, R1, R10), \ + BPF_LD_IMM64(R0, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(BPF_W, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R0, -40), \ + BPF_ALU64_REG(BPF_MOV, R0, R10), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + .stack_depth = 40, \ +} +#define BPF_ATOMIC_OP_TEST3(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test side effects, r0: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_ALU64_REG(BPF_MOV, R0, R10), \ + BPF_LD_IMM64(R1, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(width, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R1, -40), \ + BPF_ALU64_REG(BPF_SUB, R0, R10), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + .stack_depth = 40, \ +} +#define BPF_ATOMIC_OP_TEST4(width, op, logic, old, update, result) \ +{ \ + "BPF_ATOMIC | " #width ", " #op ": Test fetch: " \ + #old " " #logic " " #update " = " #result, \ + .u.insns_int = { \ + BPF_LD_IMM64(R3, (update) | BPF_ATOMIC_POISON(width)), \ + BPF_ST_MEM(width, R10, -40, old), \ + BPF_ATOMIC_OP(width, op, R10, R3, -40), \ + BPF_ALU32_REG(BPF_MOV, R0, R3), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, (op) & BPF_FETCH ? old : update } }, \ + .stack_depth = 40, \ +} + /* BPF_ATOMIC | BPF_W: BPF_ADD */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_W: BPF_ADD | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_DW: BPF_ADD */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_DW: BPF_ADD | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_ADD | BPF_FETCH, +, 0x12, 0xab, 0xbd), + /* BPF_ATOMIC | BPF_W: BPF_AND */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_W: BPF_AND | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_DW: BPF_AND */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_DW: BPF_AND | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_AND | BPF_FETCH, &, 0x12, 0xab, 0x02), + /* BPF_ATOMIC | BPF_W: BPF_OR */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_W: BPF_OR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_DW: BPF_OR */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_DW: BPF_OR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_OR | BPF_FETCH, |, 0x12, 0xab, 0xbb), + /* BPF_ATOMIC | BPF_W: BPF_XOR */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_W: BPF_XOR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_DW: BPF_XOR */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_DW: BPF_XOR | BPF_FETCH */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XOR | BPF_FETCH, ^, 0x12, 0xab, 0xb9), + /* BPF_ATOMIC | BPF_W: BPF_XCHG */ + BPF_ATOMIC_OP_TEST1(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST2(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST3(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST4(BPF_W, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + /* BPF_ATOMIC | BPF_DW: BPF_XCHG */ + BPF_ATOMIC_OP_TEST1(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST2(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST3(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), + BPF_ATOMIC_OP_TEST4(BPF_DW, BPF_XCHG, xchg, 0x12, 0xab, 0xab), +#undef BPF_ATOMIC_POISON +#undef BPF_ATOMIC_OP_TEST1 +#undef BPF_ATOMIC_OP_TEST2 +#undef BPF_ATOMIC_OP_TEST3 +#undef BPF_ATOMIC_OP_TEST4 + /* BPF_ATOMIC | BPF_W, BPF_CMPXCHG */ + { + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful return", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x01234567 } }, + .stack_depth = 40, + }, + { + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test successful store", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40), + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), BPF_LDX_MEM(BPF_W, R0, R10, -40), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x22 } }, + { { 0, 0x89abcdef } }, .stack_depth = 40, }, { - "STX_XADD_W: Test side-effects, r10: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure return", .u.insns_int = { - BPF_ALU64_REG(BPF_MOV, R1, R10), - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40), - BPF_ALU64_REG(BPF_MOV, R0, R10), - BPF_ALU64_REG(BPF_SUB, R0, R1), + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0 } }, + { { 0, 0x01234567 } }, .stack_depth = 40, }, { - "STX_XADD_W: Test side-effects, r0: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test failure store", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_W, R10, -40, 0x10), - BPF_ATOMIC_OP(BPF_W, BPF_ADD, R10, R0, -40), + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x76543210), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_LDX_MEM(BPF_W, R0, R10, -40), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x12 } }, + { { 0, 0x01234567 } }, .stack_depth = 40, }, { - "STX_XADD_W: X + 1 + 1 + 1 + ...", + "BPF_ATOMIC | BPF_W, BPF_CMPXCHG: Test side effects", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -40, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R0, 0x01234567), + BPF_ALU32_IMM(BPF_MOV, R3, 0x89abcdef), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R3, -40), + BPF_ALU32_REG(BPF_MOV, R0, R3), + BPF_EXIT_INSN(), + }, + INTERNAL, { }, + { { 0, 0x89abcdef } }, + .stack_depth = 40, + }, + /* BPF_ATOMIC | BPF_DW, BPF_CMPXCHG */ + { + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful return", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), + BPF_ALU64_REG(BPF_SUB, R0, R1), + BPF_EXIT_INSN(), + }, INTERNAL, { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxw, + { { 0, 0 } }, + .stack_depth = 40, }, { - "STX_XADD_DW: Test: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test successful store", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40), + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_STX_MEM(BPF_DW, R10, R0, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), BPF_LDX_MEM(BPF_DW, R0, R10, -40), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_REG(BPF_SUB, R0, R2), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x22 } }, + { { 0, 0 } }, .stack_depth = 40, }, { - "STX_XADD_DW: Test side-effects, r10: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure return", .u.insns_int = { - BPF_ALU64_REG(BPF_MOV, R1, R10), - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40), - BPF_ALU64_REG(BPF_MOV, R0, R10), + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_ALU64_IMM(BPF_ADD, R0, 1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), BPF_ALU64_REG(BPF_SUB, R0, R1), BPF_EXIT_INSN(), }, @@ -4378,25 +8630,552 @@ static struct bpf_test tests[] = { .stack_depth = 40, }, { - "STX_XADD_DW: Test side-effects, r0: 0x12 + 0x10 = 0x22", + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test failure store", .u.insns_int = { - BPF_ALU32_IMM(BPF_MOV, R0, 0x12), - BPF_ST_MEM(BPF_DW, R10, -40, 0x10), - BPF_ATOMIC_OP(BPF_DW, BPF_ADD, R10, R0, -40), + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_ALU64_IMM(BPF_ADD, R0, 1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_LDX_MEM(BPF_DW, R0, R10, -40), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), + BPF_ALU64_REG(BPF_SUB, R0, R1), BPF_EXIT_INSN(), }, INTERNAL, { }, - { { 0, 0x12 } }, + { { 0, 0 } }, .stack_depth = 40, }, { - "STX_XADD_DW: X + 1 + 1 + 1 + ...", + "BPF_ATOMIC | BPF_DW, BPF_CMPXCHG: Test side effects", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789abcdefULL), + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_STX_MEM(BPF_DW, R10, R1, -40), + BPF_ATOMIC_OP(BPF_DW, BPF_CMPXCHG, R10, R2, -40), + BPF_LD_IMM64(R0, 0xfedcba9876543210ULL), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_REG(BPF_SUB, R0, R2), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 40, + }, + /* BPF_JMP32 | BPF_JEQ | BPF_K */ + { + "JMP32_JEQ_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JEQ, R0, 321, 1), + BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, { }, + { { 0, 123 } } + }, + { + "JMP32_JEQ_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 12345678), + BPF_JMP32_IMM(BPF_JEQ, R0, 12345678 & 0xffff, 1), + BPF_JMP32_IMM(BPF_JEQ, R0, 12345678, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, INTERNAL, { }, - { { 0, 4134 } }, - .fill_helper = bpf_fill_stxdw, + { { 0, 12345678 } } + }, + { + "JMP32_JEQ_K: negative immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JEQ, R0, 123, 1), + BPF_JMP32_IMM(BPF_JEQ, R0, -123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + /* BPF_JMP32 | BPF_JEQ | BPF_X */ + { + "JMP32_JEQ_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 1234), + BPF_ALU32_IMM(BPF_MOV, R1, 4321), + BPF_JMP32_REG(BPF_JEQ, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 1234), + BPF_JMP32_REG(BPF_JEQ, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1234 } } + }, + /* BPF_JMP32 | BPF_JNE | BPF_K */ + { + "JMP32_JNE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), + BPF_JMP32_IMM(BPF_JNE, R0, 321, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JNE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 12345678), + BPF_JMP32_IMM(BPF_JNE, R0, 12345678, 1), + BPF_JMP32_IMM(BPF_JNE, R0, 12345678 & 0xffff, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 12345678 } } + }, + { + "JMP32_JNE_K: negative immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JNE, R0, -123, 1), + BPF_JMP32_IMM(BPF_JNE, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + /* BPF_JMP32 | BPF_JNE | BPF_X */ + { + "JMP32_JNE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 1234), + BPF_ALU32_IMM(BPF_MOV, R1, 1234), + BPF_JMP32_REG(BPF_JNE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 4321), + BPF_JMP32_REG(BPF_JNE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1234 } } + }, + /* BPF_JMP32 | BPF_JSET | BPF_K */ + { + "JMP32_JSET_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSET, R0, 2, 1), + BPF_JMP32_IMM(BPF_JSET, R0, 3, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 1 } } + }, + { + "JMP32_JSET_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0x40000000), + BPF_JMP32_IMM(BPF_JSET, R0, 0x3fffffff, 1), + BPF_JMP32_IMM(BPF_JSET, R0, 0x60000000, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0x40000000 } } + }, + { + "JMP32_JSET_K: negative immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSET, R0, -1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + /* BPF_JMP32 | BPF_JSET | BPF_X */ + { + "JMP32_JSET_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 8), + BPF_ALU32_IMM(BPF_MOV, R1, 7), + BPF_JMP32_REG(BPF_JSET, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 8 | 2), + BPF_JMP32_REG(BPF_JNE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 8 } } + }, + /* BPF_JMP32 | BPF_JGT | BPF_K */ + { + "JMP32_JGT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JGT, R0, 123, 1), + BPF_JMP32_IMM(BPF_JGT, R0, 122, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JGT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JGT, R0, 0xffffffff, 1), + BPF_JMP32_IMM(BPF_JGT, R0, 0xfffffffd, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JGT | BPF_X */ + { + "JMP32_JGT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), + BPF_JMP32_REG(BPF_JGT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), + BPF_JMP32_REG(BPF_JGT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JGE | BPF_K */ + { + "JMP32_JGE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JGE, R0, 124, 1), + BPF_JMP32_IMM(BPF_JGE, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JGE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JGE, R0, 0xffffffff, 1), + BPF_JMP32_IMM(BPF_JGE, R0, 0xfffffffe, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JGE | BPF_X */ + { + "JMP32_JGE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), + BPF_JMP32_REG(BPF_JGE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), + BPF_JMP32_REG(BPF_JGE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLT | BPF_K */ + { + "JMP32_JLT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JLT, R0, 123, 1), + BPF_JMP32_IMM(BPF_JLT, R0, 124, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JLT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JLT, R0, 0xfffffffd, 1), + BPF_JMP32_IMM(BPF_JLT, R0, 0xffffffff, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLT | BPF_X */ + { + "JMP32_JLT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), + BPF_JMP32_REG(BPF_JLT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xffffffff), + BPF_JMP32_REG(BPF_JLT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLE | BPF_K */ + { + "JMP32_JLE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 123), + BPF_JMP32_IMM(BPF_JLE, R0, 122, 1), + BPF_JMP32_IMM(BPF_JLE, R0, 123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } } + }, + { + "JMP32_JLE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffd, 1), + BPF_JMP32_IMM(BPF_JLE, R0, 0xfffffffe, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JLE | BPF_X */ + { + "JMP32_JLE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, 0xfffffffe), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffd), + BPF_JMP32_REG(BPF_JLE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, 0xfffffffe), + BPF_JMP32_REG(BPF_JLE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0xfffffffe } } + }, + /* BPF_JMP32 | BPF_JSGT | BPF_K */ + { + "JMP32_JSGT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSGT, R0, -123, 1), + BPF_JMP32_IMM(BPF_JSGT, R0, -124, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSGT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSGT, R0, -12345678, 1), + BPF_JMP32_IMM(BPF_JSGT, R0, -12345679, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSGT | BPF_X */ + { + "JMP32_JSGT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSGT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345679), + BPF_JMP32_REG(BPF_JSGT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSGE | BPF_K */ + { + "JMP32_JSGE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSGE, R0, -122, 1), + BPF_JMP32_IMM(BPF_JSGE, R0, -123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSGE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSGE, R0, -12345677, 1), + BPF_JMP32_IMM(BPF_JSGE, R0, -12345678, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSGE | BPF_X */ + { + "JMP32_JSGE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345677), + BPF_JMP32_REG(BPF_JSGE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSGE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLT | BPF_K */ + { + "JMP32_JSLT_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSLT, R0, -123, 1), + BPF_JMP32_IMM(BPF_JSLT, R0, -122, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSLT_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSLT, R0, -12345678, 1), + BPF_JMP32_IMM(BPF_JSLT, R0, -12345677, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLT | BPF_X */ + { + "JMP32_JSLT_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSLT, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345677), + BPF_JMP32_REG(BPF_JSLT, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLE | BPF_K */ + { + "JMP32_JSLE_K: Small immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -123), + BPF_JMP32_IMM(BPF_JSLE, R0, -124, 1), + BPF_JMP32_IMM(BPF_JSLE, R0, -123, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -123 } } + }, + { + "JMP32_JSLE_K: Large immediate", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_JMP32_IMM(BPF_JSLE, R0, -12345679, 1), + BPF_JMP32_IMM(BPF_JSLE, R0, -12345678, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } + }, + /* BPF_JMP32 | BPF_JSLE | BPF_K */ + { + "JMP32_JSLE_X", + .u.insns_int = { + BPF_ALU32_IMM(BPF_MOV, R0, -12345678), + BPF_ALU32_IMM(BPF_MOV, R1, -12345679), + BPF_JMP32_REG(BPF_JSLE, R0, R1, 2), + BPF_ALU32_IMM(BPF_MOV, R1, -12345678), + BPF_JMP32_REG(BPF_JSLE, R0, R1, 1), + BPF_ALU32_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, -12345678 } } }, /* BPF_JMP | BPF_EXIT */ { @@ -6436,6 +11215,2841 @@ static struct bpf_test tests[] = { {}, { { 0, 2 } }, }, + /* BPF_LDX_MEM with operand aliasing */ + { + "LDX_MEM_B: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_B, R10, -8, 123), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_B, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_H: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_H, R10, -8, 12345), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_H, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 12345 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_W: operand register aliasing", + .u.insns_int = { + BPF_ST_MEM(BPF_W, R10, -8, 123456789), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_W, R0, R0, -8), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 123456789 } }, + .stack_depth = 8, + }, + { + "LDX_MEM_DW: operand register aliasing", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x123456789abcdefULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_MOV64_REG(R0, R10), + BPF_LDX_MEM(BPF_DW, R0, R0, -8), + BPF_ALU64_REG(BPF_SUB, R0, R1), + BPF_MOV64_REG(R1, R0), + BPF_ALU64_IMM(BPF_RSH, R1, 32), + BPF_ALU64_REG(BPF_OR, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* + * Register (non-)clobbering tests for the case where a JIT implements + * complex ALU or ATOMIC operations via function calls. If so, the + * function call must be transparent to the eBPF registers. The JIT + * must therefore save and restore relevant registers across the call. + * The following tests check that the eBPF registers retain their + * values after such an operation. Mainly intended for complex ALU + * and atomic operation, but we run it for all. You never know... + * + * Note that each operations should be tested twice with different + * destinations, to check preservation for all registers. + */ +#define BPF_TEST_CLOBBER_ALU(alu, op, dst, src) \ + { \ + #alu "_" #op " to " #dst ": no clobbering", \ + .u.insns_int = { \ + BPF_ALU64_IMM(BPF_MOV, R0, R0), \ + BPF_ALU64_IMM(BPF_MOV, R1, R1), \ + BPF_ALU64_IMM(BPF_MOV, R2, R2), \ + BPF_ALU64_IMM(BPF_MOV, R3, R3), \ + BPF_ALU64_IMM(BPF_MOV, R4, R4), \ + BPF_ALU64_IMM(BPF_MOV, R5, R5), \ + BPF_ALU64_IMM(BPF_MOV, R6, R6), \ + BPF_ALU64_IMM(BPF_MOV, R7, R7), \ + BPF_ALU64_IMM(BPF_MOV, R8, R8), \ + BPF_ALU64_IMM(BPF_MOV, R9, R9), \ + BPF_##alu(BPF_ ##op, dst, src), \ + BPF_ALU32_IMM(BPF_MOV, dst, dst), \ + BPF_JMP_IMM(BPF_JNE, R0, R0, 10), \ + BPF_JMP_IMM(BPF_JNE, R1, R1, 9), \ + BPF_JMP_IMM(BPF_JNE, R2, R2, 8), \ + BPF_JMP_IMM(BPF_JNE, R3, R3, 7), \ + BPF_JMP_IMM(BPF_JNE, R4, R4, 6), \ + BPF_JMP_IMM(BPF_JNE, R5, R5, 5), \ + BPF_JMP_IMM(BPF_JNE, R6, R6, 4), \ + BPF_JMP_IMM(BPF_JNE, R7, R7, 3), \ + BPF_JMP_IMM(BPF_JNE, R8, R8, 2), \ + BPF_JMP_IMM(BPF_JNE, R9, R9, 1), \ + BPF_ALU64_IMM(BPF_MOV, R0, 1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 1 } } \ + } + /* ALU64 operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, AND, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, OR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, XOR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, LSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, RSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ARSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, ADD, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, SUB, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MUL, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, DIV, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU64_IMM, MOD, R9, 123456789), + /* ALU32 immediate operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, AND, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, OR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, XOR, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, LSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, RSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R8, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ARSH, R9, 12), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, ADD, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, SUB, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MUL, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, DIV, R9, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R8, 123456789), + BPF_TEST_CLOBBER_ALU(ALU32_IMM, MOD, R9, 123456789), + /* ALU64 register operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, AND, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, OR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, XOR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, LSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, RSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ARSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, ADD, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, SUB, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MUL, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, DIV, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU64_REG, MOD, R9, R1), + /* ALU32 register operations, register clobbering */ + BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, AND, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, OR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, XOR, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, LSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, RSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ARSH, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, ADD, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, SUB, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MUL, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, DIV, R9, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R8, R1), + BPF_TEST_CLOBBER_ALU(ALU32_REG, MOD, R9, R1), +#undef BPF_TEST_CLOBBER_ALU +#define BPF_TEST_CLOBBER_ATOMIC(width, op) \ + { \ + "Atomic_" #width " " #op ": no clobbering", \ + .u.insns_int = { \ + BPF_ALU64_IMM(BPF_MOV, R0, 0), \ + BPF_ALU64_IMM(BPF_MOV, R1, 1), \ + BPF_ALU64_IMM(BPF_MOV, R2, 2), \ + BPF_ALU64_IMM(BPF_MOV, R3, 3), \ + BPF_ALU64_IMM(BPF_MOV, R4, 4), \ + BPF_ALU64_IMM(BPF_MOV, R5, 5), \ + BPF_ALU64_IMM(BPF_MOV, R6, 6), \ + BPF_ALU64_IMM(BPF_MOV, R7, 7), \ + BPF_ALU64_IMM(BPF_MOV, R8, 8), \ + BPF_ALU64_IMM(BPF_MOV, R9, 9), \ + BPF_ST_MEM(width, R10, -8, \ + (op) == BPF_CMPXCHG ? 0 : \ + (op) & BPF_FETCH ? 1 : 0), \ + BPF_ATOMIC_OP(width, op, R10, R1, -8), \ + BPF_JMP_IMM(BPF_JNE, R0, 0, 10), \ + BPF_JMP_IMM(BPF_JNE, R1, 1, 9), \ + BPF_JMP_IMM(BPF_JNE, R2, 2, 8), \ + BPF_JMP_IMM(BPF_JNE, R3, 3, 7), \ + BPF_JMP_IMM(BPF_JNE, R4, 4, 6), \ + BPF_JMP_IMM(BPF_JNE, R5, 5, 5), \ + BPF_JMP_IMM(BPF_JNE, R6, 6, 4), \ + BPF_JMP_IMM(BPF_JNE, R7, 7, 3), \ + BPF_JMP_IMM(BPF_JNE, R8, 8, 2), \ + BPF_JMP_IMM(BPF_JNE, R9, 9, 1), \ + BPF_ALU64_IMM(BPF_MOV, R0, 1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 1 } }, \ + .stack_depth = 8, \ + } + /* 64-bit atomic operations, register clobbering */ + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_ADD | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_AND | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_OR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XOR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_XCHG), + BPF_TEST_CLOBBER_ATOMIC(BPF_DW, BPF_CMPXCHG), + /* 32-bit atomic operations, register clobbering */ + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_ADD | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_AND | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_OR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XOR | BPF_FETCH), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_XCHG), + BPF_TEST_CLOBBER_ATOMIC(BPF_W, BPF_CMPXCHG), +#undef BPF_TEST_CLOBBER_ATOMIC + /* Checking that ALU32 src is not zero extended in place */ +#define BPF_ALU32_SRC_ZEXT(op) \ + { \ + "ALU32_" #op "_X: src preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R1, 0x0123456789acbdefULL),\ + BPF_LD_IMM64(R2, 0xfedcba9876543210ULL),\ + BPF_ALU64_REG(BPF_MOV, R0, R1), \ + BPF_ALU32_REG(BPF_##op, R2, R1), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_ALU32_SRC_ZEXT(MOV), + BPF_ALU32_SRC_ZEXT(AND), + BPF_ALU32_SRC_ZEXT(OR), + BPF_ALU32_SRC_ZEXT(XOR), + BPF_ALU32_SRC_ZEXT(ADD), + BPF_ALU32_SRC_ZEXT(SUB), + BPF_ALU32_SRC_ZEXT(MUL), + BPF_ALU32_SRC_ZEXT(DIV), + BPF_ALU32_SRC_ZEXT(MOD), +#undef BPF_ALU32_SRC_ZEXT + /* Checking that ATOMIC32 src is not zero extended in place */ +#define BPF_ATOMIC32_SRC_ZEXT(op) \ + { \ + "ATOMIC_W_" #op ": src preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ST_MEM(BPF_W, R10, -4, 0), \ + BPF_ATOMIC_OP(BPF_W, BPF_##op, R10, R1, -4), \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + .stack_depth = 8, \ + } + BPF_ATOMIC32_SRC_ZEXT(ADD), + BPF_ATOMIC32_SRC_ZEXT(AND), + BPF_ATOMIC32_SRC_ZEXT(OR), + BPF_ATOMIC32_SRC_ZEXT(XOR), +#undef BPF_ATOMIC32_SRC_ZEXT + /* Checking that CMPXCHG32 src is not zero extended in place */ + { + "ATOMIC_W_CMPXCHG: src preserved in zext", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0123456789acbdefULL), + BPF_ALU64_REG(BPF_MOV, R2, R1), + BPF_ALU64_REG(BPF_MOV, R0, 0), + BPF_ST_MEM(BPF_W, R10, -4, 0), + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, R10, R1, -4), + BPF_ALU64_REG(BPF_SUB, R1, R2), + BPF_ALU64_REG(BPF_MOV, R2, R1), + BPF_ALU64_IMM(BPF_RSH, R2, 32), + BPF_ALU64_REG(BPF_OR, R1, R2), + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + /* Checking that JMP32 immediate src is not zero extended in place */ +#define BPF_JMP32_IMM_ZEXT(op) \ + { \ + "JMP32_" #op "_K: operand preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_JMP32_IMM(BPF_##op, R0, 1234, 1), \ + BPF_JMP_A(0), /* Nop */ \ + BPF_ALU64_REG(BPF_SUB, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_JMP32_IMM_ZEXT(JEQ), + BPF_JMP32_IMM_ZEXT(JNE), + BPF_JMP32_IMM_ZEXT(JSET), + BPF_JMP32_IMM_ZEXT(JGT), + BPF_JMP32_IMM_ZEXT(JGE), + BPF_JMP32_IMM_ZEXT(JLT), + BPF_JMP32_IMM_ZEXT(JLE), + BPF_JMP32_IMM_ZEXT(JSGT), + BPF_JMP32_IMM_ZEXT(JSGE), + BPF_JMP32_IMM_ZEXT(JSGT), + BPF_JMP32_IMM_ZEXT(JSLT), + BPF_JMP32_IMM_ZEXT(JSLE), +#undef BPF_JMP2_IMM_ZEXT + /* Checking that JMP32 dst & src are not zero extended in place */ +#define BPF_JMP32_REG_ZEXT(op) \ + { \ + "JMP32_" #op "_X: operands preserved in zext", \ + .u.insns_int = { \ + BPF_LD_IMM64(R0, 0x0123456789acbdefULL),\ + BPF_LD_IMM64(R1, 0xfedcba9876543210ULL),\ + BPF_ALU64_REG(BPF_MOV, R2, R0), \ + BPF_ALU64_REG(BPF_MOV, R3, R1), \ + BPF_JMP32_IMM(BPF_##op, R0, R1, 1), \ + BPF_JMP_A(0), /* Nop */ \ + BPF_ALU64_REG(BPF_SUB, R0, R2), \ + BPF_ALU64_REG(BPF_SUB, R1, R3), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_ALU64_REG(BPF_MOV, R1, R0), \ + BPF_ALU64_IMM(BPF_RSH, R1, 32), \ + BPF_ALU64_REG(BPF_OR, R0, R1), \ + BPF_EXIT_INSN(), \ + }, \ + INTERNAL, \ + { }, \ + { { 0, 0 } }, \ + } + BPF_JMP32_REG_ZEXT(JEQ), + BPF_JMP32_REG_ZEXT(JNE), + BPF_JMP32_REG_ZEXT(JSET), + BPF_JMP32_REG_ZEXT(JGT), + BPF_JMP32_REG_ZEXT(JGE), + BPF_JMP32_REG_ZEXT(JLT), + BPF_JMP32_REG_ZEXT(JLE), + BPF_JMP32_REG_ZEXT(JSGT), + BPF_JMP32_REG_ZEXT(JSGE), + BPF_JMP32_REG_ZEXT(JSGT), + BPF_JMP32_REG_ZEXT(JSLT), + BPF_JMP32_REG_ZEXT(JSLE), +#undef BPF_JMP2_REG_ZEXT + /* ALU64 K register combinations */ + { + "ALU64_MOV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_imm_regs, + }, + { + "ALU64_AND_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_imm_regs, + }, + { + "ALU64_OR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_imm_regs, + }, + { + "ALU64_XOR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_imm_regs, + }, + { + "ALU64_LSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_imm_regs, + }, + { + "ALU64_RSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_imm_regs, + }, + { + "ALU64_ARSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_imm_regs, + }, + { + "ALU64_ADD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_imm_regs, + }, + { + "ALU64_SUB_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_imm_regs, + }, + { + "ALU64_MUL_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_imm_regs, + }, + { + "ALU64_DIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_imm_regs, + }, + { + "ALU64_MOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_imm_regs, + }, + /* ALU32 K registers */ + { + "ALU32_MOV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_imm_regs, + }, + { + "ALU32_AND_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_imm_regs, + }, + { + "ALU32_OR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_imm_regs, + }, + { + "ALU32_XOR_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_imm_regs, + }, + { + "ALU32_LSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_imm_regs, + }, + { + "ALU32_RSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_imm_regs, + }, + { + "ALU32_ARSH_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_imm_regs, + }, + { + "ALU32_ADD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_imm_regs, + }, + { + "ALU32_SUB_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_imm_regs, + }, + { + "ALU32_MUL_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_imm_regs, + }, + { + "ALU32_DIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_imm_regs, + }, + { + "ALU32_MOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_imm_regs, + }, + /* ALU64 X register combinations */ + { + "ALU64_MOV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_reg_pairs, + }, + { + "ALU64_AND_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_reg_pairs, + }, + { + "ALU64_OR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_reg_pairs, + }, + { + "ALU64_XOR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_reg_pairs, + }, + { + "ALU64_LSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_reg_pairs, + }, + { + "ALU64_RSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_reg_pairs, + }, + { + "ALU64_ARSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_reg_pairs, + }, + { + "ALU64_ADD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_reg_pairs, + }, + { + "ALU64_SUB_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_reg_pairs, + }, + { + "ALU64_MUL_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_reg_pairs, + }, + { + "ALU64_DIV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_reg_pairs, + }, + { + "ALU64_MOD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_reg_pairs, + }, + /* ALU32 X register combinations */ + { + "ALU32_MOV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_reg_pairs, + }, + { + "ALU32_AND_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_reg_pairs, + }, + { + "ALU32_OR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_reg_pairs, + }, + { + "ALU32_XOR_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_reg_pairs, + }, + { + "ALU32_LSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_reg_pairs, + }, + { + "ALU32_RSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_reg_pairs, + }, + { + "ALU32_ARSH_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_reg_pairs, + }, + { + "ALU32_ADD_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_reg_pairs, + }, + { + "ALU32_SUB_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_reg_pairs, + }, + { + "ALU32_MUL_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_reg_pairs, + }, + { + "ALU32_DIV_X: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_reg_pairs, + }, + { + "ALU32_MOD_X register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_reg_pairs, + }, + /* Exhaustive test of ALU64 shift operations */ + { + "ALU64_LSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_imm, + }, + { + "ALU64_RSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_imm, + }, + { + "ALU64_ARSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_imm, + }, + { + "ALU64_LSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_reg, + }, + { + "ALU64_RSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_reg, + }, + { + "ALU64_ARSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_reg, + }, + /* Exhaustive test of ALU32 shift operations */ + { + "ALU32_LSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_imm, + }, + { + "ALU32_RSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_imm, + }, + { + "ALU32_ARSH_K: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_imm, + }, + { + "ALU32_LSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_reg, + }, + { + "ALU32_RSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_reg, + }, + { + "ALU32_ARSH_X: all shift values", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_reg, + }, + /* + * Exhaustive test of ALU64 shift operations when + * source and destination register are the same. + */ + { + "ALU64_LSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_lsh_same_reg, + }, + { + "ALU64_RSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_rsh_same_reg, + }, + { + "ALU64_ARSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_arsh_same_reg, + }, + /* + * Exhaustive test of ALU32 shift operations when + * source and destination register are the same. + */ + { + "ALU32_LSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_lsh_same_reg, + }, + { + "ALU32_RSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_rsh_same_reg, + }, + { + "ALU32_ARSH_X: all shift values with the same register", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_arsh_same_reg, + }, + /* ALU64 immediate magnitudes */ + { + "ALU64_MOV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_AND_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_OR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_XOR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_ADD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SUB_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MUL_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_DIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU32 immediate magnitudes */ + { + "ALU32_MOV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_AND_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_OR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_XOR_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_ADD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SUB_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MUL_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_DIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU64 register magnitudes */ + { + "ALU64_MOV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mov_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_AND_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_and_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_OR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_or_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_XOR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_xor_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_ADD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_add_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SUB_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sub_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MUL_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mul_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_DIV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_div_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_MOD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_mod_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* ALU32 register magnitudes */ + { + "ALU32_MOV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mov_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_AND_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_and_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_OR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_or_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_XOR_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_xor_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_ADD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_add_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SUB_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sub_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MUL_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mul_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_DIV_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_div_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_MOD_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_mod_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* LD_IMM64 immediate magnitudes and byte patterns */ + { + "LD_IMM64: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_magn, + }, + { + "LD_IMM64: checker byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_checker, + }, + { + "LD_IMM64: random positive and zero byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_pos_zero, + }, + { + "LD_IMM64: random negative and zero byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_neg_zero, + }, + { + "LD_IMM64: random positive and negative byte patterns", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_ld_imm64_pos_neg, + }, + /* 64-bit ATOMIC register combinations */ + { + "ATOMIC_DW_ADD: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_AND: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_OR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XOR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_ADD_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_AND_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_OR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XOR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_XCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xchg_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_DW_CMPXCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_cmpxchg_reg_pairs, + .stack_depth = 8, + }, + /* 32-bit ATOMIC register combinations */ + { + "ATOMIC_W_ADD: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_AND: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_OR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XOR: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_ADD_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_AND_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_OR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XOR_FETCH: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_fetch_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_XCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xchg_reg_pairs, + .stack_depth = 8, + }, + { + "ATOMIC_W_CMPXCHG: register combinations", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_cmpxchg_reg_pairs, + .stack_depth = 8, + }, + /* 64-bit ATOMIC magnitudes */ + { + "ATOMIC_DW_ADD: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_AND: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_OR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XOR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_ADD_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_add_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_AND_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_and_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_OR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_or_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XOR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xor_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_XCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic64_xchg, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_DW_CMPXCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_cmpxchg64, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* 64-bit atomic magnitudes */ + { + "ATOMIC_W_ADD: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_AND: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_OR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XOR: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_ADD_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_add_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_AND_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_and_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_OR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_or_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XOR_FETCH: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xor_fetch, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_XCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_atomic32_xchg, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ATOMIC_W_CMPXCHG: all operand magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_cmpxchg32, + .stack_depth = 8, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP immediate magnitudes */ + { + "JMP_JSET_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jset_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JEQ_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jeq_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JNE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jne_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jlt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jslt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP register magnitudes */ + { + "JMP_JSET_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jset_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JEQ_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jeq_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JNE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jne_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jlt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jslt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP_JSLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp_jsle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP32 immediate magnitudes */ + { + "JMP32_JSET_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jset_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JEQ_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jeq_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JNE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jne_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jlt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsgt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsge_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLT_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jslt_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLE_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsle_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* JMP32 register magnitudes */ + { + "JMP32_JSET_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jset_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JEQ_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jeq_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JNE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jne_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jlt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsgt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSGE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsge_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLT_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jslt_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "JMP32_JSLE_X: all register value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_jmp32_jsle_reg, + .nr_testruns = NR_PATTERN_RUNS, + }, + /* Conditional jumps with constant decision */ + { + "JMP_JSET_K: imm = 0 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JSET, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLT_K: imm = 0 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JLT, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JGE_K: imm = 0 -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JGE, R1, 0, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JGT_K: imm = 0xffffffff -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JGT, R1, U32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLE_K: imm = 0xffffffff -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_IMM(BPF_JLE, R1, U32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP32_JSGT_K: imm = 0x7fffffff -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSGT, R1, S32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP32_JSGE_K: imm = -0x80000000 -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSGE, R1, S32_MIN, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP32_JSLT_K: imm = -0x80000000 -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSLT, R1, S32_MIN, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP32_JSLE_K: imm = 0x7fffffff -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP32_IMM(BPF_JSLE, R1, S32_MAX, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JEQ_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JEQ, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JGE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JGE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JLE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JLE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JSGE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSGE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JSLE_X: dst = src -> always taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSLE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + }, + { + "JMP_JNE_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JNE, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JGT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JGT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JLT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JLT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JSGT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSGT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + { + "JMP_JSLT_X: dst = src -> never taken", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 1), + BPF_JMP_REG(BPF_JSLT, R1, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 0 } }, + }, + /* Short relative jumps */ + { + "Short relative jump: offset=0", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 0), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=1", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=2", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 2), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=3", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 3), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + { + "Short relative jump: offset=4", + .u.insns_int = { + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_JMP_IMM(BPF_JEQ, R0, 0, 4), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_ALU32_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + BPF_ALU32_IMM(BPF_MOV, R0, -1), + }, + INTERNAL | FLAG_NO_DATA | FLAG_VERIFIER_ZEXT, + { }, + { { 0, 0 } }, + }, + /* Conditional branch conversions */ + { + "Long conditional jump: taken at runtime", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_max_jmp_taken, + }, + { + "Long conditional jump: not taken at runtime", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 2 } }, + .fill_helper = bpf_fill_max_jmp_not_taken, + }, + { + "Long conditional jump: always taken, known at JIT time", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_max_jmp_always_taken, + }, + { + "Long conditional jump: never taken, known at JIT time", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 2 } }, + .fill_helper = bpf_fill_max_jmp_never_taken, + }, + /* Staggered jump sequences, immediate */ + { + "Staggered jumps: JMP_JA", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_ja, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JEQ_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JNE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSET_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, register */ + { + "Staggered jumps: JMP_JEQ_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JNE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSET_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP_JSLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, JMP32 immediate */ + { + "Staggered jumps: JMP32_JEQ_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JNE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSET_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLT_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLE_K", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle32_imm, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + /* Staggered jump sequences, JMP32 register */ + { + "Staggered jumps: JMP32_JEQ_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jeq32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JNE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jne32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSET_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jset32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jgt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jge32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jlt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jle32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsgt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSGE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsge32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLT_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jslt32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, + { + "Staggered jumps: JMP32_JSLE_X", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, MAX_STAGGERED_JMP_SIZE + 1 } }, + .fill_helper = bpf_fill_staggered_jsle32_reg, + .nr_testruns = NR_STAGGERED_JMP_RUNS, + }, }; static struct net_device dev; @@ -6599,6 +14213,8 @@ static struct bpf_prog *generate_filter(int which, int *err) fp->type = BPF_PROG_TYPE_SOCKET_FILTER; memcpy(fp->insnsi, fptr, fp->len * sizeof(struct bpf_insn)); fp->aux->stack_depth = tests[which].stack_depth; + fp->aux->verifier_zext = !!(tests[which].aux & + FLAG_VERIFIER_ZEXT); /* We cannot error here as we don't need type compatibility * checks. @@ -6639,7 +14255,7 @@ static int __run_one(const struct bpf_prog *fp, const void *data, start = ktime_get_ns(); for (i = 0; i < runs; i++) - ret = BPF_PROG_RUN(fp, data); + ret = bpf_prog_run(fp, data); finish = ktime_get_ns(); migrate_enable(); @@ -6654,12 +14270,22 @@ static int run_one(const struct bpf_prog *fp, struct bpf_test *test) { int err_cnt = 0, i, runs = MAX_TESTRUNS; + if (test->nr_testruns) + runs = min(test->nr_testruns, MAX_TESTRUNS); + for (i = 0; i < MAX_SUBTESTS; i++) { void *data; u64 duration; u32 ret; - if (test->test[i].data_size == 0 && + /* + * NOTE: Several sub-tests may be present, in which case + * a zero {data_size, result} tuple indicates the end of + * the sub-test array. The first test is always run, + * even if both data_size and result happen to be zero. + */ + if (i > 0 && + test->test[i].data_size == 0 && test->test[i].result == 0) break; @@ -6690,86 +14316,9 @@ module_param_string(test_name, test_name, sizeof(test_name), 0); static int test_id = -1; module_param(test_id, int, 0); -static int test_range[2] = { 0, ARRAY_SIZE(tests) - 1 }; +static int test_range[2] = { 0, INT_MAX }; module_param_array(test_range, int, NULL, 0); -static __init int find_test_index(const char *test_name) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (!strcmp(tests[i].descr, test_name)) - return i; - } - return -1; -} - -static __init int prepare_bpf_tests(void) -{ - int i; - - if (test_id >= 0) { - /* - * if a test_id was specified, use test_range to - * cover only that test. - */ - if (test_id >= ARRAY_SIZE(tests)) { - pr_err("test_bpf: invalid test_id specified.\n"); - return -EINVAL; - } - - test_range[0] = test_id; - test_range[1] = test_id; - } else if (*test_name) { - /* - * if a test_name was specified, find it and setup - * test_range to cover only that test. - */ - int idx = find_test_index(test_name); - - if (idx < 0) { - pr_err("test_bpf: no test named '%s' found.\n", - test_name); - return -EINVAL; - } - test_range[0] = idx; - test_range[1] = idx; - } else { - /* - * check that the supplied test_range is valid. - */ - if (test_range[0] >= ARRAY_SIZE(tests) || - test_range[1] >= ARRAY_SIZE(tests) || - test_range[0] < 0 || test_range[1] < 0) { - pr_err("test_bpf: test_range is out of bound.\n"); - return -EINVAL; - } - - if (test_range[1] < test_range[0]) { - pr_err("test_bpf: test_range is ending before it starts.\n"); - return -EINVAL; - } - } - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (tests[i].fill_helper && - tests[i].fill_helper(&tests[i]) < 0) - return -ENOMEM; - } - - return 0; -} - -static __init void destroy_bpf_tests(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (tests[i].fill_helper) - kfree(tests[i].u.ptr.insns); - } -} - static bool exclude_test(int test_id) { return test_id < test_range[0] || test_id > test_range[1]; @@ -6816,6 +14365,7 @@ static __init struct sk_buff *build_test_skb(void) skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY; skb_shinfo(skb[0])->gso_segs = 0; skb_shinfo(skb[0])->frag_list = skb[1]; + skb_shinfo(skb[0])->hwtstamps.hwtstamp = 1000; /* adjust skb[0]'s len */ skb[0]->len += skb[1]->len; @@ -6940,6 +14490,10 @@ static __init int test_skb_segment(void) for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { const struct skb_segment_test *test = &skb_segment_tests[i]; + cond_resched(); + if (exclude_test(i)) + continue; + pr_info("#%d %s ", i, test->descr); if (test_skb_segment_single(test)) { @@ -6971,7 +14525,19 @@ static __init int test_bpf(void) pr_info("#%d %s ", i, tests[i].descr); + if (tests[i].fill_helper && + tests[i].fill_helper(&tests[i]) < 0) { + pr_cont("FAIL to prog_fill\n"); + continue; + } + fp = generate_filter(i, &err); + + if (tests[i].fill_helper) { + kfree(tests[i].u.ptr.insns); + tests[i].u.ptr.insns = NULL; + } + if (fp == NULL) { if (err == 0) { pass_cnt++; @@ -7005,20 +14571,482 @@ static __init int test_bpf(void) return err_cnt ? -EINVAL : 0; } +struct tail_call_test { + const char *descr; + struct bpf_insn insns[MAX_INSNS]; + int flags; + int result; + int stack_depth; +}; + +/* Flags that can be passed to tail call test cases */ +#define FLAG_NEED_STATE BIT(0) +#define FLAG_RESULT_IN_STATE BIT(1) + +/* + * Magic marker used in test snippets for tail calls below. + * BPF_LD/MOV to R2 and R2 with this immediate value is replaced + * with the proper values by the test runner. + */ +#define TAIL_CALL_MARKER 0x7a11ca11 + +/* Special offset to indicate a NULL call target */ +#define TAIL_CALL_NULL 0x7fff + +/* Special offset to indicate an out-of-range index */ +#define TAIL_CALL_INVALID 0x7ffe + +#define TAIL_CALL(offset) \ + BPF_LD_IMM64(R2, TAIL_CALL_MARKER), \ + BPF_RAW_INSN(BPF_ALU | BPF_MOV | BPF_K, R3, 0, \ + offset, TAIL_CALL_MARKER), \ + BPF_JMP_IMM(BPF_TAIL_CALL, 0, 0, 0) + +/* + * A test function to be called from a BPF program, clobbering a lot of + * CPU registers in the process. A JITed BPF program calling this function + * must save and restore any caller-saved registers it uses for internal + * state, for example the current tail call count. + */ +BPF_CALL_1(bpf_test_func, u64, arg) +{ + char buf[64]; + long a = 0; + long b = 1; + long c = 2; + long d = 3; + long e = 4; + long f = 5; + long g = 6; + long h = 7; + + return snprintf(buf, sizeof(buf), + "%ld %lu %lx %ld %lu %lx %ld %lu %x", + a, b, c, d, e, f, g, h, (int)arg); +} +#define BPF_FUNC_test_func __BPF_FUNC_MAX_ID + +/* + * Tail call tests. Each test case may call any other test in the table, + * including itself, specified as a relative index offset from the calling + * test. The index TAIL_CALL_NULL can be used to specify a NULL target + * function to test the JIT error path. Similarly, the index TAIL_CALL_INVALID + * results in a target index that is out of range. + */ +static struct tail_call_test tail_call_tests[] = { + { + "Tail call leaf", + .insns = { + BPF_ALU64_REG(BPF_MOV, R0, R1), + BPF_ALU64_IMM(BPF_ADD, R0, 1), + BPF_EXIT_INSN(), + }, + .result = 1, + }, + { + "Tail call 2", + .insns = { + BPF_ALU64_IMM(BPF_ADD, R1, 2), + TAIL_CALL(-1), + BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_EXIT_INSN(), + }, + .result = 3, + }, + { + "Tail call 3", + .insns = { + BPF_ALU64_IMM(BPF_ADD, R1, 3), + TAIL_CALL(-1), + BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_EXIT_INSN(), + }, + .result = 6, + }, + { + "Tail call 4", + .insns = { + BPF_ALU64_IMM(BPF_ADD, R1, 4), + TAIL_CALL(-1), + BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_EXIT_INSN(), + }, + .result = 10, + }, + { + "Tail call error path, max count reached", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + TAIL_CALL(0), + BPF_EXIT_INSN(), + }, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, + }, + { + "Tail call count preserved across function calls", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_CALL_REL(BPF_FUNC_get_numa_node_id), + BPF_CALL_REL(BPF_FUNC_ktime_get_ns), + BPF_CALL_REL(BPF_FUNC_ktime_get_boot_ns), + BPF_CALL_REL(BPF_FUNC_ktime_get_coarse_ns), + BPF_CALL_REL(BPF_FUNC_jiffies64), + BPF_CALL_REL(BPF_FUNC_test_func), + BPF_LDX_MEM(BPF_DW, R1, R10, -8), + BPF_ALU32_REG(BPF_MOV, R0, R1), + TAIL_CALL(0), + BPF_EXIT_INSN(), + }, + .stack_depth = 8, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = (MAX_TAIL_CALL_CNT + 1 + 1) * MAX_TESTRUNS, + }, + { + "Tail call error path, NULL target", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + TAIL_CALL(TAIL_CALL_NULL), + BPF_EXIT_INSN(), + }, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = MAX_TESTRUNS, + }, + { + "Tail call error path, index out of range", + .insns = { + BPF_LDX_MEM(BPF_W, R2, R1, 0), + BPF_ALU64_IMM(BPF_ADD, R2, 1), + BPF_STX_MEM(BPF_W, R1, R2, 0), + TAIL_CALL(TAIL_CALL_INVALID), + BPF_EXIT_INSN(), + }, + .flags = FLAG_NEED_STATE | FLAG_RESULT_IN_STATE, + .result = MAX_TESTRUNS, + }, +}; + +static void __init destroy_tail_call_tests(struct bpf_array *progs) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) + if (progs->ptrs[i]) + bpf_prog_free(progs->ptrs[i]); + kfree(progs); +} + +static __init int prepare_tail_call_tests(struct bpf_array **pprogs) +{ + int ntests = ARRAY_SIZE(tail_call_tests); + struct bpf_array *progs; + int which, err; + + /* Allocate the table of programs to be used for tall calls */ + progs = kzalloc(sizeof(*progs) + (ntests + 1) * sizeof(progs->ptrs[0]), + GFP_KERNEL); + if (!progs) + goto out_nomem; + + /* Create all eBPF programs and populate the table */ + for (which = 0; which < ntests; which++) { + struct tail_call_test *test = &tail_call_tests[which]; + struct bpf_prog *fp; + int len, i; + + /* Compute the number of program instructions */ + for (len = 0; len < MAX_INSNS; len++) { + struct bpf_insn *insn = &test->insns[len]; + + if (len < MAX_INSNS - 1 && + insn->code == (BPF_LD | BPF_DW | BPF_IMM)) + len++; + if (insn->code == 0) + break; + } + + /* Allocate and initialize the program */ + fp = bpf_prog_alloc(bpf_prog_size(len), 0); + if (!fp) + goto out_nomem; + + fp->len = len; + fp->type = BPF_PROG_TYPE_SOCKET_FILTER; + fp->aux->stack_depth = test->stack_depth; + memcpy(fp->insnsi, test->insns, len * sizeof(struct bpf_insn)); + + /* Relocate runtime tail call offsets and addresses */ + for (i = 0; i < len; i++) { + struct bpf_insn *insn = &fp->insnsi[i]; + long addr = 0; + + switch (insn->code) { + case BPF_LD | BPF_DW | BPF_IMM: + if (insn->imm != TAIL_CALL_MARKER) + break; + insn[0].imm = (u32)(long)progs; + insn[1].imm = ((u64)(long)progs) >> 32; + break; + + case BPF_ALU | BPF_MOV | BPF_K: + if (insn->imm != TAIL_CALL_MARKER) + break; + if (insn->off == TAIL_CALL_NULL) + insn->imm = ntests; + else if (insn->off == TAIL_CALL_INVALID) + insn->imm = ntests + 1; + else + insn->imm = which + insn->off; + insn->off = 0; + break; + + case BPF_JMP | BPF_CALL: + if (insn->src_reg != BPF_PSEUDO_CALL) + break; + switch (insn->imm) { + case BPF_FUNC_get_numa_node_id: + addr = (long)&numa_node_id; + break; + case BPF_FUNC_ktime_get_ns: + addr = (long)&ktime_get_ns; + break; + case BPF_FUNC_ktime_get_boot_ns: + addr = (long)&ktime_get_boot_fast_ns; + break; + case BPF_FUNC_ktime_get_coarse_ns: + addr = (long)&ktime_get_coarse_ns; + break; + case BPF_FUNC_jiffies64: + addr = (long)&get_jiffies_64; + break; + case BPF_FUNC_test_func: + addr = (long)&bpf_test_func; + break; + default: + err = -EFAULT; + goto out_err; + } + *insn = BPF_EMIT_CALL(addr); + if ((long)__bpf_call_base + insn->imm != addr) + *insn = BPF_JMP_A(0); /* Skip: NOP */ + break; + } + } + + fp = bpf_prog_select_runtime(fp, &err); + if (err) + goto out_err; + + progs->ptrs[which] = fp; + } + + /* The last entry contains a NULL program pointer */ + progs->map.max_entries = ntests + 1; + *pprogs = progs; + return 0; + +out_nomem: + err = -ENOMEM; + +out_err: + if (progs) + destroy_tail_call_tests(progs); + return err; +} + +static __init int test_tail_calls(struct bpf_array *progs) +{ + int i, err_cnt = 0, pass_cnt = 0; + int jit_cnt = 0, run_cnt = 0; + + for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { + struct tail_call_test *test = &tail_call_tests[i]; + struct bpf_prog *fp = progs->ptrs[i]; + int *data = NULL; + int state = 0; + u64 duration; + int ret; + + cond_resched(); + if (exclude_test(i)) + continue; + + pr_info("#%d %s ", i, test->descr); + if (!fp) { + err_cnt++; + continue; + } + pr_cont("jited:%u ", fp->jited); + + run_cnt++; + if (fp->jited) + jit_cnt++; + + if (test->flags & FLAG_NEED_STATE) + data = &state; + ret = __run_one(fp, data, MAX_TESTRUNS, &duration); + if (test->flags & FLAG_RESULT_IN_STATE) + ret = state; + if (ret == test->result) { + pr_cont("%lld PASS", duration); + pass_cnt++; + } else { + pr_cont("ret %d != %d FAIL", ret, test->result); + err_cnt++; + } + } + + pr_info("%s: Summary: %d PASSED, %d FAILED, [%d/%d JIT'ed]\n", + __func__, pass_cnt, err_cnt, jit_cnt, run_cnt); + + return err_cnt ? -EINVAL : 0; +} + +static char test_suite[32]; +module_param_string(test_suite, test_suite, sizeof(test_suite), 0); + +static __init int find_test_index(const char *test_name) +{ + int i; + + if (!strcmp(test_suite, "test_bpf")) { + for (i = 0; i < ARRAY_SIZE(tests); i++) { + if (!strcmp(tests[i].descr, test_name)) + return i; + } + } + + if (!strcmp(test_suite, "test_tail_calls")) { + for (i = 0; i < ARRAY_SIZE(tail_call_tests); i++) { + if (!strcmp(tail_call_tests[i].descr, test_name)) + return i; + } + } + + if (!strcmp(test_suite, "test_skb_segment")) { + for (i = 0; i < ARRAY_SIZE(skb_segment_tests); i++) { + if (!strcmp(skb_segment_tests[i].descr, test_name)) + return i; + } + } + + return -1; +} + +static __init int prepare_test_range(void) +{ + int valid_range; + + if (!strcmp(test_suite, "test_bpf")) + valid_range = ARRAY_SIZE(tests); + else if (!strcmp(test_suite, "test_tail_calls")) + valid_range = ARRAY_SIZE(tail_call_tests); + else if (!strcmp(test_suite, "test_skb_segment")) + valid_range = ARRAY_SIZE(skb_segment_tests); + else + return 0; + + if (test_id >= 0) { + /* + * if a test_id was specified, use test_range to + * cover only that test. + */ + if (test_id >= valid_range) { + pr_err("test_bpf: invalid test_id specified for '%s' suite.\n", + test_suite); + return -EINVAL; + } + + test_range[0] = test_id; + test_range[1] = test_id; + } else if (*test_name) { + /* + * if a test_name was specified, find it and setup + * test_range to cover only that test. + */ + int idx = find_test_index(test_name); + + if (idx < 0) { + pr_err("test_bpf: no test named '%s' found for '%s' suite.\n", + test_name, test_suite); + return -EINVAL; + } + test_range[0] = idx; + test_range[1] = idx; + } else if (test_range[0] != 0 || test_range[1] != INT_MAX) { + /* + * check that the supplied test_range is valid. + */ + if (test_range[0] < 0 || test_range[1] >= valid_range) { + pr_err("test_bpf: test_range is out of bound for '%s' suite.\n", + test_suite); + return -EINVAL; + } + + if (test_range[1] < test_range[0]) { + pr_err("test_bpf: test_range is ending before it starts.\n"); + return -EINVAL; + } + } + + return 0; +} + static int __init test_bpf_init(void) { + struct bpf_array *progs = NULL; int ret; - ret = prepare_bpf_tests(); + if (strlen(test_suite) && + strcmp(test_suite, "test_bpf") && + strcmp(test_suite, "test_tail_calls") && + strcmp(test_suite, "test_skb_segment")) { + pr_err("test_bpf: invalid test_suite '%s' specified.\n", test_suite); + return -EINVAL; + } + + /* + * if test_suite is not specified, but test_id, test_name or test_range + * is specified, set 'test_bpf' as the default test suite. + */ + if (!strlen(test_suite) && + (test_id != -1 || strlen(test_name) || + (test_range[0] != 0 || test_range[1] != INT_MAX))) { + pr_info("test_bpf: set 'test_bpf' as the default test_suite.\n"); + strscpy(test_suite, "test_bpf", sizeof(test_suite)); + } + + ret = prepare_test_range(); if (ret < 0) return ret; - ret = test_bpf(); - destroy_bpf_tests(); - if (ret) - return ret; + if (!strlen(test_suite) || !strcmp(test_suite, "test_bpf")) { + ret = test_bpf(); + if (ret) + return ret; + } - return test_skb_segment(); + if (!strlen(test_suite) || !strcmp(test_suite, "test_tail_calls")) { + ret = prepare_tail_call_tests(&progs); + if (ret) + return ret; + ret = test_tail_calls(progs); + destroy_tail_call_tests(progs); + if (ret) + return ret; + } + + if (!strlen(test_suite) || !strcmp(test_suite, "test_skb_segment")) + return test_skb_segment(); + + return 0; } static void __exit test_bpf_exit(void) diff --git a/lib/test_firmware.c b/lib/test_firmware.c index b6fe89add9fe..1bccd6cd5f48 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -260,8 +260,8 @@ static ssize_t config_show(struct device *dev, len += scnprintf(buf + len, PAGE_SIZE - len, "send_uevent:\t\t%s\n", test_fw_config->send_uevent ? - "FW_ACTION_HOTPLUG" : - "FW_ACTION_NOHOTPLUG"); + "FW_ACTION_UEVENT" : + "FW_ACTION_NOUEVENT"); len += scnprintf(buf + len, PAGE_SIZE - len, "into_buf:\t\t%s\n", test_fw_config->into_buf ? "true" : "false"); @@ -729,7 +729,7 @@ static ssize_t trigger_custom_fallback_store(struct device *dev, mutex_lock(&test_fw_mutex); release_firmware(test_firmware); test_firmware = NULL; - rc = request_firmware_nowait(THIS_MODULE, FW_ACTION_NOHOTPLUG, name, + rc = request_firmware_nowait(THIS_MODULE, FW_ACTION_NOUEVENT, name, dev, GFP_KERNEL, NULL, trigger_async_request_cb); if (rc) { @@ -938,8 +938,8 @@ ssize_t trigger_batched_requests_async_store(struct device *dev, pr_info("batched loading '%s' custom fallback mechanism %u times\n", test_fw_config->name, test_fw_config->num_requests); - send_uevent = test_fw_config->send_uevent ? FW_ACTION_HOTPLUG : - FW_ACTION_NOHOTPLUG; + send_uevent = test_fw_config->send_uevent ? FW_ACTION_UEVENT : + FW_ACTION_NOUEVENT; for (i = 0; i < test_fw_config->num_requests; i++) { req = &test_fw_config->reqs[i]; diff --git a/lib/test_fortify/read_overflow-memchr.c b/lib/test_fortify/read_overflow-memchr.c new file mode 100644 index 000000000000..2743084b32af --- /dev/null +++ b/lib/test_fortify/read_overflow-memchr.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memchr(small, 0x7A, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow-memchr_inv.c b/lib/test_fortify/read_overflow-memchr_inv.c new file mode 100644 index 000000000000..b26e1f1bc217 --- /dev/null +++ b/lib/test_fortify/read_overflow-memchr_inv.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memchr_inv(small, 0x7A, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow-memcmp.c b/lib/test_fortify/read_overflow-memcmp.c new file mode 100644 index 000000000000..d5d301ff64ef --- /dev/null +++ b/lib/test_fortify/read_overflow-memcmp.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcmp(small, large, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow-memscan.c b/lib/test_fortify/read_overflow-memscan.c new file mode 100644 index 000000000000..c1a97f2df0f0 --- /dev/null +++ b/lib/test_fortify/read_overflow-memscan.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memscan(small, 0x7A, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow2-memcmp.c b/lib/test_fortify/read_overflow2-memcmp.c new file mode 100644 index 000000000000..c6091e640f76 --- /dev/null +++ b/lib/test_fortify/read_overflow2-memcmp.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcmp(large, small, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow2-memcpy.c b/lib/test_fortify/read_overflow2-memcpy.c new file mode 100644 index 000000000000..07b62e56cf16 --- /dev/null +++ b/lib/test_fortify/read_overflow2-memcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcpy(large, instance.buf, sizeof(large)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow2-memmove.c b/lib/test_fortify/read_overflow2-memmove.c new file mode 100644 index 000000000000..34edfab040a3 --- /dev/null +++ b/lib/test_fortify/read_overflow2-memmove.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memmove(large, instance.buf, sizeof(large)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/test_fortify.h b/lib/test_fortify/test_fortify.h new file mode 100644 index 000000000000..d22664fff197 --- /dev/null +++ b/lib/test_fortify/test_fortify.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include <linux/kernel.h> +#include <linux/printk.h> +#include <linux/slab.h> +#include <linux/string.h> + +void do_fortify_tests(void); + +#define __BUF_SMALL 16 +#define __BUF_LARGE 32 +struct fortify_object { + int a; + char buf[__BUF_SMALL]; + int c; +}; + +#define LITERAL_SMALL "AAAAAAAAAAAAAAA" +#define LITERAL_LARGE "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" +const char small_src[__BUF_SMALL] = LITERAL_SMALL; +const char large_src[__BUF_LARGE] = LITERAL_LARGE; + +char small[__BUF_SMALL]; +char large[__BUF_LARGE]; +struct fortify_object instance; +size_t size; + +void do_fortify_tests(void) +{ + /* Normal initializations. */ + memset(&instance, 0x32, sizeof(instance)); + memset(small, 0xA5, sizeof(small)); + memset(large, 0x5A, sizeof(large)); + + TEST; +} diff --git a/lib/test_fortify/write_overflow-memcpy.c b/lib/test_fortify/write_overflow-memcpy.c new file mode 100644 index 000000000000..3b3984e428fb --- /dev/null +++ b/lib/test_fortify/write_overflow-memcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcpy(instance.buf, large_src, sizeof(large_src)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-memmove.c b/lib/test_fortify/write_overflow-memmove.c new file mode 100644 index 000000000000..640437c3b3e0 --- /dev/null +++ b/lib/test_fortify/write_overflow-memmove.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memmove(instance.buf, large_src, sizeof(large_src)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-memset.c b/lib/test_fortify/write_overflow-memset.c new file mode 100644 index 000000000000..36e34908cfb3 --- /dev/null +++ b/lib/test_fortify/write_overflow-memset.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memset(instance.buf, 0x5A, sizeof(large_src)) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strcpy-lit.c b/lib/test_fortify/write_overflow-strcpy-lit.c new file mode 100644 index 000000000000..51effb3e50f9 --- /dev/null +++ b/lib/test_fortify/write_overflow-strcpy-lit.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strcpy(small, LITERAL_LARGE) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strcpy.c b/lib/test_fortify/write_overflow-strcpy.c new file mode 100644 index 000000000000..84f1c56a64c8 --- /dev/null +++ b/lib/test_fortify/write_overflow-strcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strcpy(small, large_src) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strlcpy-src.c b/lib/test_fortify/write_overflow-strlcpy-src.c new file mode 100644 index 000000000000..91bf83ebd34a --- /dev/null +++ b/lib/test_fortify/write_overflow-strlcpy-src.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strlcpy(small, large_src, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strlcpy.c b/lib/test_fortify/write_overflow-strlcpy.c new file mode 100644 index 000000000000..1883db7c0cd6 --- /dev/null +++ b/lib/test_fortify/write_overflow-strlcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strlcpy(instance.buf, large_src, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strncpy-src.c b/lib/test_fortify/write_overflow-strncpy-src.c new file mode 100644 index 000000000000..8dcfb8c788dd --- /dev/null +++ b/lib/test_fortify/write_overflow-strncpy-src.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strncpy(small, large_src, sizeof(small) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strncpy.c b/lib/test_fortify/write_overflow-strncpy.c new file mode 100644 index 000000000000..b85f079c815d --- /dev/null +++ b/lib/test_fortify/write_overflow-strncpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strncpy(instance.buf, large_src, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strscpy.c b/lib/test_fortify/write_overflow-strscpy.c new file mode 100644 index 000000000000..38feddf377dc --- /dev/null +++ b/lib/test_fortify/write_overflow-strscpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + strscpy(instance.buf, large_src, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 15f2e2db77bc..c259842f6d44 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -25,6 +25,7 @@ #include <linux/swapops.h> #include <linux/sched/mm.h> #include <linux/platform_device.h> +#include <linux/rmap.h> #include "test_hmm_uapi.h" @@ -46,6 +47,7 @@ struct dmirror_bounce { unsigned long cpages; }; +#define DPT_XA_TAG_ATOMIC 1UL #define DPT_XA_TAG_WRITE 3UL /* @@ -218,7 +220,7 @@ static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni, * the invalidation is handled as part of the migration process. */ if (range->event == MMU_NOTIFY_MIGRATE && - range->migrate_pgmap_owner == dmirror->mdevice) + range->owner == dmirror->mdevice) return true; if (mmu_notifier_range_blockable(range)) @@ -619,6 +621,52 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args, } } +static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start, + unsigned long end) +{ + unsigned long pfn; + + for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) { + void *entry; + + entry = xa_load(&dmirror->pt, pfn); + if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC) + return -EPERM; + } + + return 0; +} + +static int dmirror_atomic_map(unsigned long start, unsigned long end, + struct page **pages, struct dmirror *dmirror) +{ + unsigned long pfn, mapped = 0; + int i; + + /* Map the migrated pages into the device's page tables. */ + mutex_lock(&dmirror->mutex); + + for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) { + void *entry; + + if (!pages[i]) + continue; + + entry = pages[i]; + entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC); + entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC); + if (xa_is_err(entry)) { + mutex_unlock(&dmirror->mutex); + return xa_err(entry); + } + + mapped++; + } + + mutex_unlock(&dmirror->mutex); + return mapped; +} + static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, struct dmirror *dmirror) { @@ -661,6 +709,72 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args, return 0; } +static int dmirror_exclusive(struct dmirror *dmirror, + struct hmm_dmirror_cmd *cmd) +{ + unsigned long start, end, addr; + unsigned long size = cmd->npages << PAGE_SHIFT; + struct mm_struct *mm = dmirror->notifier.mm; + struct page *pages[64]; + struct dmirror_bounce bounce; + unsigned long next; + int ret; + + start = cmd->addr; + end = start + size; + if (end < start) + return -EINVAL; + + /* Since the mm is for the mirrored process, get a reference first. */ + if (!mmget_not_zero(mm)) + return -EINVAL; + + mmap_read_lock(mm); + for (addr = start; addr < end; addr = next) { + unsigned long mapped; + int i; + + if (end < addr + (ARRAY_SIZE(pages) << PAGE_SHIFT)) + next = end; + else + next = addr + (ARRAY_SIZE(pages) << PAGE_SHIFT); + + ret = make_device_exclusive_range(mm, addr, next, pages, NULL); + mapped = dmirror_atomic_map(addr, next, pages, dmirror); + for (i = 0; i < ret; i++) { + if (pages[i]) { + unlock_page(pages[i]); + put_page(pages[i]); + } + } + + if (addr + (mapped << PAGE_SHIFT) < next) { + mmap_read_unlock(mm); + mmput(mm); + return -EBUSY; + } + } + mmap_read_unlock(mm); + mmput(mm); + + /* Return the migrated data for verification. */ + ret = dmirror_bounce_init(&bounce, start, size); + if (ret) + return ret; + mutex_lock(&dmirror->mutex); + ret = dmirror_do_read(dmirror, start, end, &bounce); + mutex_unlock(&dmirror->mutex); + if (ret == 0) { + if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr, + bounce.size)) + ret = -EFAULT; + } + + cmd->cpages = bounce.cpages; + dmirror_bounce_fini(&bounce); + return ret; +} + static int dmirror_migrate(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd) { @@ -948,6 +1062,15 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp, ret = dmirror_migrate(dmirror, &cmd); break; + case HMM_DMIRROR_EXCLUSIVE: + ret = dmirror_exclusive(dmirror, &cmd); + break; + + case HMM_DMIRROR_CHECK_EXCLUSIVE: + ret = dmirror_check_atomic(dmirror, cmd.addr, + cmd.addr + (cmd.npages << PAGE_SHIFT)); + break; + case HMM_DMIRROR_SNAPSHOT: ret = dmirror_snapshot(dmirror, &cmd); break; diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h index 670b4ef2a5b6..f14dea5dcd06 100644 --- a/lib/test_hmm_uapi.h +++ b/lib/test_hmm_uapi.h @@ -33,6 +33,8 @@ struct hmm_dmirror_cmd { #define HMM_DMIRROR_WRITE _IOWR('H', 0x01, struct hmm_dmirror_cmd) #define HMM_DMIRROR_MIGRATE _IOWR('H', 0x02, struct hmm_dmirror_cmd) #define HMM_DMIRROR_SNAPSHOT _IOWR('H', 0x03, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_EXCLUSIVE _IOWR('H', 0x04, struct hmm_dmirror_cmd) +#define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x05, struct hmm_dmirror_cmd) /* * Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT. diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 44e08f4d9c52..ebed755ebf34 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -53,7 +53,6 @@ static int kasan_test_init(struct kunit *test) } multishot = kasan_save_enable_multi_shot(); - kasan_set_tagging_report_once(false); fail_data.report_found = false; kunit_add_named_resource(test, NULL, NULL, &resource, "kasan_data", &fail_data); @@ -62,7 +61,6 @@ static int kasan_test_init(struct kunit *test) static void kasan_test_exit(struct kunit *test) { - kasan_set_tagging_report_once(true); kasan_restore_multi_shot(multishot); KUNIT_EXPECT_FALSE(test, fail_data.report_found); } @@ -90,7 +88,7 @@ static void kasan_test_exit(struct kunit *test) */ #define KUNIT_EXPECT_KASAN_FAIL(test, expression) do { \ if (IS_ENABLED(CONFIG_KASAN_HW_TAGS) && \ - !kasan_async_mode_enabled()) \ + kasan_sync_fault_possible()) \ migrate_disable(); \ KUNIT_EXPECT_FALSE(test, READ_ONCE(fail_data.report_found)); \ barrier(); \ @@ -110,28 +108,40 @@ static void kasan_test_exit(struct kunit *test) } while (0) #define KASAN_TEST_NEEDS_CONFIG_ON(test, config) do { \ - if (!IS_ENABLED(config)) { \ - kunit_info((test), "skipping, " #config " required"); \ - return; \ - } \ + if (!IS_ENABLED(config)) \ + kunit_skip((test), "Test requires " #config "=y"); \ } while (0) #define KASAN_TEST_NEEDS_CONFIG_OFF(test, config) do { \ - if (IS_ENABLED(config)) { \ - kunit_info((test), "skipping, " #config " enabled"); \ - return; \ - } \ + if (IS_ENABLED(config)) \ + kunit_skip((test), "Test requires " #config "=n"); \ } while (0) static void kmalloc_oob_right(struct kunit *test) { char *ptr; - size_t size = 123; + size_t size = 128 - KASAN_GRANULE_SIZE - 5; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, ptr[size + OOB_TAG_OFF] = 'x'); + /* + * An unaligned access past the requested kmalloc size. + * Only generic KASAN can precisely detect these. + */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC)) + KUNIT_EXPECT_KASAN_FAIL(test, ptr[size] = 'x'); + + /* + * An aligned access into the first out-of-bounds granule that falls + * within the aligned kmalloc object. + */ + KUNIT_EXPECT_KASAN_FAIL(test, ptr[size + 5] = 'y'); + + /* Out-of-bounds access past the aligned kmalloc object. */ + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = + ptr[size + KASAN_GRANULE_SIZE + 5]); + kfree(ptr); } @@ -155,7 +165,7 @@ static void kmalloc_node_oob_right(struct kunit *test) ptr = kmalloc_node(size, GFP_KERNEL, 0); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, ptr[size] = 0); + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = ptr[size]); kfree(ptr); } @@ -191,7 +201,7 @@ static void kmalloc_pagealloc_uaf(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); kfree(ptr); - KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = 0); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[0]); } static void kmalloc_pagealloc_invalid_free(struct kunit *test) @@ -225,7 +235,7 @@ static void pagealloc_oob_right(struct kunit *test) ptr = page_address(pages); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, ptr[size] = 0); + KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = ptr[size]); free_pages((unsigned long)ptr, order); } @@ -240,7 +250,7 @@ static void pagealloc_uaf(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); free_pages((unsigned long)ptr, order); - KUNIT_EXPECT_KASAN_FAIL(test, ptr[0] = 0); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[0]); } static void kmalloc_large_oob_right(struct kunit *test) @@ -416,64 +426,70 @@ static void kmalloc_uaf_16(struct kunit *test) kfree(ptr1); } +/* + * Note: in the memset tests below, the written range touches both valid and + * invalid memory. This makes sure that the instrumentation does not only check + * the starting address but the whole range. + */ + static void kmalloc_oob_memset_2(struct kunit *test) { char *ptr; - size_t size = 8; + size_t size = 128 - KASAN_GRANULE_SIZE; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + 7 + OOB_TAG_OFF, 0, 2)); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 1, 0, 2)); kfree(ptr); } static void kmalloc_oob_memset_4(struct kunit *test) { char *ptr; - size_t size = 8; + size_t size = 128 - KASAN_GRANULE_SIZE; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + 5 + OOB_TAG_OFF, 0, 4)); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 3, 0, 4)); kfree(ptr); } - static void kmalloc_oob_memset_8(struct kunit *test) { char *ptr; - size_t size = 8; + size_t size = 128 - KASAN_GRANULE_SIZE; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + 1 + OOB_TAG_OFF, 0, 8)); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 7, 0, 8)); kfree(ptr); } static void kmalloc_oob_memset_16(struct kunit *test) { char *ptr; - size_t size = 16; + size_t size = 128 - KASAN_GRANULE_SIZE; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + 1 + OOB_TAG_OFF, 0, 16)); + KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr + size - 15, 0, 16)); kfree(ptr); } static void kmalloc_oob_in_memset(struct kunit *test) { char *ptr; - size_t size = 666; + size_t size = 128 - KASAN_GRANULE_SIZE; ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); - KUNIT_EXPECT_KASAN_FAIL(test, memset(ptr, 0, size + 5 + OOB_TAG_OFF)); + KUNIT_EXPECT_KASAN_FAIL(test, + memset(ptr, 0, size + KASAN_GRANULE_SIZE)); kfree(ptr); } @@ -483,11 +499,17 @@ static void kmalloc_memmove_invalid_size(struct kunit *test) size_t size = 64; volatile size_t invalid_size = -2; + /* + * Hardware tag-based mode doesn't check memmove for negative size. + * As a result, this test introduces a side-effect memory corruption, + * which can result in a crash. + */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_HW_TAGS); + ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); memset((char *)ptr, 0, 64); - KUNIT_EXPECT_KASAN_FAIL(test, memmove((char *)ptr, (char *)ptr + 4, invalid_size)); kfree(ptr); @@ -502,7 +524,7 @@ static void kmalloc_uaf(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); kfree(ptr); - KUNIT_EXPECT_KASAN_FAIL(test, *(ptr + 8) = 'x'); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[8]); } static void kmalloc_uaf_memset(struct kunit *test) @@ -510,6 +532,12 @@ static void kmalloc_uaf_memset(struct kunit *test) char *ptr; size_t size = 33; + /* + * Only generic KASAN uses quarantine, which is required to avoid a + * kernel memory corruption this test causes. + */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_GENERIC); + ptr = kmalloc(size, GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); @@ -541,7 +569,7 @@ again: goto again; } - KUNIT_EXPECT_KASAN_FAIL(test, ptr1[40] = 'x'); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr1)[40]); KUNIT_EXPECT_PTR_NE(test, ptr1, ptr2); kfree(ptr2); @@ -655,7 +683,7 @@ static void kasan_global_oob(struct kunit *test) { /* * Deliberate out-of-bounds access. To prevent CONFIG_UBSAN_LOCAL_BOUNDS - * from failing here and panicing the kernel, access the array via a + * from failing here and panicking the kernel, access the array via a * volatile pointer, which will prevent the compiler from being able to * determine the array bounds. * @@ -688,7 +716,7 @@ static void ksize_unpoisons_memory(struct kunit *test) ptr[size] = 'x'; /* This one must. */ - KUNIT_EXPECT_KASAN_FAIL(test, ptr[real_size] = 'y'); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[real_size]); kfree(ptr); } @@ -707,8 +735,8 @@ static void ksize_uaf(struct kunit *test) kfree(ptr); KUNIT_EXPECT_KASAN_FAIL(test, ksize(ptr)); - KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = *ptr); - KUNIT_EXPECT_KASAN_FAIL(test, kasan_int_result = *(ptr + size)); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[0]); + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)ptr)[size]); } static void kasan_stack_oob(struct kunit *test) diff --git a/lib/test_kasan_module.c b/lib/test_kasan_module.c index f1017f345d6c..7ebf433edef3 100644 --- a/lib/test_kasan_module.c +++ b/lib/test_kasan_module.c @@ -15,13 +15,11 @@ #include "../mm/kasan/kasan.h" -#define OOB_TAG_OFF (IS_ENABLED(CONFIG_KASAN_GENERIC) ? 0 : KASAN_GRANULE_SIZE) - static noinline void __init copy_user_test(void) { char *kmem; char __user *usermem; - size_t size = 10; + size_t size = 128 - KASAN_GRANULE_SIZE; int __maybe_unused unused; kmem = kmalloc(size, GFP_KERNEL); @@ -38,25 +36,25 @@ static noinline void __init copy_user_test(void) } pr_info("out-of-bounds in copy_from_user()\n"); - unused = copy_from_user(kmem, usermem, size + 1 + OOB_TAG_OFF); + unused = copy_from_user(kmem, usermem, size + 1); pr_info("out-of-bounds in copy_to_user()\n"); - unused = copy_to_user(usermem, kmem, size + 1 + OOB_TAG_OFF); + unused = copy_to_user(usermem, kmem, size + 1); pr_info("out-of-bounds in __copy_from_user()\n"); - unused = __copy_from_user(kmem, usermem, size + 1 + OOB_TAG_OFF); + unused = __copy_from_user(kmem, usermem, size + 1); pr_info("out-of-bounds in __copy_to_user()\n"); - unused = __copy_to_user(usermem, kmem, size + 1 + OOB_TAG_OFF); + unused = __copy_to_user(usermem, kmem, size + 1); pr_info("out-of-bounds in __copy_from_user_inatomic()\n"); - unused = __copy_from_user_inatomic(kmem, usermem, size + 1 + OOB_TAG_OFF); + unused = __copy_from_user_inatomic(kmem, usermem, size + 1); pr_info("out-of-bounds in __copy_to_user_inatomic()\n"); - unused = __copy_to_user_inatomic(usermem, kmem, size + 1 + OOB_TAG_OFF); + unused = __copy_to_user_inatomic(usermem, kmem, size + 1); pr_info("out-of-bounds in strncpy_from_user()\n"); - unused = strncpy_from_user(kmem, usermem, size + 1 + OOB_TAG_OFF); + unused = strncpy_from_user(kmem, usermem, size + 1); vm_munmap((unsigned long)usermem, PAGE_SIZE); kfree(kmem); @@ -73,7 +71,7 @@ static noinline void __init kasan_rcu_reclaim(struct rcu_head *rp) struct kasan_rcu_info, rcu); kfree(fp); - fp->i = 1; + ((volatile struct kasan_rcu_info *)fp)->i; } static noinline void __init kasan_rcu_uaf(void) diff --git a/lib/test_kmod.c b/lib/test_kmod.c index 38c250fbace3..ce1589391413 100644 --- a/lib/test_kmod.c +++ b/lib/test_kmod.c @@ -286,7 +286,7 @@ static int tally_work_test(struct kmod_test_device_info *info) * If this ran it means *all* tasks were created fine and we * are now just collecting results. * - * Only propagate errors, do not override with a subsequent sucess case. + * Only propagate errors, do not override with a subsequent success case. */ static void tally_up_work(struct kmod_test_device *test_dev) { @@ -543,7 +543,7 @@ static int trigger_config_run(struct kmod_test_device *test_dev) * wrong with the setup of the test. If the test setup went fine * then userspace must just check the result of config->test_result. * One issue with relying on the return from a call in the kernel - * is if the kernel returns a possitive value using this trigger + * is if the kernel returns a positive value using this trigger * will not return the value to userspace, it would be lost. * * By not relying on capturing the return value of tests we are using @@ -585,7 +585,7 @@ trigger_config_store(struct device *dev, * Note: any return > 0 will be treated as success * and the error value will not be available to userspace. * Do not rely on trying to send to userspace a test value - * return value as possitive return errors will be lost. + * return value as positive return errors will be lost. */ if (WARN_ON(ret > 0)) return -EINVAL; diff --git a/lib/test_kprobes.c b/lib/test_kprobes.c new file mode 100644 index 000000000000..a5edc2ebc947 --- /dev/null +++ b/lib/test_kprobes.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * test_kprobes.c - simple sanity test for *probes + * + * Copyright IBM Corp. 2008 + */ + +#include <linux/kernel.h> +#include <linux/kprobes.h> +#include <linux/random.h> +#include <kunit/test.h> + +#define div_factor 3 + +static u32 rand1, preh_val, posth_val; +static u32 (*target)(u32 value); +static u32 (*target2)(u32 value); +static struct kunit *current_test; + +static unsigned long (*internal_target)(void); +static unsigned long (*stacktrace_target)(void); +static unsigned long (*stacktrace_driver)(void); +static unsigned long target_return_address[2]; + +static noinline u32 kprobe_target(u32 value) +{ + return (value / div_factor); +} + +static int kp_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + preh_val = (rand1 / div_factor); + return 0; +} + +static void kp_post_handler(struct kprobe *p, struct pt_regs *regs, + unsigned long flags) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, preh_val, (rand1 / div_factor)); + posth_val = preh_val + div_factor; +} + +static struct kprobe kp = { + .symbol_name = "kprobe_target", + .pre_handler = kp_pre_handler, + .post_handler = kp_post_handler +}; + +static void test_kprobe(struct kunit *test) +{ + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_kprobe(&kp)); + target(rand1); + unregister_kprobe(&kp); + KUNIT_EXPECT_NE(test, 0, preh_val); + KUNIT_EXPECT_NE(test, 0, posth_val); +} + +static noinline u32 kprobe_target2(u32 value) +{ + return (value / div_factor) + 1; +} + +static noinline unsigned long kprobe_stacktrace_internal_target(void) +{ + if (!target_return_address[0]) + target_return_address[0] = (unsigned long)__builtin_return_address(0); + return target_return_address[0]; +} + +static noinline unsigned long kprobe_stacktrace_target(void) +{ + if (!target_return_address[1]) + target_return_address[1] = (unsigned long)__builtin_return_address(0); + + if (internal_target) + internal_target(); + + return target_return_address[1]; +} + +static noinline unsigned long kprobe_stacktrace_driver(void) +{ + if (stacktrace_target) + stacktrace_target(); + + /* This is for preventing inlining the function */ + return (unsigned long)__builtin_return_address(0); +} + +static int kp_pre_handler2(struct kprobe *p, struct pt_regs *regs) +{ + preh_val = (rand1 / div_factor) + 1; + return 0; +} + +static void kp_post_handler2(struct kprobe *p, struct pt_regs *regs, + unsigned long flags) +{ + KUNIT_EXPECT_EQ(current_test, preh_val, (rand1 / div_factor) + 1); + posth_val = preh_val + div_factor; +} + +static struct kprobe kp2 = { + .symbol_name = "kprobe_target2", + .pre_handler = kp_pre_handler2, + .post_handler = kp_post_handler2 +}; + +static void test_kprobes(struct kunit *test) +{ + struct kprobe *kps[2] = {&kp, &kp2}; + + current_test = test; + + /* addr and flags should be cleard for reusing kprobe. */ + kp.addr = NULL; + kp.flags = 0; + + KUNIT_EXPECT_EQ(test, 0, register_kprobes(kps, 2)); + preh_val = 0; + posth_val = 0; + target(rand1); + + KUNIT_EXPECT_NE(test, 0, preh_val); + KUNIT_EXPECT_NE(test, 0, posth_val); + + preh_val = 0; + posth_val = 0; + target2(rand1); + + KUNIT_EXPECT_NE(test, 0, preh_val); + KUNIT_EXPECT_NE(test, 0, posth_val); + unregister_kprobes(kps, 2); +} + +#ifdef CONFIG_KRETPROBES +static u32 krph_val; + +static int entry_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + krph_val = (rand1 / div_factor); + return 0; +} + +static int return_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long ret = regs_return_value(regs); + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, ret, rand1 / div_factor); + KUNIT_EXPECT_NE(current_test, krph_val, 0); + krph_val = rand1; + return 0; +} + +static struct kretprobe rp = { + .handler = return_handler, + .entry_handler = entry_handler, + .kp.symbol_name = "kprobe_target" +}; + +static void test_kretprobe(struct kunit *test) +{ + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_kretprobe(&rp)); + target(rand1); + unregister_kretprobe(&rp); + KUNIT_EXPECT_EQ(test, krph_val, rand1); +} + +static int return_handler2(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long ret = regs_return_value(regs); + + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor) + 1); + KUNIT_EXPECT_NE(current_test, krph_val, 0); + krph_val = rand1; + return 0; +} + +static struct kretprobe rp2 = { + .handler = return_handler2, + .entry_handler = entry_handler, + .kp.symbol_name = "kprobe_target2" +}; + +static void test_kretprobes(struct kunit *test) +{ + struct kretprobe *rps[2] = {&rp, &rp2}; + + current_test = test; + /* addr and flags should be cleard for reusing kprobe. */ + rp.kp.addr = NULL; + rp.kp.flags = 0; + KUNIT_EXPECT_EQ(test, 0, register_kretprobes(rps, 2)); + + krph_val = 0; + target(rand1); + KUNIT_EXPECT_EQ(test, krph_val, rand1); + + krph_val = 0; + target2(rand1); + KUNIT_EXPECT_EQ(test, krph_val, rand1); + unregister_kretprobes(rps, 2); +} + +#ifdef CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE +#define STACK_BUF_SIZE 16 +static unsigned long stack_buf[STACK_BUF_SIZE]; + +static int stacktrace_return_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long retval = regs_return_value(regs); + int i, ret; + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, retval, target_return_address[1]); + + /* + * Test stacktrace inside the kretprobe handler, this will involves + * kretprobe trampoline, but must include correct return address + * of the target function. + */ + ret = stack_trace_save(stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + + for (i = 0; i < ret; i++) { + if (stack_buf[i] == target_return_address[1]) + break; + } + KUNIT_EXPECT_NE(current_test, i, ret); + +#if !IS_MODULE(CONFIG_KPROBES_SANITY_TEST) + /* + * Test stacktrace from pt_regs at the return address. Thus the stack + * trace must start from the target return address. + */ + ret = stack_trace_save_regs(regs, stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + KUNIT_EXPECT_EQ(current_test, stack_buf[0], target_return_address[1]); +#endif + + return 0; +} + +static struct kretprobe rp3 = { + .handler = stacktrace_return_handler, + .kp.symbol_name = "kprobe_stacktrace_target" +}; + +static void test_stacktrace_on_kretprobe(struct kunit *test) +{ + unsigned long myretaddr = (unsigned long)__builtin_return_address(0); + + current_test = test; + rp3.kp.addr = NULL; + rp3.kp.flags = 0; + + /* + * Run the stacktrace_driver() to record correct return address in + * stacktrace_target() and ensure stacktrace_driver() call is not + * inlined by checking the return address of stacktrace_driver() + * and the return address of this function is different. + */ + KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + + KUNIT_ASSERT_EQ(test, 0, register_kretprobe(&rp3)); + KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + unregister_kretprobe(&rp3); +} + +static int stacktrace_internal_return_handler(struct kretprobe_instance *ri, struct pt_regs *regs) +{ + unsigned long retval = regs_return_value(regs); + int i, ret; + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + KUNIT_EXPECT_EQ(current_test, retval, target_return_address[0]); + + /* + * Test stacktrace inside the kretprobe handler for nested case. + * The unwinder will find the kretprobe_trampoline address on the + * return address, and kretprobe must solve that. + */ + ret = stack_trace_save(stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + + for (i = 0; i < ret - 1; i++) { + if (stack_buf[i] == target_return_address[0]) { + KUNIT_EXPECT_EQ(current_test, stack_buf[i + 1], target_return_address[1]); + break; + } + } + KUNIT_EXPECT_NE(current_test, i, ret); + +#if !IS_MODULE(CONFIG_KPROBES_SANITY_TEST) + /* Ditto for the regs version. */ + ret = stack_trace_save_regs(regs, stack_buf, STACK_BUF_SIZE, 0); + KUNIT_EXPECT_NE(current_test, ret, 0); + KUNIT_EXPECT_EQ(current_test, stack_buf[0], target_return_address[0]); + KUNIT_EXPECT_EQ(current_test, stack_buf[1], target_return_address[1]); +#endif + + return 0; +} + +static struct kretprobe rp4 = { + .handler = stacktrace_internal_return_handler, + .kp.symbol_name = "kprobe_stacktrace_internal_target" +}; + +static void test_stacktrace_on_nested_kretprobe(struct kunit *test) +{ + unsigned long myretaddr = (unsigned long)__builtin_return_address(0); + struct kretprobe *rps[2] = {&rp3, &rp4}; + + current_test = test; + rp3.kp.addr = NULL; + rp3.kp.flags = 0; + + //KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + + KUNIT_ASSERT_EQ(test, 0, register_kretprobes(rps, 2)); + KUNIT_ASSERT_NE(test, myretaddr, stacktrace_driver()); + unregister_kretprobes(rps, 2); +} +#endif /* CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE */ + +#endif /* CONFIG_KRETPROBES */ + +static int kprobes_test_init(struct kunit *test) +{ + target = kprobe_target; + target2 = kprobe_target2; + stacktrace_target = kprobe_stacktrace_target; + internal_target = kprobe_stacktrace_internal_target; + stacktrace_driver = kprobe_stacktrace_driver; + + do { + rand1 = prandom_u32(); + } while (rand1 <= div_factor); + return 0; +} + +static struct kunit_case kprobes_testcases[] = { + KUNIT_CASE(test_kprobe), + KUNIT_CASE(test_kprobes), +#ifdef CONFIG_KRETPROBES + KUNIT_CASE(test_kretprobe), + KUNIT_CASE(test_kretprobes), +#ifdef CONFIG_ARCH_CORRECT_STACKTRACE_ON_KRETPROBE + KUNIT_CASE(test_stacktrace_on_kretprobe), + KUNIT_CASE(test_stacktrace_on_nested_kretprobe), +#endif +#endif + {} +}; + +static struct kunit_suite kprobes_test_suite = { + .name = "kprobes_test", + .init = kprobes_test_init, + .test_cases = kprobes_testcases, +}; + +kunit_test_suites(&kprobes_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_list_sort.c b/lib/test_list_sort.c index 00daaf23316f..ade7a1ea0c8e 100644 --- a/lib/test_list_sort.c +++ b/lib/test_list_sort.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -#define pr_fmt(fmt) "list_sort_test: " fmt +#include <kunit/test.h> #include <linux/kernel.h> #include <linux/list_sort.h> @@ -23,68 +23,52 @@ struct debug_el { struct list_head list; unsigned int poison2; int value; - unsigned serial; + unsigned int serial; }; -/* Array, containing pointers to all elements in the test list */ -static struct debug_el **elts __initdata; - -static int __init check(struct debug_el *ela, struct debug_el *elb) +static void check(struct kunit *test, struct debug_el *ela, struct debug_el *elb) { - if (ela->serial >= TEST_LIST_LEN) { - pr_err("error: incorrect serial %d\n", ela->serial); - return -EINVAL; - } - if (elb->serial >= TEST_LIST_LEN) { - pr_err("error: incorrect serial %d\n", elb->serial); - return -EINVAL; - } - if (elts[ela->serial] != ela || elts[elb->serial] != elb) { - pr_err("error: phantom element\n"); - return -EINVAL; - } - if (ela->poison1 != TEST_POISON1 || ela->poison2 != TEST_POISON2) { - pr_err("error: bad poison: %#x/%#x\n", - ela->poison1, ela->poison2); - return -EINVAL; - } - if (elb->poison1 != TEST_POISON1 || elb->poison2 != TEST_POISON2) { - pr_err("error: bad poison: %#x/%#x\n", - elb->poison1, elb->poison2); - return -EINVAL; - } - return 0; + struct debug_el **elts = test->priv; + + KUNIT_EXPECT_LT_MSG(test, ela->serial, (unsigned int)TEST_LIST_LEN, "incorrect serial"); + KUNIT_EXPECT_LT_MSG(test, elb->serial, (unsigned int)TEST_LIST_LEN, "incorrect serial"); + + KUNIT_EXPECT_PTR_EQ_MSG(test, elts[ela->serial], ela, "phantom element"); + KUNIT_EXPECT_PTR_EQ_MSG(test, elts[elb->serial], elb, "phantom element"); + + KUNIT_EXPECT_EQ_MSG(test, ela->poison1, TEST_POISON1, "bad poison"); + KUNIT_EXPECT_EQ_MSG(test, ela->poison2, TEST_POISON2, "bad poison"); + + KUNIT_EXPECT_EQ_MSG(test, elb->poison1, TEST_POISON1, "bad poison"); + KUNIT_EXPECT_EQ_MSG(test, elb->poison2, TEST_POISON2, "bad poison"); } -static int __init cmp(void *priv, const struct list_head *a, - const struct list_head *b) +/* `priv` is the test pointer so check() can fail the test if the list is invalid. */ +static int cmp(void *priv, const struct list_head *a, const struct list_head *b) { struct debug_el *ela, *elb; ela = container_of(a, struct debug_el, list); elb = container_of(b, struct debug_el, list); - check(ela, elb); + check(priv, ela, elb); return ela->value - elb->value; } -static int __init list_sort_test(void) +static void list_sort_test(struct kunit *test) { - int i, count = 1, err = -ENOMEM; - struct debug_el *el; + int i, count = 1; + struct debug_el *el, **elts; struct list_head *cur; LIST_HEAD(head); - pr_debug("start testing list_sort()\n"); - - elts = kcalloc(TEST_LIST_LEN, sizeof(*elts), GFP_KERNEL); - if (!elts) - return err; + elts = kunit_kcalloc(test, TEST_LIST_LEN, sizeof(*elts), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, elts); + test->priv = elts; for (i = 0; i < TEST_LIST_LEN; i++) { - el = kmalloc(sizeof(*el), GFP_KERNEL); - if (!el) - goto exit; + el = kunit_kmalloc(test, sizeof(*el), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, el); /* force some equivalencies */ el->value = prandom_u32() % (TEST_LIST_LEN / 3); @@ -95,55 +79,44 @@ static int __init list_sort_test(void) list_add_tail(&el->list, &head); } - list_sort(NULL, &head, cmp); + list_sort(test, &head, cmp); - err = -EINVAL; for (cur = head.next; cur->next != &head; cur = cur->next) { struct debug_el *el1; int cmp_result; - if (cur->next->prev != cur) { - pr_err("error: list is corrupted\n"); - goto exit; - } + KUNIT_ASSERT_PTR_EQ_MSG(test, cur->next->prev, cur, + "list is corrupted"); - cmp_result = cmp(NULL, cur, cur->next); - if (cmp_result > 0) { - pr_err("error: list is not sorted\n"); - goto exit; - } + cmp_result = cmp(test, cur, cur->next); + KUNIT_ASSERT_LE_MSG(test, cmp_result, 0, "list is not sorted"); el = container_of(cur, struct debug_el, list); el1 = container_of(cur->next, struct debug_el, list); - if (cmp_result == 0 && el->serial >= el1->serial) { - pr_err("error: order of equivalent elements not " - "preserved\n"); - goto exit; + if (cmp_result == 0) { + KUNIT_ASSERT_LE_MSG(test, el->serial, el1->serial, + "order of equivalent elements not preserved"); } - if (check(el, el1)) { - pr_err("error: element check failed\n"); - goto exit; - } + check(test, el, el1); count++; } - if (head.prev != cur) { - pr_err("error: list is corrupted\n"); - goto exit; - } + KUNIT_EXPECT_PTR_EQ_MSG(test, head.prev, cur, "list is corrupted"); + KUNIT_EXPECT_EQ_MSG(test, count, TEST_LIST_LEN, + "list length changed after sorting!"); +} - if (count != TEST_LIST_LEN) { - pr_err("error: bad list length %d", count); - goto exit; - } +static struct kunit_case list_sort_cases[] = { + KUNIT_CASE(list_sort_test), + {} +}; + +static struct kunit_suite list_sort_suite = { + .name = "list_sort", + .test_cases = list_sort_cases, +}; + +kunit_test_suites(&list_sort_suite); - err = 0; -exit: - for (i = 0; i < TEST_LIST_LEN; i++) - kfree(elts[i]); - kfree(elts); - return err; -} -module_init(list_sort_test); MODULE_LICENSE("GPL"); diff --git a/lib/test_lockup.c b/lib/test_lockup.c index 864554e76973..906b598740a7 100644 --- a/lib/test_lockup.c +++ b/lib/test_lockup.c @@ -485,13 +485,13 @@ static int __init test_lockup_init(void) offsetof(spinlock_t, lock.wait_lock.magic), SPINLOCK_MAGIC) || test_magic(lock_rwlock_ptr, - offsetof(rwlock_t, rtmutex.wait_lock.magic), + offsetof(rwlock_t, rwbase.rtmutex.wait_lock.magic), SPINLOCK_MAGIC) || test_magic(lock_mutex_ptr, - offsetof(struct mutex, lock.wait_lock.magic), + offsetof(struct mutex, rtmutex.wait_lock.magic), SPINLOCK_MAGIC) || test_magic(lock_rwsem_ptr, - offsetof(struct rw_semaphore, rtmutex.wait_lock.magic), + offsetof(struct rw_semaphore, rwbase.rtmutex.wait_lock.magic), SPINLOCK_MAGIC)) return -EINVAL; #else @@ -502,7 +502,7 @@ static int __init test_lockup_init(void) offsetof(rwlock_t, magic), RWLOCK_MAGIC) || test_magic(lock_mutex_ptr, - offsetof(struct mutex, wait_lock.rlock.magic), + offsetof(struct mutex, wait_lock.magic), SPINLOCK_MAGIC) || test_magic(lock_rwsem_ptr, offsetof(struct rw_semaphore, wait_lock.magic), diff --git a/lib/test_printf.c b/lib/test_printf.c index 8ac71aee46af..07309c45f327 100644 --- a/lib/test_printf.c +++ b/lib/test_printf.c @@ -586,70 +586,59 @@ struct page_flags_test { int width; int shift; int mask; - unsigned long value; const char *fmt; const char *name; }; -static struct page_flags_test pft[] = { +static const struct page_flags_test pft[] = { {SECTIONS_WIDTH, SECTIONS_PGSHIFT, SECTIONS_MASK, - 0, "%d", "section"}, + "%d", "section"}, {NODES_WIDTH, NODES_PGSHIFT, NODES_MASK, - 0, "%d", "node"}, + "%d", "node"}, {ZONES_WIDTH, ZONES_PGSHIFT, ZONES_MASK, - 0, "%d", "zone"}, + "%d", "zone"}, {LAST_CPUPID_WIDTH, LAST_CPUPID_PGSHIFT, LAST_CPUPID_MASK, - 0, "%#x", "lastcpupid"}, + "%#x", "lastcpupid"}, {KASAN_TAG_WIDTH, KASAN_TAG_PGSHIFT, KASAN_TAG_MASK, - 0, "%#x", "kasantag"}, + "%#x", "kasantag"}, }; static void __init page_flags_test(int section, int node, int zone, int last_cpupid, - int kasan_tag, int flags, const char *name, char *cmp_buf) + int kasan_tag, unsigned long flags, const char *name, + char *cmp_buf) { unsigned long values[] = {section, node, zone, last_cpupid, kasan_tag}; - unsigned long page_flags = 0; - unsigned long size = 0; + unsigned long size; bool append = false; int i; - flags &= BIT(NR_PAGEFLAGS) - 1; - if (flags) { - page_flags |= flags; - snprintf(cmp_buf + size, BUF_SIZE - size, "%s", name); - size = strlen(cmp_buf); -#if SECTIONS_WIDTH || NODES_WIDTH || ZONES_WIDTH || \ - LAST_CPUPID_WIDTH || KASAN_TAG_WIDTH - /* Other information also included in page flags */ - snprintf(cmp_buf + size, BUF_SIZE - size, "|"); - size = strlen(cmp_buf); -#endif - } + for (i = 0; i < ARRAY_SIZE(values); i++) + flags |= (values[i] & pft[i].mask) << pft[i].shift; - /* Set the test value */ - for (i = 0; i < ARRAY_SIZE(pft); i++) - pft[i].value = values[i]; + size = scnprintf(cmp_buf, BUF_SIZE, "%#lx(", flags); + if (flags & PAGEFLAGS_MASK) { + size += scnprintf(cmp_buf + size, BUF_SIZE - size, "%s", name); + append = true; + } for (i = 0; i < ARRAY_SIZE(pft); i++) { if (!pft[i].width) continue; - if (append) { - snprintf(cmp_buf + size, BUF_SIZE - size, "|"); - size = strlen(cmp_buf); - } + if (append) + size += scnprintf(cmp_buf + size, BUF_SIZE - size, "|"); - page_flags |= (pft[i].value & pft[i].mask) << pft[i].shift; - snprintf(cmp_buf + size, BUF_SIZE - size, "%s=", pft[i].name); - size = strlen(cmp_buf); - snprintf(cmp_buf + size, BUF_SIZE - size, pft[i].fmt, - pft[i].value & pft[i].mask); - size = strlen(cmp_buf); + size += scnprintf(cmp_buf + size, BUF_SIZE - size, "%s=", + pft[i].name); + size += scnprintf(cmp_buf + size, BUF_SIZE - size, pft[i].fmt, + values[i] & pft[i].mask); append = true; } - test(cmp_buf, "%pGp", &page_flags); + snprintf(cmp_buf + size, BUF_SIZE - size, ")"); + + test(cmp_buf, "%pGp", &flags); } static void __init @@ -675,9 +664,8 @@ flags(void) "uptodate|dirty|lru|active|swapbacked", cmp_buffer); - flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC - | VM_DENYWRITE; - test("read|exec|mayread|maywrite|mayexec|denywrite", "%pGv", &flags); + flags = VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; + test("read|exec|mayread|maywrite|mayexec", "%pGv", &flags); gfp = GFP_TRANSHUGE; test("GFP_TRANSHUGE", "%pGg", &gfp); diff --git a/lib/test_scanf.c b/lib/test_scanf.c index 48ff5747a4da..b620cf7de503 100644 --- a/lib/test_scanf.c +++ b/lib/test_scanf.c @@ -271,7 +271,7 @@ static u32 __init next_test_random(u32 max_bits) { u32 n_bits = hweight32(prandom_u32_state(&rnd_state)) % (max_bits + 1); - return prandom_u32_state(&rnd_state) & (UINT_MAX >> (32 - n_bits)); + return prandom_u32_state(&rnd_state) & GENMASK(n_bits, 0); } static unsigned long long __init next_test_random_ull(void) @@ -280,7 +280,7 @@ static unsigned long long __init next_test_random_ull(void) u32 n_bits = (hweight32(rand1) * 3) % 64; u64 val = (u64)prandom_u32_state(&rnd_state) * rand1; - return val & (ULLONG_MAX >> (64 - n_bits)); + return val & GENMASK_ULL(n_bits, 0); } #define random_for_type(T) \ @@ -398,7 +398,7 @@ do { \ test_array_8(fn, expect, test_buffer, fmt_buffer, result); \ } while (0) -static void __init numbers_list(const char *delim) +static void __init numbers_list_ll(const char *delim) { numbers_list_8(unsigned long long, "%llu", delim, "llu", check_ull); numbers_list_8(long long, "%lld", delim, "lld", check_ll); @@ -406,28 +406,40 @@ static void __init numbers_list(const char *delim) numbers_list_8(unsigned long long, "%llx", delim, "llx", check_ull); numbers_list_8(unsigned long long, "0x%llx", delim, "llx", check_ull); numbers_list_8(long long, "0x%llx", delim, "lli", check_ll); +} +static void __init numbers_list_l(const char *delim) +{ numbers_list_8(unsigned long, "%lu", delim, "lu", check_ulong); numbers_list_8(long, "%ld", delim, "ld", check_long); numbers_list_8(long, "%ld", delim, "li", check_long); numbers_list_8(unsigned long, "%lx", delim, "lx", check_ulong); numbers_list_8(unsigned long, "0x%lx", delim, "lx", check_ulong); numbers_list_8(long, "0x%lx", delim, "li", check_long); +} +static void __init numbers_list_d(const char *delim) +{ numbers_list_8(unsigned int, "%u", delim, "u", check_uint); numbers_list_8(int, "%d", delim, "d", check_int); numbers_list_8(int, "%d", delim, "i", check_int); numbers_list_8(unsigned int, "%x", delim, "x", check_uint); numbers_list_8(unsigned int, "0x%x", delim, "x", check_uint); numbers_list_8(int, "0x%x", delim, "i", check_int); +} +static void __init numbers_list_h(const char *delim) +{ numbers_list_8(unsigned short, "%hu", delim, "hu", check_ushort); numbers_list_8(short, "%hd", delim, "hd", check_short); numbers_list_8(short, "%hd", delim, "hi", check_short); numbers_list_8(unsigned short, "%hx", delim, "hx", check_ushort); numbers_list_8(unsigned short, "0x%hx", delim, "hx", check_ushort); numbers_list_8(short, "0x%hx", delim, "hi", check_short); +} +static void __init numbers_list_hh(const char *delim) +{ numbers_list_8(unsigned char, "%hhu", delim, "hhu", check_uchar); numbers_list_8(signed char, "%hhd", delim, "hhd", check_char); numbers_list_8(signed char, "%hhd", delim, "hhi", check_char); @@ -436,11 +448,16 @@ static void __init numbers_list(const char *delim) numbers_list_8(signed char, "0x%hhx", delim, "hhi", check_char); } -/* - * List of numbers separated by delim. Each field width specifier is the - * maximum possible digits for the given type and base. - */ -static void __init numbers_list_field_width_typemax(const char *delim) +static void __init numbers_list(const char *delim) +{ + numbers_list_ll(delim); + numbers_list_l(delim); + numbers_list_d(delim); + numbers_list_h(delim); + numbers_list_hh(delim); +} + +static void __init numbers_list_field_width_ll(const char *delim) { numbers_list_fix_width(unsigned long long, "%llu", delim, 20, "llu", check_ull); numbers_list_fix_width(long long, "%lld", delim, 20, "lld", check_ll); @@ -448,7 +465,10 @@ static void __init numbers_list_field_width_typemax(const char *delim) numbers_list_fix_width(unsigned long long, "%llx", delim, 16, "llx", check_ull); numbers_list_fix_width(unsigned long long, "0x%llx", delim, 18, "llx", check_ull); numbers_list_fix_width(long long, "0x%llx", delim, 18, "lli", check_ll); +} +static void __init numbers_list_field_width_l(const char *delim) +{ #if BITS_PER_LONG == 64 numbers_list_fix_width(unsigned long, "%lu", delim, 20, "lu", check_ulong); numbers_list_fix_width(long, "%ld", delim, 20, "ld", check_long); @@ -464,21 +484,30 @@ static void __init numbers_list_field_width_typemax(const char *delim) numbers_list_fix_width(unsigned long, "0x%lx", delim, 10, "lx", check_ulong); numbers_list_fix_width(long, "0x%lx", delim, 10, "li", check_long); #endif +} +static void __init numbers_list_field_width_d(const char *delim) +{ numbers_list_fix_width(unsigned int, "%u", delim, 10, "u", check_uint); numbers_list_fix_width(int, "%d", delim, 11, "d", check_int); numbers_list_fix_width(int, "%d", delim, 11, "i", check_int); numbers_list_fix_width(unsigned int, "%x", delim, 8, "x", check_uint); numbers_list_fix_width(unsigned int, "0x%x", delim, 10, "x", check_uint); numbers_list_fix_width(int, "0x%x", delim, 10, "i", check_int); +} +static void __init numbers_list_field_width_h(const char *delim) +{ numbers_list_fix_width(unsigned short, "%hu", delim, 5, "hu", check_ushort); numbers_list_fix_width(short, "%hd", delim, 6, "hd", check_short); numbers_list_fix_width(short, "%hd", delim, 6, "hi", check_short); numbers_list_fix_width(unsigned short, "%hx", delim, 4, "hx", check_ushort); numbers_list_fix_width(unsigned short, "0x%hx", delim, 6, "hx", check_ushort); numbers_list_fix_width(short, "0x%hx", delim, 6, "hi", check_short); +} +static void __init numbers_list_field_width_hh(const char *delim) +{ numbers_list_fix_width(unsigned char, "%hhu", delim, 3, "hhu", check_uchar); numbers_list_fix_width(signed char, "%hhd", delim, 4, "hhd", check_char); numbers_list_fix_width(signed char, "%hhd", delim, 4, "hhi", check_char); @@ -489,9 +518,18 @@ static void __init numbers_list_field_width_typemax(const char *delim) /* * List of numbers separated by delim. Each field width specifier is the - * exact length of the corresponding value digits in the string being scanned. + * maximum possible digits for the given type and base. */ -static void __init numbers_list_field_width_val_width(const char *delim) +static void __init numbers_list_field_width_typemax(const char *delim) +{ + numbers_list_field_width_ll(delim); + numbers_list_field_width_l(delim); + numbers_list_field_width_d(delim); + numbers_list_field_width_h(delim); + numbers_list_field_width_hh(delim); +} + +static void __init numbers_list_field_width_val_ll(const char *delim) { numbers_list_val_width(unsigned long long, "%llu", delim, "llu", check_ull); numbers_list_val_width(long long, "%lld", delim, "lld", check_ll); @@ -499,28 +537,40 @@ static void __init numbers_list_field_width_val_width(const char *delim) numbers_list_val_width(unsigned long long, "%llx", delim, "llx", check_ull); numbers_list_val_width(unsigned long long, "0x%llx", delim, "llx", check_ull); numbers_list_val_width(long long, "0x%llx", delim, "lli", check_ll); +} +static void __init numbers_list_field_width_val_l(const char *delim) +{ numbers_list_val_width(unsigned long, "%lu", delim, "lu", check_ulong); numbers_list_val_width(long, "%ld", delim, "ld", check_long); numbers_list_val_width(long, "%ld", delim, "li", check_long); numbers_list_val_width(unsigned long, "%lx", delim, "lx", check_ulong); numbers_list_val_width(unsigned long, "0x%lx", delim, "lx", check_ulong); numbers_list_val_width(long, "0x%lx", delim, "li", check_long); +} +static void __init numbers_list_field_width_val_d(const char *delim) +{ numbers_list_val_width(unsigned int, "%u", delim, "u", check_uint); numbers_list_val_width(int, "%d", delim, "d", check_int); numbers_list_val_width(int, "%d", delim, "i", check_int); numbers_list_val_width(unsigned int, "%x", delim, "x", check_uint); numbers_list_val_width(unsigned int, "0x%x", delim, "x", check_uint); numbers_list_val_width(int, "0x%x", delim, "i", check_int); +} +static void __init numbers_list_field_width_val_h(const char *delim) +{ numbers_list_val_width(unsigned short, "%hu", delim, "hu", check_ushort); numbers_list_val_width(short, "%hd", delim, "hd", check_short); numbers_list_val_width(short, "%hd", delim, "hi", check_short); numbers_list_val_width(unsigned short, "%hx", delim, "hx", check_ushort); numbers_list_val_width(unsigned short, "0x%hx", delim, "hx", check_ushort); numbers_list_val_width(short, "0x%hx", delim, "hi", check_short); +} +static void __init numbers_list_field_width_val_hh(const char *delim) +{ numbers_list_val_width(unsigned char, "%hhu", delim, "hhu", check_uchar); numbers_list_val_width(signed char, "%hhd", delim, "hhd", check_char); numbers_list_val_width(signed char, "%hhd", delim, "hhi", check_char); @@ -530,6 +580,19 @@ static void __init numbers_list_field_width_val_width(const char *delim) } /* + * List of numbers separated by delim. Each field width specifier is the + * exact length of the corresponding value digits in the string being scanned. + */ +static void __init numbers_list_field_width_val_width(const char *delim) +{ + numbers_list_field_width_val_ll(delim); + numbers_list_field_width_val_l(delim); + numbers_list_field_width_val_d(delim); + numbers_list_field_width_val_h(delim); + numbers_list_field_width_val_hh(delim); +} + +/* * Slice a continuous string of digits without field delimiters, containing * numbers of varying length, using the field width to extract each group * of digits. For example the hex values c0,3,bf01,303 would have a @@ -600,7 +663,7 @@ static void __init numbers_prefix_overflow(void) /* * 0x prefix in a field of width 2 using %i conversion: first field * converts to 0. Next field scan starts at the character after "0x", - * which will convert if can be intepreted as decimal but will fail + * which will convert if can be interpreted as decimal but will fail * if it contains any hex digits (since no 0x prefix). */ test_number_prefix(long long, "0x67", "%2lli%lli", 0, 67, 2, check_ll); diff --git a/lib/test_sort.c b/lib/test_sort.c index 52edbe10f2e5..be02e3a098cf 100644 --- a/lib/test_sort.c +++ b/lib/test_sort.c @@ -1,4 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> + #include <linux/sort.h> #include <linux/slab.h> #include <linux/module.h> @@ -7,18 +10,17 @@ #define TEST_LEN 1000 -static int __init cmpint(const void *a, const void *b) +static int cmpint(const void *a, const void *b) { return *(int *)a - *(int *)b; } -static int __init test_sort_init(void) +static void test_sort(struct kunit *test) { - int *a, i, r = 1, err = -ENOMEM; + int *a, i, r = 1; - a = kmalloc_array(TEST_LEN, sizeof(*a), GFP_KERNEL); - if (!a) - return err; + a = kunit_kmalloc_array(test, TEST_LEN, sizeof(*a), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, a); for (i = 0; i < TEST_LEN; i++) { r = (r * 725861) % 6599; @@ -27,24 +29,20 @@ static int __init test_sort_init(void) sort(a, TEST_LEN, sizeof(*a), cmpint, NULL); - err = -EINVAL; for (i = 0; i < TEST_LEN-1; i++) - if (a[i] > a[i+1]) { - pr_err("test has failed\n"); - goto exit; - } - err = 0; - pr_info("test passed\n"); -exit: - kfree(a); - return err; + KUNIT_ASSERT_LE(test, a[i], a[i + 1]); } -static void __exit test_sort_exit(void) -{ -} +static struct kunit_case sort_test_cases[] = { + KUNIT_CASE(test_sort), + {} +}; + +static struct kunit_suite sort_test_suite = { + .name = "lib_sort", + .test_cases = sort_test_cases, +}; -module_init(test_sort_init); -module_exit(test_sort_exit); +kunit_test_suites(&sort_test_suite); MODULE_LICENSE("GPL"); diff --git a/lib/test_stackinit.c b/lib/test_stackinit.c index f93b1e145ada..a3c74e6a21ff 100644 --- a/lib/test_stackinit.c +++ b/lib/test_stackinit.c @@ -1,8 +1,13 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-or-later /* - * Test cases for compiler-based stack variable zeroing via future - * compiler flags or CONFIG_GCC_PLUGIN_STRUCTLEAK*. + * Test cases for compiler-based stack variable zeroing via + * -ftrivial-auto-var-init={zero,pattern} or CONFIG_GCC_PLUGIN_STRUCTLEAK*. + * + * External build example: + * clang -O2 -Wall -ftrivial-auto-var-init=pattern \ + * -o test_stackinit test_stackinit.c */ +#ifdef __KERNEL__ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/init.h> @@ -10,6 +15,63 @@ #include <linux/module.h> #include <linux/string.h> +#else + +/* Userspace headers. */ +#include <stdio.h> +#include <stdint.h> +#include <string.h> +#include <stdbool.h> +#include <errno.h> +#include <sys/types.h> + +/* Linux kernel-ism stubs for stand-alone userspace build. */ +#define KBUILD_MODNAME "stackinit" +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define pr_err(fmt, ...) fprintf(stderr, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warn(fmt, ...) fprintf(stderr, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_info(fmt, ...) fprintf(stdout, pr_fmt(fmt), ##__VA_ARGS__) +#define __init /**/ +#define __exit /**/ +#define __user /**/ +#define noinline __attribute__((__noinline__)) +#define __aligned(x) __attribute__((__aligned__(x))) +#ifdef __clang__ +# define __compiletime_error(message) /**/ +#else +# define __compiletime_error(message) __attribute__((__error__(message))) +#endif +#define __compiletime_assert(condition, msg, prefix, suffix) \ + do { \ + extern void prefix ## suffix(void) __compiletime_error(msg); \ + if (!(condition)) \ + prefix ## suffix(); \ + } while (0) +#define _compiletime_assert(condition, msg, prefix, suffix) \ + __compiletime_assert(condition, msg, prefix, suffix) +#define compiletime_assert(condition, msg) \ + _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__) +#define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg) +#define BUILD_BUG_ON(condition) \ + BUILD_BUG_ON_MSG(condition, "BUILD_BUG_ON failed: " #condition) +typedef uint8_t u8; +typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; + +#define module_init(func) static int (*do_init)(void) = func +#define module_exit(func) static void (*do_exit)(void) = func +#define MODULE_LICENSE(str) int main(void) { \ + int rc; \ + /* License: str */ \ + rc = do_init(); \ + if (rc == 0) \ + do_exit(); \ + return rc; \ + } + +#endif /* __KERNEL__ */ + /* Exfiltration buffer. */ #define MAX_VAR_SIZE 128 static u8 check_buf[MAX_VAR_SIZE]; @@ -33,6 +95,10 @@ static bool range_contains(char *haystack_start, size_t haystack_size, return false; } +/* Whether the test is expected to fail. */ +#define WANT_SUCCESS 0 +#define XFAIL 1 + #define DO_NOTHING_TYPE_SCALAR(var_type) var_type #define DO_NOTHING_TYPE_STRING(var_type) void #define DO_NOTHING_TYPE_STRUCT(var_type) void @@ -58,34 +124,73 @@ static bool range_contains(char *haystack_start, size_t haystack_size, #define INIT_CLONE_STRING [FILL_SIZE_STRING] #define INIT_CLONE_STRUCT /**/ -#define INIT_SCALAR_none /**/ -#define INIT_SCALAR_zero = 0 +#define ZERO_CLONE_SCALAR(zero) memset(&(zero), 0x00, sizeof(zero)) +#define ZERO_CLONE_STRING(zero) memset(&(zero), 0x00, sizeof(zero)) +/* + * For the struct, intentionally poison padding to see if it gets + * copied out in direct assignments. + * */ +#define ZERO_CLONE_STRUCT(zero) \ + do { \ + memset(&(zero), 0xFF, sizeof(zero)); \ + zero.one = 0; \ + zero.two = 0; \ + zero.three = 0; \ + zero.four = 0; \ + } while (0) + +#define INIT_SCALAR_none(var_type) /**/ +#define INIT_SCALAR_zero(var_type) = 0 -#define INIT_STRING_none [FILL_SIZE_STRING] /**/ -#define INIT_STRING_zero [FILL_SIZE_STRING] = { } +#define INIT_STRING_none(var_type) [FILL_SIZE_STRING] /**/ +#define INIT_STRING_zero(var_type) [FILL_SIZE_STRING] = { } -#define INIT_STRUCT_none /**/ -#define INIT_STRUCT_zero = { } -#define INIT_STRUCT_static_partial = { .two = 0, } -#define INIT_STRUCT_static_all = { .one = arg->one, \ - .two = arg->two, \ - .three = arg->three, \ - .four = arg->four, \ +#define INIT_STRUCT_none(var_type) /**/ +#define INIT_STRUCT_zero(var_type) = { } + + +#define __static_partial { .two = 0, } +#define __static_all { .one = 0, \ + .two = 0, \ + .three = 0, \ + .four = 0, \ } -#define INIT_STRUCT_dynamic_partial = { .two = arg->two, } -#define INIT_STRUCT_dynamic_all = { .one = arg->one, \ - .two = arg->two, \ - .three = arg->three, \ - .four = arg->four, \ +#define __dynamic_partial { .two = arg->two, } +#define __dynamic_all { .one = arg->one, \ + .two = arg->two, \ + .three = arg->three, \ + .four = arg->four, \ } -#define INIT_STRUCT_runtime_partial ; \ - var.two = 0 -#define INIT_STRUCT_runtime_all ; \ - var.one = 0; \ +#define __runtime_partial var.two = 0 +#define __runtime_all var.one = 0; \ var.two = 0; \ var.three = 0; \ - memset(&var.four, 0, \ - sizeof(var.four)) + var.four = 0 + +#define INIT_STRUCT_static_partial(var_type) \ + = __static_partial +#define INIT_STRUCT_static_all(var_type) \ + = __static_all +#define INIT_STRUCT_dynamic_partial(var_type) \ + = __dynamic_partial +#define INIT_STRUCT_dynamic_all(var_type) \ + = __dynamic_all +#define INIT_STRUCT_runtime_partial(var_type) \ + ; __runtime_partial +#define INIT_STRUCT_runtime_all(var_type) \ + ; __runtime_all + +#define INIT_STRUCT_assigned_static_partial(var_type) \ + ; var = (var_type)__static_partial +#define INIT_STRUCT_assigned_static_all(var_type) \ + ; var = (var_type)__static_all +#define INIT_STRUCT_assigned_dynamic_partial(var_type) \ + ; var = (var_type)__dynamic_partial +#define INIT_STRUCT_assigned_dynamic_all(var_type) \ + ; var = (var_type)__dynamic_all + +#define INIT_STRUCT_assigned_copy(var_type) \ + ; var = *(arg) /* * @name: unique string name for the test @@ -106,7 +211,7 @@ static noinline __init int test_ ## name (void) \ BUILD_BUG_ON(sizeof(zero) > MAX_VAR_SIZE); \ \ /* Fill clone type with zero for per-field init. */ \ - memset(&zero, 0x00, sizeof(zero)); \ + ZERO_CLONE_ ## which(zero); \ /* Clear entire check buffer for 0xFF overlap test. */ \ memset(check_buf, 0x00, sizeof(check_buf)); \ /* Fill stack with 0xFF. */ \ @@ -149,7 +254,7 @@ static noinline __init int test_ ## name (void) \ return (xfail) ? 0 : 1; \ } \ } -#define DEFINE_TEST(name, var_type, which, init_level) \ +#define DEFINE_TEST(name, var_type, which, init_level, xfail) \ /* no-op to force compiler into ignoring "uninitialized" vars */\ static noinline __init DO_NOTHING_TYPE_ ## which(var_type) \ do_nothing_ ## name(var_type *ptr) \ @@ -165,7 +270,8 @@ static noinline __init int leaf_ ## name(unsigned long sp, \ var_type *arg) \ { \ char buf[VAR_BUFFER]; \ - var_type var INIT_ ## which ## _ ## init_level; \ + var_type var \ + INIT_ ## which ## _ ## init_level(var_type); \ \ target_start = &var; \ target_size = sizeof(var); \ @@ -191,7 +297,7 @@ static noinline __init int leaf_ ## name(unsigned long sp, \ \ return (int)buf[0] | (int)buf[sizeof(buf) - 1]; \ } \ -DEFINE_TEST_DRIVER(name, var_type, which, 0) +DEFINE_TEST_DRIVER(name, var_type, which, xfail) /* Structure with no padding. */ struct test_packed { @@ -210,18 +316,13 @@ struct test_small_hole { unsigned long four; }; -/* Try to trigger unhandled padding in a structure. */ -struct test_aligned { - u32 internal1; - u64 internal2; -} __aligned(64); - +/* Trigger unhandled padding in a structure. */ struct test_big_hole { u8 one; u8 two; u8 three; /* 61 byte padding hole here. */ - struct test_aligned four; + u8 four __aligned(64); } __aligned(64); struct test_trailing_hole { @@ -240,42 +341,50 @@ struct test_user { unsigned long four; }; -#define DEFINE_SCALAR_TEST(name, init) \ - DEFINE_TEST(name ## _ ## init, name, SCALAR, init) +#define DEFINE_SCALAR_TEST(name, init, xfail) \ + DEFINE_TEST(name ## _ ## init, name, SCALAR, \ + init, xfail) -#define DEFINE_SCALAR_TESTS(init) \ - DEFINE_SCALAR_TEST(u8, init); \ - DEFINE_SCALAR_TEST(u16, init); \ - DEFINE_SCALAR_TEST(u32, init); \ - DEFINE_SCALAR_TEST(u64, init); \ - DEFINE_TEST(char_array_ ## init, unsigned char, STRING, init) +#define DEFINE_SCALAR_TESTS(init, xfail) \ + DEFINE_SCALAR_TEST(u8, init, xfail); \ + DEFINE_SCALAR_TEST(u16, init, xfail); \ + DEFINE_SCALAR_TEST(u32, init, xfail); \ + DEFINE_SCALAR_TEST(u64, init, xfail); \ + DEFINE_TEST(char_array_ ## init, unsigned char, \ + STRING, init, xfail) -#define DEFINE_STRUCT_TEST(name, init) \ +#define DEFINE_STRUCT_TEST(name, init, xfail) \ DEFINE_TEST(name ## _ ## init, \ - struct test_ ## name, STRUCT, init) + struct test_ ## name, STRUCT, init, \ + xfail) + +#define DEFINE_STRUCT_TESTS(init, xfail) \ + DEFINE_STRUCT_TEST(small_hole, init, xfail); \ + DEFINE_STRUCT_TEST(big_hole, init, xfail); \ + DEFINE_STRUCT_TEST(trailing_hole, init, xfail); \ + DEFINE_STRUCT_TEST(packed, init, xfail) -#define DEFINE_STRUCT_TESTS(init) \ - DEFINE_STRUCT_TEST(small_hole, init); \ - DEFINE_STRUCT_TEST(big_hole, init); \ - DEFINE_STRUCT_TEST(trailing_hole, init); \ - DEFINE_STRUCT_TEST(packed, init) +#define DEFINE_STRUCT_INITIALIZER_TESTS(base) \ + DEFINE_STRUCT_TESTS(base ## _ ## partial, \ + WANT_SUCCESS); \ + DEFINE_STRUCT_TESTS(base ## _ ## all, \ + WANT_SUCCESS) /* These should be fully initialized all the time! */ -DEFINE_SCALAR_TESTS(zero); -DEFINE_STRUCT_TESTS(zero); -/* Static initialization: padding may be left uninitialized. */ -DEFINE_STRUCT_TESTS(static_partial); -DEFINE_STRUCT_TESTS(static_all); -/* Dynamic initialization: padding may be left uninitialized. */ -DEFINE_STRUCT_TESTS(dynamic_partial); -DEFINE_STRUCT_TESTS(dynamic_all); -/* Runtime initialization: padding may be left uninitialized. */ -DEFINE_STRUCT_TESTS(runtime_partial); -DEFINE_STRUCT_TESTS(runtime_all); +DEFINE_SCALAR_TESTS(zero, WANT_SUCCESS); +DEFINE_STRUCT_TESTS(zero, WANT_SUCCESS); +/* Struct initializers: padding may be left uninitialized. */ +DEFINE_STRUCT_INITIALIZER_TESTS(static); +DEFINE_STRUCT_INITIALIZER_TESTS(dynamic); +DEFINE_STRUCT_INITIALIZER_TESTS(runtime); +DEFINE_STRUCT_INITIALIZER_TESTS(assigned_static); +DEFINE_STRUCT_INITIALIZER_TESTS(assigned_dynamic); +DEFINE_STRUCT_TESTS(assigned_copy, XFAIL); /* No initialization without compiler instrumentation. */ -DEFINE_SCALAR_TESTS(none); -DEFINE_STRUCT_TESTS(none); -DEFINE_TEST(user, struct test_user, STRUCT, none); +DEFINE_SCALAR_TESTS(none, WANT_SUCCESS); +DEFINE_STRUCT_TESTS(none, WANT_SUCCESS); +/* Initialization of members with __user attribute. */ +DEFINE_TEST(user, struct test_user, STRUCT, none, WANT_SUCCESS); /* * Check two uses through a variable declaration outside either path, @@ -285,6 +394,10 @@ DEFINE_TEST(user, struct test_user, STRUCT, none); static int noinline __leaf_switch_none(int path, bool fill) { switch (path) { + /* + * This is intentionally unreachable. To silence the + * warning, build with -Wno-switch-unreachable + */ uint64_t var; case 1: @@ -334,8 +447,8 @@ static noinline __init int leaf_switch_2_none(unsigned long sp, bool fill, * non-code areas (i.e. in a switch statement before the first "case"). * https://bugs.llvm.org/show_bug.cgi?id=44916 */ -DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR, 1); -DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR, 1); +DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR, XFAIL); +DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR, XFAIL); static int __init test_stackinit_init(void) { @@ -361,12 +474,18 @@ static int __init test_stackinit_init(void) test_structs(zero); /* Padding here appears to be accidentally always initialized? */ test_structs(dynamic_partial); + test_structs(assigned_dynamic_partial); /* Padding initialization depends on compiler behaviors. */ test_structs(static_partial); test_structs(static_all); test_structs(dynamic_all); test_structs(runtime_partial); test_structs(runtime_all); + test_structs(assigned_static_partial); + test_structs(assigned_static_all); + test_structs(assigned_dynamic_all); + /* Everything fails this since it effectively performs a memcpy(). */ + test_structs(assigned_copy); /* STRUCTLEAK_BYREF_ALL should cover everything from here down. */ test_scalars(none); diff --git a/lib/test_string.c b/lib/test_string.c index 7b31f4a505bf..9dfd6f52de92 100644 --- a/lib/test_string.c +++ b/lib/test_string.c @@ -179,6 +179,10 @@ static __init int strnchr_selftest(void) return 0; } +static __exit void string_selftest_remove(void) +{ +} + static __init int string_selftest_init(void) { int test, subtest; @@ -216,4 +220,5 @@ fail: } module_init(string_selftest_init); +module_exit(string_selftest_remove); MODULE_LICENSE("GPL v2"); diff --git a/lib/test_vmalloc.c b/lib/test_vmalloc.c index 01e9543de566..e14993bc84d2 100644 --- a/lib/test_vmalloc.c +++ b/lib/test_vmalloc.c @@ -35,6 +35,9 @@ __param(int, test_repeat_count, 1, __param(int, test_loop_count, 1000000, "Set test loop counter"); +__param(int, nr_pages, 0, + "Set number of pages for fix_size_alloc_test(default: 1)"); + __param(int, run_test_mask, INT_MAX, "Set tests specified in the mask.\n\n" "\t\tid: 1, name: fix_size_alloc_test\n" @@ -262,7 +265,7 @@ static int fix_size_alloc_test(void) int i; for (i = 0; i < test_loop_count; i++) { - ptr = vmalloc(3 * PAGE_SIZE); + ptr = vmalloc((nr_pages > 0 ? nr_pages:1) * PAGE_SIZE); if (!ptr) return -1; diff --git a/lib/ubsan.c b/lib/ubsan.c index 26229973049d..bdc380ff5d5c 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -14,6 +14,7 @@ #include <linux/types.h> #include <linux/sched.h> #include <linux/uaccess.h> +#include <kunit/test-bug.h> #include "ubsan.h" @@ -141,6 +142,8 @@ static void ubsan_prologue(struct source_location *loc, const char *reason) "========================================\n"); pr_err("UBSAN: %s in %s:%d:%d\n", reason, loc->file_name, loc->line & LINE_MASK, loc->column & COLUMN_MASK); + + kunit_fail_current_test("%s in %s", reason, loc->file_name); } static void ubsan_epilogue(void) diff --git a/lib/vsprintf.c b/lib/vsprintf.c index e5c7afbf7405..f90f91d83920 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -17,7 +17,7 @@ * - scnprintf and vscnprintf */ -#include <stdarg.h> +#include <linux/stdarg.h> #include <linux/build_bug.h> #include <linux/clk.h> #include <linux/clk-provider.h> @@ -86,6 +86,7 @@ static unsigned long long simple_strntoull(const char *startp, size_t max_chars, * * This function has caveats. Please use kstrtoull instead. */ +noinline unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base) { return simple_strntoull(cp, INT_MAX, endp, base); @@ -407,8 +408,9 @@ int num_to_str(char *buf, int size, unsigned long long num, unsigned int width) #define SMALL 32 /* use lowercase in hex (must be 32 == 0x20) */ #define SPECIAL 64 /* prefix hex with "0x", octal with "0" */ +static_assert(SIGN == 1); static_assert(ZEROPAD == ('0' - ' ')); -static_assert(SMALL == ' '); +static_assert(SMALL == ('a' ^ 'A')); enum format_type { FORMAT_TYPE_NONE, /* Just a string part */ @@ -992,8 +994,12 @@ char *symbol_string(char *buf, char *end, void *ptr, value = (unsigned long)ptr; #ifdef CONFIG_KALLSYMS - if (*fmt == 'B') + if (*fmt == 'B' && fmt[1] == 'b') + sprint_backtrace_build_id(sym, value); + else if (*fmt == 'B') sprint_backtrace(sym, value); + else if (*fmt == 'S' && (fmt[1] == 'b' || (fmt[1] == 'R' && fmt[2] == 'b'))) + sprint_symbol_build_id(sym, value); else if (*fmt != 's') sprint_symbol(sym, value); else @@ -2014,10 +2020,15 @@ static const struct page_flags_fields pff[] = { static char *format_page_flags(char *buf, char *end, unsigned long flags) { - unsigned long main_flags = flags & (BIT(NR_PAGEFLAGS) - 1); + unsigned long main_flags = flags & PAGEFLAGS_MASK; bool append = false; int i; + buf = number(buf, end, flags, default_flag_spec); + if (buf < end) + *buf = '('; + buf++; + /* Page flags from the main area. */ if (main_flags) { buf = format_flags(buf, end, main_flags, pageflag_names); @@ -2046,6 +2057,9 @@ char *format_page_flags(char *buf, char *end, unsigned long flags) append = true; } + if (buf < end) + *buf = ')'; + buf++; return buf; } @@ -2262,9 +2276,11 @@ early_param("no_hash_pointers", no_hash_pointers_enable); * - 'S' For symbolic direct pointers (or function descriptors) with offset * - 's' For symbolic direct pointers (or function descriptors) without offset * - '[Ss]R' as above with __builtin_extract_return_addr() translation + * - 'S[R]b' as above with module build ID (for use in backtraces) * - '[Ff]' %pf and %pF were obsoleted and later removed in favor of * %ps and %pS. Be careful when re-using these specifiers. * - 'B' For backtraced symbolic direct pointers with offset + * - 'Bb' as above with module build ID (for use in backtraces) * - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref] * - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201] * - 'b[l]' For a bitmap, the number of bits is determined by the field @@ -3416,7 +3432,7 @@ int vsscanf(const char *buf, const char *fmt, va_list args) while (*fmt) { /* skip any white space in format */ - /* white space in format matchs any amount of + /* white space in format matches any amount of * white space, including none, in the input. */ if (isspace(*fmt)) { diff --git a/lib/xz/Kconfig b/lib/xz/Kconfig index 5cb50245a878..adce22ac18d6 100644 --- a/lib/xz/Kconfig +++ b/lib/xz/Kconfig @@ -39,6 +39,19 @@ config XZ_DEC_SPARC default y select XZ_DEC_BCJ +config XZ_DEC_MICROLZMA + bool "MicroLZMA decoder" + default n + help + MicroLZMA is a header format variant where the first byte + of a raw LZMA stream (without the end of stream marker) has + been replaced with a bitwise-negation of the lc/lp/pb + properties byte. MicroLZMA was created to be used in EROFS + but can be used by other things too where wasting minimal + amount of space for headers is important. + + Unless you know that you need this, say N. + endif config XZ_DEC_BCJ diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index 72ddac6ef2ec..ef449e97d1a1 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -422,7 +422,7 @@ XZ_EXTERN enum xz_ret xz_dec_bcj_run(struct xz_dec_bcj *s, /* * Flush pending already filtered data to the output buffer. Return - * immediatelly if we couldn't flush everything, or if the next + * immediately if we couldn't flush everything, or if the next * filter in the chain had already returned XZ_STREAM_END. */ if (s->temp.filtered > 0) { diff --git a/lib/xz/xz_dec_lzma2.c b/lib/xz/xz_dec_lzma2.c index ca2603abee08..27ce34520e78 100644 --- a/lib/xz/xz_dec_lzma2.c +++ b/lib/xz/xz_dec_lzma2.c @@ -147,8 +147,8 @@ struct lzma_dec { /* * LZMA properties or related bit masks (number of literal - * context bits, a mask dervied from the number of literal - * position bits, and a mask dervied from the number + * context bits, a mask derived from the number of literal + * position bits, and a mask derived from the number * position bits) */ uint32_t lc; @@ -248,6 +248,10 @@ struct lzma2_dec { * before the first LZMA chunk. */ bool need_props; + +#ifdef XZ_DEC_MICROLZMA + bool pedantic_microlzma; +#endif }; struct xz_dec_lzma2 { @@ -387,7 +391,14 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, *left -= copy_size; - memcpy(dict->buf + dict->pos, b->in + b->in_pos, copy_size); + /* + * If doing in-place decompression in single-call mode and the + * uncompressed size of the file is larger than the caller + * thought (i.e. it is invalid input!), the buffers below may + * overlap and cause undefined behavior with memcpy(). + * With valid inputs memcpy() would be fine here. + */ + memmove(dict->buf + dict->pos, b->in + b->in_pos, copy_size); dict->pos += copy_size; if (dict->full < dict->pos) @@ -397,7 +408,11 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, if (dict->pos == dict->end) dict->pos = 0; - memcpy(b->out + b->out_pos, b->in + b->in_pos, + /* + * Like above but for multi-call mode: use memmove() + * to avoid undefined behavior with invalid input. + */ + memmove(b->out + b->out_pos, b->in + b->in_pos, copy_size); } @@ -408,6 +423,12 @@ static void dict_uncompressed(struct dictionary *dict, struct xz_buf *b, } } +#ifdef XZ_DEC_MICROLZMA +# define DICT_FLUSH_SUPPORTS_SKIPPING true +#else +# define DICT_FLUSH_SUPPORTS_SKIPPING false +#endif + /* * Flush pending data from dictionary to b->out. It is assumed that there is * enough space in b->out. This is guaranteed because caller uses dict_limit() @@ -421,8 +442,19 @@ static uint32_t dict_flush(struct dictionary *dict, struct xz_buf *b) if (dict->pos == dict->end) dict->pos = 0; - memcpy(b->out + b->out_pos, dict->buf + dict->start, - copy_size); + /* + * These buffers cannot overlap even if doing in-place + * decompression because in multi-call mode dict->buf + * has been allocated by us in this file; it's not + * provided by the caller like in single-call mode. + * + * With MicroLZMA, b->out can be NULL to skip bytes that + * the caller doesn't need. This cannot be done with XZ + * because it would break BCJ filters. + */ + if (!DICT_FLUSH_SUPPORTS_SKIPPING || b->out != NULL) + memcpy(b->out + b->out_pos, dict->buf + dict->start, + copy_size); } dict->start = dict->pos; @@ -484,11 +516,11 @@ static __always_inline void rc_normalize(struct rc_dec *rc) } /* - * Decode one bit. In some versions, this function has been splitted in three + * Decode one bit. In some versions, this function has been split in three * functions so that the compiler is supposed to be able to more easily avoid * an extra branch. In this particular version of the LZMA decoder, this * doesn't seem to be a good idea (tested with GCC 3.3.6, 3.4.6, and 4.3.3 - * on x86). Using a non-splitted version results in nicer looking code too. + * on x86). Using a non-split version results in nicer looking code too. * * NOTE: This must return an int. Do not make it return a bool or the speed * of the code generated by GCC 3.x decreases 10-15 %. (GCC 4.3 doesn't care, @@ -761,7 +793,7 @@ static bool lzma_main(struct xz_dec_lzma2 *s) } /* - * Reset the LZMA decoder and range decoder state. Dictionary is nore reset + * Reset the LZMA decoder and range decoder state. Dictionary is not reset * here, because LZMA state may be reset without resetting the dictionary. */ static void lzma_reset(struct xz_dec_lzma2 *s) @@ -774,6 +806,7 @@ static void lzma_reset(struct xz_dec_lzma2 *s) s->lzma.rep1 = 0; s->lzma.rep2 = 0; s->lzma.rep3 = 0; + s->lzma.len = 0; /* * All probabilities are initialized to the same value. This hack @@ -1157,8 +1190,6 @@ XZ_EXTERN enum xz_ret xz_dec_lzma2_reset(struct xz_dec_lzma2 *s, uint8_t props) } } - s->lzma.len = 0; - s->lzma2.sequence = SEQ_CONTROL; s->lzma2.need_dict_reset = true; @@ -1174,3 +1205,140 @@ XZ_EXTERN void xz_dec_lzma2_end(struct xz_dec_lzma2 *s) kfree(s); } + +#ifdef XZ_DEC_MICROLZMA +/* This is a wrapper struct to have a nice struct name in the public API. */ +struct xz_dec_microlzma { + struct xz_dec_lzma2 s; +}; + +enum xz_ret xz_dec_microlzma_run(struct xz_dec_microlzma *s_ptr, + struct xz_buf *b) +{ + struct xz_dec_lzma2 *s = &s_ptr->s; + + /* + * sequence is SEQ_PROPERTIES before the first input byte, + * SEQ_LZMA_PREPARE until a total of five bytes have been read, + * and SEQ_LZMA_RUN for the rest of the input stream. + */ + if (s->lzma2.sequence != SEQ_LZMA_RUN) { + if (s->lzma2.sequence == SEQ_PROPERTIES) { + /* One byte is needed for the props. */ + if (b->in_pos >= b->in_size) + return XZ_OK; + + /* + * Don't increment b->in_pos here. The same byte is + * also passed to rc_read_init() which will ignore it. + */ + if (!lzma_props(s, ~b->in[b->in_pos])) + return XZ_DATA_ERROR; + + s->lzma2.sequence = SEQ_LZMA_PREPARE; + } + + /* + * xz_dec_microlzma_reset() doesn't validate the compressed + * size so we do it here. We have to limit the maximum size + * to avoid integer overflows in lzma2_lzma(). 3 GiB is a nice + * round number and much more than users of this code should + * ever need. + */ + if (s->lzma2.compressed < RC_INIT_BYTES + || s->lzma2.compressed > (3U << 30)) + return XZ_DATA_ERROR; + + if (!rc_read_init(&s->rc, b)) + return XZ_OK; + + s->lzma2.compressed -= RC_INIT_BYTES; + s->lzma2.sequence = SEQ_LZMA_RUN; + + dict_reset(&s->dict, b); + } + + /* This is to allow increasing b->out_size between calls. */ + if (DEC_IS_SINGLE(s->dict.mode)) + s->dict.end = b->out_size - b->out_pos; + + while (true) { + dict_limit(&s->dict, min_t(size_t, b->out_size - b->out_pos, + s->lzma2.uncompressed)); + + if (!lzma2_lzma(s, b)) + return XZ_DATA_ERROR; + + s->lzma2.uncompressed -= dict_flush(&s->dict, b); + + if (s->lzma2.uncompressed == 0) { + if (s->lzma2.pedantic_microlzma) { + if (s->lzma2.compressed > 0 || s->lzma.len > 0 + || !rc_is_finished(&s->rc)) + return XZ_DATA_ERROR; + } + + return XZ_STREAM_END; + } + + if (b->out_pos == b->out_size) + return XZ_OK; + + if (b->in_pos == b->in_size + && s->temp.size < s->lzma2.compressed) + return XZ_OK; + } +} + +struct xz_dec_microlzma *xz_dec_microlzma_alloc(enum xz_mode mode, + uint32_t dict_size) +{ + struct xz_dec_microlzma *s; + + /* Restrict dict_size to the same range as in the LZMA2 code. */ + if (dict_size < 4096 || dict_size > (3U << 30)) + return NULL; + + s = kmalloc(sizeof(*s), GFP_KERNEL); + if (s == NULL) + return NULL; + + s->s.dict.mode = mode; + s->s.dict.size = dict_size; + + if (DEC_IS_MULTI(mode)) { + s->s.dict.end = dict_size; + + s->s.dict.buf = vmalloc(dict_size); + if (s->s.dict.buf == NULL) { + kfree(s); + return NULL; + } + } + + return s; +} + +void xz_dec_microlzma_reset(struct xz_dec_microlzma *s, uint32_t comp_size, + uint32_t uncomp_size, int uncomp_size_is_exact) +{ + /* + * comp_size is validated in xz_dec_microlzma_run(). + * uncomp_size can safely be anything. + */ + s->s.lzma2.compressed = comp_size; + s->s.lzma2.uncompressed = uncomp_size; + s->s.lzma2.pedantic_microlzma = uncomp_size_is_exact; + + s->s.lzma2.sequence = SEQ_PROPERTIES; + s->s.temp.size = 0; +} + +void xz_dec_microlzma_end(struct xz_dec_microlzma *s) +{ + if (DEC_IS_MULTI(s->s.dict.mode)) + vfree(s->s.dict.buf); + + kfree(s); +} +#endif diff --git a/lib/xz/xz_dec_stream.c b/lib/xz/xz_dec_stream.c index fea86deaaa01..683570b93a8c 100644 --- a/lib/xz/xz_dec_stream.c +++ b/lib/xz/xz_dec_stream.c @@ -402,12 +402,12 @@ static enum xz_ret dec_stream_header(struct xz_dec *s) * we will accept other check types too, but then the check won't * be verified and a warning (XZ_UNSUPPORTED_CHECK) will be given. */ + if (s->temp.buf[HEADER_MAGIC_SIZE + 1] > XZ_CHECK_MAX) + return XZ_OPTIONS_ERROR; + s->check_type = s->temp.buf[HEADER_MAGIC_SIZE + 1]; #ifdef XZ_DEC_ANY_CHECK - if (s->check_type > XZ_CHECK_MAX) - return XZ_OPTIONS_ERROR; - if (s->check_type > XZ_CHECK_CRC32) return XZ_UNSUPPORTED_CHECK; #else diff --git a/lib/xz/xz_dec_syms.c b/lib/xz/xz_dec_syms.c index 32eb3c03aede..61098c67a413 100644 --- a/lib/xz/xz_dec_syms.c +++ b/lib/xz/xz_dec_syms.c @@ -15,8 +15,15 @@ EXPORT_SYMBOL(xz_dec_reset); EXPORT_SYMBOL(xz_dec_run); EXPORT_SYMBOL(xz_dec_end); +#ifdef CONFIG_XZ_DEC_MICROLZMA +EXPORT_SYMBOL(xz_dec_microlzma_alloc); +EXPORT_SYMBOL(xz_dec_microlzma_reset); +EXPORT_SYMBOL(xz_dec_microlzma_run); +EXPORT_SYMBOL(xz_dec_microlzma_end); +#endif + MODULE_DESCRIPTION("XZ decompressor"); -MODULE_VERSION("1.0"); +MODULE_VERSION("1.1"); MODULE_AUTHOR("Lasse Collin <lasse.collin@tukaani.org> and Igor Pavlov"); /* diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h index 09360ebb510e..bf1e94ec7873 100644 --- a/lib/xz/xz_private.h +++ b/lib/xz/xz_private.h @@ -37,6 +37,9 @@ # ifdef CONFIG_XZ_DEC_SPARC # define XZ_DEC_SPARC # endif +# ifdef CONFIG_XZ_DEC_MICROLZMA +# define XZ_DEC_MICROLZMA +# endif # define memeq(a, b, size) (memcmp(a, b, size) == 0) # define memzero(buf, size) memset(buf, 0, size) # endif diff --git a/lib/zlib_inflate/inffast.c b/lib/zlib_inflate/inffast.c index ed1f3df27260..2843f9bb42ac 100644 --- a/lib/zlib_inflate/inffast.c +++ b/lib/zlib_inflate/inffast.c @@ -15,7 +15,7 @@ union uu { unsigned char b[2]; }; -/* Endian independed version */ +/* Endian independent version */ static inline unsigned short get_unaligned16(const unsigned short *p) { @@ -253,13 +253,12 @@ void inflate_fast(z_streamp strm, unsigned start) sfrom = (unsigned short *)(from); loops = len >> 1; - do -#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS - *sout++ = *sfrom++; -#else - *sout++ = get_unaligned16(sfrom++); -#endif - while (--loops); + do { + if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) + *sout++ = *sfrom++; + else + *sout++ = get_unaligned16(sfrom++); + } while (--loops); out = (unsigned char *)sout; from = (unsigned char *)sfrom; } else { /* dist == 1 or dist == 2 */ diff --git a/lib/zstd/huf.h b/lib/zstd/huf.h index 2143da28d952..923218d12e28 100644 --- a/lib/zstd/huf.h +++ b/lib/zstd/huf.h @@ -134,7 +134,7 @@ typedef enum { HUF_repeat_none, /**< Cannot use the previous table */ HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ - HUF_repeat_valid /**< Can use the previous table and it is asumed to be valid */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. |