diff options
Diffstat (limited to 'lib')
89 files changed, 3625 insertions, 1806 deletions
diff --git a/lib/.gitignore b/lib/.gitignore index e5e217b8307b..54596b634ecb 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only /crc32table.h /crc64table.h +/default.bconf /gen_crc32table /gen_crc64table /oid_registry_data.c diff --git a/lib/Kconfig b/lib/Kconfig index 9b5a692ce00c..eaaad4d85bf2 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -120,6 +120,9 @@ config INDIRECT_IOMEM_FALLBACK source "lib/crypto/Kconfig" +config LIB_MEMNEQ + bool + config CRC_CCITT tristate "CRC-CCITT functions" help @@ -145,6 +148,15 @@ config CRC_T10DIF kernel tree needs to calculate CRC checks for use with the SCSI data integrity subsystem. +config CRC64_ROCKSOFT + tristate "CRC calculation for the Rocksoft model CRC64" + select CRC64 + select CRYPTO + select CRYPTO_CRC64_ROCKSOFT + help + This option provides a CRC64 API to a registered crypto driver. + This is used with the block layer's data integrity subsystem. + config CRC_ITU_T tristate "CRC ITU-T V.41 functions" help @@ -728,3 +740,6 @@ config PLDMFW config ASN1_ENCODER tristate + +config POLYNOMIAL + tristate diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 14b89aa37c5c..2e24db4bff19 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -208,20 +208,87 @@ config DEBUG_BUGVERBOSE endmenu # "printk and dmesg options" +config DEBUG_KERNEL + bool "Kernel debugging" + help + Say Y here if you are developing drivers or trying to debug and + identify kernel problems. + +config DEBUG_MISC + bool "Miscellaneous debug code" + default DEBUG_KERNEL + depends on DEBUG_KERNEL + help + Say Y here if you need to enable miscellaneous debug code that should + be under a more specific debug option but isn't. + menu "Compile-time checks and compiler options" config DEBUG_INFO - bool "Compile the kernel with debug info" - depends on DEBUG_KERNEL && !COMPILE_TEST + bool help - If you say Y here the resulting kernel image will include - debugging info resulting in a larger kernel image. + A kernel debug info option other than "None" has been selected + in the "Debug information" choice below, indicating that debug + information will be generated for build targets. + +choice + prompt "Debug information" + depends on DEBUG_KERNEL + help + Selecting something other than "None" results in a kernel image + that will include debugging info resulting in a larger kernel image. This adds debug symbols to the kernel and modules (gcc -g), and is needed if you intend to use kernel crashdump or binary object tools like crash, kgdb, LKCD, gdb, etc on the kernel. - Say Y here only if you plan to debug the kernel. - If unsure, say N. + Choose which version of DWARF debug info to emit. If unsure, + select "Toolchain default". + +config DEBUG_INFO_NONE + bool "Disable debug information" + help + Do not build the kernel with debugging information, which will + result in a faster and smaller build. + +config DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT + bool "Rely on the toolchain's implicit default DWARF version" + select DEBUG_INFO + help + The implicit default version of DWARF debug info produced by a + toolchain changes over time. + + This can break consumers of the debug info that haven't upgraded to + support newer revisions, and prevent testing newer versions, but + those should be less common scenarios. + +config DEBUG_INFO_DWARF4 + bool "Generate DWARF Version 4 debuginfo" + select DEBUG_INFO + help + Generate DWARF v4 debug info. This requires gcc 4.5+ and gdb 7.0+. + + If you have consumers of DWARF debug info that are not ready for + newer revisions of DWARF, you may wish to choose this or have your + config select this. + +config DEBUG_INFO_DWARF5 + bool "Generate DWARF Version 5 debuginfo" + select DEBUG_INFO + depends on !CC_IS_CLANG || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502))) + help + Generate DWARF v5 debug info. Requires binutils 2.35.2, gcc 5.0+ (gcc + 5.0+ accepts the -gdwarf-5 flag but only had partial support for some + draft features until 7.0), and gdb 8.0+. + + Changes to the structure of debug info in Version 5 allow for around + 15-18% savings in resulting image and debug info section sizes as + compared to DWARF Version 4. DWARF Version 5 standardizes previous + extensions such as accelerators for symbol indexing and the format + for fission (.dwo/.dwp) files. Users may not want to select this + config if they rely on tooling that has not yet been updated to + support DWARF Version 5. + +endchoice # "Debug information" if DEBUG_INFO @@ -267,63 +334,27 @@ config DEBUG_INFO_SPLIT to know about the .dwo files and include them. Incompatible with older versions of ccache. -choice - prompt "DWARF version" - help - Which version of DWARF debug info to emit. - -config DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT - bool "Rely on the toolchain's implicit default DWARF version" - help - The implicit default version of DWARF debug info produced by a - toolchain changes over time. - - This can break consumers of the debug info that haven't upgraded to - support newer revisions, and prevent testing newer versions, but - those should be less common scenarios. - - If unsure, say Y. - -config DEBUG_INFO_DWARF4 - bool "Generate DWARF Version 4 debuginfo" - help - Generate DWARF v4 debug info. This requires gcc 4.5+ and gdb 7.0+. - - If you have consumers of DWARF debug info that are not ready for - newer revisions of DWARF, you may wish to choose this or have your - config select this. - -config DEBUG_INFO_DWARF5 - bool "Generate DWARF Version 5 debuginfo" - depends on !CC_IS_CLANG || (CC_IS_CLANG && (AS_IS_LLVM || (AS_IS_GNU && AS_VERSION >= 23502))) - depends on !DEBUG_INFO_BTF - help - Generate DWARF v5 debug info. Requires binutils 2.35.2, gcc 5.0+ (gcc - 5.0+ accepts the -gdwarf-5 flag but only had partial support for some - draft features until 7.0), and gdb 8.0+. - - Changes to the structure of debug info in Version 5 allow for around - 15-18% savings in resulting image and debug info section sizes as - compared to DWARF Version 4. DWARF Version 5 standardizes previous - extensions such as accelerators for symbol indexing and the format - for fission (.dwo/.dwp) files. Users may not want to select this - config if they rely on tooling that has not yet been updated to - support DWARF Version 5. - -endchoice # "DWARF version" - config DEBUG_INFO_BTF bool "Generate BTF typeinfo" depends on !DEBUG_INFO_SPLIT && !DEBUG_INFO_REDUCED depends on !GCC_PLUGIN_RANDSTRUCT || COMPILE_TEST depends on BPF_SYSCALL + depends on !DEBUG_INFO_DWARF5 || PAHOLE_VERSION >= 121 help Generate deduplicated BTF type information from DWARF debug info. Turning this on expects presence of pahole tool, which will convert DWARF type info into equivalent deduplicated BTF type info. config PAHOLE_HAS_SPLIT_BTF - def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "119") + def_bool PAHOLE_VERSION >= 119 + +config PAHOLE_HAS_BTF_TAG + def_bool PAHOLE_VERSION >= 123 + depends on CC_IS_CLANG + help + Decide whether pahole emits btf_tag attributes (btf_type_tag and + btf_decl_tag) or not. Currently only clang compiler implements + these attributes, so make the config depend on CC_IS_CLANG. config DEBUG_INFO_BTF_MODULES def_bool y @@ -331,6 +362,16 @@ config DEBUG_INFO_BTF_MODULES help Generate compact split BTF type information for kernel modules. +config MODULE_ALLOW_BTF_MISMATCH + bool "Allow loading modules with non-matching BTF type info" + depends on DEBUG_INFO_BTF_MODULES + help + For modules whose split BTF does not match vmlinux, load without + BTF rather than refusing to load. The default behavior with + module BTF enabled is to reject modules with such mismatches; + this option will still load module BTF where possible but ignore + it when a mismatch is found. + config GDB_SCRIPTS bool "Provide GDB scripts for kernel debugging" help @@ -416,7 +457,8 @@ config SECTION_MISMATCH_WARN_ONLY If unsure, say Y. config DEBUG_FORCE_FUNCTION_ALIGN_64B - bool "Force all function address 64B aligned" if EXPERT + bool "Force all function address 64B aligned" + depends on EXPERT && (X86_64 || ARM64 || PPC32 || PPC64 || ARC) help There are cases that a commit from one domain changes the function address alignment of other domains, and cause magic performance @@ -443,24 +485,25 @@ config FRAME_POINTER larger and slower, but it gives very useful debugging information in case of kernel bugs. (precise oopses/stacktraces/warnings) +config OBJTOOL + bool + config STACK_VALIDATION bool "Compile-time stack metadata validation" - depends on HAVE_STACK_VALIDATION + depends on HAVE_STACK_VALIDATION && UNWINDER_FRAME_POINTER + select OBJTOOL default n help - Add compile-time checks to validate stack metadata, including frame - pointers (if CONFIG_FRAME_POINTER is enabled). This helps ensure - that runtime stack traces are more reliable. - - This is also a prerequisite for generation of ORC unwind data, which - is needed for CONFIG_UNWINDER_ORC. + Validate frame pointer rules at compile-time. This helps ensure that + runtime stack traces are more reliable. For more information, see tools/objtool/Documentation/stack-validation.txt. -config VMLINUX_VALIDATION +config NOINSTR_VALIDATION bool - depends on STACK_VALIDATION && DEBUG_ENTRY + depends on HAVE_NOINSTR_VALIDATION && DEBUG_ENTRY + select OBJTOOL default y config VMLINUX_MAP @@ -585,20 +628,6 @@ source "lib/Kconfig.kcsan" endmenu -config DEBUG_KERNEL - bool "Kernel debugging" - help - Say Y here if you are developing drivers or trying to debug and - identify kernel problems. - -config DEBUG_MISC - bool "Miscellaneous debug code" - default DEBUG_KERNEL - depends on DEBUG_KERNEL - help - Say Y here if you need to enable miscellaneous debug code that should - be under a more specific debug option but isn't. - menu "Networking Debugging" source "net/Kconfig.debug" @@ -670,40 +699,6 @@ config DEBUG_OBJECTS_ENABLE_DEFAULT help Debug objects boot parameter default value -config DEBUG_SLAB - bool "Debug slab memory allocations" - depends on DEBUG_KERNEL && SLAB - help - Say Y here to have the kernel do limited verification on memory - allocation as well as poisoning memory on free to catch use of freed - memory. This can make kmalloc/kfree-intensive workloads much slower. - -config SLUB_DEBUG_ON - bool "SLUB debugging on by default" - depends on SLUB && SLUB_DEBUG - default n - help - Boot with debugging on by default. SLUB boots by default with - the runtime debug capabilities switched off. Enabling this is - equivalent to specifying the "slub_debug" parameter on boot. - There is no support for more fine grained debug control like - possible with slub_debug=xxx. SLUB debugging may be switched - off in a kernel built with CONFIG_SLUB_DEBUG_ON by specifying - "slub_debug=-". - -config SLUB_STATS - default n - bool "Enable SLUB performance statistics" - depends on SLUB && SYSFS - help - SLUB statistics are useful to debug SLUBs allocation behavior in - order find ways to optimize the allocator. This should never be - enabled for production use since keeping statistics slows down - the allocator by a few percentage points. The slabinfo command - supports the determination of the most active slabs to figure - out which slabs are relevant to a particular load. - Try running: slabinfo -DA - config HAVE_DEBUG_KMEMLEAK bool @@ -1043,13 +1038,6 @@ config BOOTPARAM_SOFTLOCKUP_PANIC Say N if unsure. -config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE - int - depends on SOFTLOCKUP_DETECTOR - range 0 1 - default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC - default 1 if BOOTPARAM_SOFTLOCKUP_PANIC - config HARDLOCKUP_DETECTOR_PERF bool select SOFTLOCKUP_DETECTOR @@ -1091,13 +1079,6 @@ config BOOTPARAM_HARDLOCKUP_PANIC Say N if unsure. -config BOOTPARAM_HARDLOCKUP_PANIC_VALUE - int - depends on HARDLOCKUP_DETECTOR - range 0 1 - default 0 if !BOOTPARAM_HARDLOCKUP_PANIC - default 1 if BOOTPARAM_HARDLOCKUP_PANIC - config DETECT_HUNG_TASK bool "Detect Hung Tasks" depends on DEBUG_KERNEL @@ -1145,13 +1126,6 @@ config BOOTPARAM_HUNG_TASK_PANIC Say N if unsure. -config BOOTPARAM_HUNG_TASK_PANIC_VALUE - int - depends on DETECT_HUNG_TASK - range 0 1 - default 0 if !BOOTPARAM_HUNG_TASK_PANIC - default 1 if BOOTPARAM_HUNG_TASK_PANIC - config WQ_WATCHDOG bool "Detect Workqueue Stalls" depends on DEBUG_KERNEL @@ -1565,8 +1539,7 @@ config WARN_ALL_UNSEEDED_RANDOM so architecture maintainers really need to do what they can to get the CRNG seeded sooner after the system is booted. However, since users cannot do anything actionable to - address this, by default the kernel will issue only a single - warning for the first use of unseeded randomness. + address this, by default this option is disabled. Say Y here if you want to receive warnings for all uses of unseeded randomness. This will be of use primarily for @@ -1984,10 +1957,11 @@ config KCOV bool "Code coverage for fuzzing" depends on ARCH_HAS_KCOV depends on CC_HAS_SANCOV_TRACE_PC || GCC_PLUGINS - depends on !ARCH_WANTS_NO_INSTR || STACK_VALIDATION || \ + depends on !ARCH_WANTS_NO_INSTR || HAVE_NOINSTR_HACK || \ GCC_VERSION >= 120000 || CLANG_VERSION >= 130000 select DEBUG_FS select GCC_PLUGIN_SANCOV if !CC_HAS_SANCOV_TRACE_PC + select OBJTOOL if HAVE_NOINSTR_HACK help KCOV exposes kernel code coverage information in a form suitable for coverage-guided fuzzing (randomized testing). @@ -2089,10 +2063,11 @@ config TEST_DIV64 If unsure, say N. config KPROBES_SANITY_TEST - tristate "Kprobes sanity tests" + tristate "Kprobes sanity tests" if !KUNIT_ALL_TESTS depends on DEBUG_KERNEL depends on KPROBES depends on KUNIT + default KUNIT_ALL_TESTS help This option provides for testing basic kprobes functionality on boot. Samples of kprobe and kretprobe are inserted and @@ -2100,6 +2075,18 @@ config KPROBES_SANITY_TEST Say N if you are unsure. +config FPROBE_SANITY_TEST + bool "Self test for fprobe" + depends on DEBUG_KERNEL + depends on FPROBE + depends on KUNIT=y + help + This option will enable testing the fprobe when the system boot. + A series of tests are made to verify that the fprobe is functioning + properly. + + Say N if you are unsure. + config BACKTRACE_SELF_TEST tristate "Self test for the backtrace code" depends on DEBUG_KERNEL @@ -2214,9 +2201,6 @@ config TEST_UUID config TEST_XARRAY tristate "Test the XArray code at runtime" -config TEST_OVERFLOW - tristate "Test check_*_overflow() functions at runtime" - config TEST_RHASHTABLE tristate "Perform selftest on resizable hash table" help @@ -2357,8 +2341,9 @@ config TEST_SYSCTL If unsure, say N. config BITFIELD_KUNIT - tristate "KUnit test bitfield functions at runtime" + tristate "KUnit test bitfield functions at runtime" if !KUNIT_ALL_TESTS depends on KUNIT + default KUNIT_ALL_TESTS help Enable this option to test the bitfield functions at boot. @@ -2392,8 +2377,9 @@ config HASH_KUNIT_TEST optimized versions. If unsure, say N. config RESOURCE_KUNIT_TEST - tristate "KUnit test for resource API" + tristate "KUnit test for resource API" if !KUNIT_ALL_TESTS depends on KUNIT + default KUNIT_ALL_TESTS help This builds the resource API unit test. Tests the logic of API provided by resource.c and ioport.h. @@ -2446,8 +2432,9 @@ config LINEAR_RANGES_TEST If unsure, say N. config CMDLINE_KUNIT_TEST - tristate "KUnit test for cmdline API" + tristate "KUnit test for cmdline API" if !KUNIT_ALL_TESTS depends on KUNIT + default KUNIT_ALL_TESTS help This builds the cmdline API unit test. Tests the logic of API provided by cmdline.c. @@ -2457,8 +2444,9 @@ config CMDLINE_KUNIT_TEST If unsure, say N. config BITS_TEST - tristate "KUnit test for bits.h" + tristate "KUnit test for bits.h" if !KUNIT_ALL_TESTS depends on KUNIT + default KUNIT_ALL_TESTS help This builds the bits unit test. Tests the logic of macros defined in bits.h. @@ -2501,6 +2489,30 @@ config MEMCPY_KUNIT_TEST If unsure, say N. +config OVERFLOW_KUNIT_TEST + tristate "Test check_*_overflow() functions at runtime" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Builds unit tests for the check_*_overflow(), size_*(), allocation, and + related functions. + + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + +config STACKINIT_KUNIT_TEST + tristate "Test level of stack variable initialization" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + Test if the kernel is zero-initializing stack variables and + padding. Coverage is controlled by compiler flags, + CONFIG_INIT_STACK_ALL_PATTERN, CONFIG_INIT_STACK_ALL_ZERO, + CONFIG_GCC_PLUGIN_STRUCTLEAK, CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF, + or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL. + config TEST_UDELAY tristate "udelay test driver" help @@ -2592,17 +2604,6 @@ config TEST_OBJAGG Enable this option to test object aggregation manager on boot (or module load). - -config TEST_STACKINIT - tristate "Test level of stack variable initialization" - help - Test if the kernel is zero-initializing stack variables and - padding. Coverage is controlled by compiler flags, - CONFIG_GCC_PLUGIN_STRUCTLEAK, CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF, - or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL. - - If unsure, say N. - config TEST_MEMINIT tristate "Test heap/page initialization" help diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 879757b6dd14..f0973da583e0 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only + # This config refers to the generic KASAN mode. config HAVE_ARCH_KASAN bool @@ -15,9 +16,8 @@ config HAVE_ARCH_KASAN_VMALLOC config ARCH_DISABLE_KASAN_INLINE bool help - An architecture might not support inline instrumentation. - When this option is selected, inline and stack instrumentation are - disabled. + Disables both inline and stack instrumentation. Selected by + architectures that do not support these instrumentation types. config CC_HAS_KASAN_GENERIC def_bool $(cc-option, -fsanitize=kernel-address) @@ -26,13 +26,13 @@ config CC_HAS_KASAN_SW_TAGS def_bool $(cc-option, -fsanitize=kernel-hwaddress) # This option is only required for software KASAN modes. -# Old GCC versions don't have proper support for no_sanitize_address. +# Old GCC versions do not have proper support for no_sanitize_address. # See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89124 for details. config CC_HAS_WORKING_NOSANITIZE_ADDRESS def_bool !CC_IS_GCC || GCC_VERSION >= 80300 menuconfig KASAN - bool "KASAN: runtime memory debugger" + bool "KASAN: dynamic memory safety error detector" depends on (((HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC) || \ (HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS)) && \ CC_HAS_WORKING_NOSANITIZE_ADDRESS) || \ @@ -40,10 +40,13 @@ menuconfig KASAN depends on (SLUB && SYSFS) || (SLAB && !DEBUG_SLAB) select STACKDEPOT_ALWAYS_INIT help - Enables KASAN (KernelAddressSANitizer) - runtime memory debugger, - designed to find out-of-bounds accesses and use-after-free bugs. + Enables KASAN (Kernel Address Sanitizer) - a dynamic memory safety + error detector designed to find out-of-bounds and use-after-free bugs. + See Documentation/dev-tools/kasan.rst for details. + For better error reports, also enable CONFIG_STACKTRACE. + if KASAN choice @@ -51,75 +54,71 @@ choice default KASAN_GENERIC help KASAN has three modes: - 1. generic KASAN (similar to userspace ASan, - x86_64/arm64/xtensa, enabled with CONFIG_KASAN_GENERIC), - 2. software tag-based KASAN (arm64 only, based on software - memory tagging (similar to userspace HWASan), enabled with - CONFIG_KASAN_SW_TAGS), and - 3. hardware tag-based KASAN (arm64 only, based on hardware - memory tagging, enabled with CONFIG_KASAN_HW_TAGS). - All KASAN modes are strictly debugging features. + 1. Generic KASAN (supported by many architectures, enabled with + CONFIG_KASAN_GENERIC, similar to userspace ASan), + 2. Software Tag-Based KASAN (arm64 only, based on software memory + tagging, enabled with CONFIG_KASAN_SW_TAGS, similar to userspace + HWASan), and + 3. Hardware Tag-Based KASAN (arm64 only, based on hardware memory + tagging, enabled with CONFIG_KASAN_HW_TAGS). - For better error reports enable CONFIG_STACKTRACE. + See Documentation/dev-tools/kasan.rst for details about each mode. config KASAN_GENERIC - bool "Generic mode" + bool "Generic KASAN" depends on HAVE_ARCH_KASAN && CC_HAS_KASAN_GENERIC depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS help - Enables generic KASAN mode. + Enables Generic KASAN. - This mode is supported in both GCC and Clang. With GCC it requires - version 8.3.0 or later. Any supported Clang version is compatible, - but detection of out-of-bounds accesses for global variables is - supported only since Clang 11. + Requires GCC 8.3.0+ or Clang. - This mode consumes about 1/8th of available memory at kernel start - and introduces an overhead of ~x1.5 for the rest of the allocations. + Consumes about 1/8th of available memory at kernel start and adds an + overhead of ~50% for dynamic allocations. The performance slowdown is ~x3. - Currently CONFIG_KASAN_GENERIC doesn't work with CONFIG_DEBUG_SLAB - (the resulting kernel does not boot). + (Incompatible with CONFIG_DEBUG_SLAB: the kernel does not boot.) config KASAN_SW_TAGS - bool "Software tag-based mode" + bool "Software Tag-Based KASAN" depends on HAVE_ARCH_KASAN_SW_TAGS && CC_HAS_KASAN_SW_TAGS depends on CC_HAS_WORKING_NOSANITIZE_ADDRESS select SLUB_DEBUG if SLUB select CONSTRUCTORS help - Enables software tag-based KASAN mode. + Enables Software Tag-Based KASAN. - This mode require software memory tagging support in the form of - HWASan-like compiler instrumentation. + Requires GCC 11+ or Clang. - Currently this mode is only implemented for arm64 CPUs and relies on - Top Byte Ignore. This mode requires Clang. + Supported only on arm64 CPUs and relies on Top Byte Ignore. - This mode consumes about 1/16th of available memory at kernel start - and introduces an overhead of ~20% for the rest of the allocations. - This mode may potentially introduce problems relating to pointer - casting and comparison, as it embeds tags into the top byte of each - pointer. + Consumes about 1/16th of available memory at kernel start and + add an overhead of ~20% for dynamic allocations. - Currently CONFIG_KASAN_SW_TAGS doesn't work with CONFIG_DEBUG_SLAB - (the resulting kernel does not boot). + May potentially introduce problems related to pointer casting and + comparison, as it embeds a tag into the top byte of each pointer. + + (Incompatible with CONFIG_DEBUG_SLAB: the kernel does not boot.) config KASAN_HW_TAGS - bool "Hardware tag-based mode" + bool "Hardware Tag-Based KASAN" depends on HAVE_ARCH_KASAN_HW_TAGS depends on SLUB help - Enables hardware tag-based KASAN mode. + Enables Hardware Tag-Based KASAN. + + Requires GCC 10+ or Clang 12+. + + Supported only on arm64 CPUs starting from ARMv8.5 and relies on + Memory Tagging Extension and Top Byte Ignore. - This mode requires hardware memory tagging support, and can be used - by any architecture that provides it. + Consumes about 1/32nd of available memory. - Currently this mode is only implemented for arm64 CPUs starting from - ARMv8.5 and relies on Memory Tagging Extension and Top Byte Ignore. + May potentially introduce problems related to pointer casting and + comparison, as it embeds a tag into the top byte of each pointer. endchoice @@ -131,83 +130,80 @@ choice config KASAN_OUTLINE bool "Outline instrumentation" help - Before every memory access compiler insert function call - __asan_load*/__asan_store*. These functions performs check - of shadow memory. This is slower than inline instrumentation, - however it doesn't bloat size of kernel's .text section so - much as inline does. + Makes the compiler insert function calls that check whether the memory + is accessible before each memory access. Slower than KASAN_INLINE, but + does not bloat the size of the kernel's .text section so much. config KASAN_INLINE bool "Inline instrumentation" depends on !ARCH_DISABLE_KASAN_INLINE help - Compiler directly inserts code checking shadow memory before - memory accesses. This is faster than outline (in some workloads - it gives about x2 boost over outline instrumentation), but - make kernel's .text size much bigger. + Makes the compiler directly insert memory accessibility checks before + each memory access. Faster than KASAN_OUTLINE (gives ~x2 boost for + some workloads), but makes the kernel's .text size much bigger. endchoice config KASAN_STACK - bool "Enable stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST + bool "Stack instrumentation (unsafe)" if CC_IS_CLANG && !COMPILE_TEST depends on KASAN_GENERIC || KASAN_SW_TAGS depends on !ARCH_DISABLE_KASAN_INLINE default y if CC_IS_GCC help - The LLVM stack address sanitizer has a know problem that - causes excessive stack usage in a lot of functions, see - https://bugs.llvm.org/show_bug.cgi?id=38809 - Disabling asan-stack makes it safe to run kernels build - with clang-8 with KASAN enabled, though it loses some of - the functionality. - This feature is always disabled when compile-testing with clang - to avoid cluttering the output in stack overflow warnings, - but clang users can still enable it for builds without - CONFIG_COMPILE_TEST. On gcc it is assumed to always be safe - to use and enabled by default. - If the architecture disables inline instrumentation, stack - instrumentation is also disabled as it adds inline-style - instrumentation that is run unconditionally. + Disables stack instrumentation and thus KASAN's ability to detect + out-of-bounds bugs in stack variables. + + With Clang, stack instrumentation has a problem that causes excessive + stack usage, see https://bugs.llvm.org/show_bug.cgi?id=38809. Thus, + with Clang, this option is deemed unsafe. + + This option is always disabled when compile-testing with Clang to + avoid cluttering the log with stack overflow warnings. + + With GCC, enabling stack instrumentation is assumed to be safe. + + If the architecture disables inline instrumentation via + ARCH_DISABLE_KASAN_INLINE, stack instrumentation gets disabled + as well, as it adds inline-style instrumentation that is run + unconditionally. config KASAN_TAGS_IDENTIFY - bool "Enable memory corruption identification" + bool "Memory corruption type identification" depends on KASAN_SW_TAGS || KASAN_HW_TAGS help - This option enables best-effort identification of bug type - (use-after-free or out-of-bounds) at the cost of increased - memory consumption. + Enables best-effort identification of the bug types (use-after-free + or out-of-bounds) at the cost of increased memory consumption. + Only applicable for the tag-based KASAN modes. config KASAN_VMALLOC - bool "Back mappings in vmalloc space with real shadow memory" - depends on KASAN_GENERIC && HAVE_ARCH_KASAN_VMALLOC + bool "Check accesses to vmalloc allocations" + depends on HAVE_ARCH_KASAN_VMALLOC help - By default, the shadow region for vmalloc space is the read-only - zero page. This means that KASAN cannot detect errors involving - vmalloc space. + Makes KASAN check the validity of accesses to vmalloc allocations. + + With software KASAN modes, all types vmalloc allocations are + checked. Enabling this option leads to higher memory usage. - Enabling this option will hook in to vmap/vmalloc and back those - mappings with real shadow memory allocated on demand. This allows - for KASAN to detect more sorts of errors (and to support vmapped - stacks), but at the cost of higher memory usage. + With Hardware Tag-Based KASAN, only non-executable VM_ALLOC mappings + are checked. There is no additional memory usage. config KASAN_KUNIT_TEST tristate "KUnit-compatible tests of KASAN bug detection capabilities" if !KUNIT_ALL_TESTS depends on KASAN && KUNIT default KUNIT_ALL_TESTS help - This is a KUnit test suite doing various nasty things like - out of bounds and use after free accesses. It is useful for testing - kernel debugging features like KASAN. + A KUnit-based KASAN test suite. Triggers different kinds of + out-of-bounds and use-after-free accesses. Useful for testing whether + KASAN can detect certain bug types. For more information on KUnit and unit tests in general, please refer - to the KUnit documentation in Documentation/dev-tools/kunit. + to the KUnit documentation in Documentation/dev-tools/kunit/. config KASAN_MODULE_TEST tristate "KUnit-incompatible tests of KASAN bug detection capabilities" depends on m && KASAN && !KASAN_HW_TAGS help - This is a part of the KASAN test suite that is incompatible with - KUnit. Currently includes tests that do bad copy_from/to_user - accesses. + A part of the KASAN test suite that is not integrated with KUnit. + Incompatible with Hardware Tag-Based KASAN. endif # KASAN diff --git a/lib/Kconfig.kcsan b/lib/Kconfig.kcsan index 63b70b8c5551..47a693c45864 100644 --- a/lib/Kconfig.kcsan +++ b/lib/Kconfig.kcsan @@ -10,21 +10,10 @@ config HAVE_KCSAN_COMPILER For the list of compilers that support KCSAN, please see <file:Documentation/dev-tools/kcsan.rst>. -config KCSAN_KCOV_BROKEN - def_bool KCOV && CC_HAS_SANCOV_TRACE_PC - depends on CC_IS_CLANG - depends on !$(cc-option,-Werror=unused-command-line-argument -fsanitize=thread -fsanitize-coverage=trace-pc) - help - Some versions of clang support either KCSAN and KCOV but not the - combination of the two. - See https://bugs.llvm.org/show_bug.cgi?id=45831 for the status - in newer releases. - menuconfig KCSAN bool "KCSAN: dynamic data race detector" depends on HAVE_ARCH_KCSAN && HAVE_KCSAN_COMPILER depends on DEBUG_KERNEL && !KASAN - depends on !KCSAN_KCOV_BROKEN select STACKTRACE help The Kernel Concurrency Sanitizer (KCSAN) is a dynamic @@ -198,7 +187,9 @@ config KCSAN_WEAK_MEMORY # We can either let objtool nop __tsan_func_{entry,exit}() and builtin # atomics instrumentation in .noinstr.text, or use a compiler that can # implement __no_kcsan to really remove all instrumentation. - depends on STACK_VALIDATION || CC_IS_GCC || CLANG_VERSION >= 140000 + depends on !ARCH_WANTS_NO_INSTR || HAVE_NOINSTR_HACK || \ + CC_IS_GCC || CLANG_VERSION >= 140000 + select OBJTOOL if HAVE_NOINSTR_HACK help Enable support for modeling a subset of weak memory, which allows detecting a subset of data races due to missing memory barriers. diff --git a/lib/Kconfig.kfence b/lib/Kconfig.kfence index 912f252a41fc..459dda9ef619 100644 --- a/lib/Kconfig.kfence +++ b/lib/Kconfig.kfence @@ -45,6 +45,18 @@ config KFENCE_NUM_OBJECTS pages are required; with one containing the object and two adjacent ones used as guard pages. +config KFENCE_DEFERRABLE + bool "Use a deferrable timer to trigger allocations" + help + Use a deferrable timer to trigger allocations. This avoids forcing + CPU wake-ups if the system is idle, at the risk of a less predictable + sample interval. + + Warning: The KUnit test suite fails with this option enabled - due to + the unpredictability of the sample interval! + + Say N if you are unsure. + config KFENCE_STATIC_KEYS bool "Use static keys to set up allocations" if EXPERT depends on JUMP_LABEL diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 236c5cefc4cc..a9f7eb047768 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -27,16 +27,6 @@ config UBSAN_TRAP the system. For some system builders this is an acceptable trade-off. -config UBSAN_KCOV_BROKEN - def_bool KCOV && CC_HAS_SANCOV_TRACE_PC - depends on CC_IS_CLANG - depends on !$(cc-option,-Werror=unused-command-line-argument -fsanitize=bounds -fsanitize-coverage=trace-pc) - help - Some versions of clang support either UBSAN or KCOV but not the - combination of the two. - See https://bugs.llvm.org/show_bug.cgi?id=45831 for the status - in newer releases. - config CC_HAS_UBSAN_BOUNDS def_bool $(cc-option,-fsanitize=bounds) @@ -46,7 +36,6 @@ config CC_HAS_UBSAN_ARRAY_BOUNDS config UBSAN_BOUNDS bool "Perform array index bounds checking" default UBSAN - depends on !UBSAN_KCOV_BROKEN depends on CC_HAS_UBSAN_ARRAY_BOUNDS || CC_HAS_UBSAN_BOUNDS help This option enables detection of directly indexed out of bounds @@ -72,7 +61,6 @@ config UBSAN_ARRAY_BOUNDS config UBSAN_LOCAL_BOUNDS bool "Perform array local bounds checking" depends on UBSAN_TRAP - depends on !UBSAN_KCOV_BROKEN depends on $(cc-option,-fsanitize=local-bounds) help This option enables -fsanitize=local-bounds which traps when an @@ -106,7 +94,7 @@ config UBSAN_UNREACHABLE bool "Perform checking for unreachable code" # objtool already handles unreachable checking and gets angry about # seeing UBSan instrumentation located in unreachable places. - depends on !STACK_VALIDATION + depends on !(OBJTOOL && (STACK_VALIDATION || UNWINDER_ORC || HAVE_UACCESS_VALIDATION)) depends on $(cc-option,-fsanitize=unreachable) help This option enables -fsanitize=unreachable which checks for control diff --git a/lib/Makefile b/lib/Makefile index 300f569c626b..f99bf61f8bbc 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -77,7 +77,6 @@ obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o obj-$(CONFIG_TEST_MIN_HEAP) += test_min_heap.o obj-$(CONFIG_TEST_LKM) += test_module.o obj-$(CONFIG_TEST_VMALLOC) += test_vmalloc.o -obj-$(CONFIG_TEST_OVERFLOW) += test_overflow.o obj-$(CONFIG_TEST_RHASHTABLE) += test_rhashtable.o obj-$(CONFIG_TEST_SORT) += test_sort.o obj-$(CONFIG_TEST_USER_COPY) += test_user_copy.o @@ -94,8 +93,6 @@ obj-$(CONFIG_TEST_KMOD) += test_kmod.o obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o obj-$(CONFIG_TEST_MEMCAT_P) += test_memcat_p.o obj-$(CONFIG_TEST_OBJAGG) += test_objagg.o -CFLAGS_test_stackinit.o += $(call cc-disable-warning, switch-unreachable) -obj-$(CONFIG_TEST_STACKINIT) += test_stackinit.o obj-$(CONFIG_TEST_BLACKHOLE_DEV) += test_blackhole_dev.o obj-$(CONFIG_TEST_MEMINIT) += test_meminit.o obj-$(CONFIG_TEST_LOCKUP) += test_lockup.o @@ -103,6 +100,8 @@ obj-$(CONFIG_TEST_HMM) += test_hmm.o obj-$(CONFIG_TEST_FREE_PAGES) += test_free_pages.o obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o obj-$(CONFIG_TEST_REF_TRACKER) += test_ref_tracker.o +CFLAGS_test_fprobe.o += $(CC_FLAGS_FTRACE) +obj-$(CONFIG_FPROBE_SANITY_TEST) += test_fprobe.o # # CFLAGS for compiling floating point code inside the kernel. x86/Makefile turns # off the generation of FPU/SSE* instructions for kernel proper but FPU_FLAGS @@ -175,6 +174,7 @@ obj-$(CONFIG_CRC4) += crc4.o obj-$(CONFIG_CRC7) += crc7.o obj-$(CONFIG_LIBCRC32C) += libcrc32c.o obj-$(CONFIG_CRC8) += crc8.o +obj-$(CONFIG_CRC64_ROCKSOFT) += crc64-rocksoft.o obj-$(CONFIG_XXHASH) += xxhash.o obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o @@ -251,6 +251,7 @@ obj-$(CONFIG_DIMLIB) += dim/ obj-$(CONFIG_SIGNATURE) += digsig.o lib-$(CONFIG_CLZ_TAB) += clz_tab.o +lib-$(CONFIG_LIB_MEMNEQ) += memneq.o obj-$(CONFIG_GENERIC_STRNCPY_FROM_USER) += strncpy_from_user.o obj-$(CONFIG_GENERIC_STRNLEN_USER) += strnlen_user.o @@ -263,6 +264,8 @@ obj-$(CONFIG_MEMREGION) += memregion.o obj-$(CONFIG_STMP_DEVICE) += stmp_device.o obj-$(CONFIG_IRQ_POLL) += irq_poll.o +obj-$(CONFIG_POLYNOMIAL) += polynomial.o + # stackdepot.c should not be instrumented or call instrumented functions. # Prevent the compiler from calling builtins like memcmp() or bcmp() from this # file. @@ -279,7 +282,15 @@ $(foreach file, $(libfdt_files), \ $(eval CFLAGS_$(file) = -I $(srctree)/scripts/dtc/libfdt)) lib-$(CONFIG_LIBFDT) += $(libfdt_files) -lib-$(CONFIG_BOOT_CONFIG) += bootconfig.o +obj-$(CONFIG_BOOT_CONFIG) += bootconfig.o +obj-$(CONFIG_BOOT_CONFIG_EMBED) += bootconfig-data.o + +$(obj)/bootconfig-data.o: $(obj)/default.bconf + +targets += default.bconf +filechk_defbconf = cat $(or $(real-prereqs), /dev/null) +$(obj)/default.bconf: $(CONFIG_BOOT_CONFIG_EMBED_FILE) FORCE + $(call filechk,defbconf) obj-$(CONFIG_RBTREE_TEST) += rbtree_test.o obj-$(CONFIG_INTERVAL_TREE_TEST) += interval_tree_test.o @@ -363,6 +374,9 @@ obj-$(CONFIG_BITS_TEST) += test_bits.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o +obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o +CFLAGS_stackinit_kunit.o += $(call cc-disable-warning, switch-unreachable) +obj-$(CONFIG_STACKINIT_KUNIT_TEST) += stackinit_kunit.o obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o @@ -374,7 +388,8 @@ TEST_FORTIFY_LOG = test_fortify.log quiet_cmd_test_fortify = TEST $@ cmd_test_fortify = $(CONFIG_SHELL) $(srctree)/scripts/test_fortify.sh \ $< $@ "$(NM)" $(CC) $(c_flags) \ - $(call cc-disable-warning,fortify-source) + $(call cc-disable-warning,fortify-source) \ + -DKBUILD_EXTRA_WARN1 targets += $(TEST_FORTIFY_LOGS) clean-files += $(TEST_FORTIFY_LOGS) diff --git a/lib/assoc_array.c b/lib/assoc_array.c index 079c72e26493..ca0b4f360c1a 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -1461,6 +1461,7 @@ int assoc_array_gc(struct assoc_array *array, struct assoc_array_ptr *cursor, *ptr; struct assoc_array_ptr *new_root, *new_parent, **new_ptr_pp; unsigned long nr_leaves_on_tree; + bool retained; int keylen, slot, nr_free, next_slot, i; pr_devel("-->%s()\n", __func__); @@ -1536,6 +1537,7 @@ continue_node: goto descend; } +retry_compress: pr_devel("-- compress node %p --\n", new_n); /* Count up the number of empty slots in this node and work out the @@ -1553,6 +1555,7 @@ continue_node: pr_devel("free=%d, leaves=%lu\n", nr_free, new_n->nr_leaves_on_branch); /* See what we can fold in */ + retained = false; next_slot = 0; for (slot = 0; slot < ASSOC_ARRAY_FAN_OUT; slot++) { struct assoc_array_shortcut *s; @@ -1602,9 +1605,14 @@ continue_node: pr_devel("[%d] retain node %lu/%d [nx %d]\n", slot, child->nr_leaves_on_branch, nr_free + 1, next_slot); + retained = true; } } + if (retained && new_n->nr_leaves_on_branch <= ASSOC_ARRAY_FAN_OUT) { + pr_devel("internal nodes remain despite enough space, retrying\n"); + goto retry_compress; + } pr_devel("after: %lu\n", new_n->nr_leaves_on_branch); nr_leaves_on_tree = new_n->nr_leaves_on_branch; diff --git a/lib/bitmap.c b/lib/bitmap.c index 926408883456..b18e31ea6e66 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -45,19 +45,19 @@ * for the best explanations of this ordering. */ -int __bitmap_equal(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +bool __bitmap_equal(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] != bitmap2[k]) - return 0; + return false; if (bits % BITS_PER_LONG) if ((bitmap1[k] ^ bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 0; + return false; - return 1; + return true; } EXPORT_SYMBOL(__bitmap_equal); @@ -303,33 +303,33 @@ void __bitmap_replace(unsigned long *dst, } EXPORT_SYMBOL(__bitmap_replace); -int __bitmap_intersects(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +bool __bitmap_intersects(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] & bitmap2[k]) - return 1; + return true; if (bits % BITS_PER_LONG) if ((bitmap1[k] & bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 1; - return 0; + return true; + return false; } EXPORT_SYMBOL(__bitmap_intersects); -int __bitmap_subset(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +bool __bitmap_subset(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] & ~bitmap2[k]) - return 0; + return false; if (bits % BITS_PER_LONG) if ((bitmap1[k] & ~bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 0; - return 1; + return false; + return true; } EXPORT_SYMBOL(__bitmap_subset); @@ -492,6 +492,11 @@ EXPORT_SYMBOL(bitmap_print_to_pagebuf); * @list: indicates whether the bitmap must be list * true: print in decimal list format * false: print in hexadecimal bitmask format + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print */ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, int nmaskbits, loff_t off, size_t count) @@ -512,38 +517,49 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, /** * bitmap_print_bitmask_to_buf - convert bitmap to hex bitmask format ASCII string + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print * * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal * bitmask and decimal list to userspace by sysfs ABI. * Drivers might be using a normal attribute for this kind of ABIs. A - * normal attribute typically has show entry as below: - * static ssize_t example_attribute_show(struct device *dev, + * normal attribute typically has show entry as below:: + * + * static ssize_t example_attribute_show(struct device *dev, * struct device_attribute *attr, char *buf) - * { + * { * ... * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); - * } + * } + * * show entry of attribute has no offset and count parameters and this * means the file is limited to one page only. * bitmap_print_to_pagebuf() API works terribly well for this kind of - * normal attribute with buf parameter and without offset, count: - * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, + * normal attribute with buf parameter and without offset, count:: + * + * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, * int nmaskbits) - * { - * } + * { + * } + * * The problem is once we have a large bitmap, we have a chance to get a * bitmask or list more than one page. Especially for list, it could be * as complex as 0,3,5,7,9,... We have no simple way to know it exact size. * It turns out bin_attribute is a way to break this limit. bin_attribute - * has show entry as below: - * static ssize_t - * example_bin_attribute_show(struct file *filp, struct kobject *kobj, + * has show entry as below:: + * + * static ssize_t + * example_bin_attribute_show(struct file *filp, struct kobject *kobj, * struct bin_attribute *attr, char *buf, * loff_t offset, size_t count) - * { + * { * ... - * } + * } + * * With the new offset and count parameters, this makes sysfs ABI be able * to support file size more than one page. For example, offset could be * >= 4096. @@ -553,12 +569,6 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * move to use bin_attribute. In result, we have to pass the corresponding * parameters such as off, count from bin_attribute show entry to this API. * - * @buf: buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * @off: in the string from which we are copying, We copy to @buf - * @count: the maximum number of bytes to print - * * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() * is similar with cpumap_print_to_pagebuf(), the difference is that * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption @@ -573,6 +583,7 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). * It is intended to workaround sysfs limitations discussed above and should be * used carefully in general case for the following reasons: + * * - Time complexity is O(nbits^2/count), comparing to O(nbits) for snprintf(). * - Memory complexity is O(nbits), comparing to O(1) for snprintf(). * - @off and @count are NOT offset and number of bits to print. @@ -597,6 +608,11 @@ EXPORT_SYMBOL(bitmap_print_bitmask_to_buf); /** * bitmap_print_list_to_buf - convert bitmap to decimal list format ASCII string + * @buf: buffer into which string is placed + * @maskp: pointer to bitmap to convert + * @nmaskbits: size of bitmap, in bits + * @off: in the string from which we are copying, We copy to @buf + * @count: the maximum number of bytes to print * * Everything is same with the above bitmap_print_bitmask_to_buf() except * the print format. @@ -807,7 +823,8 @@ EXPORT_SYMBOL(bitmap_parselist); /** - * bitmap_parselist_user() + * bitmap_parselist_user() - convert user buffer's list format ASCII + * string to bitmap * * @ubuf: pointer to user buffer containing string. * @ulen: buffer size in bytes. If string is smaller than this @@ -1495,5 +1512,59 @@ void bitmap_to_arr32(u32 *buf, const unsigned long *bitmap, unsigned int nbits) buf[halfwords - 1] &= (u32) (UINT_MAX >> ((-nbits) & 31)); } EXPORT_SYMBOL(bitmap_to_arr32); +#endif +#if (BITS_PER_LONG == 32) && defined(__BIG_ENDIAN) +/** + * bitmap_from_arr64 - copy the contents of u64 array of bits to bitmap + * @bitmap: array of unsigned longs, the destination bitmap + * @buf: array of u64 (in host byte order), the source bitmap + * @nbits: number of bits in @bitmap + */ +void bitmap_from_arr64(unsigned long *bitmap, const u64 *buf, unsigned int nbits) +{ + int n; + + for (n = nbits; n > 0; n -= 64) { + u64 val = *buf++; + + *bitmap++ = val; + if (n > 32) + *bitmap++ = val >> 32; + } + + /* + * Clear tail bits in the last word beyond nbits. + * + * Negative index is OK because here we point to the word next + * to the last word of the bitmap, except for nbits == 0, which + * is tested implicitly. + */ + if (nbits % BITS_PER_LONG) + bitmap[-1] &= BITMAP_LAST_WORD_MASK(nbits); +} +EXPORT_SYMBOL(bitmap_from_arr64); + +/** + * bitmap_to_arr64 - copy the contents of bitmap to a u64 array of bits + * @buf: array of u64 (in host byte order), the dest bitmap + * @bitmap: array of unsigned longs, the source bitmap + * @nbits: number of bits in @bitmap + */ +void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits) +{ + const unsigned long *end = bitmap + BITS_TO_LONGS(nbits); + + while (bitmap < end) { + *buf = *bitmap++; + if (bitmap < end) + *buf |= (u64)(*bitmap++) << 32; + buf++; + } + + /* Clear tail bits in the last element of array beyond nbits. */ + if (nbits % 64) + buf[-1] &= GENMASK_ULL(nbits % 64, 0); +} +EXPORT_SYMBOL(bitmap_to_arr64); #endif diff --git a/lib/bootconfig-data.S b/lib/bootconfig-data.S new file mode 100644 index 000000000000..ef85ba1a82f4 --- /dev/null +++ b/lib/bootconfig-data.S @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Embed default bootconfig in the kernel. + */ + .section .init.rodata, "aw" + .global embedded_bootconfig_data +embedded_bootconfig_data: + .incbin "lib/default.bconf" + .global embedded_bootconfig_data_end +embedded_bootconfig_data_end: diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 74f3201ab8e5..c59d26068a64 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -12,6 +12,19 @@ #include <linux/kernel.h> #include <linux/memblock.h> #include <linux/string.h> + +#ifdef CONFIG_BOOT_CONFIG_EMBED +/* embedded_bootconfig_data is defined in bootconfig-data.S */ +extern __visible const char embedded_bootconfig_data[]; +extern __visible const char embedded_bootconfig_data_end[]; + +const char * __init xbc_get_embedded_bootconfig(size_t *size) +{ + *size = embedded_bootconfig_data_end - embedded_bootconfig_data; + return (*size) ? embedded_bootconfig_data : NULL; +} +#endif + #else /* !__KERNEL__ */ /* * NOTE: This is only for tools/bootconfig, because tools/bootconfig will diff --git a/lib/bug.c b/lib/bug.c index 45a0584f6541..c223a2575b72 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -6,8 +6,7 @@ CONFIG_BUG - emit BUG traps. Nothing happens without this. CONFIG_GENERIC_BUG - enable this code. - CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit pointers relative to - the containing struct bug_entry for bug_addr and file. + CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable @@ -53,10 +52,10 @@ extern struct bug_entry __start___bug_table[], __stop___bug_table[]; static inline unsigned long bug_addr(const struct bug_entry *bug) { -#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - return bug->bug_addr; +#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS + return (unsigned long)&bug->bug_addr_disp + bug->bug_addr_disp; #else - return (unsigned long)bug + bug->bug_addr_disp; + return bug->bug_addr; #endif } @@ -131,10 +130,10 @@ void bug_get_file_line(struct bug_entry *bug, const char **file, unsigned int *line) { #ifdef CONFIG_DEBUG_BUGVERBOSE -#ifndef CONFIG_GENERIC_BUG_RELATIVE_POINTERS - *file = bug->file; +#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS + *file = (const char *)&bug->file_disp + bug->file_disp; #else - *file = (const char *)bug + bug->file_disp; + *file = bug->file; #endif *line = bug->line; #else diff --git a/lib/crc-itu-t.c b/lib/crc-itu-t.c index 1974b355c148..1d26a1647da5 100644 --- a/lib/crc-itu-t.c +++ b/lib/crc-itu-t.c @@ -7,7 +7,7 @@ #include <linux/module.h> #include <linux/crc-itu-t.h> -/** CRC table for the CRC ITU-T V.41 0x1021 (x^16 + x^12 + x^15 + 1) */ +/* CRC table for the CRC ITU-T V.41 0x1021 (x^16 + x^12 + x^5 + 1) */ const u16 crc_itu_t_table[256] = { 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, diff --git a/lib/crc32.c b/lib/crc32.c index 2a68dfd3b96c..5649847d0a8d 100644 --- a/lib/crc32.c +++ b/lib/crc32.c @@ -194,13 +194,11 @@ u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len) #else u32 __pure __weak crc32_le(u32 crc, unsigned char const *p, size_t len) { - return crc32_le_generic(crc, p, len, - (const u32 (*)[256])crc32table_le, CRC32_POLY_LE); + return crc32_le_generic(crc, p, len, crc32table_le, CRC32_POLY_LE); } u32 __pure __weak __crc32c_le(u32 crc, unsigned char const *p, size_t len) { - return crc32_le_generic(crc, p, len, - (const u32 (*)[256])crc32ctable_le, CRC32C_POLY_LE); + return crc32_le_generic(crc, p, len, crc32ctable_le, CRC32C_POLY_LE); } #endif EXPORT_SYMBOL(crc32_le); @@ -208,6 +206,7 @@ EXPORT_SYMBOL(__crc32c_le); u32 __pure crc32_le_base(u32, unsigned char const *, size_t) __alias(crc32_le); u32 __pure __crc32c_le_base(u32, unsigned char const *, size_t) __alias(__crc32c_le); +u32 __pure crc32_be_base(u32, unsigned char const *, size_t) __alias(crc32_be); /* * This multiplies the polynomials x and y modulo the given modulus. @@ -332,15 +331,14 @@ static inline u32 __pure crc32_be_generic(u32 crc, unsigned char const *p, } #if CRC_BE_BITS == 1 -u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_be(u32 crc, unsigned char const *p, size_t len) { return crc32_be_generic(crc, p, len, NULL, CRC32_POLY_BE); } #else -u32 __pure crc32_be(u32 crc, unsigned char const *p, size_t len) +u32 __pure __weak crc32_be(u32 crc, unsigned char const *p, size_t len) { - return crc32_be_generic(crc, p, len, - (const u32 (*)[256])crc32table_be, CRC32_POLY_BE); + return crc32_be_generic(crc, p, len, crc32table_be, CRC32_POLY_BE); } #endif EXPORT_SYMBOL(crc32_be); diff --git a/lib/crc32test.c b/lib/crc32test.c index 61ddce2cff77..9b4af79412c4 100644 --- a/lib/crc32test.c +++ b/lib/crc32test.c @@ -675,7 +675,7 @@ static int __init crc32c_test(void) /* pre-warm the cache */ for (i = 0; i < 100; i++) { - bytes += 2*test[i].length; + bytes += test[i].length; crc ^= __crc32c_le(test[i].crc, test_buf + test[i].start, test[i].length); diff --git a/lib/crc64-rocksoft.c b/lib/crc64-rocksoft.c new file mode 100644 index 000000000000..fc9ae0da5df7 --- /dev/null +++ b/lib/crc64-rocksoft.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/crc64.h> +#include <linux/err.h> +#include <linux/init.h> +#include <crypto/hash.h> +#include <crypto/algapi.h> +#include <linux/static_key.h> +#include <linux/notifier.h> + +static struct crypto_shash __rcu *crc64_rocksoft_tfm; +static DEFINE_STATIC_KEY_TRUE(crc64_rocksoft_fallback); +static DEFINE_MUTEX(crc64_rocksoft_mutex); +static struct work_struct crc64_rocksoft_rehash_work; + +static int crc64_rocksoft_notify(struct notifier_block *self, unsigned long val, void *data) +{ + struct crypto_alg *alg = data; + + if (val != CRYPTO_MSG_ALG_LOADED || + strcmp(alg->cra_name, CRC64_ROCKSOFT_STRING)) + return NOTIFY_DONE; + + schedule_work(&crc64_rocksoft_rehash_work); + return NOTIFY_OK; +} + +static void crc64_rocksoft_rehash(struct work_struct *work) +{ + struct crypto_shash *new, *old; + + mutex_lock(&crc64_rocksoft_mutex); + old = rcu_dereference_protected(crc64_rocksoft_tfm, + lockdep_is_held(&crc64_rocksoft_mutex)); + new = crypto_alloc_shash(CRC64_ROCKSOFT_STRING, 0, 0); + if (IS_ERR(new)) { + mutex_unlock(&crc64_rocksoft_mutex); + return; + } + rcu_assign_pointer(crc64_rocksoft_tfm, new); + mutex_unlock(&crc64_rocksoft_mutex); + + if (old) { + synchronize_rcu(); + crypto_free_shash(old); + } else { + static_branch_disable(&crc64_rocksoft_fallback); + } +} + +static struct notifier_block crc64_rocksoft_nb = { + .notifier_call = crc64_rocksoft_notify, +}; + +u64 crc64_rocksoft_update(u64 crc, const unsigned char *buffer, size_t len) +{ + struct { + struct shash_desc shash; + u64 crc; + } desc; + int err; + + if (static_branch_unlikely(&crc64_rocksoft_fallback)) + return crc64_rocksoft_generic(crc, buffer, len); + + rcu_read_lock(); + desc.shash.tfm = rcu_dereference(crc64_rocksoft_tfm); + desc.crc = crc; + err = crypto_shash_update(&desc.shash, buffer, len); + rcu_read_unlock(); + + BUG_ON(err); + + return desc.crc; +} +EXPORT_SYMBOL_GPL(crc64_rocksoft_update); + +u64 crc64_rocksoft(const unsigned char *buffer, size_t len) +{ + return crc64_rocksoft_update(0, buffer, len); +} +EXPORT_SYMBOL_GPL(crc64_rocksoft); + +static int __init crc64_rocksoft_mod_init(void) +{ + INIT_WORK(&crc64_rocksoft_rehash_work, crc64_rocksoft_rehash); + crypto_register_notifier(&crc64_rocksoft_nb); + crc64_rocksoft_rehash(&crc64_rocksoft_rehash_work); + return 0; +} + +static void __exit crc64_rocksoft_mod_fini(void) +{ + crypto_unregister_notifier(&crc64_rocksoft_nb); + cancel_work_sync(&crc64_rocksoft_rehash_work); + crypto_free_shash(rcu_dereference_protected(crc64_rocksoft_tfm, 1)); +} + +module_init(crc64_rocksoft_mod_init); +module_exit(crc64_rocksoft_mod_fini); + +static int crc64_rocksoft_transform_show(char *buffer, const struct kernel_param *kp) +{ + struct crypto_shash *tfm; + int len; + + if (static_branch_unlikely(&crc64_rocksoft_fallback)) + return sprintf(buffer, "fallback\n"); + + rcu_read_lock(); + tfm = rcu_dereference(crc64_rocksoft_tfm); + len = snprintf(buffer, PAGE_SIZE, "%s\n", + crypto_shash_driver_name(tfm)); + rcu_read_unlock(); + + return len; +} + +module_param_call(transform, NULL, crc64_rocksoft_transform_show, NULL, 0444); + +MODULE_AUTHOR("Keith Busch <kbusch@kernel.org>"); +MODULE_DESCRIPTION("Rocksoft model CRC64 calculation (library API)"); +MODULE_LICENSE("GPL"); +MODULE_SOFTDEP("pre: crc64"); diff --git a/lib/crc64.c b/lib/crc64.c index 9f852a89ee2a..61ae8dfb6a1c 100644 --- a/lib/crc64.c +++ b/lib/crc64.c @@ -22,6 +22,13 @@ * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 + * x^7 + x^4 + x + 1 * + * crc64rocksoft[256] table is from the Rocksoft specification polynomial + * defined as, + * + * x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 + + * x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 + + * x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1 + * * Copyright 2018 SUSE Linux. * Author: Coly Li <colyli@suse.de> */ @@ -55,3 +62,24 @@ u64 __pure crc64_be(u64 crc, const void *p, size_t len) return crc; } EXPORT_SYMBOL_GPL(crc64_be); + +/** + * crc64_rocksoft_generic - Calculate bitwise Rocksoft CRC64 + * @crc: seed value for computation. 0 for a new CRC calculation, or the + * previous crc64 value if computing incrementally. + * @p: pointer to buffer over which CRC64 is run + * @len: length of buffer @p + */ +u64 __pure crc64_rocksoft_generic(u64 crc, const void *p, size_t len) +{ + const unsigned char *_p = p; + size_t i; + + crc = ~crc; + + for (i = 0; i < len; i++) + crc = (crc >> 8) ^ crc64rocksofttable[(crc & 0xff) ^ *_p++]; + + return ~crc; +} +EXPORT_SYMBOL_GPL(crc64_rocksoft_generic); diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index e8e525650cf2..2082af43d51f 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -71,6 +71,7 @@ config CRYPTO_LIB_CURVE25519 tristate "Curve25519 scalar multiplication library" depends on CRYPTO_ARCH_HAVE_LIB_CURVE25519 || !CRYPTO_ARCH_HAVE_LIB_CURVE25519 select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n + select LIB_MEMNEQ help Enable the Curve25519 library interface. This interface may be fulfilled by either the generic implementation or an arch-specific @@ -123,7 +124,4 @@ config CRYPTO_LIB_CHACHA20POLY1305 config CRYPTO_LIB_SHA256 tristate -config CRYPTO_LIB_SM4 - tristate - endmenu diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index ed43a41f2dcc..26be2bbe09c5 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -37,9 +37,6 @@ libpoly1305-y += poly1305.o obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o libsha256-y := sha256.o -obj-$(CONFIG_CRYPTO_LIB_SM4) += libsm4.o -libsm4-y := sm4.o - ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) libblake2s-y += blake2s-selftest.o libchacha20poly1305-y += chacha20poly1305-selftest.o diff --git a/lib/crypto/sm4.c b/lib/crypto/sm4.c deleted file mode 100644 index 284e62576d0c..000000000000 --- a/lib/crypto/sm4.c +++ /dev/null @@ -1,176 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * SM4, as specified in - * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html - * - * Copyright (C) 2018 ARM Limited or its affiliates. - * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> - */ - -#include <linux/module.h> -#include <asm/unaligned.h> -#include <crypto/sm4.h> - -static const u32 fk[4] = { - 0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc -}; - -static const u32 ____cacheline_aligned ck[32] = { - 0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269, - 0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9, - 0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249, - 0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9, - 0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229, - 0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299, - 0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209, - 0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279 -}; - -static const u8 ____cacheline_aligned sbox[256] = { - 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, - 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05, - 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, - 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99, - 0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, - 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62, - 0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, - 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6, - 0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, - 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8, - 0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, - 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35, - 0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, - 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87, - 0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, - 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e, - 0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, - 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1, - 0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, - 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3, - 0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, - 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f, - 0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, - 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51, - 0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, - 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8, - 0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, - 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0, - 0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, - 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84, - 0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, - 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48 -}; - -static inline u32 sm4_t_non_lin_sub(u32 x) -{ - u32 out; - - out = (u32)sbox[x & 0xff]; - out |= (u32)sbox[(x >> 8) & 0xff] << 8; - out |= (u32)sbox[(x >> 16) & 0xff] << 16; - out |= (u32)sbox[(x >> 24) & 0xff] << 24; - - return out; -} - -static inline u32 sm4_key_lin_sub(u32 x) -{ - return x ^ rol32(x, 13) ^ rol32(x, 23); -} - -static inline u32 sm4_enc_lin_sub(u32 x) -{ - return x ^ rol32(x, 2) ^ rol32(x, 10) ^ rol32(x, 18) ^ rol32(x, 24); -} - -static inline u32 sm4_key_sub(u32 x) -{ - return sm4_key_lin_sub(sm4_t_non_lin_sub(x)); -} - -static inline u32 sm4_enc_sub(u32 x) -{ - return sm4_enc_lin_sub(sm4_t_non_lin_sub(x)); -} - -static inline u32 sm4_round(u32 x0, u32 x1, u32 x2, u32 x3, u32 rk) -{ - return x0 ^ sm4_enc_sub(x1 ^ x2 ^ x3 ^ rk); -} - - -/** - * sm4_expandkey - Expands the SM4 key as described in GB/T 32907-2016 - * @ctx: The location where the computed key will be stored. - * @in_key: The supplied key. - * @key_len: The length of the supplied key. - * - * Returns 0 on success. The function fails only if an invalid key size (or - * pointer) is supplied. - */ -int sm4_expandkey(struct sm4_ctx *ctx, const u8 *in_key, - unsigned int key_len) -{ - u32 rk[4]; - const u32 *key = (u32 *)in_key; - int i; - - if (key_len != SM4_KEY_SIZE) - return -EINVAL; - - rk[0] = get_unaligned_be32(&key[0]) ^ fk[0]; - rk[1] = get_unaligned_be32(&key[1]) ^ fk[1]; - rk[2] = get_unaligned_be32(&key[2]) ^ fk[2]; - rk[3] = get_unaligned_be32(&key[3]) ^ fk[3]; - - for (i = 0; i < 32; i += 4) { - rk[0] ^= sm4_key_sub(rk[1] ^ rk[2] ^ rk[3] ^ ck[i + 0]); - rk[1] ^= sm4_key_sub(rk[2] ^ rk[3] ^ rk[0] ^ ck[i + 1]); - rk[2] ^= sm4_key_sub(rk[3] ^ rk[0] ^ rk[1] ^ ck[i + 2]); - rk[3] ^= sm4_key_sub(rk[0] ^ rk[1] ^ rk[2] ^ ck[i + 3]); - - ctx->rkey_enc[i + 0] = rk[0]; - ctx->rkey_enc[i + 1] = rk[1]; - ctx->rkey_enc[i + 2] = rk[2]; - ctx->rkey_enc[i + 3] = rk[3]; - ctx->rkey_dec[31 - 0 - i] = rk[0]; - ctx->rkey_dec[31 - 1 - i] = rk[1]; - ctx->rkey_dec[31 - 2 - i] = rk[2]; - ctx->rkey_dec[31 - 3 - i] = rk[3]; - } - - return 0; -} -EXPORT_SYMBOL_GPL(sm4_expandkey); - -/** - * sm4_crypt_block - Encrypt or decrypt a single SM4 block - * @rk: The rkey_enc for encrypt or rkey_dec for decrypt - * @out: Buffer to store output data - * @in: Buffer containing the input data - */ -void sm4_crypt_block(const u32 *rk, u8 *out, const u8 *in) -{ - u32 x[4], i; - - x[0] = get_unaligned_be32(in + 0 * 4); - x[1] = get_unaligned_be32(in + 1 * 4); - x[2] = get_unaligned_be32(in + 2 * 4); - x[3] = get_unaligned_be32(in + 3 * 4); - - for (i = 0; i < 32; i += 4) { - x[0] = sm4_round(x[0], x[1], x[2], x[3], rk[i + 0]); - x[1] = sm4_round(x[1], x[2], x[3], x[0], rk[i + 1]); - x[2] = sm4_round(x[2], x[3], x[0], x[1], rk[i + 2]); - x[3] = sm4_round(x[3], x[0], x[1], x[2], rk[i + 3]); - } - - put_unaligned_be32(x[3 - 0], out + 0 * 4); - put_unaligned_be32(x[3 - 1], out + 1 * 4); - put_unaligned_be32(x[3 - 2], out + 2 * 4); - put_unaligned_be32(x[3 - 3], out + 3 * 4); -} -EXPORT_SYMBOL_GPL(sm4_crypt_block); - -MODULE_DESCRIPTION("Generic SM4 library"); -MODULE_LICENSE("GPL v2"); diff --git a/lib/debugobjects.c b/lib/debugobjects.c index 6946f8e204e3..337d797a7141 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -1,11 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Generic infrastructure for lifetime debugging of objects. * - * Started by Thomas Gleixner - * * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de> - * - * For licencing details see kernel-base/COPYING */ #define pr_fmt(fmt) "ODEBUG: " fmt diff --git a/lib/dim/net_dim.c b/lib/dim/net_dim.c index 06811d866775..53f6b9c6e936 100644 --- a/lib/dim/net_dim.c +++ b/lib/dim/net_dim.c @@ -12,41 +12,41 @@ * Each profile size must be of NET_DIM_PARAMS_NUM_PROFILES */ #define NET_DIM_PARAMS_NUM_PROFILES 5 -#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 -#define NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE 128 +#define NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE 256 +#define NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE 128 #define NET_DIM_DEF_PROFILE_CQE 1 #define NET_DIM_DEF_PROFILE_EQE 1 #define NET_DIM_RX_EQE_PROFILES { \ - {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {.usec = 1, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 8, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 64, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 128, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 256, .pkts = NET_DIM_DEFAULT_RX_CQ_PKTS_FROM_EQE,} \ } #define NET_DIM_RX_CQE_PROFILES { \ - {2, 256}, \ - {8, 128}, \ - {16, 64}, \ - {32, 64}, \ - {64, 64} \ + {.usec = 2, .pkts = 256,}, \ + {.usec = 8, .pkts = 128,}, \ + {.usec = 16, .pkts = 64,}, \ + {.usec = 32, .pkts = 64,}, \ + {.usec = 64, .pkts = 64,} \ } #define NET_DIM_TX_EQE_PROFILES { \ - {1, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {8, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {32, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {64, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {128, NET_DIM_DEFAULT_TX_CQ_MODERATION_PKTS_FROM_EQE} \ + {.usec = 1, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 8, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 32, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 64, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,}, \ + {.usec = 128, .pkts = NET_DIM_DEFAULT_TX_CQ_PKTS_FROM_EQE,} \ } #define NET_DIM_TX_CQE_PROFILES { \ - {5, 128}, \ - {8, 64}, \ - {16, 32}, \ - {32, 32}, \ - {64, 32} \ + {.usec = 5, .pkts = 128,}, \ + {.usec = 8, .pkts = 64,}, \ + {.usec = 16, .pkts = 32,}, \ + {.usec = 32, .pkts = 32,}, \ + {.usec = 64, .pkts = 32,} \ } static const struct dim_cq_moder diff --git a/lib/dump_stack.c b/lib/dump_stack.c index 6b7f1bf6715d..83471e81501a 100644 --- a/lib/dump_stack.c +++ b/lib/dump_stack.c @@ -102,9 +102,9 @@ asmlinkage __visible void dump_stack_lvl(const char *log_lvl) * Permit this cpu to perform nested stack dumps while serialising * against other CPUs */ - printk_cpu_lock_irqsave(flags); + printk_cpu_sync_get_irqsave(flags); __dump_stack(log_lvl); - printk_cpu_unlock_irqrestore(flags); + printk_cpu_sync_put_irqrestore(flags); } EXPORT_SYMBOL(dump_stack_lvl); diff --git a/lib/fault-inject.c b/lib/fault-inject.c index ce12621b4275..423784d9c058 100644 --- a/lib/fault-inject.c +++ b/lib/fault-inject.c @@ -41,6 +41,9 @@ EXPORT_SYMBOL_GPL(setup_fault_attr); static void fail_dump(struct fault_attr *attr) { + if (attr->no_warn) + return; + if (attr->verbose > 0 && __ratelimit(&attr->ratelimit_state)) { printk(KERN_NOTICE "FAULT_INJECTION: forcing a failure.\n" "name %pd, interval %lu, probability %lu, " diff --git a/lib/gen_crc64table.c b/lib/gen_crc64table.c index 094b43aef8db..55e222acd0b8 100644 --- a/lib/gen_crc64table.c +++ b/lib/gen_crc64table.c @@ -17,10 +17,30 @@ #include <stdio.h> #define CRC64_ECMA182_POLY 0x42F0E1EBA9EA3693ULL +#define CRC64_ROCKSOFT_POLY 0x9A6C9329AC4BC9B5ULL static uint64_t crc64_table[256] = {0}; +static uint64_t crc64_rocksoft_table[256] = {0}; -static void generate_crc64_table(void) +static void generate_reflected_crc64_table(uint64_t table[256], uint64_t poly) +{ + uint64_t i, j, c, crc; + + for (i = 0; i < 256; i++) { + crc = 0ULL; + c = i; + + for (j = 0; j < 8; j++) { + if ((crc ^ (c >> j)) & 1) + crc = (crc >> 1) ^ poly; + else + crc >>= 1; + } + table[i] = crc; + } +} + +static void generate_crc64_table(uint64_t table[256], uint64_t poly) { uint64_t i, j, c, crc; @@ -30,26 +50,22 @@ static void generate_crc64_table(void) for (j = 0; j < 8; j++) { if ((crc ^ c) & 0x8000000000000000ULL) - crc = (crc << 1) ^ CRC64_ECMA182_POLY; + crc = (crc << 1) ^ poly; else crc <<= 1; c <<= 1; } - crc64_table[i] = crc; + table[i] = crc; } } -static void print_crc64_table(void) +static void output_table(uint64_t table[256]) { int i; - printf("/* this file is generated - do not edit */\n\n"); - printf("#include <linux/types.h>\n"); - printf("#include <linux/cache.h>\n\n"); - printf("static const u64 ____cacheline_aligned crc64table[256] = {\n"); for (i = 0; i < 256; i++) { - printf("\t0x%016" PRIx64 "ULL", crc64_table[i]); + printf("\t0x%016" PRIx64 "ULL", table[i]); if (i & 0x1) printf(",\n"); else @@ -58,9 +74,22 @@ static void print_crc64_table(void) printf("};\n"); } +static void print_crc64_tables(void) +{ + printf("/* this file is generated - do not edit */\n\n"); + printf("#include <linux/types.h>\n"); + printf("#include <linux/cache.h>\n\n"); + printf("static const u64 ____cacheline_aligned crc64table[256] = {\n"); + output_table(crc64_table); + + printf("\nstatic const u64 ____cacheline_aligned crc64rocksofttable[256] = {\n"); + output_table(crc64_rocksoft_table); +} + int main(int argc, char *argv[]) { - generate_crc64_table(); - print_crc64_table(); + generate_crc64_table(crc64_table, CRC64_ECMA182_POLY); + generate_reflected_crc64_table(crc64_rocksoft_table, CRC64_ROCKSOFT_POLY); + print_crc64_tables(); return 0; } diff --git a/lib/glob.c b/lib/glob.c index 85ecbda45cd8..15b73f490720 100644 --- a/lib/glob.c +++ b/lib/glob.c @@ -45,7 +45,7 @@ bool __pure glob_match(char const *pat, char const *str) * (no exception for /), it can be easily proved that there's * never a need to backtrack multiple levels. */ - char const *back_pat = NULL, *back_str = back_str; + char const *back_pat = NULL, *back_str; /* * Loop over each token (character or class) in pat, matching diff --git a/lib/hexdump.c b/lib/hexdump.c index 9301578f98e8..06833d404398 100644 --- a/lib/hexdump.c +++ b/lib/hexdump.c @@ -22,15 +22,33 @@ EXPORT_SYMBOL(hex_asc_upper); * * hex_to_bin() converts one hex digit to its actual value or -1 in case of bad * input. + * + * This function is used to load cryptographic keys, so it is coded in such a + * way that there are no conditions or memory accesses that depend on data. + * + * Explanation of the logic: + * (ch - '9' - 1) is negative if ch <= '9' + * ('0' - 1 - ch) is negative if ch >= '0' + * we "and" these two values, so the result is negative if ch is in the range + * '0' ... '9' + * we are only interested in the sign, so we do a shift ">> 8"; note that right + * shift of a negative value is implementation-defined, so we cast the + * value to (unsigned) before the shift --- we have 0xffffff if ch is in + * the range '0' ... '9', 0 otherwise + * we "and" this value with (ch - '0' + 1) --- we have a value 1 ... 10 if ch is + * in the range '0' ... '9', 0 otherwise + * we add this value to -1 --- we have a value 0 ... 9 if ch is in the range '0' + * ... '9', -1 otherwise + * the next line is similar to the previous one, but we need to decode both + * uppercase and lowercase letters, so we use (ch & 0xdf), which converts + * lowercase to uppercase */ -int hex_to_bin(char ch) +int hex_to_bin(unsigned char ch) { - if ((ch >= '0') && (ch <= '9')) - return ch - '0'; - ch = tolower(ch); - if ((ch >= 'a') && (ch <= 'f')) - return ch - 'a' + 10; - return -1; + unsigned char cu = ch & 0xdf; + return -1 + + ((ch - '0' + 1) & (unsigned)((ch - '9' - 1) & ('0' - 1 - ch)) >> 8) + + ((cu - 'A' + 11) & (unsigned)((cu - 'F' - 1) & ('A' - 1 - cu)) >> 8); } EXPORT_SYMBOL(hex_to_bin); @@ -45,10 +63,13 @@ EXPORT_SYMBOL(hex_to_bin); int hex2bin(u8 *dst, const char *src, size_t count) { while (count--) { - int hi = hex_to_bin(*src++); - int lo = hex_to_bin(*src++); + int hi, lo; - if ((hi < 0) || (lo < 0)) + hi = hex_to_bin(*src++); + if (unlikely(hi < 0)) + return -EINVAL; + lo = hex_to_bin(*src++); + if (unlikely(lo < 0)) return -EINVAL; *dst++ = (hi << 4) | lo; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 6dd5330f7a99..0b64695ab632 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1434,7 +1434,7 @@ static ssize_t iter_xarray_get_pages(struct iov_iter *i, { unsigned nr, offset; pgoff_t index, count; - size_t size = maxsize, actual; + size_t size = maxsize; loff_t pos; if (!size || !maxpages) @@ -1461,13 +1461,7 @@ static ssize_t iter_xarray_get_pages(struct iov_iter *i, if (nr == 0) return 0; - actual = PAGE_SIZE * nr; - actual -= offset; - if (nr == count && size > 0) { - unsigned last_offset = (nr > 1) ? 0 : offset; - actual -= PAGE_SIZE - (last_offset + size); - } - return actual; + return min_t(size_t, nr * PAGE_SIZE - offset, maxsize); } /* must be done on non-empty ITER_IOVEC one */ @@ -1602,7 +1596,7 @@ static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i, struct page **p; unsigned nr, offset; pgoff_t index, count; - size_t size = maxsize, actual; + size_t size = maxsize; loff_t pos; if (!size) @@ -1631,13 +1625,7 @@ static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i, if (nr == 0) return 0; - actual = PAGE_SIZE * nr; - actual -= offset; - if (nr == count && size > 0) { - unsigned last_offset = (nr > 1) ? 0 : offset; - actual -= PAGE_SIZE - (last_offset + size); - } - return actual; + return min_t(size_t, nr * PAGE_SIZE - offset, maxsize); } ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, diff --git a/lib/irq_poll.c b/lib/irq_poll.c index 2f17b488d58e..2d5329a42105 100644 --- a/lib/irq_poll.c +++ b/lib/irq_poll.c @@ -188,14 +188,18 @@ EXPORT_SYMBOL(irq_poll_init); static int irq_poll_cpu_dead(unsigned int cpu) { /* - * If a CPU goes away, splice its entries to the current CPU - * and trigger a run of the softirq + * If a CPU goes away, splice its entries to the current CPU and + * set the POLL softirq bit. The local_bh_disable()/enable() pair + * ensures that it is handled. Otherwise the current CPU could + * reach idle with the POLL softirq pending. */ + local_bh_disable(); local_irq_disable(); list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), this_cpu_ptr(&blk_cpu_iopoll)); __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); local_irq_enable(); + local_bh_enable(); return 0; } diff --git a/lib/kobject.c b/lib/kobject.c index 56fa037501b5..5f0e71ab292c 100644 --- a/lib/kobject.c +++ b/lib/kobject.c @@ -54,32 +54,6 @@ void kobject_get_ownership(struct kobject *kobj, kuid_t *uid, kgid_t *gid) kobj->ktype->get_ownership(kobj, uid, gid); } -/* - * populate_dir - populate directory with attributes. - * @kobj: object we're working on. - * - * Most subsystems have a set of default attributes that are associated - * with an object that registers with them. This is a helper called during - * object registration that loops through the default attributes of the - * subsystem and creates attributes files for them in sysfs. - */ -static int populate_dir(struct kobject *kobj) -{ - const struct kobj_type *t = get_ktype(kobj); - struct attribute *attr; - int error = 0; - int i; - - if (t && t->default_attrs) { - for (i = 0; (attr = t->default_attrs[i]) != NULL; i++) { - error = sysfs_create_file(kobj, attr); - if (error) - break; - } - } - return error; -} - static int create_dir(struct kobject *kobj) { const struct kobj_type *ktype = get_ktype(kobj); @@ -90,12 +64,6 @@ static int create_dir(struct kobject *kobj) if (error) return error; - error = populate_dir(kobj); - if (error) { - sysfs_remove_dir(kobj); - return error; - } - if (ktype) { error = sysfs_create_groups(kobj, ktype->default_groups); if (error) { diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 886510d248e5..08c14019841a 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -340,7 +340,7 @@ EXPORT_SYMBOL(kstrtos8); * @s: input string * @res: result * - * This routine returns 0 iff the first character is one of 'Yy1Nn0', or + * This routine returns 0 iff the first character is one of 'YyTt1NnFf0', or * [oO][NnFf] for "on" and "off". Otherwise it will return -EINVAL. Value * pointed to by res is updated upon finding a match. */ @@ -353,11 +353,15 @@ int kstrtobool(const char *s, bool *res) switch (s[0]) { case 'y': case 'Y': + case 't': + case 'T': case '1': *res = true; return 0; case 'n': case 'N': + case 'f': + case 'F': case '0': *res = false; return 0; diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index c49f4ffb6273..29aff6562b42 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -1,6 +1,7 @@ obj-$(CONFIG_KUNIT) += kunit.o kunit-objs += test.o \ + resource.o \ string-stream.o \ assert.o \ try-catch.o \ diff --git a/lib/kunit/assert.c b/lib/kunit/assert.c index b972bda61c0c..d00d6d181ee8 100644 --- a/lib/kunit/assert.c +++ b/lib/kunit/assert.c @@ -10,12 +10,13 @@ #include "string-stream.h" -void kunit_base_assert_format(const struct kunit_assert *assert, +void kunit_assert_prologue(const struct kunit_loc *loc, + enum kunit_assert_type type, struct string_stream *stream) { const char *expect_or_assert = NULL; - switch (assert->type) { + switch (type) { case KUNIT_EXPECTATION: expect_or_assert = "EXPECTATION"; break; @@ -25,34 +26,33 @@ void kunit_base_assert_format(const struct kunit_assert *assert, } string_stream_add(stream, "%s FAILED at %s:%d\n", - expect_or_assert, assert->file, assert->line); + expect_or_assert, loc->file, loc->line); } -EXPORT_SYMBOL_GPL(kunit_base_assert_format); +EXPORT_SYMBOL_GPL(kunit_assert_prologue); -void kunit_assert_print_msg(const struct kunit_assert *assert, - struct string_stream *stream) +static void kunit_assert_print_msg(const struct va_format *message, + struct string_stream *stream) { - if (assert->message.fmt) - string_stream_add(stream, "\n%pV", &assert->message); + if (message->fmt) + string_stream_add(stream, "\n%pV", message); } -EXPORT_SYMBOL_GPL(kunit_assert_print_msg); void kunit_fail_assert_format(const struct kunit_assert *assert, + const struct va_format *message, struct string_stream *stream) { - kunit_base_assert_format(assert, stream); - string_stream_add(stream, "%pV", &assert->message); + string_stream_add(stream, "%pV", message); } EXPORT_SYMBOL_GPL(kunit_fail_assert_format); void kunit_unary_assert_format(const struct kunit_assert *assert, + const struct va_format *message, struct string_stream *stream) { struct kunit_unary_assert *unary_assert; unary_assert = container_of(assert, struct kunit_unary_assert, assert); - kunit_base_assert_format(assert, stream); if (unary_assert->expected_true) string_stream_add(stream, KUNIT_SUBTEST_INDENT "Expected %s to be true, but is false\n", @@ -61,11 +61,12 @@ void kunit_unary_assert_format(const struct kunit_assert *assert, string_stream_add(stream, KUNIT_SUBTEST_INDENT "Expected %s to be false, but is true\n", unary_assert->condition); - kunit_assert_print_msg(assert, stream); + kunit_assert_print_msg(message, stream); } EXPORT_SYMBOL_GPL(kunit_unary_assert_format); void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, + const struct va_format *message, struct string_stream *stream) { struct kunit_ptr_not_err_assert *ptr_assert; @@ -73,7 +74,6 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, ptr_assert = container_of(assert, struct kunit_ptr_not_err_assert, assert); - kunit_base_assert_format(assert, stream); if (!ptr_assert->value) { string_stream_add(stream, KUNIT_SUBTEST_INDENT "Expected %s is not null, but is\n", @@ -84,7 +84,7 @@ void kunit_ptr_not_err_assert_format(const struct kunit_assert *assert, ptr_assert->text, PTR_ERR(ptr_assert->value)); } - kunit_assert_print_msg(assert, stream); + kunit_assert_print_msg(message, stream); } EXPORT_SYMBOL_GPL(kunit_ptr_not_err_assert_format); @@ -112,6 +112,7 @@ static bool is_literal(struct kunit *test, const char *text, long long value, } void kunit_binary_assert_format(const struct kunit_assert *assert, + const struct va_format *message, struct string_stream *stream) { struct kunit_binary_assert *binary_assert; @@ -119,27 +120,27 @@ void kunit_binary_assert_format(const struct kunit_assert *assert, binary_assert = container_of(assert, struct kunit_binary_assert, assert); - kunit_base_assert_format(assert, stream); string_stream_add(stream, KUNIT_SUBTEST_INDENT "Expected %s %s %s, but\n", - binary_assert->left_text, - binary_assert->operation, - binary_assert->right_text); - if (!is_literal(stream->test, binary_assert->left_text, + binary_assert->text->left_text, + binary_assert->text->operation, + binary_assert->text->right_text); + if (!is_literal(stream->test, binary_assert->text->left_text, binary_assert->left_value, stream->gfp)) string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld\n", - binary_assert->left_text, + binary_assert->text->left_text, binary_assert->left_value); - if (!is_literal(stream->test, binary_assert->right_text, + if (!is_literal(stream->test, binary_assert->text->right_text, binary_assert->right_value, stream->gfp)) string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %lld", - binary_assert->right_text, + binary_assert->text->right_text, binary_assert->right_value); - kunit_assert_print_msg(assert, stream); + kunit_assert_print_msg(message, stream); } EXPORT_SYMBOL_GPL(kunit_binary_assert_format); void kunit_binary_ptr_assert_format(const struct kunit_assert *assert, + const struct va_format *message, struct string_stream *stream) { struct kunit_binary_ptr_assert *binary_assert; @@ -147,19 +148,18 @@ void kunit_binary_ptr_assert_format(const struct kunit_assert *assert, binary_assert = container_of(assert, struct kunit_binary_ptr_assert, assert); - kunit_base_assert_format(assert, stream); string_stream_add(stream, KUNIT_SUBTEST_INDENT "Expected %s %s %s, but\n", - binary_assert->left_text, - binary_assert->operation, - binary_assert->right_text); + binary_assert->text->left_text, + binary_assert->text->operation, + binary_assert->text->right_text); string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %px\n", - binary_assert->left_text, + binary_assert->text->left_text, binary_assert->left_value); string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == %px", - binary_assert->right_text, + binary_assert->text->right_text, binary_assert->right_value); - kunit_assert_print_msg(assert, stream); + kunit_assert_print_msg(message, stream); } EXPORT_SYMBOL_GPL(kunit_binary_ptr_assert_format); @@ -180,6 +180,7 @@ static bool is_str_literal(const char *text, const char *value) } void kunit_binary_str_assert_format(const struct kunit_assert *assert, + const struct va_format *message, struct string_stream *stream) { struct kunit_binary_str_assert *binary_assert; @@ -187,20 +188,19 @@ void kunit_binary_str_assert_format(const struct kunit_assert *assert, binary_assert = container_of(assert, struct kunit_binary_str_assert, assert); - kunit_base_assert_format(assert, stream); string_stream_add(stream, KUNIT_SUBTEST_INDENT "Expected %s %s %s, but\n", - binary_assert->left_text, - binary_assert->operation, - binary_assert->right_text); - if (!is_str_literal(binary_assert->left_text, binary_assert->left_value)) + binary_assert->text->left_text, + binary_assert->text->operation, + binary_assert->text->right_text); + if (!is_str_literal(binary_assert->text->left_text, binary_assert->left_value)) string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == \"%s\"\n", - binary_assert->left_text, + binary_assert->text->left_text, binary_assert->left_value); - if (!is_str_literal(binary_assert->right_text, binary_assert->right_value)) + if (!is_str_literal(binary_assert->text->right_text, binary_assert->right_value)) string_stream_add(stream, KUNIT_SUBSUBTEST_INDENT "%s == \"%s\"", - binary_assert->right_text, + binary_assert->text->right_text, binary_assert->right_value); - kunit_assert_print_msg(assert, stream); + kunit_assert_print_msg(message, stream); } EXPORT_SYMBOL_GPL(kunit_binary_str_assert_format); diff --git a/lib/kunit/debugfs.c b/lib/kunit/debugfs.c index b71db0abc12b..1048ef1b8d6e 100644 --- a/lib/kunit/debugfs.c +++ b/lib/kunit/debugfs.c @@ -52,7 +52,7 @@ static void debugfs_print_result(struct seq_file *seq, static int debugfs_print_results(struct seq_file *seq, void *v) { struct kunit_suite *suite = (struct kunit_suite *)seq->private; - bool success = kunit_suite_has_succeeded(suite); + enum kunit_status success = kunit_suite_has_succeeded(suite); struct kunit_case *test_case; if (!suite || !suite->log) diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c index 22640c9ee819..96f96e42ce06 100644 --- a/lib/kunit/executor.c +++ b/lib/kunit/executor.c @@ -71,9 +71,13 @@ kunit_filter_tests(struct kunit_suite *const suite, const char *test_glob) /* Use memcpy to workaround copy->name being const. */ copy = kmalloc(sizeof(*copy), GFP_KERNEL); + if (!copy) + return ERR_PTR(-ENOMEM); memcpy(copy, suite, sizeof(*copy)); filtered = kcalloc(n + 1, sizeof(*filtered), GFP_KERNEL); + if (!filtered) + return ERR_PTR(-ENOMEM); n = 0; kunit_suite_for_each_test_case(suite, test_case) { @@ -106,14 +110,16 @@ kunit_filter_subsuite(struct kunit_suite * const * const subsuite, filtered = kmalloc_array(n + 1, sizeof(*filtered), GFP_KERNEL); if (!filtered) - return NULL; + return ERR_PTR(-ENOMEM); n = 0; for (i = 0; subsuite[i] != NULL; ++i) { if (!glob_match(filter->suite_glob, subsuite[i]->name)) continue; filtered_suite = kunit_filter_tests(subsuite[i], filter->test_glob); - if (filtered_suite) + if (IS_ERR(filtered_suite)) + return ERR_CAST(filtered_suite); + else if (filtered_suite) filtered[n++] = filtered_suite; } filtered[n] = NULL; @@ -146,7 +152,8 @@ static void kunit_free_suite_set(struct suite_set suite_set) } static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, - const char *filter_glob) + const char *filter_glob, + int *err) { int i; struct kunit_suite * const **copy, * const *filtered_subsuite; @@ -166,6 +173,10 @@ static struct suite_set kunit_filter_suites(const struct suite_set *suite_set, for (i = 0; i < max; ++i) { filtered_subsuite = kunit_filter_subsuite(suite_set->start[i], &filter); + if (IS_ERR(filtered_subsuite)) { + *err = PTR_ERR(filtered_subsuite); + return filtered; + } if (filtered_subsuite) *copy++ = filtered_subsuite; } @@ -236,9 +247,15 @@ int kunit_run_all_tests(void) .start = __kunit_suites_start, .end = __kunit_suites_end, }; + int err = 0; - if (filter_glob_param) - suite_set = kunit_filter_suites(&suite_set, filter_glob_param); + if (filter_glob_param) { + suite_set = kunit_filter_suites(&suite_set, filter_glob_param, &err); + if (err) { + pr_err("kunit executor: error filtering suites: %d\n", err); + goto out; + } + } if (!action_param) kunit_exec_run_tests(&suite_set); @@ -251,9 +268,10 @@ int kunit_run_all_tests(void) kunit_free_suite_set(suite_set); } - kunit_handle_shutdown(); - return 0; +out: + kunit_handle_shutdown(); + return err; } #if IS_BUILTIN(CONFIG_KUNIT_TEST) diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c index 4ed57fd94e42..eac6ff480273 100644 --- a/lib/kunit/executor_test.c +++ b/lib/kunit/executor_test.c @@ -137,14 +137,16 @@ static void filter_suites_test(struct kunit *test) .end = suites + 2, }; struct suite_set filtered = {.start = NULL, .end = NULL}; + int err = 0; /* Emulate two files, each having one suite */ subsuites[0][0] = alloc_fake_suite(test, "suite0", dummy_test_cases); subsuites[1][0] = alloc_fake_suite(test, "suite1", dummy_test_cases); /* Filter out suite1 */ - filtered = kunit_filter_suites(&suite_set, "suite0"); + filtered = kunit_filter_suites(&suite_set, "suite0", &err); kfree_subsuites_at_end(test, &filtered); /* let us use ASSERTs without leaking */ + KUNIT_EXPECT_EQ(test, err, 0); KUNIT_ASSERT_EQ(test, filtered.end - filtered.start, (ptrdiff_t)1); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, filtered.start); diff --git a/lib/kunit/kunit-example-test.c b/lib/kunit/kunit-example-test.c index 51099b0ca29c..f8fe582c9e36 100644 --- a/lib/kunit/kunit-example-test.c +++ b/lib/kunit/kunit-example-test.c @@ -41,6 +41,17 @@ static int example_test_init(struct kunit *test) } /* + * This is run once before all test cases in the suite. + * See the comment on example_test_suite for more information. + */ +static int example_test_init_suite(struct kunit_suite *suite) +{ + kunit_info(suite, "initializing suite\n"); + + return 0; +} + +/* * This test should always be skipped. */ static void example_skip_test(struct kunit *test) @@ -69,6 +80,49 @@ static void example_mark_skipped_test(struct kunit *test) /* This line should run */ kunit_info(test, "You should see this line."); } + +/* + * This test shows off all the types of KUNIT_EXPECT macros. + */ +static void example_all_expect_macros_test(struct kunit *test) +{ + /* Boolean assertions */ + KUNIT_EXPECT_TRUE(test, true); + KUNIT_EXPECT_FALSE(test, false); + + /* Integer assertions */ + KUNIT_EXPECT_EQ(test, 1, 1); /* check == */ + KUNIT_EXPECT_GE(test, 1, 1); /* check >= */ + KUNIT_EXPECT_LE(test, 1, 1); /* check <= */ + KUNIT_EXPECT_NE(test, 1, 0); /* check != */ + KUNIT_EXPECT_GT(test, 1, 0); /* check > */ + KUNIT_EXPECT_LT(test, 0, 1); /* check < */ + + /* Pointer assertions */ + KUNIT_EXPECT_NOT_ERR_OR_NULL(test, test); + KUNIT_EXPECT_PTR_EQ(test, NULL, NULL); + KUNIT_EXPECT_PTR_NE(test, test, NULL); + KUNIT_EXPECT_NULL(test, NULL); + KUNIT_EXPECT_NOT_NULL(test, test); + + /* String assertions */ + KUNIT_EXPECT_STREQ(test, "hi", "hi"); + KUNIT_EXPECT_STRNEQ(test, "hi", "bye"); + + /* + * There are also ASSERT variants of all of the above that abort test + * execution if they fail. Useful for memory allocations, etc. + */ + KUNIT_ASSERT_GT(test, sizeof(char), 0); + + /* + * There are also _MSG variants of all of the above that let you include + * additional text on failure. + */ + KUNIT_EXPECT_GT_MSG(test, sizeof(int), 0, "Your ints are 0-bit?!"); + KUNIT_ASSERT_GT_MSG(test, sizeof(int), 0, "Your ints are 0-bit?!"); +} + /* * Here we make a list of all the test cases we want to add to the test suite * below. @@ -83,6 +137,7 @@ static struct kunit_case example_test_cases[] = { KUNIT_CASE(example_simple_test), KUNIT_CASE(example_skip_test), KUNIT_CASE(example_mark_skipped_test), + KUNIT_CASE(example_all_expect_macros_test), {} }; @@ -98,17 +153,20 @@ static struct kunit_case example_test_cases[] = { * may be specified which runs after every test case and can be used to for * cleanup. For clarity, running tests in a test suite would behave as follows: * + * suite.suite_init(suite); * suite.init(test); * suite.test_case[0](test); * suite.exit(test); * suite.init(test); * suite.test_case[1](test); * suite.exit(test); + * suite.suite_exit(suite); * ...; */ static struct kunit_suite example_test_suite = { .name = "example", .init = example_test_init, + .suite_init = example_test_init_suite, .test_cases = example_test_cases, }; diff --git a/lib/kunit/kunit-test.c b/lib/kunit/kunit-test.c index 555601d17f79..13d0bd8b07a9 100644 --- a/lib/kunit/kunit-test.c +++ b/lib/kunit/kunit-test.c @@ -190,6 +190,40 @@ static void kunit_resource_test_destroy_resource(struct kunit *test) KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); } +static void kunit_resource_test_remove_resource(struct kunit *test) +{ + struct kunit_test_resource_context *ctx = test->priv; + struct kunit_resource *res = kunit_alloc_and_get_resource( + &ctx->test, + fake_resource_init, + fake_resource_free, + GFP_KERNEL, + ctx); + + /* The resource is in the list */ + KUNIT_EXPECT_FALSE(test, list_empty(&ctx->test.resources)); + + /* Remove the resource. The pointer is still valid, but it can't be + * found. + */ + kunit_remove_resource(test, res); + KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); + /* We haven't been freed yet. */ + KUNIT_EXPECT_TRUE(test, ctx->is_resource_initialized); + + /* Removing the resource multiple times is valid. */ + kunit_remove_resource(test, res); + KUNIT_EXPECT_TRUE(test, list_empty(&ctx->test.resources)); + /* Despite having been removed twice (from only one reference), the + * resource still has not been freed. + */ + KUNIT_EXPECT_TRUE(test, ctx->is_resource_initialized); + + /* Free the resource. */ + kunit_put_resource(res); + KUNIT_EXPECT_FALSE(test, ctx->is_resource_initialized); +} + static void kunit_resource_test_cleanup_resources(struct kunit *test) { int i; @@ -387,6 +421,7 @@ static struct kunit_case kunit_resource_test_cases[] = { KUNIT_CASE(kunit_resource_test_init_resources), KUNIT_CASE(kunit_resource_test_alloc_resource), KUNIT_CASE(kunit_resource_test_destroy_resource), + KUNIT_CASE(kunit_resource_test_remove_resource), KUNIT_CASE(kunit_resource_test_cleanup_resources), KUNIT_CASE(kunit_resource_test_proper_free_ordering), KUNIT_CASE(kunit_resource_test_static), @@ -435,7 +470,7 @@ static void kunit_log_test(struct kunit *test) KUNIT_EXPECT_NOT_ERR_OR_NULL(test, strstr(suite.log, "along with this.")); #else - KUNIT_EXPECT_PTR_EQ(test, test->log, (char *)NULL); + KUNIT_EXPECT_NULL(test, test->log); #endif } diff --git a/lib/kunit/resource.c b/lib/kunit/resource.c new file mode 100644 index 000000000000..c414df922f34 --- /dev/null +++ b/lib/kunit/resource.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit resource API for test managed resources (allocations, etc.). + * + * Copyright (C) 2022, Google LLC. + * Author: Daniel Latypov <dlatypov@google.com> + */ + +#include <kunit/resource.h> +#include <kunit/test.h> +#include <linux/kref.h> + +/* + * Used for static resources and when a kunit_resource * has been created by + * kunit_alloc_resource(). When an init function is supplied, @data is passed + * into the init function; otherwise, we simply set the resource data field to + * the data value passed in. Doesn't initialize res->should_kfree. + */ +int __kunit_add_resource(struct kunit *test, + kunit_resource_init_t init, + kunit_resource_free_t free, + struct kunit_resource *res, + void *data) +{ + int ret = 0; + unsigned long flags; + + res->free = free; + kref_init(&res->refcount); + + if (init) { + ret = init(res, data); + if (ret) + return ret; + } else { + res->data = data; + } + + spin_lock_irqsave(&test->lock, flags); + list_add_tail(&res->node, &test->resources); + /* refcount for list is established by kref_init() */ + spin_unlock_irqrestore(&test->lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(__kunit_add_resource); + +void kunit_remove_resource(struct kunit *test, struct kunit_resource *res) +{ + unsigned long flags; + bool was_linked; + + spin_lock_irqsave(&test->lock, flags); + was_linked = !list_empty(&res->node); + list_del_init(&res->node); + spin_unlock_irqrestore(&test->lock, flags); + + if (was_linked) + kunit_put_resource(res); +} +EXPORT_SYMBOL_GPL(kunit_remove_resource); + +int kunit_destroy_resource(struct kunit *test, kunit_resource_match_t match, + void *match_data) +{ + struct kunit_resource *res = kunit_find_resource(test, match, + match_data); + + if (!res) + return -ENOENT; + + kunit_remove_resource(test, res); + + /* We have a reference also via _find(); drop it. */ + kunit_put_resource(res); + + return 0; +} +EXPORT_SYMBOL_GPL(kunit_destroy_resource); diff --git a/lib/kunit/test.c b/lib/kunit/test.c index c7ed4aabec04..a5053a07409f 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -6,10 +6,10 @@ * Author: Brendan Higgins <brendanhiggins@google.com> */ +#include <kunit/resource.h> #include <kunit/test.h> #include <kunit/test-bug.h> #include <linux/kernel.h> -#include <linux/kref.h> #include <linux/moduleparam.h> #include <linux/sched/debug.h> #include <linux/sched.h> @@ -134,7 +134,7 @@ size_t kunit_suite_num_test_cases(struct kunit_suite *suite) } EXPORT_SYMBOL_GPL(kunit_suite_num_test_cases); -static void kunit_print_subtest_start(struct kunit_suite *suite) +static void kunit_print_suite_start(struct kunit_suite *suite) { kunit_log(KERN_INFO, suite, KUNIT_SUBTEST_INDENT "# Subtest: %s", suite->name); @@ -179,6 +179,9 @@ enum kunit_status kunit_suite_has_succeeded(struct kunit_suite *suite) const struct kunit_case *test_case; enum kunit_status status = KUNIT_SKIPPED; + if (suite->suite_init_err) + return KUNIT_FAILURE; + kunit_suite_for_each_test_case(suite, test_case) { if (test_case->status == KUNIT_FAILURE) return KUNIT_FAILURE; @@ -192,7 +195,7 @@ EXPORT_SYMBOL_GPL(kunit_suite_has_succeeded); static size_t kunit_suite_counter = 1; -static void kunit_print_subtest_end(struct kunit_suite *suite) +static void kunit_print_suite_end(struct kunit_suite *suite) { kunit_print_ok_not_ok((void *)suite, false, kunit_suite_has_succeeded(suite), @@ -240,7 +243,9 @@ static void kunit_print_string_stream(struct kunit *test, } } -static void kunit_fail(struct kunit *test, struct kunit_assert *assert) +static void kunit_fail(struct kunit *test, const struct kunit_loc *loc, + enum kunit_assert_type type, const struct kunit_assert *assert, + const struct va_format *message) { struct string_stream *stream; @@ -250,12 +255,13 @@ static void kunit_fail(struct kunit *test, struct kunit_assert *assert) if (!stream) { WARN(true, "Could not allocate stream to print failed assertion in %s:%d\n", - assert->file, - assert->line); + loc->file, + loc->line); return; } - assert->format(assert, stream); + kunit_assert_prologue(loc, type, stream); + assert->format(assert, message, stream); kunit_print_string_stream(test, stream); @@ -275,29 +281,27 @@ static void __noreturn kunit_abort(struct kunit *test) WARN_ONCE(true, "Throw could not abort from test!\n"); } -void kunit_do_assertion(struct kunit *test, - struct kunit_assert *assert, - bool pass, - const char *fmt, ...) +void kunit_do_failed_assertion(struct kunit *test, + const struct kunit_loc *loc, + enum kunit_assert_type type, + const struct kunit_assert *assert, + const char *fmt, ...) { va_list args; - - if (pass) - return; - + struct va_format message; va_start(args, fmt); - assert->message.fmt = fmt; - assert->message.va = &args; + message.fmt = fmt; + message.va = &args; - kunit_fail(test, assert); + kunit_fail(test, loc, type, assert, &message); va_end(args); - if (assert->type == KUNIT_ASSERTION) + if (type == KUNIT_ASSERTION) kunit_abort(test); } -EXPORT_SYMBOL_GPL(kunit_do_assertion); +EXPORT_SYMBOL_GPL(kunit_do_failed_assertion); void kunit_init_test(struct kunit *test, const char *name, char *log) { @@ -497,7 +501,16 @@ int kunit_run_tests(struct kunit_suite *suite) struct kunit_result_stats suite_stats = { 0 }; struct kunit_result_stats total_stats = { 0 }; - kunit_print_subtest_start(suite); + if (suite->suite_init) { + suite->suite_init_err = suite->suite_init(suite); + if (suite->suite_init_err) { + kunit_err(suite, KUNIT_SUBTEST_INDENT + "# failed to initialize (%d)", suite->suite_init_err); + goto suite_end; + } + } + + kunit_print_suite_start(suite); kunit_suite_for_each_test_case(suite, test_case) { struct kunit test = { .param_value = NULL, .param_index = 0 }; @@ -550,8 +563,12 @@ int kunit_run_tests(struct kunit_suite *suite) kunit_accumulate_stats(&total_stats, param_stats); } + if (suite->suite_exit) + suite->suite_exit(suite); + kunit_print_suite_stats(suite, suite_stats, total_stats); - kunit_print_subtest_end(suite); +suite_end: + kunit_print_suite_end(suite); return 0; } @@ -561,6 +578,7 @@ static void kunit_init_suite(struct kunit_suite *suite) { kunit_debugfs_create_suite(suite); suite->status_comment[0] = '\0'; + suite->suite_init_err = 0; } int __kunit_test_suites_init(struct kunit_suite * const * const suites) @@ -591,120 +609,6 @@ void __kunit_test_suites_exit(struct kunit_suite **suites) } EXPORT_SYMBOL_GPL(__kunit_test_suites_exit); -/* - * Used for static resources and when a kunit_resource * has been created by - * kunit_alloc_resource(). When an init function is supplied, @data is passed - * into the init function; otherwise, we simply set the resource data field to - * the data value passed in. - */ -int kunit_add_resource(struct kunit *test, - kunit_resource_init_t init, - kunit_resource_free_t free, - struct kunit_resource *res, - void *data) -{ - int ret = 0; - unsigned long flags; - - res->free = free; - kref_init(&res->refcount); - - if (init) { - ret = init(res, data); - if (ret) - return ret; - } else { - res->data = data; - } - - spin_lock_irqsave(&test->lock, flags); - list_add_tail(&res->node, &test->resources); - /* refcount for list is established by kref_init() */ - spin_unlock_irqrestore(&test->lock, flags); - - return ret; -} -EXPORT_SYMBOL_GPL(kunit_add_resource); - -int kunit_add_named_resource(struct kunit *test, - kunit_resource_init_t init, - kunit_resource_free_t free, - struct kunit_resource *res, - const char *name, - void *data) -{ - struct kunit_resource *existing; - - if (!name) - return -EINVAL; - - existing = kunit_find_named_resource(test, name); - if (existing) { - kunit_put_resource(existing); - return -EEXIST; - } - - res->name = name; - - return kunit_add_resource(test, init, free, res, data); -} -EXPORT_SYMBOL_GPL(kunit_add_named_resource); - -struct kunit_resource *kunit_alloc_and_get_resource(struct kunit *test, - kunit_resource_init_t init, - kunit_resource_free_t free, - gfp_t internal_gfp, - void *data) -{ - struct kunit_resource *res; - int ret; - - res = kzalloc(sizeof(*res), internal_gfp); - if (!res) - return NULL; - - ret = kunit_add_resource(test, init, free, res, data); - if (!ret) { - /* - * bump refcount for get; kunit_resource_put() should be called - * when done. - */ - kunit_get_resource(res); - return res; - } - return NULL; -} -EXPORT_SYMBOL_GPL(kunit_alloc_and_get_resource); - -void kunit_remove_resource(struct kunit *test, struct kunit_resource *res) -{ - unsigned long flags; - - spin_lock_irqsave(&test->lock, flags); - list_del(&res->node); - spin_unlock_irqrestore(&test->lock, flags); - kunit_put_resource(res); -} -EXPORT_SYMBOL_GPL(kunit_remove_resource); - -int kunit_destroy_resource(struct kunit *test, kunit_resource_match_t match, - void *match_data) -{ - struct kunit_resource *res = kunit_find_resource(test, match, - match_data); - - if (!res) - return -ENOENT; - - kunit_remove_resource(test, res); - - /* We have a reference also via _find(); drop it. */ - kunit_put_resource(res); - - return 0; -} -EXPORT_SYMBOL_GPL(kunit_destroy_resource); - struct kunit_kmalloc_array_params { size_t n; size_t size; diff --git a/lib/kunit/try-catch.c b/lib/kunit/try-catch.c index be38a2c5ecc2..f7825991d576 100644 --- a/lib/kunit/try-catch.c +++ b/lib/kunit/try-catch.c @@ -52,7 +52,7 @@ static unsigned long kunit_test_timeout(void) * If tests timeout due to exceeding sysctl_hung_task_timeout_secs, * the task will be killed and an oops generated. */ - return 300 * MSEC_PER_SEC; /* 5 min */ + return 300 * msecs_to_jiffies(MSEC_PER_SEC); /* 5 min */ } void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context) @@ -78,6 +78,7 @@ void kunit_try_catch_run(struct kunit_try_catch *try_catch, void *context) if (time_remaining == 0) { kunit_err(test, "try timed out\n"); try_catch->try_result = -ETIMEDOUT; + kthread_stop(task_struct); } exit_code = try_catch->try_result; diff --git a/lib/list-test.c b/lib/list-test.c index ee09505df16f..d374cf5d1a57 100644 --- a/lib/list-test.c +++ b/lib/list-test.c @@ -161,6 +161,26 @@ static void list_test_list_del_init(struct kunit *test) KUNIT_EXPECT_TRUE(test, list_empty_careful(&a)); } +static void list_test_list_del_init_careful(struct kunit *test) +{ + /* NOTE: This test only checks the behaviour of this function in + * isolation. It does not verify memory model guarantees. + */ + struct list_head a, b; + LIST_HEAD(list); + + list_add_tail(&a, &list); + list_add_tail(&b, &list); + + /* before: [list] -> a -> b */ + list_del_init_careful(&a); + /* after: [list] -> b, a initialised */ + + KUNIT_EXPECT_PTR_EQ(test, list.next, &b); + KUNIT_EXPECT_PTR_EQ(test, b.prev, &list); + KUNIT_EXPECT_TRUE(test, list_empty_careful(&a)); +} + static void list_test_list_move(struct kunit *test) { struct list_head a, b; @@ -234,6 +254,24 @@ static void list_test_list_bulk_move_tail(struct kunit *test) KUNIT_EXPECT_EQ(test, i, 2); } +static void list_test_list_is_head(struct kunit *test) +{ + struct list_head a, b, c; + + /* Two lists: [a] -> b, [c] */ + INIT_LIST_HEAD(&a); + INIT_LIST_HEAD(&c); + list_add_tail(&b, &a); + + KUNIT_EXPECT_TRUE_MSG(test, list_is_head(&a, &a), + "Head element of same list"); + KUNIT_EXPECT_FALSE_MSG(test, list_is_head(&a, &b), + "Non-head element of same list"); + KUNIT_EXPECT_FALSE_MSG(test, list_is_head(&a, &c), + "Head element of different list"); +} + + static void list_test_list_is_first(struct kunit *test) { struct list_head a, b; @@ -511,6 +549,26 @@ static void list_test_list_entry(struct kunit *test) struct list_test_struct, list)); } +static void list_test_list_entry_is_head(struct kunit *test) +{ + struct list_test_struct test_struct1, test_struct2, test_struct3; + + INIT_LIST_HEAD(&test_struct1.list); + INIT_LIST_HEAD(&test_struct3.list); + + list_add_tail(&test_struct2.list, &test_struct1.list); + + KUNIT_EXPECT_TRUE_MSG(test, + list_entry_is_head((&test_struct1), &test_struct1.list, list), + "Head element of same list"); + KUNIT_EXPECT_FALSE_MSG(test, + list_entry_is_head((&test_struct2), &test_struct1.list, list), + "Non-head element of same list"); + KUNIT_EXPECT_FALSE_MSG(test, + list_entry_is_head((&test_struct3), &test_struct1.list, list), + "Head element of different list"); +} + static void list_test_list_first_entry(struct kunit *test) { struct list_test_struct test_struct1, test_struct2; @@ -707,9 +765,11 @@ static struct kunit_case list_test_cases[] = { KUNIT_CASE(list_test_list_replace_init), KUNIT_CASE(list_test_list_swap), KUNIT_CASE(list_test_list_del_init), + KUNIT_CASE(list_test_list_del_init_careful), KUNIT_CASE(list_test_list_move), KUNIT_CASE(list_test_list_move_tail), KUNIT_CASE(list_test_list_bulk_move_tail), + KUNIT_CASE(list_test_list_is_head), KUNIT_CASE(list_test_list_is_first), KUNIT_CASE(list_test_list_is_last), KUNIT_CASE(list_test_list_empty), @@ -724,6 +784,7 @@ static struct kunit_case list_test_cases[] = { KUNIT_CASE(list_test_list_splice_init), KUNIT_CASE(list_test_list_splice_tail_init), KUNIT_CASE(list_test_list_entry), + KUNIT_CASE(list_test_list_entry_is_head), KUNIT_CASE(list_test_list_first_entry), KUNIT_CASE(list_test_list_last_entry), KUNIT_CASE(list_test_list_first_entry_or_null), @@ -743,6 +804,401 @@ static struct kunit_suite list_test_module = { .test_cases = list_test_cases, }; -kunit_test_suites(&list_test_module); +struct hlist_test_struct { + int data; + struct hlist_node list; +}; + +static void hlist_test_init(struct kunit *test) +{ + /* Test the different ways of initialising a list. */ + struct hlist_head list1 = HLIST_HEAD_INIT; + struct hlist_head list2; + HLIST_HEAD(list3); + struct hlist_head *list4; + struct hlist_head *list5; + + INIT_HLIST_HEAD(&list2); + + list4 = kzalloc(sizeof(*list4), GFP_KERNEL | __GFP_NOFAIL); + INIT_HLIST_HEAD(list4); + + list5 = kmalloc(sizeof(*list5), GFP_KERNEL | __GFP_NOFAIL); + memset(list5, 0xFF, sizeof(*list5)); + INIT_HLIST_HEAD(list5); + + KUNIT_EXPECT_TRUE(test, hlist_empty(&list1)); + KUNIT_EXPECT_TRUE(test, hlist_empty(&list2)); + KUNIT_EXPECT_TRUE(test, hlist_empty(&list3)); + KUNIT_EXPECT_TRUE(test, hlist_empty(list4)); + KUNIT_EXPECT_TRUE(test, hlist_empty(list5)); + + kfree(list4); + kfree(list5); +} + +static void hlist_test_unhashed(struct kunit *test) +{ + struct hlist_node a; + HLIST_HEAD(list); + + INIT_HLIST_NODE(&a); + + /* is unhashed by default */ + KUNIT_EXPECT_TRUE(test, hlist_unhashed(&a)); + + hlist_add_head(&a, &list); + + /* is hashed once added to list */ + KUNIT_EXPECT_FALSE(test, hlist_unhashed(&a)); + + hlist_del_init(&a); + + /* is again unhashed after del_init */ + KUNIT_EXPECT_TRUE(test, hlist_unhashed(&a)); +} + +/* Doesn't test concurrency guarantees */ +static void hlist_test_unhashed_lockless(struct kunit *test) +{ + struct hlist_node a; + HLIST_HEAD(list); + + INIT_HLIST_NODE(&a); + + /* is unhashed by default */ + KUNIT_EXPECT_TRUE(test, hlist_unhashed_lockless(&a)); + + hlist_add_head(&a, &list); + + /* is hashed once added to list */ + KUNIT_EXPECT_FALSE(test, hlist_unhashed_lockless(&a)); + + hlist_del_init(&a); + + /* is again unhashed after del_init */ + KUNIT_EXPECT_TRUE(test, hlist_unhashed_lockless(&a)); +} + +static void hlist_test_del(struct kunit *test) +{ + struct hlist_node a, b; + HLIST_HEAD(list); + + hlist_add_head(&a, &list); + hlist_add_behind(&b, &a); + + /* before: [list] -> a -> b */ + hlist_del(&a); + + /* now: [list] -> b */ + KUNIT_EXPECT_PTR_EQ(test, list.first, &b); + KUNIT_EXPECT_PTR_EQ(test, b.pprev, &list.first); +} + +static void hlist_test_del_init(struct kunit *test) +{ + struct hlist_node a, b; + HLIST_HEAD(list); + + hlist_add_head(&a, &list); + hlist_add_behind(&b, &a); + + /* before: [list] -> a -> b */ + hlist_del_init(&a); + + /* now: [list] -> b */ + KUNIT_EXPECT_PTR_EQ(test, list.first, &b); + KUNIT_EXPECT_PTR_EQ(test, b.pprev, &list.first); + + /* a is now initialised */ + KUNIT_EXPECT_PTR_EQ(test, a.next, NULL); + KUNIT_EXPECT_PTR_EQ(test, a.pprev, NULL); +} + +/* Tests all three hlist_add_* functions */ +static void hlist_test_add(struct kunit *test) +{ + struct hlist_node a, b, c, d; + HLIST_HEAD(list); + + hlist_add_head(&a, &list); + hlist_add_head(&b, &list); + hlist_add_before(&c, &a); + hlist_add_behind(&d, &a); + + /* should be [list] -> b -> c -> a -> d */ + KUNIT_EXPECT_PTR_EQ(test, list.first, &b); + + KUNIT_EXPECT_PTR_EQ(test, c.pprev, &(b.next)); + KUNIT_EXPECT_PTR_EQ(test, b.next, &c); + + KUNIT_EXPECT_PTR_EQ(test, a.pprev, &(c.next)); + KUNIT_EXPECT_PTR_EQ(test, c.next, &a); + + KUNIT_EXPECT_PTR_EQ(test, d.pprev, &(a.next)); + KUNIT_EXPECT_PTR_EQ(test, a.next, &d); +} + +/* Tests both hlist_fake() and hlist_add_fake() */ +static void hlist_test_fake(struct kunit *test) +{ + struct hlist_node a; + + INIT_HLIST_NODE(&a); + + /* not fake after init */ + KUNIT_EXPECT_FALSE(test, hlist_fake(&a)); + + hlist_add_fake(&a); + + /* is now fake */ + KUNIT_EXPECT_TRUE(test, hlist_fake(&a)); +} + +static void hlist_test_is_singular_node(struct kunit *test) +{ + struct hlist_node a, b; + HLIST_HEAD(list); + + INIT_HLIST_NODE(&a); + KUNIT_EXPECT_FALSE(test, hlist_is_singular_node(&a, &list)); + + hlist_add_head(&a, &list); + KUNIT_EXPECT_TRUE(test, hlist_is_singular_node(&a, &list)); + + hlist_add_head(&b, &list); + KUNIT_EXPECT_FALSE(test, hlist_is_singular_node(&a, &list)); + KUNIT_EXPECT_FALSE(test, hlist_is_singular_node(&b, &list)); +} + +static void hlist_test_empty(struct kunit *test) +{ + struct hlist_node a; + HLIST_HEAD(list); + + /* list starts off empty */ + KUNIT_EXPECT_TRUE(test, hlist_empty(&list)); + + hlist_add_head(&a, &list); + + /* list is no longer empty */ + KUNIT_EXPECT_FALSE(test, hlist_empty(&list)); +} + +static void hlist_test_move_list(struct kunit *test) +{ + struct hlist_node a; + HLIST_HEAD(list1); + HLIST_HEAD(list2); + + hlist_add_head(&a, &list1); + + KUNIT_EXPECT_FALSE(test, hlist_empty(&list1)); + KUNIT_EXPECT_TRUE(test, hlist_empty(&list2)); + hlist_move_list(&list1, &list2); + KUNIT_EXPECT_TRUE(test, hlist_empty(&list1)); + KUNIT_EXPECT_FALSE(test, hlist_empty(&list2)); + +} + +static void hlist_test_entry(struct kunit *test) +{ + struct hlist_test_struct test_struct; + + KUNIT_EXPECT_PTR_EQ(test, &test_struct, + hlist_entry(&(test_struct.list), + struct hlist_test_struct, list)); +} + +static void hlist_test_entry_safe(struct kunit *test) +{ + struct hlist_test_struct test_struct; + + KUNIT_EXPECT_PTR_EQ(test, &test_struct, + hlist_entry_safe(&(test_struct.list), + struct hlist_test_struct, list)); + + KUNIT_EXPECT_PTR_EQ(test, NULL, + hlist_entry_safe((struct hlist_node *)NULL, + struct hlist_test_struct, list)); +} + +static void hlist_test_for_each(struct kunit *test) +{ + struct hlist_node entries[3], *cur; + HLIST_HEAD(list); + int i = 0; + + hlist_add_head(&entries[0], &list); + hlist_add_behind(&entries[1], &entries[0]); + hlist_add_behind(&entries[2], &entries[1]); + + hlist_for_each(cur, &list) { + KUNIT_EXPECT_PTR_EQ(test, cur, &entries[i]); + i++; + } + + KUNIT_EXPECT_EQ(test, i, 3); +} + + +static void hlist_test_for_each_safe(struct kunit *test) +{ + struct hlist_node entries[3], *cur, *n; + HLIST_HEAD(list); + int i = 0; + + hlist_add_head(&entries[0], &list); + hlist_add_behind(&entries[1], &entries[0]); + hlist_add_behind(&entries[2], &entries[1]); + + hlist_for_each_safe(cur, n, &list) { + KUNIT_EXPECT_PTR_EQ(test, cur, &entries[i]); + hlist_del(&entries[i]); + i++; + } + + KUNIT_EXPECT_EQ(test, i, 3); + KUNIT_EXPECT_TRUE(test, hlist_empty(&list)); +} + +static void hlist_test_for_each_entry(struct kunit *test) +{ + struct hlist_test_struct entries[5], *cur; + HLIST_HEAD(list); + int i = 0; + + entries[0].data = 0; + hlist_add_head(&entries[0].list, &list); + for (i = 1; i < 5; ++i) { + entries[i].data = i; + hlist_add_behind(&entries[i].list, &entries[i-1].list); + } + + i = 0; + + hlist_for_each_entry(cur, &list, list) { + KUNIT_EXPECT_EQ(test, cur->data, i); + i++; + } + + KUNIT_EXPECT_EQ(test, i, 5); +} + +static void hlist_test_for_each_entry_continue(struct kunit *test) +{ + struct hlist_test_struct entries[5], *cur; + HLIST_HEAD(list); + int i = 0; + + entries[0].data = 0; + hlist_add_head(&entries[0].list, &list); + for (i = 1; i < 5; ++i) { + entries[i].data = i; + hlist_add_behind(&entries[i].list, &entries[i-1].list); + } + + /* We skip the first (zero-th) entry. */ + i = 1; + + cur = &entries[0]; + hlist_for_each_entry_continue(cur, list) { + KUNIT_EXPECT_EQ(test, cur->data, i); + /* Stamp over the entry. */ + cur->data = 42; + i++; + } + + KUNIT_EXPECT_EQ(test, i, 5); + /* The first entry was not visited. */ + KUNIT_EXPECT_EQ(test, entries[0].data, 0); + /* The second (and presumably others), were. */ + KUNIT_EXPECT_EQ(test, entries[1].data, 42); +} + +static void hlist_test_for_each_entry_from(struct kunit *test) +{ + struct hlist_test_struct entries[5], *cur; + HLIST_HEAD(list); + int i = 0; + + entries[0].data = 0; + hlist_add_head(&entries[0].list, &list); + for (i = 1; i < 5; ++i) { + entries[i].data = i; + hlist_add_behind(&entries[i].list, &entries[i-1].list); + } + + i = 0; + + cur = &entries[0]; + hlist_for_each_entry_from(cur, list) { + KUNIT_EXPECT_EQ(test, cur->data, i); + /* Stamp over the entry. */ + cur->data = 42; + i++; + } + + KUNIT_EXPECT_EQ(test, i, 5); + /* The first entry was visited. */ + KUNIT_EXPECT_EQ(test, entries[0].data, 42); +} + +static void hlist_test_for_each_entry_safe(struct kunit *test) +{ + struct hlist_test_struct entries[5], *cur; + struct hlist_node *tmp_node; + HLIST_HEAD(list); + int i = 0; + + entries[0].data = 0; + hlist_add_head(&entries[0].list, &list); + for (i = 1; i < 5; ++i) { + entries[i].data = i; + hlist_add_behind(&entries[i].list, &entries[i-1].list); + } + + i = 0; + + hlist_for_each_entry_safe(cur, tmp_node, &list, list) { + KUNIT_EXPECT_EQ(test, cur->data, i); + hlist_del(&cur->list); + i++; + } + + KUNIT_EXPECT_EQ(test, i, 5); + KUNIT_EXPECT_TRUE(test, hlist_empty(&list)); +} + + +static struct kunit_case hlist_test_cases[] = { + KUNIT_CASE(hlist_test_init), + KUNIT_CASE(hlist_test_unhashed), + KUNIT_CASE(hlist_test_unhashed_lockless), + KUNIT_CASE(hlist_test_del), + KUNIT_CASE(hlist_test_del_init), + KUNIT_CASE(hlist_test_add), + KUNIT_CASE(hlist_test_fake), + KUNIT_CASE(hlist_test_is_singular_node), + KUNIT_CASE(hlist_test_empty), + KUNIT_CASE(hlist_test_move_list), + KUNIT_CASE(hlist_test_entry), + KUNIT_CASE(hlist_test_entry_safe), + KUNIT_CASE(hlist_test_for_each), + KUNIT_CASE(hlist_test_for_each_safe), + KUNIT_CASE(hlist_test_for_each_entry), + KUNIT_CASE(hlist_test_for_each_entry_continue), + KUNIT_CASE(hlist_test_for_each_entry_from), + KUNIT_CASE(hlist_test_for_each_entry_safe), + {}, +}; + +static struct kunit_suite hlist_test_module = { + .name = "hlist", + .test_cases = hlist_test_cases, +}; + +kunit_test_suites(&list_test_module, &hlist_test_module); MODULE_LICENSE("GPL v2"); diff --git a/lib/lockref.c b/lib/lockref.c index 5b34bbd3eba8..45e93ece8ba0 100644 --- a/lib/lockref.c +++ b/lib/lockref.c @@ -14,12 +14,11 @@ BUILD_BUG_ON(sizeof(old) != 8); \ old.lock_count = READ_ONCE(lockref->lock_count); \ while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \ - struct lockref new = old, prev = old; \ + struct lockref new = old; \ CODE \ - old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \ - old.lock_count, \ - new.lock_count); \ - if (likely(old.lock_count == prev.lock_count)) { \ + if (likely(try_cmpxchg64_relaxed(&lockref->lock_count, \ + &old.lock_count, \ + new.lock_count))) { \ SUCCESS; \ } \ if (!--retry) \ @@ -112,31 +111,6 @@ int lockref_put_not_zero(struct lockref *lockref) EXPORT_SYMBOL(lockref_put_not_zero); /** - * lockref_get_or_lock - Increments count unless the count is 0 or dead - * @lockref: pointer to lockref structure - * Return: 1 if count updated successfully or 0 if count was zero - * and we got the lock instead. - */ -int lockref_get_or_lock(struct lockref *lockref) -{ - CMPXCHG_LOOP( - new.count++; - if (old.count <= 0) - break; - , - return 1; - ); - - spin_lock(&lockref->lock); - if (lockref->count <= 0) - return 0; - lockref->count++; - spin_unlock(&lockref->lock); - return 1; -} -EXPORT_SYMBOL(lockref_get_or_lock); - -/** * lockref_put_return - Decrement reference count if possible * @lockref: pointer to lockref structure * diff --git a/lib/logic_iomem.c b/lib/logic_iomem.c index 8c3365f26e51..b247d412ddef 100644 --- a/lib/logic_iomem.c +++ b/lib/logic_iomem.c @@ -68,7 +68,7 @@ int logic_iomem_add_region(struct resource *resource, } EXPORT_SYMBOL(logic_iomem_add_region); -#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +#ifndef CONFIG_INDIRECT_IOMEM_FALLBACK static void __iomem *real_ioremap(phys_addr_t offset, size_t size) { WARN(1, "invalid ioremap(0x%llx, 0x%zx)\n", @@ -81,7 +81,7 @@ static void real_iounmap(volatile void __iomem *addr) WARN(1, "invalid iounmap for addr 0x%llx\n", (unsigned long long)(uintptr_t __force)addr); } -#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ +#endif /* CONFIG_INDIRECT_IOMEM_FALLBACK */ void __iomem *ioremap(phys_addr_t offset, size_t size) { @@ -168,7 +168,7 @@ void iounmap(volatile void __iomem *addr) } EXPORT_SYMBOL(iounmap); -#ifndef CONFIG_LOGIC_IOMEM_FALLBACK +#ifndef CONFIG_INDIRECT_IOMEM_FALLBACK #define MAKE_FALLBACK(op, sz) \ static u##sz real_raw_read ## op(const volatile void __iomem *addr) \ { \ @@ -213,7 +213,7 @@ static void real_memcpy_toio(volatile void __iomem *addr, const void *buffer, WARN(1, "Invalid memcpy_toio at address 0x%llx\n", (unsigned long long)(uintptr_t __force)addr); } -#endif /* CONFIG_LOGIC_IOMEM_FALLBACK */ +#endif /* CONFIG_INDIRECT_IOMEM_FALLBACK */ #define MAKE_OP(op, sz) \ u##sz __raw_read ## op(const volatile void __iomem *addr) \ diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c index 926f4823d5ea..fd1728d94bab 100644 --- a/lib/lz4/lz4_decompress.c +++ b/lib/lz4/lz4_decompress.c @@ -271,8 +271,12 @@ static FORCE_INLINE int LZ4_decompress_generic( ip += length; op += length; - /* Necessarily EOF, due to parsing restrictions */ - if (!partialDecoding || (cpy == oend)) + /* Necessarily EOF when !partialDecoding. + * When partialDecoding, it is EOF if we've either + * filled the output buffer or + * can't proceed with reading an offset for following match. + */ + if (!partialDecoding || (cpy == oend) || (ip >= (iend - 2))) break; } else { /* may overwrite up to WILDCOPYLENGTH beyond cpy */ diff --git a/lib/memneq.c b/lib/memneq.c new file mode 100644 index 000000000000..fb11608b1ec1 --- /dev/null +++ b/lib/memneq.c @@ -0,0 +1,176 @@ +/* + * Constant-time equality testing of memory regions. + * + * Authors: + * + * James Yonan <james@openvpn.net> + * Daniel Borkmann <dborkman@redhat.com> + * + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * The full GNU General Public License is included in this distribution + * in the file called LICENSE.GPL. + * + * BSD LICENSE + * + * Copyright(c) 2013 OpenVPN Technologies, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of OpenVPN Technologies nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <crypto/algapi.h> +#include <asm/unaligned.h> + +#ifndef __HAVE_ARCH_CRYPTO_MEMNEQ + +/* Generic path for arbitrary size */ +static inline unsigned long +__crypto_memneq_generic(const void *a, const void *b, size_t size) +{ + unsigned long neq = 0; + +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) + while (size >= sizeof(unsigned long)) { + neq |= get_unaligned((unsigned long *)a) ^ + get_unaligned((unsigned long *)b); + OPTIMIZER_HIDE_VAR(neq); + a += sizeof(unsigned long); + b += sizeof(unsigned long); + size -= sizeof(unsigned long); + } +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + while (size > 0) { + neq |= *(unsigned char *)a ^ *(unsigned char *)b; + OPTIMIZER_HIDE_VAR(neq); + a += 1; + b += 1; + size -= 1; + } + return neq; +} + +/* Loop-free fast-path for frequently used 16-byte size */ +static inline unsigned long __crypto_memneq_16(const void *a, const void *b) +{ + unsigned long neq = 0; + +#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (sizeof(unsigned long) == 8) { + neq |= get_unaligned((unsigned long *)a) ^ + get_unaligned((unsigned long *)b); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned long *)(a + 8)) ^ + get_unaligned((unsigned long *)(b + 8)); + OPTIMIZER_HIDE_VAR(neq); + } else if (sizeof(unsigned int) == 4) { + neq |= get_unaligned((unsigned int *)a) ^ + get_unaligned((unsigned int *)b); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned int *)(a + 4)) ^ + get_unaligned((unsigned int *)(b + 4)); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned int *)(a + 8)) ^ + get_unaligned((unsigned int *)(b + 8)); + OPTIMIZER_HIDE_VAR(neq); + neq |= get_unaligned((unsigned int *)(a + 12)) ^ + get_unaligned((unsigned int *)(b + 12)); + OPTIMIZER_HIDE_VAR(neq); + } else +#endif /* CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS */ + { + neq |= *(unsigned char *)(a) ^ *(unsigned char *)(b); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+1) ^ *(unsigned char *)(b+1); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+2) ^ *(unsigned char *)(b+2); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+3) ^ *(unsigned char *)(b+3); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+4) ^ *(unsigned char *)(b+4); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+5) ^ *(unsigned char *)(b+5); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+6) ^ *(unsigned char *)(b+6); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+7) ^ *(unsigned char *)(b+7); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+8) ^ *(unsigned char *)(b+8); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+9) ^ *(unsigned char *)(b+9); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+10) ^ *(unsigned char *)(b+10); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+11) ^ *(unsigned char *)(b+11); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+12) ^ *(unsigned char *)(b+12); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+13) ^ *(unsigned char *)(b+13); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+14) ^ *(unsigned char *)(b+14); + OPTIMIZER_HIDE_VAR(neq); + neq |= *(unsigned char *)(a+15) ^ *(unsigned char *)(b+15); + OPTIMIZER_HIDE_VAR(neq); + } + + return neq; +} + +/* Compare two areas of memory without leaking timing information, + * and with special optimizations for common sizes. Users should + * not call this function directly, but should instead use + * crypto_memneq defined in crypto/algapi.h. + */ +noinline unsigned long __crypto_memneq(const void *a, const void *b, + size_t size) +{ + switch (size) { + case 16: + return __crypto_memneq_16(a, b); + default: + return __crypto_memneq_generic(a, b, size); + } +} +EXPORT_SYMBOL(__crypto_memneq); + +#endif /* __HAVE_ARCH_CRYPTO_MEMNEQ */ diff --git a/lib/mpi/mpi-bit.c b/lib/mpi/mpi-bit.c index 142b680835df..070ba784c9f1 100644 --- a/lib/mpi/mpi-bit.c +++ b/lib/mpi/mpi-bit.c @@ -242,6 +242,7 @@ void mpi_rshift(MPI x, MPI a, unsigned int n) } MPN_NORMALIZE(x->d, x->nlimbs); } +EXPORT_SYMBOL_GPL(mpi_rshift); /**************** * Shift A by COUNT limbs to the left diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index 199ab201d501..d01aec6ae15c 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -99,7 +99,7 @@ bool nmi_cpu_backtrace(struct pt_regs *regs) * Allow nested NMI backtraces while serializing * against other CPUs. */ - printk_cpu_lock_irqsave(flags); + printk_cpu_sync_get_irqsave(flags); if (!READ_ONCE(backtrace_idle) && regs && cpu_in_idle(instruction_pointer(regs))) { pr_warn("NMI backtrace for cpu %d skipped: idling at %pS\n", cpu, (void *)instruction_pointer(regs)); @@ -110,7 +110,7 @@ bool nmi_cpu_backtrace(struct pt_regs *regs) else dump_stack(); } - printk_cpu_unlock_irqrestore(flags); + printk_cpu_sync_put_irqrestore(flags); cpumask_clear_cpu(cpu, to_cpumask(backtrace_mask)); return true; } diff --git a/lib/nodemask.c b/lib/nodemask.c index 3aa454c54c0d..e22647f5181b 100644 --- a/lib/nodemask.c +++ b/lib/nodemask.c @@ -3,9 +3,9 @@ #include <linux/module.h> #include <linux/random.h> -int __next_node_in(int node, const nodemask_t *srcp) +unsigned int __next_node_in(int node, const nodemask_t *srcp) { - int ret = __next_node(node, srcp); + unsigned int ret = __next_node(node, srcp); if (ret == MAX_NUMNODES) ret = __first_node(srcp); diff --git a/lib/test_overflow.c b/lib/overflow_kunit.c index 7a4b6f6c5473..475f0c064bf6 100644 --- a/lib/test_overflow.c +++ b/lib/overflow_kunit.c @@ -1,11 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 OR MIT /* - * Test cases for arithmetic overflow checks. + * Test cases for arithmetic overflow checks. See: + * https://www.kernel.org/doc/html/latest/dev-tools/kunit/kunit-tool.html#configuring-building-and-running-tests + * ./tools/testing/kunit/kunit.py run overflow [--raw_output] */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <kunit/test.h> #include <linux/device.h> -#include <linux/init.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/module.h> @@ -19,7 +21,7 @@ t a, b; \ t sum, diff, prod; \ bool s_of, d_of, p_of; \ - } t ## _tests[] __initconst + } t ## _tests[] DEFINE_TEST_ARRAY(u8) = { {0, 0, 0, 0, 0, false, false, false}, @@ -220,43 +222,31 @@ DEFINE_TEST_ARRAY(s64) = { bool _of; \ \ _of = check_ ## op ## _overflow(a, b, &_r); \ - if (_of != of) { \ - pr_warn("expected "fmt" "sym" "fmt \ - " to%s overflow (type %s)\n", \ - a, b, of ? "" : " not", #t); \ - err = 1; \ - } \ - if (_r != r) { \ - pr_warn("expected "fmt" "sym" "fmt" == " \ - fmt", got "fmt" (type %s)\n", \ - a, b, r, _r, #t); \ - err = 1; \ - } \ + KUNIT_EXPECT_EQ_MSG(test, _of, of, \ + "expected "fmt" "sym" "fmt" to%s overflow (type %s)\n", \ + a, b, of ? "" : " not", #t); \ + KUNIT_EXPECT_EQ_MSG(test, _r, r, \ + "expected "fmt" "sym" "fmt" == "fmt", got "fmt" (type %s)\n", \ + a, b, r, _r, #t); \ } while (0) #define DEFINE_TEST_FUNC(t, fmt) \ -static int __init do_test_ ## t(const struct test_ ## t *p) \ +static void do_test_ ## t(struct kunit *test, const struct test_ ## t *p) \ { \ - int err = 0; \ - \ check_one_op(t, fmt, add, "+", p->a, p->b, p->sum, p->s_of); \ check_one_op(t, fmt, add, "+", p->b, p->a, p->sum, p->s_of); \ check_one_op(t, fmt, sub, "-", p->a, p->b, p->diff, p->d_of); \ check_one_op(t, fmt, mul, "*", p->a, p->b, p->prod, p->p_of); \ check_one_op(t, fmt, mul, "*", p->b, p->a, p->prod, p->p_of); \ - \ - return err; \ } \ \ -static int __init test_ ## t ## _overflow(void) { \ - int err = 0; \ +static void t ## _overflow_test(struct kunit *test) { \ unsigned i; \ \ - pr_info("%-3s: %zu arithmetic tests\n", #t, \ - ARRAY_SIZE(t ## _tests)); \ for (i = 0; i < ARRAY_SIZE(t ## _tests); ++i) \ - err |= do_test_ ## t(&t ## _tests[i]); \ - return err; \ + do_test_ ## t(test, &t ## _tests[i]); \ + kunit_info(test, "%zu %s arithmetic tests finished\n", \ + ARRAY_SIZE(t ## _tests), #t); \ } DEFINE_TEST_FUNC(u8, "%d"); @@ -270,199 +260,176 @@ DEFINE_TEST_FUNC(u64, "%llu"); DEFINE_TEST_FUNC(s64, "%lld"); #endif -static int __init test_overflow_calculation(void) -{ - int err = 0; - - err |= test_u8_overflow(); - err |= test_s8_overflow(); - err |= test_u16_overflow(); - err |= test_s16_overflow(); - err |= test_u32_overflow(); - err |= test_s32_overflow(); -#if BITS_PER_LONG == 64 - err |= test_u64_overflow(); - err |= test_s64_overflow(); -#endif - - return err; -} - -static int __init test_overflow_shift(void) +static void overflow_shift_test(struct kunit *test) { - int err = 0; + int count = 0; /* Args are: value, shift, type, expected result, overflow expected */ -#define TEST_ONE_SHIFT(a, s, t, expect, of) ({ \ - int __failed = 0; \ +#define TEST_ONE_SHIFT(a, s, t, expect, of) do { \ typeof(a) __a = (a); \ typeof(s) __s = (s); \ t __e = (expect); \ t __d; \ bool __of = check_shl_overflow(__a, __s, &__d); \ if (__of != of) { \ - pr_warn("expected (%s)(%s << %s) to%s overflow\n", \ + KUNIT_EXPECT_EQ_MSG(test, __of, of, \ + "expected (%s)(%s << %s) to%s overflow\n", \ #t, #a, #s, of ? "" : " not"); \ - __failed = 1; \ } else if (!__of && __d != __e) { \ - pr_warn("expected (%s)(%s << %s) == %s\n", \ + KUNIT_EXPECT_EQ_MSG(test, __d, __e, \ + "expected (%s)(%s << %s) == %s\n", \ #t, #a, #s, #expect); \ if ((t)-1 < 0) \ - pr_warn("got %lld\n", (s64)__d); \ + kunit_info(test, "got %lld\n", (s64)__d); \ else \ - pr_warn("got %llu\n", (u64)__d); \ - __failed = 1; \ + kunit_info(test, "got %llu\n", (u64)__d); \ } \ - if (!__failed) \ - pr_info("ok: (%s)(%s << %s) == %s\n", #t, #a, #s, \ - of ? "overflow" : #expect); \ - __failed; \ -}) + count++; \ +} while (0) /* Sane shifts. */ - err |= TEST_ONE_SHIFT(1, 0, u8, 1 << 0, false); - err |= TEST_ONE_SHIFT(1, 4, u8, 1 << 4, false); - err |= TEST_ONE_SHIFT(1, 7, u8, 1 << 7, false); - err |= TEST_ONE_SHIFT(0xF, 4, u8, 0xF << 4, false); - err |= TEST_ONE_SHIFT(1, 0, u16, 1 << 0, false); - err |= TEST_ONE_SHIFT(1, 10, u16, 1 << 10, false); - err |= TEST_ONE_SHIFT(1, 15, u16, 1 << 15, false); - err |= TEST_ONE_SHIFT(0xFF, 8, u16, 0xFF << 8, false); - err |= TEST_ONE_SHIFT(1, 0, int, 1 << 0, false); - err |= TEST_ONE_SHIFT(1, 16, int, 1 << 16, false); - err |= TEST_ONE_SHIFT(1, 30, int, 1 << 30, false); - err |= TEST_ONE_SHIFT(1, 0, s32, 1 << 0, false); - err |= TEST_ONE_SHIFT(1, 16, s32, 1 << 16, false); - err |= TEST_ONE_SHIFT(1, 30, s32, 1 << 30, false); - err |= TEST_ONE_SHIFT(1, 0, unsigned int, 1U << 0, false); - err |= TEST_ONE_SHIFT(1, 20, unsigned int, 1U << 20, false); - err |= TEST_ONE_SHIFT(1, 31, unsigned int, 1U << 31, false); - err |= TEST_ONE_SHIFT(0xFFFFU, 16, unsigned int, 0xFFFFU << 16, false); - err |= TEST_ONE_SHIFT(1, 0, u32, 1U << 0, false); - err |= TEST_ONE_SHIFT(1, 20, u32, 1U << 20, false); - err |= TEST_ONE_SHIFT(1, 31, u32, 1U << 31, false); - err |= TEST_ONE_SHIFT(0xFFFFU, 16, u32, 0xFFFFU << 16, false); - err |= TEST_ONE_SHIFT(1, 0, u64, 1ULL << 0, false); - err |= TEST_ONE_SHIFT(1, 40, u64, 1ULL << 40, false); - err |= TEST_ONE_SHIFT(1, 63, u64, 1ULL << 63, false); - err |= TEST_ONE_SHIFT(0xFFFFFFFFULL, 32, u64, - 0xFFFFFFFFULL << 32, false); + TEST_ONE_SHIFT(1, 0, u8, 1 << 0, false); + TEST_ONE_SHIFT(1, 4, u8, 1 << 4, false); + TEST_ONE_SHIFT(1, 7, u8, 1 << 7, false); + TEST_ONE_SHIFT(0xF, 4, u8, 0xF << 4, false); + TEST_ONE_SHIFT(1, 0, u16, 1 << 0, false); + TEST_ONE_SHIFT(1, 10, u16, 1 << 10, false); + TEST_ONE_SHIFT(1, 15, u16, 1 << 15, false); + TEST_ONE_SHIFT(0xFF, 8, u16, 0xFF << 8, false); + TEST_ONE_SHIFT(1, 0, int, 1 << 0, false); + TEST_ONE_SHIFT(1, 16, int, 1 << 16, false); + TEST_ONE_SHIFT(1, 30, int, 1 << 30, false); + TEST_ONE_SHIFT(1, 0, s32, 1 << 0, false); + TEST_ONE_SHIFT(1, 16, s32, 1 << 16, false); + TEST_ONE_SHIFT(1, 30, s32, 1 << 30, false); + TEST_ONE_SHIFT(1, 0, unsigned int, 1U << 0, false); + TEST_ONE_SHIFT(1, 20, unsigned int, 1U << 20, false); + TEST_ONE_SHIFT(1, 31, unsigned int, 1U << 31, false); + TEST_ONE_SHIFT(0xFFFFU, 16, unsigned int, 0xFFFFU << 16, false); + TEST_ONE_SHIFT(1, 0, u32, 1U << 0, false); + TEST_ONE_SHIFT(1, 20, u32, 1U << 20, false); + TEST_ONE_SHIFT(1, 31, u32, 1U << 31, false); + TEST_ONE_SHIFT(0xFFFFU, 16, u32, 0xFFFFU << 16, false); + TEST_ONE_SHIFT(1, 0, u64, 1ULL << 0, false); + TEST_ONE_SHIFT(1, 40, u64, 1ULL << 40, false); + TEST_ONE_SHIFT(1, 63, u64, 1ULL << 63, false); + TEST_ONE_SHIFT(0xFFFFFFFFULL, 32, u64, 0xFFFFFFFFULL << 32, false); /* Sane shift: start and end with 0, without a too-wide shift. */ - err |= TEST_ONE_SHIFT(0, 7, u8, 0, false); - err |= TEST_ONE_SHIFT(0, 15, u16, 0, false); - err |= TEST_ONE_SHIFT(0, 31, unsigned int, 0, false); - err |= TEST_ONE_SHIFT(0, 31, u32, 0, false); - err |= TEST_ONE_SHIFT(0, 63, u64, 0, false); + TEST_ONE_SHIFT(0, 7, u8, 0, false); + TEST_ONE_SHIFT(0, 15, u16, 0, false); + TEST_ONE_SHIFT(0, 31, unsigned int, 0, false); + TEST_ONE_SHIFT(0, 31, u32, 0, false); + TEST_ONE_SHIFT(0, 63, u64, 0, false); /* Sane shift: start and end with 0, without reaching signed bit. */ - err |= TEST_ONE_SHIFT(0, 6, s8, 0, false); - err |= TEST_ONE_SHIFT(0, 14, s16, 0, false); - err |= TEST_ONE_SHIFT(0, 30, int, 0, false); - err |= TEST_ONE_SHIFT(0, 30, s32, 0, false); - err |= TEST_ONE_SHIFT(0, 62, s64, 0, false); + TEST_ONE_SHIFT(0, 6, s8, 0, false); + TEST_ONE_SHIFT(0, 14, s16, 0, false); + TEST_ONE_SHIFT(0, 30, int, 0, false); + TEST_ONE_SHIFT(0, 30, s32, 0, false); + TEST_ONE_SHIFT(0, 62, s64, 0, false); /* Overflow: shifted the bit off the end. */ - err |= TEST_ONE_SHIFT(1, 8, u8, 0, true); - err |= TEST_ONE_SHIFT(1, 16, u16, 0, true); - err |= TEST_ONE_SHIFT(1, 32, unsigned int, 0, true); - err |= TEST_ONE_SHIFT(1, 32, u32, 0, true); - err |= TEST_ONE_SHIFT(1, 64, u64, 0, true); + TEST_ONE_SHIFT(1, 8, u8, 0, true); + TEST_ONE_SHIFT(1, 16, u16, 0, true); + TEST_ONE_SHIFT(1, 32, unsigned int, 0, true); + TEST_ONE_SHIFT(1, 32, u32, 0, true); + TEST_ONE_SHIFT(1, 64, u64, 0, true); /* Overflow: shifted into the signed bit. */ - err |= TEST_ONE_SHIFT(1, 7, s8, 0, true); - err |= TEST_ONE_SHIFT(1, 15, s16, 0, true); - err |= TEST_ONE_SHIFT(1, 31, int, 0, true); - err |= TEST_ONE_SHIFT(1, 31, s32, 0, true); - err |= TEST_ONE_SHIFT(1, 63, s64, 0, true); + TEST_ONE_SHIFT(1, 7, s8, 0, true); + TEST_ONE_SHIFT(1, 15, s16, 0, true); + TEST_ONE_SHIFT(1, 31, int, 0, true); + TEST_ONE_SHIFT(1, 31, s32, 0, true); + TEST_ONE_SHIFT(1, 63, s64, 0, true); /* Overflow: high bit falls off unsigned types. */ /* 10010110 */ - err |= TEST_ONE_SHIFT(150, 1, u8, 0, true); + TEST_ONE_SHIFT(150, 1, u8, 0, true); /* 1000100010010110 */ - err |= TEST_ONE_SHIFT(34966, 1, u16, 0, true); + TEST_ONE_SHIFT(34966, 1, u16, 0, true); /* 10000100000010001000100010010110 */ - err |= TEST_ONE_SHIFT(2215151766U, 1, u32, 0, true); - err |= TEST_ONE_SHIFT(2215151766U, 1, unsigned int, 0, true); + TEST_ONE_SHIFT(2215151766U, 1, u32, 0, true); + TEST_ONE_SHIFT(2215151766U, 1, unsigned int, 0, true); /* 1000001000010000010000000100000010000100000010001000100010010110 */ - err |= TEST_ONE_SHIFT(9372061470395238550ULL, 1, u64, 0, true); + TEST_ONE_SHIFT(9372061470395238550ULL, 1, u64, 0, true); /* Overflow: bit shifted into signed bit on signed types. */ /* 01001011 */ - err |= TEST_ONE_SHIFT(75, 1, s8, 0, true); + TEST_ONE_SHIFT(75, 1, s8, 0, true); /* 0100010001001011 */ - err |= TEST_ONE_SHIFT(17483, 1, s16, 0, true); + TEST_ONE_SHIFT(17483, 1, s16, 0, true); /* 01000010000001000100010001001011 */ - err |= TEST_ONE_SHIFT(1107575883, 1, s32, 0, true); - err |= TEST_ONE_SHIFT(1107575883, 1, int, 0, true); + TEST_ONE_SHIFT(1107575883, 1, s32, 0, true); + TEST_ONE_SHIFT(1107575883, 1, int, 0, true); /* 0100000100001000001000000010000001000010000001000100010001001011 */ - err |= TEST_ONE_SHIFT(4686030735197619275LL, 1, s64, 0, true); + TEST_ONE_SHIFT(4686030735197619275LL, 1, s64, 0, true); /* Overflow: bit shifted past signed bit on signed types. */ /* 01001011 */ - err |= TEST_ONE_SHIFT(75, 2, s8, 0, true); + TEST_ONE_SHIFT(75, 2, s8, 0, true); /* 0100010001001011 */ - err |= TEST_ONE_SHIFT(17483, 2, s16, 0, true); + TEST_ONE_SHIFT(17483, 2, s16, 0, true); /* 01000010000001000100010001001011 */ - err |= TEST_ONE_SHIFT(1107575883, 2, s32, 0, true); - err |= TEST_ONE_SHIFT(1107575883, 2, int, 0, true); + TEST_ONE_SHIFT(1107575883, 2, s32, 0, true); + TEST_ONE_SHIFT(1107575883, 2, int, 0, true); /* 0100000100001000001000000010000001000010000001000100010001001011 */ - err |= TEST_ONE_SHIFT(4686030735197619275LL, 2, s64, 0, true); + TEST_ONE_SHIFT(4686030735197619275LL, 2, s64, 0, true); /* Overflow: values larger than destination type. */ - err |= TEST_ONE_SHIFT(0x100, 0, u8, 0, true); - err |= TEST_ONE_SHIFT(0xFF, 0, s8, 0, true); - err |= TEST_ONE_SHIFT(0x10000U, 0, u16, 0, true); - err |= TEST_ONE_SHIFT(0xFFFFU, 0, s16, 0, true); - err |= TEST_ONE_SHIFT(0x100000000ULL, 0, u32, 0, true); - err |= TEST_ONE_SHIFT(0x100000000ULL, 0, unsigned int, 0, true); - err |= TEST_ONE_SHIFT(0xFFFFFFFFUL, 0, s32, 0, true); - err |= TEST_ONE_SHIFT(0xFFFFFFFFUL, 0, int, 0, true); - err |= TEST_ONE_SHIFT(0xFFFFFFFFFFFFFFFFULL, 0, s64, 0, true); + TEST_ONE_SHIFT(0x100, 0, u8, 0, true); + TEST_ONE_SHIFT(0xFF, 0, s8, 0, true); + TEST_ONE_SHIFT(0x10000U, 0, u16, 0, true); + TEST_ONE_SHIFT(0xFFFFU, 0, s16, 0, true); + TEST_ONE_SHIFT(0x100000000ULL, 0, u32, 0, true); + TEST_ONE_SHIFT(0x100000000ULL, 0, unsigned int, 0, true); + TEST_ONE_SHIFT(0xFFFFFFFFUL, 0, s32, 0, true); + TEST_ONE_SHIFT(0xFFFFFFFFUL, 0, int, 0, true); + TEST_ONE_SHIFT(0xFFFFFFFFFFFFFFFFULL, 0, s64, 0, true); /* Nonsense: negative initial value. */ - err |= TEST_ONE_SHIFT(-1, 0, s8, 0, true); - err |= TEST_ONE_SHIFT(-1, 0, u8, 0, true); - err |= TEST_ONE_SHIFT(-5, 0, s16, 0, true); - err |= TEST_ONE_SHIFT(-5, 0, u16, 0, true); - err |= TEST_ONE_SHIFT(-10, 0, int, 0, true); - err |= TEST_ONE_SHIFT(-10, 0, unsigned int, 0, true); - err |= TEST_ONE_SHIFT(-100, 0, s32, 0, true); - err |= TEST_ONE_SHIFT(-100, 0, u32, 0, true); - err |= TEST_ONE_SHIFT(-10000, 0, s64, 0, true); - err |= TEST_ONE_SHIFT(-10000, 0, u64, 0, true); + TEST_ONE_SHIFT(-1, 0, s8, 0, true); + TEST_ONE_SHIFT(-1, 0, u8, 0, true); + TEST_ONE_SHIFT(-5, 0, s16, 0, true); + TEST_ONE_SHIFT(-5, 0, u16, 0, true); + TEST_ONE_SHIFT(-10, 0, int, 0, true); + TEST_ONE_SHIFT(-10, 0, unsigned int, 0, true); + TEST_ONE_SHIFT(-100, 0, s32, 0, true); + TEST_ONE_SHIFT(-100, 0, u32, 0, true); + TEST_ONE_SHIFT(-10000, 0, s64, 0, true); + TEST_ONE_SHIFT(-10000, 0, u64, 0, true); /* Nonsense: negative shift values. */ - err |= TEST_ONE_SHIFT(0, -5, s8, 0, true); - err |= TEST_ONE_SHIFT(0, -5, u8, 0, true); - err |= TEST_ONE_SHIFT(0, -10, s16, 0, true); - err |= TEST_ONE_SHIFT(0, -10, u16, 0, true); - err |= TEST_ONE_SHIFT(0, -15, int, 0, true); - err |= TEST_ONE_SHIFT(0, -15, unsigned int, 0, true); - err |= TEST_ONE_SHIFT(0, -20, s32, 0, true); - err |= TEST_ONE_SHIFT(0, -20, u32, 0, true); - err |= TEST_ONE_SHIFT(0, -30, s64, 0, true); - err |= TEST_ONE_SHIFT(0, -30, u64, 0, true); + TEST_ONE_SHIFT(0, -5, s8, 0, true); + TEST_ONE_SHIFT(0, -5, u8, 0, true); + TEST_ONE_SHIFT(0, -10, s16, 0, true); + TEST_ONE_SHIFT(0, -10, u16, 0, true); + TEST_ONE_SHIFT(0, -15, int, 0, true); + TEST_ONE_SHIFT(0, -15, unsigned int, 0, true); + TEST_ONE_SHIFT(0, -20, s32, 0, true); + TEST_ONE_SHIFT(0, -20, u32, 0, true); + TEST_ONE_SHIFT(0, -30, s64, 0, true); + TEST_ONE_SHIFT(0, -30, u64, 0, true); /* Overflow: shifted at or beyond entire type's bit width. */ - err |= TEST_ONE_SHIFT(0, 8, u8, 0, true); - err |= TEST_ONE_SHIFT(0, 9, u8, 0, true); - err |= TEST_ONE_SHIFT(0, 8, s8, 0, true); - err |= TEST_ONE_SHIFT(0, 9, s8, 0, true); - err |= TEST_ONE_SHIFT(0, 16, u16, 0, true); - err |= TEST_ONE_SHIFT(0, 17, u16, 0, true); - err |= TEST_ONE_SHIFT(0, 16, s16, 0, true); - err |= TEST_ONE_SHIFT(0, 17, s16, 0, true); - err |= TEST_ONE_SHIFT(0, 32, u32, 0, true); - err |= TEST_ONE_SHIFT(0, 33, u32, 0, true); - err |= TEST_ONE_SHIFT(0, 32, int, 0, true); - err |= TEST_ONE_SHIFT(0, 33, int, 0, true); - err |= TEST_ONE_SHIFT(0, 32, s32, 0, true); - err |= TEST_ONE_SHIFT(0, 33, s32, 0, true); - err |= TEST_ONE_SHIFT(0, 64, u64, 0, true); - err |= TEST_ONE_SHIFT(0, 65, u64, 0, true); - err |= TEST_ONE_SHIFT(0, 64, s64, 0, true); - err |= TEST_ONE_SHIFT(0, 65, s64, 0, true); + TEST_ONE_SHIFT(0, 8, u8, 0, true); + TEST_ONE_SHIFT(0, 9, u8, 0, true); + TEST_ONE_SHIFT(0, 8, s8, 0, true); + TEST_ONE_SHIFT(0, 9, s8, 0, true); + TEST_ONE_SHIFT(0, 16, u16, 0, true); + TEST_ONE_SHIFT(0, 17, u16, 0, true); + TEST_ONE_SHIFT(0, 16, s16, 0, true); + TEST_ONE_SHIFT(0, 17, s16, 0, true); + TEST_ONE_SHIFT(0, 32, u32, 0, true); + TEST_ONE_SHIFT(0, 33, u32, 0, true); + TEST_ONE_SHIFT(0, 32, int, 0, true); + TEST_ONE_SHIFT(0, 33, int, 0, true); + TEST_ONE_SHIFT(0, 32, s32, 0, true); + TEST_ONE_SHIFT(0, 33, s32, 0, true); + TEST_ONE_SHIFT(0, 64, u64, 0, true); + TEST_ONE_SHIFT(0, 65, u64, 0, true); + TEST_ONE_SHIFT(0, 64, s64, 0, true); + TEST_ONE_SHIFT(0, 65, s64, 0, true); /* * Corner case: for unsigned types, we fail when we've shifted @@ -473,13 +440,14 @@ static int __init test_overflow_shift(void) * signed bit). So, for now, we will test this condition but * mark it as not expected to overflow. */ - err |= TEST_ONE_SHIFT(0, 7, s8, 0, false); - err |= TEST_ONE_SHIFT(0, 15, s16, 0, false); - err |= TEST_ONE_SHIFT(0, 31, int, 0, false); - err |= TEST_ONE_SHIFT(0, 31, s32, 0, false); - err |= TEST_ONE_SHIFT(0, 63, s64, 0, false); - - return err; + TEST_ONE_SHIFT(0, 7, s8, 0, false); + TEST_ONE_SHIFT(0, 15, s16, 0, false); + TEST_ONE_SHIFT(0, 31, int, 0, false); + TEST_ONE_SHIFT(0, 31, s32, 0, false); + TEST_ONE_SHIFT(0, 63, s64, 0, false); + + kunit_info(test, "%d shift tests finished\n", count); +#undef TEST_ONE_SHIFT } /* @@ -499,7 +467,7 @@ static int __init test_overflow_shift(void) #define TEST_SIZE (5 * 4096) #define DEFINE_TEST_ALLOC(func, free_func, want_arg, want_gfp, want_node)\ -static int __init test_ ## func (void *arg) \ +static void test_ ## func (struct kunit *test, void *arg) \ { \ volatile size_t a = TEST_SIZE; \ volatile size_t b = (SIZE_MAX / TEST_SIZE) + 1; \ @@ -507,31 +475,24 @@ static int __init test_ ## func (void *arg) \ \ /* Tiny allocation test. */ \ ptr = alloc ## want_arg ## want_gfp ## want_node (func, arg, 1);\ - if (!ptr) { \ - pr_warn(#func " failed regular allocation?!\n"); \ - return 1; \ - } \ + KUNIT_ASSERT_NOT_ERR_OR_NULL_MSG(test, ptr, \ + #func " failed regular allocation?!\n"); \ free ## want_arg (free_func, arg, ptr); \ \ /* Wrapped allocation test. */ \ ptr = alloc ## want_arg ## want_gfp ## want_node (func, arg, \ a * b); \ - if (!ptr) { \ - pr_warn(#func " unexpectedly failed bad wrapping?!\n"); \ - return 1; \ - } \ + KUNIT_ASSERT_NOT_ERR_OR_NULL_MSG(test, ptr, \ + #func " unexpectedly failed bad wrapping?!\n"); \ free ## want_arg (free_func, arg, ptr); \ \ /* Saturated allocation test. */ \ ptr = alloc ## want_arg ## want_gfp ## want_node (func, arg, \ array_size(a, b)); \ if (ptr) { \ - pr_warn(#func " missed saturation!\n"); \ + KUNIT_FAIL(test, #func " missed saturation!\n"); \ free ## want_arg (free_func, arg, ptr); \ - return 1; \ } \ - pr_info(#func " detected saturation\n"); \ - return 0; \ } /* @@ -544,10 +505,7 @@ DEFINE_TEST_ALLOC(kmalloc, kfree, 0, 1, 0); DEFINE_TEST_ALLOC(kmalloc_node, kfree, 0, 1, 1); DEFINE_TEST_ALLOC(kzalloc, kfree, 0, 1, 0); DEFINE_TEST_ALLOC(kzalloc_node, kfree, 0, 1, 1); -DEFINE_TEST_ALLOC(vmalloc, vfree, 0, 0, 0); -DEFINE_TEST_ALLOC(vmalloc_node, vfree, 0, 0, 1); -DEFINE_TEST_ALLOC(vzalloc, vfree, 0, 0, 0); -DEFINE_TEST_ALLOC(vzalloc_node, vfree, 0, 0, 1); +DEFINE_TEST_ALLOC(__vmalloc, vfree, 0, 1, 0); DEFINE_TEST_ALLOC(kvmalloc, kvfree, 0, 1, 0); DEFINE_TEST_ALLOC(kvmalloc_node, kvfree, 0, 1, 1); DEFINE_TEST_ALLOC(kvzalloc, kvfree, 0, 1, 0); @@ -555,60 +513,158 @@ DEFINE_TEST_ALLOC(kvzalloc_node, kvfree, 0, 1, 1); DEFINE_TEST_ALLOC(devm_kmalloc, devm_kfree, 1, 1, 0); DEFINE_TEST_ALLOC(devm_kzalloc, devm_kfree, 1, 1, 0); -static int __init test_overflow_allocation(void) +static void overflow_allocation_test(struct kunit *test) { const char device_name[] = "overflow-test"; struct device *dev; - int err = 0; + int count = 0; + +#define check_allocation_overflow(alloc) do { \ + count++; \ + test_ ## alloc(test, dev); \ +} while (0) /* Create dummy device for devm_kmalloc()-family tests. */ dev = root_device_register(device_name); - if (IS_ERR(dev)) { - pr_warn("Cannot register test device\n"); - return 1; - } - - err |= test_kmalloc(NULL); - err |= test_kmalloc_node(NULL); - err |= test_kzalloc(NULL); - err |= test_kzalloc_node(NULL); - err |= test_kvmalloc(NULL); - err |= test_kvmalloc_node(NULL); - err |= test_kvzalloc(NULL); - err |= test_kvzalloc_node(NULL); - err |= test_vmalloc(NULL); - err |= test_vmalloc_node(NULL); - err |= test_vzalloc(NULL); - err |= test_vzalloc_node(NULL); - err |= test_devm_kmalloc(dev); - err |= test_devm_kzalloc(dev); + KUNIT_ASSERT_FALSE_MSG(test, IS_ERR(dev), + "Cannot register test device\n"); + + check_allocation_overflow(kmalloc); + check_allocation_overflow(kmalloc_node); + check_allocation_overflow(kzalloc); + check_allocation_overflow(kzalloc_node); + check_allocation_overflow(__vmalloc); + check_allocation_overflow(kvmalloc); + check_allocation_overflow(kvmalloc_node); + check_allocation_overflow(kvzalloc); + check_allocation_overflow(kvzalloc_node); + check_allocation_overflow(devm_kmalloc); + check_allocation_overflow(devm_kzalloc); device_unregister(dev); - return err; + kunit_info(test, "%d allocation overflow tests finished\n", count); +#undef check_allocation_overflow } -static int __init test_module_init(void) +struct __test_flex_array { + unsigned long flags; + size_t count; + unsigned long data[]; +}; + +static void overflow_size_helpers_test(struct kunit *test) { - int err = 0; + /* Make sure struct_size() can be used in a constant expression. */ + u8 ce_array[struct_size((struct __test_flex_array *)0, data, 55)]; + struct __test_flex_array *obj; + int count = 0; + int var; + volatile int unconst = 0; + + /* Verify constant expression against runtime version. */ + var = 55; + OPTIMIZER_HIDE_VAR(var); + KUNIT_EXPECT_EQ(test, sizeof(ce_array), struct_size(obj, data, var)); + +#define check_one_size_helper(expected, func, args...) do { \ + size_t _r = func(args); \ + KUNIT_EXPECT_EQ_MSG(test, _r, expected, \ + "expected " #func "(" #args ") to return %zu but got %zu instead\n", \ + (size_t)(expected), _r); \ + count++; \ +} while (0) - err |= test_overflow_calculation(); - err |= test_overflow_shift(); - err |= test_overflow_allocation(); + var = 4; + check_one_size_helper(20, size_mul, var++, 5); + check_one_size_helper(20, size_mul, 4, var++); + check_one_size_helper(0, size_mul, 0, 3); + check_one_size_helper(0, size_mul, 3, 0); + check_one_size_helper(6, size_mul, 2, 3); + check_one_size_helper(SIZE_MAX, size_mul, SIZE_MAX, 1); + check_one_size_helper(SIZE_MAX, size_mul, SIZE_MAX, 3); + check_one_size_helper(SIZE_MAX, size_mul, SIZE_MAX, -3); + + var = 4; + check_one_size_helper(9, size_add, var++, 5); + check_one_size_helper(9, size_add, 4, var++); + check_one_size_helper(9, size_add, 9, 0); + check_one_size_helper(9, size_add, 0, 9); + check_one_size_helper(5, size_add, 2, 3); + check_one_size_helper(SIZE_MAX, size_add, SIZE_MAX, 1); + check_one_size_helper(SIZE_MAX, size_add, SIZE_MAX, 3); + check_one_size_helper(SIZE_MAX, size_add, SIZE_MAX, -3); + + var = 4; + check_one_size_helper(1, size_sub, var--, 3); + check_one_size_helper(1, size_sub, 4, var--); + check_one_size_helper(1, size_sub, 3, 2); + check_one_size_helper(9, size_sub, 9, 0); + check_one_size_helper(SIZE_MAX, size_sub, 9, -3); + check_one_size_helper(SIZE_MAX, size_sub, 0, 9); + check_one_size_helper(SIZE_MAX, size_sub, 2, 3); + check_one_size_helper(SIZE_MAX, size_sub, SIZE_MAX, 0); + check_one_size_helper(SIZE_MAX, size_sub, SIZE_MAX, 10); + check_one_size_helper(SIZE_MAX, size_sub, 0, SIZE_MAX); + check_one_size_helper(SIZE_MAX, size_sub, 14, SIZE_MAX); + check_one_size_helper(SIZE_MAX - 2, size_sub, SIZE_MAX - 1, 1); + check_one_size_helper(SIZE_MAX - 4, size_sub, SIZE_MAX - 1, 3); + check_one_size_helper(1, size_sub, SIZE_MAX - 1, -3); + + var = 4; + check_one_size_helper(4 * sizeof(*obj->data), + flex_array_size, obj, data, var++); + check_one_size_helper(5 * sizeof(*obj->data), + flex_array_size, obj, data, var++); + check_one_size_helper(0, flex_array_size, obj, data, 0 + unconst); + check_one_size_helper(sizeof(*obj->data), + flex_array_size, obj, data, 1 + unconst); + check_one_size_helper(7 * sizeof(*obj->data), + flex_array_size, obj, data, 7 + unconst); + check_one_size_helper(SIZE_MAX, + flex_array_size, obj, data, -1 + unconst); + check_one_size_helper(SIZE_MAX, + flex_array_size, obj, data, SIZE_MAX - 4 + unconst); + + var = 4; + check_one_size_helper(sizeof(*obj) + (4 * sizeof(*obj->data)), + struct_size, obj, data, var++); + check_one_size_helper(sizeof(*obj) + (5 * sizeof(*obj->data)), + struct_size, obj, data, var++); + check_one_size_helper(sizeof(*obj), struct_size, obj, data, 0 + unconst); + check_one_size_helper(sizeof(*obj) + sizeof(*obj->data), + struct_size, obj, data, 1 + unconst); + check_one_size_helper(SIZE_MAX, + struct_size, obj, data, -3 + unconst); + check_one_size_helper(SIZE_MAX, + struct_size, obj, data, SIZE_MAX - 3 + unconst); + + kunit_info(test, "%d overflow size helper tests finished\n", count); +#undef check_one_size_helper +} - if (err) { - pr_warn("FAIL!\n"); - err = -EINVAL; - } else { - pr_info("all tests passed\n"); - } +static struct kunit_case overflow_test_cases[] = { + KUNIT_CASE(u8_overflow_test), + KUNIT_CASE(s8_overflow_test), + KUNIT_CASE(u16_overflow_test), + KUNIT_CASE(s16_overflow_test), + KUNIT_CASE(u32_overflow_test), + KUNIT_CASE(s32_overflow_test), +#if BITS_PER_LONG == 64 + KUNIT_CASE(u64_overflow_test), + KUNIT_CASE(s64_overflow_test), +#endif + KUNIT_CASE(overflow_shift_test), + KUNIT_CASE(overflow_allocation_test), + KUNIT_CASE(overflow_size_helpers_test), + {} +}; - return err; -} +static struct kunit_suite overflow_test_suite = { + .name = "overflow", + .test_cases = overflow_test_cases, +}; -static void __exit test_module_exit(void) -{ } +kunit_test_suite(overflow_test_suite); -module_init(test_module_init); -module_exit(test_module_exit); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index af9302141bcf..e5c5315da274 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -76,6 +76,7 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, data = kzalloc(sizeof(*ref->data), gfp); if (!data) { free_percpu((void __percpu *)ref->percpu_count_ptr); + ref->percpu_count_ptr = 0; return -ENOMEM; } diff --git a/lib/polynomial.c b/lib/polynomial.c new file mode 100644 index 000000000000..66d383445fec --- /dev/null +++ b/lib/polynomial.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Generic polynomial calculation using integer coefficients. + * + * Copyright (C) 2020 BAIKAL ELECTRONICS, JSC + * + * Authors: + * Maxim Kaurkin <maxim.kaurkin@baikalelectronics.ru> + * Serge Semin <Sergey.Semin@baikalelectronics.ru> + * + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/polynomial.h> + +/* + * Originally this was part of drivers/hwmon/bt1-pvt.c. + * There the following conversion is used and should serve as an example here: + * + * The original translation formulae of the temperature (in degrees of Celsius) + * to PVT data and vice-versa are following: + * + * N = 1.8322e-8*(T^4) + 2.343e-5*(T^3) + 8.7018e-3*(T^2) + 3.9269*(T^1) + + * 1.7204e2 + * T = -1.6743e-11*(N^4) + 8.1542e-8*(N^3) + -1.8201e-4*(N^2) + + * 3.1020e-1*(N^1) - 4.838e1 + * + * where T = [-48.380, 147.438]C and N = [0, 1023]. + * + * They must be accordingly altered to be suitable for the integer arithmetics. + * The technique is called 'factor redistribution', which just makes sure the + * multiplications and divisions are made so to have a result of the operations + * within the integer numbers limit. In addition we need to translate the + * formulae to accept millidegrees of Celsius. Here what they look like after + * the alterations: + * + * N = (18322e-20*(T^4) + 2343e-13*(T^3) + 87018e-9*(T^2) + 39269e-3*T + + * 17204e2) / 1e4 + * T = -16743e-12*(D^4) + 81542e-9*(D^3) - 182010e-6*(D^2) + 310200e-3*D - + * 48380 + * where T = [-48380, 147438] mC and N = [0, 1023]. + * + * static const struct polynomial poly_temp_to_N = { + * .total_divider = 10000, + * .terms = { + * {4, 18322, 10000, 10000}, + * {3, 2343, 10000, 10}, + * {2, 87018, 10000, 10}, + * {1, 39269, 1000, 1}, + * {0, 1720400, 1, 1} + * } + * }; + * + * static const struct polynomial poly_N_to_temp = { + * .total_divider = 1, + * .terms = { + * {4, -16743, 1000, 1}, + * {3, 81542, 1000, 1}, + * {2, -182010, 1000, 1}, + * {1, 310200, 1000, 1}, + * {0, -48380, 1, 1} + * } + * }; + */ + +/** + * polynomial_calc - calculate a polynomial using integer arithmetic + * + * @poly: pointer to the descriptor of the polynomial + * @data: input value of the polynimal + * + * Calculate the result of a polynomial using only integer arithmetic. For + * this to work without too much loss of precision the coefficients has to + * be altered. This is called factor redistribution. + * + * Returns the result of the polynomial calculation. + */ +long polynomial_calc(const struct polynomial *poly, long data) +{ + const struct polynomial_term *term = poly->terms; + long total_divider = poly->total_divider ?: 1; + long tmp, ret = 0; + int deg; + + /* + * Here is the polynomial calculation function, which performs the + * redistributed terms calculations. It's pretty straightforward. + * We walk over each degree term up to the free one, and perform + * the redistributed multiplication of the term coefficient, its + * divider (as for the rationale fraction representation), data + * power and the rational fraction divider leftover. Then all of + * this is collected in a total sum variable, which value is + * normalized by the total divider before being returned. + */ + do { + tmp = term->coef; + for (deg = 0; deg < term->deg; ++deg) + tmp = mult_frac(tmp, data, term->divider); + ret += tmp / term->divider_leftover; + } while ((term++)->deg); + + return ret / total_divider; +} +EXPORT_SYMBOL_GPL(polynomial_calc); + +MODULE_DESCRIPTION("Generic polynomial calculations"); +MODULE_LICENSE("GPL"); diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile index a4c7cd74cff5..4fb7700a741b 100644 --- a/lib/raid6/test/Makefile +++ b/lib/raid6/test/Makefile @@ -4,6 +4,8 @@ # from userspace. # +pound := \# + CC = gcc OPTFLAGS = -O2 # Adjust as desired CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS) @@ -42,7 +44,7 @@ else ifeq ($(HAS_NEON),yes) OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 else - HAS_ALTIVEC := $(shell printf '\#include <altivec.h>\nvector int a;\n' |\ + HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\ gcc -c -x c - >/dev/null && rm ./-.o && echo yes) ifeq ($(HAS_ALTIVEC),yes) CFLAGS += -I../../../arch/powerpc/include diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c index a3cf071941ab..841a55242aba 100644 --- a/lib/raid6/test/test.c +++ b/lib/raid6/test/test.c @@ -19,7 +19,6 @@ #define NDISKS 16 /* Including P and Q */ const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -struct raid6_calls raid6_call; char *dataptrs[NDISKS]; char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); diff --git a/lib/raid6/vpermxor.uc b/lib/raid6/vpermxor.uc index 10475dc423c1..1bfb127fbfe8 100644 --- a/lib/raid6/vpermxor.uc +++ b/lib/raid6/vpermxor.uc @@ -24,9 +24,9 @@ #ifdef CONFIG_ALTIVEC #include <altivec.h> +#include <asm/ppc-opcode.h> #ifdef __KERNEL__ #include <asm/cputable.h> -#include <asm/ppc-opcode.h> #include <asm/switch_to.h> #endif diff --git a/lib/random32.c b/lib/random32.c index a57a0e18819d..d5d9029362cb 100644 --- a/lib/random32.c +++ b/lib/random32.c @@ -41,7 +41,6 @@ #include <linux/bitops.h> #include <linux/slab.h> #include <asm/unaligned.h> -#include <trace/events/random.h> /** * prandom_u32_state - seeded pseudo-random number generator. @@ -246,25 +245,13 @@ static struct prandom_test2 { { 407983964U, 921U, 728767059U }, }; -static u32 __extract_hwseed(void) -{ - unsigned int val = 0; - - (void)(arch_get_random_seed_int(&val) || - arch_get_random_int(&val)); - - return val; -} - -static void prandom_seed_early(struct rnd_state *state, u32 seed, - bool mix_with_hwseed) +static void prandom_state_selftest_seed(struct rnd_state *state, u32 seed) { #define LCG(x) ((x) * 69069U) /* super-duper LCG */ -#define HWSEED() (mix_with_hwseed ? __extract_hwseed() : 0) - state->s1 = __seed(HWSEED() ^ LCG(seed), 2U); - state->s2 = __seed(HWSEED() ^ LCG(state->s1), 8U); - state->s3 = __seed(HWSEED() ^ LCG(state->s2), 16U); - state->s4 = __seed(HWSEED() ^ LCG(state->s3), 128U); + state->s1 = __seed(LCG(seed), 2U); + state->s2 = __seed(LCG(state->s1), 8U); + state->s3 = __seed(LCG(state->s2), 16U); + state->s4 = __seed(LCG(state->s3), 128U); } static int __init prandom_state_selftest(void) @@ -275,7 +262,7 @@ static int __init prandom_state_selftest(void) for (i = 0; i < ARRAY_SIZE(test1); i++) { struct rnd_state state; - prandom_seed_early(&state, test1[i].seed, false); + prandom_state_selftest_seed(&state, test1[i].seed); prandom_warmup(&state); if (test1[i].result != prandom_u32_state(&state)) @@ -290,7 +277,7 @@ static int __init prandom_state_selftest(void) for (i = 0; i < ARRAY_SIZE(test2); i++) { struct rnd_state state; - prandom_seed_early(&state, test2[i].seed, false); + prandom_state_selftest_seed(&state, test2[i].seed); prandom_warmup(&state); for (j = 0; j < test2[i].iteration - 1; j++) @@ -311,323 +298,3 @@ static int __init prandom_state_selftest(void) } core_initcall(prandom_state_selftest); #endif - -/* - * The prandom_u32() implementation is now completely separate from the - * prandom_state() functions, which are retained (for now) for compatibility. - * - * Because of (ab)use in the networking code for choosing random TCP/UDP port - * numbers, which open DoS possibilities if guessable, we want something - * stronger than a standard PRNG. But the performance requirements of - * the network code do not allow robust crypto for this application. - * - * So this is a homebrew Junior Spaceman implementation, based on the - * lowest-latency trustworthy crypto primitive available, SipHash. - * (The authors of SipHash have not been consulted about this abuse of - * their work.) - * - * Standard SipHash-2-4 uses 2n+4 rounds to hash n words of input to - * one word of output. This abbreviated version uses 2 rounds per word - * of output. - */ - -struct siprand_state { - unsigned long v0; - unsigned long v1; - unsigned long v2; - unsigned long v3; -}; - -static DEFINE_PER_CPU(struct siprand_state, net_rand_state) __latent_entropy; -DEFINE_PER_CPU(unsigned long, net_rand_noise); -EXPORT_PER_CPU_SYMBOL(net_rand_noise); - -/* - * This is the core CPRNG function. As "pseudorandom", this is not used - * for truly valuable things, just intended to be a PITA to guess. - * For maximum speed, we do just two SipHash rounds per word. This is - * the same rate as 4 rounds per 64 bits that SipHash normally uses, - * so hopefully it's reasonably secure. - * - * There are two changes from the official SipHash finalization: - * - We omit some constants XORed with v2 in the SipHash spec as irrelevant; - * they are there only to make the output rounds distinct from the input - * rounds, and this application has no input rounds. - * - Rather than returning v0^v1^v2^v3, return v1+v3. - * If you look at the SipHash round, the last operation on v3 is - * "v3 ^= v0", so "v0 ^ v3" just undoes that, a waste of time. - * Likewise "v1 ^= v2". (The rotate of v2 makes a difference, but - * it still cancels out half of the bits in v2 for no benefit.) - * Second, since the last combining operation was xor, continue the - * pattern of alternating xor/add for a tiny bit of extra non-linearity. - */ -static inline u32 siprand_u32(struct siprand_state *s) -{ - unsigned long v0 = s->v0, v1 = s->v1, v2 = s->v2, v3 = s->v3; - unsigned long n = raw_cpu_read(net_rand_noise); - - v3 ^= n; - PRND_SIPROUND(v0, v1, v2, v3); - PRND_SIPROUND(v0, v1, v2, v3); - v0 ^= n; - s->v0 = v0; s->v1 = v1; s->v2 = v2; s->v3 = v3; - return v1 + v3; -} - - -/** - * prandom_u32 - pseudo random number generator - * - * A 32 bit pseudo-random number is generated using a fast - * algorithm suitable for simulation. This algorithm is NOT - * considered safe for cryptographic use. - */ -u32 prandom_u32(void) -{ - struct siprand_state *state = get_cpu_ptr(&net_rand_state); - u32 res = siprand_u32(state); - - trace_prandom_u32(res); - put_cpu_ptr(&net_rand_state); - return res; -} -EXPORT_SYMBOL(prandom_u32); - -/** - * prandom_bytes - get the requested number of pseudo-random bytes - * @buf: where to copy the pseudo-random bytes to - * @bytes: the requested number of bytes - */ -void prandom_bytes(void *buf, size_t bytes) -{ - struct siprand_state *state = get_cpu_ptr(&net_rand_state); - u8 *ptr = buf; - - while (bytes >= sizeof(u32)) { - put_unaligned(siprand_u32(state), (u32 *)ptr); - ptr += sizeof(u32); - bytes -= sizeof(u32); - } - - if (bytes > 0) { - u32 rem = siprand_u32(state); - - do { - *ptr++ = (u8)rem; - rem >>= BITS_PER_BYTE; - } while (--bytes > 0); - } - put_cpu_ptr(&net_rand_state); -} -EXPORT_SYMBOL(prandom_bytes); - -/** - * prandom_seed - add entropy to pseudo random number generator - * @entropy: entropy value - * - * Add some additional seed material to the prandom pool. - * The "entropy" is actually our IP address (the only caller is - * the network code), not for unpredictability, but to ensure that - * different machines are initialized differently. - */ -void prandom_seed(u32 entropy) -{ - int i; - - add_device_randomness(&entropy, sizeof(entropy)); - - for_each_possible_cpu(i) { - struct siprand_state *state = per_cpu_ptr(&net_rand_state, i); - unsigned long v0 = state->v0, v1 = state->v1; - unsigned long v2 = state->v2, v3 = state->v3; - - do { - v3 ^= entropy; - PRND_SIPROUND(v0, v1, v2, v3); - PRND_SIPROUND(v0, v1, v2, v3); - v0 ^= entropy; - } while (unlikely(!v0 || !v1 || !v2 || !v3)); - - WRITE_ONCE(state->v0, v0); - WRITE_ONCE(state->v1, v1); - WRITE_ONCE(state->v2, v2); - WRITE_ONCE(state->v3, v3); - } -} -EXPORT_SYMBOL(prandom_seed); - -/* - * Generate some initially weak seeding values to allow - * the prandom_u32() engine to be started. - */ -static int __init prandom_init_early(void) -{ - int i; - unsigned long v0, v1, v2, v3; - - if (!arch_get_random_long(&v0)) - v0 = jiffies; - if (!arch_get_random_long(&v1)) - v1 = random_get_entropy(); - v2 = v0 ^ PRND_K0; - v3 = v1 ^ PRND_K1; - - for_each_possible_cpu(i) { - struct siprand_state *state; - - v3 ^= i; - PRND_SIPROUND(v0, v1, v2, v3); - PRND_SIPROUND(v0, v1, v2, v3); - v0 ^= i; - - state = per_cpu_ptr(&net_rand_state, i); - state->v0 = v0; state->v1 = v1; - state->v2 = v2; state->v3 = v3; - } - - return 0; -} -core_initcall(prandom_init_early); - - -/* Stronger reseeding when available, and periodically thereafter. */ -static void prandom_reseed(struct timer_list *unused); - -static DEFINE_TIMER(seed_timer, prandom_reseed); - -static void prandom_reseed(struct timer_list *unused) -{ - unsigned long expires; - int i; - - /* - * Reinitialize each CPU's PRNG with 128 bits of key. - * No locking on the CPUs, but then somewhat random results are, - * well, expected. - */ - for_each_possible_cpu(i) { - struct siprand_state *state; - unsigned long v0 = get_random_long(), v2 = v0 ^ PRND_K0; - unsigned long v1 = get_random_long(), v3 = v1 ^ PRND_K1; -#if BITS_PER_LONG == 32 - int j; - - /* - * On 32-bit machines, hash in two extra words to - * approximate 128-bit key length. Not that the hash - * has that much security, but this prevents a trivial - * 64-bit brute force. - */ - for (j = 0; j < 2; j++) { - unsigned long m = get_random_long(); - - v3 ^= m; - PRND_SIPROUND(v0, v1, v2, v3); - PRND_SIPROUND(v0, v1, v2, v3); - v0 ^= m; - } -#endif - /* - * Probably impossible in practice, but there is a - * theoretical risk that a race between this reseeding - * and the target CPU writing its state back could - * create the all-zero SipHash fixed point. - * - * To ensure that never happens, ensure the state - * we write contains no zero words. - */ - state = per_cpu_ptr(&net_rand_state, i); - WRITE_ONCE(state->v0, v0 ? v0 : -1ul); - WRITE_ONCE(state->v1, v1 ? v1 : -1ul); - WRITE_ONCE(state->v2, v2 ? v2 : -1ul); - WRITE_ONCE(state->v3, v3 ? v3 : -1ul); - } - - /* reseed every ~60 seconds, in [40 .. 80) interval with slack */ - expires = round_jiffies(jiffies + 40 * HZ + prandom_u32_max(40 * HZ)); - mod_timer(&seed_timer, expires); -} - -/* - * The random ready callback can be called from almost any interrupt. - * To avoid worrying about whether it's safe to delay that interrupt - * long enough to seed all CPUs, just schedule an immediate timer event. - */ -static void prandom_timer_start(struct random_ready_callback *unused) -{ - mod_timer(&seed_timer, jiffies); -} - -#ifdef CONFIG_RANDOM32_SELFTEST -/* Principle: True 32-bit random numbers will all have 16 differing bits on - * average. For each 32-bit number, there are 601M numbers differing by 16 - * bits, and 89% of the numbers differ by at least 12 bits. Note that more - * than 16 differing bits also implies a correlation with inverted bits. Thus - * we take 1024 random numbers and compare each of them to the other ones, - * counting the deviation of correlated bits to 16. Constants report 32, - * counters 32-log2(TEST_SIZE), and pure randoms, around 6 or lower. With the - * u32 total, TEST_SIZE may be as large as 4096 samples. - */ -#define TEST_SIZE 1024 -static int __init prandom32_state_selftest(void) -{ - unsigned int x, y, bits, samples; - u32 xor, flip; - u32 total; - u32 *data; - - data = kmalloc(sizeof(*data) * TEST_SIZE, GFP_KERNEL); - if (!data) - return 0; - - for (samples = 0; samples < TEST_SIZE; samples++) - data[samples] = prandom_u32(); - - flip = total = 0; - for (x = 0; x < samples; x++) { - for (y = 0; y < samples; y++) { - if (x == y) - continue; - xor = data[x] ^ data[y]; - flip |= xor; - bits = hweight32(xor); - total += (bits - 16) * (bits - 16); - } - } - - /* We'll return the average deviation as 2*sqrt(corr/samples), which - * is also sqrt(4*corr/samples) which provides a better resolution. - */ - bits = int_sqrt(total / (samples * (samples - 1)) * 4); - if (bits > 6) - pr_warn("prandom32: self test failed (at least %u bits" - " correlated, fixed_mask=%#x fixed_value=%#x\n", - bits, ~flip, data[0] & ~flip); - else - pr_info("prandom32: self test passed (less than %u bits" - " correlated)\n", - bits+1); - kfree(data); - return 0; -} -core_initcall(prandom32_state_selftest); -#endif /* CONFIG_RANDOM32_SELFTEST */ - -/* - * Start periodic full reseeding as soon as strong - * random numbers are available. - */ -static int __init prandom_init_late(void) -{ - static struct random_ready_callback random_ready = { - .func = prandom_timer_start - }; - int ret = add_random_ready_callback(&random_ready); - - if (ret == -EALREADY) { - prandom_timer_start(&random_ready); - ret = 0; - } - return ret; -} -late_initcall(prandom_init_late); diff --git a/lib/ref_tracker.c b/lib/ref_tracker.c index a6789c0c626b..dc7b14aa3431 100644 --- a/lib/ref_tracker.c +++ b/lib/ref_tracker.c @@ -20,6 +20,7 @@ void ref_tracker_dir_exit(struct ref_tracker_dir *dir) unsigned long flags; bool leak = false; + dir->dead = true; spin_lock_irqsave(&dir->lock, flags); list_for_each_entry_safe(tracker, n, &dir->quarantine, head) { list_del(&tracker->head); @@ -37,6 +38,7 @@ void ref_tracker_dir_exit(struct ref_tracker_dir *dir) spin_unlock_irqrestore(&dir->lock, flags); WARN_ON_ONCE(leak); WARN_ON_ONCE(refcount_read(&dir->untracked) != 1); + WARN_ON_ONCE(refcount_read(&dir->no_tracker) != 1); } EXPORT_SYMBOL(ref_tracker_dir_exit); @@ -72,6 +74,12 @@ int ref_tracker_alloc(struct ref_tracker_dir *dir, gfp_t gfp_mask = gfp; unsigned long flags; + WARN_ON_ONCE(dir->dead); + + if (!trackerp) { + refcount_inc(&dir->no_tracker); + return 0; + } if (gfp & __GFP_DIRECT_RECLAIM) gfp_mask |= __GFP_NOFAIL; *trackerp = tracker = kzalloc(sizeof(*tracker), gfp_mask); @@ -81,7 +89,6 @@ int ref_tracker_alloc(struct ref_tracker_dir *dir, return -ENOMEM; } nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); - nr_entries = filter_irq_stacks(entries, nr_entries); tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp); spin_lock_irqsave(&dir->lock, flags); @@ -95,17 +102,23 @@ int ref_tracker_free(struct ref_tracker_dir *dir, struct ref_tracker **trackerp) { unsigned long entries[REF_TRACKER_STACK_ENTRIES]; - struct ref_tracker *tracker = *trackerp; depot_stack_handle_t stack_handle; + struct ref_tracker *tracker; unsigned int nr_entries; unsigned long flags; + WARN_ON_ONCE(dir->dead); + + if (!trackerp) { + refcount_dec(&dir->no_tracker); + return 0; + } + tracker = *trackerp; if (!tracker) { refcount_dec(&dir->untracked); return -EEXIST; } nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); - nr_entries = filter_irq_stacks(entries, nr_entries); stack_handle = stack_depot_save(entries, nr_entries, GFP_ATOMIC); spin_lock_irqsave(&dir->lock, flags); diff --git a/lib/sbitmap.c b/lib/sbitmap.c index 09d293c30fd2..29eb0484215a 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -85,7 +85,6 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, bool alloc_hint) { unsigned int bits_per_word; - unsigned int i; if (shift < 0) shift = sbitmap_calculate_shift(depth); @@ -111,16 +110,12 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, sb->alloc_hint = NULL; } - sb->map = kcalloc_node(sb->map_nr, sizeof(*sb->map), flags, node); + sb->map = kvzalloc_node(sb->map_nr * sizeof(*sb->map), flags, node); if (!sb->map) { free_percpu(sb->alloc_hint); return -ENOMEM; } - for (i = 0; i < sb->map_nr; i++) { - sb->map[i].depth = min(depth, bits_per_word); - depth -= sb->map[i].depth; - } return 0; } EXPORT_SYMBOL_GPL(sbitmap_init_node); @@ -135,11 +130,6 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth) sb->depth = depth; sb->map_nr = DIV_ROUND_UP(sb->depth, bits_per_word); - - for (i = 0; i < sb->map_nr; i++) { - sb->map[i].depth = min(depth, bits_per_word); - depth -= sb->map[i].depth; - } } EXPORT_SYMBOL_GPL(sbitmap_resize); @@ -184,8 +174,8 @@ static int sbitmap_find_bit_in_index(struct sbitmap *sb, int index, int nr; do { - nr = __sbitmap_get_word(&map->word, map->depth, alloc_hint, - !sb->round_robin); + nr = __sbitmap_get_word(&map->word, __map_depth(sb, index), + alloc_hint, !sb->round_robin); if (nr != -1) break; if (!sbitmap_deferred_clear(map)) @@ -257,7 +247,9 @@ static int __sbitmap_get_shallow(struct sbitmap *sb, for (i = 0; i < sb->map_nr; i++) { again: nr = __sbitmap_get_word(&sb->map[index].word, - min(sb->map[index].depth, shallow_depth), + min_t(unsigned int, + __map_depth(sb, index), + shallow_depth), SB_NR_TO_BIT(sb, alloc_hint), true); if (nr != -1) { nr += index << sb->shift; @@ -315,11 +307,12 @@ static unsigned int __sbitmap_weight(const struct sbitmap *sb, bool set) for (i = 0; i < sb->map_nr; i++) { const struct sbitmap_word *word = &sb->map[i]; + unsigned int word_depth = __map_depth(sb, i); if (set) - weight += bitmap_weight(&word->word, word->depth); + weight += bitmap_weight(&word->word, word_depth); else - weight += bitmap_weight(&word->cleared, word->depth); + weight += bitmap_weight(&word->cleared, word_depth); } return weight; } @@ -367,7 +360,7 @@ void sbitmap_bitmap_show(struct sbitmap *sb, struct seq_file *m) for (i = 0; i < sb->map_nr; i++) { unsigned long word = READ_ONCE(sb->map[i].word); unsigned long cleared = READ_ONCE(sb->map[i].cleared); - unsigned int word_bits = READ_ONCE(sb->map[i].depth); + unsigned int word_bits = __map_depth(sb, i); word &= ~cleared; @@ -531,20 +524,23 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, for (i = 0; i < sb->map_nr; i++) { struct sbitmap_word *map = &sb->map[index]; unsigned long get_mask; + unsigned int map_depth = __map_depth(sb, index); sbitmap_deferred_clear(map); - if (map->word == (1UL << (map->depth - 1)) - 1) - continue; + if (map->word == (1UL << (map_depth - 1)) - 1) + goto next; - nr = find_first_zero_bit(&map->word, map->depth); - if (nr + nr_tags <= map->depth) { + nr = find_first_zero_bit(&map->word, map_depth); + if (nr + nr_tags <= map_depth) { atomic_long_t *ptr = (atomic_long_t *) &map->word; - int map_tags = min_t(int, nr_tags, map->depth); + int map_tags = min_t(int, nr_tags, map_depth); unsigned long val, ret; get_mask = ((1UL << map_tags) - 1) << nr; do { val = READ_ONCE(map->word); + if ((val & ~get_mask) != val) + goto next; ret = atomic_long_cmpxchg(ptr, val, get_mask | val); } while (ret != val); get_mask = (get_mask & ~ret) >> nr; @@ -555,6 +551,7 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, return get_mask; } } +next: /* Jump to next index. */ if (++index >= sb->map_nr) index = 0; @@ -563,14 +560,14 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, return 0; } -int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, - unsigned int shallow_depth) +int sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, + unsigned int shallow_depth) { WARN_ON_ONCE(shallow_depth < sbq->min_shallow_depth); return sbitmap_get_shallow(&sbq->sb, shallow_depth); } -EXPORT_SYMBOL_GPL(__sbitmap_queue_get_shallow); +EXPORT_SYMBOL_GPL(sbitmap_queue_get_shallow); void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq, unsigned int min_shallow_depth) diff --git a/lib/siphash.c b/lib/siphash.c index 72b9068ab57b..15bc5b6f368c 100644 --- a/lib/siphash.c +++ b/lib/siphash.c @@ -1,6 +1,5 @@ -/* Copyright (C) 2016 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. - * - * This file is provided under a dual BSD/GPLv2 license. +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +/* Copyright (C) 2016-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * * SipHash: a fast short-input PRF * https://131002.net/siphash/ @@ -18,19 +17,13 @@ #include <asm/word-at-a-time.h> #endif -#define SIPROUND \ - do { \ - v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \ - v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \ - v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \ - v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \ - } while (0) +#define SIPROUND SIPHASH_PERMUTATION(v0, v1, v2, v3) #define PREAMBLE(len) \ - u64 v0 = 0x736f6d6570736575ULL; \ - u64 v1 = 0x646f72616e646f6dULL; \ - u64 v2 = 0x6c7967656e657261ULL; \ - u64 v3 = 0x7465646279746573ULL; \ + u64 v0 = SIPHASH_CONST_0; \ + u64 v1 = SIPHASH_CONST_1; \ + u64 v2 = SIPHASH_CONST_2; \ + u64 v3 = SIPHASH_CONST_3; \ u64 b = ((u64)(len)) << 56; \ v3 ^= key->key[1]; \ v2 ^= key->key[0]; \ @@ -389,19 +382,13 @@ u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third, } EXPORT_SYMBOL(hsiphash_4u32); #else -#define HSIPROUND \ - do { \ - v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \ - v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \ - v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \ - v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \ - } while (0) +#define HSIPROUND HSIPHASH_PERMUTATION(v0, v1, v2, v3) #define HPREAMBLE(len) \ - u32 v0 = 0; \ - u32 v1 = 0; \ - u32 v2 = 0x6c796765U; \ - u32 v3 = 0x74656462U; \ + u32 v0 = HSIPHASH_CONST_0; \ + u32 v1 = HSIPHASH_CONST_1; \ + u32 v2 = HSIPHASH_CONST_2; \ + u32 v3 = HSIPHASH_CONST_3; \ u32 b = ((u32)(len)) << 24; \ v3 ^= key->key[1]; \ v2 ^= key->key[0]; \ diff --git a/lib/slub_kunit.c b/lib/slub_kunit.c index 8662dc6cb509..7a0564d7cb7a 100644 --- a/lib/slub_kunit.c +++ b/lib/slub_kunit.c @@ -12,7 +12,7 @@ static int slab_errors; static void test_clobber_zone(struct kunit *test) { struct kmem_cache *s = kmem_cache_create("TestSlub_RZ_alloc", 64, 0, - SLAB_RED_ZONE, NULL); + SLAB_RED_ZONE|SLAB_NO_USER_FLAGS, NULL); u8 *p = kmem_cache_alloc(s, GFP_KERNEL); kasan_disable_current(); @@ -30,7 +30,7 @@ static void test_clobber_zone(struct kunit *test) static void test_next_pointer(struct kunit *test) { struct kmem_cache *s = kmem_cache_create("TestSlub_next_ptr_free", 64, 0, - SLAB_POISON, NULL); + SLAB_POISON|SLAB_NO_USER_FLAGS, NULL); u8 *p = kmem_cache_alloc(s, GFP_KERNEL); unsigned long tmp; unsigned long *ptr_addr; @@ -75,7 +75,7 @@ static void test_next_pointer(struct kunit *test) static void test_first_word(struct kunit *test) { struct kmem_cache *s = kmem_cache_create("TestSlub_1th_word_free", 64, 0, - SLAB_POISON, NULL); + SLAB_POISON|SLAB_NO_USER_FLAGS, NULL); u8 *p = kmem_cache_alloc(s, GFP_KERNEL); kmem_cache_free(s, p); @@ -90,7 +90,7 @@ static void test_first_word(struct kunit *test) static void test_clobber_50th_byte(struct kunit *test) { struct kmem_cache *s = kmem_cache_create("TestSlub_50th_word_free", 64, 0, - SLAB_POISON, NULL); + SLAB_POISON|SLAB_NO_USER_FLAGS, NULL); u8 *p = kmem_cache_alloc(s, GFP_KERNEL); kmem_cache_free(s, p); @@ -106,7 +106,7 @@ static void test_clobber_50th_byte(struct kunit *test) static void test_clobber_redzone_free(struct kunit *test) { struct kmem_cache *s = kmem_cache_create("TestSlub_RZ_free", 64, 0, - SLAB_RED_ZONE, NULL); + SLAB_RED_ZONE|SLAB_NO_USER_FLAGS, NULL); u8 *p = kmem_cache_alloc(s, GFP_KERNEL); kasan_disable_current(); diff --git a/lib/sort.c b/lib/sort.c index aa18153864d2..b399bf10d675 100644 --- a/lib/sort.c +++ b/lib/sort.c @@ -122,16 +122,27 @@ static void swap_bytes(void *a, void *b, size_t n) * a pointer, but small integers make for the smallest compare * instructions. */ -#define SWAP_WORDS_64 (swap_func_t)0 -#define SWAP_WORDS_32 (swap_func_t)1 -#define SWAP_BYTES (swap_func_t)2 +#define SWAP_WORDS_64 (swap_r_func_t)0 +#define SWAP_WORDS_32 (swap_r_func_t)1 +#define SWAP_BYTES (swap_r_func_t)2 +#define SWAP_WRAPPER (swap_r_func_t)3 + +struct wrapper { + cmp_func_t cmp; + swap_func_t swap; +}; /* * The function pointer is last to make tail calls most efficient if the * compiler decides not to inline this function. */ -static void do_swap(void *a, void *b, size_t size, swap_func_t swap_func) +static void do_swap(void *a, void *b, size_t size, swap_r_func_t swap_func, const void *priv) { + if (swap_func == SWAP_WRAPPER) { + ((const struct wrapper *)priv)->swap(a, b, (int)size); + return; + } + if (swap_func == SWAP_WORDS_64) swap_words_64(a, b, size); else if (swap_func == SWAP_WORDS_32) @@ -139,7 +150,7 @@ static void do_swap(void *a, void *b, size_t size, swap_func_t swap_func) else if (swap_func == SWAP_BYTES) swap_bytes(a, b, size); else - swap_func(a, b, (int)size); + swap_func(a, b, (int)size, priv); } #define _CMP_WRAPPER ((cmp_r_func_t)0L) @@ -147,7 +158,7 @@ static void do_swap(void *a, void *b, size_t size, swap_func_t swap_func) static int do_cmp(const void *a, const void *b, cmp_r_func_t cmp, const void *priv) { if (cmp == _CMP_WRAPPER) - return ((cmp_func_t)(priv))(a, b); + return ((const struct wrapper *)priv)->cmp(a, b); return cmp(a, b, priv); } @@ -198,7 +209,7 @@ static size_t parent(size_t i, unsigned int lsbit, size_t size) */ void sort_r(void *base, size_t num, size_t size, cmp_r_func_t cmp_func, - swap_func_t swap_func, + swap_r_func_t swap_func, const void *priv) { /* pre-scale counters for performance */ @@ -208,6 +219,10 @@ void sort_r(void *base, size_t num, size_t size, if (!a) /* num < 2 || size == 0 */ return; + /* called from 'sort' without swap function, let's pick the default */ + if (swap_func == SWAP_WRAPPER && !((struct wrapper *)priv)->swap) + swap_func = NULL; + if (!swap_func) { if (is_aligned(base, size, 8)) swap_func = SWAP_WORDS_64; @@ -230,7 +245,7 @@ void sort_r(void *base, size_t num, size_t size, if (a) /* Building heap: sift down --a */ a -= size; else if (n -= size) /* Sorting: Extract root to --n */ - do_swap(base, base + n, size, swap_func); + do_swap(base, base + n, size, swap_func, priv); else /* Sort complete */ break; @@ -257,7 +272,7 @@ void sort_r(void *base, size_t num, size_t size, c = b; /* Where "a" belongs */ while (b != a) { /* Shift it into place */ b = parent(b, lsbit, size); - do_swap(base + b, base + c, size, swap_func); + do_swap(base + b, base + c, size, swap_func, priv); } } } @@ -267,6 +282,11 @@ void sort(void *base, size_t num, size_t size, cmp_func_t cmp_func, swap_func_t swap_func) { - return sort_r(base, num, size, _CMP_WRAPPER, swap_func, cmp_func); + struct wrapper w = { + .cmp = cmp_func, + .swap = swap_func, + }; + + return sort_r(base, num, size, _CMP_WRAPPER, SWAP_WRAPPER, &w); } EXPORT_SYMBOL(sort); diff --git a/lib/stackdepot.c b/lib/stackdepot.c index bf5ba9af0500..5ca0d086ef4a 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -66,6 +66,9 @@ struct stack_record { unsigned long entries[]; /* Variable-sized array of entries. */ }; +static bool __stack_depot_want_early_init __initdata = IS_ENABLED(CONFIG_STACKDEPOT_ALWAYS_INIT); +static bool __stack_depot_early_init_passed __initdata; + static void *stack_slabs[STACK_ALLOC_MAX_SLABS]; static int depot_index; @@ -162,38 +165,58 @@ static int __init is_stack_depot_disabled(char *str) } early_param("stack_depot_disable", is_stack_depot_disabled); -/* - * __ref because of memblock_alloc(), which will not be actually called after - * the __init code is gone, because at that point slab_is_available() is true - */ -__ref int stack_depot_init(void) +void __init stack_depot_want_early_init(void) +{ + /* Too late to request early init now */ + WARN_ON(__stack_depot_early_init_passed); + + __stack_depot_want_early_init = true; +} + +int __init stack_depot_early_init(void) +{ + size_t size; + + /* This is supposed to be called only once, from mm_init() */ + if (WARN_ON(__stack_depot_early_init_passed)) + return 0; + + __stack_depot_early_init_passed = true; + + if (!__stack_depot_want_early_init || stack_depot_disable) + return 0; + + size = (STACK_HASH_SIZE * sizeof(struct stack_record *)); + pr_info("Stack Depot early init allocating hash table with memblock_alloc, %zu bytes\n", + size); + stack_table = memblock_alloc(size, SMP_CACHE_BYTES); + + if (!stack_table) { + pr_err("Stack Depot hash table allocation failed, disabling\n"); + stack_depot_disable = true; + return -ENOMEM; + } + + return 0; +} + +int stack_depot_init(void) { static DEFINE_MUTEX(stack_depot_init_mutex); + int ret = 0; mutex_lock(&stack_depot_init_mutex); if (!stack_depot_disable && !stack_table) { - size_t size = (STACK_HASH_SIZE * sizeof(struct stack_record *)); - int i; - - if (slab_is_available()) { - pr_info("Stack Depot allocating hash table with kvmalloc\n"); - stack_table = kvmalloc(size, GFP_KERNEL); - } else { - pr_info("Stack Depot allocating hash table with memblock_alloc\n"); - stack_table = memblock_alloc(size, SMP_CACHE_BYTES); - } - if (stack_table) { - for (i = 0; i < STACK_HASH_SIZE; i++) - stack_table[i] = NULL; - } else { + pr_info("Stack Depot allocating hash table with kvcalloc\n"); + stack_table = kvcalloc(STACK_HASH_SIZE, sizeof(struct stack_record *), GFP_KERNEL); + if (!stack_table) { pr_err("Stack Depot hash table allocation failed, disabling\n"); stack_depot_disable = true; - mutex_unlock(&stack_depot_init_mutex); - return -ENOMEM; + ret = -ENOMEM; } } mutex_unlock(&stack_depot_init_mutex); - return 0; + return ret; } EXPORT_SYMBOL_GPL(stack_depot_init); diff --git a/lib/test_stackinit.c b/lib/stackinit_kunit.c index a3c74e6a21ff..35c69aa425b2 100644 --- a/lib/test_stackinit.c +++ b/lib/stackinit_kunit.c @@ -2,76 +2,21 @@ /* * Test cases for compiler-based stack variable zeroing via * -ftrivial-auto-var-init={zero,pattern} or CONFIG_GCC_PLUGIN_STRUCTLEAK*. + * For example, see: + * https://www.kernel.org/doc/html/latest/dev-tools/kunit/kunit-tool.html#configuring-building-and-running-tests + * ./tools/testing/kunit/kunit.py run stackinit [--raw_output] \ + * --make_option LLVM=1 \ + * --kconfig_add CONFIG_INIT_STACK_ALL_ZERO=y * - * External build example: - * clang -O2 -Wall -ftrivial-auto-var-init=pattern \ - * -o test_stackinit test_stackinit.c */ -#ifdef __KERNEL__ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <kunit/test.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/string.h> -#else - -/* Userspace headers. */ -#include <stdio.h> -#include <stdint.h> -#include <string.h> -#include <stdbool.h> -#include <errno.h> -#include <sys/types.h> - -/* Linux kernel-ism stubs for stand-alone userspace build. */ -#define KBUILD_MODNAME "stackinit" -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#define pr_err(fmt, ...) fprintf(stderr, pr_fmt(fmt), ##__VA_ARGS__) -#define pr_warn(fmt, ...) fprintf(stderr, pr_fmt(fmt), ##__VA_ARGS__) -#define pr_info(fmt, ...) fprintf(stdout, pr_fmt(fmt), ##__VA_ARGS__) -#define __init /**/ -#define __exit /**/ -#define __user /**/ -#define noinline __attribute__((__noinline__)) -#define __aligned(x) __attribute__((__aligned__(x))) -#ifdef __clang__ -# define __compiletime_error(message) /**/ -#else -# define __compiletime_error(message) __attribute__((__error__(message))) -#endif -#define __compiletime_assert(condition, msg, prefix, suffix) \ - do { \ - extern void prefix ## suffix(void) __compiletime_error(msg); \ - if (!(condition)) \ - prefix ## suffix(); \ - } while (0) -#define _compiletime_assert(condition, msg, prefix, suffix) \ - __compiletime_assert(condition, msg, prefix, suffix) -#define compiletime_assert(condition, msg) \ - _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__) -#define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg) -#define BUILD_BUG_ON(condition) \ - BUILD_BUG_ON_MSG(condition, "BUILD_BUG_ON failed: " #condition) -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; - -#define module_init(func) static int (*do_init)(void) = func -#define module_exit(func) static void (*do_exit)(void) = func -#define MODULE_LICENSE(str) int main(void) { \ - int rc; \ - /* License: str */ \ - rc = do_init(); \ - if (rc == 0) \ - do_exit(); \ - return rc; \ - } - -#endif /* __KERNEL__ */ - /* Exfiltration buffer. */ #define MAX_VAR_SIZE 128 static u8 check_buf[MAX_VAR_SIZE]; @@ -201,7 +146,7 @@ static bool range_contains(char *haystack_start, size_t haystack_size, */ #define DEFINE_TEST_DRIVER(name, var_type, which, xfail) \ /* Returns 0 on success, 1 on failure. */ \ -static noinline __init int test_ ## name (void) \ +static noinline void test_ ## name (struct kunit *test) \ { \ var_type zero INIT_CLONE_ ## which; \ int ignored; \ @@ -220,10 +165,8 @@ static noinline __init int test_ ## name (void) \ /* Verify all bytes overwritten with 0xFF. */ \ for (sum = 0, i = 0; i < target_size; i++) \ sum += (check_buf[i] != 0xFF); \ - if (sum) { \ - pr_err(#name ": leaf fill was not 0xFF!?\n"); \ - return 1; \ - } \ + KUNIT_ASSERT_EQ_MSG(test, sum, 0, \ + "leaf fill was not 0xFF!?\n"); \ /* Clear entire check buffer for later bit tests. */ \ memset(check_buf, 0x00, sizeof(check_buf)); \ /* Extract stack-defined variable contents. */ \ @@ -231,32 +174,29 @@ static noinline __init int test_ ## name (void) \ FETCH_ARG_ ## which(zero)); \ \ /* Validate that compiler lined up fill and target. */ \ - if (!range_contains(fill_start, fill_size, \ - target_start, target_size)) { \ - pr_err(#name ": stack fill missed target!?\n"); \ - pr_err(#name ": fill %zu wide\n", fill_size); \ - pr_err(#name ": target offset by %d\n", \ - (int)((ssize_t)(uintptr_t)fill_start - \ - (ssize_t)(uintptr_t)target_start)); \ - return 1; \ - } \ + KUNIT_ASSERT_TRUE_MSG(test, \ + range_contains(fill_start, fill_size, \ + target_start, target_size), \ + "stack fill missed target!? " \ + "(fill %zu wide, target offset by %d)\n", \ + fill_size, \ + (int)((ssize_t)(uintptr_t)fill_start - \ + (ssize_t)(uintptr_t)target_start)); \ \ /* Look for any bytes still 0xFF in check region. */ \ for (sum = 0, i = 0; i < target_size; i++) \ sum += (check_buf[i] == 0xFF); \ \ - if (sum == 0) { \ - pr_info(#name " ok\n"); \ - return 0; \ - } else { \ - pr_warn(#name " %sFAIL (uninit bytes: %d)\n", \ - (xfail) ? "X" : "", sum); \ - return (xfail) ? 0 : 1; \ - } \ + if (sum != 0 && xfail) \ + kunit_skip(test, \ + "XFAIL uninit bytes: %d\n", \ + sum); \ + KUNIT_ASSERT_EQ_MSG(test, sum, 0, \ + "uninit bytes: %d\n", sum); \ } #define DEFINE_TEST(name, var_type, which, init_level, xfail) \ /* no-op to force compiler into ignoring "uninitialized" vars */\ -static noinline __init DO_NOTHING_TYPE_ ## which(var_type) \ +static noinline DO_NOTHING_TYPE_ ## which(var_type) \ do_nothing_ ## name(var_type *ptr) \ { \ /* Will always be true, but compiler doesn't know. */ \ @@ -265,9 +205,8 @@ do_nothing_ ## name(var_type *ptr) \ else \ return DO_NOTHING_RETURN_ ## which(ptr + 1); \ } \ -static noinline __init int leaf_ ## name(unsigned long sp, \ - bool fill, \ - var_type *arg) \ +static noinline int leaf_ ## name(unsigned long sp, bool fill, \ + var_type *arg) \ { \ char buf[VAR_BUFFER]; \ var_type var \ @@ -341,6 +280,27 @@ struct test_user { unsigned long four; }; +#define ALWAYS_PASS WANT_SUCCESS +#define ALWAYS_FAIL XFAIL + +#ifdef CONFIG_INIT_STACK_NONE +# define USER_PASS XFAIL +# define BYREF_PASS XFAIL +# define STRONG_PASS XFAIL +#elif defined(CONFIG_GCC_PLUGIN_STRUCTLEAK_USER) +# define USER_PASS WANT_SUCCESS +# define BYREF_PASS XFAIL +# define STRONG_PASS XFAIL +#elif defined(CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF) +# define USER_PASS WANT_SUCCESS +# define BYREF_PASS WANT_SUCCESS +# define STRONG_PASS XFAIL +#else +# define USER_PASS WANT_SUCCESS +# define BYREF_PASS WANT_SUCCESS +# define STRONG_PASS WANT_SUCCESS +#endif + #define DEFINE_SCALAR_TEST(name, init, xfail) \ DEFINE_TEST(name ## _ ## init, name, SCALAR, \ init, xfail) @@ -364,27 +324,26 @@ struct test_user { DEFINE_STRUCT_TEST(trailing_hole, init, xfail); \ DEFINE_STRUCT_TEST(packed, init, xfail) -#define DEFINE_STRUCT_INITIALIZER_TESTS(base) \ +#define DEFINE_STRUCT_INITIALIZER_TESTS(base, xfail) \ DEFINE_STRUCT_TESTS(base ## _ ## partial, \ - WANT_SUCCESS); \ - DEFINE_STRUCT_TESTS(base ## _ ## all, \ - WANT_SUCCESS) + xfail); \ + DEFINE_STRUCT_TESTS(base ## _ ## all, xfail) /* These should be fully initialized all the time! */ -DEFINE_SCALAR_TESTS(zero, WANT_SUCCESS); -DEFINE_STRUCT_TESTS(zero, WANT_SUCCESS); +DEFINE_SCALAR_TESTS(zero, ALWAYS_PASS); +DEFINE_STRUCT_TESTS(zero, ALWAYS_PASS); /* Struct initializers: padding may be left uninitialized. */ -DEFINE_STRUCT_INITIALIZER_TESTS(static); -DEFINE_STRUCT_INITIALIZER_TESTS(dynamic); -DEFINE_STRUCT_INITIALIZER_TESTS(runtime); -DEFINE_STRUCT_INITIALIZER_TESTS(assigned_static); -DEFINE_STRUCT_INITIALIZER_TESTS(assigned_dynamic); -DEFINE_STRUCT_TESTS(assigned_copy, XFAIL); +DEFINE_STRUCT_INITIALIZER_TESTS(static, STRONG_PASS); +DEFINE_STRUCT_INITIALIZER_TESTS(dynamic, STRONG_PASS); +DEFINE_STRUCT_INITIALIZER_TESTS(runtime, STRONG_PASS); +DEFINE_STRUCT_INITIALIZER_TESTS(assigned_static, STRONG_PASS); +DEFINE_STRUCT_INITIALIZER_TESTS(assigned_dynamic, STRONG_PASS); +DEFINE_STRUCT_TESTS(assigned_copy, ALWAYS_FAIL); /* No initialization without compiler instrumentation. */ -DEFINE_SCALAR_TESTS(none, WANT_SUCCESS); -DEFINE_STRUCT_TESTS(none, WANT_SUCCESS); +DEFINE_SCALAR_TESTS(none, STRONG_PASS); +DEFINE_STRUCT_TESTS(none, BYREF_PASS); /* Initialization of members with __user attribute. */ -DEFINE_TEST(user, struct test_user, STRUCT, none, WANT_SUCCESS); +DEFINE_TEST(user, struct test_user, STRUCT, none, USER_PASS); /* * Check two uses through a variable declaration outside either path, @@ -398,7 +357,7 @@ static int noinline __leaf_switch_none(int path, bool fill) * This is intentionally unreachable. To silence the * warning, build with -Wno-switch-unreachable */ - uint64_t var; + uint64_t var[10]; case 1: target_start = &var; @@ -423,19 +382,19 @@ static int noinline __leaf_switch_none(int path, bool fill) memcpy(check_buf, target_start, target_size); break; default: - var = 5; - return var & forced_mask; + var[1] = 5; + return var[1] & forced_mask; } return 0; } -static noinline __init int leaf_switch_1_none(unsigned long sp, bool fill, +static noinline int leaf_switch_1_none(unsigned long sp, bool fill, uint64_t *arg) { return __leaf_switch_none(1, fill); } -static noinline __init int leaf_switch_2_none(unsigned long sp, bool fill, +static noinline int leaf_switch_2_none(unsigned long sp, bool fill, uint64_t *arg) { return __leaf_switch_none(2, fill); @@ -447,68 +406,56 @@ static noinline __init int leaf_switch_2_none(unsigned long sp, bool fill, * non-code areas (i.e. in a switch statement before the first "case"). * https://bugs.llvm.org/show_bug.cgi?id=44916 */ -DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR, XFAIL); -DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR, XFAIL); - -static int __init test_stackinit_init(void) -{ - unsigned int failures = 0; - -#define test_scalars(init) do { \ - failures += test_u8_ ## init (); \ - failures += test_u16_ ## init (); \ - failures += test_u32_ ## init (); \ - failures += test_u64_ ## init (); \ - failures += test_char_array_ ## init (); \ - } while (0) - -#define test_structs(init) do { \ - failures += test_small_hole_ ## init (); \ - failures += test_big_hole_ ## init (); \ - failures += test_trailing_hole_ ## init (); \ - failures += test_packed_ ## init (); \ - } while (0) - +DEFINE_TEST_DRIVER(switch_1_none, uint64_t, SCALAR, ALWAYS_FAIL); +DEFINE_TEST_DRIVER(switch_2_none, uint64_t, SCALAR, ALWAYS_FAIL); + +#define KUNIT_test_scalars(init) \ + KUNIT_CASE(test_u8_ ## init), \ + KUNIT_CASE(test_u16_ ## init), \ + KUNIT_CASE(test_u32_ ## init), \ + KUNIT_CASE(test_u64_ ## init), \ + KUNIT_CASE(test_char_array_ ## init) + +#define KUNIT_test_structs(init) \ + KUNIT_CASE(test_small_hole_ ## init), \ + KUNIT_CASE(test_big_hole_ ## init), \ + KUNIT_CASE(test_trailing_hole_ ## init),\ + KUNIT_CASE(test_packed_ ## init) \ + +static struct kunit_case stackinit_test_cases[] = { /* These are explicitly initialized and should always pass. */ - test_scalars(zero); - test_structs(zero); + KUNIT_test_scalars(zero), + KUNIT_test_structs(zero), /* Padding here appears to be accidentally always initialized? */ - test_structs(dynamic_partial); - test_structs(assigned_dynamic_partial); + KUNIT_test_structs(dynamic_partial), + KUNIT_test_structs(assigned_dynamic_partial), /* Padding initialization depends on compiler behaviors. */ - test_structs(static_partial); - test_structs(static_all); - test_structs(dynamic_all); - test_structs(runtime_partial); - test_structs(runtime_all); - test_structs(assigned_static_partial); - test_structs(assigned_static_all); - test_structs(assigned_dynamic_all); + KUNIT_test_structs(static_partial), + KUNIT_test_structs(static_all), + KUNIT_test_structs(dynamic_all), + KUNIT_test_structs(runtime_partial), + KUNIT_test_structs(runtime_all), + KUNIT_test_structs(assigned_static_partial), + KUNIT_test_structs(assigned_static_all), + KUNIT_test_structs(assigned_dynamic_all), /* Everything fails this since it effectively performs a memcpy(). */ - test_structs(assigned_copy); - + KUNIT_test_structs(assigned_copy), /* STRUCTLEAK_BYREF_ALL should cover everything from here down. */ - test_scalars(none); - failures += test_switch_1_none(); - failures += test_switch_2_none(); - + KUNIT_test_scalars(none), + KUNIT_CASE(test_switch_1_none), + KUNIT_CASE(test_switch_2_none), /* STRUCTLEAK_BYREF should cover from here down. */ - test_structs(none); - + KUNIT_test_structs(none), /* STRUCTLEAK will only cover this. */ - failures += test_user(); - - if (failures == 0) - pr_info("all tests passed!\n"); - else - pr_err("failures: %u\n", failures); + KUNIT_CASE(test_user), + {} +}; - return failures ? -EINVAL : 0; -} -module_init(test_stackinit_init); +static struct kunit_suite stackinit_test_suite = { + .name = "stackinit", + .test_cases = stackinit_test_cases, +}; -static void __exit test_stackinit_exit(void) -{ } -module_exit(test_stackinit_exit); +kunit_test_suites(&stackinit_test_suite); MODULE_LICENSE("GPL"); diff --git a/lib/string.c b/lib/string.c index 485777c9da83..6f334420f687 100644 --- a/lib/string.c +++ b/lib/string.c @@ -517,21 +517,13 @@ EXPORT_SYMBOL(strnlen); size_t strspn(const char *s, const char *accept) { const char *p; - const char *a; - size_t count = 0; for (p = s; *p != '\0'; ++p) { - for (a = accept; *a != '\0'; ++a) { - if (*p == *a) - break; - } - if (*a == '\0') - return count; - ++count; + if (!strchr(accept, *p)) + break; } - return count; + return p - s; } - EXPORT_SYMBOL(strspn); #endif @@ -544,17 +536,12 @@ EXPORT_SYMBOL(strspn); size_t strcspn(const char *s, const char *reject) { const char *p; - const char *r; - size_t count = 0; for (p = s; *p != '\0'; ++p) { - for (r = reject; *r != '\0'; ++r) { - if (*p == *r) - return count; - } - ++count; + if (strchr(reject, *p)) + break; } - return count; + return p - s; } EXPORT_SYMBOL(strcspn); #endif diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 90f9f1b7afec..5ed3beb066e6 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -757,6 +757,9 @@ char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n) return ERR_PTR(-ENOMEM); } + ptr->n = n; + devres_add(dev, ptr); + return ptr->array; } EXPORT_SYMBOL_GPL(devm_kasprintf_strarray); @@ -968,6 +971,12 @@ void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count, EXPORT_SYMBOL(memcpy_and_pad); #ifdef CONFIG_FORTIFY_SOURCE +/* These are placeholders for fortify compile-time warnings. */ +void __read_overflow2_field(size_t avail, size_t wanted) { } +EXPORT_SYMBOL(__read_overflow2_field); +void __write_overflow_field(size_t avail, size_t wanted) { } +EXPORT_SYMBOL(__write_overflow_field); + void fortify_panic(const char *name) { pr_emerg("detected buffer overflow in %s\n", name); diff --git a/lib/strncpy_from_user.c b/lib/strncpy_from_user.c index 122d8d0e253c..6432b8c3e431 100644 --- a/lib/strncpy_from_user.c +++ b/lib/strncpy_from_user.c @@ -25,7 +25,7 @@ * hit it), 'max' is the address space maximum (and we return * -EFAULT if we hit it). */ -static inline long do_strncpy_from_user(char *dst, const char __user *src, +static __always_inline long do_strncpy_from_user(char *dst, const char __user *src, unsigned long count, unsigned long max) { const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; @@ -120,7 +120,7 @@ long strncpy_from_user(char *dst, const char __user *src, long count) if (unlikely(count <= 0)) return 0; - max_addr = user_addr_max(); + max_addr = TASK_SIZE_MAX; src_addr = (unsigned long)untagged_addr(src); if (likely(src_addr < max_addr)) { unsigned long max = max_addr - src_addr; diff --git a/lib/strnlen_user.c b/lib/strnlen_user.c index 1616710b8a82..feeb935a2299 100644 --- a/lib/strnlen_user.c +++ b/lib/strnlen_user.c @@ -20,7 +20,7 @@ * if it fits in a aligned 'long'. The caller needs to check * the return value against "> max". */ -static inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max) +static __always_inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max) { const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; unsigned long align, res = 0; @@ -96,7 +96,7 @@ long strnlen_user(const char __user *str, long count) if (unlikely(count <= 0)) return 0; - max_addr = user_addr_max(); + max_addr = TASK_SIZE_MAX; src_addr = (unsigned long)untagged_addr(str); if (likely(src_addr < max_addr)) { unsigned long max = max_addr - src_addr; diff --git a/lib/test_bitmap.c b/lib/test_bitmap.c index 0c82f07f74fc..d5923a640457 100644 --- a/lib/test_bitmap.c +++ b/lib/test_bitmap.c @@ -585,6 +585,30 @@ static void __init test_bitmap_arr32(void) } } +static void __init test_bitmap_arr64(void) +{ + unsigned int nbits, next_bit; + u64 arr[EXP1_IN_BITS / 64]; + DECLARE_BITMAP(bmap2, EXP1_IN_BITS); + + memset(arr, 0xa5, sizeof(arr)); + + for (nbits = 0; nbits < EXP1_IN_BITS; ++nbits) { + memset(bmap2, 0xff, sizeof(arr)); + bitmap_to_arr64(arr, exp1, nbits); + bitmap_from_arr64(bmap2, arr, nbits); + expect_eq_bitmap(bmap2, exp1, nbits); + + next_bit = find_next_bit(bmap2, round_up(nbits, BITS_PER_LONG), nbits); + if (next_bit < round_up(nbits, BITS_PER_LONG)) + pr_err("bitmap_copy_arr64(nbits == %d:" + " tail is not safely cleared: %d\n", nbits, next_bit); + + if (nbits < EXP1_IN_BITS - 64) + expect_eq_uint(arr[DIV_ROUND_UP(nbits, 64)], 0xa5a5a5a5); + } +} + static void noinline __init test_mem_optimisations(void) { DECLARE_BITMAP(bmap1, 1024); @@ -852,6 +876,7 @@ static void __init selftest(void) test_copy(); test_replace(); test_bitmap_arr32(); + test_bitmap_arr64(); test_bitmap_parse(); test_bitmap_parselist(); test_bitmap_printlist(); diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 0c5cb2d6436a..2a7836e115b4 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -53,6 +53,7 @@ #define FLAG_EXPECTED_FAIL BIT(1) #define FLAG_SKB_FRAG BIT(2) #define FLAG_VERIFIER_ZEXT BIT(3) +#define FLAG_LARGE_MEM BIT(4) enum { CLASSIC = BIT(6), /* Old BPF instructions only. */ @@ -7838,7 +7839,7 @@ static struct bpf_test tests[] = { }, /* BPF_LDX_MEM B/H/W/DW */ { - "BPF_LDX_MEM | BPF_B", + "BPF_LDX_MEM | BPF_B, base", .u.insns_int = { BPF_LD_IMM64(R1, 0x0102030405060708ULL), BPF_LD_IMM64(R2, 0x0000000000000008ULL), @@ -7878,7 +7879,56 @@ static struct bpf_test tests[] = { .stack_depth = 8, }, { - "BPF_LDX_MEM | BPF_H", + "BPF_LDX_MEM | BPF_B, negative offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000000000088ULL), + BPF_ALU64_IMM(BPF_ADD, R1, 512), + BPF_STX_MEM(BPF_B, R1, R2, -256), + BPF_LDX_MEM(BPF_B, R0, R1, -256), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_B, small positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000000000088ULL), + BPF_STX_MEM(BPF_B, R1, R2, 256), + BPF_LDX_MEM(BPF_B, R0, R1, 256), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_B, large positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000000000088ULL), + BPF_STX_MEM(BPF_B, R1, R2, 4096), + BPF_LDX_MEM(BPF_B, R0, R1, 4096), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 4096 + 16, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_H, base", .u.insns_int = { BPF_LD_IMM64(R1, 0x0102030405060708ULL), BPF_LD_IMM64(R2, 0x0000000000000708ULL), @@ -7918,7 +7968,72 @@ static struct bpf_test tests[] = { .stack_depth = 8, }, { - "BPF_LDX_MEM | BPF_W", + "BPF_LDX_MEM | BPF_H, negative offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000000008788ULL), + BPF_ALU64_IMM(BPF_ADD, R1, 512), + BPF_STX_MEM(BPF_H, R1, R2, -256), + BPF_LDX_MEM(BPF_H, R0, R1, -256), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_H, small positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000000008788ULL), + BPF_STX_MEM(BPF_H, R1, R2, 256), + BPF_LDX_MEM(BPF_H, R0, R1, 256), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_H, large positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000000008788ULL), + BPF_STX_MEM(BPF_H, R1, R2, 8192), + BPF_LDX_MEM(BPF_H, R0, R1, 8192), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 8192 + 16, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_H, unaligned positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000000008788ULL), + BPF_STX_MEM(BPF_H, R1, R2, 13), + BPF_LDX_MEM(BPF_H, R0, R1, 13), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 32, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_W, base", .u.insns_int = { BPF_LD_IMM64(R1, 0x0102030405060708ULL), BPF_LD_IMM64(R2, 0x0000000005060708ULL), @@ -7957,6 +8072,162 @@ static struct bpf_test tests[] = { { { 0, 0 } }, .stack_depth = 8, }, + { + "BPF_LDX_MEM | BPF_W, negative offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000085868788ULL), + BPF_ALU64_IMM(BPF_ADD, R1, 512), + BPF_STX_MEM(BPF_W, R1, R2, -256), + BPF_LDX_MEM(BPF_W, R0, R1, -256), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_W, small positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000085868788ULL), + BPF_STX_MEM(BPF_W, R1, R2, 256), + BPF_LDX_MEM(BPF_W, R0, R1, 256), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_W, large positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000085868788ULL), + BPF_STX_MEM(BPF_W, R1, R2, 16384), + BPF_LDX_MEM(BPF_W, R0, R1, 16384), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 16384 + 16, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_W, unaligned positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_LD_IMM64(R3, 0x0000000085868788ULL), + BPF_STX_MEM(BPF_W, R1, R2, 13), + BPF_LDX_MEM(BPF_W, R0, R1, 13), + BPF_JMP_REG(BPF_JNE, R0, R3, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 32, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_DW, base", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x0102030405060708ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_DW, MSB set", + .u.insns_int = { + BPF_LD_IMM64(R1, 0x8182838485868788ULL), + BPF_STX_MEM(BPF_DW, R10, R1, -8), + BPF_LDX_MEM(BPF_DW, R0, R10, -8), + BPF_JMP_REG(BPF_JNE, R0, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL, + { }, + { { 0, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_DW, negative offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_ALU64_IMM(BPF_ADD, R1, 512), + BPF_STX_MEM(BPF_DW, R1, R2, -256), + BPF_LDX_MEM(BPF_DW, R0, R1, -256), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_DW, small positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_STX_MEM(BPF_DW, R1, R2, 256), + BPF_LDX_MEM(BPF_DW, R0, R1, 256), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 512, 0 } }, + .stack_depth = 8, + }, + { + "BPF_LDX_MEM | BPF_DW, large positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_STX_MEM(BPF_DW, R1, R2, 32760), + BPF_LDX_MEM(BPF_DW, R0, R1, 32760), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 32768, 0 } }, + .stack_depth = 0, + }, + { + "BPF_LDX_MEM | BPF_DW, unaligned positive offset", + .u.insns_int = { + BPF_LD_IMM64(R2, 0x8182838485868788ULL), + BPF_STX_MEM(BPF_DW, R1, R2, 13), + BPF_LDX_MEM(BPF_DW, R0, R1, 13), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + INTERNAL | FLAG_LARGE_MEM, + { }, + { { 32, 0 } }, + .stack_depth = 0, + }, /* BPF_STX_MEM B/H/W/DW */ { "BPF_STX_MEM | BPF_B", @@ -14094,6 +14365,9 @@ static void *generate_test_data(struct bpf_test *test, int sub) if (test->aux & FLAG_NO_DATA) return NULL; + if (test->aux & FLAG_LARGE_MEM) + return kmalloc(test->test[sub].data_size, GFP_KERNEL); + /* Test case expects an skb, so populate one. Various * subtests generate skbs of different sizes based on * the same data. @@ -14137,7 +14411,10 @@ static void release_test_data(const struct bpf_test *test, void *data) if (test->aux & FLAG_NO_DATA) return; - kfree_skb(data); + if (test->aux & FLAG_LARGE_MEM) + kfree(data); + else + kfree_skb(data); } static int filter_length(int which) @@ -14674,6 +14951,36 @@ static struct tail_call_test tail_call_tests[] = { .result = 10, }, { + "Tail call load/store leaf", + .insns = { + BPF_ALU64_IMM(BPF_MOV, R1, 1), + BPF_ALU64_IMM(BPF_MOV, R2, 2), + BPF_ALU64_REG(BPF_MOV, R3, BPF_REG_FP), + BPF_STX_MEM(BPF_DW, R3, R1, -8), + BPF_STX_MEM(BPF_DW, R3, R2, -16), + BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -8), + BPF_JMP_REG(BPF_JNE, R0, R1, 3), + BPF_LDX_MEM(BPF_DW, R0, BPF_REG_FP, -16), + BPF_JMP_REG(BPF_JNE, R0, R2, 1), + BPF_ALU64_IMM(BPF_MOV, R0, 0), + BPF_EXIT_INSN(), + }, + .result = 0, + .stack_depth = 32, + }, + { + "Tail call load/store", + .insns = { + BPF_ALU64_IMM(BPF_MOV, R0, 3), + BPF_STX_MEM(BPF_DW, BPF_REG_FP, R0, -8), + TAIL_CALL(-1), + BPF_ALU64_IMM(BPF_MOV, R0, -1), + BPF_EXIT_INSN(), + }, + .result = 0, + .stack_depth = 16, + }, + { "Tail call error path, max count reached", .insns = { BPF_LDX_MEM(BPF_W, R2, R1, 0), diff --git a/lib/test_firmware.c b/lib/test_firmware.c index 1bccd6cd5f48..c82b65947ce6 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -31,9 +31,12 @@ MODULE_IMPORT_NS(TEST_FIRMWARE); #define TEST_FIRMWARE_NAME "test-firmware.bin" #define TEST_FIRMWARE_NUM_REQS 4 #define TEST_FIRMWARE_BUF_SIZE SZ_1K +#define TEST_UPLOAD_MAX_SIZE SZ_2K +#define TEST_UPLOAD_BLK_SIZE 37 /* Avoid powers of two in testing */ static DEFINE_MUTEX(test_fw_mutex); static const struct firmware *test_firmware; +static LIST_HEAD(test_upload_list); struct test_batched_req { u8 idx; @@ -63,6 +66,7 @@ struct test_batched_req { * @reqs: stores all requests information * @read_fw_idx: index of thread from which we want to read firmware results * from through the read_fw trigger. + * @upload_name: firmware name to be used with upload_read sysfs node * @test_result: a test may use this to collect the result from the call * of the request_firmware*() calls used in their tests. In order of * priority we always keep first any setup error. If no setup errors were @@ -101,6 +105,7 @@ struct test_config { bool send_uevent; u8 num_requests; u8 read_fw_idx; + char *upload_name; /* * These below don't belong her but we'll move them once we create @@ -112,8 +117,34 @@ struct test_config { struct device *device); }; +struct upload_inject_err { + const char *prog; + enum fw_upload_err err_code; +}; + +struct test_firmware_upload { + char *name; + struct list_head node; + char *buf; + size_t size; + bool cancel_request; + struct upload_inject_err inject; + struct fw_upload *fwl; +}; + static struct test_config *test_fw_config; +static struct test_firmware_upload *upload_lookup_name(const char *name) +{ + struct test_firmware_upload *tst; + + list_for_each_entry(tst, &test_upload_list, node) + if (strncmp(name, tst->name, strlen(tst->name)) == 0) + return tst; + + return NULL; +} + static ssize_t test_fw_misc_read(struct file *f, char __user *buf, size_t size, loff_t *offset) { @@ -198,6 +229,7 @@ static int __test_firmware_config_init(void) test_fw_config->req_firmware = request_firmware; test_fw_config->test_result = 0; test_fw_config->reqs = NULL; + test_fw_config->upload_name = NULL; return 0; @@ -277,6 +309,13 @@ static ssize_t config_show(struct device *dev, test_fw_config->sync_direct ? "true" : "false"); len += scnprintf(buf + len, PAGE_SIZE - len, "read_fw_idx:\t%u\n", test_fw_config->read_fw_idx); + if (test_fw_config->upload_name) + len += scnprintf(buf + len, PAGE_SIZE - len, + "upload_name:\t%s\n", + test_fw_config->upload_name); + else + len += scnprintf(buf + len, PAGE_SIZE - len, + "upload_name:\tEMTPY\n"); mutex_unlock(&test_fw_mutex); @@ -392,6 +431,32 @@ static ssize_t config_name_show(struct device *dev, } static DEVICE_ATTR_RW(config_name); +static ssize_t config_upload_name_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct test_firmware_upload *tst; + int ret = count; + + mutex_lock(&test_fw_mutex); + tst = upload_lookup_name(buf); + if (tst) + test_fw_config->upload_name = tst->name; + else + ret = -EINVAL; + mutex_unlock(&test_fw_mutex); + + return ret; +} + +static ssize_t config_upload_name_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return config_test_show_str(buf, test_fw_config->upload_name); +} +static DEVICE_ATTR_RW(config_upload_name); + static ssize_t config_num_requests_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) @@ -989,6 +1054,278 @@ out: } static DEVICE_ATTR_WO(trigger_batched_requests_async); +static void upload_release(struct test_firmware_upload *tst) +{ + firmware_upload_unregister(tst->fwl); + kfree(tst->buf); + kfree(tst->name); + kfree(tst); +} + +static void upload_release_all(void) +{ + struct test_firmware_upload *tst, *tmp; + + list_for_each_entry_safe(tst, tmp, &test_upload_list, node) { + list_del(&tst->node); + upload_release(tst); + } + test_fw_config->upload_name = NULL; +} + +/* + * This table is replicated from .../firmware_loader/sysfs_upload.c + * and needs to be kept in sync. + */ +static const char * const fw_upload_err_str[] = { + [FW_UPLOAD_ERR_NONE] = "none", + [FW_UPLOAD_ERR_HW_ERROR] = "hw-error", + [FW_UPLOAD_ERR_TIMEOUT] = "timeout", + [FW_UPLOAD_ERR_CANCELED] = "user-abort", + [FW_UPLOAD_ERR_BUSY] = "device-busy", + [FW_UPLOAD_ERR_INVALID_SIZE] = "invalid-file-size", + [FW_UPLOAD_ERR_RW_ERROR] = "read-write-error", + [FW_UPLOAD_ERR_WEAROUT] = "flash-wearout", +}; + +static void upload_err_inject_error(struct test_firmware_upload *tst, + const u8 *p, const char *prog) +{ + enum fw_upload_err err; + + for (err = FW_UPLOAD_ERR_NONE + 1; err < FW_UPLOAD_ERR_MAX; err++) { + if (strncmp(p, fw_upload_err_str[err], + strlen(fw_upload_err_str[err])) == 0) { + tst->inject.prog = prog; + tst->inject.err_code = err; + return; + } + } +} + +static void upload_err_inject_prog(struct test_firmware_upload *tst, + const u8 *p) +{ + static const char * const progs[] = { + "preparing:", "transferring:", "programming:" + }; + int i; + + for (i = 0; i < ARRAY_SIZE(progs); i++) { + if (strncmp(p, progs[i], strlen(progs[i])) == 0) { + upload_err_inject_error(tst, p + strlen(progs[i]), + progs[i]); + return; + } + } +} + +#define FIVE_MINUTES_MS (5 * 60 * 1000) +static enum fw_upload_err +fw_upload_wait_on_cancel(struct test_firmware_upload *tst) +{ + int ms_delay; + + for (ms_delay = 0; ms_delay < FIVE_MINUTES_MS; ms_delay += 100) { + msleep(100); + if (tst->cancel_request) + return FW_UPLOAD_ERR_CANCELED; + } + return FW_UPLOAD_ERR_NONE; +} + +static enum fw_upload_err test_fw_upload_prepare(struct fw_upload *fwl, + const u8 *data, u32 size) +{ + struct test_firmware_upload *tst = fwl->dd_handle; + enum fw_upload_err ret = FW_UPLOAD_ERR_NONE; + const char *progress = "preparing:"; + + tst->cancel_request = false; + + if (!size || size > TEST_UPLOAD_MAX_SIZE) { + ret = FW_UPLOAD_ERR_INVALID_SIZE; + goto err_out; + } + + if (strncmp(data, "inject:", strlen("inject:")) == 0) + upload_err_inject_prog(tst, data + strlen("inject:")); + + memset(tst->buf, 0, TEST_UPLOAD_MAX_SIZE); + tst->size = size; + + if (tst->inject.err_code == FW_UPLOAD_ERR_NONE || + strncmp(tst->inject.prog, progress, strlen(progress)) != 0) + return FW_UPLOAD_ERR_NONE; + + if (tst->inject.err_code == FW_UPLOAD_ERR_CANCELED) + ret = fw_upload_wait_on_cancel(tst); + else + ret = tst->inject.err_code; + +err_out: + /* + * The cleanup op only executes if the prepare op succeeds. + * If the prepare op fails, it must do it's own clean-up. + */ + tst->inject.err_code = FW_UPLOAD_ERR_NONE; + tst->inject.prog = NULL; + + return ret; +} + +static enum fw_upload_err test_fw_upload_write(struct fw_upload *fwl, + const u8 *data, u32 offset, + u32 size, u32 *written) +{ + struct test_firmware_upload *tst = fwl->dd_handle; + const char *progress = "transferring:"; + u32 blk_size; + + if (tst->cancel_request) + return FW_UPLOAD_ERR_CANCELED; + + blk_size = min_t(u32, TEST_UPLOAD_BLK_SIZE, size); + memcpy(tst->buf + offset, data + offset, blk_size); + + *written = blk_size; + + if (tst->inject.err_code == FW_UPLOAD_ERR_NONE || + strncmp(tst->inject.prog, progress, strlen(progress)) != 0) + return FW_UPLOAD_ERR_NONE; + + if (tst->inject.err_code == FW_UPLOAD_ERR_CANCELED) + return fw_upload_wait_on_cancel(tst); + + return tst->inject.err_code; +} + +static enum fw_upload_err test_fw_upload_complete(struct fw_upload *fwl) +{ + struct test_firmware_upload *tst = fwl->dd_handle; + const char *progress = "programming:"; + + if (tst->cancel_request) + return FW_UPLOAD_ERR_CANCELED; + + if (tst->inject.err_code == FW_UPLOAD_ERR_NONE || + strncmp(tst->inject.prog, progress, strlen(progress)) != 0) + return FW_UPLOAD_ERR_NONE; + + if (tst->inject.err_code == FW_UPLOAD_ERR_CANCELED) + return fw_upload_wait_on_cancel(tst); + + return tst->inject.err_code; +} + +static void test_fw_upload_cancel(struct fw_upload *fwl) +{ + struct test_firmware_upload *tst = fwl->dd_handle; + + tst->cancel_request = true; +} + +static void test_fw_cleanup(struct fw_upload *fwl) +{ + struct test_firmware_upload *tst = fwl->dd_handle; + + tst->inject.err_code = FW_UPLOAD_ERR_NONE; + tst->inject.prog = NULL; +} + +static const struct fw_upload_ops upload_test_ops = { + .prepare = test_fw_upload_prepare, + .write = test_fw_upload_write, + .poll_complete = test_fw_upload_complete, + .cancel = test_fw_upload_cancel, + .cleanup = test_fw_cleanup +}; + +static ssize_t upload_register_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct test_firmware_upload *tst; + struct fw_upload *fwl; + char *name; + int ret; + + name = kstrndup(buf, count, GFP_KERNEL); + if (!name) + return -ENOMEM; + + mutex_lock(&test_fw_mutex); + tst = upload_lookup_name(name); + if (tst) { + ret = -EEXIST; + goto free_name; + } + + tst = kzalloc(sizeof(*tst), GFP_KERNEL); + if (!tst) { + ret = -ENOMEM; + goto free_name; + } + + tst->name = name; + tst->buf = kzalloc(TEST_UPLOAD_MAX_SIZE, GFP_KERNEL); + if (!tst->buf) { + ret = -ENOMEM; + goto free_tst; + } + + fwl = firmware_upload_register(THIS_MODULE, dev, tst->name, + &upload_test_ops, tst); + if (IS_ERR(fwl)) { + ret = PTR_ERR(fwl); + goto free_buf; + } + + tst->fwl = fwl; + list_add_tail(&tst->node, &test_upload_list); + mutex_unlock(&test_fw_mutex); + return count; + +free_buf: + kfree(tst->buf); + +free_tst: + kfree(tst); + +free_name: + mutex_unlock(&test_fw_mutex); + kfree(name); + + return ret; +} +static DEVICE_ATTR_WO(upload_register); + +static ssize_t upload_unregister_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct test_firmware_upload *tst; + int ret = count; + + mutex_lock(&test_fw_mutex); + tst = upload_lookup_name(buf); + if (!tst) { + ret = -EINVAL; + goto out; + } + + if (test_fw_config->upload_name == tst->name) + test_fw_config->upload_name = NULL; + + list_del(&tst->node); + upload_release(tst); + +out: + mutex_unlock(&test_fw_mutex); + return ret; +} +static DEVICE_ATTR_WO(upload_unregister); + static ssize_t test_result_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -1051,6 +1388,45 @@ out: } static DEVICE_ATTR_RO(read_firmware); +static ssize_t upload_read_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct test_firmware_upload *tst = NULL; + struct test_firmware_upload *tst_iter; + int ret = -EINVAL; + + if (!test_fw_config->upload_name) { + pr_err("Set config_upload_name before using upload_read\n"); + return -EINVAL; + } + + mutex_lock(&test_fw_mutex); + list_for_each_entry(tst_iter, &test_upload_list, node) + if (tst_iter->name == test_fw_config->upload_name) { + tst = tst_iter; + break; + } + + if (!tst) { + pr_err("Firmware name not found: %s\n", + test_fw_config->upload_name); + goto out; + } + + if (tst->size > PAGE_SIZE) { + pr_err("Testing interface must use PAGE_SIZE firmware for now\n"); + goto out; + } + + memcpy(buf, tst->buf, tst->size); + ret = tst->size; +out: + mutex_unlock(&test_fw_mutex); + return ret; +} +static DEVICE_ATTR_RO(upload_read); + #define TEST_FW_DEV_ATTR(name) &dev_attr_##name.attr static struct attribute *test_dev_attrs[] = { @@ -1066,6 +1442,7 @@ static struct attribute *test_dev_attrs[] = { TEST_FW_DEV_ATTR(config_sync_direct), TEST_FW_DEV_ATTR(config_send_uevent), TEST_FW_DEV_ATTR(config_read_fw_idx), + TEST_FW_DEV_ATTR(config_upload_name), /* These don't use the config at all - they could be ported! */ TEST_FW_DEV_ATTR(trigger_request), @@ -1082,6 +1459,9 @@ static struct attribute *test_dev_attrs[] = { TEST_FW_DEV_ATTR(release_all_firmware), TEST_FW_DEV_ATTR(test_result), TEST_FW_DEV_ATTR(read_firmware), + TEST_FW_DEV_ATTR(upload_read), + TEST_FW_DEV_ATTR(upload_register), + TEST_FW_DEV_ATTR(upload_unregister), NULL, }; @@ -1128,6 +1508,7 @@ static void __exit test_firmware_exit(void) mutex_lock(&test_fw_mutex); release_firmware(test_firmware); misc_deregister(&test_fw_misc_device); + upload_release_all(); __test_firmware_config_free(); kfree(test_fw_config); mutex_unlock(&test_fw_mutex); diff --git a/lib/test_fortify/read_overflow2_field-memcpy.c b/lib/test_fortify/read_overflow2_field-memcpy.c new file mode 100644 index 000000000000..de9569266223 --- /dev/null +++ b/lib/test_fortify/read_overflow2_field-memcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcpy(large, instance.buf, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/read_overflow2_field-memmove.c b/lib/test_fortify/read_overflow2_field-memmove.c new file mode 100644 index 000000000000..6cc2724c8f62 --- /dev/null +++ b/lib/test_fortify/read_overflow2_field-memmove.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memmove(large, instance.buf, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow_field-memcpy.c b/lib/test_fortify/write_overflow_field-memcpy.c new file mode 100644 index 000000000000..28cc81058dd3 --- /dev/null +++ b/lib/test_fortify/write_overflow_field-memcpy.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memcpy(instance.buf, large, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow_field-memmove.c b/lib/test_fortify/write_overflow_field-memmove.c new file mode 100644 index 000000000000..377fcf9bb2fd --- /dev/null +++ b/lib/test_fortify/write_overflow_field-memmove.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memmove(instance.buf, large, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow_field-memset.c b/lib/test_fortify/write_overflow_field-memset.c new file mode 100644 index 000000000000..2331da26909e --- /dev/null +++ b/lib/test_fortify/write_overflow_field-memset.c @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define TEST \ + memset(instance.buf, 0x42, sizeof(instance.buf) + 1) + +#include "test_fortify.h" diff --git a/lib/test_fprobe.c b/lib/test_fprobe.c new file mode 100644 index 000000000000..ed70637a2ffa --- /dev/null +++ b/lib/test_fprobe.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * test_fprobe.c - simple sanity test for fprobe + */ + +#include <linux/kernel.h> +#include <linux/fprobe.h> +#include <linux/random.h> +#include <kunit/test.h> + +#define div_factor 3 + +static struct kunit *current_test; + +static u32 rand1, entry_val, exit_val; + +/* Use indirect calls to avoid inlining the target functions */ +static u32 (*target)(u32 value); +static u32 (*target2)(u32 value); +static unsigned long target_ip; +static unsigned long target2_ip; + +static noinline u32 fprobe_selftest_target(u32 value) +{ + return (value / div_factor); +} + +static noinline u32 fprobe_selftest_target2(u32 value) +{ + return (value / div_factor) + 1; +} + +static notrace void fp_entry_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs) +{ + KUNIT_EXPECT_FALSE(current_test, preemptible()); + /* This can be called on the fprobe_selftest_target and the fprobe_selftest_target2 */ + if (ip != target_ip) + KUNIT_EXPECT_EQ(current_test, ip, target2_ip); + entry_val = (rand1 / div_factor); +} + +static notrace void fp_exit_handler(struct fprobe *fp, unsigned long ip, struct pt_regs *regs) +{ + unsigned long ret = regs_return_value(regs); + + KUNIT_EXPECT_FALSE(current_test, preemptible()); + if (ip != target_ip) { + KUNIT_EXPECT_EQ(current_test, ip, target2_ip); + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor) + 1); + } else + KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor)); + KUNIT_EXPECT_EQ(current_test, entry_val, (rand1 / div_factor)); + exit_val = entry_val + div_factor; +} + +/* Test entry only (no rethook) */ +static void test_fprobe_entry(struct kunit *test) +{ + struct fprobe fp_entry = { + .entry_handler = fp_entry_handler, + }; + + current_test = test; + + /* Before register, unregister should be failed. */ + KUNIT_EXPECT_NE(test, 0, unregister_fprobe(&fp_entry)); + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp_entry, "fprobe_selftest_target*", NULL)); + + entry_val = 0; + exit_val = 0; + target(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, 0, exit_val); + + entry_val = 0; + exit_val = 0; + target2(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, 0, exit_val); + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp_entry)); +} + +static void test_fprobe(struct kunit *test) +{ + struct fprobe fp = { + .entry_handler = fp_entry_handler, + .exit_handler = fp_exit_handler, + }; + + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp, "fprobe_selftest_target*", NULL)); + + entry_val = 0; + exit_val = 0; + target(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + entry_val = 0; + exit_val = 0; + target2(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); +} + +static void test_fprobe_syms(struct kunit *test) +{ + static const char *syms[] = {"fprobe_selftest_target", "fprobe_selftest_target2"}; + struct fprobe fp = { + .entry_handler = fp_entry_handler, + .exit_handler = fp_exit_handler, + }; + + current_test = test; + KUNIT_EXPECT_EQ(test, 0, register_fprobe_syms(&fp, syms, 2)); + + entry_val = 0; + exit_val = 0; + target(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + entry_val = 0; + exit_val = 0; + target2(rand1); + KUNIT_EXPECT_NE(test, 0, entry_val); + KUNIT_EXPECT_EQ(test, entry_val + div_factor, exit_val); + + KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp)); +} + +static unsigned long get_ftrace_location(void *func) +{ + unsigned long size, addr = (unsigned long)func; + + if (!kallsyms_lookup_size_offset(addr, &size, NULL) || !size) + return 0; + + return ftrace_location_range(addr, addr + size - 1); +} + +static int fprobe_test_init(struct kunit *test) +{ + do { + rand1 = prandom_u32(); + } while (rand1 <= div_factor); + + target = fprobe_selftest_target; + target2 = fprobe_selftest_target2; + target_ip = get_ftrace_location(target); + target2_ip = get_ftrace_location(target2); + + return 0; +} + +static struct kunit_case fprobe_testcases[] = { + KUNIT_CASE(test_fprobe_entry), + KUNIT_CASE(test_fprobe), + KUNIT_CASE(test_fprobe_syms), + {} +}; + +static struct kunit_suite fprobe_test_suite = { + .name = "fprobe_test", + .init = fprobe_test_init, + .test_cases = fprobe_testcases, +}; + +kunit_test_suites(&fprobe_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 767538089a62..cfe632047839 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -12,6 +12,7 @@ #include <linux/kernel.h> #include <linux/cdev.h> #include <linux/device.h> +#include <linux/memremap.h> #include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/sched.h> @@ -26,6 +27,8 @@ #include <linux/sched/mm.h> #include <linux/platform_device.h> #include <linux/rmap.h> +#include <linux/mmu_notifier.h> +#include <linux/migrate.h> #include "test_hmm_uapi.h" @@ -563,7 +566,6 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice) } dpage->zone_device_data = rpage; - get_page(dpage); lock_page(dpage); return dpage; diff --git a/lib/test_kasan.c b/lib/test_kasan.c index 3b413f8c8a71..c233b1a4e984 100644 --- a/lib/test_kasan.c +++ b/lib/test_kasan.c @@ -19,6 +19,7 @@ #include <linux/uaccess.h> #include <linux/io.h> #include <linux/vmalloc.h> +#include <linux/set_memory.h> #include <asm/page.h> @@ -36,7 +37,7 @@ void *kasan_ptr_result; int kasan_int_result; static struct kunit_resource resource; -static struct kunit_kasan_expectation fail_data; +static struct kunit_kasan_status test_status; static bool multishot; /* @@ -53,58 +54,63 @@ static int kasan_test_init(struct kunit *test) } multishot = kasan_save_enable_multi_shot(); - fail_data.report_found = false; + test_status.report_found = false; + test_status.sync_fault = false; kunit_add_named_resource(test, NULL, NULL, &resource, - "kasan_data", &fail_data); + "kasan_status", &test_status); return 0; } static void kasan_test_exit(struct kunit *test) { kasan_restore_multi_shot(multishot); - KUNIT_EXPECT_FALSE(test, fail_data.report_found); + KUNIT_EXPECT_FALSE(test, test_status.report_found); } /** * KUNIT_EXPECT_KASAN_FAIL() - check that the executed expression produces a * KASAN report; causes a test failure otherwise. This relies on a KUnit - * resource named "kasan_data". Do not use this name for KUnit resources + * resource named "kasan_status". Do not use this name for KUnit resources * outside of KASAN tests. * - * For hardware tag-based KASAN in sync mode, when a tag fault happens, tag + * For hardware tag-based KASAN, when a synchronous tag fault happens, tag * checking is auto-disabled. When this happens, this test handler reenables * tag checking. As tag checking can be only disabled or enabled per CPU, * this handler disables migration (preemption). * - * Since the compiler doesn't see that the expression can change the fail_data + * Since the compiler doesn't see that the expression can change the test_status * fields, it can reorder or optimize away the accesses to those fields. * Use READ/WRITE_ONCE() for the accesses and compiler barriers around the * expression to prevent that. * - * In between KUNIT_EXPECT_KASAN_FAIL checks, fail_data.report_found is kept as - * false. This allows detecting KASAN reports that happen outside of the checks - * by asserting !fail_data.report_found at the start of KUNIT_EXPECT_KASAN_FAIL - * and in kasan_test_exit. + * In between KUNIT_EXPECT_KASAN_FAIL checks, test_status.report_found is kept + * as false. This allows detecting KASAN reports that happen outside of the + * checks by asserting !test_status.report_found at the start of + * KUNIT_EXPECT_KASAN_FAIL and in kasan_test_exit. */ #define KUNIT_EXPECT_KASAN_FAIL(test, expression) do { \ if (IS_ENABLED(CONFIG_KASAN_HW_TAGS) && \ kasan_sync_fault_possible()) \ migrate_disable(); \ - KUNIT_EXPECT_FALSE(test, READ_ONCE(fail_data.report_found)); \ + KUNIT_EXPECT_FALSE(test, READ_ONCE(test_status.report_found)); \ barrier(); \ expression; \ barrier(); \ - if (!READ_ONCE(fail_data.report_found)) { \ + if (kasan_async_fault_possible()) \ + kasan_force_async_fault(); \ + if (!READ_ONCE(test_status.report_found)) { \ KUNIT_FAIL(test, KUNIT_SUBTEST_INDENT "KASAN failure " \ "expected in \"" #expression \ "\", but none occurred"); \ } \ - if (IS_ENABLED(CONFIG_KASAN_HW_TAGS)) { \ - if (READ_ONCE(fail_data.report_found)) \ - kasan_enable_tagging_sync(); \ + if (IS_ENABLED(CONFIG_KASAN_HW_TAGS) && \ + kasan_sync_fault_possible()) { \ + if (READ_ONCE(test_status.report_found) && \ + READ_ONCE(test_status.sync_fault)) \ + kasan_enable_tagging(); \ migrate_enable(); \ } \ - WRITE_ONCE(fail_data.report_found, false); \ + WRITE_ONCE(test_status.report_found, false); \ } while (0) #define KASAN_TEST_NEEDS_CONFIG_ON(test, config) do { \ @@ -385,7 +391,7 @@ static void krealloc_uaf(struct kunit *test) kfree(ptr1); KUNIT_EXPECT_KASAN_FAIL(test, ptr2 = krealloc(ptr1, size2, GFP_KERNEL)); - KUNIT_ASSERT_PTR_EQ(test, (void *)ptr2, NULL); + KUNIT_ASSERT_NULL(test, ptr2); KUNIT_EXPECT_KASAN_FAIL(test, *(volatile char *)ptr1); } @@ -780,7 +786,7 @@ static void ksize_uaf(struct kunit *test) static void kasan_stack_oob(struct kunit *test) { char stack_array[10]; - /* See comment in kasan_global_oob. */ + /* See comment in kasan_global_oob_right. */ char *volatile array = stack_array; char *p = &array[ARRAY_SIZE(stack_array) + OOB_TAG_OFF]; @@ -793,7 +799,7 @@ static void kasan_alloca_oob_left(struct kunit *test) { volatile int i = 10; char alloca_array[i]; - /* See comment in kasan_global_oob. */ + /* See comment in kasan_global_oob_right. */ char *volatile array = alloca_array; char *p = array - 1; @@ -808,7 +814,7 @@ static void kasan_alloca_oob_right(struct kunit *test) { volatile int i = 10; char alloca_array[i]; - /* See comment in kasan_global_oob. */ + /* See comment in kasan_global_oob_right. */ char *volatile array = alloca_array; char *p = array + i; @@ -1057,21 +1063,186 @@ static void kmalloc_double_kzfree(struct kunit *test) KUNIT_EXPECT_KASAN_FAIL(test, kfree_sensitive(ptr)); } +static void vmalloc_helpers_tags(struct kunit *test) +{ + void *ptr; + + /* This test is intended for tag-based modes. */ + KASAN_TEST_NEEDS_CONFIG_OFF(test, CONFIG_KASAN_GENERIC); + + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_VMALLOC); + + ptr = vmalloc(PAGE_SIZE); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + + /* Check that the returned pointer is tagged. */ + KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + + /* Make sure exported vmalloc helpers handle tagged pointers. */ + KUNIT_ASSERT_TRUE(test, is_vmalloc_addr(ptr)); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, vmalloc_to_page(ptr)); + +#if !IS_MODULE(CONFIG_KASAN_KUNIT_TEST) + { + int rv; + + /* Make sure vmalloc'ed memory permissions can be changed. */ + rv = set_memory_ro((unsigned long)ptr, 1); + KUNIT_ASSERT_GE(test, rv, 0); + rv = set_memory_rw((unsigned long)ptr, 1); + KUNIT_ASSERT_GE(test, rv, 0); + } +#endif + + vfree(ptr); +} + static void vmalloc_oob(struct kunit *test) { - void *area; + char *v_ptr, *p_ptr; + struct page *page; + size_t size = PAGE_SIZE / 2 - KASAN_GRANULE_SIZE - 5; KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_VMALLOC); + v_ptr = vmalloc(size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, v_ptr); + + OPTIMIZER_HIDE_VAR(v_ptr); + /* - * We have to be careful not to hit the guard page. + * We have to be careful not to hit the guard page in vmalloc tests. * The MMU will catch that and crash us. */ - area = vmalloc(3000); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, area); - KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)area)[3100]); - vfree(area); + /* Make sure in-bounds accesses are valid. */ + v_ptr[0] = 0; + v_ptr[size - 1] = 0; + + /* + * An unaligned access past the requested vmalloc size. + * Only generic KASAN can precisely detect these. + */ + if (IS_ENABLED(CONFIG_KASAN_GENERIC)) + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)v_ptr)[size]); + + /* An aligned access into the first out-of-bounds granule. */ + KUNIT_EXPECT_KASAN_FAIL(test, ((volatile char *)v_ptr)[size + 5]); + + /* Check that in-bounds accesses to the physical page are valid. */ + page = vmalloc_to_page(v_ptr); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, page); + p_ptr = page_address(page); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p_ptr); + p_ptr[0] = 0; + + vfree(v_ptr); + + /* + * We can't check for use-after-unmap bugs in this nor in the following + * vmalloc tests, as the page might be fully unmapped and accessing it + * will crash the kernel. + */ +} + +static void vmap_tags(struct kunit *test) +{ + char *p_ptr, *v_ptr; + struct page *p_page, *v_page; + + /* + * This test is specifically crafted for the software tag-based mode, + * the only tag-based mode that poisons vmap mappings. + */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_SW_TAGS); + + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_VMALLOC); + + p_page = alloc_pages(GFP_KERNEL, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p_page); + p_ptr = page_address(p_page); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p_ptr); + + v_ptr = vmap(&p_page, 1, VM_MAP, PAGE_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, v_ptr); + + /* + * We can't check for out-of-bounds bugs in this nor in the following + * vmalloc tests, as allocations have page granularity and accessing + * the guard page will crash the kernel. + */ + + KUNIT_EXPECT_GE(test, (u8)get_tag(v_ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(v_ptr), (u8)KASAN_TAG_KERNEL); + + /* Make sure that in-bounds accesses through both pointers work. */ + *p_ptr = 0; + *v_ptr = 0; + + /* Make sure vmalloc_to_page() correctly recovers the page pointer. */ + v_page = vmalloc_to_page(v_ptr); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, v_page); + KUNIT_EXPECT_PTR_EQ(test, p_page, v_page); + + vunmap(v_ptr); + free_pages((unsigned long)p_ptr, 1); +} + +static void vm_map_ram_tags(struct kunit *test) +{ + char *p_ptr, *v_ptr; + struct page *page; + + /* + * This test is specifically crafted for the software tag-based mode, + * the only tag-based mode that poisons vm_map_ram mappings. + */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_SW_TAGS); + + page = alloc_pages(GFP_KERNEL, 1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, page); + p_ptr = page_address(page); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p_ptr); + + v_ptr = vm_map_ram(&page, 1, -1); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, v_ptr); + + KUNIT_EXPECT_GE(test, (u8)get_tag(v_ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(v_ptr), (u8)KASAN_TAG_KERNEL); + + /* Make sure that in-bounds accesses through both pointers work. */ + *p_ptr = 0; + *v_ptr = 0; + + vm_unmap_ram(v_ptr, 1); + free_pages((unsigned long)p_ptr, 1); +} + +static void vmalloc_percpu(struct kunit *test) +{ + char __percpu *ptr; + int cpu; + + /* + * This test is specifically crafted for the software tag-based mode, + * the only tag-based mode that poisons percpu mappings. + */ + KASAN_TEST_NEEDS_CONFIG_ON(test, CONFIG_KASAN_SW_TAGS); + + ptr = __alloc_percpu(PAGE_SIZE, PAGE_SIZE); + + for_each_possible_cpu(cpu) { + char *c_ptr = per_cpu_ptr(ptr, cpu); + + KUNIT_EXPECT_GE(test, (u8)get_tag(c_ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(c_ptr), (u8)KASAN_TAG_KERNEL); + + /* Make sure that in-bounds accesses don't crash the kernel. */ + *c_ptr = 0; + } + + free_percpu(ptr); } /* @@ -1105,6 +1276,18 @@ static void match_all_not_assigned(struct kunit *test) KUNIT_EXPECT_LT(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); free_pages((unsigned long)ptr, order); } + + if (!IS_ENABLED(CONFIG_KASAN_VMALLOC)) + return; + + for (i = 0; i < 256; i++) { + size = (get_random_int() % 1024) + 1; + ptr = vmalloc(size); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); + KUNIT_EXPECT_GE(test, (u8)get_tag(ptr), (u8)KASAN_TAG_MIN); + KUNIT_EXPECT_LT(test, (u8)get_tag(ptr), (u8)KASAN_TAG_KERNEL); + vfree(ptr); + } } /* Check that 0xff works as a match-all pointer tag for tag-based modes. */ @@ -1210,7 +1393,11 @@ static struct kunit_case kasan_kunit_test_cases[] = { KUNIT_CASE(kasan_bitops_generic), KUNIT_CASE(kasan_bitops_tags), KUNIT_CASE(kmalloc_double_kzfree), + KUNIT_CASE(vmalloc_helpers_tags), KUNIT_CASE(vmalloc_oob), + KUNIT_CASE(vmap_tags), + KUNIT_CASE(vm_map_ram_tags), + KUNIT_CASE(vmalloc_percpu), KUNIT_CASE(match_all_not_assigned), KUNIT_CASE(match_all_ptr_tag), KUNIT_CASE(match_all_mem_tag), diff --git a/lib/test_kmod.c b/lib/test_kmod.c index ce1589391413..cb800b1d0d99 100644 --- a/lib/test_kmod.c +++ b/lib/test_kmod.c @@ -1149,6 +1149,7 @@ static struct kmod_test_device *register_test_dev_kmod(void) if (ret) { pr_err("could not register misc device: %d\n", ret); free_test_dev_kmod(test_dev); + test_dev = NULL; goto out; } diff --git a/lib/test_lockup.c b/lib/test_lockup.c index 906b598740a7..c3fd87d6c2dd 100644 --- a/lib/test_lockup.c +++ b/lib/test_lockup.c @@ -417,9 +417,14 @@ static bool test_kernel_ptr(unsigned long addr, int size) return false; /* should be at least readable kernel address */ - if (access_ok(ptr, 1) || - access_ok(ptr + size - 1, 1) || - get_kernel_nofault(buf, ptr) || + if (!IS_ENABLED(CONFIG_ALTERNATE_USER_ADDRESS_SPACE) && + (access_ok((void __user *)ptr, 1) || + access_ok((void __user *)ptr + size - 1, 1))) { + pr_err("user space ptr invalid in kernel: %#lx\n", addr); + return true; + } + + if (get_kernel_nofault(buf, ptr) || get_kernel_nofault(buf, ptr + size - 1)) { pr_err("invalid kernel ptr: %#lx\n", addr); return true; diff --git a/lib/test_meminit.c b/lib/test_meminit.c index 3ca717f11397..c95db11a6906 100644 --- a/lib/test_meminit.c +++ b/lib/test_meminit.c @@ -279,13 +279,18 @@ static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) c = kmem_cache_create("test_cache", size, size, SLAB_TYPESAFE_BY_RCU, NULL); buf = kmem_cache_alloc(c, GFP_KERNEL); + if (!buf) + goto out; saved_ptr = buf; fill_with_garbage(buf, size); buf_contents = kmalloc(size, GFP_KERNEL); - if (!buf_contents) + if (!buf_contents) { + kmem_cache_free(c, buf); goto out; + } used_objects = kmalloc_array(maxiter, sizeof(void *), GFP_KERNEL); if (!used_objects) { + kmem_cache_free(c, buf); kfree(buf_contents); goto out; } @@ -306,11 +311,14 @@ static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) } } + for (iter = 0; iter < maxiter; iter++) + kmem_cache_free(c, used_objects[iter]); + free_out: - kmem_cache_destroy(c); kfree(buf_contents); kfree(used_objects); out: + kmem_cache_destroy(c); *total_failures += fail; return 1; } diff --git a/lib/test_siphash.c b/lib/test_siphash.c index a6d854d933bf..a96788d0141d 100644 --- a/lib/test_siphash.c +++ b/lib/test_siphash.c @@ -1,8 +1,7 @@ -/* Test cases for siphash.c +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +/* Copyright (C) 2016-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. * - * Copyright (C) 2016 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. - * - * This file is provided under a dual BSD/GPLv2 license. + * Test cases for siphash.c * * SipHash: a fast short-input PRF * https://131002.net/siphash/ diff --git a/lib/test_string.c b/lib/test_string.c index 9dfd6f52de92..c5cb92fb710e 100644 --- a/lib/test_string.c +++ b/lib/test_string.c @@ -179,6 +179,34 @@ static __init int strnchr_selftest(void) return 0; } +static __init int strspn_selftest(void) +{ + static const struct strspn_test { + const char str[16]; + const char accept[16]; + const char reject[16]; + unsigned a; + unsigned r; + } tests[] __initconst = { + { "foobar", "", "", 0, 6 }, + { "abba", "abc", "ABBA", 4, 4 }, + { "abba", "a", "b", 1, 1 }, + { "", "abc", "abc", 0, 0}, + }; + const struct strspn_test *s = tests; + size_t i, res; + + for (i = 0; i < ARRAY_SIZE(tests); ++i, ++s) { + res = strspn(s->str, s->accept); + if (res != s->a) + return 0x100 + 2*i; + res = strcspn(s->str, s->reject); + if (res != s->r) + return 0x100 + 2*i + 1; + } + return 0; +} + static __exit void string_selftest_remove(void) { } @@ -212,6 +240,11 @@ static __init int string_selftest_init(void) if (subtest) goto fail; + test = 6; + subtest = strspn_selftest(); + if (subtest) + goto fail; + pr_info("String selftests succeeded\n"); return 0; fail: diff --git a/lib/test_sysctl.c b/lib/test_sysctl.c index a5a3d6c27e1f..9a564971f539 100644 --- a/lib/test_sysctl.c +++ b/lib/test_sysctl.c @@ -38,6 +38,7 @@ static int i_zero; static int i_one_hundred = 100; +static int match_int_ok = 1; struct test_sysctl_data { int int_0001; @@ -96,6 +97,13 @@ static struct ctl_table test_table[] = { .proc_handler = proc_dointvec, }, { + .procname = "match_int", + .data = &match_int_ok, + .maxlen = sizeof(match_int_ok), + .mode = 0444, + .proc_handler = proc_dointvec, + }, + { .procname = "boot_int", .data = &test_data.boot_int, .maxlen = sizeof(test_data.boot_int), @@ -132,6 +140,30 @@ static struct ctl_table_header *test_sysctl_header; static int __init test_sysctl_init(void) { + int i; + + struct { + int defined; + int wanted; + } match_int[] = { + {.defined = *(int *)SYSCTL_ZERO, .wanted = 0}, + {.defined = *(int *)SYSCTL_ONE, .wanted = 1}, + {.defined = *(int *)SYSCTL_TWO, .wanted = 2}, + {.defined = *(int *)SYSCTL_THREE, .wanted = 3}, + {.defined = *(int *)SYSCTL_FOUR, .wanted = 4}, + {.defined = *(int *)SYSCTL_ONE_HUNDRED, .wanted = 100}, + {.defined = *(int *)SYSCTL_TWO_HUNDRED, .wanted = 200}, + {.defined = *(int *)SYSCTL_ONE_THOUSAND, .wanted = 1000}, + {.defined = *(int *)SYSCTL_THREE_THOUSAND, .wanted = 3000}, + {.defined = *(int *)SYSCTL_INT_MAX, .wanted = INT_MAX}, + {.defined = *(int *)SYSCTL_MAXOLDUID, .wanted = 65535}, + {.defined = *(int *)SYSCTL_NEG_ONE, .wanted = -1}, + }; + + for (i = 0; i < ARRAY_SIZE(match_int); i++) + if (match_int[i].defined != match_int[i].wanted) + match_int_ok = 0; + test_data.bitmap_0001 = kzalloc(SYSCTL_TEST_BITMAP_SIZE/8, GFP_KERNEL); if (!test_data.bitmap_0001) return -ENOMEM; diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 8b1c318189ce..e77d4856442c 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1463,6 +1463,25 @@ unlock: XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_create_range_5(struct xarray *xa, + unsigned long index, unsigned int order) +{ + XA_STATE_ORDER(xas, xa, index, order); + unsigned int i; + + xa_store_order(xa, index, order, xa_mk_index(index), GFP_KERNEL); + + for (i = 0; i < order + 10; i++) { + do { + xas_lock(&xas); + xas_create_range(&xas); + xas_unlock(&xas); + } while (xas_nomem(&xas, GFP_KERNEL)); + } + + xa_destroy(xa); +} + static noinline void check_create_range(struct xarray *xa) { unsigned int order; @@ -1490,6 +1509,9 @@ static noinline void check_create_range(struct xarray *xa) check_create_range_4(xa, (3U << order) + 1, order); check_create_range_4(xa, (3U << order) - 1, order); check_create_range_4(xa, (1U << 24) + 1, order); + + check_create_range_5(xa, 0, order); + check_create_range_5(xa, (1U << order), order); } check_create_range_3(); diff --git a/lib/ubsan.c b/lib/ubsan.c index bdc380ff5d5c..36bd75e33426 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -154,16 +154,8 @@ static void ubsan_epilogue(void) current->in_ubsan--; - if (panic_on_warn) { - /* - * This thread may hit another WARN() in the panic path. - * Resetting this prevents additional WARN() from panicking the - * system on this thread. Other threads are blocked by the - * panic_mutex in panic(). - */ - panic_on_warn = 0; + if (panic_on_warn) panic("panic_on_warn set ...\n"); - } } void __ubsan_handle_divrem_overflow(void *_data, void *lhs, void *rhs) diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 3b8129dd374c..3c1853a9d1c0 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -49,10 +49,15 @@ #include <asm/page.h> /* for PAGE_SIZE */ #include <asm/byteorder.h> /* cpu_to_le16 */ +#include <asm/unaligned.h> #include <linux/string_helpers.h> #include "kstrtox.h" +/* Disable pointer hashing if requested */ +bool no_hash_pointers __ro_after_init; +EXPORT_SYMBOL_GPL(no_hash_pointers); + static noinline unsigned long long simple_strntoull(const char *startp, size_t max_chars, char **endp, unsigned int base) { const char *cp; @@ -745,59 +750,37 @@ static int __init debug_boot_weak_hash_enable(char *str) } early_param("debug_boot_weak_hash", debug_boot_weak_hash_enable); -static DEFINE_STATIC_KEY_TRUE(not_filled_random_ptr_key); -static siphash_key_t ptr_key __read_mostly; +static DEFINE_STATIC_KEY_FALSE(filled_random_ptr_key); static void enable_ptr_key_workfn(struct work_struct *work) { - get_random_bytes(&ptr_key, sizeof(ptr_key)); - /* Needs to run from preemptible context */ - static_branch_disable(¬_filled_random_ptr_key); + static_branch_enable(&filled_random_ptr_key); } -static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn); - -static void fill_random_ptr_key(struct random_ready_callback *unused) +/* Maps a pointer to a 32 bit unique identifier. */ +static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out) { - /* This may be in an interrupt handler. */ - queue_work(system_unbound_wq, &enable_ptr_key_work); -} + static siphash_key_t ptr_key __read_mostly; + unsigned long hashval; -static struct random_ready_callback random_ready = { - .func = fill_random_ptr_key -}; + if (!static_branch_likely(&filled_random_ptr_key)) { + static bool filled = false; + static DEFINE_SPINLOCK(filling); + static DECLARE_WORK(enable_ptr_key_work, enable_ptr_key_workfn); + unsigned long flags; -static int __init initialize_ptr_random(void) -{ - int key_size = sizeof(ptr_key); - int ret; - - /* Use hw RNG if available. */ - if (get_random_bytes_arch(&ptr_key, key_size) == key_size) { - static_branch_disable(¬_filled_random_ptr_key); - return 0; - } + if (!system_unbound_wq || !rng_is_initialized() || + !spin_trylock_irqsave(&filling, flags)) + return -EAGAIN; - ret = add_random_ready_callback(&random_ready); - if (!ret) { - return 0; - } else if (ret == -EALREADY) { - /* This is in preemptible context */ - enable_ptr_key_workfn(&enable_ptr_key_work); - return 0; + if (!filled) { + get_random_bytes(&ptr_key, sizeof(ptr_key)); + queue_work(system_unbound_wq, &enable_ptr_key_work); + filled = true; + } + spin_unlock_irqrestore(&filling, flags); } - return ret; -} -early_initcall(initialize_ptr_random); - -/* Maps a pointer to a 32 bit unique identifier. */ -static inline int __ptr_to_hashval(const void *ptr, unsigned long *hashval_out) -{ - unsigned long hashval; - - if (static_branch_unlikely(¬_filled_random_ptr_key)) - return -EAGAIN; #ifdef CONFIG_64BIT hashval = (unsigned long)siphash_1u64((u64)ptr, &ptr_key); @@ -848,6 +831,19 @@ static char *ptr_to_id(char *buf, char *end, const void *ptr, return pointer_string(buf, end, (const void *)hashval, spec); } +static char *default_pointer(char *buf, char *end, const void *ptr, + struct printf_spec spec) +{ + /* + * default is to _not_ leak addresses, so hash before printing, + * unless no_hash_pointers is specified on the command line. + */ + if (unlikely(no_hash_pointers)) + return pointer_string(buf, end, ptr, spec); + + return ptr_to_id(buf, end, ptr, spec); +} + int kptr_restrict __read_mostly; static noinline_for_stack @@ -857,7 +853,7 @@ char *restricted_pointer(char *buf, char *end, const void *ptr, switch (kptr_restrict) { case 0: /* Handle as %p, hash and do _not_ leak addresses. */ - return ptr_to_id(buf, end, ptr, spec); + return default_pointer(buf, end, ptr, spec); case 1: { const struct cred *cred; @@ -1761,7 +1757,7 @@ char *fourcc_string(char *buf, char *end, const u32 *fourcc, char output[sizeof("0123 little-endian (0x01234567)")]; char *p = output; unsigned int i; - u32 val; + u32 orig, val; if (fmt[1] != 'c' || fmt[2] != 'c') return error_string(buf, end, "(%p4?)", spec); @@ -1769,21 +1765,23 @@ char *fourcc_string(char *buf, char *end, const u32 *fourcc, if (check_pointer(&buf, end, fourcc, spec)) return buf; - val = *fourcc & ~BIT(31); + orig = get_unaligned(fourcc); + val = orig & ~BIT(31); - for (i = 0; i < sizeof(*fourcc); i++) { + for (i = 0; i < sizeof(u32); i++) { unsigned char c = val >> (i * 8); /* Print non-control ASCII characters as-is, dot otherwise */ *p++ = isascii(c) && isprint(c) ? c : '.'; } - strcpy(p, *fourcc & BIT(31) ? " big-endian" : " little-endian"); + *p++ = ' '; + strcpy(p, orig & BIT(31) ? "big-endian" : "little-endian"); p += strlen(p); *p++ = ' '; *p++ = '('; - p = special_hex_number(p, output + sizeof(output) - 2, *fourcc, sizeof(u32)); + p = special_hex_number(p, output + sizeof(output) - 2, orig, sizeof(u32)); *p++ = ')'; *p = '\0'; @@ -2223,10 +2221,6 @@ char *fwnode_string(char *buf, char *end, struct fwnode_handle *fwnode, return widen_string(buf, buf - buf_start, end, spec); } -/* Disable pointer hashing if requested */ -bool no_hash_pointers __ro_after_init; -EXPORT_SYMBOL_GPL(no_hash_pointers); - int __init no_hash_pointers_enable(char *str) { if (no_hash_pointers) @@ -2455,7 +2449,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, case 'e': /* %pe with a non-ERR_PTR gets treated as plain %p */ if (!IS_ERR(ptr)) - break; + return default_pointer(buf, end, ptr, spec); return err_ptr(buf, end, ptr, spec); case 'u': case 'k': @@ -2465,16 +2459,9 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, default: return error_string(buf, end, "(einval)", spec); } + default: + return default_pointer(buf, end, ptr, spec); } - - /* - * default is to _not_ leak addresses, so hash before printing, - * unless no_hash_pointers is specified on the command line. - */ - if (unlikely(no_hash_pointers)) - return pointer_string(buf, end, ptr, spec); - else - return ptr_to_id(buf, end, ptr, spec); } /* @@ -2895,13 +2882,15 @@ int vscnprintf(char *buf, size_t size, const char *fmt, va_list args) { int i; + if (unlikely(!size)) + return 0; + i = vsnprintf(buf, size, fmt, args); if (likely(i < size)) return i; - if (size != 0) - return size - 1; - return 0; + + return size - 1; } EXPORT_SYMBOL(vscnprintf); diff --git a/lib/xarray.c b/lib/xarray.c index 6f47f6375808..ea9ce1f0b386 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -207,6 +207,8 @@ static void *xas_descend(struct xa_state *xas, struct xa_node *node) if (xa_is_sibling(entry)) { offset = xa_to_sibling(entry); entry = xa_entry(xas->xa, node, offset); + if (node->shift && xa_is_node(entry)) + entry = XA_RETRY_ENTRY; } xas->xa_offset = offset; @@ -262,9 +264,10 @@ static void xa_node_free(struct xa_node *node) * xas_destroy() - Free any resources allocated during the XArray operation. * @xas: XArray operation state. * - * This function is now internal-only. + * Most users will not need to call this function; it is called for you + * by xas_nomem(). */ -static void xas_destroy(struct xa_state *xas) +void xas_destroy(struct xa_state *xas) { struct xa_node *next, *node = xas->xa_alloc; @@ -302,7 +305,7 @@ bool xas_nomem(struct xa_state *xas, gfp_t gfp) } if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) gfp |= __GFP_ACCOUNT; - xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); + xas->xa_alloc = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp); if (!xas->xa_alloc) return false; xas->xa_alloc->parent = NULL; @@ -334,10 +337,10 @@ static bool __xas_nomem(struct xa_state *xas, gfp_t gfp) gfp |= __GFP_ACCOUNT; if (gfpflags_allow_blocking(gfp)) { xas_unlock_type(xas, lock_type); - xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); + xas->xa_alloc = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp); xas_lock_type(xas, lock_type); } else { - xas->xa_alloc = kmem_cache_alloc(radix_tree_node_cachep, gfp); + xas->xa_alloc = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp); } if (!xas->xa_alloc) return false; @@ -371,7 +374,7 @@ static void *xas_alloc(struct xa_state *xas, unsigned int shift) if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) gfp |= __GFP_ACCOUNT; - node = kmem_cache_alloc(radix_tree_node_cachep, gfp); + node = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp); if (!node) { xas_set_err(xas, -ENOMEM); return NULL; @@ -722,6 +725,8 @@ void xas_create_range(struct xa_state *xas) for (;;) { struct xa_node *node = xas->xa_node; + if (node->shift >= shift) + break; xas->xa_node = xa_parent_locked(xas->xa, node); xas->xa_offset = node->offset - 1; if (node->offset != 0) @@ -1014,7 +1019,7 @@ void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, void *sibling = NULL; struct xa_node *node; - node = kmem_cache_alloc(radix_tree_node_cachep, gfp); + node = kmem_cache_alloc_lru(radix_tree_node_cachep, xas->xa_lru, gfp); if (!node) goto nomem; node->array = xas->xa; @@ -1079,6 +1084,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order) xa_mk_node(child)); if (xa_is_value(curr)) values--; + xas_update(xas, child); } else { unsigned int canon = offset - xas->xa_sibs; @@ -1093,6 +1099,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order) } while (offset-- > xas->xa_offset); node->nr_values += values; + xas_update(xas, node); } EXPORT_SYMBOL_GPL(xas_split); #endif |